From b1b7408468273727c2a56a7947dca2d5c7c7f2d5 Mon Sep 17 00:00:00 2001 From: Quang Son Le Date: Wed, 10 Jul 2024 17:52:36 +0200 Subject: [PATCH 01/55] POLIO-1610: Add IM endpoint - Abstract logic from refresh_lqas_data - use inheritance to set up IM endpoint --- iaso/api/tasks/__init__.py | 5 +- plugins/polio/api/urls.py | 2 + plugins/polio/tasks/api/refresh_im_data.py | 6 + plugins/polio/tasks/api/refresh_lqas_data.py | 148 +----------------- .../polio/tasks/api/refresh_lqas_im_data.py | 146 +++++++++++++++++ 5 files changed, 160 insertions(+), 147 deletions(-) create mode 100644 plugins/polio/tasks/api/refresh_im_data.py create mode 100644 plugins/polio/tasks/api/refresh_lqas_im_data.py diff --git a/iaso/api/tasks/__init__.py b/iaso/api/tasks/__init__.py index df1d7f6723..4f75e418b1 100644 --- a/iaso/api/tasks/__init__.py +++ b/iaso/api/tasks/__init__.py @@ -189,8 +189,7 @@ def get_serializer_class(self): return ExternalTaskSerializer def create(self, request): - serializer_class = self.get_serializer_class() - serializer = serializer_class(data=request.data, context={"request": request}) + serializer = self.get_serializer(data=request.data) serializer.is_valid(raise_exception=True) task = serializer.save() data = serializer.validated_data @@ -230,7 +229,7 @@ def launch_task(self, slug, config={}, task_id=None, id_field=None): except: logger.exception(f"Bad id_field configuration.Expected non-empty dict, got {id_field}") return ERRORED - run_statuses = ["queued", "success", "failed"] + run_statuses = ["queued", "success", "failed", "stopped"] transport = RequestsHTTPTransport( url=openhexa_url, verify=True, diff --git a/plugins/polio/api/urls.py b/plugins/polio/api/urls.py index 589d6b2bdd..aa8becc47e 100644 --- a/plugins/polio/api/urls.py +++ b/plugins/polio/api/urls.py @@ -41,6 +41,7 @@ IncidentReportViewSet, ) +from plugins.polio.tasks.api.refresh_im_data import RefreshIMDataViewset from plugins.polio.tasks.api.refresh_lqas_data import RefreshLQASDataViewset router = routers.SimpleRouter() @@ -71,6 +72,7 @@ router.register(r"polio/rounds", RoundViewSet, basename="rounds") router.register(r"polio/reasonsfordelay", ReasonForDelayViewSet, basename="reasonsfordelay") router.register(r"polio/tasks/refreshlqas", RefreshLQASDataViewset, basename="refreshlqas") +router.register(r"polio/tasks/refreshim", RefreshIMDataViewset, basename="refreshim") router.register(r"polio/vaccine/request_forms", VaccineRequestFormViewSet, basename="vaccine_request_forms") router.register(r"polio/vaccine/vaccine_stock", VaccineStockManagementViewSet, basename="vaccine_stocks") router.register( diff --git a/plugins/polio/tasks/api/refresh_im_data.py b/plugins/polio/tasks/api/refresh_im_data.py new file mode 100644 index 0000000000..268cec115a --- /dev/null +++ b/plugins/polio/tasks/api/refresh_im_data.py @@ -0,0 +1,6 @@ +from plugins.polio.tasks.api.refresh_lqas_im_data import IM_CONFIG_SLUG, IM_TASK_NAME, RefreshLQASIMDataViewset + + +class RefreshIMDataViewset(RefreshLQASIMDataViewset): + task_name = IM_TASK_NAME + config_slug = IM_CONFIG_SLUG \ No newline at end of file diff --git a/plugins/polio/tasks/api/refresh_lqas_data.py b/plugins/polio/tasks/api/refresh_lqas_data.py index f7650318eb..dc5bb47817 100644 --- a/plugins/polio/tasks/api/refresh_lqas_data.py +++ b/plugins/polio/tasks/api/refresh_lqas_data.py @@ -1,146 +1,6 @@ -from iaso.api.tasks import ExternalTaskModelViewSet, ExternalTaskPostSerializer, ExternalTaskSerializer, TaskSerializer -from iaso.models.base import RUNNING, SKIPPED, ERRORED, SUCCESS, Task -from iaso.models.org_unit import OrgUnit -from rest_framework import permissions, serializers, filters -from hat.menupermissions import models as permission -from iaso.api.common import HasPermission -from rest_framework.response import Response -from django_filters.rest_framework import DjangoFilterBackend # type:ignore -from django.db.models import Q -from rest_framework.decorators import action -import jsonschema +from plugins.polio.tasks.api.refresh_lqas_im_data import LQAS_CONFIG_SLUG, LQAS_TASK_NAME, RefreshLQASIMDataViewset -TASK_NAME = "Refresh LQAS data" -NO_AUTHORIZED_COUNTRY_ERROR_MESSAGE = "No authorised org unit found for user" -NO_AUTHORIZED_COUNTRY_ERROR = {"country_id": NO_AUTHORIZED_COUNTRY_ERROR_MESSAGE} -LQAS_CONFIG_SLUG = "lqas-pipeline-config" -NO_AUTHORIZED_COUNTRY_ERROR = {"country_id": "No authorised org unit found for user"} -pipeline_config_schema = { - "type": "object", - "properties": { - "openhexa_token": {"type": "string"}, - "openhexa_url": {"type": "string"}, - "lqas_pipeline": {"type": "string"}, - "oh_pipeline_target": {"type": "string"}, - "lqas_pipeline_version": {"type": "number"}, - }, - "required": ["openhexa_token", "openhexa_url", "lqas_pipeline", "oh_pipeline_target", "lqas_pipeline_version"], -} - -class RefreshLQASDataGetSerializer(serializers.Serializer): - country_id = serializers.IntegerField(required=False) - - def validate(self, attrs): - validated_data = super().validate(attrs) - request = self.context["request"] - country_id = request.query_params.get("country_id", None) - # It seems a bit stange to limit the access on country id - # but to launch the refresh for all countries if no id is passed - if country_id is not None: - user = request.user - country_id = int(country_id) - user_has_access = OrgUnit.objects.filter_for_user(user).filter(id=country_id).count() > 0 - if not user_has_access: - raise serializers.ValidationError(NO_AUTHORIZED_COUNTRY_ERROR) - return validated_data - - -class RefreshLQASDataPostSerializer(ExternalTaskPostSerializer): - def validate(self, attrs): - validated_data = super().validate(attrs) - request = self.context["request"] - slug = validated_data.get("slug", None) - config = validated_data.get("config", None) - id_field = validated_data.get("id_field", None) - error = {} - if slug is None or slug != LQAS_CONFIG_SLUG: - error["slug"] = f"Wrong config slug. Expected {LQAS_CONFIG_SLUG}, got {slug}" - if config is None: - error["config"] = "This field is mandatory" - if id_field is None: - error["id_field"] = "This field is mandatory" - country_id = id_field.get("country_id", None) - if country_id is None: - error["id_field"] = "id_field should contain field 'country_id" - # It seems a bit stange to limit the access on country id - # but to launch the refresh for all countries if no id is passed - if country_id is not None: - user = request.user - try: - country_id = int(country_id) - user_has_access = OrgUnit.objects.filter_for_user(user).filter(id=country_id).count() > 0 - if not user_has_access: - error["id_field"] = NO_AUTHORIZED_COUNTRY_ERROR_MESSAGE - except: - error["id_field"] = f"Expected int, got {country_id}" - if error: - raise serializers.ValidationError(error) - res = {**validated_data} - res["config"] = config # is this safe? - res["id_field"] = id_field - return res - - -class CustomTaskSearchFilterBackend(filters.BaseFilterBackend): - def filter_queryset(self, request, queryset, view): - search = request.query_params.get("search") - if search: - query = ( - Q(name__icontains=search) - | Q(status__icontains=search) - | Q(launcher__first_name__icontains=search) - | Q(launcher__username__icontains=search) - | Q(launcher__last_name__icontains=search) - ) - return queryset.filter(query) - - return queryset - - -class RefreshLQASDataViewset(ExternalTaskModelViewSet): - permission_classes = [permissions.IsAuthenticated, HasPermission(permission.POLIO, permission.POLIO_CONFIG)] # type: ignore - http_method_names = ["get", "post", "patch"] - model = Task - filter_backends = [ - filters.OrderingFilter, - CustomTaskSearchFilterBackend, - DjangoFilterBackend, - ] - filterset_fields = {"created_at": ["gte"], "ended_at": ["exact"], "started_at": ["gte"], "status": ["exact"]} - ordering_fields = ["created_at", "ended_at", "name", "started_at", "status"] - ordering = ["-started_at"] - - def get_serializer_class(self): - if self.request.method == "POST": - return RefreshLQASDataPostSerializer - return ExternalTaskSerializer - - def get_queryset(self): - user = self.request.user - account = user.iaso_profile.account - queryset = Task.objects.filter(account=account).filter(external=True) - authorized_countries = user.iaso_profile.org_units.filter(org_unit_type_id__category="COUNTRY") - if authorized_countries.count() > 0: - authorized_names = [f"{TASK_NAME}-{id}" for id in authorized_countries] - queryset = queryset.filter(name__in=authorized_names) - return queryset - - @action(detail=False, methods=["get"], serializer_class=TaskSerializer) - def last_run_for_country(self, request): - serializer = RefreshLQASDataGetSerializer(data=request.data, context={"request": request}) - serializer.is_valid(raise_exception=True) - country_id = request.query_params.get("country_id", None) - status_query = Q(status=SUCCESS) | Q(status=RUNNING) | Q(status=ERRORED) - queryset = self.get_queryset().filter(status_query).exclude(started_at__isnull=True) - # The filter is based on how the task name is generated by ExternalTaskPostSerializer - query = ( - Q(name=LQAS_CONFIG_SLUG) | Q(name=f"{LQAS_CONFIG_SLUG}-{country_id}") - if country_id is not None - else Q(name=LQAS_CONFIG_SLUG) - ) - queryset = queryset.filter(query).order_by("-started_at") - if queryset.count() == 0: - return Response({"task": {}}) - result = queryset.first() - return Response({"task": TaskSerializer(instance=result).data}) +class RefreshLQASDataViewset(RefreshLQASIMDataViewset): + task_name = LQAS_TASK_NAME + config_slug = LQAS_CONFIG_SLUG diff --git a/plugins/polio/tasks/api/refresh_lqas_im_data.py b/plugins/polio/tasks/api/refresh_lqas_im_data.py new file mode 100644 index 0000000000..74680727ec --- /dev/null +++ b/plugins/polio/tasks/api/refresh_lqas_im_data.py @@ -0,0 +1,146 @@ +from iaso.api.tasks import ExternalTaskModelViewSet, ExternalTaskPostSerializer, ExternalTaskSerializer, TaskSerializer +from iaso.models.base import RUNNING, ERRORED, SUCCESS, Task +from iaso.models.org_unit import OrgUnit +from rest_framework import permissions, serializers, filters +from hat.menupermissions import models as permission +from iaso.api.common import HasPermission +from rest_framework.response import Response +from django_filters.rest_framework import DjangoFilterBackend # type:ignore +from django.db.models import Q +from rest_framework.decorators import action + +LQAS_TASK_NAME = "Refresh LQAS data" +LQAS_CONFIG_SLUG = "lqas-pipeline-config" +IM_TASK_NAME = "Refresh IM data" +IM_CONFIG_SLUG = 'im-pipeline-config' +NO_AUTHORIZED_COUNTRY_ERROR_MESSAGE = "No authorised org unit found for user" +NO_AUTHORIZED_COUNTRY_ERROR = {"country_id": NO_AUTHORIZED_COUNTRY_ERROR_MESSAGE} +NO_AUTHORIZED_COUNTRY_ERROR = {"country_id": "No authorised org unit found for user"} + + +class RefreshLQASIMDataGetSerializer(serializers.Serializer): + country_id = serializers.IntegerField(required=False) + + def validate(self, attrs): + validated_data = super().validate(attrs) + request = self.context["request"] + country_id = request.query_params.get("country_id", None) + # It seems a bit stange to limit the access on country id + # but to launch the refresh for all countries if no id is passed + if country_id is not None: + user = request.user + country_id = int(country_id) + user_has_access = OrgUnit.objects.filter_for_user(user).filter(id=country_id).count() > 0 + if not user_has_access: + raise serializers.ValidationError(NO_AUTHORIZED_COUNTRY_ERROR) + return validated_data + + +class RefreshLQASIMDataPostSerializer(ExternalTaskPostSerializer): + def validate(self, attrs): + validated_data = super().validate(attrs) + request = self.context["request"] + config_slug = self.context.get("config_slug", None) + print("DOES IT WORK", config_slug) + slug = validated_data.get("slug", None) + config = validated_data.get("config", None) + id_field = validated_data.get("id_field", None) + error = {} + if slug is None or slug != config_slug: + error["slug"] = f"Wrong config slug. Expected {config_slug}, got {slug}" + if config is None: + error["config"] = "This field is mandatory" + if id_field is None: + error["id_field"] = "This field is mandatory" + country_id = id_field.get("country_id", None) + if country_id is None: + error["id_field"] = "id_field should contain field 'country_id" + # It seems a bit stange to limit the access on country id + # but to launch the refresh for all countries if no id is passed + if country_id is not None: + user = request.user + try: + country_id = int(country_id) + user_has_access = OrgUnit.objects.filter_for_user(user).filter(id=country_id).count() > 0 + if not user_has_access: + error["id_field"] = NO_AUTHORIZED_COUNTRY_ERROR_MESSAGE + except: + error["id_field"] = f"Expected int, got {country_id}" + if error: + raise serializers.ValidationError(error) + res = {**validated_data} + res["config"] = config # is this safe? + res["id_field"] = id_field + return res + + +class CustomTaskSearchFilterBackend(filters.BaseFilterBackend): + def filter_queryset(self, request, queryset, view): + search = request.query_params.get("search") + if search: + query = ( + Q(name__icontains=search) + | Q(status__icontains=search) + | Q(launcher__first_name__icontains=search) + | Q(launcher__username__icontains=search) + | Q(launcher__last_name__icontains=search) + ) + return queryset.filter(query) + + return queryset + + +class RefreshLQASIMDataViewset(ExternalTaskModelViewSet): + permission_classes = [permissions.IsAuthenticated, HasPermission(permission.POLIO, permission.POLIO_CONFIG)] # type: ignore + http_method_names = ["get", "post", "patch"] + model = Task + filter_backends = [ + filters.OrderingFilter, + CustomTaskSearchFilterBackend, + DjangoFilterBackend, + ] + filterset_fields = {"created_at": ["gte"], "ended_at": ["exact"], "started_at": ["gte"], "status": ["exact"]} + ordering_fields = ["created_at", "ended_at", "name", "started_at", "status"] + ordering = ["-started_at"] + # defaulting to LQAS + task_name = LQAS_TASK_NAME + config_slug = LQAS_CONFIG_SLUG + + def get_serializer_class(self): + if self.request.method == "POST": + return RefreshLQASIMDataPostSerializer + return ExternalTaskSerializer + + def get_serializer_context(self): + context = super().get_serializer_context() + context.update({"config_slug": self.config_slug}) + return context + + def get_queryset(self): + user = self.request.user + account = user.iaso_profile.account + queryset = Task.objects.filter(account=account).filter(external=True) + authorized_countries = user.iaso_profile.org_units.filter(org_unit_type_id__category="COUNTRY") + if authorized_countries.count() > 0: + authorized_names = [f"{self.task_name}-{id}" for id in authorized_countries] + queryset = queryset.filter(name__in=authorized_names) + return queryset + + @action(detail=False, methods=["get"], serializer_class=TaskSerializer) + def last_run_for_country(self, request): + serializer = RefreshLQASIMDataGetSerializer(data=request.data, context={"request": request}) + serializer.is_valid(raise_exception=True) + country_id = request.query_params.get("country_id", None) + status_query = Q(status=SUCCESS) | Q(status=RUNNING) | Q(status=ERRORED) + queryset = self.get_queryset().filter(status_query).exclude(started_at__isnull=True) + # The filter is based on how the task name is generated by ExternalTaskPostSerializer + query = ( + Q(name=self.config_slug) | Q(name=f"{self.config_slug}-{country_id}") + if country_id is not None + else Q(name=self.config_slug) + ) + queryset = queryset.filter(query).order_by("-started_at") + if queryset.count() == 0: + return Response({"task": {}}) + result = queryset.first() + return Response({"task": TaskSerializer(instance=result).data}) From 4dd8f68bff6c47924ae782aa5a30c0f595ced651 Mon Sep 17 00:00:00 2001 From: Quang Son Le Date: Wed, 10 Jul 2024 17:53:59 +0200 Subject: [PATCH 02/55] POLIO-1610: add refresh button in IM UI - Rename RefreshLQASData --> RefreshLqasIMData - Add IM specific code --- .../js/src/domains/LQAS-IM/shared/Filters.tsx | 22 +++++++++---------- ...reshLqasData.tsx => RefreshLqasIMData.tsx} | 11 ++++++---- .../js/src/hooks/useGetLatestLQASIMUpdate.ts | 6 +++-- plugins/polio/tasks/api/refresh_im_data.py | 2 +- .../polio/tasks/api/refresh_lqas_im_data.py | 2 +- 5 files changed, 23 insertions(+), 20 deletions(-) rename plugins/polio/js/src/domains/LQAS-IM/shared/{RefreshLqasData.tsx => RefreshLqasIMData.tsx} (92%) diff --git a/plugins/polio/js/src/domains/LQAS-IM/shared/Filters.tsx b/plugins/polio/js/src/domains/LQAS-IM/shared/Filters.tsx index e03a0147de..42587500c2 100644 --- a/plugins/polio/js/src/domains/LQAS-IM/shared/Filters.tsx +++ b/plugins/polio/js/src/domains/LQAS-IM/shared/Filters.tsx @@ -17,7 +17,7 @@ import { DisplayIfUserHasPerm } from '../../../../../../../hat/assets/js/apps/Ia import MESSAGES from '../../../constants/messages'; import { makeCampaignsDropDown } from '../../../utils/index'; import { useGetLqasImCountriesOptions } from './hooks/api/useGetLqasImCountriesOptions'; -import { RefreshLqasData } from './RefreshLqasData'; +import { RefreshLqasIMData } from './RefreshLqasIMData'; import { baseUrls } from '../../../constants/urls'; import { POLIO_ADMIN } from '../../../constants/permissions'; @@ -142,17 +142,15 @@ export const Filters: FunctionComponent = ({ /> )} - {/* remove condition when IM pipeline is ready */} - {!imType && ( - - - - - - )} + + + + + + ); diff --git a/plugins/polio/js/src/domains/LQAS-IM/shared/RefreshLqasData.tsx b/plugins/polio/js/src/domains/LQAS-IM/shared/RefreshLqasIMData.tsx similarity index 92% rename from plugins/polio/js/src/domains/LQAS-IM/shared/RefreshLqasData.tsx rename to plugins/polio/js/src/domains/LQAS-IM/shared/RefreshLqasIMData.tsx index e046ad72ab..5c18db191e 100644 --- a/plugins/polio/js/src/domains/LQAS-IM/shared/RefreshLqasData.tsx +++ b/plugins/polio/js/src/domains/LQAS-IM/shared/RefreshLqasIMData.tsx @@ -21,6 +21,8 @@ type Props = { const LQAS_TASK_ENDPOINT = '/api/polio/tasks/refreshlqas/'; const LQAS_CONFIG_SLUG = 'lqas-pipeline-config'; +const IM_TASK_ENDPOINT = '/api/polio/tasks/refreshim/'; +const IM_CONFIG_SLUG = 'im-pipeline-config'; const useLastUpdate = ( lastUpdate: Task, @@ -46,11 +48,12 @@ const useLastUpdate = ( return result; }; -export const RefreshLqasData: FunctionComponent = ({ +export const RefreshLqasIMData: FunctionComponent = ({ countryId, isLqas, }) => { - const taskUrl = isLqas ? LQAS_TASK_ENDPOINT : undefined; + const taskUrl = isLqas ? LQAS_TASK_ENDPOINT : IM_TASK_ENDPOINT; + const slug = isLqas ? LQAS_CONFIG_SLUG : IM_CONFIG_SLUG; const { formatMessage } = useSafeIntl(); const [lastTaskStatus, setlastTaskStatus] = useState(); const queryClient = useQueryClient(); @@ -68,11 +71,11 @@ export const RefreshLqasData: FunctionComponent = ({ if (countryId) { createRefreshTask({ config: { country_id: parseInt(countryId, 10) }, - slug: LQAS_CONFIG_SLUG, + slug, id_field: { country_id: parseInt(countryId, 10) }, }); } - }, [countryId, createRefreshTask]); + }, [countryId, createRefreshTask, slug]); useEffect(() => { if (lastTaskStatus !== latestManualRefresh?.status) { diff --git a/plugins/polio/js/src/hooks/useGetLatestLQASIMUpdate.ts b/plugins/polio/js/src/hooks/useGetLatestLQASIMUpdate.ts index 6f7aab11f0..6c63be27b8 100644 --- a/plugins/polio/js/src/hooks/useGetLatestLQASIMUpdate.ts +++ b/plugins/polio/js/src/hooks/useGetLatestLQASIMUpdate.ts @@ -3,11 +3,13 @@ import { useSnackQuery } from '../../../../../hat/assets/js/apps/Iaso/libs/apiHo import { getRequest } from '../../../../../hat/assets/js/apps/Iaso/libs/Api'; import { Optional } from '../../../../../hat/assets/js/apps/Iaso/types/utils'; -const endpoint = '/api/polio/tasks/refreshlqas/last_run_for_country/'; +const lqasEndpoint = '/api/polio/tasks/refreshlqas/last_run_for_country/'; +const imEndpoint = '/api/polio/tasks/refreshim/last_run_for_country/'; const getLatestRefresh = (isLqas: boolean, countryId: Optional) => { + const endpoint = isLqas ? lqasEndpoint : imEndpoint; const url = countryId ? `${endpoint}?country_id=${countryId}` : endpoint; - if (isLqas && countryId !== undefined) { + if (countryId !== undefined) { return getRequest(url); } return null; diff --git a/plugins/polio/tasks/api/refresh_im_data.py b/plugins/polio/tasks/api/refresh_im_data.py index 268cec115a..939aa85b3d 100644 --- a/plugins/polio/tasks/api/refresh_im_data.py +++ b/plugins/polio/tasks/api/refresh_im_data.py @@ -3,4 +3,4 @@ class RefreshIMDataViewset(RefreshLQASIMDataViewset): task_name = IM_TASK_NAME - config_slug = IM_CONFIG_SLUG \ No newline at end of file + config_slug = IM_CONFIG_SLUG diff --git a/plugins/polio/tasks/api/refresh_lqas_im_data.py b/plugins/polio/tasks/api/refresh_lqas_im_data.py index 74680727ec..b89985adb0 100644 --- a/plugins/polio/tasks/api/refresh_lqas_im_data.py +++ b/plugins/polio/tasks/api/refresh_lqas_im_data.py @@ -12,7 +12,7 @@ LQAS_TASK_NAME = "Refresh LQAS data" LQAS_CONFIG_SLUG = "lqas-pipeline-config" IM_TASK_NAME = "Refresh IM data" -IM_CONFIG_SLUG = 'im-pipeline-config' +IM_CONFIG_SLUG = "im-pipeline-config" NO_AUTHORIZED_COUNTRY_ERROR_MESSAGE = "No authorised org unit found for user" NO_AUTHORIZED_COUNTRY_ERROR = {"country_id": NO_AUTHORIZED_COUNTRY_ERROR_MESSAGE} NO_AUTHORIZED_COUNTRY_ERROR = {"country_id": "No authorised org unit found for user"} From f67d3385c9442581d2059a00851061c0ddc7ecf2 Mon Sep 17 00:00:00 2001 From: Quang Son Le Date: Wed, 10 Jul 2024 17:56:44 +0200 Subject: [PATCH 03/55] POLIO-1610: update button text --- plugins/polio/js/src/constants/messages.ts | 4 ++++ plugins/polio/js/src/constants/translations/en.json | 1 + plugins/polio/js/src/constants/translations/fr.json | 1 + .../js/src/domains/LQAS-IM/shared/RefreshLqasIMData.tsx | 5 ++++- 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/plugins/polio/js/src/constants/messages.ts b/plugins/polio/js/src/constants/messages.ts index 61cb102b47..09d2a1252f 100644 --- a/plugins/polio/js/src/constants/messages.ts +++ b/plugins/polio/js/src/constants/messages.ts @@ -2210,6 +2210,10 @@ const MESSAGES = defineMessages({ id: 'iaso.polio.label.refreshLqasData', defaultMessage: 'Refresh LQAS data', }, + refreshIMData: { + id: 'iaso.polio.label.refreshIMData', + defaultMessage: 'Refresh IM data', + }, districtsInScope: { id: 'iaso.polio.label.districtsInScope', defaultMessage: 'Districts in scope', diff --git a/plugins/polio/js/src/constants/translations/en.json b/plugins/polio/js/src/constants/translations/en.json index 5740dfd26b..4045dc9401 100644 --- a/plugins/polio/js/src/constants/translations/en.json +++ b/plugins/polio/js/src/constants/translations/en.json @@ -520,6 +520,7 @@ "iaso.polio.label.receptionPreAlert": "Pre-alert", "iaso.polio.label.receptionVaccineArrivalReport": "Arrival report", "iaso.polio.label.refreshedAt": "Refreshed at", + "iaso.polio.label.refreshIMData": "Refresh IM data", "iaso.polio.label.refreshLqasData": "Refresh LQAS data", "iaso.polio.label.refreshPage": "Refresh the page", "iaso.polio.label.regional": "Regional", diff --git a/plugins/polio/js/src/constants/translations/fr.json b/plugins/polio/js/src/constants/translations/fr.json index e59ecd3cb9..37453666a4 100644 --- a/plugins/polio/js/src/constants/translations/fr.json +++ b/plugins/polio/js/src/constants/translations/fr.json @@ -519,6 +519,7 @@ "iaso.polio.label.receptionPreAlert": "Pré-alerte", "iaso.polio.label.receptionVaccineArrivalReport": "Rapport d'arrivée", "iaso.polio.label.refreshedAt": "Actualisé le", + "iaso.polio.label.refreshIMData": "Rafraîchir les données IM", "iaso.polio.label.refreshLqasData": "Rafraîchir les données LQAS", "iaso.polio.label.refreshPage": "Rafraîchir la page", "iaso.polio.label.regional": "Régional", diff --git a/plugins/polio/js/src/domains/LQAS-IM/shared/RefreshLqasIMData.tsx b/plugins/polio/js/src/domains/LQAS-IM/shared/RefreshLqasIMData.tsx index 5c18db191e..7b36ea00d5 100644 --- a/plugins/polio/js/src/domains/LQAS-IM/shared/RefreshLqasIMData.tsx +++ b/plugins/polio/js/src/domains/LQAS-IM/shared/RefreshLqasIMData.tsx @@ -87,6 +87,9 @@ export const RefreshLqasIMData: FunctionComponent = ({ }, [lastTaskStatus, latestManualRefresh?.status, queryClient]); const disableButton = Boolean(latestManualRefresh?.status === 'RUNNING'); // TODO make enum with statuses + const buttonText = isLqas + ? formatMessage(MESSAGES.refreshLqasData) + : formatMessage(MESSAGES.refreshIMData); if (!countryId) return null; return ( <> @@ -101,7 +104,7 @@ export const RefreshLqasIMData: FunctionComponent = ({ - {formatMessage(MESSAGES.refreshLqasData)} + {buttonText} {disableButton && ( Date: Thu, 11 Jul 2024 09:29:38 +0200 Subject: [PATCH 04/55] POLIO-1610: add tests for refresh im endpoint --- plugins/polio/tests/test_im_refresh_data.py | 211 ++++++++++++++++++++ 1 file changed, 211 insertions(+) create mode 100644 plugins/polio/tests/test_im_refresh_data.py diff --git a/plugins/polio/tests/test_im_refresh_data.py b/plugins/polio/tests/test_im_refresh_data.py new file mode 100644 index 0000000000..a777973d9c --- /dev/null +++ b/plugins/polio/tests/test_im_refresh_data.py @@ -0,0 +1,211 @@ +from datetime import datetime +import json +from iaso import models as m +from iaso.test import APITestCase +from iaso.models.base import RUNNING, KILLED, SUCCESS, SKIPPED + +from iaso.models.json_config import Config +from unittest.mock import patch + +from plugins.polio.tasks.api.refresh_im_data import RefreshIMDataViewset +from plugins.polio.tasks.api.refresh_lqas_im_data import IM_CONFIG_SLUG + + +class RefreshLQASDataTestCase(APITestCase): + @classmethod + def setUp(cls): + cls.url = "/api/polio/tasks/refreshim/" + cls.action_url = f"{cls.url}last_run_for_country/" + cls.account = account = m.Account.objects.create(name="test account") + cls.user = cls.create_user_with_profile(username="test user", account=account, permissions=["iaso_polio"]) + + cls.external_task1 = m.Task.objects.create( + status=RUNNING, account=account, launcher=cls.user, name="external task 1", external=True + ) + cls.external_task2 = m.Task.objects.create( + status=RUNNING, account=account, launcher=cls.user, name="external task 2", external=True + ) + cls.country = m.OrgUnitType.objects.create(name="Country", depth=1, category="COUNTRY") + cls.project = m.Project.objects.create(name="Polio", app_id="polio.rapid.outbreak.taskforce", account=account) + cls.data_source = m.DataSource.objects.create(name="Default source") + cls.data_source.projects.add(cls.project) + cls.data_source.save() + cls.source_version = m.SourceVersion.objects.create(data_source=cls.data_source, number=1) + cls.account.default_version = cls.source_version + cls.account.save() + cls.country_org_unit = m.OrgUnit.objects.create( + name="Country1", + validation_status=m.OrgUnit.VALIDATION_VALID, + source_ref="PvtAI4RUMkr", + org_unit_type=cls.country, + version=cls.source_version, + ) + cls.other_country_org_unit = m.OrgUnit.objects.create( + name="Country2", + validation_status=m.OrgUnit.VALIDATION_VALID, + source_ref="PvtAI4RUMkr", + org_unit_type=cls.country, + version=cls.source_version, + ) + cls.task1 = m.Task.objects.create( + status=RUNNING, + account=account, + launcher=cls.user, + name=IM_CONFIG_SLUG, + started_at=datetime.now(), + external=True, + ) + cls.task2 = m.Task.objects.create( + status=RUNNING, account=account, launcher=cls.user, name="task 2", external=True + ) + cls.limited_user = cls.create_user_with_profile( + username="other user", account=account, permissions=["iaso_polio"], org_units=[cls.other_country_org_unit] + ) + cls.json_config = { + "pipeline": "pipeline", + "openhexa_url": "https://yippie.openhexa.xyz/", + "openhexa_token": "token", + "pipeline_version": 1, + "oh_pipeline_target": "staging", + } + cls.lqas_config = Config.objects.create(slug=IM_CONFIG_SLUG, content=cls.json_config) + + def mock_openhexa_call_success(self, slug=None, config=None, id_field=None, task_id=None): + return SUCCESS + + def mock_openhexa_call_skipped(self, slug=None, config=None, id_field=None, task_id=None): + return SKIPPED + + def mock_openhexa_call_running(self, slug=None, config=None, id_field=None, task_id=None): + return RUNNING + + def test_no_perm(self): + user_no_perm = self.create_user_with_profile(username="test user2", account=self.account, permissions=[]) + self.client.force_authenticate(user_no_perm) + response = self.client.get( + self.url, + format="json", + ) + jr = self.assertJSONResponse(response, 403) + self.assertEqual({"detail": "You do not have permission to perform this action."}, jr) + + response = self.client.post( + self.url, + format="json", + data={ + "id_field": {"country_id": self.country_org_unit.id}, + "config": {"country_id": self.country_org_unit.id}, + "slug": IM_CONFIG_SLUG, + }, + ) + jr = self.assertJSONResponse(response, 403) + self.assertEqual({"detail": "You do not have permission to perform this action."}, jr) + + @patch.object(RefreshIMDataViewset, "launch_task", mock_openhexa_call_running) + def test_create_external_task(self): + self.client.force_authenticate(self.user) + response = self.client.post( + self.url, + format="json", + data={ + "id_field": {"country_id": self.country_org_unit.id}, + "config": {"country_id": self.country_org_unit.id}, + "slug": IM_CONFIG_SLUG, + }, + ) + response = self.assertJSONResponse(response, 200) + task = response["task"] + self.assertEqual(task["status"], RUNNING) + self.assertEqual(task["launcher"]["username"], self.user.username) + self.assertEqual(task["name"], f"{IM_CONFIG_SLUG}-{self.country_org_unit.id}") + + @patch.object(RefreshIMDataViewset, "launch_task", mock_openhexa_call_skipped) + def test_create_external_task_pipeline_already_running(self): + self.client.force_authenticate(self.user) + response = self.client.post( + self.url, + format="json", + data={ + "id_field": {"country_id": self.country_org_unit.id}, + "config": {"country_id": self.country_org_unit.id}, + "slug": IM_CONFIG_SLUG, + }, + ) + response = self.assertJSONResponse(response, 200) + task = response["task"] + self.assertEqual(task["status"], SKIPPED) + self.assertEqual(task["launcher"]["username"], self.user.username) + self.assertEqual(task["name"], f"{IM_CONFIG_SLUG}-{self.country_org_unit.id}") + + @patch.object(RefreshIMDataViewset, "launch_task", mock_openhexa_call_running) + def test_patch_external_task(self): + self.client.force_authenticate(self.user) + response = self.client.post( + self.url, + format="json", + data={ + "id_field": {"country_id": self.country_org_unit.id}, + "config": {"country_id": self.country_org_unit.id}, + "slug": IM_CONFIG_SLUG, + }, + ) + response = self.assertJSONResponse(response, 200) + task = response["task"] + task_id = task["id"] + self.assertEqual(task["status"], RUNNING) + self.assertEqual(task["progress_value"], 0) + self.assertEqual(task["end_value"], 0) + + response = self.client.patch( + f"{self.url}{task_id}/", format="json", data={"status": SUCCESS, "progress_value": 21} + ) + response = self.assertJSONResponse(response, 200) + task = response + self.assertEqual(task["id"], task_id) + self.assertEqual(task["status"], SUCCESS) + self.assertEqual(task["progress_value"], 21) + self.assertEqual(task["end_value"], 0) + + @patch.object(RefreshIMDataViewset, "launch_task", mock_openhexa_call_running) + def test_kill_external_task(self): + self.client.force_authenticate(self.user) + response = self.client.post( + self.url, + format="json", + data={ + "id_field": {"country_id": self.country_org_unit.id}, + "config": {"country_id": self.country_org_unit.id}, + "slug": IM_CONFIG_SLUG, + }, + ) + response = self.assertJSONResponse(response, 200) + task = response["task"] + task_id = task["id"] + self.assertEqual(task["status"], RUNNING) + + response = self.client.patch( + f"{self.url}{task_id}/", + format="json", + data={ + "should_be_killed": True, + }, + ) + response = self.assertJSONResponse(response, 200) + task = response + self.assertEqual(task["id"], task_id) + self.assertEqual(task["status"], KILLED) + + def test_get_latest_for_country(self): + self.client.force_authenticate(self.user) + response = self.client.get(f"{self.action_url}?country_id={self.country_org_unit.id}") + response = self.assertJSONResponse(response, 200) + task = response["task"] + self.assertEqual(task["id"], self.task1.id) + self.assertEqual(task["ended_at"], self.task1.ended_at) + self.assertEqual(task["status"], self.task1.status) + + def test_restrict_user_country(self): + self.client.force_authenticate(self.limited_user) + response = self.client.get(f"{self.action_url}?country_id={self.country_org_unit.id}") + response = self.assertJSONResponse(response, 400) + self.assertEqual(response, {"country_id": ["No authorised org unit found for user"]}) From cd4288842d923142228d676881bfd5cb62acfe13 Mon Sep 17 00:00:00 2001 From: Quang Son Le Date: Thu, 22 Aug 2024 18:13:04 +0200 Subject: [PATCH 05/55] POLIO-1610: fix bug in launch_task method --- iaso/api/tasks/__init__.py | 2 +- plugins/polio/tasks/api/refresh_lqas_im_data.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/iaso/api/tasks/__init__.py b/iaso/api/tasks/__init__.py index 4f75e418b1..b83be28e81 100644 --- a/iaso/api/tasks/__init__.py +++ b/iaso/api/tasks/__init__.py @@ -283,7 +283,7 @@ def launch_task(self, slug, config={}, task_id=None, id_field=None): mutation_input = ( {"id": pipeline, "versionId": pipeline_version, "config": oh_config} if pipeline_version - else {"id": pipeline, "config": config} + else {"id": pipeline, "config": oh_config} ) try: run_mutation = gql( diff --git a/plugins/polio/tasks/api/refresh_lqas_im_data.py b/plugins/polio/tasks/api/refresh_lqas_im_data.py index b89985adb0..21913094d4 100644 --- a/plugins/polio/tasks/api/refresh_lqas_im_data.py +++ b/plugins/polio/tasks/api/refresh_lqas_im_data.py @@ -41,7 +41,6 @@ def validate(self, attrs): validated_data = super().validate(attrs) request = self.context["request"] config_slug = self.context.get("config_slug", None) - print("DOES IT WORK", config_slug) slug = validated_data.get("slug", None) config = validated_data.get("config", None) id_field = validated_data.get("id_field", None) From 3de6e30f04f084e7ab61e7d6441f95e59b84e400 Mon Sep 17 00:00:00 2001 From: Quang Son Le Date: Thu, 22 Aug 2024 18:13:45 +0200 Subject: [PATCH 06/55] POLIO-1610: add viewsets + urls for all 3 IM types --- plugins/polio/api/urls.py | 11 +++++++- plugins/polio/tasks/api/refresh_im_data.py | 32 ++++++++++++++++++++-- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/plugins/polio/api/urls.py b/plugins/polio/api/urls.py index aa8becc47e..068c022e6d 100644 --- a/plugins/polio/api/urls.py +++ b/plugins/polio/api/urls.py @@ -41,7 +41,12 @@ IncidentReportViewSet, ) -from plugins.polio.tasks.api.refresh_im_data import RefreshIMDataViewset +from plugins.polio.tasks.api.refresh_im_data import ( + RefreshIMAllDataViewset, + RefreshIMDataViewset, + RefreshIMHouseholdDataViewset, + RefreshIMOutOfHouseholdDataViewset, +) from plugins.polio.tasks.api.refresh_lqas_data import RefreshLQASDataViewset router = routers.SimpleRouter() @@ -72,7 +77,11 @@ router.register(r"polio/rounds", RoundViewSet, basename="rounds") router.register(r"polio/reasonsfordelay", ReasonForDelayViewSet, basename="reasonsfordelay") router.register(r"polio/tasks/refreshlqas", RefreshLQASDataViewset, basename="refreshlqas") +# TODO delete router.register(r"polio/tasks/refreshim", RefreshIMDataViewset, basename="refreshim") +router.register(r"polio/tasks/refreshim/hh", RefreshIMHouseholdDataViewset, basename="refreshimhh") +router.register(r"polio/tasks/refreshim/ohh", RefreshIMOutOfHouseholdDataViewset, basename="refreshimohh") +router.register(r"polio/tasks/refreshim/hh_ohh", RefreshIMAllDataViewset, basename="refreshimhhohh") router.register(r"polio/vaccine/request_forms", VaccineRequestFormViewSet, basename="vaccine_request_forms") router.register(r"polio/vaccine/vaccine_stock", VaccineStockManagementViewSet, basename="vaccine_stocks") router.register( diff --git a/plugins/polio/tasks/api/refresh_im_data.py b/plugins/polio/tasks/api/refresh_im_data.py index 939aa85b3d..2b17fd2296 100644 --- a/plugins/polio/tasks/api/refresh_im_data.py +++ b/plugins/polio/tasks/api/refresh_im_data.py @@ -1,6 +1,34 @@ -from plugins.polio.tasks.api.refresh_lqas_im_data import IM_CONFIG_SLUG, IM_TASK_NAME, RefreshLQASIMDataViewset +from plugins.polio.tasks.api.refresh_lqas_im_data import IM_TASK_NAME, RefreshLQASIMDataViewset +from django.shortcuts import get_object_or_404 +from iaso.models.json_config import Config +import logging +from iaso.models.base import ERRORED, RUNNING, SKIPPED +from gql.transport.requests import RequestsHTTPTransport +from gql import Client, gql +logger = logging.getLogger(__name__) +IM_HH_CONFIG_SLUG = "im_hh-pipeline-config" +IM_OHH_CONFIG_SLUG = "im_ohh-pipeline-config" +IM_HH_OHH_CONFIG_SLUG = "im_hh_ohh-pipeline-config" + + +# deprecated class RefreshIMDataViewset(RefreshLQASIMDataViewset): task_name = IM_TASK_NAME - config_slug = IM_CONFIG_SLUG + config_slug = IM_HH_CONFIG_SLUG + + +class RefreshIMHouseholdDataViewset(RefreshLQASIMDataViewset): + task_name = IM_TASK_NAME + config_slug = IM_HH_CONFIG_SLUG + + +class RefreshIMOutOfHouseholdDataViewset(RefreshLQASIMDataViewset): + task_name = IM_TASK_NAME + config_slug = IM_OHH_CONFIG_SLUG + + +class RefreshIMAllDataViewset(RefreshLQASIMDataViewset): + task_name = IM_TASK_NAME + config_slug = IM_HH_OHH_CONFIG_SLUG From 4da4d1400d916875b2cb537f7de30bc48ca890f2 Mon Sep 17 00:00:00 2001 From: Quang Son Le Date: Mon, 26 Aug 2024 16:05:59 +0200 Subject: [PATCH 07/55] POLIO-1610: remove deprecated endpoint, update tests --- plugins/polio/api/urls.py | 3 - plugins/polio/tasks/api/refresh_im_data.py | 12 +- ...esh_data.py => test_im_refresh_hh_data.py} | 38 ++-- .../tests/test_im_refresh_hh_ohh_data.py | 207 ++++++++++++++++++ .../polio/tests/test_im_refresh_ohh_data.py | 207 ++++++++++++++++++ 5 files changed, 432 insertions(+), 35 deletions(-) rename plugins/polio/tests/{test_im_refresh_data.py => test_im_refresh_hh_data.py} (87%) create mode 100644 plugins/polio/tests/test_im_refresh_hh_ohh_data.py create mode 100644 plugins/polio/tests/test_im_refresh_ohh_data.py diff --git a/plugins/polio/api/urls.py b/plugins/polio/api/urls.py index a4cc275b17..881af9012e 100644 --- a/plugins/polio/api/urls.py +++ b/plugins/polio/api/urls.py @@ -48,7 +48,6 @@ from plugins.polio.tasks.api.refresh_im_data import ( RefreshIMAllDataViewset, - RefreshIMDataViewset, RefreshIMHouseholdDataViewset, RefreshIMOutOfHouseholdDataViewset, ) @@ -87,8 +86,6 @@ router.register(r"polio/rounds", RoundViewSet, basename="rounds") router.register(r"polio/reasonsfordelay", ReasonForDelayViewSet, basename="reasonsfordelay") router.register(r"polio/tasks/refreshlqas", RefreshLQASDataViewset, basename="refreshlqas") -# TODO delete -router.register(r"polio/tasks/refreshim", RefreshIMDataViewset, basename="refreshim") router.register(r"polio/tasks/refreshim/hh", RefreshIMHouseholdDataViewset, basename="refreshimhh") router.register(r"polio/tasks/refreshim/ohh", RefreshIMOutOfHouseholdDataViewset, basename="refreshimohh") router.register(r"polio/tasks/refreshim/hh_ohh", RefreshIMAllDataViewset, basename="refreshimhhohh") diff --git a/plugins/polio/tasks/api/refresh_im_data.py b/plugins/polio/tasks/api/refresh_im_data.py index 2b17fd2296..01bc0cfd65 100644 --- a/plugins/polio/tasks/api/refresh_im_data.py +++ b/plugins/polio/tasks/api/refresh_im_data.py @@ -1,10 +1,6 @@ from plugins.polio.tasks.api.refresh_lqas_im_data import IM_TASK_NAME, RefreshLQASIMDataViewset -from django.shortcuts import get_object_or_404 -from iaso.models.json_config import Config import logging -from iaso.models.base import ERRORED, RUNNING, SKIPPED -from gql.transport.requests import RequestsHTTPTransport -from gql import Client, gql + logger = logging.getLogger(__name__) @@ -13,12 +9,6 @@ IM_HH_OHH_CONFIG_SLUG = "im_hh_ohh-pipeline-config" -# deprecated -class RefreshIMDataViewset(RefreshLQASIMDataViewset): - task_name = IM_TASK_NAME - config_slug = IM_HH_CONFIG_SLUG - - class RefreshIMHouseholdDataViewset(RefreshLQASIMDataViewset): task_name = IM_TASK_NAME config_slug = IM_HH_CONFIG_SLUG diff --git a/plugins/polio/tests/test_im_refresh_data.py b/plugins/polio/tests/test_im_refresh_hh_data.py similarity index 87% rename from plugins/polio/tests/test_im_refresh_data.py rename to plugins/polio/tests/test_im_refresh_hh_data.py index a777973d9c..6d62f4fcbf 100644 --- a/plugins/polio/tests/test_im_refresh_data.py +++ b/plugins/polio/tests/test_im_refresh_hh_data.py @@ -1,20 +1,15 @@ from datetime import datetime -import json from iaso import models as m from iaso.test import APITestCase from iaso.models.base import RUNNING, KILLED, SUCCESS, SKIPPED - -from iaso.models.json_config import Config from unittest.mock import patch - -from plugins.polio.tasks.api.refresh_im_data import RefreshIMDataViewset -from plugins.polio.tasks.api.refresh_lqas_im_data import IM_CONFIG_SLUG +from plugins.polio.tasks.api.refresh_im_data import IM_HH_CONFIG_SLUG, RefreshIMHouseholdDataViewset -class RefreshLQASDataTestCase(APITestCase): +class RefreshIMHouseholdDataTestCase(APITestCase): @classmethod def setUp(cls): - cls.url = "/api/polio/tasks/refreshim/" + cls.url = "/api/polio/tasks/refreshim/hh/" cls.action_url = f"{cls.url}last_run_for_country/" cls.account = account = m.Account.objects.create(name="test account") cls.user = cls.create_user_with_profile(username="test user", account=account, permissions=["iaso_polio"]) @@ -51,7 +46,7 @@ def setUp(cls): status=RUNNING, account=account, launcher=cls.user, - name=IM_CONFIG_SLUG, + name=IM_HH_CONFIG_SLUG, started_at=datetime.now(), external=True, ) @@ -68,7 +63,6 @@ def setUp(cls): "pipeline_version": 1, "oh_pipeline_target": "staging", } - cls.lqas_config = Config.objects.create(slug=IM_CONFIG_SLUG, content=cls.json_config) def mock_openhexa_call_success(self, slug=None, config=None, id_field=None, task_id=None): return SUCCESS @@ -95,13 +89,13 @@ def test_no_perm(self): data={ "id_field": {"country_id": self.country_org_unit.id}, "config": {"country_id": self.country_org_unit.id}, - "slug": IM_CONFIG_SLUG, + "slug": IM_HH_CONFIG_SLUG, }, ) jr = self.assertJSONResponse(response, 403) self.assertEqual({"detail": "You do not have permission to perform this action."}, jr) - @patch.object(RefreshIMDataViewset, "launch_task", mock_openhexa_call_running) + @patch.object(RefreshIMHouseholdDataViewset, "launch_task", mock_openhexa_call_running) def test_create_external_task(self): self.client.force_authenticate(self.user) response = self.client.post( @@ -110,16 +104,16 @@ def test_create_external_task(self): data={ "id_field": {"country_id": self.country_org_unit.id}, "config": {"country_id": self.country_org_unit.id}, - "slug": IM_CONFIG_SLUG, + "slug": IM_HH_CONFIG_SLUG, }, ) response = self.assertJSONResponse(response, 200) task = response["task"] self.assertEqual(task["status"], RUNNING) self.assertEqual(task["launcher"]["username"], self.user.username) - self.assertEqual(task["name"], f"{IM_CONFIG_SLUG}-{self.country_org_unit.id}") + self.assertEqual(task["name"], f"{IM_HH_CONFIG_SLUG}-{self.country_org_unit.id}") - @patch.object(RefreshIMDataViewset, "launch_task", mock_openhexa_call_skipped) + @patch.object(RefreshIMHouseholdDataViewset, "launch_task", mock_openhexa_call_skipped) def test_create_external_task_pipeline_already_running(self): self.client.force_authenticate(self.user) response = self.client.post( @@ -128,16 +122,16 @@ def test_create_external_task_pipeline_already_running(self): data={ "id_field": {"country_id": self.country_org_unit.id}, "config": {"country_id": self.country_org_unit.id}, - "slug": IM_CONFIG_SLUG, + "slug": IM_HH_CONFIG_SLUG, }, ) response = self.assertJSONResponse(response, 200) task = response["task"] self.assertEqual(task["status"], SKIPPED) self.assertEqual(task["launcher"]["username"], self.user.username) - self.assertEqual(task["name"], f"{IM_CONFIG_SLUG}-{self.country_org_unit.id}") + self.assertEqual(task["name"], f"{IM_HH_CONFIG_SLUG}-{self.country_org_unit.id}") - @patch.object(RefreshIMDataViewset, "launch_task", mock_openhexa_call_running) + @patch.object(RefreshIMHouseholdDataViewset, "launch_task", mock_openhexa_call_running) def test_patch_external_task(self): self.client.force_authenticate(self.user) response = self.client.post( @@ -146,7 +140,7 @@ def test_patch_external_task(self): data={ "id_field": {"country_id": self.country_org_unit.id}, "config": {"country_id": self.country_org_unit.id}, - "slug": IM_CONFIG_SLUG, + "slug": IM_HH_CONFIG_SLUG, }, ) response = self.assertJSONResponse(response, 200) @@ -166,7 +160,7 @@ def test_patch_external_task(self): self.assertEqual(task["progress_value"], 21) self.assertEqual(task["end_value"], 0) - @patch.object(RefreshIMDataViewset, "launch_task", mock_openhexa_call_running) + @patch.object(RefreshIMHouseholdDataViewset, "launch_task", mock_openhexa_call_running) def test_kill_external_task(self): self.client.force_authenticate(self.user) response = self.client.post( @@ -175,7 +169,7 @@ def test_kill_external_task(self): data={ "id_field": {"country_id": self.country_org_unit.id}, "config": {"country_id": self.country_org_unit.id}, - "slug": IM_CONFIG_SLUG, + "slug": IM_HH_CONFIG_SLUG, }, ) response = self.assertJSONResponse(response, 200) @@ -200,6 +194,8 @@ def test_get_latest_for_country(self): response = self.client.get(f"{self.action_url}?country_id={self.country_org_unit.id}") response = self.assertJSONResponse(response, 200) task = response["task"] + print("RESPONSE", response) + print("TASK", task) self.assertEqual(task["id"], self.task1.id) self.assertEqual(task["ended_at"], self.task1.ended_at) self.assertEqual(task["status"], self.task1.status) diff --git a/plugins/polio/tests/test_im_refresh_hh_ohh_data.py b/plugins/polio/tests/test_im_refresh_hh_ohh_data.py new file mode 100644 index 0000000000..c0facfca5a --- /dev/null +++ b/plugins/polio/tests/test_im_refresh_hh_ohh_data.py @@ -0,0 +1,207 @@ +from datetime import datetime +from iaso import models as m +from iaso.test import APITestCase +from iaso.models.base import RUNNING, KILLED, SUCCESS, SKIPPED +from unittest.mock import patch +from plugins.polio.tasks.api.refresh_im_data import IM_HH_OHH_CONFIG_SLUG, RefreshIMAllDataViewset + + +class RefreshIMHouseholdDataTestCase(APITestCase): + @classmethod + def setUp(cls): + cls.url = "/api/polio/tasks/refreshim/hh_ohh/" + cls.action_url = f"{cls.url}last_run_for_country/" + cls.account = account = m.Account.objects.create(name="test account") + cls.user = cls.create_user_with_profile(username="test user", account=account, permissions=["iaso_polio"]) + + cls.external_task1 = m.Task.objects.create( + status=RUNNING, account=account, launcher=cls.user, name="external task 1", external=True + ) + cls.external_task2 = m.Task.objects.create( + status=RUNNING, account=account, launcher=cls.user, name="external task 2", external=True + ) + cls.country = m.OrgUnitType.objects.create(name="Country", depth=1, category="COUNTRY") + cls.project = m.Project.objects.create(name="Polio", app_id="polio.rapid.outbreak.taskforce", account=account) + cls.data_source = m.DataSource.objects.create(name="Default source") + cls.data_source.projects.add(cls.project) + cls.data_source.save() + cls.source_version = m.SourceVersion.objects.create(data_source=cls.data_source, number=1) + cls.account.default_version = cls.source_version + cls.account.save() + cls.country_org_unit = m.OrgUnit.objects.create( + name="Country1", + validation_status=m.OrgUnit.VALIDATION_VALID, + source_ref="PvtAI4RUMkr", + org_unit_type=cls.country, + version=cls.source_version, + ) + cls.other_country_org_unit = m.OrgUnit.objects.create( + name="Country2", + validation_status=m.OrgUnit.VALIDATION_VALID, + source_ref="PvtAI4RUMkr", + org_unit_type=cls.country, + version=cls.source_version, + ) + cls.task1 = m.Task.objects.create( + status=RUNNING, + account=account, + launcher=cls.user, + name=IM_HH_OHH_CONFIG_SLUG, + started_at=datetime.now(), + external=True, + ) + cls.task2 = m.Task.objects.create( + status=RUNNING, account=account, launcher=cls.user, name="task 2", external=True + ) + cls.limited_user = cls.create_user_with_profile( + username="other user", account=account, permissions=["iaso_polio"], org_units=[cls.other_country_org_unit] + ) + cls.json_config = { + "pipeline": "pipeline", + "openhexa_url": "https://yippie.openhexa.xyz/", + "openhexa_token": "token", + "pipeline_version": 1, + "oh_pipeline_target": "staging", + } + + def mock_openhexa_call_success(self, slug=None, config=None, id_field=None, task_id=None): + return SUCCESS + + def mock_openhexa_call_skipped(self, slug=None, config=None, id_field=None, task_id=None): + return SKIPPED + + def mock_openhexa_call_running(self, slug=None, config=None, id_field=None, task_id=None): + return RUNNING + + def test_no_perm(self): + user_no_perm = self.create_user_with_profile(username="test user2", account=self.account, permissions=[]) + self.client.force_authenticate(user_no_perm) + response = self.client.get( + self.url, + format="json", + ) + jr = self.assertJSONResponse(response, 403) + self.assertEqual({"detail": "You do not have permission to perform this action."}, jr) + + response = self.client.post( + self.url, + format="json", + data={ + "id_field": {"country_id": self.country_org_unit.id}, + "config": {"country_id": self.country_org_unit.id}, + "slug": IM_HH_OHH_CONFIG_SLUG, + }, + ) + jr = self.assertJSONResponse(response, 403) + self.assertEqual({"detail": "You do not have permission to perform this action."}, jr) + + @patch.object(RefreshIMAllDataViewset, "launch_task", mock_openhexa_call_running) + def test_create_external_task(self): + self.client.force_authenticate(self.user) + response = self.client.post( + self.url, + format="json", + data={ + "id_field": {"country_id": self.country_org_unit.id}, + "config": {"country_id": self.country_org_unit.id}, + "slug": IM_HH_OHH_CONFIG_SLUG, + }, + ) + response = self.assertJSONResponse(response, 200) + task = response["task"] + self.assertEqual(task["status"], RUNNING) + self.assertEqual(task["launcher"]["username"], self.user.username) + self.assertEqual(task["name"], f"{IM_HH_OHH_CONFIG_SLUG}-{self.country_org_unit.id}") + + @patch.object(RefreshIMAllDataViewset, "launch_task", mock_openhexa_call_skipped) + def test_create_external_task_pipeline_already_running(self): + self.client.force_authenticate(self.user) + response = self.client.post( + self.url, + format="json", + data={ + "id_field": {"country_id": self.country_org_unit.id}, + "config": {"country_id": self.country_org_unit.id}, + "slug": IM_HH_OHH_CONFIG_SLUG, + }, + ) + response = self.assertJSONResponse(response, 200) + task = response["task"] + self.assertEqual(task["status"], SKIPPED) + self.assertEqual(task["launcher"]["username"], self.user.username) + self.assertEqual(task["name"], f"{IM_HH_OHH_CONFIG_SLUG}-{self.country_org_unit.id}") + + @patch.object(RefreshIMAllDataViewset, "launch_task", mock_openhexa_call_running) + def test_patch_external_task(self): + self.client.force_authenticate(self.user) + response = self.client.post( + self.url, + format="json", + data={ + "id_field": {"country_id": self.country_org_unit.id}, + "config": {"country_id": self.country_org_unit.id}, + "slug": IM_HH_OHH_CONFIG_SLUG, + }, + ) + response = self.assertJSONResponse(response, 200) + task = response["task"] + task_id = task["id"] + self.assertEqual(task["status"], RUNNING) + self.assertEqual(task["progress_value"], 0) + self.assertEqual(task["end_value"], 0) + + response = self.client.patch( + f"{self.url}{task_id}/", format="json", data={"status": SUCCESS, "progress_value": 21} + ) + response = self.assertJSONResponse(response, 200) + task = response + self.assertEqual(task["id"], task_id) + self.assertEqual(task["status"], SUCCESS) + self.assertEqual(task["progress_value"], 21) + self.assertEqual(task["end_value"], 0) + + @patch.object(RefreshIMAllDataViewset, "launch_task", mock_openhexa_call_running) + def test_kill_external_task(self): + self.client.force_authenticate(self.user) + response = self.client.post( + self.url, + format="json", + data={ + "id_field": {"country_id": self.country_org_unit.id}, + "config": {"country_id": self.country_org_unit.id}, + "slug": IM_HH_OHH_CONFIG_SLUG, + }, + ) + response = self.assertJSONResponse(response, 200) + task = response["task"] + task_id = task["id"] + self.assertEqual(task["status"], RUNNING) + + response = self.client.patch( + f"{self.url}{task_id}/", + format="json", + data={ + "should_be_killed": True, + }, + ) + response = self.assertJSONResponse(response, 200) + task = response + self.assertEqual(task["id"], task_id) + self.assertEqual(task["status"], KILLED) + + def test_get_latest_for_country(self): + self.client.force_authenticate(self.user) + response = self.client.get(f"{self.action_url}?country_id={self.country_org_unit.id}") + response = self.assertJSONResponse(response, 200) + task = response["task"] + print("RESPONSE", response) + print("TASK", task) + self.assertEqual(task["id"], self.task1.id) + self.assertEqual(task["ended_at"], self.task1.ended_at) + self.assertEqual(task["status"], self.task1.status) + + def test_restrict_user_country(self): + self.client.force_authenticate(self.limited_user) + response = self.client.get(f"{self.action_url}?country_id={self.country_org_unit.id}") + response = self.assertJSONResponse(response, 400) + self.assertEqual(response, {"country_id": ["No authorised org unit found for user"]}) diff --git a/plugins/polio/tests/test_im_refresh_ohh_data.py b/plugins/polio/tests/test_im_refresh_ohh_data.py new file mode 100644 index 0000000000..19a790a6ab --- /dev/null +++ b/plugins/polio/tests/test_im_refresh_ohh_data.py @@ -0,0 +1,207 @@ +from datetime import datetime +from iaso import models as m +from iaso.test import APITestCase +from iaso.models.base import RUNNING, KILLED, SUCCESS, SKIPPED +from unittest.mock import patch +from plugins.polio.tasks.api.refresh_im_data import IM_OHH_CONFIG_SLUG, RefreshIMOutOfHouseholdDataViewset + + +class RefreshIMOutOfHouseholdDataTestCase(APITestCase): + @classmethod + def setUp(cls): + cls.url = "/api/polio/tasks/refreshim/ohh/" + cls.action_url = f"{cls.url}last_run_for_country/" + cls.account = account = m.Account.objects.create(name="test account") + cls.user = cls.create_user_with_profile(username="test user", account=account, permissions=["iaso_polio"]) + + cls.external_task1 = m.Task.objects.create( + status=RUNNING, account=account, launcher=cls.user, name="external task 1", external=True + ) + cls.external_task2 = m.Task.objects.create( + status=RUNNING, account=account, launcher=cls.user, name="external task 2", external=True + ) + cls.country = m.OrgUnitType.objects.create(name="Country", depth=1, category="COUNTRY") + cls.project = m.Project.objects.create(name="Polio", app_id="polio.rapid.outbreak.taskforce", account=account) + cls.data_source = m.DataSource.objects.create(name="Default source") + cls.data_source.projects.add(cls.project) + cls.data_source.save() + cls.source_version = m.SourceVersion.objects.create(data_source=cls.data_source, number=1) + cls.account.default_version = cls.source_version + cls.account.save() + cls.country_org_unit = m.OrgUnit.objects.create( + name="Country1", + validation_status=m.OrgUnit.VALIDATION_VALID, + source_ref="PvtAI4RUMkr", + org_unit_type=cls.country, + version=cls.source_version, + ) + cls.other_country_org_unit = m.OrgUnit.objects.create( + name="Country2", + validation_status=m.OrgUnit.VALIDATION_VALID, + source_ref="PvtAI4RUMkr", + org_unit_type=cls.country, + version=cls.source_version, + ) + cls.task1 = m.Task.objects.create( + status=RUNNING, + account=account, + launcher=cls.user, + name=IM_OHH_CONFIG_SLUG, + started_at=datetime.now(), + external=True, + ) + cls.task2 = m.Task.objects.create( + status=RUNNING, account=account, launcher=cls.user, name="task 2", external=True + ) + cls.limited_user = cls.create_user_with_profile( + username="other user", account=account, permissions=["iaso_polio"], org_units=[cls.other_country_org_unit] + ) + cls.json_config = { + "pipeline": "pipeline", + "openhexa_url": "https://yippie.openhexa.xyz/", + "openhexa_token": "token", + "pipeline_version": 1, + "oh_pipeline_target": "staging", + } + + def mock_openhexa_call_success(self, slug=None, config=None, id_field=None, task_id=None): + return SUCCESS + + def mock_openhexa_call_skipped(self, slug=None, config=None, id_field=None, task_id=None): + return SKIPPED + + def mock_openhexa_call_running(self, slug=None, config=None, id_field=None, task_id=None): + return RUNNING + + def test_no_perm(self): + user_no_perm = self.create_user_with_profile(username="test user2", account=self.account, permissions=[]) + self.client.force_authenticate(user_no_perm) + response = self.client.get( + self.url, + format="json", + ) + jr = self.assertJSONResponse(response, 403) + self.assertEqual({"detail": "You do not have permission to perform this action."}, jr) + + response = self.client.post( + self.url, + format="json", + data={ + "id_field": {"country_id": self.country_org_unit.id}, + "config": {"country_id": self.country_org_unit.id}, + "slug": IM_OHH_CONFIG_SLUG, + }, + ) + jr = self.assertJSONResponse(response, 403) + self.assertEqual({"detail": "You do not have permission to perform this action."}, jr) + + @patch.object(RefreshIMOutOfHouseholdDataViewset, "launch_task", mock_openhexa_call_running) + def test_create_external_task(self): + self.client.force_authenticate(self.user) + response = self.client.post( + self.url, + format="json", + data={ + "id_field": {"country_id": self.country_org_unit.id}, + "config": {"country_id": self.country_org_unit.id}, + "slug": IM_OHH_CONFIG_SLUG, + }, + ) + response = self.assertJSONResponse(response, 200) + task = response["task"] + self.assertEqual(task["status"], RUNNING) + self.assertEqual(task["launcher"]["username"], self.user.username) + self.assertEqual(task["name"], f"{IM_OHH_CONFIG_SLUG}-{self.country_org_unit.id}") + + @patch.object(RefreshIMOutOfHouseholdDataViewset, "launch_task", mock_openhexa_call_skipped) + def test_create_external_task_pipeline_already_running(self): + self.client.force_authenticate(self.user) + response = self.client.post( + self.url, + format="json", + data={ + "id_field": {"country_id": self.country_org_unit.id}, + "config": {"country_id": self.country_org_unit.id}, + "slug": IM_OHH_CONFIG_SLUG, + }, + ) + response = self.assertJSONResponse(response, 200) + task = response["task"] + self.assertEqual(task["status"], SKIPPED) + self.assertEqual(task["launcher"]["username"], self.user.username) + self.assertEqual(task["name"], f"{IM_OHH_CONFIG_SLUG}-{self.country_org_unit.id}") + + @patch.object(RefreshIMOutOfHouseholdDataViewset, "launch_task", mock_openhexa_call_running) + def test_patch_external_task(self): + self.client.force_authenticate(self.user) + response = self.client.post( + self.url, + format="json", + data={ + "id_field": {"country_id": self.country_org_unit.id}, + "config": {"country_id": self.country_org_unit.id}, + "slug": IM_OHH_CONFIG_SLUG, + }, + ) + response = self.assertJSONResponse(response, 200) + task = response["task"] + task_id = task["id"] + self.assertEqual(task["status"], RUNNING) + self.assertEqual(task["progress_value"], 0) + self.assertEqual(task["end_value"], 0) + + response = self.client.patch( + f"{self.url}{task_id}/", format="json", data={"status": SUCCESS, "progress_value": 21} + ) + response = self.assertJSONResponse(response, 200) + task = response + self.assertEqual(task["id"], task_id) + self.assertEqual(task["status"], SUCCESS) + self.assertEqual(task["progress_value"], 21) + self.assertEqual(task["end_value"], 0) + + @patch.object(RefreshIMOutOfHouseholdDataViewset, "launch_task", mock_openhexa_call_running) + def test_kill_external_task(self): + self.client.force_authenticate(self.user) + response = self.client.post( + self.url, + format="json", + data={ + "id_field": {"country_id": self.country_org_unit.id}, + "config": {"country_id": self.country_org_unit.id}, + "slug": IM_OHH_CONFIG_SLUG, + }, + ) + response = self.assertJSONResponse(response, 200) + task = response["task"] + task_id = task["id"] + self.assertEqual(task["status"], RUNNING) + + response = self.client.patch( + f"{self.url}{task_id}/", + format="json", + data={ + "should_be_killed": True, + }, + ) + response = self.assertJSONResponse(response, 200) + task = response + self.assertEqual(task["id"], task_id) + self.assertEqual(task["status"], KILLED) + + def test_get_latest_for_country(self): + self.client.force_authenticate(self.user) + response = self.client.get(f"{self.action_url}?country_id={self.country_org_unit.id}") + response = self.assertJSONResponse(response, 200) + task = response["task"] + print("RESPONSE", response) + print("TASK", task) + self.assertEqual(task["id"], self.task1.id) + self.assertEqual(task["ended_at"], self.task1.ended_at) + self.assertEqual(task["status"], self.task1.status) + + def test_restrict_user_country(self): + self.client.force_authenticate(self.limited_user) + response = self.client.get(f"{self.action_url}?country_id={self.country_org_unit.id}") + response = self.assertJSONResponse(response, 400) + self.assertEqual(response, {"country_id": ["No authorised org unit found for user"]}) From 5ee294252ae5ede284405ec572b5b86dcda02161 Mon Sep 17 00:00:00 2001 From: Quang Son Le Date: Mon, 26 Aug 2024 16:06:53 +0200 Subject: [PATCH 08/55] POLIO-1610: remove unused logger --- plugins/polio/tasks/api/refresh_im_data.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/plugins/polio/tasks/api/refresh_im_data.py b/plugins/polio/tasks/api/refresh_im_data.py index 01bc0cfd65..273e1550e0 100644 --- a/plugins/polio/tasks/api/refresh_im_data.py +++ b/plugins/polio/tasks/api/refresh_im_data.py @@ -1,8 +1,4 @@ from plugins.polio.tasks.api.refresh_lqas_im_data import IM_TASK_NAME, RefreshLQASIMDataViewset -import logging - - -logger = logging.getLogger(__name__) IM_HH_CONFIG_SLUG = "im_hh-pipeline-config" IM_OHH_CONFIG_SLUG = "im_ohh-pipeline-config" From bfd57fb33a89765a84bc3b90d9fbfc0c4ec44fd7 Mon Sep 17 00:00:00 2001 From: Beygorghor Date: Mon, 26 Aug 2024 16:37:26 +0200 Subject: [PATCH 09/55] adding default image to model and api --- .../Iaso/domains/instances/types/instance.ts | 45 ++++++++++++++++++- iaso/admin.py | 30 ++++++------- iaso/api/instances.py | 15 ++++++- iaso/api/serializers.py | 3 +- iaso/migrations/0296_orgunit_default_image.py | 24 ++++++++++ iaso/models/base.py | 21 ++++++++- iaso/models/org_unit.py | 7 ++- 7 files changed, 123 insertions(+), 22 deletions(-) create mode 100644 iaso/migrations/0296_orgunit_default_image.py diff --git a/hat/assets/js/apps/Iaso/domains/instances/types/instance.ts b/hat/assets/js/apps/Iaso/domains/instances/types/instance.ts index 2400eb9344..a1973e2579 100644 --- a/hat/assets/js/apps/Iaso/domains/instances/types/instance.ts +++ b/hat/assets/js/apps/Iaso/domains/instances/types/instance.ts @@ -1,8 +1,8 @@ /* eslint-disable camelcase */ import { Pagination } from 'bluesquare-components'; -import { OrgUnit } from '../../orgUnits/types/orgUnit'; import { User } from '../../../utils/usersUtils'; import { Beneficiary } from '../../entities/types/beneficiary'; +import { OrgUnit } from '../../orgUnits/types/orgUnit'; type Lock = { id: number; @@ -89,3 +89,46 @@ export type FileContent = { export interface PaginatedInstances extends Pagination { instances: Instance[]; } +export type MimeType = + // Text + | 'text/plain' + | 'text/html' + | 'text/css' + | 'text/javascript' + // Image + | 'image/jpeg' + | 'image/png' + | 'image/gif' + | 'image/svg+xml' + | 'image/webp' + // Audio + | 'audio/mpeg' + | 'audio/ogg' + | 'audio/wav' + // Video + | 'video/mp4' + | 'video/mpeg' + | 'video/webm' + | 'video/ogg' + // Application + | 'application/json' + | 'application/xml' + | 'application/zip' + | 'application/pdf' + | 'application/sql' + | 'application/graphql' + | 'application/ld+json' + | 'application/msword' + | 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' + | 'application/vnd.ms-excel' + | 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' + | 'application/vnd.ms-powerpoint' + | 'application/vnd.openxmlformats-officedocument.presentationml.presentation' + // Font + | 'font/ttf' + | 'font/woff' + | 'font/woff2' + // Other + | 'application/octet-stream' + | 'multipart/form-data' + | 'application/x-www-form-urlencoded'; diff --git a/iaso/admin.py b/iaso/admin.py index 4d0ef80561..a367ed7d4b 100644 --- a/iaso/admin.py +++ b/iaso/admin.py @@ -1,5 +1,4 @@ -from typing import Any -from typing import Protocol +from typing import Any, Protocol from django import forms as django_forms from django.contrib.admin import widgets @@ -7,11 +6,12 @@ from django.contrib.gis.db import models as geomodels from django.contrib.postgres.fields import ArrayField from django.db import models -from django.utils.html import format_html_join, format_html +from django.utils.html import format_html, format_html_join from django.utils.safestring import mark_safe from django_json_widget.widgets import JSONEditorWidget from iaso.models.json_config import Config # type: ignore + from .models import ( Account, AccountFeatureFlag, @@ -45,8 +45,12 @@ MatchingAlgorithm, OrgUnit, OrgUnitChangeRequest, + OrgUnitReferenceInstance, OrgUnitType, Page, + Payment, + PaymentLot, + PotentialPayment, Profile, Project, Report, @@ -61,13 +65,9 @@ WorkflowChange, WorkflowFollowup, WorkflowVersion, - OrgUnitReferenceInstance, - PotentialPayment, - Payment, - PaymentLot, ) from .models.data_store import JsonDataStore -from .models.microplanning import Team, Planning, Assignment +from .models.microplanning import Assignment, Planning, Team from .utils.gis import convert_2d_point_to_3d @@ -145,7 +145,7 @@ def has_add_permission(self, request, obj=None): @admin.register(OrgUnit) @admin_attr_decorator class OrgUnitAdmin(admin.GeoModelAdmin): - raw_id_fields = ("parent", "reference_instances") + raw_id_fields = ("parent", "reference_instances", "default_image") list_filter = ("org_unit_type", "custom", "validated", "sub_source", "version") search_fields = ("name", "source_ref", "uuid") readonly_fields = ("path",) @@ -454,9 +454,9 @@ class EntityAdmin(admin.ModelAdmin): def get_form(self, request, obj=None, **kwargs): # In the for the entity type, we also want to indicate the account name form = super().get_form(request, obj, **kwargs) - form.base_fields[ - "entity_type" - ].label_from_instance = lambda entity: f"{entity.name} (Account: {entity.account.name})" + form.base_fields["entity_type"].label_from_instance = ( + lambda entity: f"{entity.name} (Account: {entity.account.name})" + ) return form def get_queryset(self, request): diff --git a/iaso/api/instances.py b/iaso/api/instances.py index c78a7ae626..a61934e320 100644 --- a/iaso/api/instances.py +++ b/iaso/api/instances.py @@ -1,4 +1,5 @@ import json +import mimetypes import ntpath from time import gmtime, strftime from typing import Any, Dict, Union @@ -6,9 +7,10 @@ import pandas as pd from django.contrib.auth.models import User from django.contrib.gis.geos import Point +from django.core.files.storage import default_storage from django.core.paginator import Paginator from django.db import connection, transaction -from django.db.models import Count, Q, QuerySet, Prefetch +from django.db.models import Count, Prefetch, Q, QuerySet from django.http import HttpResponse, StreamingHttpResponse from django.utils.timezone import now from rest_framework import permissions, serializers, status, viewsets @@ -37,11 +39,11 @@ from iaso.utils import timestamp_to_datetime from ..models.forms import CR_MODE_IF_REFERENCE_FORM +from ..utils.models.common import get_creator_name from . import common from .comment import UserSerializerForComment from .common import CONTENT_TYPE_CSV, CONTENT_TYPE_XLSX, FileFormatEnum, TimestampField, safe_api_import from .instance_filters import get_form_from_instance_filters, parse_instance_filters -from ..utils.models.common import get_creator_name class InstanceSerializer(serializers.ModelSerializer): @@ -101,9 +103,18 @@ def has_object_permission(self, request: Request, view, obj: Instance): class InstanceFileSerializer(serializers.Serializer): + id = serializers.IntegerField(read_only=True) instance_id = serializers.IntegerField() file = serializers.FileField(use_url=True) created_at = TimestampField(read_only=True) + file_type = serializers.SerializerMethodField() + + def get_file_type(self, obj): + if obj.file: + file_path = default_storage.path(obj.file.name) + mime_type, _ = mimetypes.guess_type(file_path) + return mime_type + return None class OrgUnitNestedSerializer(OrgUnitSerializer): diff --git a/iaso/api/serializers.py b/iaso/api/serializers.py index 76ef63094e..4b1b74a2c0 100644 --- a/iaso/api/serializers.py +++ b/iaso/api/serializers.py @@ -4,7 +4,7 @@ from iaso.api.common import TimestampField from iaso.api.query_params import APP_ID -from iaso.models import OrgUnit, OrgUnitType, Group +from iaso.models import Group, OrgUnit, OrgUnitType class TimestampSerializerMixin: @@ -223,6 +223,7 @@ class Meta: "groups", "creator", "projects", + "default_image", ] diff --git a/iaso/migrations/0296_orgunit_default_image.py b/iaso/migrations/0296_orgunit_default_image.py new file mode 100644 index 0000000000..06a746f72b --- /dev/null +++ b/iaso/migrations/0296_orgunit_default_image.py @@ -0,0 +1,24 @@ +# Generated by Django 4.2.11 on 2024-08-26 12:52 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ("iaso", "0295_merge_20240819_1249"), + ] + + operations = [ + migrations.AddField( + model_name="orgunit", + name="default_image", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="default_for_org_units", + to="iaso.instancefile", + ), + ), + ] diff --git a/iaso/models/base.py b/iaso/models/base.py index 6d96833b80..5d570ec16e 100644 --- a/iaso/models/base.py +++ b/iaso/models/base.py @@ -1,4 +1,5 @@ import datetime +import mimetypes import operator import random import re @@ -22,6 +23,7 @@ from django.contrib.postgres.aggregates import ArrayAgg from django.contrib.postgres.fields import ArrayField from django.core.exceptions import ObjectDoesNotExist, ValidationError +from django.core.files.storage import default_storage from django.core.paginator import Paginator from django.core.validators import MinLengthValidator from django.db import models @@ -1120,7 +1122,8 @@ def get_form_version(self): def export(self, launcher=None, force_export=False): from iaso.dhis2.datavalue_exporter import DataValueExporter - from iaso.dhis2.export_request_builder import ExportRequestBuilder, NothingToExportError + from iaso.dhis2.export_request_builder import (ExportRequestBuilder, + NothingToExportError) try: export_request = ExportRequestBuilder().build_export_request( @@ -1348,6 +1351,22 @@ class InstanceFile(models.Model): def __str__(self): return "%s " % (self.name,) + def as_dict(self): + return { + "id": self.id, + "instance_id": self.instance_id, + "file": self.file.url if self.file else None, + "created_at": self.created_at.timestamp() if self.created_at else None, + "file_type": self.get_file_type(), + } + + def get_file_type(self): + if self.file: + file_path = default_storage.path(self.file.name) + mime_type, _ = mimetypes.guess_type(file_path) + return mime_type + return None + class Profile(models.Model): account = models.ForeignKey(Account, on_delete=models.CASCADE) diff --git a/iaso/models/org_unit.py b/iaso/models/org_unit.py index 5945dae085..2754927af1 100644 --- a/iaso/models/org_unit.py +++ b/iaso/models/org_unit.py @@ -14,7 +14,6 @@ from django.db import models, transaction from django.db.models import Q, QuerySet from django.db.models.expressions import RawSQL -from django.utils import timezone from django.utils.translation import gettext_lazy as _ from django_ltree.fields import PathField # type: ignore from django_ltree.models import TreeModel # type: ignore @@ -27,7 +26,7 @@ from .project import Project try: # for typing - from .base import Account, Instance + from .base import Account except: pass @@ -301,6 +300,9 @@ class OrgUnit(TreeModel): opening_date = models.DateField(blank=True, null=True) # Start date of activities of the organisation unit closed_date = models.DateField(blank=True, null=True) # End date of activities of the organisation unit objects = OrgUnitManager.from_queryset(OrgUnitQuerySet)() # type: ignore + default_image: "InstanceFile" = models.ForeignKey( + "InstanceFile", on_delete=models.SET_NULL, null=True, blank=True, related_name="default_for_org_units" + ) class Meta: indexes = [ @@ -471,6 +473,7 @@ def as_dict_with_parents(self, light=False, light_parents=True): "creator": get_creator_name(self.creator), "opening_date": self.opening_date.strftime("%d/%m/%Y") if self.opening_date else None, "closed_date": self.closed_date.strftime("%d/%m/%Y") if self.closed_date else None, + "default_image": self.default_image.as_dict() if self.default_image else None, } if not light: # avoiding joins here res["groups"] = [group.as_dict(with_counts=False) for group in self.groups.all()] From 6b818fa71a3924c7dbfa5c85ec0c565a9eaaf702 Mon Sep 17 00:00:00 2001 From: Quang Son Le Date: Mon, 26 Aug 2024 16:48:30 +0200 Subject: [PATCH 10/55] POLIO-1610: update front-end - add specific endpoints for each im type --- plugins/polio/js/src/constants/types.ts | 2 +- .../js/src/domains/LQAS-IM/shared/Filters.tsx | 8 +-- .../LQAS-IM/shared/RefreshLqasIMData.tsx | 50 ++++++++++++++++--- .../api/useGetLqasImCountriesOptions.tsx | 17 +++---- .../js/src/hooks/useGetLatestLQASIMUpdate.ts | 28 ++++++++--- 5 files changed, 76 insertions(+), 29 deletions(-) diff --git a/plugins/polio/js/src/constants/types.ts b/plugins/polio/js/src/constants/types.ts index 6a4f39a49e..8e84c63da6 100644 --- a/plugins/polio/js/src/constants/types.ts +++ b/plugins/polio/js/src/constants/types.ts @@ -67,7 +67,7 @@ export type ConvertedLqasImData = { rounds: { number: number; data: LqasImDistrictDataWithNameAndRegion[] }[]; }; -export type IMType = 'imGlobal' | 'imIHH' | 'imOHH'; +export type IMType = 'imGlobal' | 'imHH' | 'imOHH'; export type LqasIMtype = IMType | 'lqas'; diff --git a/plugins/polio/js/src/domains/LQAS-IM/shared/Filters.tsx b/plugins/polio/js/src/domains/LQAS-IM/shared/Filters.tsx index 42587500c2..ee945bd7ba 100644 --- a/plugins/polio/js/src/domains/LQAS-IM/shared/Filters.tsx +++ b/plugins/polio/js/src/domains/LQAS-IM/shared/Filters.tsx @@ -20,6 +20,7 @@ import { useGetLqasImCountriesOptions } from './hooks/api/useGetLqasImCountriesO import { RefreshLqasIMData } from './RefreshLqasIMData'; import { baseUrls } from '../../../constants/urls'; import { POLIO_ADMIN } from '../../../constants/permissions'; +import { IMType } from '../../../constants/types'; export type Params = { campaign: string | undefined; @@ -37,7 +38,7 @@ type Props = { campaigns: any[]; campaignsFetching: boolean; params: Params; - imType?: 'imGlobal' | 'imHH' | 'imOHH'; + imType?: IMType; }; const getCurrentUrl = (imType?: 'imGlobal' | 'imHH' | 'imOHH'): string => { @@ -62,7 +63,6 @@ export const Filters: FunctionComponent = ({ }) => { const { formatMessage } = useSafeIntl(); const redirectToReplace = useRedirectToReplace(); - const isLqas = !imType; const currentUrl = getCurrentUrl(imType); const [filters, setFilters] = useState({ campaign: params?.campaign, @@ -71,7 +71,7 @@ export const Filters: FunctionComponent = ({ const { campaign, country } = params; const { data: countriesOptions, isFetching: countriesLoading } = - useGetLqasImCountriesOptions(isLqas); + useGetLqasImCountriesOptions(); const dropDownOptions = useMemo(() => { const displayedCampaigns = country @@ -146,7 +146,7 @@ export const Filters: FunctionComponent = ({ diff --git a/plugins/polio/js/src/domains/LQAS-IM/shared/RefreshLqasIMData.tsx b/plugins/polio/js/src/domains/LQAS-IM/shared/RefreshLqasIMData.tsx index 7b36ea00d5..8f86c07fb3 100644 --- a/plugins/polio/js/src/domains/LQAS-IM/shared/RefreshLqasIMData.tsx +++ b/plugins/polio/js/src/domains/LQAS-IM/shared/RefreshLqasIMData.tsx @@ -13,16 +13,50 @@ import { Task } from '../../../../../../../hat/assets/js/apps/Iaso/domains/tasks import { useGetLatestLQASIMUpdate } from '../../../hooks/useGetLatestLQASIMUpdate'; import { useCreateTask } from '../../../../../../../hat/assets/js/apps/Iaso/hooks/taskMonitor'; import MESSAGES from '../../../constants/messages'; +import { IMType } from '../../../constants/types'; type Props = { countryId?: string; - isLqas: boolean; + imType?: IMType; }; const LQAS_TASK_ENDPOINT = '/api/polio/tasks/refreshlqas/'; const LQAS_CONFIG_SLUG = 'lqas-pipeline-config'; -const IM_TASK_ENDPOINT = '/api/polio/tasks/refreshim/'; -const IM_CONFIG_SLUG = 'im-pipeline-config'; +const IM_HH_TASK_ENDPOINT = '/api/polio/tasks/refreshim/hh/'; +const IM_HH_CONFIG_SLUG = 'im_hh-pipeline-config'; +const IM_OHH_TASK_ENDPOINT = '/api/polio/tasks/refreshim/ohh/'; +const IM_OHH_CONFIG_SLUG = 'im_ohh-pipeline-config'; +const IM_HH_OHH_TASK_ENDPOINT = '/api/polio/tasks/refreshim/hh_ohh/'; +const IM_HH_OHH_CONFIG_SLUG = 'im_hh_ohh-pipeline-config'; + +const getImEndpoint = (imType: IMType): string => { + switch (imType) { + case 'imHH': + return IM_HH_TASK_ENDPOINT; + case 'imOHH': + return IM_OHH_TASK_ENDPOINT; + case 'imGlobal': + return IM_HH_OHH_TASK_ENDPOINT; + default: + console.warn( + `Expected IM type to be "imHH", "imOHH" or "imGlobal", got ${imType}`, + ); + return ''; + } +}; + +const getImConfigSlug = (imType: IMType): string => { + switch (imType) { + case 'imHH': + return IM_HH_CONFIG_SLUG; + case 'imOHH': + return IM_OHH_CONFIG_SLUG; + case 'imGlobal': + return IM_HH_OHH_CONFIG_SLUG; + default: + return ''; + } +}; const useLastUpdate = ( lastUpdate: Task, @@ -50,10 +84,10 @@ const useLastUpdate = ( export const RefreshLqasIMData: FunctionComponent = ({ countryId, - isLqas, + imType, }) => { - const taskUrl = isLqas ? LQAS_TASK_ENDPOINT : IM_TASK_ENDPOINT; - const slug = isLqas ? LQAS_CONFIG_SLUG : IM_CONFIG_SLUG; + const taskUrl = !imType ? LQAS_TASK_ENDPOINT : getImEndpoint(imType); + const slug = !imType ? LQAS_CONFIG_SLUG : getImConfigSlug(imType); const { formatMessage } = useSafeIntl(); const [lastTaskStatus, setlastTaskStatus] = useState(); const queryClient = useQueryClient(); @@ -61,8 +95,8 @@ export const RefreshLqasIMData: FunctionComponent = ({ endpoint: taskUrl, }); const { data: latestManualRefresh } = useGetLatestLQASIMUpdate( - isLqas, countryId, + imType, ); const { message: lastUpdate, updateStatus } = @@ -87,7 +121,7 @@ export const RefreshLqasIMData: FunctionComponent = ({ }, [lastTaskStatus, latestManualRefresh?.status, queryClient]); const disableButton = Boolean(latestManualRefresh?.status === 'RUNNING'); // TODO make enum with statuses - const buttonText = isLqas + const buttonText = !imType ? formatMessage(MESSAGES.refreshLqasData) : formatMessage(MESSAGES.refreshIMData); if (!countryId) return null; diff --git a/plugins/polio/js/src/domains/LQAS-IM/shared/hooks/api/useGetLqasImCountriesOptions.tsx b/plugins/polio/js/src/domains/LQAS-IM/shared/hooks/api/useGetLqasImCountriesOptions.tsx index 2b67f829fe..4eedf42905 100644 --- a/plugins/polio/js/src/domains/LQAS-IM/shared/hooks/api/useGetLqasImCountriesOptions.tsx +++ b/plugins/polio/js/src/domains/LQAS-IM/shared/hooks/api/useGetLqasImCountriesOptions.tsx @@ -3,19 +3,16 @@ import { useSnackQuery } from '../../../../../../../../../hat/assets/js/apps/Ias import { getRequest } from '../../../../../../../../../hat/assets/js/apps/Iaso/libs/Api'; import { DropdownOptions } from '../../../../../../../../../hat/assets/js/apps/Iaso/types/utils'; -const getLqasImCountriesOptions = (isLqas: boolean) => { - const category = isLqas ? 'lqas' : 'im'; - return getRequest( - `/api/polio/lqasim/countries/?category=${category}&order=name`, - ); +const getLqasImCountriesOptions = () => { + return getRequest(`/api/polio/lqasim/countries/?order=name`); }; -export const useGetLqasImCountriesOptions = ( - isLqas: boolean, -): UseQueryResult[]> => { +export const useGetLqasImCountriesOptions = (): UseQueryResult< + DropdownOptions[] +> => { return useSnackQuery({ - queryKey: ['lqasimcountries', isLqas], - queryFn: () => getLqasImCountriesOptions(isLqas), + queryKey: ['lqasimcountries'], + queryFn: () => getLqasImCountriesOptions(), options: { select: data => (data?.results ?? []).map(result => ({ diff --git a/plugins/polio/js/src/hooks/useGetLatestLQASIMUpdate.ts b/plugins/polio/js/src/hooks/useGetLatestLQASIMUpdate.ts index 6c63be27b8..58bb9bba32 100644 --- a/plugins/polio/js/src/hooks/useGetLatestLQASIMUpdate.ts +++ b/plugins/polio/js/src/hooks/useGetLatestLQASIMUpdate.ts @@ -2,12 +2,28 @@ import { UseQueryResult } from 'react-query'; import { useSnackQuery } from '../../../../../hat/assets/js/apps/Iaso/libs/apiHooks'; import { getRequest } from '../../../../../hat/assets/js/apps/Iaso/libs/Api'; import { Optional } from '../../../../../hat/assets/js/apps/Iaso/types/utils'; +import { IMType } from '../constants/types'; const lqasEndpoint = '/api/polio/tasks/refreshlqas/last_run_for_country/'; -const imEndpoint = '/api/polio/tasks/refreshim/last_run_for_country/'; +const imHHEndpoint = '/api/polio/tasks/refreshim/last_run_for_country/'; +const imOHHEndpoint = '/api/polio/tasks/refreshim/last_run_for_country/'; +const imGlobalEndpoint = '/api/polio/tasks/refreshim/last_run_for_country/'; -const getLatestRefresh = (isLqas: boolean, countryId: Optional) => { - const endpoint = isLqas ? lqasEndpoint : imEndpoint; +const getImEndpoint = (imType: IMType): string => { + switch (imType) { + case 'imHH': + return imHHEndpoint; + case 'imOHH': + return imOHHEndpoint; + case 'imGlobal': + return imGlobalEndpoint; + default: + return ''; + } +}; + +const getLatestRefresh = (countryId: Optional, imType?: IMType) => { + const endpoint = !imType ? lqasEndpoint : getImEndpoint(imType); const url = countryId ? `${endpoint}?country_id=${countryId}` : endpoint; if (countryId !== undefined) { return getRequest(url); @@ -16,12 +32,12 @@ const getLatestRefresh = (isLqas: boolean, countryId: Optional) => { }; export const useGetLatestLQASIMUpdate = ( - isLqas: boolean, countryId: Optional, + imType?: IMType, ): UseQueryResult => { return useSnackQuery({ - queryKey: ['get-latest-task-run', isLqas, countryId], - queryFn: () => getLatestRefresh(isLqas, countryId), + queryKey: ['get-latest-task-run', imType, countryId], + queryFn: () => getLatestRefresh(countryId, imType), options: { select: data => data?.task ?? {}, refetchInterval: 5000, From 3a60904711d8f5aab7c3852c7892b75ca36ab80a Mon Sep 17 00:00:00 2001 From: Quang Son Le Date: Mon, 26 Aug 2024 16:55:31 +0200 Subject: [PATCH 11/55] POLIO-1610: fix url typo --- plugins/polio/js/src/hooks/useGetLatestLQASIMUpdate.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/plugins/polio/js/src/hooks/useGetLatestLQASIMUpdate.ts b/plugins/polio/js/src/hooks/useGetLatestLQASIMUpdate.ts index 58bb9bba32..3a0a5720db 100644 --- a/plugins/polio/js/src/hooks/useGetLatestLQASIMUpdate.ts +++ b/plugins/polio/js/src/hooks/useGetLatestLQASIMUpdate.ts @@ -5,9 +5,10 @@ import { Optional } from '../../../../../hat/assets/js/apps/Iaso/types/utils'; import { IMType } from '../constants/types'; const lqasEndpoint = '/api/polio/tasks/refreshlqas/last_run_for_country/'; -const imHHEndpoint = '/api/polio/tasks/refreshim/last_run_for_country/'; -const imOHHEndpoint = '/api/polio/tasks/refreshim/last_run_for_country/'; -const imGlobalEndpoint = '/api/polio/tasks/refreshim/last_run_for_country/'; +const imHHEndpoint = '/api/polio/tasks/refreshim/hh/last_run_for_country/'; +const imOHHEndpoint = '/api/polio/tasks/refreshim/ohh/last_run_for_country/'; +const imGlobalEndpoint = + '/api/polio/tasks/refreshim/hh_ohh/last_run_for_country/'; const getImEndpoint = (imType: IMType): string => { switch (imType) { From b6abfa72bf74e37c1122c59ad0813cf7761a5c7a Mon Sep 17 00:00:00 2001 From: Christophe Gerard Date: Wed, 28 Aug 2024 13:42:00 +0200 Subject: [PATCH 12/55] move file mime type function --- iaso/api/instances.py | 7 ++----- iaso/models/base.py | 13 +++---------- iaso/utils/file_utils.py | 11 +++++++++++ 3 files changed, 16 insertions(+), 15 deletions(-) create mode 100644 iaso/utils/file_utils.py diff --git a/iaso/api/instances.py b/iaso/api/instances.py index 98d62872fb..89bfa204b9 100644 --- a/iaso/api/instances.py +++ b/iaso/api/instances.py @@ -37,6 +37,7 @@ Project, ) from iaso.utils import timestamp_to_datetime +from iaso.utils.file_utils import get_file_type from ..models.forms import CR_MODE_IF_REFERENCE_FORM from ..utils.models.common import get_creator_name @@ -110,11 +111,7 @@ class InstanceFileSerializer(serializers.Serializer): file_type = serializers.SerializerMethodField() def get_file_type(self, obj): - if obj.file: - file_path = default_storage.path(obj.file.name) - mime_type, _ = mimetypes.guess_type(file_path) - return mime_type - return None + return get_file_type(obj.file) class OrgUnitNestedSerializer(OrgUnitSerializer): diff --git a/iaso/models/base.py b/iaso/models/base.py index e4db2a4906..840dfe2f89 100644 --- a/iaso/models/base.py +++ b/iaso/models/base.py @@ -40,6 +40,7 @@ from iaso.models.data_source import DataSource, SourceVersion from iaso.models.org_unit import OrgUnit, OrgUnitReferenceInstance from iaso.utils import extract_form_version_id, flat_parse_xml_soup +from iaso.utils.file_utils import get_file_type from .. import periods from ..utils.jsonlogic import jsonlogic_to_q @@ -1121,8 +1122,7 @@ def get_form_version(self): def export(self, launcher=None, force_export=False): from iaso.dhis2.datavalue_exporter import DataValueExporter - from iaso.dhis2.export_request_builder import (ExportRequestBuilder, - NothingToExportError) + from iaso.dhis2.export_request_builder import ExportRequestBuilder, NothingToExportError try: export_request = ExportRequestBuilder().build_export_request( @@ -1337,16 +1337,9 @@ def as_dict(self): "instance_id": self.instance_id, "file": self.file.url if self.file else None, "created_at": self.created_at.timestamp() if self.created_at else None, - "file_type": self.get_file_type(), + "file_type": get_file_type(self.file), } - def get_file_type(self): - if self.file: - file_path = default_storage.path(self.file.name) - mime_type, _ = mimetypes.guess_type(file_path) - return mime_type - return None - class Profile(models.Model): account = models.ForeignKey(Account, on_delete=models.CASCADE) diff --git a/iaso/utils/file_utils.py b/iaso/utils/file_utils.py new file mode 100644 index 0000000000..f375f7bc6f --- /dev/null +++ b/iaso/utils/file_utils.py @@ -0,0 +1,11 @@ +import mimetypes + +from django.core.files.storage import default_storage + + +def get_file_type(file): + if file: + file_path = default_storage.path(file.name) + mime_type, _ = mimetypes.guess_type(file_path) + return mime_type + return None From f722046e1016be26a76a87be2a5063554730f3b2 Mon Sep 17 00:00:00 2001 From: Quang Son Le Date: Wed, 28 Aug 2024 15:02:13 +0200 Subject: [PATCH 13/55] POLIO-1610: pass imType to API --- .../js/src/domains/LQAS-IM/shared/RefreshLqasIMData.tsx | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/plugins/polio/js/src/domains/LQAS-IM/shared/RefreshLqasIMData.tsx b/plugins/polio/js/src/domains/LQAS-IM/shared/RefreshLqasIMData.tsx index 8f86c07fb3..ecbd9c672b 100644 --- a/plugins/polio/js/src/domains/LQAS-IM/shared/RefreshLqasIMData.tsx +++ b/plugins/polio/js/src/domains/LQAS-IM/shared/RefreshLqasIMData.tsx @@ -102,14 +102,19 @@ export const RefreshLqasIMData: FunctionComponent = ({ const { message: lastUpdate, updateStatus } = useLastUpdate(latestManualRefresh); const launchRefresh = useCallback(() => { + const config: any = {}; + if (imType) { + config.im_type = imType; + } if (countryId) { + config.country_id = parseInt(countryId, 10); createRefreshTask({ - config: { country_id: parseInt(countryId, 10) }, + config, slug, id_field: { country_id: parseInt(countryId, 10) }, }); } - }, [countryId, createRefreshTask, slug]); + }, [countryId, createRefreshTask, slug, imType]); useEffect(() => { if (lastTaskStatus !== latestManualRefresh?.status) { From 9e8c72a5d227b8498210d974e431ddb101acd81f Mon Sep 17 00:00:00 2001 From: Quang Son Le Date: Wed, 28 Aug 2024 15:16:24 +0200 Subject: [PATCH 14/55] POLIO-1610: format pipeline config correctly --- .../LQAS-IM/shared/RefreshLqasIMData.tsx | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/plugins/polio/js/src/domains/LQAS-IM/shared/RefreshLqasIMData.tsx b/plugins/polio/js/src/domains/LQAS-IM/shared/RefreshLqasIMData.tsx index ecbd9c672b..2c65e5867b 100644 --- a/plugins/polio/js/src/domains/LQAS-IM/shared/RefreshLqasIMData.tsx +++ b/plugins/polio/js/src/domains/LQAS-IM/shared/RefreshLqasIMData.tsx @@ -28,6 +28,9 @@ const IM_OHH_TASK_ENDPOINT = '/api/polio/tasks/refreshim/ohh/'; const IM_OHH_CONFIG_SLUG = 'im_ohh-pipeline-config'; const IM_HH_OHH_TASK_ENDPOINT = '/api/polio/tasks/refreshim/hh_ohh/'; const IM_HH_OHH_CONFIG_SLUG = 'im_hh_ohh-pipeline-config'; +const IM_HH_API_PARAM = 'HH'; +const IM_OHH_API_PARAM = 'OHH'; +const IM_HH_OHH_API_PARAM = 'HH_OHH'; const getImEndpoint = (imType: IMType): string => { switch (imType) { @@ -58,6 +61,19 @@ const getImConfigSlug = (imType: IMType): string => { } }; +const getPipelineParam = (imType: IMType): string => { + switch (imType) { + case 'imHH': + return IM_HH_API_PARAM; + case 'imOHH': + return IM_OHH_API_PARAM; + case 'imGlobal': + return IM_HH_OHH_API_PARAM; + default: + return ''; + } +}; + const useLastUpdate = ( lastUpdate: Task, ): { message: string; updateStatus: string } => { @@ -104,7 +120,7 @@ export const RefreshLqasIMData: FunctionComponent = ({ const launchRefresh = useCallback(() => { const config: any = {}; if (imType) { - config.im_type = imType; + config.im_type = getPipelineParam(imType); } if (countryId) { config.country_id = parseInt(countryId, 10); From 0a81f856c36f3b00e19dfa49d9d127a296212454 Mon Sep 17 00:00:00 2001 From: Christophe Gerard Date: Wed, 28 Aug 2024 15:24:39 +0200 Subject: [PATCH 15/55] wip on table --- hat/assets/js/apps/Iaso/constants/urls.ts | 9 +-- .../domains/forms/components/FilesTable.tsx | 56 +++++++++++++++++++ .../js/apps/Iaso/domains/orgUnits/details.js | 25 ++++++++- .../js/apps/Iaso/domains/orgUnits/messages.ts | 4 ++ 4 files changed, 89 insertions(+), 5 deletions(-) create mode 100644 hat/assets/js/apps/Iaso/domains/forms/components/FilesTable.tsx diff --git a/hat/assets/js/apps/Iaso/constants/urls.ts b/hat/assets/js/apps/Iaso/constants/urls.ts index 5fcda03c5b..84edafd1f2 100644 --- a/hat/assets/js/apps/Iaso/constants/urls.ts +++ b/hat/assets/js/apps/Iaso/constants/urls.ts @@ -5,6 +5,7 @@ export const paginationPathParamsWithPrefix = (prefix: string): string[] => paginationPathParams.map(p => `${prefix}${capitalize(p, true)}`); export const FORMS_PREFIX = 'formsParams'; +export const FILES_PREFIX = 'filesParams'; export const LINKS_PREFIX = 'linksParams'; export const LOGS_PREFIX = 'logsParams'; export const OU_CHILDREN_PREFIX = 'childrenParams'; @@ -120,13 +121,13 @@ export const baseRouteConfigs: Record = { mappings: { url: 'forms/mappings', params: [ - 'accountId', - 'formId', + 'accountId', + 'formId', 'mappingTypes', 'orgUnitTypeIds', 'projectsIds', - 'search', - ...paginationPathParams + 'search', + ...paginationPathParams, ], }, mappingDetail: { diff --git a/hat/assets/js/apps/Iaso/domains/forms/components/FilesTable.tsx b/hat/assets/js/apps/Iaso/domains/forms/components/FilesTable.tsx new file mode 100644 index 0000000000..817394bcca --- /dev/null +++ b/hat/assets/js/apps/Iaso/domains/forms/components/FilesTable.tsx @@ -0,0 +1,56 @@ +import { Column } from 'bluesquare-components'; +import React, { FunctionComponent } from 'react'; +import { TableWithDeepLink } from '../../../components/tables/TableWithDeepLink'; +import { usePrefixedParams } from '../../../routing/hooks/usePrefixedParams'; +import { useFormsTableColumns } from '../config'; +import { tableDefaults, useGetForms } from '../hooks/useGetForms'; + +type Props = { + baseUrl: string; + params: Record; + paramsPrefix?: string; + tableDefaults?: { + order?: string; + limit?: number; + page?: number; + }; +}; + +export const FilesTable: FunctionComponent = ({ + baseUrl, + params, + paramsPrefix, + tableDefaults: tableDefaultsProp, +}) => { + const columns = useFormsTableColumns({ + showDeleted: params?.showDeleted === 'true', + orgUnitId: params?.orgUnitId, + }) as Column[]; + + const apiParams = usePrefixedParams(paramsPrefix, params); + + const { data: forms, isLoading: isLoadingForms } = useGetForms( + apiParams, + tableDefaultsProp + ? { ...tableDefaults, ...tableDefaultsProp } + : tableDefaults, + ); + const defaultLimit = tableDefaultsProp?.limit ?? tableDefaults.limit; + return ( + + ); +}; diff --git a/hat/assets/js/apps/Iaso/domains/orgUnits/details.js b/hat/assets/js/apps/Iaso/domains/orgUnits/details.js index e7e0a58129..749c9be063 100644 --- a/hat/assets/js/apps/Iaso/domains/orgUnits/details.js +++ b/hat/assets/js/apps/Iaso/domains/orgUnits/details.js @@ -15,6 +15,7 @@ import { useQueryClient } from 'react-query'; import { useDispatch } from 'react-redux'; import TopBar from '../../components/nav/TopBarComponent'; import { + FILES_PREFIX, FORMS_PREFIX, LINKS_PREFIX, OU_CHILDREN_PREFIX, @@ -22,6 +23,7 @@ import { } from '../../constants/urls.ts'; import { useParamsObject } from '../../routing/hooks/useParamsObject.tsx'; import { fetchAssociatedOrgUnits } from '../../utils/requests'; +import { FilesTable } from '../forms/components/FilesTable'; import { FormsTable } from '../forms/components/FormsTable.tsx'; import { resetOrgUnits } from './actions'; import { OrgUnitForm } from './components/OrgUnitForm.tsx'; @@ -30,6 +32,7 @@ import { OrgUnitsMapComments } from './components/orgUnitMap/OrgUnitsMapComments import { OrgUnitChildren } from './details/Children/OrgUnitChildren.tsx'; import { OrgUnitLinks } from './details/Links/OrgUnitLinks.tsx'; import { Logs } from './history/LogsComponent.tsx'; +import { wktToGeoJSON } from './history/LogValue.tsx'; import { useOrgUnitDetailData, useOrgUnitTabParams, @@ -42,7 +45,6 @@ import { getLinksSources, getOrgUnitsTree, } from './utils'; -import { wktToGeoJSON } from './history/LogValue.tsx'; const baseUrl = baseUrls.orgUnitDetails; const useStyles = makeStyles(theme => ({ @@ -96,6 +98,7 @@ const tabs = [ 'links', 'history', 'forms', + 'files', 'comments', ]; @@ -475,6 +478,26 @@ const OrgUnitDetail = () => { /> )} + {params.tab === 'files' && ( + + + + )} {params.tab === 'forms' && ( Date: Thu, 29 Aug 2024 12:32:46 +0200 Subject: [PATCH 16/55] Front-end --- .../dialogs/ImageGalleryComponent.tsx | 5 +- .../components/dialogs/ImageGalleryLink.tsx | 6 +- .../files/LazyImagesListComponent.tsx | 82 +++++++++++++---- .../domains/forms/components/FilesTable.tsx | 56 ------------ .../apps/Iaso/domains/forms/config/index.js | 10 +- .../Iaso/domains/forms/hooks/useGetImages.tsx | 32 +++++++ .../Iaso/domains/instances/types/instance.ts | 11 +++ .../orgUnits/components/OrgUnitImages.tsx | 91 +++++++++++++++++++ .../js/apps/Iaso/domains/orgUnits/details.js | 22 ++--- .../js/apps/Iaso/domains/orgUnits/messages.ts | 4 + .../Iaso/domains/orgUnits/types/orgUnit.ts | 3 +- iaso/api/instances.py | 13 ++- 12 files changed, 231 insertions(+), 104 deletions(-) delete mode 100644 hat/assets/js/apps/Iaso/domains/forms/components/FilesTable.tsx create mode 100644 hat/assets/js/apps/Iaso/domains/forms/hooks/useGetImages.tsx create mode 100644 hat/assets/js/apps/Iaso/domains/orgUnits/components/OrgUnitImages.tsx diff --git a/hat/assets/js/apps/Iaso/components/dialogs/ImageGalleryComponent.tsx b/hat/assets/js/apps/Iaso/components/dialogs/ImageGalleryComponent.tsx index 88931a3bbd..9163fb844e 100644 --- a/hat/assets/js/apps/Iaso/components/dialogs/ImageGalleryComponent.tsx +++ b/hat/assets/js/apps/Iaso/components/dialogs/ImageGalleryComponent.tsx @@ -107,8 +107,8 @@ type Props = { currentIndex: number; // eslint-disable-next-line no-unused-vars setCurrentIndex?: (index: number) => void; - url: string | null; - urlLabel: { id: string; defaultMessage: string } | undefined; + url?: string | null; + urlLabel?: { id: string; defaultMessage: string } | undefined; // eslint-disable-next-line no-unused-vars getExtraInfos?: (image: ShortFile) => React.ReactNode; }; @@ -157,7 +157,6 @@ const ImageGallery: FunctionComponent = ({ )} closeLightbox()} > diff --git a/hat/assets/js/apps/Iaso/components/dialogs/ImageGalleryLink.tsx b/hat/assets/js/apps/Iaso/components/dialogs/ImageGalleryLink.tsx index e73dd95c22..1f08ea61ca 100644 --- a/hat/assets/js/apps/Iaso/components/dialogs/ImageGalleryLink.tsx +++ b/hat/assets/js/apps/Iaso/components/dialogs/ImageGalleryLink.tsx @@ -1,6 +1,6 @@ import { Box } from '@mui/material'; import { makeStyles } from '@mui/styles'; -import { useSafeIntl, LinkWithLocation } from 'bluesquare-components'; +import { LinkWithLocation, useSafeIntl } from 'bluesquare-components'; import React, { FunctionComponent } from 'react'; const useStyles = makeStyles(theme => ({ @@ -13,8 +13,8 @@ const useStyles = makeStyles(theme => ({ })); type Props = { - url: string | null; - urlLabel: { id: string; defaultMessage: string } | undefined; + url?: string | null; + urlLabel?: { id: string; defaultMessage: string } | undefined; }; export const ImageGalleryLink: FunctionComponent = ({ diff --git a/hat/assets/js/apps/Iaso/components/files/LazyImagesListComponent.tsx b/hat/assets/js/apps/Iaso/components/files/LazyImagesListComponent.tsx index d0b5066db3..8bed653006 100644 --- a/hat/assets/js/apps/Iaso/components/files/LazyImagesListComponent.tsx +++ b/hat/assets/js/apps/Iaso/components/files/LazyImagesListComponent.tsx @@ -1,10 +1,11 @@ -import React, { FunctionComponent, useRef, useState, useEffect } from 'react'; -import { Box, Grid } from '@mui/material'; +import FavoriteIcon from '@mui/icons-material/Favorite'; +import FavoriteBorderIcon from '@mui/icons-material/FavoriteBorder'; +import { Box, Grid, IconButton } from '@mui/material'; import { grey } from '@mui/material/colors'; - -import { LoadingSpinner, LazyImage } from 'bluesquare-components'; -import { getFileName } from '../../utils/filesUtils'; +import { LazyImage, LoadingSpinner } from 'bluesquare-components'; +import React, { FunctionComponent, useEffect, useRef, useState } from 'react'; import { ShortFile } from '../../domains/instances/types/instance'; +import { getFileName } from '../../utils/filesUtils'; const styles = { imageItem: { @@ -28,11 +29,17 @@ type Props = { imageList: ShortFile[]; // eslint-disable-next-line no-unused-vars onImageClick: (index: number) => void; + // eslint-disable-next-line no-unused-vars + onImageFavoriteClick?: (id: number) => void; + // eslint-disable-next-line no-unused-vars + isDefaultImage?: (id: number) => boolean; }; const LazyImagesList: FunctionComponent = ({ imageList, onImageClick, + onImageFavoriteClick, + isDefaultImage, }) => { const containerRef = useRef(null); const [width, setWidth] = useState(undefined); @@ -62,24 +69,63 @@ const LazyImagesList: FunctionComponent = ({ > {(src, loading, isVisible) => ( onImageClick(index)} - role="button" - tabIndex={0} - sx={styles.imageContainer} - style={{ + sx={{ + ...styles.imageContainer, + position: 'relative', backgroundImage: loading ? 'none' : `url('${src}')`, }} > - {loading && isVisible && ( - - )} + {onImageFavoriteClick && + isDefaultImage && ( + + onImageFavoriteClick( + file.itemId, + ) + } + > + {!isDefaultImage( + file.itemId, + ) && } + {isDefaultImage( + file.itemId, + ) && } + + )} + onImageClick(index)} + role="button" + tabIndex={0} + sx={{ + width: '100%', + height: '100%', + }} + > + {loading && isVisible && ( + + )} + )} diff --git a/hat/assets/js/apps/Iaso/domains/forms/components/FilesTable.tsx b/hat/assets/js/apps/Iaso/domains/forms/components/FilesTable.tsx deleted file mode 100644 index 817394bcca..0000000000 --- a/hat/assets/js/apps/Iaso/domains/forms/components/FilesTable.tsx +++ /dev/null @@ -1,56 +0,0 @@ -import { Column } from 'bluesquare-components'; -import React, { FunctionComponent } from 'react'; -import { TableWithDeepLink } from '../../../components/tables/TableWithDeepLink'; -import { usePrefixedParams } from '../../../routing/hooks/usePrefixedParams'; -import { useFormsTableColumns } from '../config'; -import { tableDefaults, useGetForms } from '../hooks/useGetForms'; - -type Props = { - baseUrl: string; - params: Record; - paramsPrefix?: string; - tableDefaults?: { - order?: string; - limit?: number; - page?: number; - }; -}; - -export const FilesTable: FunctionComponent = ({ - baseUrl, - params, - paramsPrefix, - tableDefaults: tableDefaultsProp, -}) => { - const columns = useFormsTableColumns({ - showDeleted: params?.showDeleted === 'true', - orgUnitId: params?.orgUnitId, - }) as Column[]; - - const apiParams = usePrefixedParams(paramsPrefix, params); - - const { data: forms, isLoading: isLoadingForms } = useGetForms( - apiParams, - tableDefaultsProp - ? { ...tableDefaults, ...tableDefaultsProp } - : tableDefaults, - ); - const defaultLimit = tableDefaultsProp?.limit ?? tableDefaults.limit; - return ( - - ); -}; diff --git a/hat/assets/js/apps/Iaso/domains/forms/config/index.js b/hat/assets/js/apps/Iaso/domains/forms/config/index.js index 3eab1d931f..fdbda1828f 100644 --- a/hat/assets/js/apps/Iaso/domains/forms/config/index.js +++ b/hat/assets/js/apps/Iaso/domains/forms/config/index.js @@ -1,13 +1,13 @@ -import React, { useMemo } from 'react'; import { IconButton, useSafeIntl } from 'bluesquare-components'; -import FormVersionsDialog from '../components/FormVersionsDialogComponent'; -import { baseUrls } from '../../../constants/urls.ts'; -import { userHasPermission, userHasOneOfPermissions } from '../../users/utils'; -import MESSAGES from '../messages'; +import React, { useMemo } from 'react'; import { DateTimeCell } from '../../../components/Cells/DateTimeCell.tsx'; +import { baseUrls } from '../../../constants/urls.ts'; import * as Permission from '../../../utils/permissions.ts'; import { useCurrentUser } from '../../../utils/usersUtils.ts'; +import { userHasOneOfPermissions, userHasPermission } from '../../users/utils'; import { FormActions } from '../components/FormActions.tsx'; +import FormVersionsDialog from '../components/FormVersionsDialogComponent'; +import MESSAGES from '../messages'; export const baseUrl = baseUrls.forms; diff --git a/hat/assets/js/apps/Iaso/domains/forms/hooks/useGetImages.tsx b/hat/assets/js/apps/Iaso/domains/forms/hooks/useGetImages.tsx new file mode 100644 index 0000000000..0c5aeaadc3 --- /dev/null +++ b/hat/assets/js/apps/Iaso/domains/forms/hooks/useGetImages.tsx @@ -0,0 +1,32 @@ +import { UseQueryResult } from 'react-query'; +import { getRequest } from '../../../libs/Api'; +import { useSnackQuery } from '../../../libs/apiHooks'; +import { File, ShortFile } from '../../instances/types/instance'; + +const getFiles = params => { + const queryString = new URLSearchParams({ + ...params, + image_only: 'true', + }).toString(); + return getRequest(`/api/instances/attachments/?${queryString}`); +}; + +export const useGetImages = (params): UseQueryResult => { + return useSnackQuery({ + queryKey: ['files', params], + queryFn: () => getFiles(params), + options: { + staleTime: 60000, + cacheTime: 60000, + keepPreviousData: true, + select: data => { + return data.map((file: File) => ({ + itemId: file.id, + createdAt: file.created_at, + path: file.file, + file_type: file.file_type, + })); + }, + }, + }); +}; diff --git a/hat/assets/js/apps/Iaso/domains/instances/types/instance.ts b/hat/assets/js/apps/Iaso/domains/instances/types/instance.ts index a1973e2579..77a673eea8 100644 --- a/hat/assets/js/apps/Iaso/domains/instances/types/instance.ts +++ b/hat/assets/js/apps/Iaso/domains/instances/types/instance.ts @@ -16,7 +16,17 @@ export type ShortFile = { itemId: number; createdAt: number; path: string; + file_type?: MimeType; }; + +export type File = { + id: number; + instance_id: number; + file: string; + created_at: number; + file_type: MimeType; +}; + export type Instance = { uuid: string; id: number; @@ -89,6 +99,7 @@ export type FileContent = { export interface PaginatedInstances extends Pagination { instances: Instance[]; } + export type MimeType = // Text | 'text/plain' diff --git a/hat/assets/js/apps/Iaso/domains/orgUnits/components/OrgUnitImages.tsx b/hat/assets/js/apps/Iaso/domains/orgUnits/components/OrgUnitImages.tsx new file mode 100644 index 0000000000..2f534c8dbd --- /dev/null +++ b/hat/assets/js/apps/Iaso/domains/orgUnits/components/OrgUnitImages.tsx @@ -0,0 +1,91 @@ +import { Box, Typography } from '@mui/material'; +import { LoadingSpinner } from 'bluesquare-components'; +import React, { FunctionComponent, useCallback, useState } from 'react'; +import ImageGallery from '../../../components/dialogs/ImageGalleryComponent'; +import LazyImagesList from '../../../components/files/LazyImagesListComponent'; +import { useGetImages } from '../../forms/hooks/useGetImages'; +import { ShortFile } from '../../instances/types/instance'; +import { useSaveOrgUnit } from '../hooks'; +import { OrgUnit } from '../types/orgUnit'; + +type Props = { + params: Record; + orgUnit?: OrgUnit; +}; + +const ExtraInfos: FunctionComponent<{ file: ShortFile }> = ({ file }) => { + return ( + + + {file?.file_type} + + + ); +}; + +export const OrgUnitImages: FunctionComponent = ({ + params, + orgUnit, +}) => { + const [viewerIsOpen, setViewerIsOpen] = useState(false); + + const { mutateAsync: saveOu, isLoading: savingOu } = useSaveOrgUnit(); + const [currentImageIndex, setCurrentImageIndex] = useState(0); + const { data: files, isLoading: isLoadingFiles } = useGetImages({ + orgUnitId: params.orgUnitId, + }); + const handleOpenLightbox = index => { + setCurrentImageIndex(index); + setViewerIsOpen(true); + }; + + const handleCloseLightbox = () => { + setCurrentImageIndex(0); + setViewerIsOpen(false); + }; + + const getExtraInfos = useCallback( + (file: ShortFile) => , + [], + ); + const isLoading = savingOu || isLoadingFiles; + const isDefaultImage = useCallback( + (imageId: number) => { + return imageId === orgUnit?.default_image?.id; + }, + [orgUnit?.default_image?.id], + ); + const handleImageFavoriteClick = useCallback( + (imageid: number) => { + saveOu({ + id: params.orgUnitId, + default_image: isDefaultImage(imageid) ? null : imageid, + }); + }, + [saveOu, params.orgUnitId, isDefaultImage], + ); + + return ( + <> + {isLoading && } + {!isLoadingFiles && files?.length === 0 && 'NO IMAGES'} + {!isLoadingFiles && ( + handleOpenLightbox(index)} + onImageFavoriteClick={handleImageFavoriteClick} + isDefaultImage={isDefaultImage} + /> + )} + {files && viewerIsOpen && ( + setCurrentImageIndex(newIndex)} + getExtraInfos={getExtraInfos} + /> + )} + + ); +}; diff --git a/hat/assets/js/apps/Iaso/domains/orgUnits/details.js b/hat/assets/js/apps/Iaso/domains/orgUnits/details.js index 749c9be063..c79ae13c4e 100644 --- a/hat/assets/js/apps/Iaso/domains/orgUnits/details.js +++ b/hat/assets/js/apps/Iaso/domains/orgUnits/details.js @@ -15,7 +15,6 @@ import { useQueryClient } from 'react-query'; import { useDispatch } from 'react-redux'; import TopBar from '../../components/nav/TopBarComponent'; import { - FILES_PREFIX, FORMS_PREFIX, LINKS_PREFIX, OU_CHILDREN_PREFIX, @@ -23,10 +22,10 @@ import { } from '../../constants/urls.ts'; import { useParamsObject } from '../../routing/hooks/useParamsObject.tsx'; import { fetchAssociatedOrgUnits } from '../../utils/requests'; -import { FilesTable } from '../forms/components/FilesTable'; import { FormsTable } from '../forms/components/FormsTable.tsx'; import { resetOrgUnits } from './actions'; import { OrgUnitForm } from './components/OrgUnitForm.tsx'; +import { OrgUnitImages } from './components/OrgUnitImages.tsx'; import { OrgUnitMap } from './components/orgUnitMap/OrgUnitMap/OrgUnitMap.tsx'; import { OrgUnitsMapComments } from './components/orgUnitMap/OrgUnitsMapComments'; import { OrgUnitChildren } from './details/Children/OrgUnitChildren.tsx'; @@ -98,7 +97,7 @@ const tabs = [ 'links', 'history', 'forms', - 'files', + 'images', 'comments', ]; @@ -478,23 +477,16 @@ const OrgUnitDetail = () => { /> )} - {params.tab === 'files' && ( + {params.tab === 'images' && ( - )} diff --git a/hat/assets/js/apps/Iaso/domains/orgUnits/messages.ts b/hat/assets/js/apps/Iaso/domains/orgUnits/messages.ts index 47e88e6888..adfcb4adf4 100644 --- a/hat/assets/js/apps/Iaso/domains/orgUnits/messages.ts +++ b/hat/assets/js/apps/Iaso/domains/orgUnits/messages.ts @@ -1,6 +1,10 @@ import { defineMessages } from 'react-intl'; const MESSAGES = defineMessages({ + images: { + defaultMessage: 'Images', + id: 'iaso.label.images', + }, files: { defaultMessage: 'Files', id: 'iaso.instance.files', diff --git a/hat/assets/js/apps/Iaso/domains/orgUnits/types/orgUnit.ts b/hat/assets/js/apps/Iaso/domains/orgUnits/types/orgUnit.ts index bc2eee8322..699453cb9a 100644 --- a/hat/assets/js/apps/Iaso/domains/orgUnits/types/orgUnit.ts +++ b/hat/assets/js/apps/Iaso/domains/orgUnits/types/orgUnit.ts @@ -2,7 +2,7 @@ import { Pagination, UrlParams } from 'bluesquare-components'; import { ReactNode } from 'react'; import { GeoJson } from '../../../components/maps/types'; import { Nullable } from '../../../types/utils'; -import { Instance } from '../../instances/types/instance'; +import { File, Instance } from '../../instances/types/instance'; import { DataSource } from './dataSources'; import { OrgunitType } from './orgunitTypes'; import { Shape } from './shapes'; @@ -105,6 +105,7 @@ export type OrgUnit = { reference_instance_action?: string; opening_date?: Date; closed_date?: Date; + default_image?: File; }; export interface PaginatedOrgUnits extends Pagination { orgunits: OrgUnit[]; diff --git a/iaso/api/instances.py b/iaso/api/instances.py index 89bfa204b9..d8d2d665f8 100644 --- a/iaso/api/instances.py +++ b/iaso/api/instances.py @@ -10,7 +10,7 @@ from django.core.files.storage import default_storage from django.core.paginator import Paginator from django.db import connection, transaction -from django.db.models import Count, Prefetch, Q, QuerySet +from django.db.models import Count, F, Func, Prefetch, Q, QuerySet from django.http import HttpResponse, StreamingHttpResponse from django.utils.timezone import now from rest_framework import permissions, serializers, status, viewsets @@ -172,8 +172,15 @@ def get_queryset(self): def attachments(self, request): instances = self.get_queryset() filters = parse_instance_filters(request.GET) - instances = instances.for_filters(**filters) - queryset = InstanceFile.objects.filter(instance__in=instances) + instances = instances.for_filters(**filters) # Annotate queryset with file extension + queryset = InstanceFile.objects.filter(instance__in=instances).annotate( + file_extension=Func(F("file"), function="LOWER", template="SUBSTRING(%(expressions)s, '\.([^\.]+)$')") + ) + + image_only = request.GET.get("image_only", "false").lower() == "true" + if image_only: + image_extensions = ["jpg", "jpeg", "png", "gif", "bmp", "tiff"] + queryset = queryset.filter(file_extension__in=image_extensions) paginator = common.Paginator() page = paginator.paginate_queryset(queryset, request) From d31167ab7023eec15f2787b07d097bae61a32a01 Mon Sep 17 00:00:00 2001 From: Beygorghor Date: Thu, 29 Aug 2024 13:17:49 +0200 Subject: [PATCH 17/55] save default image --- .../orgUnits/components/OrgUnitImages.tsx | 8 ++++++-- .../js/apps/Iaso/domains/orgUnits/details.js | 3 ++- iaso/api/org_units.py | 19 +++++++++++++++++-- iaso/models/org_unit.py | 4 ++-- 4 files changed, 27 insertions(+), 7 deletions(-) diff --git a/hat/assets/js/apps/Iaso/domains/orgUnits/components/OrgUnitImages.tsx b/hat/assets/js/apps/Iaso/domains/orgUnits/components/OrgUnitImages.tsx index 2f534c8dbd..27b7cce919 100644 --- a/hat/assets/js/apps/Iaso/domains/orgUnits/components/OrgUnitImages.tsx +++ b/hat/assets/js/apps/Iaso/domains/orgUnits/components/OrgUnitImages.tsx @@ -11,6 +11,7 @@ import { OrgUnit } from '../types/orgUnit'; type Props = { params: Record; orgUnit?: OrgUnit; + isFetchingDetail: boolean; }; const ExtraInfos: FunctionComponent<{ file: ShortFile }> = ({ file }) => { @@ -26,10 +27,13 @@ const ExtraInfos: FunctionComponent<{ file: ShortFile }> = ({ file }) => { export const OrgUnitImages: FunctionComponent = ({ params, orgUnit, + isFetchingDetail, }) => { const [viewerIsOpen, setViewerIsOpen] = useState(false); - const { mutateAsync: saveOu, isLoading: savingOu } = useSaveOrgUnit(); + const { mutateAsync: saveOu, isLoading: savingOu } = useSaveOrgUnit(null, [ + 'currentOrgUnit', + ]); const [currentImageIndex, setCurrentImageIndex] = useState(0); const { data: files, isLoading: isLoadingFiles } = useGetImages({ orgUnitId: params.orgUnitId, @@ -48,7 +52,7 @@ export const OrgUnitImages: FunctionComponent = ({ (file: ShortFile) => , [], ); - const isLoading = savingOu || isLoadingFiles; + const isLoading = savingOu || isLoadingFiles || isFetchingDetail; const isDefaultImage = useCallback( (imageId: number) => { return imageId === orgUnit?.default_image?.id; diff --git a/hat/assets/js/apps/Iaso/domains/orgUnits/details.js b/hat/assets/js/apps/Iaso/domains/orgUnits/details.js index c79ae13c4e..c76de1c14a 100644 --- a/hat/assets/js/apps/Iaso/domains/orgUnits/details.js +++ b/hat/assets/js/apps/Iaso/domains/orgUnits/details.js @@ -486,7 +486,8 @@ const OrgUnitDetail = () => { > )} diff --git a/iaso/api/org_units.py b/iaso/api/org_units.py index f715ead78f..a64a9d8784 100644 --- a/iaso/api/org_units.py +++ b/iaso/api/org_units.py @@ -23,13 +23,12 @@ from iaso.api.org_unit_search import annotate_query, build_org_units_queryset from iaso.api.serializers import OrgUnitSearchSerializer, OrgUnitSmallSearchSerializer, OrgUnitTreeSearchSerializer from iaso.gpkg import org_units_to_gpkg_bytes -from iaso.models import DataSource, Form, Group, Instance, OrgUnit, OrgUnitType, Project, SourceVersion +from iaso.models import DataSource, Form, Group, Instance, InstanceFile, OrgUnit, OrgUnitType, Project, SourceVersion from iaso.utils import geojson_queryset from iaso.utils.gis import simplify_geom from ..utils.models.common import get_creator_name, get_org_unit_parents_ref - # noinspection PyMethodMayBeStatic @@ -551,6 +550,22 @@ def partial_update(self, request, pk=None): continue new_groups.append(temp_group) + if "default_image" in request.data: + default_image_id = request.data["default_image"] + if default_image_id is not None: + try: + default_image = InstanceFile.objects.get(id=default_image_id) + org_unit.default_image = default_image + except InstanceFile.DoesNotExist: + errors.append( + { + "errorKey": "default_image", + "errorMessage": _("InstanceFile with id {} does not exist").format(default_image_id), + } + ) + else: + org_unit.default_image = None + opening_date = request.data.get("opening_date", None) org_unit.opening_date = None if not opening_date else self.get_date(opening_date) diff --git a/iaso/models/org_unit.py b/iaso/models/org_unit.py index 1a426db9af..ade6ddf5a6 100644 --- a/iaso/models/org_unit.py +++ b/iaso/models/org_unit.py @@ -300,8 +300,8 @@ class OrgUnit(TreeModel): opening_date = models.DateField(blank=True, null=True) # Start date of activities of the organisation unit closed_date = models.DateField(blank=True, null=True) # End date of activities of the organisation unit objects = OrgUnitManager.from_queryset(OrgUnitQuerySet)() # type: ignore - default_image: "InstanceFile" = models.ForeignKey( - "InstanceFile", on_delete=models.SET_NULL, null=True, blank=True, related_name="default_for_org_units" + default_image = models.ForeignKey( + "iaso.InstanceFile", on_delete=models.SET_NULL, null=True, blank=True, related_name="default_for_org_units" ) class Meta: From afbf404fe28b6808e9fd2bf7ee52e410acba1bb6 Mon Sep 17 00:00:00 2001 From: Beygorghor Date: Thu, 29 Aug 2024 14:42:07 +0200 Subject: [PATCH 18/55] fine tuning --- .../apps/Iaso/components/files/FavButton.tsx | 62 +++++++++++++++++++ .../files/LazyImagesListComponent.tsx | 40 +++--------- .../Iaso/domains/app/translations/en.json | 4 +- .../Iaso/domains/app/translations/fr.json | 4 +- .../orgUnits/components/ImageInfos.tsx | 32 ++++++++++ .../orgUnits/components/OrgUnitImages.tsx | 48 ++++++++------ .../js/apps/Iaso/domains/orgUnits/messages.ts | 4 ++ 7 files changed, 144 insertions(+), 50 deletions(-) create mode 100644 hat/assets/js/apps/Iaso/components/files/FavButton.tsx create mode 100644 hat/assets/js/apps/Iaso/domains/orgUnits/components/ImageInfos.tsx diff --git a/hat/assets/js/apps/Iaso/components/files/FavButton.tsx b/hat/assets/js/apps/Iaso/components/files/FavButton.tsx new file mode 100644 index 0000000000..4387adc92d --- /dev/null +++ b/hat/assets/js/apps/Iaso/components/files/FavButton.tsx @@ -0,0 +1,62 @@ +import FavoriteIcon from '@mui/icons-material/Favorite'; +import FavoriteBorderIcon from '@mui/icons-material/FavoriteBorder'; +import { IconButton, Tooltip } from '@mui/material'; +import { useSafeIntl } from 'bluesquare-components'; +import React, { FunctionComponent } from 'react'; +import { defineMessages } from 'react-intl'; +import { ShortFile } from '../../domains/instances/types/instance'; + +const styles = { + favButton: { + position: 'absolute', + top: 8, + right: 8, + backgroundColor: 'rgba(255, 255, 255, 0.7)', + boxShadow: '0 0 10px rgba(0, 0, 0, 0.2)', + '&:hover': { + backgroundColor: 'rgba(255, 255, 255, 0.9)', + }, + }, +}; +const MESSAGES = defineMessages({ + setAsDefault: { + id: 'iaso.orgUnit.setAsDefaultImage', + defaultMessage: 'Set as default image', + }, + removeAsDefault: { + id: 'iaso.orgUnit.removeAsDefaultImage', + defaultMessage: 'Remove as default image', + }, +}); + +type Props = { + file: ShortFile; + // eslint-disable-next-line no-unused-vars + onImageFavoriteClick: (id: number) => void; + // eslint-disable-next-line no-unused-vars + isDefaultImage: (id: number) => boolean; +}; + +export const FavButton: FunctionComponent = ({ + file, + onImageFavoriteClick, + isDefaultImage, +}) => { + const isDefault = isDefaultImage(file.itemId); + const { formatMessage } = useSafeIntl(); + const title = isDefault + ? formatMessage(MESSAGES.removeAsDefault) + : formatMessage(MESSAGES.setAsDefault); + return ( + + onImageFavoriteClick(file.itemId)} + > + {!isDefault && } + {isDefault && } + + + ); +}; diff --git a/hat/assets/js/apps/Iaso/components/files/LazyImagesListComponent.tsx b/hat/assets/js/apps/Iaso/components/files/LazyImagesListComponent.tsx index 8bed653006..1e68cae7bd 100644 --- a/hat/assets/js/apps/Iaso/components/files/LazyImagesListComponent.tsx +++ b/hat/assets/js/apps/Iaso/components/files/LazyImagesListComponent.tsx @@ -1,11 +1,10 @@ -import FavoriteIcon from '@mui/icons-material/Favorite'; -import FavoriteBorderIcon from '@mui/icons-material/FavoriteBorder'; -import { Box, Grid, IconButton } from '@mui/material'; +import { Box, Grid } from '@mui/material'; import { grey } from '@mui/material/colors'; import { LazyImage, LoadingSpinner } from 'bluesquare-components'; import React, { FunctionComponent, useEffect, useRef, useState } from 'react'; import { ShortFile } from '../../domains/instances/types/instance'; import { getFileName } from '../../utils/filesUtils'; +import { FavButton } from './FavButton'; const styles = { imageItem: { @@ -79,34 +78,15 @@ const LazyImagesList: FunctionComponent = ({ > {onImageFavoriteClick && isDefaultImage && ( - - onImageFavoriteClick( - file.itemId, - ) + - {!isDefaultImage( - file.itemId, - ) && } - {isDefaultImage( - file.itemId, - ) && } - + isDefaultImage={ + isDefaultImage + } + /> )} onImageClick(index)} diff --git a/hat/assets/js/apps/Iaso/domains/app/translations/en.json b/hat/assets/js/apps/Iaso/domains/app/translations/en.json index f37ef7c2ce..9e409f31a3 100644 --- a/hat/assets/js/apps/Iaso/domains/app/translations/en.json +++ b/hat/assets/js/apps/Iaso/domains/app/translations/en.json @@ -852,12 +852,14 @@ "iaso.orgUnits.project": "Project", "iaso.orgUnits.referenceForm": "Reference form", "iaso.orgUnits.referenceForms": "Reference forms", + "iaso.orgUnits.removeAsDefaultImage": "Remove as default image", "iaso.orgUnits.removeFromGroups": "Remove from group(s)", "iaso.orgUnits.search": "Search org unit", "iaso.orgUnits.searchParams": "Use prefix “refs:” for external org unit ID search. Use prefix “ids:” for internal org unit ID search. You can also search multiple IDs at once, separated by a comma or a space. E.g. “ids: 123456, 654321” or “refs: O6uvpzGd5pu, ImspTQPwCqd”", "iaso.orgUnits.selectionAction": "With selected org unit", "iaso.orgUnits.selectOrgUnit": "Please select an org Unit", "iaso.orgUnits.selectProjects": "Select a project", + "iaso.orgUnits.setAsDefaultImage": "Set as default image", "iaso.orgUnits.shortName": "Short name", "iaso.orgUnits.source": "Source", "iaso.orgUnits.sourceLower": "source", @@ -1462,4 +1464,4 @@ "trypelim.permissions.zones": "Zones", "trypelim.permissions.zones_edit": "Edit zones", "trypelim.permissions.zones_shapes_edit": "Edit zone shapes" -} +} \ No newline at end of file diff --git a/hat/assets/js/apps/Iaso/domains/app/translations/fr.json b/hat/assets/js/apps/Iaso/domains/app/translations/fr.json index a3110646df..1e897a5d0f 100644 --- a/hat/assets/js/apps/Iaso/domains/app/translations/fr.json +++ b/hat/assets/js/apps/Iaso/domains/app/translations/fr.json @@ -852,12 +852,14 @@ "iaso.orgUnits.project": "Projet", "iaso.orgUnits.referenceForm": "Formulaire de référence", "iaso.orgUnits.referenceForms": "Formulaires de référence", + "iaso.orgUnits.removeAsDefaultImage": "Retirer l'image par défaut", "iaso.orgUnits.removeFromGroups": "Retirer du(des) groupe(s)", "iaso.orgUnits.search": "Rechercher une unité d'org.", "iaso.orgUnits.searchParams": "Utilisez le préfixe “refs:” pour la recherche d'ID externe d'unité d’organisation. Utilisez le préfixe “ids:” pour la recherche d'ID interne d'unité d’organisation. Vous pouvez également rechercher plusieurs ID à la fois, séparés par une virgule ou un espace. Par exemple: “ids: 123456, 654321” ou “refs: O6uvpzGd5pu, ImspTQPwCqd”", "iaso.orgUnits.selectionAction": "Avec la sélection d'unités d'organisation", "iaso.orgUnits.selectOrgUnit": "Merci de sélectionner une unité d'organisation", "iaso.orgUnits.selectProjects": "Selectionner un projet", + "iaso.orgUnits.setAsDefaultImage": "Définir comme image par défaut", "iaso.orgUnits.shortName": "Nom raccourci", "iaso.orgUnits.source": "Source", "iaso.orgUnits.sourceLower": "source", @@ -1461,4 +1463,4 @@ "trypelim.permissions.zones": "Zones", "trypelim.permissions.zones_edit": "Edit zones", "trypelim.permissions.zones_shapes_edit": "Edit zone shapes" -} +} \ No newline at end of file diff --git a/hat/assets/js/apps/Iaso/domains/orgUnits/components/ImageInfos.tsx b/hat/assets/js/apps/Iaso/domains/orgUnits/components/ImageInfos.tsx new file mode 100644 index 0000000000..8d0e73b6d0 --- /dev/null +++ b/hat/assets/js/apps/Iaso/domains/orgUnits/components/ImageInfos.tsx @@ -0,0 +1,32 @@ +import { LoadingSpinner } from 'bluesquare-components'; +import React, { FunctionComponent } from 'react'; +import { FavButton } from '../../../components/files/FavButton'; +import { ShortFile } from '../../instances/types/instance'; + + +type ImageInfosProps = { + file: ShortFile; + // eslint-disable-next-line no-unused-vars + onImageFavoriteClick: (imageId: number) => void; + // eslint-disable-next-line no-unused-vars + isDefaultImage: (imageId: number) => boolean; + isLoading: boolean; +}; + +export const ImageInfos: FunctionComponent = ({ + file, + onImageFavoriteClick, + isDefaultImage, + isLoading, +}) => { + return ( + <> + {isLoading && } + + + ); +}; diff --git a/hat/assets/js/apps/Iaso/domains/orgUnits/components/OrgUnitImages.tsx b/hat/assets/js/apps/Iaso/domains/orgUnits/components/OrgUnitImages.tsx index 27b7cce919..5cbc7c5836 100644 --- a/hat/assets/js/apps/Iaso/domains/orgUnits/components/OrgUnitImages.tsx +++ b/hat/assets/js/apps/Iaso/domains/orgUnits/components/OrgUnitImages.tsx @@ -1,12 +1,15 @@ -import { Box, Typography } from '@mui/material'; -import { LoadingSpinner } from 'bluesquare-components'; +import { Paper } from '@mui/material'; +import { LoadingSpinner, useSafeIntl } from 'bluesquare-components'; import React, { FunctionComponent, useCallback, useState } from 'react'; import ImageGallery from '../../../components/dialogs/ImageGalleryComponent'; import LazyImagesList from '../../../components/files/LazyImagesListComponent'; +import { SxStyles } from '../../../types/general'; import { useGetImages } from '../../forms/hooks/useGetImages'; import { ShortFile } from '../../instances/types/instance'; import { useSaveOrgUnit } from '../hooks'; +import MESSAGES from '../messages'; import { OrgUnit } from '../types/orgUnit'; +import { ImageInfos } from './ImageInfos'; type Props = { params: Record; @@ -14,14 +17,12 @@ type Props = { isFetchingDetail: boolean; }; -const ExtraInfos: FunctionComponent<{ file: ShortFile }> = ({ file }) => { - return ( - - - {file?.file_type} - - - ); +const styles: SxStyles = { + noResult: { + padding: theme => theme.spacing(2), + textAlign: 'center', + backgroundColor: 'rgba(0,0,0,0.03)', + }, }; export const OrgUnitImages: FunctionComponent = ({ @@ -30,7 +31,7 @@ export const OrgUnitImages: FunctionComponent = ({ isFetchingDetail, }) => { const [viewerIsOpen, setViewerIsOpen] = useState(false); - + const { formatMessage } = useSafeIntl(); const { mutateAsync: saveOu, isLoading: savingOu } = useSaveOrgUnit(null, [ 'currentOrgUnit', ]); @@ -47,12 +48,6 @@ export const OrgUnitImages: FunctionComponent = ({ setCurrentImageIndex(0); setViewerIsOpen(false); }; - - const getExtraInfos = useCallback( - (file: ShortFile) => , - [], - ); - const isLoading = savingOu || isLoadingFiles || isFetchingDetail; const isDefaultImage = useCallback( (imageId: number) => { return imageId === orgUnit?.default_image?.id; @@ -69,10 +64,27 @@ export const OrgUnitImages: FunctionComponent = ({ [saveOu, params.orgUnitId, isDefaultImage], ); + const isLoading = savingOu || isLoadingFiles || isFetchingDetail; + const getExtraInfos = useCallback( + (file: ShortFile) => ( + + ), + [handleImageFavoriteClick, isDefaultImage, isLoading], + ); + return ( <> {isLoading && } - {!isLoadingFiles && files?.length === 0 && 'NO IMAGES'} + {!isLoadingFiles && files?.length === 0 && ( + + {formatMessage(MESSAGES.noResult)} + + )} {!isLoadingFiles && ( Date: Thu, 29 Aug 2024 15:00:58 +0200 Subject: [PATCH 19/55] ase star instead of heart --- hat/assets/js/apps/Iaso/components/files/FavButton.tsx | 8 ++++---- iaso/api/instances.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/hat/assets/js/apps/Iaso/components/files/FavButton.tsx b/hat/assets/js/apps/Iaso/components/files/FavButton.tsx index 4387adc92d..59449d4263 100644 --- a/hat/assets/js/apps/Iaso/components/files/FavButton.tsx +++ b/hat/assets/js/apps/Iaso/components/files/FavButton.tsx @@ -1,5 +1,5 @@ -import FavoriteIcon from '@mui/icons-material/Favorite'; -import FavoriteBorderIcon from '@mui/icons-material/FavoriteBorder'; +import StarBorderIcon from '@mui/icons-material/StarBorder'; +import StarRateIcon from '@mui/icons-material/StarRate'; import { IconButton, Tooltip } from '@mui/material'; import { useSafeIntl } from 'bluesquare-components'; import React, { FunctionComponent } from 'react'; @@ -54,8 +54,8 @@ export const FavButton: FunctionComponent = ({ sx={styles.favButton} onClick={() => onImageFavoriteClick(file.itemId)} > - {!isDefault && } - {isDefault && } + {!isDefault && } + {isDefault && } ); diff --git a/iaso/api/instances.py b/iaso/api/instances.py index d8d2d665f8..b634d0d7e3 100644 --- a/iaso/api/instances.py +++ b/iaso/api/instances.py @@ -172,7 +172,7 @@ def get_queryset(self): def attachments(self, request): instances = self.get_queryset() filters = parse_instance_filters(request.GET) - instances = instances.for_filters(**filters) # Annotate queryset with file extension + instances = instances.for_filters(**filters) queryset = InstanceFile.objects.filter(instance__in=instances).annotate( file_extension=Func(F("file"), function="LOWER", template="SUBSTRING(%(expressions)s, '\.([^\.]+)$')") ) From d2c4094aec0dd24176809b9c0e45c675d544413b Mon Sep 17 00:00:00 2001 From: Beygorghor Date: Thu, 29 Aug 2024 15:02:52 +0200 Subject: [PATCH 20/55] black --- iaso/admin.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/iaso/admin.py b/iaso/admin.py index a367ed7d4b..05f8927fc5 100644 --- a/iaso/admin.py +++ b/iaso/admin.py @@ -454,9 +454,9 @@ class EntityAdmin(admin.ModelAdmin): def get_form(self, request, obj=None, **kwargs): # In the for the entity type, we also want to indicate the account name form = super().get_form(request, obj, **kwargs) - form.base_fields["entity_type"].label_from_instance = ( - lambda entity: f"{entity.name} (Account: {entity.account.name})" - ) + form.base_fields[ + "entity_type" + ].label_from_instance = lambda entity: f"{entity.name} (Account: {entity.account.name})" return form def get_queryset(self, request): From a6c2663ff445e079e8e8859f13b058121885c62c Mon Sep 17 00:00:00 2001 From: Beygorghor Date: Thu, 29 Aug 2024 15:08:04 +0200 Subject: [PATCH 21/55] translation problem --- hat/assets/js/apps/Iaso/components/files/FavButton.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hat/assets/js/apps/Iaso/components/files/FavButton.tsx b/hat/assets/js/apps/Iaso/components/files/FavButton.tsx index 59449d4263..0a6bcd1ba6 100644 --- a/hat/assets/js/apps/Iaso/components/files/FavButton.tsx +++ b/hat/assets/js/apps/Iaso/components/files/FavButton.tsx @@ -20,11 +20,11 @@ const styles = { }; const MESSAGES = defineMessages({ setAsDefault: { - id: 'iaso.orgUnit.setAsDefaultImage', + id: 'iaso.orgUnits.setAsDefaultImage', defaultMessage: 'Set as default image', }, removeAsDefault: { - id: 'iaso.orgUnit.removeAsDefaultImage', + id: 'iaso.orgUnits.removeAsDefaultImage', defaultMessage: 'Remove as default image', }, }); From 27e1e1d0f047fde9002174054b90a4abd9f41219 Mon Sep 17 00:00:00 2001 From: Beygorghor Date: Thu, 29 Aug 2024 15:46:21 +0200 Subject: [PATCH 22/55] fix tests --- iaso/tests/api/test_serializers.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/iaso/tests/api/test_serializers.py b/iaso/tests/api/test_serializers.py index 5d54c07ea9..139537e0a9 100644 --- a/iaso/tests/api/test_serializers.py +++ b/iaso/tests/api/test_serializers.py @@ -1,14 +1,14 @@ import datetime from collections import OrderedDict from unittest import mock -from django.test import TestCase import pytz -from django.contrib.gis.geos import Polygon, Point, MultiPolygon +from django.contrib.gis.geos import MultiPolygon, Point, Polygon +from django.test import TestCase from iaso import models as m from iaso.api.query_params import APP_ID -from iaso.api.serializers import OrgUnitSearchSerializer, OrgUnitSmallSearchSerializer, AppIdSerializer +from iaso.api.serializers import AppIdSerializer, OrgUnitSearchSerializer, OrgUnitSmallSearchSerializer from iaso.test import APITestCase @@ -140,6 +140,7 @@ def test_serialize_search(self): "aliases": None, "created_at": 1522800000.0, "creator": None, + "default_image": None, "groups": [ OrderedDict( [ @@ -241,6 +242,7 @@ def test_serialize_search(self): "longitude": 4.0, "altitude": 100.0, "creator": None, + "default_image": None, "projects": [], }, ) From f35679eb877b74f4ab8c45a03bb8681e02ecb096 Mon Sep 17 00:00:00 2001 From: Christophe Gerard Date: Tue, 3 Sep 2024 14:50:18 +0200 Subject: [PATCH 23/55] test serializer with default image --- iaso/tests/api/test_serializers.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/iaso/tests/api/test_serializers.py b/iaso/tests/api/test_serializers.py index 139537e0a9..f7a4f3bd52 100644 --- a/iaso/tests/api/test_serializers.py +++ b/iaso/tests/api/test_serializers.py @@ -4,6 +4,7 @@ import pytz from django.contrib.gis.geos import MultiPolygon, Point, Polygon +from django.core.files.uploadedfile import SimpleUploadedFile from django.test import TestCase from iaso import models as m @@ -129,6 +130,17 @@ def setUpTestData(cls): cls.create_form_instance( form=cls.form_1, period="202003", org_unit=cls.jedi_council_corruscant, project=cls.project ) + cls.image_file = m.InstanceFile.objects.create( + file=SimpleUploadedFile("test_image.jpg", b"file_content", content_type="image/jpeg"), + name="test_image.jpg", + ) + cls.org_unit_with_image = m.OrgUnit.objects.create( + org_unit_type=cls.jedi_council, + version=sw_version_1, + name="Jedi Council with Image", + validation_status=m.OrgUnit.VALIDATION_VALID, + default_image=cls.image_file, # Assign the InstanceFile here + ) def test_serialize_search(self): orgunit = m.OrgUnit.objects.first() @@ -325,6 +337,22 @@ def test_creator_org_unit(self): f"{self.yoda.username} ({self.yoda.first_name} {self.yoda.last_name})", response.json().get("creator") ) + def test_serialize_search_with_default_image(self): + org_unit = self.org_unit_with_image + serializer = OrgUnitSearchSerializer(org_unit) + serialized_data = serializer.data + + self.assertIsNotNone(serialized_data.get("default_image")) + self.assertIsInstance(serialized_data["default_image"], int) + self.assertEqual(serialized_data["default_image"], self.image_file.id) + + # Test org unit without default image + org_unit_without_image = self.jedi_council_corruscant + serializer = OrgUnitSearchSerializer(org_unit_without_image) + serialized_data = serializer.data + + self.assertIsNone(serialized_data.get("default_image")) + class AppIdSerializerTestCase(TestCase): def test_app_id_serializer(self): From d0c1fdfebb97382fa259d145feb5b8380bffef67 Mon Sep 17 00:00:00 2001 From: Christophe Gerard Date: Tue, 3 Sep 2024 15:22:47 +0200 Subject: [PATCH 24/55] testing api --- iaso/tests/api/test_orgunits.py | 38 +++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/iaso/tests/api/test_orgunits.py b/iaso/tests/api/test_orgunits.py index 8e260141b8..a0777aa68f 100644 --- a/iaso/tests/api/test_orgunits.py +++ b/iaso/tests/api/test_orgunits.py @@ -1,14 +1,14 @@ +import csv +import io import typing -from django.contrib.gis.geos import Polygon, Point, MultiPolygon, GEOSGeometry +from django.contrib.gis.geos import GEOSGeometry, MultiPolygon, Point, Polygon from django.db import connection from hat.audit.models import Modification from iaso import models as m -from iaso.models import OrgUnitType, OrgUnit +from iaso.models import OrgUnit, OrgUnitType from iaso.test import APITestCase -import csv -import io class OrgUnitAPITestCase(APITestCase): @@ -1253,3 +1253,33 @@ def test_org_unit_search_only_direct_children_true(self): # list of all direct children of the jedi_council_endor OU ou_ids_list = [self.jedi_squad_endor.pk, self.jedi_squad_endor_2.pk] self.assertEqual(sorted(ids_in_response), sorted(ou_ids_list)) + + def test_edit_org_unit_add_default_image(self): + old_ou = self.jedi_council_corruscant + self.client.force_authenticate(self.yoda) + image = m.InstanceFile.objects.create(file="path/to/image.jpg") + response = self.client.patch( + f"/api/orgunits/{old_ou.id}/", + format="json", + data={"default_image": image.id}, + ) + jr = self.assertJSONResponse(response, 200) + self.assertValidOrgUnitData(jr) + ou = m.OrgUnit.objects.get(id=jr["id"]) + self.assertEqual(ou.default_image.id, image.id) + + def test_edit_org_unit_remove_default_image(self): + old_ou = self.jedi_council_corruscant + image = m.InstanceFile.objects.create(file="path/to/image.jpg") + old_ou.default_image = image + old_ou.save() + self.client.force_authenticate(self.yoda) + response = self.client.patch( + f"/api/orgunits/{old_ou.id}/", + format="json", + data={"default_image": None}, + ) + jr = self.assertJSONResponse(response, 200) + self.assertValidOrgUnitData(jr) + ou = m.OrgUnit.objects.get(id=jr["id"]) + self.assertIsNone(ou.default_image) From 0f0f82871c84cdac4f8ef1eba8ab7f5e768dd5ad Mon Sep 17 00:00:00 2001 From: Christophe Gerard Date: Tue, 3 Sep 2024 16:11:18 +0200 Subject: [PATCH 25/55] some test on attachments api --- iaso/tests/api/test_instances.py | 47 ++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/iaso/tests/api/test_instances.py b/iaso/tests/api/test_instances.py index 3cf429fc5d..92b93dbe6e 100644 --- a/iaso/tests/api/test_instances.py +++ b/iaso/tests/api/test_instances.py @@ -1,4 +1,6 @@ +import csv import datetime +import io import json import typing from unittest import mock @@ -16,11 +18,9 @@ from hat.audit.models import Modification from iaso import models as m from iaso.api import query_params as query +from iaso.models import FormVersion, Instance, InstanceLock from iaso.models.microplanning import Planning, Team -from iaso.models import Instance, InstanceLock, FormVersion from iaso.test import APITestCase -import csv -import io MOCK_DATE = datetime.datetime(2020, 2, 2, 2, 2, 2, tzinfo=pytz.utc) @@ -1800,3 +1800,44 @@ def test_instances_filter_from_date_to_date(self): another_instance.id, ], ) + + def test_attachments_list(self): + self.client.force_authenticate(self.yoda) + instance = self.create_form_instance(form=self.form_1, project=self.project) + attachment1 = m.InstanceFile.objects.create(instance=instance, file="test1.jpg") + attachment2 = m.InstanceFile.objects.create(instance=instance, file="test2.pdf") + + response = self.client.get("/api/instances/attachments/") + self.assertJSONResponse(response, 200) + + data = response.json() + self.assertEqual(len(data), 2) + self.assertEqual(data[0]["id"], attachment1.id) + self.assertEqual(data[1]["id"], attachment2.id) + + def test_attachments_filter_image_only(self): + self.client.force_authenticate(self.yoda) + instance = self.create_form_instance(form=self.form_1, project=self.project) + m.InstanceFile.objects.create(instance=instance, file="test1.jpg") + m.InstanceFile.objects.create(instance=instance, file="test2.pdf") + + response = self.client.get("/api/instances/attachments/?image_only=true") + self.assertJSONResponse(response, 200) + + data = response.json() + self.assertEqual(len(data), 1) + self.assertTrue(data[0]["file"].endswith(".jpg")) + + def test_attachments_pagination(self): + self.client.force_authenticate(self.yoda) + instance = self.create_form_instance(form=self.form_1, project=self.project) + for i in range(30): # Assuming default page size is less than 30 + m.InstanceFile.objects.create(instance=instance, file=f"test{i}.jpg") + + response = self.client.get("/api/instances/attachments/?limit=30") + self.assertJSONResponse(response, 200) + + data = response.json() + self.assertEqual(len(data["results"]), 30) + self.assertFalse(data["has_next"]) + self.assertFalse(data["has_previous"]) From 244ede37382fa1b835cf19c2470888f5620d8fea Mon Sep 17 00:00:00 2001 From: Bram Jans Date: Mon, 29 Jul 2024 10:51:46 +0200 Subject: [PATCH 26/55] SLEEP-1458 Add backend support for search on instance fields --- iaso/api/entity.py | 10 ++ iaso/tests/api/test_entities.py | 235 ++++++++++++++++++++------------ iaso/utils/jsonlogic.py | 15 +- 3 files changed, 170 insertions(+), 90 deletions(-) diff --git a/iaso/api/entity.py b/iaso/api/entity.py index 5b1dcd3e22..d82892ad3c 100644 --- a/iaso/api/entity.py +++ b/iaso/api/entity.py @@ -1,6 +1,7 @@ import csv import datetime import io +import json import math from time import gmtime, strftime from typing import Any, List, Union @@ -29,6 +30,7 @@ ) from iaso.models import Entity, EntityType, Instance, OrgUnit from iaso.models.deduplication import ValidationStatus +from iaso.utils.jsonlogic import jsonlogic_to_q class EntitySerializer(serializers.ModelSerializer): @@ -228,6 +230,14 @@ def retrieve(self, request, pk=None): def list(self, request: Request, *args, **kwargs): queryset = self.filter_queryset(self.get_queryset()) + + json_content = request.GET.get("jsonContent", None) + # json_content = '{"and": [{"==": [{"var": "trypelim_CATT.result"}, "positive"]}]}' + if json_content: + q = jsonlogic_to_q(json.loads(json_content), field_prefix="json__") + queryset = queryset.filter(q) + print(queryset.query) + csv_format = request.GET.get("csv", None) xlsx_format = request.GET.get("xlsx", None) is_export = any([csv_format, xlsx_format]) diff --git a/iaso/tests/api/test_entities.py b/iaso/tests/api/test_entities.py index da8295f341..656d9cdf8b 100644 --- a/iaso/tests/api/test_entities.py +++ b/iaso/tests/api/test_entities.py @@ -1,3 +1,4 @@ +import json import time import uuid from unittest import mock @@ -48,7 +49,9 @@ def setUpTestData(cls): username="user_without_ou", account=star_wars, permissions=["iaso_entities"] ) - cls.form_1 = m.Form.objects.create(name="Hydroponics study", period_type=m.MONTH, single_per_period=True) + cls.form_1 = m.Form.objects.create( + name="Hydroponics study", period_type=m.MONTH, single_per_period=True, form_id="form_1" + ) cls.create_form_instance( form=cls.form_1, period="202001", org_unit=cls.jedi_council_corruscant, project=cls.project, uuid=uuid.uuid4 @@ -63,11 +66,15 @@ def setUpTestData(cls): form=cls.form_1, period="202003", org_unit=cls.jedi_council_corruscant, project=cls.project, uuid=uuid.uuid4 ) + cls.entity_type = EntityType.objects.create( + name="Type 1", + reference_form=cls.form_1, + account=cls.star_wars, + ) + def test_create_single_entity(self): self.client.force_authenticate(self.yoda) - entity_type = EntityType.objects.create(name="Type 1", reference_form=self.form_1) - instance = Instance.objects.create( org_unit=self.jedi_council_corruscant, form=self.form_1, @@ -76,7 +83,7 @@ def test_create_single_entity(self): payload = { "name": "New Client", - "entity_type": entity_type.pk, + "entity_type": self.entity_type.pk, "attributes": instance.uuid, "account": self.yoda.iaso_profile.account.pk, } @@ -88,8 +95,6 @@ def test_create_single_entity(self): def test_create_multiples_entity(self): self.client.force_authenticate(self.yoda) - entity_type = EntityType.objects.create(name="Type 1", reference_form=self.form_1) - instance = Instance.objects.create( org_unit=self.jedi_council_corruscant, form=self.form_1, period="202002", uuid=uuid.uuid4() ) @@ -100,12 +105,12 @@ def test_create_multiples_entity(self): payload = { "name": "New Client", - "entity_type": entity_type.pk, + "entity_type": self.entity_type.pk, "attributes": instance.uuid, "account": self.yoda.iaso_profile.account.pk, }, { "name": "New Client 2", - "entity_type": entity_type.pk, + "entity_type": self.entity_type.pk, "attributes": second_instance.uuid, "account": self.yoda.iaso_profile.account.pk, } @@ -118,20 +123,18 @@ def test_create_multiples_entity(self): def test_create_entity_same_attributes(self): self.client.force_authenticate(self.yoda) - entity_type = EntityType.objects.create(name="Type 1", reference_form=self.form_1) - instance = Instance.objects.create( org_unit=self.jedi_council_corruscant, form=self.form_1, period="202002", uuid=uuid.uuid4() ) payload = { "name": "New Client", - "entity_type": entity_type.pk, + "entity_type": self.entity_type.pk, "attributes": instance.uuid, "account": self.yoda.iaso_profile.account.pk, }, { "name": "New Client 2", - "entity_type": entity_type.pk, + "entity_type": self.entity_type.pk, "attributes": instance.uuid, "account": self.yoda.iaso_profile.account.pk, } @@ -143,8 +146,6 @@ def test_create_entity_same_attributes(self): def test_retrieve_entity(self): self.client.force_authenticate(self.yoda) - entity_type = EntityType.objects.create(name="Type 1", reference_form=self.form_1, account=self.star_wars) - instance = Instance.objects.create( org_unit=self.jedi_council_corruscant, form=self.form_1, period="202002", uuid=uuid.uuid4() ) @@ -155,12 +156,12 @@ def test_retrieve_entity(self): payload = { "name": "New Client", - "entity_type": entity_type.pk, + "entity_type": self.entity_type.pk, "attributes": instance.uuid, "account": self.yoda.iaso_profile.account.pk, }, { "name": "New Client 2", - "entity_type": entity_type.pk, + "entity_type": self.entity_type.pk, "attributes": second_instance.uuid, "account": self.yoda.iaso_profile.account.pk, } @@ -175,8 +176,6 @@ def test_retrieve_entity(self): def test_retrieve_entity_without_attributes(self): self.client.force_authenticate(self.yoda) - entity_type = EntityType.objects.create(name="Type 1", reference_form=self.form_1, account=self.star_wars) - instance = Instance.objects.create( org_unit=self.jedi_council_corruscant, form=self.form_1, period="202002", uuid=uuid.uuid4() ) @@ -187,12 +186,12 @@ def test_retrieve_entity_without_attributes(self): payload = { "name": "New Client", - "entity_type": entity_type.pk, + "entity_type": self.entity_type.pk, "attributes": instance.uuid, "account": self.yoda.iaso_profile.account.pk, }, { "name": "New Client 2", - "entity_type": entity_type.pk, + "entity_type": self.entity_type.pk, "attributes": second_instance.uuid, "account": self.yoda.iaso_profile.account.pk, } @@ -200,7 +199,7 @@ def test_retrieve_entity_without_attributes(self): self.client.post("/api/entities/bulk_create/", data=payload, format="json") entity = Entity.objects.create( - name="New Client 3", entity_type=entity_type, account=self.yoda.iaso_profile.account + name="New Client 3", entity_type=self.entity_type, account=self.yoda.iaso_profile.account ) entity.refresh_from_db() @@ -210,7 +209,7 @@ def test_retrieve_entity_without_attributes(self): self.assertEqual(response.status_code, 200) self.assertEqual(len(response.json()["result"]), 3) - response = self.client.get(f"/api/entities/?entity_type_id={entity_type.pk}", format="json") + response = self.client.get(f"/api/entities/?entity_type_id={self.entity_type.pk}", format="json") self.assertEqual(response.status_code, 200) self.assertEqual(len(response.json()["result"]), 3) @@ -227,9 +226,6 @@ def test_get_entity_search_filter(self): """ self.client.force_authenticate(self.yoda) - # Let's first create the test data - entity_type = EntityType.objects.create(name="Type 1", reference_form=self.form_1) - instance = Instance.objects.create( org_unit=self.jedi_council_corruscant, form=self.form_1, @@ -239,7 +235,7 @@ def test_get_entity_search_filter(self): payload = { "name": "New Client", - "entity_type": entity_type.pk, + "entity_type": self.entity_type.pk, "attributes": instance.uuid, "account": self.yoda.iaso_profile.account.pk, } @@ -270,10 +266,102 @@ def test_get_entity_search_filter(self): self.assertEqual(len(response.json()["result"]), 1) self.assertEqual(the_result["id"], newly_added_entity.id) - def test_get_entity_by_id(self): + def test_get_entity_search_in_instances(self): + """ + Test the 'searchInInstances' filter of /api/entities + + This parameter allows to filter entities based on the presence of a + form instance with certain attributes. + """ self.client.force_authenticate(self.yoda) + self.form_2 = m.Form.objects.create(name="Form 2", form_id="form_2") + + # Entity 1 - Female from Bujumbura + ent1_instance1 = Instance.objects.create( + org_unit=self.jedi_council_corruscant, + form=self.form_1, + json={"gender": "F"}, + ) + ent1 = Entity.objects.create( + name="Ent 1", + entity_type=self.entity_type, + attributes=ent1_instance1, + account=self.star_wars, + ) + ent1_instance1.entity = ent1 + ent1_instance1.save() + Instance.objects.create( + org_unit=self.jedi_council_corruscant, + form=self.form_2, + json={"residence": "Bujumbura"}, + entity=ent1, + ) + + # Entity 2 - Male from Kinshasa + ent2_instance1 = Instance.objects.create( + org_unit=self.jedi_council_corruscant, + form=self.form_1, + json={"gender": "M"}, + ) + ent2 = Entity.objects.create( + name="Ent 1", + entity_type=self.entity_type, + attributes=ent2_instance1, + account=self.star_wars, + ) + ent2_instance1.entity = ent2 + ent2_instance1.save() + Instance.objects.create( + org_unit=self.jedi_council_corruscant, + form=self.form_2, + json={"residence": "Kinshasa"}, + entity=ent2, + ) - entity_type = EntityType.objects.create(name="Type 1", reference_form=self.form_1) + response = self.client.get( + "/api/entities/", + {"jsonContent": self._generate_json_filter("and", "F", "Bujumbura")}, + ) + self.assertEqual(len(response.json()["result"]), 1) + the_result = response.json()["result"][0] + self.assertEqual(the_result["id"], ent1.id) + + response = self.client.get( + "/api/entities/", + {"jsonContent": self._generate_json_filter("and", "M", "Bujumbura")}, + ) + self.assertEqual(len(response.json()["result"]), 0) + + response = self.client.get( + "/api/entities/", + {"jsonContent": self._generate_json_filter("or", "F", "Kinshasa")}, + ) + self.assertEqual(len(response.json()["result"]), 2) + result_ids = [r["id"] for r in response.json()["result"]] + self.assertEqual(sorted(result_ids), sorted([ent1.id, ent2.id])) + + def _generate_json_filter(self, operator, gender, residence): + return json.dumps( + { + operator: [ + { + "==": [ + {"var": f"{self.form_1.form_id}.gender"}, + gender, + ] + }, + { + "==": [ + {"var": f"{self.form_2.form_id}.residence"}, + residence, + ] + }, + ] + } + ) + + def test_get_entity_by_id(self): + self.client.force_authenticate(self.yoda) instance = Instance.objects.create( org_unit=self.jedi_council_corruscant, @@ -283,7 +371,7 @@ def test_get_entity_by_id(self): payload = { "name": "New Client", - "entity_type": entity_type.pk, + "entity_type": self.entity_type.pk, "attributes": instance.uuid, "account": self.yoda.iaso_profile.account.pk, } @@ -297,11 +385,10 @@ def test_get_entity_by_id(self): def test_handle_wrong_attributes(self): self.client.force_authenticate(self.yoda) - entity_type = EntityType.objects.create(name="Type 1", reference_form=self.form_1) payload = { "name": "New Client", - "entity_type": entity_type.pk, + "entity_type": self.entity_type.pk, "attributes": 2324, "account": self.yoda.iaso_profile.account.pk, } @@ -313,22 +400,20 @@ def test_handle_wrong_attributes(self): def test_update_entity(self): self.client.force_authenticate(self.yoda) - entity_type = EntityType.objects.create(name="Type 1", reference_form=self.form_1) - instance = Instance.objects.create( org_unit=self.jedi_council_corruscant, form=self.form_1, period="202002", uuid=uuid.uuid4() ) payload_post = { "name": "New Client", - "entity_type": entity_type.pk, + "entity_type": self.entity_type.pk, "attributes": instance.uuid, "account": self.yoda.iaso_profile.account.pk, } self.client.post("/api/entities/", data=payload_post, format="json") - payload = {"name": "New Client-2", "entity_type": entity_type.pk, "attributes": instance.pk} + payload = {"name": "New Client-2", "entity_type": self.entity_type.pk, "attributes": instance.pk} response = self.client.patch("/api/entities/{0}/".format(Entity.objects.last().pk), data=payload, format="json") @@ -337,8 +422,6 @@ def test_update_entity(self): def test_retrieve_only_non_deleted_entity(self): self.client.force_authenticate(self.yoda) - entity_type = EntityType.objects.create(name="Type 1", reference_form=self.form_1, account=self.star_wars) - instance = Instance.objects.create( org_unit=self.jedi_council_corruscant, form=self.form_1, period="202002", uuid=uuid.uuid4() ) @@ -349,12 +432,12 @@ def test_retrieve_only_non_deleted_entity(self): payload = { "name": "New Client", - "entity_type": entity_type.pk, + "entity_type": self.entity_type.pk, "attributes": instance.uuid, "account": self.yoda.iaso_profile.account.pk, }, { "name": "New Client 2", - "entity_type": entity_type.pk, + "entity_type": self.entity_type.pk, "attributes": second_instance.uuid, "account": self.yoda.iaso_profile.account.pk, } @@ -369,11 +452,9 @@ def test_retrieve_only_non_deleted_entity(self): def test_cant_create_entity_without_attributes(self): self.client.force_authenticate(self.yoda) - entity_type = EntityType.objects.create(name="Type 1", reference_form=self.form_1) - payload = { "name": "New Client", - "entity_type": entity_type.pk, + "entity_type": self.entity_type.pk, "attributes": None, "account": self.yoda.iaso_profile.account.pk, } @@ -385,8 +466,6 @@ def test_cant_create_entity_without_attributes(self): def test_retrieve_entity_only_same_account(self): self.client.force_authenticate(self.yoda) - entity_type = EntityType.objects.create(name="Type 1", reference_form=self.form_1, account=self.star_wars) - instance = Instance.objects.create( org_unit=self.jedi_council_corruscant, form=self.form_1, @@ -401,14 +480,14 @@ def test_retrieve_entity_only_same_account(self): Entity.objects.create( name="New Client", - entity_type=entity_type, + entity_type=self.entity_type, attributes=instance, account=self.yop_solo.iaso_profile.account, ) Entity.objects.create( name="New Client", - entity_type=entity_type, + entity_type=self.entity_type, attributes=second_instance, account=self.yoda.iaso_profile.account, ) @@ -421,12 +500,12 @@ def test_retrieve_entity_only_same_account(self): @mock.patch("iaso.api.entity.gmtime", lambda: time.struct_time((2021, 7, 18, 14, 57, 0, 1, 291, 0))) def test_export_entity(self): self.client.force_authenticate(self.yoda) - entity_type = EntityType.objects.create( - name="Type 1", - reference_form=self.form_1, - fields_detail_info_view=["something", "else"], - fields_list_view=["something", "else"], - ) + # entity_type = EntityType.objects.create( + # name="Type 1", + # reference_form=self.form_1, + # fields_detail_info_view=["something", "else"], + # fields_list_view=["something", "else"], + # ) instance = Instance.objects.create( org_unit=self.jedi_council_corruscant, @@ -442,14 +521,14 @@ def test_export_entity(self): Entity.objects.create( name="New Client", - entity_type=entity_type, + entity_type=self.entity_type, attributes=instance, account=self.yop_solo.iaso_profile.account, ) Entity.objects.create( name="New Client", - entity_type=entity_type, + entity_type=self.entity_type, attributes=second_instance, account=self.yoda.iaso_profile.account, ) @@ -465,12 +544,12 @@ def test_export_entity(self): self.assertEqual(response.get("Content-Disposition"), "attachment; filename=entities-2021-07-18-14-57.xlsx") # export specific entity type as xlsx - response = self.client.get(f"/api/entities/?entity_type_ids={entity_type.pk}&xlsx=true/") + response = self.client.get(f"/api/entities/?entity_type_ids={self.entity_type.pk}&xlsx=true/") self.assertEqual(response.status_code, 200) self.assertEqual(response.get("Content-Disposition"), "attachment; filename=entities-2021-07-18-14-57.xlsx") # export specific entity type as csv - response = self.client.get(f"/api/entities/?entity_type_ids={entity_type.pk}&csv=true/") + response = self.client.get(f"/api/entities/?entity_type_ids={self.entity_type.pk}&csv=true/") self.assertEqual(response.status_code, 200) self.assertEqual(response.get("Content-Disposition"), "attachment; filename=entities-2021-07-18-14-57.csv") @@ -479,11 +558,6 @@ def test_export_entity(self): def test_handle_export_entity_type_empty_field_list(self): self.client.force_authenticate(self.yoda) - entity_type = EntityType.objects.create( - name="Type 1", - reference_form=self.form_1, - ) - instance = Instance.objects.create( org_unit=self.jedi_council_corruscant, form=self.form_1, @@ -492,7 +566,7 @@ def test_handle_export_entity_type_empty_field_list(self): entity = Entity.objects.create( name="New Client", - entity_type=entity_type, + entity_type=self.entity_type, account=self.yop_solo.iaso_profile.account, ) @@ -516,11 +590,6 @@ def test_entity_mobile(self): FormVersion.objects.create(form=self.form_1, version_id="A_FORM_ID") - entity_type = EntityType.objects.create( - name="Type 1", - reference_form=self.form_1, - ) - instance = Instance.objects.create( org_unit=self.jedi_council_corruscant, form=self.form_1, @@ -534,7 +603,7 @@ def test_entity_mobile(self): entity = Entity.objects.create( name="New Client", - entity_type=entity_type, + entity_type=self.entity_type, attributes=instance, account=self.yoda.iaso_profile.account, ) @@ -567,11 +636,6 @@ def test_entity_mobile_user(self): FormVersion.objects.create(form=self.form_1, version_id="A_FORM_ID") - entity_type = EntityType.objects.create( - name="Type 1", - reference_form=self.form_1, - ) - instance = Instance.objects.create( org_unit=self.jedi_council_corruscant, form=self.form_1, @@ -657,21 +721,21 @@ def test_entity_mobile_user(self): entity = Entity.objects.create( name="New Client", - entity_type=entity_type, + entity_type=self.entity_type, attributes=instance, account=self.yoda.iaso_profile.account, ) entity_unvalidated = Entity.objects.create( name="New Client", - entity_type=entity_type, + entity_type=self.entity_type, attributes=instance_unvalidated_ou, account=self.yoda.iaso_profile.account, ) entity_no_attributes = Entity.objects.create( name="New Client_2", - entity_type=entity_type, + entity_type=self.entity_type, account=self.yop_solo.iaso_profile.account, ) @@ -692,9 +756,9 @@ def test_entity_mobile_user(self): self.assertEqual(response.status_code, 200) self.assertEqual(response_json.get("count"), 2) - self.assertEqual(response_json.get("results")[0].get("entity_type_id"), str(entity_type.id)) + self.assertEqual(response_json.get("results")[0].get("entity_type_id"), str(self.entity_type.id)) self.assertEqual(len(response_json.get("results")[0].get("instances")), 1) - self.assertEqual(response_json.get("results")[1].get("entity_type_id"), str(entity_type.id)) + self.assertEqual(response_json.get("results")[1].get("entity_type_id"), str(self.entity_type.id)) self.assertEqual(len(response_json.get("results")[1].get("instances")), 0) def test_access_respect_appid_mobile(self): @@ -730,14 +794,9 @@ def test_access_respect_appid_mobile(self): } instance_app_id.save() - entity_type, created = EntityType.objects.get_or_create( - name="Type 1", - reference_form=self.form_1, - ) - entity_app_id = Entity.objects.create( name="New Client", - entity_type=entity_type, + entity_type=self.entity_type, attributes=instance_app_id, account=self.yoda.iaso_profile.account, ) @@ -755,7 +814,7 @@ def test_access_respect_appid_mobile(self): response_json = response.json() self.assertEqual(response_json["count"], 1) - self.assertEqual(response_json["results"][0]["entity_type_id"], str(entity_type.id)) + self.assertEqual(response_json["results"][0]["entity_type_id"], str(self.entity_type.id)) response_entity_instance = response_json["results"][0]["instances"] @@ -789,17 +848,15 @@ def test_retrieve_entities_user_geo_restrictions(self): user_manager.refresh_from_db() self.client.force_authenticate(user_manager) - entity_type = EntityType.objects.create(name="Type 1", reference_form=self.form_1, account=self.star_wars) - # Village 1 (district 1): instance + entity instance_1 = Instance.objects.create(org_unit=village_1, form=self.form_1, period="202002") - entity = Entity.objects.create(entity_type=entity_type, attributes=instance_1, account=self.star_wars) + entity = Entity.objects.create(entity_type=self.entity_type, attributes=instance_1, account=self.star_wars) instance_1.entity = entity instance_1.save() # Village 2 (district 2): instance + entity instance_2 = Instance.objects.create(org_unit=village_2, form=self.form_1, period="202002") - entity_2 = Entity.objects.create(entity_type=entity_type, attributes=instance_2, account=self.star_wars) + entity_2 = Entity.objects.create(entity_type=self.entity_type, attributes=instance_2, account=self.star_wars) instance_2.entity = entity_2 instance_2.save() diff --git a/iaso/utils/jsonlogic.py b/iaso/utils/jsonlogic.py index 0d62782355..1efdbf35d2 100644 --- a/iaso/utils/jsonlogic.py +++ b/iaso/utils/jsonlogic.py @@ -4,7 +4,7 @@ from functools import reduce from typing import Dict, Any -from django.db.models import Q, Transform +from django.db.models import Exists, Q, Transform, OuterRef from django.db.models.fields.json import KeyTransformTextLookupMixin, JSONField @@ -99,8 +99,21 @@ def jsonlogic_to_q(jsonlogic: Dict[str, Any], field_prefix: str = "") -> Q: extract = "__forcefloat" lookup = lookups[op] + + form_id = None + field_name_arr = field_name.split(".") + if len(field_name_arr) == 2: + form_id, field_name = field_name_arr + f = f"{field_prefix}{field_name}{extract}__{lookup}" q = Q(**{f: value}) + + if form_id: + from iaso.models import Instance + + subquery = Instance.objects.filter(Q(entity_id=OuterRef("id")) & Q(form__form_id=form_id) & q) + q = Exists(subquery) + if op == "!=": # invert the filter q = ~q From 694d1a5ac8ad14014ff81710012ed74357a4edad Mon Sep 17 00:00:00 2001 From: Bram Jans Date: Thu, 1 Aug 2024 17:59:30 +0200 Subject: [PATCH 27/55] SLEEP-1458 Only fetch location when changing to maps tab Avoid unnecessary queries. --- hat/assets/js/apps/Iaso/domains/entities/hooks/requests.ts | 5 +++++ hat/assets/js/apps/Iaso/domains/entities/index.tsx | 1 + 2 files changed, 6 insertions(+) diff --git a/hat/assets/js/apps/Iaso/domains/entities/hooks/requests.ts b/hat/assets/js/apps/Iaso/domains/entities/hooks/requests.ts index a130c1d0be..f29fe4a92a 100644 --- a/hat/assets/js/apps/Iaso/domains/entities/hooks/requests.ts +++ b/hat/assets/js/apps/Iaso/domains/entities/hooks/requests.ts @@ -43,6 +43,7 @@ type Params = { entityTypeIds?: string; locationLimit?: string; groups?: string; + tab?: string; }; type ApiParams = { @@ -59,6 +60,7 @@ type ApiParams = { asLocation?: boolean; locationLimit?: string; groups?: string; + tab: string; }; type GetAPiParams = { @@ -85,6 +87,7 @@ export const useGetBeneficiariesApiParams = ( limit: params.pageSize || '20', page: params.page || '1', groups: params.groups, + tab: params.tab || 'list', }; if (asLocation) { apiParams.asLocation = true; @@ -105,6 +108,7 @@ export const useGetBeneficiariesPaginated = ( queryKey: ['beneficiaries', apiParams], queryFn: () => getRequest(url), options: { + enabled: apiParams.tab === 'list', staleTime: 60000, cacheTime: 1000 * 60 * 5, keepPreviousData: true, @@ -120,6 +124,7 @@ export const useGetBeneficiariesLocations = ( queryKey: ['beneficiariesLocations', apiParams], queryFn: () => getRequest(url), options: { + enabled: apiParams.tab === 'map', staleTime: 60000, select: data => data?.result?.map(beneficiary => ({ diff --git a/hat/assets/js/apps/Iaso/domains/entities/index.tsx b/hat/assets/js/apps/Iaso/domains/entities/index.tsx index 3ca57d0131..8918d31dd2 100644 --- a/hat/assets/js/apps/Iaso/domains/entities/index.tsx +++ b/hat/assets/js/apps/Iaso/domains/entities/index.tsx @@ -106,6 +106,7 @@ export const Beneficiaries: FunctionComponent = () => { } const { data: locations, isFetching: isFetchingLocations } = useGetBeneficiariesLocations(params, displayedLocation); + return ( <> {isLoading && tab === 'map' && } From ff432e70ad65dda518fbfe5377c585798d0b24d6 Mon Sep 17 00:00:00 2001 From: Bram Jans Date: Mon, 5 Aug 2024 14:33:06 +0200 Subject: [PATCH 28/55] SLEEP-1458 Entities filter on instances: frontend v1 --- hat/assets/js/apps/Iaso/constants/urls.ts | 9 ++-- .../domains/entities/components/Filters.tsx | 52 ++++++++++++++++++- .../entityTypes/hooks/requests/forms.ts | 8 ++- .../js/apps/Iaso/domains/entities/messages.ts | 4 ++ .../Iaso/domains/entities/types/filters.ts | 22 +++----- .../fields/hooks/useGetFormDescriptor.ts | 4 +- .../fields/hooks/useGetQueryBuildersFields.ts | 27 ++++++++++ .../forms/hooks/useGetPossibleFields.ts | 39 +++++++++++++- 8 files changed, 139 insertions(+), 26 deletions(-) diff --git a/hat/assets/js/apps/Iaso/constants/urls.ts b/hat/assets/js/apps/Iaso/constants/urls.ts index 5fcda03c5b..34058979d3 100644 --- a/hat/assets/js/apps/Iaso/constants/urls.ts +++ b/hat/assets/js/apps/Iaso/constants/urls.ts @@ -120,13 +120,13 @@ export const baseRouteConfigs: Record = { mappings: { url: 'forms/mappings', params: [ - 'accountId', - 'formId', + 'accountId', + 'formId', 'mappingTypes', 'orgUnitTypeIds', 'projectsIds', - 'search', - ...paginationPathParams + 'search', + ...paginationPathParams, ], }, mappingDetail: { @@ -326,6 +326,7 @@ export const baseRouteConfigs: Record = { 'entityTypeIds', 'locationLimit', 'groups', + 'fieldsSearch', ...paginationPathParams, ], }, diff --git a/hat/assets/js/apps/Iaso/domains/entities/components/Filters.tsx b/hat/assets/js/apps/Iaso/domains/entities/components/Filters.tsx index d99841adaf..beb5d57067 100644 --- a/hat/assets/js/apps/Iaso/domains/entities/components/Filters.tsx +++ b/hat/assets/js/apps/Iaso/domains/entities/components/Filters.tsx @@ -11,9 +11,11 @@ import { makeStyles } from '@mui/styles'; import SearchIcon from '@mui/icons-material/Search'; import { + QueryBuilderInput, commonStyles, - useSafeIntl, + useHumanReadableJsonLogic, useRedirectTo, + useSafeIntl, } from 'bluesquare-components'; // @ts-ignore @@ -47,6 +49,13 @@ import { hasFeatureFlag, } from '../../../utils/featureFlags'; +import { Popper } from '../../forms/fields/components/Popper'; +import { parseJson } from '../../instances/utils/jsonLogicParse'; +import { useGetAllPossibleFields } from '../../forms/hooks/useGetPossibleFields'; +import { useGetFormDescriptor } from '../../forms/fields/hooks/useGetFormDescriptor'; +import { useGetQueryBuilderFieldsForAllForms } from '../../forms/fields/hooks/useGetQueryBuildersFields'; +import { useGetQueryBuilderListToReplace } from '../../forms/fields/hooks/useGetQueryBuilderListToReplace'; + const useStyles = makeStyles(theme => ({ ...commonStyles(theme), })); @@ -89,6 +98,7 @@ const Filters: FunctionComponent = ({ params, isFetching }) => { entityTypeIds: params.entityTypeIds, locationLimit: params.locationLimit, groups: params.groups, + fieldsSearch: params.fieldsSearch, }); useEffect(() => { @@ -102,6 +112,7 @@ const Filters: FunctionComponent = ({ params, isFetching }) => { entityTypeIds: params.entityTypeIds, locationLimit: params.locationLimit, groups: params.groups, + fieldsSearch: params.fieldsSearch, }); }, [params]); const [filtersUpdated, setFiltersUpdated] = useState(false); @@ -127,6 +138,31 @@ const Filters: FunctionComponent = ({ params, isFetching }) => { sourceVersionId, }); + // Load QueryBuilder resources + const { allPossibleFields } = useGetAllPossibleFields(); + const { data: formDescriptors } = useGetFormDescriptor(); + const fields = useGetQueryBuilderFieldsForAllForms( + formDescriptors, + allPossibleFields, + ); + const queryBuilderListToReplace = useGetQueryBuilderListToReplace(); + const getHumanReadableJsonLogic = useHumanReadableJsonLogic( + fields, + queryBuilderListToReplace, + ); + const fieldsSearchJson = filters.fieldsSearch + ? JSON.parse(filters.fieldsSearch) + : undefined; + + const handleChangeQueryBuilder = value => { + if (value) { + const parsedValue = parseJson({ value, fields }); + handleChange('fieldsSearch', JSON.stringify(parsedValue)); + } else { + handleChange('fieldsSearch', undefined); + } + }; + const handleSearch = useCallback(() => { if (filtersUpdated) { setFiltersUpdated(false); @@ -219,7 +255,19 @@ const Filters: FunctionComponent = ({ params, isFetching }) => { initialSelection={initialOrgUnit} /> - + + handleChange('fieldsSearch', undefined), + }} + InfoPopper={} + /> {params.tab === 'map' && ( => { + let url = '/api/forms/?fields=id,name,latest_form_version'; + if (fields) { + url += `,${fields.join(',')}`; + } return useSnackQuery({ queryKey: ['forms'], - queryFn: () => - getRequest('/api/forms/?fields=id,name,latest_form_version'), + queryFn: () => getRequest(url), options: { staleTime: 60000, enabled, diff --git a/hat/assets/js/apps/Iaso/domains/entities/messages.ts b/hat/assets/js/apps/Iaso/domains/entities/messages.ts index ab5668d791..8194b36b5b 100644 --- a/hat/assets/js/apps/Iaso/domains/entities/messages.ts +++ b/hat/assets/js/apps/Iaso/domains/entities/messages.ts @@ -253,6 +253,10 @@ const MESSAGES = defineMessages({ defaultMessage: 'Group', id: 'iaso.label.group', }, + queryBuilder: { + id: 'iaso.instance.queryBuilder', + defaultMessage: 'Search in submitted fields', + }, }); export default MESSAGES; diff --git a/hat/assets/js/apps/Iaso/domains/entities/types/filters.ts b/hat/assets/js/apps/Iaso/domains/entities/types/filters.ts index 6f7479485e..c7cb733179 100644 --- a/hat/assets/js/apps/Iaso/domains/entities/types/filters.ts +++ b/hat/assets/js/apps/Iaso/domains/entities/types/filters.ts @@ -1,27 +1,19 @@ export type Filters = { - search?: string; - location?: string; dateFrom?: string; dateTo?: string; - submitterId?: string; - submitterTeamId?: string; entityTypeIds?: string; + fieldsSearch?: string; + location?: string; locationLimit?: string; groups?: string; + search?: string; + submitterId?: string; + submitterTeamId?: string; }; -export type Params = { - pageSize: string; +export type Params = Filters & { order: string; page: string; + pageSize: string; tab: string; - search?: string; - location?: string; - dateFrom?: string; - dateTo?: string; - submitterId?: string; - submitterTeamId?: string; - entityTypeIds?: string; - locationLimit?: string; - groups?: string; }; diff --git a/hat/assets/js/apps/Iaso/domains/forms/fields/hooks/useGetFormDescriptor.ts b/hat/assets/js/apps/Iaso/domains/forms/fields/hooks/useGetFormDescriptor.ts index 4365fd7524..a94c467cdc 100644 --- a/hat/assets/js/apps/Iaso/domains/forms/fields/hooks/useGetFormDescriptor.ts +++ b/hat/assets/js/apps/Iaso/domains/forms/fields/hooks/useGetFormDescriptor.ts @@ -12,7 +12,7 @@ type FormVersionsList = { }; const getVersion = (formId: number | undefined): Promise => { - return getRequest(`/api/formversions/?form_id=${formId}&fields=descriptor`); + return getRequest(`/api/formversions/?form_id=&fields=descriptor`); }; export const useGetFormDescriptor = ( formId?: number, @@ -25,7 +25,7 @@ export const useGetFormDescriptor = ( queryKey, queryFn: () => getVersion(formId), options: { - enabled: Boolean(formId), + // enabled: Boolean(formId), TODO select: ( data: FormVersionsList | undefined, ): FormDescriptor[] | undefined => { diff --git a/hat/assets/js/apps/Iaso/domains/forms/fields/hooks/useGetQueryBuildersFields.ts b/hat/assets/js/apps/Iaso/domains/forms/fields/hooks/useGetQueryBuildersFields.ts index 56ae43093c..cb3d73f243 100644 --- a/hat/assets/js/apps/Iaso/domains/forms/fields/hooks/useGetQueryBuildersFields.ts +++ b/hat/assets/js/apps/Iaso/domains/forms/fields/hooks/useGetQueryBuildersFields.ts @@ -54,3 +54,30 @@ export const useGetQueryBuildersFields = ( }); return fields; }; + +export const useGetQueryBuilderFieldsForAllForms = ( + formDescriptors?: FormDescriptor[], + allPossibleFields?: PossibleField[], +): QueryBuilderFields => { + if (!allPossibleFields || !formDescriptors) return {}; + const fields: QueryBuilderFields = {}; + + for (const [form_id, possibleFields] of Object.entries(allPossibleFields)) { + const subfields = useGetQueryBuildersFields( + formDescriptors, + possibleFields, + ); + + fields[form_id] = { + label: form_id, + type: '!group', + mode: 'array', + conjunctions: ['AND', 'OR'], + operators: ['some', 'all', 'none'], + defaultOperator: 'some', + subfields: subfields, + }; + } + + return fields; +}; diff --git a/hat/assets/js/apps/Iaso/domains/forms/hooks/useGetPossibleFields.ts b/hat/assets/js/apps/Iaso/domains/forms/hooks/useGetPossibleFields.ts index bdb7ebdebf..815dd35d2d 100644 --- a/hat/assets/js/apps/Iaso/domains/forms/hooks/useGetPossibleFields.ts +++ b/hat/assets/js/apps/Iaso/domains/forms/hooks/useGetPossibleFields.ts @@ -3,7 +3,10 @@ import { useMemo } from 'react'; import { UseQueryResult } from 'react-query'; import { cloneDeep } from 'lodash'; import { DropdownOptions } from '../../../types/utils'; -import { useGetForm } from '../../entities/entityTypes/hooks/requests/forms'; +import { + useGetForm, + useGetForms, +} from '../../entities/entityTypes/hooks/requests/forms'; import { useSnackQuery } from '../../../libs/apiHooks'; import { getRequest } from '../../../libs/Api'; @@ -14,6 +17,10 @@ type Result = { possibleFields: PossibleField[]; isFetchingForm: boolean; }; +type AllResults = { + allPossibleFields: any; // TODO + isFetchingForms: boolean; +}; export const usePossibleFields = ( isFetchingForm: boolean, @@ -42,6 +49,36 @@ export const useGetPossibleFields = (formId?: number, appId?: string): Result => return usePossibleFields(isFetchingForm, currentForm); }; +export const useAllPossibleFields = ( + isFetchingForms: boolean, + allForms: Form[] = [], +): AllResults => { + return useMemo(() => { + let allPossibleFields = {}; + allForms.forEach(form => { + const possibleFields = + form?.possible_fields?.map(field => ({ + ...field, + fieldKey: field.name.replace('.', ''), + })) || []; + allPossibleFields[form.form_id] = possibleFields; + }); + + return { + allPossibleFields, + isFetchingForms, + }; + }, [isFetchingForms]); +}; + +export const useGetAllPossibleFields = (): AllResults => { + const { data: allForms, isFetching: isFetchingForms } = useGetForms(true, [ + 'form_id', + 'possible_fields', + ]); + return useAllPossibleFields(isFetchingForms, allForms); +}; + type PossibleFieldsDropdown = { isFetching: boolean; dropdown: DropdownOptions[]; From c99c293d2b0f53233176692522f65368f6bbfb55 Mon Sep 17 00:00:00 2001 From: Bram Jans Date: Tue, 6 Aug 2024 16:53:41 +0200 Subject: [PATCH 29/55] SLEEP-1458 Implement "some" filter into JSON logic to Q --- .../Iaso/domains/entities/hooks/requests.ts | 27 +++++-------------- .../js/apps/Iaso/domains/entities/index.tsx | 1 + iaso/api/entity.py | 12 ++++----- iaso/tests/api/test_entities.py | 20 +++++++------- iaso/utils/jsonlogic.py | 22 +++++++-------- 5 files changed, 33 insertions(+), 49 deletions(-) diff --git a/hat/assets/js/apps/Iaso/domains/entities/hooks/requests.ts b/hat/assets/js/apps/Iaso/domains/entities/hooks/requests.ts index f29fe4a92a..c99184228a 100644 --- a/hat/assets/js/apps/Iaso/domains/entities/hooks/requests.ts +++ b/hat/assets/js/apps/Iaso/domains/entities/hooks/requests.ts @@ -14,38 +14,23 @@ import { } from '../../../libs/Api'; import MESSAGES from '../messages'; +import { Location } from '../components/ListMap'; +import getDisplayName, { Profile } from '../../../utils/usersUtils'; import { makeUrlWithParams } from '../../../libs/utils'; import { Beneficiary } from '../types/beneficiary'; -import { PaginatedInstances } from '../../instances/types/instance'; +import { DisplayedLocation } from '../types/locations'; import { DropdownOptions } from '../../../types/utils'; -import getDisplayName, { Profile } from '../../../utils/usersUtils'; import { DropdownTeamsOptions, Team } from '../../teams/types/team'; import { ExtraColumn } from '../types/fields'; -import { DisplayedLocation } from '../types/locations'; -import { Location } from '../components/ListMap'; +import { PaginatedInstances } from '../../instances/types/instance'; +import { Params } from '../types/filters'; export interface PaginatedBeneficiaries extends Pagination { result: Array; columns: Array; } -type Params = { - pageSize: string; - order: string; - page: string; - search?: string; - location?: string; - dateFrom?: string; - dateTo?: string; - submitterId?: string; - submitterTeamId?: string; - entityTypeIds?: string; - locationLimit?: string; - groups?: string; - tab?: string; -}; - type ApiParams = { limit?: string; order_columns: string; @@ -61,6 +46,7 @@ type ApiParams = { locationLimit?: string; groups?: string; tab: string; + fields_search?: string; }; type GetAPiParams = { @@ -88,6 +74,7 @@ export const useGetBeneficiariesApiParams = ( page: params.page || '1', groups: params.groups, tab: params.tab || 'list', + fields_search: params.fieldsSearch, }; if (asLocation) { apiParams.asLocation = true; diff --git a/hat/assets/js/apps/Iaso/domains/entities/index.tsx b/hat/assets/js/apps/Iaso/domains/entities/index.tsx index 8918d31dd2..3522b9c009 100644 --- a/hat/assets/js/apps/Iaso/domains/entities/index.tsx +++ b/hat/assets/js/apps/Iaso/domains/entities/index.tsx @@ -48,6 +48,7 @@ type Params = { entityTypeIds?: string; locationLimit?: string; groups?: string; + fieldsSearch?: string; }; export const Beneficiaries: FunctionComponent = () => { diff --git a/iaso/api/entity.py b/iaso/api/entity.py index d82892ad3c..ca9a85b6fa 100644 --- a/iaso/api/entity.py +++ b/iaso/api/entity.py @@ -133,6 +133,7 @@ def get_queryset(self): created_by_id = self.request.query_params.get("created_by_id", None) created_by_team_id = self.request.query_params.get("created_by_team_id", None) groups = self.request.query_params.get("groups", None) + fields_search = self.request.GET.get("fields_search", None) queryset = Entity.objects.filter_for_user(self.request.user) @@ -183,6 +184,10 @@ def get_queryset(self): if groups: queryset = queryset.filter(attributes__org_unit__groups__in=groups.split(",")) + if fields_search: + q = jsonlogic_to_q(json.loads(fields_search), field_prefix="json__") + queryset = queryset.filter(q) + # location return queryset @@ -231,13 +236,6 @@ def retrieve(self, request, pk=None): def list(self, request: Request, *args, **kwargs): queryset = self.filter_queryset(self.get_queryset()) - json_content = request.GET.get("jsonContent", None) - # json_content = '{"and": [{"==": [{"var": "trypelim_CATT.result"}, "positive"]}]}' - if json_content: - q = jsonlogic_to_q(json.loads(json_content), field_prefix="json__") - queryset = queryset.filter(q) - print(queryset.query) - csv_format = request.GET.get("csv", None) xlsx_format = request.GET.get("xlsx", None) is_export = any([csv_format, xlsx_format]) diff --git a/iaso/tests/api/test_entities.py b/iaso/tests/api/test_entities.py index 656d9cdf8b..f85150016b 100644 --- a/iaso/tests/api/test_entities.py +++ b/iaso/tests/api/test_entities.py @@ -268,7 +268,7 @@ def test_get_entity_search_filter(self): def test_get_entity_search_in_instances(self): """ - Test the 'searchInInstances' filter of /api/entities + Test the 'fields_search' filter of /api/entities This parameter allows to filter entities based on the presence of a form instance with certain attributes. @@ -320,7 +320,7 @@ def test_get_entity_search_in_instances(self): response = self.client.get( "/api/entities/", - {"jsonContent": self._generate_json_filter("and", "F", "Bujumbura")}, + {"fields_search": self._generate_json_filter("and", "F", "Bujumbura")}, ) self.assertEqual(len(response.json()["result"]), 1) the_result = response.json()["result"][0] @@ -328,13 +328,13 @@ def test_get_entity_search_in_instances(self): response = self.client.get( "/api/entities/", - {"jsonContent": self._generate_json_filter("and", "M", "Bujumbura")}, + {"fields_search": self._generate_json_filter("and", "M", "Bujumbura")}, ) self.assertEqual(len(response.json()["result"]), 0) response = self.client.get( "/api/entities/", - {"jsonContent": self._generate_json_filter("or", "F", "Kinshasa")}, + {"fields_search": self._generate_json_filter("or", "F", "Kinshasa")}, ) self.assertEqual(len(response.json()["result"]), 2) result_ids = [r["id"] for r in response.json()["result"]] @@ -345,15 +345,15 @@ def _generate_json_filter(self, operator, gender, residence): { operator: [ { - "==": [ - {"var": f"{self.form_1.form_id}.gender"}, - gender, + "some": [ + {"var": self.form_1.form_id}, + {"==": [{"var": "gender"}, gender]}, ] }, { - "==": [ - {"var": f"{self.form_2.form_id}.residence"}, - residence, + "some": [ + {"var": self.form_2.form_id}, + {"==": [{"var": "residence"}, residence]}, ] }, ] diff --git a/iaso/utils/jsonlogic.py b/iaso/utils/jsonlogic.py index 1efdbf35d2..3f4888ba72 100644 --- a/iaso/utils/jsonlogic.py +++ b/iaso/utils/jsonlogic.py @@ -55,9 +55,18 @@ def jsonlogic_to_q(jsonlogic: Dict[str, Any], field_prefix: str = "") -> Q: operator.or_, (jsonlogic_to_q(subquery, field_prefix) for subquery in jsonlogic["or"]), ) - elif "!" in jsonlogic: return ~jsonlogic_to_q(jsonlogic["!"], field_prefix) + elif "some" in jsonlogic: + from iaso.models import Instance + + form_var, conditions = jsonlogic["some"] + form_id = form_var["var"] + + subquery = Instance.objects.filter( + Q(entity_id=OuterRef("id")) & Q(form__form_id=form_id) & jsonlogic_to_q(conditions, field_prefix) + ) + return Exists(subquery) if not jsonlogic.keys(): return Q() @@ -100,20 +109,9 @@ def jsonlogic_to_q(jsonlogic: Dict[str, Any], field_prefix: str = "") -> Q: lookup = lookups[op] - form_id = None - field_name_arr = field_name.split(".") - if len(field_name_arr) == 2: - form_id, field_name = field_name_arr - f = f"{field_prefix}{field_name}{extract}__{lookup}" q = Q(**{f: value}) - if form_id: - from iaso.models import Instance - - subquery = Instance.objects.filter(Q(entity_id=OuterRef("id")) & Q(form__form_id=form_id) & q) - q = Exists(subquery) - if op == "!=": # invert the filter q = ~q From dc5534cb477a2b1a3a23d21abb7999ef7e4bb3f7 Mon Sep 17 00:00:00 2001 From: Bram Jans Date: Wed, 7 Aug 2024 11:27:17 +0200 Subject: [PATCH 30/55] SLEEP-1458 Add ALL and NONE filters, extract function... for entities. Since we directly reference the Instances and entity_id, I feel it's better off in it's own function. --- iaso/api/entity.py | 4 +- iaso/tests/api/test_entities.py | 37 +++++++++++--- iaso/utils/jsonlogic.py | 86 +++++++++++++++++++++++++++------ 3 files changed, 104 insertions(+), 23 deletions(-) diff --git a/iaso/api/entity.py b/iaso/api/entity.py index ca9a85b6fa..c6bc9b7e3c 100644 --- a/iaso/api/entity.py +++ b/iaso/api/entity.py @@ -30,7 +30,7 @@ ) from iaso.models import Entity, EntityType, Instance, OrgUnit from iaso.models.deduplication import ValidationStatus -from iaso.utils.jsonlogic import jsonlogic_to_q +from iaso.utils.jsonlogic import entities_jsonlogic_to_q class EntitySerializer(serializers.ModelSerializer): @@ -185,7 +185,7 @@ def get_queryset(self): queryset = queryset.filter(attributes__org_unit__groups__in=groups.split(",")) if fields_search: - q = jsonlogic_to_q(json.loads(fields_search), field_prefix="json__") + q = entities_jsonlogic_to_q(json.loads(fields_search)) queryset = queryset.filter(q) # location diff --git a/iaso/tests/api/test_entities.py b/iaso/tests/api/test_entities.py index f85150016b..51614be893 100644 --- a/iaso/tests/api/test_entities.py +++ b/iaso/tests/api/test_entities.py @@ -317,41 +317,66 @@ def test_get_entity_search_in_instances(self): json={"residence": "Kinshasa"}, entity=ent2, ) + Instance.objects.create( + org_unit=self.jedi_council_corruscant, + form=self.form_2, + json={"residence": "Accra"}, + entity=ent2, + ) + # gender f AND SOME residence Bujumbura response = self.client.get( "/api/entities/", - {"fields_search": self._generate_json_filter("and", "F", "Bujumbura")}, + {"fields_search": self._generate_json_filter("and", "some", "F", "Bujumbura")}, ) self.assertEqual(len(response.json()["result"]), 1) the_result = response.json()["result"][0] self.assertEqual(the_result["id"], ent1.id) + # gender m AND SOME residence Bujumbura response = self.client.get( "/api/entities/", - {"fields_search": self._generate_json_filter("and", "M", "Bujumbura")}, + {"fields_search": self._generate_json_filter("and", "some", "M", "Bujumbura")}, ) self.assertEqual(len(response.json()["result"]), 0) + # gender f OR SOME residence Kinshasa response = self.client.get( "/api/entities/", - {"fields_search": self._generate_json_filter("or", "F", "Kinshasa")}, + {"fields_search": self._generate_json_filter("or", "some", "F", "Kinshasa")}, ) self.assertEqual(len(response.json()["result"]), 2) result_ids = [r["id"] for r in response.json()["result"]] self.assertEqual(sorted(result_ids), sorted([ent1.id, ent2.id])) - def _generate_json_filter(self, operator, gender, residence): + # gender m AND SOME residence Kinshasa + response = self.client.get( + "/api/entities/", + {"fields_search": self._generate_json_filter("and", "some", "M", "Kinshasa")}, + ) + self.assertEqual(len(response.json()["result"]), 1) + the_result = response.json()["result"][0] + self.assertEqual(the_result["id"], ent2.id) + + # gender M AND ALL residence Kinshasa + response = self.client.get( + "/api/entities/", + {"fields_search": self._generate_json_filter("and", "all", "M", "Kinshasa")}, + ) + self.assertEqual(len(response.json()["result"]), 0) + + def _generate_json_filter(self, operator, some_or_all, gender, residence): return json.dumps( { operator: [ { - "some": [ + some_or_all: [ {"var": self.form_1.form_id}, {"==": [{"var": "gender"}, gender]}, ] }, { - "some": [ + some_or_all: [ {"var": self.form_2.form_id}, {"==": [{"var": "residence"}, residence]}, ] diff --git a/iaso/utils/jsonlogic.py b/iaso/utils/jsonlogic.py index 3f4888ba72..1c9f032770 100644 --- a/iaso/utils/jsonlogic.py +++ b/iaso/utils/jsonlogic.py @@ -2,7 +2,7 @@ import operator from functools import reduce -from typing import Dict, Any +from typing import Any, Callable, Dict from django.db.models import Exists, Q, Transform, OuterRef from django.db.models.fields.json import KeyTransformTextLookupMixin, JSONField @@ -36,37 +36,34 @@ def as_sql(self, compiler, connection): JSONField.register_lookup(ExtractForceFloat) -def jsonlogic_to_q(jsonlogic: Dict[str, Any], field_prefix: str = "") -> Q: +def jsonlogic_to_q( + jsonlogic: Dict[str, Any], + field_prefix: str = "", + recursion_func: Callable = None, +) -> Q: """Converts a JsonLogic query to a Django Q object. :param jsonlogic: The JsonLogic query to convert, stored in a Python dict. Example: {"and": [{"==": [{"var": "gender"}, "F"]}, {"<": [{"var": "age"}, 25]}]} :param field_prefix: A prefix to add to all fields in the generated query. Useful to follow a relationship or to dig in a JSONField + :param recursion_func: Optionally specify a function to call for recursion, allowing this method to be "wrapped". By default, when no function is specified, it calls itself. :return: A Django Q object. """ + func = jsonlogic_to_q if recursion_func is None else recursion_func + if "and" in jsonlogic: return reduce( operator.and_, - (jsonlogic_to_q(subquery, field_prefix) for subquery in jsonlogic["and"]), + (func(subquery, field_prefix) for subquery in jsonlogic["and"]), ) elif "or" in jsonlogic: return reduce( operator.or_, - (jsonlogic_to_q(subquery, field_prefix) for subquery in jsonlogic["or"]), + (func(subquery, field_prefix) for subquery in jsonlogic["or"]), ) elif "!" in jsonlogic: - return ~jsonlogic_to_q(jsonlogic["!"], field_prefix) - elif "some" in jsonlogic: - from iaso.models import Instance - - form_var, conditions = jsonlogic["some"] - form_id = form_var["var"] - - subquery = Instance.objects.filter( - Q(entity_id=OuterRef("id")) & Q(form__form_id=form_id) & jsonlogic_to_q(conditions, field_prefix) - ) - return Exists(subquery) + return ~func(jsonlogic["!"], field_prefix) if not jsonlogic.keys(): return Q() @@ -116,3 +113,62 @@ def jsonlogic_to_q(jsonlogic: Dict[str, Any], field_prefix: str = "") -> Q: # invert the filter q = ~q return q + + +def entities_jsonlogic_to_q(jsonlogic: Dict[str, Any], field_prefix: str = "") -> Q: + """This enhances the jsonlogic_to_q() method to allow filtering entities on + the submitted values of their instances. + It also converts a JsonLogic query to a Django Q object. + + :param jsonlogic: The JsonLogic query to convert, stored in a Python dict. Example: + { + "and": [ + { + "some": [ + { "var": "form_1_id" }, + { + "and": [ + { "==": [{"var": "gender"}, "female"] }, + { "==": [{"var": "serie_id"}, "2"] } + ] + } + ] + }, + { + "some": [ + { "var": "form_2_id" }, + { "==": [{"var": "result"}, "negative"] } + ] + } + ] + } + + :return: A Django Q object. + """ + from iaso.models import Instance + + if "some" in jsonlogic or "all" in jsonlogic or "none" in jsonlogic: + operator = list(jsonlogic.keys())[0] # there's only 1 key + form_var, conditions = jsonlogic[operator] + form_id = form_var["var"] + + form_id_filter = Q(entity_id=OuterRef("id")) & Q(form__form_id=form_id) + + if operator == "some": + return Exists(Instance.objects.filter(form_id_filter & entities_jsonlogic_to_q(conditions, field_prefix))) + elif operator == "all": + # In case of "all", we do a double filter: + # - EXIST on the form without conditions to exclude entities that don't have the form + # - NOT EXIST on the form with inverted conditions, so only get forms that only have + # the desired conditions + return Exists(Instance.objects.filter(form_id_filter)) & ~Exists( + Instance.objects.filter(form_id_filter & ~entities_jsonlogic_to_q(conditions, field_prefix)) + ) + elif operator == "none": + return ~Exists(Instance.objects.filter(form_id_filter & entities_jsonlogic_to_q(conditions, field_prefix))) + else: + return jsonlogic_to_q( + jsonlogic, + field_prefix="json__", + recursion_func=entities_jsonlogic_to_q, + ) From 5430f21693613068da04bdbf2161eaf482c9dac2 Mon Sep 17 00:00:00 2001 From: Bram Jans Date: Wed, 7 Aug 2024 11:58:04 +0200 Subject: [PATCH 31/55] SLEEP-1458 Small design improvements of filters --- .../domains/entities/components/Filters.tsx | 90 +++++++++++-------- .../hooks/useGetQueryBuilderListToReplace.ts | 10 ++- .../components/InstancesFiltersComponent.js | 8 +- 3 files changed, 64 insertions(+), 44 deletions(-) diff --git a/hat/assets/js/apps/Iaso/domains/entities/components/Filters.tsx b/hat/assets/js/apps/Iaso/domains/entities/components/Filters.tsx index beb5d57067..ebc7451d80 100644 --- a/hat/assets/js/apps/Iaso/domains/entities/components/Filters.tsx +++ b/hat/assets/js/apps/Iaso/domains/entities/components/Filters.tsx @@ -255,19 +255,6 @@ const Filters: FunctionComponent = ({ params, isFetching }) => { initialSelection={initialOrgUnit} /> - - handleChange('fieldsSearch', undefined), - }} - InfoPopper={} - /> {params.tab === 'map' && ( = ({ params, isFetching }) => { options={usersOptions} /> + - - - - - - + + + + handleChange('fieldsSearch', undefined), + }} + InfoPopper={} /> - + + + + + + + + + - + ); }; diff --git a/hat/assets/js/apps/Iaso/domains/forms/fields/hooks/useGetQueryBuilderListToReplace.ts b/hat/assets/js/apps/Iaso/domains/forms/fields/hooks/useGetQueryBuilderListToReplace.ts index 0a59c0568f..57509707b7 100644 --- a/hat/assets/js/apps/Iaso/domains/forms/fields/hooks/useGetQueryBuilderListToReplace.ts +++ b/hat/assets/js/apps/Iaso/domains/forms/fields/hooks/useGetQueryBuilderListToReplace.ts @@ -1,5 +1,5 @@ import { QueryBuilderListToReplace } from 'bluesquare-components'; -import { purple, blue } from '@mui/material/colors'; +import { blue, purple, green, red } from '@mui/material/colors'; export const useGetQueryBuilderListToReplace = (): QueryBuilderListToReplace[] => { @@ -8,6 +8,14 @@ export const useGetQueryBuilderListToReplace = color: purple[700], items: ['AND', 'OR'], }, + { + color: green[700], + items: ['SOME OF', 'ALL OF', 'HAVE'], + }, + { + color: red[700], + items: ['NONE OF'], + }, { color: blue[700], items: [ diff --git a/hat/assets/js/apps/Iaso/domains/instances/components/InstancesFiltersComponent.js b/hat/assets/js/apps/Iaso/domains/instances/components/InstancesFiltersComponent.js index e3c2efbe06..5427501582 100644 --- a/hat/assets/js/apps/Iaso/domains/instances/components/InstancesFiltersComponent.js +++ b/hat/assets/js/apps/Iaso/domains/instances/components/InstancesFiltersComponent.js @@ -268,7 +268,7 @@ const InstancesFiltersComponent = ({ - + - + - + - + Date: Thu, 8 Aug 2024 13:57:38 +0200 Subject: [PATCH 32/55] SLEEP-1458 Improvement: Use form names instead of form_id for the entities filter. --- .../fields/hooks/useGetQueryBuildersFields.ts | 7 ++++--- .../domains/forms/hooks/useGetPossibleFields.ts | 15 ++++++++++++--- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/hat/assets/js/apps/Iaso/domains/forms/fields/hooks/useGetQueryBuildersFields.ts b/hat/assets/js/apps/Iaso/domains/forms/fields/hooks/useGetQueryBuildersFields.ts index cb3d73f243..aa5a0205bb 100644 --- a/hat/assets/js/apps/Iaso/domains/forms/fields/hooks/useGetQueryBuildersFields.ts +++ b/hat/assets/js/apps/Iaso/domains/forms/fields/hooks/useGetQueryBuildersFields.ts @@ -3,6 +3,7 @@ import { QueryBuilderFields } from 'bluesquare-components'; import { formatLabel } from '../../../instances/utils'; import { FormDescriptor, PossibleField } from '../../types/forms'; +import { PossibleFieldsForForm } from '../../hooks/useGetPossibleFields'; import { iasoFields, Field } from '../constants'; import { findDescriptorInChildren } from '../../../../utils'; @@ -57,19 +58,19 @@ export const useGetQueryBuildersFields = ( export const useGetQueryBuilderFieldsForAllForms = ( formDescriptors?: FormDescriptor[], - allPossibleFields?: PossibleField[], + allPossibleFields?: PossibleFieldsForForm[], ): QueryBuilderFields => { if (!allPossibleFields || !formDescriptors) return {}; const fields: QueryBuilderFields = {}; - for (const [form_id, possibleFields] of Object.entries(allPossibleFields)) { + for (const { form_id, name, possibleFields } of allPossibleFields) { const subfields = useGetQueryBuildersFields( formDescriptors, possibleFields, ); fields[form_id] = { - label: form_id, + label: name, type: '!group', mode: 'array', conjunctions: ['AND', 'OR'], diff --git a/hat/assets/js/apps/Iaso/domains/forms/hooks/useGetPossibleFields.ts b/hat/assets/js/apps/Iaso/domains/forms/hooks/useGetPossibleFields.ts index 815dd35d2d..d078430ce8 100644 --- a/hat/assets/js/apps/Iaso/domains/forms/hooks/useGetPossibleFields.ts +++ b/hat/assets/js/apps/Iaso/domains/forms/hooks/useGetPossibleFields.ts @@ -17,8 +17,13 @@ type Result = { possibleFields: PossibleField[]; isFetchingForm: boolean; }; +export type PossibleFieldsForForm = { + form_id: string; + name: string; + possibleFields: PossibleField[]; +}; type AllResults = { - allPossibleFields: any; // TODO + allPossibleFields: PossibleFieldsForForm[]; isFetchingForms: boolean; }; @@ -54,14 +59,18 @@ export const useAllPossibleFields = ( allForms: Form[] = [], ): AllResults => { return useMemo(() => { - let allPossibleFields = {}; + let allPossibleFields: PossibleFieldsForForm[] = []; allForms.forEach(form => { const possibleFields = form?.possible_fields?.map(field => ({ ...field, fieldKey: field.name.replace('.', ''), })) || []; - allPossibleFields[form.form_id] = possibleFields; + allPossibleFields.push({ + form_id: form.form_id, + name: form.name, + possibleFields, + }); }); return { From 44d774adbc94b29a17a482505f50228b18e0a901 Mon Sep 17 00:00:00 2001 From: Bram Jans Date: Thu, 8 Aug 2024 15:15:19 +0200 Subject: [PATCH 33/55] SLEEP-1458 Update bluesquare-components To get updates on the QueryBuilder. --- package-lock.json | 48 +++++++++++++++++++++++------------------------ package.json | 4 ++-- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/package-lock.json b/package-lock.json index ea64829765..600ad79646 100644 --- a/package-lock.json +++ b/package-lock.json @@ -30,7 +30,7 @@ "abortcontroller-polyfill": "^1.7.5", "array-flat-polyfill": "^1.0.1", "babel-plugin-formatjs": "^10.3.4", - "bluesquare-components": "github:BLSQ/bluesquare-components#IA-2850_react-router_update_dev", + "bluesquare-components": "github:BLSQ/bluesquare-components#SLEEP-1458-entities-filtering", "classnames": "^2.2.6", "color": "^3.1.2", "dom-to-pdf": "^0.3.1", @@ -3909,9 +3909,9 @@ } }, "node_modules/@react-awesome-query-builder/core": { - "version": "6.5.2", - "resolved": "https://registry.npmjs.org/@react-awesome-query-builder/core/-/core-6.5.2.tgz", - "integrity": "sha512-Ijpo/uO6upsclgIRLI8YLf+coFWkfjAS/WRB7+qeXmMX9MsaqKlYa8tk9dzYtGKK3msr6bdwCiMPlsd9CNDhrg==", + "version": "6.6.3", + "resolved": "https://registry.npmjs.org/@react-awesome-query-builder/core/-/core-6.6.3.tgz", + "integrity": "sha512-+631G0kojyUfJYNiMU/Su/bJPiwFA9o5XEQXr5VGxIJDnfs+zU+iqg2CeUlqknRKOdRgqylJbi5ZXKVqRLDsxA==", "dependencies": { "@babel/runtime": "^7.24.5", "clone": "^2.1.2", @@ -3925,11 +3925,11 @@ } }, "node_modules/@react-awesome-query-builder/mui": { - "version": "6.5.2", - "resolved": "https://registry.npmjs.org/@react-awesome-query-builder/mui/-/mui-6.5.2.tgz", - "integrity": "sha512-5iYxZnBRAbUwbV6DJiI2+0FYdAJybRZmWd/qZqrXCxq/0DN4mVqp7/AF2x4Wk2ggDy0bo5JLHPIgpUONVKBg9A==", + "version": "6.6.3", + "resolved": "https://registry.npmjs.org/@react-awesome-query-builder/mui/-/mui-6.6.3.tgz", + "integrity": "sha512-WI1sRwdooseztVCyvO+vWLx3KFI9LPiM2041qpVfWWP3Ot2lrTAxRuyxVOGOVZOuMClFhps3kqVGF/72QdD/Dg==", "dependencies": { - "@react-awesome-query-builder/ui": "^6.5.2", + "@react-awesome-query-builder/ui": "^6.6.3", "lodash": "^4.17.21", "material-ui-confirm": "^3.0.5" }, @@ -3939,17 +3939,17 @@ "@mui/base": "^5.0.0-alpha.87", "@mui/icons-material": "^5.2.4", "@mui/material": "^5.2.4", - "@mui/x-date-pickers": "^5.0.0-beta.2 || ^6.0.0", + "@mui/x-date-pickers": "^5.0.0-beta.2 || ^6.0.0 || ^7.0.0", "react": "^16.8.4 || ^17.0.1 || ^18.0.0", "react-dom": "^16.8.4 || ^17.0.1 || ^18.0.0" } }, "node_modules/@react-awesome-query-builder/ui": { - "version": "6.5.2", - "resolved": "https://registry.npmjs.org/@react-awesome-query-builder/ui/-/ui-6.5.2.tgz", - "integrity": "sha512-onex0K8Ji/nZC+LrR2z7WwuUvo/selzb9aphFBRgGbzrdjkavag68h6sf1tMdaNv1Fki2fuvQcl8q4glCkVOeg==", + "version": "6.6.3", + "resolved": "https://registry.npmjs.org/@react-awesome-query-builder/ui/-/ui-6.6.3.tgz", + "integrity": "sha512-cuCKn8Mto5vTjfmvJrFiM/ETopLXcPnyQ2PKc8gt+dpfLs7UzYLWBF+V72aYMiPfAMA27Jf2+aPNEGD2x0LemA==", "dependencies": { - "@react-awesome-query-builder/core": "^6.5.2", + "@react-awesome-query-builder/core": "^6.6.3", "classnames": "^2.5.1", "lodash": "^4.17.21", "prop-types": "^15.8.1", @@ -6437,7 +6437,7 @@ }, "node_modules/bluesquare-components": { "version": "1.0.0", - "resolved": "git+ssh://git@github.com/BLSQ/bluesquare-components.git#c11a6f8d00363868960e77393cc3b896c934ac74", + "resolved": "git+ssh://git@github.com/BLSQ/bluesquare-components.git#7d971cd44fd0f8203587570fd3ee37f326708558", "dependencies": { "@babel/plugin-transform-class-properties": "^7.24.6", "@babel/plugin-transform-object-rest-spread": "^7.24.6", @@ -6447,7 +6447,7 @@ "@dnd-kit/core": "^6.0.8", "@dnd-kit/modifiers": "^6.0.0", "@dnd-kit/sortable": "^7.0.1", - "@react-awesome-query-builder/mui": "^6.4.2", + "@react-awesome-query-builder/mui": "^6.6.2", "babel-plugin-formatjs": "^10.3.18", "dayjs": "^1.11.10", "install": "^0.13.0", @@ -11275,9 +11275,9 @@ "integrity": "sha512-fedL7PRwmeVkgyhu9hLeTBaI6wcGk7JGJswdaRsa5aUbkXI1kr1xZwTPBtaYPpwf56878iDek6VbVnuWMebJmw==" }, "node_modules/i18next": { - "version": "23.11.5", - "resolved": "https://registry.npmjs.org/i18next/-/i18next-23.11.5.tgz", - "integrity": "sha512-41pvpVbW9rhZPk5xjCX2TPJi2861LEig/YRhUkY+1FQ2IQPS0bKUDYnEqY8XPPbB48h1uIwLnP9iiEfuSl20CA==", + "version": "23.14.0", + "resolved": "https://registry.npmjs.org/i18next/-/i18next-23.14.0.tgz", + "integrity": "sha512-Y5GL4OdA8IU2geRrt2+Uc1iIhsjICdHZzT9tNwQ3TVqdNzgxHToGCKf/TPRP80vTCAP6svg2WbbJL+Gx5MFQVA==", "funding": [ { "type": "individual", @@ -11350,9 +11350,9 @@ } }, "node_modules/immutable": { - "version": "4.3.6", - "resolved": "https://registry.npmjs.org/immutable/-/immutable-4.3.6.tgz", - "integrity": "sha512-Ju0+lEMyzMVZarkTn/gqRpdqd5dOPaz1mCZ0SH3JV6iFw81PldE/PEB1hWVEA288HPt4WXW8O7AWxB10M+03QQ==" + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/immutable/-/immutable-4.3.7.tgz", + "integrity": "sha512-1hqclzwYwjRDFLjcFxOM5AYkkG0rpFPpr1RLPMEuGczoS7YA8gLhy8SWXYRAA/XwfEHpfo3cw5JGioS32fnMRw==" }, "node_modules/import-fresh": { "version": "3.3.0", @@ -12518,9 +12518,9 @@ "dev": true }, "node_modules/json-logic-js": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/json-logic-js/-/json-logic-js-2.0.2.tgz", - "integrity": "sha512-ZBtBdMJieqQcH7IX/LaBsr5pX+Y5JIW+EhejtM3Ffg2jdN9Iwf+Ht6TbHnvAZ/YtwyuhPaCBlnvzrwVeWdvGDQ==" + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/json-logic-js/-/json-logic-js-2.0.5.tgz", + "integrity": "sha512-rTT2+lqcuUmj4DgWfmzupZqQDA64AdmYqizzMPWj3DxGdfFNsxPpcNVSaTj4l8W2tG/+hg7/mQhxjU3aPacO6g==" }, "node_modules/json-parse-even-better-errors": { "version": "2.3.1", diff --git a/package.json b/package.json index fc7e573645..2f178f19f8 100644 --- a/package.json +++ b/package.json @@ -69,7 +69,7 @@ "abortcontroller-polyfill": "^1.7.5", "array-flat-polyfill": "^1.0.1", "babel-plugin-formatjs": "^10.3.4", - "bluesquare-components": "github:BLSQ/bluesquare-components#IA-2850_react-router_update_dev", + "bluesquare-components": "github:BLSQ/bluesquare-components#SLEEP-1458-entities-filtering", "classnames": "^2.2.6", "color": "^3.1.2", "dom-to-pdf": "^0.3.1", @@ -165,4 +165,4 @@ "webpack-cli": "^4.10.0", "webpack-dev-server": "^4.15.1" } -} \ No newline at end of file +} From 22ccc951f62ba82da53f7ebf56866728427c59ac Mon Sep 17 00:00:00 2001 From: Bram Jans Date: Wed, 4 Sep 2024 13:38:46 +0200 Subject: [PATCH 34/55] SLEEP-1458 PR feedback from @quang-le --- .../domains/entities/components/Filters.tsx | 4 +-- .../entityTypes/hooks/requests/forms.ts | 8 ++--- .../fields/hooks/useGetFormDescriptor.ts | 32 ++++++++++++++----- .../forms/hooks/useGetPossibleFields.ts | 9 ++++-- 4 files changed, 35 insertions(+), 18 deletions(-) diff --git a/hat/assets/js/apps/Iaso/domains/entities/components/Filters.tsx b/hat/assets/js/apps/Iaso/domains/entities/components/Filters.tsx index ebc7451d80..92e7368d26 100644 --- a/hat/assets/js/apps/Iaso/domains/entities/components/Filters.tsx +++ b/hat/assets/js/apps/Iaso/domains/entities/components/Filters.tsx @@ -52,7 +52,7 @@ import { import { Popper } from '../../forms/fields/components/Popper'; import { parseJson } from '../../instances/utils/jsonLogicParse'; import { useGetAllPossibleFields } from '../../forms/hooks/useGetPossibleFields'; -import { useGetFormDescriptor } from '../../forms/fields/hooks/useGetFormDescriptor'; +import { useGetAllFormDescriptors } from '../../forms/fields/hooks/useGetFormDescriptor'; import { useGetQueryBuilderFieldsForAllForms } from '../../forms/fields/hooks/useGetQueryBuildersFields'; import { useGetQueryBuilderListToReplace } from '../../forms/fields/hooks/useGetQueryBuilderListToReplace'; @@ -140,7 +140,7 @@ const Filters: FunctionComponent = ({ params, isFetching }) => { // Load QueryBuilder resources const { allPossibleFields } = useGetAllPossibleFields(); - const { data: formDescriptors } = useGetFormDescriptor(); + const { data: formDescriptors } = useGetAllFormDescriptors(); const fields = useGetQueryBuilderFieldsForAllForms( formDescriptors, allPossibleFields, diff --git a/hat/assets/js/apps/Iaso/domains/entities/entityTypes/hooks/requests/forms.ts b/hat/assets/js/apps/Iaso/domains/entities/entityTypes/hooks/requests/forms.ts index 9e4fcbdc70..d27011eebe 100644 --- a/hat/assets/js/apps/Iaso/domains/entities/entityTypes/hooks/requests/forms.ts +++ b/hat/assets/js/apps/Iaso/domains/entities/entityTypes/hooks/requests/forms.ts @@ -6,7 +6,6 @@ import { getRequest } from '../../../../../libs/Api'; import { Form, PossibleField } from '../../../../forms/types/forms'; import { usePossibleFields } from '../../../../forms/hooks/useGetPossibleFields'; - export const useGetForm = ( formId: number | undefined, enabled: boolean, @@ -37,10 +36,9 @@ export const useGetForms = ( enabled: boolean, fields?: string[] | undefined, ): UseQueryResult => { - let url = '/api/forms/?fields=id,name,latest_form_version'; - if (fields) { - url += `,${fields.join(',')}`; - } + const apiUrl = '/api/forms/?fields=id,name,latest_form_version'; + const url = fields ? `${apiUrl},${fields.join(',')}` : apiUrl; + return useSnackQuery({ queryKey: ['forms'], queryFn: () => getRequest(url), diff --git a/hat/assets/js/apps/Iaso/domains/forms/fields/hooks/useGetFormDescriptor.ts b/hat/assets/js/apps/Iaso/domains/forms/fields/hooks/useGetFormDescriptor.ts index a94c467cdc..1c542c0097 100644 --- a/hat/assets/js/apps/Iaso/domains/forms/fields/hooks/useGetFormDescriptor.ts +++ b/hat/assets/js/apps/Iaso/domains/forms/fields/hooks/useGetFormDescriptor.ts @@ -12,8 +12,16 @@ type FormVersionsList = { }; const getVersion = (formId: number | undefined): Promise => { - return getRequest(`/api/formversions/?form_id=&fields=descriptor`); + return getRequest(`/api/formversions/?form_id=${formId}&fields=descriptor`); }; + +const processResult = ( + data: FormVersionsList | undefined, +): FormDescriptor[] | undefined => { + if (!data) return data; + return data.form_versions?.map(version => version.descriptor); +}; + export const useGetFormDescriptor = ( formId?: number, ): UseQueryResult => { @@ -25,13 +33,21 @@ export const useGetFormDescriptor = ( queryKey, queryFn: () => getVersion(formId), options: { - // enabled: Boolean(formId), TODO - select: ( - data: FormVersionsList | undefined, - ): FormDescriptor[] | undefined => { - if (!data) return data; - return data.form_versions?.map(version => version.descriptor); - }, + enabled: Boolean(formId), + select: data => processResult(data), + }, + }); +}; + +export const useGetAllFormDescriptors = (): UseQueryResult< + FormDescriptor[] | undefined, + Error +> => { + return useSnackQuery({ + queryKey: ['instanceDescriptors'], + queryFn: () => getRequest('/api/formversions/?fields=descriptor'), + options: { + select: data => processResult(data), }, }); }; diff --git a/hat/assets/js/apps/Iaso/domains/forms/hooks/useGetPossibleFields.ts b/hat/assets/js/apps/Iaso/domains/forms/hooks/useGetPossibleFields.ts index d078430ce8..3fe780dd8f 100644 --- a/hat/assets/js/apps/Iaso/domains/forms/hooks/useGetPossibleFields.ts +++ b/hat/assets/js/apps/Iaso/domains/forms/hooks/useGetPossibleFields.ts @@ -44,7 +44,10 @@ export const usePossibleFields = ( }, [form?.possible_fields, isFetchingForm]); }; -export const useGetPossibleFields = (formId?: number, appId?: string): Result => { +export const useGetPossibleFields = ( + formId?: number, + appId?: string, +): Result => { const { data: currentForm, isFetching: isFetchingForm } = useGetForm( formId, Boolean(formId), @@ -59,7 +62,7 @@ export const useAllPossibleFields = ( allForms: Form[] = [], ): AllResults => { return useMemo(() => { - let allPossibleFields: PossibleFieldsForForm[] = []; + const allPossibleFields: PossibleFieldsForForm[] = []; allForms.forEach(form => { const possibleFields = form?.possible_fields?.map(field => ({ @@ -77,7 +80,7 @@ export const useAllPossibleFields = ( allPossibleFields, isFetchingForms, }; - }, [isFetchingForms]); + }, [isFetchingForms, allForms]); }; export const useGetAllPossibleFields = (): AllResults => { From 5cd686444ff5f9f450d2a1c67156c8da92ecbf50 Mon Sep 17 00:00:00 2001 From: Bram Jans Date: Wed, 4 Sep 2024 13:39:31 +0200 Subject: [PATCH 35/55] SLEEP-1458 PR feedback from @kemar --- iaso/utils/jsonlogic.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/iaso/utils/jsonlogic.py b/iaso/utils/jsonlogic.py index 1c9f032770..db8bf301c5 100644 --- a/iaso/utils/jsonlogic.py +++ b/iaso/utils/jsonlogic.py @@ -1,7 +1,6 @@ """JsonLogic(https://jsonlogic.com/)-related utilities.""" import operator -from functools import reduce from typing import Any, Callable, Dict from django.db.models import Exists, Q, Transform, OuterRef @@ -53,15 +52,15 @@ def jsonlogic_to_q( func = jsonlogic_to_q if recursion_func is None else recursion_func if "and" in jsonlogic: - return reduce( - operator.and_, - (func(subquery, field_prefix) for subquery in jsonlogic["and"]), - ) + sub_query = Q() + for lookup in jsonlogic["and"]: + sub_query = operator.and_(sub_query, func(lookup, field_prefix)) + return sub_query elif "or" in jsonlogic: - return reduce( - operator.or_, - (func(subquery, field_prefix) for subquery in jsonlogic["or"]), - ) + sub_query = Q() + for lookup in jsonlogic["or"]: + sub_query = operator.or_(sub_query, func(lookup, field_prefix)) + return sub_query elif "!" in jsonlogic: return ~func(jsonlogic["!"], field_prefix) From 3ee5a4eadf5e27206f2c6c200c74a996af0afa7b Mon Sep 17 00:00:00 2001 From: Beygorghor Date: Tue, 10 Sep 2024 09:26:51 +0200 Subject: [PATCH 36/55] Use dedicated page for details --- hat/assets/js/apps/Iaso/constants/routes.tsx | 11 +- hat/assets/js/apps/Iaso/constants/urls.ts | 5 + .../components/InstanceFileContentRich.js | 2 + .../Dialogs/HighlightFieldsChanges.tsx | 2 +- .../Dialogs/ReviewOrgUnitChangesDialog.tsx | 135 ------------------ .../Tables/ReviewOrgUnitChangesTable.tsx | 90 ++++-------- .../ReviewOrgUnitChangesDetailsTable.tsx | 10 +- .../ReviewOrgUnitChangesDetailsTableRow.tsx | 8 +- .../orgUnits/reviewChanges/details.tsx | 132 +++++++++++++++++ .../hooks/api/useSaveChangeRequest.ts | 4 +- .../{ReviewOrgUnitChanges.tsx => index.tsx} | 0 .../domains/orgUnits/reviewChanges/types.ts | 7 +- hat/webpack.dev.js | 1 - 13 files changed, 198 insertions(+), 209 deletions(-) delete mode 100644 hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Dialogs/ReviewOrgUnitChangesDialog.tsx create mode 100644 hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/details.tsx rename hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/{ReviewOrgUnitChanges.tsx => index.tsx} (100%) diff --git a/hat/assets/js/apps/Iaso/constants/routes.tsx b/hat/assets/js/apps/Iaso/constants/routes.tsx index 1d51c0f876..83269f9d34 100644 --- a/hat/assets/js/apps/Iaso/constants/routes.tsx +++ b/hat/assets/js/apps/Iaso/constants/routes.tsx @@ -28,7 +28,8 @@ import { OrgUnits } from '../domains/orgUnits'; import OrgUnitDetail from '../domains/orgUnits/details'; import Groups from '../domains/orgUnits/groups'; import Types from '../domains/orgUnits/orgUnitTypes'; -import { ReviewOrgUnitChanges } from '../domains/orgUnits/reviewChanges/ReviewOrgUnitChanges'; +import { ReviewOrgUnitChanges } from '../domains/orgUnits/reviewChanges'; +import { ReviewOrgUnitChangesDetail } from '../domains/orgUnits/reviewChanges/details'; import Pages from '../domains/pages'; import { LotsPayments } from '../domains/payments/LotsPayments'; import { PotentialPayments } from '../domains/payments/PotentialPayments'; @@ -170,6 +171,13 @@ export const orgUnitChangeRequestPath = { element: , }; +export const orgUnitChangeRequestDetailPath = { + baseUrl: baseUrls.orgUnitsChangeRequestDetail, + routerUrl: `${baseUrls.orgUnitsChangeRequestDetail}/*`, + permissions: [Permission.ORG_UNITS_CHANGE_REQUEST_REVIEW], + element: , +}; + export const registryPath = { baseUrl: baseUrls.registry, routerUrl: `${baseUrls.registry}/*`, @@ -439,6 +447,7 @@ export const routeConfigs: (RoutePath | AnonymousRoutePath)[] = [ workflowsPath, workflowsDetailPath, orgUnitChangeRequestPath, + orgUnitChangeRequestDetailPath, registryPath, modulesPath, potentialPaymentsPath, diff --git a/hat/assets/js/apps/Iaso/constants/urls.ts b/hat/assets/js/apps/Iaso/constants/urls.ts index 3bab09452d..0c771f94fa 100644 --- a/hat/assets/js/apps/Iaso/constants/urls.ts +++ b/hat/assets/js/apps/Iaso/constants/urls.ts @@ -184,6 +184,10 @@ export const baseRouteConfigs: Record = { 'potentialPaymentIds', ], }, + orgUnitsChangeRequestDetail: { + url: `${ORG_UNITS_CHANGE_REQUEST}/detail`, + params: ['accountId', 'changeRequestId'], + }, registry: { url: 'orgunits/registry', params: [ @@ -521,6 +525,7 @@ type IasoBaseUrls = { orgUnits: string; orgUnitDetails: string; orgUnitsChangeRequest: string; + orgUnitsChangeRequestDetail: string; registry: string; registryDetail: string; links: string; diff --git a/hat/assets/js/apps/Iaso/domains/instances/components/InstanceFileContentRich.js b/hat/assets/js/apps/Iaso/domains/instances/components/InstanceFileContentRich.js index 0b1f057634..3d6017e840 100644 --- a/hat/assets/js/apps/Iaso/domains/instances/components/InstanceFileContentRich.js +++ b/hat/assets/js/apps/Iaso/domains/instances/components/InstanceFileContentRich.js @@ -196,6 +196,8 @@ const PhotoField = ({ descriptor, data, showQuestionKey, files }) => { maxWidth: '35vw', maxHeight: '35vh', cursor: 'pointer', + width: '100%', + height: 'auto', }} onClick={() => setOpen(true)} /> diff --git a/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Dialogs/HighlightFieldsChanges.tsx b/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Dialogs/HighlightFieldsChanges.tsx index 7d7bb5a41d..8115a135cd 100644 --- a/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Dialogs/HighlightFieldsChanges.tsx +++ b/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Dialogs/HighlightFieldsChanges.tsx @@ -66,7 +66,7 @@ export const HighlightFields: FunctionComponent = ({ }, [fieldType, oldFieldValues, newFieldValues]); return ( - + {field.label} {(fieldType && fieldType === 'array' && ( diff --git a/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Dialogs/ReviewOrgUnitChangesDialog.tsx b/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Dialogs/ReviewOrgUnitChangesDialog.tsx deleted file mode 100644 index 03265bdb7a..0000000000 --- a/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Dialogs/ReviewOrgUnitChangesDialog.tsx +++ /dev/null @@ -1,135 +0,0 @@ -import React, { - FunctionComponent, - useCallback, - Dispatch, - SetStateAction, - useMemo, -} from 'react'; -import { - useSafeIntl, - SimpleModal, - IconButton as IconButtonBlsq, -} from 'bluesquare-components'; -import MESSAGES from '../messages'; -import { SelectedChangeRequest } from '../Tables/ReviewOrgUnitChangesTable'; -import { useNewFields } from '../hooks/useNewFields'; -import { ApproveOrgUnitChangesButtons } from './ReviewOrgUnitChangesButtons'; -import { ChangeRequestValidationStatus } from '../types'; -import { useSaveChangeRequest } from '../hooks/api/useSaveChangeRequest'; -import { ReviewOrgUnitChangesDetailsTable } from '../Tables/details/ReviewOrgUnitChangesDetailsTable'; -import { ReviewOrgUnitChangesDialogTitle } from './ReviewOrgUnitChangesDialogTitle'; -import { useGetApprovalProposal } from '../hooks/api/useGetApprovalProposal'; - -type Props = { - isOpen: boolean; - closeDialog: () => void; - selectedChangeRequest: SelectedChangeRequest; -}; - -export const ReviewOrgUnitChangesDialog: FunctionComponent = ({ - isOpen, - closeDialog, - selectedChangeRequest, -}) => { - const { formatMessage } = useSafeIntl(); - - const { data: changeRequest, isFetching: isFetchingChangeRequest } = - useGetApprovalProposal(selectedChangeRequest.id); - const isNew: boolean = - !isFetchingChangeRequest && changeRequest?.status === 'new'; - const isNewOrgUnit = changeRequest - ? changeRequest.org_unit.validation_status === 'NEW' - : false; - const { newFields, setSelected } = useNewFields(changeRequest); - const titleMessage = useMemo(() => { - if (changeRequest?.status === 'rejected') { - return formatMessage(MESSAGES.seeRejectedChanges); - } - if (changeRequest?.status === 'approved') { - return formatMessage(MESSAGES.seeApprovedChanges); - } - if (isNewOrgUnit) { - return formatMessage(MESSAGES.validateOrRejectNewOrgUnit); - } - return formatMessage(MESSAGES.validateOrRejectChanges); - }, [changeRequest?.status, formatMessage, isNewOrgUnit]); - const { mutate: submitChangeRequest, isLoading: isSaving } = - useSaveChangeRequest(closeDialog, selectedChangeRequest.id); - - return ( - null} - id="approve-orgunit-changes-dialog" - dataTestId="approve-orgunit-changes-dialog" - titleMessage={ - - } - closeDialog={closeDialog} - buttons={() => - isFetchingChangeRequest ? ( - <> - ) : ( - - ) - } - > - - - ); -}; - -type PropsIcon = { - changeRequestId: number; - setSelectedChangeRequest: Dispatch>; - index: number; - status: ChangeRequestValidationStatus; -}; - -export const IconButton: FunctionComponent = ({ - setSelectedChangeRequest, - changeRequestId, - index, - status, -}) => { - const handleClick = useCallback(() => { - setSelectedChangeRequest({ - id: changeRequestId, - index, - }); - }, [changeRequestId, index, setSelectedChangeRequest]); - let message = MESSAGES.validateOrRejectChanges; - if (status === 'rejected') { - message = MESSAGES.seeRejectedChanges; - } - if (status === 'approved') { - message = MESSAGES.seeApprovedChanges; - } - return ( - - ); -}; diff --git a/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Tables/ReviewOrgUnitChangesTable.tsx b/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Tables/ReviewOrgUnitChangesTable.tsx index c65f0b6951..d9e7f61aa1 100644 --- a/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Tables/ReviewOrgUnitChangesTable.tsx +++ b/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Tables/ReviewOrgUnitChangesTable.tsx @@ -1,37 +1,25 @@ -import React, { - FunctionComponent, - ReactElement, - useState, - Dispatch, - SetStateAction, - useMemo, -} from 'react'; -import Color from 'color'; -import { Column, textPlaceholder, useSafeIntl } from 'bluesquare-components'; import { Box } from '@mui/material'; +import { Column, textPlaceholder, useSafeIntl } from 'bluesquare-components'; +import Color from 'color'; +import React, { FunctionComponent, ReactElement, useMemo } from 'react'; +import { BreakWordCell } from '../../../../components/Cells/BreakWordCell'; +import { DateTimeCell } from '../../../../components/Cells/DateTimeCell'; +import { UserCell } from '../../../../components/Cells/UserCell'; import { TableWithDeepLink } from '../../../../components/tables/TableWithDeepLink'; import { baseUrls } from '../../../../constants/urls'; +import { ColumnCell } from '../../../../types/general'; +import { LinkToOrgUnit } from '../../components/LinkToOrgUnit'; +import { IconButton } from '../details'; +import { colorCodes } from '../Dialogs/ReviewOrgUnitChangesInfos'; +import MESSAGES from '../messages'; import { - OrgUnitChangeRequestsPaginated, ApproveOrgUnitParams, - OrgUnitChangeRequest, ChangeRequestValidationStatus, + OrgUnitChangeRequest, + OrgUnitChangeRequestsPaginated, } from '../types'; -import MESSAGES from '../messages'; -import { LinkToOrgUnit } from '../../components/LinkToOrgUnit'; -import { DateTimeCell } from '../../../../components/Cells/DateTimeCell'; -import { - ReviewOrgUnitChangesDialog, - IconButton, -} from '../Dialogs/ReviewOrgUnitChangesDialog'; -import { UserCell } from '../../../../components/Cells/UserCell'; -import { colorCodes } from '../Dialogs/ReviewOrgUnitChangesInfos'; -import { ColumnCell } from '../../../../types/general'; -import { BreakWordCell } from '../../../../components/Cells/BreakWordCell'; -const useColumns = ( - setSelectedChangeRequest: Dispatch>, -): Column[] => { +const useColumns = (): Column[] => { const { formatMessage } = useSafeIntl(); return useMemo( () => [ @@ -154,20 +142,18 @@ const useColumns = ( accessor: 'actions', sortable: false, Cell: ({ - row: { original: changeRequest, index }, + row: { original: changeRequest }, }: ColumnCell): ReactElement => { return ( ); }, }, ], - [formatMessage, setSelectedChangeRequest], + [formatMessage], ); }; @@ -205,37 +191,21 @@ export const ReviewOrgUnitChangesTable: FunctionComponent = ({ isFetching, params, }) => { - const [selectedChangeRequest, setSelectedChangeRequest] = useState< - SelectedChangeRequest | undefined - >(); - const columns = useColumns(setSelectedChangeRequest); - const handleCloseDialog = () => { - setSelectedChangeRequest(undefined); - }; + const columns = useColumns(); return ( - <> - {/* This dialog is at this level to keep selected request in state and allow further multiaction/pagination feature */} - {selectedChangeRequest && ( - - )} - - + ); }; diff --git a/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Tables/details/ReviewOrgUnitChangesDetailsTable.tsx b/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Tables/details/ReviewOrgUnitChangesDetailsTable.tsx index c450efbe07..2caf50af6a 100644 --- a/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Tables/details/ReviewOrgUnitChangesDetailsTable.tsx +++ b/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Tables/details/ReviewOrgUnitChangesDetailsTable.tsx @@ -1,11 +1,11 @@ /* eslint-disable camelcase */ -import React, { FunctionComponent } from 'react'; +import { Table, TableContainer } from '@mui/material'; import { LoadingSpinner } from 'bluesquare-components'; -import { TableContainer, Table } from '@mui/material'; +import React, { FunctionComponent } from 'react'; import { NewOrgUnitField } from '../../hooks/useNewFields'; -import { ReviewOrgUnitChangesDetailsTableHead } from './ReviewOrgUnitChangesDetailsTableHead'; -import { ReviewOrgUnitChangesDetailsTableBody } from './ReviewOrgUnitChangesDetailsTableBody'; import { OrgUnitChangeRequestDetails } from '../../types'; +import { ReviewOrgUnitChangesDetailsTableBody } from './ReviewOrgUnitChangesDetailsTableBody'; +import { ReviewOrgUnitChangesDetailsTableHead } from './ReviewOrgUnitChangesDetailsTableHead'; type Props = { isSaving: boolean; @@ -28,7 +28,7 @@ export const ReviewOrgUnitChangesDetailsTable: FunctionComponent = ({ const isNew: boolean = !isFetchingChangeRequest && changeRequest?.status === 'new'; return ( - + {(isFetchingChangeRequest || isSaving) && ( )} diff --git a/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Tables/details/ReviewOrgUnitChangesDetailsTableRow.tsx b/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Tables/details/ReviewOrgUnitChangesDetailsTableRow.tsx index f4ffb8e298..759c840376 100644 --- a/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Tables/details/ReviewOrgUnitChangesDetailsTableRow.tsx +++ b/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Tables/details/ReviewOrgUnitChangesDetailsTableRow.tsx @@ -20,7 +20,6 @@ const useStyles = makeStyles(theme => ({ color: 'inherit', }, cellRejected: { - maxWidth: 350, '& > a': { color: `${theme.palette.error.main} !important`, }, @@ -38,7 +37,6 @@ const useStyles = makeStyles(theme => ({ }, }, cellApproved: { - maxWidth: 350, '& > a': { color: `${theme.palette.success.main} !important`, }, @@ -92,11 +90,15 @@ export const ReviewOrgUnitChangesDetailsTableRow: FunctionComponent = ({ return ( <> {!isNewOrgUnit && ( - + {field.oldValue} )} ({ + width: '100%', + height: `calc(100vh - 65px)`, + padding: theme.spacing(4), + margin: 0, + overflow: 'hidden', + backgroundColor: 'white', + [theme.breakpoints.down('md')]: { + padding: theme.spacing(2), + }, + }), + body: theme => ({ + width: '100%', + maxHeight: `calc(100vh - 200px)`, + padding: 0, + margin: 0, + overflow: 'auto', + border: `1px solid ${(theme.palette as any).border.main}`, + borderRadius: 2, + }), +}; + +export const ReviewOrgUnitChangesDetail: FunctionComponent = () => { + const { formatMessage } = useSafeIntl(); + + const params = useParamsObject( + baseUrls.orgUnitsChangeRequestDetail, + ) as unknown as OrgUnitChangeRequestDetailParams; + + const { data: changeRequest, isFetching: isFetchingChangeRequest } = + useGetApprovalProposal(Number(params.changeRequestId)); + const isNew: boolean = + !isFetchingChangeRequest && changeRequest?.status === 'new'; + const isNewOrgUnit = changeRequest + ? changeRequest.org_unit.validation_status === 'NEW' + : false; + const { newFields, setSelected } = useNewFields(changeRequest); + const goBack = useGoBack(baseUrls.orgUnitsChangeRequest); + const titleMessage = useMemo(() => { + if (changeRequest?.status === 'rejected') { + return formatMessage(MESSAGES.seeRejectedChanges); + } + if (changeRequest?.status === 'approved') { + return formatMessage(MESSAGES.seeApprovedChanges); + } + if (isNewOrgUnit) { + return formatMessage(MESSAGES.validateOrRejectNewOrgUnit); + } + return formatMessage(MESSAGES.validateOrRejectChanges); + }, [changeRequest?.status, formatMessage, isNewOrgUnit]); + const { mutate: submitChangeRequest, isLoading: isSaving } = + useSaveChangeRequest(() => goBack(), Number(params.changeRequestId)); + + return ( + + + + + + + + + + + ); +}; + +type PropsIcon = { + changeRequestId: number; + status: ChangeRequestValidationStatus; +}; + +export const IconButton: FunctionComponent = ({ + changeRequestId, + status, +}) => { + let message = MESSAGES.validateOrRejectChanges; + if (status === 'rejected') { + message = MESSAGES.seeRejectedChanges; + } + if (status === 'approved') { + message = MESSAGES.seeApprovedChanges; + } + return ( + + ); +}; diff --git a/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/hooks/api/useSaveChangeRequest.ts b/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/hooks/api/useSaveChangeRequest.ts index 30e7721173..290ca5caed 100644 --- a/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/hooks/api/useSaveChangeRequest.ts +++ b/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/hooks/api/useSaveChangeRequest.ts @@ -10,12 +10,12 @@ export type UseSaveChangeRequestQueryData = { }; export const useSaveChangeRequest = ( - closeDialog: () => void, + onSuccess: () => void, id: number, ): UseMutationResult => useSnackMutation({ mutationFn: (data: UseSaveChangeRequestQueryData) => patchRequest(`/api/orgunits/changes/${id}/`, data), invalidateQueryKey: ['getApprovalProposal', 'getApprovalProposals'], - options: { onSuccess: () => closeDialog() }, + options: { onSuccess }, }); diff --git a/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/ReviewOrgUnitChanges.tsx b/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/index.tsx similarity index 100% rename from hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/ReviewOrgUnitChanges.tsx rename to hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/index.tsx diff --git a/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/types.ts b/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/types.ts index e2aa4ebf2c..0110312d05 100644 --- a/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/types.ts +++ b/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/types.ts @@ -1,8 +1,8 @@ /* eslint-disable camelcase */ import { Pagination, UrlParams } from 'bluesquare-components'; import { User } from '../../../utils/usersUtils'; -import { OrgunitType } from '../types/orgunitTypes'; import { OrgUnitStatus, ShortOrgUnit } from '../types/orgUnit'; +import { OrgunitType } from '../types/orgunitTypes'; export type ChangeRequestValidationStatus = 'new' | 'rejected' | 'approved'; export type ApproveOrgUnitParams = UrlParams & { @@ -21,6 +21,11 @@ export type ApproveOrgUnitParams = UrlParams & { paymentIds?: string; // comma separated ids potentialPaymentIds?: string; // comma separated ids }; + +export type OrgUnitChangeRequestDetailParams = UrlParams & { + changeRequestId: string; +}; + export type Group = { id: number; name: string; diff --git a/hat/webpack.dev.js b/hat/webpack.dev.js index c30df611f7..cb3756e52a 100644 --- a/hat/webpack.dev.js +++ b/hat/webpack.dev.js @@ -8,7 +8,6 @@ const BundleTracker = require('webpack-bundle-tracker'); // remember to switch in webpack.prod.js and // django settings as well const LOCALE = 'fr'; - // If you launch the dev server with `WEBPACK_HOST=192.168.1.XXX npm run dev` // where 192.168.1.XXX is your local IP address, you can access the dev server // from another device on the same network, typically from a mobile device or tablet From 815d4bccadc2671cebb3b81b94930bf102d70330 Mon Sep 17 00:00:00 2001 From: Beygorghor Date: Tue, 10 Sep 2024 13:29:14 +0200 Subject: [PATCH 37/55] adapt title --- .../Iaso/domains/app/translations/en.json | 1 + .../Iaso/domains/app/translations/fr.json | 1 + .../ReviewOrgUnitChangesDialogTitle.tsx | 12 ++++++++--- .../orgUnits/reviewChanges/details.tsx | 21 ++++++++++++++----- .../orgUnits/reviewChanges/messages.ts | 4 ++++ 5 files changed, 31 insertions(+), 8 deletions(-) diff --git a/hat/assets/js/apps/Iaso/domains/app/translations/en.json b/hat/assets/js/apps/Iaso/domains/app/translations/en.json index f37ef7c2ce..d6352b2aca 100644 --- a/hat/assets/js/apps/Iaso/domains/app/translations/en.json +++ b/hat/assets/js/apps/Iaso/domains/app/translations/en.json @@ -639,6 +639,7 @@ "iaso.label.renderError": "Error rendering value", "iaso.label.restore": "Restore", "iaso.label.resultsLower": "result(s)", + "iaso.label.reviewChangeProposal": "Change proposals for {name}", "iaso.label.reviewChangeProposals": "Review change proposals", "iaso.label.save": "Save", "iaso.label.search": "Search", diff --git a/hat/assets/js/apps/Iaso/domains/app/translations/fr.json b/hat/assets/js/apps/Iaso/domains/app/translations/fr.json index a3110646df..b5807af107 100644 --- a/hat/assets/js/apps/Iaso/domains/app/translations/fr.json +++ b/hat/assets/js/apps/Iaso/domains/app/translations/fr.json @@ -639,6 +639,7 @@ "iaso.label.renderError": "Erreur de rendu de la valeur", "iaso.label.restore": "Restaurer", "iaso.label.resultsLower": "résultat(s)", + "iaso.label.reviewChangeProposal": "Propositions de changement pour {name}", "iaso.label.reviewChangeProposals": "Examiner les propositions de changement", "iaso.label.save": "Sauver", "iaso.label.search": "Recherche", diff --git a/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Dialogs/ReviewOrgUnitChangesDialogTitle.tsx b/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Dialogs/ReviewOrgUnitChangesDialogTitle.tsx index 6154490273..706bfd961d 100644 --- a/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Dialogs/ReviewOrgUnitChangesDialogTitle.tsx +++ b/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Dialogs/ReviewOrgUnitChangesDialogTitle.tsx @@ -1,6 +1,6 @@ -import React, { FunctionComponent, useState } from 'react'; -import { Box, IconButton, Popover } from '@mui/material'; import InfoIcon from '@mui/icons-material/Info'; +import { Box, IconButton, Popover } from '@mui/material'; +import React, { FunctionComponent, useState } from 'react'; import { OrgUnitChangeRequestDetails } from '../types'; import { ReviewOrgUnitChangesInfos, @@ -42,7 +42,13 @@ export const ReviewOrgUnitChangesDialogTitle: FunctionComponent = ({ }} > {titleMessage} - + { return ( - + + {!isFetchingChangeRequest && ( + + )} Date: Tue, 10 Sep 2024 14:02:20 +0200 Subject: [PATCH 38/55] rename folder dialogs --- .../{Dialogs => Components}/HighlightFieldsChanges.tsx | 0 .../{Dialogs => Components}/ReviewOrgUnitChangesButtons.tsx | 0 .../ReviewOrgUnitChangesCommentDialog.tsx | 0 .../ReviewOrgUnitChangesCommentDialogButtons.tsx | 0 .../{Dialogs => Components}/ReviewOrgUnitChangesInfos.tsx | 0 .../ReviewOrgUnitChangesTitle.tsx} | 2 +- .../{Dialogs => Components}/ReviewOrgUnitFieldChanges.tsx | 0 .../reviewChanges/Tables/ReviewOrgUnitChangesTable.tsx | 2 +- .../Tables/details/ReviewOrgUnitChangesDetailsTableBody.tsx | 2 +- .../js/apps/Iaso/domains/orgUnits/reviewChanges/details.tsx | 6 +++--- 10 files changed, 6 insertions(+), 6 deletions(-) rename hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/{Dialogs => Components}/HighlightFieldsChanges.tsx (100%) rename hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/{Dialogs => Components}/ReviewOrgUnitChangesButtons.tsx (100%) rename hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/{Dialogs => Components}/ReviewOrgUnitChangesCommentDialog.tsx (100%) rename hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/{Dialogs => Components}/ReviewOrgUnitChangesCommentDialogButtons.tsx (100%) rename hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/{Dialogs => Components}/ReviewOrgUnitChangesInfos.tsx (100%) rename hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/{Dialogs/ReviewOrgUnitChangesDialogTitle.tsx => Components/ReviewOrgUnitChangesTitle.tsx} (96%) rename hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/{Dialogs => Components}/ReviewOrgUnitFieldChanges.tsx (100%) diff --git a/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Dialogs/HighlightFieldsChanges.tsx b/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Components/HighlightFieldsChanges.tsx similarity index 100% rename from hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Dialogs/HighlightFieldsChanges.tsx rename to hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Components/HighlightFieldsChanges.tsx diff --git a/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Dialogs/ReviewOrgUnitChangesButtons.tsx b/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Components/ReviewOrgUnitChangesButtons.tsx similarity index 100% rename from hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Dialogs/ReviewOrgUnitChangesButtons.tsx rename to hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Components/ReviewOrgUnitChangesButtons.tsx diff --git a/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Dialogs/ReviewOrgUnitChangesCommentDialog.tsx b/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Components/ReviewOrgUnitChangesCommentDialog.tsx similarity index 100% rename from hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Dialogs/ReviewOrgUnitChangesCommentDialog.tsx rename to hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Components/ReviewOrgUnitChangesCommentDialog.tsx diff --git a/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Dialogs/ReviewOrgUnitChangesCommentDialogButtons.tsx b/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Components/ReviewOrgUnitChangesCommentDialogButtons.tsx similarity index 100% rename from hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Dialogs/ReviewOrgUnitChangesCommentDialogButtons.tsx rename to hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Components/ReviewOrgUnitChangesCommentDialogButtons.tsx diff --git a/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Dialogs/ReviewOrgUnitChangesInfos.tsx b/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Components/ReviewOrgUnitChangesInfos.tsx similarity index 100% rename from hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Dialogs/ReviewOrgUnitChangesInfos.tsx rename to hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Components/ReviewOrgUnitChangesInfos.tsx diff --git a/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Dialogs/ReviewOrgUnitChangesDialogTitle.tsx b/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Components/ReviewOrgUnitChangesTitle.tsx similarity index 96% rename from hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Dialogs/ReviewOrgUnitChangesDialogTitle.tsx rename to hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Components/ReviewOrgUnitChangesTitle.tsx index 706bfd961d..2eb6e7c835 100644 --- a/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Dialogs/ReviewOrgUnitChangesDialogTitle.tsx +++ b/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Components/ReviewOrgUnitChangesTitle.tsx @@ -13,7 +13,7 @@ type Props = { isFetchingChangeRequest: boolean; }; -export const ReviewOrgUnitChangesDialogTitle: FunctionComponent = ({ +export const ReviewOrgUnitChangesTitle: FunctionComponent = ({ titleMessage, changeRequest, isFetchingChangeRequest, diff --git a/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Dialogs/ReviewOrgUnitFieldChanges.tsx b/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Components/ReviewOrgUnitFieldChanges.tsx similarity index 100% rename from hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Dialogs/ReviewOrgUnitFieldChanges.tsx rename to hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Components/ReviewOrgUnitFieldChanges.tsx diff --git a/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Tables/ReviewOrgUnitChangesTable.tsx b/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Tables/ReviewOrgUnitChangesTable.tsx index d9e7f61aa1..a1476eece8 100644 --- a/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Tables/ReviewOrgUnitChangesTable.tsx +++ b/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Tables/ReviewOrgUnitChangesTable.tsx @@ -10,7 +10,7 @@ import { baseUrls } from '../../../../constants/urls'; import { ColumnCell } from '../../../../types/general'; import { LinkToOrgUnit } from '../../components/LinkToOrgUnit'; import { IconButton } from '../details'; -import { colorCodes } from '../Dialogs/ReviewOrgUnitChangesInfos'; +import { colorCodes } from '../Components/ReviewOrgUnitChangesInfos'; import MESSAGES from '../messages'; import { ApproveOrgUnitParams, diff --git a/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Tables/details/ReviewOrgUnitChangesDetailsTableBody.tsx b/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Tables/details/ReviewOrgUnitChangesDetailsTableBody.tsx index 07dbf25ac7..9501d64b92 100644 --- a/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Tables/details/ReviewOrgUnitChangesDetailsTableBody.tsx +++ b/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Tables/details/ReviewOrgUnitChangesDetailsTableBody.tsx @@ -1,6 +1,6 @@ import { TableBody } from '@mui/material'; import React, { FunctionComponent } from 'react'; -import { HighlightFields } from '../../Dialogs/HighlightFieldsChanges'; +import { HighlightFields } from '../../Components/HighlightFieldsChanges'; import { NewOrgUnitField } from '../../hooks/useNewFields'; import { OrgUnitChangeRequestDetails } from '../../types'; diff --git a/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/details.tsx b/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/details.tsx index cea80500b1..e1eebfe2cb 100644 --- a/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/details.tsx +++ b/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/details.tsx @@ -9,8 +9,8 @@ import TopBar from '../../../components/nav/TopBarComponent'; import { baseUrls } from '../../../constants/urls'; import { useParamsObject } from '../../../routing/hooks/useParamsObject'; import { SxStyles } from '../../../types/general'; -import { ApproveOrgUnitChangesButtons } from './Dialogs/ReviewOrgUnitChangesButtons'; -import { ReviewOrgUnitChangesDialogTitle } from './Dialogs/ReviewOrgUnitChangesDialogTitle'; +import { ApproveOrgUnitChangesButtons } from './Components/ReviewOrgUnitChangesButtons'; +import { ReviewOrgUnitChangesTitle } from './Components/ReviewOrgUnitChangesTitle'; import { useGetApprovalProposal } from './hooks/api/useGetApprovalProposal'; import { useSaveChangeRequest } from './hooks/api/useSaveChangeRequest'; import { useNewFields } from './hooks/useNewFields'; @@ -86,7 +86,7 @@ export const ReviewOrgUnitChangesDetail: FunctionComponent = () => { /> {!isFetchingChangeRequest && ( - Date: Wed, 11 Sep 2024 09:26:34 +0200 Subject: [PATCH 39/55] merge migrations --- iaso/migrations/0297_merge_20240911_0725.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 iaso/migrations/0297_merge_20240911_0725.py diff --git a/iaso/migrations/0297_merge_20240911_0725.py b/iaso/migrations/0297_merge_20240911_0725.py new file mode 100644 index 0000000000..acba3601b9 --- /dev/null +++ b/iaso/migrations/0297_merge_20240911_0725.py @@ -0,0 +1,12 @@ +# Generated by Django 4.2.13 on 2024-09-11 07:25 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("iaso", "0296_alter_account_modules"), + ("iaso", "0296_orgunit_default_image"), + ] + + operations = [] From 8a3daace6540eaf66e60ec11e1cece5c0b4a72b5 Mon Sep 17 00:00:00 2001 From: Benjamin Monjoie Date: Wed, 11 Sep 2024 12:15:55 +0200 Subject: [PATCH 40/55] Fix Form accepts invalid parameter for periods When the mobile receives a value for the `period_type` and the sum of `periods_before_allowed` and `periods_after_allowed` is less than 1, the mobile application doesn't know how to handle that and crashes. IA-3377 --- iaso/api/forms.py | 19 +++++++---- iaso/tests/api/test_forms.py | 52 ++++++++++++++++++++++++++++++ iaso/tests/api/test_mobileforms.py | 4 +++ 3 files changed, 69 insertions(+), 6 deletions(-) diff --git a/iaso/api/forms.py b/iaso/api/forms.py index bc2a8f131a..28a74235c9 100644 --- a/iaso/api/forms.py +++ b/iaso/api/forms.py @@ -172,15 +172,22 @@ def validate(self, data: typing.Mapping): raise serializers.ValidationError({"org_unit_type_ids": "Invalid org unit type ids"}) # If the period type is None, some period-specific fields must have specific values - if "period_type" in data and data["period_type"] is None: + if "period_type" in data: tracker_errors = {} - if data["periods_before_allowed"] != 0: - tracker_errors["periods_before_allowed"] = "Should be 0" - if data["periods_after_allowed"] != 0: - tracker_errors["periods_after_allowed"] = "Should be 0" + if data["period_type"] is None: + if data["periods_before_allowed"] != 0: + tracker_errors["periods_before_allowed"] = "Should be 0 when period type is not specified" + if data["periods_after_allowed"] != 0: + tracker_errors["periods_after_allowed"] = "Should be 0 when period type is not specified" + else: + before = data.get("periods_before_allowed", 0) + after = data.get("periods_after_allowed", 0) + if before + after < 1: + tracker_errors[ + "periods_allowed" + ] = "periods_before_allowed + periods_after_allowed should be greater than or equal to 1" if tracker_errors: raise serializers.ValidationError(tracker_errors) - return data def update(self, form, validated_data): diff --git a/iaso/tests/api/test_forms.py b/iaso/tests/api/test_forms.py index 1e2f4c2af5..2b02e406f0 100644 --- a/iaso/tests/api/test_forms.py +++ b/iaso/tests/api/test_forms.py @@ -303,6 +303,8 @@ def test_forms_create_ok(self): data={ "name": "test form 1", "period_type": "MONTH", + "periods_before_allowed": 1, + "periods_after_allowed": 0, "project_ids": [self.project_1.id], "org_unit_type_ids": [self.jedi_council.id, self.jedi_academy.id], }, @@ -316,6 +318,54 @@ def test_forms_create_ok(self): self.assertEqual(1, form.projects.count()) self.assertEqual(2, form.org_unit_types.count()) + def test_forms_create_ok_without_period_type(self): + """POST /forms/ happy path without period type""" + + self.client.force_authenticate(self.yoda) + response = self.client.post( + f"/api/forms/", + data={ + "name": "test form 1", + "period_type": None, + "periods_before_allowed": 0, + "periods_after_allowed": 0, + "project_ids": [self.project_1.id], + "org_unit_type_ids": [self.jedi_council.id, self.jedi_academy.id], + }, + format="json", + ) + self.assertJSONResponse(response, 201) + + response_data = response.json() + self.assertValidFormData(response_data) + form = m.Form.objects.get(pk=response_data["id"]) + self.assertEqual(1, form.projects.count()) + self.assertEqual(2, form.org_unit_types.count()) + + def test_forms_create_not_ok_with_period_type_and_wrong_period_before_and_after(self): + """POST /forms/ with wrong period before and after""" + + self.client.force_authenticate(self.yoda) + response = self.client.post( + f"/api/forms/", + data={ + "name": "test form 1", + "period_type": "MONTH", + "periods_before_allowed": 0, + "periods_after_allowed": 0, + "project_ids": [self.project_1.id], + "org_unit_type_ids": [self.jedi_council.id, self.jedi_academy.id], + }, + format="json", + ) + self.assertJSONResponse(response, 400) + + response_data = response.json() + self.assertEqual( + response_data["periods_allowed"][0], + "periods_before_allowed + periods_after_allowed should be greater than or equal to 1", + ) + def test_forms_create_ok_extended(self): """POST /forms/ happy path (more fields)""" @@ -455,6 +505,8 @@ def test_forms_update_ok(self): data={ "name": "test form 1 (updated)", "period_type": "QUARTER", + "pperiods_before_allowed": "0", + "periods_after_allowed": "1", "single_per_period": True, "device_field": "deviceid", "location_field": "location", diff --git a/iaso/tests/api/test_mobileforms.py b/iaso/tests/api/test_mobileforms.py index f0831cf155..704c7dd631 100644 --- a/iaso/tests/api/test_mobileforms.py +++ b/iaso/tests/api/test_mobileforms.py @@ -202,6 +202,8 @@ def test_forms_create_ok(self): data={ "name": "test form 1", "period_type": "MONTH", + "periods_before_allowed": 0, + "periods_after_allowed": 1, "project_ids": [self.project_1.id], "org_unit_type_ids": [self.jedi_council.id, self.jedi_academy.id], }, @@ -354,6 +356,8 @@ def test_forms_update_ok(self): data={ "name": "test form 2 (updated)", "period_type": "QUARTER", + "periods_before_allowed": 0, + "periods_after_allowed": 1, "single_per_period": True, "device_field": "deviceid", "location_field": "location", From af8f24aa5eecc38c83cb2b023f37611dd83faf90 Mon Sep 17 00:00:00 2001 From: Christophe Gerard Date: Wed, 11 Sep 2024 14:31:01 +0200 Subject: [PATCH 41/55] remove cancel/close button --- .../Components/ReviewOrgUnitChangesButtons.tsx | 13 ------------- .../Iaso/domains/orgUnits/reviewChanges/details.tsx | 1 - 2 files changed, 14 deletions(-) diff --git a/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Components/ReviewOrgUnitChangesButtons.tsx b/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Components/ReviewOrgUnitChangesButtons.tsx index 0d9b7ff2e8..380c5620fe 100644 --- a/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Components/ReviewOrgUnitChangesButtons.tsx +++ b/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/Components/ReviewOrgUnitChangesButtons.tsx @@ -19,7 +19,6 @@ type SubmitChangeRequest = ( ) => void; type Props = { - closeDialog: () => void; newFields: NewOrgUnitField[]; isNew: boolean; isNewOrgUnit: boolean; @@ -28,7 +27,6 @@ type Props = { }; export const ApproveOrgUnitChangesButtons: FunctionComponent = ({ - closeDialog, newFields, isNew, isNewOrgUnit, @@ -85,17 +83,6 @@ export const ApproveOrgUnitChangesButtons: FunctionComponent = ({ titleMessage={dialogTitleMessage} /> - {isNew && ( <> diff --git a/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/details.tsx b/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/details.tsx index e1eebfe2cb..677ba08b00 100644 --- a/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/details.tsx +++ b/hat/assets/js/apps/Iaso/domains/orgUnits/reviewChanges/details.tsx @@ -104,7 +104,6 @@ export const ReviewOrgUnitChangesDetail: FunctionComponent = () => { Date: Thu, 12 Sep 2024 11:18:45 +0200 Subject: [PATCH 42/55] IA-3418: remove double API call in potential payments page - use url i.o params as queryKey, because the 'account' redirection would cause a second call to the APi with a race condition. This would sometimes result in 2 potential payments being created, which allowed the creation of empty payment lots down the line --- .../requests/useGetPotentialPayments.tsx | 26 +++++++++---------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/hat/assets/js/apps/Iaso/domains/payments/hooks/requests/useGetPotentialPayments.tsx b/hat/assets/js/apps/Iaso/domains/payments/hooks/requests/useGetPotentialPayments.tsx index 2649e39ebd..8128f33f58 100644 --- a/hat/assets/js/apps/Iaso/domains/payments/hooks/requests/useGetPotentialPayments.tsx +++ b/hat/assets/js/apps/Iaso/domains/payments/hooks/requests/useGetPotentialPayments.tsx @@ -8,7 +8,13 @@ import { apiDateFormat, formatDateString } from '../../../../utils/dates'; const apiUrl = '/api/potential_payments/'; -const getPotentialPayments = (options: PotentialPaymentParams) => { +const getPotentialPayments = (url: string) => { + return getRequest(url) as Promise; +}; + +export const useGetPotentialPayments = ( + params: PotentialPaymentParams, +): UseQueryResult => { const { change_requests__created_at_after, change_requests__created_at_before, @@ -17,10 +23,10 @@ const getPotentialPayments = (options: PotentialPaymentParams) => { users, user_roles, page, - } = options; + } = params; const apiParams = { - order: options.order || 'user__last_name', - limit: options.pageSize || 10, + order: params.order || 'user__last_name', + limit: params.pageSize || 10, page, change_requests__created_at_after: formatDateString( change_requests__created_at_after, @@ -37,18 +43,10 @@ const getPotentialPayments = (options: PotentialPaymentParams) => { users, user_roles, }; - const url = makeUrlWithParams(apiUrl, apiParams); - - return getRequest(url) as Promise; -}; - -export const useGetPotentialPayments = ( - params: PotentialPaymentParams, -): UseQueryResult => { return useSnackQuery({ - queryKey: ['potentialPayments', params], - queryFn: () => getPotentialPayments(params), + queryKey: ['potentialPayments', url], + queryFn: () => getPotentialPayments(url), options: { staleTime: 1000 * 60 * 15, // in MS cacheTime: 1000 * 60 * 5, From d4400bf7e2d1f4ecb121d4a27c36cffc040bda0f Mon Sep 17 00:00:00 2001 From: Quang Son Le Date: Thu, 12 Sep 2024 11:19:22 +0200 Subject: [PATCH 43/55] IA-3418: show change request id in payment lot list (admin) --- iaso/admin.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/iaso/admin.py b/iaso/admin.py index 4d0ef80561..71d45e4690 100644 --- a/iaso/admin.py +++ b/iaso/admin.py @@ -710,6 +710,7 @@ class OrgUnitChangeRequestAdmin(admin.ModelAdmin): "old_reference_instances", "old_opening_date", "old_closed_date", + "potential_payment", ) raw_id_fields = ( "org_unit", @@ -801,6 +802,19 @@ class ConfigAdmin(admin.ModelAdmin): @admin.register(PotentialPayment) class PotentialPaymentAdmin(admin.ModelAdmin): formfield_overrides = {models.JSONField: {"widget": IasoJSONEditorWidget}} + list_display = ("id", "change_request_ids") + + def change_request_ids(self, obj): + change_requests = obj.change_requests.all() + if change_requests: + return format_html( + ", ".join( + f'{cr.id}' for cr in change_requests + ) + ) + return "-" + + change_request_ids.short_description = "Change Request IDs" @admin.register(Payment) From 1afa00d0efca36346daac99db6496b0ca308a0a7 Mon Sep 17 00:00:00 2001 From: Christophe Gerard Date: Thu, 12 Sep 2024 12:18:45 +0200 Subject: [PATCH 44/55] adding payments infos in admin list view --- iaso/admin.py | 54 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 40 insertions(+), 14 deletions(-) diff --git a/iaso/admin.py b/iaso/admin.py index 71d45e4690..d09f081be9 100644 --- a/iaso/admin.py +++ b/iaso/admin.py @@ -1,5 +1,4 @@ -from typing import Any -from typing import Protocol +from typing import Any, Protocol from django import forms as django_forms from django.contrib.admin import widgets @@ -7,11 +6,12 @@ from django.contrib.gis.db import models as geomodels from django.contrib.postgres.fields import ArrayField from django.db import models -from django.utils.html import format_html_join, format_html +from django.utils.html import format_html, format_html_join from django.utils.safestring import mark_safe from django_json_widget.widgets import JSONEditorWidget from iaso.models.json_config import Config # type: ignore + from .models import ( Account, AccountFeatureFlag, @@ -45,8 +45,12 @@ MatchingAlgorithm, OrgUnit, OrgUnitChangeRequest, + OrgUnitReferenceInstance, OrgUnitType, Page, + Payment, + PaymentLot, + PotentialPayment, Profile, Project, Report, @@ -61,13 +65,9 @@ WorkflowChange, WorkflowFollowup, WorkflowVersion, - OrgUnitReferenceInstance, - PotentialPayment, - Payment, - PaymentLot, ) from .models.data_store import JsonDataStore -from .models.microplanning import Team, Planning, Assignment +from .models.microplanning import Assignment, Planning, Team from .utils.gis import convert_2d_point_to_3d @@ -454,9 +454,9 @@ class EntityAdmin(admin.ModelAdmin): def get_form(self, request, obj=None, **kwargs): # In the for the entity type, we also want to indicate the account name form = super().get_form(request, obj, **kwargs) - form.base_fields[ - "entity_type" - ].label_from_instance = lambda entity: f"{entity.name} (Account: {entity.account.name})" + form.base_fields["entity_type"].label_from_instance = ( + lambda entity: f"{entity.name} (Account: {entity.account.name})" + ) return form def get_queryset(self, request): @@ -820,11 +820,37 @@ def change_request_ids(self, obj): @admin.register(Payment) class PaymentAdmin(admin.ModelAdmin): formfield_overrides = {models.JSONField: {"widget": IasoJSONEditorWidget}} + list_display = ("id", "status", "created_at", "updated_at", "change_request_ids") + + def change_request_ids(self, obj): + change_requests = obj.change_requests.all() + if change_requests: + return format_html( + ", ".join( + f'{cr.id}' for cr in change_requests + ) + ) + return "-" + + change_request_ids.short_description = "Change Request IDs" @admin.register(PaymentLot) class PaymentLotAdmin(admin.ModelAdmin): formfield_overrides = {models.JSONField: {"widget": IasoJSONEditorWidget}} + list_display = ("id", "status", "created_at", "updated_at", "payment_ids") + + def payment_ids(self, obj): + payments = obj.payments.all() + if payments: + return format_html( + ", ".join( + f'{payment.id}' for payment in payments + ) + ) + return "-" + + payment_ids.short_description = "Payment IDs" @admin.register(DataSource) From f5f38c1590680fbf7cc67d3557ca9f34c75d2e54 Mon Sep 17 00:00:00 2001 From: Beygorghor Date: Thu, 12 Sep 2024 12:33:54 +0200 Subject: [PATCH 45/55] black --- iaso/admin.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/iaso/admin.py b/iaso/admin.py index d09f081be9..91de2190df 100644 --- a/iaso/admin.py +++ b/iaso/admin.py @@ -454,9 +454,9 @@ class EntityAdmin(admin.ModelAdmin): def get_form(self, request, obj=None, **kwargs): # In the for the entity type, we also want to indicate the account name form = super().get_form(request, obj, **kwargs) - form.base_fields["entity_type"].label_from_instance = ( - lambda entity: f"{entity.name} (Account: {entity.account.name})" - ) + form.base_fields[ + "entity_type" + ].label_from_instance = lambda entity: f"{entity.name} (Account: {entity.account.name})" return form def get_queryset(self, request): From df2de25c08f74b194a041c65d85579d77c2dc8bb Mon Sep 17 00:00:00 2001 From: Quang Son Le Date: Thu, 12 Sep 2024 15:30:40 +0200 Subject: [PATCH 46/55] IA-3365: typo --- iaso/api/profiles/audit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iaso/api/profiles/audit.py b/iaso/api/profiles/audit.py index fc1c62607b..03c0f3585e 100644 --- a/iaso/api/profiles/audit.py +++ b/iaso/api/profiles/audit.py @@ -53,7 +53,7 @@ class Meta: "deleted_at", ] - # TODO delete this method when user soft delete is implemented + # TODO delete this method when User soft delete is implemented def get_deleted_at(self, profile): return None From fb7ca99f982ded9bc1d128a387a0c7e08b0bed5e Mon Sep 17 00:00:00 2001 From: Christophe Gerard Date: Thu, 12 Sep 2024 16:29:36 +0200 Subject: [PATCH 47/55] fix migration conflict --- iaso/migrations/0298_merge_20240912_1428.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 iaso/migrations/0298_merge_20240912_1428.py diff --git a/iaso/migrations/0298_merge_20240912_1428.py b/iaso/migrations/0298_merge_20240912_1428.py new file mode 100644 index 0000000000..67ff81b5fb --- /dev/null +++ b/iaso/migrations/0298_merge_20240912_1428.py @@ -0,0 +1,12 @@ +# Generated by Django 4.2.13 on 2024-09-12 14:28 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("iaso", "0297_merge_20240911_0725"), + ("iaso", "0297_merge_20240911_0935"), + ] + + operations = [] From 1dea00abfaca46d531b6645d33874a01ec7c9aba Mon Sep 17 00:00:00 2001 From: Quang Son Le <38907762+quang-le@users.noreply.github.com> Date: Thu, 12 Sep 2024 16:31:26 +0200 Subject: [PATCH 48/55] Update iaso/api/profiles/profile_logs.py Co-authored-by: Thibault Dethier <84660492+tdethier@users.noreply.github.com> --- iaso/api/profiles/profile_logs.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/iaso/api/profiles/profile_logs.py b/iaso/api/profiles/profile_logs.py index 684bafcb44..b44ab1002d 100644 --- a/iaso/api/profiles/profile_logs.py +++ b/iaso/api/profiles/profile_logs.py @@ -16,8 +16,6 @@ from iaso.api.common import Paginator from iaso.models.org_unit import OrgUnit -from django.db.models import F, Func, Value, CharField, JSONField, IntegerField -from django.db.models.functions import Cast class ProfileLogsListPagination(Paginator): From 1d03d29f65de337bb97cafb7b7b688045736d7c5 Mon Sep 17 00:00:00 2001 From: Quang Son Le <38907762+quang-le@users.noreply.github.com> Date: Thu, 12 Sep 2024 16:31:54 +0200 Subject: [PATCH 49/55] Update iaso/api/profiles/profile_logs.py Co-authored-by: Thibault Dethier <84660492+tdethier@users.noreply.github.com> --- iaso/api/profiles/profile_logs.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/iaso/api/profiles/profile_logs.py b/iaso/api/profiles/profile_logs.py index b44ab1002d..e030158746 100644 --- a/iaso/api/profiles/profile_logs.py +++ b/iaso/api/profiles/profile_logs.py @@ -168,13 +168,13 @@ def get_queryset(self): if "created_at" in order: queryset = queryset.order_by(order) - if order == "modified_by": + elif order == "modified_by": queryset = queryset.order_by("user__username") - if order == "-modified_by": + elif order == "-modified_by": queryset = queryset.order_by("-user__username") - if order == "user": + elif order == "user": queryset = queryset.order_by("new_value__0__fields__user__username") - if order == "-user": + elif order == "-user": queryset = queryset.order_by("-new_value__0__fields__user__username") return queryset From 6f090d03f8868ab24b6b0d1812732fafd7b94cfb Mon Sep 17 00:00:00 2001 From: Quang Son Le <38907762+quang-le@users.noreply.github.com> Date: Thu, 12 Sep 2024 16:33:29 +0200 Subject: [PATCH 50/55] Update iaso/tests/api/test_profile_logs.py Co-authored-by: Thibault Dethier <84660492+tdethier@users.noreply.github.com> --- iaso/tests/api/test_profile_logs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iaso/tests/api/test_profile_logs.py b/iaso/tests/api/test_profile_logs.py index bb52e90c23..7892c6d74f 100644 --- a/iaso/tests/api/test_profile_logs.py +++ b/iaso/tests/api/test_profile_logs.py @@ -129,7 +129,7 @@ def setUpTestData(cls): date1 = datetime.datetime(2020, 2, 10, 0, 0, 5) date2 = datetime.datetime(2020, 2, 15, 0, 0, 5, tzinfo=pytz.UTC) # Logs - # by user 1 for editabe user 1 with org unit 1 before date + # by user 1 for editable user 1 with org unit 1 before date with patch("django.utils.timezone.now", lambda: date1): cls.log_1 = Modification.objects.create( user=cls.user_with_permission_1, From edc622d291e189328c124fe7d656625880b7e545 Mon Sep 17 00:00:00 2001 From: Quang Son Le Date: Thu, 12 Sep 2024 16:47:04 +0200 Subject: [PATCH 51/55] IA-3402: code review - add some tests --- iaso/tests/api/test_profile_logs.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/iaso/tests/api/test_profile_logs.py b/iaso/tests/api/test_profile_logs.py index bb52e90c23..0fabae4d52 100644 --- a/iaso/tests/api/test_profile_logs.py +++ b/iaso/tests/api/test_profile_logs.py @@ -126,8 +126,9 @@ def setUpTestData(cls): app_label="iaso", model="profile", ) - date1 = datetime.datetime(2020, 2, 10, 0, 0, 5) + date1 = datetime.datetime(2020, 2, 10, 0, 0, 5, tzinfo=pytz.UTC) date2 = datetime.datetime(2020, 2, 15, 0, 0, 5, tzinfo=pytz.UTC) + # date3 = datetime.datetime(2020, 2, 15, 0, 0, 5, tzinfo=pytz.UTC) # Logs # by user 1 for editabe user 1 with org unit 1 before date with patch("django.utils.timezone.now", lambda: date1): @@ -449,6 +450,11 @@ def setUpTestData(cls): ], ) + def test_unauthenticated_user(self): + """GET /api/userlogs/ anonymous user --> 401""" + response = self.client.get("/api/userlogs/") + self.assertJSONResponse(response, 401) + def test_user_no_permission(self): """GET /api/userlogs/ without USERS_ADMIN permission --> 403""" self.client.force_authenticate(self.user_without_permission) @@ -520,6 +526,7 @@ def test_filters(self): jsonschema.validate(instance=data, schema=PROFILE_LOG_LIST_SCHEMA) except jsonschema.exceptions.ValidationError as ex: self.fail(msg=str(ex)) + results = data["results"] self.assertEquals(len(results), 1) user = results[0]["user"] @@ -540,3 +547,16 @@ def test_filters(self): new_location = results[0]["new_location"][0] self.assertEquals(new_location["name"], self.org_unit_1.name) self.assertEquals(new_location["id"], self.org_unit_1.id) + + response = self.client.get( + f"/api/userlogs/?user_ids={self.edited_user_1.id}&modified_by={self.user_with_permission_1.id}&created_at_after=2020-02-14&created_at_before=2020-02-09" + ) + data = self.assertJSONResponse(response, 200) + self.assertEquals(data["count"], 0) + try: + jsonschema.validate(instance=data, schema=PROFILE_LOG_LIST_SCHEMA) + except jsonschema.exceptions.ValidationError as ex: + self.fail(msg=str(ex)) + + results = data["results"] + self.assertEquals(results, []) From 3bc66e55faad0dccbe161cf263821a998c572c33 Mon Sep 17 00:00:00 2001 From: Quang Son Le Date: Thu, 12 Sep 2024 16:53:37 +0200 Subject: [PATCH 52/55] IA-3402: black --- iaso/api/profiles/profile_logs.py | 1 - 1 file changed, 1 deletion(-) diff --git a/iaso/api/profiles/profile_logs.py b/iaso/api/profiles/profile_logs.py index e030158746..70038aa7bb 100644 --- a/iaso/api/profiles/profile_logs.py +++ b/iaso/api/profiles/profile_logs.py @@ -17,7 +17,6 @@ from iaso.models.org_unit import OrgUnit - class ProfileLogsListPagination(Paginator): page_size = 20 From 9a400d741e7cc7fa56f23727481cf07578961746 Mon Sep 17 00:00:00 2001 From: Christophe Gerard Date: Thu, 12 Sep 2024 17:36:11 +0200 Subject: [PATCH 53/55] fix mime type on S3 --- iaso/api/instances.py | 23 +++++++++-------------- iaso/models/base.py | 12 ++++++------ iaso/utils/file_utils.py | 5 +---- 3 files changed, 16 insertions(+), 24 deletions(-) diff --git a/iaso/api/instances.py b/iaso/api/instances.py index 24767ab1d2..020e4efb42 100644 --- a/iaso/api/instances.py +++ b/iaso/api/instances.py @@ -7,7 +7,6 @@ import pandas as pd from django.contrib.auth.models import User from django.contrib.gis.geos import Point -from django.core.files.storage import default_storage from django.core.paginator import Paginator from django.db import connection, transaction from django.db.models import Count, F, Func, Prefetch, Q, QuerySet @@ -21,21 +20,15 @@ from typing_extensions import Annotated, TypedDict import iaso.periods as periods -from hat.api.export_utils import Echo, generate_xlsx, iter_items, timestamp_to_utc_datetime +from hat.api.export_utils import (Echo, generate_xlsx, iter_items, + timestamp_to_utc_datetime) from hat.audit.models import INSTANCE_API, log_modification from hat.common.utils import queryset_iterator from hat.menupermissions import models as permission from iaso.api.serializers import OrgUnitSerializer -from iaso.models import ( - Entity, - Instance, - InstanceFile, - InstanceLock, - InstanceQuerySet, - OrgUnit, - OrgUnitChangeRequest, - Project, -) +from iaso.models import (Entity, Instance, InstanceFile, InstanceLock, + InstanceQuerySet, OrgUnit, OrgUnitChangeRequest, + Project) from iaso.utils import timestamp_to_datetime from iaso.utils.file_utils import get_file_type @@ -43,8 +36,10 @@ from ..utils.models.common import get_creator_name from . import common from .comment import UserSerializerForComment -from .common import CONTENT_TYPE_CSV, CONTENT_TYPE_XLSX, FileFormatEnum, TimestampField, safe_api_import -from .instance_filters import get_form_from_instance_filters, parse_instance_filters +from .common import (CONTENT_TYPE_CSV, CONTENT_TYPE_XLSX, FileFormatEnum, + TimestampField, safe_api_import) +from .instance_filters import (get_form_from_instance_filters, + parse_instance_filters) class InstanceSerializer(serializers.ModelSerializer): diff --git a/iaso/models/base.py b/iaso/models/base.py index 0a0816388c..feb4296cc4 100644 --- a/iaso/models/base.py +++ b/iaso/models/base.py @@ -11,20 +11,18 @@ from logging import getLogger from urllib.error import HTTPError from urllib.request import urlopen -from .project import Project + import django_cte from bs4 import BeautifulSoup as Soup # type: ignore from django import forms as dj_forms from django.contrib import auth from django.contrib.auth import models as authModels -from django.contrib.auth.models import User +from django.contrib.auth.models import AnonymousUser, User from django.contrib.gis.db.models.fields import PointField from django.contrib.gis.geos import Point from django.contrib.postgres.aggregates import ArrayAgg from django.contrib.postgres.fields import ArrayField -from django.contrib.auth.models import AnonymousUser, User from django.core.exceptions import ObjectDoesNotExist, ValidationError -from django.core.files.storage import default_storage from django.core.paginator import Paginator from django.core.validators import MinLengthValidator from django.db import models @@ -44,11 +42,12 @@ from iaso.utils.file_utils import get_file_type from .. import periods +from ..utils.emoji import fix_emoji from ..utils.jsonlogic import jsonlogic_to_q from ..utils.models.common import get_creator_name from .device import Device, DeviceOwnership from .forms import Form, FormVersion -from ..utils.emoji import fix_emoji +from .project import Project logger = getLogger(__name__) @@ -1187,7 +1186,8 @@ def get_form_version(self): def export(self, launcher=None, force_export=False): from iaso.dhis2.datavalue_exporter import DataValueExporter - from iaso.dhis2.export_request_builder import ExportRequestBuilder, NothingToExportError + from iaso.dhis2.export_request_builder import (ExportRequestBuilder, + NothingToExportError) try: export_request = ExportRequestBuilder().build_export_request( diff --git a/iaso/utils/file_utils.py b/iaso/utils/file_utils.py index f375f7bc6f..f4986ef354 100644 --- a/iaso/utils/file_utils.py +++ b/iaso/utils/file_utils.py @@ -1,11 +1,8 @@ import mimetypes -from django.core.files.storage import default_storage - def get_file_type(file): if file: - file_path = default_storage.path(file.name) - mime_type, _ = mimetypes.guess_type(file_path) + mime_type, _ = mimetypes.guess_type(file.name) return mime_type return None From 66776b4bd3d458a2814275d2efc6b20dadce06b4 Mon Sep 17 00:00:00 2001 From: Math VDH Date: Fri, 13 Sep 2024 10:00:58 +0100 Subject: [PATCH 54/55] IA-3365 Fix migrations conflicts --- iaso/migrations/0297_merge_20240911_0725.py | 12 ------------ ..._20240911_0921.py => 0297_merge_20240913_0900.py} | 3 ++- iaso/migrations/0298_merge_20240912_1428.py | 12 ------------ 3 files changed, 2 insertions(+), 25 deletions(-) delete mode 100644 iaso/migrations/0297_merge_20240911_0725.py rename iaso/migrations/{0297_merge_20240911_0921.py => 0297_merge_20240913_0900.py} (69%) delete mode 100644 iaso/migrations/0298_merge_20240912_1428.py diff --git a/iaso/migrations/0297_merge_20240911_0725.py b/iaso/migrations/0297_merge_20240911_0725.py deleted file mode 100644 index acba3601b9..0000000000 --- a/iaso/migrations/0297_merge_20240911_0725.py +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by Django 4.2.13 on 2024-09-11 07:25 - -from django.db import migrations - - -class Migration(migrations.Migration): - dependencies = [ - ("iaso", "0296_alter_account_modules"), - ("iaso", "0296_orgunit_default_image"), - ] - - operations = [] diff --git a/iaso/migrations/0297_merge_20240911_0921.py b/iaso/migrations/0297_merge_20240913_0900.py similarity index 69% rename from iaso/migrations/0297_merge_20240911_0921.py rename to iaso/migrations/0297_merge_20240913_0900.py index 69354195ab..da5a1962e2 100644 --- a/iaso/migrations/0297_merge_20240911_0921.py +++ b/iaso/migrations/0297_merge_20240913_0900.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.13 on 2024-09-11 09:21 +# Generated by Django 4.2.11 on 2024-09-13 09:00 from django.db import migrations @@ -7,6 +7,7 @@ class Migration(migrations.Migration): dependencies = [ ("iaso", "0296_alter_account_modules"), ("iaso", "0296_groupset_group_belonging"), + ("iaso", "0296_orgunit_default_image"), ] operations = [] diff --git a/iaso/migrations/0298_merge_20240912_1428.py b/iaso/migrations/0298_merge_20240912_1428.py deleted file mode 100644 index 67ff81b5fb..0000000000 --- a/iaso/migrations/0298_merge_20240912_1428.py +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by Django 4.2.13 on 2024-09-12 14:28 - -from django.db import migrations - - -class Migration(migrations.Migration): - dependencies = [ - ("iaso", "0297_merge_20240911_0725"), - ("iaso", "0297_merge_20240911_0935"), - ] - - operations = [] From 0bd156a2ae7e7ab83af8afda0b1122e1ff180498 Mon Sep 17 00:00:00 2001 From: Math VDH Date: Fri, 13 Sep 2024 10:05:29 +0100 Subject: [PATCH 55/55] IA-3365 fix formatting --- iaso/api/instances.py | 23 ++++++++++++++--------- iaso/models/base.py | 4 ++-- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/iaso/api/instances.py b/iaso/api/instances.py index 020e4efb42..aa0a0d69c4 100644 --- a/iaso/api/instances.py +++ b/iaso/api/instances.py @@ -4,6 +4,7 @@ from time import gmtime, strftime from typing import Any, Dict, Union + import pandas as pd from django.contrib.auth.models import User from django.contrib.gis.geos import Point @@ -20,15 +21,21 @@ from typing_extensions import Annotated, TypedDict import iaso.periods as periods -from hat.api.export_utils import (Echo, generate_xlsx, iter_items, - timestamp_to_utc_datetime) +from hat.api.export_utils import Echo, generate_xlsx, iter_items, timestamp_to_utc_datetime from hat.audit.models import INSTANCE_API, log_modification from hat.common.utils import queryset_iterator from hat.menupermissions import models as permission from iaso.api.serializers import OrgUnitSerializer -from iaso.models import (Entity, Instance, InstanceFile, InstanceLock, - InstanceQuerySet, OrgUnit, OrgUnitChangeRequest, - Project) +from iaso.models import ( + Entity, + Instance, + InstanceFile, + InstanceLock, + InstanceQuerySet, + OrgUnit, + OrgUnitChangeRequest, + Project, +) from iaso.utils import timestamp_to_datetime from iaso.utils.file_utils import get_file_type @@ -36,10 +43,8 @@ from ..utils.models.common import get_creator_name from . import common from .comment import UserSerializerForComment -from .common import (CONTENT_TYPE_CSV, CONTENT_TYPE_XLSX, FileFormatEnum, - TimestampField, safe_api_import) -from .instance_filters import (get_form_from_instance_filters, - parse_instance_filters) +from .common import CONTENT_TYPE_CSV, CONTENT_TYPE_XLSX, FileFormatEnum, TimestampField, safe_api_import +from .instance_filters import get_form_from_instance_filters, parse_instance_filters class InstanceSerializer(serializers.ModelSerializer): diff --git a/iaso/models/base.py b/iaso/models/base.py index feb4296cc4..8ef7d35922 100644 --- a/iaso/models/base.py +++ b/iaso/models/base.py @@ -12,6 +12,7 @@ from urllib.error import HTTPError from urllib.request import urlopen + import django_cte from bs4 import BeautifulSoup as Soup # type: ignore from django import forms as dj_forms @@ -1186,8 +1187,7 @@ def get_form_version(self): def export(self, launcher=None, force_export=False): from iaso.dhis2.datavalue_exporter import DataValueExporter - from iaso.dhis2.export_request_builder import (ExportRequestBuilder, - NothingToExportError) + from iaso.dhis2.export_request_builder import ExportRequestBuilder, NothingToExportError try: export_request = ExportRequestBuilder().build_export_request(