✨ [#4993] Implement fetching select(boxes) options from Referentielij…

…sten

this was previously possible with logic and service fetch, but this functionality provides a shortcut to more easily integrate with Re ferentielijsten API

src/openforms/contrib/referentielijsten/client.py

@@ -0,0 +1,35 @@
+from functools import partial
+from typing import Any, TypedDict
+
+from django.core.cache import cache
+
+from ape_pie import APIClient
+from zgw_consumers.service import pagination_helper
+
+REFERENTIELIJSTEN_LOOKUP_CACHE_TIMEOUT = 5 * 60
+
+
+class TabelItem(TypedDict):
+    code: str
+    naam: str
+    begindatumGeldigheid: str  # ISO 8601 datetime string
+    einddatumGeldigheid: str | None  # ISO 8601 datetime string
+    aanvullendeGegevens: Any
+
+
+class ReferentielijstenClient(APIClient):
+    def get_items_for_tabel(self, code: str) -> list[TabelItem]:
+        response = self.get("items", params={"tabel__code": code}, timeout=10)
+        response.raise_for_status()
+        data = response.json()
+        all_data = list(pagination_helper(self, data))
+        return all_data
+
+    def get_items_for_tabel_cached(self, code: str) -> list[TabelItem]:
+        result = cache.get_or_set(
+            key=f"referentielijsten|get_items_for_tabel|code:{code}",
+            default=partial(self.get_items_for_tabel, code),
+            timeout=REFERENTIELIJSTEN_LOOKUP_CACHE_TIMEOUT,
+        )
+        assert result is not None
+        return result

src/openforms/formio/dynamic_config/dynamic_options.py

@@ -11,13 +11,21 @@
 from openforms.typing import DataMapping, JSONValue
 
 from ..typing import Component
+from .referentielijsten import fetch_options_from_referentielijsten
 
 
-def normalise_option(option: JSONValue) -> JSONValue:
+class DynamicOptionException(Exception):
+    def __init__(self, msg: str, detail: str):
+        self.detail = detail
+
+        super().__init__(msg)
+
+
+def normalise_option(option: JSONValue) -> tuple[JSONValue, JSONValue]:
     if not isinstance(option, list):
-        return [option, option]
+        return (option, option)
 
-    return option[:2]
+    return (option[0], option[1])
 
 
 def is_or_contains_none(option: JSONValue) -> bool:
@@ -26,29 +34,23 @@ def is_or_contains_none(option: JSONValue) -> bool:
     return option is None
 
 
-def escape_option(option: JSONValue) -> list[str]:
-    return [escape(item) for item in option]
+def escape_option(option: tuple[JSONValue, JSONValue]) -> tuple[str, str]:
+    return (escape(str(option[0])), escape(str(option[1])))
 
 
 def deduplicate_options(
-    options: JSONValue,
-) -> JSONValue:
+    options: list[tuple[str, str]],
+) -> list[tuple[str, str]]:
     new_options = []
     for option in options:
         if option not in new_options:
             new_options.append(option)
     return new_options
 
 
-def add_options_to_config(
-    component: Component,
-    data: DataMapping,
-    submission: Submission,
-    options_path: str = "values",
-) -> None:
-    if glom(component, "openForms.dataSrc", default=None) != "variable":
-        return
-
+def get_options_from_variable(
+    component: Component, data: DataMapping, submission: Submission
+) -> list[tuple[str, str]] | None:
     items_expression = glom(component, "openForms.itemsExpression")
     items_array = jsonLogic(items_expression, data)
     if not items_array:
@@ -80,7 +82,10 @@ def add_options_to_config(
             % {"items_expression": json.dumps(items_expression)},
         )
 
-    normalised_options = [normalise_option(option) for option in not_none_options]
+    normalised_options: list[tuple[JSONValue, JSONValue]] = [
+        normalise_option(option) for option in not_none_options
+    ]
+
     if any(
         isinstance(item_key, (dict, list)) or isinstance(item_label, (dict, list))
         for item_key, item_label in normalised_options
@@ -97,12 +102,40 @@ def add_options_to_config(
 
     escaped_options = [escape_option(option) for option in normalised_options]
     deduplicated_options = deduplicate_options(escaped_options)
+
+    return deduplicated_options
+
+
+def add_options_to_config(
+    component: Component,
+    data: DataMapping,
+    submission: Submission,
+    options_path: str = "values",
+) -> None:
+    data_src = glom(component, "openForms.dataSrc", default=None)
+    match data_src:
+        case "referentielijsten":
+            items_array = fetch_options_from_referentielijsten(component, submission)
+            if not items_array:
+                raise DynamicOptionException(
+                    "Could not retrieve options from Referentielijsten API",
+                    detail=_(
+                        "Loading the form failed due to problems with an external system, please try again later"
+                    ),
+                )
+        case "variable":
+            items_array = get_options_from_variable(component, data, submission)
+            if items_array is None:
+                return
+        case _:
+            return
+
     assign(
         component,
         options_path,
         [
             {"label": escaped_label, "value": escaped_key}
-            for escaped_key, escaped_label in deduplicated_options
+            for escaped_key, escaped_label in items_array
         ],
         missing=dict,
     )

src/openforms/formio/dynamic_config/referentielijsten.py

@@ -0,0 +1,71 @@
+from django.utils.translation import gettext as _
+
+from glom import glom
+from requests.exceptions import RequestException
+from zgw_consumers.client import build_client
+from zgw_consumers.models import Service
+
+from openforms.contrib.referentielijsten.client import ReferentielijstenClient
+from openforms.logging import logevent
+from openforms.submissions.models import Submission
+
+from ..typing import Component
+
+
+def fetch_options_from_referentielijsten(
+    component: Component, submission: Submission
+) -> list[tuple[str, str]] | None:
+    service_slug = glom(component, "openForms.service", default=None)
+    code = glom(component, "openForms.code", default=None)
+    if not service_slug:
+        logevent.form_configuration_error(
+            submission.form,
+            component,
+            _(
+                "Cannot fetch from Referentielijsten API, because no `service` is configured."
+            ),
+        )
+        return
+
+    if not code:
+        logevent.form_configuration_error(
+            submission.form,
+            component,
+            _(
+                "Cannot fetch from Referentielijsten API, because no `code` is configured."
+            ),
+        )
+        return
+
+    try:
+        service = Service.objects.get(slug=service_slug)
+    except Service.DoesNotExist:
+        logevent.form_configuration_error(
+            submission.form,
+            component,
+            _(
+                "Cannot fetch from Referentielijsten API, service with {service_slug} does not exist."
+            ).format(service_slug=service_slug),
+        )
+        return
+
+    try:
+        with build_client(service, client_factory=ReferentielijstenClient) as client:
+            result = client.get_items_for_tabel_cached(code)
+    except RequestException as e:
+        logevent.referentielijsten_failure_response(
+            submission.form,
+            component,
+            _(
+                "Exception occurred while fetching from Referentielijsten API: {exception}."
+            ).format(exception=e),
+        )
+        return
+    else:
+        if not result:
+            logevent.referentielijsten_failure_response(
+                submission.form,
+                component,
+                _("No results found from Referentielijsten API."),
+            )
+        return [[item["code"], item["naam"]] for item in result]

src/openforms/logging/logevent.py

@@ -613,6 +613,19 @@ def stuf_bg_response(service: StufService, url):
 # - - -
 
 
+def referentielijsten_failure_response(
+    form: Form, component: JSONObject, error_message: str
+):
+    _create_log(
+        form,
+        "referentielijsten_failure_response",
+        extra_data={"component": component, "error": error_message},
+    )
+
+
+# - - -
+
+
 def hijack_started(hijacker, hijacked):
     _create_log(
         hijacked,

src/openforms/logging/templates/logging/events/referentielijsten_failure_response.txt

@@ -0,0 +1,4 @@
+{% load i18n %}
+{% blocktrans trimmed with lead=log.fmt_lead url=log.fmt_url %}
+   {{ lead }}: Failed to fetch items from Referentielijsten API: {{ error }}
+{% endblocktrans %}

src/openforms/submissions/api/serializers.py

@@ -13,6 +13,7 @@
 from drf_spectacular.types import OpenApiTypes
 from drf_spectacular.utils import extend_schema_field
 from rest_framework import serializers
+from rest_framework.exceptions import APIException
 from rest_framework.reverse import reverse
 from rest_framework_nested.serializers import NestedHyperlinkedModelSerializer
 
@@ -267,12 +268,17 @@ class Meta:
     @elasticapm.capture_span(span_type="app.api.serialization")
     def to_representation(self, instance):
         # invoke the configured form logic to dynamically update the Formio.js configuration
-        new_configuration = evaluate_form_logic(
-            instance.submission,
-            instance,
-            instance.submission.data,
-            **self.context,
-        )
+        try:
+            new_configuration = evaluate_form_logic(
+                instance.submission,
+                instance,
+                instance.submission.data,
+                **self.context,
+            )
+        except Exception as e:
+            if detail := getattr(e, "detail", None):
+                raise APIException(detail)
+            raise e  # pragma: no cover
         # update the config for serialization
         instance.form_step.form_definition.configuration = new_configuration
         return super().to_representation(instance)