IA-3401 enketo emoji support

iaso/enketo/enketo_xml.py

@@ -1,6 +1,10 @@
 from typing import Tuple
+import re
 
 from lxml import etree  # type: ignore
+from lxml.etree import XMLParser
+
+from iaso.utils.emoji import fix_emoji
 
 ENKETO_FORM_ID_SEPARATOR = "-"
 
@@ -80,10 +84,12 @@ def inject_xml_find_uuid(instance_xml, instance_id, version_id, user_id) -> Tupl
     """ "Inject the attribute in different place in the xml
     Return the uuid found in the xml
     """
+    # use custom parser to match the recover flag as in beautifulsoup while flattening
+    lxml_parser = XMLParser(huge_tree=True, recover=True)
     xml_str = instance_xml.decode("utf-8")
     #  Get the instanceID (uuid) from the //meta/instanceID
     #  We have an uuid on instance. but it seems not always filled?
-    root = etree.fromstring(xml_str)
+    root = etree.fromstring(fix_emoji(xml_str), parser=lxml_parser)
     instance_id_tag = root.find(".//meta/instanceID")
     instance_uuid = instance_id_tag.text.replace("uuid:", "")  # type: ignore
 

iaso/models/base.py

@@ -44,6 +44,7 @@
 from ..utils.models.common import get_creator_name
 from .device import Device, DeviceOwnership
 from .forms import Form, FormVersion
+from ..utils.emoji import fix_emoji
 
 logger = getLogger(__name__)
 
@@ -1046,8 +1047,10 @@ def convert_correlation(self):
             self.save()
 
     def xml_file_to_json(self, file: typing.IO) -> typing.Dict[str, typing.Any]:
-        copy_io_utf8 = StringIO(file.read().decode("utf-8"))
-        soup = Soup(copy_io_utf8, "xml", from_encoding="utf-8")
+        raw_content = file.read().decode("utf-8")
+        fixed_content = fix_emoji(raw_content).decode("utf-8")
+        copy_io_utf8 = StringIO(fixed_content)
+        soup = Soup(copy_io_utf8, "lxml-xml", from_encoding="utf-8")
 
         form_version_id = extract_form_version_id(soup)
         if form_version_id:

iaso/tests/enketo/test_enketo_lib.py

@@ -23,6 +23,13 @@ def test_inject_userid_update_tag_text_if_present(self):
         self.assertEqual(str(xml), str(expectedInjected))
         self.assertEqual(uuid, "demo")
 
+    def test_inject_user_id_with_emoji_content(self):
+        original_xml = b'<data xmlns:jr="http://openrosa.org/javarosa" xmlns:orx="http://openrosa.org/xforms" xmlns:ev="http://www.w3.org/2001/xml-events" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:odk="http://www.opendatakit.org/xforms" xmlns:h="http://www.w3.org/1999/xhtml" id="quality_pca_2.31.8" version="1"><meta><instanceID>uuid:demo</instanceID><editUserID>546</editUserID></meta><prevous_muac_color>&#55357;&#57313;Yellow</prevous_muac_color></data>'
+        uuid, xml = inject_xml_find_uuid(original_xml, 123, 2012010601, 977)
+        expectedInjected = b'<data xmlns:jr="http://openrosa.org/javarosa" xmlns:orx="http://openrosa.org/xforms" xmlns:ev="http://www.w3.org/2001/xml-events" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:odk="http://www.opendatakit.org/xforms" xmlns:h="http://www.w3.org/1999/xhtml" id="quality_pca_2.31.8" version="2012010601" iasoInstance="123"><meta><instanceID>uuid:demo</instanceID><editUserID>977</editUserID></meta><prevous_muac_color>\xf0\x9f\x9f\xa1Yellow</prevous_muac_color></data>'
+        self.assertEqual(str(xml), str(expectedInjected))
+        self.assertEqual(uuid, "demo")
+
     def test_to_xforms_xml(self):
         form = m.Form.objects.create(name="name < with entity", form_id="odk_form_id")
         m.FormVersion.objects.create(form=form, version_id="2012010601")

iaso/tests/fixtures/submission_with_emoji.xml

@@ -0,0 +1 @@
+<?xml version='1.0' ?><data id="entity_registration" version="2024080903" xmlns:h="http://www.w3.org/1999/xhtml" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:jr="http://openrosa.org/javarosa" xmlns:ev="http://www.w3.org/2001/xml-events" xmlns:orx="http://openrosa.org/xforms" xmlns:odk="http://www.opendatakit.org/xforms"><registration_date>2024-07-01T00:00:00.000+02:00</registration_date><registration_date_note /><generated_note_name_4 /><consent><consent_one>yes</consent_one><consent_two /></consent><consent_given><beneficiary><first_name>Test 1_2</first_name><middle_name>Test 1_2</middle_name><last_name>Test 1_2</last_name><gender /><gender_note /></beneficiary><age_group><age_entry>years</age_entry><age_years>25</age_years><age__int__>25</age__int__><actual_birthday__date__ /><birth_date /></age_group><card><record_book_or_vaccination_card /></card><id><document_id /><has_number /><location /></id><food_assistance><general_food_assistance>no</general_food_assistance></food_assistance><physiology><physiology_status>pregnant</physiology_status></physiology><summary><beneficiary_name /><beneficiary_gender /><beneficiary_age /><beneficiary_status /></summary></consent_given><previous_muac>19.0</previous_muac><anthropometric_visit_done>0</anthropometric_visit_done><medical_visit_done>0</medical_visit_done><followup_visits>0</followup_visits><prevous_muac_color>&#55357;&#57313;Yellow</prevous_muac_color><program>TSFP</program><next_visit__date__>2024-07-15</next_visit__date__><previous_soap_given>yes</previous_soap_given><previous_net_given>yes</previous_net_given><previous_number_of_green_visits>0</previous_number_of_green_visits><previous_number_of_yellow_visits>0</previous_number_of_yellow_visits><previous_total_number_of_visits>0</previous_total_number_of_visits><is_exiting>0</is_exiting><discharged_date__date__ /><is_pregnant>yes</is_pregnant><is_breastfeeding /><child_age>0</child_age><actual_child_birthday__date__ /><cured__bool__>0</cured__bool__><defaulter>0</defaulter><pregnancy_child_age>26</pregnancy_child_age><transfer_to_ou_id>0</transfer_to_ou_id><transfer_to_ou_name /><org_unit_id>124</org_unit_id><org_unit_name>Namur</org_unit_name><meta><instanceID>uuid:61edabd0-4759-4560-b0d7-bedd1be38675</instanceID></meta></data>

iaso/tests/models/test_instance.py

@@ -194,6 +194,20 @@ def assertStatusIs(self, instance: m.Instance, status: str):
         instance_with_status = m.Instance.objects.with_status().get(pk=instance.pk)
         self.assertEqual(instance_with_status.status, status)
 
+    def test_xml_to_json_should_contains_emoji(self):
+        self.maxDiff = None
+        instance = m.Instance.objects.create(
+            form=self.form_1,
+            period="202001",
+            org_unit=self.jedi_council_coruscant,
+            file=UploadedFile(open("iaso/tests/fixtures/submission_with_emoji.xml")),
+        )
+        json_instance = instance.get_and_save_json_of_xml()
+
+        self.assertEqual(json_instance["_version"], "2024080903")
+        # assert flattened and  lowered case keys
+        self.assertEqual(json_instance["prevous_muac_color"], "🟡Yellow")
+
     def test_xml_to_json_should_contains_chars_encoding(self):
         instance = m.Instance.objects.create(
             form=self.form_1,

iaso/utils/emoji.py

@@ -0,0 +1,17 @@
+import re
+
+
+def fix_emoji(payload):
+    chrifnotspecial = lambda dec: (
+        "&#%d;" % dec if dec in [10, 13, 35, 38, 59, 60, 62] else chr(dec)
+    )  # don't convert `\n\r#&;<>`
+    payload = re.sub(r"&#(\d+);", lambda x: chrifnotspecial(int(x.group(1))), payload)
+    # No idea why 'INFORMATION SEPARATOR's ended up in some messages,
+    # but I decide that I don't need them, and they make the parser barf out...
+    for dec in [28, 29, 30, 31]:
+        payload = payload.replace(chr(dec), "")
+
+    # combine surrogate pairs
+    payload = payload.encode("utf-16", "surrogatepass").decode("utf-16")
+    payload = payload.encode("utf-8")
+    return payload