diff --git a/src/formpack/constants.py b/src/formpack/constants.py index f992940f..c8112962 100644 --- a/src/formpack/constants.py +++ b/src/formpack/constants.py @@ -154,3 +154,13 @@ 'form_appearance', 'form_meta_edit', ] + +# Analysis types +ANALYSIS_TYPE_CODING = 'coding' +ANALYSIS_TYPE_TRANSCRIPT = 'transcript' +ANALYSIS_TYPE_TRANSLATION = 'translation' +ANALYSIS_TYPES = [ + ANALYSIS_TYPE_CODING, + ANALYSIS_TYPE_TRANSCRIPT, + ANALYSIS_TYPE_TRANSLATION, +] diff --git a/src/formpack/pack.py b/src/formpack/pack.py index 0eec0270..7cf6e1f4 100644 --- a/src/formpack/pack.py +++ b/src/formpack/pack.py @@ -3,9 +3,10 @@ import json from collections import OrderedDict from copy import deepcopy +from typing import Dict from formpack.schema.fields import CopyField -from .version import FormVersion +from .version import FormVersion, AnalysisForm from .reporting import Export, AutoReport from .utils.expand_content import expand_content from .utils.replace_aliases import replace_aliases @@ -52,6 +53,8 @@ def __init__( self.asset_type = asset_type + self.analysis_form = None + self.load_all_versions(versions) # FIXME: Find a safe way to use this. Wrapping with try/except isn't enough @@ -176,6 +179,9 @@ def load_version(self, schema): self.versions[form_version.id] = form_version + def extend_survey(self, analysis_form: Dict) -> None: + self.analysis_form = AnalysisForm(self, analysis_form) + def version_diff(self, vn1, vn2): v1 = self.versions[vn1] v2 = self.versions[vn2] diff --git a/src/formpack/reporting/export.py b/src/formpack/reporting/export.py index 5388a1a3..069d55be 100644 --- a/src/formpack/reporting/export.py +++ b/src/formpack/reporting/export.py @@ -4,16 +4,23 @@ import zipfile from collections import defaultdict, OrderedDict from inspect import isclass -from typing import Iterator, Generator, Optional +from typing import ( + Dict, + Generator, + Iterator, + Optional, +) import xlsxwriter from ..constants import ( + ANALYSIS_TYPE_TRANSCRIPT, + ANALYSIS_TYPE_TRANSLATION, GEO_QUESTION_TYPES, TAG_COLUMNS_AND_SEPARATORS, UNSPECIFIED_TRANSLATION, ) -from ..schema import CopyField +from ..schema import CopyField, FormField from ..submission import FormSubmission from ..utils.exceptions import FormPackGeoJsonError from ..utils.flatten_content import flatten_tag_list @@ -60,9 +67,11 @@ def __init__( :param tag_cols_for_header: list :param filter_fields: list :param xls_types_as_text: bool + :param include_media_url: bool """ self.formpack = formpack + self.analysis_form = formpack.analysis_form self.lang = lang self.group_sep = group_sep self.title = title @@ -81,6 +90,12 @@ def __init__( tag_cols_for_header = [] self.tag_cols_for_header = tag_cols_for_header + _filter_fields = [] + for item in self.filter_fields: + item = re.sub(r'^_supplementalDetails/', '', item) + _filter_fields.append(item) + self.filter_fields = _filter_fields + # If some fields need to be arbitrarily copied, add them # to the first section if copy_fields: @@ -224,6 +239,9 @@ def get_fields_labels_tags_for_all_versions( # Ensure that fields are filtered if they've been specified, otherwise # carry on as usual + if self.analysis_form: + all_fields = self.analysis_form.insert_analysis_fields(all_fields) + if self.filter_fields: all_fields = [ field @@ -320,6 +338,7 @@ def format_one_submission( submission, current_section, attachments=None, + supplemental_details=None, ): # 'current_section' is the name of what will become sheets in xls. @@ -382,17 +401,46 @@ def _get_attachment(val, field, attachments): if re.match(fr'^.*/{_val}$', f['filename']) is not None ] - def _get_value_from_entry(entry, field): + def _get_value_from_supplemental_details( + field: FormField, supplemental_details: Dict + ) -> Optional[str]: + source, name = field.analysis_path + _sup_details = supplemental_details.get(source, {}) + + if not _sup_details: + return + + # The names for translation and transcript fields are in the format + # of `translated_` which must be stripped to get the + # value from the supplemental details dict + if _name := re.match(r'^(translation|transcript)_', name): + name = _name.groups()[0] + + val = _sup_details.get(name) + if val is None: + return '' + + return val + + def _get_value_from_entry( + entry: Dict, field: FormField, supplemental_details: Dict + ) -> Optional[str]: + if field.analysis_question and supplemental_details: + return _get_value_from_supplemental_details( + field, supplemental_details + ) + suffix = 'meta/' if field.data_type == 'audit' else '' return entry.get(f'{suffix}{field.path}') + if self.analysis_form: + _fields = self.analysis_form.insert_analysis_fields(_fields) + # Ensure that fields are filtered if they've been specified, otherwise # carry on as usual if self.filter_fields: _fields = tuple( - field - for field in current_section.fields.values() - if field.path in self.filter_fields + field for field in _fields if field.path in self.filter_fields ) # 'rows' will contain all the formatted entries for the current @@ -423,13 +471,17 @@ def _get_value_from_entry(entry, field): row.update(_empty_row) attachments = entry.get('_attachments') or attachments + supplemental_details = ( + entry.get('_supplementalDetails') or supplemental_details + ) for field in _fields: # TODO: pass a context to fields so they can all format ? if field.can_format: - # get submission value for this field - val = _get_value_from_entry(entry, field) + val = _get_value_from_entry( + entry, field, supplemental_details + ) # get the attachment for this field attachment = _get_attachment(val, field, attachments) # get a mapping of {"col_name": "val", ...} @@ -493,7 +545,8 @@ def _get_value_from_entry(entry, field): chunk = self.format_one_submission( entry[child_section.path], child_section, - attachments, + attachments=attachments, + supplemental_details=supplemental_details, ) for key, value in iter(chunk.items()): if key in chunks: diff --git a/src/formpack/schema/fields.py b/src/formpack/schema/fields.py index 42f6c28f..1ac05c57 100644 --- a/src/formpack/schema/fields.py +++ b/src/formpack/schema/fields.py @@ -8,7 +8,13 @@ import statistics from .datadef import FormDataDef, FormChoice -from ..constants import UNSPECIFIED_TRANSLATION +from ..constants import ( + ANALYSIS_TYPES, + ANALYSIS_TYPE_CODING, + ANALYSIS_TYPE_TRANSCRIPT, + ANALYSIS_TYPE_TRANSLATION, + UNSPECIFIED_TRANSLATION, +) from ..utils import singlemode from ..utils.ordered_collection import OrderedDefaultdict @@ -35,6 +41,20 @@ def __init__( self.section = section self.can_format = can_format self.tags = kwargs.get('tags', []) + self.analysis_question = False + + source = kwargs.get('source') + if source is not None: + self.source = source + self.analysis_question = True + self.analysis_type = kwargs.get('analysis_type') + self.analysis_path = kwargs.get('analysis_path') + self.settings = kwargs.get('settings') + if self.analysis_type in [ + ANALYSIS_TYPE_TRANSCRIPT, + ANALYSIS_TYPE_TRANSLATION, + ]: + self.language = kwargs['language'] hierarchy = list(hierarchy) if hierarchy is not None else [None] self.hierarchy = hierarchy + [self] @@ -45,11 +65,15 @@ def __init__( if has_stats is not None: self.has_stats = has_stats else: - self.has_stats = data_type != 'note' + self.has_stats = data_type != 'note' and not self.analysis_question # do not include the root section in the path self.path = '/'.join(info.name for info in self.hierarchy[1:]) + @property + def qpath(self): + return self.path.replace('/', '-') + def get_labels( self, lang=UNSPECIFIED_TRANSLATION, @@ -139,7 +163,7 @@ def _get_label( # even if `lang` can be None, we don't want the `label` to be None. label = self.labels.get(lang, self.name) # If `label` is None, no matches are found, so return `field` name. - return self.name if label is None else label + return label or self.name def __repr__(self): args = (self.__class__.__name__, self.name, self.data_type) @@ -178,6 +202,12 @@ def from_json_definition( labels = cls._extract_json_labels(definition, translations) appearance = definition.get('appearance') or_other = definition.get('_or_other', False) + source = definition.get('source') + analysis_type = definition.get('analysis_type', ANALYSIS_TYPE_CODING) + settings = definition.get('settings', {}) + analysis_path = definition.get('path') + languages = definition.get('languages') + language = definition.get('language') # normalize spaces data_type = definition['type'] @@ -185,6 +215,9 @@ def from_json_definition( if ' ' in data_type: raise ValueError('invalid data_type: %s' % data_type) + if analysis_type not in ANALYSIS_TYPES: + raise ValueError(f'Invalid analysis data type: {analysis_type}') + if data_type in ('select_one', 'select_multiple'): choice_id = definition['select_from_list_name'] # pyxform#472 introduced dynamic list_names for select_one with the @@ -246,6 +279,12 @@ def from_json_definition( 'section': section, 'choice': choice, 'or_other': or_other, + 'source': source, + 'analysis_type': analysis_type, + 'settings': settings, + 'analysis_path': analysis_path, + 'language': language, + 'languages': languages, } if data_type == 'select_multiple' and appearance == 'literacy': @@ -424,21 +463,6 @@ def get_substats( class TextField(ExtendedFormField): - def get_stats(self, metrics, lang=UNSPECIFIED_TRANSLATION, limit=100): - - stats = super().get_stats(metrics, lang, limit) - - top = metrics.most_common(limit) - total = stats['total_count'] - - percentage = [] - for key, val in top: - percentage.append((key, self._get_percentage(val, total))) - - stats.update({'frequency': top, 'percentage': percentage}) - - return stats - def get_disaggregated_stats( self, metrics, top_splitters, lang=UNSPECIFIED_TRANSLATION, limit=100 ): @@ -459,6 +483,75 @@ def sum_frequencies(element): return stats + def get_labels( + self, + lang=UNSPECIFIED_TRANSLATION, + group_sep='/', + hierarchy_in_labels=False, + multiple_select='both', + *args, + **kwargs, + ): + args = lang, group_sep, hierarchy_in_labels, multiple_select + if getattr(self, 'analysis_type', None) in [ + ANALYSIS_TYPE_TRANSCRIPT, + ANALYSIS_TYPE_TRANSLATION, + ]: + source_label = self.source_field._get_label(*args) + _type = 'translation' if self._is_translation else 'transcript' + return [f'{source_label} - {_type} ({self.language})'] + return [self._get_label(*args)] + + def get_stats(self, metrics, lang=UNSPECIFIED_TRANSLATION, limit=100): + + stats = super().get_stats(metrics, lang, limit) + + top = metrics.most_common(limit) + total = stats['total_count'] + + percentage = [] + for key, val in top: + percentage.append((key, self._get_percentage(val, total))) + + stats.update({'frequency': top, 'percentage': percentage}) + + return stats + + @property + def _is_transcript(self): + return getattr(self, 'analysis_type', '') == ANALYSIS_TYPE_TRANSCRIPT + + @property + def _is_translation(self): + return getattr(self, 'analysis_type', '') == ANALYSIS_TYPE_TRANSLATION + + def format( + self, + val, + lang=UNSPECIFIED_TRANSLATION, + group_sep='/', + hierarchy_in_labels=False, + multiple_select='both', + xls_types_as_text=True, + *args, + **kwargs, + ): + if val is None: + val = '' + + if isinstance(val, dict): + if self._is_translation: + try: + val = val[self.language]['value'] + except KeyError: + val = '' + elif self._is_transcript: + val = ( + val['value'] if val['languageCode'] == self.language else '' + ) + + return {self.name: val} + class MediaField(TextField): def get_labels(self, include_media_url=False, *args, **kwargs): diff --git a/src/formpack/version.py b/src/formpack/version.py index 9bb1e32a..c08b0c13 100644 --- a/src/formpack/version.py +++ b/src/formpack/version.py @@ -1,9 +1,18 @@ # coding: utf-8 -from collections import OrderedDict +from collections import OrderedDict, defaultdict +from typing import ( + Dict, + List, + Union, +) from pyxform import aliases as pyxform_aliases -from .constants import UNTRANSLATED +from .constants import ( + ANALYSIS_TYPE_TRANSCRIPT, + ANALYSIS_TYPE_TRANSLATION, + UNTRANSLATED, +) from .errors import SchemaError from .errors import TranslationError from .schema import FormField, FormGroup, FormSection, FormChoice @@ -42,7 +51,113 @@ def get(self, key, default=None): return self._vals.get(key, default) -class FormVersion: +class BaseForm: + @staticmethod + def _get_field_labels( + field: FormField, + translations: List[str], + ) -> LabelStruct: + if 'label' in field: + if not isinstance(field['label'], list): + field['label'] = [field['label']] + return LabelStruct(labels=field['label'], translations=translations) + return LabelStruct() + + @staticmethod + def _get_fields_by_name( + survey: Dict[str, Union[str, List]] + ) -> Dict[str, Dict[str, Union[str, List]]]: + return {row['name']: row for row in survey if 'name' in row} + + @staticmethod + def _get_translations(content: Dict[str, List]) -> List[str]: + return [ + t if t is not None else UNTRANSLATED + for t in content.get('translations', [None]) + ] + + +class AnalysisForm(BaseForm): + def __init__( + self, + formpack: 'FormPack', + schema: Dict[str, Union[str, List]], + ) -> None: + + self.schema = schema + self.formpack = formpack + + survey = self.schema.get('additional_fields', []) + fields_by_name = self._get_fields_by_name(survey) + section = FormSection(name=formpack.title) + + self.translations = self._get_translations(schema) + + choices_definition = schema.get('additional_choices', ()) + field_choices = FormChoice.all_from_json_definition( + choices_definition, self.translations + ) + + for data_def in survey: + data_type = data_def['type'] + if data_type in [ + ANALYSIS_TYPE_TRANSCRIPT, + ANALYSIS_TYPE_TRANSLATION, + ]: + data_def.update( + { + 'type': 'text', + 'analysis_type': data_type, + } + ) + + field = FormField.from_json_definition( + definition=data_def, + field_choices=field_choices, + section=section, + translations=self.translations, + ) + + field.labels = self._get_field_labels( + field=fields_by_name[field.name], + translations=self.translations, + ) + section.fields[field.name] = field + + self.fields = list(section.fields.values()) + self.fields_by_source = self._get_fields_by_source() + + def __repr__(self) -> str: + return f"" + + def _get_fields_by_source(self) -> Dict[str, List[FormField]]: + fields_by_source = defaultdict(list) + for field in self.fields: + fields_by_source[field.source].append(field) + return fields_by_source + + def _map_sections_to_analysis_fields( + self, survey_field: FormField + ) -> List[FormField]: + _fields = [] + for analysis_field in self.fields_by_source[survey_field.qpath]: + analysis_field.section = survey_field.section + analysis_field.source_field = survey_field + _fields.append(analysis_field) + return _fields + + def insert_analysis_fields( + self, fields: List[FormField] + ) -> List[FormField]: + _fields = [] + for field in fields: + _fields.append(field) + if field.qpath in self.fields_by_source: + _fields += self._map_sections_to_analysis_fields(field) + return _fields + + +class FormVersion(BaseForm): @classmethod def verify_schema_structure(cls, struct): if 'content' not in struct: @@ -97,17 +212,14 @@ def __init__(self, form_pack, schema): content = self.schema['content'] - self.translations = [ - t if t is not None else UNTRANSLATED - for t in content.get('translations', [None]) - ] + self.translations = self._get_translations(content) # TODO: put those parts in a separate method and unit test it survey = content.get('survey', []) survey = self._append_pseudo_questions(survey) - fields_by_name = dict([(row.get('name'), row) for row in survey]) + fields_by_name = self._get_fields_by_name(survey) # Analyze the survey schema and extract the informations we need # to build the export: the sections, the choices, the fields @@ -211,16 +323,7 @@ def __init__(self, form_pack, schema): section.fields[field.name] = field _f = fields_by_name[field.name] - _labels = LabelStruct() - - if 'label' in _f: - if not isinstance(_f['label'], list): - _f['label'] = [_f['label']] - _labels = LabelStruct( - labels=_f['label'], translations=self.translations - ) - - field.labels = _labels + field.labels = self._get_field_labels(_f, self.translations) assert 'labels' not in _f # FIXME: Find a safe way to use this. Wrapping with try/except isn't enough diff --git a/tests/fixtures/analysis_form/__init__.py b/tests/fixtures/analysis_form/__init__.py new file mode 100644 index 00000000..c2e63cb6 --- /dev/null +++ b/tests/fixtures/analysis_form/__init__.py @@ -0,0 +1,15 @@ +# coding: utf-8 +''' +analysis_form +''' + +from ..load_fixture_json import load_fixture_json + +DATA = { + 'title': 'Simple Clerk Interaction', + 'id_string': 'cerk_interaction', + 'versions': [ + load_fixture_json('analysis_form/v1'), + load_fixture_json('analysis_form/v2'), + ], +} diff --git a/tests/fixtures/analysis_form/analysis_form.json b/tests/fixtures/analysis_form/analysis_form.json new file mode 100644 index 00000000..9c5819d3 --- /dev/null +++ b/tests/fixtures/analysis_form/analysis_form.json @@ -0,0 +1,108 @@ +{ + "engines": { + "engines/transcript_manual": { + "details": "A human provided transcription" + } + }, + "additional_fields": [ + { + "type": "transcript", + "name": "record_a_note/transcript_en", + "label": "record_a_note Transcript (en)", + "path": [ + "record_a_note", + "transcript_en" + ], + "language": "en", + "source": "record_a_note", + "settings": { + "mode": "manual", + "engine": "engines/transcript_manual" + } + }, + { + "type": "transcript", + "name": "record_a_note/transcript_es", + "label": "record_a_note Transcript (es)", + "path": [ + "record_a_note", + "transcript_es" + ], + "language": "es", + "source": "record_a_note", + "settings": { + "mode": "manual", + "engine": "engines/transcript_manual" + } + }, + { + "type": "translation", + "name": "record_a_note/translation_en", + "label": "record_a_note Translated (en)", + "language": "en", + "path": [ + "record_a_note", + "translation_en" + ], + "source": "record_a_note", + "settings": { + "mode": "manual", + "engine": "engines/translation_manual" + } + }, + { + "type": "translation", + "name": "record_a_note/translation_es", + "label": "record_a_note Translated (es)", + "language": "es", + "path": [ + "record_a_note", + "translation_es" + ], + "source": "record_a_note", + "settings": { + "mode": "manual", + "engine": "engines/translation_manual" + } + }, + { + "type": "datetime", + "name": "record_a_note/acme_timestamp", + "path": [ + "record_a_note", + "acme_timestamp" + ], + "label": [ + "Transcription Timestamp" + ], + "source": "record_a_note" + }, + { + "type": "text", + "name": "name_of_clerk/comment", + "path": [ + "clerk_details-name_of_clerk", + "comment" + ], + "label": [ + "Comment on the name of the clerk" + ], + "source": "clerk_details-name_of_clerk" + }, + { + "type": "text", + "name": "name_of_shop/comment", + "path": [ + "clerk_details-name_of_shop", + "comment" + ], + "label": [ + "Comment on the name of the shop" + ], + "source": "clerk_details-name_of_shop" + } + ], + "translations": [ + "English (en)" + ] +} diff --git a/tests/fixtures/analysis_form/v1.json b/tests/fixtures/analysis_form/v1.json new file mode 100644 index 00000000..f411725c --- /dev/null +++ b/tests/fixtures/analysis_form/v1.json @@ -0,0 +1,102 @@ +{ + "version": "v1", + "content": { + "survey": [ + { + "type": "audio", + "name": "record_a_note", + "label": [ + "Record a clerk saying something", + "Registri oficiston dirantan ion" + ] + }, + { + "type": "begin_group", + "name": "clerk_details", + "label": [ + "Some details of the clerk", + "Kelkaj detaloj de la oficisto" + ] + }, + { + "type": "text", + "name": "name_of_clerk", + "label": [ + "What is the clerk's name?", + "" + ] + }, + { + "type": "end_group" + } + ], + "settings": { + "version": "v1" + }, + "translated": [ + "label" + ], + "translations": [ + "English (en)", + "Esperanto (es)" + ] + }, + "submissions": [ + { + "record_a_note": "clerk_interaction_1.mp3", + "clerk_details/name_of_clerk": "John", + "_attachments": [ + { + "mimetype": "audio/mpeg", + "filename": "clerk_interaction_1.mp3" + } + ], + "_supplementalDetails": { + "record_a_note": { + "transcript": { + "value": "Saluton, kiel mi povas helpi vin?", + "languageCode": "es" + }, + "translation": { + "en": { + "value": "Hello how may I help you?" + } + }, + "acme_timestamp": "2021-11-01Z" + }, + "clerk_details-name_of_clerk": { + "comment": "Sounds like an interesting person" + } + } + }, + { + "record_a_note": "clerk_interaction_2.mp3", + "clerk_details/name_of_clerk": "Alex", + "_attachments": [ + { + "mimetype": "audio/mpeg", + "filename": "clerk_interaction_2.mp3" + } + ], + "_supplementalDetails": { + "record_a_note": { + "transcript": { + "value": "Thank you for your business", + "languageCode": "en" + } + } + } + }, + { + "record_a_note": "clerk_interaction_3.mp3", + "clerk_details/name_of_clerk": "Olivier", + "_attachments": [ + { + "mimetype": "audio/mpeg", + "filename": "clerk_interaction_3.mp3" + } + ], + "_supplementalDetails": {} + } + ] +} diff --git a/tests/fixtures/analysis_form/v2.json b/tests/fixtures/analysis_form/v2.json new file mode 100644 index 00000000..bdb0cfc1 --- /dev/null +++ b/tests/fixtures/analysis_form/v2.json @@ -0,0 +1,97 @@ +{ + "version": "v2", + "content": { + "survey": [ + { + "type": "audio", + "name": "record_a_note", + "label": [ + "Record a clerk saying something", + "Registri oficiston dirantan ion" + ] + }, + { + "type": "begin_group", + "name": "clerk_details", + "label": [ + "Some details of the clerk", + "Kelkaj detaloj de la oficisto" + ] + }, + { + "type": "text", + "name": "name_of_shop", + "label": [ + "What is the shop's name?", + "Kio estas la nomo de la butiko?" + ] + }, + { + "type": "end_group" + } + ], + "settings": { + "version": "v2" + }, + "translated": [ + "label" + ], + "translations": [ + "English (en)", + "Esperanto (es)" + ] + }, + "submissions": [ + { + "record_a_note": "clerk_interaction_4.mp3", + "clerk_details/name_of_shop": "Save On", + "_attachments": [ + { + "mimetype": "audio/mpeg", + "filename": "clerk_interaction_4.mp3" + } + ], + "_supplementalDetails": { + "record_a_note": { + "transcript": { + "value": "Hello how may I help you?", + "languageCode": "en" + }, + "acme_timestamp": "2021-11-01Z" + }, + "clerk_details-name_of_shop": { + "comment": "Pretty cliche" + } + } + }, + { + "record_a_note": "clerk_interaction_5.mp3", + "clerk_details/name_of_shop": "Walmart", + "_attachments": [ + { + "mimetype": "audio/mpeg", + "filename": "clerk_interaction_5.mp3" + } + ], + "_supplementalDetails": { + "record_a_note": { + "transcript": { + "value": "Thank you for your business", + "languageCode": "en" + } + } + } + }, + { + "record_a_note": "clerk_interaction_6.mp3", + "clerk_details/name_of_shop": "Costco", + "_attachments": [ + { + "mimetype": "audio/mpeg", + "filename": "clerk_interaction_6.mp3" + } + ], + "_supplementalDetails": {} + } + ] +} diff --git a/tests/fixtures/analysis_form_advanced/__init__.py b/tests/fixtures/analysis_form_advanced/__init__.py new file mode 100644 index 00000000..cfa87315 --- /dev/null +++ b/tests/fixtures/analysis_form_advanced/__init__.py @@ -0,0 +1,14 @@ +# coding: utf-8 +''' +analysis_form_advanced +''' + +from ..load_fixture_json import load_fixture_json + +DATA = { + 'title': 'Advanced Clerk Interaction', + 'id_string': 'cerk_interaction_advanced', + 'versions': [ + load_fixture_json('analysis_form_advanced/v1'), + ], +} diff --git a/tests/fixtures/analysis_form_advanced/analysis_form.json b/tests/fixtures/analysis_form_advanced/analysis_form.json new file mode 100644 index 00000000..5da3a4b0 --- /dev/null +++ b/tests/fixtures/analysis_form_advanced/analysis_form.json @@ -0,0 +1,118 @@ +{ + "engines": { + "transcript": { + "details": "an external service provided by ACME, Inc." + } + }, + "additional_fields": [ + { + "type": "transcript", + "name": "record_a_note/transcript_en", + "path": [ + "clerk_interactions-record_a_note", + "transcript_en" + ], + "source": "clerk_interactions-record_a_note", + "language": "en", + "settings": { + "mode": "auto", + "engine": "engines/acme_1_speech2text" + } + }, + { + "type": "select_multiple", + "select_from_list_name": "record_a_note_tones", + "name": "record_a_note/tone_of_voice", + "path": [ + "clerk_interactions-record_a_note", + "tone_of_voice" + ], + "label": [ + "How was the tone of the clerk's voice?", + "Kiel estis la tono de la voĉo de la oficisto?" + ], + "source": "clerk_interactions-record_a_note" + }, + { + "type": "text", + "name": "goods_sold/comment", + "path": [ + "clerk_interactions-goods_sold", + "comment" + ], + "label": [ + "Comment on the goods sold at the store", + "Komentu la varojn venditajn en la vendejo" + ], + "source": "clerk_interactions-goods_sold" + }, + { + "type": "select_one", + "select_from_list_name": "goods_sold_ratings", + "name": "goods_sold/rating", + "path": [ + "clerk_interactions-goods_sold", + "rating" + ], + "label": [ + "Rate the quality of the goods sold at the store", + "Komentu la varojn vendojn en la vendejo" + ], + "source": "clerk_interactions-goods_sold" + } + ], + "additional_choices": [ + { + "list_name": "goods_sold_ratings", + "name": 1, + "label": [ + "Poor quality", + "Malbona kvalito" + ] + }, + { + "list_name": "goods_sold_ratings", + "name": 2, + "label": [ + "Average quality", + "Meza kvalito" + ] + }, + { + "list_name": "ratings", + "name": 3, + "label": [ + "High quality", + "Alta kvalito" + ] + }, + { + "list_name": "record_a_note_tones", + "name": "anxious", + "label": [ + "Anxious", + "Maltrankvila" + ] + }, + { + "list_name": "record_a_note_tones", + "name": "excited", + "label": [ + "Excited", + "Ekscitita" + ] + }, + { + "list_name": "record_a_note_tones", + "name": "confused", + "label": [ + "Confused", + "Konfuzita" + ] + } + ], + "translations": [ + "English (en)", + "Esperanto (es)" + ] +} diff --git a/tests/fixtures/analysis_form_advanced/v1.json b/tests/fixtures/analysis_form_advanced/v1.json new file mode 100644 index 00000000..69a56222 --- /dev/null +++ b/tests/fixtures/analysis_form_advanced/v1.json @@ -0,0 +1,132 @@ +{ + "version": "v1", + "content": { + "survey": [ + { + "type": "begin_group", + "name": "clerk_interactions", + "label": [ + "Clerk interactions", + "Komizo-interagoj" + ] + }, + { + "type": "audio", + "name": "record_a_note", + "label": [ + "Record a clerk saying something", + "Registri oficiston dirantan ion" + ] + }, + { + "type": "select_multiple goods", + "name": "goods_sold", + "label": [ + "What are some goods sold at the store?", + "Kio estas iuj varoj venditaj en la vendejo?" + ] + }, + { + "type": "end_group" + } + ], + "choices": [ + { + "list_name": "goods", + "name": "chocolate", + "label": [ + "Chocolate", + "Ĉokolado" + ] + }, + { + "list_name": "goods", + "name": "fruit", + "label": [ + "Fruit", + "Frukto" + ] + }, + { + "list_name": "goods", + "name": "pasta", + "label": [ + "Pasta", + "Pasto" + ] + } + ], + "settings": { + "version": "v1" + }, + "translated": [ + "label" + ], + "translations": [ + "English (en)", + "Esperanto (es)" + ] + }, + "submissions": [ + { + "clerk_interactions/record_a_note": "clerk_interaction_1.mp3", + "clerk_interactions/goods_sold": "chocolate", + "_attachments": [ + { + "mimetype": "audio/mpeg", + "filename": "clerk_interaction_1.mp3" + } + ], + "_supplementalDetails": { + "clerk_interactions-record_a_note": { + "transcript": { + "value": "Hello how may I help you?", + "languageCode": "en" + }, + "tone_of_voice": "excited confused" + }, + "clerk_interactions-goods_sold": { + "comment": "Not much diversity", + "rating": "3" + } + } + }, + { + "clerk_interactions/record_a_note": "clerk_interaction_2.mp3", + "clerk_interactions/goods_sold": "chocolate fruit pasta", + "_attachments": [ + { + "mimetype": "audio/mpeg", + "filename": "clerk_interaction_2.mp3" + } + ], + "_supplementalDetails": { + "clerk_interactions-record_a_note": { + "transcript": { + "value": "Thank you for your business", + "languageCode": "en" + }, + "tone_of_voice": "anxious excited" + }, + "clerk_interactions-goods_sold": { + "rating": "2" + } + } + }, + { + "clerk_interactions/record_a_note": "clerk_interaction_3.mp3", + "clerk_interactions/goods_sold": "pasta", + "_attachments": [ + { + "mimetype": "audio/mpeg", + "filename": "clerk_interaction_3.mp3" + } + ], + "_supplementalDetails": { + "clerk_interactions-goods_sold": { + "rating": "3" + } + } + } + ] +} diff --git a/tests/fixtures/analysis_form_repeat_groups/__init__.py b/tests/fixtures/analysis_form_repeat_groups/__init__.py new file mode 100644 index 00000000..e484b9b2 --- /dev/null +++ b/tests/fixtures/analysis_form_repeat_groups/__init__.py @@ -0,0 +1,14 @@ +# coding: utf-8 +''' +analysis_form_repeat_groups +''' + +from ..load_fixture_json import load_fixture_json + +DATA = { + 'title': 'Clerk Interaction Repeat Groups', + 'id_string': 'cerk_interaction_repeat_groups', + 'versions': [ + load_fixture_json('analysis_form_repeat_groups/v1'), + ], +} diff --git a/tests/fixtures/analysis_form_repeat_groups/analysis_form.json b/tests/fixtures/analysis_form_repeat_groups/analysis_form.json new file mode 100644 index 00000000..8ccbe323 --- /dev/null +++ b/tests/fixtures/analysis_form_repeat_groups/analysis_form.json @@ -0,0 +1,35 @@ +{ + "engines": { + "transcript": { + "details": "an external service provided by ACME, Inc." + } + }, + "additional_fields": [ + { + "type": "text", + "name": "record_a_note/transcript", + "path": ["record_a_note", "transcript"], + "source": "record_a_note", + "analysis_type": "transcript", + "settings": { + "mode": "auto", + "engine": "engines/acme_1_speech2text" + } + }, + { + "type": "text", + "name": "record_a_noise/comment_on_noise_level", + "path": ["record_a_noise", "comment_on_noise_level"], + "label": [ + "Comment on noise level", + "Komentu pri brunivelo" + ], + "source": "record_a_noise", + "analysis_type": "coding" + } + ], + "translations": [ + "English (en)", + "Esperanto (es)" + ] +} diff --git a/tests/fixtures/analysis_form_repeat_groups/v1.json b/tests/fixtures/analysis_form_repeat_groups/v1.json new file mode 100644 index 00000000..94b65319 --- /dev/null +++ b/tests/fixtures/analysis_form_repeat_groups/v1.json @@ -0,0 +1,170 @@ +{ + "version": "v1", + "content": { + "survey": [ + { + "type": "text", + "name": "enumerator_name", + "label": [ + "What is your name?", + "Kio estas via nomo?" + ] + }, + { + "type": "begin_repeat", + "name": "stores", + "label": [ + "Stores", + "Vendejoj" + ] + }, + { + "type": "text", + "name": "store_name", + "label": [ + "What is the store name?", + "Kio estas la nomo de la vendejo?" + ] + }, + { + "type": "begin_group", + "name": "recordings", + "label": [ + "Recordings", + "Registradoj" + ] + }, + { + "type": "begin_repeat", + "name": "record_interactions", + "label": [ + "Record interactions", + "Registri interagojn" + ] + }, + { + "type": "audio", + "name": "record_a_note", + "label": [ + "Record a clerk saying something", + "Registri oficiston dirantan ion" + ] + }, + { + "type": "end_repeat" + }, + { + "type": "begin_repeat", + "name": "record_ambient_noises", + "label": [ + "Record ambient noises", + "Registri ĉirkaŭajn bruojn" + ] + }, + { + "type": "audio", + "name": "record_a_noise", + "label": [ + "Record some abient noise", + "Registru iun ĉirkaŭan bruon" + ] + }, + { + "type": "end_repeat" + }, + { + "type": "end_group" + }, + { + "type": "end_repeat" + } + ], + "settings": { + "version": "v1" + }, + "translated": [ + "label" + ], + "translations": [ + "English (en)", + "Esperanto (es)" + ] + }, + "submissions": [ + { + "enumerator_name": "John Doe", + "stores": [ + { + "stores/store_name": "Costco", + "stores/recordings/record_interactions": [ + { + "stores/recordings/record_interactions/record_a_note": "clerk_interaction_1.mp3" + }, + { + "stores/recordings/record_interactions/record_a_note": "clerk_interaction_2.mp3" + }, + { + "stores/recordings/record_interactions/record_a_note": "clerk_interaction_3.mp3" + } + ], + "stores/recordings/record_ambient_noises": [ + { + "stores/recordings/record_ambient_noises/record_a_noise": "noise_1.mp3" + }, + { + "stores/recordings/record_ambient_noises/record_a_noise": "noise_2.mp3" + } + ] + } + ], + "_attachments": [ + { + "mimetype": "audio/mpeg", + "filename": "clerk_interaction_1.mp3" + }, + { + "mimetype": "audio/mpeg", + "filename": "clerk_interaction_2.mp3" + }, + { + "mimetype": "audio/mpeg", + "filename": "clerk_interaction_3.mp3" + }, + { + "mimetype": "audio/mpeg", + "filename": "noise_1.mp3" + }, + { + "mimetype": "audio/mpeg", + "filename": "noise_2.mp3" + } + ], + "_supplementalDetails": { + "record_a_note": { + "transcript": [ + { + "_index": 0, + "value": "Hello how may I help you?" + }, + { + "_index": 2, + "value": "Thank you for your business" + } + ] + }, + "record_a_noise": { + "comment_on_noise_level": [ + { + "_index": 0, + "value": "Lot's of noise" + }, + { + "_index": 1, + "value": "Quiet" + } + ] + } + } + } + ] +} diff --git a/tests/fixtures/build_fixture.py b/tests/fixtures/build_fixture.py index a36bbba4..64ac360b 100644 --- a/tests/fixtures/build_fixture.py +++ b/tests/fixtures/build_fixture.py @@ -31,6 +31,7 @@ def build_fixture(modulename, data_variable_name='DATA'): for submission in schema.pop('submissions'): submission.update({version_id_key: version}) submissions.append(submission) + return title, schemas, submissions diff --git a/tests/fixtures/load_fixture_json.py b/tests/fixtures/load_fixture_json.py index 24a46824..482974c9 100644 --- a/tests/fixtures/load_fixture_json.py +++ b/tests/fixtures/load_fixture_json.py @@ -9,3 +9,8 @@ def load_fixture_json(fname): with open(os.path.join(CUR_DIR, '%s.json' % fname)) as ff: content_ = ff.read() return json.loads(content_) + + +def load_analysis_form_json(path): + with open(os.path.join(CUR_DIR, path, 'analysis_form.json')) as f: + return json.loads(f.read()) diff --git a/tests/test_additional_field_exports.py b/tests/test_additional_field_exports.py new file mode 100644 index 00000000..b9c190ec --- /dev/null +++ b/tests/test_additional_field_exports.py @@ -0,0 +1,538 @@ +# coding: utf-8 +import unittest +from formpack import FormPack +from .fixtures import build_fixture +from .fixtures.load_fixture_json import load_analysis_form_json + + +def test_additional_field_exports_without_labels(): + title, schemas, submissions = build_fixture('analysis_form') + analysis_form = load_analysis_form_json('analysis_form') + pack = FormPack(schemas, title=title) + pack.extend_survey(analysis_form) + + options = { + 'versions': 'v1', + 'filter_fields': [ + 'record_a_note', + '_supplementalDetails/record_a_note/transcript_en', + '_supplementalDetails/record_a_note/transcript_es', + '_supplementalDetails/record_a_note/translation_en', + '_supplementalDetails/record_a_note/translation_es', + '_supplementalDetails/record_a_note/acme_timestamp', + ], + } + export = pack.export(**options) + values = export.to_dict(submissions) + main_export_sheet = values['Simple Clerk Interaction'] + + assert 3 == len(main_export_sheet['data']) + assert main_export_sheet['fields'] == [ + 'record_a_note', + 'record_a_note - transcript (en)', + 'record_a_note - transcript (es)', + 'record_a_note - translation (en)', + 'record_a_note - translation (es)', + 'record_a_note/acme_timestamp', + ] + response0 = main_export_sheet['data'][0] + assert response0 == [ + 'clerk_interaction_1.mp3', + '', + 'Saluton, kiel mi povas helpi vin?', + 'Hello how may I help you?', + '', + '2021-11-01Z', + ] + + +def test_additional_field_exports_with_labels(): + title, schemas, submissions = build_fixture('analysis_form') + analysis_form = load_analysis_form_json('analysis_form') + pack = FormPack(schemas, title=title) + pack.extend_survey(analysis_form) + + options = { + 'versions': 'v1', + 'filter_fields': [ + 'record_a_note', + '_supplementalDetails/record_a_note/transcript_en', + '_supplementalDetails/record_a_note/transcript_es', + '_supplementalDetails/record_a_note/translation_en', + '_supplementalDetails/record_a_note/translation_es', + '_supplementalDetails/record_a_note/acme_timestamp', + ], + 'lang': 'English (en)', + } + export = pack.export(**options) + values = export.to_dict(submissions) + main_export_sheet = values['Simple Clerk Interaction'] + + assert 3 == len(main_export_sheet['data']) + assert main_export_sheet['fields'] == [ + 'Record a clerk saying something', + 'Record a clerk saying something - transcript (en)', + 'Record a clerk saying something - transcript (es)', + 'Record a clerk saying something - translation (en)', + 'Record a clerk saying something - translation (es)', + 'Transcription Timestamp', + ] + response0 = main_export_sheet['data'][0] + assert response0 == [ + 'clerk_interaction_1.mp3', + '', + 'Saluton, kiel mi povas helpi vin?', + 'Hello how may I help you?', + '', + '2021-11-01Z', + ] + + +@unittest.skip('Currently not supporting repeat groups') +def test_additional_field_exports_repeat_groups(): + title, schemas, submissions = build_fixture('analysis_form_repeat_groups') + analysis_form = load_analysis_form_json('analysis_form_repeat_groups') + pack = FormPack(schemas, title=title) + pack.extend_survey(analysis_form) + + options = { + 'versions': 'v1', + } + export = pack.export(**options) + values = export.to_dict(submissions) + assert [ + 'Clerk Interaction Repeat Groups', + 'stores', + 'record_interactions', + 'record_ambient_noises', + ] == list(values.keys()) + + main_export_sheet = values['Clerk Interaction Repeat Groups'] + assert ['enumerator_name', '_index'] == main_export_sheet['fields'] + main_response0 = main_export_sheet['data'][0] + assert 'John Doe' == main_response0[0] + + repeat_sheet_0 = values['stores'] + assert 'Costco' == repeat_sheet_0['data'][0][0] + + repeat_sheet_1 = values['record_interactions'] + assert [ + 'record_a_note', + 'record_a_note/transcript_acme_1_speech2text', + ] == repeat_sheet_1['fields'][:2] + assert 3 == len(repeat_sheet_1['data']) + repeat_data_response_1 = [res[:2] for res in repeat_sheet_1['data']] + repeat_data_expected_1 = [ + [ + 'clerk_interaction_1.mp3', + 'Hello how may I help you?', + ], + [ + 'clerk_interaction_2.mp3', + '', + ], + [ + 'clerk_interaction_3.mp3', + 'Thank you for your business', + ], + ] + assert repeat_data_expected_1 == repeat_data_response_1 + + repeat_sheet_2 = values['record_ambient_noises'] + assert [ + 'record_a_noise', + 'record_a_noise/comment_on_noise_level', + ] == repeat_sheet_2['fields'][:2] + assert 2 == len(repeat_sheet_2['data']) + repeat_data_response_2 = [res[:2] for res in repeat_sheet_2['data']] + repeat_data_expected_2 = [ + [ + 'noise_1.mp3', + "Lot's of noise", + ], + [ + 'noise_2.mp3', + 'Quiet', + ], + ] + assert repeat_data_expected_2 == repeat_data_response_2 + + +def test_additional_field_exports_advanced(): + title, schemas, submissions = build_fixture('analysis_form_advanced') + analysis_form = load_analysis_form_json('analysis_form_advanced') + pack = FormPack(schemas, title=title) + pack.extend_survey(analysis_form) + + options = { + 'versions': 'v1', + 'multiple_select': 'both', + } + export = pack.export(**options) + values = export.to_dict(submissions) + main_export_sheet = values['Advanced Clerk Interaction'] + + assert 3 == len(main_export_sheet['data']) + assert main_export_sheet['fields'] == [ + 'record_a_note', + 'record_a_note - transcript (en)', + 'record_a_note/tone_of_voice', + 'record_a_note/tone_of_voice/anxious', + 'record_a_note/tone_of_voice/excited', + 'record_a_note/tone_of_voice/confused', + 'goods_sold', + 'goods_sold/chocolate', + 'goods_sold/fruit', + 'goods_sold/pasta', + 'goods_sold/comment', + 'goods_sold/rating', + ] + assert main_export_sheet['data'] == [ + [ + 'clerk_interaction_1.mp3', + 'Hello how may I help you?', + 'excited confused', + '0', + '1', + '1', + 'chocolate', + '1', + '0', + '0', + 'Not much diversity', + '3', + ], + [ + 'clerk_interaction_2.mp3', + 'Thank you for your business', + 'anxious excited', + '1', + '1', + '0', + 'chocolate fruit pasta', + '1', + '1', + '1', + '', + '2', + ], + [ + 'clerk_interaction_3.mp3', + '', + '', + '', + '', + '', + 'pasta', + '0', + '0', + '1', + '', + '3', + ], + ] + + options['multiple_select'] = 'details' + export = pack.export(**options) + values = export.to_dict(submissions) + main_export_sheet = values['Advanced Clerk Interaction'] + + assert main_export_sheet['fields'] == [ + 'record_a_note', + 'record_a_note - transcript (en)', + 'record_a_note/tone_of_voice/anxious', + 'record_a_note/tone_of_voice/excited', + 'record_a_note/tone_of_voice/confused', + 'goods_sold/chocolate', + 'goods_sold/fruit', + 'goods_sold/pasta', + 'goods_sold/comment', + 'goods_sold/rating', + ] + assert main_export_sheet['data'] == [ + [ + 'clerk_interaction_1.mp3', + 'Hello how may I help you?', + '0', + '1', + '1', + '1', + '0', + '0', + 'Not much diversity', + '3', + ], + [ + 'clerk_interaction_2.mp3', + 'Thank you for your business', + '1', + '1', + '0', + '1', + '1', + '1', + '', + '2', + ], + [ + 'clerk_interaction_3.mp3', + '', + '', + '', + '', + '0', + '0', + '1', + '', + '3', + ], + ] + + options['multiple_select'] = 'summary' + export = pack.export(**options) + values = export.to_dict(submissions) + main_export_sheet = values['Advanced Clerk Interaction'] + + assert main_export_sheet['fields'] == [ + 'record_a_note', + 'record_a_note - transcript (en)', + 'record_a_note/tone_of_voice', + 'goods_sold', + 'goods_sold/comment', + 'goods_sold/rating', + ] + assert main_export_sheet['data'] == [ + [ + 'clerk_interaction_1.mp3', + 'Hello how may I help you?', + 'excited confused', + 'chocolate', + 'Not much diversity', + '3', + ], + [ + 'clerk_interaction_2.mp3', + 'Thank you for your business', + 'anxious excited', + 'chocolate fruit pasta', + '', + '2', + ], + [ + 'clerk_interaction_3.mp3', + '', + '', + 'pasta', + '', + '3', + ], + ] + + +def test_additional_field_exports_v2(): + title, schemas, submissions = build_fixture('analysis_form') + analysis_form = load_analysis_form_json('analysis_form') + pack = FormPack(schemas, title=title) + pack.extend_survey(analysis_form) + + options = {'versions': 'v2'} + export = pack.export(**options) + values = export.to_dict(submissions) + main_export_sheet = values['Simple Clerk Interaction'] + + assert 3 == len(main_export_sheet['data']) + assert main_export_sheet['fields'] == [ + 'record_a_note', + 'record_a_note - transcript (en)', + 'record_a_note - transcript (es)', + 'record_a_note - translation (en)', + 'record_a_note - translation (es)', + 'record_a_note/acme_timestamp', + 'name_of_shop', + 'name_of_shop/comment', + ] + response0 = main_export_sheet['data'][0] + assert response0 == [ + 'clerk_interaction_4.mp3', + 'Hello how may I help you?', + '', + '', + '', + '2021-11-01Z', + 'Save On', + 'Pretty cliche', + ] + + +def test_additional_field_exports_all_versions(): + title, schemas, submissions = build_fixture('analysis_form') + analysis_form = load_analysis_form_json('analysis_form') + pack = FormPack(schemas, title=title) + pack.extend_survey(analysis_form) + + options = {'versions': pack.versions} + export = pack.export(**options) + values = export.to_dict(submissions) + main_export_sheet = values['Simple Clerk Interaction'] + + assert 6 == len(main_export_sheet['data']) + assert main_export_sheet['fields'] == [ + 'record_a_note', + 'record_a_note - transcript (en)', + 'record_a_note - transcript (es)', + 'record_a_note - translation (en)', + 'record_a_note - translation (es)', + 'record_a_note/acme_timestamp', + 'name_of_shop', + 'name_of_shop/comment', + 'name_of_clerk', + 'name_of_clerk/comment', + ] + response0 = main_export_sheet['data'][0] + assert response0 == [ + 'clerk_interaction_1.mp3', + '', + 'Saluton, kiel mi povas helpi vin?', + 'Hello how may I help you?', + '', + '2021-11-01Z', + '', + '', + 'John', + 'Sounds like an interesting person', + ] + response3 = main_export_sheet['data'][3] + assert response3 == [ + 'clerk_interaction_4.mp3', + 'Hello how may I help you?', + '', + '', + '', + '2021-11-01Z', + 'Save On', + 'Pretty cliche', + '', + '', + ] + + +def test_additional_field_exports_all_versions_exclude_fields(): + title, schemas, submissions = build_fixture('analysis_form') + analysis_form = load_analysis_form_json('analysis_form') + pack = FormPack(schemas, title=title) + pack.extend_survey(analysis_form) + + options = { + 'versions': pack.versions, + 'filter_fields': [ + 'record_a_note', + '_supplementalDetails/clerk_details/name_of_shop', + '_supplementalDetails/clerk_details/name_of_clerk', + ], + } + export = pack.export(**options) + values = export.to_dict(submissions) + main_export_sheet = values['Simple Clerk Interaction'] + + assert 6 == len(main_export_sheet['data']) + assert main_export_sheet['fields'] == [ + 'record_a_note', + 'name_of_shop', + 'name_of_clerk', + ] + response0 = main_export_sheet['data'][0] + assert response0 == [ + 'clerk_interaction_1.mp3', + '', + 'John', + ] + response3 = main_export_sheet['data'][3] + assert response3 == [ + 'clerk_interaction_4.mp3', + 'Save On', + '', + ] + + +def test_additional_field_exports_all_versions_langs(): + title, schemas, submissions = build_fixture('analysis_form') + analysis_form = load_analysis_form_json('analysis_form') + pack = FormPack(schemas, title=title) + pack.extend_survey(analysis_form) + + options = { + 'versions': pack.versions, + 'lang': 'English (en)', + } + export = pack.export(**options) + values = export.to_dict(submissions) + main_export_sheet = values['Simple Clerk Interaction'] + + assert main_export_sheet['fields'] == [ + 'Record a clerk saying something', + 'Record a clerk saying something - transcript (en)', + 'Record a clerk saying something - transcript (es)', + 'Record a clerk saying something - translation (en)', + 'Record a clerk saying something - translation (es)', + 'Transcription Timestamp', + "What is the shop's name?", + 'Comment on the name of the shop', + "What is the clerk's name?", + 'Comment on the name of the clerk', + ] + + options['lang'] = 'Esperanto (es)' + export = pack.export(**options) + values = export.to_dict(submissions) + main_export_sheet = values['Simple Clerk Interaction'] + + assert main_export_sheet['fields'] == [ + 'Registri oficiston dirantan ion', + 'Registri oficiston dirantan ion - transcript (en)', + 'Registri oficiston dirantan ion - transcript (es)', + 'Registri oficiston dirantan ion - translation (en)', + 'Registri oficiston dirantan ion - translation (es)', + 'record_a_note/acme_timestamp', + 'Kio estas la nomo de la butiko?', + 'name_of_shop/comment', + 'name_of_clerk', + 'name_of_clerk/comment', + ] + + options['lang'] = None + export = pack.export(**options) + values = export.to_dict(submissions) + main_export_sheet = values['Simple Clerk Interaction'] + + assert main_export_sheet['fields'] == [ + 'record_a_note', + 'record_a_note - transcript (en)', + 'record_a_note - transcript (es)', + 'record_a_note - translation (en)', + 'record_a_note - translation (es)', + 'record_a_note/acme_timestamp', + 'name_of_shop', + 'name_of_shop/comment', + 'name_of_clerk', + 'name_of_clerk/comment', + ] + + +def test_simple_report_with_analysis_form(): + title, schemas, submissions = build_fixture('analysis_form') + analysis_form = load_analysis_form_json('analysis_form') + pack = FormPack(schemas, title) + pack.extend_survey(analysis_form) + + lang = 'English (en)' + report = pack.autoreport(versions=pack.versions.keys()) + stats = report.get_stats(submissions, lang=lang) + + assert stats.submissions_count == 6 + + stats = set([n for f, n, d in stats]) + analysis_fields = set( + [f._get_label(lang=lang) for f in pack.analysis_form.fields] + ) + # Ensure analysis fields aren't making it into the report + assert not stats.intersection(analysis_fields) diff --git a/tests/test_fixtures_valid.py b/tests/test_fixtures_valid.py index f40581cd..0d647b26 100644 --- a/tests/test_fixtures_valid.py +++ b/tests/test_fixtures_valid.py @@ -3,6 +3,8 @@ from formpack import FormPack from .fixtures import build_fixture +from .fixtures.load_fixture_json import load_analysis_form_json +from formpack.constants import ANALYSIS_TYPES class TestFormPackFixtures(unittest.TestCase): @@ -117,3 +119,31 @@ def test_xml_instances_loaded(self): """ fp = FormPack(**build_fixture('favcolor')) self.assertEqual(len(fp.versions), 2) + + def test_analysis_form(self): + fixture = build_fixture('analysis_form') + assert 3 == len(fixture) + + title, schemas, submissions = fixture + analysis_form = load_analysis_form_json('analysis_form') + fp = FormPack(schemas, title) + fp.extend_survey(analysis_form) + + assert 2 == len(fp.versions) + assert 'Simple Clerk Interaction' == title + + expected_analysis_questions = sorted( + [f['name'] for f in analysis_form['additional_fields']] + ) + actual_analysis_questions = sorted( + [f.name for f in fp.analysis_form.fields] + ) + assert expected_analysis_questions == actual_analysis_questions + + f1 = fp.analysis_form.fields[0] + assert hasattr(f1, 'source') and f1.source + assert hasattr(f1, 'has_stats') and not f1.has_stats + assert ( + hasattr(f1, 'analysis_type') and f1.analysis_type in ANALYSIS_TYPES + ) + assert hasattr(f1, 'settings') diff --git a/tests/test_kobo_locking.py b/tests/test_kobo_locking.py index 705c6d25..ef15500b 100644 --- a/tests/test_kobo_locking.py +++ b/tests/test_kobo_locking.py @@ -50,7 +50,7 @@ def _construct_xlsx_for_import(self, sheet_name, sheet_content): for row_num, row_list in enumerate(sheet_content): for col_num, cell_value in enumerate(row_list): if cell_value and cell_value is not None: - worksheet.cell(row_num+1, col_num+1).value = cell_value + worksheet.cell(row_num + 1, col_num + 1).value = cell_value xlsx_import_io = BytesIO() workbook_to_import.save(xlsx_import_io) xlsx_import_io.seek(0)