From b0c64592b645b12e376d0843366bf5e6c4757f04 Mon Sep 17 00:00:00 2001 From: Alex Dorey Date: Tue, 14 Feb 2017 08:45:25 -0800 Subject: [PATCH 01/14] utility method to pull remote data into a local directory for testing formpack reports --- src/formpack/remote_pack.py | 109 ++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 src/formpack/remote_pack.py diff --git a/src/formpack/remote_pack.py b/src/formpack/remote_pack.py new file mode 100644 index 00000000..e512acf0 --- /dev/null +++ b/src/formpack/remote_pack.py @@ -0,0 +1,109 @@ +# coding: utf-8 + +from __future__ import (unicode_literals, print_function, + absolute_import, division) + +from .pack import FormPack + +import os +import json +import errno +import requests + +from urlparse import urlparse + + +def mkdir_p(path): + try: + os.makedirs(path) + except OSError as exc: # Python >2.5 + if exc.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise + + +def _get_kobo_environ_vars(): + for env_var in [ + 'KOBO_API_TOKEN', + 'KOBO_API_URL', + ]: + if not env_var in os.environ: + raise ValueError('Configuration value not present: {}'.format(env_var)) + _data_dir = os.environ.get('FORMPACK_DATA_DIRECTORY', None) + if not _data_dir: + _data_dir = os.path.join(os.path.expanduser('~'), + '.formpack') + mkdir_p(_data_dir) + return ( + os.environ['KOBO_API_TOKEN'], + os.environ['KOBO_API_URL'], + _data_dir, + ) + + +class RemoteFormPack: + def __init__(self, uid): + self.uid = uid + (self.api_token, + self.api_url, + self._data_dir) = _get_kobo_environ_vars() + + self.data_dir = os.path.join(self._data_dir, self.uid) + mkdir_p(self.data_dir) + self._versions_dir = os.path.join(self.data_dir, 'versions') + self._data_path = os.path.join(self.data_dir, 'data.json') + mkdir_p(self._versions_dir) + + _url = '{}{}/?format=json'.format(self.api_url, self.uid) + r1 = requests.get(_url, + headers=self._headers(), + ).json() + self._deployment_identifier = r1['deployment__identifier'] + version_id = r1['version_id'] + _version_file_path = self._version_file_path(version_id) + + if not os.path.exists(_version_file_path): + with open(_version_file_path, 'w') as ff: + ff.write(json.dumps(resp_data['content'], indent=4)) + + _deployment = urlparse(self._deployment_identifier) + self._kc_api_url = '{}://{}/api/v1'.format(_deployment.scheme, + _deployment.netloc) + r2 = requests.get('{}/forms?id_string={}'.format(self._kc_api_url, self.uid), + headers=self._headers()).json() + self.kc_formid = r2[0]['formid'] + + def pull(self): + _data_url = '{}/data/{}'.format(self._kc_api_url, self.kc_formid) + resp = requests.get('{}{}'.format(_data_url, '?format=json'), + headers=self._headers()) + with open(self._data_path, 'w') as ff: + ff.write(resp.content) + _version_ids = set([i['__version__'] for i in resp.json()]) + for _version_id in _version_ids: + self._ensure_version(_version_id) + + def _ensure_version(self, version_id): + _f = self._version_file_path(version_id) + if not os.path.exists(_f): + raise Exception('version content not found. please write' + ' content to file and rerun script\n{}'.format(_f)) + + def _version_file_path(self, version_id): + return os.path.join(self._versions_dir, '{}.json'.format(version_id)) + + def _headers(self, upd={}): + return dict({'Content-Type': 'application/json', + 'Authorization': 'Token {}'.format(self.api_token), + }, **upd) + + def pack(self): + with open(self._data_path, 'r') as ff: + self.submissions = json.loads(ff.read()) + _version_ids = set([s['__version__'] for s in self.submissions]) + self.versions = [] + for version_id in _version_ids: + with open(self._version_file_path(version_id), 'r') as ff: + self.versions.append({'content': json.loads(ff.read())}) + return FormPack(versions=self.versions, id_string=self.uid) From 073f5054ceac4dc8774c740268ad9dc68d96826f Mon Sep 17 00:00:00 2001 From: Alex Dorey Date: Mon, 6 Mar 2017 11:04:33 -0800 Subject: [PATCH 02/14] * moved `from_json_definition` class methods out to the methods that use them because they are not reused very much, and hide the transformation that goes into building the Fields, Sections, and Versions * added a `src` parameter to many of the fields / sections to facilitate debugging the content that was used to create the python objects in the form `version.Version` * added a `pull.py` script to expose options to the CLI * `select_one_external` is now the preferred alias throughout the code (underscores instead of spaces) * added comment about xform_tools.normalize_data_type (obsolete) with `replace_aliases`, which is run on the content before the version is initialized. --- pull.py | 82 ++++++++++++ src/formpack/pack.py | 45 +++++-- src/formpack/remote_pack.py | 203 +++++++++++++++++++----------- src/formpack/schema/__init__.py | 74 ++++++++++- src/formpack/schema/datadef.py | 73 +++-------- src/formpack/schema/fields.py | 71 ----------- src/formpack/utils/xform_tools.py | 9 +- src/formpack/version.py | 115 ++++++++++++++--- 8 files changed, 445 insertions(+), 227 deletions(-) create mode 100644 pull.py diff --git a/pull.py b/pull.py new file mode 100644 index 00000000..d1d3b241 --- /dev/null +++ b/pull.py @@ -0,0 +1,82 @@ +from formpack.remote_pack import RemoteFormPack, FORMPACK_DATA_DIR + +import os +import json +import argparse + + +parser = argparse.ArgumentParser(description='Initialize RemoteFormPack.') + +parser.add_argument('--refresh-data', + dest='refresh_data', + help='flush data cache', + action='store_true') + +parser.add_argument('--print-stats', + dest='print_stats', + help='print stats from the dataset', + action='store_true') + +parser.add_argument('--print-survey', + dest='print_survey', + help='print survey questions', + action='store_true') + +parser.add_argument('--print-submissions', + dest='print_submissions', + help='print submissions from the dataset', + action='store_true') + +parser.add_argument('--account', + dest='account', + help='server:account corresponding to local config', + action='store') + +parser.add_argument('uid', + nargs=1, + help='formid', + action='store') + + +def load_pack(uid, account): + accounts_file = os.environ.get('KOBO_ACCOUNTS', False) + if not accounts_file: + accounts_file = os.path.join(FORMPACK_DATA_DIR, 'accounts.json') + if not os.path.exists(accounts_file): + raise ValueError('need an accounts json file in {}'.format( + accounts_file)) + with open(accounts_file, 'r') as ff: + accounts = json.loads(ff.read()) + try: + _account = accounts[account] + except KeyError: + raise ValueError('accounts.json needs a configuration for {}'.format( + account)) + return RemoteFormPack(uid=uid, + token=_account['token'], + api_url=_account['api_url'], + ) + + +def run(args): + rpack = load_pack(uid=args.uid[0], + account=args.account) + + if args.refresh_data: + print('clearing submissions') + rpack.clear_submissions() + + rpack.pull() + formpk = rpack.create_pack() + + if args.print_stats: + print(json.dumps(formpk._stats, indent=2)) + + if args.print_survey: + print(json.dumps(formpk.get_survey(), indent=2)) + + if args.print_submissions: + print(json.dumps(list(rpack.submissions), indent=2)) + +if __name__ == '__main__': + run(parser.parse_args()) diff --git a/src/formpack/pack.py b/src/formpack/pack.py index 5dc3273c..79bde4c3 100644 --- a/src/formpack/pack.py +++ b/src/formpack/pack.py @@ -47,6 +47,7 @@ def __init__(self, versions=None, title='Submissions', id_string=None, self.root_node_name = root_node_name self.title = title + self.strict_schema = strict_schema # excel sheet name size limit @@ -54,11 +55,10 @@ def __init__(self, versions=None, title='Submissions', id_string=None, self.title = self.title[:28] + '...' self.asset_type = asset_type - self.load_all_versions(versions) def __repr__(self): - return '' % self._stats() + return '' % self._stats_str def version_id_keys(self, _versions=None): # if no parameter is passed, default to 'all' @@ -71,6 +71,12 @@ def version_id_keys(self, _versions=None): _id_keys.append(_id_key) return _id_keys + @property + def latest_version(self): + if len(self.versions) > 0: + return self.versions.values()[-1] + else: + raise ValueError('No versions available.') @property def available_translations(self): @@ -97,15 +103,27 @@ def __getitem__(self, index): except IndexError: raise IndexError('version at index %d is not available' % index) + @property def _stats(self): _stats = OrderedDict() + _stats['title'] = self.title _stats['id_string'] = self.id_string - _stats['versions'] = len(self.versions) - # _stats['submissions'] = self.submissions_count() - _stats['row_count'] = len(self[-1].schema.get('content', {}) - .get('survey', [])) + _vs = self.versions.values() + if len(_vs) > 0: + _content = _vs[-1].schema.get('content', {}) + _survey = _content.get('survey', []) + _stats['row_count'] = len(_survey) + + _versions = [] + for (vid, version) in self.versions.items(): + _versions.append(version._stats()) + _stats['versions'] = _versions + return _stats + + @property + def _stats_str(self): # returns stats in the format [ key="value" ] - return '\n\t'.join('%s="%s"' % item for item in _stats.items()) + return '\n\t'.join('%s="%s"' % item for item in self._stats.items()) def load_all_versions(self, versions): for schema in versions: @@ -165,6 +183,14 @@ def load_version(self, schema): self.versions[form_version.id] = form_version + def _latest_change(self): + _lvs = len(self.versions) + _keys = self.versions.keys() + if _lvs > 1: + v1 = _keys[_lvs - 2] + v2 = _keys[_lvs - 1] + return self.version_diff(v1, v2) + def version_diff(self, vn1, vn2): v1 = self.versions[vn1] v2 = self.versions[vn2] @@ -270,6 +296,11 @@ def to_dict(self, **kwargs): out[u'asset_type'] = self.asset_type return out + def get_survey(self): + return [ + row for row in self.latest_version.rows(include_groups=True) + ] + def to_json(self, **kwargs): return json.dumps(self.to_dict(), **kwargs) diff --git a/src/formpack/remote_pack.py b/src/formpack/remote_pack.py index e512acf0..6aea7a21 100644 --- a/src/formpack/remote_pack.py +++ b/src/formpack/remote_pack.py @@ -5,105 +5,158 @@ from .pack import FormPack -import os import json import errno import requests - from urlparse import urlparse +from argparse import Namespace as Ns +from os import (path, makedirs, unlink) + +FORMPACK_DATA_DIR = path.join(path.expanduser('~'), + '.formpack') -def mkdir_p(path): +def mkdir_p(_path): try: - os.makedirs(path) + makedirs(_path) except OSError as exc: # Python >2.5 - if exc.errno == errno.EEXIST and os.path.isdir(path): + if exc.errno == errno.EEXIST and path.isdir(_path): pass else: raise -def _get_kobo_environ_vars(): - for env_var in [ - 'KOBO_API_TOKEN', - 'KOBO_API_URL', - ]: - if not env_var in os.environ: - raise ValueError('Configuration value not present: {}'.format(env_var)) - _data_dir = os.environ.get('FORMPACK_DATA_DIRECTORY', None) - if not _data_dir: - _data_dir = os.path.join(os.path.expanduser('~'), - '.formpack') - mkdir_p(_data_dir) - return ( - os.environ['KOBO_API_TOKEN'], - os.environ['KOBO_API_URL'], - _data_dir, - ) - - class RemoteFormPack: - def __init__(self, uid): + def __init__(self, uid, + token, + api_url, + data_dir=None): self.uid = uid - (self.api_token, - self.api_url, - self._data_dir) = _get_kobo_environ_vars() + self.api_token = token + self.api_url = api_url + self._data_dir = data_dir or FORMPACK_DATA_DIR - self.data_dir = os.path.join(self._data_dir, self.uid) + self.data_dir = path.join(self._data_dir, self.uid) mkdir_p(self.data_dir) - self._versions_dir = os.path.join(self.data_dir, 'versions') - self._data_path = os.path.join(self.data_dir, 'data.json') - mkdir_p(self._versions_dir) - - _url = '{}{}/?format=json'.format(self.api_url, self.uid) - r1 = requests.get(_url, - headers=self._headers(), - ).json() - self._deployment_identifier = r1['deployment__identifier'] - version_id = r1['version_id'] - _version_file_path = self._version_file_path(version_id) - - if not os.path.exists(_version_file_path): - with open(_version_file_path, 'w') as ff: - ff.write(json.dumps(resp_data['content'], indent=4)) - - _deployment = urlparse(self._deployment_identifier) - self._kc_api_url = '{}://{}/api/v1'.format(_deployment.scheme, - _deployment.netloc) - r2 = requests.get('{}/forms?id_string={}'.format(self._kc_api_url, self.uid), - headers=self._headers()).json() - self.kc_formid = r2[0]['formid'] + self.paths = { + 'versions/': self.path('versions'), + 'data': self.path('data.json'), + 'context': self.path('context.json'), + 'asset': self.path('asset.json'), + } + self._versions_dir = self.path('versions') + self._data_path = self.path('data.json') + self._context_path = self.path('context.json') + mkdir_p(self.path('versions')) + self._asset_url = '{}{}'.format(self.api_url, self.uid) + self.asset = Ns(**self._query_asset()) + self.context = Ns(**self._query_kcform()) + + def path(self, *args): + return path.join(self.data_dir, *args) + + def _query_asset(self): + if not path.exists(self.path('asset.json')): + ad = requests.get('{}/?format=json'.format(self._asset_url), + headers=self._headers(), + ).json() + if 'detail' in ad and ad['detail'] == 'Invalid token.': + raise ValueError('Invalid token. Is it the correct server?') + elif 'detail' in ad: + raise ValueError("Error querying API: {}".format( + ad['detail'])) + with open(self.path('asset.json'), 'w') as ff: + ff.write(json.dumps(ad)) + return ad + else: + with open(self.path('asset.json'), 'r') as ff: + return json.loads(ff.read()) + + def _query_kcform(self): + asset = self.asset + if not path.exists(self.path('context.json')): + _deployment_identifier = asset.deployment__identifier + _deployment = urlparse(_deployment_identifier) + ctx = { + 'kc_api_url': '{}://{}/api/v1'.format(_deployment.scheme, + _deployment.netloc), + } + _url = '{}/forms?id_string={}'.format(ctx['kc_api_url'], + self.uid) + r2 = requests.get(_url, headers=self._headers()).json() + ctx['kc_formid'] = r2[0]['formid'] + with open(self.path('context.json'), 'w') as ff: + ff.write(json.dumps(ctx)) + return ctx + else: + with open(self.path('context.json'), 'r') as ff: + return json.loads(ff.read()) def pull(self): - _data_url = '{}/data/{}'.format(self._kc_api_url, self.kc_formid) - resp = requests.get('{}{}'.format(_data_url, '?format=json'), - headers=self._headers()) - with open(self._data_path, 'w') as ff: - ff.write(resp.content) - _version_ids = set([i['__version__'] for i in resp.json()]) - for _version_id in _version_ids: - self._ensure_version(_version_id) - - def _ensure_version(self, version_id): - _f = self._version_file_path(version_id) - if not os.path.exists(_f): - raise Exception('version content not found. please write' - ' content to file and rerun script\n{}'.format(_f)) - - def _version_file_path(self, version_id): - return os.path.join(self._versions_dir, '{}.json'.format(version_id)) + if not path.exists(self.path('data.json')): + _data_url = '{}/data/{}?{}'.format(self.context.kc_api_url, + self.context.kc_formid, + 'format=json', + ) + _data = requests.get(_data_url, headers=self._headers()).json() + with open(self.path('data.json'), 'w') as ff: + ff.write(json.dumps(_data)) + _version_ids = set([i['__version__'] for i in _data]) + for version_id in _version_ids: + self.load_version(version_id) + + def load_version(self, version_id): + _version_path = path.join(self.path('versions'), + '{}.json'.format(version_id) + ) + if not path.exists(_version_path): + _version_url = '{}/{}/{}/?format=json'.format( + self._asset_url, + 'versions', + version_id) + vd = requests.get(_version_url, headers=self._headers()).json() + with open(_version_path, 'w') as ff: + ff.write(json.dumps(vd)) + return vd + else: + with open(_version_path, 'r') as ff: + return json.loads(ff.read()) def _headers(self, upd={}): return dict({'Content-Type': 'application/json', 'Authorization': 'Token {}'.format(self.api_token), - }, **upd) + }, **upd) - def pack(self): - with open(self._data_path, 'r') as ff: - self.submissions = json.loads(ff.read()) + def create_pack(self): + if not path.exists(self._data_path): + raise Exception('cannot generate formpack without running ' + 'remote_pack.pull()') _version_ids = set([s['__version__'] for s in self.submissions]) self.versions = [] for version_id in _version_ids: - with open(self._version_file_path(version_id), 'r') as ff: - self.versions.append({'content': json.loads(ff.read())}) - return FormPack(versions=self.versions, id_string=self.uid) + _v = self.load_version(version_id) + _v['version'] = version_id + _v['date'] = _v.pop('date_deployed', None) + self.versions.append(_v) + return FormPack(versions=self.versions, id_string=self.uid, + title=self.asset.name, ellipsize_title=False, + ) + + def stats(self): + pck = self.create_pack() + _stats = pck._stats() + return _stats + + def _submissions(self): + with open(self.path('data.json'), 'r') as ff: + return json.loads(ff.read()) + + def clear_submissions(self): + _data_path = self.path('data.json') + if path.exists(_data_path): + unlink(_data_path) + + @property + def submissions(self): + for submission in self._submissions(): + yield submission diff --git a/src/formpack/schema/__init__.py b/src/formpack/schema/__init__.py index d7c756e8..8f0211ef 100644 --- a/src/formpack/schema/__init__.py +++ b/src/formpack/schema/__init__.py @@ -4,5 +4,77 @@ absolute_import, division) +from functools import partial + from .fields import * # noqa -from .datadef import * # noqa \ No newline at end of file +from .datadef import * # noqa + + +def _field_from_dict(definition, hierarchy=None, + section=None, field_choices={}, + translations=None): + """Return an instance of a Field class matching this JSON field def + + Depending of the data datype extracted from the field definition, + this method will return an instance of a different class. + + Args: + definition (dict): Description + group (FormGroup, optional): The group this field is into + section (FormSection, optional): The section this field is into + field_choices (dict, optional): + A mapping of all the FormChoice instances available for + this form. + + Returns: + Union[FormChoiceField, FormChoiceField, + FormChoiceFieldWithMultipleSelect, FormField]: + The FormField instance matching this definiton. + """ + name = definition.get('$autoname', definition.get('name')) + label = definition.get('label') + if label: + labels = OrderedDict(zip(translations, label)) + else: + labels = {} + + # normalize spaces + data_type = definition['type'] + choice = None + + if ' ' in data_type: + raise ValueError('invalid data_type: %s' % data_type) + + if data_type in ('select_one', 'select_multiple'): + choice_id = definition['select_from_list_name'] + choice = field_choices[choice_id] + + data_type_classes = { + "select_one": FormChoiceField, + "select_multiple": FormChoiceFieldWithMultipleSelect, + "geopoint": FormGPSField, + "date": DateField, + "text": TextField, + "barcode": TextField, + + # calculate is usually not text but for our purpose it's good + # enough + "calculate": TextField, + "acknowledge": TextField, + "integer": NumField, + 'decimal': NumField, + + # legacy type, treat them as text + "select_one_external": partial(TextField, data_type=data_type), + "cascading_select": partial(TextField, data_type=data_type), + } + + cls = data_type_classes.get(data_type, FormField) + return cls(name=name, + labels=labels, + data_type=data_type, + hierarchy=hierarchy, + section=section, + choice=choice, + src=definition, + ) diff --git a/src/formpack/schema/datadef.py b/src/formpack/schema/datadef.py index 73982605..34fa59a2 100644 --- a/src/formpack/schema/datadef.py +++ b/src/formpack/schema/datadef.py @@ -21,11 +21,14 @@ class FormDataDef(object): """ Any object composing a form. It's only used with a subclass. """ - def __init__(self, name, labels=None, has_stats=False, *args, **kwargs): + def __init__(self, name, labels=None, + has_stats=False, src=None, + *args, **kwargs): self.name = name self.labels = labels or {} self.value_names = self.get_value_names() self.has_stats = has_stats + self.src = src def __repr__(self): return "<%s name='%s'>" % (self.__class__.__name__, self.name) @@ -33,25 +36,6 @@ def __repr__(self): def get_value_names(self): return [self.name] - @classmethod - def from_json_definition(cls, definition): - labels = cls._extract_json_labels(definition) - return cls(definition['name'], labels) - - @classmethod - def _extract_json_labels(cls, definition): - """ Extract translation labels from the JSON data definition """ - labels = OrderedDict() - if "label" in definition: - labels[UNTRANSLATED] = definition['label'] - - for key, val in definition.items(): - if key.startswith('label:'): - # sometime the label can be separated with 2 :: - _, lang = re.split(r'::?', key, maxsplit=1, flags=re.U) - labels[lang] = val - return labels - class FormGroup(FormDataDef): # useful to get __repr__ pass @@ -76,10 +60,18 @@ def __init__(self, name="submissions", labels=None, fields=None, # do not include the root section in the path self.path = '/'.join(info.name for info in self.hierarchy[1:]) - @classmethod - def from_json_definition(cls, definition, hierarchy=(None,), parent=None): - labels = cls._extract_json_labels(definition) - return cls(definition['name'], labels, hierarchy=hierarchy, parent=parent) + @property + def rows(self, include_groups=False): + for (name, field) in self.fields.items(): + if include_groups and hasattr(self, 'begin_rows'): + for row in self.begin_rows: + yield row.src + + yield field.src + + if include_groups and hasattr(self, 'end_rows'): + for row in self.end_rows: + yield row.src def get_label(self, lang=UNTRANSLATED): return [self.labels.get(lang) or self.name] @@ -91,38 +83,9 @@ def __repr__(self): class FormChoice(FormDataDef): def __init__(self, name, *args, **kwargs): - super(FormChoice, self).__init__(name, *args, **kwargs) self.name = name - self.options = OrderedDict() - - @classmethod - def all_from_json_definition(cls, definition, translation_list): - all_choices = {} - for choice_definition in definition: - choice_name = choice_definition.get('name') - choice_key = choice_definition.get('list_name') - if not choice_name or not choice_key: - continue - - if choice_key not in all_choices: - all_choices[choice_key] = FormChoice(choice_key) - choices = all_choices[choice_key] - - option = choices.options[choice_name] = {} - - # apparently choices dont need a label if they have an image - if 'label' in choice_definition: - _label = choice_definition['label'] - else: - _label = choice_definition.get('image') - if isinstance(_label, basestring): - _label = [_label] - elif _label is None and len(translation_list) == 1: - _label = [None] - option['labels'] = OrderedDict(zip(translation_list, _label)) - option['name'] = choice_name - return all_choices - + self.options = kwargs.pop('options', OrderedDict()) + super(FormChoice, self).__init__(name, *args, **kwargs) @property def translations(self): diff --git a/src/formpack/schema/fields.py b/src/formpack/schema/fields.py index 9657a587..31b13495 100644 --- a/src/formpack/schema/fields.py +++ b/src/formpack/schema/fields.py @@ -6,7 +6,6 @@ import re from operator import itemgetter -from functools import partial try: xrange = xrange @@ -106,76 +105,6 @@ def __repr__(self): args = (self.__class__.__name__, self.name, self.data_type) return "<%s name='%s' type='%s'>" % args - @classmethod - def from_json_definition(cls, definition, hierarchy=None, - section=None, field_choices={}, - translations=None): - """Return an instance of a Field class matching this JSON field def - - Depending of the data datype extracted from the field definition, - this method will return an instance of a different class. - - Args: - definition (dict): Description - group (FormGroup, optional): The group this field is into - section (FormSection, optional): The section this field is into - field_choices (dict, optional): - A mapping of all the FormChoice instances available for - this form. - - Returns: - Union[FormChoiceField, FormChoiceField, - FormChoiceFieldWithMultipleSelect, FormField]: - The FormField instance matching this definiton. - """ - name = definition['name'] - label = definition.get('label') - if label: - labels = OrderedDict(zip(translations, label)) - else: - labels = {} - - # normalize spaces - data_type = definition['type'] - choice = None - - if ' ' in data_type: - raise ValueError('invalid data_type: %s' % data_type) - - if data_type in ('select_one', 'select_multiple'): - choice_id = definition['select_from_list_name'] - choice = field_choices[choice_id] - - data_type_classes = { - "select_one": FormChoiceField, - "select_multiple": FormChoiceFieldWithMultipleSelect, - "geopoint": FormGPSField, - "date": DateField, - "text": TextField, - "barcode": TextField, - - # calculate is usually not text but for our purpose it's good - # enough - "calculate": TextField, - "acknowledge": TextField, - "integer": NumField, - 'decimal': NumField, - - # legacy type, treat them as text - "select_one_external": partial(TextField, data_type=data_type), - "cascading_select": partial(TextField, data_type=data_type), - } - - args = { - 'name': name, - 'labels': labels, - 'data_type': data_type, - 'hierarchy': hierarchy, - 'section': section, - 'choice': choice - } - return data_type_classes.get(data_type, cls)(**args) - def format(self, val, lang=UNSPECIFIED_TRANSLATION, context=None): return {self.name: val} diff --git a/src/formpack/utils/xform_tools.py b/src/formpack/utils/xform_tools.py index 2da4c4b5..b33dac47 100644 --- a/src/formpack/utils/xform_tools.py +++ b/src/formpack/utils/xform_tools.py @@ -11,6 +11,7 @@ from pyquery import PyQuery +# made obsolete by replace_aliases DATA_TYPE_ALIASES = ( ("add select one prompt using", 'select_one'), ("select one from", 'select_one'), @@ -20,7 +21,7 @@ ("select all that apply from", 'select_multiple'), ("select multiple", 'select_multiple'), ("select all that apply", 'select_multiple'), - ("select_one_external", "select one external"), + ("select one external", "select_one_external"), ('cascading select', 'cascading_select'), ('location', 'geopoint'), ("begin lgroup", 'begin_repeat'), @@ -79,8 +80,12 @@ def parse_xml_to_data(xml_str): def normalize_data_type(data_type): - """ Normalize spaces and aliases for field data types """ + """ + Normalize spaces and aliases for field data types + note: this method is made obsolete by the pre-processing + "replace_aliases" step. + """ # normalize spaces data_type = ' '.join(data_type.split()) diff --git a/src/formpack/version.py b/src/formpack/version.py index 62ac6c4b..4cd88796 100644 --- a/src/formpack/version.py +++ b/src/formpack/version.py @@ -18,6 +18,7 @@ from .errors import SchemaError from .utils.flatten_content import flatten_content from .schema import (FormField, FormGroup, FormSection, FormChoice) +from .schema import _field_from_dict from .errors import TranslationError @@ -41,6 +42,56 @@ def get(self, key, default=None): return self._vals.get(key, default) +def get_labels(choice_definition, translation_list): + # choices dont need a label if they have an image + if 'label' in choice_definition: + _label = choice_definition['label'] + elif 'image' in choice_definition: + _label = choice_definition['image'] + else: + _label = None + + if isinstance(_label, basestring): + _label = [_label] + elif _label is None and len(translation_list) == 1: + _label = [None] + + return OrderedDict(zip(translation_list, _label)) + + +def choices_from_structures(definition, translation_list): + all_choices = {} + for choice_definition in definition: + choice_name = choice_definition.get('$autovalue', + choice_definition.get('name')) + choice_key = choice_definition.get('list_name') + if not choice_name or not choice_key: + continue + + if choice_key not in all_choices: + all_choices[choice_key] = { + 'name': choice_key, + 'options': OrderedDict(), + } + + all_choices[choice_key]['options'][choice_name] = { + 'labels': get_labels(choice_definition, translation_list), + 'name': choice_name, + } + return all_choices.items() + + +def extract_json_labels(definition, column, translations): + _ld = OrderedDict() + labels = definition.get(column, []) + for (i, translation) in enumerate(translations): + if i < len(labels): + _ld[translation] = labels[i] + else: + continue + return _ld + + class FormVersion(object): @classmethod def verify_schema_structure(cls, struct): @@ -50,10 +101,7 @@ def verify_schema_structure(cls, struct): raise SchemaError('version content must have "survey"') validate_content(struct['content']) - # QUESTION FOR ALEX: get rid off _root_node_name ? What is it for ? def __init__(self, form_pack, schema): - - # QUESTION FOR ALEX: why this check ? if 'name' in schema: raise ValueError('FormVersion should not have a name parameter. ' 'consider using "title" or "id_string"') @@ -65,6 +113,7 @@ def __init__(self, form_pack, schema): # form version id, unique to this version of the form self.id = schema.get('version') + self.date = schema.get('date') self.version_id_key = schema.get('version_id_key', form_pack.default_version_id_key) @@ -98,7 +147,12 @@ def __init__(self, form_pack, schema): # TODO: put those parts in a separate method and unit test it survey = content.get('survey', []) - fields_by_name = dict(map(lambda row: (row.get('name'), row), survey)) + + fields_by_name = dict(map(lambda row: + (row.get('$autoname', row.get('name')), + row, + ), + survey)) # Analyze the survey schema and extract the informations we need # to build the export: the sections, the choices, the fields @@ -108,12 +162,17 @@ def __init__(self, form_pack, schema): # Choices are the list of values you can choose from to answer a # specific question. They can have translatable labels. choices_definition = content.get('choices', ()) - field_choices = FormChoice.all_from_json_definition(choices_definition, - self.translations) + + field_choices = dict([ + (key, FormChoice(key, options=itm['options'])) + for (key, itm) in choices_from_structures(choices_definition, + list(self.translations)) + ]) # Extract fields data group = None - section = FormSection(name=form_pack.title) + section = FormSection(name=form_pack.title, src=False) + self._main_section = section self.sections[form_pack.title] = section # Those will keep track of were we are while traversing the @@ -131,6 +190,8 @@ def __init__(self, form_pack, schema): continue data_type = normalize_data_type(data_type) + if '$autoname' in data_definition: + data_definition['name'] = data_definition.get('$autoname') name = data_definition.get('name') # parse closing groups and repeat @@ -159,7 +220,12 @@ def __init__(self, form_pack, schema): if data_type == 'begin_group': group_stack.append(group) - group = FormGroup.from_json_definition(data_definition) + + labels = extract_json_labels(data_definition, 'label', + self.translations) + group = FormGroup(data_definition['name'], labels, + src=data_definition) + # We go down in one level on nesting, so save the parent group. # Parent maybe None, in that case we are at the top level. hierarchy.append(group) @@ -170,9 +236,17 @@ def __init__(self, form_pack, schema): # Parent maybe None, in that case we are at the top level. parent_section = section - section = FormSection.from_json_definition(data_definition, - hierarchy, - parent=parent_section) + labels = extract_json_labels(data_definition, + 'label', + self.translations) + _repeat_name = data_definition.get('$autoname', data_definition.get('name')) + section = FormSection(_repeat_name, + labels, + hierarchy=hierarchy, + src=data_definition, + parent=parent_section, + ) + self.sections[section.name] = section hierarchy.append(section) section_stack.append(parent_section) @@ -181,10 +255,10 @@ def __init__(self, form_pack, schema): # If we are here, it's a regular field # Get the the data name and type - field = FormField.from_json_definition(data_definition, - hierarchy, section, - field_choices, - translations=self.translations) + field = _field_from_dict(data_definition, + hierarchy, section, + field_choices, + translations=self.translations) section.fields[field.name] = field _f = fields_by_name[field.name] @@ -200,14 +274,23 @@ def __init__(self, form_pack, schema): assert 'labels' not in _f def __repr__(self): - return '' % self._stats() + return '' % self._stats_str() + + def rows(self, include_groups=False): + for row in self._main_section.rows: + yield row def _stats(self): _stats = OrderedDict() _stats['id_string'] = self._get_id_string() _stats['version'] = self.id + _stats['date'] = self.date _stats['row_count'] = len(self.schema.get('content', {}).get('survey', [])) # returns stats in the format [ key="value" ] + return _stats + + def _stats_str(self): + _stats = self._stats() return '\n\t'.join(map(lambda key: '%s="%s"' % (key, str(_stats[key])), _stats.keys())) From 4f1487538cb1d90adeff02990b210e8988f457af Mon Sep 17 00:00:00 2001 From: Alex Dorey Date: Wed, 8 Mar 2017 12:36:57 -0800 Subject: [PATCH 03/14] added __init__ file and README to support loading production forms + data as tests --- tests/fixtures/remote_pack/README | 0 tests/fixtures/remote_pack/README.rst | 7 ++++ tests/fixtures/remote_pack/__init__.py | 48 ++++++++++++++++++++++++++ 3 files changed, 55 insertions(+) create mode 100644 tests/fixtures/remote_pack/README create mode 100644 tests/fixtures/remote_pack/README.rst create mode 100644 tests/fixtures/remote_pack/__init__.py diff --git a/tests/fixtures/remote_pack/README b/tests/fixtures/remote_pack/README new file mode 100644 index 00000000..e69de29b diff --git a/tests/fixtures/remote_pack/README.rst b/tests/fixtures/remote_pack/README.rst new file mode 100644 index 00000000..7b21255e --- /dev/null +++ b/tests/fixtures/remote_pack/README.rst @@ -0,0 +1,7 @@ +This directory provides an `__init__.py` file which can be put in the same +directory as a backup of production data via the remote_pack.py CLI utility + +Simply copy or move the directory containing data (example: "~/.formpack/aFoRmId654321") +to the parent directory "tests/fixtures" and copy the accompanying `__init__.py` +file. The data can then be tested with formpack through the "build_fixtures(...)" +method. diff --git a/tests/fixtures/remote_pack/__init__.py b/tests/fixtures/remote_pack/__init__.py new file mode 100644 index 00000000..b89a866e --- /dev/null +++ b/tests/fixtures/remote_pack/__init__.py @@ -0,0 +1,48 @@ +# coding: utf-8 + +from __future__ import (unicode_literals, print_function, + absolute_import, division) + +''' +This file provides a way to open production data and +write a test based on that data. +''' +import os +import glob +import json +from collections import defaultdict + + +DIR = os.path.dirname(os.path.abspath(__file__)) + + +def _path(*args): + return os.path.join(DIR, *args) + +def _load_version_file(vf): + with open(vf, 'r') as version_f: + vx = json.loads(version_f.read()) + uid = vx.pop('uid') + vx.update({ + 'version': uid, + 'submissions': submissions.get(uid, []) + }) + return vx + +with open(_path('asset.json'), 'r') as asset_file: + asset = json.loads(asset_file.read()) + +with open(_path('data.json'), 'r') as submission_f: + submissions = defaultdict(list) + for submission in json.loads(submission_f.read()): + vkey = submission['__version__'] + submissions[vkey].append(submission) + +_version_file_list = glob.glob(_path('versions', '*')) + + +DATA = { + 'title': asset['name'], + 'versions': [_load_version_file(vf) for vf in _version_file_list], + 'submissions': submissions, +} From 041295970a03e13540b0c61d9c2d663b015cbaf4 Mon Sep 17 00:00:00 2001 From: Alex Dorey Date: Fri, 10 Mar 2017 11:10:22 -0800 Subject: [PATCH 04/14] removing extra README --- tests/fixtures/remote_pack/README | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 tests/fixtures/remote_pack/README diff --git a/tests/fixtures/remote_pack/README b/tests/fixtures/remote_pack/README deleted file mode 100644 index e69de29b..00000000 From b6f5517299ff92d5f26c92ffc31310a50f574d83 Mon Sep 17 00:00:00 2001 From: Alex Dorey Date: Fri, 10 Mar 2017 11:15:18 -0800 Subject: [PATCH 05/14] added missing requirement for remote-pack work --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b26f8d14..6fb8a8ee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,5 @@ path.py==8.1.2 PyExcelerate==0.6.7 pyquery==1.2.11 pyxform==0.9.22 -statistics==1.0.3.5 \ No newline at end of file +requests==2.13.0 +statistics==1.0.3.5 From 75b0f82e4604a7153d9227a0cf507ddf959a9293 Mon Sep 17 00:00:00 2001 From: Alex Dorey Date: Sat, 11 Mar 2017 23:32:38 -0800 Subject: [PATCH 06/14] with a cli argument "--run-module=importable_module" with a method "run" that receives the formpack object --- pull.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pull.py b/pull.py index d1d3b241..6589259d 100644 --- a/pull.py +++ b/pull.py @@ -3,6 +3,7 @@ import os import json import argparse +import importlib parser = argparse.ArgumentParser(description='Initialize RemoteFormPack.') @@ -27,6 +28,11 @@ help='print submissions from the dataset', action='store_true') +parser.add_argument('--run-module', + dest='run_module', + help='import and run a module', + action='store') + parser.add_argument('--account', dest='account', help='server:account corresponding to local config', @@ -78,5 +84,13 @@ def run(args): if args.print_submissions: print(json.dumps(list(rpack.submissions), indent=2)) + if args.run_module: + _mod = importlib.import_module(args.run_module) + if not hasattr(_mod, 'run') and hasattr(_mod.run, '__call__'): + _mod.run(formpk, submissions=rpack.submissions) + else: + raise ValueError('run-module parameter must be an importable ' + 'module with a method "run"') + if __name__ == '__main__': run(parser.parse_args()) From 4f1789b96f863a3629631100fbfeb6bf05f22af2 Mon Sep 17 00:00:00 2001 From: Alex Dorey Date: Sun, 12 Mar 2017 10:18:07 -0700 Subject: [PATCH 07/14] * indent json files on save * strip "content" field from asset.json (redundant with versions json stored separately) * use `date_deployed` rather than `date` (for now) --- src/formpack/remote_pack.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/formpack/remote_pack.py b/src/formpack/remote_pack.py index 6aea7a21..507a9b91 100644 --- a/src/formpack/remote_pack.py +++ b/src/formpack/remote_pack.py @@ -65,8 +65,10 @@ def _query_asset(self): elif 'detail' in ad: raise ValueError("Error querying API: {}".format( ad['detail'])) + # content is ultimately pulled form the "version" file + del ad['content'] with open(self.path('asset.json'), 'w') as ff: - ff.write(json.dumps(ad)) + ff.write(json.dumps(ad, indent=2)) return ad else: with open(self.path('asset.json'), 'r') as ff: @@ -86,7 +88,7 @@ def _query_kcform(self): r2 = requests.get(_url, headers=self._headers()).json() ctx['kc_formid'] = r2[0]['formid'] with open(self.path('context.json'), 'w') as ff: - ff.write(json.dumps(ctx)) + ff.write(json.dumps(ctx, indent=2)) return ctx else: with open(self.path('context.json'), 'r') as ff: @@ -100,7 +102,7 @@ def pull(self): ) _data = requests.get(_data_url, headers=self._headers()).json() with open(self.path('data.json'), 'w') as ff: - ff.write(json.dumps(_data)) + ff.write(json.dumps(_data, indent=2)) _version_ids = set([i['__version__'] for i in _data]) for version_id in _version_ids: self.load_version(version_id) @@ -116,7 +118,7 @@ def load_version(self, version_id): version_id) vd = requests.get(_version_url, headers=self._headers()).json() with open(_version_path, 'w') as ff: - ff.write(json.dumps(vd)) + ff.write(json.dumps(vd, indent=2)) return vd else: with open(_version_path, 'r') as ff: @@ -136,7 +138,7 @@ def create_pack(self): for version_id in _version_ids: _v = self.load_version(version_id) _v['version'] = version_id - _v['date'] = _v.pop('date_deployed', None) + _v['date_deployed'] = _v.pop('date_deployed', None) self.versions.append(_v) return FormPack(versions=self.versions, id_string=self.uid, title=self.asset.name, ellipsize_title=False, From d6543ed3615df01160a17fa95fcda791e614b879 Mon Sep 17 00:00:00 2001 From: Alex Dorey Date: Sun, 12 Mar 2017 13:20:33 -0700 Subject: [PATCH 08/14] * allow submissions to be stored in FormPack object. * at first, this will be used in just tests; eventually it could be a way to store the submissions that fed to the formpack via generators. --- src/formpack/pack.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/formpack/pack.py b/src/formpack/pack.py index 79bde4c3..f0e018c1 100644 --- a/src/formpack/pack.py +++ b/src/formpack/pack.py @@ -23,12 +23,11 @@ class FormPack(object): - - # TODO: make a clear signature for __init__ def __init__(self, versions=None, title='Submissions', id_string=None, default_version_id_key='__version__', strict_schema=False, root_node_name='data', + submissions=None, asset_type=None, submissions_xml=None, ellipsize_title=True): if not versions: @@ -46,6 +45,8 @@ def __init__(self, versions=None, title='Submissions', id_string=None, self.id_string = id_string self.root_node_name = root_node_name + self.submissions = submissions + self.title = title self.strict_schema = strict_schema From 340432fb0c579fb56681036509955321a917cdcb Mon Sep 17 00:00:00 2001 From: Alex Dorey Date: Tue, 14 Mar 2017 17:12:13 -0700 Subject: [PATCH 09/14] * using date_deployed as in previous commit --- src/formpack/version.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/formpack/version.py b/src/formpack/version.py index 4cd88796..13a7e0e5 100644 --- a/src/formpack/version.py +++ b/src/formpack/version.py @@ -113,7 +113,8 @@ def __init__(self, form_pack, schema): # form version id, unique to this version of the form self.id = schema.get('version') - self.date = schema.get('date') + self.date = schema.get('date_deployed') + self.version_id_key = schema.get('version_id_key', form_pack.default_version_id_key) @@ -207,8 +208,8 @@ def __init__(self, form_pack, schema): continue if data_type == 'end_repeat': - # We go up in one level of nesting, so we set the current section - # to be what used to be the parent section + # We go up in one level of nesting, so we set the current + # section to be what used to be the parent section hierarchy.pop() section = section_stack.pop() continue @@ -222,7 +223,7 @@ def __init__(self, form_pack, schema): group_stack.append(group) labels = extract_json_labels(data_definition, 'label', - self.translations) + self.translations) group = FormGroup(data_definition['name'], labels, src=data_definition) @@ -237,8 +238,8 @@ def __init__(self, form_pack, schema): parent_section = section labels = extract_json_labels(data_definition, - 'label', - self.translations) + 'label', + self.translations) _repeat_name = data_definition.get('$autoname', data_definition.get('name')) section = FormSection(_repeat_name, labels, From c8ef353df689b33e16495320dab993a090508bb1 Mon Sep 17 00:00:00 2001 From: Alex Dorey Date: Thu, 16 Mar 2017 12:28:40 -0700 Subject: [PATCH 10/14] * moved and renamed "LabelStruct" into a module translated_item * passing TranslatedItem() object instead of dict when fields and sections are created * added assertions which, when uncommented, should not fail (but don't want to risk AssertionErrors in production) --- src/formpack/schema/__init__.py | 10 ++++--- src/formpack/schema/datadef.py | 8 ++++-- src/formpack/schema/fields.py | 2 +- src/formpack/translated_item.py | 31 ++++++++++++++++++++ src/formpack/version.py | 51 +++++++-------------------------- tests/test_translated_item.py | 45 +++++++++++++++++++++++++++++ 6 files changed, 99 insertions(+), 48 deletions(-) create mode 100644 src/formpack/translated_item.py create mode 100644 tests/test_translated_item.py diff --git a/src/formpack/schema/__init__.py b/src/formpack/schema/__init__.py index 8f0211ef..3b5050bc 100644 --- a/src/formpack/schema/__init__.py +++ b/src/formpack/schema/__init__.py @@ -8,6 +8,7 @@ from .fields import * # noqa from .datadef import * # noqa +from ..translated_item import TranslatedItem def _field_from_dict(definition, hierarchy=None, @@ -32,11 +33,12 @@ def _field_from_dict(definition, hierarchy=None, The FormField instance matching this definiton. """ name = definition.get('$autoname', definition.get('name')) - label = definition.get('label') - if label: - labels = OrderedDict(zip(translations, label)) + labels = definition.get('label') + if labels: + labels = TranslatedItem(labels, + translations=translations) else: - labels = {} + labels = TranslatedItem() # normalize spaces data_type = definition['type'] diff --git a/src/formpack/schema/datadef.py b/src/formpack/schema/datadef.py index 34fa59a2..cc89d747 100644 --- a/src/formpack/schema/datadef.py +++ b/src/formpack/schema/datadef.py @@ -16,6 +16,7 @@ from collections import OrderedDict from ..constants import UNTRANSLATED +from ..translated_item import TranslatedItem class FormDataDef(object): @@ -25,7 +26,8 @@ def __init__(self, name, labels=None, has_stats=False, src=None, *args, **kwargs): self.name = name - self.labels = labels or {} + # assert labels is None or isinstance(labels, TranslatedItem) + self.labels = labels or TranslatedItem() self.value_names = self.get_value_names() self.has_stats = has_stats self.src = src @@ -49,11 +51,11 @@ def __init__(self, name="submissions", labels=None, fields=None, *args, **kwargs): if labels is None: - labels = {UNTRANSLATED: 'submissions'} + labels = TranslatedItem() + self.parent = parent super(FormSection, self).__init__(name, labels, *args, **kwargs) self.fields = fields or OrderedDict() - self.parent = parent self.children = list(children) self.hierarchy = list(hierarchy) + [self] diff --git a/src/formpack/schema/fields.py b/src/formpack/schema/fields.py index 31b13495..e77d7c83 100644 --- a/src/formpack/schema/fields.py +++ b/src/formpack/schema/fields.py @@ -32,7 +32,7 @@ class FormField(FormDataDef): def __init__(self, name, labels, data_type, hierarchy=None, section=None, can_format=True, has_stats=None, *args, **kwargs): - + # assert not isinstance(labels, dict): self.data_type = data_type self.section = section self.can_format = can_format diff --git a/src/formpack/translated_item.py b/src/formpack/translated_item.py new file mode 100644 index 00000000..88b9b37d --- /dev/null +++ b/src/formpack/translated_item.py @@ -0,0 +1,31 @@ +# coding: utf-8 + +from __future__ import (unicode_literals, print_function, absolute_import, + division) + +from collections import OrderedDict +from .errors import TranslationError + + +class TranslatedItem(object): + def __init__(self, values=[], translations=[], strict=False, context=''): + if len(translations) == 1 and translations[0] is None and \ + len(values) == 0: + values = [None] + if len(values) > len(translations): + raise TranslationError('String count exceeds translation count. {}' + .format(context)) + if strict and len(values) < len(translations): + raise TranslationError('Translation count does not match' + ' string count. {}'.format(context)) + else: + while len(values) < len(translations): + values = values + [None] + + self._translations = OrderedDict(zip(translations, values)) + + def __getitem__(self, index): + return self._translations.values()[index] + + def get(self, key, default=None): + return self._translations.get(key, default) diff --git a/src/formpack/version.py b/src/formpack/version.py index 13a7e0e5..ccd5c478 100644 --- a/src/formpack/version.py +++ b/src/formpack/version.py @@ -18,28 +18,8 @@ from .errors import SchemaError from .utils.flatten_content import flatten_content from .schema import (FormField, FormGroup, FormSection, FormChoice) +from .translated_item import TranslatedItem from .schema import _field_from_dict -from .errors import TranslationError - - -class LabelStruct(object): - ''' - LabelStruct stores labels + translations assigned to `field.labels` - ''' - - def __init__(self, labels=[], translations=[]): - if len(labels) != len(translations): - errmsg = 'Mismatched labels and translations: [{}] [{}] ' \ - '{}!={}'.format(', '.join(labels), - ', '.join(translations), len(labels), - len(translations)) - raise TranslationError(errmsg) - self._labels = labels - self._translations = translations - self._vals = dict(zip(translations, labels)) - - def get(self, key, default=None): - return self._vals.get(key, default) def get_labels(choice_definition, translation_list): @@ -81,17 +61,6 @@ def choices_from_structures(definition, translation_list): return all_choices.items() -def extract_json_labels(definition, column, translations): - _ld = OrderedDict() - labels = definition.get(column, []) - for (i, translation) in enumerate(translations): - if i < len(labels): - _ld[translation] = labels[i] - else: - continue - return _ld - - class FormVersion(object): @classmethod def verify_schema_structure(cls, struct): @@ -222,8 +191,10 @@ def __init__(self, form_pack, schema): if data_type == 'begin_group': group_stack.append(group) - labels = extract_json_labels(data_definition, 'label', - self.translations) + labels = TranslatedItem(data_definition.get('label', []), + translations=self.translations, + ) + group = FormGroup(data_definition['name'], labels, src=data_definition) @@ -237,9 +208,9 @@ def __init__(self, form_pack, schema): # Parent maybe None, in that case we are at the top level. parent_section = section - labels = extract_json_labels(data_definition, - 'label', - self.translations) + labels = TranslatedItem(data_definition.get('label', []), + translations=self.translations, + ) _repeat_name = data_definition.get('$autoname', data_definition.get('name')) section = FormSection(_repeat_name, labels, @@ -263,13 +234,13 @@ def __init__(self, form_pack, schema): section.fields[field.name] = field _f = fields_by_name[field.name] - _labels = LabelStruct() + _labels = TranslatedItem() if 'label' in _f: if not isinstance(_f['label'], list): _f['label'] = [_f['label']] - _labels = LabelStruct(labels=_f['label'], - translations=self.translations) + _labels = TranslatedItem(_f['label'], + translations=self.translations) field.labels = _labels assert 'labels' not in _f diff --git a/tests/test_translated_item.py b/tests/test_translated_item.py new file mode 100644 index 00000000..f0554c17 --- /dev/null +++ b/tests/test_translated_item.py @@ -0,0 +1,45 @@ +# coding: utf-8 + +from __future__ import (unicode_literals, print_function, + absolute_import, division) +import json +import pytest + +from formpack.translated_item import TranslatedItem +from formpack.errors import TranslationError + + +def test_simple_translated(): + labels = TranslatedItem(['x', 'y'], + translations=['langx', 'langy']) + expected = '{"langx": "x", "langy": "y"}' + assert json.dumps(labels._translations) == expected + + +def test_invalid_translateds(): + with pytest.raises(TranslationError): + TranslatedItem(['two', 'translations'], + translations=['onelang']) + + ti = TranslatedItem(['one translation'], + # strict=False, by default + translations=['lang1', 'lang2']) + assert ti._translations['lang2'] is None + + # this happens when a null translation is created on an + # incomplete form. If we were to make this invalid, we should + # enforce it at a different step + ti = TranslatedItem([], translations=[None]) + assert json.dumps(ti._translations) == '{"null": null}' + + +def test_strict_translateds(): + with pytest.raises(TranslationError): + TranslatedItem(['one translation'], + strict=True, + translations=['two', 'langs']) + + with pytest.raises(TranslationError): + TranslatedItem(['two', 'translations'], + translations=['onelang'], + strict=True) From ff42d04109095d52ee89fbe54f7d4d800eb9b689 Mon Sep 17 00:00:00 2001 From: Alex Dorey Date: Thu, 16 Mar 2017 13:11:11 -0700 Subject: [PATCH 11/14] * an OrderedDict can also be used to initialize a TranslatedItem --- src/formpack/translated_item.py | 4 ++++ tests/test_translated_item.py | 14 +++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/formpack/translated_item.py b/src/formpack/translated_item.py index 88b9b37d..451ae4b6 100644 --- a/src/formpack/translated_item.py +++ b/src/formpack/translated_item.py @@ -9,6 +9,10 @@ class TranslatedItem(object): def __init__(self, values=[], translations=[], strict=False, context=''): + if isinstance(values, OrderedDict): + translations = values.keys() + values = values.values() + if len(translations) == 1 and translations[0] is None and \ len(values) == 0: values = [None] diff --git a/tests/test_translated_item.py b/tests/test_translated_item.py index f0554c17..3e952db7 100644 --- a/tests/test_translated_item.py +++ b/tests/test_translated_item.py @@ -4,16 +4,24 @@ absolute_import, division) import json import pytest +from collections import OrderedDict from formpack.translated_item import TranslatedItem from formpack.errors import TranslationError def test_simple_translated(): - labels = TranslatedItem(['x', 'y'], - translations=['langx', 'langy']) + t1 = TranslatedItem(['x', 'y'], + translations=['langx', 'langy']) expected = '{"langx": "x", "langy": "y"}' - assert json.dumps(labels._translations) == expected + assert json.dumps(t1._translations) == expected + + # an OrderedDict can also be used to initialize a TranslatedItem + t2 = TranslatedItem(OrderedDict([ + ('langx', 'x'), + ('langy', 'y'), + ])) + assert json.dumps(t1._translations) == json.dumps(t2._translations) def test_invalid_translateds(): From 0b6648b09f691c62f5ab7d7b1c34f96658ed077b Mon Sep 17 00:00:00 2001 From: Alex Dorey Date: Tue, 16 May 2017 21:26:11 -0700 Subject: [PATCH 12/14] removing unused attribute on FormPack (pack.submissions will be used in future tests) --- src/formpack/pack.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/formpack/pack.py b/src/formpack/pack.py index 87137507..12e7e881 100644 --- a/src/formpack/pack.py +++ b/src/formpack/pack.py @@ -44,8 +44,6 @@ def __init__(self, versions=None, title='Submissions', id_string=None, self.id_string = id_string self.root_node_name = root_node_name - self.submissions = submissions - self.title = title self.strict_schema = strict_schema From b25677ba662496ae60918bcf48fd58b085216bd3 Mon Sep 17 00:00:00 2001 From: Alex Dorey Date: Wed, 17 May 2017 09:30:36 -0700 Subject: [PATCH 13/14] misc errors when version not found --- src/formpack/pack.py | 2 ++ src/formpack/remote_pack.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/formpack/pack.py b/src/formpack/pack.py index 12e7e881..519ee589 100644 --- a/src/formpack/pack.py +++ b/src/formpack/pack.py @@ -139,6 +139,8 @@ def load_version(self, schema): unique accross an entire FormPack. It can be None, but only for one version in the FormPack. """ + if 'content' not in schema: + raise ValueError('''"content" is not an available key: {}'''.format(schema.keys())) replace_aliases(schema['content'], in_place=True) expand_content(schema['content'], in_place=True) diff --git a/src/formpack/remote_pack.py b/src/formpack/remote_pack.py index 507a9b91..287f6b07 100644 --- a/src/formpack/remote_pack.py +++ b/src/formpack/remote_pack.py @@ -117,6 +117,8 @@ def load_version(self, version_id): 'versions', version_id) vd = requests.get(_version_url, headers=self._headers()).json() + if vd.get('detail') == 'Not found.': + raise Exception('Version not found') with open(_version_path, 'w') as ff: ff.write(json.dumps(vd, indent=2)) return vd From 429f1254bcdd80a997827f6b56a0ce7c6b661d5d Mon Sep 17 00:00:00 2001 From: "John N. Milner" Date: Thu, 28 Mar 2024 22:40:30 -0400 Subject: [PATCH 14/14] Remove obsolete `ellipsize_title` argument --- src/formpack/remote_pack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/formpack/remote_pack.py b/src/formpack/remote_pack.py index 287f6b07..d55c8c63 100644 --- a/src/formpack/remote_pack.py +++ b/src/formpack/remote_pack.py @@ -143,7 +143,7 @@ def create_pack(self): _v['date_deployed'] = _v.pop('date_deployed', None) self.versions.append(_v) return FormPack(versions=self.versions, id_string=self.uid, - title=self.asset.name, ellipsize_title=False, + title=self.asset.name, ) def stats(self):