diff --git a/onadata/apps/api/viewsets/data_viewset.py b/onadata/apps/api/viewsets/data_viewset.py index ee3dbabcd..f5eba795f 100644 --- a/onadata/apps/api/viewsets/data_viewset.py +++ b/onadata/apps/api/viewsets/data_viewset.py @@ -109,7 +109,6 @@ class DataViewSet(AnonymousUserPublicFormsMixin, ModelViewSet): > [ > { > "_id": 4503, -> "_deleted_at": null, > "expense_type": "service", > "_xform_id_string": "exp", > "_geolocation": [ @@ -157,7 +156,6 @@ class DataViewSet(AnonymousUserPublicFormsMixin, ModelViewSet): > > { > "_id": 4503, -> "_deleted_at": null, > "expense_type": "service", > "_xform_id_string": "exp", > "_geolocation": [ @@ -213,7 +211,6 @@ class DataViewSet(AnonymousUserPublicFormsMixin, ModelViewSet): > [ > { > "_id": 4503, -> "_deleted_at": null, > "expense_type": "service", > "_xform_id_string": "exp", > "_geolocation": [ diff --git a/onadata/apps/logger/management/commands/clean_duplicated_submissions.py b/onadata/apps/logger/management/commands/clean_duplicated_submissions.py index a543acd27..6050a51a9 100644 --- a/onadata/apps/logger/management/commands/clean_duplicated_submissions.py +++ b/onadata/apps/logger/management/commands/clean_duplicated_submissions.py @@ -39,19 +39,9 @@ def add_arguments(self, parser): help="Specify a XForm's `id_string` to clean up only this form", ) - parser.add_argument( - "--purge", - action='store_true', - default=False, - help="Erase duplicate `Instance`s from the database entirely instead " - "of marking them as deleted using the `deleted_at` attribute. " - "Default is False", - ) - def handle(self, *args, **options): username = options['user'] xform_id_string = options['xform'] - purge = options['purge'] # Retrieve all instances with the same `uuid`. query = Instance.objects @@ -61,13 +51,6 @@ def handle(self, *args, **options): if username: query = query.filter(xform__user__username=username) - # if we don't purge, we don't want to see instances - # that have been marked as deleted. However, if we do purge - # we do need these instances to be in the list in order - # to delete them permanently - if not purge: - query = query.filter(deleted_at=None) - query = query.values_list('uuid', flat=True)\ .annotate(count_uuid=Count('uuid'))\ .filter(count_uuid__gt=1)\ @@ -77,13 +60,6 @@ def handle(self, *args, **options): duplicated_query = Instance.objects.filter(uuid=uuid) - # if we don't purge, we don't want to see instances - # that have been marked as deleted. However, if we do purge - # we do need these instances to be in the list in order - # to delete them permanently - if not purge: - duplicated_query = duplicated_query.filter(deleted_at=None) - instances_with_same_uuid = duplicated_query.values_list('id', 'xml_hash')\ .order_by('xml_hash', 'date_created') @@ -97,8 +73,7 @@ def handle(self, *args, **options): if instance_xml_hash != xml_hash_ref: self.__clean_up(instance_id_ref, - duplicated_instance_ids, - purge) + duplicated_instance_ids) xml_hash_ref = instance_xml_hash instance_id_ref = instance_id duplicated_instance_ids = [] @@ -107,14 +82,10 @@ def handle(self, *args, **options): duplicated_instance_ids.append(instance_id) self.__clean_up(instance_id_ref, - duplicated_instance_ids, - purge) + duplicated_instance_ids) if not self.__vaccuum: - if purge: - self.stdout.write('No instances have been purged.') - else: - self.stdout.write('No instances have been marked as deleted.') + self.stdout.write('No instances have been purged.') else: # Update number of submissions for each user. for user_ in list(self.__users): @@ -128,7 +99,7 @@ def handle(self, *args, **options): self.stdout.write( '\t\tDone! New number: {}'.format(result['count'])) - def __clean_up(self, instance_id_ref, duplicated_instance_ids, purge): + def __clean_up(self, instance_id_ref, duplicated_instance_ids): if instance_id_ref is not None and len(duplicated_instance_ids) > 0: self.__vaccuum = True with transaction.atomic(): @@ -144,30 +115,15 @@ def __clean_up(self, instance_id_ref, duplicated_instance_ids, purge): .get(id=instance_id_ref) main_instance.parsed_instance.save() - if purge: - self.stdout.write('\tPurging instances: {}'.format( - duplicated_instance_ids)) - Instance.objects.select_for_update()\ - .filter(id__in=duplicated_instance_ids).delete() - ParsedInstance.objects.select_for_update()\ - .filter(instance_id__in=duplicated_instance_ids).delete() - settings.MONGO_DB.instances.remove( - {'_id': {'$in': duplicated_instance_ids}} - ) - else: - self.stdout.write('\tMarking instances as deleted: {}'.format( - duplicated_instance_ids)) - # We could loop through instances and use `Instance.set_deleted()` - # but it would be way slower. - Instance.objects.select_for_update()\ - .filter(id__in=duplicated_instance_ids)\ - .update(deleted_at=timezone.now()) - settings.MONGO_DB.instances.update_many( - {'_id': {'$in': duplicated_instance_ids}}, - {'$set': { - '_deleted_at': timezone.now().strftime(MONGO_STRFTIME) - }} - ) + self.stdout.write('\tPurging instances: {}'.format( + duplicated_instance_ids)) + Instance.objects.select_for_update()\ + .filter(id__in=duplicated_instance_ids).delete() + ParsedInstance.objects.select_for_update()\ + .filter(instance_id__in=duplicated_instance_ids).delete() + settings.MONGO_DB.instances.remove( + {'_id': {'$in': duplicated_instance_ids}} + ) # Update number of submissions xform = main_instance.xform self.stdout.write( diff --git a/onadata/apps/logger/management/commands/sync_deleted_instances_fix.py b/onadata/apps/logger/management/commands/sync_deleted_instances_fix.py deleted file mode 100644 index c67bea984..000000000 --- a/onadata/apps/logger/management/commands/sync_deleted_instances_fix.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python -# vim: ai ts=4 sts=4 et sw=4 fileencoding=utf-8 -# coding: utf-8 -import json - -from django.conf import settings -from django.core.management import BaseCommand -from django.utils import timezone -from django.utils.dateparse import parse_datetime -from django.utils.translation import ugettext_lazy - -from onadata.apps.logger.models import Instance - - -class Command(BaseCommand): - help = ugettext_lazy("Fixes deleted instances by syncing " - "deleted items from mongo.") - - def handle(self, *args, **kwargs): - # Reset all sql deletes to None - Instance.objects.exclude( - deleted_at=None, xform__downloadable=True).update(deleted_at=None) - - # Get all mongo deletes - query = '{"$and": [{"_deleted_at": {"$exists": true}}, ' \ - '{"_deleted_at": {"$ne": null}}]}' - query = json.loads(query) - xform_instances = settings.MONGO_DB.instances - cursor = xform_instances.find(query) - for record in cursor: - # update sql instance with deleted_at datetime from mongo - try: - i = Instance.objects.get( - uuid=record["_uuid"], xform__downloadable=True) - except Instance.DoesNotExist: - continue - else: - deleted_at = parse_datetime(record["_deleted_at"]) - if not timezone.is_aware(deleted_at): - deleted_at = timezone.make_aware( - deleted_at, timezone.utc) - i.set_deleted(deleted_at) diff --git a/onadata/apps/logger/models/instance.py b/onadata/apps/logger/models/instance.py index 70f440cdc..0be54e917 100644 --- a/onadata/apps/logger/models/instance.py +++ b/onadata/apps/logger/models/instance.py @@ -24,7 +24,6 @@ clean_and_parse_xml, get_uuid_from_xml from onadata.libs.utils.common_tags import ( ATTACHMENTS, - DELETEDAT, GEOLOCATION, ID, MONGO_STRFTIME, @@ -91,6 +90,28 @@ def update_xform_submission_count(sender, instance, created, **kwargs): ) +def nullify_exports_time_of_last_submission(sender, instance, **kwargs): + """ + Formerly, "deleting" a submission would set a flag on the `Instance`, + causing the `date_modified` attribute to be set to the current timestamp. + `Export.exports_outdated()` relied on this to detect when a new `Export` + needed to be generated due to submission deletion, but now that we always + delete `Instance`s outright, this trick doesn't work. This kludge simply + makes every `Export` for a form appear stale by nulling out its + `time_of_last_submission` attribute. + """ + # Avoid circular import + try: + export_model = instance.xform.export_set.model + except XForm.DoesNotExist: + return + f = instance.xform.export_set.filter( + # Match the statuses considered by `Export.exports_outdated()` + internal_status__in=[export_model.SUCCESSFUL, export_model.PENDING], + ) + f.update(time_of_last_submission=None) + + def update_user_submissions_counter(sender, instance, created, **kwargs): if not created: return @@ -157,7 +178,8 @@ class Instance(models.Model): # this will end up representing "date last parsed" date_modified = models.DateTimeField(auto_now=True) - # this will end up representing "date instance was deleted" + # this formerly represented "date instance was deleted". + # do not use it anymore. deleted_at = models.DateTimeField(null=True, default=None) # ODK keeps track of three statuses for an instance: @@ -195,15 +217,6 @@ def asset(self): """ return self.xform - @classmethod - def set_deleted_at(cls, instance_id, deleted_at=timezone.now()): - try: - instance = cls.objects.get(id=instance_id) - except cls.DoesNotExist: - pass - else: - instance.set_deleted(deleted_at) - def _check_active(self, force): """Check that form is active and raise exception if not. @@ -359,9 +372,6 @@ def get_full_dict(self): NOTES: self.get_notes() } - if isinstance(self.instance.deleted_at, datetime): - data[DELETEDAT] = self.deleted_at.strftime(MONGO_STRFTIME) - d.update(data) return d @@ -413,13 +423,6 @@ def save(self, *args, **kwargs): super().save(*args, **kwargs) - def set_deleted(self, deleted_at=timezone.now()): - self.deleted_at = deleted_at - self.save() - # force submission count re-calculation - self.xform.submission_count(force_update=True) - self.parsed_instance.save() - def get_validation_status(self): """ Returns instance validation status. @@ -436,6 +439,9 @@ def get_validation_status(self): post_save.connect(update_xform_submission_count, sender=Instance, dispatch_uid='update_xform_submission_count') +post_delete.connect(nullify_exports_time_of_last_submission, sender=Instance, + dispatch_uid='nullify_exports_time_of_last_submission') + post_save.connect(update_user_submissions_counter, sender=Instance, dispatch_uid='update_user_submissions_counter') diff --git a/onadata/apps/logger/models/xform.py b/onadata/apps/logger/models/xform.py index 4f0b0f6b6..b082a7be5 100644 --- a/onadata/apps/logger/models/xform.py +++ b/onadata/apps/logger/models/xform.py @@ -193,7 +193,7 @@ def __str__(self): def submission_count(self, force_update=False): if self.num_of_submissions == 0 or force_update: - count = self.instances.filter(deleted_at__isnull=True).count() + count = self.instances.count() self.num_of_submissions = count self.save(update_fields=['num_of_submissions']) return self.num_of_submissions @@ -201,14 +201,12 @@ def submission_count(self, force_update=False): def geocoded_submission_count(self): """Number of geocoded submissions.""" - return self.instances.filter(deleted_at__isnull=True, - geom__isnull=False).count() + return self.instances.filter(geom__isnull=False).count() def time_of_last_submission(self): if self.last_submission_time is None and self.num_of_submissions > 0: try: - last_submission = self.instances.\ - filter(deleted_at__isnull=True).latest("date_created") + last_submission = self.instances.latest("date_created") except ObjectDoesNotExist: pass else: diff --git a/onadata/apps/logger/tests/models/test_xform.py b/onadata/apps/logger/tests/models/test_xform.py index 5094d2664..ede18e99a 100644 --- a/onadata/apps/logger/tests/models/test_xform.py +++ b/onadata/apps/logger/tests/models/test_xform.py @@ -8,22 +8,6 @@ class TestXForm(TestBase): - def test_submission_count_filters_deleted(self): - self._publish_transportation_form_and_submit_instance() - - # update the xform object the num_submissions seems to be cached in - # the in-memory xform object as zero - self.xform = XForm.objects.get(pk=self.xform.id) - self.assertEqual(self.xform.submission_count(), 1) - instance = Instance.objects.get(xform=self.xform) - instance.set_deleted() - self.assertIsNotNone(instance.deleted_at) - - # update the xform object, the num_submissions seems to be cached in - # the in-memory xform object as one - self.xform = XForm.objects.get(pk=self.xform.id) - self.assertEqual(self.xform.submission_count(), 0) - def test_set_title_in_xml_unicode_error(self): xls_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), diff --git a/onadata/apps/logger/tests/test_command_syncd_deleted_instances_fix.py b/onadata/apps/logger/tests/test_command_syncd_deleted_instances_fix.py deleted file mode 100644 index 8dddcc4a6..000000000 --- a/onadata/apps/logger/tests/test_command_syncd_deleted_instances_fix.py +++ /dev/null @@ -1,73 +0,0 @@ -# coding: utf-8 -from datetime import datetime -from django.utils import timezone -from django.core.management import call_command - -from onadata.apps.main.tests.test_base import TestBase -from onadata.apps.logger.models import Instance - - -class CommandSyncDeletedTests(TestBase): - - def setUp(self): - TestBase.setUp(self) - self._create_user_and_login() - self._publish_transportation_form_and_submit_instance() - - def test_command(self): - count = Instance.objects.filter(deleted_at=None).count() - self.assertTrue(count > 0) - deleted_at = timezone.now() - deleted_at = datetime.strptime( - deleted_at.strftime("%Y-%m-%dT%H:%M:%S"), "%Y-%m-%dT%H:%M:%S") - deleted_at = timezone.make_aware(deleted_at, timezone.utc) - instance = Instance.objects.filter(deleted_at=None)[0] - instance.deleted_at = deleted_at - instance.save() - - # ensure mongo has deleted_at by calling remongo - call_command('remongo') - - # reset deleted_at to None - instance.deleted_at = None - instance.save() - - same_instance = Instance.objects.get(pk=instance.pk) - self.assertIsNone(same_instance.deleted_at) - - # reset the deleted_at time from datetime in mongo - call_command('sync_deleted_instances_fix') - same_instance = Instance.objects.get(pk=instance.pk) - - # deleted_at should now have a datetime value - self.assertIsNotNone(same_instance.deleted_at) - self.assertTrue(isinstance(same_instance.deleted_at, datetime)) - self.assertEqual(same_instance.deleted_at, deleted_at) - - def test_command_on_inactive_form(self): - count = Instance.objects.filter(deleted_at=None).count() - self.assertTrue(count > 0) - deleted_at = timezone.now() - instance = Instance.objects.filter(deleted_at=None)[0] - instance.deleted_at = deleted_at - instance.save() - - # ensure mongo has deleted_at by calling remongo - call_command('remongo') - - # reset deleted_at to None - instance.deleted_at = None - instance.save() - - # make xform inactive - self.xform.downloadable = False - self.xform.save() - same_instance = Instance.objects.get(pk=instance.pk) - self.assertIsNone(same_instance.deleted_at) - - # reset the deleted_at time from datetime in mongo - call_command('sync_deleted_instances_fix') - same_instance = Instance.objects.get(pk=instance.pk) - - # deleted_at should now have a datetime value - self.assertIsNone(same_instance.deleted_at) diff --git a/onadata/apps/logger/views.py b/onadata/apps/logger/views.py index 573a2b13d..63e5cb4d6 100644 --- a/onadata/apps/logger/views.py +++ b/onadata/apps/logger/views.py @@ -434,7 +434,7 @@ def view_submission_list(request, username): return HttpResponseForbidden('Not shared.') num_entries = request.GET.get('numEntries', None) cursor = request.GET.get('cursor', None) - instances = xform.instances.filter(deleted_at=None).order_by('pk') + instances = xform.instances.order_by('pk') cursor = _parse_int(cursor) if cursor: @@ -480,7 +480,7 @@ def view_download_submission(request, username): uuid = _extract_uuid(form_id_parts[1]) instance = get_object_or_404( Instance, xform__id_string__exact=id_string, uuid=uuid, - xform__user__username=username, deleted_at=None) + xform__user__username=username) xform = instance.xform if not has_permission(xform, form_user, request, xform.shared_data): return HttpResponseForbidden('Not shared.') diff --git a/onadata/apps/main/views.py b/onadata/apps/main/views.py index ae7f45195..143790467 100644 --- a/onadata/apps/main/views.py +++ b/onadata/apps/main/views.py @@ -551,6 +551,7 @@ def _get_migrate_url(username): kf_url=settings.KOBOFORM_URL, username=username ) + def make_kpi_data_redirect_view(kpi_data_route): def view_func(request, username, id_string): owner = get_object_or_404(User, username__iexact=username) diff --git a/onadata/apps/viewer/management/commands/rewrite_attachments_urls_in_mongo.py b/onadata/apps/viewer/management/commands/rewrite_attachments_urls_in_mongo.py index c83ef6534..d719a8a99 100644 --- a/onadata/apps/viewer/management/commands/rewrite_attachments_urls_in_mongo.py +++ b/onadata/apps/viewer/management/commands/rewrite_attachments_urls_in_mongo.py @@ -76,10 +76,6 @@ def handle(self, *args, **kwargs): def __get_data(self): query = {"$and": [ - {"$or": [ - {"_deleted_at": {"$exists": False}}, - {"_deleted_at": None} - ]}, {"_attachments": {"$ne": ""}}, {"_attachments": {"$ne": []}}, {"$or": [ diff --git a/onadata/apps/viewer/management/commands/update_delete_from_mongo.py b/onadata/apps/viewer/management/commands/update_delete_from_mongo.py deleted file mode 100644 index 3daab42a2..000000000 --- a/onadata/apps/viewer/management/commands/update_delete_from_mongo.py +++ /dev/null @@ -1,26 +0,0 @@ -# coding: utf-8 -from django.core.management.base import BaseCommand -from django.utils.translation import ugettext_lazy - -from onadata.apps.logger.models.instance import Instance -from onadata.apps.viewer.models.parsed_instance import xform_instances,\ - datetime_from_str -from onadata.libs.utils.common_tags import DELETEDAT, ID - - -class Command(BaseCommand): - help = ugettext_lazy("Update deleted records from mongo to sql instances") - - def handle(self, *args, **kwargs): - q = {"$and": [{"_deleted_at": {"$exists": True}}, - {"_deleted_at": {"$ne": None}}]} - cursor = xform_instances.find(q) - c = 0 - for record in cursor: - date_deleted = datetime_from_str(record[DELETEDAT]) - id = record[ID] - if Instance.set_deleted_at(id, deleted_at=date_deleted): - c += 1 - print("deleted on ", date_deleted) - print("-------------------------------") - print("Updated %d records." % c) diff --git a/onadata/apps/viewer/models/parsed_instance.py b/onadata/apps/viewer/models/parsed_instance.py index c970e055e..345be03cc 100644 --- a/onadata/apps/viewer/models/parsed_instance.py +++ b/onadata/apps/viewer/models/parsed_instance.py @@ -23,7 +23,6 @@ GEOLOCATION, SUBMISSION_TIME, MONGO_STRFTIME, - DELETEDAT, TAGS, NOTES, SUBMITTED_BY, @@ -92,9 +91,9 @@ def get_base_query(cls, username, id_string): @classmethod @apply_form_field_names def query_mongo(cls, username, id_string, query, fields, sort, start=0, - limit=DEFAULT_LIMIT, count=False, hide_deleted=True): + limit=DEFAULT_LIMIT, count=False): - query = cls._get_mongo_cursor_query(query, hide_deleted, username, id_string) + query = cls._get_mongo_cursor_query(query, username, id_string) if count: return [ @@ -118,7 +117,7 @@ def query_mongo(cls, username, id_string, query, fields, sort, start=0, @classmethod @apply_form_field_names - def mongo_aggregate(cls, query, pipeline, hide_deleted=True): + def mongo_aggregate(cls, query, pipeline): """Perform mongo aggregate queries query - is a dict which is to be passed to $match, a pipeline operator pipeline - list of dicts or dict of mongodb pipeline operators, @@ -132,13 +131,6 @@ def mongo_aggregate(cls, query, pipeline, hide_deleted=True): if not isinstance(query, dict): raise Exception(_("Invalid query! %s" % query)) query = MongoHelper.to_safe_dict(query) - if hide_deleted: - # display only active elements - deleted_at_query = { - "$or": [{"_deleted_at": {"$exists": False}}, - {"_deleted_at": None}]} - # join existing query with deleted_at_query on an $and - query = {"$and": [query, deleted_at_query]} k = [{'$match': query}] if isinstance(pipeline, list): k.extend(pipeline) @@ -151,9 +143,10 @@ def mongo_aggregate(cls, query, pipeline, hide_deleted=True): @apply_form_field_names def query_mongo_minimal( cls, query, fields, sort, start=0, limit=DEFAULT_LIMIT, - count=False, hide_deleted=True): + count=False): + + query = cls._get_mongo_cursor_query(query) - query = cls._get_mongo_cursor_query(query, hide_deleted) if count: return [ { @@ -179,9 +172,9 @@ def query_mongo_minimal( @classmethod @apply_form_field_names - def query_mongo_no_paging(cls, query, fields, count=False, hide_deleted=True): + def query_mongo_no_paging(cls, query, fields, count=False): - query = cls._get_mongo_cursor_query(query, hide_deleted) + query = cls._get_mongo_cursor_query(query) if count: return [ @@ -201,9 +194,6 @@ def _get_mongo_cursor(cls, query, fields): :param query: JSON string :param fields: Array string - :param hide_deleted: boolean - :param username: string - :param id_string: string :return: pymongo Cursor """ fields_to_select = {cls.USERFORM_ID: 0} @@ -226,12 +216,11 @@ def _get_mongo_cursor(cls, query, fields): ) @classmethod - def _get_mongo_cursor_query(cls, query, hide_deleted, username=None, id_string=None): + def _get_mongo_cursor_query(cls, query, username=None, id_string=None): """ Returns the query to get a Mongo cursor. :param query: JSON string - :param hide_deleted: boolean :param username: string :param id_string: string :return: dict @@ -246,11 +235,6 @@ def _get_mongo_cursor_query(cls, query, hide_deleted, username=None, id_string=N if username and id_string: query.update(cls.get_base_query(username, id_string)) - if hide_deleted: - # display only active elements - # join existing query with deleted_at_query on an $and - query = {"$and": [query, {"_deleted_at": None}]} - return query @classmethod @@ -296,9 +280,6 @@ def to_dict_for_mongo(self): if self.instance.user else None } - if isinstance(self.instance.deleted_at, datetime.datetime): - data[DELETEDAT] = self.instance.deleted_at.strftime(MONGO_STRFTIME) - d.update(data) return MongoHelper.to_safe_dict(d) @@ -320,7 +301,9 @@ def update_mongo(self, asynchronous=True): if success != self.instance.is_synced_with_mongo: # Skip the labor-intensive stuff in Instance.save() to gain performance # Use .update() instead of .save() - Instance.objects.filter(pk=self.instance.id).update(is_synced_with_mongo=success) + Instance.objects.filter(pk=self.instance.id).update( + is_synced_with_mongo=success + ) return True diff --git a/onadata/apps/viewer/pandas_mongo_bridge.py b/onadata/apps/viewer/pandas_mongo_bridge.py index de164bae0..64a5aa5ca 100644 --- a/onadata/apps/viewer/pandas_mongo_bridge.py +++ b/onadata/apps/viewer/pandas_mongo_bridge.py @@ -87,8 +87,15 @@ def get_prefix_from_xpath(xpath): class AbstractDataFrameBuilder: - IGNORED_COLUMNS = [XFORM_ID_STRING, STATUS, ID, ATTACHMENTS, GEOLOCATION, - DELETEDAT, SUBMITTED_BY] + IGNORED_COLUMNS = [ + XFORM_ID_STRING, + STATUS, + ID, + ATTACHMENTS, + GEOLOCATION, + DELETEDAT, # no longer used but may persist in old submissions + SUBMITTED_BY, + ] # fields NOT within the form def that we want to include ADDITIONAL_COLUMNS = [UUID, SUBMISSION_TIME, TAGS, NOTES, VALIDATION_STATUS] BINARY_SELECT_MULTIPLES = False diff --git a/onadata/apps/viewer/tests/test_pandas_mongo_bridge.py b/onadata/apps/viewer/tests/test_pandas_mongo_bridge.py index f5e0ed066..d599dd0dd 100644 --- a/onadata/apps/viewer/tests/test_pandas_mongo_bridge.py +++ b/onadata/apps/viewer/tests/test_pandas_mongo_bridge.py @@ -252,6 +252,7 @@ def test_csv_columns_for_gps_within_groups(self): ] + AbstractDataFrameBuilder.ADDITIONAL_COLUMNS +\ AbstractDataFrameBuilder.IGNORED_COLUMNS try: + # `_deleted_at` is no longer used but may persist in old submissions expected_columns.remove('_deleted_at') except ValueError: pass diff --git a/onadata/libs/data/query.py b/onadata/libs/data/query.py index ee073f0a2..2be63934b 100644 --- a/onadata/libs/data/query.py +++ b/onadata/libs/data/query.py @@ -65,7 +65,6 @@ def _postgres_count_group(field, name, xform): return "SELECT %(json)s AS \"%(name)s\", COUNT(*) AS count FROM "\ "%(table)s WHERE %(restrict_field)s=%(restrict_value)s "\ - "AND %(exclude_deleted)s "\ "GROUP BY %(json)s" % string_args @@ -73,8 +72,7 @@ def _postgres_select_key(field, name, xform): string_args = _query_args(field, name, xform) return "SELECT %(json)s AS \"%(name)s\" FROM %(table)s WHERE "\ - "%(restrict_field)s=%(restrict_value)s "\ - "AND %(exclude_deleted)s" % string_args + "%(restrict_field)s=%(restrict_value)s" % string_args def _query_args(field, name, xform): @@ -83,8 +81,7 @@ def _query_args(field, name, xform): 'json': _json_query(field), 'name': name, 'restrict_field': 'xform_id', - 'restrict_value': xform.pk, - 'exclude_deleted': 'deleted_at IS NULL'} + 'restrict_value': xform.pk} def _select_key(field, name, xform): diff --git a/onadata/libs/utils/common_tags.py b/onadata/libs/utils/common_tags.py index 9a5f635c0..cc98a8d25 100644 --- a/onadata/libs/utils/common_tags.py +++ b/onadata/libs/utils/common_tags.py @@ -33,7 +33,7 @@ DATE = "_date" GEOLOCATION = "_geolocation" SUBMISSION_TIME = '_submission_time' -DELETEDAT = "_deleted_at" # marker for delete surveys +DELETEDAT = "_deleted_at" # no longer used but may persist in old submissions SUBMITTED_BY = "_submitted_by" VALIDATION_STATUS = "_validation_status" diff --git a/onadata/libs/utils/export_tools.py b/onadata/libs/utils/export_tools.py index 224179a4e..a308132dc 100644 --- a/onadata/libs/utils/export_tools.py +++ b/onadata/libs/utils/export_tools.py @@ -177,8 +177,13 @@ def dict_to_joined_export(data, index, indices, name): class ExportBuilder: - IGNORED_COLUMNS = [XFORM_ID_STRING, STATUS, ATTACHMENTS, GEOLOCATION, - DELETEDAT] + IGNORED_COLUMNS = [ + XFORM_ID_STRING, + STATUS, + ATTACHMENTS, + GEOLOCATION, + DELETEDAT, # no longer used but may persist in old submissions + ] # fields we export but are not within the form's structure EXTRA_FIELDS = [ID, UUID, SUBMISSION_TIME, INDEX, PARENT_TABLE_NAME, PARENT_INDEX, TAGS, NOTES] @@ -609,22 +614,19 @@ def generate_export(export_type, extension, username, id_string, return export -def query_mongo(username, id_string, query=None, hide_deleted=True): +def query_mongo(username, id_string, query=None): query = json.loads(query, object_hook=json_util.object_hook)\ if query else {} query = MongoHelper.to_safe_dict(query) query[USERFORM_ID] = '{0}_{1}'.format(username, id_string) - if hide_deleted: - # display only active elements - # join existing query with deleted_at_query on an $and - query = {"$and": [query, {"_deleted_at": None}]} return xform_instances.find(query, max_time_ms=settings.MONGO_DB_MAX_TIME_MS) def should_create_new_export(xform, export_type): - if Export.objects.filter( - xform=xform, export_type=export_type).count() == 0 \ - or Export.exports_outdated(xform, export_type=export_type): + if ( + not Export.objects.filter(xform=xform, export_type=export_type).exists() + or Export.exports_outdated(xform, export_type=export_type) + ): return True return False @@ -746,7 +748,6 @@ def kml_export_data(id_string, user): instances = Instance.objects.filter( xform__user=user, xform__id_string=id_string, - deleted_at=None, geom__isnull=False ).order_by('id') data_for_template = []