Skip to content

Commit

Permalink
Stop querying deleted_at
Browse files Browse the repository at this point in the history
See #696
  • Loading branch information
jnm committed Jun 14, 2021
1 parent c7f90af commit ad11065
Show file tree
Hide file tree
Showing 7 changed files with 16 additions and 41 deletions.
8 changes: 3 additions & 5 deletions onadata/apps/logger/models/xform.py
Original file line number Diff line number Diff line change
Expand Up @@ -223,22 +223,20 @@ def __unicode__(self):

def submission_count(self, force_update=False):
if self.num_of_submissions == 0 or force_update:
count = self.instances.filter(deleted_at__isnull=True).count()
count = self.instances.count()
self.num_of_submissions = count
self.save(update_fields=['num_of_submissions'])
return self.num_of_submissions
submission_count.short_description = ugettext_lazy("Submission Count")

def geocoded_submission_count(self):
"""Number of geocoded submissions."""
return self.instances.filter(deleted_at__isnull=True,
geom__isnull=False).count()
return self.instances.filter(geom__isnull=False).count()

def time_of_last_submission(self):
if self.last_submission_time is None and self.num_of_submissions > 0:
try:
last_submission = self.instances.\
filter(deleted_at__isnull=True).latest("date_created")
last_submission = self.instances.latest("date_created")
except ObjectDoesNotExist:
pass
else:
Expand Down
4 changes: 2 additions & 2 deletions onadata/apps/logger/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -579,7 +579,7 @@ def view_submission_list(request, username):
return HttpResponseForbidden('Not shared.')
num_entries = request.GET.get('numEntries', None)
cursor = request.GET.get('cursor', None)
instances = xform.instances.filter(deleted_at=None).order_by('pk')
instances = xform.instances.order_by('pk')

cursor = _parse_int(cursor)
if cursor:
Expand Down Expand Up @@ -625,7 +625,7 @@ def view_download_submission(request, username):
uuid = _extract_uuid(form_id_parts[1])
instance = get_object_or_404(
Instance, xform__id_string__exact=id_string, uuid=uuid,
xform__user__username=username, deleted_at=None)
xform__user__username=username)
xform = instance.xform
if not has_permission(xform, form_user, request, xform.shared_data):
return HttpResponseForbidden('Not shared.')
Expand Down
2 changes: 1 addition & 1 deletion onadata/apps/survey_report/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ def _wrapper(request, username, id_string, *args, **kwargs):

def get_instances_for_user_and_form(user, form_id, submission=None):
userform_id = '{}_{}'.format(user, form_id)
query = {'_userform_id': userform_id, '_deleted_at': {'$exists': False}}
query = {'_userform_id': userform_id}
if submission:
query['_id'] = submission
return settings.MONGO_DB.instances.find(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,10 +78,6 @@ def handle(self, *args, **kwargs):

def __get_data(self):
query = {"$and": [
{"$or": [
{"_deleted_at": {"$exists": False}},
{"_deleted_at": None}
]},
{"_attachments": {"$ne": ""}},
{"_attachments": {"$ne": []}},
{"$or": [
Expand Down
29 changes: 8 additions & 21 deletions onadata/apps/viewer/models/parsed_instance.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,9 +94,9 @@ def get_base_query(cls, username, id_string):
@classmethod
@apply_form_field_names
def query_mongo(cls, username, id_string, query, fields, sort, start=0,
limit=DEFAULT_LIMIT, count=False, hide_deleted=True):
limit=DEFAULT_LIMIT, count=False):

cursor = cls._get_mongo_cursor(query, fields, hide_deleted, username, id_string)
cursor = cls._get_mongo_cursor(query, fields, username, id_string)

if count:
return [{"count": cursor.count()}]
Expand All @@ -112,7 +112,7 @@ def query_mongo(cls, username, id_string, query, fields, sort, start=0,

@classmethod
@apply_form_field_names
def mongo_aggregate(cls, query, pipeline, hide_deleted=True):
def mongo_aggregate(cls, query, pipeline):
"""Perform mongo aggregate queries
query - is a dict which is to be passed to $match, a pipeline operator
pipeline - list of dicts or dict of mongodb pipeline operators,
Expand All @@ -126,13 +126,6 @@ def mongo_aggregate(cls, query, pipeline, hide_deleted=True):
if not isinstance(query, dict):
raise Exception(_("Invalid query! %s" % query))
query = MongoHelper.to_safe_dict(query)
if hide_deleted:
# display only active elements
deleted_at_query = {
"$or": [{"_deleted_at": {"$exists": False}},
{"_deleted_at": None}]}
# join existing query with deleted_at_query on an $and
query = {"$and": [query, deleted_at_query]}
k = [{'$match': query}]
if isinstance(pipeline, list):
k.extend(pipeline)
Expand All @@ -145,9 +138,9 @@ def mongo_aggregate(cls, query, pipeline, hide_deleted=True):
@apply_form_field_names
def query_mongo_minimal(
cls, query, fields, sort, start=0, limit=DEFAULT_LIMIT,
count=False, hide_deleted=True):
count=False):

cursor = cls._get_mongo_cursor(query, fields, hide_deleted)
cursor = cls._get_mongo_cursor(query, fields)

if count:
return [{"count": cursor.count()}]
Expand All @@ -166,23 +159,22 @@ def query_mongo_minimal(

@classmethod
@apply_form_field_names
def query_mongo_no_paging(cls, query, fields, count=False, hide_deleted=True):
def query_mongo_no_paging(cls, query, fields, count=False):

cursor = cls._get_mongo_cursor(query, fields, hide_deleted)
cursor = cls._get_mongo_cursor(query, fields)

if count:
return [{"count": cursor.count()}]
else:
return cursor

@classmethod
def _get_mongo_cursor(cls, query, fields, hide_deleted, username=None, id_string=None):
def _get_mongo_cursor(cls, query, fields, username=None, id_string=None):
"""
Returns a Mongo cursor based on the query.
:param query: JSON string
:param fields: Array string
:param hide_deleted: boolean
:param username: string
:param id_string: string
:return: pymongo Cursor
Expand All @@ -198,11 +190,6 @@ def _get_mongo_cursor(cls, query, fields, hide_deleted, username=None, id_string
if username and id_string:
query.update(cls.get_base_query(username, id_string))

if hide_deleted:
# display only active elements
# join existing query with deleted_at_query on an $and
query = {"$and": [query, {"_deleted_at": None}]}

# fields must be a string array i.e. '["name", "age"]'
if isinstance(fields, basestring):
fields = json.loads(fields, object_hook=json_util.object_hook)
Expand Down
3 changes: 1 addition & 2 deletions onadata/libs/data/query.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,8 +72,7 @@ def _postgres_select_key(field, name, xform):
string_args = _query_args(field, name, xform)

return "SELECT %(json)s AS \"%(name)s\" FROM %(table)s WHERE "\
"%(restrict_field)s=%(restrict_value)s "\
"AND %(exclude_deleted)s" % string_args
"%(restrict_field)s=%(restrict_value)s" % string_args


def _query_args(field, name, xform):
Expand Down
7 changes: 1 addition & 6 deletions onadata/libs/utils/export_tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -758,15 +758,11 @@ def generate_export(export_type, extension, username, id_string,
return export


def query_mongo(username, id_string, query=None, hide_deleted=True):
def query_mongo(username, id_string, query=None):
query = json.loads(query, object_hook=json_util.object_hook)\
if query else {}
query = MongoHelper.to_safe_dict(query)
query[USERFORM_ID] = '{0}_{1}'.format(username, id_string)
if hide_deleted:
# display only active elements
# join existing query with deleted_at_query on an $and
query = {"$and": [query, {"_deleted_at": None}]}
return xform_instances.find(query, max_time_ms=settings.MONGO_DB_MAX_TIME_MS)


Expand Down Expand Up @@ -895,7 +891,6 @@ def kml_export_data(id_string, user):
instances = Instance.objects.filter(
xform__user=user,
xform__id_string=id_string,
deleted_at=None,
geom__isnull=False
).order_by('id')
data_for_template = []
Expand Down

0 comments on commit ad11065

Please sign in to comment.