From c74426fc72f521bfcaa0e06922856a111e738a42 Mon Sep 17 00:00:00 2001 From: "John N. Milner" Date: Wed, 30 Jun 2021 15:42:19 -0400 Subject: [PATCH] Add data migration to purge deleted submissions WARNING: may be slow on large databases. Mention this in release notes. Closes #733. --- .../0019_purge_deleted_instances.py | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 onadata/apps/logger/migrations/0019_purge_deleted_instances.py diff --git a/onadata/apps/logger/migrations/0019_purge_deleted_instances.py b/onadata/apps/logger/migrations/0019_purge_deleted_instances.py new file mode 100644 index 000000000..fa6903c1e --- /dev/null +++ b/onadata/apps/logger/migrations/0019_purge_deleted_instances.py @@ -0,0 +1,42 @@ +# Generated by Django 2.2.14 on 2021-06-30 19:00 +from django.conf import settings +from django.db import migrations + + +def purge_deleted_instances(apps, schema_editor): + """ + Remove all submissions that have been already marked as deleted from both + PostgreSQL and MongoDB. If this is too slow, revert to a previous release + and run the code in + https://github.com/kobotoolbox/kobocat/issues/696#issuecomment-809622367 + using `manage.py shell_plus`. + """ + Instance = apps.get_model('logger', 'Instance') + to_purge = Instance.objects.exclude(deleted_at=None).only('pk') + if not to_purge.exists(): + return + print( + f'Purging {to_purge.count()} deleted instances...', end='', flush=True + ) + for instance in to_purge.iterator(): + # Manually delete from MongoDB because signals are not called in + # migrations (that would require freezing all application code, not + # just the models!) + settings.MONGO_DB.instances.delete_one({'_id': instance.pk}) + instance.delete() + print('Done!', flush=True) + + +def do_nothing(apps, schema_editor): + pass + + +class Migration(migrations.Migration): + dependencies = [ + ('logger', '0018_add_submission_counter'), + ] + operations = [ + migrations.RunPython( + purge_deleted_instances, reverse_code=do_nothing + ), + ]