From 5d5fe86bf720dbeed104a6b9982fb4c805146b1a Mon Sep 17 00:00:00 2001 From: Graham Herceg Date: Tue, 14 May 2024 14:09:59 -0400 Subject: [PATCH 1/2] Increase timeout from 5 to 30 minutes The default in serial_task is 5 minutes. Increasing this value ensures that the reprocess_archive_stub task has time to finish and organically release its lock, rather than the lock timing out while a task is still active, resulting in multiple tasks running simultaneously. --- corehq/form_processor/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corehq/form_processor/tasks.py b/corehq/form_processor/tasks.py index d8546e57e92b..28bc9de9245c 100644 --- a/corehq/form_processor/tasks.py +++ b/corehq/form_processor/tasks.py @@ -40,7 +40,7 @@ def _reprocess_archive_stubs(): reprocess_archive_stubs.delay() -@serial_task("reprocess_archive_stubs", queue=settings.CELERY_PERIODIC_QUEUE) +@serial_task("reprocess_archive_stubs", queue=settings.CELERY_PERIODIC_QUEUE, timeout=30 * 60) def reprocess_archive_stubs(): # Check for archive stubs from couchforms.models import UnfinishedArchiveStub From 4318fce2e9b596491423aa8dd29cab9907f01166 Mon Sep 17 00:00:00 2001 From: Graham Herceg Date: Tue, 14 May 2024 16:32:21 -0400 Subject: [PATCH 2/2] Increase to 24 hours This task takes way longer than 30 minutes to run on some occassions, so we need the timeout to be larger than the max runtime of this task. --- corehq/form_processor/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corehq/form_processor/tasks.py b/corehq/form_processor/tasks.py index 28bc9de9245c..f901589192b2 100644 --- a/corehq/form_processor/tasks.py +++ b/corehq/form_processor/tasks.py @@ -40,7 +40,7 @@ def _reprocess_archive_stubs(): reprocess_archive_stubs.delay() -@serial_task("reprocess_archive_stubs", queue=settings.CELERY_PERIODIC_QUEUE, timeout=30 * 60) +@serial_task("reprocess_archive_stubs", timeout=24 * 60 * 60, queue=settings.CELERY_PERIODIC_QUEUE) def reprocess_archive_stubs(): # Check for archive stubs from couchforms.models import UnfinishedArchiveStub