chore: celery #206

backend/api/logic/check_folder_structure.py

@@ -7,6 +7,7 @@
 from django.utils.translation import gettext
 
 
+# TODO: Move all to tasks module
 def parse_zip_file(project, dir_path):  # TODO block paths that start with ..
     dir_path = os.path.normpath(os.path.join(settings.MEDIA_ROOT, dir_path))
     struct = get_zip_structure(dir_path)

backend/api/models/checks.py

@@ -42,6 +42,8 @@ class StructureCheck(models.Model):
 
     # ID check should be generated automatically
 
+# TODO: Add state, queued, running, done
+
 
 class ExtraCheck(models.Model):
     """Model that represents an extra check for a project.

backend/api/tasks/test.py

@@ -0,0 +1,24 @@
+import time
+
+from celery import shared_task
+from celery.result import AsyncResult
+
+
+# TODO
+# ! This works
+# ! But not async | I can obviouly do this in the same function dummy
+def test_print(id, result):
+    print("result: " + str(result), flush=True)
+
+
+def test():
+    a: AsyncResult[int] = testing.apply_async()
+    # Use propagate=False to avoid raising exceptions. Check if failed by a.failed()
+    a.get(propagate=False, callback=test_print)
+    print("async?")
+
+
+@shared_task
+def testing():
+    time.sleep(2)
+    return 5

backend/notifications/logic.py

@@ -1,6 +1,5 @@
 import threading
 from collections import defaultdict
-from os import error
 from smtplib import SMTPException
 from typing import DefaultDict, Dict, List
 
@@ -37,8 +36,12 @@ def _send_mail(mail: mail.EmailMessage, result: List[bool]):
         result[0] = False
 
 
+# TODO: Maybe convert to a bunch of celery tasks
+# TODO: Move to tasks module
+# TODO: Retry 3
+# https://docs.celeryq.dev/en/v5.3.6/getting-started/next-steps.html#next-steps
 # Send all unsent emails
-@shared_task
+@shared_task(ignore_result=True)
 def _send_mails():
     # All notifications that need to be sent
     notifications = Notification.objects.filter(is_sent=False)

backend/pyproject.toml

@@ -26,6 +26,7 @@ celery-types = "^0.22.0"
 docker = "^7.0.0"
 faker = "^24.7.1"
 django-seed = "^0.3.1"
+django-celery-results = "^2.5.1"
 
 
 [build-system]

backend/setup.sh

@@ -4,6 +4,7 @@ poetry install > /dev/null
 
 echo "Migrating database..."
 python manage.py migrate > /dev/null
+python manage.py migrate django_celery_results > /dev/null
 
 echo "Populating database..."
 python manage.py loaddata */fixtures/* > /dev/null

backend/ypovoli/celery.py

@@ -2,6 +2,7 @@
 
 from celery import Celery
 from celery.app.task import Task
+from django.conf import settings
 
 Task.__class_getitem__ = classmethod(lambda cls, *args, **kwargs: cls)  # type: ignore[attr-defined]
 
@@ -17,4 +18,16 @@
 app.config_from_object("django.conf:settings", namespace="CELERY")
 
 # Load task modules from all registered Django apps.
+# Will only load tasks defined in a tasks.py file, not a module
 app.autodiscover_tasks()
+
+# Load tasks from all installed apps defined inside a module called tasks
+for app_name in settings.INSTALLED_APPS:
+    if app_name.startswith('django'):
+        continue
+    for root, dirs, files in os.walk(app_name + '/tasks'):
+        for file in files:
+            if file.startswith('__') or file.endswith('.pyc') or not file.endswith('.py'):
+                continue
+            file = file[:-3]
+            app.autodiscover_tasks([app_name + '.tasks'], related_name=file)

backend/ypovoli/settings.py

@@ -48,6 +48,7 @@
     "authentication",  # Ypovoli authentication
     "api",  # Ypovoli logic of the base application
     "notifications",  # Ypovoli notifications
+    "django_celery_results",  # Celery results
 ]
 
 MIDDLEWARE = [
@@ -168,7 +169,10 @@
 }
 
 CELERY_BROKER_URL = f"redis://@{REDIS_CUSTOM['host']}:{REDIS_CUSTOM['port']}/{REDIS_CUSTOM['db_celery']}"
-CELERY_RESULT_BACKEND = f"redis://@{REDIS_CUSTOM['host']}:{REDIS_CUSTOM['port']}/{REDIS_CUSTOM['db_celery']}"
+CELERY_CACHE_BACKEND = "default"
+# TODO: Test if works
+CELERY_RESULT_BACKEND = "django-db"
+CELERY_IMPORTS = ("api.tasks",)
 
 FILE_PATHS = {
     "docker_images": "../data/docker_images/",

development.yml

@@ -55,13 +55,14 @@ services:
     volumes:
       - ${BACKEND_DIR}:/code
 
+  # TODO: Is this an entire different container worthy?
   celery:
     <<: *common-keys-selab
     container_name: celery
     build:
       context: $BACKEND_DIR
       dockerfile: Dockerfile
-    command: celery -A ypovoli worker -l DEBUG
+    command: sh -c "./setup.sh && celery -A ypovoli worker -l DEBUG"
     volumes:
       - ${BACKEND_DIR}:/code
     depends_on: