From 4ef4d9f3061d648395d87fe7d58cd5112e8b8c02 Mon Sep 17 00:00:00 2001 From: Topvennie Date: Tue, 9 Apr 2024 16:38:22 +0200 Subject: [PATCH] chore: celery #206 --- backend/api/logic/check_folder_structure.py | 1 + backend/api/logic/run_extra_checks.py | 1 - backend/api/models/checks.py | 2 ++ backend/api/tasks/__init__.py | 0 backend/api/tasks/extra_checks.py | 0 backend/api/tasks/test.py | 24 +++++++++++++++++++++ backend/notifications/logic.py | 7 ++++-- backend/poetry.lock | 17 ++++++++++++++- backend/pyproject.toml | 1 + backend/setup.sh | 1 + backend/ypovoli/celery.py | 13 +++++++++++ backend/ypovoli/settings.py | 6 +++++- development.yml | 3 ++- 13 files changed, 70 insertions(+), 6 deletions(-) delete mode 100644 backend/api/logic/run_extra_checks.py create mode 100644 backend/api/tasks/__init__.py create mode 100644 backend/api/tasks/extra_checks.py create mode 100644 backend/api/tasks/test.py diff --git a/backend/api/logic/check_folder_structure.py b/backend/api/logic/check_folder_structure.py index 3b8813dd..073f7d4d 100644 --- a/backend/api/logic/check_folder_structure.py +++ b/backend/api/logic/check_folder_structure.py @@ -7,6 +7,7 @@ from django.utils.translation import gettext +# TODO: Move all to tasks module def parse_zip_file(project, dir_path): # TODO block paths that start with .. dir_path = os.path.normpath(os.path.join(settings.MEDIA_ROOT, dir_path)) struct = get_zip_structure(dir_path) diff --git a/backend/api/logic/run_extra_checks.py b/backend/api/logic/run_extra_checks.py deleted file mode 100644 index a10eabaf..00000000 --- a/backend/api/logic/run_extra_checks.py +++ /dev/null @@ -1 +0,0 @@ -# TODO: Send mail when submission fails diff --git a/backend/api/models/checks.py b/backend/api/models/checks.py index 975f46aa..732ef3b2 100644 --- a/backend/api/models/checks.py +++ b/backend/api/models/checks.py @@ -42,6 +42,8 @@ class StructureCheck(models.Model): # ID check should be generated automatically +# TODO: Add state, queued, running, done + class ExtraCheck(models.Model): """Model that represents an extra check for a project. diff --git a/backend/api/tasks/__init__.py b/backend/api/tasks/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/api/tasks/extra_checks.py b/backend/api/tasks/extra_checks.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/api/tasks/test.py b/backend/api/tasks/test.py new file mode 100644 index 00000000..c1c65e97 --- /dev/null +++ b/backend/api/tasks/test.py @@ -0,0 +1,24 @@ +import time + +from celery import shared_task +from celery.result import AsyncResult + + +# TODO +# ! This works +# ! But not async | I can obviouly do this in the same function dummy +def test_print(id, result): + print("result: " + str(result), flush=True) + + +def test(): + a: AsyncResult[int] = testing.apply_async() + # Use propagate=False to avoid raising exceptions. Check if failed by a.failed() + a.get(propagate=False, callback=test_print) + print("async?") + + +@shared_task +def testing(): + time.sleep(2) + return 5 diff --git a/backend/notifications/logic.py b/backend/notifications/logic.py index 0d0b1f92..e2061a87 100644 --- a/backend/notifications/logic.py +++ b/backend/notifications/logic.py @@ -1,6 +1,5 @@ import threading from collections import defaultdict -from os import error from smtplib import SMTPException from typing import DefaultDict, Dict, List @@ -37,8 +36,12 @@ def _send_mail(mail: mail.EmailMessage, result: List[bool]): result[0] = False +# TODO: Maybe convert to a bunch of celery tasks +# TODO: Move to tasks module +# TODO: Retry 3 +# https://docs.celeryq.dev/en/v5.3.6/getting-started/next-steps.html#next-steps # Send all unsent emails -@shared_task +@shared_task(ignore_result=True) def _send_mails(): # All notifications that need to be sent notifications = Notification.objects.filter(is_sent=False) diff --git a/backend/poetry.lock b/backend/poetry.lock index f2f23dd2..c1feeb70 100644 --- a/backend/poetry.lock +++ b/backend/poetry.lock @@ -394,6 +394,21 @@ tzdata = {version = "*", markers = "sys_platform == \"win32\""} argon2 = ["argon2-cffi (>=19.1.0)"] bcrypt = ["bcrypt"] +[[package]] +name = "django-celery-results" +version = "2.5.1" +description = "Celery result backends for Django." +optional = false +python-versions = "*" +files = [ + {file = "django_celery_results-2.5.1-py3-none-any.whl", hash = "sha256:0da4cd5ecc049333e4524a23fcfc3460dfae91aa0a60f1fae4b6b2889c254e01"}, + {file = "django_celery_results-2.5.1.tar.gz", hash = "sha256:3ecb7147f773f34d0381bac6246337ce4cf88a2ea7b82774ed48e518b67bb8fd"}, +] + +[package.dependencies] +celery = ">=5.2.7,<6.0" +Django = ">=3.2.18" + [[package]] name = "django-redis" version = "5.4.0" @@ -1439,4 +1454,4 @@ brotli = ["Brotli"] [metadata] lock-version = "2.0" python-versions = "^3.11.4" -content-hash = "f695b0795b84d554a050594eda382a6d6e2df4392bffd8239baa894ec6944a15" +content-hash = "9f9d8237ba14cd1c4b230f9553971149a0650eeb9cf836d9474530def1462aaf" diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 9ed30719..f4806246 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -26,6 +26,7 @@ celery-types = "^0.22.0" docker = "^7.0.0" faker = "^24.7.1" django-seed = "^0.3.1" +django-celery-results = "^2.5.1" [build-system] diff --git a/backend/setup.sh b/backend/setup.sh index 4124f4b0..e3948c30 100755 --- a/backend/setup.sh +++ b/backend/setup.sh @@ -4,6 +4,7 @@ poetry install > /dev/null echo "Migrating database..." python manage.py migrate > /dev/null +python manage.py migrate django_celery_results > /dev/null echo "Populating database..." python manage.py loaddata */fixtures/* > /dev/null diff --git a/backend/ypovoli/celery.py b/backend/ypovoli/celery.py index 489d93b2..ebf81f02 100644 --- a/backend/ypovoli/celery.py +++ b/backend/ypovoli/celery.py @@ -2,6 +2,7 @@ from celery import Celery from celery.app.task import Task +from django.conf import settings Task.__class_getitem__ = classmethod(lambda cls, *args, **kwargs: cls) # type: ignore[attr-defined] @@ -17,4 +18,16 @@ app.config_from_object("django.conf:settings", namespace="CELERY") # Load task modules from all registered Django apps. +# Will only load tasks defined in a tasks.py file, not a module app.autodiscover_tasks() + +# Load tasks from all installed apps defined inside a module called tasks +for app_name in settings.INSTALLED_APPS: + if app_name.startswith('django'): + continue + for root, dirs, files in os.walk(app_name + '/tasks'): + for file in files: + if file.startswith('__') or file.endswith('.pyc') or not file.endswith('.py'): + continue + file = file[:-3] + app.autodiscover_tasks([app_name + '.tasks'], related_name=file) diff --git a/backend/ypovoli/settings.py b/backend/ypovoli/settings.py index f136aad2..c5d1b9e7 100644 --- a/backend/ypovoli/settings.py +++ b/backend/ypovoli/settings.py @@ -48,6 +48,7 @@ "authentication", # Ypovoli authentication "api", # Ypovoli logic of the base application "notifications", # Ypovoli notifications + "django_celery_results", # Celery results ] MIDDLEWARE = [ @@ -168,7 +169,10 @@ } CELERY_BROKER_URL = f"redis://@{REDIS_CUSTOM['host']}:{REDIS_CUSTOM['port']}/{REDIS_CUSTOM['db_celery']}" -CELERY_RESULT_BACKEND = f"redis://@{REDIS_CUSTOM['host']}:{REDIS_CUSTOM['port']}/{REDIS_CUSTOM['db_celery']}" +CELERY_CACHE_BACKEND = "default" +# TODO: Test if works +CELERY_RESULT_BACKEND = "django-db" +CELERY_IMPORTS = ("api.tasks",) FILE_PATHS = { "docker_images": "../data/docker_images/", diff --git a/development.yml b/development.yml index 49cc24a4..ff346004 100644 --- a/development.yml +++ b/development.yml @@ -55,13 +55,14 @@ services: volumes: - ${BACKEND_DIR}:/code + # TODO: Is this an entire different container worthy? celery: <<: *common-keys-selab container_name: celery build: context: $BACKEND_DIR dockerfile: Dockerfile - command: celery -A ypovoli worker -l DEBUG + command: sh -c "./setup.sh && celery -A ypovoli worker -l DEBUG" volumes: - ${BACKEND_DIR}:/code depends_on: