From 6f96e4aa9f60b4c0d47c02b8ccb45ecd7c9b6601 Mon Sep 17 00:00:00 2001 From: francis Date: Tue, 21 May 2024 23:12:38 +0200 Subject: [PATCH] fix: backend linting --- backend/api/tasks/docker_image.py | 29 +++++++++++++++--- backend/api/tasks/extra_check.py | 51 ++++++++++++++++++++++++++++++- 2 files changed, 74 insertions(+), 6 deletions(-) diff --git a/backend/api/tasks/docker_image.py b/backend/api/tasks/docker_image.py index cf614a0d..246d5934 100644 --- a/backend/api/tasks/docker_image.py +++ b/backend/api/tasks/docker_image.py @@ -3,25 +3,44 @@ from api.logic.get_file_path import get_docker_image_tag from api.models.docker import DockerImage, StateEnum from celery import shared_task +from notifications.signals import NotificationType, notification_create from ypovoli.settings import MEDIA_ROOT -# TODO: Remove built image when it's deleted from the database @shared_task def task_docker_image_build(docker_image: DockerImage): # Set state docker_image.state = StateEnum.BUILDING docker_image.save() + notification_type = NotificationType.DOCKER_IMAGE_BUILD_SUCCESS + # Build the image try: client = docker.from_env() client.images.build(path=MEDIA_ROOT, dockerfile=docker_image.file.path, tag=get_docker_image_tag(docker_image), rm=True, quiet=True, forcerm=True) docker_image.state = StateEnum.READY - except (docker.errors.APIError, docker.errors.BuildError, docker.errors.APIError, TypeError): + except (docker.errors.APIError, docker.errors.BuildError, TypeError): docker_image.state = StateEnum.ERROR - # TODO: Sent notification + notification_type = NotificationType.DOCKER_IMAGE_BUILD_ERROR + finally: + # Update the state + docker_image.save() - # Update the state - docker_image.save() + # Send notification + notification_create.send( + sender=DockerImage, + type=notification_type, + queryset=[docker_image.owner], + arguments={"name": docker_image.name}, + ) + + +@shared_task +def task_docker_image_remove(docker_image: DockerImage): + try: + client = docker.from_env() + client.images.remove(get_docker_image_tag(docker_image)) + except docker.errors.APIError: + pass diff --git a/backend/api/tasks/extra_check.py b/backend/api/tasks/extra_check.py index 7ad94afc..1f63e87a 100644 --- a/backend/api/tasks/extra_check.py +++ b/backend/api/tasks/extra_check.py @@ -1,3 +1,5 @@ +import io +import os import shutil import zipfile from time import sleep @@ -14,6 +16,7 @@ from django.core.files.base import ContentFile from docker.models.containers import Container from docker.types import LogConfig +from notifications.signals import NotificationType, notification_create from requests.exceptions import ConnectionError @@ -33,12 +36,22 @@ def task_extra_check_start(structure_check_result: bool, extra_check_result: Ext extra_check_result.error_message = ErrorMessageEnum.DOCKER_IMAGE_ERROR extra_check_result.save() + notification_create.send( + sender=ExtraCheckResult, + type=NotificationType.EXTRA_CHECK_FAIL, + queryset=list(extra_check_result.submission.group.students.all()), + arguments={"name": extra_check_result.extra_check.name}, + ) + return structure_check_result # Will probably never happen but doesn't hurt to check while extra_check_result.submission.running_checks: sleep(1) + # Notification type + notification_type = NotificationType.EXTRA_CHECK_SUCCESS + # Lock extra_check_result.submission.running_checks = True @@ -48,6 +61,7 @@ def task_extra_check_start(structure_check_result: bool, extra_check_result: Ext submission_directory = "/".join(extra_check_result.submission.zip.path.split("/") [:-1]) + "/submission/" # Directory where the files will be extracted + artifacts_directory = f"{submission_directory}/artifacts" # Directory where the artifacts will be stored extra_check_name = extra_check_result.extra_check.file.name.split("/")[-1] # Name of the extra check file submission_uuid = extra_check_result.submission.zip.path.split("/")[-2] # Uuid of the submission @@ -55,6 +69,9 @@ def task_extra_check_start(structure_check_result: bool, extra_check_result: Ext with zipfile.ZipFile(extra_check_result.submission.zip.path, 'r') as zip: zip.extractall(submission_directory) + # Create artifacts directory + os.makedirs(artifacts_directory, exist_ok=True) + container: Container | None = None try: @@ -76,6 +93,9 @@ def task_extra_check_start(structure_check_result: bool, extra_check_result: Ext f"{get_submission_full_dir_path(extra_check_result.submission)}/submission": { "bind": "/submission", "mode": "rw" }, + f"{get_submission_full_dir_path(extra_check_result.submission)}/submission/artifacts": { + "bind": "/submission/artifacts", "mode": "rw" + }, get_extra_check_file_full_path(extra_check_result.extra_check): { "bind": f"/submission/{extra_check_name}", "mode": "ro" } @@ -104,43 +124,52 @@ def task_extra_check_start(structure_check_result: bool, extra_check_result: Ext case 1: extra_check_result.result = StateEnum.FAILED extra_check_result.error_message = ErrorMessageEnum.CHECK_ERROR + notification_type = NotificationType.EXTRA_CHECK_FAIL # Time limit case 2: extra_check_result.result = StateEnum.FAILED extra_check_result.error_message = ErrorMessageEnum.TIME_LIMIT + notification_type = NotificationType.EXTRA_CHECK_FAIL # Memory limit case 3: extra_check_result.result = StateEnum.FAILED extra_check_result.error_message = ErrorMessageEnum.MEMORY_LIMIT + notification_type = NotificationType.EXTRA_CHECK_FAIL # Catch all non zero exit codes case _: extra_check_result.result = StateEnum.FAILED extra_check_result.error_message = ErrorMessageEnum.RUNTIME_ERROR + notification_type = NotificationType.EXTRA_CHECK_FAIL # Docker image error except (docker.errors.APIError, docker.errors.ImageNotFound): extra_check_result.result = StateEnum.FAILED extra_check_result.error_message = ErrorMessageEnum.DOCKER_IMAGE_ERROR + notification_type = NotificationType.EXTRA_CHECK_FAIL # Runtime error except docker.errors.ContainerError: extra_check_result.result = StateEnum.FAILED extra_check_result.error_message = ErrorMessageEnum.RUNTIME_ERROR + notification_type = NotificationType.EXTRA_CHECK_FAIL # Timeout error except ConnectionError: extra_check_result.result = StateEnum.FAILED extra_check_result.error_message = ErrorMessageEnum.TIME_LIMIT + notification_type = NotificationType.EXTRA_CHECK_FAIL # Unknown error except Exception: extra_check_result.result = StateEnum.FAILED extra_check_result.error_message = ErrorMessageEnum.UNKNOWN + notification_type = NotificationType.EXTRA_CHECK_FAIL # Cleanup and data saving + # Start by saving any logs finally: logs: str if container: @@ -153,7 +182,27 @@ def task_extra_check_start(structure_check_result: bool, extra_check_result: Ext logs = "Container error" extra_check_result.log_file.save(submission_uuid, content=ContentFile(logs), save=False) - extra_check_result.save() + + # Send notification + notification_create.send( + sender=ExtraCheckResult, + type=notification_type, + queryset=list(extra_check_result.submission.group.students.all()), + arguments={"name": extra_check_result.extra_check.name}, + ) + + # Zip and save any possible artifacts + memory_zip = io.BytesIO() + if os.listdir(artifacts_directory): + with zipfile.ZipFile(memory_zip, 'w') as zip: + for root, _, files in os.walk(artifacts_directory): + for file in files: + zip.write(os.path.join(root, file), os.path.relpath(os.path.join(root, file), artifacts_directory)) + + memory_zip.seek(0) + extra_check_result.artifact.save(submission_uuid, ContentFile(memory_zip.read()), False) + + extra_check_result.save() # Remove directory try: