diff --git a/backend/api/fixtures/realistic/realistic.yaml b/backend/api/fixtures/realistic/realistic.yaml index 84cf1c2a..bf2f4f15 100644 --- a/backend/api/fixtures/realistic/realistic.yaml +++ b/backend/api/fixtures/realistic/realistic.yaml @@ -172,6 +172,7 @@ time_limit: 10 memory_limit: 50 show_log: false + show_artifact: false - model: api.extracheck pk: 1 fields: @@ -182,6 +183,7 @@ time_limit: 30 memory_limit: 128 show_log: true + show_artifact: true # MARK: Students - model: api.student @@ -457,41 +459,49 @@ fields: extra_check: 0 log_file: fixtures/realistic/projects/0/0/submissions/0/submission_1/logs/log_extra_check_0.txt + artifact: "" - model: api.extracheckresult pk: 3 fields: extra_check: 1 log_file: fixtures/realistic/projects/0/0/submissions/0/submission_1/logs/log_extra_check_1.txt + artifact: fixtures/realistic/projects/0/0/submissions/0/submission_1/artifacts/artifact_extra_check_1.zip - model: api.extracheckresult pk: 5 fields: extra_check: 0 log_file: fixtures/realistic/projects/0/0/submissions/0/submission_2/logs/log_extra_check_0.txt + artifact: "" - model: api.extracheckresult pk: 6 fields: extra_check: 1 log_file: fixtures/realistic/projects/0/0/submissions/0/submission_2/logs/log_extra_check_1.txt + artifact: fixtures/realistic/projects/0/0/submissions/0/submission_2/artifacts/artifact_extra_check_1.zip - model: api.extracheckresult pk: 8 fields: extra_check: 0 log_file: "" + artifact: "" - model: api.extracheckresult pk: 9 fields: extra_check: 1 log_file: "" + artifact: "" - model: api.extracheckresult pk: 11 fields: extra_check: 0 log_file: fixtures/realistic/projects/0/0/submissions/1/submission_2/logs/log_extra_check_0.txt + artifact: "" - model: api.extracheckresult pk: 12 fields: extra_check: 1 log_file: fixtures/realistic/projects/0/0/submissions/1/submission_2/logs/log_extra_check_1.txt + artifact: "" # MARK: Teachers - model: api.teacher diff --git a/backend/api/locale/en/LC_MESSAGES/django.po b/backend/api/locale/en/LC_MESSAGES/django.po index 5eda2f33..dbf3b355 100755 --- a/backend/api/locale/en/LC_MESSAGES/django.po +++ b/backend/api/locale/en/LC_MESSAGES/django.po @@ -34,61 +34,61 @@ msgstr "Docker image is ready" msgid "dockerimage.state.error" msgstr "Docker image failed to build" -#: models/submission.py:61 +#: models/submission.py:62 msgid "submission.state.queued" msgstr "Queued" -#: models/submission.py:62 +#: models/submission.py:63 msgid "submission.state.running" msgstr "Running" -#: models/submission.py:63 +#: models/submission.py:64 msgid "submission.state.success" msgstr "Success" -#: models/submission.py:64 +#: models/submission.py:65 msgid "submission.state.failed" msgstr "Failed" -#: models/submission.py:69 +#: models/submission.py:70 msgid "submission.error.blockedextension" msgstr "The zip file contains a file with a non-allowed extension." -#: models/submission.py:70 +#: models/submission.py:71 msgid "submission.error.obligatedextensionnotfound" msgstr "" "The submitted zip file doesn't have any file with an obligated file " "extension." -#: models/submission.py:71 +#: models/submission.py:72 msgid "submission.error.filedirnotfound" msgstr "The submitted zip file lacks an obligated directory." -#: models/submission.py:74 +#: models/submission.py:75 msgid "submission.error.dockerimageerror" msgstr "try again later." -#: models/submission.py:75 +#: models/submission.py:76 msgid "submission.error.timelimit" msgstr "Timelimit exceeded." -#: models/submission.py:76 +#: models/submission.py:77 msgid "submission.error.memorylimit" msgstr "Memorylimit exceeded." -#: models/submission.py:77 +#: models/submission.py:78 msgid "submission.error.checkerror" msgstr "A check failed." -#: models/submission.py:78 +#: models/submission.py:79 msgid "submission.error.runtimeerror" msgstr "Crashed." -#: models/submission.py:79 +#: models/submission.py:80 msgid "submission.error.unknown" msgstr "Unkown error." -#: models/submission.py:80 +#: models/submission.py:81 msgid "submission.error.failedstructurecheck" msgstr "The zip file doesn't have the right structure." @@ -287,3 +287,7 @@ msgstr "No zip file available." #: views/submission_view.py:50 msgid "extra_check_result.download.log" msgstr "No log file available." + +#: views/submission_view.py:60 +msgid "extra_check_result.download.artifact" +msgstr "No artifact available." diff --git a/backend/api/locale/nl/LC_MESSAGES/django.po b/backend/api/locale/nl/LC_MESSAGES/django.po index 648e58f9..7acb1b3d 100755 --- a/backend/api/locale/nl/LC_MESSAGES/django.po +++ b/backend/api/locale/nl/LC_MESSAGES/django.po @@ -34,61 +34,61 @@ msgstr "Docker image is klaar." msgid "dockerimage.state.error" msgstr "Docker image is gefaald om te bouwen." -#: models/submission.py:61 +#: models/submission.py:62 msgid "submission.state.queued" msgstr "wachten" -#: models/submission.py:62 +#: models/submission.py:63 msgid "submission.state.running" msgstr "lopen" -#: models/submission.py:63 +#: models/submission.py:64 msgid "submission.state.success" msgstr "succes" -#: models/submission.py:64 +#: models/submission.py:65 msgid "submission.state.failed" msgstr "gefaald" -#: models/submission.py:69 +#: models/submission.py:70 msgid "submission.error.blockedextension" msgstr "De zip file bevat een niet toegelaten bestandstype." -#: models/submission.py:70 +#: models/submission.py:71 msgid "submission.error.obligatedextensionnotfound" msgstr "" "Er is geen enkel bestand met een bepaalde bestandstype die verplicht is in " "het ingediende zip-bestand." -#: models/submission.py:71 +#: models/submission.py:72 msgid "submission.error.filedirnotfound" msgstr "De ingediende zip file mankeerd een verplichtte map." -#: models/submission.py:74 +#: models/submission.py:75 msgid "submission.error.dockerimageerror" msgstr "Probeer later opnieuw." -#: models/submission.py:75 +#: models/submission.py:76 msgid "submission.error.timelimit" msgstr "Tijdslimit bereikt." -#: models/submission.py:76 +#: models/submission.py:77 msgid "submission.error.memorylimit" msgstr "Geheugenlimiet bereikt." -#: models/submission.py:77 +#: models/submission.py:78 msgid "submission.error.checkerror" msgstr "Een check faalde." -#: models/submission.py:78 +#: models/submission.py:79 msgid "submission.error.runtimeerror" msgstr "Crashed." -#: models/submission.py:79 +#: models/submission.py:80 msgid "submission.error.unknown" msgstr "Onbekende fout." -#: models/submission.py:80 +#: models/submission.py:81 msgid "submission.error.failedstructurecheck" msgstr "De ingediende zip file heeft niet de juiste structuur." @@ -288,3 +288,9 @@ msgstr "Geen zip bestand beschikbaar." #: views/submission_view.py:50 msgid "extra_check_result.download.log" msgstr "Geen log bestand beschikbaar." + +#: views/submission_view.py:60 +#, fuzzy +#| msgid "extra_check_result.download.log" +msgid "extra_check_result.download.artifact" +msgstr "Geen artifact beschikbaar." diff --git a/backend/api/logic/get_file_path.py b/backend/api/logic/get_file_path.py index f0b735d0..c75adb0f 100644 --- a/backend/api/logic/get_file_path.py +++ b/backend/api/logic/get_file_path.py @@ -56,3 +56,8 @@ def get_docker_image_file_path(instance: DockerImage, _: str) -> str: def get_docker_image_tag(instance: DockerImage) -> str: return f"{DOCKER_BUILD_ROOT_NAME}_{instance.id}" + + +def get_extra_check_artifact_file_path(instance: ExtraCheckResult, uuid: str) -> str: + return (f"{_get_project_dir_path(instance.submission.group.project)}" + f"submissions/{instance.submission.group.id}/{uuid}/artifacts/{_get_uuid()}.zip") diff --git a/backend/api/migrations/0025_extracheckresult_artifact.py b/backend/api/migrations/0025_extracheckresult_artifact.py new file mode 100644 index 00000000..7e3984aa --- /dev/null +++ b/backend/api/migrations/0025_extracheckresult_artifact.py @@ -0,0 +1,24 @@ +# Generated by Django 5.0.4 on 2024-05-13 21:33 + +from api.logic.get_file_path import get_extra_check_artifact_file_path +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0024_alter_dockerimage_state'), + ] + + operations = [ + migrations.AddField( + model_name='extracheckresult', + name='artifact', + field=models.FileField(max_length=256, null=True, upload_to=get_extra_check_artifact_file_path), + ), + migrations.AddField( + model_name='extracheck', + name='show_artifact', + field=models.BooleanField(default=True), + ), + ] diff --git a/backend/api/models/checks.py b/backend/api/models/checks.py index efc7b42d..7f5aa570 100644 --- a/backend/api/models/checks.py +++ b/backend/api/models/checks.py @@ -113,3 +113,10 @@ class ExtraCheck(models.Model): blank=False, null=False ) + + # Whether the artifacts should made available to the student + show_artifact = models.BooleanField( + default=True, + blank=False, + null=False + ) diff --git a/backend/api/models/submission.py b/backend/api/models/submission.py index 20294604..ee3afdb1 100644 --- a/backend/api/models/submission.py +++ b/backend/api/models/submission.py @@ -1,6 +1,7 @@ from typing import TYPE_CHECKING -from api.logic.get_file_path import (get_extra_check_log_file_path, +from api.logic.get_file_path import (get_extra_check_artifact_file_path, + get_extra_check_log_file_path, get_submission_file_path) from api.models.checks import ExtraCheck, StructureCheck from api.models.group import Group @@ -137,3 +138,11 @@ class ExtraCheckResult(CheckResult): blank=False, null=True ) + + # File path for the artifact of the extra checks + artifact = models.FileField( + upload_to=get_extra_check_artifact_file_path, + max_length=256, + blank=False, + null=True + ) diff --git a/backend/api/permissions/submission_permissions.py b/backend/api/permissions/submission_permissions.py index 7bc7f642..7180a0fa 100644 --- a/backend/api/permissions/submission_permissions.py +++ b/backend/api/permissions/submission_permissions.py @@ -57,3 +57,18 @@ def has_object_permission(self, request: Request, view: APIView, obj: ExtraCheck return obj.extra_check.show_log return True + + +class ExtraCheckResultArtifactPermission(ExtraCheckResultPermission): + def has_object_permission(self, request: Request, view: APIView, obj: ExtraCheckResult) -> bool: + result = super().has_object_permission(request, view, obj) + + if not result: + return False + + user: User = cast(User, request.user) + + if is_student(user): + return obj.extra_check.show_artifact + + return True diff --git a/backend/api/serializers/submission_serializer.py b/backend/api/serializers/submission_serializer.py index db2d505c..acdf26f8 100644 --- a/backend/api/serializers/submission_serializer.py +++ b/backend/api/serializers/submission_serializer.py @@ -39,6 +39,9 @@ def to_representation(self, instance: ExtraCheckResult) -> dict | None: representation["log_file"] = request.build_absolute_uri( reverse("extra-check-result-detail", args=[str(instance.id)]) + "log/" ) + representation["artifact"] = request.build_absolute_uri( + reverse("extra-check-result-detail", args=[str(instance.id)]) + "artifact/" + ) return representation return None diff --git a/backend/api/tasks/extra_check.py b/backend/api/tasks/extra_check.py index 7ad94afc..5ddb1565 100644 --- a/backend/api/tasks/extra_check.py +++ b/backend/api/tasks/extra_check.py @@ -1,3 +1,5 @@ +import io +import os import shutil import zipfile from time import sleep @@ -11,6 +13,7 @@ from api.models.docker import StateEnum as DockerStateEnum from api.models.submission import ErrorMessageEnum, ExtraCheckResult, StateEnum from celery import shared_task +from django.core.files import File from django.core.files.base import ContentFile from docker.models.containers import Container from docker.types import LogConfig @@ -48,6 +51,7 @@ def task_extra_check_start(structure_check_result: bool, extra_check_result: Ext submission_directory = "/".join(extra_check_result.submission.zip.path.split("/") [:-1]) + "/submission/" # Directory where the files will be extracted + artifacts_directory = f"{submission_directory}/artifacts" # Directory where the artifacts will be stored extra_check_name = extra_check_result.extra_check.file.name.split("/")[-1] # Name of the extra check file submission_uuid = extra_check_result.submission.zip.path.split("/")[-2] # Uuid of the submission @@ -55,6 +59,9 @@ def task_extra_check_start(structure_check_result: bool, extra_check_result: Ext with zipfile.ZipFile(extra_check_result.submission.zip.path, 'r') as zip: zip.extractall(submission_directory) + # Create artifacts directory + os.makedirs(artifacts_directory, exist_ok=True) + container: Container | None = None try: @@ -76,6 +83,9 @@ def task_extra_check_start(structure_check_result: bool, extra_check_result: Ext f"{get_submission_full_dir_path(extra_check_result.submission)}/submission": { "bind": "/submission", "mode": "rw" }, + f"{get_submission_full_dir_path(extra_check_result.submission)}/submission/artifacts": { + "bind": "/submission/artifacts", "mode": "rw" + }, get_extra_check_file_full_path(extra_check_result.extra_check): { "bind": f"/submission/{extra_check_name}", "mode": "ro" } @@ -141,6 +151,7 @@ def task_extra_check_start(structure_check_result: bool, extra_check_result: Ext extra_check_result.error_message = ErrorMessageEnum.UNKNOWN # Cleanup and data saving + # Start by saving any logs finally: logs: str if container: @@ -153,7 +164,19 @@ def task_extra_check_start(structure_check_result: bool, extra_check_result: Ext logs = "Container error" extra_check_result.log_file.save(submission_uuid, content=ContentFile(logs), save=False) - extra_check_result.save() + + # Zip and save any possible artifacts + memory_zip = io.BytesIO() + if os.listdir(artifacts_directory): + with zipfile.ZipFile(memory_zip, 'w') as zip: + for root, _, files in os.walk(artifacts_directory): + for file in files: + zip.write(os.path.join(root, file), os.path.relpath(os.path.join(root, file), artifacts_directory)) + + memory_zip.seek(0) + extra_check_result.artifact.save(submission_uuid, ContentFile(memory_zip.read()), False) + + extra_check_result.save() # Remove directory try: diff --git a/backend/api/views/submission_view.py b/backend/api/views/submission_view.py index 9010b64e..f1f42a58 100644 --- a/backend/api/views/submission_view.py +++ b/backend/api/views/submission_view.py @@ -1,8 +1,9 @@ from api.models.submission import (ExtraCheckResult, StructureCheckResult, Submission) from api.permissions.submission_permissions import ( - ExtraCheckResultLogPermission, ExtraCheckResultPermission, - StructureCheckResultPermission, SubmissionPermission) + ExtraCheckResultArtifactPermission, ExtraCheckResultLogPermission, + ExtraCheckResultPermission, StructureCheckResultPermission, + SubmissionPermission) from api.serializers.submission_serializer import ( ExtraCheckResultSerializer, StructureCheckResultSerializer, SubmissionSerializer) @@ -42,11 +43,20 @@ class ExtraCheckResultViewSet(RetrieveModelMixin, GenericViewSet): serializer_class = ExtraCheckResultSerializer permission_classes = [ExtraCheckResultPermission] - @action(detail=True, permission_classes=[IsAdminUser | ExtraCheckResultLogPermission]) + @action(detail=True, permission_classes=[IsAdminUser | ExtraCheckResultArtifactPermission]) def log(self, request, **__): extra_check_result: ExtraCheckResult = self.get_object() if not extra_check_result.log_file: return Response({"message": _("extra_check_result.download.log")}, status=404) - return FileResponse(open(extra_check_result.log_file.path, "rb"), as_attachment=True) + return FileResponse(open(extra_check_result.log_file.path, "rb"), as_attachment=True, filename="log.txt") + + @action(detail=True, permission_classes=[IsAdminUser | ExtraCheckResultLogPermission]) + def artifact(self, request, **__): + extra_check_result: ExtraCheckResult = self.get_object() + + if not extra_check_result.artifact: + return Response({"message": _("extra_check_result.download.artifact")}, status=404) + + return FileResponse(open(extra_check_result.artifact.path, "rb"), as_attachment=True, filename="artifact.zip") diff --git a/backend/data/fixtures/realistic/projects/0/0/checks/generate_gibberish.sh b/backend/data/fixtures/realistic/projects/0/0/checks/generate_gibberish.sh index 182840c5..9690ec1e 100755 --- a/backend/data/fixtures/realistic/projects/0/0/checks/generate_gibberish.sh +++ b/backend/data/fixtures/realistic/projects/0/0/checks/generate_gibberish.sh @@ -1,5 +1,7 @@ #!/bin/bash +# generate gibberish logs + # Function to generate a random sequence of characters generate_sequence() { cat /dev/urandom | tr -dc 'a-zA-Z0-9' | head -c 50 @@ -14,3 +16,7 @@ while [ $count -le 50 ]; do generate_sequence count=$((count+1)) done + +# Generate an artifact + +wget https://golang.org/doc/gopher/modelsheet.jpg -P artifacts diff --git a/backend/data/fixtures/realistic/projects/0/0/submissions/0/submission_1/artifacts/artifact_extra_check_0.zip b/backend/data/fixtures/realistic/projects/0/0/submissions/0/submission_1/artifacts/artifact_extra_check_0.zip new file mode 100644 index 00000000..e69de29b diff --git a/backend/data/fixtures/realistic/projects/0/0/submissions/0/submission_1/artifacts/artifact_extra_check_1.zip b/backend/data/fixtures/realistic/projects/0/0/submissions/0/submission_1/artifacts/artifact_extra_check_1.zip new file mode 100644 index 00000000..912548a7 Binary files /dev/null and b/backend/data/fixtures/realistic/projects/0/0/submissions/0/submission_1/artifacts/artifact_extra_check_1.zip differ diff --git a/backend/data/fixtures/realistic/projects/0/0/submissions/0/submission_2/artifacts/artifact_extra_check_0.zip b/backend/data/fixtures/realistic/projects/0/0/submissions/0/submission_2/artifacts/artifact_extra_check_0.zip new file mode 100644 index 00000000..e69de29b diff --git a/backend/data/fixtures/realistic/projects/0/0/submissions/0/submission_2/artifacts/artifact_extra_check_1.zip b/backend/data/fixtures/realistic/projects/0/0/submissions/0/submission_2/artifacts/artifact_extra_check_1.zip new file mode 100644 index 00000000..912548a7 Binary files /dev/null and b/backend/data/fixtures/realistic/projects/0/0/submissions/0/submission_2/artifacts/artifact_extra_check_1.zip differ