Merge branch 'dev' into edit_project

# Conflicts:
#	frontend/src/i18n/locales/en.ts
#	frontend/src/i18n/locales/nl.ts
#	frontend/src/views/CreateProjectView.vue

backend/src/docker_tests/docker_tests.py

@@ -1,8 +1,10 @@
 import os
 import shutil
+from dataclasses import dataclass
 from pathlib import Path
 
 from docker import DockerClient
+from docker.errors import APIError, NotFound
 from docker.models.containers import Container
 from sqlalchemy.ext.asyncio import AsyncSession
 
@@ -15,6 +17,18 @@
 # if a container exits with this code, the test failed (exit 0 means the test succeeded)
 EXIT_TEST_FAILED = 10
 
+# mark test runner containers with this label
+REV_DOMAIN = "be.ugent.sel2-5"
+TEST_RUNNER_LABEL = "test_runner"
+
+
+@dataclass
+class DockerResult:
+    status: Status
+    test_results: list[TestResult]
+    stdout: str | None
+    stderr: str | None
+
 
 def read_feedback_file(path: str) -> list[str]:
     with open(path, 'r') as f:
@@ -37,50 +51,25 @@ async def launch_docker_tests(
     os.makedirs(feedback_dir)
     touch(os.path.join(feedback_dir, "correct"), os.path.join(feedback_dir, "failed"))
 
-    # TODO: zorgen dat tests niet gemount worden als custom docker image gemaakt wordt
-
     if using_default_docker_image(tests_uuid):
         # relative path independent of working dir (tests will break otherwise)
         # path = "./docker_default"
         path = os.path.join(Path(__file__).parent, "docker_default")
         image_tag = "default_image"
+        tests_dir = tests_path(tests_uuid)
 
         # rebuild default image if changes were made
         await build_docker_image(path, image_tag, client)
     else:
         image_tag = tests_uuid
+        tests_dir = None
 
-    container = run_docker_tests(
-        image_tag,
-        submission_path(submission_uuid),
-        artifact_dir,
-        feedback_dir,
-        tests_path(tests_uuid),
-        client,
-    )
-    exit_code = (await wait_until_exit(container))['StatusCode']
-
-    if exit_code == 0:
-        status = Status.Accepted
-    elif exit_code == EXIT_TEST_FAILED:
-        status = Status.Rejected
-    else:
-        status = Status.Crashed
-
-    test_results = []
-    for line in read_feedback_file(os.path.join(feedback_dir, "correct")):
-        test_results.append(TestResult(succeeded=True, value=line))
-    for line in read_feedback_file(os.path.join(feedback_dir, "failed")):
-        test_results.append(TestResult(succeeded=False, value=line))
-
-    stdout = container.logs(stdout=True, stderr=False).decode("utf-8")
-    stderr = container.logs(stdout=False, stderr=True).decode("utf-8")
-    container.remove()
+    result = await run_docker_tests(image_tag, submission_uuid, artifact_dir, feedback_dir, tests_dir, client)
 
     await update_submission_status(
-        db, submission_id, status, test_results,
-        stdout=stdout if stdout else None,
-        stderr=stderr if stderr else None,
+        db, submission_id, result.status, result.test_results,
+        stdout=result.stdout,
+        stderr=result.stderr,
     )
 
     await db.close()
@@ -89,6 +78,52 @@ async def launch_docker_tests(
     shutil.rmtree(feedback_dir)
 
 
+async def run_docker_tests(image_tag: str, submission_uuid: str, artifact_dir: str, feedback_dir: str, tests_dir: str | None,
+                           client: DockerClient) -> DockerResult:
+    try:
+        container = create_container(
+            image_tag,
+            submission_path(submission_uuid),
+            artifact_dir,
+            feedback_dir,
+            tests_dir,
+            client,
+        )
+    except APIError as e:
+        return DockerResult(status=Status.Crashed, test_results=[], stdout=None, stderr=str(e))
+
+    try:
+        container.start()
+        exit_code = (await wait_until_exit(container))['StatusCode']
+
+        if exit_code == 0:
+            status = Status.Accepted
+        elif exit_code == EXIT_TEST_FAILED:
+            status = Status.Rejected
+        else:
+            status = Status.Crashed
+
+        test_results = []
+        for line in read_feedback_file(os.path.join(feedback_dir, "correct")):
+            test_results.append(TestResult(succeeded=True, value=line))
+        for line in read_feedback_file(os.path.join(feedback_dir, "failed")):
+            test_results.append(TestResult(succeeded=False, value=line))
+
+        stdout = container.logs(stdout=True, stderr=False).decode("utf-8")
+        stderr = container.logs(stdout=False, stderr=True).decode("utf-8")
+
+        return DockerResult(status=status, test_results=test_results, stdout=stdout if stdout else None,
+                            stderr=stderr if stderr else None)
+
+    except APIError as e:
+        return DockerResult(status=Status.Crashed, test_results=[], stdout=None, stderr=str(e))
+
+    finally:
+        container.remove(force=True)
+        # remove all stopped containers with test runner tag
+        client.containers.prune(filters={"label": f"{REV_DOMAIN}={TEST_RUNNER_LABEL}"})
+
+
 @to_async
 def build_docker_image(path: str, tag: str, client: DockerClient):
     """Build a docker image from a directory where a file 'Dockerfile' is present"""
@@ -97,24 +132,43 @@ def build_docker_image(path: str, tag: str, client: DockerClient):
         tag=tag,
         forcerm=True
     )
-    client.images.prune()  # cleanup dangling images
+
+    # clean up dangling images
+    client.images.prune()
+
+
+def remove_docker_image_if_exists(tag: str, client: DockerClient):
+    try:
+        client.images.remove(image=tag, force=True)
+    except NotFound:
+        pass
+
+    # clean up dangling images
+    client.images.prune()
 
 
 def using_default_docker_image(tests_uuid: str) -> bool:
     return not os.path.isfile(os.path.join(tests_path(tests_uuid), "Dockerfile"))
 
 
-def run_docker_tests(
-    image_tag: str, submission_dir: str, artifact_dir: str, feedback_dir: str, tests_dir: str, client: DockerClient
+def create_container(
+    image_tag: str, submission_dir: str, artifact_dir: str, feedback_dir: str, tests_dir: str | None,
+    client: DockerClient
 ) -> Container:
-    return client.containers.run(
+    volumes = {
+        submission_dir: {'bind': '/submission', 'mode': 'ro'},
+        artifact_dir: {'bind': '/artifacts', 'mode': 'rw'},
+        feedback_dir: {'bind': '/feedback', 'mode': 'rw'},
+    }
+
+    # only mount test files for default image
+    if tests_dir is not None:
+        volumes[tests_dir] = {'bind': '/tests', 'mode': 'ro'}
+
+    return client.containers.create(
         image=image_tag,
-        volumes={
-            submission_dir: {'bind': '/submission', 'mode': 'ro'},
-            artifact_dir: {'bind': '/artifacts', 'mode': 'rw'},
-            feedback_dir: {'bind': '/feedback', 'mode': 'rw'},
-            tests_dir: {'bind': '/tests', 'mode': 'ro'},
-        },
+        volumes=volumes,
+        labels={REV_DOMAIN: TEST_RUNNER_LABEL},
         environment={
             'SUBMISSION_DIR': '/submission',
             'ARTIFACT_DIR': '/artifacts',
@@ -124,8 +178,6 @@ def run_docker_tests(
             'EXIT_TEST_FAILED': EXIT_TEST_FAILED,
         },
         detach=True,
-        stdout=True,
-        stderr=True,
     )  # pyright: ignore
 
 

backend/src/docker_tests/utils.py

@@ -59,6 +59,7 @@ def write_and_unpack_files(files: list[UploadFile], uuid: str | None) -> str:
 
             if upload_file.content_type == "application/zip":
                 shutil.unpack_archive(path, files_path)
+                os.remove(path)  # don't store zip file
 
     return uuid
 

backend/src/project/router.py

@@ -2,12 +2,15 @@
 
 from docker import DockerClient
 from fastapi import APIRouter, Depends, UploadFile, BackgroundTasks
+from fastapi.responses import StreamingResponse
 from sqlalchemy.ext.asyncio import AsyncSession
 
+from src import dependencies
 from src.auth.dependencies import authentication_validation
 from src.dependencies import get_async_db
 from src.group.dependencies import retrieve_groups_by_project
 from src.group.schemas import GroupList
+from src.project.utils import project_zip_stream
 from src.submission.schemas import Submission
 from src.submission.service import get_submissions_by_project
 from . import service
@@ -23,7 +26,7 @@
     update_project, update_test_files,
 )
 from ..docker_tests.dependencies import get_docker_client
-from ..docker_tests.docker_tests import using_default_docker_image, build_docker_image
+from ..docker_tests.docker_tests import using_default_docker_image, build_docker_image, remove_docker_image_if_exists
 from ..docker_tests.utils import get_files_from_dir, tests_path, write_and_unpack_files, remove_test_files
 
 router = APIRouter(
@@ -84,6 +87,14 @@ async def list_submissions(project_id: int,
     return await get_submissions_by_project(db, project_id)
 
 
+@router.get("/{project_id}/zip", response_class=StreamingResponse, dependencies=[Depends(patch_permission_validation)])
+async def get_submissions_dump(project_id: int, db: AsyncSession = Depends(get_async_db)):
+    """Return zip file containing all submission files and csv"""
+    submissions = await get_submissions_by_project(db, project_id)
+    data = await project_zip_stream(db, submissions, project_id)
+    return StreamingResponse(data, media_type="application/zip")
+
+
 @router.get("/{project_id}/test_files")
 async def get_test_files(test_files_uuid: Optional[str] = Depends(retrieve_test_files_uuid)):
     if not test_files_uuid:
@@ -117,7 +128,10 @@ async def put_test_files(
 async def delete_test_files(
     project: Project = Depends(retrieve_project),
     uuid: str = Depends(retrieve_test_files_uuid),
-    db: AsyncSession = Depends(get_async_db)
+    db: AsyncSession = Depends(get_async_db),
+    client: DockerClient = Depends(get_docker_client)
 ):
+    remove_docker_image_if_exists(uuid, client)
     remove_test_files(uuid)
+
     return await service.update_test_files(db, project.id, None)

backend/src/project/utils.py

@@ -0,0 +1,45 @@
+
+from sqlalchemy.ext.asyncio import AsyncSession
+from src.docker_tests.utils import submission_path
+import pathlib
+import zipfile
+import csv
+import io
+
+from src.submission.models import Status
+from src.submission.schemas import Submission
+from typing import Sequence
+
+from src.group.service import get_group_by_id
+from src.group.exceptions import GroupNotFound
+
+
+async def project_zip_stream(db: AsyncSession, submissions: Sequence[Submission], project_id):
+    data = io.BytesIO()
+    csvdata = io.StringIO()
+    writer = csv.DictWriter(csvdata, fieldnames=[
+                            "project", "group", "date", "status", "remarks", "stdout", "stderr"])
+    writer.writeheader()
+
+    with zipfile.ZipFile(data, mode='w') as z:
+
+        for submission in submissions:
+            path = pathlib.Path(submission_path(submission.files_uuid, ""))
+
+            group = await get_group_by_id(db, submission.group_id)
+
+            if not group:
+                raise GroupNotFound()
+
+            writer.writerow({'project': submission.project_id, 'group': group.num, 'date': submission.date, 'status': Status(
+                submission.status).name, 'remarks': submission.remarks, 'stdout': submission.stdout, 'stderr': submission.stderr})
+
+            for f_name in path.iterdir():
+                name = f"project_{
+                    project_id}/group_{group.num}/{str(f_name).replace(str(path), "")}"
+                z.write(f_name, arcname=name)
+
+        z.writestr(f"project_{project_id}/submissions.csv", csvdata.getvalue())
+
+    data.seek(0)
+    return data

backend/src/submission/router.py

@@ -59,7 +59,8 @@ async def create_submission(background_tasks: BackgroundTasks,
     status = Status.InProgress if docker_tests_present else Status.Accepted
 
     submission = await service.create_submission(
-        db, uuid=submission_uuid, remarks=submission_in.remarks, status=status, group_id=group.id, project_id=group.project_id
+        db, uuid=submission_uuid, remarks=submission_in.remarks, status=status, group_id=group.id,
+        project_id=group.project_id
     )
 
     # launch docker tests
@@ -73,7 +74,8 @@ async def create_submission(background_tasks: BackgroundTasks,
 @router.delete("/{submission_id}",
                dependencies=[Depends(admin_user_validation)],
                status_code=200)
-async def delete_submision(submission: Submission = Depends(retrieve_submission), db: AsyncSession = Depends(get_async_db)):
+async def delete_submission(submission: Submission = Depends(retrieve_submission),
+                            db: AsyncSession = Depends(get_async_db)):
     remove_files(submission.files_uuid)
     await service.delete_submission(db, submission.id)
 
@@ -95,9 +97,9 @@ async def get_file(path: str, submission: Submission = Depends(retrieve_submissi
 
 
 @router.get("/{submission_id}/zip", response_class=StreamingResponse)
-async def get_all_files(submission: Submission = Depends(retrieve_submission)):
-    path = submission_path(submission.files_uuid, "")
-    return StreamingResponse(zip_stream(path, submission.group_id), media_type="application/zip")
+async def get_all_files(submission: Submission = Depends(retrieve_submission), db: AsyncSession = Depends(get_async_db)):
+    data = await zip_stream(db, submission)
+    return StreamingResponse(data, media_type="application/zip")
 
 
 @router.get("/{submission_id}/artifacts", response_model=list[File])

backend/src/submission/utils.py

@@ -7,10 +7,16 @@
 from uuid import uuid4
 
 from fastapi import UploadFile
+from sqlalchemy.ext.asyncio import AsyncSession
 
-from src.docker_tests.utils import submission_path, submissions_path
+from src.docker_tests.utils import submissions_path
 from src.project.schemas import Project
 from src.submission.exceptions import UnMetRequirements
+from src.submission.schemas import Submission
+
+from src.group.service import get_group_by_id
+from src.group.exceptions import GroupNotFound
+from ..docker_tests.utils import submission_path, get_files_from_dir, artifacts_path
 
 
 def upload_files(files: list[UploadFile], project: Project) -> str:
@@ -41,21 +47,26 @@ def upload_files(files: list[UploadFile], project: Project) -> str:
                           "msg": f"Required file not found: {r.value}"})
 
     if len(errors):
-        shutil.rmtree(files_path)
+        remove_files(uuid)
         raise UnMetRequirements(errors)
 
     return uuid
 
 
-def zip_stream(path, group_id: int):
-    base_path = pathlib.Path(path)
+async def zip_stream(db: AsyncSession, submission: Submission):
+    base_path = pathlib.Path(submission_path(submission.files_uuid))
+    group = await get_group_by_id(db, submission.group_id)
+
+    if not group:
+        raise GroupNotFound()
+
     data = io.BytesIO()
     with zipfile.ZipFile(data, mode='w') as z:
         for f_name in base_path.iterdir():
-            name = f"group_{group_id}/{str(f_name).replace(path, "")}"
+            name = f"group_{group.num}/{str(f_name).replace(str(base_path), "")}"
             z.write(f_name, arcname=name)
     data.seek(0)
-    yield from data
+    return data
 
 
 def remove_files(uuid: str):