From d2ba9abc8aa04c7b1a9a7c898b85d469a2ad9b6b Mon Sep 17 00:00:00 2001 From: Xander Bil Date: Sun, 19 May 2024 20:56:34 +0200 Subject: [PATCH 1/3] backend: implement project zip --- backend/src/project/router.py | 11 +++++++++++ backend/src/project/utils.py | 34 ++++++++++++++++++++++++++++++++ backend/src/submission/router.py | 6 +++--- backend/src/submission/utils.py | 21 +++++++++++++++----- 4 files changed, 64 insertions(+), 8 deletions(-) create mode 100644 backend/src/project/utils.py diff --git a/backend/src/project/router.py b/backend/src/project/router.py index cba03df1..519d0bed 100644 --- a/backend/src/project/router.py +++ b/backend/src/project/router.py @@ -2,12 +2,15 @@ from docker import DockerClient from fastapi import APIRouter, Depends, UploadFile, BackgroundTasks +from fastapi.responses import StreamingResponse from sqlalchemy.ext.asyncio import AsyncSession +from src import dependencies from src.auth.dependencies import authentication_validation from src.dependencies import get_async_db from src.group.dependencies import retrieve_groups_by_project from src.group.schemas import GroupList +from src.project.utils import project_zip_stream from src.submission.schemas import Submission from src.submission.service import get_submissions_by_project from . import service @@ -84,6 +87,14 @@ async def list_submissions(project_id: int, return await get_submissions_by_project(db, project_id) +@router.get("/{project_id}/zip", response_class=StreamingResponse, dependencies=[Depends(patch_permission_validation)]) +async def get_submissions_dump(project_id: int, db: AsyncSession = Depends(get_async_db)): + """Return zip file containing all submission files and csv""" + submissions = await get_submissions_by_project(db, project_id) + data = await project_zip_stream(db, submissions) + return StreamingResponse(data, media_type="application/zip") + + @router.get("/{project_id}/test_files") async def get_test_files(test_files_uuid: str = Depends(retrieve_test_files_uuid)): return get_files_from_dir(tests_path(test_files_uuid)) diff --git a/backend/src/project/utils.py b/backend/src/project/utils.py new file mode 100644 index 00000000..bdde2336 --- /dev/null +++ b/backend/src/project/utils.py @@ -0,0 +1,34 @@ + +from sqlalchemy.ext.asyncio import AsyncSession +from src.docker_tests.utils import submission_path +import pathlib +import zipfile +import csv +import io + +from src.submission.schemas import Submission +from typing import List + +from src.group.service import get_group_by_id +from src.group.exceptions import GroupNotFound + + +async def project_zip_stream(db: AsyncSession, submissions: List[Submission]): + data = io.BytesIO() + + with zipfile.ZipFile(data, mode='w') as z: + for submission in submissions: + path = pathlib.Path(submission_path(submission.files_uuid, "")) + + group = await get_group_by_id(db, submission.group_id) + + if not group: + raise GroupNotFound() + + for f_name in path.iterdir(): + name = f"project_{ + submission.project_id}/group_{group.num}/{str(f_name).replace(str(path), "")}" + z.write(f_name, arcname=name) + + data.seek(0) + return data diff --git a/backend/src/submission/router.py b/backend/src/submission/router.py index 683001cb..8f07cd37 100644 --- a/backend/src/submission/router.py +++ b/backend/src/submission/router.py @@ -95,9 +95,9 @@ async def get_file(path: str, submission: Submission = Depends(retrieve_submissi @router.get("/{submission_id}/zip", response_class=StreamingResponse) -async def get_all_files(submission: Submission = Depends(retrieve_submission)): - path = submission_path(submission.files_uuid, "") - return StreamingResponse(zip_stream(path, submission.group_id), media_type="application/zip") +async def get_all_files(submission: Submission = Depends(retrieve_submission), db: AsyncSession = Depends(get_async_db)): + data = await zip_stream(db, submission) + return StreamingResponse(data, media_type="application/zip") @router.get("/{submission_id}/artifacts", response_model=list[File]) diff --git a/backend/src/submission/utils.py b/backend/src/submission/utils.py index 7e0afe1d..c96d1656 100644 --- a/backend/src/submission/utils.py +++ b/backend/src/submission/utils.py @@ -7,10 +7,16 @@ from uuid import uuid4 from fastapi import UploadFile +from sqlalchemy.ext.asyncio import AsyncSession -from src.docker_tests.utils import submission_path, submissions_path +from src.docker_tests.utils import submissions_path from src.project.schemas import Project from src.submission.exceptions import UnMetRequirements +from src.submission.schemas import Submission + +from src.group.service import get_group_by_id +from src.group.exceptions import GroupNotFound +from ..docker_tests.utils import submission_path, get_files_from_dir, artifacts_path def upload_files(files: list[UploadFile], project: Project) -> str: @@ -47,15 +53,20 @@ def upload_files(files: list[UploadFile], project: Project) -> str: return uuid -def zip_stream(path, group_id: int): - base_path = pathlib.Path(path) +async def zip_stream(db: AsyncSession, submission: Submission): + base_path = pathlib.Path(submission_path(submission.files_uuid)) + group = await get_group_by_id(db, submission.group_id) + + if not group: + raise GroupNotFound() + data = io.BytesIO() with zipfile.ZipFile(data, mode='w') as z: for f_name in base_path.iterdir(): - name = f"group_{group_id}/{str(f_name).replace(path, "")}" + name = f"group_{group.num}/{str(f_name).replace(str(base_path), "")}" z.write(f_name, arcname=name) data.seek(0) - yield from data + return data def remove_files(uuid: str): From 8a9511440649976049f5e78400f7fb7cf88b5035 Mon Sep 17 00:00:00 2001 From: Xander Bil Date: Mon, 20 May 2024 12:32:10 +0200 Subject: [PATCH 2/3] backend: implement csv --- backend/src/project/router.py | 2 +- backend/src/project/utils.py | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/backend/src/project/router.py b/backend/src/project/router.py index 519d0bed..099fb6a8 100644 --- a/backend/src/project/router.py +++ b/backend/src/project/router.py @@ -91,7 +91,7 @@ async def list_submissions(project_id: int, async def get_submissions_dump(project_id: int, db: AsyncSession = Depends(get_async_db)): """Return zip file containing all submission files and csv""" submissions = await get_submissions_by_project(db, project_id) - data = await project_zip_stream(db, submissions) + data = await project_zip_stream(db, submissions, project_id) return StreamingResponse(data, media_type="application/zip") diff --git a/backend/src/project/utils.py b/backend/src/project/utils.py index bdde2336..b81211bf 100644 --- a/backend/src/project/utils.py +++ b/backend/src/project/utils.py @@ -6,17 +6,23 @@ import csv import io +from src.submission.models import Status from src.submission.schemas import Submission -from typing import List +from typing import Sequence from src.group.service import get_group_by_id from src.group.exceptions import GroupNotFound -async def project_zip_stream(db: AsyncSession, submissions: List[Submission]): +async def project_zip_stream(db: AsyncSession, submissions: Sequence[Submission], project_id): data = io.BytesIO() + csvdata = io.StringIO() + writer = csv.DictWriter(csvdata, fieldnames=[ + "project", "group", "date", "status", "remarks", "stdout", "stderr"]) + writer.writeheader() with zipfile.ZipFile(data, mode='w') as z: + for submission in submissions: path = pathlib.Path(submission_path(submission.files_uuid, "")) @@ -25,10 +31,15 @@ async def project_zip_stream(db: AsyncSession, submissions: List[Submission]): if not group: raise GroupNotFound() + writer.writerow({'project': submission.project_id, 'group': group.num, 'date': submission.date, 'status': Status( + submission.status).name, 'remarks': submission.remarks, 'stdout': submission.stdout, 'stderr': submission.stderr}) + for f_name in path.iterdir(): name = f"project_{ - submission.project_id}/group_{group.num}/{str(f_name).replace(str(path), "")}" + project_id}/group_{group.num}/{str(f_name).replace(str(path), "")}" z.write(f_name, arcname=name) + z.writestr(f"project_{project_id}/submissions.csv", csvdata.getvalue()) + data.seek(0) return data From fa8e2ac8ec25fac69a1ae27e7dc5f0114c80a168 Mon Sep 17 00:00:00 2001 From: Xander Bil Date: Mon, 20 May 2024 12:58:42 +0200 Subject: [PATCH 3/3] frontend: download all submissions --- frontend/src/i18n/locales/en.ts | 1 + frontend/src/i18n/locales/nl.ts | 1 + frontend/src/views/SubmissionsTeacherView.vue | 19 +++++++++++++++++++ 3 files changed, 21 insertions(+) diff --git a/frontend/src/i18n/locales/en.ts b/frontend/src/i18n/locales/en.ts index 1c9c4c55..27bb87b9 100644 --- a/frontend/src/i18n/locales/en.ts +++ b/frontend/src/i18n/locales/en.ts @@ -61,6 +61,7 @@ export default { edit: "Edit project", submissions_list: "All submissions", submissions_list_teacher: "All submissions for this project", + submissions_zip: "Download all submissions", not_found: "No projects found.", finished: "Finished", not_found2: "Project not found", diff --git a/frontend/src/i18n/locales/nl.ts b/frontend/src/i18n/locales/nl.ts index 37693491..930a2e43 100644 --- a/frontend/src/i18n/locales/nl.ts +++ b/frontend/src/i18n/locales/nl.ts @@ -61,6 +61,7 @@ export default { edit: "Bewerk project", submissions_list: "Alle indieningen", submissions_list_teacher: "Alle indieningen voor dit project", + submissions_zip: "Download alle indieningen", not_found: "Geen projecten teruggevonden.", finished: "Afgerond", not_found2: "Project niet teruggevonden", diff --git a/frontend/src/views/SubmissionsTeacherView.vue b/frontend/src/views/SubmissionsTeacherView.vue index 1a6af32c..346b4ba9 100644 --- a/frontend/src/views/SubmissionsTeacherView.vue +++ b/frontend/src/views/SubmissionsTeacherView.vue @@ -4,6 +4,11 @@

{{ $t("submission.submissions_title", { project: project.name }) }}

+ + + {{ $t("project.submissions_zip") }} + + {{ $t("submission.no_submissions") }} @@ -28,6 +33,7 @@ import { useProjectQuery } from "@/queries/Project"; import { useProjectSubmissionsQuery } from "@/queries/Submission"; import { toRefs } from "vue"; import SubmissionTeacherCard from "@/components/submission/SubmissionTeacherCard.vue"; +import { download_file } from "@/utils"; const props = defineProps<{ projectId: number; @@ -42,4 +48,17 @@ const { error, } = useProjectSubmissionsQuery(projectId); const { data: project, isLoading: projectLoading } = useProjectQuery(projectId); + +const downloadAll = () => { + download_file(`/api/projects/${projectId.value}/zip`, `project_${projectId.value}`); +}; + +