From 956e486c8d1891af9e0e6ac6b27f3b4bd3d4961b Mon Sep 17 00:00:00 2001 From: vmanawat Date: Tue, 24 Sep 2024 11:17:45 -0700 Subject: [PATCH 1/9] Updating dashboard page which allows the user to download an uploaded file --- backend/src/aqi_api/aqi_api.service.ts | 3 +- backend/src/cron-job/cron-job.service.ts | 10 +- .../file_parse_and_validation.module.ts | 3 +- .../file_submissions.controller.ts | 22 +- .../file_submissions.module.ts | 2 + .../file_submissions.service.ts | 77 ++--- frontend/src/common/api.ts | 138 +++++---- frontend/src/common/manage-files.ts | 7 + frontend/src/pages/Dashboard.tsx | 284 ++++++++++-------- frontend/src/pages/FileUpload.tsx | 2 +- 10 files changed, 275 insertions(+), 273 deletions(-) diff --git a/backend/src/aqi_api/aqi_api.service.ts b/backend/src/aqi_api/aqi_api.service.ts index 21de3b0a..93e80916 100644 --- a/backend/src/aqi_api/aqi_api.service.ts +++ b/backend/src/aqi_api/aqi_api.service.ts @@ -125,8 +125,7 @@ export class AqiApiService { }); await this.wait(9); - console.log(response.data.id) - + const obsResultResponse = await axios.get( `${process.env.AQI_BASE_URL}/v2/observationimports/${response.data.id}/result`, { diff --git a/backend/src/cron-job/cron-job.service.ts b/backend/src/cron-job/cron-job.service.ts index faa8e502..0b56d326 100644 --- a/backend/src/cron-job/cron-job.service.ts +++ b/backend/src/cron-job/cron-job.service.ts @@ -4,7 +4,6 @@ import { error } from "winston"; import { Cron, CronExpression } from "@nestjs/schedule"; import { PrismaService } from "nestjs-prisma"; import { FileParseValidateService } from "src/file_parse_and_validation/file_parse_and_validation.service"; -import { FileSubmissionsService } from "src/file_submissions/file_submissions.service"; import { ObjectStoreService } from "src/objectStore/objectStore.service"; /** @@ -21,7 +20,6 @@ export class CronJobService { constructor( private prisma: PrismaService, private readonly fileParser: FileParseValidateService, - private readonly fileSubmissionsService: FileSubmissionsService, private readonly objectStore: ObjectStoreService, ) { this.tableModels = new Map([ @@ -433,10 +431,10 @@ export class CronJobService { grab all the files from the DB and S3 bucket that have a status of QUEUED for each file returned, change the status to INPROGRESS and go to the parser */ - // if (!this.dataPullDownComplete) { - // this.logger.warn("Data pull down from AQSS did not complete"); - // return; - // } + if (!this.dataPullDownComplete) { + this.logger.warn("Data pull down from AQSS did not complete"); + return; + } let filesToValidate = await this.fileParser.getQueuedFiles(); diff --git a/backend/src/file_parse_and_validation/file_parse_and_validation.module.ts b/backend/src/file_parse_and_validation/file_parse_and_validation.module.ts index 65ec219f..c425c295 100644 --- a/backend/src/file_parse_and_validation/file_parse_and_validation.module.ts +++ b/backend/src/file_parse_and_validation/file_parse_and_validation.module.ts @@ -3,10 +3,11 @@ import { HttpModule } from '@nestjs/axios'; import { FileParseValidateService } from './file_parse_and_validation.service'; import { AqiApiService } from 'src/aqi_api/aqi_api.service'; import { FileSubmissionsService } from 'src/file_submissions/file_submissions.service'; +import { ObjectStoreModule } from 'src/objectStore/objectStore.module'; @Module({ providers: [FileParseValidateService, FileSubmissionsService, AqiApiService], exports: [FileParseValidateService], - imports: [HttpModule] + imports: [HttpModule, ObjectStoreModule] }) export class FileParseValidateModule {} diff --git a/backend/src/file_submissions/file_submissions.controller.ts b/backend/src/file_submissions/file_submissions.controller.ts index d6358a64..45559bb0 100644 --- a/backend/src/file_submissions/file_submissions.controller.ts +++ b/backend/src/file_submissions/file_submissions.controller.ts @@ -11,6 +11,7 @@ import { ParseFilePipe, MaxFileSizeValidator, UseGuards, + Res, } from "@nestjs/common"; import { FileSubmissionsService } from "./file_submissions.service"; import { CreateFileSubmissionDto } from "./dto/create-file_submission.dto"; @@ -35,7 +36,7 @@ import { FileInfo } from "src/types/types"; export class FileSubmissionsController { constructor( private readonly fileSubmissionsService: FileSubmissionsService, - private readonly sanitizeService: SanitizeService + private readonly sanitizeService: SanitizeService, ) {} @Post() @@ -44,41 +45,36 @@ export class FileSubmissionsController { @UploadedFile( new ParseFilePipe({ validators: [new MaxFileSizeValidator({ maxSize: 10000000 })], - }) + }), ) file: Express.Multer.File, - @Body() body: any + @Body() body: any, ) { return this.fileSubmissionsService.create(body, file); } @Get() - findByCode( - @Param("submissionCode") submissionCode: string - ) { + findByCode(@Param("submissionCode") submissionCode: string) { return this.fileSubmissionsService.findByCode(submissionCode); } @Post("search") @UseInterceptors(FileInterceptor("file")) async findByQuery( - @Body() body: any + @Body() body: any, ): Promise> { return this.fileSubmissionsService.findBySearch(body); } @Get(":fileName") - findOne( - @Param("fileName") fileName: string - ): Promise> { - const sanitizedParam = this.sanitizeService.sanitizeInput(fileName); - return this.fileSubmissionsService.findOne(sanitizedParam); + getFromS3(@Param("fileName") fileName: string) { + return this.fileSubmissionsService.getFromS3(fileName); } @Patch(":id") update( @Param("id") id: string, - @Body() updateFileSubmissionDto: UpdateFileSubmissionDto + @Body() updateFileSubmissionDto: UpdateFileSubmissionDto, ) { return this.fileSubmissionsService.update(+id, updateFileSubmissionDto); } diff --git a/backend/src/file_submissions/file_submissions.module.ts b/backend/src/file_submissions/file_submissions.module.ts index 1560f275..177951fd 100644 --- a/backend/src/file_submissions/file_submissions.module.ts +++ b/backend/src/file_submissions/file_submissions.module.ts @@ -2,10 +2,12 @@ import { Module } from "@nestjs/common"; import { FileSubmissionsService } from "./file_submissions.service"; import { FileSubmissionsController } from "./file_submissions.controller"; import { SanitizeService } from "src/sanitize/sanitize.service"; +import { ObjectStoreModule } from "src/objectStore/objectStore.module"; @Module({ controllers: [FileSubmissionsController], providers: [FileSubmissionsService, SanitizeService], exports: [FileSubmissionsService], + imports: [ObjectStoreModule], }) export class FileSubmissionsModule {} diff --git a/backend/src/file_submissions/file_submissions.service.ts b/backend/src/file_submissions/file_submissions.service.ts index 99e8bef8..fc63d2bf 100644 --- a/backend/src/file_submissions/file_submissions.service.ts +++ b/backend/src/file_submissions/file_submissions.service.ts @@ -6,10 +6,14 @@ import { FileResultsWithCount } from "src/interface/fileResultsWithCount"; import { file_submission, Prisma } from "@prisma/client"; import { FileInfo } from "src/types/types"; import { randomUUID } from "crypto"; +import { ObjectStoreService } from "src/objectStore/objectStore.service"; @Injectable() export class FileSubmissionsService { - constructor(private prisma: PrismaService) {} + constructor( + private prisma: PrismaService, + private readonly objectStore: ObjectStoreService, + ) {} async create(body: any, file: Express.Multer.File) { const createFileSubmissionDto = new CreateFileSubmissionDto(); @@ -51,7 +55,11 @@ export class FileSubmissionsService { submission_date: createFileSubmissionDto.submission_date, submitter_user_id: createFileSubmissionDto.submitter_user_id, submission_status: { connect: { submission_status_code: "QUEUED" } }, - file_operation_codes: { connect: { file_operation_code: createFileSubmissionDto.file_operation_code } }, + file_operation_codes: { + connect: { + file_operation_code: createFileSubmissionDto.file_operation_code, + }, + }, submitter_agency_name: createFileSubmissionDto.submitter_agency_name, sample_count: createFileSubmissionDto.sample_count, results_count: createFileSubmissionDto.result_count, @@ -173,42 +181,6 @@ export class FileSubmissionsService { return records; } - async findOne( - fileName: string, - ): Promise> { - let records: FileResultsWithCount = { - count: 0, - results: [], - }; - /* - TODO: - - Find the file_submission record with the submission_id = id - - Grab the file from the S3 bucket - - Do the initial validation first (validate the fields in the file that are not in the AQI API). if failed, set the submission_status_code to FAILED, populate the error_log column and return with the error message. - - Once the initial validation has passed, then call the AQI API on the rest of the fields. If failed, set the submission_status_code to FAILED, populate error_log column and return with error message. - - Once the AQI API call is done, then update the submission_status_code field in the database to PASSED. If failed, set the submission_status_code to FAILED, populate error_log column and return with error message. - */ - - const query = { - where: { - file_name: { - contains: fileName, - }, - }, - }; - - const [results, count] = await this.prisma.$transaction([ - this.prisma.file_submission.findMany(query), - - this.prisma.file_submission.count({ - where: query.where, - }), - ]); - - records = { ...results, count, results }; - return records; - } - async updateFileStatus(submission_id: string, status: string) { await this.prisma.file_submission.update({ where: { @@ -227,6 +199,16 @@ export class FileSubmissionsService { remove(id: number) { return `This action removes a #${id} fileSubmission`; } + + async getFromS3(fileName: string) { + try{ + const fileBinary = await this.objectStore.getFileData(fileName) + return fileBinary + } catch (err){ + console.error(`Error fetching file from S3: ${err.message}`); + throw err; + } + } } async function saveToS3(token: any, file: Express.Multer.File) { @@ -259,22 +241,3 @@ async function saveToS3(token: any, file: Express.Multer.File) { return [fileGUID, newFileName]; } - -async function getFromS3(submission_id: string) { - const axios = require("axios"); - - let config = { - method: "get", - maxBodyLength: Infinity, - url: `${process.env.COMS_URI}/v1/object/${submission_id}`, - headers: { - Accept: "application/json", - }, - }; - - await axios.request(config).then((response) => { - console.log(response); - }); - - return null; -} diff --git a/frontend/src/common/api.ts b/frontend/src/common/api.ts index a430f460..d68f3508 100644 --- a/frontend/src/common/api.ts +++ b/frontend/src/common/api.ts @@ -1,6 +1,6 @@ -import axios, { AxiosResponse, AxiosError, AxiosRequestConfig } from 'axios' -import config from '../config' -import { AUTH_TOKEN } from '../service/user-service' +import axios, { AxiosResponse, AxiosError, AxiosRequestConfig } from "axios"; +import config from "../config"; +import { AUTH_TOKEN } from "../service/user-service"; const STATUS_CODES = { Ok: 200, @@ -12,123 +12,135 @@ const STATUS_CODES = { InternalServerError: 500, BadGateway: 502, ServiceUnavailable: 503, -} +}; -const { KEYCLOAK_URL } = config +const { KEYCLOAK_URL } = config; interface ApiRequestParameters { - url: string - requiresAuthentication?: boolean - params?: T + url: string; + requiresAuthentication?: boolean; + params?: T; } -export const get = (parameters: ApiRequestParameters, headers?: {}): Promise => { - let config: AxiosRequestConfig = { headers: headers } +export const get = ( + parameters: ApiRequestParameters, + headers?: {}, +): Promise => { + let config: AxiosRequestConfig = { headers: headers }; return new Promise((resolve, reject) => { - const { url, requiresAuthentication, params } = parameters + const { url, requiresAuthentication, params } = parameters; if (requiresAuthentication) { - axios.defaults.headers.common['Authorization'] = `Bearer ${localStorage.getItem(AUTH_TOKEN)}` + axios.defaults.headers.common["Authorization"] = + `Bearer ${localStorage.getItem(AUTH_TOKEN)}`; } if (params) { - config.params = params + config.params = params; } axios .get(url, config) .then((response: AxiosResponse) => { - const { data, status } = response + const { data, status } = response; if (status === STATUS_CODES.Unauthorized) { - window.location = KEYCLOAK_URL + window.location = KEYCLOAK_URL; } - resolve(data as T) + resolve(data as T); }) .catch((error: AxiosError) => { - console.log(error.message) - reject(error) - }) - }) -} - -export const post = (parameters: ApiRequestParameters): Promise => { + console.log(error.message); + reject(error); + }); + }); +}; + +export const post = ( + parameters: ApiRequestParameters, +): Promise => { let config: AxiosRequestConfig = { headers: {}, - } + }; return new Promise((resolve, reject) => { - const { url, requiresAuthentication, params } = parameters + const { url, requiresAuthentication, params } = parameters; if (requiresAuthentication) { - axios.defaults.headers.common['Authorization'] = `Bearer ${localStorage.getItem(AUTH_TOKEN)}` + axios.defaults.headers.common["Authorization"] = + `Bearer ${localStorage.getItem(AUTH_TOKEN)}`; } - axios .post(url, params, config) .then((response: AxiosResponse) => { - resolve(response.data as T) + resolve(response.data as T); }) .catch((error: AxiosError) => { - console.log(error.message) - reject(error) - }) - }) -} - -export const patch = (parameters: ApiRequestParameters): Promise => { - let config: AxiosRequestConfig = { headers: {} } + console.log(error.message); + reject(error); + }); + }); +}; + +export const patch = ( + parameters: ApiRequestParameters, +): Promise => { + let config: AxiosRequestConfig = { headers: {} }; return new Promise((resolve, reject) => { - const { url, requiresAuthentication, params: data } = parameters + const { url, requiresAuthentication, params: data } = parameters; if (requiresAuthentication) { - axios.defaults.headers.common['Authorization'] = `Bearer ${localStorage.getItem(AUTH_TOKEN)}` + axios.defaults.headers.common["Authorization"] = + `Bearer ${localStorage.getItem(AUTH_TOKEN)}`; } axios .patch(url, data, config) .then((response: AxiosResponse) => { - const { status } = response + const { status } = response; if (status === STATUS_CODES.Unauthorized) { - window.location = KEYCLOAK_URL + window.location = KEYCLOAK_URL; } - resolve(response.data as T) + resolve(response.data as T); }) .catch((error: AxiosError) => { - console.log(error.message) - reject(error) - }) - }) -} - -export const put = (parameters: ApiRequestParameters): Promise => { - let config: AxiosRequestConfig = { headers: {} } + console.log(error.message); + reject(error); + }); + }); +}; + +export const put = ( + parameters: ApiRequestParameters, +): Promise => { + let config: AxiosRequestConfig = { headers: {} }; return new Promise((resolve, reject) => { - const { url, requiresAuthentication, params: data } = parameters + const { url, requiresAuthentication, params: data } = parameters; if (requiresAuthentication) { - axios.defaults.headers.common['Authorization'] = `Bearer ${localStorage.getItem(AUTH_TOKEN)}` + axios.defaults.headers.common["Authorization"] = + `Bearer ${localStorage.getItem(AUTH_TOKEN)}`; } axios .put(url, data, config) .then((response: AxiosResponse) => { - const { status } = response + const { status } = response; if (status === STATUS_CODES.Unauthorized) { - window.location = KEYCLOAK_URL + window.location = KEYCLOAK_URL; } - resolve(response.data as T) + resolve(response.data as T); }) .catch((error: AxiosError) => { - console.log(error.message) - reject(error) - }) - }) -} + console.log(error.message); + reject(error); + }); + }); +}; export const generateApiParameters = ( url: string, @@ -138,11 +150,11 @@ export const generateApiParameters = ( let result = { url, requiresAuthentication, - } as ApiRequestParameters + } as ApiRequestParameters; if (params) { - return { ...result, params } + return { ...result, params }; } - return result -} + return result; +}; diff --git a/frontend/src/common/manage-files.ts b/frontend/src/common/manage-files.ts index 5f8553d6..a25205c2 100644 --- a/frontend/src/common/manage-files.ts +++ b/frontend/src/common/manage-files.ts @@ -29,4 +29,11 @@ export const searchFiles = async (formData: FormData): Promise => { const getParameters = api.generateApiParameters(url, formData) const response: FileInfo[] = await api.post(getParameters) return response; +} + +export const downloadFile = async (fileName: String) => { + const url = `${config.API_BASE_URL}/v1/file_submissions/${fileName}`; + const getParameters = api.generateApiParameters(url) + const response = await api.get(getParameters) + return response } \ No newline at end of file diff --git a/frontend/src/pages/Dashboard.tsx b/frontend/src/pages/Dashboard.tsx index 77a8e357..1bfb8a33 100644 --- a/frontend/src/pages/Dashboard.tsx +++ b/frontend/src/pages/Dashboard.tsx @@ -1,17 +1,17 @@ -import apiService from '@/service/api-service' -import Button from '@mui/material/Button' -import Dialog from '@mui/material/Dialog' -import DialogActions from '@mui/material/DialogActions' -import DialogContent from '@mui/material/DialogContent' -import DialogTitle from '@mui/material/DialogTitle' -import Table from '@mui/material/Table' -import TableBody from '@mui/material/TableBody' -import TableCell from '@mui/material/TableCell' -import TableRow from '@mui/material/TableRow' -import { DataGrid, GridToolbar } from '@mui/x-data-grid' -import { useEffect, useState } from 'react' -import { DeleteRounded, Description } from '@mui/icons-material' -import _kc from '@/keycloak' +import apiService from "@/service/api-service"; +import Button from "@mui/material/Button"; +import Dialog from "@mui/material/Dialog"; +import DialogActions from "@mui/material/DialogActions"; +import DialogContent from "@mui/material/DialogContent"; +import DialogTitle from "@mui/material/DialogTitle"; +import Table from "@mui/material/Table"; +import TableBody from "@mui/material/TableBody"; +import TableCell from "@mui/material/TableCell"; +import TableRow from "@mui/material/TableRow"; +import { DataGrid, GridToolbar } from "@mui/x-data-grid"; +import { useEffect, useState } from "react"; +import { DeleteRounded, Description } from "@mui/icons-material"; +import _kc from "@/keycloak"; import { Box, FormControl, @@ -22,25 +22,24 @@ import { Select, TextField, Typography, -} from '@mui/material' -import { FileStatusCode } from '@/types/types' -import { getFileStatusCodes } from '@/common/manage-dropdowns' -import { searchFiles } from '@/common/manage-files' -import userEvent from '@testing-library/user-event' +} from "@mui/material"; +import { FileStatusCode } from "@/types/types"; +import { getFileStatusCodes } from "@/common/manage-dropdowns"; +import { downloadFile, searchFiles } from "@/common/manage-files"; const columns = [ { - field: 'file_name', - headerName: 'File Name', + field: "file_name", + headerName: "File Name", sortable: true, filterable: true, flex: 1.5, renderCell: (params) => ( handleDownload(params.row.file_name, params.row.submission_id) @@ -51,50 +50,50 @@ const columns = [ ), }, { - field: 'submission_date', - headerName: 'Submission Date', + field: "submission_date", + headerName: "Submission Date", sortable: true, filterable: true, flex: 2, }, { - field: 'submitter_user_id', - headerName: 'Submitter Username', + field: "submitter_user_id", + headerName: "Submitter Username", sortable: true, filterable: true, flex: 2, }, { - field: 'submitter_agency_name', - headerName: 'Submitter Agency', + field: "submitter_agency_name", + headerName: "Submitter Agency", sortable: true, filterable: true, flex: 2, }, { - field: 'submission_status_code', - headerName: 'Status', + field: "submission_status_code", + headerName: "Status", sortable: true, filterable: true, flex: 1.5, }, { - field: 'sample_count', - headerName: '# Samples', + field: "sample_count", + headerName: "# Samples", sortable: true, filterable: true, flex: 1, }, { - field: 'results_count', - headerName: '# Results', + field: "results_count", + headerName: "# Results", sortable: true, filterable: true, flex: 1, }, { - field: 'delete', - headerName: 'Delete', + field: "delete", + headerName: "Delete", flex: 0.75, renderCell: (params) => ( ( ), }, -] +]; export default function Dashboard() { const [formData, setFormData] = useState({ - fileName: '', - submissionDateTo: '', - submissionDateFrom: '', - submitterUsername: '', - submitterAgency: '', - fileStatus: '', - }) + fileName: "", + submissionDateTo: "", + submissionDateFrom: "", + submitterUsername: "", + submitterAgency: "", + fileStatus: "", + }); const handleFormInputChange = (key, event) => { setFormData({ ...formData, [key]: event.target.value, - }) - } + }); + }; const [data, setData] = useState({ items: [], totalRows: 0, - }) + }); const [paginationModel, setPaginationModel] = useState({ page: 0, pageSize: 10, - }) + }); const handlePaginationChange = (params) => { setTimeout(() => { if (params.pageSize != paginationModel.pageSize) { - setPaginationModel({ page: 0, pageSize: params.pageSize }) - }else{ - setPaginationModel({...paginationModel, page: params.page }) + setPaginationModel({ page: 0, pageSize: params.pageSize }); + } else { + setPaginationModel({ ...paginationModel, page: params.page }); } - }, 10) - } + }, 10); + }; const handleSearch = async (event) => { if (event != null) { - event.preventDefault() - setPaginationModel({ page: 0, pageSize: 10 }) + event.preventDefault(); + setPaginationModel({ page: 0, pageSize: 10 }); } - const requestData = new FormData() + const requestData = new FormData(); for (var key in formData) { - requestData.append(key, formData[key]) + requestData.append(key, formData[key]); } - requestData.append('page', paginationModel.page) - requestData.append('pageSize', paginationModel.pageSize) + requestData.append("page", paginationModel.page); + requestData.append("pageSize", paginationModel.pageSize); await searchFiles(requestData).then((response) => { - const dataValues = Object.values(response.results) - const totalRecFound = response.count + const dataValues = Object.values(response.results); + const totalRecFound = response.count; setData({ items: dataValues, totalRows: totalRecFound, - }) - }) - } + }); + }); + }; const [submissionStatusCodes, setSubmissionStatusCodes] = useState({ items: [], - }) + }); - const [selectedStatusCode, setSelectedStatusCode] = useState('ALL') + const [selectedStatusCode, setSelectedStatusCode] = useState("ALL"); const [selectedSubmitterUserName, setSelectedSubmitterUserName] = - useState('ALL') + useState("ALL"); const [selectedSubmitterAgencyName, setSelectedSubmitterAgencyName] = - useState('ALL') + useState("ALL"); const handleStatusChange = (event) => { - setSelectedStatusCode(event.target.value) - handleFormInputChange('fileStatus', event) - } + setSelectedStatusCode(event.target.value); + handleFormInputChange("fileStatus", event); + }; const handleUsernameChange = (event) => { - setSelectedSubmitterUserName(event.target.value) - handleFormInputChange('submitterUsername', event) - } + setSelectedSubmitterUserName(event.target.value); + handleFormInputChange("submitterUsername", event); + }; const handleAgencyChange = (event) => { - setSelectedSubmitterAgencyName(event.target.value) - handleFormInputChange('submitterAgency', event) - } + setSelectedSubmitterAgencyName(event.target.value); + handleFormInputChange("submitterAgency", event); + }; useEffect(() => { async function fetchFileStatusCodes() { await getFileStatusCodes().then((response) => { - const newSubmissionCodes = submissionStatusCodes.items + const newSubmissionCodes = submissionStatusCodes.items; Object.keys(response).map((key) => { - newSubmissionCodes[key] = response[key] - }) + newSubmissionCodes[key] = response[key]; + }); setSubmissionStatusCodes({ items: newSubmissionCodes, - }) - }) + }); + }); } - fetchFileStatusCodes() - }, []) + fetchFileStatusCodes(); + }, []); useEffect(() => { if (data.items.length > 0) { - handleSearch(null) + handleSearch(null); } - }, [paginationModel]) + }, [paginationModel]); - const [selectedRow, setSelectedRow] = useState(null) + const [selectedRow, setSelectedRow] = useState(null); const handleClose = () => { - setSelectedRow(null) - } + setSelectedRow(null); + }; return ( <>
@@ -252,14 +251,14 @@ export default function Dashboard() { - +
- + File Name @@ -267,23 +266,23 @@ export default function Dashboard() { id="file-name-input" variant="outlined" size="small" - sx={{ width: '520px' }} + sx={{ width: "520px" }} onChange={(event) => - handleFormInputChange('fileName', event) + handleFormInputChange("fileName", event) } /> - + Submission Date - + From: @@ -293,15 +292,15 @@ export default function Dashboard() { size="small" type="date" onChange={(event) => - handleFormInputChange('submissionDateFrom', event) + handleFormInputChange("submissionDateFrom", event) } /> - + To: @@ -310,15 +309,15 @@ export default function Dashboard() { size="small" type="date" onChange={(event) => - handleFormInputChange('submissionDateTo', event) + handleFormInputChange("submissionDateTo", event) } /> - + Submitting Agency @@ -327,7 +326,7 @@ export default function Dashboard() { name="dropdown-agency" variant="outlined" size="small" - sx={{ width: '515px' }} + sx={{ width: "515px" }} onChange={handleAgencyChange} value={selectedSubmitterAgencyName} > @@ -341,10 +340,10 @@ export default function Dashboard() { - + Submitter Username @@ -353,7 +352,7 @@ export default function Dashboard() { name="dropdown-user" variant="outlined" size="small" - sx={{ width: '515px' }} + sx={{ width: "515px" }} onChange={handleUsernameChange} value={selectedSubmitterUserName} > @@ -367,10 +366,10 @@ export default function Dashboard() { - + Status @@ -379,7 +378,7 @@ export default function Dashboard() { name="dropdown-status" variant="outlined" size="small" - sx={{ width: '515px' }} + sx={{ width: "515px" }} onChange={handleStatusChange} value={selectedStatusCode} > @@ -395,13 +394,13 @@ export default function Dashboard() { {option.submission_status_code} )) - : ''} + : ""} - +
- ) + ); } -function handleDownload(fileName: string, submission_id: string): void { - console.log(fileName) - console.log(submission_id) +async function handleDownload(fileName: string): Promise { + const fileMimeType = getMimeType(fileName); + await downloadFile(fileName).then((response) => { + const fileBuffer = new Uint8Array(response.data); + const blob = new Blob([fileBuffer], { type: fileMimeType }); + const link = document.createElement("a"); + link.href = URL.createObjectURL(blob); + link.download = fileName; + document.body.appendChild(link); // Append link to body + link.click(); // Click the link + document.body.removeChild(link); // Clean up + }); } function handleDelete(fileName: string, submission_id: string): void { - console.log(fileName) - console.log(submission_id) + console.log(fileName); + console.log(submission_id); } function handleMessages(fileName: string, submission_id: string): void { - console.log(fileName) - console.log(submission_id) + console.log(fileName); + console.log(submission_id); +} + +function getMimeType(fileName: string) { + const fileNameParts = fileName.split("."); + const ext = fileNameParts.length > 1 ? fileNameParts.pop() || "" : ""; + + switch (ext) { + case "xlsx": + return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; + case "csv": + return "text/csv"; + case "txt": + return "text/plain"; + default: + return "application/octet-stream"; + } } diff --git a/frontend/src/pages/FileUpload.tsx b/frontend/src/pages/FileUpload.tsx index 63e6a34d..b748cce7 100644 --- a/frontend/src/pages/FileUpload.tsx +++ b/frontend/src/pages/FileUpload.tsx @@ -28,7 +28,7 @@ import { } from "@mui/icons-material"; import "@/index.css"; import { jwtDecode } from "jwt-decode"; -import { getFiles, insertFile, validationRequest } from "@/common/manage-files"; +import { insertFile } from "@/common/manage-files"; import UserService from "@/service/user-service"; const fileTypes = ["xlsx", "csv", "txt"]; From fb35c237457ea9efe2471cf293620b195da22ac3 Mon Sep 17 00:00:00 2001 From: vmanawat Date: Tue, 24 Sep 2024 11:55:40 -0700 Subject: [PATCH 2/9] New service to allow users to download file logs --- backend/src/app.module.ts | 4 +- .../dto/create-file_error_log.dto.ts | 9 +++ .../dto/file_error_logs.dto.ts | 23 +++++++ .../dto/update-file_error_log.dto.ts | 3 + .../entities/file_error_log.entity.ts | 1 + .../file_error_logs.controller.spec.ts | 20 ++++++ .../file_error_logs.controller.ts | 55 +++++++++++++++ .../file_error_logs/file_error_logs.module.ts | 9 +++ .../file_error_logs.service.spec.ts | 18 +++++ .../file_error_logs.service.ts | 38 +++++++++++ frontend/src/common/manage-files.ts | 68 +++++++++++-------- frontend/src/pages/Dashboard.tsx | 10 +-- 12 files changed, 222 insertions(+), 36 deletions(-) create mode 100644 backend/src/file_error_logs/dto/create-file_error_log.dto.ts create mode 100644 backend/src/file_error_logs/dto/file_error_logs.dto.ts create mode 100644 backend/src/file_error_logs/dto/update-file_error_log.dto.ts create mode 100644 backend/src/file_error_logs/entities/file_error_log.entity.ts create mode 100644 backend/src/file_error_logs/file_error_logs.controller.spec.ts create mode 100644 backend/src/file_error_logs/file_error_logs.controller.ts create mode 100644 backend/src/file_error_logs/file_error_logs.module.ts create mode 100644 backend/src/file_error_logs/file_error_logs.service.spec.ts create mode 100644 backend/src/file_error_logs/file_error_logs.service.ts diff --git a/backend/src/app.module.ts b/backend/src/app.module.ts index f774f990..f44ebc9b 100644 --- a/backend/src/app.module.ts +++ b/backend/src/app.module.ts @@ -25,6 +25,7 @@ import { FileParseValidateModule } from "./file_parse_and_validation/file_parse_ import { FtpModule } from "./ftp/ftp.module"; import { FileValidationModule } from './file_validation/file_validation.module'; import { ObjectStoreModule } from "./objectStore/objectStore.module"; +import { FileErrorLogsModule } from './file_error_logs/file_error_logs.module'; const DB_HOST = process.env.POSTGRES_HOST || "localhost"; const DB_USER = process.env.POSTGRES_USER || "postgres"; @@ -72,7 +73,8 @@ function getMiddlewares() { AqiApiModule, FtpModule, FileValidationModule, - ObjectStoreModule + ObjectStoreModule, + FileErrorLogsModule ], controllers: [AppController, MetricsController, HealthController], providers: [AppService, CronJobService], diff --git a/backend/src/file_error_logs/dto/create-file_error_log.dto.ts b/backend/src/file_error_logs/dto/create-file_error_log.dto.ts new file mode 100644 index 00000000..4d3a5e05 --- /dev/null +++ b/backend/src/file_error_logs/dto/create-file_error_log.dto.ts @@ -0,0 +1,9 @@ +import { PickType } from "@nestjs/swagger"; +import { FileErrorLogDto } from "./file_error_logs.dto"; + +export class CreateFileErrorLogDto extends PickType(FileErrorLogDto, [ + 'file_error_log_id', + 'file_submission_id', + 'file_name', + 'error_log' +] as const) {} diff --git a/backend/src/file_error_logs/dto/file_error_logs.dto.ts b/backend/src/file_error_logs/dto/file_error_logs.dto.ts new file mode 100644 index 00000000..4ac99405 --- /dev/null +++ b/backend/src/file_error_logs/dto/file_error_logs.dto.ts @@ -0,0 +1,23 @@ +import { ApiProperty } from "@nestjs/swagger"; + +export class FileErrorLogDto { + @ApiProperty({ + description: "File error log ID", + }) + file_error_log_id: string; + + @ApiProperty({ + description: "File submission ID", + }) + file_submission_id: string; + + @ApiProperty({ + description: "File name", + }) + file_name: string; + + @ApiProperty({ + description: "Error log data", + }) + error_log: string; +} diff --git a/backend/src/file_error_logs/dto/update-file_error_log.dto.ts b/backend/src/file_error_logs/dto/update-file_error_log.dto.ts new file mode 100644 index 00000000..2f1139bf --- /dev/null +++ b/backend/src/file_error_logs/dto/update-file_error_log.dto.ts @@ -0,0 +1,3 @@ +import { CreateFileErrorLogDto } from './create-file_error_log.dto'; + +export class UpdateFileErrorLogDto extends (CreateFileErrorLogDto) {} diff --git a/backend/src/file_error_logs/entities/file_error_log.entity.ts b/backend/src/file_error_logs/entities/file_error_log.entity.ts new file mode 100644 index 00000000..5fdd6c80 --- /dev/null +++ b/backend/src/file_error_logs/entities/file_error_log.entity.ts @@ -0,0 +1 @@ +export class FileErrorLog {} diff --git a/backend/src/file_error_logs/file_error_logs.controller.spec.ts b/backend/src/file_error_logs/file_error_logs.controller.spec.ts new file mode 100644 index 00000000..5440f226 --- /dev/null +++ b/backend/src/file_error_logs/file_error_logs.controller.spec.ts @@ -0,0 +1,20 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { FileErrorLogsController } from './file_error_logs.controller'; +import { FileErrorLogsService } from './file_error_logs.service'; + +describe('FileErrorLogsController', () => { + let controller: FileErrorLogsController; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + controllers: [FileErrorLogsController], + providers: [FileErrorLogsService], + }).compile(); + + controller = module.get(FileErrorLogsController); + }); + + it('should be defined', () => { + expect(controller).toBeDefined(); + }); +}); diff --git a/backend/src/file_error_logs/file_error_logs.controller.ts b/backend/src/file_error_logs/file_error_logs.controller.ts new file mode 100644 index 00000000..1c5efeee --- /dev/null +++ b/backend/src/file_error_logs/file_error_logs.controller.ts @@ -0,0 +1,55 @@ +import { + Controller, + Get, + Post, + Body, + Patch, + Param, + Delete, + UseGuards, +} from "@nestjs/common"; +import { FileErrorLogsService } from "./file_error_logs.service"; +import { CreateFileErrorLogDto } from "./dto/create-file_error_log.dto"; +import { UpdateFileErrorLogDto } from "./dto/update-file_error_log.dto"; +import { ApiTags } from "@nestjs/swagger"; +import { JwtRoleGuard } from "src/auth/jwtrole.guard"; +import { JwtAuthGuard } from "src/auth/jwtauth.guard"; +import { Roles } from "src/auth/decorators/roles.decorators"; +import { Role } from "src/enum/role.enum"; + +@ApiTags("file_error_logs") +@Controller({ path: "file_error_logs", version: "1" }) +@UseGuards(JwtAuthGuard) +@UseGuards(JwtRoleGuard) +@Roles(Role.ENMODS_ADMIN) +export class FileErrorLogsController { + constructor(private readonly fileErrorLogsService: FileErrorLogsService) {} + + @Post() + create(@Body() createFileErrorLogDto: CreateFileErrorLogDto) { + return this.fileErrorLogsService.create(createFileErrorLogDto); + } + + @Get() + findAll() { + return this.fileErrorLogsService.findAll(); + } + + @Get(":file_submission_id") + findOne(@Param("file_submission_id") id: string) { + return this.fileErrorLogsService.findOne(id); + } + + @Patch(":id") + update( + @Param("id") id: string, + @Body() updateFileErrorLogDto: UpdateFileErrorLogDto, + ) { + return this.fileErrorLogsService.update(+id, updateFileErrorLogDto); + } + + @Delete(":id") + remove(@Param("id") id: string) { + return this.fileErrorLogsService.remove(+id); + } +} diff --git a/backend/src/file_error_logs/file_error_logs.module.ts b/backend/src/file_error_logs/file_error_logs.module.ts new file mode 100644 index 00000000..510f3f47 --- /dev/null +++ b/backend/src/file_error_logs/file_error_logs.module.ts @@ -0,0 +1,9 @@ +import { Module } from '@nestjs/common'; +import { FileErrorLogsService } from './file_error_logs.service'; +import { FileErrorLogsController } from './file_error_logs.controller'; + +@Module({ + controllers: [FileErrorLogsController], + providers: [FileErrorLogsService], +}) +export class FileErrorLogsModule {} diff --git a/backend/src/file_error_logs/file_error_logs.service.spec.ts b/backend/src/file_error_logs/file_error_logs.service.spec.ts new file mode 100644 index 00000000..28595789 --- /dev/null +++ b/backend/src/file_error_logs/file_error_logs.service.spec.ts @@ -0,0 +1,18 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { FileErrorLogsService } from './file_error_logs.service'; + +describe('FileErrorLogsService', () => { + let service: FileErrorLogsService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [FileErrorLogsService], + }).compile(); + + service = module.get(FileErrorLogsService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/backend/src/file_error_logs/file_error_logs.service.ts b/backend/src/file_error_logs/file_error_logs.service.ts new file mode 100644 index 00000000..ddfa0c5d --- /dev/null +++ b/backend/src/file_error_logs/file_error_logs.service.ts @@ -0,0 +1,38 @@ +import { Injectable } from "@nestjs/common"; +import { CreateFileErrorLogDto } from "./dto/create-file_error_log.dto"; +import { UpdateFileErrorLogDto } from "./dto/update-file_error_log.dto"; +import { PrismaService } from "nestjs-prisma"; + +@Injectable() +export class FileErrorLogsService { + constructor(private prisma: PrismaService) {} + + create(createFileErrorLogDto: CreateFileErrorLogDto) { + return "This action adds a new fileErrorLog"; + } + + findAll() { + return `This action returns all fileErrorLogs`; + } + + async findOne(file_submission_id: string) { + const fileLogs = await this.prisma.file_error_logs.findMany({ + where: { + file_submission_id: file_submission_id, + }, + select: { + error_log: true + } + }) + + console.log(fileLogs[0].error_log) + } + + update(id: number, updateFileErrorLogDto: UpdateFileErrorLogDto) { + return `This action updates a #${id} fileErrorLog`; + } + + remove(id: number) { + return `This action removes a #${id} fileErrorLog`; + } +} diff --git a/frontend/src/common/manage-files.ts b/frontend/src/common/manage-files.ts index a25205c2..6c2d8087 100644 --- a/frontend/src/common/manage-files.ts +++ b/frontend/src/common/manage-files.ts @@ -1,39 +1,47 @@ -import { FileInfo } from '@/types/types'; -import * as api from './api'; -import config from '@/config'; +import { FileInfo } from "@/types/types"; +import * as api from "./api"; +import config from "@/config"; export const insertFile = async (formData: FormData): Promise => { - const url = `${config.API_BASE_URL}/v1/file_submissions`; - const postParameters = api.generateApiParameters(url, formData) - const fileID: String = await api.post(postParameters) - return fileID; + const url = `${config.API_BASE_URL}/v1/file_submissions`; + const postParameters = api.generateApiParameters(url, formData); + const fileID: String = await api.post(postParameters); + return fileID; +}; -} - -export const validationRequest = async (submission_id: String): Promise => { - const url = `${config.API_BASE_URL}/v1/file_submissions/${submission_id}`; - const getParameters = api.generateApiParameters(url) - const response: String = await api.get(getParameters) - return response; -} +export const validationRequest = async ( + submission_id: String, +): Promise => { + const url = `${config.API_BASE_URL}/v1/file_submissions/${submission_id}`; + const getParameters = api.generateApiParameters(url); + const response: String = await api.get(getParameters); + return response; +}; export const getFiles = async (substring: String): Promise => { - const url = `${config.API_BASE_URL}/v1/file_submissions/${substring}`; - const getParameters = api.generateApiParameters(url) - const response: FileInfo = await api.get(getParameters) - return response; -} + const url = `${config.API_BASE_URL}/v1/file_submissions/${substring}`; + const getParameters = api.generateApiParameters(url); + const response: FileInfo = await api.get(getParameters); + return response; +}; export const searchFiles = async (formData: FormData): Promise => { - const url = `${config.API_BASE_URL}/v1/file_submissions/search`; - const getParameters = api.generateApiParameters(url, formData) - const response: FileInfo[] = await api.post(getParameters) - return response; -} + const url = `${config.API_BASE_URL}/v1/file_submissions/search`; + const getParameters = api.generateApiParameters(url, formData); + const response: FileInfo[] = await api.post(getParameters); + return response; +}; export const downloadFile = async (fileName: String) => { - const url = `${config.API_BASE_URL}/v1/file_submissions/${fileName}`; - const getParameters = api.generateApiParameters(url) - const response = await api.get(getParameters) - return response -} \ No newline at end of file + const url = `${config.API_BASE_URL}/v1/file_submissions/${fileName}`; + const getParameters = api.generateApiParameters(url); + const response = await api.get(getParameters); + return response; +}; + +export const downloadFileLogs = async (fileID: String) => { + const url = `${config.API_BASE_URL}/v1/file_error_logs/${fileID}`; + const getParameters = api.generateApiParameters(url); + const response = await api.get(getParameters); + return response; +}; diff --git a/frontend/src/pages/Dashboard.tsx b/frontend/src/pages/Dashboard.tsx index 1bfb8a33..dab2e3d4 100644 --- a/frontend/src/pages/Dashboard.tsx +++ b/frontend/src/pages/Dashboard.tsx @@ -25,7 +25,7 @@ import { } from "@mui/material"; import { FileStatusCode } from "@/types/types"; import { getFileStatusCodes } from "@/common/manage-dropdowns"; -import { downloadFile, searchFiles } from "@/common/manage-files"; +import { downloadFile, downloadFileLogs, searchFiles } from "@/common/manage-files"; const columns = [ { @@ -481,12 +481,12 @@ async function handleDownload(fileName: string): Promise { }); } -function handleDelete(fileName: string, submission_id: string): void { - console.log(fileName); - console.log(submission_id); +async function handleMessages(fileName: string, submission_id: string): Promise { + + await downloadFileLogs(submission_id) } -function handleMessages(fileName: string, submission_id: string): void { +function handleDelete(fileName: string, submission_id: string): void { console.log(fileName); console.log(submission_id); } From b3640233328228c3c39fd1f86521fa67f6c0a57e Mon Sep 17 00:00:00 2001 From: vmanawat Date: Tue, 24 Sep 2024 14:00:36 -0700 Subject: [PATCH 3/9] some UI changes and parsing the errorlogs --- .../file_error_logs.service.ts | 22 +++++++++++++++---- frontend/src/App.tsx | 6 ++--- frontend/src/components/Header.tsx | 2 +- frontend/src/pages/Dashboard.tsx | 14 ++++++------ 4 files changed, 28 insertions(+), 16 deletions(-) diff --git a/backend/src/file_error_logs/file_error_logs.service.ts b/backend/src/file_error_logs/file_error_logs.service.ts index ddfa0c5d..699aaa17 100644 --- a/backend/src/file_error_logs/file_error_logs.service.ts +++ b/backend/src/file_error_logs/file_error_logs.service.ts @@ -21,11 +21,11 @@ export class FileErrorLogsService { file_submission_id: file_submission_id, }, select: { - error_log: true - } - }) + error_log: true, + }, + }); - console.log(fileLogs[0].error_log) + formulateErrorFile(fileLogs[0].error_log); } update(id: number, updateFileErrorLogDto: UpdateFileErrorLogDto) { @@ -36,3 +36,17 @@ export class FileErrorLogsService { return `This action removes a #${id} fileErrorLog`; } } + +function formulateErrorFile(logs: any) { + let formattedMessages = ""; + + logs.forEach((log) => { + const rowNum = log.rowNum; + + for (const [key, msg] of Object.entries(log.message)) { + formattedMessages += `Row ${rowNum}: ${key} - ${msg}\n`; + } + }); + + console.log(formattedMessages); +} diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index bcf4219d..7bb84d80 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -20,15 +20,13 @@ const styles = { }, content: { display: 'flex', - width: '1200px', + width: '1800px', bgcolor: '#ffffff', }, sidebar: { paddingTop: '8em', - paddingLeft: '2em', - // width: '28%', + paddingLeft: '0.5em', width: '20%', - // bgcolor: '#efefff', }, mainContent: { marginTop: '8em', diff --git a/frontend/src/components/Header.tsx b/frontend/src/components/Header.tsx index e23afde5..6587ce76 100644 --- a/frontend/src/components/Header.tsx +++ b/frontend/src/components/Header.tsx @@ -19,7 +19,7 @@ const styles = { alignItems: 'center', }, innerContent: { - maxWidth: '1200px', + maxWidth: '1800px', width: '100%', margin: '0 auto', display: 'flex', diff --git a/frontend/src/pages/Dashboard.tsx b/frontend/src/pages/Dashboard.tsx index dab2e3d4..3e255c90 100644 --- a/frontend/src/pages/Dashboard.tsx +++ b/frontend/src/pages/Dashboard.tsx @@ -266,20 +266,20 @@ export default function Dashboard() { id="file-name-input" variant="outlined" size="small" - sx={{ width: "520px" }} + sx={{ width: "650px" }} onChange={(event) => handleFormInputChange("fileName", event) } /> - + Submission Date - + @@ -352,7 +352,7 @@ export default function Dashboard() { name="dropdown-user" variant="outlined" size="small" - sx={{ width: "515px" }} + sx={{ width: "645px" }} onChange={handleUsernameChange} value={selectedSubmitterUserName} > @@ -378,7 +378,7 @@ export default function Dashboard() { name="dropdown-status" variant="outlined" size="small" - sx={{ width: "515px" }} + sx={{ width: "645px" }} onChange={handleStatusChange} value={selectedStatusCode} > @@ -400,7 +400,7 @@ export default function Dashboard() { - +