Add max file size constraint per user

.env.example

@@ -10,6 +10,7 @@ HOST_SERVER_IP=localhost
 HOST_SERVER_PORT=5000
 HOST_SERVER_PROTOCOL=http
 DB_FORCE_TABLES_RECREATION=false
+RESERVED_DISK_SPACE_BYTES_PER_USER=1e8
 NODE_ENV=development
 SECRET_KEY=SECRET
 POSTFIX_ENABLED=false

docker-compose.yml

@@ -77,6 +77,7 @@ services:
       DB_HOST: postgres
       DB_PORT: 5432
       DB_FORCE_TABLES_RECREATION: ${DB_FORCE_TABLES_RECREATION}
+      RESERVED_DISK_SPACE_BYTES_PER_USER: ${RESERVED_DISK_SPACE_BYTES_PER_USER}
       NODE_ENV: ${NODE_ENV}
       HOST_SERVER_PORT: "${HOST_SERVER_PORT}"
       HOST_SERVER_IP: "${HOST_SERVER_IP}"

web-app/server/src/config/index.ts

@@ -32,7 +32,9 @@ export const config = {
         fileConfig: {
             allowedFileFormats: ["text/csv", "application/vnd.ms-excel"],
             allowedDelimiters: [",", "|", ";"],
-            maxFileSize: 1e10,
+            reservedDiskSpacePerUser: Number(
+                process.env.RESERVED_DISK_SPACE_BYTES_PER_USER || 50 * 1e3
+            ),
         },
         maxThreadsCount: Number(process.env.MAX_THREADS_COUNT || "4"),
     },

web-app/server/src/db/models/FileData/FileInfo.ts

@@ -1,4 +1,13 @@
-import sequelize, { BOOLEAN, DATE, INTEGER, STRING, TEXT, UUID, UUIDV4 } from "sequelize";
+import sequelize, {
+    BOOLEAN,
+    DATE,
+    INTEGER,
+    Op,
+    STRING,
+    TEXT,
+    UUID,
+    UUIDV4,
+} from "sequelize";
 import {
     BelongsTo,
     Column,
@@ -14,6 +23,7 @@ import {
     builtInDatasets,
     getPathToBuiltInDataset,
 } from "../../initBuiltInDatasets";
+import { v4 as uuidv4 } from "uuid";
 import { CsvParserStream, parse } from "fast-csv";
 import { FileProps, Column as SchemaColumn } from "../../../graphql/types/types";
 import {
@@ -23,12 +33,14 @@ import {
     findRowsAndColumnsNumber,
 } from "../../../graphql/schema/TaskCreating/csvValidator";
 import { ApolloError } from "apollo-server-core";
+import { FileUpload } from "graphql-upload";
 import { FileFormat } from "./FileFormat";
-import { GeneralTaskConfig } from "../TaskData/configs/GeneralTaskConfig";
+import { GeneralTaskConfig, mainPrimitives } from "../TaskData/configs/GeneralTaskConfig";
 import { Row } from "@fast-csv/parse";
 import { User } from "../UserData/User";
 import config from "../../../config";
-import { finished } from "stream/promises";
+import { FileSizeLimiter } from "./streamDataLimiter";
+import { pipeline } from "stream/promises";
 import fs from "fs";
 import { generateHeaderByPath } from "../../../graphql/schema/TaskCreating/generateHeader";
 import path from "path";
@@ -81,6 +93,9 @@ export class FileInfo extends Model implements FileInfoModelMethods {
     @Column({ type: STRING, allowNull: true })
     mimeType!: string | null;
 
+    @Column({ type: INTEGER, allowNull: false })
+    fileSize!: number;
+
     @Column({ type: STRING })
     encoding!: string | null;
 
@@ -105,9 +120,23 @@ export class FileInfo extends Model implements FileInfoModelMethods {
     @Column({ type: INTEGER, allowNull: true })
     countOfColumns?: number;
 
+    @Column({ type: INTEGER, defaultValue: 0, allowNull: false })
+    numberOfUses!: number;
+
     @Column({ type: STRING, unique: true })
     path!: string;
 
+    recomputeNumberOfUses = async () => {
+        const numberOfUses = await GeneralTaskConfig.count({
+            where: {
+                type: { [Op.in]: mainPrimitives },
+                fileID: this.fileID,
+            },
+        });
+
+        return this.update({ numberOfUses });
+    };
+
     static getPathToMainFile = () => {
         if (!require.main) {
             throw new ApolloError("Cannot find main");
@@ -162,6 +191,7 @@ export class FileInfo extends Model implements FileInfoModelMethods {
         const dbPath = getPathToBuiltInDataset(props.fileName);
         const path = FileInfo.resolvePath(dbPath, isBuiltIn);
         const { hasHeader, delimiter } = datasetProps;
+        const { size: fileSize } = await fs.promises.stat(path);
         const header = await generateHeaderByPath(path, hasHeader, delimiter);
         const renamedHeader = JSON.stringify(header);
 
@@ -173,6 +203,7 @@ export class FileInfo extends Model implements FileInfoModelMethods {
                 renamedHeader,
                 hasHeader,
                 delimiter,
+                fileSize,
             },
         });
         console.log(
@@ -192,7 +223,7 @@ export class FileInfo extends Model implements FileInfoModelMethods {
 
     static uploadDataset = async (
         datasetProps: FileProps,
-        table: any,
+        table: Promise<FileUpload>,
         userID: string | null = null
     ) => {
         const isBuiltIn = false;
@@ -204,29 +235,48 @@ export class FileInfo extends Model implements FileInfoModelMethods {
         } = await table;
 
         const stream = createReadStream();
+        const fileID = uuidv4();
+        const fileName = `${fileID}.csv`;
+        const newFilePath = `${
+            (!config.inContainer && "../../volumes/") || ""
+        }uploads/${fileName}`;
+        const out = fs.createWriteStream(newFilePath);
+
+        const user = userID === null ? null : await User.findByPk(userID);
+        const maxFileSize = user?.remainingDiskSpace ?? Infinity;
+
+        const fileSizeLimiter = new FileSizeLimiter(maxFileSize);
+        fileSizeLimiter.on("error", (error) => {
+            stream.destroy(error);
+            out.destroy(error);
+        });
+
+        await pipeline(stream, fileSizeLimiter, out);
+
+        const path = FileInfo.getPathToUploadedDataset(fileName);
+        const { size: fileSize } = await fs.promises.stat(newFilePath);
+
         const file = await FileInfo.create({
             ...datasetProps,
+            fileID,
             encoding,
             mimeType,
+            fileName,
             originalFileName,
+            path,
             userID,
             isValid: false,
+            fileSize,
         });
 
-        const { fileID } = file;
-        const fileName = `${fileID}.csv`;
-        const path = FileInfo.getPathToUploadedDataset(fileName);
-        await file.update({ fileName, path });
-
-        const out = fs.createWriteStream(
-            `${(!config.inContainer && "../../volumes/") || ""}uploads/${fileName}`
-        );
-        stream.pipe(out);
-        await finished(out);
         await file.update({
             renamedHeader: JSON.stringify(await file.generateHeader()),
         });
 
+        if (user) {
+            await user.recomputeRemainingDiskSpace();
+        }
+
         if (datasetProps.inputFormat) {
             const fileFormat = await FileFormat.createFileFormatIfPropsValid(
                 file,

web-app/server/src/db/models/FileData/streamDataLimiter.ts

@@ -0,0 +1,32 @@
+import { UserInputError } from "apollo-server-core";
+import { Transform, TransformCallback } from "stream";
+
+export class FileSizeLimiter extends Transform {
+    maxSize: number;
+    currentSize = 0;
+
+    constructor(maxSize: number) {
+        super();
+        this.maxSize = maxSize;
+    }
+
+    override _transform(
+        chunk: any,
+        encoding: BufferEncoding,
+        callback: TransformCallback
+    ) {
+        this.currentSize += chunk.length;
+
+        if (this.currentSize > this.maxSize) {
+            this.destroy(
+                new UserInputError(
+                    `Remaining file size of ${this.maxSize} bytes exceeded`
+                )
+            );
+            return;
+        }
+
+        this.push(chunk);
+        callback();
+    }
+}

web-app/server/src/db/models/UserData/User.ts

@@ -1,18 +1,12 @@
-import {
-    Column,
-    HasMany,
-    IsEmail,
-    IsUUID,
-    Model,
-    Table,
-} from "sequelize-typescript";
+import { Column, HasMany, IsEmail, IsUUID, Model, Table } from "sequelize-typescript";
 import { Role, RoleType } from "./Role";
-import { STRING, UUID, UUIDV4 } from "sequelize";
+import { INTEGER, STRING, UUID, UUIDV4, VIRTUAL } from "sequelize";
 import { Session, SessionStatusType } from "./Session";
 import { Feedback } from "./Feedback";
 import { FileInfo } from "../FileData/FileInfo";
 import { Permission } from "./Permission";
 import { TaskState } from "../TaskData/TaskState";
+import config from "../../../config";
 
 const ALL_ACCOUNT_STATUS = ["EMAIL_VERIFICATION", "EMAIL_VERIFIED"] as const;
 export type AccountStatusType = typeof ALL_ACCOUNT_STATUS[number];
@@ -26,6 +20,8 @@ interface UserModelMethods {
     createSession: (deviceID: string) => Promise<Session>;
 }
 
+const defaultReservedDiskSpace = config.appConfig.fileConfig.reservedDiskSpacePerUser;
+
 @Table({
     tableName: "Users",
     updatedAt: false,
@@ -48,6 +44,20 @@ export class User extends Model implements UserModelMethods {
     @HasMany(() => FileInfo)
     files?: [FileInfo];
 
+    @Column({
+        type: INTEGER,
+        allowNull: false,
+        defaultValue: defaultReservedDiskSpace,
+    })
+    reservedDiskSpace!: number;
+
+    @Column({
+        type: INTEGER,
+        allowNull: false,
+        defaultValue: defaultReservedDiskSpace,
+    })
+    remainingDiskSpace!: number;
+
     @HasMany(() => Role)
     roles?: [Role];
 
@@ -97,9 +107,7 @@ export class User extends Model implements UserModelMethods {
         if (~roleIdx) {
             return roles[roleIdx];
         }
-        const permissionIndices = JSON.stringify(
-            Role.getPermissionIndicesForRole(role)
-        );
+        const permissionIndices = JSON.stringify(Role.getPermissionIndicesForRole(role));
         return (await this.$create("role", {
             type: role,
             permissionIndices,
@@ -113,4 +121,18 @@ export class User extends Model implements UserModelMethods {
         const status: SessionStatusType = "VALID";
         return await Session.create({ userID: this.userID, status, deviceID });
     };
+
+    recomputeRemainingDiskSpace = async () => {
+        const files: FileInfo[] | null = await this.$get("files");
+
+        let occupiedSpace = 0;
+
+        if (files) {
+            occupiedSpace = files.reduce((acc, curr) => acc + curr.fileSize, 0);
+        }
+
+        return this.update({
+            remainingDiskSpace: this.reservedDiskSpace - occupiedSpace,
+        });
+    };
 }

web-app/server/src/graphql/schema/TaskCreating/Creator/AbstractCreator.ts

@@ -75,9 +75,13 @@ export abstract class AbstractCreator<
         protected readonly type: DBTaskPrimitiveType,
         protected context: Context,
         protected forceCreate: boolean,
-        protected fileInfo: FileInfo,
-        protected userID?: string
-    ) {}
+        protected fileInfo: FileInfo
+    ) {
+        this.type = type;
+        this.context = context;
+        this.forceCreate = forceCreate;
+        this.fileInfo = fileInfo;
+    }
 
     protected models = () => this.context.models;
     protected logger = this.context.logger;
@@ -155,7 +159,7 @@ export abstract class AbstractCreator<
 
     private saveToDB = async () => {
         const status: TaskStatusType = "ADDING_TO_DB";
-        const userID = this.userID;
+        const userID = this.context.sessionInfo?.userID;
         const { fileID } = this.fileInfo;
         const taskState = await this.context.models.TaskState.create({
             status,
@@ -190,6 +194,7 @@ export abstract class AbstractCreator<
         const task = await this.saveToDB();
         const { taskID } = task;
 
+        await this.fileInfo.recomputeNumberOfUses();
         await produce({ taskID }, this.context.topicNames.tasks);
         task.status = "ADDED_TO_THE_TASK_QUEUE";
         await task.save();

web-app/server/src/graphql/schema/TaskInfo/resolvers.ts

@@ -621,14 +621,6 @@ export const TaskInfoResolvers: Resolvers = {
             }
             return fileInfo.getColumnNames();
         },
-        numberOfUses: async ({ fileID }, _, { models }) => {
-            return await models.GeneralTaskConfig.count({
-                where: {
-                    type: { [Op.in]: mainPrimitives },
-                    fileID,
-                },
-            });
-        },
     },
     Query: {
         datasetInfo: async (_, { fileID }, { models, sessionInfo }) => {