feat: add all database models, pydantic models for tasks, fixtures and tests for database

.env.example

@@ -1,4 +1,4 @@
-POSTGRES_DIALECT_DRIVER=postgresql
+POSTGRES_DIALECT_DRIVER=postgresql+psycopg
 
 POSTGRES_USER=admin
 POSTGRES_PASSWORD=admin

.github/workflows/run-linter-and-tests.yaml

@@ -24,5 +24,8 @@ jobs:
       - name: Run all linters and formatters
         run: make lint
 
+      - name: Up all containers
+        run: docker compose -f dev-docker-compose.yaml up --build --force-recreate --remove-orphans -d
+
       - name: Run all tests and count coverage
         run: make test

Makefile

@@ -1,4 +1,4 @@
-.PHONY: env install-deps up open-db revision migrate worker app init lint test
+.PHONY: env install-deps up open-db revision migrate downgrade worker app init lint test
 
 ifeq ($(shell test -e '.env' && echo -n yes), yes)
 	include .env
@@ -20,7 +20,7 @@ install-deps:
 ## Up development-only docker containers
 up:
 	(trap 'docker compose -f dev-docker-compose.yaml down' INT; \
-	docker compose -f dev-docker-compose.yaml up --build --force-recreate --remove-orphans)
+	docker compose -f dev-docker-compose.yaml up --build --force-recreate --remove-orphans $(args))
 
 ## Open database with docker-compose
 open-db:
@@ -34,6 +34,10 @@ revision:
 migrate:
 	poetry run alembic -c app/settings/alembic.ini upgrade $(args)
 
+## Downgrade database
+downgrade:
+	poetry run alembic -c app/settings/alembic.ini downgrade $(args)
+
 ## Run celery worker in watch mode
 worker:
 	. .venv/bin/activate && watchmedo auto-restart --directory=./ --pattern='*.py' --recursive -- celery -A app.worker worker --loglevel=info --concurrency=1

app/db/migrations/env.py

@@ -1,10 +1,18 @@
 from logging.config import fileConfig
-
 from sqlalchemy import engine_from_config, pool
 from alembic import context
-
 from app.settings import get_settings
 from app.db import Base
+from app.db.models.user.code import Code
+from app.db.models.user.device import Device
+from app.db.models.user.feedback import Feedback
+from app.db.models.user.permission import Permission
+from app.db.models.user.role import Role
+from app.db.models.user.session import Session
+from app.db.models.user.user import User
+from app.db.models.file.file_info import FileInfo
+from app.db.models.file.file_format import FileFormat
+from app.tasks.models.task import Task
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.

app/db/migrations/versions/03c0f0f4b98e_first_empty_migration.py

@@ -0,0 +1,30 @@
+"""First empty migration
+
+Revision ID: 03c0f0f4b98e
+Revises:
+Create Date: 2023-12-22 15:55:23.335330
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = "03c0f0f4b98e"
+down_revision: Union[str, None] = None
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

app/db/migrations/versions/399d81bfa440_defining_the_database_structure.py

@@ -0,0 +1,268 @@
+"""Defining the Database Structure
+
+Revision ID: 399d81bfa440
+Revises: 03c0f0f4b98e
+Create Date: 2023-12-22 15:57:31.889415
+
+"""
+from typing import Sequence, Union
+import fastapi_users_db_sqlalchemy
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = "399d81bfa440"
+down_revision: Union[str, None] = "03c0f0f4b98e"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "Device",
+        sa.Column("id", sa.String(), nullable=False),
+        sa.Column("user_agent", sa.String(), nullable=True),
+        sa.Column("browser", sa.String(), nullable=True),
+        sa.Column("engine", sa.String(), nullable=True),
+        sa.Column("os", sa.String(), nullable=True),
+        sa.Column("os_version", sa.String(), nullable=True),
+        sa.Column("device", sa.String(), nullable=True),
+        sa.Column("cpu", sa.String(), nullable=True),
+        sa.Column("screen", sa.String(), nullable=True),
+        sa.Column("plugins", sa.String(), nullable=True),
+        sa.Column("time_zone", sa.String(), nullable=True),
+        sa.Column("language", sa.String(), nullable=True),
+        sa.Column("created_at", sa.DateTime(), nullable=False),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(op.f("ix_Device_id"), "Device", ["id"], unique=False)
+    op.create_table(
+        "Permission",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column(
+            "permission",
+            sa.Enum(
+                "CAN_USE_BUILTIN_DATASETS",
+                "CAN_USE_OWN_DATASETS",
+                "CAN_USE_USERS_DATASETS",
+                "CAN_VIEW_ADMIN_INFO",
+                "CAN_MANAGE_USERS_SESSIONS",
+                "CAN_MANAGE_APP_CONFIG",
+                name="allpermissions",
+            ),
+            nullable=False,
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "User",
+        sa.Column("full_name", sa.String(), nullable=False),
+        sa.Column("country", sa.String(), nullable=False),
+        sa.Column("company_or_affiliation", sa.String(), nullable=False),
+        sa.Column("occupation", sa.String(), nullable=False),
+        sa.Column(
+            "account_status",
+            sa.Enum("EMAIL_VERIFICATION", "EMAIl_VERIFIED", name="accountstatustype"),
+            nullable=False,
+        ),
+        sa.Column("created_at", sa.DateTime(), nullable=False),
+        sa.Column("deleted_at", sa.DateTime(), nullable=True),
+        sa.Column("id", fastapi_users_db_sqlalchemy.generics.GUID(), nullable=False),
+        sa.Column("email", sa.String(length=320), nullable=False),
+        sa.Column("hashed_password", sa.String(length=1024), nullable=False),
+        sa.Column("is_active", sa.Boolean(), nullable=False),
+        sa.Column("is_superuser", sa.Boolean(), nullable=False),
+        sa.Column("is_verified", sa.Boolean(), nullable=False),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(op.f("ix_User_email"), "User", ["email"], unique=True)
+    op.create_table(
+        "Code",
+        sa.Column("id", sa.Uuid(), nullable=False),
+        sa.Column(
+            "type",
+            sa.Enum(
+                "EMAIL_VERIFICATION_REQUIRED",
+                "PASSWORD_RECOVERY_PENDING",
+                "PASSWORD_RECOVERY_APPROVED",
+                name="codetype",
+            ),
+            nullable=False,
+        ),
+        sa.Column("value", sa.Integer(), nullable=False),
+        sa.Column("expiring_date", sa.DateTime(), nullable=False),
+        sa.Column(
+            "user_id", fastapi_users_db_sqlalchemy.generics.GUID(), nullable=False
+        ),
+        sa.Column("device_id", sa.String(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["device_id"], ["Device.id"], onupdate="CASCADE", ondelete="SET NULL"
+        ),
+        sa.ForeignKeyConstraint(["user_id"], ["User.id"], onupdate="CASCADE"),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "Feedback",
+        sa.Column("id", sa.Uuid(), nullable=False),
+        sa.Column(
+            "user_id", fastapi_users_db_sqlalchemy.generics.GUID(), nullable=True
+        ),
+        sa.Column("rating", sa.Integer(), nullable=False),
+        sa.Column("subject", sa.String(), nullable=True),
+        sa.Column("text", sa.String(), nullable=False),
+        sa.Column("created_at", sa.DateTime(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["user_id"], ["User.id"], onupdate="CASCADE", ondelete="SET NULL"
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(op.f("ix_Feedback_id"), "Feedback", ["id"], unique=False)
+    op.create_table(
+        "FileInfo",
+        sa.Column("id", sa.Uuid(), nullable=False),
+        sa.Column(
+            "created_at", sa.DateTime(), server_default=sa.text("now()"), nullable=False
+        ),
+        sa.Column(
+            "user_id", fastapi_users_db_sqlalchemy.generics.GUID(), nullable=True
+        ),
+        sa.Column("is_built_in", sa.Boolean(), nullable=False),
+        sa.Column("is_valid", sa.Boolean(), nullable=False),
+        sa.Column("mime_type", sa.String(), nullable=True),
+        sa.Column("encoding", sa.String(), nullable=True),
+        sa.Column("file_name", sa.String(), nullable=False),
+        sa.Column("original_file_name", sa.String(), nullable=False),
+        sa.Column("has_header", sa.Boolean(), nullable=False),
+        sa.Column("delimiter", sa.String(), nullable=False),
+        sa.Column("renamed_header", sa.String(), nullable=False),
+        sa.Column("rows_count", sa.Integer(), nullable=False),
+        sa.Column("count_of_columns", sa.Integer(), nullable=False),
+        sa.Column("path", sa.String(), nullable=False),
+        sa.Column("deleted_at", sa.DateTime(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["user_id"], ["User.id"], onupdate="CASCADE", ondelete="SET NULL"
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("path"),
+    )
+    op.create_table(
+        "Role",
+        sa.Column("id", sa.Uuid(), nullable=False),
+        sa.Column(
+            "user_id", fastapi_users_db_sqlalchemy.generics.GUID(), nullable=False
+        ),
+        sa.Column(
+            "type",
+            sa.Enum(
+                "ANONYMOUS", "USER", "SUPPORT", "ADMIN", "DEVELOPER", name="roletype"
+            ),
+            nullable=False,
+        ),
+        sa.Column("permission_indices", sa.String(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["user_id"], ["User.id"], onupdate="CASCADE", ondelete="CASCADE"
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "Session",
+        sa.Column("id", sa.Uuid(), nullable=False),
+        sa.Column(
+            "user_id", fastapi_users_db_sqlalchemy.generics.GUID(), nullable=False
+        ),
+        sa.Column("device_id", sa.String(), nullable=False),
+        sa.Column(
+            "status",
+            sa.Enum("INVALID", "VALID", name="sessionstatustype"),
+            nullable=False,
+        ),
+        sa.Column("access_token_iat", sa.Integer(), nullable=True),
+        sa.Column("refresh_token_iat", sa.Integer(), nullable=True),
+        sa.Column("created_at", sa.DateTime(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["device_id"], ["Device.id"], onupdate="CASCADE", ondelete="CASCADE"
+        ),
+        sa.ForeignKeyConstraint(
+            ["user_id"], ["User.id"], onupdate="CASCADE", ondelete="CASCADE"
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "FileFormat",
+        sa.Column("id", sa.Uuid(), nullable=False),
+        sa.Column("file_id", sa.Uuid(), nullable=False),
+        sa.Column(
+            "input_format",
+            sa.Enum("SINGULAR", "TABULAR", name="inputformat"),
+            nullable=False,
+        ),
+        sa.Column("singular_tid_column_index", sa.Integer(), nullable=True),
+        sa.Column("singular_item_column_index", sa.Integer(), nullable=True),
+        sa.Column("tabular_has_tid", sa.Boolean(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["file_id"], ["FileInfo.id"], onupdate="CASCADE", ondelete="CASCADE"
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "Task",
+        sa.Column("id", sa.Uuid(), nullable=False),
+        sa.Column(
+            "user_id", fastapi_users_db_sqlalchemy.generics.GUID(), nullable=True
+        ),
+        sa.Column("file_id", sa.Uuid(), nullable=True),
+        sa.Column("is_private", sa.Boolean(), nullable=False),
+        sa.Column("attempt_number", sa.Integer(), nullable=False),
+        sa.Column(
+            "status",
+            sa.Enum(
+                "IN_PROCESS",
+                "COMPLETED",
+                "INTERNAL_SERVER_ERROR",
+                "RESOURCE_LIMIT_IS_REACHED",
+                "ADDED_TO_THE_TASK_QUEUE",
+                "ADDING_TO_DB",
+                name="taskstatus",
+            ),
+            nullable=False,
+        ),
+        sa.Column("phase_name", sa.String(), nullable=True),
+        sa.Column("current_phase", sa.Integer(), nullable=True),
+        sa.Column("progres", sa.Float(), nullable=False),
+        sa.Column("max_phase", sa.Integer(), nullable=True),
+        sa.Column("error_msg", sa.Integer(), nullable=True),
+        sa.Column("id_executed", sa.Boolean(), nullable=False),
+        sa.Column("elapsed_time", sa.Float(), nullable=True),
+        sa.Column("config", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
+        sa.Column("result", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
+        sa.Column("created_at", sa.DateTime(), nullable=False),
+        sa.Column("deleted_at", sa.DateTime(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["file_id"], ["FileInfo.id"], onupdate="CASCADE", ondelete="SET NULL"
+        ),
+        sa.ForeignKeyConstraint(
+            ["user_id"], ["User.id"], onupdate="CASCADE", ondelete="SET NULL"
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table("Task")
+    op.drop_table("FileFormat")
+    op.drop_table("Session")
+    op.drop_table("Role")
+    op.drop_table("FileInfo")
+    op.drop_index(op.f("ix_Feedback_id"), table_name="Feedback")
+    op.drop_table("Feedback")
+    op.drop_table("Code")
+    op.drop_index(op.f("ix_User_email"), table_name="User")
+    op.drop_table("User")
+    op.drop_table("Permission")
+    op.drop_index(op.f("ix_Device_id"), table_name="Device")
+    op.drop_table("Device")
+    # ### end Alembic commands ###

app/db/models/file/file_format.py

@@ -0,0 +1,23 @@
+from enum import StrEnum, auto
+from uuid import UUID, uuid4
+from sqlalchemy import ForeignKey
+from sqlalchemy.orm import Mapped, mapped_column
+from app.db import Base
+
+
+class InputFormat(StrEnum):
+    SINGULAR = auto()
+    TABULAR = auto()
+
+
+class FileFormat(Base):
+    __tablename__ = "FileFormat"
+
+    id: Mapped[UUID] = mapped_column(primary_key=True, default=uuid4)
+    file_id: Mapped[UUID] = mapped_column(
+        ForeignKey("FileInfo.id", onupdate="CASCADE", ondelete="CASCADE")
+    )
+    input_format: Mapped[InputFormat]
+    singular_tid_column_index: Mapped[int | None] = mapped_column(default=None)
+    singular_item_column_index: Mapped[int | None] = mapped_column(default=None)
+    tabular_has_tid: Mapped[bool | None] = mapped_column(default=None)