Feature/add last modified for all entities (#51)

* Updated schema for family, event, doc, and collection.

* Added created and last modified to entity read DTOs.

* Added created and last modified columns to each entity.

* Check for existence of new columns on each entity but not values.

* Updated test schema with latest schema.

* Fixed slug bug where get always returns original family slug.

* Isolated collection and slug update tests.

app/clients/db/models/law_policy/collection.py

@@ -1,7 +1,7 @@
 import sqlalchemy as sa
-from app.clients.db.models.law_policy import Family
-from app.clients.db.models.app import Organisation
 
+from app.clients.db.models.app import Organisation
+from app.clients.db.models.law_policy import Family
 from app.clients.db.session import Base
 
 
@@ -13,6 +13,15 @@ class Collection(Base):
     import_id = sa.Column(sa.Text, primary_key=True)
     title = sa.Column(sa.Text, nullable=False)
     description = sa.Column(sa.Text, nullable=False)
+    created = sa.Column(
+        sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
+    )
+    last_modified = sa.Column(
+        sa.DateTime(timezone=True),
+        server_default=sa.func.now(),
+        onupdate=sa.func.now(),
+        nullable=False,
+    )
 
 
 class CollectionFamily(Base):

app/clients/db/models/law_policy/family.py

@@ -5,12 +5,13 @@
 from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy.orm import relationship
 
-from .geography import Geography
 from app.clients.db.models.app import Organisation
 from app.clients.db.models.app.enum import BaseModelEnum
 from app.clients.db.models.document import PhysicalDocument
 from app.clients.db.session import Base
 
+from .geography import Geography
+
 
 class FamilyCategory(BaseModelEnum):
     """Family categories as understood in the context of law/policy."""
@@ -63,6 +64,15 @@ class Family(Base):
         lazy="joined",
         order_by="FamilyEvent.date",
     )
+    created = sa.Column(
+        sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
+    )
+    last_modified = sa.Column(
+        sa.DateTime(timezone=True),
+        server_default=sa.func.now(),
+        onupdate=sa.func.now(),
+        nullable=False,
+    )
 
     @hybrid_property
     def family_status(self) -> FamilyStatus:  # type: ignore
@@ -202,6 +212,15 @@ class FamilyDocument(Base):
     )
     document_type = sa.Column(sa.ForeignKey(FamilyDocumentType.name), nullable=True)
     document_role = sa.Column(sa.ForeignKey(FamilyDocumentRole.name), nullable=True)
+    created = sa.Column(
+        sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
+    )
+    last_modified = sa.Column(
+        sa.DateTime(timezone=True),
+        server_default=sa.func.now(),
+        onupdate=sa.func.now(),
+        nullable=False,
+    )
 
     slugs: list["Slug"] = relationship("Slug", lazy="joined")
     physical_document: PhysicalDocument = relationship(
@@ -272,3 +291,12 @@ class FamilyEvent(Base):
         nullable=True,
     )
     status = sa.Column(sa.Enum(EventStatus), nullable=False)
+    created = sa.Column(
+        sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
+    )
+    last_modified = sa.Column(
+        sa.DateTime(timezone=True),
+        server_default=sa.func.now(),
+        onupdate=sa.func.now(),
+        nullable=False,
+    )

app/model/collection.py

@@ -1,3 +1,4 @@
+from datetime import datetime
 from pydantic import BaseModel
 
 
@@ -9,6 +10,8 @@ class CollectionReadDTO(BaseModel):
     description: str
     families: list[str]
     organisation: str
+    created: datetime
+    last_modified: datetime
 
 
 class CollectionWriteDTO(BaseModel):

app/model/document.py

@@ -1,4 +1,6 @@
+from datetime import datetime
 from typing import Optional
+
 from pydantic import BaseModel
 
 from app.clients.db.models.law_policy.family import (
@@ -17,6 +19,8 @@ class DocumentReadDTO(BaseModel):
     role: str
     type: Optional[str]
     slug: str
+    created: datetime
+    last_modified: datetime
     # From PhysicalDocument
     physical_id: int
     title: str

app/model/event.py

@@ -19,6 +19,8 @@ class EventReadDTO(BaseModel):
     date: datetime
     event_type_value: str
     event_status: EventStatus
+    created: datetime
+    last_modified: datetime
 
     # From FamilyDocument
     family_import_id: str

app/model/family.py

@@ -1,6 +1,8 @@
 from datetime import datetime
 from typing import Optional
+
 from pydantic import BaseModel
+
 from app.model.general import Json
 
 
@@ -21,6 +23,8 @@ class FamilyReadDTO(BaseModel):
     documents: list[str]
     collections: list[str]
     organisation: str
+    created: datetime
+    last_modified: datetime
 
 
 class FamilyWriteDTO(BaseModel):

app/repository/collection.py

@@ -1,12 +1,19 @@
 """Operations on the repository for the Collection entity."""
 
 import logging
+from datetime import datetime
 from typing import Optional, Tuple, cast
 
-from sqlalchemy.orm import Session
-from sqlalchemy import Column
+from sqlalchemy import Column, or_
+from sqlalchemy import delete as db_delete
+from sqlalchemy import update as db_update
+from sqlalchemy.exc import NoResultFound
+from sqlalchemy.orm import Query, Session
+from sqlalchemy_utils import escape_like
+
 from app.clients.db.models.app.counters import CountedEntity
 from app.clients.db.models.app.users import Organisation
+from app.clients.db.models.law_policy import Collection
 from app.clients.db.models.law_policy.collection import (
     CollectionFamily,
     CollectionOrganisation,
@@ -18,16 +25,8 @@
     CollectionReadDTO,
     CollectionWriteDTO,
 )
-from app.clients.db.models.law_policy import Collection
-from sqlalchemy.exc import NoResultFound
-from sqlalchemy.orm import Query
-
-from sqlalchemy import or_, update as db_update, delete as db_delete
-from sqlalchemy_utils import escape_like
-
 from app.repository.helpers import generate_import_id
 
-
 _LOGGER = logging.getLogger(__name__)
 
 CollectionOrg = Tuple[Collection, Organisation]
@@ -83,6 +82,8 @@ def _collection_to_dto(db: Session, co: CollectionOrg) -> CollectionReadDTO:
         description=str(collection.description),
         organisation=cast(str, org.name),
         families=families,
+        created=cast(datetime, collection.created),
+        last_modified=cast(datetime, collection.last_modified),
     )
 
 

app/repository/document.py

@@ -1,13 +1,15 @@
-from typing import Tuple
 import logging
-from typing import Optional, cast
+from typing import Optional, Tuple, cast
+
+from sqlalchemy import Column, and_, func, or_
+from sqlalchemy import insert as db_insert
+from sqlalchemy import update as db_update
+from sqlalchemy.exc import NoResultFound
+from sqlalchemy.orm import Query, Session, aliased
+from sqlalchemy.sql.functions import concat
+from sqlalchemy_utils import escape_like
+
 from app.clients.db.models.app.counters import CountedEntity
-from app.clients.db.models.law_policy.family import (
-    DocumentStatus,
-    Slug,
-)
-from app.errors import RepositoryError, ValidationError
-from app.model.document import DocumentCreateDTO, DocumentReadDTO, DocumentWriteDTO
 from app.clients.db.models.document.physical_document import (
     Language,
     LanguageSource,
@@ -17,17 +19,14 @@
 from app.clients.db.models.law_policy import (
     FamilyDocument,
 )
-
-from sqlalchemy import Column, or_, and_, update as db_update, insert as db_insert
-from sqlalchemy_utils import escape_like
-from sqlalchemy.exc import NoResultFound
-from sqlalchemy.orm import Session, Query, aliased
-from sqlalchemy.sql.functions import concat
-from sqlalchemy import func
-
-from app.repository.helpers import generate_import_id, generate_slug
+from app.clients.db.models.law_policy.family import (
+    DocumentStatus,
+    Slug,
+)
+from app.errors import RepositoryError, ValidationError
+from app.model.document import DocumentCreateDTO, DocumentReadDTO, DocumentWriteDTO
 from app.repository import family as family_repo
-
+from app.repository.helpers import generate_import_id, generate_slug
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -57,7 +56,9 @@ def _get_query(db: Session) -> Query:
             FamilyDocument.document_status.label("status"),
             FamilyDocument.document_role.label("role"),
             FamilyDocument.document_type.label("type"),
-            concat(sq_slug.c.name).label("slug"),
+            FamilyDocument.created.label("created"),
+            FamilyDocument.last_modified.label("last_modified"),
+            concat(sq_slug.c.name).label("slug"),  # type: ignore
             PhysicalDocument.id.label("physical_id"),
             PhysicalDocument.title.label("title"),
             PhysicalDocument.md5_sum.label("md5_sum"),

app/repository/event.py

@@ -4,24 +4,25 @@
 from datetime import datetime
 from typing import Optional, Tuple, cast
 
-from sqlalchemy import or_, Column, update as db_update, delete as db_delete
-from sqlalchemy.orm import Query, Session
+from sqlalchemy import Column, or_
+from sqlalchemy import delete as db_delete
+from sqlalchemy import update as db_update
 from sqlalchemy.exc import NoResultFound
+from sqlalchemy.orm import Query, Session
 from sqlalchemy_utils import escape_like
 
 from app.clients.db.models.app.counters import CountedEntity
 from app.clients.db.models.law_policy import (
     EventStatus,
-    FamilyEvent,
     Family,
     FamilyDocument,
+    FamilyEvent,
 )
-from app.errors import ValidationError, RepositoryError
+from app.errors import RepositoryError, ValidationError
 from app.model.event import EventCreateDTO, EventReadDTO, EventWriteDTO
 from app.repository import family as family_repo
 from app.repository.helpers import generate_import_id
 
-
 _LOGGER = logging.getLogger(__name__)
 
 FamilyEventTuple = Tuple[FamilyEvent, Family, FamilyDocument]
@@ -58,6 +59,8 @@ def _event_to_dto(family_event_meta: FamilyEventTuple) -> EventReadDTO:
         family_document_import_id=family_document_import_id,
         event_type_value=cast(str, family_event.event_type_name),
         event_status=cast(EventStatus, family_event.status),
+        created=cast(datetime, family_event.created),
+        last_modified=cast(datetime, family_event.last_modified),
     )
 
 

app/repository/family.py

@@ -1,6 +1,7 @@
 """Operations on the repository for the Family entity."""
 
 import logging
+from datetime import datetime
 from typing import Optional, Tuple, Union, cast
 
 from sqlalchemy import Column, and_, or_
@@ -78,7 +79,7 @@ def _family_to_dto(db: Session, fam_geo_meta_org: FamilyGeoMetaOrg) -> FamilyRea
         category=str(f.family_category),
         status=str(f.family_status),
         metadata=metadata,
-        slug=str(f.slugs[-1].name if len(f.slugs) > 0 else ""),
+        slug=str(f.slugs[0].name if len(f.slugs) > 0 else ""),
         events=[str(e.import_id) for e in f.events],
         published_date=f.published_date,
         last_updated_date=f.last_updated_date,
@@ -90,6 +91,8 @@ def _family_to_dto(db: Session, fam_geo_meta_org: FamilyGeoMetaOrg) -> FamilyRea
             )
         ],
         organisation=org,
+        created=cast(datetime, f.created),
+        last_modified=cast(datetime, f.last_modified),
     )