PROD-2700 Implement Soft Delete for PrivacyRequests (#5321)

.fides/db_dataset.yml

@@ -1306,6 +1306,10 @@ dataset:
       data_categories: [ system.operations ]
     - name: submitted_by
       data_categories: [ system.operations ]
+    - name: deleted_at
+      data_categories: [ system.operations ]
+    - name: deleted_by
+      data_categories: [ system.operations ]
   - name: privacyrequesterror
     data_categories: []
     fields:
@@ -2308,4 +2312,4 @@ dataset:
       - name: single_row
         data_categories: [system.operations]
       - name: updated_at
-        data_categories: [system.operations]
+        data_categories: [system.operations]

CHANGELOG.md

@@ -22,6 +22,9 @@ The types of changes are:
 - Make all "Description" table columns expandable in Admin UI tables [#5340](https://github.com/ethyca/fides/pull/5340)
 - Added new RDS MySQL Connector [#5343](https://github.com/ethyca/fides/pull/5343)
 
+### Added
+- Implement Soft Delete for PrivacyRequests [#5321](https://github.com/ethyca/fides/pull/5321/files)
+
 ### Developer Experience
 - Migrate toggle switches from Chakra to Ant Design [#5323](https://github.com/ethyca/fides/pull/5323)
 - Replace `debugLog` with global scoped `fidesDebugger` for better debug experience and optimization of prod code [#5335](https://github.com/ethyca/fides/pull/5335)

clients/admin-ui/src/types/api/models/BulkSoftDeletePrivacyRequests.ts

@@ -0,0 +1,10 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { BulkUpdateFailed } from "./BulkUpdateFailed";
+
+export type BulkSoftDeletePrivacyRequests = {
+  succeeded: Array<string>;
+  failed: Array<BulkUpdateFailed>;
+};

clients/admin-ui/src/types/api/models/DynamicErasureEmailDocsSchema.ts

@@ -0,0 +1,22 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+import type { AdvancedSettings } from "./AdvancedSettings";
+import type { FidesDatasetReference } from "./FidesDatasetReference";
+
+/**
+ * DynamicErasureEmailDocsSchema Secrets Schema for API Docs
+ */
+export type DynamicErasureEmailDocsSchema = {
+  test_email_address?: string | null;
+  advanced_settings?: AdvancedSettings;
+  /**
+   * Dataset reference to the field containing the third party vendor name. Both third_party_vendor_name and recipient_email_address must reference the same dataset and collection.
+   */
+  third_party_vendor_name: FidesDatasetReference;
+  /**
+   * Dataset reference to the field containing the recipient email address. Both third_party_vendor_name and recipient_email_address must reference the same dataset and collection.
+   */
+  recipient_email_address: FidesDatasetReference;
+};

clients/admin-ui/src/types/api/models/PrivacyRequestFilter.ts

@@ -25,6 +25,7 @@ export type PrivacyRequestFilter = {
   verbose?: boolean | null;
   include_identities?: boolean | null;
   include_custom_privacy_request_fields?: boolean | null;
+  include_deleted_requests?: boolean | null;
   download_csv?: boolean | null;
   sort_field?: string;
   sort_direction?: ColumnSort;

clients/admin-ui/src/types/api/models/PrivacyRequestResponse.ts

@@ -32,4 +32,6 @@ export type PrivacyRequestResponse = {
   custom_privacy_request_fields_approved_by?: string | null;
   custom_privacy_request_fields_approved_at?: string | null;
   source?: PrivacyRequestSource | null;
+  deleted_at?: string | null;
+  deleted_by?: string | null;
 };

clients/admin-ui/src/types/api/models/PrivacyRequestVerboseResponse.ts

@@ -34,5 +34,7 @@ export type PrivacyRequestVerboseResponse = {
   custom_privacy_request_fields_approved_by?: string | null;
   custom_privacy_request_fields_approved_at?: string | null;
   source?: PrivacyRequestSource | null;
+  deleted_at?: string | null;
+  deleted_by?: string | null;
   results: Record<string, Array<ExecutionAndAuditLogResponse>>;
 };

src/fides/api/alembic/migrations/versions/75bb9ee843f5_add_soft_delete_for_privacy_request.py

@@ -0,0 +1,33 @@
+"""Adds deleted_at and deleted_by to PrivacyRequest
+
+Revision ID: 75bb9ee843f5
+Revises: 68c590ff6e89
+Create Date: 2024-09-19 14:35:30.510909
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "75bb9ee843f5"
+down_revision = "68c590ff6e89"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "privacyrequest",
+        sa.Column("deleted_at", sa.DateTime(timezone=True), nullable=True),
+    )
+    op.add_column("privacyrequest", sa.Column("deleted_by", sa.String(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("privacyrequest", "deleted_by")
+    op.drop_column("privacyrequest", "deleted_at")
+    # ### end Alembic commands ###

src/fides/api/api/v1/endpoints/privacy_request_endpoints.py

@@ -104,6 +104,7 @@
 from fides.api.schemas.privacy_request import (
     BulkPostPrivacyRequests,
     BulkReviewResponse,
+    BulkSoftDeletePrivacyRequests,
     DenyPrivacyRequests,
     ExecutionLogDetailResponse,
     ManualWebhookData,
@@ -147,6 +148,7 @@
 from fides.common.api.scope_registry import (
     PRIVACY_REQUEST_CALLBACK_RESUME,
     PRIVACY_REQUEST_CREATE,
+    PRIVACY_REQUEST_DELETE,
     PRIVACY_REQUEST_NOTIFICATIONS_CREATE_OR_UPDATE,
     PRIVACY_REQUEST_NOTIFICATIONS_READ,
     PRIVACY_REQUEST_READ,
@@ -159,6 +161,7 @@
     PRIVACY_REQUEST_APPROVE,
     PRIVACY_REQUEST_AUTHENTICATED,
     PRIVACY_REQUEST_BULK_RETRY,
+    PRIVACY_REQUEST_BULK_SOFT_DELETE,
     PRIVACY_REQUEST_DENY,
     PRIVACY_REQUEST_MANUAL_WEBHOOK_ACCESS_INPUT,
     PRIVACY_REQUEST_MANUAL_WEBHOOK_ERASURE_INPUT,
@@ -170,6 +173,7 @@
     PRIVACY_REQUEST_RESUME_FROM_REQUIRES_INPUT,
     PRIVACY_REQUEST_RETRY,
     PRIVACY_REQUEST_SEARCH,
+    PRIVACY_REQUEST_SOFT_DELETE,
     PRIVACY_REQUEST_TRANSFER_TO_PARENT,
     PRIVACY_REQUEST_VERIFY_IDENTITY,
     PRIVACY_REQUESTS,
@@ -188,7 +192,7 @@
 
 
 def get_privacy_request_or_error(
-    db: Session, privacy_request_id: str
+    db: Session, privacy_request_id: str, error_if_deleted: Optional[bool] = True
 ) -> PrivacyRequest:
     """Load the privacy request or throw a 404"""
     logger.info("Finding privacy request with id '{}'", privacy_request_id)
@@ -201,6 +205,12 @@ def get_privacy_request_or_error(
             detail=f"No privacy request found with id '{privacy_request_id}'.",
         )
 
+    if error_if_deleted and privacy_request.deleted_at is not None:
+        raise HTTPException(
+            status_code=HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=f"Privacy request with id {privacy_request_id} has been deleted.",
+        )
+
     return privacy_request
 
 
@@ -434,6 +444,7 @@ def _filter_privacy_request_queryset(
     external_id: Optional[str] = None,
     action_type: Optional[ActionType] = None,
     include_consent_webhook_requests: Optional[bool] = False,
+    include_deleted_requests: Optional[bool] = False,
 ) -> Query:
     """
     Utility method to apply filters to our privacy request query.
@@ -588,6 +599,10 @@ def _filter_privacy_request_queryset(
             )
         )
 
+    # Filter out deleted requests
+    if not include_deleted_requests:
+        query = query.filter(PrivacyRequest.deleted_at.is_(None))
+
     return query
 
 
@@ -664,6 +679,7 @@ def _shared_privacy_request_search(
     verbose: Optional[bool] = False,
     include_identities: Optional[bool] = False,
     include_custom_privacy_request_fields: Optional[bool] = False,
+    include_deleted_requests: Optional[bool] = False,
     download_csv: Optional[bool] = False,
     sort_field: str = "created_at",
     sort_direction: ColumnSort = ColumnSort.DESC,
@@ -698,6 +714,8 @@ def _shared_privacy_request_search(
         errored_gt,
         external_id,
         action_type,
+        None,
+        include_deleted_requests,
     )
 
     logger.info(
@@ -772,6 +790,7 @@ def get_request_status(
     include_identities: Optional[bool] = False,
     include_custom_privacy_request_fields: Optional[bool] = False,
     download_csv: Optional[bool] = False,
+    include_deleted_requests: Optional[bool] = False,
     sort_field: str = "created_at",
     sort_direction: ColumnSort = ColumnSort.DESC,
 ) -> Union[StreamingResponse, AbstractPage[PrivacyRequest]]:
@@ -810,6 +829,7 @@ def get_request_status(
         verbose=verbose,
         include_identities=include_identities,
         include_custom_privacy_request_fields=include_custom_privacy_request_fields,
+        include_deleted_requests=include_deleted_requests,
         download_csv=download_csv,
         sort_field=sort_field,
         sort_direction=sort_direction,
@@ -863,6 +883,7 @@ def privacy_request_search(
         verbose=privacy_request_filter.verbose,
         include_identities=privacy_request_filter.include_identities,
         include_custom_privacy_request_fields=privacy_request_filter.include_custom_privacy_request_fields,
+        include_deleted_requests=privacy_request_filter.include_deleted_requests,
         download_csv=privacy_request_filter.download_csv,
         sort_field=privacy_request_filter.sort_field,
         sort_direction=privacy_request_filter.sort_direction,
@@ -882,7 +903,7 @@ def get_request_status_logs(
 ) -> AbstractPage[ExecutionLog]:
     """Returns all the execution logs associated with a given privacy request ordered by updated asc."""
 
-    get_privacy_request_or_error(db, privacy_request_id)
+    get_privacy_request_or_error(db, privacy_request_id, error_if_deleted=False)
 
     logger.info(
         "Finding all execution logs for privacy request {} with params '{}'",
@@ -1159,6 +1180,15 @@ def bulk_restart_privacy_request_from_failure(
             )
             continue
 
+        if privacy_request.deleted_at is not None:
+            failed.append(
+                {
+                    "message": "Cannot restart a deleted privacy request",
+                    "data": {"privacy_request_id": privacy_request_id},
+                }
+            )
+            continue
+
         if privacy_request.status != PrivacyRequestStatus.error:
             failed.append(
                 {
@@ -1242,6 +1272,17 @@ def review_privacy_request(
             )
             continue
 
+        if privacy_request.deleted_at is not None:
+            failed.append(
+                {
+                    "message": "Cannot transition status for a deleted request",
+                    "data": PrivacyRequestResponse.model_validate(
+                        privacy_request
+                    ).model_dump(mode="json"),
+                }
+            )
+            continue
+
         if privacy_request.status != PrivacyRequestStatus.pending:
             failed.append(
                 {
@@ -1871,7 +1912,9 @@ def view_uploaded_manual_webhook_data(
     If checked=False, data must be reviewed before submission. The privacy request should not be submitted as-is.
     """
     privacy_request: PrivacyRequest = get_privacy_request_or_error(
-        db, privacy_request_id
+        db,
+        privacy_request_id,
+        error_if_deleted=False,
     )
     access_manual_webhook: AccessManualWebhook = get_access_manual_webhook_or_404(
         connection_config
@@ -1929,7 +1972,9 @@ def view_uploaded_erasure_manual_webhook_data(
     If checked=False, data must be reviewed before submission. The privacy request should not be submitted as-is.
     """
     privacy_request: PrivacyRequest = get_privacy_request_or_error(
-        db, privacy_request_id
+        db,
+        privacy_request_id,
+        error_if_deleted=False,
     )
     manual_webhook: AccessManualWebhook = get_access_manual_webhook_or_404(
         connection_config
@@ -2307,7 +2352,9 @@ def get_individual_privacy_request_tasks(
 ) -> List[RequestTask]:
     """Returns individual Privacy Request Tasks created by DSR 3.0 scheduler
     in order by creation and collection address"""
-    pr: PrivacyRequest = get_privacy_request_or_error(db, privacy_request_id)
+    pr: PrivacyRequest = get_privacy_request_or_error(
+        db, privacy_request_id, error_if_deleted=False
+    )
 
     logger.info(f"Getting Request Tasks for '{privacy_request_id}'")
 
@@ -2441,3 +2488,86 @@ def request_task_async_callback(
     queue_request_task(request_task, privacy_request_proceed=True)
 
     return {"task_queued": True}
+
+
+@router.post(
+    PRIVACY_REQUEST_BULK_SOFT_DELETE,
+    dependencies=[Security(verify_oauth_client, scopes=[PRIVACY_REQUEST_DELETE])],
+    status_code=HTTP_200_OK,
+    response_model=BulkSoftDeletePrivacyRequests,
+)
+def bulk_soft_delete_privacy_requests(
+    *,
+    db: Session = Depends(deps.get_db),
+    client: ClientDetail = Security(
+        verify_oauth_client,
+        scopes=[PRIVACY_REQUEST_DELETE],
+    ),
+    privacy_requests: ReviewPrivacyRequestIds,
+) -> BulkSoftDeletePrivacyRequests:
+    """
+    Soft delete a list of privacy requests. The requests' deleted_at field will be populated with the current datetime
+    and its deleted_by field will be populated with the user_id of the user who initiated the deletion. Returns an
+    object with the list of successfully deleted privacy requests and the list of failed deletions.
+    """
+    succeeded: List[str] = []
+    failed: List[Dict[str, Any]] = []
+
+    user_id = client.user_id
+    if client.id == CONFIG.security.oauth_root_client_id:
+        user_id = "root"
+
+    for privacy_request_id in privacy_requests.request_ids:
+        privacy_request = PrivacyRequest.get(db, object_id=privacy_request_id)
+
+        if not privacy_request:
+            failed.append(
+                {
+                    "message": f"No privacy request found with id '{privacy_request_id}'",
+                    "data": {"privacy_request_id": privacy_request_id},
+                }
+            )
+            continue
+
+        if privacy_request.deleted_at is not None:
+            failed.append(
+                {
+                    "message": f"Privacy request '{privacy_request_id}' has already been deleted.",
+                    "data": {"privacy_request_id": privacy_request_id},
+                }
+            )
+            continue
+
+        privacy_request.soft_delete(db, user_id)
+        succeeded.append(privacy_request.id)
+
+    return BulkSoftDeletePrivacyRequests(succeeded=succeeded, failed=failed)
+
+
+@router.post(
+    PRIVACY_REQUEST_SOFT_DELETE,
+    dependencies=[Security(verify_oauth_client, scopes=[PRIVACY_REQUEST_DELETE])],
+    status_code=HTTP_200_OK,
+    response_model=None,
+)
+def soft_delete_privacy_request(
+    privacy_request_id: str,
+    *,
+    db: Session = Depends(deps.get_db),
+    client: ClientDetail = Security(
+        verify_oauth_client,
+        scopes=[PRIVACY_REQUEST_DELETE],
+    ),
+) -> None:
+    """
+    Endpoint for soft deleting a privacy request. The request's deleted_at field will be populated with the current datetime
+    and its deleted_by field will be populated with the user_id of the user who initiated the deletion.
+    """
+    privacy_request: PrivacyRequest = get_privacy_request_or_error(
+        db, privacy_request_id
+    )
+    user_id = client.user_id
+    if client.id == CONFIG.security.oauth_root_client_id:
+        user_id = "root"
+
+    privacy_request.soft_delete(db, user_id)