Tidy ETL (#998)

split etl: applications, nois, documents
provide better menu building structure

bin/migrate-oats-data/applications/__init__.py

@@ -1,2 +1,8 @@
 from .base_applications import *
 from .app_prep import process_alcs_application_prep_fields
+from .submissions import process_alcs_app_submissions
+from .application_submission_status_email import (
+    process_application_submission_status_emails,
+    clean_application_submission_status_emails,
+)
+from .base_applications import process_applications

bin/migrate-oats-data/applications/application_submission_status_email/submission_status_email.py

@@ -23,7 +23,7 @@ def process_application_submission_status_emails(conn=None):
 
 
 @inject_conn_pool
-def clean_notice_of_intent_submission_status_emails(conn=None):
+def clean_application_submission_status_emails(conn=None):
     update_query = f"""
                         UPDATE alcs.application_submission_to_submission_status status
                         SET email_sent_date = NULL

bin/migrate-oats-data/menu/__init__.py

@@ -0,0 +1,2 @@
+from .menu import setup_menu_args_parser
+from .commands import *

bin/migrate-oats-data/menu/command_parser/__init__.py

@@ -0,0 +1,3 @@
+from .application_command_parser import *
+from .notice_of_intent_command_parser import *
+from .document_command_parser import *

bin/migrate-oats-data/menu/command_parser/application_command_parser.py

@@ -0,0 +1,40 @@
+def application_import_command_parser(import_batch_size, subparsers):
+    application_import_command = subparsers.add_parser(
+        "application-import",
+        help=f"Import application with specified batch size: (default: {import_batch_size})",
+    )
+    application_import_command.add_argument(
+        "--batch-size",
+        type=int,
+        default=import_batch_size,
+        metavar="",
+        help=f"batch size (default: {import_batch_size})",
+    )
+
+
+def app_prep_import_command_parser(import_batch_size, subparsers):
+    app_prep_import_command = subparsers.add_parser(
+        "app-prep-import",
+        help=f"Import App prep into ALCS (update applications table) in specified batch size: (default: {import_batch_size})",
+    )
+    app_prep_import_command.add_argument(
+        "--batch-size",
+        type=int,
+        default=import_batch_size,
+        metavar="",
+        help=f"batch size (default: {import_batch_size})",
+    )
+
+
+def app_sub_import_command_parser(import_batch_size, subparsers):
+    app_sub_import_command = subparsers.add_parser(
+        "app-sub-import",
+        help=f"Import App submission into ALCS (update application_submission table) in specified batch size: (default: {import_batch_size})",
+    )
+    app_sub_import_command.add_argument(
+        "--batch-size",
+        type=int,
+        default=import_batch_size,
+        metavar="",
+        help=f"batch size (default: {import_batch_size})",
+    )

bin/migrate-oats-data/menu/command_parser/document_command_parser.py

@@ -0,0 +1,55 @@
+# Liam question which of the following document commands need to be deleted
+def document_import_command_parser(import_batch_size, subparsers):
+    document_import_command = subparsers.add_parser(
+        "document-import",
+        help=f"Import documents with specified batch size: (default: {import_batch_size})",
+    )
+    document_import_command.add_argument(
+        "--batch-size",
+        type=int,
+        default=import_batch_size,
+        metavar="",
+        help=f"batch size (default: {import_batch_size})",
+    )
+
+
+def document_noi_import_command_parser(import_batch_size, subparsers):
+    document_noi_import_command = subparsers.add_parser(
+        "document-noi-import",
+        help=f"Import documents_noi with specified batch size: (default: {import_batch_size})",
+    )
+    document_noi_import_command.add_argument(
+        "--batch-size",
+        type=int,
+        default=import_batch_size,
+        metavar="",
+        help=f"batch size (default: {import_batch_size})",
+    )
+
+
+def application_document_import_command_parser(import_batch_size, subparsers):
+    application_document_import_command = subparsers.add_parser(
+        "app-document-import",
+        help=f"Links imported documents with application documents  specified batch size: (default: {import_batch_size})",
+    )
+    application_document_import_command.add_argument(
+        "--batch-size",
+        type=int,
+        default=import_batch_size,
+        metavar="",
+        help=f"batch size (default: {import_batch_size})",
+    )
+
+
+def noi_document_import_command_parser(import_batch_size, subparsers):
+    noi_document_import_command = subparsers.add_parser(
+        "noi-document-import",
+        help=f"Links imported documents with noi documents  specified batch size: (default: {import_batch_size})",
+    )
+    noi_document_import_command.add_argument(
+        "--batch-size",
+        type=int,
+        default=import_batch_size,
+        metavar="",
+        help=f"batch size (default: {import_batch_size})",
+    )

bin/migrate-oats-data/menu/command_parser/notice_of_intent_command_parser.py

@@ -0,0 +1,12 @@
+def noi_import_command_parser(import_batch_size, subparsers):
+    noi_import_command = subparsers.add_parser(
+        "noi-import",
+        help=f"Import NOI with specified batch size: (default: {import_batch_size})",
+    )
+    noi_import_command.add_argument(
+        "--batch-size",
+        type=int,
+        default=import_batch_size,
+        metavar="",
+        help=f"batch size (default: {import_batch_size})",
+    )

bin/migrate-oats-data/menu/commands/__init__.py

@@ -0,0 +1,5 @@
+from .import_all import import_all
+from .clean_all import clean_all
+from .documents import *
+from .applications import *
+from .notice_of_intents import *

bin/migrate-oats-data/menu/commands/applications.py

@@ -0,0 +1,46 @@
+from applications import (
+    process_applications,
+    process_alcs_application_prep_fields,
+    process_alcs_app_submissions,
+)
+
+
+def app_prep_import(console, args):
+    console.log("Beginning OATS -> ALCS app-prep import process")
+    with console.status(
+        "[bold green]App prep import (applications table update in ALCS)..."
+    ) as status:
+        if args.batch_size:
+            import_batch_size = args.batch_size
+
+        console.log(f"Processing app-prep import in batch size = {import_batch_size}")
+
+        process_alcs_application_prep_fields(batch_size=import_batch_size)
+
+
+def application_import(console, args):
+    console.log("Beginning OATS -> ALCS application import process")
+    with console.status(
+        "[bold green]application import (application table update in ALCS)..."
+    ) as status:
+        if args.batch_size:
+            import_batch_size = args.batch_size
+
+        console.log(
+            f"Processing applications import in batch size = {import_batch_size}"
+        )
+
+        process_applications(batch_size=import_batch_size)
+
+
+def application_submission_import(console, args):
+    console.log("Beginning OATS -> ALCS app-sub import process")
+    with console.status(
+        "[bold green]App submission import (application_submission table update in ALCS)..."
+    ) as status:
+        if args.batch_size:
+            import_batch_size = args.batch_size
+
+        console.log(f"Processing app-sub import in batch size = {import_batch_size}")
+
+        process_alcs_app_submissions(batch_size=import_batch_size)

bin/migrate-oats-data/menu/commands/clean_all.py

@@ -0,0 +1,32 @@
+from applications import clean_applications
+from noi.notice_of_intent_migration import clean_notice_of_intent
+from applications.submissions import clean_application_submission
+from applications.application_submission_status_email import (
+    clean_application_submission_status_emails,
+)
+from noi.noi_submission_status_email import (
+    clean_application_submission_status_emails,
+)
+from noi import clean_notice_of_intents
+from documents import (
+    clean_application_documents,
+    clean_documents,
+    clean_noi_documents,
+)
+
+
+def clean_all(console, args):
+    with console.status("[bold green]Cleaning previous ETL...") as status:
+        console.log("Cleaning data:")
+        # this will be enabled once application import is ready
+
+        clean_application_documents()
+        clean_noi_documents()
+        clean_documents()
+        clean_application_submission()
+        clean_applications()
+        clean_notice_of_intents()
+        clean_application_submission_status_emails(),
+        clean_application_submission_status_emails(),
+
+        console.log("Done")

bin/migrate-oats-data/menu/commands/documents.py

@@ -0,0 +1,42 @@
+from documents import (
+    process_documents,
+    process_documents_noi,
+    process_application_documents,
+    process_noi_documents,
+)
+
+
+def document_import(console, args):
+    console.log("Beginning OATS -> ALCS document-import process")
+    with console.status("[bold green]Import OATS into ALCS...") as status:
+        if args.batch_size:
+            import_batch_size = args.batch_size
+
+        console.log(f"Processing documents in batch size = {import_batch_size}")
+
+        process_documents(batch_size=import_batch_size)
+        process_documents_noi(batch_size=import_batch_size)
+
+
+def app_document_import(console, args):
+    console.log("Beginning OATS -> ALCS app-document-import process")
+    with console.status("[bold green]Link application documents in ALCS...") as status:
+        if args.batch_size:
+            import_batch_size = args.batch_size
+
+        console.log(
+            f"Processing application documents in batch size = {import_batch_size}"
+        )
+
+        process_application_documents(batch_size=import_batch_size)
+
+
+def noi_document_import(console, args):
+    console.log("Beginning OATS -> ALCS noi-document-import process")
+    with console.status("[bold green]Link application documents in ALCS...") as status:
+        if args.batch_size:
+            import_batch_size = args.batch_size
+
+        console.log(f"Processing NOI documents in batch size = {import_batch_size}")
+
+        process_noi_documents(batch_size=import_batch_size)

bin/migrate-oats-data/menu/commands/import_all.py

@@ -0,0 +1,68 @@
+from applications import (
+    process_applications,
+    process_alcs_application_prep_fields,
+)
+from noi.notice_of_intent_migration import (
+    init_notice_of_intents,
+    process_notice_of_intent,
+)
+from applications.submissions import process_alcs_app_submissions
+from applications.application_submission_status_email import (
+    process_application_submission_status_emails,
+)
+from noi.noi_submission_status_email import (
+    process_notice_of_intent_submission_status_emails,
+)
+from noi import (
+    process_notice_of_intent,
+)
+from documents import (
+    process_application_documents,
+    process_noi_documents,
+)
+
+
+def import_all(console, args):
+    console.log("Beginning OATS -> ALCS import process")
+
+    with console.status("[bold green]Import OATS into ALCS...") as status:
+        console.log("Init applications:")
+        # this will be enabled once application import is ready
+
+        if args and args.batch_size:
+            import_batch_size = args.batch_size
+
+        console.log("Batching applications:")
+        process_applications(batch_size=import_batch_size)
+
+        console.log("Init NOIs:")
+        init_notice_of_intents(batch_size=import_batch_size)
+
+        # TODO Liam question: which process_documents_noi or process_noi_documents is the correct one to keep?
+        # console.log("Processing NOI specific documents:")
+        # process_documents_noi(batch_size=import_batch_size)
+
+        # console.log("Processing documents:")
+        # process_documents(batch_size=import_batch_size)
+
+        console.log("Processing application documents:")
+        process_application_documents(batch_size=import_batch_size)
+
+        console.log("Processing NOI documents:")
+        process_noi_documents(batch_size=import_batch_size)
+
+        console.log("Processing application prep:")
+        process_alcs_application_prep_fields(batch_size=import_batch_size)
+
+        console.log("Processing application submission:")
+        process_alcs_app_submissions(batch_size=import_batch_size)
+
+        console.log("Processing notice of intents")
+        process_notice_of_intent(batch_size=import_batch_size)
+
+        # NOTE: both process_application_submission_status_emails(), process_notice_of_intent_submission_status_emails()
+        #       must be the last ones in the migrate etl
+        console.log("Processing submission status emails")
+        process_application_submission_status_emails()
+        process_notice_of_intent_submission_status_emails()
+        console.log("Done")

bin/migrate-oats-data/menu/commands/notice_of_intents.py

@@ -0,0 +1,18 @@
+from noi.notice_of_intent_migration import (
+    init_notice_of_intents,
+    process_notice_of_intent,
+)
+
+
+def notice_of_intent_import(console, args):
+    console.log("Beginning OATS -> ALCS NOI import process")
+    with console.status(
+        "[bold green]NOI import (notice_of_intent table update in ALCS)..."
+    ) as status:
+        if args.batch_size:
+            import_batch_size = args.batch_size
+
+        console.log(f"Processing NOI import in batch size = {import_batch_size}")
+
+        init_notice_of_intents(batch_size=import_batch_size)
+        process_notice_of_intent(batch_size=import_batch_size)