From 722ce38167881b664aa1126a131faa861b26797d Mon Sep 17 00:00:00 2001 From: mhuseinov <61513701+mhuseinov@users.noreply.github.com> Date: Fri, 22 Sep 2023 13:21:11 -0700 Subject: [PATCH] Tidy ETL (#998) split etl: applications, nois, documents provide better menu building structure --- .../applications/__init__.py | 6 + .../submission_status_email.py | 2 +- bin/migrate-oats-data/menu/__init__.py | 2 + .../menu/command_parser/__init__.py | 3 + .../application_command_parser.py | 40 ++ .../command_parser/document_command_parser.py | 55 +++ .../notice_of_intent_command_parser.py | 12 + .../menu/commands/__init__.py | 5 + .../menu/commands/applications.py | 46 +++ .../menu/commands/clean_all.py | 32 ++ .../menu/commands/documents.py | 42 ++ .../menu/commands/import_all.py | 68 ++++ .../menu/commands/notice_of_intents.py | 18 + bin/migrate-oats-data/menu/menu.py | 58 +++ bin/migrate-oats-data/migrate.py | 365 ++---------------- bin/migrate-oats-data/noi/__init__.py | 3 +- .../noi/notice_of_intent_init.py | 32 +- .../noi/notice_of_intent_migration.py | 17 + .../__init__.py | 4 +- ...oats_to_alcs_notice_of_intent_table_etl.py | 2 +- 20 files changed, 447 insertions(+), 365 deletions(-) create mode 100644 bin/migrate-oats-data/menu/__init__.py create mode 100644 bin/migrate-oats-data/menu/command_parser/__init__.py create mode 100644 bin/migrate-oats-data/menu/command_parser/application_command_parser.py create mode 100644 bin/migrate-oats-data/menu/command_parser/document_command_parser.py create mode 100644 bin/migrate-oats-data/menu/command_parser/notice_of_intent_command_parser.py create mode 100644 bin/migrate-oats-data/menu/commands/__init__.py create mode 100644 bin/migrate-oats-data/menu/commands/applications.py create mode 100644 bin/migrate-oats-data/menu/commands/clean_all.py create mode 100644 bin/migrate-oats-data/menu/commands/documents.py create mode 100644 bin/migrate-oats-data/menu/commands/import_all.py create mode 100644 bin/migrate-oats-data/menu/commands/notice_of_intents.py create mode 100644 bin/migrate-oats-data/menu/menu.py create mode 100644 bin/migrate-oats-data/noi/notice_of_intent_migration.py diff --git a/bin/migrate-oats-data/applications/__init__.py b/bin/migrate-oats-data/applications/__init__.py index 958d38f1d4..8e0663d4c5 100644 --- a/bin/migrate-oats-data/applications/__init__.py +++ b/bin/migrate-oats-data/applications/__init__.py @@ -1,2 +1,8 @@ from .base_applications import * from .app_prep import process_alcs_application_prep_fields +from .submissions import process_alcs_app_submissions +from .application_submission_status_email import ( + process_application_submission_status_emails, + clean_application_submission_status_emails, +) +from .base_applications import process_applications diff --git a/bin/migrate-oats-data/applications/application_submission_status_email/submission_status_email.py b/bin/migrate-oats-data/applications/application_submission_status_email/submission_status_email.py index 6955d24626..52f35f5e81 100644 --- a/bin/migrate-oats-data/applications/application_submission_status_email/submission_status_email.py +++ b/bin/migrate-oats-data/applications/application_submission_status_email/submission_status_email.py @@ -23,7 +23,7 @@ def process_application_submission_status_emails(conn=None): @inject_conn_pool -def clean_notice_of_intent_submission_status_emails(conn=None): +def clean_application_submission_status_emails(conn=None): update_query = f""" UPDATE alcs.application_submission_to_submission_status status SET email_sent_date = NULL diff --git a/bin/migrate-oats-data/menu/__init__.py b/bin/migrate-oats-data/menu/__init__.py new file mode 100644 index 0000000000..29f9f954c8 --- /dev/null +++ b/bin/migrate-oats-data/menu/__init__.py @@ -0,0 +1,2 @@ +from .menu import setup_menu_args_parser +from .commands import * diff --git a/bin/migrate-oats-data/menu/command_parser/__init__.py b/bin/migrate-oats-data/menu/command_parser/__init__.py new file mode 100644 index 0000000000..a442dfeeef --- /dev/null +++ b/bin/migrate-oats-data/menu/command_parser/__init__.py @@ -0,0 +1,3 @@ +from .application_command_parser import * +from .notice_of_intent_command_parser import * +from .document_command_parser import * diff --git a/bin/migrate-oats-data/menu/command_parser/application_command_parser.py b/bin/migrate-oats-data/menu/command_parser/application_command_parser.py new file mode 100644 index 0000000000..91894fb298 --- /dev/null +++ b/bin/migrate-oats-data/menu/command_parser/application_command_parser.py @@ -0,0 +1,40 @@ +def application_import_command_parser(import_batch_size, subparsers): + application_import_command = subparsers.add_parser( + "application-import", + help=f"Import application with specified batch size: (default: {import_batch_size})", + ) + application_import_command.add_argument( + "--batch-size", + type=int, + default=import_batch_size, + metavar="", + help=f"batch size (default: {import_batch_size})", + ) + + +def app_prep_import_command_parser(import_batch_size, subparsers): + app_prep_import_command = subparsers.add_parser( + "app-prep-import", + help=f"Import App prep into ALCS (update applications table) in specified batch size: (default: {import_batch_size})", + ) + app_prep_import_command.add_argument( + "--batch-size", + type=int, + default=import_batch_size, + metavar="", + help=f"batch size (default: {import_batch_size})", + ) + + +def app_sub_import_command_parser(import_batch_size, subparsers): + app_sub_import_command = subparsers.add_parser( + "app-sub-import", + help=f"Import App submission into ALCS (update application_submission table) in specified batch size: (default: {import_batch_size})", + ) + app_sub_import_command.add_argument( + "--batch-size", + type=int, + default=import_batch_size, + metavar="", + help=f"batch size (default: {import_batch_size})", + ) diff --git a/bin/migrate-oats-data/menu/command_parser/document_command_parser.py b/bin/migrate-oats-data/menu/command_parser/document_command_parser.py new file mode 100644 index 0000000000..fc175314a9 --- /dev/null +++ b/bin/migrate-oats-data/menu/command_parser/document_command_parser.py @@ -0,0 +1,55 @@ +# Liam question which of the following document commands need to be deleted +def document_import_command_parser(import_batch_size, subparsers): + document_import_command = subparsers.add_parser( + "document-import", + help=f"Import documents with specified batch size: (default: {import_batch_size})", + ) + document_import_command.add_argument( + "--batch-size", + type=int, + default=import_batch_size, + metavar="", + help=f"batch size (default: {import_batch_size})", + ) + + +def document_noi_import_command_parser(import_batch_size, subparsers): + document_noi_import_command = subparsers.add_parser( + "document-noi-import", + help=f"Import documents_noi with specified batch size: (default: {import_batch_size})", + ) + document_noi_import_command.add_argument( + "--batch-size", + type=int, + default=import_batch_size, + metavar="", + help=f"batch size (default: {import_batch_size})", + ) + + +def application_document_import_command_parser(import_batch_size, subparsers): + application_document_import_command = subparsers.add_parser( + "app-document-import", + help=f"Links imported documents with application documents specified batch size: (default: {import_batch_size})", + ) + application_document_import_command.add_argument( + "--batch-size", + type=int, + default=import_batch_size, + metavar="", + help=f"batch size (default: {import_batch_size})", + ) + + +def noi_document_import_command_parser(import_batch_size, subparsers): + noi_document_import_command = subparsers.add_parser( + "noi-document-import", + help=f"Links imported documents with noi documents specified batch size: (default: {import_batch_size})", + ) + noi_document_import_command.add_argument( + "--batch-size", + type=int, + default=import_batch_size, + metavar="", + help=f"batch size (default: {import_batch_size})", + ) diff --git a/bin/migrate-oats-data/menu/command_parser/notice_of_intent_command_parser.py b/bin/migrate-oats-data/menu/command_parser/notice_of_intent_command_parser.py new file mode 100644 index 0000000000..617199d846 --- /dev/null +++ b/bin/migrate-oats-data/menu/command_parser/notice_of_intent_command_parser.py @@ -0,0 +1,12 @@ +def noi_import_command_parser(import_batch_size, subparsers): + noi_import_command = subparsers.add_parser( + "noi-import", + help=f"Import NOI with specified batch size: (default: {import_batch_size})", + ) + noi_import_command.add_argument( + "--batch-size", + type=int, + default=import_batch_size, + metavar="", + help=f"batch size (default: {import_batch_size})", + ) diff --git a/bin/migrate-oats-data/menu/commands/__init__.py b/bin/migrate-oats-data/menu/commands/__init__.py new file mode 100644 index 0000000000..e2a01516ad --- /dev/null +++ b/bin/migrate-oats-data/menu/commands/__init__.py @@ -0,0 +1,5 @@ +from .import_all import import_all +from .clean_all import clean_all +from .documents import * +from .applications import * +from .notice_of_intents import * \ No newline at end of file diff --git a/bin/migrate-oats-data/menu/commands/applications.py b/bin/migrate-oats-data/menu/commands/applications.py new file mode 100644 index 0000000000..3fc2bce7d5 --- /dev/null +++ b/bin/migrate-oats-data/menu/commands/applications.py @@ -0,0 +1,46 @@ +from applications import ( + process_applications, + process_alcs_application_prep_fields, + process_alcs_app_submissions, +) + + +def app_prep_import(console, args): + console.log("Beginning OATS -> ALCS app-prep import process") + with console.status( + "[bold green]App prep import (applications table update in ALCS)..." + ) as status: + if args.batch_size: + import_batch_size = args.batch_size + + console.log(f"Processing app-prep import in batch size = {import_batch_size}") + + process_alcs_application_prep_fields(batch_size=import_batch_size) + + +def application_import(console, args): + console.log("Beginning OATS -> ALCS application import process") + with console.status( + "[bold green]application import (application table update in ALCS)..." + ) as status: + if args.batch_size: + import_batch_size = args.batch_size + + console.log( + f"Processing applications import in batch size = {import_batch_size}" + ) + + process_applications(batch_size=import_batch_size) + + +def application_submission_import(console, args): + console.log("Beginning OATS -> ALCS app-sub import process") + with console.status( + "[bold green]App submission import (application_submission table update in ALCS)..." + ) as status: + if args.batch_size: + import_batch_size = args.batch_size + + console.log(f"Processing app-sub import in batch size = {import_batch_size}") + + process_alcs_app_submissions(batch_size=import_batch_size) diff --git a/bin/migrate-oats-data/menu/commands/clean_all.py b/bin/migrate-oats-data/menu/commands/clean_all.py new file mode 100644 index 0000000000..a3562d4d28 --- /dev/null +++ b/bin/migrate-oats-data/menu/commands/clean_all.py @@ -0,0 +1,32 @@ +from applications import clean_applications +from noi.notice_of_intent_migration import clean_notice_of_intent +from applications.submissions import clean_application_submission +from applications.application_submission_status_email import ( + clean_application_submission_status_emails, +) +from noi.noi_submission_status_email import ( + clean_application_submission_status_emails, +) +from noi import clean_notice_of_intents +from documents import ( + clean_application_documents, + clean_documents, + clean_noi_documents, +) + + +def clean_all(console, args): + with console.status("[bold green]Cleaning previous ETL...") as status: + console.log("Cleaning data:") + # this will be enabled once application import is ready + + clean_application_documents() + clean_noi_documents() + clean_documents() + clean_application_submission() + clean_applications() + clean_notice_of_intents() + clean_application_submission_status_emails(), + clean_application_submission_status_emails(), + + console.log("Done") diff --git a/bin/migrate-oats-data/menu/commands/documents.py b/bin/migrate-oats-data/menu/commands/documents.py new file mode 100644 index 0000000000..ab62657b75 --- /dev/null +++ b/bin/migrate-oats-data/menu/commands/documents.py @@ -0,0 +1,42 @@ +from documents import ( + process_documents, + process_documents_noi, + process_application_documents, + process_noi_documents, +) + + +def document_import(console, args): + console.log("Beginning OATS -> ALCS document-import process") + with console.status("[bold green]Import OATS into ALCS...") as status: + if args.batch_size: + import_batch_size = args.batch_size + + console.log(f"Processing documents in batch size = {import_batch_size}") + + process_documents(batch_size=import_batch_size) + process_documents_noi(batch_size=import_batch_size) + + +def app_document_import(console, args): + console.log("Beginning OATS -> ALCS app-document-import process") + with console.status("[bold green]Link application documents in ALCS...") as status: + if args.batch_size: + import_batch_size = args.batch_size + + console.log( + f"Processing application documents in batch size = {import_batch_size}" + ) + + process_application_documents(batch_size=import_batch_size) + + +def noi_document_import(console, args): + console.log("Beginning OATS -> ALCS noi-document-import process") + with console.status("[bold green]Link application documents in ALCS...") as status: + if args.batch_size: + import_batch_size = args.batch_size + + console.log(f"Processing NOI documents in batch size = {import_batch_size}") + + process_noi_documents(batch_size=import_batch_size) diff --git a/bin/migrate-oats-data/menu/commands/import_all.py b/bin/migrate-oats-data/menu/commands/import_all.py new file mode 100644 index 0000000000..6c6eea70df --- /dev/null +++ b/bin/migrate-oats-data/menu/commands/import_all.py @@ -0,0 +1,68 @@ +from applications import ( + process_applications, + process_alcs_application_prep_fields, +) +from noi.notice_of_intent_migration import ( + init_notice_of_intents, + process_notice_of_intent, +) +from applications.submissions import process_alcs_app_submissions +from applications.application_submission_status_email import ( + process_application_submission_status_emails, +) +from noi.noi_submission_status_email import ( + process_notice_of_intent_submission_status_emails, +) +from noi import ( + process_notice_of_intent, +) +from documents import ( + process_application_documents, + process_noi_documents, +) + + +def import_all(console, args): + console.log("Beginning OATS -> ALCS import process") + + with console.status("[bold green]Import OATS into ALCS...") as status: + console.log("Init applications:") + # this will be enabled once application import is ready + + if args and args.batch_size: + import_batch_size = args.batch_size + + console.log("Batching applications:") + process_applications(batch_size=import_batch_size) + + console.log("Init NOIs:") + init_notice_of_intents(batch_size=import_batch_size) + + # TODO Liam question: which process_documents_noi or process_noi_documents is the correct one to keep? + # console.log("Processing NOI specific documents:") + # process_documents_noi(batch_size=import_batch_size) + + # console.log("Processing documents:") + # process_documents(batch_size=import_batch_size) + + console.log("Processing application documents:") + process_application_documents(batch_size=import_batch_size) + + console.log("Processing NOI documents:") + process_noi_documents(batch_size=import_batch_size) + + console.log("Processing application prep:") + process_alcs_application_prep_fields(batch_size=import_batch_size) + + console.log("Processing application submission:") + process_alcs_app_submissions(batch_size=import_batch_size) + + console.log("Processing notice of intents") + process_notice_of_intent(batch_size=import_batch_size) + + # NOTE: both process_application_submission_status_emails(), process_notice_of_intent_submission_status_emails() + # must be the last ones in the migrate etl + console.log("Processing submission status emails") + process_application_submission_status_emails() + process_notice_of_intent_submission_status_emails() + console.log("Done") diff --git a/bin/migrate-oats-data/menu/commands/notice_of_intents.py b/bin/migrate-oats-data/menu/commands/notice_of_intents.py new file mode 100644 index 0000000000..3033686765 --- /dev/null +++ b/bin/migrate-oats-data/menu/commands/notice_of_intents.py @@ -0,0 +1,18 @@ +from noi.notice_of_intent_migration import ( + init_notice_of_intents, + process_notice_of_intent, +) + + +def notice_of_intent_import(console, args): + console.log("Beginning OATS -> ALCS NOI import process") + with console.status( + "[bold green]NOI import (notice_of_intent table update in ALCS)..." + ) as status: + if args.batch_size: + import_batch_size = args.batch_size + + console.log(f"Processing NOI import in batch size = {import_batch_size}") + + init_notice_of_intents(batch_size=import_batch_size) + process_notice_of_intent(batch_size=import_batch_size) diff --git a/bin/migrate-oats-data/menu/menu.py b/bin/migrate-oats-data/menu/menu.py new file mode 100644 index 0000000000..f26f446288 --- /dev/null +++ b/bin/migrate-oats-data/menu/menu.py @@ -0,0 +1,58 @@ +import sys, argparse +from .command_parser.application_command_parser import ( + application_import_command_parser, + app_prep_import_command_parser, + app_sub_import_command_parser, +) +from .command_parser.notice_of_intent_command_parser import noi_import_command_parser +from .command_parser.document_command_parser import ( + document_import_command_parser, + application_document_import_command_parser, + document_noi_import_command_parser, + noi_document_import_command_parser, +) +from common import BATCH_UPLOAD_SIZE + +import_batch_size = BATCH_UPLOAD_SIZE + + +def _import_command_parser(subparsers): + import_command = subparsers.add_parser( + "import", + help=f"Starts OATS -> ALCS import process with specified batch size: (default: {import_batch_size})", + ) + import_command.add_argument( + "--batch-size", + type=int, + default=import_batch_size, + metavar="", + help=f"batch size (default: {import_batch_size})", + ) + import_command.set_defaults(func=import_batch_size) + + +def setup_menu_args_parser(import_batch_size): + parser = argparse.ArgumentParser(description="OATS ETL utility") + + # Add subcommands to parser + subparsers = parser.add_subparsers(dest="command") + application_import_command_parser(import_batch_size, subparsers) + document_import_command_parser(import_batch_size, subparsers) + application_document_import_command_parser(import_batch_size, subparsers) + app_prep_import_command_parser(import_batch_size, subparsers) + noi_import_command_parser(import_batch_size, subparsers) + document_noi_import_command_parser(import_batch_size, subparsers) + noi_document_import_command_parser(import_batch_size, subparsers) + _import_command_parser(subparsers) + app_sub_import_command_parser(import_batch_size, subparsers) + + subparsers.add_parser("clean", help="Clean all imported data") + + # Print help message if user specifies --help or -h flag + if "-h" in sys.argv or "--help" in sys.argv: + parser.print_help() + sys.exit(0) + + # Parse arguments and call command function + args = parser.parse_args() + return args diff --git a/bin/migrate-oats-data/migrate.py b/bin/migrate-oats-data/migrate.py index 0a7b88ae44..aea6fd9cb4 100644 --- a/bin/migrate-oats-data/migrate.py +++ b/bin/migrate-oats-data/migrate.py @@ -1,210 +1,24 @@ -import sys, logging, argparse +import logging +from rich.console import Console -from documents import ( - process_documents, - process_application_documents, - clean_application_documents, - clean_documents, - process_documents_noi, - process_noi_documents, - clean_noi_documents, +from menu import setup_menu_args_parser +from menu.commands import ( + import_all, + clean_all, + document_import, + app_document_import, + app_prep_import, + application_import, + application_submission_import, + noi_document_import, + notice_of_intent_import, ) -from rich.console import Console from db import connection_pool -from applications import ( - process_applications, - clean_applications, - process_alcs_application_prep_fields, -) -from noi.notice_of_intent_init import ( - process_nois, - clean_nois, -) -from applications.submissions import ( - process_alcs_app_submissions, - clean_application_submission, -) from common import BATCH_UPLOAD_SIZE -from applications.application_submission_status_email import ( - process_application_submission_status_emails, - clean_notice_of_intent_submission_status_emails, -) -from noi.noi_submission_status_email import ( - process_notice_of_intent_submission_status_emails, - clean_application_submission_status_emails, -) -from noi.oats_to_alcs_notice_of_intent_table_etl import ( - process_alcs_notice_of_intent_fee_fields, -) import_batch_size = BATCH_UPLOAD_SIZE -# TODO tidy import menu setup - - -def application_import_command_parser(import_batch_size, subparsers): - application_import_command = subparsers.add_parser( - "application-import", - help=f"Import application with specified batch size: (default: {import_batch_size})", - ) - application_import_command.add_argument( - "--batch-size", - type=int, - default=import_batch_size, - metavar="", - help=f"batch size (default: {import_batch_size})", - ) - application_import_command.set_defaults(func=process_applications) - - -def document_import_command_parser(import_batch_size, subparsers): - document_import_command = subparsers.add_parser( - "document-import", - help=f"Import documents with specified batch size: (default: {import_batch_size})", - ) - document_import_command.add_argument( - "--batch-size", - type=int, - default=import_batch_size, - metavar="", - help=f"batch size (default: {import_batch_size})", - ) - document_import_command.set_defaults(func=process_documents) - - -def document_noi_import_command_parser(import_batch_size, subparsers): - document_noi_import_command = subparsers.add_parser( - "document-noi-import", - help=f"Import documents_noi with specified batch size: (default: {import_batch_size})", - ) - document_noi_import_command.add_argument( - "--batch-size", - type=int, - default=import_batch_size, - metavar="", - help=f"batch size (default: {import_batch_size})", - ) - document_noi_import_command.set_defaults(func=process_documents_noi) - - -def application_document_import_command_parser(import_batch_size, subparsers): - application_document_import_command = subparsers.add_parser( - "app-document-import", - help=f"Links imported documents with application documents specified batch size: (default: {import_batch_size})", - ) - application_document_import_command.add_argument( - "--batch-size", - type=int, - default=import_batch_size, - metavar="", - help=f"batch size (default: {import_batch_size})", - ) - application_document_import_command.set_defaults(func=import_batch_size) - - -def noi_document_import_command_parser(import_batch_size, subparsers): - noi_document_import_command = subparsers.add_parser( - "noi-document-import", - help=f"Links imported documents with noi documents specified batch size: (default: {import_batch_size})", - ) - noi_document_import_command.add_argument( - "--batch-size", - type=int, - default=import_batch_size, - metavar="", - help=f"batch size (default: {import_batch_size})", - ) - noi_document_import_command.set_defaults(func=import_batch_size) - - -def app_prep_import_command_parser(import_batch_size, subparsers): - app_prep_import_command = subparsers.add_parser( - "app-prep-import", - help=f"Import App prep into ALCS (update applications table) in specified batch size: (default: {import_batch_size})", - ) - app_prep_import_command.add_argument( - "--batch-size", - type=int, - default=import_batch_size, - metavar="", - help=f"batch size (default: {import_batch_size})", - ) - app_prep_import_command.set_defaults(func=import_batch_size) - - -def noi_import_command_parser(import_batch_size, subparsers): - noi_import_command = subparsers.add_parser( - "noi-import", - help=f"Import NOI with specified batch size: (default: {import_batch_size})", - ) - noi_import_command.add_argument( - "--batch-size", - type=int, - default=import_batch_size, - metavar="", - help=f"batch size (default: {import_batch_size})", - ) - noi_import_command.set_defaults(func=import_batch_size) - - -def import_command_parser(subparsers): - import_command = subparsers.add_parser( - "import", - help=f"Starts OATS -> ALCS import process with specified batch size: (default: {import_batch_size})", - ) - import_command.add_argument( - "--batch-size", - type=int, - default=import_batch_size, - metavar="", - help=f"batch size (default: {import_batch_size})", - ) - import_command.set_defaults(func=import_batch_size) - - -def app_sub_import_command_parser(import_batch_size, subparsers): - app_sub_import_command = subparsers.add_parser( - "app-sub-import", - help=f"Import App submission into ALCS (update application_submission table) in specified batch size: (default: {import_batch_size})", - ) - app_sub_import_command.add_argument( - "--batch-size", - type=int, - default=import_batch_size, - metavar="", - help=f"batch size (default: {import_batch_size})", - ) - app_sub_import_command.set_defaults(func=import_batch_size) - - -def setup_menu_args_parser(import_batch_size): - parser = argparse.ArgumentParser(description="OATS ETL utility") - - # Add subcommands to parser - subparsers = parser.add_subparsers(dest="command") - application_import_command_parser(import_batch_size, subparsers) - document_import_command_parser(import_batch_size, subparsers) - application_document_import_command_parser(import_batch_size, subparsers) - app_prep_import_command_parser(import_batch_size, subparsers) - noi_import_command_parser(import_batch_size, subparsers) - document_noi_import_command_parser(import_batch_size, subparsers) - noi_document_import_command_parser(import_batch_size, subparsers) - import_command_parser(subparsers) - app_sub_import_command_parser(import_batch_size, subparsers) - - subparsers.add_parser("clean", help="Clean all imported data") - - # Print help message if user specifies --help or -h flag - if "-h" in sys.argv or "--help" in sys.argv: - parser.print_help() - sys.exit(0) - - # Parse arguments and call command function - args = parser.parse_args() - return args - - if __name__ == "__main__": args = setup_menu_args_parser(import_batch_size) @@ -217,158 +31,23 @@ def setup_menu_args_parser(import_batch_size): # Call function corresponding to selected action using match statement match args.command: case "import": - console.log("Beginning OATS -> ALCS import process") - with console.status("[bold green]Import OATS into ALCS...") as status: - console.log("Processing applications:") - # this will be enabled once application import is ready - - if args and args.batch_size: - import_batch_size = args.batch_size - - console.log("Batching applications:") - process_applications(batch_size=import_batch_size) - - console.log("Processing NOIs:") - process_nois(batch_size=import_batch_size) - - # TODO Liam question which process_documents_noi or process_noi_documents is the correct one to keep? - # console.log("Processing NOI specific documents:") - # process_documents_noi(batch_size=import_batch_size) - - # console.log("Processing documents:") - # process_documents(batch_size=import_batch_size) - - console.log("Processing application documents:") - process_application_documents(batch_size=import_batch_size) - - console.log("Processing NOI documents:") - process_noi_documents(batch_size=import_batch_size) - - console.log("Processing application prep:") - process_alcs_application_prep_fields(batch_size=import_batch_size) - - console.log("Processing application submission:") - process_alcs_app_submissions(batch_size=import_batch_size) - - console.log("Processing notice of intent fields") - process_alcs_notice_of_intent_fee_fields( - batch_size=import_batch_size - ) - - # NOTE: both process_application_submission_status_emails(), process_notice_of_intent_submission_status_emails() - # must be the last ones in the migrate etl - console.log("Processing submission status emails") - process_application_submission_status_emails() - process_notice_of_intent_submission_status_emails() - console.log("Done") + import_all(console, args) case "clean": - with console.status("[bold green]Cleaning previous ETL...") as status: - console.log("Cleaning data:") - # this will be enabled once application import is ready - - clean_application_documents() - clean_noi_documents() - clean_documents() - clean_application_submission() - clean_applications() - clean_nois() - clean_notice_of_intent_submission_status_emails(), - clean_application_submission_status_emails(), - - console.log("Done") + clean_all(console, args) case "document-import": - console.log("Beginning OATS -> ALCS document-import process") - with console.status("[bold green]Import OATS into ALCS...") as status: - if args.batch_size: - import_batch_size = args.batch_size - - console.log( - f"Processing documents in batch size = {import_batch_size}" - ) - - process_documents(batch_size=import_batch_size) - process_documents_noi(batch_size=import_batch_size) + document_import(console, args) case "app-document-import": - console.log("Beginning OATS -> ALCS app-document-import process") - with console.status( - "[bold green]Link application documents in ALCS..." - ) as status: - if args.batch_size: - import_batch_size = args.batch_size - - console.log( - f"Processing application documents in batch size = {import_batch_size}" - ) - - process_application_documents(batch_size=import_batch_size) + app_document_import(console, args) case "app-prep-import": - console.log("Beginning OATS -> ALCS app-prep import process") - with console.status( - "[bold green]App prep import (applications table update in ALCS)..." - ) as status: - if args.batch_size: - import_batch_size = args.batch_size - - console.log( - f"Processing app-prep import in batch size = {import_batch_size}" - ) - - process_alcs_application_prep_fields(batch_size=import_batch_size) + app_prep_import(console, args) case "noi-import": - console.log("Beginning OATS -> ALCS NOI import process") - with console.status( - "[bold green]NOI import (notice_of_intent table update in ALCS)..." - ) as status: - if args.batch_size: - import_batch_size = args.batch_size - - console.log( - f"Processing NOI import in batch size = {import_batch_size}" - ) - - process_nois(batch_size=import_batch_size) - process_alcs_notice_of_intent_fee_fields( - batch_size=import_batch_size - ) + notice_of_intent_import(console, args) case "application-import": - console.log("Beginning OATS -> ALCS application import process") - with console.status( - "[bold green]application import (application table update in ALCS)..." - ) as status: - if args.batch_size: - import_batch_size = args.batch_size - - console.log( - f"Processing applications import in batch size = {import_batch_size}" - ) - - process_applications(batch_size=import_batch_size) + application_import(console, args) case "noi-document-import": - console.log("Beginning OATS -> ALCS noi-document-import process") - with console.status( - "[bold green]Link application documents in ALCS..." - ) as status: - if args.batch_size: - import_batch_size = args.batch_size - - console.log( - f"Processing NOI documents in batch size = {import_batch_size}" - ) - - process_noi_documents(batch_size=import_batch_size) + noi_document_import(console, args) case "app-sub-import": - console.log("Beginning OATS -> ALCS app-sub import process") - with console.status( - "[bold green]App submission import (application_submission table update in ALCS)..." - ) as status: - if args.batch_size: - import_batch_size = args.batch_size - - console.log( - f"Processing app-sub import in batch size = {import_batch_size}" - ) - - process_alcs_app_submissions(batch_size=import_batch_size) + application_submission_import(console, args) finally: if connection_pool: diff --git a/bin/migrate-oats-data/noi/__init__.py b/bin/migrate-oats-data/noi/__init__.py index 4b8338fdd6..9348bb9dcd 100644 --- a/bin/migrate-oats-data/noi/__init__.py +++ b/bin/migrate-oats-data/noi/__init__.py @@ -1 +1,2 @@ -from .notice_of_intent_init import * \ No newline at end of file +from .notice_of_intent_init import init_notice_of_intents, clean_notice_of_intents +from .notice_of_intent_migration import process_notice_of_intent diff --git a/bin/migrate-oats-data/noi/notice_of_intent_init.py b/bin/migrate-oats-data/noi/notice_of_intent_init.py index 82a6094c2f..4febbd3958 100644 --- a/bin/migrate-oats-data/noi/notice_of_intent_init.py +++ b/bin/migrate-oats-data/noi/notice_of_intent_init.py @@ -1,5 +1,6 @@ from db import inject_conn_pool + def noi_insert_query(number_of_rows_to_insert): nois_to_insert = ",".join(["%s"] * number_of_rows_to_insert) return f""" @@ -16,13 +17,11 @@ def noi_insert_query(number_of_rows_to_insert): type_code = EXCLUDED.type_code """ + @inject_conn_pool -def process_nois(conn=None, batch_size=10000): +def init_notice_of_intents(conn=None, batch_size=10000): with conn.cursor() as cursor: - - with open( - "noi/sql/insert_noi_count.sql", "r", encoding="utf-8" - ) as sql_file: + with open("noi/sql/insert_noi_count.sql", "r", encoding="utf-8") as sql_file: count_query = sql_file.read() cursor.execute(count_query) count_total = cursor.fetchone()[0] @@ -38,19 +37,17 @@ def process_nois(conn=None, batch_size=10000): cursor.execute( f"{application_sql} WHERE ng.noi_application_id > {last_application_id} ORDER by ng.noi_application_id;" ) - + rows = cursor.fetchmany(batch_size) if not rows: break - try: + try: applications_to_be_inserted_count = len(rows) - insert_query = noi_insert_query( - applications_to_be_inserted_count - ) + insert_query = noi_insert_query(applications_to_be_inserted_count) cursor.execute(insert_query, rows) conn.commit() - + last_application_id = rows[-1][0] successful_inserts_count = ( successful_inserts_count + applications_to_be_inserted_count @@ -58,23 +55,22 @@ def process_nois(conn=None, batch_size=10000): print( f"retrieved/inserted items count: {applications_to_be_inserted_count}; total successfully inserted/updated NOIs so far {successful_inserts_count}; last inserted noi_applidation_id: {last_application_id}" - ) except Exception as e: conn.rollback() print("Error", e) - failed_inserts = count_total - successful_inserts_count - last_application_id = last_application_id +1 - + failed_inserts = count_total - successful_inserts_count + last_application_id = last_application_id + 1 print("Total amount of successful inserts:", successful_inserts_count) - print("Total failed inserts:", failed_inserts) + print("Total failed inserts:", failed_inserts) + @inject_conn_pool -def clean_nois(conn=None): +def clean_notice_of_intents(conn=None): print("Start NOI cleaning") with conn.cursor() as cursor: cursor.execute( "DELETE FROM alcs.notice_of_intent a WHERE a.audit_created_by = 'oats_etl'" ) - print(f"Deleted items count = {cursor.rowcount}") \ No newline at end of file + print(f"Deleted items count = {cursor.rowcount}") diff --git a/bin/migrate-oats-data/noi/notice_of_intent_migration.py b/bin/migrate-oats-data/noi/notice_of_intent_migration.py new file mode 100644 index 0000000000..7f8e058562 --- /dev/null +++ b/bin/migrate-oats-data/noi/notice_of_intent_migration.py @@ -0,0 +1,17 @@ +from .oats_to_alcs_notice_of_intent_table_etl.oats_to_alcs_notice_of_intent_table_etl import ( + process_alcs_notice_of_intent_base_fields, +) +from .notice_of_intent_init import init_notice_of_intents, clean_notice_of_intents + + +def init_notice_of_intent(batch_size): + init_notice_of_intents(batch_size=batch_size) + + +def clean_notice_of_intent(): + clean_notice_of_intents() + + +def process_notice_of_intent(batch_size): + process_alcs_notice_of_intent_base_fields(batch_size=batch_size) + # place the rest notice of intent processing function here diff --git a/bin/migrate-oats-data/noi/oats_to_alcs_notice_of_intent_table_etl/__init__.py b/bin/migrate-oats-data/noi/oats_to_alcs_notice_of_intent_table_etl/__init__.py index 4b40e6e129..c47d9f3610 100644 --- a/bin/migrate-oats-data/noi/oats_to_alcs_notice_of_intent_table_etl/__init__.py +++ b/bin/migrate-oats-data/noi/oats_to_alcs_notice_of_intent_table_etl/__init__.py @@ -1 +1,3 @@ -from .oats_to_alcs_notice_of_intent_table_etl import process_alcs_notice_of_intent_fee_fields +from .oats_to_alcs_notice_of_intent_table_etl import ( + process_alcs_notice_of_intent_base_fields, +) diff --git a/bin/migrate-oats-data/noi/oats_to_alcs_notice_of_intent_table_etl/oats_to_alcs_notice_of_intent_table_etl.py b/bin/migrate-oats-data/noi/oats_to_alcs_notice_of_intent_table_etl/oats_to_alcs_notice_of_intent_table_etl.py index 3898d44659..e2d4a2f55d 100644 --- a/bin/migrate-oats-data/noi/oats_to_alcs_notice_of_intent_table_etl/oats_to_alcs_notice_of_intent_table_etl.py +++ b/bin/migrate-oats-data/noi/oats_to_alcs_notice_of_intent_table_etl/oats_to_alcs_notice_of_intent_table_etl.py @@ -26,7 +26,7 @@ class OatsToAlcsAgCap(Enum): @inject_conn_pool -def process_alcs_notice_of_intent_fee_fields(conn=None, batch_size=BATCH_UPLOAD_SIZE): +def process_alcs_notice_of_intent_base_fields(conn=None, batch_size=BATCH_UPLOAD_SIZE): """ decision_date is imported separately """