From b5eddf82ed731df724576ea096f01b169e87bfdc Mon Sep 17 00:00:00 2001 From: Mekhti Date: Mon, 18 Sep 2023 13:49:46 -0700 Subject: [PATCH 1/5] set email_sent date on statuses for submission imported by etl --- .../applications/app_prep.py | 11 +++-- .../submission_status_email/__init__.py | 1 + .../submission_status_email.py | 40 +++++++++++++++++++ bin/migrate-oats-data/common/__init__.py | 1 + bin/migrate-oats-data/common/constants.py | 1 + .../noi/submission_status_email/__init__.py | 1 + .../submission_status_email.py | 40 +++++++++++++++++++ 7 files changed, 91 insertions(+), 4 deletions(-) create mode 100644 bin/migrate-oats-data/applications/submission_status_email/__init__.py create mode 100644 bin/migrate-oats-data/applications/submission_status_email/submission_status_email.py create mode 100644 bin/migrate-oats-data/common/constants.py create mode 100644 bin/migrate-oats-data/noi/submission_status_email/__init__.py create mode 100644 bin/migrate-oats-data/noi/submission_status_email/submission_status_email.py diff --git a/bin/migrate-oats-data/applications/app_prep.py b/bin/migrate-oats-data/applications/app_prep.py index 73840819f4..01d346eef9 100644 --- a/bin/migrate-oats-data/applications/app_prep.py +++ b/bin/migrate-oats-data/applications/app_prep.py @@ -1,9 +1,6 @@ from common import ( ALRChangeCode, AlcsNfuTypeCode, - AlcsNfuSubTypeCode, - OatsCapabilitySourceCode, - OatsAgriCapabilityCodes, OATS_NFU_SUBTYPES, AlcsAgCap, AlcsAgCapSource, @@ -47,12 +44,14 @@ class OatsToAlcsAgCap(Enum): S = AlcsAgCap.Secondary.value U = AlcsAgCap.Unclassified.value + class OatsLegislationCodes(Enum): SEC_30_1 = AlcsApplicantType.Land_owner.value SEC_29_1 = AlcsApplicantType.LFNG.value SEC_17_3 = AlcsApplicantType.Land_owner.value SEC_17_1 = AlcsApplicantType.LFNG.value + @inject_conn_pool def process_alcs_application_prep_fields(conn=None, batch_size=BATCH_UPLOAD_SIZE): """ @@ -242,8 +241,8 @@ def mapOatsToAlcsAppPrep(data): return data -def mapOatsToAlcsLegislationCode(data): +def mapOatsToAlcsLegislationCode(data): if data["legislation_code"]: data["legislation_code"] = str( OatsLegislationCodes[data["legislation_code"]].value @@ -251,6 +250,7 @@ def mapOatsToAlcsLegislationCode(data): return data + def get_update_query(unique_fields): # unique_fields takes input from called function and appends to query query = """ @@ -271,6 +271,7 @@ def get_update_query(unique_fields): """ return query.format(unique_fields=unique_fields) + def get_update_query_for_nfu(): unique_fields = """, nfu_use_type = %(nonfarm_use_type_code)s, @@ -278,6 +279,7 @@ def get_update_query_for_nfu(): proposal_end_date = %(nonfarm_use_end_date)s""" return get_update_query(unique_fields) + def get_update_query_for_nar(): # naruSubtype is a part of submission, import there unique_fields = """, @@ -305,6 +307,7 @@ def get_update_query_for_other(): unique_fields = """""" return get_update_query(unique_fields) + def map_oats_to_alcs_nfu_subtypes(nfu_type_code, nfu_subtype_code): for dict_obj in OATS_NFU_SUBTYPES: if str(dict_obj["type_key"]) == str(nfu_type_code) and str( diff --git a/bin/migrate-oats-data/applications/submission_status_email/__init__.py b/bin/migrate-oats-data/applications/submission_status_email/__init__.py new file mode 100644 index 0000000000..5d23579b3f --- /dev/null +++ b/bin/migrate-oats-data/applications/submission_status_email/__init__.py @@ -0,0 +1 @@ +from .submission_status_email import * diff --git a/bin/migrate-oats-data/applications/submission_status_email/submission_status_email.py b/bin/migrate-oats-data/applications/submission_status_email/submission_status_email.py new file mode 100644 index 0000000000..928ebb095f --- /dev/null +++ b/bin/migrate-oats-data/applications/submission_status_email/submission_status_email.py @@ -0,0 +1,40 @@ +from db import inject_conn_pool +import traceback +from common import OATS_ETL_USER + + +@inject_conn_pool +def process_application_submission_status_emails(conn=None): + update_query = f""" + UPDATE application_submission_to_submission_status status + SET email_sent_date = '0001-01-01 01:00:00.000 -0700' + FROM application_submission app_sub + WHERE status.submission_uuid = app_sub.uuid + AND app_sub.audit_created_by = {OATS_ETL_USER} + """ + + try: + with conn.cursor() as cursor: + cursor.execute(update_query) + except Exception as error: + print("".join(traceback.format_exception(None, error, error.__traceback__))) + cursor.close() + conn.close() + + +@inject_conn_pool +def clean_application_submission_status_emails(conn=None): + update_query = f""" + UPDATE application_submission_to_submission_status status + SET email_sent_date = NULL + FROM application_submission app_sub + WHERE status.submission_uuid = app_sub.uuid + AND app_sub.audit_created_by = {OATS_ETL_USER} + """ + try: + with conn.cursor() as cursor: + cursor.execute(update_query) + except Exception as error: + print("".join(traceback.format_exception(None, error, error.__traceback__))) + cursor.close() + conn.close() diff --git a/bin/migrate-oats-data/common/__init__.py b/bin/migrate-oats-data/common/__init__.py index 26c07e6d74..1ef347bddc 100644 --- a/bin/migrate-oats-data/common/__init__.py +++ b/bin/migrate-oats-data/common/__init__.py @@ -1,3 +1,4 @@ from .oats_application_code_values import * from .alcs_application_enum import * from .etl_logger import log_start, log_end +from .constants import * diff --git a/bin/migrate-oats-data/common/constants.py b/bin/migrate-oats-data/common/constants.py new file mode 100644 index 0000000000..46497ef9c6 --- /dev/null +++ b/bin/migrate-oats-data/common/constants.py @@ -0,0 +1 @@ +OATS_ETL_USER = "oats_etl" diff --git a/bin/migrate-oats-data/noi/submission_status_email/__init__.py b/bin/migrate-oats-data/noi/submission_status_email/__init__.py new file mode 100644 index 0000000000..5d23579b3f --- /dev/null +++ b/bin/migrate-oats-data/noi/submission_status_email/__init__.py @@ -0,0 +1 @@ +from .submission_status_email import * diff --git a/bin/migrate-oats-data/noi/submission_status_email/submission_status_email.py b/bin/migrate-oats-data/noi/submission_status_email/submission_status_email.py new file mode 100644 index 0000000000..c58f771c9b --- /dev/null +++ b/bin/migrate-oats-data/noi/submission_status_email/submission_status_email.py @@ -0,0 +1,40 @@ +from db import inject_conn_pool +import traceback +from common import OATS_ETL_USER + + +@inject_conn_pool +def process_application_submission_status_emails(conn=None): + update_query = f""" + UPDATE notice_of_intent_submission_to_submission_status status + SET email_sent_date = '0001-01-01 01:00:00.000 -0700' + FROM notice_of_intent_submission noi_sub + WHERE status.submission_uuid = noi_sub.uuid + AND noi_sub.audit_created_by = {OATS_ETL_USER} + """ + + try: + with conn.cursor() as cursor: + cursor.execute(update_query) + except Exception as error: + print("".join(traceback.format_exception(None, error, error.__traceback__))) + cursor.close() + conn.close() + + +@inject_conn_pool +def clean_application_submission_status_emails(conn=None): + update_query = f""" + UPDATE notice_of_intent_submission_to_submission_status status + SET email_sent_date = '0001-01-01 01:00:00.000 -0700' + FROM notice_of_intent_submission noi_sub + WHERE status.submission_uuid = noi_sub.uuid + AND noi_sub.audit_created_by = {OATS_ETL_USER} + """ + try: + with conn.cursor() as cursor: + cursor.execute(update_query) + except Exception as error: + print("".join(traceback.format_exception(None, error, error.__traceback__))) + cursor.close() + conn.close() From 7ec1c4a09d4edb4e018c5a91f5ef30ec53ab2022 Mon Sep 17 00:00:00 2001 From: Mekhti Date: Mon, 18 Sep 2023 15:13:56 -0700 Subject: [PATCH 2/5] rename modules, add email commands to migrate.py --- .../__init__.py | 0 .../submission_status_email.py | 2 +- bin/migrate-oats-data/migrate.py | 25 +++++++++++++++++-- .../__init__.py | 0 .../submission_status_email.py | 2 +- 5 files changed, 25 insertions(+), 4 deletions(-) rename bin/migrate-oats-data/applications/{submission_status_email => application_submission_status_email}/__init__.py (100%) rename bin/migrate-oats-data/applications/{submission_status_email => application_submission_status_email}/submission_status_email.py (95%) rename bin/migrate-oats-data/noi/{submission_status_email => noi_submission_status_email}/__init__.py (100%) rename bin/migrate-oats-data/noi/{submission_status_email => noi_submission_status_email}/submission_status_email.py (95%) diff --git a/bin/migrate-oats-data/applications/submission_status_email/__init__.py b/bin/migrate-oats-data/applications/application_submission_status_email/__init__.py similarity index 100% rename from bin/migrate-oats-data/applications/submission_status_email/__init__.py rename to bin/migrate-oats-data/applications/application_submission_status_email/__init__.py diff --git a/bin/migrate-oats-data/applications/submission_status_email/submission_status_email.py b/bin/migrate-oats-data/applications/application_submission_status_email/submission_status_email.py similarity index 95% rename from bin/migrate-oats-data/applications/submission_status_email/submission_status_email.py rename to bin/migrate-oats-data/applications/application_submission_status_email/submission_status_email.py index 928ebb095f..71fe72f222 100644 --- a/bin/migrate-oats-data/applications/submission_status_email/submission_status_email.py +++ b/bin/migrate-oats-data/applications/application_submission_status_email/submission_status_email.py @@ -23,7 +23,7 @@ def process_application_submission_status_emails(conn=None): @inject_conn_pool -def clean_application_submission_status_emails(conn=None): +def clean_notice_of_intent_submission_status_emails(conn=None): update_query = f""" UPDATE application_submission_to_submission_status status SET email_sent_date = NULL diff --git a/bin/migrate-oats-data/migrate.py b/bin/migrate-oats-data/migrate.py index 01501d78ad..1bfee1c74f 100644 --- a/bin/migrate-oats-data/migrate.py +++ b/bin/migrate-oats-data/migrate.py @@ -20,12 +20,21 @@ process_nois, clean_nois, ) -from submissions import( +from submissions import ( process_alcs_app_submissions, clean_application_submission, ) from constants import BATCH_UPLOAD_SIZE +from application_submission_status_email import ( + process_application_submission_status_emails, + clean_notice_of_intent_submission_status_emails, +) +from noi_submission_status_email import ( + process_notice_of_intent_submission_status_emails, + clean_application_submission_status_emails, +) + import_batch_size = BATCH_UPLOAD_SIZE # TODO tidy import menu setup @@ -60,6 +69,7 @@ def document_import_command_parser(import_batch_size, subparsers): ) document_import_command.set_defaults(func=process_documents) + def document_noi_import_command_parser(import_batch_size, subparsers): document_noi_import_command = subparsers.add_parser( "document-noi-import", @@ -89,6 +99,7 @@ def application_document_import_command_parser(import_batch_size, subparsers): ) application_document_import_command.set_defaults(func=import_batch_size) + def noi_document_import_command_parser(import_batch_size, subparsers): noi_document_import_command = subparsers.add_parser( "noi-document-import", @@ -118,6 +129,7 @@ def app_prep_import_command_parser(import_batch_size, subparsers): ) app_prep_import_command.set_defaults(func=import_batch_size) + def noi_import_command_parser(import_batch_size, subparsers): noi_import_command = subparsers.add_parser( "noi-import", @@ -132,6 +144,7 @@ def noi_import_command_parser(import_batch_size, subparsers): ) noi_import_command.set_defaults(func=import_batch_size) + def import_command_parser(subparsers): import_command = subparsers.add_parser( "import", @@ -146,6 +159,7 @@ def import_command_parser(subparsers): ) import_command.set_defaults(func=import_batch_size) + def app_sub_import_command_parser(import_batch_size, subparsers): app_sub_import_command = subparsers.add_parser( "app-sub-import", @@ -218,7 +232,7 @@ def setup_menu_args_parser(import_batch_size): process_documents(batch_size=import_batch_size) console.log("Processing NOI specific documents:") - process_documents_noi(batch_size=import_batch_size) + process_documents_noi(batch_size=import_batch_size) console.log("Processing application documents:") process_application_documents(batch_size=import_batch_size) @@ -232,6 +246,11 @@ def setup_menu_args_parser(import_batch_size): console.log("Processing application submission:") process_alcs_app_submissions(batch_size=import_batch_size) + # NOTE: both process_application_submission_status_emails(), process_notice_of_intent_submission_status_emails() + # must be the last ones in the migrate etl + console.log("Processing submission status emails") + process_application_submission_status_emails() + process_notice_of_intent_submission_status_emails() console.log("Done") case "clean": with console.status("[bold green]Cleaning previous ETL...") as status: @@ -244,6 +263,8 @@ def setup_menu_args_parser(import_batch_size): clean_application_submission() clean_applications() clean_nois() + clean_notice_of_intent_submission_status_emails(), + clean_notice_of_intent_submission_status_emails(), console.log("Done") case "document-import": diff --git a/bin/migrate-oats-data/noi/submission_status_email/__init__.py b/bin/migrate-oats-data/noi/noi_submission_status_email/__init__.py similarity index 100% rename from bin/migrate-oats-data/noi/submission_status_email/__init__.py rename to bin/migrate-oats-data/noi/noi_submission_status_email/__init__.py diff --git a/bin/migrate-oats-data/noi/submission_status_email/submission_status_email.py b/bin/migrate-oats-data/noi/noi_submission_status_email/submission_status_email.py similarity index 95% rename from bin/migrate-oats-data/noi/submission_status_email/submission_status_email.py rename to bin/migrate-oats-data/noi/noi_submission_status_email/submission_status_email.py index c58f771c9b..da2155c3a9 100644 --- a/bin/migrate-oats-data/noi/submission_status_email/submission_status_email.py +++ b/bin/migrate-oats-data/noi/noi_submission_status_email/submission_status_email.py @@ -4,7 +4,7 @@ @inject_conn_pool -def process_application_submission_status_emails(conn=None): +def process_notice_of_intent_submission_status_emails(conn=None): update_query = f""" UPDATE notice_of_intent_submission_to_submission_status status SET email_sent_date = '0001-01-01 01:00:00.000 -0700' From b211c76ca42314a29ce3d2970e6e19754a2cfb08 Mon Sep 17 00:00:00 2001 From: Mekhti Date: Mon, 18 Sep 2023 15:18:25 -0700 Subject: [PATCH 3/5] remove fileNumber length limit on search --- services/apps/alcs/src/alcs/search/search.dto.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/services/apps/alcs/src/alcs/search/search.dto.ts b/services/apps/alcs/src/alcs/search/search.dto.ts index ff084814b4..b617813718 100644 --- a/services/apps/alcs/src/alcs/search/search.dto.ts +++ b/services/apps/alcs/src/alcs/search/search.dto.ts @@ -85,7 +85,6 @@ export class PagingRequestDto { export class SearchRequestDto extends PagingRequestDto { @IsString() @IsOptional() - @MinLength(3) fileNumber?: string; @IsString() From 031f5d683ba452191aa14eeffae82d4ade7dd7cc Mon Sep 17 00:00:00 2001 From: Mekhti Date: Mon, 18 Sep 2023 15:25:44 -0700 Subject: [PATCH 4/5] adjust query --- .../submission_status_email.py | 14 ++++---- bin/migrate-oats-data/migrate.py | 36 +++++++++---------- .../submission_status_email.py | 16 ++++----- 3 files changed, 33 insertions(+), 33 deletions(-) diff --git a/bin/migrate-oats-data/applications/application_submission_status_email/submission_status_email.py b/bin/migrate-oats-data/applications/application_submission_status_email/submission_status_email.py index 71fe72f222..6955d24626 100644 --- a/bin/migrate-oats-data/applications/application_submission_status_email/submission_status_email.py +++ b/bin/migrate-oats-data/applications/application_submission_status_email/submission_status_email.py @@ -6,11 +6,11 @@ @inject_conn_pool def process_application_submission_status_emails(conn=None): update_query = f""" - UPDATE application_submission_to_submission_status status - SET email_sent_date = '0001-01-01 01:00:00.000 -0700' - FROM application_submission app_sub + UPDATE alcs.application_submission_to_submission_status status + SET email_sent_date = '0001-01-01 07:00:00.000 -0700' + FROM alcs.application_submission app_sub WHERE status.submission_uuid = app_sub.uuid - AND app_sub.audit_created_by = {OATS_ETL_USER} + AND app_sub.audit_created_by = '{OATS_ETL_USER}' """ try: @@ -25,11 +25,11 @@ def process_application_submission_status_emails(conn=None): @inject_conn_pool def clean_notice_of_intent_submission_status_emails(conn=None): update_query = f""" - UPDATE application_submission_to_submission_status status + UPDATE alcs.application_submission_to_submission_status status SET email_sent_date = NULL - FROM application_submission app_sub + FROM alcs.application_submission app_sub WHERE status.submission_uuid = app_sub.uuid - AND app_sub.audit_created_by = {OATS_ETL_USER} + AND app_sub.audit_created_by = '{OATS_ETL_USER}' """ try: with conn.cursor() as cursor: diff --git a/bin/migrate-oats-data/migrate.py b/bin/migrate-oats-data/migrate.py index 1bfee1c74f..1507dc3ab7 100644 --- a/bin/migrate-oats-data/migrate.py +++ b/bin/migrate-oats-data/migrate.py @@ -26,11 +26,11 @@ ) from constants import BATCH_UPLOAD_SIZE -from application_submission_status_email import ( +from applications.application_submission_status_email import ( process_application_submission_status_emails, clean_notice_of_intent_submission_status_emails, ) -from noi_submission_status_email import ( +from noi.noi_submission_status_email import ( process_notice_of_intent_submission_status_emails, clean_application_submission_status_emails, ) @@ -222,29 +222,29 @@ def setup_menu_args_parser(import_batch_size): if args and args.batch_size: import_batch_size = args.batch_size - console.log("Batching applications:") - process_applications(batch_size=import_batch_size) + # console.log("Batching applications:") + # process_applications(batch_size=import_batch_size) - console.log("Processing NOIs:") - process_nois(batch_size=import_batch_size) + # console.log("Processing NOIs:") + # process_nois(batch_size=import_batch_size) - console.log("Processing documents:") - process_documents(batch_size=import_batch_size) + # console.log("Processing documents:") + # process_documents(batch_size=import_batch_size) - console.log("Processing NOI specific documents:") - process_documents_noi(batch_size=import_batch_size) + # console.log("Processing NOI specific documents:") + # process_documents_noi(batch_size=import_batch_size) - console.log("Processing application documents:") - process_application_documents(batch_size=import_batch_size) + # console.log("Processing application documents:") + # process_application_documents(batch_size=import_batch_size) - console.log("Processing NOI documents:") - process_noi_documents(batch_size=import_batch_size) + # console.log("Processing NOI documents:") + # process_noi_documents(batch_size=import_batch_size) - console.log("Processing application prep:") - process_alcs_application_prep_fields(batch_size=import_batch_size) + # console.log("Processing application prep:") + # process_alcs_application_prep_fields(batch_size=import_batch_size) - console.log("Processing application submission:") - process_alcs_app_submissions(batch_size=import_batch_size) + # console.log("Processing application submission:") + # process_alcs_app_submissions(batch_size=import_batch_size) # NOTE: both process_application_submission_status_emails(), process_notice_of_intent_submission_status_emails() # must be the last ones in the migrate etl diff --git a/bin/migrate-oats-data/noi/noi_submission_status_email/submission_status_email.py b/bin/migrate-oats-data/noi/noi_submission_status_email/submission_status_email.py index da2155c3a9..748f078943 100644 --- a/bin/migrate-oats-data/noi/noi_submission_status_email/submission_status_email.py +++ b/bin/migrate-oats-data/noi/noi_submission_status_email/submission_status_email.py @@ -6,11 +6,11 @@ @inject_conn_pool def process_notice_of_intent_submission_status_emails(conn=None): update_query = f""" - UPDATE notice_of_intent_submission_to_submission_status status - SET email_sent_date = '0001-01-01 01:00:00.000 -0700' - FROM notice_of_intent_submission noi_sub + UPDATE alcs.notice_of_intent_submission_to_submission_status status + SET email_sent_date = '0001-01-01 07:00:00.000 -0700' + FROM alcs.notice_of_intent_submission noi_sub WHERE status.submission_uuid = noi_sub.uuid - AND noi_sub.audit_created_by = {OATS_ETL_USER} + AND noi_sub.audit_created_by = '{OATS_ETL_USER}' """ try: @@ -25,11 +25,11 @@ def process_notice_of_intent_submission_status_emails(conn=None): @inject_conn_pool def clean_application_submission_status_emails(conn=None): update_query = f""" - UPDATE notice_of_intent_submission_to_submission_status status - SET email_sent_date = '0001-01-01 01:00:00.000 -0700' - FROM notice_of_intent_submission noi_sub + UPDATE alcs.notice_of_intent_submission_to_submission_status status + SET email_sent_date = NULL + FROM alcs.notice_of_intent_submission noi_sub WHERE status.submission_uuid = noi_sub.uuid - AND noi_sub.audit_created_by = {OATS_ETL_USER} + AND noi_sub.audit_created_by = '{OATS_ETL_USER}' """ try: with conn.cursor() as cursor: From b49342ad1d5371cc86f58920d6a1724339147bbd Mon Sep 17 00:00:00 2001 From: Mekhti Date: Mon, 18 Sep 2023 15:41:41 -0700 Subject: [PATCH 5/5] uncomment migrate --- bin/migrate-oats-data/migrate.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/bin/migrate-oats-data/migrate.py b/bin/migrate-oats-data/migrate.py index 1507dc3ab7..12407d9042 100644 --- a/bin/migrate-oats-data/migrate.py +++ b/bin/migrate-oats-data/migrate.py @@ -222,29 +222,29 @@ def setup_menu_args_parser(import_batch_size): if args and args.batch_size: import_batch_size = args.batch_size - # console.log("Batching applications:") - # process_applications(batch_size=import_batch_size) + console.log("Batching applications:") + process_applications(batch_size=import_batch_size) - # console.log("Processing NOIs:") - # process_nois(batch_size=import_batch_size) + console.log("Processing NOIs:") + process_nois(batch_size=import_batch_size) - # console.log("Processing documents:") - # process_documents(batch_size=import_batch_size) + console.log("Processing documents:") + process_documents(batch_size=import_batch_size) - # console.log("Processing NOI specific documents:") - # process_documents_noi(batch_size=import_batch_size) + console.log("Processing NOI specific documents:") + process_documents_noi(batch_size=import_batch_size) - # console.log("Processing application documents:") - # process_application_documents(batch_size=import_batch_size) + console.log("Processing application documents:") + process_application_documents(batch_size=import_batch_size) - # console.log("Processing NOI documents:") - # process_noi_documents(batch_size=import_batch_size) + console.log("Processing NOI documents:") + process_noi_documents(batch_size=import_batch_size) - # console.log("Processing application prep:") - # process_alcs_application_prep_fields(batch_size=import_batch_size) + console.log("Processing application prep:") + process_alcs_application_prep_fields(batch_size=import_batch_size) - # console.log("Processing application submission:") - # process_alcs_app_submissions(batch_size=import_batch_size) + console.log("Processing application submission:") + process_alcs_app_submissions(batch_size=import_batch_size) # NOTE: both process_application_submission_status_emails(), process_notice_of_intent_submission_status_emails() # must be the last ones in the migrate etl