Feature/alcs 1129 - import data from OATS to ALCS -> notice_of_intent…

… table part 1 (#994)

import into the notice_of_intent table
fix the import issue with the document
bonus: open links in a new tab

bin/migrate-oats-data/applications/app_prep.py

@@ -10,7 +10,7 @@
     AlcsApplicantType,
 )
 from db import inject_conn_pool
-from constants import BATCH_UPLOAD_SIZE
+from common import BATCH_UPLOAD_SIZE
 from psycopg2.extras import execute_batch, RealDictCursor
 import traceback
 from enum import Enum
@@ -288,15 +288,13 @@ def get_update_query_for_nar():
 
 
 def get_update_query_for_exc():
-    # TODO Will be finalized in ALCS-834.
     # exclsn_app_type_code is out of scope. It is a part of submission
     unique_fields = """,
             incl_excl_applicant_type = %(legislation_code)s"""
     return get_update_query(unique_fields)
 
 
 def get_update_query_for_inc():
-    # TODO Will be finalized in ALCS-834.
     unique_fields = """,
             incl_excl_applicant_type = %(legislation_code)s"""
     return get_update_query(unique_fields)

bin/migrate-oats-data/submissions/app_submissions.py → bin/migrate-oats-data/applications/submissions/app_submissions.py

@@ -18,14 +18,15 @@
     add_soil_field,
 )
 from db import inject_conn_pool
-from constants import BATCH_UPLOAD_SIZE
+from common import BATCH_UPLOAD_SIZE
 from psycopg2.extras import execute_batch, RealDictCursor
 import traceback
 from enum import Enum
 import json
 
 etl_name = "alcs_app_sub"
 
+
 @inject_conn_pool
 def process_alcs_app_submissions(conn=None, batch_size=BATCH_UPLOAD_SIZE):
     """
@@ -39,7 +40,7 @@ def process_alcs_app_submissions(conn=None, batch_size=BATCH_UPLOAD_SIZE):
     log_start(etl_name)
     with conn.cursor(cursor_factory=RealDictCursor) as cursor:
         with open(
-            "submissions/sql/app_submission_count.sql",
+            "applications/submissions/sql/app_submission_count.sql",
             "r",
             encoding="utf-8",
         ) as sql_file:
@@ -53,7 +54,7 @@ def process_alcs_app_submissions(conn=None, batch_size=BATCH_UPLOAD_SIZE):
         last_submission_id = 0
 
         with open(
-            "submissions/sql/app_submission.sql",
+            "applications/submissions/sql/app_submission.sql",
             "r",
             encoding="utf-8",
         ) as sql_file:
@@ -68,14 +69,21 @@ def process_alcs_app_submissions(conn=None, batch_size=BATCH_UPLOAD_SIZE):
                 if not rows:
                     break
                 try:
-
                     direction_data = get_direction_data(rows, cursor)
                     subdiv_data = get_subdiv_data(rows, cursor)
                     soil_data = get_soil_data(rows, cursor)
 
                     submissions_to_be_inserted_count = len(rows)
 
-                    insert_app_sub_records(conn, batch_size, cursor, rows, direction_data, subdiv_data, soil_data)
+                    insert_app_sub_records(
+                        conn,
+                        batch_size,
+                        cursor,
+                        rows,
+                        direction_data,
+                        subdiv_data,
+                        soil_data,
+                    )
 
                     successful_inserts_count = (
                         successful_inserts_count + submissions_to_be_inserted_count
@@ -99,7 +107,10 @@ def process_alcs_app_submissions(conn=None, batch_size=BATCH_UPLOAD_SIZE):
     print("Total failed inserts:", failed_inserts)
     log_end(etl_name)
 
-def insert_app_sub_records(conn, batch_size, cursor, rows, direction_data, subdiv_data, soil_data):
+
+def insert_app_sub_records(
+    conn, batch_size, cursor, rows, direction_data, subdiv_data, soil_data
+):
     """
     Function to insert submission records in batches.
 
@@ -110,7 +121,7 @@ def insert_app_sub_records(conn, batch_size, cursor, rows, direction_data, subdi
     rows (list): Rows of data to insert in the database.
     direction_data (dict): Dictionary of adjacent parcel data
     subdiv_data: dictionary of subdivision data lists
-    soil_data: dictonary of soil element data.
+    soil_data: dictionary of soil element data.
 
     Returns:
     None: Commits the changes to the database.
@@ -140,21 +151,22 @@ def insert_app_sub_records(conn, batch_size, cursor, rows, direction_data, subdi
     if len(other_data_list) > 0:
         execute_batch(
             cursor,
-            get_insert_query("",""),
+            get_insert_query("", ""),
             other_data_list,
             page_size=batch_size,
         )
 
     conn.commit()
 
+
 def prepare_app_sub_data(app_sub_raw_data_list, direction_data, subdiv_data, soil_data):
     """
     This function prepares different lists of data based on the 'alr_change_code' field of each data dict in 'app_sub_raw_data_list'.
 
     :param app_sub_raw_data_list: A list of raw data dictionaries.
     :param direction_data: A dictionary of adjacent parcel data.
     :param subdiv_data: dictionary of subdivision data lists.
-    :param soil_data: dictonary of soil element data.
+    :param soil_data: dictionary of soil element data.
     :return: Five lists, each containing dictionaries from 'app_sub_raw_data_list' and 'direction_data' grouped based on the 'alr_change_code' field
 
     Detailed Workflow:
@@ -173,25 +185,28 @@ def prepare_app_sub_data(app_sub_raw_data_list, direction_data, subdiv_data, soi
     for row in app_sub_raw_data_list:
         data = dict(row)
         data = add_direction_field(data)
-        data = add_subdiv(data,json)
+        data = add_subdiv(data, json)
         data = add_soil_field(data)
-        if data['alr_appl_component_id'] in subdiv_data:
+        if data["alr_appl_component_id"] in subdiv_data:
             data = map_subdiv_lots(data, subdiv_data, json)
         if data["alr_application_id"] in direction_data:
             data = map_direction_values(data, direction_data)
         if data["alr_appl_component_id"] in soil_data:
             data = map_soil_data(data, soil_data)
         if data["alr_change_code"] == ALRChangeCode.NFU.value:
             nfu_data_list.append(data)
-        elif data["alr_change_code"] == ALRChangeCode.EXC.value or data["alr_change_code"] == ALRChangeCode.INC.value:
+        elif (
+            data["alr_change_code"] == ALRChangeCode.EXC.value
+            or data["alr_change_code"] == ALRChangeCode.INC.value
+        ):
             inc_exc_data_list.append(data)
         else:
             other_data_list.append(data)
 
     return nfu_data_list, other_data_list, inc_exc_data_list
 
 
-def get_insert_query(unique_fields,unique_values):
+def get_insert_query(unique_fields, unique_values):
     # unique_fields takes input from called function and appends to query
     query = """
                 INSERT INTO alcs.application_submission (
@@ -233,6 +248,7 @@ def get_insert_query(unique_fields,unique_values):
     """
     return query.format(unique_fields=unique_fields, unique_values=unique_values)
 
+
 def get_insert_query_for_nfu():
     unique_fields = """, nfu_hectares,
                         nfu_will_import_fill,
@@ -254,35 +270,40 @@ def get_insert_query_for_nfu():
                         %(fill_duration_unit)s,
                         %(fill_area)s
                     """
-    return get_insert_query(unique_fields,unique_values)
+    return get_insert_query(unique_fields, unique_values)
+
 
 def get_insert_query_for_inc_exc():
     unique_fields = ", incl_excl_hectares"
     unique_values = ", %(alr_area)s"
-    return get_insert_query(unique_fields,unique_values)
+    return get_insert_query(unique_fields, unique_values)
+
 
 def get_direction_data(rows, cursor):
     # runs query to get direction data and creates a dict based on alr_application_id
     adj_rows = get_directions_rows(rows, cursor)
     direction_data = create_direction_dict(adj_rows)
     return direction_data
 
+
 def get_subdiv_data(rows, cursor):
-    # runs query to get subdivision data and creates a dictionaly based on alr_appl_component_id with a list of plots
+    # runs query to get subdivision data and creates a dictionary based on alr_appl_component_id with a list of plots
     subdiv_rows = get_subdiv_rows(rows, cursor)
     subdiv_data = create_subdiv_dict(subdiv_rows)
     return subdiv_data
 
+
 def get_soil_data(rows, cursor):
     soil_rows = get_soil_rows(rows, cursor)
     soil_data = create_soil_dict(soil_rows)
     return soil_data
 
+
 @inject_conn_pool
 def clean_application_submission(conn=None):
     print("Start application_submission cleaning")
     with conn.cursor() as cursor:
         cursor.execute(
             "DELETE FROM alcs.application_submission a WHERE a.audit_created_by = 'oats_etl'"
         )
-        print(f"Deleted items count = {cursor.rowcount}")
+        print(f"Deleted items count = {cursor.rowcount}")

bin/migrate-oats-data/common/constants.py

@@ -1 +1,2 @@
 OATS_ETL_USER = "oats_etl"
+BATCH_UPLOAD_SIZE = 1000

bin/migrate-oats-data/common/etl_logger.py

@@ -4,9 +4,7 @@
 etl_log_file_name = "etl_log.txt"
 
 
-def log_start(
-    etl_name="Not specified",
-):
+def log_start(etl_name="Not specified", etl_log_file_name=etl_log_file_name):
     data = {
         "etl_name": etl_name,
         "start_time": datetime.now().isoformat(),
@@ -21,6 +19,7 @@ def log_end(
     etl_name="Not specified",
     error_msg=None,
     error_log=None,
+    etl_log_file_name=etl_log_file_name,
 ):
     data = {
         "etl_name": etl_name,

bin/migrate-oats-data/migrate.py

@@ -16,15 +16,15 @@
     clean_applications,
     process_alcs_application_prep_fields,
 )
-from noi import (
+from noi.notice_of_intent_init import (
     process_nois,
     clean_nois,
 )
-from submissions import (
+from applications.submissions import (
     process_alcs_app_submissions,
     clean_application_submission,
 )
-from constants import BATCH_UPLOAD_SIZE
+from common import BATCH_UPLOAD_SIZE
 
 from applications.application_submission_status_email import (
     process_application_submission_status_emails,
@@ -34,6 +34,9 @@
     process_notice_of_intent_submission_status_emails,
     clean_application_submission_status_emails,
 )
+from noi.oats_to_alcs_notice_of_intent_table_etl import (
+    process_alcs_notice_of_intent_fee_fields,
+)
 
 import_batch_size = BATCH_UPLOAD_SIZE
 
@@ -228,11 +231,12 @@ def setup_menu_args_parser(import_batch_size):
                     console.log("Processing NOIs:")
                     process_nois(batch_size=import_batch_size)
 
-                    console.log("Processing documents:")
-                    process_documents(batch_size=import_batch_size)
+                    # TODO Liam question which process_documents_noi or process_noi_documents is the correct one to keep?
+                    # console.log("Processing NOI specific documents:")
+                    # process_documents_noi(batch_size=import_batch_size)
 
-                    console.log("Processing NOI specific documents:")
-                    process_documents_noi(batch_size=import_batch_size)
+                    # console.log("Processing documents:")
+                    # process_documents(batch_size=import_batch_size)
 
                     console.log("Processing application documents:")
                     process_application_documents(batch_size=import_batch_size)
@@ -246,6 +250,11 @@ def setup_menu_args_parser(import_batch_size):
                     console.log("Processing application submission:")
                     process_alcs_app_submissions(batch_size=import_batch_size)
 
+                    console.log("Processing notice of intent fields")
+                    process_alcs_notice_of_intent_fee_fields(
+                        batch_size=import_batch_size
+                    )
+
                     # NOTE: both process_application_submission_status_emails(), process_notice_of_intent_submission_status_emails()
                     #       must be the last ones in the migrate etl
                     console.log("Processing submission status emails")
@@ -264,7 +273,7 @@ def setup_menu_args_parser(import_batch_size):
                     clean_applications()
                     clean_nois()
                     clean_notice_of_intent_submission_status_emails(),
-                    clean_notice_of_intent_submission_status_emails(),
+                    clean_application_submission_status_emails(),
 
                     console.log("Done")
             case "document-import":
@@ -318,6 +327,9 @@ def setup_menu_args_parser(import_batch_size):
                     )
 
                     process_nois(batch_size=import_batch_size)
+                    process_alcs_notice_of_intent_fee_fields(
+                        batch_size=import_batch_size
+                    )
             case "application-import":
                 console.log("Beginning OATS -> ALCS application import process")
                 with console.status(

bin/migrate-oats-data/noi/__init__.py

@@ -1 +1 @@
-from .noi import *
+from .notice_of_intent_init import *

bin/migrate-oats-data/noi/oats_to_alcs_notice_of_intent_table_etl/__init__.py

@@ -0,0 +1 @@
+from .oats_to_alcs_notice_of_intent_table_etl import process_alcs_notice_of_intent_fee_fields