From 63f0c83cef7f3c0204f2e0eda0fe9ac2dde3fd0c Mon Sep 17 00:00:00 2001 From: abimichel <136123575+abimichel@users.noreply.github.com> Date: Tue, 13 Feb 2024 17:49:47 -0800 Subject: [PATCH] update data_replication_parametrized_audit_os.py Signed-off-by: abimichel <136123575+abimichel@users.noreply.github.com> --- data_replication_parametrized_audit_os.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/data_replication_parametrized_audit_os.py b/data_replication_parametrized_audit_os.py index e1a6b8d..ae22c45 100644 --- a/data_replication_parametrized_audit_os.py +++ b/data_replication_parametrized_audit_os.py @@ -114,6 +114,8 @@ def extract_from_oracle(table_name,source_schema,customsql_ind,customsql_query): oracle_connection = OrcPool.acquire() oracle_cursor = oracle_connection.cursor() try: + current_time = datetime.datetime.now() + print("Replication Start:", current_time) if customsql_ind == "Y": # Use placeholders in the query and bind the table name as a parameter sql_query=customsql_query @@ -148,6 +150,9 @@ def load_into_postgres(table_name, data,target_schema): insert_query = f'INSERT INTO {target_schema}.{table_name} VALUES ({", ".join(["%s"] * len(data[0]))})' #insert_query = f'INSERT INTO {target_schema}.{table_name} VALUES %s' + current_time = datetime.datetime.now() + print("Replication End:", current_time) + # Use execute_batch for efficient batch insert with postgres_connection.cursor() as cursor: # Prepare the data as a list of tuples @@ -185,7 +190,7 @@ def load_data_from_src_tgt(table_name,source_schema,target_schema,customsql_ind, # Main ETL process active_tables_rows =get_active_tables(mstr_schema,app_name) print(active_tables_rows) - tables_to_extract = [(row[2],row[1],row[3],row[9],row[10]) for row in active_tables_rows] + tables_to_extract = [(row[2],row[1],row[3],row[8],row[9]) for row in active_tables_rows] print(f"tables to extract are {tables_to_extract}") print(f'No of concurrent tasks:{concurrent_tasks}')