Skip to content

Commit

Permalink
Merge pull request #43 from ianco/master
Browse files Browse the repository at this point in the history 
Filter relationship audit based on active orgbook companies only
  • Loading branch information
@ianco
ianco authored Sep 5, 2024
2 parents 845cdcc + de93113 commit 51faabf
Show file tree
Hide file tree
Showing 2 changed files with 94 additions and 79 deletions.
142 changes: 78 additions & 64 deletions scripts/orgbook_data_audit.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,12 @@
import requests
import csv

from config import CORP_TYPES_IN_SCOPE, corp_num_with_prefix, bare_corp_num
from config import (
CORP_TYPES_IN_SCOPE,
LEAR_CORP_TYPES_IN_SCOPE,
corp_num_with_prefix,
bare_corp_num,
)
from rocketchat_hooks import log_error, log_warning, log_info


Expand Down Expand Up @@ -105,6 +110,11 @@ def compare_bc_reg_orgbook(
wrong_corp_reg_dt = []
wrong_corp_juris = []

if USE_LEAR:
corp_types_filter = LEAR_CORP_TYPES_IN_SCOPE
else:
corp_types_filter = CORP_TYPES_IN_SCOPE

# check if all the BC Reg corps are in orgbook (with the same corp type)
if USE_LEAR:
cmd_pfx = "Lear"
Expand All @@ -116,56 +126,58 @@ def compare_bc_reg_orgbook(
bc_reg_corp_type = bc_reg_corp_types[bc_reg_corp_num]
bc_reg_corp_name = bc_reg_corp_names[bc_reg_corp_num]
bc_reg_corp_info = bc_reg_corp_infos[bc_reg_corp_num]
if bare_corp_num(bc_reg_corp_num) in future_corps:
#print("Future corp ignore:", row["corp_num"])
pass
elif not bc_reg_corp_num in orgbook_corp_types:
# not in orgbook
error_msgs += "Topic not found for: " + bc_reg_corp_num + "\n"
missing_in_orgbook.append(bc_reg_corp_num)
error_cmds += "./manage -e prod queueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n"
pass
elif (not orgbook_corp_types[bc_reg_corp_num]) or (orgbook_corp_types[bc_reg_corp_num] != bc_reg_corp_type):
# in orgbook but has the wrong corp type in orgbook
error_msgs += "Corp Type mis-match for: " + bc_reg_corp_num + '; BC Reg: "'+bc_reg_corp_type+'", OrgBook: "'+orgbook_corp_types[bc_reg_corp_num]+'"' + "\n"
wrong_corp_type.append(bc_reg_corp_num)
error_cmds += "./manage -p bc -e prod deleteTopic " + bc_reg_corp_num + "\n"
error_cmds += "./manage -e prod requeueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n"
elif (orgbook_corp_names[bc_reg_corp_num].strip() != bc_reg_corp_name.strip()):
# in orgbook but has the wrong corp name in orgbook
error_msgs += "Corp Name mis-match for: " + bc_reg_corp_num + ' BC Reg: "'+bc_reg_corp_name+'", OrgBook: "'+orgbook_corp_names[bc_reg_corp_num]+'"' + "\n"
wrong_corp_name.append(bc_reg_corp_num)
error_cmds += "./manage -p bc -e prod deleteTopic " + bc_reg_corp_num + "\n"
error_cmds += "./manage -e prod requeueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n"
elif (orgbook_corp_infos[bc_reg_corp_num]["entity_status"] != bc_reg_corp_info["op_state_typ_cd"]):
# wrong entity status
error_msgs += "Corp Status mis-match for: " + bc_reg_corp_num + ' BC Reg: "'+bc_reg_corp_info["op_state_typ_cd"]+'", OrgBook: "'+orgbook_corp_infos[bc_reg_corp_num]["entity_status"]+'"' + "\n"
wrong_corp_status.append(bc_reg_corp_num)
error_cmds += "./manage -p bc -e prod deleteTopic " + bc_reg_corp_num + "\n"
error_cmds += "./manage -e prod requeueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n"
elif (orgbook_corp_infos[bc_reg_corp_num]["bus_num"].strip() != bc_reg_corp_info["bn_9"].strip()):
# wrong BN9 business number
error_msgs += "Business Number mis-match for: " + bc_reg_corp_num + ' BC Reg: "'+bc_reg_corp_info["bn_9"]+'", OrgBook: "'+orgbook_corp_infos[bc_reg_corp_num]["bus_num"]+'"' + "\n"
wrong_bus_num.append(bc_reg_corp_num)
error_cmds += "./manage -p bc -e prod deleteTopic " + bc_reg_corp_num + "\n"
error_cmds += "./manage -e prod requeueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n"
elif (not compare_dates(orgbook_corp_infos[bc_reg_corp_num]["registration_date"], bc_reg_corp_info["recognition_dts"], USE_LEAR=USE_LEAR)):
# wrong registration date
error_msgs += "Corp Registration Date mis-match for: " + bc_reg_corp_num + ' BC Reg: "'+bc_reg_corp_info["recognition_dts"]+'", OrgBook: "'+orgbook_corp_infos[bc_reg_corp_num]["registration_date"]+'"' + "\n"
wrong_corp_reg_dt.append(bc_reg_corp_num)
error_cmds += "./manage -p bc -e prod deleteTopic " + bc_reg_corp_num + "\n"
error_cmds += "./manage -e prod requeueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n"
elif (orgbook_corp_infos[bc_reg_corp_num]["home_jurisdiction"] != get_corp_jurisdiction(bc_reg_corp_info["corp_type"], bc_reg_corp_info["corp_class"], bc_reg_corp_info["can_jur_typ_cd"], bc_reg_corp_info["othr_juris_desc"])):
# wrong jurisdiction
calc_juris = get_corp_jurisdiction(bc_reg_corp_info["corp_type"], bc_reg_corp_info["corp_class"], bc_reg_corp_info["can_jur_typ_cd"], bc_reg_corp_info["othr_juris_desc"])
error_msgs += "Corp Jurisdiction mis-match for: " + bc_reg_corp_num + ' BC Reg: "'+calc_juris+'", OrgBook: "'+orgbook_corp_infos[bc_reg_corp_num]["home_jurisdiction"]+'"' + "\n"
wrong_corp_juris.append(bc_reg_corp_num)
error_cmds += "./manage -p bc -e prod deleteTopic " + bc_reg_corp_num + "\n"
error_cmds += "./manage -e prod requeueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n"
if bc_reg_corp_type in corp_types_filter:
if bare_corp_num(bc_reg_corp_num) in future_corps:
#print("Future corp ignore:", row["corp_num"])
pass
elif not bc_reg_corp_num in orgbook_corp_types:
# not in orgbook
error_msgs += "Topic not found for: " + bc_reg_corp_num + "\n"
missing_in_orgbook.append(bc_reg_corp_num)
error_cmds += "./manage -e prod queueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n"
pass
elif (not orgbook_corp_types[bc_reg_corp_num]) or (orgbook_corp_types[bc_reg_corp_num] != bc_reg_corp_type):
# in orgbook but has the wrong corp type in orgbook
error_msgs += "Corp Type mis-match for: " + bc_reg_corp_num + '; BC Reg: "'+bc_reg_corp_type+'", OrgBook: "'+orgbook_corp_types[bc_reg_corp_num]+'"' + "\n"
wrong_corp_type.append(bc_reg_corp_num)
error_cmds += "./manage -p bc -e prod deleteTopic " + bc_reg_corp_num + "\n"
error_cmds += "./manage -e prod requeueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n"
elif (orgbook_corp_names[bc_reg_corp_num].strip() != bc_reg_corp_name.strip()):
# in orgbook but has the wrong corp name in orgbook
error_msgs += "Corp Name mis-match for: " + bc_reg_corp_num + ' BC Reg: "'+bc_reg_corp_name+'", OrgBook: "'+orgbook_corp_names[bc_reg_corp_num]+'"' + "\n"
wrong_corp_name.append(bc_reg_corp_num)
error_cmds += "./manage -p bc -e prod deleteTopic " + bc_reg_corp_num + "\n"
error_cmds += "./manage -e prod requeueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n"
elif (orgbook_corp_infos[bc_reg_corp_num]["entity_status"] != bc_reg_corp_info["op_state_typ_cd"]):
# wrong entity status
error_msgs += "Corp Status mis-match for: " + bc_reg_corp_num + ' BC Reg: "'+bc_reg_corp_info["op_state_typ_cd"]+'", OrgBook: "'+orgbook_corp_infos[bc_reg_corp_num]["entity_status"]+'"' + "\n"
wrong_corp_status.append(bc_reg_corp_num)
error_cmds += "./manage -p bc -e prod deleteTopic " + bc_reg_corp_num + "\n"
error_cmds += "./manage -e prod requeueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n"
elif (orgbook_corp_infos[bc_reg_corp_num]["bus_num"].strip() != bc_reg_corp_info["bn_9"].strip()):
# wrong BN9 business number
error_msgs += "Business Number mis-match for: " + bc_reg_corp_num + ' BC Reg: "'+bc_reg_corp_info["bn_9"]+'", OrgBook: "'+orgbook_corp_infos[bc_reg_corp_num]["bus_num"]+'"' + "\n"
wrong_bus_num.append(bc_reg_corp_num)
error_cmds += "./manage -p bc -e prod deleteTopic " + bc_reg_corp_num + "\n"
error_cmds += "./manage -e prod requeueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n"
elif (not compare_dates(orgbook_corp_infos[bc_reg_corp_num]["registration_date"], bc_reg_corp_info["recognition_dts"], USE_LEAR=USE_LEAR)):
# wrong registration date
error_msgs += "Corp Registration Date mis-match for: " + bc_reg_corp_num + ' BC Reg: "'+bc_reg_corp_info["recognition_dts"]+'", OrgBook: "'+orgbook_corp_infos[bc_reg_corp_num]["registration_date"]+'"' + "\n"
wrong_corp_reg_dt.append(bc_reg_corp_num)
error_cmds += "./manage -p bc -e prod deleteTopic " + bc_reg_corp_num + "\n"
error_cmds += "./manage -e prod requeueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n"
elif (orgbook_corp_infos[bc_reg_corp_num]["home_jurisdiction"] != get_corp_jurisdiction(bc_reg_corp_info["corp_type"], bc_reg_corp_info["corp_class"], bc_reg_corp_info["can_jur_typ_cd"], bc_reg_corp_info["othr_juris_desc"])):
# wrong jurisdiction
calc_juris = get_corp_jurisdiction(bc_reg_corp_info["corp_type"], bc_reg_corp_info["corp_class"], bc_reg_corp_info["can_jur_typ_cd"], bc_reg_corp_info["othr_juris_desc"])
error_msgs += "Corp Jurisdiction mis-match for: " + bc_reg_corp_num + ' BC Reg: "'+calc_juris+'", OrgBook: "'+orgbook_corp_infos[bc_reg_corp_num]["home_jurisdiction"]+'"' + "\n"
wrong_corp_juris.append(bc_reg_corp_num)
error_cmds += "./manage -p bc -e prod deleteTopic " + bc_reg_corp_num + "\n"
error_cmds += "./manage -e prod requeueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n"

# now check if there are corps in orgbook that are *not* in BC Reg database
for orgbook_corp in orgbook_corp_types:
if not (orgbook_corp in bc_reg_corp_types):
bc_reg_corp_type = bc_reg_corp_types.get(orgbook_corp)
if (bc_reg_corp_type and bc_reg_corp_type in corp_types_filter) and not (orgbook_corp in bc_reg_corp_types):
missing_in_bcreg.append(orgbook_corp)
error_msgs += "OrgBook corp not in BC Reg: " + orgbook_corp + "\n"
error_cmds += "./manage -p bc -e prod deleteTopic " + orgbook_corp + "\n"
Expand Down Expand Up @@ -196,22 +208,24 @@ def compare_bc_reg_orgbook(
o_hash = source_id_1 + ":" + source_id_2
active_reln_hash[o_hash] = o_hash
for relation in bc_reg_owners:
f_hash = relation["firm"] + ":" + relation["owner"]
o_hash = relation["owner"] + ":" + relation["firm"]
if (not f_hash in reln_hash) and (not f_hash in active_reln_hash):
active_reln_list.append(f_hash)
error_msgs += "Missing relationship in OrgBook:" + f_hash + "\n"
reg_cmd = "queueOrgForRelnsUpdate"
corp_num = relation["owner"]
if corp_num.startswith('BC'):
corp_num = corp_num[2:]
error_cmds += "./manage -e prod " + reg_cmd + " " + corp_num + " " + relation['firm'] + "\n"
if (not f_hash in reln_hash) and (not o_hash in active_reln_hash):
active_reln_list.append(o_hash)
error_msgs += "Missing relationship in OrgBook:" + o_hash + "\n"
reg_cmd = "queueOrgForRelnsUpdateLear"
corp_num = relation["owner"]
error_cmds += "./manage -e prod " + reg_cmd + " " + relation['firm'] + " " + corp_num + "\n"
# only check if both firms are already in OrgBook
if relation["firm"] in bc_reg_corp_types and relation["owner"] in bc_reg_corp_types:
f_hash = relation["firm"] + ":" + relation["owner"]
o_hash = relation["owner"] + ":" + relation["firm"]
if (not f_hash in reln_hash) and (not f_hash in active_reln_hash):
active_reln_list.append(f_hash)
error_msgs += "Missing relationship in OrgBook:" + f_hash + "\n"
reg_cmd = "queueOrgForRelnsUpdate"
corp_num = relation["owner"]
if corp_num.startswith('BC'):
corp_num = corp_num[2:]
error_cmds += "./manage -e prod " + reg_cmd + " " + corp_num + " " + relation['firm'] + "\n"
if (not f_hash in reln_hash) and (not o_hash in active_reln_hash):
active_reln_list.append(o_hash)
error_msgs += "Missing relationship in OrgBook:" + o_hash + "\n"
reg_cmd = "queueOrgForRelnsUpdateLear"
corp_num = relation["owner"]
error_cmds += "./manage -e prod " + reg_cmd + " " + relation['firm'] + " " + corp_num + "\n"

corp_errors = (len(missing_in_orgbook) +
len(missing_in_bcreg) +
Expand Down
31 changes: 16 additions & 15 deletions scripts/orgbook_data_load.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -388,21 +388,22 @@ def get_orgbook_all_corps(USE_LEAR: bool = False):
cur.execute(sql4)
for row in cur:
# row[1] is the corp_type
if row[1] in corp_types_filter:
orgbook_corp_types[row[0]] = row[1]
corp_name = row[4] if (row[4] and 0 < len(row[4])) else row[3]
orgbook_corp_names[row[0]] = corp_name
write_corp = {
"corp_num": row[0],
"corp_type": row[1],
"registration_date": row[2],
"corp_name":corp_name,
"home_jurisdiction": row[5],
"entity_status": row[6],
"bus_num": row[7],
}
corp_writer.writerow(write_corp)
orgbook_corp_infos[row[0]] = write_corp
# if row[1] in corp_types_filter:
# load all orgs and check the filter when running the audit report
orgbook_corp_types[row[0]] = row[1]
corp_name = row[4] if (row[4] and 0 < len(row[4])) else row[3]
orgbook_corp_names[row[0]] = corp_name
write_corp = {
"corp_num": row[0],
"corp_type": row[1],
"registration_date": row[2],
"corp_name":corp_name,
"home_jurisdiction": row[5],
"entity_status": row[6],
"bus_num": row[7],
}
corp_writer.writerow(write_corp)
orgbook_corp_infos[row[0]] = write_corp
cur.close()
except (Exception) as error:
print(error)
Expand Down

0 comments on commit 51faabf

Please sign in to comment.