Merge branch 'main' into PIMS-1968

.gitignore

@@ -45,3 +45,6 @@ yarn-error.log*
 
 jest/latestTestRun.json
 package-lock.json
+
+# .csv and txt files created by script
+tools/propertyCheck/data/*

express-api/package.json

@@ -29,7 +29,7 @@
     "cors": "2.8.5",
     "dotenv": "16.4.1",
     "express": "4.19.2",
-    "express-rate-limit": "7.3.0",
+    "express-rate-limit": "7.4.0",
     "morgan": "1.10.0",
     "multer": "1.4.5-lts.1",
     "node-cron": "3.0.3",
@@ -55,13 +55,13 @@
     "@types/jest": "29.5.10",
     "@types/morgan": "1.9.9",
     "@types/multer": "1.4.11",
-    "@types/node": "20.14.1",
+    "@types/node": "22.0.0",
     "@types/node-cron": "3.0.11",
     "@types/nunjucks": "3.2.6",
     "@types/supertest": "6.0.2",
     "@types/swagger-ui-express": "4.1.6",
-    "@typescript-eslint/eslint-plugin": "7.17.0",
-    "@typescript-eslint/parser": "7.17.0",
+    "@typescript-eslint/eslint-plugin": "7.18.0",
+    "@typescript-eslint/parser": "7.18.0",
     "cross-env": "7.0.3",
     "eslint": "8.57.0",
     "eslint-config-prettier": "9.1.0",

react-app/package.json

@@ -23,8 +23,8 @@
     "@mui/icons-material": "5.16.0",
     "@mui/lab": "5.0.0-alpha.170",
     "@mui/material": "5.16.0",
-    "@mui/x-data-grid": "7.11.0",
-    "@mui/x-date-pickers": "7.11.0",
+    "@mui/x-data-grid": "7.12.0",
+    "@mui/x-date-pickers": "7.12.0",
     "@turf/turf": "7.0.0",
     "dayjs": "1.11.10",
     "node-xlsx": "0.24.0",
@@ -33,12 +33,12 @@
     "react-error-boundary": "4.0.12",
     "react-hook-form": "7.52.0",
     "react-leaflet": "4.2.1",
-    "react-router-dom": "6.25.0",
+    "react-router-dom": "6.26.0",
     "supercluster": "8.0.1",
     "use-supercluster": "1.2.0"
   },
   "devDependencies": {
-    "@babel/preset-env": "7.24.0",
+    "@babel/preset-env": "7.25.2",
     "@babel/preset-react": "7.24.1",
     "@testing-library/jest-dom": "6.4.2",
     "@testing-library/react": "16.0.0",
@@ -47,8 +47,8 @@
     "@types/leaflet": "1.9.9",
     "@types/react": "18.3.1",
     "@types/react-dom": "18.3.0",
-    "@typescript-eslint/eslint-plugin": "7.17.0",
-    "@typescript-eslint/parser": "7.17.0",
+    "@typescript-eslint/eslint-plugin": "7.18.0",
+    "@typescript-eslint/parser": "7.18.0",
     "@vitejs/plugin-react": "4.3.0",
     "eslint": "8.57.0",
     "eslint-config-prettier": "9.1.0",

tools/propertyCheck/README.md

@@ -0,0 +1,88 @@
+# Property Check Scripts
+
+## Purpose
+
+This script allows users to perform numerous tasks on a property dataset given. See flags section for full information on all the operations this script can perform. 
+This script was created to help resolve some bad data stored within PIMS. [Firework Cluster Ticket](https://citz-imb.atlassian.net/browse/PIMS-1681) was the start of this work and it spiraled from there.
+
+
+## Setup
+
+1. Python must be installed on your local system for this to work.
+   - To check if Python is installed on your system try running the following on a command line:
+
+     `python --version` OR `python3 --version`
+
+     If you get a result similar to "Python x.x.x" you may procede.
+
+     If you get a result similar to "python: command not found" please navigate to the [Python download site](https://www.python.org/downloads/) to download.
+
+2. shapely must be installed to run with flags: run, p, and l. To install please run
+
+    `pip install shapely` OR `pip3 install shapely`
+
+    If you are unsure if the package is correctly installed try running the script. If shapely is not installed you will get a message similar to "Must have the following packages to run: ['shapely']." 
+
+Place the file you would like to process in the 'data' folder.
+3. Run the script with the following structure. Use python3 if that is what worked in step 1: 
+
+    `python(3) <path_to_property_check.py>property_check.py.py <flag1> <file1>`
+
+    If you would like to run with multiple flags and files include them in the same pattern proceeding the first flag and file. eg: 
+
+    `python3 property_check.py.py -run data.csv -m other_data.csv`
+
+    This command will run process 'run' on the file data.csv and the process 'm' on the file other_data.csv. 
+
+## Flags
+
+The following flags can be used when running the script. Note that each of the flags must be followed with a file name of a file stored within the 'data' folder. 
+
+### -run 
+
+Running the script followed by a '-run' flag will start the proess laid out in process_data.py. The scructure of this process can be viewed in [process_data.drawio.png](./process_data.drawio.png)
+
+Required Column(s): "Address1", "City", "Point", "PID"
+
+Example: `python3 property_check.py.py -run data_to_check.csv`
+
+### -m
+
+Running the script followed by a '-m' flag will go through the given file and remove any rows that have Ministries matching those found on line 21 in file property_check.py
+
+Required Column(s): "Ministry"
+
+Example: `python3 property_check.py.py -m data_with_many_ministries.csv`
+
+### -p
+
+Running the script followed by a '-p' flag will go through the given file and check the polygon returned from openmaps.gov.bc.ca by sending in the stored PID. Lines will be removed if the point stored falls within the polygon.
+
+Required Column(s): "Point", "PID"
+
+Example: `python3 property_check.py.py -p check_point_by_PID.csv`
+
+### -l
+
+Running the script followed by a '-l' flag will go through the given file and check the PID returned from openmaps.gov.bc.ca by sending in the stored point. Lines will be removed if the PID stored doesn't match the returned PID.
+
+Required Column(s): "Point", "PID"
+
+Example: `python3 property_check.py.py -l check_PID_by_point.csv`
+
+### -c
+
+Running the script followed by a '-c' flag will go through the given file and check the city (municipality) returned from openmaps.gov.bc.ca by sending in the stored PID. Lines will be removed if the city stored doesn't match the returned city.
+
+Required Column(s): "Point", "PID", "City"
+
+Example: `python3 property_check.py.py -c check_city_by_pid.csv`
+
+
+## Notes
+
+Any output files will be written into the data folder. 
+
+Any files used in the command line to run the script need to be in the data folder at the time that you run the script. 
+
+Don't include the path with '/data/' in the filename when running the script.

tools/propertyCheck/data/testing_data.csv

@@ -0,0 +1,12 @@
+PID,Name,Ministry,Point,Address1,City
+31117252,BLD 1000054 Port Alberni District Office,BC Hydro,POINT (-124.806 49.2499),4820 Wallace Street,Port Alberni
+1451227,01-234-11759-022 Gorge Road Hospital,Vancouver Island Health Authority,POINT (-123.106 49.2481),61 & 63 Gorge Road East,Victoria
+13635107,B0034603 Oil Storage Shed,Real Property Division,POINT (-116.835 49.6828),16332 Hwy. 3a,Crawford Bay
+4676351,Chetwynd Hosp Chetwynd General Hospital,Northern Health Authority,POINT (-121.6445 55.6963),5426 Hospital Road,Chetwynd
+293458,4206 BCT Facility,BC Transit ,POINT (-123.392 48.4816),4206 Commerce Circle,Victoria
+31414010,NE4 - Carpentry Shop,British Columbia Institute of Technology,POINT (-123.0008912086487 49.253366734828042),3700 Willingdon Avenue,Burnaby
+7266251,6 Burrard West Building,Vancouver Coastal Health Authority,POINT (-123.129 49.2806),1081 - Burrard Street,Vancouver
+6870571,"1,698.80 NSM Fort Nelson Campus",Northern Lights College,POINT (-122.708 58.8155),Box 8605504 Simpson Trail,Fort Nelson
+7678355,400 Gold Creek Timberframe Building/First Aid Centre--Gold Creek Campus--Classroom/Training,College of the Rockies,POINT (-126.646 54.3973),13th St. S,Cranbrook
+17903149,Green College - B East,University of BC,POINT (-123.256004 49.270833),6201 Cecil Green Park Road, Vancouver
+7553684,68 Adam Robertson Elementary,School District 8 Kootenay Lake,POINT (-119.828 49.203),PO Box 1549 421 9th Ave N,Creston

tools/propertyCheck/geocoder_connection.py

@@ -0,0 +1,89 @@
+""" Makes a call to the geo coder with the given address """
+import http.client
+import helpers
+
+def format_address(address):
+    """Takes in a string address and replaces all spaces with %20"""
+    mod_address = address.replace(" ", '%20')
+    return mod_address
+
+def call_geocoder(street_address, city, set_back):
+    """Makes a call to the geocoder and returns the result"""
+    conn = http.client.HTTPSConnection("geocoder.api.gov.bc.ca")
+    payload = ''
+    headers = {}
+    encoded_address = format_address(street_address + " " + city)
+    req_str = "/addresses.json?addressString=" + encoded_address + \
+        "BC&locationDescriptor=accessPoint&maxResults=1" + \
+        "&setBack=" + str(set_back) + "%0A&provinceCode=BC"
+    conn.request("GET", req_str , payload, headers)
+    res = conn.getresponse()
+    data = res.read()
+    return data.decode("utf-8")
+
+def process_data(in_props, good_data):
+    """If given two files that you want to pull 'good' data into"""
+    count = 0
+    log_li = []
+    prop_headers = in_props[0]
+    prop_address_index = prop_headers.index("Address1")
+    prop_city_index = prop_headers.index("City")
+    prop_pid_index = prop_headers.index("PID")
+    data_headers = good_data[0]
+    data_address_index = data_headers.index("Address")
+    data_city_index = data_headers.index("CITY")
+    data_pid_index = data_headers.index("PID")
+
+    for prop_index, prop_row in enumerate(in_props):
+        if prop_index == 0:
+            continue
+
+        p_address = prop_row[prop_address_index].lower()
+        p_city = prop_row[prop_city_index].lower()
+        p_pid = prop_row[prop_pid_index]
+
+        for data_index, data_row in enumerate(good_data):
+            if data_index == 0:
+                continue
+            d_address = data_row[data_address_index].lower()
+            d_city = data_row[data_city_index].lower()
+            d_pid = data_row[data_pid_index].replace("-", "")
+            d_pid = helpers.remove_leading_zeros(d_pid)
+
+            if p_address == d_address:
+
+                if p_city == d_city and p_pid == d_pid:
+                    break
+                key_word = ''
+                key_p = ""
+                key_d = ""
+                if p_city != d_city:
+                    key_word = 'city'
+                    key_p = p_city
+                    key_d = d_city
+                elif p_pid != d_pid:
+                    key_word = 'pid'
+                    key_p = p_pid
+                    key_d = d_pid
+                p = str(count) + ". Address: " + p_address + " match. " + key_word + " mismatch"
+                log_li.append(p)
+                log_li.append("  PIMS stored " + key_word + " " + key_p)
+                log_li.append("  Verified dataset " + key_word + " " + key_d)
+                count = count + 1
+
+    return log_li
+
+def get_point_from_geo_data(geo_data):
+    """Takes json object from geocoder and returns the point in (x, y) format"""
+
+    precision_points = geo_data["features"][0]["properties"]["precisionPoints"]
+    precision_point_int = int(precision_points)
+
+    if precision_point_int < 95:
+        return 0
+    point = geo_data["features"][0]["geometry"]["coordinates"]
+
+    x = point[0]
+    y = point[1]
+
+    return(x, y)