From ffb2ff84d1c57f2225b0a2cef0bfa9f798646683 Mon Sep 17 00:00:00 2001 From: Yingting Chen Date: Thu, 31 Oct 2024 15:12:27 +1300 Subject: [PATCH] feat: plugin changes for nz_imagery_survey_index --- buildings/gui/reference_data.py | 49 ++++- buildings/gui/reference_data.ui | 25 +++ buildings/reference_data/other_reference.py | 197 ++++++++++++++++++++ 3 files changed, 267 insertions(+), 4 deletions(-) create mode 100644 buildings/reference_data/other_reference.py diff --git a/buildings/gui/reference_data.py b/buildings/gui/reference_data.py index f121a9cc..5532da2f 100644 --- a/buildings/gui/reference_data.py +++ b/buildings/gui/reference_data.py @@ -19,7 +19,7 @@ from qgis.PyQt.QtGui import QIcon from buildings.gui.error_dialog import ErrorDialog -from buildings.reference_data import topo50, admin_bdys +from buildings.reference_data import topo50, admin_bdys, other_reference from buildings.sql import buildings_bulk_load_select_statements as bulk_load_select from buildings.sql import buildings_reference_select_statements as reference_select from buildings.utilities import database as db @@ -52,6 +52,7 @@ "protected_areas_polygons", "coastlines_and_islands", "suburb_locality", + "nz_imagery_survey_index", ] DATASET_STATSNZ = ["territorial_authority"] @@ -70,6 +71,8 @@ ] DATASET_ADMIN_BDYS = ["suburb_locality", "territorial_authority"] +DATASET_OTHER = ["nz_imagery_survey_index"] + class UpdateReferenceData(QFrame, FORM_CLASS): def __init__(self, dockwidget, parent=None): @@ -100,6 +103,7 @@ def __init__(self, dockwidget, parent=None): # set up signals and slots self.grbx_topo.toggled.connect(self.check_all_topo) self.grbx_admin.toggled.connect(self.check_all_admin) + self.grbx_other.toggled.connect(self.check_all_other) self.btn_exit.clicked.connect(self.exit_clicked) self.btn_update.clicked.connect( partial(self.update_clicked, commit_status=True) @@ -116,6 +120,7 @@ def enable_checkboxes(self): """Enable frame""" self.grbx_topo.setEnabled(1) self.grbx_admin.setEnabled(1) + self.grbx_other.setEnabled(1) self.chbx_canals.setEnabled(1) self.chbx_coastline_and_islands.setEnabled(1) self.chbx_lagoons.setEnabled(1) @@ -129,6 +134,7 @@ def enable_checkboxes(self): self.chbx_protected_areas.setEnabled(1) self.chbx_suburbs.setEnabled(1) self.chbx_ta.setEnabled(1) + self.chbx_imagery.setEnabled(1) self.btn_update.setEnabled(1) # clear message self.lb_message.setText("") @@ -137,6 +143,7 @@ def disable_checkboxes(self): """Disable frame (when outlines dataset in progress)""" self.grbx_topo.setDisabled(1) self.grbx_admin.setDisabled(1) + self.grbx_other.setDisabled(1) self.chbx_canals.setDisabled(1) self.chbx_coastline_and_islands.setDisabled(1) self.chbx_lagoons.setDisabled(1) @@ -150,6 +157,7 @@ def disable_checkboxes(self): self.chbx_protected_areas.setDisabled(1) self.chbx_suburbs.setDisabled(1) self.chbx_ta.setDisabled(1) + self.chbx_imagery.setDisabled(1) self.btn_update.setDisabled(1) # add message self.lb_message.setText( @@ -171,8 +179,13 @@ def btn_status_clicked(self): return if dataset in DATASET_TOPO50: status = topo50.check_status_topo50(api_key, dataset) - else: + elif dataset in DATASET_ADMIN_BDYS: status = admin_bdys.check_status_admin_bdys(api_key, dataset) + else: + status = other_reference.check_status_imagery_survey_index( + api_key, dataset + ) + row_tbl = tbl.rowCount() tbl.setRowCount(row_tbl + 1) tbl.setItem(row_tbl, 0, QTableWidgetItem(status["dataset"])) @@ -241,6 +254,9 @@ def update_clicked(self, commit_status=True): # territorial authority and grid if self.chbx_ta.isChecked(): self.admin_bdy_layer_processing("territorial_authority") + # nz imagery survey index + if self.chbx_imagery.isChecked(): + self.other_layer_processing("nz_imagery_survey_index") # create log for this update if len(self.updates) > 0: @@ -251,10 +267,10 @@ def update_clicked(self, commit_status=True): # final message box if self.message == "": self.message = "No layers were updated." - self.msgbox.setText(self.message) - self.msgbox.exec_() if commit_status: self.db.commit_open_cursor() + self.msgbox.setText(self.message) + self.msgbox.exec_() @pyqtSlot() def exit_clicked(self): @@ -299,6 +315,18 @@ def check_all_admin(self): box.setChecked(False) box.setEnabled(1) + @pyqtSlot() + def check_all_other(self): + """Called when combobox to check all admin layers is toggled""" + if self.grbx_other.isChecked(): + for box in self.grbx_other.findChildren(QCheckBox): + box.setChecked(True) + box.setEnabled(1) + else: + for box in self.grbx_other.findChildren(QCheckBox): + box.setChecked(False) + box.setEnabled(1) + def message_box(self): return QMessageBox( QMessageBox.Information, "Note", self.message, buttons=QMessageBox.Ok @@ -349,6 +377,19 @@ def admin_bdy_layer_processing(self, dataset): else: self.update_message(status, dataset) + def other_layer_processing(self, dataset): + """Processes to run for other layers""" + api_key = self.check_api_key(dataset) + if api_key is None: + return + if dataset == "nz_imagery_survey_index": + status = other_reference.update_imagery_survey_index( + api_key, dataset, self.db + ) + self.update_message(status, dataset) + if status == "updated": + self.updates.append(dataset) + def check_api_key(self, layer): # check for API key if layer in DATASET_LINZ: diff --git a/buildings/gui/reference_data.ui b/buildings/gui/reference_data.ui index e427666a..afdb1532 100644 --- a/buildings/gui/reference_data.ui +++ b/buildings/gui/reference_data.ui @@ -224,6 +224,31 @@ + + + + Other reference + + + false + + + true + + + false + + + + + + NZ Imagery Survey Index + + + + + + diff --git a/buildings/reference_data/other_reference.py b/buildings/reference_data/other_reference.py new file mode 100644 index 00000000..c72a8021 --- /dev/null +++ b/buildings/reference_data/other_reference.py @@ -0,0 +1,197 @@ +from builtins import str +from collections import Counter + + +# script to update canal data + +from buildings.sql import buildings_reference_select_statements as reference_select +from buildings.utilities import database as db +from qgis.core import QgsVectorLayer +from qgis.PyQt.QtCore import Qt, QVariant + +LDS_LAYER_IDS = { + "nz_imagery_survey_index": 95677, +} + + +URI = "https://data.linz.govt.nz/services;key={1}/wfs/layer-{0}-changeset?SERVICE=WFS&VERSION=2.0.0&REQUEST=GetFeature&typeNames=layer-{0}-changeset&viewparams=from:{2};to:{3}{4}&SRSNAME=EPSG:2193&outputFormat=json" + + +def last_update(column_name): + + # get last update of layer date from log + from_var = db.execute_return( + reference_select.log_select_last_update.format(column_name) + ) + from_var = from_var.fetchone() + if from_var is None: + # default to beginning of 2018 + from_var = "2018-01-01T02:15:47.317439" + else: + from_var = str(from_var[0]).split("+")[0] + from_var = from_var.split(" ") + from_var = from_var[0] + "T" + from_var[1] + return from_var + + +def current_date(): + to_var = db.execute_return("SELECT now();") + to_var = to_var.fetchone()[0] + to_var = str(to_var).split("+")[0] + to_var = to_var.split(" ") + to_var = to_var[0] + "T" + to_var[1] + return to_var + + +def check_status_imagery_survey_index(kx_api_key, dataset): + # get last update of layer date from log + from_var = last_update("imagery_survey_index") + + # current date + to_var = current_date() + + cql_filter = "" + layer = QgsVectorLayer( + URI.format(LDS_LAYER_IDS[dataset], kx_api_key, from_var, to_var, cql_filter) + ) + if not layer.isValid(): + return { + "dataset": dataset, + "last_updated": from_var, + "new_updates": "error", + "insert": "error", + "update": "error", + "delete": "error", + } + + if layer.featureCount() == 0: + return { + "dataset": dataset, + "last_updated": from_var, + "new_updates": "", + "insert": "0", + "update": "0", + "delete": "0", + } + counts = Counter([feat["__change__"] for feat in layer.getFeatures()]) + return { + "dataset": dataset, + "last_updated": from_var, + "new_updates": "Available", + "insert": str(counts["INSERT"]), + "update": str(counts["UPDATE"]), + "delete": str(counts["DELETE"]), + } + + +def update_imagery_survey_index(kx_api_key, dataset, dbconn): + if dataset != "nz_imagery_survey_index": + return "error" + + # get last update of layer date from log + from_var = last_update("imagery_survey_index") + + # current date + to_var = current_date() + + cql_filter = "" + external_id = "imagery_survey_id" + + layer = QgsVectorLayer( + URI.format(LDS_LAYER_IDS[dataset], kx_api_key, from_var, to_var, cql_filter) + ) + if not layer.isValid(): + # something went wrong + return "error" + + if layer.featureCount() == 0: + return "current" + + for feature in layer.getFeatures(): + if feature.attribute("__change__") == "DELETE": + sql = "DELETE FROM buildings_reference.nz_imagery_survey_index WHERE imagery_survey_id = %s;" + dbconn.execute_no_commit(sql, (feature[external_id],)) + + elif feature.attribute("__change__") == "INSERT": + sql = "SELECT True FROM buildings_reference.nz_imagery_survey_index WHERE imagery_survey_id = %s;" + result = dbconn.execute_return( + sql, + (feature[external_id],), + ) + result = result.fetchone() + if result is None: + sql = """ + INSERT INTO buildings_reference.nz_imagery_survey_index ( + imagery_survey_id, + name, + imagery_id, + index_id, + set_order, + ground_sample_distance, + accuracy, + supplier, + licensor, + flown_from, + flown_to, + shape) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, ST_SetSRID(ST_GeometryFromText(%s), 2193)) + """ + dbconn.execute_no_commit( + sql, + ( + feature[external_id], + correct_attribute_format(feature["name"]), + correct_attribute_format(feature["imagery_id"]), + correct_attribute_format(feature["index_id"]), + correct_attribute_format(feature["set_order"]), + correct_attribute_format(feature["ground_sample_distance"]), + correct_attribute_format(feature["accuracy"]), + correct_attribute_format(feature["supplier"]), + correct_attribute_format(feature["licensor"]), + feature["flown_from"].toString(Qt.ISODate), + feature["flown_to"].toString(Qt.ISODate), + feature.geometry().asWkt(), + ), + ) + + elif feature.attribute("__change__") == "UPDATE": + sql = """ + UPDATE buildings_reference.nz_imagery_survey_index + SET + name = %s, + imagery_id = %s, + index_id = %s, + set_order = %s, + ground_sample_distance = %s, + accuracy = %s, + supplier = %s, + licensor = %s, + flown_from = %s, + flown_to = %s, + shape = ST_SetSRID(ST_GeometryFromText(%s), 2193) + WHERE imagery_survey_id = %s; + """ + dbconn.execute_no_commit( + sql, + ( + correct_attribute_format(feature["name"]), + correct_attribute_format(feature["imagery_id"]), + correct_attribute_format(feature["index_id"]), + correct_attribute_format(feature["set_order"]), + correct_attribute_format(feature["ground_sample_distance"]), + correct_attribute_format(feature["accuracy"]), + correct_attribute_format(feature["supplier"]), + correct_attribute_format(feature["licensor"]), + feature["flown_from"].toString(Qt.ISODate), + feature["flown_to"].toString(Qt.ISODate), + feature.geometry().asWkt(), + feature[external_id], + ), + ) + return "updated" + + +def correct_attribute_format(value): + if type(value) == QVariant and value.isNull(): + return None + return value