From db08734a5de09f3b7049182c7cd796f80556af34 Mon Sep 17 00:00:00 2001 From: Martijn Govers Date: Tue, 14 Jan 2025 14:10:57 +0100 Subject: [PATCH 1/3] fix bug where validation errors blew up quadratically Signed-off-by: Martijn Govers --- src/power_grid_model/validation/validation.py | 2 +- .../unit/validation/test_batch_validation.py | 94 +++++++++++++++---- 2 files changed, 78 insertions(+), 18 deletions(-) diff --git a/src/power_grid_model/validation/validation.py b/src/power_grid_model/validation/validation.py index 5e9fca790..7dd532931 100644 --- a/src/power_grid_model/validation/validation.py +++ b/src/power_grid_model/validation/validation.py @@ -158,7 +158,7 @@ def validate_batch_data( batch_errors = input_errors + id_errors if not id_errors: - batch_errors = input_errors + batch_errors = input_errors.copy() merged_data = _update_input_data(input_data_copy, row_update_data) batch_errors += validate_required_values(merged_data, calculation_type, symmetric) batch_errors += validate_values(merged_data, calculation_type) diff --git a/tests/unit/validation/test_batch_validation.py b/tests/unit/validation/test_batch_validation.py index 064f9853d..c9879f399 100644 --- a/tests/unit/validation/test_batch_validation.py +++ b/tests/unit/validation/test_batch_validation.py @@ -6,19 +6,19 @@ import numpy as np import pytest -from power_grid_model import DatasetType, LoadGenType, initialize_array +from power_grid_model import ComponentType, DatasetType, LoadGenType, initialize_array from power_grid_model._utils import compatibility_convert_row_columnar_dataset, is_columnar from power_grid_model.enum import ComponentAttributeFilterOptions from power_grid_model.validation import validate_batch_data -from power_grid_model.validation.errors import MultiComponentNotUniqueError, NotBooleanError +from power_grid_model.validation.errors import MultiComponentNotUniqueError, NotBetweenOrAtError, NotBooleanError @pytest.fixture def original_input_data() -> dict[str, np.ndarray]: - node = initialize_array("input", "node", 4) + node = initialize_array(DatasetType.input, ComponentType.node, 4) node["id"] = [1, 2, 3, 4] node["u_rated"] = 10.5e3 - line = initialize_array("input", "line", 4) + line = initialize_array(DatasetType.input, ComponentType.line, 4) line["id"] = [5, 6, 7, 8] line["from_node"] = [1, 2, 3, 1] @@ -31,7 +31,7 @@ def original_input_data() -> dict[str, np.ndarray]: line["tan1"] = 4.0 line["i_n"] = 5.0 - asym_load = initialize_array("input", "asym_load", 2) + asym_load = initialize_array(DatasetType.input, ComponentType.asym_load, 2) asym_load["id"] = [9, 10] asym_load["node"] = [1, 2] asym_load["status"] = [1, 1] @@ -39,13 +39,19 @@ def original_input_data() -> dict[str, np.ndarray]: asym_load["p_specified"] = [[11e6, 12e6, 13e6], [21e6, 22e6, 23e6]] asym_load["q_specified"] = [[11e5, 12e5, 13e5], [21e5, 22e5, 23e5]] - return {"node": node, "line": line, "asym_load": asym_load} + return { + ComponentType.node: node, + ComponentType.line: line, + ComponentType.asym_load: asym_load, + } @pytest.fixture def original_input_data_columnar_all(original_input_data): return compatibility_convert_row_columnar_dataset( - original_input_data, ComponentAttributeFilterOptions.everything, DatasetType.input + original_input_data, + ComponentAttributeFilterOptions.everything, + DatasetType.input, ) @@ -57,7 +63,11 @@ def original_input_data_columnar_relevant(original_input_data): @pytest.fixture( - params=["original_input_data", "original_input_data_columnar_all", "original_input_data_columnar_relevant"] + params=[ + "original_input_data", + "original_input_data_columnar_all", + "original_input_data_columnar_relevant", + ] ) def input_data(request): return request.getfixturevalue(request.param) @@ -65,12 +75,12 @@ def input_data(request): @pytest.fixture def original_batch_data() -> dict[str, np.ndarray]: - line = initialize_array("update", "line", (3, 2)) + line = initialize_array(DatasetType.update, ComponentType.line, (3, 2)) line["id"] = [[5, 6], [6, 7], [7, 5]] line["from_status"] = [[1, 1], [1, 1], [1, 1]] # Add batch for asym_load, which has 2-D array for p_specified - asym_load = initialize_array("update", "asym_load", (3, 2)) + asym_load = initialize_array(DatasetType.update, ComponentType.asym_load, (3, 2)) asym_load["id"] = [[9, 10], [9, 10], [9, 10]] return {"line": line, "asym_load": asym_load} @@ -79,19 +89,27 @@ def original_batch_data() -> dict[str, np.ndarray]: @pytest.fixture def original_batch_data_columnar_all(original_batch_data): return compatibility_convert_row_columnar_dataset( - original_batch_data, ComponentAttributeFilterOptions.everything, DatasetType.update + original_batch_data, + ComponentAttributeFilterOptions.everything, + DatasetType.update, ) @pytest.fixture def original_batch_data_columnar_relevant(original_batch_data): return compatibility_convert_row_columnar_dataset( - original_batch_data, ComponentAttributeFilterOptions.relevant, DatasetType.update + original_batch_data, + ComponentAttributeFilterOptions.relevant, + DatasetType.update, ) @pytest.fixture( - params=["original_batch_data", "original_batch_data_columnar_all", "original_batch_data_columnar_relevant"] + params=[ + "original_batch_data", + "original_batch_data_columnar_all", + "original_batch_data_columnar_relevant", + ] ) def batch_data(request): return request.getfixturevalue(request.param) @@ -119,7 +137,49 @@ def test_validate_batch_data_input_error(input_data, batch_data): def test_validate_batch_data_update_error(input_data, batch_data): batch_data["line"]["from_status"] = np.array([[12, 34], [0, -128], [56, 78]]) errors = validate_batch_data(input_data, batch_data) - assert len(errors) == 3 - assert NotBooleanError("line", "from_status", [5, 6]) == errors[0][0] - assert NotBooleanError("line", "from_status", [5, 7]) == errors[1][1] - assert NotBooleanError("line", "from_status", [5, 6]) == errors[2][0] + assert len(errors) == 2 + assert 1 not in errors + assert len(errors[0]) == 1 + assert len(errors[2]) == 1 + assert errors[0] == [NotBooleanError("line", "from_status", [5, 6])] + assert errors[2] == [NotBooleanError("line", "from_status", [5, 7])] + + +def test_validate_batch_data_transformer_tap_nom(): + node_input = initialize_array(DatasetType.input, ComponentType.node, 2) + node_input["id"] = [1, 2] + node_input["u_rated"] = [400, 10500] + + transformer_input = initialize_array(DatasetType.input, ComponentType.transformer, 1) + transformer_input["id"] = 3 + transformer_input["from_node"] = 1 + transformer_input["to_node"] = 2 + transformer_input["from_status"] = 1 + transformer_input["to_status"] = 1 + transformer_input["u1"] = 10750 + transformer_input["u2"] = 420 + transformer_input["sn"] = 400000 + transformer_input["uk"] = 0.04 + transformer_input["pk"] = 3750 + transformer_input["i0"] = 0.002230015414744929 + transformer_input["p0"] = 515 + transformer_input["winding_from"] = 2 + transformer_input["winding_to"] = 1 + transformer_input["clock"] = 5 + transformer_input["tap_side"] = 0 + transformer_input["tap_pos"] = 1 + transformer_input["tap_min"] = 1 + transformer_input["tap_max"] = 5 + transformer_input["tap_size"] = 250 + test_input_data = { + ComponentType.node: node_input, + ComponentType.transformer: transformer_input, + } + test_update_data = {ComponentType.sym_load: initialize_array(DatasetType.update, ComponentType.sym_load, (2, 0))} + result = validate_batch_data(test_input_data, test_update_data) + assert result is not None + assert len(result) == test_update_data[ComponentType.sym_load].shape[0] + assert len(result[0]) == test_input_data[ComponentType.transformer].shape[0] + + error = NotBetweenOrAtError(ComponentType.transformer, "tap_nom", [3], ("tap_min", "tap_max")) + assert result == {0: [error], 1: [error]} From 03ef8778953b97b16faf6cfb15928c912858cd44 Mon Sep 17 00:00:00 2001 From: Martijn Govers Date: Tue, 14 Jan 2025 15:09:11 +0100 Subject: [PATCH 2/3] process comments Signed-off-by: Martijn Govers --- tests/unit/validation/test_batch_validation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/validation/test_batch_validation.py b/tests/unit/validation/test_batch_validation.py index c9879f399..5d976e513 100644 --- a/tests/unit/validation/test_batch_validation.py +++ b/tests/unit/validation/test_batch_validation.py @@ -180,6 +180,7 @@ def test_validate_batch_data_transformer_tap_nom(): assert result is not None assert len(result) == test_update_data[ComponentType.sym_load].shape[0] assert len(result[0]) == test_input_data[ComponentType.transformer].shape[0] + assert len(result[1]) == test_input_data[ComponentType.transformer].shape[0] error = NotBetweenOrAtError(ComponentType.transformer, "tap_nom", [3], ("tap_min", "tap_max")) assert result == {0: [error], 1: [error]} From 197335d3a5d84f5138b4be57073cf43db8375f4c Mon Sep 17 00:00:00 2001 From: Martijn Govers Date: Tue, 14 Jan 2025 15:12:08 +0100 Subject: [PATCH 3/3] improve fix Signed-off-by: Martijn Govers --- src/power_grid_model/validation/validation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/power_grid_model/validation/validation.py b/src/power_grid_model/validation/validation.py index 7dd532931..69baa895f 100644 --- a/src/power_grid_model/validation/validation.py +++ b/src/power_grid_model/validation/validation.py @@ -158,7 +158,6 @@ def validate_batch_data( batch_errors = input_errors + id_errors if not id_errors: - batch_errors = input_errors.copy() merged_data = _update_input_data(input_data_copy, row_update_data) batch_errors += validate_required_values(merged_data, calculation_type, symmetric) batch_errors += validate_values(merged_data, calculation_type)