diff --git a/src/calliope/attrdict.py b/src/calliope/attrdict.py index c056c3b4..3c171090 100644 --- a/src/calliope/attrdict.py +++ b/src/calliope/attrdict.py @@ -5,14 +5,11 @@ import copy import io import logging -from pathlib import Path import numpy as np import ruamel.yaml as ruamel_yaml from typing_extensions import Self -from calliope.util.tools import relative_path - logger = logging.getLogger(__name__) diff --git a/src/calliope/config/config_schema.yaml b/src/calliope/config/config_schema.yaml index 7348bd18..f6fbeb51 100644 --- a/src/calliope/config/config_schema.yaml +++ b/src/calliope/config/config_schema.yaml @@ -228,14 +228,6 @@ properties: additionalProperties: false patternProperties: *nested_pattern - # templates: - # type: [object, "null"] - # description: >- - # Abstract technology/node templates from which techs/nodes can `inherit`. - # See the model definition schema for more guidance on content. - # additionalProperties: false - # patternProperties: *nested_pattern - overrides: type: [object, "null"] description: >- diff --git a/src/calliope/io.py b/src/calliope/io.py index 93ec19a0..275ed98d 100644 --- a/src/calliope/io.py +++ b/src/calliope/io.py @@ -217,10 +217,7 @@ def load_config(filename: str): return loaded -def read_rich_yaml( - yaml: str | Path, - allow_override: bool = False, -) -> AttrDict: +def read_rich_yaml(yaml: str | Path, allow_override: bool = False) -> AttrDict: """Returns an AttrDict initialised from the given YAML file or string. Uses calliope's "flavour" for YAML files. diff --git a/src/calliope/preprocess/__init__.py b/src/calliope/preprocess/__init__.py index f1998f20..b5bd90c8 100644 --- a/src/calliope/preprocess/__init__.py +++ b/src/calliope/preprocess/__init__.py @@ -2,5 +2,5 @@ from calliope.preprocess.data_tables import DataTable from calliope.preprocess.model_data import ModelDataFactory +from calliope.preprocess.model_definition import prepare_model_definition from calliope.preprocess.model_math import CalliopeMath -from calliope.preprocess.scenarios import prepare_model_definition diff --git a/src/calliope/preprocess/scenarios.py b/src/calliope/preprocess/model_definition.py similarity index 95% rename from src/calliope/preprocess/scenarios.py rename to src/calliope/preprocess/model_definition.py index d4e1950b..4fcf1ba3 100644 --- a/src/calliope/preprocess/scenarios.py +++ b/src/calliope/preprocess/model_definition.py @@ -37,7 +37,9 @@ def prepare_model_definition( model_def = AttrDict(data) else: model_def = read_rich_yaml(data) - model_def, applied_overrides = _load_scenario_overrides(model_def, scenario, override_dict) + model_def, applied_overrides = _load_scenario_overrides( + model_def, scenario, override_dict + ) template_solver = TemplateSolver(model_def) model_def = template_solver.resolved_data @@ -47,7 +49,7 @@ def prepare_model_definition( def _load_scenario_overrides( model_definition: dict, scenario: str | None = None, - override_dict: dict | None = None + override_dict: dict | None = None, ) -> tuple[AttrDict, str]: """Apply user-defined overrides to the model definition. @@ -214,7 +216,9 @@ def _resolve_template(self, name: str, stack: None | set[str] = None) -> AttrDic if stack is None: stack = set() elif name in stack: - raise exceptions.ModelError(f"Circular template reference detected for '{name}'.") + raise exceptions.ModelError( + f"Circular template reference detected for '{name}'." + ) stack.add(name) result = AttrDict() @@ -238,7 +242,9 @@ def _resolve_data(self, section, level: int = 0): if isinstance(section, dict): if self.TEMPLATES_SECTION in section: if level != 0: - raise exceptions.ModelError("Template definitions must be placed at the top level of the YAML file.") + raise exceptions.ModelError( + "Template definitions must be placed at the top level of the YAML file." + ) if self.TEMPLATE_CALL in section: template = self.resolved_templates[section[self.TEMPLATE_CALL]].copy() else: @@ -246,7 +252,7 @@ def _resolve_data(self, section, level: int = 0): local = AttrDict() for key in section.keys() - {self.TEMPLATE_CALL, self.TEMPLATES_SECTION}: - local[key] = self._resolve_data(section[key], level=level+1) + local[key] = self._resolve_data(section[key], level=level + 1) # Local values have priority. template.union(local, allow_override=True) @@ -254,4 +260,3 @@ def _resolve_data(self, section, level: int = 0): else: result = section return result - diff --git a/src/calliope/util/tools.py b/src/calliope/util/tools.py index 8575409b..51920d88 100644 --- a/src/calliope/util/tools.py +++ b/src/calliope/util/tools.py @@ -2,15 +2,11 @@ # Licensed under the Apache 2.0 License (see LICENSE file). """Assorted helper tools.""" -from copy import deepcopy from pathlib import Path -from typing import TYPE_CHECKING, Any, TypeVar +from typing import Any, TypeVar from typing_extensions import ParamSpec -if TYPE_CHECKING: - from calliope import AttrDict - P = ParamSpec("P") T = TypeVar("T") diff --git a/tests/test_core_attrdict.py b/tests/test_core_attrdict.py index 32a8dd5b..340e72e2 100644 --- a/tests/test_core_attrdict.py +++ b/tests/test_core_attrdict.py @@ -16,7 +16,6 @@ def regular_dict(self): } return d - @pytest.fixture def attr_dict(self, regular_dict): d = regular_dict @@ -211,4 +210,3 @@ def test_del_key_nested(self, attr_dict): def test_to_yaml_string(self, attr_dict): result = attr_dict.to_yaml() assert "a: 1" in result - diff --git a/tests/test_core_preprocess.py b/tests/test_core_preprocess.py index 6122ab4d..0ee2f38c 100644 --- a/tests/test_core_preprocess.py +++ b/tests/test_core_preprocess.py @@ -5,7 +5,6 @@ import calliope import calliope.exceptions as exceptions -from calliope.attrdict import AttrDict from calliope.io import read_rich_yaml from .common.util import build_test_model as build_model @@ -17,8 +16,8 @@ def test_model_from_dict(self, data_source_dir): """Test creating a model from dict/AttrDict instead of from YAML""" model_dir = data_source_dir.parent model_location = model_dir / "model.yaml" - model_dict = read_rich_yaml(model_location) - node_dict = AttrDict( + model_dict = calliope.io.read_rich_yaml(model_location) + node_dict = calliope.AttrDict( { "nodes": { "a": {"techs": {"test_supply_elec": {}, "test_demand_elec": {}}}, @@ -35,108 +34,6 @@ def test_model_from_dict(self, data_source_dir): # test as dict calliope.Model(model_dict.as_dict()) - @pytest.mark.filterwarnings( - "ignore:(?s).*(links, test_link_a_b_elec) | Deactivated:calliope.exceptions.ModelWarning" - ) - def test_valid_scenarios(self, dummy_int): - """Test that valid scenario definition from overrides raises no error and results in applied scenario.""" - override = read_rich_yaml( - f""" - scenarios: - scenario_1: ['one', 'two'] - - overrides: - one: - techs.test_supply_gas.flow_cap_max: {dummy_int} - two: - techs.test_supply_elec.flow_cap_max: {dummy_int/2} - - nodes: - a: - techs: - test_supply_gas: - test_supply_elec: - test_demand_elec: - """ - ) - model = build_model(override_dict=override, scenario="scenario_1") - - assert ( - model._model_data.sel(techs="test_supply_gas")["flow_cap_max"] == dummy_int - ) - assert ( - model._model_data.sel(techs="test_supply_elec")["flow_cap_max"] - == dummy_int / 2 - ) - - def test_valid_scenario_of_scenarios(self, dummy_int): - """Test that valid scenario definition which groups scenarios and overrides raises - no error and results in applied scenario. - """ - override = read_rich_yaml( - f""" - scenarios: - scenario_1: ['one', 'two'] - scenario_2: ['scenario_1', 'new_location'] - - overrides: - one: - techs.test_supply_gas.flow_cap_max: {dummy_int} - two: - techs.test_supply_elec.flow_cap_max: {dummy_int/2} - new_location: - nodes.b.techs: - test_supply_elec: - - nodes: - a: - techs: - test_supply_gas: - test_supply_elec: - test_demand_elec: - """ - ) - model = build_model(override_dict=override, scenario="scenario_2") - - assert ( - model._model_data.sel(techs="test_supply_gas")["flow_cap_max"] == dummy_int - ) - assert ( - model._model_data.sel(techs="test_supply_elec")["flow_cap_max"] - == dummy_int / 2 - ) - - def test_invalid_scenarios_dict(self): - """Test that invalid scenario definition raises appropriate error""" - override = read_rich_yaml( - """ - scenarios: - scenario_1: - techs.foo.bar: 1 - """ - ) - with pytest.raises(exceptions.ModelError) as excinfo: - build_model(override_dict=override, scenario="scenario_1") - - assert check_error_or_warning( - excinfo, "(scenarios, scenario_1) | Unrecognised override name: techs." - ) - - def test_invalid_scenarios_str(self): - """Test that invalid scenario definition raises appropriate error""" - override = read_rich_yaml( - """ - scenarios: - scenario_1: 'foo' - """ - ) - with pytest.raises(exceptions.ModelError) as excinfo: - build_model(override_dict=override, scenario="scenario_1") - - assert check_error_or_warning( - excinfo, "(scenarios, scenario_1) | Unrecognised override name: foo." - ) - def test_undefined_carriers(self): """Test that user has input either carrier or carrier_in/_out for each tech""" override = read_rich_yaml( diff --git a/tests/test_core_util.py b/tests/test_core_util.py index a0423d37..e1c2baff 100644 --- a/tests/test_core_util.py +++ b/tests/test_core_util.py @@ -184,9 +184,7 @@ def test_invalid_dict(self, to_validate, expected_path): @pytest.fixture def base_math(self): - return read_rich_yaml( - Path(calliope.__file__).parent / "math" / "plan.yaml" - ) + return read_rich_yaml(Path(calliope.__file__).parent / "math" / "plan.yaml") @pytest.mark.parametrize( "dict_path", glob.glob(str(Path(calliope.__file__).parent / "math" / "*.yaml")) @@ -196,8 +194,7 @@ def test_validate_math(self, base_math, dict_path): Path(calliope.__file__).parent / "config" / "math_schema.yaml" ) to_validate = base_math.union( - read_rich_yaml(dict_path, allow_override=True), - allow_override=True, + read_rich_yaml(dict_path, allow_override=True), allow_override=True ) schema.validate_dict(to_validate, math_schema, "") diff --git a/tests/test_preprocess_model_definition.py b/tests/test_preprocess_model_definition.py index bb1eb015..1501165e 100644 --- a/tests/test_preprocess_model_definition.py +++ b/tests/test_preprocess_model_definition.py @@ -2,11 +2,117 @@ from calliope.exceptions import ModelError from calliope.io import read_rich_yaml -from calliope.preprocess.scenarios import TemplateSolver +from calliope.preprocess.model_definition import TemplateSolver +from .common.util import build_test_model as build_model +from .common.util import check_error_or_warning -class TestTemplateSolver: +class TestScenarioOverrides: + @pytest.mark.filterwarnings( + "ignore:(?s).*(links, test_link_a_b_elec) | Deactivated:calliope.exceptions.ModelWarning" + ) + def test_valid_scenarios(self, dummy_int): + """Test that valid scenario definition from overrides raises no error and results in applied scenario.""" + override = read_rich_yaml( + f""" + scenarios: + scenario_1: ['one', 'two'] + + overrides: + one: + techs.test_supply_gas.flow_cap_max: {dummy_int} + two: + techs.test_supply_elec.flow_cap_max: {dummy_int/2} + + nodes: + a: + techs: + test_supply_gas: + test_supply_elec: + test_demand_elec: + """ + ) + model = build_model(override_dict=override, scenario="scenario_1") + + assert ( + model._model_data.sel(techs="test_supply_gas")["flow_cap_max"] == dummy_int + ) + assert ( + model._model_data.sel(techs="test_supply_elec")["flow_cap_max"] + == dummy_int / 2 + ) + + def test_valid_scenario_of_scenarios(self, dummy_int): + """Test that valid scenario definition which groups scenarios and overrides raises + no error and results in applied scenario. + """ + override = read_rich_yaml( + f""" + scenarios: + scenario_1: ['one', 'two'] + scenario_2: ['scenario_1', 'new_location'] + + overrides: + one: + techs.test_supply_gas.flow_cap_max: {dummy_int} + two: + techs.test_supply_elec.flow_cap_max: {dummy_int/2} + new_location: + nodes.b.techs: + test_supply_elec: + + nodes: + a: + techs: + test_supply_gas: + test_supply_elec: + test_demand_elec: + """ + ) + model = build_model(override_dict=override, scenario="scenario_2") + + assert ( + model._model_data.sel(techs="test_supply_gas")["flow_cap_max"] == dummy_int + ) + assert ( + model._model_data.sel(techs="test_supply_elec")["flow_cap_max"] + == dummy_int / 2 + ) + + def test_invalid_scenarios_dict(self): + """Test that invalid scenario definition raises appropriate error""" + override = read_rich_yaml( + """ + scenarios: + scenario_1: + techs.foo.bar: 1 + """ + ) + with pytest.raises(ModelError) as excinfo: + build_model(override_dict=override, scenario="scenario_1") + + assert check_error_or_warning( + excinfo, "(scenarios, scenario_1) | Unrecognised override name: techs." + ) + + def test_invalid_scenarios_str(self): + """Test that invalid scenario definition raises appropriate error""" + override = read_rich_yaml( + """ + scenarios: + scenario_1: 'foo' + """ + ) + with pytest.raises(ModelError) as excinfo: + build_model(override_dict=override, scenario="scenario_1") + + assert check_error_or_warning( + excinfo, "(scenarios, scenario_1) | Unrecognised override name: foo." + ) + + +class TestTemplateSolver: @pytest.fixture def dummy_solved_template(self) -> TemplateSolver: text = """ @@ -45,7 +151,8 @@ def test_inheritance_templates(self, dummy_solved_template): templates.T1 == {"A": ["foo", "bar"], "B": 1}, templates.T2 == {"A": ["foo", "bar"], "B": 1, "C": "bar"}, templates.T3 == {"A": ["foo", "bar"], "B": 11}, - templates.T4 == {"A": ["bar", "foobar"], "B": "1", "C": {"foo": "bar"}, "D": True} + templates.T4 + == {"A": ["bar", "foobar"], "B": "1", "C": {"foo": "bar"}, "D": True}, ] ) @@ -55,7 +162,8 @@ def test_template_inheritance_data(self, dummy_solved_template): [ data.a == {"A": ["foo", "bar"], "B": 1, "a1": 1}, data.b == {"A": ["foo", "bar"], "B": 11}, - data.c == {"A": ["bar", "foobar"], "B": "1", "C": {"foo": "bar"}, "D": False} + data.c + == {"A": ["bar", "foobar"], "B": "1", "C": {"foo": "bar"}, "D": False}, ] ) @@ -70,7 +178,9 @@ def test_invalid_template_error(self): template: T2 """ ) - with pytest.raises(ModelError, match="Template definitions must be YAML blocks."): + with pytest.raises( + ModelError, match="Template definitions must be YAML blocks." + ): TemplateSolver(text) def test_circular_template_error(self): @@ -106,5 +216,8 @@ def test_incorrect_template_placement_error(self): this: "should not be here" """ ) - with pytest.raises(ModelError, match="Template definitions must be placed at the top level of the YAML file."): + with pytest.raises( + ModelError, + match="Template definitions must be placed at the top level of the YAML file.", + ): TemplateSolver(text) diff --git a/tests/test_preprocess_time.py b/tests/test_preprocess_time.py index e5c4e037..dab2cbd6 100644 --- a/tests/test_preprocess_time.py +++ b/tests/test_preprocess_time.py @@ -1,7 +1,7 @@ import pandas as pd import pytest # noqa: F401 -from calliope import AttrDict, exceptions +from calliope import exceptions from calliope.io import read_rich_yaml from .common.util import build_test_model