diff --git a/docs/hooks/generate_readable_schema.py b/docs/hooks/generate_readable_schema.py index 89ae232e..76f574a8 100644 --- a/docs/hooks/generate_readable_schema.py +++ b/docs/hooks/generate_readable_schema.py @@ -14,12 +14,13 @@ import jsonschema2md from mkdocs.structure.files import File +from calliope.schemas import config_schema from calliope.util import schema TEMPDIR = tempfile.TemporaryDirectory() SCHEMAS = { - "config_schema": schema.CONFIG_SCHEMA, + "config_schema": config_schema.CalliopeConfig().model_no_ref_schema(), "model_schema": schema.MODEL_SCHEMA, "math_schema": schema.MATH_SCHEMA, "data_table_schema": schema.DATA_TABLE_SCHEMA, diff --git a/pyproject.toml b/pyproject.toml index c0d5839f..5d4a5b0b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ max-complexity = 10 # Ignore `E402` (import violations) and `F401` (unused imports) in all `__init__.py` files [tool.ruff.lint.per-file-ignores] -"__init__.py" = ["E402", "F401"] +"__init__.py" = ["E402", "F401", "D104"] "*.ipynb" = ["E402"] "tests/*" = ["D"] "docs/examples/*" = ["D"] diff --git a/requirements/base.txt b/requirements/base.txt index 2bf5f664..65e0713e 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -4,6 +4,7 @@ geographiclib >= 2, < 3 ipdb >= 0.13, < 0.14 ipykernel < 7 jinja2 >= 3, < 4 +jsonref >= 1.1, < 2 jsonschema >= 4, < 5 natsort >= 8, < 9 netcdf4 >= 1.2, < 1.7 @@ -13,4 +14,5 @@ pyomo >= 6.5, < 6.7.2 pyparsing >= 3.0, < 3.1 ruamel.yaml >= 0.18, < 0.19 typing-extensions >= 4, < 5 -xarray >= 2024.1, < 2024.4 \ No newline at end of file +xarray >= 2024.1, < 2024.4 +pydantic >= 2.9.2 diff --git a/src/calliope/attrdict.py b/src/calliope/attrdict.py index bd94df7b..3e12e402 100644 --- a/src/calliope/attrdict.py +++ b/src/calliope/attrdict.py @@ -9,6 +9,7 @@ import numpy as np import ruamel.yaml as ruamel_yaml +from ruamel.yaml.scalarstring import walk_tree from typing_extensions import Self from calliope.util.tools import relative_path @@ -328,12 +329,8 @@ def as_dict_flat(self): d[k] = self.get_key(k) return d - def to_yaml(self, path=None): - """Conversion to YAML. - - Saves the AttrDict to the ``path`` as a YAML file or returns a YAML string - if ``path`` is None. - """ + def to_yaml(self, path: str | None = None) -> str: + """Return a serialised YAML string.""" result = self.copy() yaml_ = ruamel_yaml.YAML() yaml_.indent = 2 @@ -355,13 +352,16 @@ def to_yaml(self, path=None): result = result.as_dict() - if path is not None: + # handle multi-line strings. + walk_tree(result) + + stream = io.StringIO() + yaml_.dump(result, stream) + yaml_str = stream.getvalue() + if path: with open(path, "w") as f: - yaml_.dump(result, f) - else: - stream = io.StringIO() - yaml_.dump(result, stream) - return stream.getvalue() + f.write(yaml_str) + return yaml_str def keys_nested(self, subkeys_as="list"): """Returns all keys in the AttrDict, including nested keys. diff --git a/src/calliope/backend/__init__.py b/src/calliope/backend/__init__.py index d37395d8..1b663ae3 100644 --- a/src/calliope/backend/__init__.py +++ b/src/calliope/backend/__init__.py @@ -16,18 +16,18 @@ if TYPE_CHECKING: from calliope.backend.backend_model import BackendModel + from calliope.schemas import config_schema def get_model_backend( - name: str, data: xr.Dataset, math: CalliopeMath, **kwargs + build_config: "config_schema.Build", data: xr.Dataset, math: CalliopeMath ) -> "BackendModel": """Assign a backend using the given configuration. Args: - name (str): name of the backend to use. + build_config: Build configuration options. data (Dataset): model data for the backend. math (CalliopeMath): Calliope math. - **kwargs: backend keyword arguments corresponding to model.config.build. Raises: exceptions.BackendError: If invalid backend was requested. @@ -35,10 +35,10 @@ def get_model_backend( Returns: BackendModel: Initialized backend object. """ - match name: + match build_config.backend: case "pyomo": - return PyomoBackendModel(data, math, **kwargs) + return PyomoBackendModel(data, math, build_config) case "gurobi": - return GurobiBackendModel(data, math, **kwargs) + return GurobiBackendModel(data, math, build_config) case _: - raise BackendError(f"Incorrect backend '{name}' requested.") + raise BackendError(f"Incorrect backend '{build_config.backend}' requested.") diff --git a/src/calliope/backend/backend_model.py b/src/calliope/backend/backend_model.py index c52d74ab..40a687ef 100644 --- a/src/calliope/backend/backend_model.py +++ b/src/calliope/backend/backend_model.py @@ -32,11 +32,8 @@ from calliope.exceptions import warn as model_warn from calliope.io import load_config from calliope.preprocess.model_math import ORDERED_COMPONENTS_T, CalliopeMath -from calliope.util.schema import ( - MODEL_SCHEMA, - extract_from_schema, - update_then_validate_config, -) +from calliope.schemas import config_schema +from calliope.util.schema import MODEL_SCHEMA, extract_from_schema if TYPE_CHECKING: from calliope.backend.parsing import T as Tp @@ -69,20 +66,20 @@ class BackendModelGenerator(ABC): _PARAM_UNITS = extract_from_schema(MODEL_SCHEMA, "x-unit") _PARAM_TYPE = extract_from_schema(MODEL_SCHEMA, "x-type") - def __init__(self, inputs: xr.Dataset, math: CalliopeMath, **kwargs): + def __init__( + self, inputs: xr.Dataset, math: CalliopeMath, build_config: config_schema.Build + ): """Abstract base class to build a representation of the optimisation problem. Args: inputs (xr.Dataset): Calliope model data. math (CalliopeMath): Calliope math. - **kwargs (Any): build configuration overrides. + build_config: Build configuration options. """ self._dataset = xr.Dataset() self.inputs = inputs.copy() self.inputs.attrs = deepcopy(inputs.attrs) - self.inputs.attrs["config"]["build"] = update_then_validate_config( - "build", self.inputs.attrs["config"], **kwargs - ) + self.config = build_config self.math: CalliopeMath = deepcopy(math) self._solve_logger = logging.getLogger(__name__ + ".") @@ -200,6 +197,7 @@ def _check_inputs(self): "equation_name": "", "backend_interface": self, "input_data": self.inputs, + "build_config": self.config, "helper_functions": helper_functions._registry["where"], "apply_where": True, "references": set(), @@ -246,7 +244,7 @@ def add_optimisation_components(self) -> None: # The order of adding components matters! # 1. Variables, 2. Global Expressions, 3. Constraints, 4. Objectives self._add_all_inputs_as_parameters() - if self.inputs.attrs["config"]["build"]["pre_validate_math_strings"]: + if self.config.pre_validate_math_strings: self._validate_math_string_parsing() for components in typing.get_args(ORDERED_COMPONENTS_T): component = components.removesuffix("s") @@ -399,7 +397,7 @@ def _add_all_inputs_as_parameters(self) -> None: if param_name in self.parameters.keys(): continue elif ( - self.inputs.attrs["config"]["build"]["mode"] != "operate" + self.config.mode != "operate" and param_name in extract_from_schema(MODEL_SCHEMA, "x-operate-param").keys() ): @@ -606,7 +604,11 @@ class BackendModel(BackendModelGenerator, Generic[T]): """Calliope's backend model functionality.""" def __init__( - self, inputs: xr.Dataset, math: CalliopeMath, instance: T, **kwargs + self, + inputs: xr.Dataset, + math: CalliopeMath, + instance: T, + build_config: config_schema.Build, ) -> None: """Abstract base class to build backend models that interface with solvers. @@ -614,9 +616,9 @@ def __init__( inputs (xr.Dataset): Calliope model data. math (CalliopeMath): Calliope math. instance (T): Interface model instance. - **kwargs: build configuration overrides. + build_config: Build configuration options. """ - super().__init__(inputs, math, **kwargs) + super().__init__(inputs, math, build_config) self._instance = instance self.shadow_prices: ShadowPrices self._has_verbose_strings: bool = False diff --git a/src/calliope/backend/gurobi_backend_model.py b/src/calliope/backend/gurobi_backend_model.py index 2d2e0a48..a36d3ac0 100644 --- a/src/calliope/backend/gurobi_backend_model.py +++ b/src/calliope/backend/gurobi_backend_model.py @@ -18,6 +18,7 @@ from calliope.exceptions import BackendError, BackendWarning from calliope.exceptions import warn as model_warn from calliope.preprocess import CalliopeMath +from calliope.schemas import config_schema if importlib.util.find_spec("gurobipy") is not None: import gurobipy @@ -41,19 +42,21 @@ class GurobiBackendModel(backend_model.BackendModel): """gurobipy-specific backend functionality.""" - def __init__(self, inputs: xr.Dataset, math: CalliopeMath, **kwargs) -> None: + def __init__( + self, inputs: xr.Dataset, math: CalliopeMath, build_config: config_schema.Build + ) -> None: """Gurobi solver interface class. Args: inputs (xr.Dataset): Calliope model data. math (CalliopeMath): Calliope math. - **kwargs: passed directly to the solver. + build_config: Build configuration options. """ if importlib.util.find_spec("gurobipy") is None: raise ImportError( "Install the `gurobipy` package to build the optimisation problem with the Gurobi backend." ) - super().__init__(inputs, math, gurobipy.Model(), **kwargs) + super().__init__(inputs, math, gurobipy.Model(), build_config) self._instance: gurobipy.Model self.shadow_prices = GurobiShadowPrices(self) @@ -144,7 +147,7 @@ def _objective_setter( ) -> xr.DataArray: expr = element.evaluate_expression(self, references=references) - if name == self.inputs.attrs["config"].build.objective: + if name == self.config.objective: self._instance.setObjective(expr.item(), sense=sense) self.log("objectives", name, "Objective activated.") diff --git a/src/calliope/backend/parsing.py b/src/calliope/backend/parsing.py index 33c9ea47..5cdd0808 100644 --- a/src/calliope/backend/parsing.py +++ b/src/calliope/backend/parsing.py @@ -311,6 +311,7 @@ def evaluate_where( helper_functions=helper_functions._registry["where"], input_data=backend_interface.inputs, backend_interface=backend_interface, + build_config=backend_interface.config, references=references if references is not None else set(), apply_where=True, ) diff --git a/src/calliope/backend/pyomo_backend_model.py b/src/calliope/backend/pyomo_backend_model.py index 5ba41ba0..2a439672 100644 --- a/src/calliope/backend/pyomo_backend_model.py +++ b/src/calliope/backend/pyomo_backend_model.py @@ -29,6 +29,7 @@ from calliope.exceptions import BackendError, BackendWarning from calliope.exceptions import warn as model_warn from calliope.preprocess import CalliopeMath +from calliope.schemas import config_schema from calliope.util.logging import LogWriter from . import backend_model, parsing @@ -58,15 +59,17 @@ class PyomoBackendModel(backend_model.BackendModel): """Pyomo-specific backend functionality.""" - def __init__(self, inputs: xr.Dataset, math: CalliopeMath, **kwargs) -> None: + def __init__( + self, inputs: xr.Dataset, math: CalliopeMath, build_config: config_schema.Build + ) -> None: """Pyomo solver interface class. Args: inputs (xr.Dataset): Calliope model data. math (CalliopeMath): Calliope math. - **kwargs: passed directly to the solver. + build_config: Build configuration options. """ - super().__init__(inputs, math, pmo.block(), **kwargs) + super().__init__(inputs, math, pmo.block(), build_config) self._instance.parameters = pmo.parameter_dict() self._instance.variables = pmo.variable_dict() @@ -185,7 +188,7 @@ def _objective_setter( ) -> xr.DataArray: expr = element.evaluate_expression(self, references=references) objective = pmo.objective(expr.item(), sense=sense) - if name == self.inputs.attrs["config"].build.objective: + if name == self.config.objective: text = "activated" objective.activate() else: diff --git a/src/calliope/backend/where_parser.py b/src/calliope/backend/where_parser.py index f434a9bf..f4b3ff6c 100644 --- a/src/calliope/backend/where_parser.py +++ b/src/calliope/backend/where_parser.py @@ -15,13 +15,13 @@ from calliope.backend import expression_parser from calliope.exceptions import BackendError +from calliope.schemas import config_schema if TYPE_CHECKING: from calliope.backend.backend_model import BackendModel pp.ParserElement.enablePackrat() - BOOLEANTYPE = np.bool_ | np.typing.NDArray[np.bool_] @@ -34,6 +34,7 @@ class EvalAttrs(TypedDict): helper_functions: dict[str, Callable] apply_where: NotRequired[bool] references: NotRequired[set] + build_config: config_schema.Build class EvalWhere(expression_parser.EvalToArrayStr): @@ -118,9 +119,7 @@ def as_math_string(self) -> str: # noqa: D102, override return rf"\text{{config.{self.config_option}}}" def as_array(self) -> xr.DataArray: # noqa: D102, override - config_val = ( - self.eval_attrs["input_data"].attrs["config"].build[self.config_option] - ) + config_val = getattr(self.eval_attrs["build_config"], self.config_option) if not isinstance(config_val, int | float | str | bool | np.bool_): raise BackendError( diff --git a/src/calliope/cli.py b/src/calliope/cli.py index a9d811d2..4059de7e 100644 --- a/src/calliope/cli.py +++ b/src/calliope/cli.py @@ -278,9 +278,9 @@ def run( # Else run the model, then save outputs else: click.secho("Starting model run...") - + kwargs = {} if save_logs: - model.config.set_key("solve.save_logs", save_logs) + kwargs["solve.save_logs"] = save_logs if save_csv is None and save_netcdf is None: click.secho( @@ -292,14 +292,13 @@ def run( # If save_netcdf is used, override the 'save_per_spore_path' to point to a # directory of the same name as the planned netcdf - if save_netcdf and model.config.solve.spores_save_per_spore: - model.config.set_key( - "solve.spores_save_per_spore_path", + if save_netcdf and model.config.solve.spores.save_per_spore: + kwargs["solve.spores_save_per_spore_path"] = ( save_netcdf.replace(".nc", "/spore_{}.nc"), ) model.build() - model.solve() + model.solve(**kwargs) termination = model._model_data.attrs.get( "termination_condition", "unknown" ) diff --git a/src/calliope/config/config_schema.yaml b/src/calliope/config/config_schema.yaml index b9ebe627..41a8c06e 100644 --- a/src/calliope/config/config_schema.yaml +++ b/src/calliope/config/config_schema.yaml @@ -15,172 +15,16 @@ properties: init: type: object description: All configuration options used when initialising a Calliope model - additionalProperties: false - properties: - name: - type: ["null", string] - default: null - description: Model name - calliope_version: - type: ["null", string] - default: null - description: Calliope framework version this model is intended for - time_subset: - oneOf: - - type: "null" - - type: array - minItems: 2 - maxItems: 2 - items: - type: string - description: ISO8601 format datetime strings of the form `YYYY-mm-dd HH:MM:SS` (e.g, '2005-01', '2005-01-01', '2005-01-01 00:00', ...) - default: null - description: >- - Subset of timesteps as an two-element list giving the **inclusive** range. - For example, ['2005-01', '2005-04'] will create a time subset from '2005-01-01 00:00:00' to '2005-04-31 23:59:59'. - time_resample: - type: ["null", string] - default: null - description: setting to adjust time resolution, e.g. "2h" for 2-hourly - pattern: "^[0-9]+[a-zA-Z]" - time_cluster: - type: ["null", string] - default: null - description: setting to cluster the timeseries, must be a path to a file where each date is linked to a representative date that also exists in the timeseries. - time_format: - type: string - default: "ISO8601" - description: Timestamp format of all time series data when read from file. "ISO8601" means "%Y-%m-%d %H:%M:%S". - distance_unit: - type: string - default: km - description: >- - Unit of transmission link `distance` (m - metres, km - kilometres). - Automatically derived distances from lat/lon coordinates will be given in this unit. - enum: [m, km] build: type: object description: > All configuration options used when building a Calliope optimisation problem (`calliope.Model.build`). Additional configuration items will be passed onto math string parsing and can therefore be accessed in the `where` strings by `config.[item-name]`, where "[item-name]" is the name of your own configuration item. - additionalProperties: true - properties: - add_math: - type: array - default: [] - description: List of references to files which contain additional mathematical formulations to be applied on top of or instead of the base mode math. - uniqueItems: true - items: - type: string - description: > - If referring to an pre-defined Calliope math file (see documentation for available files), do not append the reference with ".yaml". - If referring to your own math file, ensure the file type is given as a suffix (".yaml" or ".yml"). - Relative paths will be assumed to be relative to the model definition file given when creating a calliope Model (`calliope.Model(model_definition=...)`). - ignore_mode_math: - type: boolean - default: false - description: >- - If True, do not initialise the mathematical formulation with the pre-defined math for the given run `mode`. - This option can be used to completely re-define the Calliope mathematical formulation. - backend: - type: string - default: pyomo - description: Module with which to build the optimisation problem - ensure_feasibility: - type: boolean - default: false - description: > - whether to include decision variables in the model which will meet unmet demand or consume unused supply in the model so that the optimisation solves successfully. - This should only be used as a debugging option (as any unmet demand/unused supply is a sign of improper model formulation). - mode: - type: string - default: plan - description: Mode in which to run the optimisation. - enum: [plan, spores, operate] - objective: - type: string - default: min_cost_optimisation - description: Name of internal objective function to use, from those defined in the pre-defined math and any applied additional math. - operate_window: - type: string - description: >- - Operate mode rolling `window`, given as a pandas frequency string. - See [here](https://pandas.pydata.org/docs/user_guide/timeseries.html#offset-aliases) for a list of frequency aliases. - operate_horizon: - type: string - description: >- - Operate mode rolling `horizon`, given as a pandas frequency string. - See [here](https://pandas.pydata.org/docs/user_guide/timeseries.html#offset-aliases) for a list of frequency aliases. - Must be ≥ `operate_window` - operate_use_cap_results: - type: boolean - default: false - description: If the model already contains `plan` mode results, use those optimal capacities as input parameters to the `operate` mode run. - pre_validate_math_strings: - type: boolean - default: true - description: >- - If true, the Calliope math definition will be scanned for parsing errors _before_ undertaking the much more expensive operation of building the optimisation problem. - You can switch this off (e.g., if you know there are no parsing errors) to reduce overall build time. solve: type: object description: All configuration options used when solving a Calliope optimisation problem (`calliope.Model.solve`). - additionalProperties: false - properties: - spores_number: - type: integer - default: 3 - description: SPORES mode number of iterations after the initial base run. - spores_score_cost_class: - type: string - default: spores_score - description: SPORES mode cost class to vary between iterations after the initial base run. - spores_slack_cost_group: - type: string - description: SPORES mode cost class to keep below the given `slack` (usually "monetary"). - spores_save_per_spore: - type: boolean - default: false - description: Whether or not to save the result of each SPORES mode run between iterations. If False, will consolidate all iterations into one dataset after completion of N iterations (defined by `spores_number`) and save that one dataset. - spores_save_per_spore_path: - type: string - description: If saving per spore, the path to save to. - spores_skip_cost_op: - type: boolean - default: false - description: If the model already contains `plan` mode results, use those as the initial base run results and start with SPORES iterations immediately. - save_logs: - type: ["null", string] - default: null - description: If given, should be a path to a directory in which to save optimisation logs. - solver_io: - type: ["null", string] - default: null - description: > - Some solvers have different interfaces that perform differently. - For instance, setting `solver_io="python"` when using the solver `gurobi` tends to reduce the time to send the optimisation problem to the solver. - solver_options: - type: ["null", object] - default: null - description: Any solver options, as key-value pairs, to pass to the chosen solver - solver: - type: string - default: cbc - description: Solver to use. Any solvers that have Pyomo interfaces can be used. Refer to the Pyomo documentation for the latest list. - zero_threshold: - type: number - default: 1e-10 - description: On postprocessing the optimisation results, values smaller than this threshold will be considered as optimisation artefacts and will be set to zero. - shadow_prices: - type: array - uniqueItems: true - items: - type: string - description: Names of model constraints. - default: [] - description: List of constraints for which to extract shadow prices. Shadow prices will be added as variables to the model results as `shadow_price_{constraintname}`. parameters: type: [object, "null"] diff --git a/src/calliope/example_models/national_scale/scenarios.yaml b/src/calliope/example_models/national_scale/scenarios.yaml index 58a3dc81..0e34f8f9 100644 --- a/src/calliope/example_models/national_scale/scenarios.yaml +++ b/src/calliope/example_models/national_scale/scenarios.yaml @@ -70,8 +70,9 @@ overrides: init.time_subset: ["2005-01-01", "2005-01-10"] build: mode: operate - operate_window: 12h - operate_horizon: 24h + operate: + window: 12h + horizon: 24h nodes: region1.techs.ccgt.flow_cap: 30000 diff --git a/src/calliope/example_models/urban_scale/scenarios.yaml b/src/calliope/example_models/urban_scale/scenarios.yaml index 12d114cb..d754496d 100644 --- a/src/calliope/example_models/urban_scale/scenarios.yaml +++ b/src/calliope/example_models/urban_scale/scenarios.yaml @@ -51,8 +51,9 @@ overrides: init.time_subset: ["2005-07-01", "2005-07-10"] build: mode: operate - operate_window: 2h - operate_horizon: 48h + operate: + window: 2h + horizon: 48h nodes: X1: diff --git a/src/calliope/model.py b/src/calliope/model.py index ee8c5a77..ffc20e4b 100644 --- a/src/calliope/model.py +++ b/src/calliope/model.py @@ -17,15 +17,15 @@ from calliope.postprocess import postprocess as postprocess_results from calliope.preprocess.data_tables import DataTable from calliope.preprocess.model_data import ModelDataFactory +from calliope.schemas import config_schema from calliope.util.logging import log_time from calliope.util.schema import ( CONFIG_SCHEMA, MODEL_SCHEMA, extract_from_schema, - update_then_validate_config, validate_dict, ) -from calliope.util.tools import climb_template_tree, relative_path +from calliope.util.tools import climb_template_tree if TYPE_CHECKING: from calliope.backend.backend_model import BackendModel @@ -43,7 +43,7 @@ class Model: """A Calliope Model.""" _TS_OFFSET = pd.Timedelta(1, unit="nanoseconds") - ATTRS_SAVED = ("_def_path", "applied_math") + ATTRS_SAVED = ("applied_math", "config") def __init__( self, @@ -74,10 +74,9 @@ def __init__( **kwargs: initialisation overrides. """ self._timings: dict = {} - self.config: AttrDict + self.config: config_schema.CalliopeConfig self.defaults: AttrDict self.applied_math: preprocess.CalliopeMath - self._def_path: str | None = None self.backend: BackendModel self._is_built: bool = False self._is_solved: bool = False @@ -88,20 +87,24 @@ def __init__( LOGGER, self._timings, "model_creation", comment="Model: initialising" ) if isinstance(model_definition, xr.Dataset): + if kwargs: + raise exceptions.ModelError( + "Cannot apply initialisation configuration overrides when loading data from an xarray Dataset." + ) self._init_from_model_data(model_definition) else: if isinstance(model_definition, dict): model_def_dict = AttrDict(model_definition) else: - self._def_path = str(model_definition) + kwargs["def_path"] = str(model_definition) model_def_dict = AttrDict.from_yaml(model_definition) (model_def, applied_overrides) = preprocess.load_scenario_overrides( - model_def_dict, scenario, override_dict, **kwargs + model_def_dict, scenario, override_dict ) self._init_from_model_def_dict( - model_def, applied_overrides, scenario, data_table_dfs + model_def, applied_overrides, scenario, data_table_dfs, **kwargs ) self._model_data.attrs["timestamp_model_creation"] = timestamp_model_creation @@ -144,6 +147,7 @@ def _init_from_model_def_dict( applied_overrides: str, scenario: str | None, data_table_dfs: dict[str, pd.DataFrame] | None = None, + **kwargs, ) -> None: """Initialise the model using pre-processed YAML files and optional dataframes/dicts. @@ -152,6 +156,7 @@ def _init_from_model_def_dict( applied_overrides (str): overrides specified by users scenario (str | None): scenario specified by users data_table_dfs (dict[str, pd.DataFrame] | None, optional): files with additional model information. Defaults to None. + **kwargs: Initialisation configuration overrides. """ # First pass to check top-level keys are all good validate_dict(model_definition, CONFIG_SCHEMA, "Model definition") @@ -162,19 +167,13 @@ def _init_from_model_def_dict( "model_run_creation", comment="Model: preprocessing stage 1 (model_run)", ) - model_config = AttrDict(extract_from_schema(CONFIG_SCHEMA, "default")) - model_config.union(model_definition.pop("config"), allow_override=True) - - init_config = update_then_validate_config("init", model_config) - if init_config["time_cluster"] is not None: - init_config["time_cluster"] = relative_path( - self._def_path, init_config["time_cluster"] - ) + model_config = config_schema.CalliopeConfig(**model_definition.pop("config")) + init_config = model_config.update({"init": kwargs}).init param_metadata = {"default": extract_from_schema(MODEL_SCHEMA, "default")} attributes = { - "calliope_version_defined": init_config["calliope_version"], + "calliope_version_defined": init_config.calliope_version, "calliope_version_initialised": calliope.__version__, "applied_overrides": applied_overrides, "scenario": scenario, @@ -185,11 +184,8 @@ def _init_from_model_def_dict( for table_name, table_dict in model_definition.pop("data_tables", {}).items(): table_dict, _ = climb_template_tree(table_dict, templates, table_name) data_tables.append( - DataTable( - init_config, table_name, table_dict, data_table_dfs, self._def_path - ) + DataTable(table_name, table_dict, data_table_dfs, init_config.def_path) ) - model_data_factory = ModelDataFactory( init_config, model_definition, data_tables, attributes, param_metadata ) @@ -204,9 +200,12 @@ def _init_from_model_def_dict( comment="Model: preprocessing stage 2 (model_data)", ) - self._add_observed_dict("config", model_config) + self._model_data.attrs["name"] = init_config.name + + # Unlike at the build and solve phases, we store the init config overrides in the main model config. + model_config.init = init_config + self.config = model_config - self._model_data.attrs["name"] = init_config["name"] log_time( LOGGER, self._timings, @@ -223,15 +222,15 @@ def _init_from_model_data(self, model_data: xr.Dataset) -> None: model_data (xr.Dataset): Model dataset with input parameters as arrays and configuration stored in the dataset attributes dictionary. """ - if "_def_path" in model_data.attrs: - self._def_path = model_data.attrs.pop("_def_path") if "applied_math" in model_data.attrs: self.applied_math = preprocess.CalliopeMath.from_dict( model_data.attrs.pop("applied_math") ) + if "config" in model_data.attrs: + self.config = config_schema.CalliopeConfig(**model_data.attrs.pop("config")) + self.config.update(model_data.attrs.pop("config_kwarg_overrides")) self._model_data = model_data - self._add_model_data_methods() if self.results: self._is_solved = True @@ -243,47 +242,6 @@ def _init_from_model_data(self, model_data: xr.Dataset) -> None: comment="Model: loaded model_data", ) - def _add_model_data_methods(self): - """Add observed data to `model`. - - 1. Filter model dataset to produce views on the input/results data - 2. Add top-level configuration dictionaries simultaneously to the model data attributes and as attributes of this class. - - """ - self._add_observed_dict("config") - - def _add_observed_dict(self, name: str, dict_to_add: dict | None = None) -> None: - """Add the same dictionary as property of model object and an attribute of the model xarray dataset. - - Args: - name (str): - Name of dictionary which will be set as the model property name and - (if necessary) the dataset attribute name. - dict_to_add (dict | None, optional): - If given, set as both the model property and the dataset attribute, - otherwise set an existing dataset attribute as a model property of the - same name. Defaults to None. - - Raises: - exceptions.ModelError: If `dict_to_add` is not given, it must be an attribute of model data. - TypeError: `dict_to_add` must be a dictionary. - """ - if dict_to_add is None: - try: - dict_to_add = self._model_data.attrs[name] - except KeyError: - raise exceptions.ModelError( - f"Expected the model property `{name}` to be a dictionary attribute of the model dataset. If you are loading the model from a NetCDF file, ensure it is a valid Calliope model." - ) - if not isinstance(dict_to_add, dict): - raise TypeError( - f"Attempted to add dictionary property `{name}` to model, but received argument of type `{type(dict_to_add).__name__}`" - ) - else: - dict_to_add = AttrDict(dict_to_add) - self._model_data.attrs[name] = dict_to_add - setattr(self, name, dict_to_add) - def build( self, force: bool = False, add_math_dict: dict | None = None, **kwargs ) -> None: @@ -310,30 +268,26 @@ def build( comment="Model: backend build starting", ) - backend_config = {**self.config["build"], **kwargs} - mode = backend_config["mode"] + build_config = self.config.update({"build": kwargs}).build + mode = build_config.mode if mode == "operate": if not self._model_data.attrs["allow_operate_mode"]: raise exceptions.ModelError( "Unable to run this model in operate (i.e. dispatch) mode, probably because " "there exist non-uniform timesteps (e.g. from time clustering)" ) - start_window_idx = backend_config.pop("start_window_idx", 0) - backend_input = self._prepare_operate_mode_inputs( - start_window_idx, **backend_config - ) + backend_input = self._prepare_operate_mode_inputs(build_config.operate) else: backend_input = self._model_data - init_math_list = [] if backend_config.get("ignore_mode_math") else [mode] + init_math_list = [] if build_config.ignore_mode_math else [mode] end_math_list = [] if add_math_dict is None else [add_math_dict] - full_math_list = init_math_list + backend_config["add_math"] + end_math_list + full_math_list = init_math_list + build_config.add_math + end_math_list LOGGER.debug(f"Math preprocessing | Loading math: {full_math_list}") - model_math = preprocess.CalliopeMath(full_math_list, self._def_path) + model_math = preprocess.CalliopeMath(full_math_list, self.config.init.def_path) - backend_name = backend_config.pop("backend") self.backend = backend.get_model_backend( - backend_name, backend_input, model_math, **backend_config + build_config, backend_input, model_math ) self.backend.add_optimisation_components() @@ -370,7 +324,7 @@ def solve(self, force: bool = False, warmstart: bool = False, **kwargs) -> None: exceptions.ModelError: Some preprocessing steps will stop a run mode of "operate" from being possible. """ # Check that results exist and are non-empty - if not self._is_built: + if not self.is_built: raise exceptions.ModelError( "You must build the optimisation problem (`.build()`) " "before you can run it." @@ -388,23 +342,25 @@ def solve(self, force: bool = False, warmstart: bool = False, **kwargs) -> None: else: to_drop = [] - run_mode = self.backend.inputs.attrs["config"]["build"]["mode"] + solve_config = self.config.update({"solve": kwargs}).solve + # FIXME: find a way to avoid overcomplicated passing of settings between modes + mode = self.config.update(self.config.applied_keyword_overrides).build.mode self._model_data.attrs["timestamp_solve_start"] = log_time( LOGGER, self._timings, "solve_start", - comment=f"Optimisation model | starting model in {run_mode} mode.", + comment=f"Optimisation model | starting model in {mode} mode.", ) - solver_config = update_then_validate_config("solve", self.config, **kwargs) - - shadow_prices = solver_config.get("shadow_prices", []) + shadow_prices = solve_config.shadow_prices self.backend.shadow_prices.track_constraints(shadow_prices) - if run_mode == "operate": - results = self._solve_operate(**solver_config) + if mode == "operate": + results = self._solve_operate(**solve_config.model_dump()) else: - results = self.backend._solve(warmstart=warmstart, **solver_config) + results = self.backend._solve( + warmstart=warmstart, **solve_config.model_dump() + ) log_time( LOGGER, @@ -417,7 +373,7 @@ def solve(self, force: bool = False, warmstart: bool = False, **kwargs) -> None: # Add additional post-processed result variables to results if results.attrs["termination_condition"] in ["optimal", "feasible"]: results = postprocess_results.postprocess_model_results( - results, self._model_data + results, self._model_data, self.config.solve.zero_threshold ) log_time( @@ -434,7 +390,6 @@ def solve(self, force: bool = False, warmstart: bool = False, **kwargs) -> None: self._model_data = xr.merge( [results, self._model_data], compat="override", combine_attrs="no_conflicts" ) - self._add_model_data_methods() self._model_data.attrs["timestamp_solve_complete"] = log_time( LOGGER, @@ -469,6 +424,7 @@ def to_netcdf(self, path): saved_attrs[attr] = dict(getattr(self, attr)) else: saved_attrs[attr] = getattr(self, attr) + saved_attrs["config_kwarg_overrides"] = self.config.applied_keyword_overrides io.save_netcdf(self._model_data, path, **saved_attrs) @@ -507,28 +463,24 @@ def info(self) -> str: return "\n".join(info_strings) def _prepare_operate_mode_inputs( - self, start_window_idx: int = 0, **config_kwargs + self, operate_config: config_schema.BuildOperate ) -> xr.Dataset: """Slice the input data to just the length of operate mode time horizon. Args: - start_window_idx (int, optional): - Set the operate `window` to start at, based on integer index. - This is used when re-initialising the backend model for shorter time horizons close to the end of the model period. - Defaults to 0. - **config_kwargs: kwargs related to operate mode configuration. + operate_config (config.BuildOperate): operate mode configuration options. Returns: xr.Dataset: Slice of input data. """ - window = config_kwargs["operate_window"] - horizon = config_kwargs["operate_horizon"] self._model_data.coords["windowsteps"] = pd.date_range( self.inputs.timesteps[0].item(), self.inputs.timesteps[-1].item(), - freq=window, + freq=operate_config.window, + ) + horizonsteps = self._model_data.coords["windowsteps"] + pd.Timedelta( + operate_config.horizon ) - horizonsteps = self._model_data.coords["windowsteps"] + pd.Timedelta(horizon) # We require an offset because pandas / xarray slicing is _inclusive_ of both endpoints # where we only want it to be inclusive of the left endpoint. # Except in the last time horizon, where we want it to include the right endpoint. @@ -538,11 +490,11 @@ def _prepare_operate_mode_inputs( self._model_data.coords["horizonsteps"] = clipped_horizonsteps - self._TS_OFFSET sliced_inputs = self._model_data.sel( timesteps=slice( - self._model_data.windowsteps[start_window_idx], - self._model_data.horizonsteps[start_window_idx], + self._model_data.windowsteps[operate_config.start_window_idx], + self._model_data.horizonsteps[operate_config.start_window_idx], ) ) - if config_kwargs.get("operate_use_cap_results", False): + if operate_config.use_cap_results: to_parameterise = extract_from_schema(MODEL_SCHEMA, "x-operate-param") if not self._is_solved: raise exceptions.ModelError( @@ -565,10 +517,7 @@ def _solve_operate(self, **solver_config) -> xr.Dataset: """ if self.backend.inputs.timesteps[0] != self._model_data.timesteps[0]: LOGGER.info("Optimisation model | Resetting model to first time window.") - self.build( - force=True, - **{"mode": "operate", **self.backend.inputs.attrs["config"]["build"]}, - ) + self.build(force=True, **self.config.build.applied_keyword_overrides) LOGGER.info("Optimisation model | Running first time window.") @@ -595,11 +544,9 @@ def _solve_operate(self, **solver_config) -> xr.Dataset: "Optimisation model | Reaching the end of the timeseries. " "Re-building model with shorter time horizon." ) - self.build( - force=True, - start_window_idx=idx + 1, - **self.backend.inputs.attrs["config"]["build"], - ) + build_kwargs = AttrDict(self.config.build.applied_keyword_overrides) + build_kwargs.set_key("operate.start_window_idx", idx + 1) + self.build(force=True, **build_kwargs) else: self.backend._dataset.coords["timesteps"] = new_inputs.timesteps self.backend.inputs.coords["timesteps"] = new_inputs.timesteps diff --git a/src/calliope/postprocess/postprocess.py b/src/calliope/postprocess/postprocess.py index 402b928e..327b1ce2 100644 --- a/src/calliope/postprocess/postprocess.py +++ b/src/calliope/postprocess/postprocess.py @@ -11,7 +11,7 @@ def postprocess_model_results( - results: xr.Dataset, model_data: xr.Dataset + results: xr.Dataset, model_data: xr.Dataset, zero_threshold: float ) -> xr.Dataset: """Post-processing of model results. @@ -22,11 +22,11 @@ def postprocess_model_results( Args: results (xarray.Dataset): Output from the solver backend. model_data (xarray.Dataset): Calliope model data. + zero_threshold (float): Numbers below this value will be assumed to be zero Returns: xarray.Dataset: input-results dataset. """ - zero_threshold = model_data.config.solve.zero_threshold results["capacity_factor"] = capacity_factor(results, model_data) results["systemwide_capacity_factor"] = capacity_factor( results, model_data, systemwide=True diff --git a/src/calliope/preprocess/data_tables.py b/src/calliope/preprocess/data_tables.py index 4a90fbf3..a9e7acf2 100644 --- a/src/calliope/preprocess/data_tables.py +++ b/src/calliope/preprocess/data_tables.py @@ -51,22 +51,20 @@ class DataTable: def __init__( self, - model_config: dict, table_name: str, data_table: DataTableDict, data_table_dfs: dict[str, pd.DataFrame] | None = None, - model_definition_path: Path | None = None, + model_definition_path: Path = Path("."), ): """Load and format a data table from file / in-memory object. Args: - model_config (dict): Model initialisation configuration dictionary. table_name (str): name of the data table. data_table (DataTableDict): Data table definition dictionary. data_table_dfs (dict[str, pd.DataFrame] | None, optional): If given, a dictionary mapping table names in `data_table` to in-memory pandas DataFrames. Defaults to None. - model_definition_path (Path | None, optional): + model_definition_path (Path, optional): If given, the path to the model definition YAML file, relative to which data table filepaths will be set. If None, relative data table filepaths will be considered relative to the current working directory. Defaults to None. @@ -75,7 +73,6 @@ def __init__( self.input = data_table self.dfs = data_table_dfs if data_table_dfs is not None else dict() self.model_definition_path = model_definition_path - self.config = model_config self.columns = self._listify_if_defined("columns") self.index = self._listify_if_defined("rows") diff --git a/src/calliope/preprocess/model_data.py b/src/calliope/preprocess/model_data.py index 7c6d6cc3..b4b3bad0 100644 --- a/src/calliope/preprocess/model_data.py +++ b/src/calliope/preprocess/model_data.py @@ -16,6 +16,7 @@ from calliope import exceptions from calliope.attrdict import AttrDict from calliope.preprocess import data_tables, time +from calliope.schemas.config_schema import Init from calliope.util.schema import MODEL_SCHEMA, validate_dict from calliope.util.tools import climb_template_tree, listify @@ -70,7 +71,7 @@ class ModelDataFactory: def __init__( self, - model_config: dict, + init_config: Init, model_definition: ModelDefinition, data_tables: list[data_tables.DataTable], attributes: dict, @@ -81,13 +82,13 @@ def __init__( This includes resampling/clustering timeseries data as necessary. Args: - model_config (dict): Model initialisation configuration (i.e., `config.init`). + init_config (Init): Model initialisation configuration (i.e., `config.init`). model_definition (ModelDefinition): Definition of model nodes and technologies, and their potential `templates`. data_tables (list[data_tables.DataTable]): Pre-loaded data tables that will be used to initialise the dataset before handling definitions given in `model_definition`. attributes (dict): Attributes to attach to the model Dataset. param_attributes (dict[str, dict]): Attributes to attach to the generated model DataArrays. """ - self.config: dict = model_config + self.config: Init = init_config self.model_definition: ModelDefinition = model_definition.copy() self.dataset = xr.Dataset(attrs=AttrDict(attributes)) self.tech_data_from_tables = AttrDict() @@ -244,7 +245,7 @@ def update_time_dimension_and_params(self): raise exceptions.ModelError( "Must define at least one timeseries parameter in a Calliope model." ) - time_subset = self.config.get("time_subset", None) + time_subset = self.config.time_subset if time_subset is not None: self.dataset = time.subset_timeseries(self.dataset, time_subset) self.dataset = time.add_inferred_time_params(self.dataset) @@ -252,11 +253,11 @@ def update_time_dimension_and_params(self): # By default, the model allows operate mode self.dataset.attrs["allow_operate_mode"] = 1 - if self.config["time_resample"] is not None: - self.dataset = time.resample(self.dataset, self.config["time_resample"]) - if self.config["time_cluster"] is not None: + if self.config.time_resample is not None: + self.dataset = time.resample(self.dataset, self.config.time_resample) + if self.config.time_cluster is not None: self.dataset = time.cluster( - self.dataset, self.config["time_cluster"], self.config["time_format"] + self.dataset, self.config.time_cluster, self.config.time_format ) def clean_data_from_undefined_members(self): @@ -324,7 +325,7 @@ def add_link_distances(self): self.dataset.longitude.sel(nodes=node2).item(), )["s12"] distance_array = pd.Series(distances).rename_axis(index="techs").to_xarray() - if self.config["distance_unit"] == "km": + if self.config.distance_unit == "km": distance_array /= 1000 else: LOGGER.debug( @@ -660,7 +661,7 @@ def _add_to_dataset(self, to_add: xr.Dataset, id_: str): """ to_add_numeric_dims = self._update_numeric_dims(to_add, id_) to_add_numeric_ts_dims = time.timeseries_to_datetime( - to_add_numeric_dims, self.config["time_format"], id_ + to_add_numeric_dims, self.config.time_format, id_ ) self.dataset = xr.merge( [to_add_numeric_ts_dims, self.dataset], diff --git a/src/calliope/preprocess/scenarios.py b/src/calliope/preprocess/scenarios.py index 473544fb..9673defb 100644 --- a/src/calliope/preprocess/scenarios.py +++ b/src/calliope/preprocess/scenarios.py @@ -15,7 +15,6 @@ def load_scenario_overrides( model_definition: dict, scenario: str | None = None, override_dict: dict | None = None, - **kwargs, ) -> tuple[AttrDict, str]: """Apply user-defined overrides to the model definition. @@ -28,8 +27,6 @@ def load_scenario_overrides( override_dict (dict | None, optional): Overrides to apply _after_ `scenario` overrides. Defaults to None. - **kwargs: - initialisation overrides. Returns: tuple[AttrDict, str]: @@ -88,14 +85,10 @@ def load_scenario_overrides( _log_overrides(model_def_dict, model_def_with_overrides) - model_def_with_overrides.union( - AttrDict({"config.init": kwargs}), allow_override=True - ) - return (model_def_with_overrides, ";".join(applied_overrides)) -def _combine_overrides(overrides: AttrDict, scenario_overrides: list): +def _combine_overrides(overrides: AttrDict, scenario_overrides: list[AttrDict]): combined_override_dict = AttrDict() for override in scenario_overrides: try: diff --git a/src/calliope/schemas/__init__.py b/src/calliope/schemas/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/calliope/schemas/config_schema.py b/src/calliope/schemas/config_schema.py new file mode 100644 index 00000000..4a6415d7 --- /dev/null +++ b/src/calliope/schemas/config_schema.py @@ -0,0 +1,302 @@ +# Copyright (C) since 2013 Calliope contributors listed in AUTHORS. +# Licensed under the Apache 2.0 License (see LICENSE file). +"""Implements the Calliope configuration class.""" + +from datetime import datetime +from pathlib import Path +from typing import Literal, Self + +import jsonref +from pydantic import BaseModel, Field, model_validator + +from calliope.attrdict import AttrDict +from calliope.util import tools +from calliope.util.schema import UniqueList + +MODES_T = Literal["plan", "operate", "spores"] +CONFIG_T = Literal["init", "build", "solve"] + + +def hide_from_schema(to_hide: list[str]): + """Hide fields from the generated schema. + + Args: + to_hide (list[str]): List of fields to hide. + """ + + def _hide_from_schema(schema: dict): + for hide in to_hide: + schema.get("properties", {}).pop(hide, None) + return schema + + return _hide_from_schema + + +class ConfigBaseModel(BaseModel): + """A base class for creating pydantic models for Calliope configuration options.""" + + _kwargs: dict = {} + + def update(self, update_dict: dict, deep: bool = False) -> Self: + """Return a new iteration of the model with updated fields. + + Updates are validated and stored in the parent class in the `_kwargs` key. + + Args: + update_dict (dict): Dictionary with which to update the base model. + deep (bool, optional): Set to True to make a deep copy of the model. Defaults to False. + + Returns: + BaseModel: New model instance. + """ + new_dict: dict = {} + # Iterate through dict to be updated and convert any sub-dicts into their respective pydantic model objects + for key, val in update_dict.items(): + key_class = getattr(self, key) + if isinstance(key_class, ConfigBaseModel): + new_dict[key] = key_class.update(val) + key_class._kwargs = val + else: + new_dict[key] = val + updated = super().model_copy(update=new_dict, deep=deep) + updated.model_validate(updated) + self._kwargs = update_dict + return updated + + def model_no_ref_schema(self) -> AttrDict: + """Generate an AttrDict with the schema replacing $ref/$def for better readability. + + Returns: + AttrDict: class schema. + """ + schema_dict = AttrDict(super().model_json_schema()) + schema_dict = AttrDict(jsonref.replace_refs(schema_dict)) + schema_dict.del_key("$defs") + return schema_dict + + @property + def applied_keyword_overrides(self) -> dict: + """Most recently applied keyword overrides used to update this configuration. + + Returns: + dict: Description of applied overrides. + """ + return self._kwargs + + +class Init(ConfigBaseModel): + """All configuration options used when initialising a Calliope model.""" + + model_config = { + "title": "init", + "extra": "forbid", + "frozen": True, + "json_schema_extra": hide_from_schema(["def_path"]), + "revalidate_instances": "always", + "use_attribute_docstrings": True, + } + + def_path: Path = Field(default=".", repr=False, exclude=True) + """The path to the main model definition YAML file, if one has been used to instantiate the Calliope Model class.""" + + name: str | None = Field(default=None) + """Model name""" + + calliope_version: str | None = Field(default=None) + """Calliope framework version this model is intended for""" + + time_subset: tuple[datetime, datetime] | None = Field(default=None) + """ + Subset of timesteps as an two-element list giving the **inclusive** range. + For example, ["2005-01", "2005-04"] will create a time subset from "2005-01-01 00:00:00" to "2005-04-31 23:59:59". + + Strings must be ISO8601-compatible, i.e. of the form `YYYY-mm-dd HH:MM:SS` (e.g, '2005-01 ', '2005-01-01', '2005-01-01 00:00', ...) + """ + + time_resample: str | None = Field(default=None, pattern="^[0-9]+[a-zA-Z]") + """Setting to adjust time resolution, e.g. '2h' for 2-hourly""" + + time_cluster: Path | None = Field(default=None) + """ + Setting to cluster the timeseries. + Must be a path to a file where each date is linked to a representative date that also exists in the timeseries. + """ + + time_format: str = Field(default="ISO8601") + """ + Timestamp format of all time series data when read from file. + 'ISO8601' means '%Y-%m-%d %H:%M:%S'. + """ + + distance_unit: Literal["km", "m"] = Field(default="km") + """ + Unit of transmission link `distance` (m - metres, km - kilometres). + Automatically derived distances from lat/lon coordinates will be given in this unit. + """ + + @model_validator(mode="before") + @classmethod + def abs_path(cls, data): + """Add model definition path.""" + if data.get("time_cluster", None) is not None: + data["time_cluster"] = tools.relative_path( + data["def_path"], data["time_cluster"] + ) + return data + + +class BuildOperate(ConfigBaseModel): + """Operate mode configuration options used when building a Calliope optimisation problem (`calliope.Model.build`).""" + + model_config = { + "title": "operate", + "extra": "forbid", + "json_schema_extra": hide_from_schema(["start_window_idx"]), + "revalidate_instances": "always", + "use_attribute_docstrings": True, + } + + window: str = Field(default="24h") + """ + Operate mode rolling `window`, given as a pandas frequency string. + See [here](https://pandas.pydata.org/docs/user_guide/timeseries.html#offset-aliases) for a list of frequency aliases. + """ + + horizon: str = Field(default="48h") + """ + Operate mode rolling `horizon`, given as a pandas frequency string. + See [here](https://pandas.pydata.org/docs/user_guide/timeseries.html#offset-aliases) for a list of frequency aliases. + Must be ≥ `window` + """ + + use_cap_results: bool = Field(default=False) + """If the model already contains `plan` mode results, use those optimal capacities as input parameters to the `operate` mode run.""" + + start_window_idx: int = Field(default=0, repr=False, exclude=True) + """Which time window to build. This is used to track the window when re-building the model part way through solving in `operate` mode.""" + + +class Build(ConfigBaseModel): + """Base configuration options used when building a Calliope optimisation problem (`calliope.Model.build`).""" + + model_config = { + "title": "build", + "extra": "allow", + "revalidate_instances": "always", + } + + mode: MODES_T = Field(default="plan") + """Mode in which to run the optimisation.""" + + add_math: UniqueList[str] = Field(default=[]) + """ + List of references to files which contain additional mathematical formulations to be applied on top of or instead of the base mode math. + If referring to an pre-defined Calliope math file (see documentation for available files), do not append the reference with ".yaml". + If referring to your own math file, ensure the file type is given as a suffix (".yaml" or ".yml"). + Relative paths will be assumed to be relative to the model definition file given when creating a calliope Model (`calliope.Model(model_definition=...)`) + """ + + ignore_mode_math: bool = Field(default=False) + """ + If True, do not initialise the mathematical formulation with the pre-defined math for the given run `mode`. + This option can be used to completely re-define the Calliope mathematical formulation. + """ + + backend: Literal["pyomo", "gurobi"] = Field(default="pyomo") + """Module with which to build the optimisation problem.""" + + ensure_feasibility: bool = Field(default=False) + """ + Whether to include decision variables in the model which will meet unmet demand or consume unused supply in the model so that the optimisation solves successfully. + This should only be used as a debugging option (as any unmet demand/unused supply is a sign of improper model formulation). + """ + + objective: str = Field(default="min_cost_optimisation") + """Name of internal objective function to use, from those defined in the pre-defined math and any applied additional math.""" + + pre_validate_math_strings: bool = Field(default=True) + """ + If true, the Calliope math definition will be scanned for parsing errors _before_ undertaking the much more expensive operation of building the optimisation problem. + You can switch this off (e.g., if you know there are no parsing errors) to reduce overall build time. + """ + + operate: BuildOperate = BuildOperate() + + +class SolveSpores(ConfigBaseModel): + """SPORES configuration options used when solving a Calliope optimisation problem (`calliope.Model.solve`).""" + + number: int = Field(default=3) + """SPORES mode number of iterations after the initial base run.""" + + score_cost_class: str = Field(default="score") + """SPORES mode cost class to vary between iterations after the initial base run.""" + + slack_cost_group: str = Field(default=None) + """SPORES mode cost class to keep below the given `slack` (usually "monetary").""" + + save_per_spore: bool = Field(default=False) + """ + Whether or not to save the result of each SPORES mode run between iterations. + If False, will consolidate all iterations into one dataset after completion of N iterations (defined by `number`) and save that one dataset. + """ + + save_per_spore_path: Path | None = Field(default=None) + """If saving per spore, the path to save to.""" + + skip_cost_op: bool = Field(default=False) + """If the model already contains `plan` mode results, use those as the initial base run results and start with SPORES iterations immediately.""" + + @model_validator(mode="after") + def require_save_per_spore_path(self) -> Self: + """Ensure that path is given if saving per spore.""" + if self.save_per_spore: + if self.save_per_spore_path is None: + raise ValueError( + "Must define `save_per_spore_path` if you want to save each SPORES result separately." + ) + elif not self.save_per_spore_path.is_dir(): + raise ValueError("`save_per_spore_path` must be a directory.") + return self + + +class Solve(ConfigBaseModel): + """Base configuration options used when solving a Calliope optimisation problem (`calliope.Model.solve`).""" + + model_config = { + "title": "solve", + "extra": "forbid", + "revalidate_instances": "always", + } + + save_logs: Path | None = Field(default=None) + """If given, should be a path to a directory in which to save optimisation logs.""" + + solver_io: str | None = Field(default=None) + """ + Some solvers have different interfaces that perform differently. + For instance, setting `solver_io="python"` when using the solver `gurobi` tends to reduce the time to send the optimisation problem to the solver. + """ + + solver_options: dict = Field(default={}) + """Any solver options, as key-value pairs, to pass to the chosen solver""" + + solver: str = Field(default="cbc") + """Solver to use. Any solvers that have Pyomo interfaces can be used. Refer to the Pyomo documentation for the latest list.""" + + zero_threshold: float = Field(default=1e-10) + """On postprocessing the optimisation results, values smaller than this threshold will be considered as optimisation artefacts and will be set to zero.""" + + shadow_prices: UniqueList[str] = Field(default=[]) + """Names of model constraints.""" + + spores: SolveSpores = SolveSpores() + + +class CalliopeConfig(ConfigBaseModel): + """Calliope configuration class.""" + + model_config = {"title": "config"} + init: Init = Init() + build: Build = Build() + solve: Solve = Solve() diff --git a/src/calliope/schemas/data_table_schema.py b/src/calliope/schemas/data_table_schema.py new file mode 100644 index 00000000..0638d226 --- /dev/null +++ b/src/calliope/schemas/data_table_schema.py @@ -0,0 +1,80 @@ +"""Implements the data table configuration class.""" + +from pydantic import BaseModel, model_validator +from typing_extensions import Self + +from calliope.util.schema import AttrStr, UniqueList +from calliope.util.tools import listify + + +class DataTable(BaseModel): + """Data table validation model.""" + + data: str + """ + Absolute or relative filepath. + Relative paths are based on the model config file used to initialise the model. + """ + rows: None | AttrStr | UniqueList[AttrStr] = None + """ + Names of dimensions defined row-wise. + Each name should correspond to a column in your data that contains index items. + These columns must be to the left of the columns containing your data. + """ + columns: None | AttrStr | UniqueList[AttrStr] = None + """ + Names of dimensions defined column-wise. + Each name should correspond to a row in your data that contains index items. + These rows must be above the rows containing your data. + """ + select: None | dict[AttrStr, AttrStr | UniqueList[AttrStr]] = None + """ + Select one or more index item from a dimension. + Selection takes place before `drop` and `add_dims`, so you can select a single + value from a data dimension and then drop the dimension so it doesn't find its way + through to the final dataset. + """ + drop: None | AttrStr | UniqueList[AttrStr] = None + """ + Enables removing rows and/or columns that contain irrelevant data/metadata. + These could include comments on the source of the data, the data license, or the parameter units. + You can also drop a dimension and then reintroduce it in `add_dims`, but with different index items. + """ + add_dims: None | dict[AttrStr, AttrStr] = None + """ + Data dimensions to add after loading in the array. + These allow you to use the same file to assign values to different parameters/dimension index items + (e.g., setting `flow_cap_min` and `flow_cap_max` to the same value), + or to add a dimension which would otherwise be a column containing the same information in each row + (e.g., assigning the cost class to monetary for a file containing cost data). + """ + rename_dims: None | dict[AttrStr, AttrStr] = None + """ + Mapping between dimension names in the data table being loaded to equivalent Calliope dimension names. + For instance, the "time" column in the data table would need to be mapped to "timesteps": `{"time": "timesteps"}`. + """ + template: None | AttrStr = None + """ + Reference to a template from which to inherit common configuration options. + """ + + @model_validator(mode="after") + def check_row_and_columns(self) -> Self: + """Ensure users specify a valid data table shape.""" + rows = set(listify(self.rows)) + columns = set(listify(self.columns)) + if not rows and not columns: + raise ValueError("Either row or columns must be defined for data_table.") + elif rows & columns: + raise ValueError("Rows and columns must not overlap.") + + if self.add_dims: + if self.add_dims.keys() & (rows | columns): + raise ValueError("Added dimensions must not be in columns or rows.") + + if self.rename_dims: + if set(self.rename_dims.values()) - (rows | columns): + raise ValueError( + "Renamed dimensions must be in either rows or columns." + ) + return self diff --git a/src/calliope/util/schema.py b/src/calliope/util/schema.py index bd98cc77..f8cb7b6c 100644 --- a/src/calliope/util/schema.py +++ b/src/calliope/util/schema.py @@ -5,10 +5,13 @@ import importlib import re import sys +from collections.abc import Hashable from copy import deepcopy -from typing import Literal +from typing import Annotated, Literal, TypeVar import jsonschema +from pydantic import AfterValidator, Field +from pydantic_core import PydanticCustomError from calliope.attrdict import AttrDict from calliope.exceptions import print_warnings_and_raise_errors @@ -19,26 +22,34 @@ DATA_TABLE_SCHEMA = load_config("data_table_schema.yaml") MATH_SCHEMA = load_config("math_schema.yaml") +# Regular string pattern for most calliope attributes +FIELD_REGEX = r"^[^_^\d][\w]*$" +AttrStr = Annotated[str, Field(pattern=FIELD_REGEX)] + +# == +# Taken from https://github.com/pydantic/pydantic-core/pull/820#issuecomment-1670475909 +T = TypeVar("T", bound=Hashable) + + +def _validate_unique_list(v: list[T]) -> list[T]: + if len(v) != len(set(v)): + raise PydanticCustomError("unique_list", "List must be unique") + return v + + +UniqueList = Annotated[ + list[T], + AfterValidator(_validate_unique_list), + Field(json_schema_extra={"uniqueItems": True}), +] +# == + def reset(): """Reset all module-level schema to the pre-defined dictionaries.""" importlib.reload(sys.modules[__name__]) -def update_then_validate_config( - config_key: str, config_dict: AttrDict, **update_kwargs -) -> AttrDict: - """Return an updated version of the configuration schema.""" - to_validate = deepcopy(config_dict[config_key]) - to_validate.union(AttrDict(update_kwargs), allow_override=True) - validate_dict( - {"config": {config_key: to_validate}}, - CONFIG_SCHEMA, - f"`{config_key}` configuration", - ) - return to_validate - - def update_model_schema( top_level_property: Literal["nodes", "techs", "parameters"], new_entries: dict, diff --git a/src/calliope/util/tools.py b/src/calliope/util/tools.py index dee2f6ca..3d8d4320 100644 --- a/src/calliope/util/tools.py +++ b/src/calliope/util/tools.py @@ -15,7 +15,7 @@ T = TypeVar("T") -def relative_path(base_path_file, path) -> Path: +def relative_path(base_path_file: str | Path, path: str | Path) -> Path: """Path standardization. If ``path`` is not absolute, it is interpreted as relative to the @@ -23,7 +23,7 @@ def relative_path(base_path_file, path) -> Path: """ # Check if base_path_file is a string because it might be an AttrDict path = Path(path) - if path.is_absolute() or base_path_file is None: + if path.is_absolute(): return path else: base_path_file = Path(base_path_file) diff --git a/tests/common/util.py b/tests/common/util.py index 8ae70da8..94f90dc2 100644 --- a/tests/common/util.py +++ b/tests/common/util.py @@ -95,9 +95,7 @@ def build_lp( math (dict | None, optional): All constraint/global expression/objective math to apply. Defaults to None. backend_name (Literal["pyomo"], optional): Backend to use to create the LP file. Defaults to "pyomo". """ - math = calliope.preprocess.CalliopeMath( - ["plan", *model.config.build.get("add_math", [])] - ) + math = calliope.preprocess.CalliopeMath(["plan", *model.config.build.add_math]) math_to_add = calliope.AttrDict() if isinstance(math_data, dict): diff --git a/tests/conftest.py b/tests/conftest.py index 3d4694c5..f85c7593 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,7 +8,8 @@ from calliope.attrdict import AttrDict from calliope.backend import latex_backend_model, pyomo_backend_model from calliope.preprocess import CalliopeMath -from calliope.util.schema import CONFIG_SCHEMA, MODEL_SCHEMA, extract_from_schema +from calliope.schemas import config_schema +from calliope.util.schema import MODEL_SCHEMA, extract_from_schema from .common.util import build_test_model as build_model @@ -33,7 +34,7 @@ def foreach(request): @pytest.fixture(scope="session") def config_defaults(): - return AttrDict(extract_from_schema(CONFIG_SCHEMA, "default")) + return AttrDict(config_schema.CalliopeConfig().model_dump()) @pytest.fixture(scope="session") diff --git a/tests/test_core_model.py b/tests/test_core_model.py index e16ebfa4..ddd97800 100644 --- a/tests/test_core_model.py +++ b/tests/test_core_model.py @@ -9,7 +9,6 @@ import calliope.preprocess from .common.util import build_test_model as build_model -from .common.util import check_error_or_warning LOGGER = "calliope.model" @@ -32,40 +31,6 @@ def test_info(self, national_scale_example): def test_info_simple_model(self, simple_supply): simple_supply.info() - def test_update_observed_dict(self, national_scale_example): - national_scale_example.config.build["backend"] = "foo" - assert national_scale_example._model_data.attrs["config"].build.backend == "foo" - - def test_add_observed_dict_from_model_data( - self, national_scale_example, dict_to_add - ): - national_scale_example._model_data.attrs["foo"] = dict_to_add - national_scale_example._add_observed_dict("foo") - assert national_scale_example.foo == dict_to_add - assert national_scale_example._model_data.attrs["foo"] == dict_to_add - - def test_add_observed_dict_from_dict(self, national_scale_example, dict_to_add): - national_scale_example._add_observed_dict("bar", dict_to_add) - assert national_scale_example.bar == dict_to_add - assert national_scale_example._model_data.attrs["bar"] == dict_to_add - - def test_add_observed_dict_not_available(self, national_scale_example): - with pytest.raises(calliope.exceptions.ModelError) as excinfo: - national_scale_example._add_observed_dict("baz") - assert check_error_or_warning( - excinfo, - "Expected the model property `baz` to be a dictionary attribute of the model dataset", - ) - assert not hasattr(national_scale_example, "baz") - - def test_add_observed_dict_not_dict(self, national_scale_example): - with pytest.raises(TypeError) as excinfo: - national_scale_example._add_observed_dict("baz", "bar") - assert check_error_or_warning( - excinfo, - "Attempted to add dictionary property `baz` to model, but received argument of type `str`", - ) - class TestOperateMode: @contextmanager @@ -127,9 +92,7 @@ def rerun_operate_log(self, request, operate_model_and_log): def test_backend_build_mode(self, operate_model_and_log): """Verify that we have run in operate mode""" operate_model, _ = operate_model_and_log - assert ( - operate_model.backend.inputs.attrs["config"]["build"]["mode"] == "operate" - ) + assert operate_model.backend.config.mode == "operate" def test_operate_mode_success(self, operate_model_and_log): """Solving in operate mode should lead to an optimal solution.""" @@ -153,8 +116,8 @@ def test_reset_model_window(self, rerun_operate_log): def test_end_of_horizon(self, operate_model_and_log): """Check that increasingly shorter time horizons are logged as model rebuilds.""" operate_model, log = operate_model_and_log - config = operate_model.backend.inputs.attrs["config"]["build"] - if config["operate_window"] != config["operate_horizon"]: + config = operate_model.backend.config.operate + if config.operate_window != config.operate_horizon: assert "Reaching the end of the timeseries." in log else: assert "Reaching the end of the timeseries." not in log diff --git a/tests/test_data_table_schema.py b/tests/test_data_table_schema.py new file mode 100644 index 00000000..ba4d76b4 --- /dev/null +++ b/tests/test_data_table_schema.py @@ -0,0 +1,96 @@ +"""Test data table schema validation.""" + +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from calliope.attrdict import AttrDict +from calliope.schemas.data_table_schema import DataTable + +from .common.util import check_error_or_warning + + +@pytest.fixture +def full_data_table_config(): + return """data: time_varying_df +rows: timesteps +columns: [comment, nodes, techs] +select: + nodes: [node1, node2] + techs: pv +drop: comment +add_dims: + parameters: something + costs: monetary +rename_dims: + location: nodes +template: some_template +""" + + +@pytest.fixture +def model_yaml_data_tables() -> AttrDict: + return AttrDict.from_yaml( + Path(__file__).parent + / "common" + / "national_scale_from_data_tables" + / "model.yaml" + ) + + +@pytest.mark.parametrize( + "data_table", + [{"rows": "timesteps"}, {"rows": "timesteps", "columns": ["techs", "nodes"]}], +) +def test_path_not_provided(data_table): + """Not providing the path should result in a failure.""" + with pytest.raises(ValidationError): + DataTable(**data_table) + + +@pytest.mark.parametrize("data_table", [{"data": "foo"}]) +def test_incomplete_column_or_row(data_table): + """Not providing either rows or columns is invalid.""" + with pytest.raises(ValidationError) as excinfo: + DataTable(**data_table) + assert check_error_or_warning( + excinfo, "Either row or columns must be defined for data_table." + ) + + +@pytest.mark.parametrize( + ("rows", "columns"), + [ + ("nodes", "nodes"), + (["nodes", "techs"], "techs"), + (["nodes", "techs", "params"], ["params", "costs"]), + ], +) +def test_row_column_overlap(rows, columns): + """Rows and columns must not share any similar values.""" + with pytest.raises(ValidationError) as excinfo: + DataTable(data="foobar", rows=rows, columns=columns) + assert check_error_or_warning(excinfo, "Rows and columns must not overlap.") + + +@pytest.mark.parametrize( + ("rows", "columns", "add_dims"), [("nodes", None, {"nodes": "MEX"})] +) +def test_add_dims_overlap(rows, columns, add_dims): + with pytest.raises(ValidationError) as excinfo: + DataTable(data="foo", rows=rows, columns=columns, add_dims=add_dims) + assert check_error_or_warning( + excinfo, "Added dimensions must not be in columns or rows." + ) + + +def test_full_table_config(full_data_table_config): + """Test a fully fledged data table configuration.""" + DataTable(**AttrDict.from_yaml_string(full_data_table_config)) + + +def test_data_table_model(model_yaml_data_tables): + """Data table validation must conform to expected usage.""" + for data_table in model_yaml_data_tables["data_tables"].values(): + DataTable(**data_table) diff --git a/tests/test_preprocess_model_data.py b/tests/test_preprocess_model_data.py index 48bc519c..e3208e1a 100644 --- a/tests/test_preprocess_model_data.py +++ b/tests/test_preprocess_model_data.py @@ -202,10 +202,14 @@ def test_add_link_distances_missing_distance( @pytest.mark.parametrize(("unit", "expected"), [("m", 343834), ("km", 343.834)]) def test_add_link_distances_no_da( - self, my_caplog, model_data_factory_w_params: ModelDataFactory, unit, expected + self, + mocker, + my_caplog, + model_data_factory_w_params: ModelDataFactory, + unit, + expected, ): - _default_distance_unit = model_data_factory_w_params.config["distance_unit"] - model_data_factory_w_params.config["distance_unit"] = unit + mocker.patch.object(ModelDataFactory, "config.distance_unit", return_value=unit) model_data_factory_w_params.clean_data_from_undefined_members() model_data_factory_w_params.dataset["latitude"] = ( pd.Series({"A": 51.507222, "B": 48.8567}) @@ -220,7 +224,6 @@ def test_add_link_distances_no_da( del model_data_factory_w_params.dataset["distance"] model_data_factory_w_params.add_link_distances() - model_data_factory_w_params.config["distance_unit"] = _default_distance_unit assert "Link distance matrix automatically computed" in my_caplog.text assert ( model_data_factory_w_params.dataset["distance"].dropna("techs")