diff --git a/docs/advanced/mode.md b/docs/advanced/mode.md index 8a7701f4..5ffb2835 100644 --- a/docs/advanced/mode.md +++ b/docs/advanced/mode.md @@ -65,9 +65,6 @@ For this reason, `horizon` must always be equal to or larger than `window`. ## SPORES mode -!!! warning - SPORES mode has not yet been re-implemented in Calliope v0.7. - `SPORES` refers to Spatially-explicit Practically Optimal REsultS. This run mode allows a user to generate any number of alternative results which are within a certain range of the optimal cost. It follows on from previous work in the field of `modelling to generate alternatives` (MGA), with a particular emphasis on alternatives that vary maximally in the spatial dimension. @@ -80,15 +77,9 @@ config.build.mode: spores config.solve: # The number of SPORES to generate: spores_number: 10 - # The cost class to optimise against when generating SPORES: - spores_score_cost_class: spores_score - # The initial system cost to limit the SPORES to fit within: - spores_cost_max: .inf - # The cost class to constrain to be less than or equal to `spores_cost_max`: - spores_slack_cost_group: monetary parameters: # The fraction above the cost-optimal cost to set the maximum cost during SPORES: - slack: 0.1 + spores_slack: 0.1 ``` You will now also need a `spores_score` cost class in your model. diff --git a/docs/examples/loading_tabular_data.py b/docs/examples/loading_tabular_data.py index dc41a7ea..1157e101 100644 --- a/docs/examples/loading_tabular_data.py +++ b/docs/examples/loading_tabular_data.py @@ -1,7 +1,6 @@ # --- # jupyter: # jupytext: -# custom_cell_magics: kql # text_representation: # extension: .py # format_name: percent diff --git a/docs/examples/modes.py b/docs/examples/modes.py new file mode 100644 index 00000000..9b16a45d --- /dev/null +++ b/docs/examples/modes.py @@ -0,0 +1,190 @@ +# --- +# jupyter: +# jupytext: +# text_representation: +# extension: .py +# format_name: percent +# format_version: '1.3' +# jupytext_version: 1.16.4 +# kernelspec: +# display_name: calliope_docs_build +# language: python +# name: calliope_docs_build +# --- + +# %% [markdown] +# # Running models in different modes +# +# Models can be built and solved in different modes: + +# - `plan` mode. +# In `plan` mode, the user defines upper and lower boundaries for technology capacities and the model decides on an optimal system configuration. +# In this configuration, the total cost of investing in technologies and then using them to meet demand in every _timestep_ (e.g., every hour) is as low as possible. +# - `operate` mode. +# In `operate` mode, all capacity constraints are fixed and the system is operated with a receding horizon control algorithm. +# This is sometimes known as a `dispatch` model - we're only concerned with the _dispatch_ of technologies whose capacities are already fixed. +# Optimisation is limited to a time horizon which +# - `spores` mode. +# `SPORES` refers to Spatially-explicit Practically Optimal REsultS. +# This run mode allows a user to generate any number of alternative results which are within a certain range of the optimal cost. + +# In this notebook we will run the Calliope national scale example model in these three modes. + +# More detail on these modes is given in the [_advanced_ section of the Calliope documentation](https://calliope.readthedocs.io/en/latest/advanced/mode/). + +# %% + +import plotly.express as px +import plotly.graph_objects as go +import xarray as xr + +import calliope + +# We update logging to show a bit more information but to hide the solver output, which can be long. +calliope.set_log_verbosity("INFO", include_solver_output=False) + +# %% [markdown] +# ## Running in `plan` mode. + +# %% +# We subset to the same time range as operate mode +model_plan = calliope.examples.national_scale(time_subset=["2005-01-01", "2005-01-10"]) +model_plan.build() +model_plan.solve() + +# %% [markdown] +# ## Running in `operate` mode. + +# %% +model_operate = calliope.examples.national_scale(scenario="operate") +model_operate.build() +model_operate.solve() + +# %% [markdown] +# Note how we have capacity variables as parameters in the inputs and only dispatch variables in the results + +# %% +model_operate.inputs[["flow_cap", "storage_cap", "area_use"]] + +# %% +model_operate.results + +# %% [markdown] +# ## Running in `spores` mode. + +# %% +# We subset to the same time range as operate/plan mode +model_spores = calliope.examples.national_scale( + scenario="spores", time_subset=["2005-01-01", "2005-01-10"] +) +model_spores.build() +model_spores.solve() + +# %% [markdown] +# Note how we have a new `spores` dimension in our results. + +# %% +model_spores.results + +# %% [markdown] +# We can track the SPORES scores used between iterations using the `spores_score_cumulative` result. +# This scoring mechanism is based on increasing the score of any technology-node combination where the + +# %% +# We do some prettification of the outputs +model_spores.results.spores_score_cumulative.to_series().where( + lambda x: x > 0 +).dropna().unstack("spores") + +# %% [markdown] +# ## Visualising results +# +# We can use [plotly](https://plotly.com/) to quickly examine our results. +# These are just some examples of how to visualise Calliope data. + +# %% +# We set the color mapping to use in all our plots by extracting the colors defined in the technology definitions of our model. +# We also create some reusable plotting functions. +colors = model_plan.inputs.color.to_series().to_dict() + + +def plot_flows(results: xr.Dataset) -> go.Figure: + df_electricity = ( + (results.flow_out.fillna(0) - results.flow_in.fillna(0)) + .sel(carriers="power") + .sum("nodes") + .to_series() + .where(lambda x: x != 0) + .dropna() + .to_frame("Flow in/out (kWh)") + .reset_index() + ) + df_electricity_demand = df_electricity[df_electricity.techs == "demand_power"] + df_electricity_other = df_electricity[df_electricity.techs != "demand_power"] + + fig = px.bar( + df_electricity_other, + x="timesteps", + y="Flow in/out (kWh)", + color="techs", + color_discrete_map=colors, + ) + fig.add_scatter( + x=df_electricity_demand.timesteps, + y=-1 * df_electricity_demand["Flow in/out (kWh)"], + marker_color="black", + name="demand", + ) + return fig + + +def plot_capacity(results: xr.Dataset, **plotly_kwargs) -> go.Figure: + df_capacity = ( + results.flow_cap.where(results.techs != "demand_power") + .sel(carriers="power") + .to_series() + .where(lambda x: x != 0) + .dropna() + .to_frame("Flow capacity (kW)") + .reset_index() + ) + + fig = px.bar( + df_capacity, + x="nodes", + y="Flow capacity (kW)", + color="techs", + color_discrete_map=colors, + **plotly_kwargs, + ) + return fig + + +# %% [markdown] +# ## `plan` vs `operate` +# Here, we compare flows over the 10 days. +# Note how flows do not match as the rolling horizon makes it difficult to make the correct storage charge/discharge decisions. + +# %% +fig_flows_plan = plot_flows( + model_plan.results.sel(timesteps=model_operate.results.timesteps) +) +fig_flows_plan.update_layout(title="Plan mode flows") + + +# %% +fig_flows_operate = plot_flows(model_operate.results) +fig_flows_operate.update_layout(title="Operate mode flows") + +# %% [markdown] +# ## `plan` vs `spores` +# Here, we compare installed capacities between the baseline run (== `plan` mode) and the SPORES. +# Note how the baseline SPORE is the same as `plan` mode and then results deviate considerably. + +# %% +fig_flows_plan = plot_capacity(model_plan.results) +fig_flows_plan.update_layout(title="Plan mode capacities") + +# %% +fig_flows_spores = plot_capacity(model_spores.results, facet_col="spores") +fig_flows_spores.update_layout(title="SPORES mode capacities") diff --git a/docs/examples/national_scale/notebook.py b/docs/examples/national_scale/notebook.py index 70e6d114..28b0684d 100644 --- a/docs/examples/national_scale/notebook.py +++ b/docs/examples/national_scale/notebook.py @@ -109,7 +109,7 @@ # %% [markdown] # #### Plotting flows -# We do this by combinging in- and out-flows and separating demand from other technologies. +# We do this by combining in- and out-flows and separating demand from other technologies. # First, we look at the aggregated result across all nodes, then we look at each node separately. # %% diff --git a/docs/examples/national_scale/spores_run.py b/docs/examples/national_scale/spores_run.py deleted file mode 100644 index 7613b53b..00000000 --- a/docs/examples/national_scale/spores_run.py +++ /dev/null @@ -1,136 +0,0 @@ -# --- -# jupyter: -# jupytext: -# text_representation: -# extension: .py -# format_name: percent -# format_version: '1.3' -# jupytext_version: 1.16.4 -# kernelspec: -# display_name: Python 3 (ipykernel) -# language: python -# name: python3 -# --- - -# %% [markdown] -# # Generating SPORES -# An interactive example of how to generate near-optimal system designs (or SPORES) out of a Calliope v0.7.0 model. This example relies solely on default software functionality and a custom Python function to determine how to assign penalties (scores) to previously explored system design options. - -# %% -# Importing the required packages -import calliope -import xarray as xr - -# %% [markdown] -# ## Cost-optimal model run and extraction of SPORES-relevant outputs - -# %% -# Loading model files and building the model -model = calliope.examples.national_scale(scenario="spores") -model.build() - -# Solving -model.solve() - -# Extracting SPORES-relevant data -least_feasible_cost = model.results.cost.loc[{"costs": "monetary"}].sum().sum() -print("The minimum cost for a feasible system design is {}".format(least_feasible_cost.values)) - - -# %% [markdown] -# ## SPORES model run -# ### Definition of the penalty-assignment methods - -# %% -def scoring_integer(results, backend): - # Filter for technologies of interest - spores_techs = backend.inputs["spores_tracker"].notnull() - # Look at capacity deployment in the previous iteration - previous_cap = results.flow_cap - # Make sure that penalties are applied only to non-negligible deployments of capacity - min_relevant_size = 0.1 * previous_cap.where(spores_techs).max( - ["nodes", "carriers", "techs"] - ) - # Where capacity was deployed more than the minimal relevant size, assign an integer penalty (score) - new_score = previous_cap.copy() - new_score = new_score.where(spores_techs, other=0) - new_score = new_score.where(new_score > min_relevant_size, other=0) - new_score = new_score.where(new_score == 0, other=1000) - # Transform the score into a "cost" parameter - new_score.rename("cost_flow_cap") - new_score = new_score.expand_dims(costs=["spores_score"]).copy() - new_score = new_score.sum("carriers") - # Extract the existing cost parameters from the backend - all_costs = backend.get_parameter("cost_flow_cap", as_backend_objs=False) - try: - all_costs = all_costs.expand_dims(nodes=results.nodes).copy() - except: - pass - # Create a new version of the cost parameters by adding up the calculated scores - new_all_costs = all_costs - new_all_costs.loc[{"costs":"spores_score"}] += new_score.loc[{"costs":"spores_score"}] - - return new_all_costs - - -# %% [markdown] -# ### Iterating over the desired number of alternatives - -# %% -# Create some lists to store results as they get generated -spores = [] # full results -scores = [] # scores only -spores_counter = 1 -number_of_spores = 5 - -# %% -for i in range(spores_counter, spores_counter + number_of_spores): - - if spores_counter == 1: - # Store the cost-optimal results - spores.append(model.results.expand_dims(spores=[0])) - scores.append( - model.backend.get_parameter("cost_flow_cap", as_backend_objs=False) - .sel(costs="spores_score") - .expand_dims(spores=[0]) - ) - # Update the slack-cost backend parameter based on the calculated minimum feasible system design cost - model.backend.update_parameter("spores_cost_max", least_feasible_cost) - # Update the objective_cost_weights to reflect the ones defined for the SPORES mode - model.backend.update_parameter( - "objective_cost_weights", model.inputs.spores_objective_cost_weights - ) - else: - pass - - # Calculate weights based on a scoring method - spores_score = scoring_integer(model.results, model.backend) - # Assign a new score based on the calculated penalties - model.backend.update_parameter( - "cost_flow_cap", spores_score.reindex_like(model.inputs.cost_flow_cap) - ) - # Run the model again to get a solution that reflects the new penalties - model.solve(force=True) - # Store the results - spores.append(model.results.expand_dims(spores=[i])) - scores.append( - model.backend.get_parameter("cost_flow_cap", as_backend_objs=False) - .sel(costs="spores_score") - .expand_dims(spores=[i]) - ) - - spores_counter += 1 - -# Concatenate the results in the storage lists into xarray objects -spore_ds = xr.concat(spores, dim="spores") -score_da = xr.concat(scores, dim="spores") - -# %% [markdown] -# ## Sense-check - -# %% -# Extract the deployed capacities across SPORES, which we want to inspect -flow_caps = spore_ds.flow_cap.where( - model.backend.inputs["spores_tracker"].notnull()).sel( - carriers='power').to_series().dropna().unstack("spores") -flow_caps diff --git a/mkdocs.yml b/mkdocs.yml index 2db9eee6..1ac4a8d9 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -132,6 +132,7 @@ nav: - examples/milp/index.md - examples/milp/notebook.py - examples/loading_tabular_data.py + - examples/modes.py - examples/piecewise_constraints.py - examples/calliope_model_object.py - examples/calliope_logging.py diff --git a/src/calliope/backend/backend_model.py b/src/calliope/backend/backend_model.py index 4bac6969..6a71d601 100644 --- a/src/calliope/backend/backend_model.py +++ b/src/calliope/backend/backend_model.py @@ -69,6 +69,8 @@ class BackendModelGenerator(ABC): _PARAM_DESCRIPTIONS = extract_from_schema(MODEL_SCHEMA, "description") _PARAM_UNITS = extract_from_schema(MODEL_SCHEMA, "x-unit") _PARAM_TYPE = extract_from_schema(MODEL_SCHEMA, "x-type") + objective: str + """Optimisation problem objective name.""" def __init__(self, inputs: xr.Dataset, math: CalliopeMath, **kwargs): """Abstract base class to build a representation of the optimisation problem. diff --git a/src/calliope/backend/gurobi_backend_model.py b/src/calliope/backend/gurobi_backend_model.py index 46c5fd08..b67c72eb 100644 --- a/src/calliope/backend/gurobi_backend_model.py +++ b/src/calliope/backend/gurobi_backend_model.py @@ -146,7 +146,7 @@ def _objective_setter( if name == self.inputs.attrs["config"].build.objective: self._instance.setObjective(expr.item(), sense=sense) - + self.objective = name self.log("objectives", name, "Objective activated.") return xr.DataArray(expr) @@ -157,7 +157,8 @@ def set_objective(self, name: str) -> None: # noqa: D102, override to_set = self.objectives[name] sense = self.OBJECTIVE_SENSE_DICT[to_set.attrs["sense"]] self._instance.setObjective(to_set.item(), sense=sense) - self.log("objectives", name, "Objective activated.") + self.objective = name + self.log("objectives", name, "Objective activated.", level="info") def get_parameter( # noqa: D102, override self, name: str, as_backend_objs: bool = True diff --git a/src/calliope/backend/latex_backend_model.py b/src/calliope/backend/latex_backend_model.py index c33229b0..2a344d30 100644 --- a/src/calliope/backend/latex_backend_model.py +++ b/src/calliope/backend/latex_backend_model.py @@ -476,6 +476,12 @@ def _objective_setter( equations=equation_strings, sense=sense_dict[objective_dict["sense"]], ) + if name == self.inputs.attrs["config"].build.objective: + self.objective = name + + def set_objective(self, name: str): # noqa: D102, override + self.objective = name + self.log("objectives", name, "Objective activated.", level="info") def _create_obj_list( self, key: str, component_type: backend_model._COMPONENTS_T @@ -534,7 +540,8 @@ def generate_math_doc( "yaml_snippet": da.attrs.get("yaml_snippet", None), } for name, da in sorted(getattr(self, objtype).data_vars.items()) - if "math_string" in da.attrs + if (objtype == "objectives" and name == self.objective) + or (objtype != "objectives" and "math_string" in da.attrs) or (objtype == "parameters" and da.attrs["references"]) ] for objtype in [ diff --git a/src/calliope/backend/pyomo_backend_model.py b/src/calliope/backend/pyomo_backend_model.py index ba0b83a5..975dd9a4 100644 --- a/src/calliope/backend/pyomo_backend_model.py +++ b/src/calliope/backend/pyomo_backend_model.py @@ -189,6 +189,7 @@ def _objective_setter( if name == self.inputs.attrs["config"].build.objective: text = "activated" objective.activate() + self.objective = name else: text = "deactivated" objective.deactivate() @@ -202,13 +203,12 @@ def _objective_setter( def set_objective(self, name: str) -> None: # noqa: D102, override for obj_name, obj in self.objectives.items(): if obj.item().active and obj_name != name: - self.log("objectives", obj_name, "Objective deactivated.") + self.log("objectives", obj_name, "Objective deactivated.", level="info") obj.item().deactivate() - elif obj.item().active and obj_name == name: - self.log("objectives", obj_name, "Objective already activated.") - elif not obj.item().active and obj_name == name: - self.log("objectives", obj_name, "Objective activated.") + if obj_name == name: obj.item().activate() + self.log("objectives", obj_name, "Objective activated.", level="info") + self.objective = name def get_parameter( # noqa: D102, override self, name: str, as_backend_objs: bool = True diff --git a/src/calliope/model.py b/src/calliope/model.py index d8f83ae2..71dbddee 100644 --- a/src/calliope/model.py +++ b/src/calliope/model.py @@ -320,7 +320,7 @@ def build( backend_input = self._prepare_operate_mode_inputs( start_window_idx, **backend_config ) - if mode == "spores": + elif mode == "spores": backend_input = self._model_data.copy() if "spores_score" not in backend_input: backend_input["spores_score"] = xr.DataArray(0).assign_attrs( diff --git a/tests/test_backend_gurobi.py b/tests/test_backend_gurobi.py index d7c1beff..4491e2b3 100755 --- a/tests/test_backend_gurobi.py +++ b/tests/test_backend_gurobi.py @@ -1,3 +1,5 @@ +import logging + import gurobipy import pytest # noqa: F401 import xarray as xr @@ -96,6 +98,31 @@ def test_add_valid_obj(self, simple_supply_gurobi): ) assert "foo" in simple_supply_gurobi.backend.objectives + def test_default_objective_set(self, simple_supply_longnames): + obj = simple_supply_longnames.backend._instance.getObjective() + + assert "flow_cap" in str(obj) + assert simple_supply_longnames.backend.objective == "min_cost_optimisation" + + def test_new_objective_set(self, simple_supply_gurobi_func): + simple_supply_gurobi_func.backend.add_objective( + "foo", {"equations": [{"expression": "bigM"}], "sense": "minimise"} + ) + simple_supply_gurobi_func.backend.set_objective("foo") + simple_supply_gurobi_func.backend.verbose_strings() + obj = simple_supply_gurobi_func.backend._instance.getObjective() + assert simple_supply_gurobi_func.backend.objective == "foo" + + assert "flow_cap" not in str(obj) + + def test_new_objective_set_log(self, caplog, simple_supply_gurobi_func): + caplog.set_level(logging.INFO) + simple_supply_gurobi_func.backend.add_objective( + "foo", {"equations": [{"expression": "bigM"}], "sense": "minimise"} + ) + simple_supply_gurobi_func.backend.set_objective("foo") + assert ":foo | Objective activated." in caplog.text + def test_object_string_representation(self, simple_supply_gurobi): assert ( simple_supply_gurobi.backend.variables.flow_out.sel( diff --git a/tests/test_backend_latex_backend.py b/tests/test_backend_latex_backend.py index e28b0830..7c100ffe 100644 --- a/tests/test_backend_latex_backend.py +++ b/tests/test_backend_latex_backend.py @@ -1,3 +1,4 @@ +import logging import textwrap import pytest @@ -198,6 +199,21 @@ def test_add_objective(self, dummy_latex_backend_model): assert "obj" not in dummy_latex_backend_model.valid_component_names assert len(dummy_latex_backend_model.objectives.data_vars) == 1 + def test_default_objective_set(self, dummy_latex_backend_model): + dummy_latex_backend_model.objective == "min_cost_optimisation" + + def test_new_objective_set(self, dummy_latex_backend_model): + dummy_latex_backend_model.add_objective( + "foo", {"equations": [{"expression": "bigM"}], "sense": "minimise"} + ) + dummy_latex_backend_model.set_objective("foo") + assert dummy_latex_backend_model.objective == "foo" + + def test_new_objective_set_log(self, caplog, dummy_latex_backend_model): + caplog.set_level(logging.INFO) + dummy_latex_backend_model.set_objective("foo") + assert ":foo | Objective activated." in caplog.text + def test_add_piecewise_constraint(self, dummy_latex_backend_model): dummy_latex_backend_model.add_parameter( "piecewise_x", diff --git a/tests/test_backend_pyomo.py b/tests/test_backend_pyomo.py index 710d147a..f47f458b 100755 --- a/tests/test_backend_pyomo.py +++ b/tests/test_backend_pyomo.py @@ -1842,6 +1842,29 @@ def test_add_valid_obj(self, simple_supply): assert "foo" in simple_supply.backend.objectives assert not simple_supply.backend.objectives.foo.item().active + def test_default_objective_set(self, simple_supply): + assert simple_supply.backend.objectives.min_cost_optimisation.item().active + assert simple_supply.backend.objective == "min_cost_optimisation" + + def test_new_objective_set(self, simple_supply_build_func): + simple_supply_build_func.backend.add_objective( + "foo", {"equations": [{"expression": "bigM"}], "sense": "minimise"} + ) + simple_supply_build_func.backend.set_objective("foo") + + assert simple_supply_build_func.backend.objectives.foo.item().active + assert not simple_supply_build_func.backend.objectives.min_cost_optimisation.item().active + assert simple_supply_build_func.backend.objective == "foo" + + def test_new_objective_set_log(self, caplog, simple_supply_build_func): + caplog.set_level(logging.INFO) + simple_supply_build_func.backend.add_objective( + "foo", {"equations": [{"expression": "bigM"}], "sense": "minimise"} + ) + simple_supply_build_func.backend.set_objective("foo") + assert ":foo | Objective activated." in caplog.text + assert ":min_cost_optimisation | Objective deactivated." in caplog.text + @staticmethod def _is_fixed(val): return val.fixed