From 718cf22d3a761160ebc312df83f788579b8b9503 Mon Sep 17 00:00:00 2001 From: pesap Date: Fri, 23 Aug 2024 13:17:35 -0600 Subject: [PATCH 1/2] chore: Fixing dependencies for runners and fixing testing (#9) --- .../workflows/{python-app.yaml => CI.yaml} | 29 +- pyproject.toml | 32 +-- src/r2x/__version__.py | 2 +- src/r2x/parser/plexos.py | 2 + tests/data/2-bus_example.xml | 259 ++++++++++++++++++ tests/test_plexos_parser.py | 4 +- 6 files changed, 292 insertions(+), 36 deletions(-) rename .github/workflows/{python-app.yaml => CI.yaml} (79%) create mode 100755 tests/data/2-bus_example.xml diff --git a/.github/workflows/python-app.yaml b/.github/workflows/CI.yaml similarity index 79% rename from .github/workflows/python-app.yaml rename to .github/workflows/CI.yaml index bb8cb796..90ac1614 100644 --- a/.github/workflows/python-app.yaml +++ b/.github/workflows/CI.yaml @@ -5,9 +5,9 @@ on: branches: - main -permissions: - pull-requests: write - contents: write +env: + DEFAULT_PYTHON: "3.12" + DEFAULT_OS: ubuntu-latest jobs: pytest: @@ -15,6 +15,9 @@ jobs: matrix: python-version: ["3.11", "3.12"] os: [ubuntu-latest, windows-latest] + permissions: + pull-requests: write + contents: write runs-on: ${{ matrix.os }} steps: @@ -31,20 +34,16 @@ jobs: python -m pip install '.[dev]' - name: Running package tests run: | - python -m pytest -vvl + python -m pytest -vvl --cov --cov-report=xml - # - name: Coverage comment - # id: coverage_comment - # uses: py-cov-action/python-coverage-comment-action@v3 + # - name: codecov + # uses: codecov/codecov-action@v4.2.0 + # if: ${{ matrix.os == env.DEFAULT_OS && matrix.python-version == env.DEFAULT_PYTHON }} # with: - # GITHUB_TOKEN: ${{ github.token }} - # - # - name: Store Pull Request comment to be posted - # uses: actions/upload-artifact@v4 - # if: steps.coverage_comment.outputs.COMMENT_FILE_WRITTEN == 'true' - # with: - # name: python-coverage-comment-action - # path: python-coverage-comment-action.txt + # token: ${{ secrets.CODECOV_TOKEN }} + # name: r2x-test + # fail_ci_if_error: false + # verbose: true pre-commit: runs-on: ubuntu-latest diff --git a/pyproject.toml b/pyproject.toml index 00847375..91f63781 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,13 @@ [build-system] -requires = ["setuptools>=61.0.0", "wheel"] -build-backend = "setuptools.build_meta" +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.version] +path = "src/r2x/__version__.py" [project] name = "r2x" -version = "v1.0.0" +dynamic = ["version"] authors = [ { name = "Pedro Sanchez", email = "psanchez@nrel.gov" }, { name = "Obika Kodi", email = "kodi.obika@nrel.gov" }, @@ -13,7 +16,12 @@ authors = [ description = "ReEDS to X parser" requires-python = ">=3.11" classifiers = [ - "Programming Language :: Python :: 3", + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD License", + "Topic :: Software Development :: Build Tools", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Operating System :: OS Independent", ] dependencies = [ @@ -21,6 +29,7 @@ dependencies = [ "loguru~=0.7.2", "pandas~=2.2", "infrasys>=0.0.4", + "plexosdb>=0.0.2", "polars~=1.1.0", "pyyaml~=6.0.1", "rich~=13.7.1", @@ -65,21 +74,6 @@ ignore_missing_imports=true module = "polars" follow_imports = "skip" -# Setuptools configuration -[tool.setuptools] -include-package-data = true - -[tool.setuptools.packages.find] -where = ["src"] -include = ["*"] - -# Setuptools configuration -[tool.setuptools.dynamic] -readme = { file = ["README.md"], content-type = "text/markdown" } - -[tool.setuptools.package-data] -"*" = ["*.json", "*.csv", "*.xml", ".sql"] - [tool.ruff] line-length = 110 target-version = "py311" diff --git a/src/r2x/__version__.py b/src/r2x/__version__.py index e33950af..d7060612 100644 --- a/src/r2x/__version__.py +++ b/src/r2x/__version__.py @@ -1,3 +1,3 @@ # This file containts the version # noqa: D100 -__version__ = "v0.5.2" +__version__ = "v1.0.0rc0" __data_model_version__ = "0.0.1" diff --git a/src/r2x/parser/plexos.py b/src/r2x/parser/plexos.py index 18d9085e..c1d9b8be 100644 --- a/src/r2x/parser/plexos.py +++ b/src/r2x/parser/plexos.py @@ -823,6 +823,8 @@ def _construct_interfaces(self, default_model=TransmissionInterface): return def _select_model_name(self): + # TODO(pesap): Add fail mechanism + # https://github.com/NREL/R2X/issues/10 query = f""" select obj.name from t_object as obj diff --git a/tests/data/2-bus_example.xml b/tests/data/2-bus_example.xml new file mode 100755 index 00000000..c5501b15 --- /dev/null +++ b/tests/data/2-bus_example.xml @@ -0,0 +1,259 @@ + + + 1 + 1 + 0 + - + + + 2 + 2 + 0 + - + + + + 1 + System + 1 + true + 1 + The integrated energy system + + + 2 + Generator + 2 + true + 22 + Generating unit, or collection of like generating units + + + 4 + Fuel + 2 + true + 18 + Fuel for a thermal generating unit + + + 1 + 43 + 47 + 100 + 169864276 + + + 2 + 47 + 46 + 1 + 169864277 + + + 3 + 47 + 47 + 100 + 169864278 + + + 4 + 47 + 47 + 120 + 169864279 + + + 78 + Scenario + 11 + true + 35 + Data scenario + + + 43 + 1 + 1 + 1 + 2 + 20 + + + 44 + 1 + 1 + 261 + 22 + 21 + + + 45 + 1 + 1 + 261 + 22 + 22 + + + 46 + 1 + 1 + 196 + 19 + 23 + + + 47 + 1 + 1 + 1 + 2 + 24 + + + 52 + 2 + 20 + 12 + 22 + 21 + + + 53 + 1 + 1 + 39 + 4 + 25 + + + 54 + 2 + 24 + 7 + 4 + 25 + + + 55 + 2 + 24 + 12 + 22 + 22 + + + 56 + 22 + 21 + 264 + 19 + 23 + + + 57 + 22 + 22 + 264 + 19 + 23 + + + 58 + 1 + 1 + 707 + 80 + 26 + + + 59 + 1 + 1 + 700 + 78 + 27 + + + 60 + 80 + 26 + 708 + 78 + 27 + + + 1 + 1 + System + 1 + the system object + 98bd24fd-e92b-4738-92d4-3f03a8a09cc3 + + + 20 + 2 + SolarPV_01 + 2 + + 0b6116f3-28c3-4cb1-a3c9-bad59585db4e + + + 21 + 22 + node_01 + 22 + + b4652dbe-a597-44e5-a36e-b1c9d53e0b18 + + + 22 + 22 + node_02 + 22 + + f926e130-4719-4249-bf32-f0dded4a93bd + + + 23 + 19 + region_1 + 19 + + 5db56664-18a1-4e9e-a9f2-31fc816d8dda + + + 24 + 2 + ThermalCC_01 + 2 + + 6659cbb9-2851-483c-a5bc-1d03659f2ca2 + + + 25 + 4 + gas + 4 + + 71917e41-f45b-44ad-81db-d88efcc34aea + + + 26 + 80 + main_model + 80 + + baec2eea-16a4-4f3a-bd6f-c24664b0069e + + + 27 + 78 + MoreCapacity + 78 + + 331cc8ab-bff8-4416-91d2-748012abef75 + + diff --git a/tests/test_plexos_parser.py b/tests/test_plexos_parser.py index fd016805..b1aa7ae5 100644 --- a/tests/test_plexos_parser.py +++ b/tests/test_plexos_parser.py @@ -4,7 +4,8 @@ from r2x.parser.handler import get_parser_data from r2x.parser.plexos import PlexosParser -DB_NAME = "plexos_example.xml" +DB_NAME = "2-bus_example.xml" +MODEL_NAME = "main_model" @pytest.fixture @@ -14,6 +15,7 @@ def plexos_scenario(tmp_path, data_folder): input_model="plexos", run_folder=data_folder, output_folder=tmp_path, + model=MODEL_NAME, solve_year=2035, weather_year=2012, fmap={"xml_file": {"fname": DB_NAME, "model": "default"}}, From dee6edc71d10b2fe54fc07d1c9daeaad57d4ba4a Mon Sep 17 00:00:00 2001 From: pesap Date: Fri, 23 Aug 2024 14:32:24 -0600 Subject: [PATCH 2/2] fix: Cleaning configuration file for plexos and adding more testing (#11) --- src/r2x/defaults/config.json | 1 + src/r2x/defaults/plexos_input.json | 1 + src/r2x/exceptions.py | 4 +++ src/r2x/parser/plexos.py | 51 ++++++++++++++++++++++-------- src/r2x/utils.py | 1 + tests/test_plexos_parser.py | 27 ++++++++++++++++ 6 files changed, 71 insertions(+), 14 deletions(-) diff --git a/src/r2x/defaults/config.json b/src/r2x/defaults/config.json index 59c1fd88..4ce3e7b9 100644 --- a/src/r2x/defaults/config.json +++ b/src/r2x/defaults/config.json @@ -30,6 +30,7 @@ "Flexibility", "Regulation" ], + "device_inference_string": {}, "distribution_losses": 1, "generator_map": {}, "heatrate_fits_file": "heatrate_generic_fits.csv", diff --git a/src/r2x/defaults/plexos_input.json b/src/r2x/defaults/plexos_input.json index c7ed69d1..765508d6 100644 --- a/src/r2x/defaults/plexos_input.json +++ b/src/r2x/defaults/plexos_input.json @@ -1,5 +1,6 @@ { "plexos_device_map": {}, + "plexos_fuel_map": {}, "plexos_property_map": { "Charge Efficiency": "charge_efficiency", "Commit": "must_run", diff --git a/src/r2x/exceptions.py b/src/r2x/exceptions.py index 083e319f..ecdeaaca 100644 --- a/src/r2x/exceptions.py +++ b/src/r2x/exceptions.py @@ -24,3 +24,7 @@ class ModelError(Exception): class MultipleFilesError(Exception): pass + + +class ParserError(Exception): + pass diff --git a/src/r2x/parser/plexos.py b/src/r2x/parser/plexos.py index c1d9b8be..e3a1ba8c 100644 --- a/src/r2x/parser/plexos.py +++ b/src/r2x/parser/plexos.py @@ -19,7 +19,7 @@ from r2x.api import System from r2x.config import Scenario from r2x.enums import ACBusTypes, ReserveDirection, ReserveType, PrimeMoversType -from r2x.exceptions import ModelError +from r2x.exceptions import ModelError, ParserError from plexosdb import PlexosSQLite from plexosdb.enums import ClassEnum, CollectionEnum from r2x.model import ( @@ -126,12 +126,22 @@ class PlexosParser(PCMParser): def __init__(self, *args, xml_file: str | None = None, **kwargs) -> None: super().__init__(*args, **kwargs) assert self.config.run_folder - self.run_folder = Path(self.config.run_folder) self.system = System(name=self.config.name) self.property_map = self.config.defaults["plexos_property_map"] self.device_map = self.config.defaults["plexos_device_map"] - self.prime_mover_map = self.config.defaults["tech_fuel_pm_map"] + self.fuel_map = self.config.defaults["plexos_fuel_map"] + self.device_match_string = self.config.defaults["device_inference_string"] + + # TODO(pesap): Rename exceptions to include R2X + # https://github.com/NREL/R2X/issues/5 + # R2X needs at least one of this maps defined to correctly work. + if not self.fuel_map and not self.device_map and not self.device_match_string: + msg = ( + "Neither `plexos_fuel_map` or `plexos_device_map` or `device_match_string` was provided. " + "To fix, provide any of the mappings." + ) + raise ParserError(msg) # Populate databse from XML file. xml_file = xml_file or self.run_folder / self.config.fmap["xml_file"]["fname"] @@ -183,7 +193,6 @@ def _collect_horizon_data(self, model_name: str) -> dict[str, float]: def build_system(self) -> System: """Create infrasys system.""" logger.info("Building infrasys system using {}", self.__class__.__name__) - # self.append_to_db = self.config.defaults.get("append_to_existing_database", False) # If we decide to change the engine for handling the data we can do it here. object_data = self._plexos_table_data() @@ -478,10 +487,16 @@ def _construct_generators(self): generator_name = generator_name[0] generator_fuel_type = generator_fuel_map.get(generator_name) logger.trace("Parsing generator = {} with fuel type = {}", generator_name, generator_fuel_type) + + # Get prime mover map from fuel + generator_prime_mover = self.fuel_map.get(generator_fuel_type, "") + + # First we check if there is a mapping one the name, if not, we try to use the prime + # mover map for the fuel and if not we infer the model by the name and a string. model_map = ( - self.config.device_map.get(generator_name, "") - or self.config.fuel_map.get(generator_fuel_type, "") - or self._infer_model_type(generator_name) + self.device_map.get(generator_name, "") or generator_prime_mover["device"] + if generator_prime_mover + else None or self._infer_model_type(generator_name) ) if getattr(R2X_MODELS, model_map, None) is None: @@ -519,15 +534,15 @@ def _construct_generators(self): # Add prime mover mapping mapped_records["prime_mover_type"] = ( - self.prime_mover_map[generator_fuel_type].get("type") - if generator_fuel_type in self.prime_mover_map.keys() - else self.prime_mover_map["default"].get("type") + self.fuel_map[generator_fuel_type].get("type") + if generator_fuel_type in self.fuel_map.keys() + else self.fuel_map["default"].get("type") ) mapped_records["prime_mover_type"] = PrimeMoversType[mapped_records["prime_mover_type"]] mapped_records["fuel"] = ( - self.prime_mover_map[generator_fuel_type].get("fuel") - if generator_fuel_type in self.prime_mover_map.keys() - else self.prime_mover_map["default"].get("fuel") + self.fuel_map[generator_fuel_type].get("fuel") + if generator_fuel_type in self.fuel_map.keys() + else self.fuel_map["default"].get("fuel") ) match model_map: @@ -696,6 +711,10 @@ def _construct_batteries(self): def _add_buses_to_batteries(self): batteries = [battery["name"] for battery in self.system.to_records(GenericBattery)] + if not batteries: + msg = "No battery objects found on the system. Skipping adding membership to buses." + logger.warning(msg) + return generator_memberships = self.db.get_memberships( *batteries, object_class=ClassEnum.Battery, @@ -720,6 +739,10 @@ def _add_buses_to_batteries(self): def _add_battery_reserves(self): reserve_map = self.system.get_component(ReserveMap, name="contributing_generators") batteries = [battery["name"] for battery in self.system.to_records(GenericBattery)] + if not batteries: + msg = "No battery objects found on the system. Skipping adding membership to buses." + logger.warning(msg) + return generator_memberships = self.db.get_memberships( *batteries, object_class=ClassEnum.Battery, @@ -823,7 +846,7 @@ def _construct_interfaces(self, default_model=TransmissionInterface): return def _select_model_name(self): - # TODO(pesap): Add fail mechanism + # TODO(pesap): Handle exception if no model name found # https://github.com/NREL/R2X/issues/10 query = f""" select obj.name diff --git a/src/r2x/utils.py b/src/r2x/utils.py index 3b742bd4..8e9f2c45 100644 --- a/src/r2x/utils.py +++ b/src/r2x/utils.py @@ -147,6 +147,7 @@ def update_dict(base_dict: dict, override_dict: ChainMap | dict | None = None) - "model_map", "tech_fuel_pm_map", "device_map", + "plexos_fuel_map", ] for key, value in override_dict.items(): if key in base_dict and all(replace_key not in key for replace_key in _replace_keys): diff --git a/tests/test_plexos_parser.py b/tests/test_plexos_parser.py index b1aa7ae5..735133c1 100644 --- a/tests/test_plexos_parser.py +++ b/tests/test_plexos_parser.py @@ -1,6 +1,9 @@ +from plexosdb.sqlite import PlexosSQLite import pytest from plexosdb import XMLHandler +from r2x.api import System from r2x.config import Scenario +from r2x.exceptions import ParserError from r2x.parser.handler import get_parser_data from r2x.parser.plexos import PlexosParser @@ -24,6 +27,8 @@ def plexos_scenario(tmp_path, data_folder): @pytest.fixture def plexos_parser_instance(plexos_scenario): + plexos_device_map = {"SolarPV_01": "RenewableFix", "ThermalCC": "ThermalStandard"} + plexos_scenario.defaults["plexos_device_map"] = plexos_device_map return get_parser_data(plexos_scenario, parser_class=PlexosParser) @@ -31,3 +36,25 @@ def test_plexos_parser_instance(plexos_parser_instance): assert isinstance(plexos_parser_instance, PlexosParser) assert len(plexos_parser_instance.data) == 1 # Plexos parser just parses a single file assert isinstance(plexos_parser_instance.data["xml_file"], XMLHandler) + assert isinstance(plexos_parser_instance.db, PlexosSQLite) + + +@pytest.mark.skip +def test_build_system(plexos_parser_instance): + system = plexos_parser_instance.build_system() + assert isinstance(system, System) + + +def test_raise_if_no_map_provided(tmp_path, data_folder): + scenario = Scenario.from_kwargs( + name="plexos_test", + input_model="plexos", + run_folder=data_folder, + output_folder=tmp_path, + solve_year=2035, + model=MODEL_NAME, + weather_year=2012, + fmap={"xml_file": {"fname": DB_NAME}}, + ) + with pytest.raises(ParserError): + _ = get_parser_data(scenario, parser_class=PlexosParser)