Notebook_12_run_MOO.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d19b77a3",
   "metadata": {},
   "source": [
    "## Run multiobjective optimization of dam porftolios and process the optimization outputs\n",
    "### T. Janus\n",
    "### Created: 15/01/2024\n",
    "### Modified: 06/11/2024"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4ca8e2b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import ClassVar, Dict, List, Any, Tuple, Set, Tuple, Sequence\n",
    "from typing import TypeAlias, TypeVar, Generic\n",
    "import subprocess\n",
    "import pathlib\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import string\n",
    "from datetime import datetime\n",
    "from parse import parse\n",
    "import json\n",
    "import gc\n",
    "import pprint\n",
    "import re\n",
    "import ast\n",
    "import plotly.express as px\n",
    "import plotly.graph_objects as go\n",
    "import networkx as nx\n",
    "import pygmo as pg\n",
    "# Apply the deao sorting\n",
    "from deap import base, tools, creator\n",
    "# Define the fitness and individual classes\n",
    "from tqdm import tqdm\n",
    "import seaborn as sns # for Data visualization\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt # for Data visualization\n",
    "from IPython.display import display, HTML\n",
    "from jinja2 import Template\n",
    "from lib.notebook12 import (\n",
    "    reduce_mem_usage, read_id_ifc_map, set_remap, get_every_n_row, \n",
    "    SolutionFileParser, OutputVisualiser, ObjectiveCalculator, \n",
    "    map_ids, find_solution_by_dam_numbers, return_row_by_criterion\n",
    ")\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "762bec2d-0535-4091-9ef3-ce91f5436098",
   "metadata": {},
   "source": [
    "## Custom functions and classes\n",
    "### Functions\n",
    "* `reduce_mem_usage` - Used for reducing the memory occupied by a dataframe and reduces data type based on content and range of values.\n",
    "* `read_id_ifc_map` - Reads a dictionary that maps optimization dam IDs to IFC dam IDs.\n",
    "* `set_remap` - Maps values in a set to new values based on a provided mapping dictionary.\n",
    "* `get_every_n_row` - Gets every n-th row of a dataframe and returns a reduced dataframe.\n",
    "### Classes\n",
    "* `SolutionFileParser`\n",
    "* `OutputVisualiser`\n",
    "* `ObjectiveCalculator`"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "732d6319",
   "metadata": {},
   "source": [
    "### Define file names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "152a42f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# optimization solution files with optimizations from two scenarios:\n",
    "# * nobuilt - all dams are considered for optimization as if no dams that currently exist had been built\n",
    "# * built - current state with built dams in place\n",
    "#-------------------------------------------------------\n",
    "# 5 objective optimization with GHG emissions calculated using G-res methodology\n",
    "mya_nobuilt_5obj_filename = 'mya_5_obj_nobuilt.sol'\n",
    "mya_built_5obj_filename = 'mya_5_obj_built.sol'\n",
    "# 5 objective optimization with GHG emissions calculated using emission factors\n",
    "mya_nobuilt_5obj_soued_filename = 'mya_5_obj_nobuilt_soued.sol'\n",
    "mya_built_5obj_soued_filename = 'mya_5_obj_built_soued.sol'\n",
    "# 3 objective optimization with GHG emissions calculated using emission factors\n",
    "mya_nobuilt_3obj_soued_filename = 'mya_3_obj_nobuilt_soued.sol'\n",
    "mya_built_3obj_soued_filename = 'mya_3_obj_built_soued.sol'\n",
    "#-------------------------------------------------------\n",
    "# Paths to output files from the algorithm with expansion / compression\n",
    "sol_file_folder_5obj = pathlib.Path('moo_solver_CPAIOR/outputs/epsilon2_5obj')\n",
    "sol_file_folder_3obj = pathlib.Path('moo_solver_CPAIOR/outputs/epsilon2_3obj_soued')\n",
    "#-------------------------------------------------------\n",
    "mya_nobuilt_5obj_path_cpaior = sol_file_folder_5obj / pathlib.Path(mya_nobuilt_5obj_filename)\n",
    "mya_built_5obj_path_cpaior = sol_file_folder_5obj / pathlib.Path(mya_built_5obj_filename)\n",
    "mya_nobuilt_soued_5obj_path_cpaior = sol_file_folder_5obj / pathlib.Path(mya_nobuilt_5obj_soued_filename)\n",
    "mya_built_soued5obj_path_cpaior = sol_file_folder_5obj / pathlib.Path(mya_built_5obj_soued_filename)\n",
    "mya_nobuilt_soud_3obj_path_cpaior = sol_file_folder_3obj / pathlib.Path(mya_nobuilt_3obj_soued_filename)\n",
    "mya_built_soued_3obj_path_cpaior = sol_file_folder_3obj / pathlib.Path(mya_built_3obj_soued_filename)\n",
    "#-------------------------------------------------------\n",
    "# Load dataframe with dam ids and objective values for each dam\n",
    "dam_data_filename = pathlib.Path(\"outputs/moo/all_hp.csv\")\n",
    "# Load the mapping between ids used in the MOO algorithm and the IDs in the IFC database\n",
    "map_file_path = pathlib.Path('outputs/moo/id_to_ifc.json')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "facf955a-6e83-4ce1-8359-a335ff19d57d",
   "metadata": {},
   "source": [
    "### Read data about dams and dam identifiers (different for the water model and for the IFC database)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4a6adac8-6d6d-4a42-bf4f-5821af97ad64",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read file with all hydroelectric plants for optimization and their objective values\n",
    "dam_data: pd.DataFrame = pd.read_csv(dam_data_filename)\n",
    "# Find IFC ids of built dams\n",
    "built_dam_ifc_ids: Set[int] = set(\n",
    "    dam_data[dam_data['status'] == 'Existing']['ifc_id'].to_list())\n",
    "# Read the ID (water model identifiers) to IFC map\n",
    "id_ifc_map: Dict[int, int] = read_id_ifc_map(map_file_path)\n",
    "# List IFC IDs in a sorted order of dams included in the analysis\n",
    "ifc_ids: List[int] = sorted([id_ifc_map[_id+1] for _id, _ in enumerate(id_ifc_map) ])\n",
    "# Some repetition here, but left for now in fear of breaking the code\n",
    "with open(map_file_path, 'r') as file:\n",
    "    id_map = json.load(file)\n",
    "id_map: Dict[int, int] = {int(key): value for key, value in id_map.copy().items()} # Maps optim ids to ifc ids\n",
    "ifc_to_id_map: Dict[int, int] = {value: key for key, value in id_map.items()}\n",
    "# ----------------------------------------------------------------------------------------------------------------\n",
    "dam_df = pd.read_csv(dam_data_filename, index_col=0).set_index('ifc_id')\n",
    "built_dam_ids_ifc: Set[int] = set(dam_df[dam_df['status_int'] == 1].index.to_list())\n",
    "# Set of dams used for optimization\n",
    "built_dam_ids_opt: Set[int] = set_remap(built_dam_ids_ifc, ifc_to_id_map)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ee13c738-3ca1-480a-a81a-393bb3f86502",
   "metadata": {},
   "source": [
    "### Find the number of existing dams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e1bcaae1-a9a7-4328-b9c9-e202b562c540",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of existing dams: 29\n",
      "Existing dam ids: {129, 2, 4, 133, 6, 134, 135, 15, 17, 35, 38, 45, 47, 49, 62, 63, 65, 66, 86, 96, 98, 101, 105, 107, 236, 108, 118, 120, 127}\n"
     ]
    }
   ],
   "source": [
    "# Print a number of existing dams\n",
    "num_existing_dams = dam_df[dam_df['status'] == 'Existing']['name'].count()\n",
    "print(f\"Number of existing dams: {num_existing_dams}\")\n",
    "# Print a set of existing dam ids\n",
    "existing_dam_ids = set(dam_df[dam_df['status'] == 'Existing'].index.to_list())\n",
    "print(\"Existing dam ids:\", existing_dam_ids)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2f92f7a9",
   "metadata": {},
   "source": [
    "## Call the optimizer by calling external script using subprocess. Alternatively, run the script `.sh` files (Linux/MacOS X), or `.bat` files (Windows), directly from Terminal/Console.\n",
    "\n",
    "### RUNS optimization as a subprocess from within this notebook\n",
    "#### Switch the `rerun_CPAIOR` flag to True if you want to run the optimization yourself (may take 3hr+ to execute). \n",
    "#### Otherwise keep it as False and read pre-saved optimization results in .sol files."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "208f2dc7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# SET THIS TO TRUE TO RUN THE OPTIMIZATION\n",
    "rerun_CPAIOR: bool = False\n",
    "\n",
    "script_paths = [\n",
    "    'moo_solver_CPAIOR/run_myanmar_dam_selection.sh',\n",
    "    'moo_solver_CPAIOR/run_myanmar_dam_selection_soued.sh'\n",
    "]\n",
    "\n",
    "# Use subprocess to call and execute the shell script\n",
    "if rerun_CPAIOR:\n",
    "    # Specify the path to your shell script\n",
    "    for script_path in script_paths:\n",
    "        try:\n",
    "            subprocess.run(['bash', script_path], check=True)\n",
    "            print(\"Optimization runs successful.\")\n",
    "        except subprocess.CalledProcessError as e:\n",
    "            print(f\"Error executing script: {e}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c5a0f5fa-534e-4a33-8c9f-d1486e6fd1e4",
   "metadata": {},
   "source": [
    "## Parse optimization outputs - fetches the results from `.sol` text files and saves in `.csv` and `.json` formats\n",
    "### Processes the following optimization scenarios:\n",
    "1. 5-objective optimization with emissions calculated with ReEmission\n",
    "2. 5-objective optimization with emissions calculated using emission factors (Soued et al.)\n",
    "3. 3-objective optimization with emissions calculated using emission factors (Soued et al.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ae1ea7cd-a070-4c19-8092-2489596f692e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sets paths and arguments for processing optimization results from different optimization scenarios\n",
    "exec_options = {\n",
    "    '5obj' : {\n",
    "        'nobj': 5,\n",
    "        'row_fraction': 1/3,\n",
    "        'sol_path_nobuilt' : mya_nobuilt_5obj_path_cpaior,\n",
    "        'sol_path_built': mya_built_5obj_path_cpaior,\n",
    "        'output_json_file_nobuilt': sol_file_folder_5obj / pathlib.Path('mya_5_obj_nobuilt.json'),\n",
    "        'output_csv_file_nobuilt': sol_file_folder_5obj / pathlib.Path('mya_5_obj_nobuilt.csv'),\n",
    "        'output_json_file_built': sol_file_folder_5obj / pathlib.Path('mya_5_obj_built.json'),\n",
    "        'output_csv_file_built': sol_file_folder_5obj / pathlib.Path('mya_5_obj_built.csv'),\n",
    "        'merged_csv_file': sol_file_folder_5obj / pathlib.Path('merged_df_5obj.csv'),\n",
    "        'nondom_csv_file': sol_file_folder_5obj / pathlib.Path('em_int_nondom_df_5obj.csv')\n",
    "    },\n",
    "    '5obj_soued': {\n",
    "        'nobj': 5,\n",
    "        'row_fraction': 1/3,\n",
    "        'sol_path_nobuilt' :mya_nobuilt_soued_5obj_path_cpaior,\n",
    "        'sol_path_built': mya_built_soued5obj_path_cpaior,\n",
    "        'output_json_file_nobuilt': sol_file_folder_5obj / pathlib.Path('mya_5_obj_nobuilt_soued.json'),\n",
    "        'output_csv_file_nobuilt': sol_file_folder_5obj / pathlib.Path('mya_5_obj_nobuilt_soued.csv'),\n",
    "        'output_json_file_built': sol_file_folder_5obj / pathlib.Path('mya_5_obj_built_soued.json'),\n",
    "        'output_csv_file_built': sol_file_folder_5obj / pathlib.Path('mya_5_obj_built_soued.csv'),\n",
    "        'merged_csv_file': sol_file_folder_5obj / pathlib.Path('merged_df_5obj_soued.csv'),\n",
    "        'nondom_csv_file': sol_file_folder_5obj / pathlib.Path('em_int_nondom_df_5obj_soued.csv')\n",
    "    },\n",
    "    '3obj_soued' : {\n",
    "        'nobj': 3,\n",
    "        'row_fraction': 1,\n",
    "        'sol_path_nobuilt' : mya_nobuilt_soud_3obj_path_cpaior,\n",
    "        'sol_path_built': mya_built_soued_3obj_path_cpaior,\n",
    "        'output_json_file_nobuilt': sol_file_folder_3obj / pathlib.Path('mya_3_obj_nobuilt_soued.json'),\n",
    "        'output_csv_file_nobuilt': sol_file_folder_3obj / pathlib.Path('mya_3_obj_nobuilt_soued.csv'),\n",
    "        'output_json_file_built': sol_file_folder_3obj / pathlib.Path('mya_3_obj_built_soued.json'),\n",
    "        'output_csv_file_built': sol_file_folder_3obj / pathlib.Path('mya_3_obj_built_soued.csv'),\n",
    "        'merged_csv_file': sol_file_folder_3obj / pathlib.Path('merged_df_3obj_soued.csv'),\n",
    "        'nondom_csv_file': sol_file_folder_3obj / pathlib.Path('em_int_nondom_df_3obj_soued.csv')\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1587f0fe-2746-432f-a774-dae1b52bbdda",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Select option - needs to be done one-by-one as on smaller laptops, running all three options in a row \n",
    "# can lead to memory overflow followed by a kernel crash\n",
    "options_batch = ['3obj_soued', '5obj_soued', '5obj']\n",
    "filter_dataframe: bool = True # Remove some solution rows to reduce the amount of data for saving and visualising\n",
    "# Save data from .sol files to json and/or csv if boolean flags for each are set to True\n",
    "convert_to_json, convert_to_csv = False, False\n",
    "process_results: bool = True\n",
    "save_to_file: bool = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "5050bac8-54a4-40ed-9bdb-b1bdd3685535",
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_sol_and_reduce_mem_usage(\n",
    "        exec_options: Dict[str, Dict[str, str]], \n",
    "        option: str, \n",
    "        convert_to_json: bool, \n",
    "        convert_to_csv: bool) -> Tuple[pd.DataFrame, pd.DataFrame]:\n",
    "    \"\"\" \"\"\"\n",
    "    # Nobuilt scenario\n",
    "    sol_parser_nobuilt = SolutionFileParser(exec_options[option]['sol_path_nobuilt'])\n",
    "    sol_parser_nobuilt.parse()\n",
    "    if convert_to_json:\n",
    "        sol_parser_nobuilt.to_json(exec_options[option]['output_json_file_nobuilt'])\n",
    "    if convert_to_csv:\n",
    "        sol_parser_nobuilt.to_csv(exec_options[option]['output_csv_file_nobuilt'])\n",
    "    # Concetenate dataframes into `merged_df`\n",
    "    df_nobuilt = reduce_mem_usage(sol_parser_nobuilt.solutions_df)\n",
    "    # Remove the unwanted objects from memory\n",
    "    del sol_parser_nobuilt\n",
    "    # Built scenario\n",
    "    sol_parser_built = SolutionFileParser(exec_options[option]['sol_path_built'])\n",
    "    sol_parser_built.parse()\n",
    "    if convert_to_json:\n",
    "        sol_parser_built.to_json(exec_options[option]['output_json_file_built'])\n",
    "    if convert_to_csv:\n",
    "        sol_parser_built.to_csv(exec_options[option]['output_csv_file_built'])\n",
    "    # Concetenate dataframes into `merged_df`\n",
    "    df_built = reduce_mem_usage(sol_parser_built.solutions_df)\n",
    "    # Remove the unwanted objects from memory\n",
    "    del sol_parser_built\n",
    "    return df_built, df_nobuilt\n",
    "\n",
    "def filter_parsed_dataframes(\n",
    "        df_built: pd.DataFrame, \n",
    "        df_nobuilt: pd.DataFrame,\n",
    "        exec_options: Dict[str, Dict[str, str]], \n",
    "        option: str) -> Tuple[pd.DataFrame, pd.DataFrame]:\n",
    "    \"\"\" \"\"\"\n",
    "    # Filter the dataframes before merging in the next step\n",
    "    df_built = df_built.sort_values(by=['energy'], ascending=True)\n",
    "    df_nobuilt = df_nobuilt.sort_values(by=['energy'], ascending=True)\n",
    "    # Filter the dataframe if filtering is selected\n",
    "    df_built_filt = get_every_n_row(df_built, int(1/exec_options[option]['row_fraction']))\n",
    "    df_nobuilt_filt = get_every_n_row(df_nobuilt, int(1/exec_options[option]['row_fraction']))\n",
    "    return df_built_filt, df_nobuilt_filt\n",
    "\n",
    "def make_merged_df(\n",
    "        df_built_filt: pd.DataFrame, \n",
    "        df_nobuilt_filt: pd.DataFrame, \n",
    "        exec_options: Dict[str, Dict[str, str]], \n",
    "        option: str) -> pd.DataFrame:\n",
    "    \"\"\" \"\"\"\n",
    "    df_nobuilt_filt['Scenario'] = \"Not Built\"\n",
    "    df_built_filt['Scenario'] = \"Built\"\n",
    "    # Merge the built and nobuilt dataframes. They are concatenated by the row dimension in\n",
    "    # order \"Not Built\" -> \"Built\"\n",
    "    merged_df = pd.concat([df_built_filt, df_nobuilt_filt], ignore_index=True)\n",
    "    old_new_col_map = {\n",
    "        'energy': \"Mean annual HP, [MW]\",\n",
    "        'ghg': 'GHG emissions [tonne CO<sub>2,eq</sub>/year]', \n",
    "        'firm_energy': 'Firm HP, [MW]',\n",
    "        'loss_agri': 'Agricultural land loss, [km<sup>2</sup>]',\n",
    "        'loss_forest': 'Deforestation, [km<sup>2</sup>]',\n",
    "        'num_dams': 'No. of selected dams',\n",
    "        'dam_ids': 'Dam IDs',\n",
    "        'land_loss': 'Land loss, [km<sup>2</sup>]',\n",
    "        'ghg_intensity': 'GHG intensity [gCO<sub>2,eq</sub>/kWh]'}\n",
    "    merged_df.rename(columns=old_new_col_map, inplace=True)\n",
    "    merged_df['Dam IDs'] = merged_df['Dam IDs'].apply(set)\n",
    "    merged_df['Firm Power Ratio, [%]'] = \\\n",
    "        merged_df['Firm HP, [MW]'] / merged_df['Mean annual HP, [MW]'] * 100\n",
    "    merged_df['Scenario, [1/0]'] = merged_df['Scenario'].map({'Built': 1, 'Not Built': 0})\n",
    "    \n",
    "    if exec_options[option]['nobj'] == 5:\n",
    "        # Define bin edges for land loss\n",
    "        bins = [0, 300, 600, 1000, 1500, 2000]\n",
    "        # Define labels for the bins\n",
    "        labels = ['0-300 km2', '300-500 km2', '500-1000 km2', '1000-1500 km2', '1500-2000 km2']\n",
    "        merged_df[\"Loss of Land [km<sup>2</sup>]\"] = pd.cut(\n",
    "            merged_df['Land loss, [km<sup>2</sup>]'], bins=bins, labels=labels, right=False)\n",
    "    # Arrange by status and energy in ascending order\n",
    "    merged_df = merged_df.sort_values(by=['Scenario', 'Mean annual HP, [MW]'], ascending=True)\n",
    "    # Introduce new columns\n",
    "    merged_df['HP Production [GWh/year]'] = merged_df[\"Mean annual HP, [MW]\"] * 365.25 * 24 / 1_000\n",
    "    merged_df['Mean HP [GWh/d]'] = merged_df[\"Mean annual HP, [MW]\"] * 24 / 1_000\n",
    "    merged_df['Firm HP [GWh/d]'] = merged_df['Firm HP, [MW]'] * 24 / 1_000\n",
    "    \n",
    "    # Reduce size of some data in merged_df\n",
    "    merged_df['Scenario, [1/0]'] = merged_df['Scenario, [1/0]'].astype('uint8')\n",
    "    if exec_options[option]['nobj'] == 5:\n",
    "        merged_df[\"Loss of Land [km<sup>2</sup>]\"] = merged_df[\"Loss of Land [km<sup>2</sup>]\"].astype('category')\n",
    "    merged_df['No. of selected dams'] = merged_df['No. of selected dams'].astype('int8')\n",
    "    # Use an automated method\n",
    "    merged_df = reduce_mem_usage(merged_df)\n",
    "    return merged_df\n",
    "\n",
    "def nondom_sort_in_2_objectives(\n",
    "        merged_df: pd.DataFrame, \n",
    "        exec_options: Dict[str, Dict[str, str]], \n",
    "        option: str, method: str = \"pygmo\") -> Tuple[np.ndarray, np.ndarray, np.ndarray]:\n",
    "    \"\"\" \"\"\"\n",
    "    # Perform non-dominated sorting in 2D for two 5D nondominated fronts: for Built and NotBuilt scenarios\n",
    "    # Create a pareto dominant front for not built data\n",
    "    if exec_options[option]['nobj'] == 5:\n",
    "        col_names = [\n",
    "            \"Mean annual HP, [MW]\", 'GHG emissions [tonne CO<sub>2,eq</sub>/year]', \n",
    "            'GHG intensity [gCO<sub>2,eq</sub>/kWh]', 'Firm Power Ratio, [%]', \n",
    "            'Land loss, [km<sup>2</sup>]', 'Dam IDs']\n",
    "    if exec_options[option]['nobj'] == 3:\n",
    "        col_names = [\n",
    "            \"Mean annual HP, [MW]\", 'GHG emissions [tonne CO<sub>2,eq</sub>/year]', \n",
    "            'GHG intensity [gCO<sub>2,eq</sub>/kWh]', 'Firm Power Ratio, [%]', \n",
    "            'Dam IDs']\n",
    "    xy_pairs_built = merged_df\\\n",
    "        .loc[merged_df['Scenario'] =='Built', col_names]\n",
    "    xy_pairs_built[\"Mean annual HP, [MW]\"] = xy_pairs_built[\"Mean annual HP, [MW]\"] * -1\n",
    "    xy_pairs_built_list = xy_pairs_built.to_numpy().tolist()\n",
    "    \n",
    "    xy_pairs_nobuilt = merged_df\\\n",
    "        .loc[merged_df['Scenario'] =='Not Built', col_names]\n",
    "    xy_pairs_nobuilt[\"Mean annual HP, [MW]\"] = xy_pairs_nobuilt[\"Mean annual HP, [MW]\"] * -1\n",
    "    xy_pairs_nobuilt_list = xy_pairs_nobuilt.to_numpy().tolist()\n",
    "    \n",
    "    xy_pair_array_built_np = np.array(xy_pairs_built_list)\n",
    "    xy_pair_array_nobuilt_np = np.array(xy_pairs_nobuilt_list)\n",
    "\n",
    "    if method == \"deap\":\n",
    "        creator.create(\"FitnessMin\", base.Fitness, weights=(-1.0, -1.0))  # Minimize both objectives\n",
    "        creator.create(\"Individual\", list, fitness=creator.FitnessMin)\n",
    "        individuals_built = [creator.Individual(point) for point in xy_pair_array_built_np[:,:2]]\n",
    "        for i_built, ind_built in enumerate(individuals_built):\n",
    "            ind_built.fitness.values = ind_built  # Set the point as the fitness value\n",
    "            ind_built.index = i_built  # Store the original index\n",
    "        fronts_built = tools.sortNondominated(individuals_built, len(individuals_built), first_front_only=True)\n",
    "        # Extract indices of non-dominated front\n",
    "        non_dom_front_built = np.array([ind_built.index for ind_built in fronts_built[0]])\n",
    "    \n",
    "        individuals_nobuilt = [creator.Individual(point) for point in xy_pair_array_nobuilt_np[:,:2]]\n",
    "        for i_nobuilt, ind_nobuilt in enumerate(individuals_nobuilt):\n",
    "            ind_nobuilt.fitness.values = ind_nobuilt  # Set the point as the fitness value\n",
    "            ind_nobuilt.index = i_nobuilt  # Store the original index\n",
    "        fronts_nobuilt = tools.sortNondominated(individuals_nobuilt, len(individuals_nobuilt), first_front_only=True)\n",
    "        # Extract indices of non-dominated front\n",
    "        non_dom_front_nobuilt = np.array([ind_nobuilt.index for ind_nobuilt in fronts_nobuilt[0]])\n",
    "    if method == \"pygmo\":\n",
    "        # Find non-dominated fronts, i.e. indexes of nondominated points. List them in the order of\n",
    "        # increasing HP production\n",
    "        non_dom_front_built = pg.non_dominated_front_2d(points=xy_pair_array_built_np[:,:2])[::-1]\n",
    "        non_dom_front_nobuilt = pg.non_dominated_front_2d(points=xy_pair_array_nobuilt_np[:,:2])[::-1]\n",
    "        \n",
    "    # Convert back from negative to positive values\n",
    "    xy_pair_array_built_np[:,0] = xy_pair_array_built_np[:,0] * -1\n",
    "    xy_pair_array_nobuilt_np[:,0] = xy_pair_array_nobuilt_np[:,0] * -1\n",
    "    # Select nondominated points\n",
    "    xy_nondom_built_np = xy_pair_array_built_np[non_dom_front_built]\n",
    "    xy_nondom_nobuilt_np = xy_pair_array_nobuilt_np[non_dom_front_nobuilt]\n",
    "    xy_nondom_all = np.concatenate((xy_nondom_built_np, xy_nondom_nobuilt_np), axis=0)\n",
    "    return xy_nondom_built_np, xy_nondom_nobuilt_np, xy_nondom_all\n",
    "\n",
    "def convert_nondom_to_dfs(xy_nondom_built_np, xy_nondom_nobuilt_np, exec_options, option) -> pd.DataFrame:\n",
    "    \"\"\" \"\"\"\n",
    "    if exec_options[option]['nobj'] == 5:\n",
    "        col_names = [\n",
    "            \"Mean annual HP, [MW]\", 'GHG emissions [tonne CO<sub>2,eq</sub>/year]', \n",
    "            'GHG intensity [gCO<sub>2,eq</sub>/kWh]', 'Firm Power Ratio, [%]', \n",
    "            'Land loss, [km<sup>2</sup>]', 'Dam IDs']\n",
    "    if exec_options[option]['nobj'] == 3:\n",
    "        col_names = [\n",
    "            \"Mean annual HP, [MW]\", 'GHG emissions [tonne CO<sub>2,eq</sub>/year]', \n",
    "            'GHG intensity [gCO<sub>2,eq</sub>/kWh]', 'Firm Power Ratio, [%]', \n",
    "            'Dam IDs']\n",
    "    xy_nondom_built_df = pd.DataFrame(xy_nondom_built_np, columns=col_names)\n",
    "    xy_nondom_built_df['Scenario'] = \"Built\"\n",
    "    xy_nondom_nobuilt_df = pd.DataFrame(xy_nondom_nobuilt_np, columns=col_names)\n",
    "    xy_nondom_nobuilt_df['Scenario'] = \"Not Built\"\n",
    "    xy_nondom_df = pd.concat([xy_nondom_built_df, xy_nondom_nobuilt_df], ignore_index=True)\n",
    "    xy_nondom_df['HP Production [GWh/year]'] = xy_nondom_df[\"Mean annual HP, [MW]\"] * 365.25 * 24 / 1_000\n",
    "    if exec_options[option]['nobj'] == 5:\n",
    "        # Define bin edges for land loss\n",
    "        bins = [0, 300, 600, 1000, 1500, 2000]\n",
    "        # Define labels for the bins\n",
    "        labels = ['0-300 km2', '300-500 km2', '500-1000 km2', '1000-1500 km2', '1500-2000 km2']\n",
    "        xy_nondom_df[\"Loss of Land [km<sup>2</sup>]\"] = pd.cut(\n",
    "            xy_nondom_df['Land loss, [km<sup>2</sup>]'], bins=bins, labels=labels, right=False)\n",
    "    em_int_nondom_df = xy_nondom_df\n",
    "    return em_int_nondom_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "efcc80e7-401c-457f-b14d-64d05158a65d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Memory usage of dataframe is 115.01 MB\n",
      "Memory usage after optimization is: 105.42 MB\n",
      "Decreased by 8.3%\n",
      "Memory usage of dataframe is 105.42 MB\n",
      "Memory usage after optimization is: 105.42 MB\n",
      "Decreased by 0.0%\n",
      "Memory usage of dataframe is 78.27 MB\n",
      "Memory usage after optimization is: 71.75 MB\n",
      "Decreased by 8.3%\n",
      "Memory usage of dataframe is 71.75 MB\n",
      "Memory usage after optimization is: 71.75 MB\n",
      "Decreased by 0.0%\n",
      "Memory usage of dataframe is 97.53 MB\n",
      "Memory usage after optimization is: 92.17 MB\n",
      "Decreased by 5.5%\n",
      "Performing nondominated sorting\n",
      "Finished nondominated sorting\n",
      "Processed the option 5obj\n",
      "Total number of solutions : 2814823\n",
      "Scenario with built constructed dams 1139898 solutions\n",
      "Scenario with zero constructed dams 1674925 solutions\n"
     ]
    }
   ],
   "source": [
    "if process_results:\n",
    "    for option in options_batch:\n",
    "        # Parse and reduce size of solutions from .sol files\n",
    "        df_built, df_nobuilt = parse_sol_and_reduce_mem_usage(\n",
    "            exec_options, \n",
    "            option,\n",
    "            convert_to_json=convert_to_json, \n",
    "            convert_to_csv=convert_to_csv)\n",
    "        # Filter the results by removing some solutions\n",
    "        df_built_filt, df_nobuilt_filt = filter_parsed_dataframes(\n",
    "            df_built, \n",
    "            df_nobuilt,\n",
    "            exec_options,\n",
    "            option)\n",
    "        # Merge data from two optimization scenarios (with build status and without build statuses)\n",
    "        merged_df = make_merged_df(df_built_filt, df_nobuilt_filt, exec_options, option)\n",
    "        # Perform nondominated sorting\n",
    "        print(\"Performing nondominated sorting\")\n",
    "        xy_nondom_built_np, xy_nondom_nobuilt_np, xy_nondom_all = nondom_sort_in_2_objectives(merged_df, exec_options, option)\n",
    "        print(\"Finished nondominated sorting\")\n",
    "        em_int_nondom_df = convert_nondom_to_dfs(xy_nondom_built_np, xy_nondom_nobuilt_np, exec_options, option)\n",
    "        if save_to_file:\n",
    "            em_int_nondom_df.to_csv(exec_options[option]['nondom_csv_file'], index=False)\n",
    "            merged_df.to_csv(exec_options[option]['merged_csv_file'], index = False)\n",
    "        #Statistics\n",
    "        number_of_solutions = len(df_nobuilt) + len(df_built)\n",
    "        print(f\"Processed the option {option}\")\n",
    "        print(f\"Total number of solutions : {number_of_solutions}\")\n",
    "        print(f\"Scenario with built constructed dams {len(df_built)} solutions\")\n",
    "        print(f\"Scenario with zero constructed dams {len(df_nobuilt)} solutions\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "08f55964-6bcc-423a-aafb-d63d281263f7",
   "metadata": {},
   "source": [
    "## The End"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}