Notebook_1_run_batch_simulation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "576aa5b8",
   "metadata": {},
   "source": [
    "# Run batch simulation with RE-EMISSION\n",
    "\n",
    "### Produces a range of GHG estimates for different combinations of parameters\n",
    "\n",
    "#### T.Janus 14/10/2023"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3ee8e678",
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import Any, Tuple, Dict\n",
    "from itertools import product\n",
    "import os\n",
    "import re\n",
    "try:\n",
    "    import reemission\n",
    "except ModuleNotFoundError:\n",
    "    !pip install reemission@git+https://github.com/tomjanus/reemission\n",
    "    import reemission\n",
    "import subprocess\n",
    "import json\n",
    "from reemission.input import Inputs\n",
    "from reemission.model import EmissionModel"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f22ea67b",
   "metadata": {},
   "source": [
    "### Define utility functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5739f5f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def access_nested_dict(nested_dict: Dict, keys: Tuple[Any, ...]) -> Dict | Any:\n",
    "    \"\"\"Returns a value (can be any type including sub (nested) directory) from a nested dictionary for\n",
    "    a number of nested keys provided in a typle `keys`\"\"\"\n",
    "    try:\n",
    "        for key in keys:\n",
    "            nested_dict = nested_dict[key]\n",
    "        return nested_dict\n",
    "    except (KeyError, TypeError) as e:\n",
    "        return None\n",
    "    \n",
    "\n",
    "def modify_dict_fields(data: Dict, field_name: Tuple[str, ...], field_value: Any) -> None:\n",
    "    \"\"\" \"\"\"\n",
    "    for key, key_data in data.items():\n",
    "        data_slice = access_nested_dict(key_data, field_name[:-1])\n",
    "        data_slice[field_name[-1]] = field_value\n",
    "\n",
    "        \n",
    "def load_json(json_file: str) -> Dict:\n",
    "    \"\"\" \"\"\"\n",
    "    with open(json_file, 'r') as file:\n",
    "        data = json.load(file)\n",
    "    return data  \n",
    "\n",
    "\n",
    "def save_to_json(data: Dict, file_name: str) -> None:\n",
    "    \"\"\" \"\"\"\n",
    "    with open(file_name, \"w\") as file:\n",
    "        json.dump(data, file, indent=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "012c10f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Config dictionary for running simulations with combinations (combinatorial) of different options\n",
    "batch_simu_config = (\n",
    "    {   \"field\": (\"catchment\", \"biogenic_factors\", \"soil_type\"),\n",
    "        \"values\": (\"mineral\", \"organic\"),\n",
    "        \"acronyms\": (\"MIN\", \"ORG\"),\n",
    "    },\n",
    "    {   \"field\": (\"catchment\", \"biogenic_factors\", \"landuse_intensity\"),\n",
    "        \"values\": (\"low intensity\", \"high intensity\"),\n",
    "        \"acronyms\": (\"LOW\", \"HIGH\")\n",
    "    },\n",
    "    {   \"field\": (\"catchment\", \"biogenic_factors\", \"treatment_factor\"),\n",
    "        \"values\": (\n",
    "            \"no treatment\", \"primary (mechanical)\", \"secondary biological treatment\", \n",
    "            \"tertiary\"),\n",
    "        \"acronyms\": (\"NO\", \"PRIM\", \"SEC\", \"TER\")\n",
    "    },\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9420f050",
   "metadata": {},
   "source": [
    "## 1. Create a set of input files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bdd5d25b",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_directory = os.path.join('inputs/reemission')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a609baad",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set combinations\n",
    "value_combination_combo = []\n",
    "acronym_combination_combo = []\n",
    "# Generate all combinations of values and acronyms\n",
    "value_combinations = product(*(item[\"values\"] for item in batch_simu_config))\n",
    "acronym_combinations = product(*(item[\"acronyms\"] for item in batch_simu_config))\n",
    "\n",
    "for combo in acronym_combinations:\n",
    "    acronym_combination_combo.append(combo)\n",
    "for combo in value_combinations:\n",
    "    value_combination_combo.append(combo)\n",
    "\n",
    "input_json_file = os.path.join(input_directory, \"reemission_inputs.json\")\n",
    "for value_combo, acronym_combo in zip(value_combination_combo, acronym_combination_combo):\n",
    "    file_suffix = \"_\".join(acronym_combo)\n",
    "    # Load a json file\n",
    "    input_data: Dict = load_json(input_json_file)\n",
    "    for ix, item in enumerate(batch_simu_config):\n",
    "        modify_dict_fields(input_data, item['field'], value_combo[ix])\n",
    "    # Create a new file name\n",
    "    base_name, extension = os.path.splitext(input_json_file)\n",
    "    new_file_name = f\"{base_name}_{file_suffix}{extension}\"\n",
    "    print(f\"Saving file: {new_file_name}\")\n",
    "    save_to_json(input_data, os.path.join(new_file_name))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4ac3e700",
   "metadata": {},
   "source": [
    "## 2. Discover input files in a folder and run simulation for each input file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a929d797",
   "metadata": {},
   "outputs": [],
   "source": [
    "file_pattern = r'^reemission_inputs_(.+)\\.json$'\n",
    "file_regex = re.compile(file_pattern)\n",
    "# List to store matching filenames\n",
    "matching_files = []\n",
    "suffixes = []\n",
    "# Iterate through the files in the directory\n",
    "for root, dirs, files in os.walk(input_directory):\n",
    "    for file in files:\n",
    "        match = file_regex.match(file)\n",
    "        if match:\n",
    "            suffix = match.group(1)\n",
    "            matching_files.append(os.path.join(root, file))\n",
    "            suffixes.append(suffix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9824295b",
   "metadata": {},
   "outputs": [],
   "source": [
    "outputs_folder = os.path.join(\"outputs\", \"reemission\")\n",
    "for ix, (file, suffix) in enumerate(zip(matching_files, suffixes)):\n",
    "    xls_output_file = \"outputs_\"+suffix+\".xlsx\"\n",
    "    json_output_file = \"outputs_\"+suffix+\".json\"\n",
    "    print(f\"Running simulation no. {ix+1}\")\n",
    "    cmd = [\n",
    "        \"reemission\", \"calculate\", file, \"-o\", os.path.join(outputs_folder, xls_output_file), \n",
    "        \"-o\", os.path.join(outputs_folder, json_output_file)]\n",
    "    result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}