Experiments_yearly_evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a667537d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from evaluate import *\n",
    "from interpolate import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "621aba20",
   "metadata": {},
   "outputs": [],
   "source": [
    "base_folder = \"Output/Reproduce_new\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7ab36ccf",
   "metadata": {},
   "source": [
    "# Evaluating the results of the experiments with yearly data\n",
    "\n",
    "In this notebook no plots are created, this is done in a separate notebook."
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "123c8a1e",
   "metadata": {},
   "source": [
    "### Testing the effect of modifications to flat UNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1e2c2f75",
   "metadata": {},
   "outputs": [],
   "source": [
    "keywords_blacklist = [\"RESULTS_INTERPOLATED\"]  #  this is used to exclude results that were interpolated from loading if needed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5258775b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10 matching runs found\n",
      "Loss: Masked_MSELoss, CoordConv: True, CylindricalPadding: True, R2 = 0.375 +/- 0.005\n",
      "10 matching runs found\n",
      "Loss: Masked_MSELoss, CoordConv: True, CylindricalPadding: False, R2 = 0.367 +/- 0.010\n",
      "10 matching runs found\n",
      "Loss: Masked_MSELoss, CoordConv: False, CylindricalPadding: True, R2 = 0.357 +/- 0.015\n",
      "10 matching runs found\n",
      "Loss: Masked_MSELoss, CoordConv: False, CylindricalPadding: False, R2 = 0.352 +/- 0.015\n",
      "10 matching runs found\n",
      "Loss: Masked_AreaWeightedMSELoss, CoordConv: True, CylindricalPadding: True, R2 = 0.377 +/- 0.005\n",
      "10 matching runs found\n",
      "Loss: Masked_AreaWeightedMSELoss, CoordConv: True, CylindricalPadding: False, R2 = 0.376 +/- 0.005\n",
      "10 matching runs found\n",
      "Loss: Masked_AreaWeightedMSELoss, CoordConv: False, CylindricalPadding: True, R2 = 0.366 +/- 0.004\n",
      "10 matching runs found\n",
      "Loss: Masked_AreaWeightedMSELoss, CoordConv: False, CylindricalPadding: False, R2 = 0.365 +/- 0.005\n"
     ]
    }
   ],
   "source": [
    "for loss in [\"Masked_MSELoss\", \"Masked_AreaWeightedMSELoss\"]:\n",
    "    for use_coord_conv in [True, False]:\n",
    "        for use_cylindrical_padding in [True, False]:\n",
    "            # results can be loaded by specifying a configuration of data set, model and training.\n",
    "            conditions = {\n",
    "                \"DATASET_DESCRIPTION\": {\"PREDICTOR_VARIABLES\" : {\"tsurf\": [\"tsurf\"],\n",
    "                                                                 \"prec\": [\"prec\"]},\n",
    "                                        \"GRID_TYPE\": \"Flat\",\n",
    "                                        \"PRECIP_WEIGHTING\" : False\n",
    "                                       },\n",
    "                \"MODEL_TRAINING_DESCRIPTION\": {\"MODEL_TYPE\": \"UNet_Flat\",\n",
    "                                              \"USE_COORD_CONV\": use_coord_conv,\n",
    "                                              \"USE_CYLINDRICAL_PADDING\": use_cylindrical_padding,\n",
    "                                              \"LOSS\": loss,\n",
    "                                              \"LEARNING_RATE\": 1e-3,\n",
    "                                              \"DEPTH\": 3,\n",
    "                                               \"FMAPS\": (32,32,64,64)\n",
    "                                              }\n",
    "            }\n",
    "            descriptions, predictions, gt, masks = load_data_for_comparison(base_folder, conditions, keywords_blacklist=keywords_blacklist)\n",
    "            r2 = np.zeros(len(predictions))\n",
    "            for i in range(len(predictions)):\n",
    "                r2[i] = get_weighted_average(get_r2(predictions[i], gt[i]), descriptions[i][\"DATASET_DESCRIPTION\"])\n",
    "            print('Loss: {}, CoordConv: {}, CylindricalPadding: {}, R2 = {:.3f} +/- {:.3f}'.format(loss, use_coord_conv, use_cylindrical_padding, np.mean(r2), np.std(r2)))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "a0d46c13",
   "metadata": {},
   "source": [
    "### Comparing different ML methods:"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7c16dac2",
   "metadata": {},
   "source": [
    "Flat grid:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "fedc591a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# configurations of the runs we want to load:\n",
    "\n",
    "linreg_baseline_c = {\n",
    "    \"DATASET_DESCRIPTION\": {\"GRID_TYPE\": \"Flat\",\n",
    "                            \"PREDICTOR_VARIABLES\" : {\"tsurf\": [\"tsurf\"],\n",
    "                                                     \"prec\": [\"prec\"]},\n",
    "                            \"DATASETS_NO_GAPS\": ['isotopes', 'tsurf', 'prec']},\n",
    "    \"MODEL_TRAINING_DESCRIPTION\": {\"MODEL_TYPE\": \"LinReg_Pixelwise\"}\n",
    "}\n",
    "\n",
    "randomforest_baseline_c = {\n",
    "    \"DATASET_DESCRIPTION\": {\"GRID_TYPE\": \"Flat\",\n",
    "                            \"PREDICTOR_VARIABLES\" : {\"tsurf\": [\"tsurf\"],\n",
    "                                                     \"prec\": [\"prec\"]},\n",
    "                            \"DATASETS_NO_GAPS\": ['isotopes', 'tsurf', 'prec']},\n",
    "                            \n",
    "    \"MODEL_TRAINING_DESCRIPTION\": {\"MODEL_TYPE\": \"RandomForest_Pixelwise\"}\n",
    "}\n",
    "\n",
    "pca_baseline_c = {\n",
    "    \"DATASET_DESCRIPTION\": {\"GRID_TYPE\": \"Flat\",\n",
    "                            \"PREDICTOR_VARIABLES\" : {\"tsurf\": [\"tsurf\"],\n",
    "                                                     \"prec\": [\"prec\"]},\n",
    "                            \"DATASETS_NO_GAPS\": ['isotopes', 'tsurf', 'prec']},\n",
    "    \"MODEL_TRAINING_DESCRIPTION\": {\"MODEL_TYPE\": \"PCA_Flat\",\n",
    "                                  \"REGTYPE\": \"linreg\"}\n",
    "}\n",
    "\n",
    "unet_unmodified_c = {\n",
    "    \"DATASET_DESCRIPTION\": {\"PREDICTOR_VARIABLES\" : {\"tsurf\": [\"tsurf\"],\n",
    "                                                     \"prec\": [\"prec\"]},\n",
    "                            \"DATASETS_NO_GAPS\": ['isotopes', 'tsurf', 'prec'],\n",
    "                            \"GRID_TYPE\": \"Flat\",\n",
    "                            \"PRECIP_WEIGHTING\" : False},\n",
    "    \"MODEL_TRAINING_DESCRIPTION\": {\"MODEL_TYPE\": \"UNet_Flat\",\n",
    "                                  \"USE_COORD_CONV\": False,\n",
    "                                  \"USE_CYLINDRICAL_PADDING\": False,\n",
    "                                  \"LEARNING_RATE\": 1e-3,                                   \n",
    "                                  \"LOSS\": \"Masked_MSELoss\"}\n",
    "}\n",
    "\n",
    "unet_modified_c = {\n",
    "    \"DATASET_DESCRIPTION\": {\"PREDICTOR_VARIABLES\" : {\"tsurf\": [\"tsurf\"],\n",
    "                                                     \"prec\": [\"prec\"]},\n",
    "                            \"DATASETS_NO_GAPS\": ['isotopes', 'tsurf', 'prec'],\n",
    "                            \"PRECIP_WEIGHTING\" : False,\n",
    "                            \"GRID_TYPE\": \"Flat\"\n",
    "                                       },\n",
    "    \"MODEL_TRAINING_DESCRIPTION\": {\"MODEL_TYPE\": \"UNet_Flat\",\n",
    "                                  \"USE_COORD_CONV\": True,\n",
    "                                  \"USE_CYLINDRICAL_PADDING\": True,\n",
    "                                  \"LEARNING_RATE\": 1e-3,\n",
    "                                  \"LOSS\": \"Masked_AreaWeightedMSELoss\",\n",
    "                                  \"DEPTH\": 3,\n",
    "                                  \"FMAPS\": (32,32,64,64)}\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "362cb82e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10 matching runs found\n",
      "10 matching runs found\n",
      "1 matching runs found\n",
      "1 matching runs found\n",
      "1 matching runs found\n",
      "Unmodified UNet: $R2 = 0.352 \\pm 0.015$\n",
      "Modified UNet: $R2 = 0.377 \\pm 0.005$\n",
      "Random Forest: $R2 = 0.212 \\pm 0.000$\n",
      "Pixelwise Linear regression: $R2 = 0.251 \\pm 0.000$\n",
      "PCA Regression: $R2 = 0.303 \\pm 0.000$\n"
     ]
    }
   ],
   "source": [
    "descriptions_u_u, predictions_u_u, gt_u_u, masks_u_u = load_data_for_comparison(base_folder, unet_unmodified_c, keywords_blacklist=keywords_blacklist)\n",
    "descriptions_m_u, predictions_m_u, gt_m_u, masks_m_u = load_data_for_comparison(base_folder, unet_modified_c, keywords_blacklist=keywords_blacklist)\n",
    "descriptions_rf, predictions_rf, gt_rf, masks_rf = load_data_for_comparison(base_folder, randomforest_baseline_c, keywords_blacklist=keywords_blacklist)\n",
    "\n",
    "descriptions_lr, predictions_lr, gt_lr, masks_lr = load_data_for_comparison(base_folder, linreg_baseline_c, keywords_blacklist=keywords_blacklist)\n",
    "descriptions_pca, predictions_pca, gt_pca, masks_pca = load_data_for_comparison(base_folder, pca_baseline_c, keywords_blacklist=keywords_blacklist)\n",
    "\n",
    "\n",
    "r2_u_u = np.zeros(len(predictions_u_u))\n",
    "r2_m_u = np.zeros(len(predictions_m_u))\n",
    "r2_rf = np.zeros(len(predictions_rf))\n",
    "\n",
    "r2_lr = np.zeros(len(predictions_lr))\n",
    "r2_pca = np.zeros(len(predictions_pca))\n",
    "\n",
    "\n",
    "for i in range(len(predictions_u_u)):\n",
    "    r2_u_u[i] = get_weighted_average(get_r2(predictions_u_u[i], gt_u_u[i]), descriptions_u_u[i][\"DATASET_DESCRIPTION\"])\n",
    "\n",
    "for i in range(len(predictions_m_u)):\n",
    "    r2_m_u[i] = get_weighted_average(get_r2(predictions_m_u[i], gt_m_u[i]), descriptions_m_u[i][\"DATASET_DESCRIPTION\"])\n",
    "    \n",
    "for i in range(len(predictions_rf)):\n",
    "    r2_rf[i] = get_weighted_average(get_r2(predictions_rf[i], gt_rf[i]), descriptions_rf[i][\"DATASET_DESCRIPTION\"])\n",
    "    \n",
    "for i in range(len(predictions_lr)):\n",
    "    r2_lr[i] = get_weighted_average(get_r2(predictions_lr[i], gt_lr[i]), descriptions_lr[i][\"DATASET_DESCRIPTION\"])\n",
    "    \n",
    "for i in range(len(predictions_pca)):\n",
    "    r2_pca[i] = get_weighted_average(get_r2(predictions_pca[i], gt_pca[i]), descriptions_pca[i][\"DATASET_DESCRIPTION\"])\n",
    "\n",
    "\n",
    "print(r\"Unmodified UNet: $R2 = {:.3f} \\pm {:.3f}$\".format(np.mean(r2_u_u), np.std(r2_u_u)))\n",
    "print(r\"Modified UNet: $R2 = {:.3f} \\pm {:.3f}$\".format(np.mean(r2_m_u), np.std(r2_m_u)))\n",
    "print(r\"Random Forest: $R2 = {:.3f} \\pm {:.3f}$\".format(np.mean(r2_rf), np.std(r2_rf)))\n",
    "\n",
    "print(r\"Pixelwise Linear regression: $R2 = {:.3f} \\pm {:.3f}$\".format(np.mean(r2_lr), np.std(r2_lr)))\n",
    "print(r\"PCA Regression: $R2 = {:.3f} \\pm {:.3f}$\".format(np.mean(r2_pca), np.std(r2_pca)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e3963426",
   "metadata": {},
   "source": [
    "icosahedral grid:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "5cd5cbfe",
   "metadata": {},
   "outputs": [],
   "source": [
    "# configurations of the ml models on the icosahedral grid, whose results we want to load:\n",
    "\n",
    "ico_baseline_c = {\n",
    "    \"DATASET_DESCRIPTION\": {\"GRID_TYPE\": \"Ico\",},\n",
    "    \"MODEL_TRAINING_DESCRIPTION\": {\"MODEL_TYPE\": \"PCA_Ico\",\n",
    "                                  \"REGTYPE\": \"linreg\"}\n",
    "}\n",
    "\n",
    "ico_unet_c = {\n",
    "    \"DATASET_DESCRIPTION\": {\"GRID_TYPE\": \"Ico\",\n",
    "                            \"PREDICTOR_VARIABLES\" : {\"tsurf\": [\"tsurf\"],\n",
    "                                                                 \"prec\": [\"prec\"]},\n",
    "                            \"PRECIP_WEIGHTING\" : False\n",
    "                                       },\n",
    "    \"MODEL_TRAINING_DESCRIPTION\": {\"MODEL_TYPE\": \"UNet_Ico\"}\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "0c21e2b5",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10 matching runs found\n",
      "1 matching runs found\n",
      "Ico Unet: R2 = 0.396 +/- 0.009\n",
      "Ico Baseline: R2 = 0.339 +/- 0.000\n"
     ]
    }
   ],
   "source": [
    "descriptions_ico_unet, predictions_ico_unet, gt_ico_unet, masks_ico_unet = load_data_for_comparison(base_folder, ico_unet_c)\n",
    "descriptions_ico_bl, predictions_ico_bl, gt_ico_bl, masks_ico_bl = load_data_for_comparison(base_folder, ico_baseline_c)\n",
    "\n",
    "r2_ico_unet = np.zeros(len(predictions_ico_unet))\n",
    "r2_ico_bl = np.zeros(len(predictions_ico_bl))\n",
    "\n",
    "for i in range(len(predictions_ico_unet)):\n",
    "    r2_ico_unet[i] = np.mean(get_r2(predictions_ico_unet[i], gt_ico_unet[i]), axis=(1,2))\n",
    "\n",
    "for i in range(len(predictions_ico_bl)):\n",
    "    r2_ico_bl[i] = np.mean(get_r2(predictions_ico_bl[i], gt_ico_bl[i]), axis=(1,2))\n",
    "    \n",
    "print('Ico Unet: R2 = {:.3f} +/- {:.3f}'.format(np.mean(r2_ico_unet), np.std(r2_ico_unet)))\n",
    "print('Ico Baseline: R2 = {:.3f} +/- {:.3f}'.format(np.mean(r2_ico_bl), np.std(r2_ico_bl)))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "8a093163",
   "metadata": {},
   "source": [
    "To be able to compare methods from flat and icosahedral grids, we need to interpolate the results between grids. This is done in the following cells."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "008ab95d",
   "metadata": {},
   "source": [
    "Flat interpolated to icosahedral grid:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "a676682c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "1 matching runs found\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "1 matching runs found\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "1 matching runs found\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "1 matching runs found\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "1 matching runs found\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "1 matching runs found\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "1 matching runs found\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "1 matching runs found\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "1 matching runs found\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "1 matching runs found\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "Output/Reproduce_new\\f58b0b708787c6906882857c7c3ada11d2785e8c47b247b8c98f74dfcb6d\\predictions.gz\n",
      "1 matching runs found\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "Output/Reproduce_new\\f58b0b708787c6906882857c7c3ada1708a5d6620bde0a91abd5bd0d4167\\predictions.gz\n",
      "1 matching runs found\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "Output/Reproduce_new\\f58b0b708787c6906882857c7c3ada1b331d90fb3bb8f686e60fb4694f84\\predictions.gz\n",
      "1 matching runs found\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "Output/Reproduce_new\\f58b0b708787c6906882857c7c3ada4c0f235c5b8790926dfac2dee1fcd4\\predictions.gz\n",
      "1 matching runs found\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "Output/Reproduce_new\\f58b0b708787c6906882857c7c3ada576a18f36590d3531ffca37b6de968\\predictions.gz\n",
      "1 matching runs found\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "Output/Reproduce_new\\f58b0b708787c6906882857c7c3ada7463a0580a1b8f672aa4e06304d509\\predictions.gz\n",
      "1 matching runs found\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "Output/Reproduce_new\\f58b0b708787c6906882857c7c3ada7f04ab0809263865da4a5cdc237fa9\\predictions.gz\n",
      "1 matching runs found\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "Output/Reproduce_new\\f58b0b708787c6906882857c7c3ada96d9124892228444d980843459fe8d\\predictions.gz\n",
      "1 matching runs found\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "Output/Reproduce_new\\f58b0b708787c6906882857c7c3adaaac5dd7ec5619ea3e514d8867fb915\\predictions.gz\n",
      "1 matching runs found\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "Output/Reproduce_new\\f58b0b708787c6906882857c7c3adac43d21071365fbd967ac53ca58c39a\\predictions.gz\n",
      "1 matching runs found\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "Output/Reproduce_new\\f58b0b708787c6906882857c7c3ada716c63e8186d96b82c72abbafaf658\\predictions.gz\n",
      "1 matching runs found\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "Output/Reproduce_new\\f58b0b708787c6906882857c7c3adaa5c975cb0fb8c5b6aad7388577b31e\\predictions.gz\n",
      "1 matching runs found\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "Output/Reproduce_new\\f58b0b708787c6906882857c7c3adab1797cbd05550fc1e8d0c7907fa9cd\\predictions.gz\n",
      "1 matching runs found\n"
     ]
    }
   ],
   "source": [
    "data_u_u = []\n",
    "gt_u_u = []\n",
    "\n",
    "for i in range(len(descriptions_u_u)):\n",
    "    data, gt = get_interpolated_data_and_gt(descriptions_u_u[i], predictions_u_u[i], output_folder=base_folder, \n",
    "                                            script_folder=\"Scripts/\", do_scaling=False)\n",
    "    data_u_u.append(data)\n",
    "    gt_u_u.append(gt)\n",
    "\n",
    "data_m_u = []\n",
    "gt_m_u = []\n",
    "\n",
    "for i in range(len(descriptions_m_u)):\n",
    "    data, gt = get_interpolated_data_and_gt(descriptions_m_u[i], predictions_m_u[i], output_folder=base_folder, \n",
    "                                            script_folder=\"Scripts/\", print_folder_names=True, do_scaling=False)\n",
    "    data_m_u.append(data)\n",
    "    gt_m_u.append(gt)\n",
    "    \n",
    "data_rf = []\n",
    "gt_rf = []\n",
    "\n",
    "for i in range(len(descriptions_rf)):\n",
    "    data, gt = get_interpolated_data_and_gt(descriptions_rf[i], predictions_rf[i], output_folder=base_folder, \n",
    "                                            script_folder=\"Scripts/\", print_folder_names=True, do_scaling=False)\n",
    "    data_rf.append(data)\n",
    "    gt_rf.append(gt)\n",
    "    \n",
    "data_lr = []\n",
    "gt_lr = []\n",
    "\n",
    "for i in range(len(descriptions_lr)):\n",
    "    data, gt = get_interpolated_data_and_gt(descriptions_lr[i], predictions_lr[i], output_folder=base_folder, \n",
    "                                            script_folder=\"Scripts/\", print_folder_names=True, do_scaling=False)\n",
    "    data_lr.append(data)\n",
    "    gt_lr.append(gt)\n",
    "    \n",
    "data_pca = []\n",
    "gt_pca = []\n",
    "\n",
    "for i in range(len(descriptions_pca)):\n",
    "    data, gt = get_interpolated_data_and_gt(descriptions_pca[i], predictions_pca[i], output_folder=base_folder, \n",
    "                                            script_folder=\"Scripts/\", print_folder_names=True, do_scaling=False)\n",
    "    data_pca.append(data)\n",
    "    gt_pca.append(gt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d600ecb7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unmodified UNet: $R2 = 0.374 \\pm 0.017$\n",
      "Modified UNet: $R2 = 0.402 \\pm 0.006$\n",
      "Random Forest: $R2 = 0.256 \\pm 0.000$\n",
      "Pixelwise Linear regression: $R2 = 0.274 \\pm 0.000$\n",
      "PCA Regression: $R2 = 0.332 \\pm 0.000$\n"
     ]
    }
   ],
   "source": [
    "r2_u_u = np.zeros(len(data_u_u))\n",
    "r2_m_u = np.zeros(len(data_m_u))\n",
    "r2_rf = np.zeros(len(data_rf))\n",
    "\n",
    "r2_lr = np.zeros(len(data_lr))\n",
    "r2_pca = np.zeros(len(data_pca))\n",
    "\n",
    "\n",
    "for i in range(len(predictions_u_u)):\n",
    "    r2_u_u[i] = np.mean(get_r2(data_u_u[i], gt_u_u[i]), axis=(1,2))\n",
    "\n",
    "for i in range(len(predictions_m_u)):\n",
    "    r2_m_u[i] = np.mean(get_r2(data_m_u[i], gt_m_u[i]), axis=(1,2))\n",
    "    \n",
    "for i in range(len(predictions_rf)):\n",
    "    r2_rf[i] = np.mean(get_r2(data_rf[i], gt_rf[i]), axis=(1,2))\n",
    "    \n",
    "for i in range(len(predictions_lr)):\n",
    "    r2_lr[i] = np.mean(get_r2(data_lr[i], gt_lr[i]), axis=(1,2))\n",
    "    \n",
    "for i in range(len(predictions_pca)):\n",
    "    r2_pca[i] = np.mean(get_r2(data_pca[i], gt_pca[i]), axis=(1,2))\n",
    "\n",
    "\n",
    "print(r\"Unmodified UNet: $R2 = {:.3f} \\pm {:.3f}$\".format(np.mean(r2_u_u), np.std(r2_u_u)))\n",
    "print(r\"Modified UNet: $R2 = {:.3f} \\pm {:.3f}$\".format(np.mean(r2_m_u), np.std(r2_m_u)))\n",
    "print(r\"Random Forest: $R2 = {:.3f} \\pm {:.3f}$\".format(np.mean(r2_rf), np.std(r2_rf)))\n",
    "\n",
    "print(r\"Pixelwise Linear regression: $R2 = {:.3f} \\pm {:.3f}$\".format(np.mean(r2_lr), np.std(r2_lr)))\n",
    "print(r\"PCA Regression: $R2 = {:.3f} \\pm {:.3f}$\".format(np.mean(r2_pca), np.std(r2_pca)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5c26ae1d",
   "metadata": {},
   "source": [
    "Icosahedral interpolated to flat grid:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "42395dce",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "1 matching runs found\n",
      "When interpolating to model grid, currently the iHadCM3 specifics are used.\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "1 matching runs found\n",
      "When interpolating to model grid, currently the iHadCM3 specifics are used.\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "1 matching runs found\n",
      "When interpolating to model grid, currently the iHadCM3 specifics are used.\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "1 matching runs found\n",
      "When interpolating to model grid, currently the iHadCM3 specifics are used.\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "1 matching runs found\n",
      "When interpolating to model grid, currently the iHadCM3 specifics are used.\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "1 matching runs found\n",
      "When interpolating to model grid, currently the iHadCM3 specifics are used.\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "1 matching runs found\n",
      "When interpolating to model grid, currently the iHadCM3 specifics are used.\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "1 matching runs found\n",
      "When interpolating to model grid, currently the iHadCM3 specifics are used.\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "1 matching runs found\n",
      "When interpolating to model grid, currently the iHadCM3 specifics are used.\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "1 matching runs found\n",
      "When interpolating to model grid, currently the iHadCM3 specifics are used.\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "1 matching runs found\n",
      "When interpolating to model grid, currently the iHadCM3 specifics are used.\n"
     ]
    }
   ],
   "source": [
    "data_ico_bl = []\n",
    "gt_ico_bl = []\n",
    "\n",
    "for i in range(len(descriptions_ico_bl)):\n",
    "    data, gt = get_interpolated_data_and_gt(descriptions_ico_bl[i], predictions_ico_bl[i], output_folder=base_folder, \n",
    "                                            script_folder=\"Scripts/\", do_scaling=False)\n",
    "    data_ico_bl.append(data)\n",
    "    gt_ico_bl.append(gt)\n",
    "\n",
    "data_ico_unet = []\n",
    "gt_ico_unet = []\n",
    "\n",
    "for i in range(len(descriptions_m_u)):\n",
    "    data, gt = get_interpolated_data_and_gt(descriptions_ico_unet[i], predictions_ico_unet[i], output_folder=base_folder, \n",
    "                                            script_folder=\"Scripts/\", do_scaling=False)\n",
    "    data_ico_unet.append(data)\n",
    "    gt_ico_unet.append(gt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "6ce766be",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ico baseline, flat grid: $R2 = 0.076 \\pm 0.000$\n",
      "Ico UNet, flat grid: $R2 = 0.126 \\pm 0.011$\n"
     ]
    }
   ],
   "source": [
    "r2_ico_bl = np.zeros(len(data_ico_bl))\n",
    "r2_ico_unet = np.zeros(len(data_ico_unet))\n",
    "\n",
    "for i in range(len(data_ico_bl)):\n",
    "    r2_ico_bl[i] = get_weighted_average(get_r2(data_ico_bl[i], gt_ico_bl[i]), descriptions_lr[0][\"DATASET_DESCRIPTION\"])\n",
    "\n",
    "for i in range(len(data_ico_unet)):\n",
    "    r2_ico_unet[i] = get_weighted_average(get_r2(data_ico_unet[i], gt_ico_unet[i]), descriptions_lr[0][\"DATASET_DESCRIPTION\"])\n",
    "    \n",
    "\n",
    "print(r\"Ico baseline, flat grid: $R2 = {:.3f} \\pm {:.3f}$\".format(np.mean(r2_ico_bl), np.std(r2_ico_bl)))\n",
    "print(r\"Ico UNet, flat grid: $R2 = {:.3f} \\pm {:.3f}$\".format(np.mean(r2_ico_unet), np.std(r2_ico_unet)))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "104065a0",
   "metadata": {},
   "source": [
    "### Evaluating Experiment for Comparing different predictor variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "647f58dc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10 matching runs found\n",
      "10 matching runs found\n",
      "Tas: R2 = 0.251 \\pm 0.004\n",
      "Precip: R2 = 0.327 \\pm 0.006\n"
     ]
    }
   ],
   "source": [
    "conditions_tas = {\n",
    "    \"DATASET_DESCRIPTION\": {\"PREDICTOR_VARIABLES\" : {\"tsurf\": [\"tsurf\"]},\n",
    "                            \"PRECIP_WEIGHTING\" : False\n",
    "                           },\n",
    "    \"MODEL_TRAINING_DESCRIPTION\": {\"MODEL_TYPE\": \"UNet_Flat\",\n",
    "                                  \"USE_COORD_CONV\": True,\n",
    "                                  \"USE_CYLINDRICAL_PADDING\": True,\n",
    "                                  \"LOSS\": \"Masked_AreaWeightedMSELoss\",\n",
    "                                  }\n",
    "}\n",
    "\n",
    "conditions_pr = {\n",
    "    \"DATASET_DESCRIPTION\": {\"PREDICTOR_VARIABLES\" : {\"prec\": [\"prec\"]},\n",
    "                            \"PRECIP_WEIGHTING\" : False\n",
    "                           },\n",
    "    \"MODEL_TRAINING_DESCRIPTION\": {\"MODEL_TYPE\": \"UNet_Flat\",\n",
    "                                  \"USE_COORD_CONV\": True,\n",
    "                                  \"USE_CYLINDRICAL_PADDING\": True,\n",
    "                                  \"LOSS\": \"Masked_AreaWeightedMSELoss\",\n",
    "                                  }\n",
    "}\n",
    "\n",
    "descriptions_tas, predictions_tas, gt_tas, masks_tas = load_data_for_comparison(base_folder, conditions_tas)\n",
    "descriptions_pr, predictions_pr, gt_pr, masks_pr = load_data_for_comparison(base_folder, conditions_pr)\n",
    "\n",
    "r2_tas = np.zeros(len(predictions_tas))\n",
    "r2_pr = np.zeros(len(predictions_pr))\n",
    "\n",
    "\n",
    "for i in range(len(predictions_tas)):\n",
    "    r2_tas[i] = get_weighted_average(get_r2(predictions_tas[i], gt_tas[i]), descriptions_tas[i][\"DATASET_DESCRIPTION\"])\n",
    "\n",
    "for i in range(len(predictions_pr)):\n",
    "    r2_pr[i] = get_weighted_average(get_r2(predictions_pr[i], gt_pr[i]), descriptions_pr[i][\"DATASET_DESCRIPTION\"])\n",
    "    \n",
    "print('Tas: R2 = {:.3f} \\pm {:.3f}'.format(np.mean(r2_tas), np.std(r2_tas)))\n",
    "print('Precip: R2 = {:.3f} \\pm {:.3f}'.format(np.mean(r2_pr), np.std(r2_pr)))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "54842bab",
   "metadata": {},
   "source": [
    "# Hyperparameter tuning\n",
    "\n",
    "Evaluate performance for the architectures with modified hyperparameters, as described in the `Experiments_yearly.ipynb` notebook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "490ecc37",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10 matching runs found\n",
      "10 matching runs found\n",
      "Deeper: R2 = 0.386 +/- 0.008\n",
      "Wider: R2 = 0.389 +/- 0.006\n"
     ]
    }
   ],
   "source": [
    "conditions_wider = {\n",
    "    \"DATASET_DESCRIPTION\": {\"PREDICTOR_VARIABLES\" : {\"tsurf\": [\"tsurf\"],\n",
    "                                                     \"prec\": [\"prec\"]},\n",
    "                            \"PRECIP_WEIGHTING\" : False\n",
    "                           },\n",
    "    \"MODEL_TRAINING_DESCRIPTION\": {\"MODEL_TYPE\": \"UNet_Flat\",\n",
    "                                  \"USE_COORD_CONV\": True,\n",
    "                                  \"USE_CYLINDRICAL_PADDING\": True,\n",
    "                                  \"LOSS\": \"Masked_AreaWeightedMSELoss\",\n",
    "                                  \"LEARNING_RATE\": 1e-3,\n",
    "                                  \"DEPTH\": 3,\n",
    "                                   \"FMAPS\": (64,64,128,128)\n",
    "                                  }\n",
    "}\n",
    "\n",
    "conditions_deeper = {\n",
    "    \"DATASET_DESCRIPTION\": {\"PREDICTOR_VARIABLES\" : {\"tsurf\": [\"tsurf\"],\n",
    "                                                     \"prec\": [\"prec\"]},\n",
    "                            \"PRECIP_WEIGHTING\" : False\n",
    "                           },\n",
    "    \"MODEL_TRAINING_DESCRIPTION\": {\"MODEL_TYPE\": \"UNet_Flat\",\n",
    "                                  \"USE_COORD_CONV\": True,\n",
    "                                  \"USE_CYLINDRICAL_PADDING\": True,\n",
    "                                  \"LOSS\": \"Masked_AreaWeightedMSELoss\",\n",
    "                                  \"LEARNING_RATE\": 1e-3,\n",
    "                                  \"DEPTH\": 4\n",
    "                                  }\n",
    "}\n",
    "\n",
    "descriptions_deeper, predictions_deeper, gt_deeper, masks_deeper = load_data_for_comparison(base_folder, conditions_deeper)\n",
    "descriptions_wider, predictions_wider, gt_wider, masks_wider = load_data_for_comparison(base_folder, conditions_wider)\n",
    "\n",
    "r2_deeper = np.zeros(len(predictions_deeper))\n",
    "r2_wider = np.zeros(len(predictions_wider))\n",
    "\n",
    "for i in range(len(predictions_deeper)):\n",
    "    r2_deeper[i] = get_weighted_average(get_r2(predictions_deeper[i], gt_deeper[i]), descriptions_deeper[i][\"DATASET_DESCRIPTION\"])\n",
    "\n",
    "for i in range(len(predictions_wider)):\n",
    "    r2_wider[i] = get_weighted_average(get_r2(predictions_wider[i], gt_wider[i]), descriptions_wider[i][\"DATASET_DESCRIPTION\"])\n",
    "    \n",
    "print('Deeper: R2 = {:.3f} +/- {:.3f}'.format(np.mean(r2_deeper), np.std(r2_deeper)))\n",
    "print('Wider: R2 = {:.3f} +/- {:.3f}'.format(np.mean(r2_wider), np.std(r2_wider)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "268eacab",
   "metadata": {},
   "source": [
    "### learning rate tuning:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "6edba356",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "60 matching runs found\n"
     ]
    }
   ],
   "source": [
    "conditions_lrs = {\n",
    "    \"DATASET_DESCRIPTION\": {\"PREDICTOR_VARIABLES\" : {\"tsurf\": [\"tsurf\"],\n",
    "                                                     \"prec\": [\"prec\"]},\n",
    "                            \"PRECIP_WEIGHTING\" : False,\n",
    "                            \"GRID_TYPE\": \"Flat\"\n",
    "                           },\n",
    "    \"MODEL_TRAINING_DESCRIPTION\": {\"MODEL_TYPE\": \"UNet_Flat\",\n",
    "                                  \"USE_COORD_CONV\": True,\n",
    "                                  \"USE_CYLINDRICAL_PADDING\": True,\n",
    "                                  \"LOSS\": \"Masked_AreaWeightedMSELoss\",\n",
    "                                  \"FMAPS\": (32,32,64,64),\n",
    "                                  \"DEPTH\": 3\n",
    "                                  }\n",
    "}\n",
    "\n",
    "descriptions_lrs, predictions_lrs, gt_lrs, masks_lrs = load_data_for_comparison(base_folder, conditions_lrs, keywords_blacklist=keywords_blacklist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "a4b19c36",
   "metadata": {},
   "outputs": [],
   "source": [
    "r2_lrs = np.zeros(len(predictions_lrs))\n",
    "\n",
    "for i in range(len(predictions_lrs)):\n",
    "    r2_lrs[i] = get_weighted_average(get_r2(predictions_lrs[i], gt_lrs[i]), descriptions_lrs[i][\"DATASET_DESCRIPTION\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "8e985f5e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lr: 0.0001 R2 = 0.352 +/- 0.004\n",
      "lr: 0.0001438449888287663 R2 = 0.357 +/- 0.005\n",
      "lr: 0.00020691380811147902 R2 = 0.362 +/- 0.005\n",
      "lr: 0.00029763514416313193 R2 = 0.365 +/- 0.002\n",
      "lr: 0.00042813323987193956 R2 = 0.369 +/- 0.005\n",
      "lr: 0.0006158482110660267 R2 = 0.376 +/- 0.001\n",
      "lr: 0.0008858667904100823 R2 = 0.377 +/- 0.008\n",
      "lr: 0.0012742749857031334 R2 = 0.379 +/- 0.002\n",
      "lr: 0.0018329807108324356 R2 = 0.386 +/- 0.003\n",
      "lr: 0.0026366508987303583 R2 = 0.382 +/- 0.001\n",
      "lr: 0.00379269019073225 R2 = 0.382 +/- 0.007\n",
      "lr: 0.005455594781168515 R2 = 0.381 +/- 0.009\n",
      "lr: 0.007847599703514606 R2 = 0.382 +/- 0.003\n",
      "lr: 0.011288378916846883 R2 = 0.385 +/- 0.001\n",
      "lr: 0.01623776739188721 R2 = 0.372 +/- 0.001\n",
      "lr: 0.023357214690901212 R2 = 0.374 +/- 0.004\n",
      "lr: 0.03359818286283781 R2 = 0.378 +/- 0.004\n",
      "lr: 0.04832930238571752 R2 = 0.374 +/- 0.001\n",
      "lr: 0.06951927961775606 R2 = 0.379 +/- 0.004\n",
      "lr: 0.1 R2 = 0.363 +/- 0.003\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, '$R^2$ score')"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEKCAYAAAAB0GKPAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABGbUlEQVR4nO3deXSb93ng++8DgAAIgPu+Uwu1WZZtWba8O06atTN1ttZJ42abJDdd5nbu0rm5587tdKbTdnLOnc5MZzJ10yZp0mma3a1rO0kT2/Eq25IlWSslkdTCfd9AEMT2u38ApEASJAEQAEHy+ZzjYxLA++JHviKe97c9jxhjUEoppdZi2egGKKWU2hw0YCillEqKBgyllFJJ0YChlFIqKRowlFJKJUUDhlJKqaTYNroB2VJZWWlaW1s3uhlKKbWpvPXWWyPGmKpEz23ZgNHa2sqJEyc2uhlKKbWpiMj1lZ7TISmllFJJ0YChlFIqKTkNGCLyPhG5JCIdIvKlBM8/KiJnROS0iJwQkQfinvtdETknIudF5F/lst1KKaVyGDBExAp8BXg/cAD4uIgcWPKy54DbjDG3A58F/ip27EHg88DdwG3APxORthw1XSmlFLntYdwNdBhjuowxAeA7wKPxLzDGeM3NbIhuYP7r/cDrxhifMSYEvAh8KEftVkopRW4DRgPQHfd9T+yxRUTkQyLSDjxDtJcBcA54SEQqRMQFfABoSnDsF2JDWSeGh4cz/gMopdR2lsuAIQkeW5Zb3RjzpDFmH/BB4A9jj10Evgz8DPgJ8DYQSnDsV40xR4wxR6qqEi4jVkoplaZc7sPoYXGvoBHoW+nFxpiXRGSXiFQaY0aMMV8DvgYgIn8cO59S25o/GGYuGMEfCjMbCOMPhQmGDAfqi7FaEt2jKZW+XAaM40CbiOwAeoGPAb8e/wIR2Q10GmOMiBwG7MBo7LlqY8yQiDQDHwbuzWHblcqZcMQw7Q/iD0bwB8PMBsP4g+Ho96Ewc7HH5oIRIivUP/MFQhzdWZHbhqstL2cBwxgTEpHfAX4KWIGvG2POi8gXY88/AXwE+KSIBIFZ4LG4SfAfikgFEAR+2xgznqu2q+1lZi6E27FxSRDO9U5yvm9qXefoHJ6hutjJjkp3hlqlFMhWLdF65MgRo6lBVDpe7RihrdpDdbEz5+8dCEX4h9O9BMPr/7u0WYT33lJLiasgAy1T24WIvGWMOZLoOd3prVQcfzBM95iPs72TG/L+lwamMxIsAEIRw8sdw4TCkYycTykNGErF6Rz2EjEwODXH0JQ/p+8dCEW4NDid0XNOzYZ48+pYRs+pti8NGErF6RyeWfg6172My4PTBEKZ7w1cG/XRMZTZQKS2Jw0YSsX0T87i9d/c3pPLXkYwHOHSQPY+1N+6Ps7YTCBr51fbgwYMpWI6hrzLHstVL+PKoJe5LPQu5oUj8ErHSFZ6MGr70IChFDAbCNM7Prvs8Vz0MkLhCO0D61tGmwyvP8TrXaNZfx+1dWnAUIqbk92JZLuXcWXIiz+Ymzv/nvHZnAQntTVpwFDbnjGGzuHlw1HzBqfmGMxSLyMcMTn/AD99Y4Lh6bmcvqfaGjRgqG2vb9LPzFx41dec7clOL6NjyMtsILfzChEDr3WO4A+u/jMrtZQGDLXtJZrsXmpoOvO9jHDEcLF/Y4aHZubCHNP5DJUiDRhqW/MFQvRNLJ/sTiTTvYyuYS++wMbd5fdP+Dm3QTva1eakAUNta51DMySbTi2TvYxIxHBhg3oX8c72Tma85zQ5G8zo+VT+0IChtq1Ek92hSIS/feM6b/dMJDwmU72MrpGZNedNcsHE5jNm19nT8QVCnO+b5Jkz/Tx7tp+50Mb/bCrzNi6Hs1IbrGd8dtmQ0JnuaGrx9oFpyl12mspdi56f72XUrCOTbb70LubNBiK81jnCO/dVI5J80aVgOEL3mI9rozMMTs0t6qkNTc0t+92pzU97GGrb6ljSu4gYw4tXhqkuclDstPHtN2/gnVtWCXjdvYxrozOLUpDkg8GpOc4k8XMZY+ifnOW1jhGePNnL611jDEzOLRvWG8hx4kaVGxow1LbknQsxMLn4Q629f4rh6Tke2VvNJ462MDMX4u/evEF4yY6+9cxlGGPWXRwpW873Ta24AGDCF+DkjXH+/nQvL7QPc23UR2ilnY6w7HertgYNGGpb6hzyLrorNsbwi8vDlLvtHGwoob60kA/d0cDVkRl+en5g2fHp9jKuj/qYzrPeRbxjnaPMxHpVs4EwF/un+PHZfp49O0B7/3TSe0am/aGF86itQ+cw1LYTiRi6RhYPR3WNzNAzPsujt9djtUTH8e9oLqN7fJZXOkZoLCvkUGPpwuvTmcswxnCuL7+Xsc6FIrx8ZRiHzcrAlD/pFWTzAqEIz10cZHeNh4EpP7uqPNlpqNoQ2sNQ207P+OyyO+UXLw/jcdg43Fy26PEP3FpLS7mLH57sWTYun2ov48aYj6nZ/L/rHpsJ0j+ZerCY9gf5q1e6eLljhBcvDTOow1JbjgYMte10DC+uO9E7PkvHkJf7d1dSYF38J2GzWPj40WacNit/+/r1RctPU53LyNe5i0wYnPLz5y92Mjjlp7XCzfUxHzfGfBvdLJVhGjBU3nm7e4Kh6ezcnU77gwxMLk689+LlIZwFFo7uKE94TLGzgF8/2sy4L8D33+omEnfrnWwvo3vMx4Rva25o6xjy8hcvdRIOG77w4C7esbeKcMRwaWCaCZ8WbdpKNGCovDJf1/qlyyNZ2TG8NG/UyPQc5/umOLqjAmeBdcXjWirc/PKhetoHpnnh0tDC48n2MrZqCo4T18b469euUlpo5zffsYuGskJaK9xYRegY9ury2i1GA4bKKx1DXkJhQyAU4ReXhjKaUTUSMXTF1ewGeOnKMFaLcN+uijWPv2dHOXc0lfL8xaFFKcnX6mX0jPsY32K9i4gx/PT8AD861cuuKg9feGgnpS47AHabheYKF51DXl1eu8VowFB5IxIxXB68Ob8wMxfmF5eGCYUzk/67e9y3qAzq5GyQUzcmuLOljCJnwZrHiwgfvKOB2hIn3zvRzag3OrS1Vi/jXO/WmrsIhiN893g3L14e5u7Wcj55b+uy3tmuKg99k36ujswQWWW/htpcNGCovHF9zLcsVcfYTIBXOkYwqS7ZSeDK4OLhqFc7RjAYHmyrSvocBVYLnzjagiD87Rs3Fmpkr9TL6JuYZWxm64zje+dCfO2Vq5ztneT9B2sXLUOeV+Gxs7s6upz28qCXkRkt1rRVaMBQeaN9hfxKfRN+TlwfX9e5J2eDDMVVmfPNhXjz6hiHGkspd9tTOle5287H7mpicMrPj071YIxZsZeR7fKuuTQ8PccTL3bSNzHLr9/dzINtVctyTxXaLTy8p4p9tUU4bBY6hrwMTmrA2Co0YKi8MDDpX3Wc/8qgl/Pr2PS2dLL72NVRAuEID6XQu4jXVlPEuw/UcKZnktc6o4WIlvYyBib9jHq3Ru+ia9jLEy92MheK8PkHd3KwoWTZa0Tg3p2VOAusNJQVsrPKQ6dOfG8pGjBUXkim8tzb3ZNcG5lZ83VLhSOGq3HHBUIRjnWOsq+2iNqS5Tu1G8oKaSgrXPO8D++p4kBdMT8+10/XiHdZL2Or9C5O3hjnG69ew+O08ZsP71oxC+2BuuKF32dtsZNdVW7GZgJcHpwmmKF5KLWxNGCoDTfhC9Cf5Gqa17tGU078d2PMtzDXAHD82hi+QJiH9yzvXdgswt2t5Ty8p4pfOlBNVZFjxfOKCB+9s5Fyt4O/e7ObydngQi9jcMrP8PTmHooxxvDzi4P84K0eWipdfPGhXSsO31UVOTjUeLPXUV3soC02j9Ex6F00HKg2Lw0YasNd7J9e+0UxEQMvXR5mMoVlqvHDUaFIhFc6RmitcNFS4V722gP1xRTaoyt+qoucvPtADQ/tqaTUlXgVlbPAyuNHmwmGI3z7jev0TcwyOLX5S5+GwhG+/1YPz7cPcWdzGZ++r3Xh97KU3Wbhvl0Vi+YzHDYre2qKKHbaovsxdHlt0uJvbvKNBgy1oWYDYa6PpjbMFAwbfnF5KKkqcZO+4KI7/be7J5mcDfLwnuplr3XZreyrLVr2eGOZi/cfrOWeneW4Hcs/NKuLnXz0cCPd47P845n+WC9o895R++ZCfP3Vq5zunuA9B2r48OEGbJaVPyqiv5fleUzrSgvZFZvH6J9Mrm66gvaBKab9+blvR7PVqg11aXCadJbpz8yFefHyEO/aX7Ms/1O8K0M3ey8RY3jp8jB1JU721CzPonpbUym2Fc4lIuys8tBS4eby4DQX+qYW7ek42FDCw3uqePHyMI1lhdzVujjNiDHRzYjeuWjab+9cOPr/wPz30f/PzIWZCYSo8jg4UF/M/rpiylypreJKRygS4erIDBf6pjjfN4U/GOZjdzUtytCbyN5aD41liec0aoud7K72cKp7gvb+aX5pf3jV3fQq2rO7POilpLAgqb1BuaYBQ22YYDiybPVSKsZmgrzSMcLDbVVYLMtLi4bCkUWT3Rf7pxj2zvHYXU3LloOWu+3sqFw+RLWU1SLsrytmV5WHi/1TXBqYXigk9O4DNfSOz/LU231cG5lhJhANAPPBYKWCQw6bBbfDhsdho8xtp760kJ5xH0+f6efpM/3UlTg5UBcNHnUlzpTKqK5mLhjm8pCXC32TXBqcxh+MUGAV9tQU8VBb1ZolVsvdBdzRVLbi81VFjoXA3BkblmpN4ne8nXUMewmEIozOBBIOmW40DRhqw3QNz6x7vLZ/ws/xa2Mc3bk8tcf1MR/BcPRD2hjDi/MFkuqXLwk93FKa0vvabRZuayplT00R5/om6RzyAsJjdzXxrWPX6BqZweOw4XZYqSl24HbYcNttscdsC8+5HbYVe0gj3jku9k9xoW+K59uHeK59iFJXAfvrijlQVxzN2ZQgUK5m2h+kvX+aC/1TdAx7CUcMbruVg/UlHKgrZle1Z9Ue2zybVbh/d2XCQD3PahF2VXuoKnLQMRRdXqsBY2WRWMJGIG+XY2vAUBvCGLMoH9N6dA7P4HbYlu0NiO+9JCqQNK+53EV1UfKFkOIV2q3c1VrO3toiznRPcmPMx2++Y3da51qq0uPgwbYqHmyrwjsXor1/igv9Uxy/OsaxzlEKC6JzLvvrimmr8eCwJR7uGfHOcaEvemz3mA9DtEd1784K9tcV01LhwpJir+Xu1vKkhkxqip3srvJw4voYveM+YO2cXdvVtdEZZuai83LjMwGMMRnrTWaKBgy1IbrHZhf+ODLhTM8kLruVnbEKb+MzgUV3aS9eGqYoQYEkqwVuby5d9/sXOwt4oK2SUe8cp7snMj7p7XHYONJazpHWcgKhCFeGovMo7QPTnOqewGYRdlV5OFBfzL7aIiZng1yI9U7ml7TWlzp51/5qDtSVUFPsSPvDaFeVO+mewvw8xrGuUS72TzPlD1Kch2Pz+SB+tWAoYpicDS4kdMwXGjDUhriYod5FvDevjuGy26gtcdIxfLN30TPuo2PYy/tuqV023LKnpghPghU+6arwOHjH3mqePtOX0YAYz26zcEt9CbfUlxCOGK6PzXAx1oO4dOrmh45FoLXSzd07yjM2eV5SWMCdLSvPWyxV7razp7YIi0TnMQYn/XkfMLrHfFQVOXI6Qd8z7luWzn/EG9CAodTQVHZSZkQMvHxlmHfsrV402f3i5WGcBRbuXlIgyRH74M00q0W4taGE17vGMn7uRO+1s9LDzkoPH7i1joEpP5cHvRQ5beyrLcJlz9yfuM0iPLC7csWVZImICK0VLhrLXAvzGG01y5cu54uBST+vdoywo9KdcF4sWy4kqMaYj0krc7oPQ0TeJyKXRKRDRL6U4PlHReSMiJwWkRMi8kDcc/+biJwXkXMi8ncikt6gs9pwFweS36iXqmDY8NzFQUKxye7h6ej4/T0JCiQdaizBbsvOn8COSveKm/2yRUSoKynk4T1VHG4uy2iwADjcUkZJGj/TfJqQnvFZro36MpJ5OBvGZgK8dGWYiInOeeXqA3to2s9IghuosTzM8puzgCEiVuArwPuBA8DHReTAkpc9B9xmjLkd+CzwV7FjG4D/FThijDkIWIGP5ajpKoOm/EF6x7O7iSt+9epCgaTdlYteU+oqWEjBnQ0iwm1NpVk7f661VLjS/n3VljjZVe3BAJf6p/PyznlyNsgL7UMLNxrGwFvrzJCcrES9C4AJX5BwntUSyWUP426gwxjTZYwJAN8BHo1/gTHGa27efriB+N+WDSgUERvgAvpy0GaVYe0ppAFZr8nZIKdvTHCktWzZPMUdzaVZX4HSUFq4ai6qzcLjtC3biJiKImcB+2uLKLDmZ9lWXyDELy4NLdqICdHeaTrJLlMx6QvSN5H49xEx+TcslcuA0QB0x33fE3tsERH5kIi0A88Q7WVgjOkF/j/gBtAPTBpj/inBsV+IDWWdGB4ezsKPoNbDHwxzdST9jXqpeuXKcLRA0u7FSQbrSp3UlaydjTYTbt/kvQyLwAO7K9c9dNdQ5mJHpZvOIW/KySOzaS4U5oX24RUXKJzunshYxcdEzvevnnNsOweMRLdzy/pbxpgnjTH7gA8CfwggImVEeyM7gHrALSKPJzj2q8aYI8aYI1VV6dU5UNlzZdBLrrJc++ZCHL82zm2NpZTFZVi1CMuW1mZTVZGDxiRSpeer25tTLzCVSHQew8Owd46OIW9eDLWEwhFevDS8bHVSPF8gzIUkUu+nY2YuxI1R36qvmS8DnC9yGTB6gKa47xtZZVjJGPMSsEtEKoFfAq4aY4aNMUHgR8B92WysyqxojpzcDUcd64oVSFqSwnxXtYeSwtxORt/WVEqe7b9KSkNZIftqizNyrtoS56KyrRud+j0SMbzcMZJwsnmp9v5pZuZCGW9D+8DUmnnURrdxD+M40CYiO0TETnTS+qn4F4jIbokNLIvIYcAOjBIdirpHRFyx598FXMxh29U6XR2ZWTZGnC2BUITXOkfZX1tETfHNxXQF1uhy11wrKSxIKk9VPnHZrRzdkf68xVLOAit7a4tw2a10Dm38PMbrXaP0rzB3sFQoYjh1YyKj7+8PhukcWnt+ZNofyqt05zkLGMaYEPA7wE+Jfth/zxhzXkS+KCJfjL3sI8A5ETlNdEXVYybqDeAHwEngbKzdX81V29X6RNOA5K53cfzaGLPB5QWSDjaUbFi21EONJaSwfWFDicB9u5cvQ16v+li6845hL/0TG5fu/K3rY1xbYyhoqRtjPoYyGOSuDHpXTEa51GgeLa/N6T9hY8yzxpg9xphdxpg/ij32hDHmidjXXzbG3GKMud0Yc68x5pW4Y/+tMWafMeagMeY3jDH581tUq+oZn2Xan/kufSJXR2Z46fIwOyrdNMdl+/Q4bezdwA1jLrstrzesxdtd7Uk7t9Zq5tOETPtDXBqcZi6UnZ3wqznXO8mlgfQWXrx1fTwje0hSHZ7Np0SEm+SeR21muehdTM0G+e7xG/zly11YLcIHbq1b9PwdTaWrZlbNhVvqiymw5vdkhtUSbWc2VBfFlW0d8jKU4yJTHUPTnOlJvxLiuC9I5/D6V/l1Dq8+PHt1ZGZRhcJ8WimlqUFUVo1457I6wRmKRHitY5TnLw0RiRge2VvNw3uqFi0DrS5yrFnbIRccNisH6ot5uzt/y7furs5sOpF4NquFtpoiyt32hXmMXF2X7jEfx6+tfyPe292TNJe7015mHImsnqV5YNLPN169SlO5i88/uBPIryEpDRgqq7K5Ue/y4DRPn+lnxDvH/toifvlQfcIloIdTSJaXbXtrirg8OM1sIH8mMufZLJK13sW8upJouvO3eyboHZ/lrtasvh0Ag1N+XuscIRMZSeZCEc72TnBnS3oLAq6P+Vbc8xEMR/jO8RuEIoaecR/hiMFqEWYDEXyBUNYCeSp0SEplzbQ/SPd4apOLyRibCfA/X7/OX792DWMMn7q3ld+4tzVhsNhR6c7IPoJMsVktG7JSKxl7aouyviigpjiaJmQuFKG9fyory1Xjjc0EePHycEb3/1wZ9K66d2M1F1fZ0/HjcwMMTc9xZ0sZwbBZNCyVL/MYGjBU1lwenM7IXd28QCjCzy8O8l9+fpmOIS/vvaWW331XG3trE08m2yySlzutd1V5KC7c+LvFeAVWYX9d9iflK9x29tZ4EMh6mpAp/+L8UJkSMXAyjTxTvROzTPgSB5r2gSle7xrlvl0VvGtfNQDXx24uu82X/Rj59a9WbRlzoeTWmSfDGMOF/imeOdvPhC/IocYS3n+wbs0NeAfqiym0b8wy2tWICLc1lvLylZGNbsqCfbXFK1bsyySLRdhR5aGu1EnH0AyDk352VWU+CeRsIMwL7cvzQyXj1I1xSgoLFopxJdI/6ad3YpaG0uR38a+UZHDaH+SHb/VQW+zkvbGaLSWFBdwY83Hfruhr8iVzrQYMlRWprDNfzdC0n6fP9NMx5KW22MnnHmxkZ+XaHzAue7R8ab5qKndR4bHnxVCD3WZZsZeWDbWxsq2vdoxyfWyG+6hc+6AUzIXCvHBpKK0CVq91jvD0mX5sFuHT97WuGjROXh+nrtiZ1Oq74enEiz+MMfzwZA9zoQifu6tpocBXU7mL7rGbw7n58O8EdEhKZUEkYrgytL7Jbn8wzI/P9vNnz12hZ9zHPztUx28/sjupYAHRdBypFPrZCPkyXLa/rihrdUESqY3NY4SN4dKAlwlf5j4MIxHDy5dHVhz6Wc3p7nGePtPPvtroSq6/ef06fatsMJz2h5JeMr5SPqpjXaNcHvTy/lvrFmUlaC53Me4LMuWP/hzBsFn4eiPl91+U2pTO9U2mvQrIHwxz4toY//lnl3mlY4TDzWX87+/ey327KrEmuY+i3L05UnHUFDupK93YOmDOAkvONzSWuArYV1uEzSILVfgy5fWu0YUa5qloH5jiB2/1sLPSzcfvbuYz9+/AWWDlG69dWzUB4Pm+SfzB1Xsyk77ENWAGJv385NwA+2qLuGdJGpaW2HLj+OSE+dDL0IChMurt7gnO9Saf3dMYw9CUn5evDPOXL3fxH565wI9O9VLiKuA337GLDx9uTLnmdqYS5uXC7Y2lG/r+B+qLN6Qn1lTuorncReewd9FqoPV4u3si5ZQfEN0o9+03blBXUsjj97QszCF85v5WjDF847VrK97dB8OG090Tq54/Ue9ifgmts8DKhw83LqvNUlfqxGaRRcNS+TCPoXMYKmPeuj6WVNqFYDjC1ZEZLg1Mc2nwZgW22mInD7ZVsa+2iKZyF5Y0UrwW2i0058EmvWSVue20VrjS+qBbL5fdSlv1xszzzKcJ+acLg3QNz/BQm1nXTvzOYS/nV5hUXk3/5Cx/8/o1Sl0FfOq+1kXLiquLnHzq3la+9spVvvnaNT73wM6Eiyi6hmdoq/ZQ4VleLMsXCHF9dPnij/kltJ++rzXhDZHNYqG+tJDrcQEjmcy62aYBQ62bMYY3r47RObzyqqjJ2WA0QAxM0THsJRg2FFiFXVUeHmyrZG9NEaWu9e+XaKsu2vAUIKk61FTKjTHfmqmuM+2W+uKkh/kyrbbEGVsdNcilwWlGZubSzl/VPznL8atjKR836p3jG69ew2Gz8tn7dyT84G4qd/GJo81869h1/ub163zm/taFiel4b10f5z231C57/GL/9LLrOr+E9v5dFexZZTiwpdzFsa5RQpEINouFCV+ASGR9gXW9NGCodYlEDK93jS67Q44YQ8+Yj/bBaS4NTNMfG3YodRVwuLmMfbVF7KzyJPzjS5fVQlbrdGeLx2GjrcaTdlK8dLgd1qwsZ02Wy25jf10RzgJLtArfZHoBY8IX4JUrIykH26nZIF9/9SoRY/jcfTtWvVlpqynio3c28r0T3XzneDe/fnfzskA74g1wbWSG1ri5s7lQeFnuqfgltIkCTLymchcvd4zQPxFNoRKOwMRscEM3omrAUGmLRAyvdIzQEzehFzGGp8/0c6ZnAl8gjEWiKz7ee0st+2qLqC5yZK2WdnO5e8PSl6/XLfUldA7PZHyT2UoONpRseE+svrSQnZWxdOeTs9zamNoO+NlAmBcvDxNM8Xc2GwjzjdeuMhMI87kHdlBdvHaguq2plJlAiKfP9PP3p3v58B0Ny/4dn+6eoLGscGFO6Mqgd9H1jKywhHYlzRXRodXrY76FnFuj3jkNGGrzCYUjvHxlZKHnMO9MzwSvd41yS30xBxtK2FNdlLPNc7ncS5BpzgIr+2uLOdub/cSERU4bO/NgFVlNbB7jQv8UVwaneWRfddI9zmA4wouXU99rEQhF+Oaxa4x4A3zq3lYay5Kf77pvVyUzc9E9Hh6Hjfcu6SHMl3M91FhKKBzh0pIlt8c6o0tof+W2+kVLaFdS7CygzBXdwDdvdCZAW9ItzjwNGCplwVgt5KXLFyPG8Hz7ELXFTj5+d3Nak9bpqvTY8ypnVDr21UUTE2a7MuGtDSVZ6+WlYj5gAFwe8jI0PZfUzmljoj3bsZnU9iWEIhG+/eZ1usd8fPzu5rSGL39pfzUzcyFevDyM22Hjgd2LNx1e7J9iV5WH3onZRdexf3KWn5yPLqFNpZJhU7mL66PxK6U2duJbl9WqlMyFwjzfPpRwrfuZnglGvAHeua86p8ECNnfvYl6B1cLBLCcmLHUV0FKRH6vI7DYLe2qiNdY7h5JfXnv82njS5VXnRYzhB2/1cHnQywfvaEj79ywi/Mrt9dxSX8yzZ/s5dWNxTqlwJDoBHp9kMBiO8N3j3RSusIR2Nc3lLiZngwubGydng4QymUkxRRowVNL8wTDPXxxKuIEoHLnZuziQ5RTZS7nsVppSGFrIZ23VHtyO7A3h5UvvYl5dSbRsa+fwDP2Ta5dtvdA3RcdQaosDjDE8faaPMz2TvO+WWu5qXV+tcosIjx1pYmeVmx+e7OHSkvoWPeOzi4bK5pfQfvTO1PcUtZRHhw7nh6WM2dhehgYMlRRfIMTPLw4yvkLKhY3sXeyu9mz4BG6mWCzCHU3Zqd9R7i7Ii0JS8WpLnOyudjMbDHOxf3rVXdM3Rn1rbpJL5Ln2IV7vGuPBtkoeWlLnPV02q4XHj7ZQW+Lk22/e4EaCvRaQ/BLaldSWOCmwLt7At5GZazVgqDV550L87MIgU7OJaxeEI4YXLt3sXeQyXmzWpbSraa5w8ci+Kgrtmf3zvHWDd5UnUuVxLKQmWW1Yanh6jmNdqWf3fa1zhOfbh7izpYz3rbGMNVXOAiufvm8Hxc4CvnnsOoNLUpyksoR2JVaL0FDqWrSBbyNThGjAUKuanA3y8wuDq65GWdq7uKW+OKvDKvFaKjbvUtrV1JUU8v6DdTSWJZ8+ezWVHntKqbhzxWIRdlZ5qCl2rFgfY8of5KU0iiDNJxM8UFfMB29fvgw2EzwOG5+5fwc2i/CNV68yHptriF9C+1gSS2jnuexWHtpTuaj2e0uFi/4JP8HYL2AjS7ZqwFArmvAFeO7iIL7AysFifu6iruTm3MWOSjdH1jlOnKxcJ87LJWeBlYf2VHH3jjJs6xxyO5SHvYt587u+r43M0LOkQqM/GOYXl4ZTXjkWn0zwsbuasrqjvdxt59P3txIIR/jGq1fxzoUWltB+YEkW2tXsqnLzgVvraCxzUVV0M81Ic7mLsDELmXNn5sJrJjzMFg0YKqFR7xw/vziEP7j6H+qZnglGZ272LspcBRQ5C2goLaSpPLt3tNVFDso2+VLaZOyuLuK9B2spd69eMGolNcUOaks2NivuaubrY4Qihov90wuJ/sIRw0uXh/H6UyvjmiiZYLbVlRTyG/e0MuEL8vVXrvLTFJbQepw23rW/mqM7KxbSzMcHjPl5p3xYXqsBQy0z6p3jufYhAmvc1cX3LvbXRXsX8ZOqd7aUYbNm785uKyylTVZJYQHvOVCbVhnVVHdQ51qZ287e2iIsAh1DXgZj8xjHOkdTSrg34p3j+ye6+dorXZS67MuSCSbrYEMx9+2qoMyVWoDeEUuNPjTtT2oJrUj03/AHDtYu64XEp0nxOGxUuO2LN/Bt0DyGbtxTy5y8MZFUior53sUnjt7cpNcct8bfZbdxa0MJp25MZLyNboc1Y+P7m4XFItzRXEZ9aSHHOkdXHSqcV1fqTDupXy61VLhpKoulO5/y450LLfqAXM3w9BwvXBri7e4JbFbh3p0VPLy3OuUlrAB7az0Lw3etlW76J2e50DfF4FRy8wb764r53AM7cdmtq75/caGNozsqFvUk4lW47dgsslC1srncRceQF2MMIrJh8xgaMNQig1P+hKUkl1o0dxHrXZS6Cih2Lr4r21dbxLWRmRWX46Zrd7Unr/YT5FJNsZP331rLm1fH6B5bfe/CoSxvBMyU2pJoFb4X2oe4MuhNqmcwNO3nhfYhzvRMYrMK9++u5MG2Soqc6Q3dtVa6uLNl8RBSXUkhdSWFjM0EuNA3Rfe4D7PGvVTrKmlXLAL76oq5taFk1XkVi0Wo8NgXAlVTuYtT3RNM+IKUue0bNiSlAUMtcrYnuVxGb8d6F48fbV744E5Uh0JEuGtHOf90fjBjbbRZZMstpU2Vw2blwbYqOoe9vHV9PGGPsLGsMGGNhnxUWxKdx3i+fYjOYS+31K8c6Aan/LxwaYizsUDxQFslD7ZVpdWjmNdQVsg9OypWfL7cbeeBtkqm/UHaB6bpGvamvGqrzFXA0Z0VSaewqS5yLgSMlrhEhGVuO/5ghJm5EO51/Mzp0IChFgxO+ZMqbxmOGF5YMncBrLjbutLjYHe1J+UduitpqXDhsG29pbTp2FXlobrIwasdo8vuOg/l+dxFPI/Dxr66IuxWCx1DiQPGwFS0R3Gud5ICq4UH26p4oK1yXYECoosCHthdmdTmzyJnAXe1lnNrQwmXB6e5POhdc67PaolmIz5QV5zSBtPqYgf0zrfRid1m4caYb6EW/Kg3oAFDbZz19C6KC22UrDJJeFtTCT3jvjVXXSVjO012J6PIWcB7DtRwtneSC/1TGBMNqpkoSJVLjWUudlS6l9WQGJj08/ylaKCw2yw8tKeKB3ZXZuTDstxdwEN7qlJedusssHKosZQDdcV0DHu5NDCdcK9ShcfOPTsrKClMfZiswm3HIhAx0XQkjWWFS3Z8zy2aM8yFlH7jEv10+ASw0xjz70WkGag1xryZldapnFlv72KtsqgOm5U7mss41jm6rnbWFDs23QdhLlgswm1NpdSVODnWNZr1JIbZUFscnce4dHaaCV+A2WA00eX5vikcNgvv2FvFA7sqcWXorrq40MY79iafUj0Rm9XCvtpi9lQXcX3Mx8X+KSZ8QWwW4VBTCXtritKea7NZLZS57QsroprLXbx0eZhAKILdZtmQlVKp/ub/BxAB3gn8e2Aa+CFwV4bbpXLsXJJ1GN7unu9dtCz6Q0imjvaOSjddw96kV5wkkk4+nu2kutjJPz9Uvylza1UXO9gdqwL4169dY2h6DofNwiN7q7h/dyUue+YGRNwOK+/cV52xLAEWi7Cj0s2OSjd9E7MUOW1pT77Hqy5yLASGlnIXEQM9Ez52VnoY8wUWVk3lSqqh9agx5rcBP4AxZhzQ271NbmjKn9SHeDhieP7SfO/i5gd3caEt6bv+I63lpPtZth2X0qZjMwYLiBWRqvNQWljAlD/IO/dV86/fu493H6jNaLBwFlh4ZF91Rs8Zr760MCPBAlhUDXB+j1N3bANfKGxWzO+WLan+xoIiYgUMgIhUEe1xqE0s2Spvb3dPMJagd5FKavGSwgL21xVzvm9q7RcvsWcd3Xu1OdSWFPJbj+zGZpGs5AgrsArv2Fu9bPl3vqryOBCJpjV32W1UehyLExHOzK06d5hpqfYw/gx4EqgWkT8CXgH+OOOtUjkzNJ1a76J+Se8CkhuOindLfTEeZ2r3KjaLsKtqey+l3Q5qS5x4HLasBAubRXh4b9Wmqsxot1kojZswbyl3cWPMh4ltBsl1qvOkA0Zswvsl4F8DfwL0Ax80xnw/S21TOZDs3MXpWO/iXftrFt3le5y2lPM52awWjrSkVvNhR5V7Ic+O2rqqPA6ykfrJInB/W+Wm2PW+VHXx4kSEvkB4YQl1rie+k77NM8YYEfl7Y8ydQHsW26RyZGjaz8BkkiujYr2LfbXr613Mqy8tpDl2t5QMnezeHmxWC5Uex7oWRiwlAvfsrMjL9O7JqPI4uUR0qXFT3Aa+Co+DCV+AcMRkNRtvvFRj+esioiuitoj19i4AmtYxCX24pTSp5IS1JY601rGrzSnZdODJOtJStmq6jnwX38OoLnLgiG3gg+gejfkaHLmQasB4hGjQ6BSRMyJyVkTOZKNhKrtS7l2ULu9duB3WdaWecNltSe1G1t7F9tJQWojHactI5cZDjSW0bfJ/P84CK0WxOT+LCM3lrkUb+HKZVyrVVVLvX8+bicj7gP8KWIG/Msb8xyXPPwr8IdGVVyHgXxljXhGRvcB34166E/h9Y8x/WU97trPzvcmtUjrdPc7YTIDfuKdlee8iA/Wh99ZEkxOOzSROTuhx2jbtUIJKT5nbzq/cVk84Ypj2B5mcDTI1G2Iq9vW0P5hUHqd9dUWbcgNjItVFDqZjdUGayl280D7EXDCMo8AanceoyU07UgoYxpjrInIb8GDsoZeNMW8nc2xsOe5XgHcDPcBxEXnKGHMh7mXPAU/F5ksOAd8D9hljLgG3x52nl+hqLZWG4ek5+leonRwv2rsYTti7gPTnL+KJCEday/nZhcGEWUD31GzfrLTbndUilLrsy/b4GGPwzoWY8oeYmg1G//OHmJwNLuR12lnl5nBzagsr8llVkYPO4RkgulLKAN3js+yu9uQ01XmqqUF+F/g88KPYQ/9TRL5qjPlvSRx+N9BhjOmKnes7wKPAQsAwxsQnkXET2++xxLuATmPM9VTarm5Kfu5i5d6F22GlMkOZUOeTE14ZXJxDyGYVdlbqUlq1mIhQ5LxZ2TGePxhmyh+kapNk6U3W0g18AtwY87G72sPUbGghXUi2pfoO/4Lobu/fN8b8PnAP0QCSjAagO+77nthji4jIh0SkHXgG+GyC83wM+LuUWq0WjHiT71083z5EQ2lhwt5FYwqb9ZJxW2MpzoLF/xx3VupSWpUaZ4GV6iLnluuVehw23I7o3hRngZWqIgc3xmYWns/VxHeqf40CxKdkDMceS/bYpZb1IIwxTxpj9gEfJDqfcfMEInbgV4CEez9E5AsickJETgwPDyfZrO0l2V3dp26MM+4L8q591Qn/+DIxHBXPbrNwx5IhhM0+WalUJsX3mloqXHSPzRKJjeOOeHMzLJVqwPgG8IaI/IGI/AHwOvC1JI/tAZrivm8E+lZ6sTHmJWCXiFTGPfx+4KQxJmE1HmPMV40xR4wxR6qqqpJs1vYx4p2jfyLZuYto7yJRKnGX3bpiacn12FHppia2hLCuxKlLaZWKs3QD32wwvBAocrVSKqWAYYz5U+AzwBgwDnwmhZVKx4E2EdkR6yl8DHgq/gUisju2oxwROUw0sWF8PuyPo8NRactU76KpPHurluaTE2rNC6UWqypanojwRiwRYa4CRsrpGo0xJ4GTaRwXEpHfAX5KdFnt140x50Xki7HnnwA+AnxSRILALPCYiSVNEREX0RVW/0uq762S710EQpFVexeQmeW0KykpLODuHeXU61JapRYpKSzAYbMwF4pQ6XFQWGDlxpiPI63lzMyFmQ2EKbRntxJlqqukvgn8rjFmIvZ9GfCfjDGJJqeXMcY8Czy75LEn4r7+MvDlFY71ASsX3VWrSmZlVMQYvneimwlfkA8fbkzYuyi0W7K+AmWnJhlUKqGqIgc947MLG/huLMlc22jPbgW+VOcwDs0HC1ioh3FHRlukMm7UO0dfEr2LH5/t50L/FL98qG7FzLCNZa4ttwJFqc0ifh6jqdzF0PQcs4HoOqRcDEulGjAssV4FACJSjtYFz3vJzF0c6xrl1c5R7t1VwX27Kld8XaZXRymlkhefbbclloiwezzay8hF5tpUP+z/E/CaiPwg9v2vAn+U2SapTEqmd9HeP8XTb/exv7aIX761bsXXOQssVGdhdZRSKjllrgIKrEIwbGgsK1zYwLenpigntTFSXSX1LaIT04Ox/z5kjPmbbDRMZca5NSrb9U7M8p3j3dSVOnnsrmYsqww36XCUUhtLRKiM3bQ5bFZqS5wL8xiBUIRpf+KcbJmSUsAQkV8Fuo0x/x0oA/4otvxV5aGxmQC947MrPj/hC/CtY9cotFv55L2ta+6q1uEopTZefC9/PnPt/Aa+bA9LpTqH8f8aY6ZF5AGiS1y/Cfx55pulMmG1uQt/MMy3jl0nEIrwqXtb16xx7LBZFjbVKaU2TtWSgDEXijA0Hd3Al+1hqVQDxnxakF8GnjDG/APRzXUqz6zWuwhHDN85foOhaT+/fncztSVrF6xpLCvU4Sil8kCl+2YZ2/lef3eONvClGjB6ReQvgF8DnhURRxrnUDmw0r4LYwz/+HYflwe9PHp7Q9L5mpordDhKqXxgsQgV7mgvo9xtx223cj02jzE+EyASSZTkO0PvneLrf43oTu33xfZjlAO/l+lGqfSEwhFmA2H6JmbpWaF38fKVEd68NsbDe6q4q7U8qfPabRZqijJbNlMplb75/RiyZANfKGKYnM3exHeqBZR83KyFgTGmH+jPdKO2O38wzLQ/RCAcIRiKEAxHmIv9PxCKEAybBI9FWOvG4mzvJD85P8CtDSW8+0DyJboaywqx5KjIvFJqbUvnMS4OTOObC+Fy2BidCVDmzs5MgW66yzPGGH5xaTjjY5E3xnx8/0Q3zeUuPnpn46rLZ5fKZu4opVTqqjwOLAIRA02x4eIb4z721RYz6p1jd3V20uvo/EOeuTQ4nfFgMTYT4G+OXaO4sIDH72mhwJr8ZS+wCnXFOhylVD6xWS0LvYjGUhcWYWFYKpsT3xow8oh3LsSZ7uRSkCdrNhDmm69dI2LgU/e24nGk1qlsLHPpcJRSeWh+WMpus1BXUrgQMKb9oay955oBQ0TeLSJ/KSK3x77/QtZas80dvzpGKIMrHEKRCP/zjeuM+QI8fk9LWkWPsln7QimVvvgNfE3lLnrGZglncYUUJNfD+C2iK6EeF5F3ArdntUXbVNewN6la28kyxvDkyV6ujszwkcMN7Kh0p3yOAqtQV6IBQ6l8FH8D2FLuIhCOMDiVuc+QRJIJGMPGmAljzP8JvAe4K6st2ob8wTAnb0xk9JzPXxriVPcEv7S/mtubytY+IIGG0kKsOhylVF5y2KyUuqIZGuY38MXXx8iGZALGM/NfGGO+BHwre83Znk5cGycQimTsfKdujPPcxSEON5fyyN7qtM+jq6OUym/zvYxSVwFFDtvGB4xY+o/47/9b9pqz/fSM+zJ6kS/2T/Gjk73srHTzwTsa0k7nYbOKlklVKs/Nz2OICM0Vro0PGLHG/IaIDItIj4h8MvbYPSLyH0Tkray2cAsLhCKcuDaesfNd6Jvi22/coK7UySeOtmCzpL8IToejlMp/8QWVmstdjM0E8G7kKqmY3wc+QHTCe6eI/Az4PtHEg/8qKy3bBt7umcAXCK/9wiSc653k229ep77UyWfv37HuYvBNZTocpVS+K7Rb8TijS+Xn5zGuj81k7f2SXZTvNcYcBxCRf0e0eNKe+PreKjVD036uDHozcq6zvZN89/gNGstcfPq+VpwF6wsWNotQX6qb9ZTaDKqLHHj9IepLC7GKcH00e8NSyfYwakXkCyLyMFAD9GiwSF84Ynjz6lhGznWmZ4LvHr9BU5mLz2QgWHicNm5tLMGWwm5wpdTGmZ/HKLBaqC91LmSuzYZkexj/FjgEfAK4FSgSkZ8Dp4BTxphvZ6l9W9K53kmmZtc/zni6e4Lvn+impcLNp+5rwWFLL1iUu+00lhXSWFZIqUvLmyi1mSxNRPjmtTGC4UhKKYCSlVTAMMZ8Nf57EWkkGkBuBd4PaMBI0oQvwMX+1etsJ+PUjXF+8FYPrZVuPpVEedV4FoGaYieNZYU0lBXismsOSqU2qyJnAS67FV8gTHOFm1c7R7nYP8WhxtKMv1danxTGmB6gB3g2s83Z2owxvN41tmYa8rWcvD7OD0/2sKPKzSfvSS5YFFiFhtJogKgrKUwpwCil8ltVkYPro76Fie+T18fzJ2Co9LQPrD8T7YlrYzx5qpddVR4ev6dl1Q9+t8NKQ2khjWUuqoscmkRQqS2qOhYwSgoLKCks4K0bE3z6/sy/jwaMHJn2Bznbs75MtMevjvHk6V7aqj2rpimvK3VyW2Mp5VkqoqKUyi/x+zHu31XBvTsrsvI+GjBy5Pi19WWifePqKP9wuo89NR4+cXTlYGERuKu1POU05kqpzavEVYDDZmEuFOEde6v5tbuasvI+OpCdA13DXgYm59I+/lhXNFjsrSni8VWCBcCuao8GC6W2oXTKF6RKA0aWrTcT7WudI/zj233sqy3iE0ebV90fYbMIB+tL0n4vpdTmlYuAobeiWbaeTLSvdozwzNl+DtQV87G7m9bMDdVW41l3ShCl1OZUrQFjc1tPJtqXrwzz43MD3FJfzMfual4zEWCBVThQX5zWeymlNr9ytx2bVSCLRfd0SCpL1pOJ9qXL0WBxsKEkqWABsL+uOO2d3kqpzU9EqPJkt5ehASNL0s1E2z4wxU/OD3BrQwmPHWlKKlg4bBb21hal00yl1BaS7XkMDRhZkG4m2tlAmCdP9VJT7OBX72xMuh7FgfrirOSNUUptLtmex9BPmQwLRwxvdKWXifaZs33MzIX46OGmpLPFuuxW9tRo70IpBRWe7GZ00ICRYWd6JphOo+JV+8AUJ29M8PCeKhrKki+NerChWCvjKaUAsFqECk/2MjxowMigUe8c7QPTKR8XPxT1yN7qpI/zOG3srPSk/H5Kqa0rmxPfGjAyJBIxvHF1DJPGkranz6Q+FAVwqKFEEwoqpRapLt4iAUNE3icil0SkQ0S+lOD5R0XkjIicFpETIvJA3HOlIvIDEWkXkYsicm8u276W831TTPiCKR/X3j/Fqe7Uh6JKXQW0VGjdbaXUYpXu7AWMnG3cExEr8BXg3URraRwXkaeMMRfiXvYc8JQxxojIIeB7wL7Yc/8V+Ikx5qMiYgfy5tNywhfgfF/qmWhnA2GePN1LbbGTR/YlPxQFcGtDCSLau1BKLbZVJr3vBjqMMV3GmADwHeDR+BcYY7zGLAzquIntWRSRYuAh4Gux1wXypab4eooizQ9FfeRw45ppP+JVeOw0ledNvFRKbRO5DBgNQHfc9z2xxxYRkQ+JSDvwDPDZ2MM7gWHgGyJySkT+SkTcCY79Qmwo68Tw8HDmf4IELvanVxTpYppDUQC3ZaGSllJKrSWXASNRP2nZfbkx5kljzD7gg8Afxh62AYeBPzfG3AHMAMvmQIwxXzXGHDHGHKmqqspYw1cy5Q9yrje9oai/T3MoqqbYQW2Jc+0XKqVUhuUyYPQA8VU9GoG+lV5sjHkJ2CUilbFje4wxb8Se/gHRALKh3uhKryhSukNRQFbq9CqlVDJyGTCOA20isiM2af0x4Kn4F4jIbonN5IrIYcAOjBpjBoBuEdkbe+m7gPjJ8py7MjjN8HTqRZHWMxRVX+rMSc57pZRKJGerpIwxIRH5HeCngBX4ujHmvIh8Mfb8E8BHgE+KSBCYBR6LmwT/l8DfxoJNF/CZXLV9qZm5EKe6J1I+bj1DUQC3N5WmfIxSSmVKTuthGGOeBZ5d8tgTcV9/GfjyCseeBo5ks33JevPqGKFw+kNRn7y3NeWhqJYKF6Wu7G35V0qptehO7xR1DXvpn/SnfNyioajS1IaiLAK3NmrpVaXUxtKAkYLZQHr1udc7FLWj0k2xsyDl45RSKpM0YKTg+LWxtOpzL6yKujP1VVFWCxxs0N6FUmrjacBI0o1RHz3jsykft56hKIDd1R7cDi29rpTaeBowkuAPhjlxPfWiSOsdirJZhVvqtXehlMoPGjCScPL6OP5gboeiAPbWFOEssKZ8nFJKZYMGjDX0TsxybdSX8nE3h6Kq0xqKstss7K8rTvk4pZTKFg0YqwiEIhy/ut6hqPRyWu2vK8Ju08ujlMof+om0ilM3xvEFwikft96hKGeBhb01RSkfp5RS2aQBYwWDU346h2dSPu5c7+S6hqIA7mguS6lUq1JK5YJ+KiUQCkd4I42hqAlfgB+d6qGxrJB3prEqCqC53MWOymWlPpRSasNpwEjg+pgPrz+U0jERY/jeiR4iBh470oQ1jTKJhXYLd+0oS/k4pZTKBQ0YCUTSqHHxi0tDXBud4Vduq6fCk14K8qM7KnDYdBmtUio/acDIgOujMzzfPsRtjSXckWYK8rYaD/VpznkopVQuaMBYJ38wzHdPdFNSWMCjtzcQq/+UkiKnLe1Ao5RSuaIBYx2MMfz96V6mZoM8dldzWruyReDeXRW6Kkoplff0U2odTt6Y4EzPJO/aX0NzuSutc9xSX0xlmnMeSimVSxow0jTineMf3+5jR6Wbh/ekt5u73F3AQU0uqJTaJDRgpCEUifDd491YLcKvHWnCksa8hdUC9+6qxJLG8lullNoIGjDS8LMLg/ROzPLhww2UFKZXCe/2prK0j1VKqY2gASNFV4amefnKCHe3lqddq6K2xMHeWs0VpZTaXDRgpMA7F+IHJ3qoLnLwgVvr0jpHgVU4uqMiwy1TSqns04CRJGMMP3yrh9lgmMfuako79fhdreVaclUptSlpwEjSsa5RLg1O876DtdSVpLcju7ncRasmFlRKbVIaMJLQPznLT84NsLemiHt3pjecpIkFlVKbnQaMNQRC0SW0hQVWPnJnY1qpP0ATCyqlNj8NGGt49lw/Q9NzfPRII5405x40saBSaivQgLGK832TvHl1jAd3V9JWnd4yWE0sqJTaKjRgrGByNsiPTvbSUFrIu2+pSescmlhQKbWV6CdZAuGI4XsnuglHDI/d1YTNkt6vSRMLKqW2Eg0YCXz3RDdXR2b457fVp/2Br4kFlVJbjQaMJc71TvLN165xa0MJh5tL0zqHzSKaWFApteXoluMl2mo8PH5PCy3l7rSX0B5uKdXEgkqpLUd7GEs4bFZ+454WCu3p7ZnYXe1hd5orqpRSKp9pwMigqiIHR1p0N7dSamvSgJEhLruVB9t03kIptXVpwMgAqwUebKvEWaCpP5RSW5cGjAy4q7WcCt1voZTa4nIaMETkfSJySUQ6RORLCZ5/VETOiMhpETkhIg/EPXdNRM7OP5fLdq9mb62HnVWejW6GUkplXc6W1YqIFfgK8G6gBzguIk8ZYy7Evew54CljjBGRQ8D3gH1xzz9ijBnJVZvXUlPs4I4mneRWSm0Puexh3A10GGO6jDEB4DvAo/EvMMZ4jTEm9q0bMOQpt8PK/bt1klsptX3kMmA0AN1x3/fEHltERD4kIu3AM8Bn454ywD+JyFsi8oWstnQNNovwUFuVTnIrpbaVXAaMRLfiy3oQxpgnjTH7gA8Cfxj31P3GmMPA+4HfFpGHlr2ByBdicx8nhoeHM9Ts5Y7uLKfMbc/a+ZVSKh/lMmD0AE1x3zcCfSu92BjzErBLRCpj3/fF/j8EPEl0iGvpMV81xhwxxhypqqrKZNsX7K8roqVC63IrpbafXAaM40CbiOwQETvwMeCp+BeIyG6JJXASkcOAHRgVEbeIFMUedwPvAc7lsO0A1JU4uV2LISmltqmcrZIyxoRE5HeAnwJW4OvGmPMi8sXY808AHwE+KSJBYBZ4LLZiqgZ4MhZLbMC3jTE/yVXbATxOG/ftrkg7IaFSSm12Oc1Wa4x5Fnh2yWNPxH39ZeDLCY7rAm7LegNXYLMKD7dV4bDpJLdSavvSnd5JuHdnBSUuTVeulNreNGCs4WBDMU3lro1uhlJKbTgNGKtoKCvkUGPpRjdDKaXyggaMFRQX2rhvV8VGN0MppfKGBowE7DYLD+2posCqvx6llJqnNb0T0I15Sim1nN5CK6WUSooGDKWUUknRgKGUUiopGjCUUkolRQOGUkqppGjAUEoplRQNGEoppZKiAUMppVRSNGAopZRKihizrKz2liAiw8D1BE+VAJNJPFYJjGShaWtJ1JZcnSfZY9Z63UrPp/J4Pl0TyMx1ycdrstpz+X5d8v1vZbNekxZjTOIa18aYbfUf8NUkHzuRL+3L1XmSPWat1630fCqP59M1ydR1ycdrspmvS77/rWzFa7Idh6T+McnHNkqm2pLOeZI9Zq3XrfR8Ko/n0zWBzLQnH6/Jas/l+3XJ97+VLXdNtuyQ1HqJyAljzJGNboe6Sa9JftLrkn+ydU22Yw8jWV/d6AaoZfSa5Ce9LvknK9dEexhKKaWSoj0MpZRSSdGAoZRSKikaMJRSSiVFA0YaRMQtIm+JyD/b6LaoKBHZLyJPiMgPROQ3N7o9CkTkgyLylyLyDyLyno1uj4oSkZ0i8jUR+UGqx26rgCEiXxeRIRE5t+Tx94nIJRHpEJEvJXGq/wv4XnZauf1k4roYYy4aY74I/BqgSzzXKUPX5O+NMZ8HPg08lsXmbhsZui5dxph/kdb7b6dVUiLyEOAFvmWMORh7zApcBt4N9ADHgY8DVuBPlpzis8AhotvuncCIMebp3LR+68rEdTHGDInIrwBfAv67MebbuWr/VpSpaxI77j8Bf2uMOZmj5m9ZGb4uPzDGfDSV97etr/mbizHmJRFpXfLw3UCHMaYLQES+AzxqjPkTYNmQk4g8AriBA8CsiDxrjIlkt+VbWyauS+w8TwFPicgzgAaMdcjQ34oA/xH4sQaLzMjU30q6tlXAWEED0B33fQ9wdKUXG2P+HwAR+TTRHoYGi+xI6bqIyDuADwMO4NlsNmwbS+maAP8S+CWgRER2G2OeyGbjtrFU/1YqgD8C7hCR/zsWWJKiAQMkwWNrjtMZY/46801RcVK6LsaYXwC/yFZjFJD6Nfkz4M+y1xwVk+p1GQW+mM4bbatJ7xX0AE1x3zcCfRvUFnWTXpf8o9ckP+XsumjAiE4QtYnIDhGxAx8DntrgNim9LvlIr0l+ytl12VYBQ0T+DjgG7BWRHhH5F8aYEPA7wE+Bi8D3jDHnN7Kd241el/yj1yQ/bfR12VbLapVSSqVvW/UwlFJKpU8DhlJKqaRowFBKKZUUDRhKKaWSogFDKaVUUjRgKKWUSooGDLXtiIg3x+/3Wo7fr1REfiuX76m2Bw0YSq2TiKyak80Yc1+O37MU0IChMk6TDyoFiMgu4CtAFeADPm+MaReRfw78G8AOjAKfMMYMisgfAPVAKzAiIpeBZmBn7P//JZZ8DxHxGmM8sYy6fwCMAAeBt4DHjTFGRD4A/GnsuZPATmPMotTUsQzJv0y0Fos7Vv/jH4AyoAD4N8aYfyCaUnyXiJwGfmaM+T0R+T2ixaUcwJPGmH+bud+e2i40YCgV9VXgi8aYKyJyFPgfwDuBV4B7Yh/qnwP+NfB/xI65E3jAGDMbCyD7gEeAIuCSiPy5MSa45H3uAG4hmhzuVeB+ETkB/AXwkDHmaiz9w0ruBQ4ZY8ZivYwPGWOmRKQSeF1EniJaROqgMeZ2gFh51DaidROEaM2Qh4wxL6X7y1LbkwYMte2JiAe4D/h+tOYPEL0Th2jmz++KSB3RXsbVuEOfMsbMxn3/jDFmDpgTkSGghmgm0XhvGmN6Yu97mmgPxQt0GWPmz/13wBdWaO7PjDFj800H/jhWhS1CtC5CTYJj3hP771Tsew/RAKIBQ6VEA4ZS0bm8ifk78iX+G/Cnxpin4oaU5s0see1c3NdhEv99JXpNonoGK4l/z08QHUK70xgTFJFrRIerlhLgT4wxf5HC+yi1jE56q23PGDMFXBWRX4VoaVERuS32dAnQG/v6U1lqQjuwM6705mNJHlcCDMWCxSNAS+zxaaLDYvN+Cnw21pNCRBpEpHr9zVbbjfYw1HbkEpH4oaI/JXq3/uci8m+ITiB/B3ibaI/i+yLSC7wO7Mh0Y2JzIL8F/ERERoA3kzz0b4F/jM2BnCYaeDDGjIrIqyJyjmg97d8Tkf3AsdiQmxd4HBjK8I+itjhNb65UHhARjzHGK9FP9K8AV4wx/3mj26VUPB2SUio/fD42CX6e6FCTzjeovKM9DKWUUknRHoZSSqmkaMBQSimVFA0YSimlkqIBQymlVFI0YCillEqKBgyllFJJ+f8BKpiekeHnFo8AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "lrs = [d[\"MODEL_TRAINING_DESCRIPTION\"][\"LEARNING_RATE\"] for d in descriptions_lrs]\n",
    "\n",
    "r2_lrs_mean = []\n",
    "r2_lrs_std = []\n",
    "\n",
    "ls = np.logspace(-4,-1,20)  # learning rate configurations - needs to be changed if configuration in experiment gets changed!\n",
    "for i, l in enumerate(ls):\n",
    "    indices = np.where(l==lrs)[0]\n",
    "    print('lr: {} R2 = {:.3f} +/- {:.3f}'.format(l, np.mean(r2_lrs[indices]), np.std(r2_lrs[indices])))\n",
    "    r2_lrs_mean.append(np.mean(r2_lrs[indices]))\n",
    "    r2_lrs_std.append(np.std(r2_lrs[indices]))\n",
    "    \n",
    "r2_lrs_mean = np.array(r2_lrs_mean)\n",
    "r2_lrs_std = np.array(r2_lrs_std)\n",
    "\n",
    "plt.plot(ls, r2_lrs_mean)\n",
    "plt.fill_between(ls, r2_lrs_mean-r2_lrs_std, r2_lrs_mean+r2_lrs_std, alpha=0.4)\n",
    "\n",
    "plt.xscale(\"log\")\n",
    "plt.xlabel(\"Learning rate\")\n",
    "plt.ylabel(r\"$R^2$ score\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4683b6a5",
   "metadata": {},
   "source": [
    "# Precipitation weighting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "a6e111b5",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10 matching runs found\n",
      "10 matching runs found\n",
      "No precip weighting: R2 = 0.377 \\pm 0.005\n",
      "Precip weighting: R2 = 0.416 \\pm 0.004\n"
     ]
    }
   ],
   "source": [
    "conditions_pw = {\n",
    "    \"DATASET_DESCRIPTION\": {\"PRECIP_WEIGHTING\" : True},\n",
    "    \"MODEL_TRAINING_DESCRIPTION\": {}\n",
    "}\n",
    "\n",
    "conditions_no_pw = {\n",
    "    \"DATASET_DESCRIPTION\": {\"PREDICTOR_VARIABLES\" : {\"tsurf\": [\"tsurf\"],\n",
    "                                                     \"prec\": [\"prec\"]},\n",
    "                            \"PRECIP_WEIGHTING\" : False,\n",
    "                            \"GRID_TYPE\": \"Flat\"},\n",
    "    \"MODEL_TRAINING_DESCRIPTION\": {\"MODEL_TYPE\": \"UNet_Flat\",\n",
    "                                  \"USE_COORD_CONV\": True,\n",
    "                                  \"USE_CYLINDRICAL_PADDING\": True,\n",
    "                                  \"LOSS\": \"Masked_AreaWeightedMSELoss\",\n",
    "                                  \"LEARNING_RATE\": 1e-3,\n",
    "                                  \"DEPTH\": 3,\n",
    "                                  \"FMAPS\": (32,32,64,64)\n",
    "                                  }\n",
    "}\n",
    "\n",
    "descriptions_pw, predictions_pw, gt_pw, masks_pw = load_data_for_comparison(base_folder, conditions_pw)\n",
    "descriptions_no_pw, predictions_no_pw, gt_no_pw, masks_no_pw = load_data_for_comparison(base_folder, conditions_no_pw, keywords_blacklist=keywords_blacklist)\n",
    "\n",
    "r2_pw = np.zeros(len(predictions_pw))\n",
    "r2_no_pw = np.zeros(len(predictions_no_pw))\n",
    "\n",
    "for i in range(len(predictions_pw)):\n",
    "    r2_pw[i] = get_weighted_average(get_r2(predictions_pw[i], gt_pw[i]), descriptions_pw[i][\"DATASET_DESCRIPTION\"])\n",
    "\n",
    "for i in range(len(predictions_no_pw)):\n",
    "    r2_no_pw[i] = get_weighted_average(get_r2(predictions_no_pw[i], gt_no_pw[i]), descriptions_no_pw[i][\"DATASET_DESCRIPTION\"])\n",
    "    \n",
    "print('No precip weighting: R2 = {:.3f} \\pm {:.3f}'.format(np.mean(r2_no_pw), np.std(r2_no_pw)))\n",
    "print('Precip weighting: R2 = {:.3f} \\pm {:.3f}'.format(np.mean(r2_pw), np.std(r2_pw)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1e006b78",
   "metadata": {},
   "source": [
    "### Interpolate groundtruth Flat->Ico->Flat\n",
    "\n",
    "To demonstrate that the comparatively bad performance of the icosahedral UNet on the flat grid is mostly due to interpolation artifacts, we do the following: We interpolate the predictions of a flat UNet network to the icosahedral grid and back to the flat grid."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "1fd7eac4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10 matching runs found\n"
     ]
    }
   ],
   "source": [
    "gt_c = {\n",
    "    \"DATASET_DESCRIPTION\": {\"PREDICTOR_VARIABLES\" : {\"tsurf\": [\"tsurf\"],\n",
    "                                                     \"prec\": [\"prec\"]},\n",
    "                            \"DATASETS_NO_GAPS\": ['isotopes', 'tsurf', 'prec'],\n",
    "                            \"GRID_TYPE\": \"Flat\",\n",
    "                            \"PRECIP_WEIGHTING\" : False},\n",
    "    \"MODEL_TRAINING_DESCRIPTION\": {\"MODEL_TYPE\": \"UNet_Flat\",\n",
    "                                  \"USE_COORD_CONV\": False,\n",
    "                                  \"USE_CYLINDRICAL_PADDING\": False,\n",
    "                                  \"LEARNING_RATE\": 5e-3,                                   \n",
    "                                  \"LOSS\": \"Masked_MSELoss\"}\n",
    "}\n",
    "descriptions_flat, predictions_flat, gt_flat, masks_flat = load_data_for_comparison(base_folder, gt_c, keywords_blacklist=[\"RESULTS_INTERPOLATED\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "1a0c3e7f",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "Output/Reproduce_new\\f58b0b708787c6906882857c7c3ada0cc8df3a3a2a798f0471e383386859\\predictions.gz\n",
      "1 matching runs found\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "Output/Reproduce_new\\f58b0b708787c6906882857c7c3ada2f1d2e4c5f4d251e3c041ae24d8712\\predictions.gz\n",
      "1 matching runs found\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "Output/Reproduce_new\\f58b0b708787c6906882857c7c3ada629092eb06efbfc80f4f095210d896\\predictions.gz\n",
      "1 matching runs found\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "Output/Reproduce_new\\f58b0b708787c6906882857c7c3ada6f13a68c321a47247432909ca13e42\\predictions.gz\n",
      "1 matching runs found\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "Output/Reproduce_new\\f58b0b708787c6906882857c7c3ada723b02705ae6c6b3abf404455966ee\\predictions.gz\n",
      "1 matching runs found\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "Output/Reproduce_new\\f58b0b708787c6906882857c7c3adab3070be62c3dc8fd8c96adfa39467f\\predictions.gz\n",
      "1 matching runs found\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "Output/Reproduce_new\\f58b0b708787c6906882857c7c3adabc3e063a85bda51497e2e88264709b\\predictions.gz\n",
      "1 matching runs found\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "Output/Reproduce_new\\f58b0b708787c6906882857c7c3adabc4eea002f3fb30fcdbc7327cc7306\\predictions.gz\n",
      "1 matching runs found\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "Output/Reproduce_new\\f58b0b708787c6906882857c7c3adae43f5eb16698845429d3490a534d8d\\predictions.gz\n",
      "1 matching runs found\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "Output/Reproduce_new\\f58b0b708787c6906882857c7c3adaf66d20ef369a881bba3c10743fe76a\\predictions.gz\n",
      "1 matching runs found\n"
     ]
    }
   ],
   "source": [
    "data_l = []\n",
    "gt_l = []\n",
    "\n",
    "for i in range(len(descriptions_flat)):\n",
    "    data, gt = get_interpolated_data_and_gt(descriptions_flat[i], gt_flat[i], output_folder=base_folder, \n",
    "                                            script_folder=\"Scripts/\", do_scaling=False, print_folder_names=True)\n",
    "    data_l.append(data)\n",
    "    gt_l.append(gt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "aba0520e",
   "metadata": {},
   "outputs": [],
   "source": [
    "gt_c = {\n",
    "    \"DATASET_DESCRIPTION\": {\"RESULTS_INTERPOLATED\": True,\n",
    "                            \"RESULTS_RESCALED\": False,\n",
    "                            \"GRID_TYPE\": \"Ico\"},\n",
    "    \"MODEL_TRAINING_DESCRIPTION\": {\"MODEL_TYPE\": \"UNet_Flat\",\n",
    "                                    \"USE_CYLINDRICAL_PADDING\": True\n",
    "                                  }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "86abf35c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10 matching runs found\n"
     ]
    }
   ],
   "source": [
    "descriptions_interp, predictions_interp, gt_interp, masks_interp = load_data_for_comparison(base_folder, gt_c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "9b0091f0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "1 matching runs found\n",
      "When interpolating to model grid, currently the iHadCM3 specifics are used.\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "1 matching runs found\n",
      "When interpolating to model grid, currently the iHadCM3 specifics are used.\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "1 matching runs found\n",
      "When interpolating to model grid, currently the iHadCM3 specifics are used.\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "1 matching runs found\n",
      "When interpolating to model grid, currently the iHadCM3 specifics are used.\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "1 matching runs found\n",
      "When interpolating to model grid, currently the iHadCM3 specifics are used.\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "1 matching runs found\n",
      "When interpolating to model grid, currently the iHadCM3 specifics are used.\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "1 matching runs found\n",
      "When interpolating to model grid, currently the iHadCM3 specifics are used.\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "1 matching runs found\n",
      "When interpolating to model grid, currently the iHadCM3 specifics are used.\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "1 matching runs found\n",
      "When interpolating to model grid, currently the iHadCM3 specifics are used.\n",
      "OSError: Probably CDO is not installed. Test if we have exiting interpolated results.\n",
      "1 matching runs found\n",
      "When interpolating to model grid, currently the iHadCM3 specifics are used.\n"
     ]
    }
   ],
   "source": [
    "# do second interpolation:\n",
    "\n",
    "data_r = []\n",
    "gt_r = []\n",
    "\n",
    "for i in range(len(descriptions_interp)):\n",
    "    data, gt = get_interpolated_data_and_gt(descriptions_interp[i], predictions_interp[i], output_folder=base_folder, \n",
    "                                            script_folder=\"Scripts/\", do_scaling=False)\n",
    "    data_r.append(data)\n",
    "    gt_r.append(gt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "cfba06c1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Twice interpolated: R2 = 0.130 +/- 0.011\n"
     ]
    }
   ],
   "source": [
    "r2_r = np.zeros(len(data_r))\n",
    "\n",
    "for i in range(len(data_r)):\n",
    "    r2_r[i] = get_weighted_average(get_r2(data_r[i], gt_r[i]), descriptions_interp[i][\"DATASET_DESCRIPTION\"])  # need a dataset description of a flat grid.\n",
    "\n",
    "    \n",
    "print('Twice interpolated: R2 = {:.3f} +/- {:.3f}'.format(np.mean(r2_r), np.std(r2_r)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  },
  "vscode": {
   "interpreter": {
    "hash": "90b409070de10f2c3c5a8c504ecede612c695480a2027d21529fbe1a33a534d8"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}