project/01_2_performance_plots.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "188fc02b-2e9b-4b72-b906-77d35ba50dab",
   "metadata": {},
   "source": [
    "# Compare models\n",
    "\n",
    "1. Load available configurations\n",
    "2. Load validation predictions\n",
    "    - calculate absolute error\n",
    "    - select top N for plotting by MAE from smallest (best) to largest (worst) (top N as specified, default 5)\n",
    "    - correlation per sample, correlation per feat, correlation overall\n",
    "    - MAE plots\n",
    "3. Load test data predictions\n",
    "    - as for validation data\n",
    "    - top N based on validation data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a1e5f978-a0cb-4bb6-98d1-467eda257165",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "import logging\n",
    "import random\n",
    "from pathlib import Path\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import yaml\n",
    "from IPython.display import display\n",
    "\n",
    "import pimmslearn\n",
    "import pimmslearn.imputation\n",
    "import pimmslearn.models\n",
    "import pimmslearn.nb\n",
    "from pimmslearn.analyzers import compare_predictions\n",
    "from pimmslearn.io import datasplits\n",
    "from pimmslearn.models.collect_dumps import collect, select_content\n",
    "\n",
    "pd.options.display.max_rows = 30\n",
    "pd.options.display.min_rows = 10\n",
    "pd.options.display.max_colwidth = 100\n",
    "\n",
    "plt.rcParams.update({'figure.figsize': (4, 2)})\n",
    "pimmslearn.plotting.make_large_descriptors(7)\n",
    "\n",
    "logger = pimmslearn.logging.setup_nb_logger()\n",
    "logging.getLogger('fontTools').setLevel(logging.WARNING)\n",
    "\n",
    "\n",
    "def load_config_file(fname: Path, first_split='config_') -> dict:\n",
    "    with open(fname) as f:\n",
    "        loaded = yaml.safe_load(f)\n",
    "    key = f\"{select_content(fname.stem, first_split=first_split)}\"\n",
    "    return key, loaded\n",
    "\n",
    "\n",
    "def build_text(s):\n",
    "    ret = ''\n",
    "    if not np.isnan(s[\"latent_dim\"]):\n",
    "        ret += f'LD: {int(s[\"latent_dim\"])} '\n",
    "    try:\n",
    "        if len(s[\"hidden_layers\"]):\n",
    "            t = \",\".join(str(x) for x in s[\"hidden_layers\"])\n",
    "            ret += f\"HL: {t}\"\n",
    "    except TypeError:\n",
    "        # nan\n",
    "        pass\n",
    "    return ret"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "67f5161a",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "# catch passed parameters\n",
    "args = None\n",
    "args = dict(globals()).keys()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c962bc4",
   "metadata": {},
   "source": [
    "Papermill script parameters:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e6e91c6b-20d6-402c-9577-a2bfd8ba592e",
   "metadata": {
    "lines_to_next_cell": 2,
    "tags": [
     "parameters"
    ]
   },
   "outputs": [],
   "source": [
    "# files and folders\n",
    "# Datasplit folder with data for experiment\n",
    "folder_experiment: str = 'runs/example'\n",
    "folder_data: str = ''  # specify data directory if needed\n",
    "file_format: str = 'csv'  # change default to pickled files\n",
    "# Machine parsed metadata from rawfile workflow\n",
    "fn_rawfile_metadata: str = 'data/dev_datasets/HeLa_6070/files_selected_metadata_N50.csv'\n",
    "models: str = 'Median,CF,DAE,VAE'  # picked models to compare (comma separated)\n",
    "sel_models: str = ''  # user defined comparison (comma separated)\n",
    "# Restrict plotting to top N methods for imputation based on error of validation data, maximum 10\n",
    "plot_to_n: int = 5\n",
    "feat_name_display: str = None  # display name for feature name in plural (e.g. 'protein groups')\n",
    "save_agg_pred: bool = False  # save aggregated predictions of validation and test data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bc38df73",
   "metadata": {},
   "source": [
    "Some argument transformations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ec1509e8-6908-43c3-8909-efbb0229c324",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "args = pimmslearn.nb.get_params(args, globals=globals())\n",
    "args"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "19b33594",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "args = pimmslearn.nb.args_from_dict(args)\n",
    "args"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "59081f60",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "figures = {}\n",
    "dumps = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c3e124fb",
   "metadata": {
    "lines_to_next_cell": 2,
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "TARGET_COL = 'observed'\n",
    "METRIC = 'MAE'\n",
    "MIN_FREQ = None\n",
    "MODELS_PASSED = args.models.split(',')\n",
    "MODELS = MODELS_PASSED.copy()\n",
    "FEAT_NAME_DISPLAY = args.feat_name_display\n",
    "SEL_MODELS = None\n",
    "if args.sel_models:\n",
    "    SEL_MODELS = args.sel_models.split(',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "747d5e4a",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "# list(sns.color_palette().as_hex()) # string representation of colors\n",
    "if args.plot_to_n > 10:\n",
    "    logger.warning(\"Set maximum of models to 10 (maximum)\")\n",
    "    args.overwrite_entry('plot_to_n', 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4ba2a48-dedc-47a9-b2ea-79936dfc48ef",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "data = datasplits.DataSplits.from_folder(\n",
    "    args.data, file_format=args.file_format)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "611a8edf",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(1, 2, sharey=True, sharex=True)\n",
    "\n",
    "pimmslearn.plotting.data.plot_observations(data.val_y.unstack(), ax=axes[0],\n",
    "                                     title='Validation split', size=1, xlabel='')\n",
    "pimmslearn.plotting.data.plot_observations(data.test_y.unstack(), ax=axes[1],\n",
    "                                     title='Test split', size=1, xlabel='')\n",
    "fig.suptitle(\"Simulated missing values per sample\", size=8)\n",
    "# hide axis and use only for common x label\n",
    "fig.add_subplot(111, frameon=False)\n",
    "plt.tick_params(labelcolor='none', which='both', top=False, bottom=False, left=False, right=False)\n",
    "plt.xlabel(f'Samples ordered by identified {data.val_y.index.names[-1]}')\n",
    "group = 1\n",
    "fname = args.out_figures / f'2_{group}_fake_na_val_test_splits.png'\n",
    "figures[fname.stem] = fname\n",
    "pimmslearn.savefig(fig, name=fname)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ffc6d140-f48e-4477-84f3-47a196e0a3d8",
   "metadata": {},
   "source": [
    "## data completeness across entire data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2d043b40-5c74-40cc-a5cf-8d22ac5538a8",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "# load frequency of training features...\n",
    "# needs to be pickle -> index.name needed\n",
    "freq_feat = pimmslearn.io.datasplits.load_freq(args.data, file='freq_features.json')\n",
    "freq_feat.head()  # training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d8f8c3f4-9896-4f0e-8f93-780f90b22573",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "prop = freq_feat / len(data.train_X.index.levels[0])\n",
    "prop.sort_values().to_frame().plot(\n",
    "    xlabel=f'{data.val_y.index.names[-1]}',\n",
    "    ylabel='Proportion of identification in samples')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "19e5adfb",
   "metadata": {},
   "source": [
    "View training data in wide format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9a94ad00-78fd-4541-be5d-68391af99bd5",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "data.to_wide_format()\n",
    "data.train_X"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "21102a1d",
   "metadata": {},
   "source": [
    "Number of samples and features:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "526626c0-98c7-4741-abae-b6fc8c218f23",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "N_SAMPLES, M_FEAT = data.train_X.shape\n",
    "print(f\"N samples: {N_SAMPLES:,d}, M features: {M_FEAT}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "61186a4e",
   "metadata": {},
   "source": [
    "Collect outputs in excel file:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f3e738bd-79e9-4714-af4d-f3d0d2893353",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "fname = args.folder_experiment / '01_2_performance_summary.xlsx'\n",
    "dumps[fname.stem] = fname\n",
    "writer = pd.ExcelWriter(fname)\n",
    "print(f\"Saving to: {fname}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bbe028c4-190d-4d50-b8a7-d109817d7b98",
   "metadata": {},
   "source": [
    "## Model specifications\n",
    "- used for bar plot annotations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "91bc1e12-8477-4eda-a4c2-1f132e468616",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "# model_key could be used as key from config file\n",
    "# ? load only specified configs?\n",
    "# ? case: no config file available?\n",
    "all_configs = collect(\n",
    "    paths=(fname for fname in args.out_models.iterdir()\n",
    "           if fname.suffix == '.yaml'\n",
    "           and 'model_config' in fname.name),\n",
    "    load_fn=load_config_file\n",
    ")\n",
    "model_configs = pd.DataFrame(all_configs).set_index('id')\n",
    "model_configs.T.to_excel(writer, sheet_name='model_params')\n",
    "model_configs.T"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "40a64b82-a99e-48c9-b5bf-591f48d19fe8",
   "metadata": {},
   "source": [
    "Set Feature name (columns are features, rows are samples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "af8c112f-fb4f-4dcd-b729-9c9558715d88",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "# index name\n",
    "freq_feat.index.name = data.train_X.columns.name\n",
    "# sample index name\n",
    "sample_index_name = data.train_X.index.name"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e4edc203-3fac-4321-8193-a5cc39590665",
   "metadata": {},
   "source": [
    "# Load predictions on validation and test data split\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "808110cf",
   "metadata": {},
   "source": [
    "## Validation data\n",
    "- set top N models to plot based on validation data split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4efc3fe6",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "pred_val = compare_predictions.load_split_prediction_by_modelkey(\n",
    "    experiment_folder=args.folder_experiment,\n",
    "    split='val',\n",
    "    model_keys=MODELS_PASSED,\n",
    "    shared_columns=[TARGET_COL])\n",
    "SAMPLE_ID, FEAT_NAME = pred_val.index.names\n",
    "if not FEAT_NAME_DISPLAY:\n",
    "    FEAT_NAME_DISPLAY = FEAT_NAME\n",
    "pred_val[MODELS]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ad1e732c-235f-4fd9-95cc-a64a2ec09f6c",
   "metadata": {},
   "source": [
    "Describe absolute error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d5196bcc",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "errors_val = (pred_val\n",
    "              .drop(TARGET_COL, axis=1)\n",
    "              .sub(pred_val[TARGET_COL], axis=0)\n",
    "              [MODELS])\n",
    "errors_val  # over all samples and all features"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b11fc897",
   "metadata": {
    "lines_to_next_cell": 0
   },
   "source": [
    "### Select top N for plotting and set colors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e94d9dd6-d97d-4e1c-b877-48dc1ae9c7c7",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "ORDER_MODELS = (errors_val\n",
    "                .abs()\n",
    "                .mean()\n",
    "                .sort_values()\n",
    "                .index\n",
    "                .to_list())\n",
    "ORDER_MODELS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f5865679",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "pred_val = pred_val[[TARGET_COL] + ORDER_MODELS]\n",
    "if args.save_agg_pred:\n",
    "    fname = args.folder_experiment / '01_2_agg_pred_val.csv'\n",
    "    dumps[fname.stem] = fname\n",
    "    pred_val.to_csv(fname)\n",
    "    logger.info(f\"Saved aggregated predictions to: {fname}\")\n",
    "pred_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4d6417fc",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "mae_stats_ordered_val = errors_val.abs().describe()[ORDER_MODELS]\n",
    "mae_stats_ordered_val.to_excel(writer, sheet_name='mae_stats_ordered_val', float_format='%.5f')\n",
    "mae_stats_ordered_val.T"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f5b33f93",
   "metadata": {},
   "source": [
    "Some model have fixed colors, others are assigned randomly\n",
    "\n",
    "> Note\n",
    ">\n",
    "> 1. The order of \"new\" models is important for the color assignment.\n",
    "> 2. User defined model keys for the same model with two configuration will yield different colors."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "36e078fb-2268-41dd-a069-4ca3dc5ca6cf",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "COLORS_TO_USE = pimmslearn.plotting.defaults.assign_colors(list(k.upper() for k in ORDER_MODELS))\n",
    "pimmslearn.plotting.defaults.ModelColorVisualizer(ORDER_MODELS, COLORS_TO_USE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2440887-b5f2-45a1-90cd-d15ef9bfa0a7",
   "metadata": {
    "lines_to_next_cell": 2,
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "TOP_N_ORDER = ORDER_MODELS[:args.plot_to_n]\n",
    "TOP_N_COLOR_PALETTE = {model: color for model,\n",
    "                       color in zip(TOP_N_ORDER, COLORS_TO_USE)}\n",
    "TOP_N_ORDER"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0ac5f058-c580-4676-83c8-768bdb30f526",
   "metadata": {},
   "source": [
    "### Correlation per sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cea24eb1",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "corr_per_sample_val = (pred_val\n",
    "                       .groupby(sample_index_name)\n",
    "                       .apply(\n",
    "                           lambda df: df.corr().loc[TARGET_COL]\n",
    "                       )[ORDER_MODELS])\n",
    "\n",
    "min_corr = int(corr_per_sample_val.min().min() * 10) / 10\n",
    "kwargs = dict(ylim=(min_corr, 1), rot=90,\n",
    "              #     boxprops=dict(linewidth=1.5),\n",
    "              flierprops=dict(markersize=3),\n",
    "              # title='Corr. betw. fake NA and model pred. per sample on validation data',\n",
    "              ylabel='correlation per sample')\n",
    "ax = corr_per_sample_val[TOP_N_ORDER].plot.box(**kwargs)\n",
    "ax.set_xticklabels(ax.get_xticklabels(), rotation=45,\n",
    "                   horizontalalignment='right')\n",
    "fname = args.out_figures / f'2_{group}_pred_corr_val_per_sample.pdf'\n",
    "figures[fname.stem] = fname\n",
    "pimmslearn.savefig(ax.get_figure(), name=fname)\n",
    "\n",
    "fname = args.out_figures / f'2_{group}_pred_corr_val_per_sample.xlsx'\n",
    "dumps[fname.stem] = fname\n",
    "with pd.ExcelWriter(fname) as w:\n",
    "    corr_per_sample_val.describe().to_excel(w, sheet_name='summary')\n",
    "    corr_per_sample_val.to_excel(w, sheet_name='correlations')\n",
    "    corr_per_sample_val[TOP_N_ORDER].to_excel(w, sheet_name='correlations_plotted')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6682b3e4-7841-4348-8be2-b63bfd371f65",
   "metadata": {},
   "source": [
    "identify samples which are below lower whisker for models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4068d91f-856e-4aa6-9c62-5f1f77a77c4c",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "treshold = pimmslearn.pandas.get_lower_whiskers(\n",
    "    corr_per_sample_val[TOP_N_ORDER]).min()\n",
    "mask = (corr_per_sample_val[TOP_N_ORDER] < treshold).any(axis=1)\n",
    "corr_per_sample_val.loc[mask].style.highlight_min(\n",
    "    axis=1) if mask.sum() else 'Nothing to display'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d5575e14-3b49-4821-90e2-17ad0bc2acb1",
   "metadata": {},
   "source": [
    "### Error plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "52298acd-73c5-4574-b7fe-8fb6544708cf",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "c_error_min = 4.5\n",
    "mask = (errors_val[MODELS].abs() > c_error_min).any(axis=1)\n",
    "errors_val.loc[mask].sort_index(level=1).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "570fc505-ab27-4710-b4c2-adbe72b33898",
   "metadata": {
    "lines_to_next_cell": 2,
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "errors_val = errors_val.abs().groupby(\n",
    "    freq_feat.index.name).mean()  # absolute error\n",
    "errors_val = errors_val.join(freq_feat)\n",
    "errors_val = errors_val.sort_values(by=freq_feat.name, ascending=True)\n",
    "errors_val.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ddc98a9f",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "errors_val.describe()[ORDER_MODELS].T  # mean of means"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "af4f0e81-e9af-4763-908d-f7bdf4a4fed7",
   "metadata": {
    "lines_to_next_cell": 2,
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "c_avg_error = 2\n",
    "mask = (errors_val[TOP_N_ORDER] >= c_avg_error).any(axis=1)\n",
    "errors_val.loc[mask]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6696849c-15ee-47ba-b1b8-cbf2312d66bb",
   "metadata": {},
   "source": [
    "### Error by non-decimal number of intensity\n",
    "- number of observations in parentheses."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "df6923c5-e6f7-4a14-aa8e-d55bf66cf817",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(8, 3))\n",
    "ax, errors_binned = pimmslearn.plotting.errors.plot_errors_by_median(\n",
    "    pred_val[\n",
    "        [TARGET_COL] + TOP_N_ORDER\n",
    "    ],\n",
    "    feat_medians=data.train_X.median(),\n",
    "    ax=ax,\n",
    "    feat_name=FEAT_NAME_DISPLAY,\n",
    "    palette=TOP_N_COLOR_PALETTE,\n",
    "    metric_name=METRIC,)\n",
    "ax.set_ylabel(f\"Average error ({METRIC})\")\n",
    "ax.legend(loc='best', ncols=len(TOP_N_ORDER))\n",
    "fname = args.out_figures / f'2_{group}_errors_binned_by_feat_median_val.pdf'\n",
    "figures[fname.stem] = fname\n",
    "pimmslearn.savefig(ax.get_figure(), name=fname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6f6ffdd5",
   "metadata": {
    "lines_to_next_cell": 2,
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "# ! only used for reporting\n",
    "plotted = pimmslearn.plotting.errors.get_data_for_errors_by_median(\n",
    "    errors=errors_binned,\n",
    "    feat_name=FEAT_NAME_DISPLAY,\n",
    "    metric_name=METRIC\n",
    ")\n",
    "plotted.to_excel(fname.with_suffix('.xlsx'), index=False)\n",
    "plotted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6122a309-5435-44d2-a6f8-8e9d46b5afae",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "errors_binned.head()\n",
    "dumps[fname.stem] = fname.with_suffix('.csv')\n",
    "errors_binned.to_csv(fname.with_suffix('.csv'))\n",
    "errors_binned.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "15313cc9-5f1c-4c25-a22b-6c93b2e4364f",
   "metadata": {},
   "source": [
    "## test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1dc848c6-d39e-4092-9b72-3f6a0e1949e2",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "pred_test = compare_predictions.load_split_prediction_by_modelkey(\n",
    "    experiment_folder=args.folder_experiment,\n",
    "    split='test',\n",
    "    model_keys=MODELS_PASSED,\n",
    "    shared_columns=[TARGET_COL])\n",
    "pred_test = pred_test[[TARGET_COL] + ORDER_MODELS]\n",
    "pred_test = pred_test.join(freq_feat, on=freq_feat.index.name)\n",
    "if args.save_agg_pred:\n",
    "    fname = args.folder_experiment / '01_2_agg_pred_test.csv'\n",
    "    dumps[fname.stem] = fname\n",
    "    pred_test.to_csv(fname)\n",
    "    logger.info(f\"Saved aggregated predictions to: {fname}\")\n",
    "pred_test"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "38d02407",
   "metadata": {},
   "source": [
    "Write averages for all models to excel (from before?)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8bce941c",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "errors_test_mae = pimmslearn.pandas.calc_errors.get_absolute_error(\n",
    "    pred_test\n",
    ")\n",
    "mae_stats_ordered_test = errors_test_mae.describe()[ORDER_MODELS]\n",
    "mae_stats_ordered_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff722dae",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "mae_stats_ordered_test.to_excel(writer, sheet_name='mae_stats_ordered_test', float_format='%.5f')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "629eddae",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "cp_mean_perf = pd.concat([\n",
    "    mae_stats_ordered_val.loc['mean'],\n",
    "    mae_stats_ordered_test.loc['mean'],\n",
    "],\n",
    "    axis=1,\n",
    "    keys=['val', 'test']\n",
    ").sort_values(by='val')\n",
    "cp_mean_perf.to_excel(writer, sheet_name='cp_mean_perf', float_format='%.5f')\n",
    "cp_mean_perf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f639cd92",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "writer.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7921c819",
   "metadata": {
    "lines_to_next_cell": 0
   },
   "source": [
    "### Intensity distribution as histogram\n",
    "Plot top 4 models predictions for intensities in test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "99f7951f",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "min_max = pimmslearn.plotting.data.min_max(pred_test[TARGET_COL])\n",
    "top_n = 4\n",
    "fig, axes = plt.subplots(ncols=top_n, figsize=(8, 2), sharey=True)\n",
    "\n",
    "for model, color, ax in zip(\n",
    "        ORDER_MODELS[:top_n],\n",
    "        COLORS_TO_USE[:top_n],\n",
    "        axes):\n",
    "\n",
    "    ax, bins = pimmslearn.plotting.data.plot_histogram_intensities(\n",
    "        pred_test[TARGET_COL],\n",
    "        color='grey',\n",
    "        min_max=min_max,\n",
    "        ax=ax\n",
    "    )\n",
    "    ax, _ = pimmslearn.plotting.data.plot_histogram_intensities(\n",
    "        pred_test[model],\n",
    "        color=color,\n",
    "        min_max=min_max,\n",
    "        ax=ax,\n",
    "        alpha=0.5,\n",
    "    )\n",
    "    _ = [(l_.set_rotation(90))\n",
    "         for l_ in ax.get_xticklabels()]\n",
    "    ax.legend()\n",
    "\n",
    "axes[0].set_ylabel('Number of observations')\n",
    "\n",
    "fname = args.out_figures / f'2_{group}_intensity_binned_top_{top_n}_models_test.pdf'\n",
    "figures[fname.stem] = fname\n",
    "pimmslearn.savefig(fig, name=fname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "843a917f",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "counts_per_bin = pimmslearn.pandas.get_counts_per_bin(df=pred_test,\n",
    "                                                bins=bins,\n",
    "                                                columns=[TARGET_COL, *ORDER_MODELS[:top_n]])\n",
    "\n",
    "counts_per_bin.to_excel(fname.with_suffix('.xlsx'))\n",
    "counts_per_bin"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a1dc6066-441d-40c9-be3b-45529a6fd41e",
   "metadata": {},
   "source": [
    "### Correlation per sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee088a12-ee60-45d1-bf5a-e07b76413c56",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "corr_per_sample_test = (pred_test\n",
    "                        .groupby(sample_index_name)\n",
    "                        .apply(lambda df: df.corr().loc[TARGET_COL])\n",
    "                        [ORDER_MODELS])\n",
    "corr_per_sample_test = corr_per_sample_test.join(\n",
    "    pred_test\n",
    "    .groupby(sample_index_name)[TARGET_COL]\n",
    "    .count()\n",
    "    .rename('n_obs')\n",
    ")\n",
    "too_few_obs = corr_per_sample_test['n_obs'] < 3\n",
    "corr_per_sample_test.loc[~too_few_obs].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "825efac2",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "# ! add minimum\n",
    "kwargs = dict(ylim=(0.7, 1), rot=90,\n",
    "              flierprops=dict(markersize=3),\n",
    "              # title='Corr. betw. fake NA and model predictions per sample on test data',\n",
    "              ylabel='correlation per sample')\n",
    "ax = (corr_per_sample_test\n",
    "      .loc[~too_few_obs, TOP_N_ORDER]\n",
    "      .plot\n",
    "      .box(**kwargs))\n",
    "ax.set_xticklabels(ax.get_xticklabels(), rotation=45,\n",
    "                   horizontalalignment='right')\n",
    "fname = args.out_figures / f'2_{group}_pred_corr_test_per_sample.pdf'\n",
    "figures[fname.stem] = fname\n",
    "pimmslearn.savefig(ax.get_figure(), name=fname)\n",
    "\n",
    "dumps[fname.stem] = fname.with_suffix('.xlsx')\n",
    "with pd.ExcelWriter(fname.with_suffix('.xlsx')) as w:\n",
    "    corr_per_sample_test.describe().to_excel(w, sheet_name='summary')\n",
    "    corr_per_sample_test.to_excel(w, sheet_name='correlations')\n",
    "    corr_per_sample_test.loc[~too_few_obs, TOP_N_ORDER].to_excel(w, sheet_name='correlations_plotted')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "742ed838-c46c-431e-a3ae-a05c8285a7a7",
   "metadata": {},
   "source": [
    "identify samples which are below lower whisker for models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77b846e1-00b8-4f61-b5cd-cdc1692787de",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "treshold = pimmslearn.pandas.get_lower_whiskers(\n",
    "    corr_per_sample_test[TOP_N_ORDER]).min()\n",
    "mask = (corr_per_sample_test[TOP_N_ORDER] < treshold).any(axis=1)\n",
    "corr_per_sample_test.loc[mask].style.highlight_min(\n",
    "    axis=1) if mask.sum() else 'Nothing to display'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7bff3764-5063-4399-a182-3ba795fbe99d",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "feature_names = pred_test.index.levels[-1]\n",
    "N_SAMPLES = pred_test.index\n",
    "M = len(feature_names)\n",
    "pred_test.loc[pd.IndexSlice[:, feature_names[random.randint(0, M - 1)]], :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c6145bd0-9b59-490e-9a0e-89475c18663b",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "options = random.sample(sorted(set(feature_names)), 1)\n",
    "pred_test.loc[pd.IndexSlice[:, options[0]], :]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "53038e08-e7df-4758-bff0-1e1b6e61cdd2",
   "metadata": {},
   "source": [
    "### Correlation per feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6ee92128-4f78-45e9-a607-8e6c4163181a",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "corr_per_feat_test = pred_test.groupby(FEAT_NAME).apply(\n",
    "    lambda df: df.corr().loc[TARGET_COL])[ORDER_MODELS]\n",
    "corr_per_feat_test = corr_per_feat_test.join(pred_test.groupby(FEAT_NAME)[\n",
    "    TARGET_COL].count().rename('n_obs'))\n",
    "\n",
    "too_few_obs = corr_per_feat_test['n_obs'] < 3\n",
    "corr_per_feat_test.loc[~too_few_obs].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8e45b324-eaa0-43e4-b28b-b0f839f91955",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "corr_per_feat_test.loc[too_few_obs].dropna(thresh=3, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4c9a9ecc-526a-41ac-8a4d-d3a389ea6c07",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "kwargs = dict(rot=90,\n",
    "              flierprops=dict(markersize=1),\n",
    "              ylabel=f'correlation per {FEAT_NAME_DISPLAY}')\n",
    "ax = (corr_per_feat_test\n",
    "      .loc[~too_few_obs, TOP_N_ORDER]\n",
    "      .plot\n",
    "      .box(**kwargs)\n",
    "      )\n",
    "_ = ax.set_xticklabels(ax.get_xticklabels(), rotation=45,\n",
    "                       horizontalalignment='right')\n",
    "fname = args.out_figures / f'2_{group}_pred_corr_test_per_feat.pdf'\n",
    "figures[fname.stem] = fname\n",
    "pimmslearn.savefig(ax.get_figure(), name=fname)\n",
    "dumps[fname.stem] = fname.with_suffix('.xlsx')\n",
    "with pd.ExcelWriter(fname.with_suffix('.xlsx')) as w:\n",
    "    corr_per_feat_test.loc[~too_few_obs].describe().to_excel(\n",
    "        w, sheet_name='summary')\n",
    "    corr_per_feat_test.to_excel(w, sheet_name='correlations')\n",
    "    corr_per_feat_test.loc[~too_few_obs, TOP_N_ORDER].to_excel(w, sheet_name='correlations_plotted')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b38ffdfc-b1b0-4ae0-a47d-5881c534881f",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "feat_count_test = data.test_y.stack().groupby(FEAT_NAME).count()\n",
    "feat_count_test.name = 'count'\n",
    "feat_count_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9993d145-8b78-4769-838a-01721900a3c7",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "treshold = pimmslearn.pandas.get_lower_whiskers(\n",
    "    corr_per_feat_test[TOP_N_ORDER]).min()\n",
    "mask = (corr_per_feat_test[TOP_N_ORDER] < treshold).any(axis=1)\n",
    "\n",
    "\n",
    "def highlight_min(s, color, tolerence=0.00001):\n",
    "    return np.where((s - s.min()).abs() < tolerence, f\"background-color: {color};\", None)\n",
    "\n",
    "\n",
    "view = (corr_per_feat_test\n",
    "        .join(feat_count_test)\n",
    "        .loc[mask]\n",
    "        .sort_values('count'))\n",
    "\n",
    "if not view.empty:\n",
    "    display(view\n",
    "            .style.\n",
    "            apply(highlight_min, color='yellow', axis=1,\n",
    "                  subset=corr_per_feat_test.columns)\n",
    "            )\n",
    "else:\n",
    "    print(\"None found\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7cbb1b61-0499-4c29-90dd-ddf69a177a11",
   "metadata": {},
   "source": [
    "### Error plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "829ebc82-587d-47c6-8422-03c610855211",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "metrics = pimmslearn.models.Metrics()\n",
    "test_metrics = metrics.add_metrics(\n",
    "    pred_test[['observed', *TOP_N_ORDER]], key='test data')\n",
    "test_metrics = pd.DataFrame(test_metrics)[TOP_N_ORDER]\n",
    "test_metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f8269d00-9048-4e70-9f39-dab95e103c32",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "n_in_comparison = int(test_metrics.loc['N'].unique()[0])\n",
    "n_in_comparison"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "096083d1-bcd2-44a2-94fe-a89b7d204b66",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "_to_plot = test_metrics.loc[METRIC].to_frame().T\n",
    "_to_plot.index = [feature_names.name]\n",
    "_to_plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "05a259ef-48bd-4dd0-8dfe-9e2750579383",
   "metadata": {
    "lines_to_next_cell": 2,
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "try:\n",
    "    text = model_configs[[\"latent_dim\", \"hidden_layers\"]].apply(\n",
    "        build_text,\n",
    "        axis=1)\n",
    "except KeyError:\n",
    "    logger.warning(\"No PIMMS models in comparsion. Using empty text\")\n",
    "    text = pd.Series('', index=model_configs.columns)\n",
    "\n",
    "_to_plot.loc[\"text\"] = text\n",
    "_to_plot = _to_plot.fillna('')\n",
    "_to_plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d3dd53c0-4068-4eac-a5c3-7aaa608e5f8f",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(4, 2))  # size of the plot can be adjusted\n",
    "ax = _to_plot.loc[[feature_names.name]].plot.bar(\n",
    "    rot=0,\n",
    "    ylabel=f\"{METRIC} for {FEAT_NAME_DISPLAY}\\n({n_in_comparison:,} intensities)\",\n",
    "    # title=f'performance on test data (based on {n_in_comparison:,} measurements)',\n",
    "    color=COLORS_TO_USE,\n",
    "    ax=ax,\n",
    "    width=.7)\n",
    "ax = pimmslearn.plotting.add_height_to_barplot(ax, size=7)\n",
    "ax = pimmslearn.plotting.add_text_to_barplot(ax, _to_plot.loc[\"text\"], size=7)\n",
    "ax.set_xticklabels([])\n",
    "fname = args.out_figures / f'2_{group}_performance_test.pdf'\n",
    "figures[fname.stem] = fname\n",
    "pimmslearn.savefig(fig, name=fname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ef92551d",
   "metadata": {
    "lines_to_next_cell": 2,
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "dumps[fname.stem] = fname.with_suffix('.csv')\n",
    "_to_plot_long = _to_plot.T\n",
    "_to_plot_long = _to_plot_long.rename(\n",
    "    {feature_names.name: 'metric_value'}, axis=1)\n",
    "_to_plot_long['data level'] = feature_names.name\n",
    "_to_plot_long = _to_plot_long.set_index('data level', append=True)\n",
    "_to_plot_long.to_csv(fname.with_suffix('.csv'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d88c21c7",
   "metadata": {},
   "source": [
    "### Plot error by median feature intensity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "588f7bf3",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "pimmslearn.plotting.make_large_descriptors(7)\n",
    "fig, ax = plt.subplots(figsize=(8, 2))\n",
    "\n",
    "ax, errors_binned = pimmslearn.plotting.errors.plot_errors_by_median(\n",
    "    pred=pred_test[\n",
    "        [TARGET_COL] + TOP_N_ORDER\n",
    "    ],\n",
    "    feat_medians=data.train_X.median(),\n",
    "    ax=ax,\n",
    "    feat_name=FEAT_NAME_DISPLAY,\n",
    "    metric_name=METRIC,\n",
    "    palette=COLORS_TO_USE\n",
    ")\n",
    "ax.legend(loc='best', ncols=len(TOP_N_ORDER))\n",
    "pimmslearn.plotting.make_large_descriptors(6)\n",
    "fname = args.out_figures / f'2_{group}_test_errors_binned_by_feat_medians.pdf'\n",
    "figures[fname.stem] = fname\n",
    "pimmslearn.savefig(ax.get_figure(), name=fname)\n",
    "\n",
    "dumps[fname.stem] = fname.with_suffix('.csv')\n",
    "errors_binned.to_csv(fname.with_suffix('.csv'))\n",
    "errors_binned"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e1455bcc",
   "metadata": {
    "lines_to_next_cell": 2,
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "# ! only used for reporting\n",
    "plotted = pimmslearn.plotting.errors.get_data_for_errors_by_median(\n",
    "    errors=errors_binned,\n",
    "    feat_name=FEAT_NAME_DISPLAY,\n",
    "    metric_name=METRIC\n",
    ")\n",
    "plotted.to_excel(fname.with_suffix('.xlsx'), index=False)\n",
    "plotted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b13ecd37",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "(errors_binned\n",
    " .set_index(\n",
    "     ['model', errors_binned.columns[-1]]\n",
    " )\n",
    " .loc[ORDER_MODELS[0]]\n",
    " .sort_values(by=METRIC))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "26370a1a",
   "metadata": {},
   "source": [
    "### Custom model selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "712faf9a",
   "metadata": {
    "lines_to_next_cell": 2,
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "if SEL_MODELS:\n",
    "    metrics = pimmslearn.models.Metrics()\n",
    "    test_metrics = metrics.add_metrics(\n",
    "        pred_test[['observed', *SEL_MODELS]], key='test data')\n",
    "    test_metrics = pd.DataFrame(test_metrics)[SEL_MODELS]\n",
    "    test_metrics\n",
    "\n",
    "    n_in_comparison = int(test_metrics.loc['N'].unique()[0])\n",
    "    n_in_comparison\n",
    "\n",
    "    _to_plot = test_metrics.loc[METRIC].to_frame().T\n",
    "    _to_plot.index = [feature_names.name]\n",
    "    _to_plot\n",
    "\n",
    "    try:\n",
    "        text = model_configs[[\"latent_dim\", \"hidden_layers\"]].apply(\n",
    "            build_text,\n",
    "            axis=1)\n",
    "    except KeyError:\n",
    "        logger.warning(\"No PIMMS models in comparsion. Using empty text\")\n",
    "        text = pd.Series('', index=model_configs.columns)\n",
    "\n",
    "    _to_plot.loc[\"text\"] = text\n",
    "    _to_plot = _to_plot.fillna('')\n",
    "    _to_plot\n",
    "\n",
    "    fig, ax = plt.subplots(figsize=(4, 2))\n",
    "    ax = _to_plot.loc[[feature_names.name]].plot.bar(\n",
    "        rot=0,\n",
    "        ylabel=f\"{METRIC} for {FEAT_NAME_DISPLAY} ({n_in_comparison:,} intensities)\",\n",
    "        # title=f'performance on test data (based on {n_in_comparison:,} measurements)',\n",
    "        color=pimmslearn.plotting.defaults.assign_colors(\n",
    "            list(k.upper() for k in SEL_MODELS)),\n",
    "        ax=ax,\n",
    "        width=.7)\n",
    "    ax.legend(loc='best', ncols=len(SEL_MODELS))\n",
    "    ax = pimmslearn.plotting.add_height_to_barplot(ax, size=5)\n",
    "    ax = pimmslearn.plotting.add_text_to_barplot(ax, _to_plot.loc[\"text\"], size=5)\n",
    "    ax.set_xticklabels([])\n",
    "\n",
    "    fname = args.out_figures / f'2_{group}_performance_test_sel.pdf'\n",
    "    figures[fname.stem] = fname\n",
    "    pimmslearn.savefig(fig, name=fname)\n",
    "\n",
    "    dumps[fname.stem] = fname.with_suffix('.csv')\n",
    "    _to_plot_long = _to_plot.T\n",
    "    _to_plot_long = _to_plot_long.rename(\n",
    "        {feature_names.name: 'metric_value'}, axis=1)\n",
    "    _to_plot_long['data level'] = feature_names.name\n",
    "    _to_plot_long = _to_plot_long.set_index('data level', append=True)\n",
    "    _to_plot_long.to_csv(fname.with_suffix('.csv'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2a578570",
   "metadata": {
    "lines_to_next_cell": 2,
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "# custom selection\n",
    "if SEL_MODELS:\n",
    "    pimmslearn.plotting.make_large_descriptors(7)\n",
    "    fig, ax = plt.subplots(figsize=(8, 2))\n",
    "\n",
    "    ax, errors_binned = pimmslearn.plotting.errors.plot_errors_by_median(\n",
    "        pred=pred_test[\n",
    "            [TARGET_COL] + SEL_MODELS\n",
    "        ],\n",
    "        feat_medians=data.train_X.median(),\n",
    "        ax=ax,\n",
    "        metric_name=METRIC,\n",
    "        feat_name=FEAT_NAME_DISPLAY,\n",
    "        palette=pimmslearn.plotting.defaults.assign_colors(\n",
    "            list(k.upper() for k in SEL_MODELS))\n",
    "    )\n",
    "    # ax.set_ylim(0, 1.5)\n",
    "    ax.legend(loc='best', ncols=len(SEL_MODELS))\n",
    "    # for text in ax.legend().get_texts():\n",
    "    #     text.set_fontsize(6)\n",
    "    fname = args.out_figures / f'2_{group}_test_errors_binned_by_feat_medians_sel.pdf'\n",
    "    figures[fname.stem] = fname\n",
    "    pimmslearn.savefig(ax.get_figure(), name=fname)\n",
    "    plt.show(fig)\n",
    "\n",
    "    dumps[fname.stem] = fname.with_suffix('.csv')\n",
    "    errors_binned.to_csv(fname.with_suffix('.csv'))\n",
    "    pimmslearn.plotting.make_large_descriptors(6)\n",
    "    # ax.xaxis.set_tick_params(rotation=0) # horizontal\n",
    "\n",
    "    # ! only used for reporting\n",
    "    plotted = pimmslearn.plotting.errors.get_data_for_errors_by_median(\n",
    "        errors=errors_binned,\n",
    "        feat_name=FEAT_NAME_DISPLAY,\n",
    "        metric_name=METRIC\n",
    "    )\n",
    "    plotted.to_excel(fname.with_suffix('.xlsx'), index=False)\n",
    "    display(plotted)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "549236ca-9e89-47aa-905c-c97a45d4dc2b",
   "metadata": {},
   "source": [
    "### Error by non-decimal number of intensity\n",
    "\n",
    "- number of observations in parentheses."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3339df97-230f-4cbd-b61d-7aef9a7495e8",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(8, 2))\n",
    "ax, errors_binned = pimmslearn.plotting.errors.plot_errors_binned(\n",
    "    pred_test[\n",
    "        [TARGET_COL] + TOP_N_ORDER\n",
    "    ],\n",
    "    ax=ax,\n",
    "    palette=TOP_N_COLOR_PALETTE,\n",
    "    metric_name=METRIC,\n",
    ")\n",
    "ax.legend(loc='best', ncols=len(TOP_N_ORDER))\n",
    "fname = args.out_figures / f'2_{group}_test_errors_binned_by_int.pdf'\n",
    "figures[fname.stem] = fname\n",
    "pimmslearn.savefig(ax.get_figure(), name=fname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "095f64eb-1c4f-47ae-9a01-d5b05a795779",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "dumps[fname.stem] = fname.with_suffix('.csv')\n",
    "errors_binned.to_csv(fname.with_suffix('.csv'))\n",
    "errors_binned.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a05f8957",
   "metadata": {
    "lines_to_next_cell": 0
   },
   "source": [
    "## Figures dumped to disk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c8f67ae1-40e9-4c2a-af0a-41e627703518",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "figures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b08b442f",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "dumps\n",
    "print(\"done\")"
   ]
  }
 ],
 "metadata": {
  "jupytext": {
   "formats": "ipynb,py:percent"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.17"
  },
  "toc-autonumbering": true,
  "vscode": {
   "interpreter": {
    "hash": "cf83e9cb890c7f96eb0ae04f39a82254555f56a1a0ed2f03b23a8b40fe6cd31c"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}