eval_demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation demo\n",
    "This notebook will present how to evalute the intent recognition including macro f1 scores for each subsets.\n",
    "Download the predicted scores from our baseline models at [this link](https://cornell.box.com/s/5g5q7tnak1le5cxa3nv69xep6o7e7uwi) and save them to `ROOT`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "from collections import defaultdict\n",
    "\n",
    "from eval_utils import eval_all_metrics, SUBSET2IDS\n",
    "\n",
    "ROOT = \"\"  # the folder which you place the downloaded scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "organize the evaluation results in a table\n",
    "\"\"\"\n",
    "def get_allresults_df(root: str) -> pd.DataFrame:\n",
    "    data_dict = defaultdict(list)\n",
    "    # these are the three baseline model ablations\n",
    "    for model_type in [\"image\", \"image_cam\", \"image_hs_cam\"]:\n",
    "        data_dict[\"model\"].append(model_type)\n",
    "        val_f1s = []  # 5 x 28\n",
    "        all_f1s = defaultdict(list)\n",
    "\n",
    "        # get results for each run\n",
    "        for run_num in range(5):\n",
    "            d_dict = torch.load(f\"{root}/{model_type}_{run_num}.pth\")\n",
    "            f1_dict = eval_all_metrics(\n",
    "                d_dict[\"val_scores\"], d_dict[\"test_scores\"],\n",
    "                d_dict[\"val_targets\"], d_dict[\"test_targets\"]\n",
    "            )\n",
    "            for k, v in f1_dict.items():\n",
    "                if isinstance(v, float):\n",
    "                    all_f1s[k].append(v * 100)\n",
    "                else:\n",
    "                    all_f1s[k].append(np.array(v)[np.newaxis, :] * 100)\n",
    "\n",
    "        val_f1s = np.vstack(all_f1s[\"val_none\"])\n",
    "\n",
    "        for e_type, c_ids in SUBSET2IDS.items():\n",
    "            e_f1s = np.mean(np.hstack([val_f1s[:, c:c+1] for c in c_ids]), 1)\n",
    "            data_dict[f\"val-{e_type}\"].append(\"{:.2f} +- {:.2f}\".format(\n",
    "                np.mean(e_f1s), np.std(e_f1s)\n",
    "            ))\n",
    "\n",
    "        for k, values in all_f1s.items():\n",
    "            if not k.endswith(\"none\"):\n",
    "                data_dict[k].append(\"{:.2f} +- {:.2f}\".format(\n",
    "                    np.mean(values), np.std(values)\n",
    "                ))\n",
    "    df = pd.DataFrame(data_dict)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>val-easy</th>\n",
       "      <th>val-medium</th>\n",
       "      <th>val-hard</th>\n",
       "      <th>val-object</th>\n",
       "      <th>val-context</th>\n",
       "      <th>val-other</th>\n",
       "      <th>val_micro</th>\n",
       "      <th>val_samples</th>\n",
       "      <th>val_macro</th>\n",
       "      <th>test_micro</th>\n",
       "      <th>test_samples</th>\n",
       "      <th>test_macro</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>image</td>\n",
       "      <td>54.64 +- 2.54</td>\n",
       "      <td>24.92 +- 1.18</td>\n",
       "      <td>10.71 +- 1.33</td>\n",
       "      <td>25.58 +- 2.51</td>\n",
       "      <td>30.16 +- 2.97</td>\n",
       "      <td>21.34 +- 0.74</td>\n",
       "      <td>31.36 +- 1.16</td>\n",
       "      <td>29.91 +- 1.73</td>\n",
       "      <td>23.03 +- 0.79</td>\n",
       "      <td>30.23 +- 0.73</td>\n",
       "      <td>28.45 +- 1.71</td>\n",
       "      <td>22.77 +- 0.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>image_cam</td>\n",
       "      <td>57.10 +- 1.84</td>\n",
       "      <td>25.68 +- 1.24</td>\n",
       "      <td>12.72 +- 2.31</td>\n",
       "      <td>28.15 +- 1.94</td>\n",
       "      <td>28.62 +- 2.13</td>\n",
       "      <td>22.60 +- 1.40</td>\n",
       "      <td>32.87 +- 1.13</td>\n",
       "      <td>32.46 +- 1.18</td>\n",
       "      <td>24.42 +- 0.95</td>\n",
       "      <td>32.07 +- 0.84</td>\n",
       "      <td>30.91 +- 1.27</td>\n",
       "      <td>24.37 +- 0.65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>image_hs_cam</td>\n",
       "      <td>58.86 +- 2.56</td>\n",
       "      <td>26.30 +- 1.42</td>\n",
       "      <td>13.11 +- 2.15</td>\n",
       "      <td>29.66 +- 2.19</td>\n",
       "      <td>32.48 +- 1.34</td>\n",
       "      <td>22.61 +- 0.48</td>\n",
       "      <td>32.94 +- 1.16</td>\n",
       "      <td>33.61 +- 0.92</td>\n",
       "      <td>25.07 +- 0.52</td>\n",
       "      <td>31.28 +- 0.36</td>\n",
       "      <td>31.39 +- 0.78</td>\n",
       "      <td>23.98 +- 0.85</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          model       val-easy     val-medium       val-hard     val-object  \\\n",
       "0         image  54.64 +- 2.54  24.92 +- 1.18  10.71 +- 1.33  25.58 +- 2.51   \n",
       "1     image_cam  57.10 +- 1.84  25.68 +- 1.24  12.72 +- 2.31  28.15 +- 1.94   \n",
       "2  image_hs_cam  58.86 +- 2.56  26.30 +- 1.42  13.11 +- 2.15  29.66 +- 2.19   \n",
       "\n",
       "     val-context      val-other      val_micro    val_samples      val_macro  \\\n",
       "0  30.16 +- 2.97  21.34 +- 0.74  31.36 +- 1.16  29.91 +- 1.73  23.03 +- 0.79   \n",
       "1  28.62 +- 2.13  22.60 +- 1.40  32.87 +- 1.13  32.46 +- 1.18  24.42 +- 0.95   \n",
       "2  32.48 +- 1.34  22.61 +- 0.48  32.94 +- 1.16  33.61 +- 0.92  25.07 +- 0.52   \n",
       "\n",
       "      test_micro   test_samples     test_macro  \n",
       "0  30.23 +- 0.73  28.45 +- 1.71  22.77 +- 0.59  \n",
       "1  32.07 +- 0.84  30.91 +- 1.27  24.37 +- 0.65  \n",
       "2  31.28 +- 0.36  31.39 +- 0.78  23.98 +- 0.85  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# these are the same results reported in README.md\n",
    "get_allresults_df(ROOT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}