results_baselines.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import dill\n",
    "from IPython.core import display as ICD  # to print multiple nice pandas tables\n",
    "import json\n",
    "import logging\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import seaborn as sns\n",
    "import tqdm\n",
    "from tqdm import tqdm_notebook, tnrange\n",
    "sns.set()\n",
    "import pyperclip\n",
    "import utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import subtract_baseline, tabular, multireplace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "dump_file = 'clean_dumps/results_baselines_simple_model.pickle'\n",
    "loaded_results = pickle.load(open(dump_file,'br'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(loaded_results)\n",
    "df = df.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "nice = [x for x in df['level_1'] if 'mylogi' not in x]\n",
    "df = df[df['level_1'].isin(nice)]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['classifier'] = [q.split('_')[0] for q in df['level_1'].values]\n",
    "df['lsa'] = [q.split('_')[1] for q in df['level_1'].values]\n",
    "df['scheme'] = [q.split('_')[2] for q in df['level_1'].values]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.drop(['level_0','level_1'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.set_index(['classifier','lsa','scheme','level_2'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>CRDataset</th>\n",
       "      <th>MPQADataset</th>\n",
       "      <th>MRDataset</th>\n",
       "      <th>SUBJDataset</th>\n",
       "      <th>TRECDataset-ABBR</th>\n",
       "      <th>TRECDataset-DESC</th>\n",
       "      <th>TRECDataset-ENTY</th>\n",
       "      <th>TRECDataset-HUM</th>\n",
       "      <th>TRECDataset-LOC</th>\n",
       "      <th>TRECDataset-NUM</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lsa</th>\n",
       "      <th>scheme</th>\n",
       "      <th>level_2</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\"></th>\n",
       "      <th rowspan=\"3\" valign=\"top\"></th>\n",
       "      <th>test</th>\n",
       "      <td>-0.000050</td>\n",
       "      <td>-0.000636</td>\n",
       "      <td>-0.000469</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.000818</td>\n",
       "      <td>0.000293</td>\n",
       "      <td>-0.000704</td>\n",
       "      <td>-0.000045</td>\n",
       "      <td>-0.000465</td>\n",
       "      <td>0.000060</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train</th>\n",
       "      <td>0.000132</td>\n",
       "      <td>0.000041</td>\n",
       "      <td>0.000059</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.000002</td>\n",
       "      <td>-0.000028</td>\n",
       "      <td>0.000014</td>\n",
       "      <td>0.000057</td>\n",
       "      <td>-0.000061</td>\n",
       "      <td>0.000021</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>valid</th>\n",
       "      <td>-0.001011</td>\n",
       "      <td>0.000306</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000835</td>\n",
       "      <td>-0.000073</td>\n",
       "      <td>0.000596</td>\n",
       "      <td>-0.000409</td>\n",
       "      <td>0.000957</td>\n",
       "      <td>-0.000225</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    CRDataset  MPQADataset  MRDataset  SUBJDataset  \\\n",
       "lsa scheme level_2                                                   \n",
       "           test     -0.000050    -0.000636  -0.000469          0.0   \n",
       "           train     0.000132     0.000041   0.000059          0.0   \n",
       "           valid    -0.001011     0.000306   0.000000          0.0   \n",
       "\n",
       "                    TRECDataset-ABBR  TRECDataset-DESC  TRECDataset-ENTY  \\\n",
       "lsa scheme level_2                                                         \n",
       "           test            -0.000818          0.000293         -0.000704   \n",
       "           train           -0.000002         -0.000028          0.000014   \n",
       "           valid            0.000835         -0.000073          0.000596   \n",
       "\n",
       "                    TRECDataset-HUM  TRECDataset-LOC  TRECDataset-NUM  \n",
       "lsa scheme level_2                                                     \n",
       "           test           -0.000045        -0.000465         0.000060  \n",
       "           train           0.000057        -0.000061         0.000021  \n",
       "           valid          -0.000409         0.000957        -0.000225  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "constant = df.loc['constant'].copy()\n",
    "subtract_baseline(constant)\n",
    "constant"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.loc['logistic']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_hdf(key='default', path_or_buf='clean_dumps/baseline_results.hdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "nottrec = [x for x in df.columns if 'TREC' not in x] \n",
    "trec = [x for x in df.columns if 'TREC' in x]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>CRDataset</th>\n",
       "      <th>MPQADataset</th>\n",
       "      <th>MRDataset</th>\n",
       "      <th>SUBJDataset</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lsa</th>\n",
       "      <th>scheme</th>\n",
       "      <th>level_2</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"14\" valign=\"top\"></th>\n",
       "      <th rowspan=\"2\" valign=\"top\">None</th>\n",
       "      <th>test</th>\n",
       "      <td>0.787302</td>\n",
       "      <td>0.837512</td>\n",
       "      <td>0.761387</td>\n",
       "      <td>0.907200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train</th>\n",
       "      <td>0.970861</td>\n",
       "      <td>0.911763</td>\n",
       "      <td>0.977489</td>\n",
       "      <td>0.994925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">tfchi2</th>\n",
       "      <th>test</th>\n",
       "      <td>0.751323</td>\n",
       "      <td>0.806409</td>\n",
       "      <td>0.683974</td>\n",
       "      <td>0.835600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train</th>\n",
       "      <td>0.788543</td>\n",
       "      <td>0.854361</td>\n",
       "      <td>0.706179</td>\n",
       "      <td>0.844700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">tfgr</th>\n",
       "      <th>test</th>\n",
       "      <td>0.746561</td>\n",
       "      <td>0.812630</td>\n",
       "      <td>0.667854</td>\n",
       "      <td>0.840400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train</th>\n",
       "      <td>0.788543</td>\n",
       "      <td>0.855705</td>\n",
       "      <td>0.708805</td>\n",
       "      <td>0.845300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">tfidf</th>\n",
       "      <th>test</th>\n",
       "      <td>0.771429</td>\n",
       "      <td>0.837889</td>\n",
       "      <td>0.747329</td>\n",
       "      <td>0.903600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train</th>\n",
       "      <td>0.999470</td>\n",
       "      <td>0.978619</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">tfig</th>\n",
       "      <th>test</th>\n",
       "      <td>0.768783</td>\n",
       "      <td>0.814703</td>\n",
       "      <td>0.674977</td>\n",
       "      <td>0.839000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train</th>\n",
       "      <td>0.789735</td>\n",
       "      <td>0.854408</td>\n",
       "      <td>0.712135</td>\n",
       "      <td>0.847400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">tfor</th>\n",
       "      <th>test</th>\n",
       "      <td>0.792063</td>\n",
       "      <td>0.837889</td>\n",
       "      <td>0.769447</td>\n",
       "      <td>0.907600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train</th>\n",
       "      <td>0.892781</td>\n",
       "      <td>0.885667</td>\n",
       "      <td>0.903482</td>\n",
       "      <td>0.935850</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">tfrf</th>\n",
       "      <th>test</th>\n",
       "      <td>0.762434</td>\n",
       "      <td>0.826956</td>\n",
       "      <td>0.728022</td>\n",
       "      <td>0.884400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train</th>\n",
       "      <td>0.837285</td>\n",
       "      <td>0.868505</td>\n",
       "      <td>0.822699</td>\n",
       "      <td>0.918700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"14\" valign=\"top\">lsa200</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">None</th>\n",
       "      <th>test</th>\n",
       "      <td>0.753439</td>\n",
       "      <td>0.743827</td>\n",
       "      <td>0.661668</td>\n",
       "      <td>0.871200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train</th>\n",
       "      <td>0.797815</td>\n",
       "      <td>0.756931</td>\n",
       "      <td>0.696049</td>\n",
       "      <td>0.886675</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">tfchi2</th>\n",
       "      <th>test</th>\n",
       "      <td>0.748677</td>\n",
       "      <td>0.770217</td>\n",
       "      <td>0.666354</td>\n",
       "      <td>0.833000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train</th>\n",
       "      <td>0.784305</td>\n",
       "      <td>0.786752</td>\n",
       "      <td>0.686904</td>\n",
       "      <td>0.842550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">tfgr</th>\n",
       "      <th>test</th>\n",
       "      <td>0.754497</td>\n",
       "      <td>0.771725</td>\n",
       "      <td>0.670665</td>\n",
       "      <td>0.831400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train</th>\n",
       "      <td>0.786887</td>\n",
       "      <td>0.785266</td>\n",
       "      <td>0.689952</td>\n",
       "      <td>0.844875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">tfidf</th>\n",
       "      <th>test</th>\n",
       "      <td>0.752910</td>\n",
       "      <td>0.750424</td>\n",
       "      <td>0.682849</td>\n",
       "      <td>0.889800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train</th>\n",
       "      <td>0.801192</td>\n",
       "      <td>0.761127</td>\n",
       "      <td>0.709696</td>\n",
       "      <td>0.900375</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">tfig</th>\n",
       "      <th>test</th>\n",
       "      <td>0.757672</td>\n",
       "      <td>0.772290</td>\n",
       "      <td>0.658294</td>\n",
       "      <td>0.836600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train</th>\n",
       "      <td>0.785364</td>\n",
       "      <td>0.785337</td>\n",
       "      <td>0.691734</td>\n",
       "      <td>0.844700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">tfor</th>\n",
       "      <th>test</th>\n",
       "      <td>0.779894</td>\n",
       "      <td>0.776060</td>\n",
       "      <td>0.729147</td>\n",
       "      <td>0.877800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train</th>\n",
       "      <td>0.844371</td>\n",
       "      <td>0.786917</td>\n",
       "      <td>0.772681</td>\n",
       "      <td>0.899025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">tfrf</th>\n",
       "      <th>test</th>\n",
       "      <td>0.752910</td>\n",
       "      <td>0.781527</td>\n",
       "      <td>0.675539</td>\n",
       "      <td>0.873600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train</th>\n",
       "      <td>0.802252</td>\n",
       "      <td>0.786869</td>\n",
       "      <td>0.715559</td>\n",
       "      <td>0.889075</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"14\" valign=\"top\">lsa300</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">None</th>\n",
       "      <th>test</th>\n",
       "      <td>0.759788</td>\n",
       "      <td>0.763054</td>\n",
       "      <td>0.691659</td>\n",
       "      <td>0.892800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train</th>\n",
       "      <td>0.822252</td>\n",
       "      <td>0.782885</td>\n",
       "      <td>0.721069</td>\n",
       "      <td>0.900200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">tfchi2</th>\n",
       "      <th>test</th>\n",
       "      <td>0.762963</td>\n",
       "      <td>0.784543</td>\n",
       "      <td>0.678913</td>\n",
       "      <td>0.829000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train</th>\n",
       "      <td>0.786225</td>\n",
       "      <td>0.802947</td>\n",
       "      <td>0.696377</td>\n",
       "      <td>0.843250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">tfgr</th>\n",
       "      <th>test</th>\n",
       "      <td>0.764550</td>\n",
       "      <td>0.787747</td>\n",
       "      <td>0.666167</td>\n",
       "      <td>0.841600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train</th>\n",
       "      <td>0.786556</td>\n",
       "      <td>0.801768</td>\n",
       "      <td>0.697690</td>\n",
       "      <td>0.845350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">tfidf</th>\n",
       "      <th>test</th>\n",
       "      <td>0.778307</td>\n",
       "      <td>0.763054</td>\n",
       "      <td>0.703280</td>\n",
       "      <td>0.889600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train</th>\n",
       "      <td>0.830927</td>\n",
       "      <td>0.788190</td>\n",
       "      <td>0.738586</td>\n",
       "      <td>0.912025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">tfig</th>\n",
       "      <th>test</th>\n",
       "      <td>0.750794</td>\n",
       "      <td>0.780019</td>\n",
       "      <td>0.667854</td>\n",
       "      <td>0.839000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train</th>\n",
       "      <td>0.788742</td>\n",
       "      <td>0.804055</td>\n",
       "      <td>0.697901</td>\n",
       "      <td>0.846925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">tfor</th>\n",
       "      <th>test</th>\n",
       "      <td>0.797884</td>\n",
       "      <td>0.781338</td>\n",
       "      <td>0.743768</td>\n",
       "      <td>0.892000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train</th>\n",
       "      <td>0.853510</td>\n",
       "      <td>0.804809</td>\n",
       "      <td>0.792074</td>\n",
       "      <td>0.906050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">tfrf</th>\n",
       "      <th>test</th>\n",
       "      <td>0.775661</td>\n",
       "      <td>0.782846</td>\n",
       "      <td>0.691097</td>\n",
       "      <td>0.867000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train</th>\n",
       "      <td>0.809470</td>\n",
       "      <td>0.802004</td>\n",
       "      <td>0.733310</td>\n",
       "      <td>0.895200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"14\" valign=\"top\">lsa400</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">None</th>\n",
       "      <th>test</th>\n",
       "      <td>0.751852</td>\n",
       "      <td>0.775495</td>\n",
       "      <td>0.711528</td>\n",
       "      <td>0.887400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train</th>\n",
       "      <td>0.845099</td>\n",
       "      <td>0.798774</td>\n",
       "      <td>0.746817</td>\n",
       "      <td>0.910725</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">tfchi2</th>\n",
       "      <th>test</th>\n",
       "      <td>0.760317</td>\n",
       "      <td>0.790952</td>\n",
       "      <td>0.680037</td>\n",
       "      <td>0.834400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train</th>\n",
       "      <td>0.787881</td>\n",
       "      <td>0.812730</td>\n",
       "      <td>0.699238</td>\n",
       "      <td>0.845875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">tfgr</th>\n",
       "      <th>test</th>\n",
       "      <td>0.753968</td>\n",
       "      <td>0.789821</td>\n",
       "      <td>0.679663</td>\n",
       "      <td>0.827400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train</th>\n",
       "      <td>0.789603</td>\n",
       "      <td>0.815064</td>\n",
       "      <td>0.701700</td>\n",
       "      <td>0.848175</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">tfidf</th>\n",
       "      <th>test</th>\n",
       "      <td>0.766667</td>\n",
       "      <td>0.784166</td>\n",
       "      <td>0.730834</td>\n",
       "      <td>0.892200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train</th>\n",
       "      <td>0.850662</td>\n",
       "      <td>0.802098</td>\n",
       "      <td>0.758237</td>\n",
       "      <td>0.918475</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">tfig</th>\n",
       "      <th>test</th>\n",
       "      <td>0.742328</td>\n",
       "      <td>0.789444</td>\n",
       "      <td>0.676476</td>\n",
       "      <td>0.838400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train</th>\n",
       "      <td>0.789735</td>\n",
       "      <td>0.813060</td>\n",
       "      <td>0.702193</td>\n",
       "      <td>0.845900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">tfor</th>\n",
       "      <th>test</th>\n",
       "      <td>0.793122</td>\n",
       "      <td>0.791894</td>\n",
       "      <td>0.755576</td>\n",
       "      <td>0.879000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train</th>\n",
       "      <td>0.863311</td>\n",
       "      <td>0.820038</td>\n",
       "      <td>0.803353</td>\n",
       "      <td>0.911700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">tfrf</th>\n",
       "      <th>test</th>\n",
       "      <td>0.765079</td>\n",
       "      <td>0.794910</td>\n",
       "      <td>0.705717</td>\n",
       "      <td>0.871200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train</th>\n",
       "      <td>0.815232</td>\n",
       "      <td>0.814451</td>\n",
       "      <td>0.746981</td>\n",
       "      <td>0.900600</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       CRDataset  MPQADataset  MRDataset  SUBJDataset\n",
       "lsa    scheme level_2                                                \n",
       "       None   test      0.787302     0.837512   0.761387     0.907200\n",
       "              train     0.970861     0.911763   0.977489     0.994925\n",
       "       tfchi2 test      0.751323     0.806409   0.683974     0.835600\n",
       "              train     0.788543     0.854361   0.706179     0.844700\n",
       "       tfgr   test      0.746561     0.812630   0.667854     0.840400\n",
       "              train     0.788543     0.855705   0.708805     0.845300\n",
       "       tfidf  test      0.771429     0.837889   0.747329     0.903600\n",
       "              train     0.999470     0.978619   1.000000     1.000000\n",
       "       tfig   test      0.768783     0.814703   0.674977     0.839000\n",
       "              train     0.789735     0.854408   0.712135     0.847400\n",
       "       tfor   test      0.792063     0.837889   0.769447     0.907600\n",
       "              train     0.892781     0.885667   0.903482     0.935850\n",
       "       tfrf   test      0.762434     0.826956   0.728022     0.884400\n",
       "              train     0.837285     0.868505   0.822699     0.918700\n",
       "lsa200 None   test      0.753439     0.743827   0.661668     0.871200\n",
       "              train     0.797815     0.756931   0.696049     0.886675\n",
       "       tfchi2 test      0.748677     0.770217   0.666354     0.833000\n",
       "              train     0.784305     0.786752   0.686904     0.842550\n",
       "       tfgr   test      0.754497     0.771725   0.670665     0.831400\n",
       "              train     0.786887     0.785266   0.689952     0.844875\n",
       "       tfidf  test      0.752910     0.750424   0.682849     0.889800\n",
       "              train     0.801192     0.761127   0.709696     0.900375\n",
       "       tfig   test      0.757672     0.772290   0.658294     0.836600\n",
       "              train     0.785364     0.785337   0.691734     0.844700\n",
       "       tfor   test      0.779894     0.776060   0.729147     0.877800\n",
       "              train     0.844371     0.786917   0.772681     0.899025\n",
       "       tfrf   test      0.752910     0.781527   0.675539     0.873600\n",
       "              train     0.802252     0.786869   0.715559     0.889075\n",
       "lsa300 None   test      0.759788     0.763054   0.691659     0.892800\n",
       "              train     0.822252     0.782885   0.721069     0.900200\n",
       "       tfchi2 test      0.762963     0.784543   0.678913     0.829000\n",
       "              train     0.786225     0.802947   0.696377     0.843250\n",
       "       tfgr   test      0.764550     0.787747   0.666167     0.841600\n",
       "              train     0.786556     0.801768   0.697690     0.845350\n",
       "       tfidf  test      0.778307     0.763054   0.703280     0.889600\n",
       "              train     0.830927     0.788190   0.738586     0.912025\n",
       "       tfig   test      0.750794     0.780019   0.667854     0.839000\n",
       "              train     0.788742     0.804055   0.697901     0.846925\n",
       "       tfor   test      0.797884     0.781338   0.743768     0.892000\n",
       "              train     0.853510     0.804809   0.792074     0.906050\n",
       "       tfrf   test      0.775661     0.782846   0.691097     0.867000\n",
       "              train     0.809470     0.802004   0.733310     0.895200\n",
       "lsa400 None   test      0.751852     0.775495   0.711528     0.887400\n",
       "              train     0.845099     0.798774   0.746817     0.910725\n",
       "       tfchi2 test      0.760317     0.790952   0.680037     0.834400\n",
       "              train     0.787881     0.812730   0.699238     0.845875\n",
       "       tfgr   test      0.753968     0.789821   0.679663     0.827400\n",
       "              train     0.789603     0.815064   0.701700     0.848175\n",
       "       tfidf  test      0.766667     0.784166   0.730834     0.892200\n",
       "              train     0.850662     0.802098   0.758237     0.918475\n",
       "       tfig   test      0.742328     0.789444   0.676476     0.838400\n",
       "              train     0.789735     0.813060   0.702193     0.845900\n",
       "       tfor   test      0.793122     0.791894   0.755576     0.879000\n",
       "              train     0.863311     0.820038   0.803353     0.911700\n",
       "       tfrf   test      0.765079     0.794910   0.705717     0.871200\n",
       "              train     0.815232     0.814451   0.746981     0.900600"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.loc[(slice(None), slice(None),['train','test']),nottrec]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "bow = df.loc[''].loc[(slice(None),['train','test']),nottrec]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\\begin{table}[H]\n",
      "\\begin{center}\n",
      "\n",
      "\\begin{tabular}{llrrrr}\n",
      "\\toprule\n",
      "{} &      &  CR &  MPQA &  MR &  SUBJ \\\\\n",
      "scheme &  &            &              &            &              \\\\\n",
      "\\midrule\n",
      "None & test &      0.787 &        0.838 &      0.761 &        0.907 \\\\\n",
      "{} & train &      0.971 &        0.912 &      0.977 &        0.995 \\\\\n",
      "tfchi2 & test &      0.751 &        0.806 &      0.684 &        0.836 \\\\\n",
      "{} & train &      0.789 &        0.854 &      0.706 &        0.845 \\\\\n",
      "tfgr & test &      0.747 &        0.813 &      0.668 &        0.840 \\\\\n",
      "{} & train &      0.789 &        0.856 &      0.709 &        0.845 \\\\\n",
      "tfidf & test &      0.771 &        0.838 &      0.747 &        0.904 \\\\\n",
      "{} & train &      0.999 &        0.979 &      1.000 &        1.000 \\\\\n",
      "tfig & test &      0.769 &        0.815 &      0.675 &        0.839 \\\\\n",
      "{} & train &      0.790 &        0.854 &      0.712 &        0.847 \\\\\n",
      "tfor & test &      0.792 &        0.838 &      0.769 &        0.908 \\\\\n",
      "{} & train &      0.893 &        0.886 &      0.903 &        0.936 \\\\\n",
      "tfrf & test &      0.762 &        0.827 &      0.728 &        0.884 \\\\\n",
      "{} & train &      0.837 &        0.869 &      0.823 &        0.919 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n",
      "\\caption[Accuracy for BOW baseline]{Accuracy for BOW baseline}\n",
      "\\label{}\n",
      "\\end{center}\n",
      "\\end{table}\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "latex = tabular(bow.round(3).to_latex(), 'Accuracy for BOW baseline')\n",
    "latex = latex.replace('level\\_2','')\n",
    "latex = latex.replace('Dataset','')\n",
    "pyperclip.copy(latex)\n",
    "print(latex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\\begin{table}[H]\n",
      "\\begin{center}\n",
      "\n",
      "\\begin{tabular}{llrrrr}\n",
      "\\toprule\n",
      "{} &      &  CR &  MPQA &  MR &  SUBJ \\\\\n",
      "scheme &  &            &              &            &              \\\\\n",
      "\\midrule\n",
      "None & test &      0.150 &        0.150 &      0.261 &        0.407 \\\\\n",
      "{} & train &      0.333 &        0.224 &      0.477 &        0.495 \\\\\n",
      "tfchi2 & test &      0.114 &        0.119 &      0.184 &        0.336 \\\\\n",
      "{} & train &      0.151 &        0.167 &      0.206 &        0.345 \\\\\n",
      "tfgr & test &      0.109 &        0.125 &      0.168 &        0.340 \\\\\n",
      "{} & train &      0.151 &        0.168 &      0.209 &        0.345 \\\\\n",
      "tfidf & test &      0.134 &        0.150 &      0.247 &        0.404 \\\\\n",
      "{} & train &      0.362 &        0.291 &      0.500 &        0.500 \\\\\n",
      "tfig & test &      0.131 &        0.127 &      0.175 &        0.339 \\\\\n",
      "{} & train &      0.152 &        0.167 &      0.212 &        0.347 \\\\\n",
      "tfor & test &      0.154 &        0.150 &      0.269 &        0.408 \\\\\n",
      "{} & train &      0.255 &        0.198 &      0.403 &        0.436 \\\\\n",
      "tfrf & test &      0.125 &        0.139 &      0.228 &        0.384 \\\\\n",
      "{} & train &      0.200 &        0.181 &      0.323 &        0.419 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n",
      "\\caption[Accuracy improvements for BOW baseline]{Accuracy improvements for BOW baseline}\n",
      "\\label{}\n",
      "\\end{center}\n",
      "\\end{table}\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "subtract_baseline(bow)\n",
    "latex = tabular(bow.round(3).to_latex(), 'Accuracy improvements for BOW baseline')\n",
    "latex = latex.replace('level\\_2','')\n",
    "latex = latex.replace('Dataset','')\n",
    "pyperclip.copy(latex)\n",
    "print(latex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "bow=None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "bow_trec = df.loc[''].loc[(slice(None),['train','test']),trec]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\\begin{table}[H]\n",
      "\\begin{center}\n",
      "\n",
      "\\begin{tabular}{llrrrrrr}\n",
      "\\toprule\n",
      "{} &&  ABBR &  DESC &  ENTY &  HUM &  LOC &  NUM \\\\\n",
      "scheme &  & & & &&&\\\\\n",
      "\\midrule\n",
      "None & test & 0.995 & 0.925 & 0.874 &0.920 &0.960 &0.959 \\\\\n",
      "{} & train & 0.995 & 0.981 & 0.983 &0.986 &0.987 &0.990 \\\\\n",
      "tfchi2 & test & 0.993 & 0.878 & 0.789 &0.899 &0.945 &0.923 \\\\\n",
      "{} & train & 0.993 & 0.872 & 0.787 &0.895 &0.949 &0.919 \\\\\n",
      "tfgr & test & 0.991 & 0.866 & 0.784 &0.889 &0.942 &0.924 \\\\\n",
      "{} & train & 0.991 & 0.878 & 0.788 &0.896 &0.946 &0.926 \\\\\n",
      "tfidf & test & 0.991 & 0.925 & 0.885 &0.930 &0.966 &0.968 \\\\\n",
      "{} & train & 1.000 & 1.000 & 1.000 &1.000 &1.000 &1.000 \\\\\n",
      "tfig & test & 0.991 & 0.872 & 0.784 &0.887 &0.949 &0.922 \\\\\n",
      "{} & train & 0.991 & 0.878 & 0.788 &0.896 &0.947 &0.924 \\\\\n",
      "tfor & test & 0.990 & 0.864 & 0.852 &0.930 &0.952 &0.948 \\\\\n",
      "{} & train & 0.991 & 0.954 & 0.940 &0.969 &0.969 &0.971 \\\\\n",
      "tfrf & test & 0.989 & 0.855 & 0.837 &0.908 &0.930 &0.943 \\\\\n",
      "{} & train & 0.991 & 0.926 & 0.906 &0.942 &0.951 &0.957 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n",
      "\\caption[Accuracy for BOW baseline on TREC datasets]{Accuracy for BOW baseline on TREC datasets}\n",
      "\\label{}\n",
      "\\end{center}\n",
      "\\end{table}\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "latex = tabular(bow_trec.round(3).to_latex(), 'Accuracy for BOW baseline on TREC datasets')\n",
    "latex = latex.replace('level\\_2','')\n",
    "latex = latex.replace('   ','')\n",
    "latex = latex.replace('TRECDataset-','')\n",
    "pyperclip.copy(latex)\n",
    "print(latex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\\begin{table}[H]\n",
      "\\begin{center}\n",
      "\n",
      "\\begin{tabular}{llrrrrrr}\n",
      "\\toprule\n",
      "{} &&  ABBR &  DESC &  ENTY &  HUM &  LOC &  NUM \\\\\n",
      "scheme &  & & & &&&\\\\\n",
      "\\midrule\n",
      "None & test & 0.011 & 0.144 & 0.100 &0.137 &0.114 &0.128 \\\\\n",
      "{} & train & 0.011 & 0.199 & 0.209 &0.203 &0.141 &0.159 \\\\\n",
      "tfchi2 & test & 0.009 & 0.097 & 0.014 &0.115 &0.099 &0.093 \\\\\n",
      "{} & train & 0.009 & 0.091 & 0.013 &0.112 &0.103 &0.088 \\\\\n",
      "tfgr & test & 0.007 & 0.085 & 0.009 &0.105 &0.096 &0.094 \\\\\n",
      "{} & train & 0.007 & 0.096 & 0.013 &0.112 &0.100 &0.095 \\\\\n",
      "tfidf & test & 0.007 & 0.144 & 0.110 &0.146 &0.120 &0.138 \\\\\n",
      "{} & train & 0.016 & 0.218 & 0.226 &0.216 &0.154 &0.170 \\\\\n",
      "tfig & test & 0.007 & 0.091 & 0.009 &0.104 &0.103 &0.092 \\\\\n",
      "{} & train & 0.007 & 0.096 & 0.014 &0.112 &0.101 &0.093 \\\\\n",
      "tfor & test & 0.006 & 0.082 & 0.077 &0.146 &0.106 &0.118 \\\\\n",
      "{} & train & 0.007 & 0.172 & 0.166 &0.185 &0.123 &0.141 \\\\\n",
      "tfrf & test & 0.005 & 0.074 & 0.062 &0.125 &0.084 &0.113 \\\\\n",
      "{} & train & 0.007 & 0.144 & 0.132 &0.159 &0.105 &0.126 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n",
      "\\caption[Accuracy improvements for BOW baseline on TREC datasets]{Accuracy improvements for BOW baseline on TREC datasets}\n",
      "\\label{}\n",
      "\\end{center}\n",
      "\\end{table}\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "subtract_baseline(bow_trec)\n",
    "latex = tabular(bow_trec.round(3).to_latex(), 'Accuracy improvements for BOW baseline on TREC datasets')\n",
    "latex = latex.replace('level\\_2','')\n",
    "latex = latex.replace('   ','')\n",
    "latex = latex.replace('TRECDataset-','')\n",
    "pyperclip.copy(latex)\n",
    "print(latex)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LSA baseline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyperclip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n",
      "\\begin{table}[H]\n",
      "\\begin{center}\n",
      "\n",
      "\\begin{tabular}{llrrrr}\n",
      "\\toprule\n",
      "{} &      &  CR &  MPQA &  MR &  SUBJ \\\\\n",
      "scheme &  &            &              &            &              \\\\\n",
      "\\midrule\n",
      "None & test &      0.753 &        0.744 &      0.662 &        0.871 \\\\\n",
      "{} & train &      0.798 &        0.757 &      0.696 &        0.887 \\\\\n",
      "tfchi2 & test &      0.749 &        0.770 &      0.666 &        0.833 \\\\\n",
      "{} & train &      0.784 &        0.787 &      0.687 &        0.843 \\\\\n",
      "tfgr & test &      0.754 &        0.772 &      0.671 &        0.831 \\\\\n",
      "{} & train &      0.787 &        0.785 &      0.690 &        0.845 \\\\\n",
      "tfidf & test &      0.753 &        0.750 &      0.683 &        0.890 \\\\\n",
      "{} & train &      0.801 &        0.761 &      0.710 &        0.900 \\\\\n",
      "tfig & test &      0.758 &        0.772 &      0.658 &        0.837 \\\\\n",
      "{} & train &      0.785 &        0.785 &      0.692 &        0.845 \\\\\n",
      "tfor & test &      0.780 &        0.776 &      0.729 &        0.878 \\\\\n",
      "{} & train &      0.844 &        0.787 &      0.773 &        0.899 \\\\\n",
      "tfrf & test &      0.753 &        0.782 &      0.676 &        0.874 \\\\\n",
      "{} & train &      0.802 &        0.787 &      0.716 &        0.889 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n",
      "\\caption[Accuracy for LSA baseline with 200 dimensions]{Accuracy for LSA baseline with 200 dimensions}\n",
      "\\label{tab:lsa:resuts:abs:200}\n",
      "\\end{center}\n",
      "\\end{table}\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\\begin{table}[H]\n",
      "\\begin{center}\n",
      "\n",
      "\\begin{tabular}{llrrrr}\n",
      "\\toprule\n",
      "{} &      &  CR &  MPQA &  MR &  SUBJ \\\\\n",
      "scheme &  &            &              &            &              \\\\\n",
      "\\midrule\n",
      "None & test &      0.760 &        0.763 &      0.692 &        0.893 \\\\\n",
      "{} & train &      0.822 &        0.783 &      0.721 &        0.900 \\\\\n",
      "tfchi2 & test &      0.763 &        0.785 &      0.679 &        0.829 \\\\\n",
      "{} & train &      0.786 &        0.803 &      0.696 &        0.843 \\\\\n",
      "tfgr & test &      0.765 &        0.788 &      0.666 &        0.842 \\\\\n",
      "{} & train &      0.787 &        0.802 &      0.698 &        0.845 \\\\\n",
      "tfidf & test &      0.778 &        0.763 &      0.703 &        0.890 \\\\\n",
      "{} & train &      0.831 &        0.788 &      0.739 &        0.912 \\\\\n",
      "tfig & test &      0.751 &        0.780 &      0.668 &        0.839 \\\\\n",
      "{} & train &      0.789 &        0.804 &      0.698 &        0.847 \\\\\n",
      "tfor & test &      0.798 &        0.781 &      0.744 &        0.892 \\\\\n",
      "{} & train &      0.854 &        0.805 &      0.792 &        0.906 \\\\\n",
      "tfrf & test &      0.776 &        0.783 &      0.691 &        0.867 \\\\\n",
      "{} & train &      0.809 &        0.802 &      0.733 &        0.895 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n",
      "\\caption[Accuracy for LSA baseline with 300 dimensions]{Accuracy for LSA baseline with 300 dimensions}\n",
      "\\label{tab:lsa:resuts:abs:300}\n",
      "\\end{center}\n",
      "\\end{table}\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\\begin{table}[H]\n",
      "\\begin{center}\n",
      "\n",
      "\\begin{tabular}{llrrrr}\n",
      "\\toprule\n",
      "{} &      &  CR &  MPQA &  MR &  SUBJ \\\\\n",
      "scheme &  &            &              &            &              \\\\\n",
      "\\midrule\n",
      "None & test &      0.752 &        0.775 &      0.712 &        0.887 \\\\\n",
      "{} & train &      0.845 &        0.799 &      0.747 &        0.911 \\\\\n",
      "tfchi2 & test &      0.760 &        0.791 &      0.680 &        0.834 \\\\\n",
      "{} & train &      0.788 &        0.813 &      0.699 &        0.846 \\\\\n",
      "tfgr & test &      0.754 &        0.790 &      0.680 &        0.827 \\\\\n",
      "{} & train &      0.790 &        0.815 &      0.702 &        0.848 \\\\\n",
      "tfidf & test &      0.767 &        0.784 &      0.731 &        0.892 \\\\\n",
      "{} & train &      0.851 &        0.802 &      0.758 &        0.918 \\\\\n",
      "tfig & test &      0.742 &        0.789 &      0.676 &        0.838 \\\\\n",
      "{} & train &      0.790 &        0.813 &      0.702 &        0.846 \\\\\n",
      "tfor & test &      0.793 &        0.792 &      0.756 &        0.879 \\\\\n",
      "{} & train &      0.863 &        0.820 &      0.803 &        0.912 \\\\\n",
      "tfrf & test &      0.765 &        0.795 &      0.706 &        0.871 \\\\\n",
      "{} & train &      0.815 &        0.814 &      0.747 &        0.901 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n",
      "\\caption[Accuracy for LSA baseline with 400 dimensions]{Accuracy for LSA baseline with 400 dimensions}\n",
      "\\label{tab:lsa:resuts:abs:400}\n",
      "\\end{center}\n",
      "\\end{table}\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\\begin{table}[H]\n",
      "\\begin{center}\n",
      "\n",
      "\\begin{tabular}{llrrrr}\n",
      "\\toprule\n",
      "{} &      &  CR &  MPQA &  MR &  SUBJ \\\\\n",
      "scheme &  &            &              &            &              \\\\\n",
      "\\midrule\n",
      "None & test &      0.116 &        0.056 &      0.162 &        0.371 \\\\\n",
      "{} & train &      0.160 &        0.069 &      0.196 &        0.387 \\\\\n",
      "tfchi2 & test &      0.111 &        0.082 &      0.166 &        0.333 \\\\\n",
      "{} & train &      0.147 &        0.099 &      0.187 &        0.343 \\\\\n",
      "tfgr & test &      0.117 &        0.084 &      0.171 &        0.331 \\\\\n",
      "{} & train &      0.149 &        0.098 &      0.190 &        0.345 \\\\\n",
      "tfidf & test &      0.115 &        0.063 &      0.183 &        0.390 \\\\\n",
      "{} & train &      0.164 &        0.073 &      0.210 &        0.400 \\\\\n",
      "tfig & test &      0.120 &        0.085 &      0.158 &        0.337 \\\\\n",
      "{} & train &      0.148 &        0.098 &      0.192 &        0.345 \\\\\n",
      "tfor & test &      0.142 &        0.088 &      0.229 &        0.378 \\\\\n",
      "{} & train &      0.207 &        0.099 &      0.273 &        0.399 \\\\\n",
      "tfrf & test &      0.115 &        0.094 &      0.176 &        0.374 \\\\\n",
      "{} & train &      0.165 &        0.099 &      0.216 &        0.389 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n",
      "\\caption[Accuracy improvements for LSA baseline with 200 dimensions]{Accuracy improvements for LSA baseline with 200 dimensions}\n",
      "\\label{tab:lsa:resuts:200}\n",
      "\\end{center}\n",
      "\\end{table}\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\\begin{table}[H]\n",
      "\\begin{center}\n",
      "\n",
      "\\begin{tabular}{llrrrr}\n",
      "\\toprule\n",
      "{} &      &  CR &  MPQA &  MR &  SUBJ \\\\\n",
      "scheme &  &            &              &            &              \\\\\n",
      "\\midrule\n",
      "None & test &      0.122 &        0.075 &      0.192 &        0.393 \\\\\n",
      "{} & train &      0.185 &        0.095 &      0.221 &        0.400 \\\\\n",
      "tfchi2 & test &      0.125 &        0.097 &      0.179 &        0.329 \\\\\n",
      "{} & train &      0.149 &        0.115 &      0.196 &        0.343 \\\\\n",
      "tfgr & test &      0.127 &        0.100 &      0.166 &        0.342 \\\\\n",
      "{} & train &      0.149 &        0.114 &      0.198 &        0.345 \\\\\n",
      "tfidf & test &      0.141 &        0.075 &      0.203 &        0.390 \\\\\n",
      "{} & train &      0.193 &        0.100 &      0.239 &        0.412 \\\\\n",
      "tfig & test &      0.113 &        0.092 &      0.168 &        0.339 \\\\\n",
      "{} & train &      0.151 &        0.116 &      0.198 &        0.347 \\\\\n",
      "tfor & test &      0.160 &        0.094 &      0.244 &        0.392 \\\\\n",
      "{} & train &      0.216 &        0.117 &      0.292 &        0.406 \\\\\n",
      "tfrf & test &      0.138 &        0.095 &      0.191 &        0.367 \\\\\n",
      "{} & train &      0.172 &        0.114 &      0.233 &        0.395 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n",
      "\\caption[Accuracy improvements for LSA baseline with 300 dimensions]{Accuracy improvements for LSA baseline with 300 dimensions}\n",
      "\\label{tab:lsa:resuts:300}\n",
      "\\end{center}\n",
      "\\end{table}\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\\begin{table}[H]\n",
      "\\begin{center}\n",
      "\n",
      "\\begin{tabular}{llrrrr}\n",
      "\\toprule\n",
      "{} &      &  CR &  MPQA &  MR &  SUBJ \\\\\n",
      "scheme &  &            &              &            &              \\\\\n",
      "\\midrule\n",
      "None & test &      0.114 &        0.088 &      0.212 &        0.387 \\\\\n",
      "{} & train &      0.207 &        0.111 &      0.247 &        0.411 \\\\\n",
      "tfchi2 & test &      0.123 &        0.103 &      0.180 &        0.334 \\\\\n",
      "{} & train &      0.150 &        0.125 &      0.199 &        0.346 \\\\\n",
      "tfgr & test &      0.116 &        0.102 &      0.180 &        0.327 \\\\\n",
      "{} & train &      0.152 &        0.127 &      0.202 &        0.348 \\\\\n",
      "tfidf & test &      0.129 &        0.096 &      0.231 &        0.392 \\\\\n",
      "{} & train &      0.213 &        0.114 &      0.258 &        0.418 \\\\\n",
      "tfig & test &      0.105 &        0.102 &      0.176 &        0.338 \\\\\n",
      "{} & train &      0.152 &        0.125 &      0.202 &        0.346 \\\\\n",
      "tfor & test &      0.156 &        0.104 &      0.256 &        0.379 \\\\\n",
      "{} & train &      0.226 &        0.132 &      0.303 &        0.412 \\\\\n",
      "tfrf & test &      0.127 &        0.107 &      0.206 &        0.371 \\\\\n",
      "{} & train &      0.178 &        0.127 &      0.247 &        0.401 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n",
      "\\caption[Accuracy improvements for LSA baseline with 400 dimensions]{Accuracy improvements for LSA baseline with 400 dimensions}\n",
      "\\label{tab:lsa:resuts:400}\n",
      "\\end{center}\n",
      "\\end{table}\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "all_res = df.loc[(slice(None), slice(None),['train','test']),nottrec]\n",
    "lsas = all_res.loc[['lsa200','lsa300','lsa400']]\n",
    "al = ''\n",
    "for dim in [200,300,400]:\n",
    "    tex = lsas.loc['lsa'+str(dim)].round(3).to_latex()\n",
    "    tex = multireplace(tex, [\n",
    "        ('Dataset',''),\n",
    "        ('level\\_2',''),\n",
    "        ('lsa','')\n",
    "    ])\n",
    "    pom = tabular(tex, 'Accuracy for LSA baseline with {} dimensions'.format(dim), 'tab:lsa:resuts:abs:'+str(dim))\n",
    "    al += '\\n\\n\\n' + pom\n",
    "\n",
    "nobaseline = lsas.copy()\n",
    "subtract_baseline(nobaseline)\n",
    "for dim in [200,300,400]:\n",
    "    tex = nobaseline.loc['lsa'+str(dim)].round(3).to_latex()\n",
    "    tex = multireplace(tex, [\n",
    "        ('Dataset',''),\n",
    "        ('level\\_2',''),\n",
    "        ('lsa','')\n",
    "    ])\n",
    "    pom = tabular(tex, 'Accuracy improvements for LSA baseline with {} dimensions'.format(dim), \n",
    "                  'tab:lsa:resuts:'+str(dim))\n",
    "    al += '\\n\\n\\n' + pom  \n",
    "    \n",
    "pyperclip.copy(al)\n",
    "print(al)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n",
      "\\begin{table}[H]\n",
      "\\begin{center}\n",
      "\n",
      "\\begin{tabular}{llrrrrrr}\n",
      "\\toprule\n",
      "{} &  &  ABBR &  DESC &  ENTY &  HUM &  LOC &  NUM \\\\\n",
      "scheme &  &       &       &       &      &      &      \\\\\n",
      "\\midrule\n",
      "None & test &     0.992 &     0.900 &     0.841 &    0.912 &    0.946 &    0.949 \\\\\n",
      "{} & train &     0.992 &     0.909 &     0.872 &    0.930 &    0.956 &    0.954 \\\\\n",
      "tfchi2 & test &     0.992 &     0.866 &     0.785 &    0.889 &    0.948 &    0.915 \\\\\n",
      "{} & train &     0.994 &     0.875 &     0.787 &    0.897 &    0.949 &    0.919 \\\\\n",
      "tfgr & test &     0.991 &     0.877 &     0.787 &    0.895 &    0.942 &    0.933 \\\\\n",
      "{} & train &     0.991 &     0.879 &     0.787 &    0.895 &    0.947 &    0.926 \\\\\n",
      "tfidf & test &     0.992 &     0.890 &     0.853 &    0.914 &    0.949 &    0.948 \\\\\n",
      "{} & train &     0.995 &     0.906 &     0.877 &    0.936 &    0.961 &    0.966 \\\\\n",
      "tfig & test &     0.989 &     0.872 &     0.787 &    0.891 &    0.939 &    0.923 \\\\\n",
      "{} & train &     0.991 &     0.881 &     0.788 &    0.893 &    0.947 &    0.921 \\\\\n",
      "tfor & test &     0.992 &     0.864 &     0.850 &    0.916 &    0.948 &    0.956 \\\\\n",
      "{} & train &     0.990 &     0.941 &     0.926 &    0.963 &    0.963 &    0.968 \\\\\n",
      "tfrf & test &     0.990 &     0.856 &     0.831 &    0.903 &    0.941 &    0.943 \\\\\n",
      "{} & train &     0.991 &     0.899 &     0.880 &    0.932 &    0.948 &    0.952 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n",
      "\\caption[Accuracy for LSA baseline with 200 dimensions on TREC datasets]{Accuracy for LSA baseline with 200 dimensions on TREC datasets}\n",
      "\\label{tab:lsa:resuts:abs:200:TREC}\n",
      "\\end{center}\n",
      "\\end{table}\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\\begin{table}[H]\n",
      "\\begin{center}\n",
      "\n",
      "\\begin{tabular}{llrrrrrr}\n",
      "\\toprule\n",
      "{} &  &  ABBR &  DESC &  ENTY &  HUM &  LOC &  NUM \\\\\n",
      "scheme &  &       &       &       &      &      &      \\\\\n",
      "\\midrule\n",
      "None & test &     0.992 &     0.906 &     0.847 &    0.917 &    0.953 &    0.955 \\\\\n",
      "{} & train &     0.993 &     0.922 &     0.885 &    0.944 &    0.959 &    0.963 \\\\\n",
      "tfchi2 & test &     0.994 &     0.884 &     0.787 &    0.895 &    0.944 &    0.921 \\\\\n",
      "{} & train &     0.992 &     0.879 &     0.789 &    0.896 &    0.949 &    0.918 \\\\\n",
      "tfgr & test &     0.992 &     0.869 &     0.786 &    0.886 &    0.944 &    0.926 \\\\\n",
      "{} & train &     0.991 &     0.880 &     0.787 &    0.896 &    0.949 &    0.923 \\\\\n",
      "tfidf & test &     0.993 &     0.903 &     0.859 &    0.925 &    0.951 &    0.954 \\\\\n",
      "{} & train &     0.995 &     0.923 &     0.896 &    0.954 &    0.970 &    0.975 \\\\\n",
      "tfig & test &     0.990 &     0.876 &     0.787 &    0.886 &    0.941 &    0.925 \\\\\n",
      "{} & train &     0.991 &     0.879 &     0.787 &    0.895 &    0.948 &    0.930 \\\\\n",
      "tfor & test &     0.990 &     0.865 &     0.857 &    0.924 &    0.955 &    0.957 \\\\\n",
      "{} & train &     0.991 &     0.943 &     0.931 &    0.966 &    0.965 &    0.971 \\\\\n",
      "tfrf & test &     0.990 &     0.865 &     0.832 &    0.910 &    0.942 &    0.949 \\\\\n",
      "{} & train &     0.991 &     0.906 &     0.887 &    0.935 &    0.949 &    0.953 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n",
      "\\caption[Accuracy for LSA baseline with 300 dimensions on TREC datasets]{Accuracy for LSA baseline with 300 dimensions on TREC datasets}\n",
      "\\label{tab:lsa:resuts:abs:300:TREC}\n",
      "\\end{center}\n",
      "\\end{table}\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\\begin{table}[H]\n",
      "\\begin{center}\n",
      "\n",
      "\\begin{tabular}{llrrrrrr}\n",
      "\\toprule\n",
      "{} &  &  ABBR &  DESC &  ENTY &  HUM &  LOC &  NUM \\\\\n",
      "scheme &  &       &       &       &      &      &      \\\\\n",
      "\\midrule\n",
      "None & test &     0.992 &     0.913 &     0.865 &    0.920 &    0.943 &    0.956 \\\\\n",
      "{} & train &     0.994 &     0.932 &     0.900 &    0.948 &    0.965 &    0.967 \\\\\n",
      "tfchi2 & test &     0.993 &     0.879 &     0.785 &    0.895 &    0.945 &    0.915 \\\\\n",
      "{} & train &     0.993 &     0.879 &     0.787 &    0.898 &    0.948 &    0.917 \\\\\n",
      "tfgr & test &     0.987 &     0.875 &     0.786 &    0.891 &    0.945 &    0.925 \\\\\n",
      "{} & train &     0.992 &     0.881 &     0.787 &    0.896 &    0.948 &    0.928 \\\\\n",
      "tfidf & test &     0.994 &     0.905 &     0.859 &    0.923 &    0.953 &    0.957 \\\\\n",
      "{} & train &     0.995 &     0.933 &     0.909 &    0.965 &    0.980 &    0.985 \\\\\n",
      "tfig & test &     0.990 &     0.878 &     0.785 &    0.895 &    0.944 &    0.922 \\\\\n",
      "{} & train &     0.991 &     0.880 &     0.787 &    0.896 &    0.947 &    0.923 \\\\\n",
      "tfor & test &     0.993 &     0.867 &     0.847 &    0.928 &    0.949 &    0.959 \\\\\n",
      "{} & train &     0.991 &     0.946 &     0.934 &    0.965 &    0.965 &    0.970 \\\\\n",
      "tfrf & test &     0.991 &     0.863 &     0.832 &    0.917 &    0.944 &    0.946 \\\\\n",
      "{} & train &     0.991 &     0.906 &     0.887 &    0.936 &    0.950 &    0.954 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n",
      "\\caption[Accuracy for LSA baseline with 400 dimensions on TREC datasets]{Accuracy for LSA baseline with 400 dimensions on TREC datasets}\n",
      "\\label{tab:lsa:resuts:abs:400:TREC}\n",
      "\\end{center}\n",
      "\\end{table}\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\\begin{table}[H]\n",
      "\\begin{center}\n",
      "\n",
      "\\begin{tabular}{llrrrrrr}\n",
      "\\toprule\n",
      "{} &  &  ABBR &  DESC &  ENTY &  HUM &  LOC &  NUM \\\\\n",
      "scheme &  &       &       &       &      &      &      \\\\\n",
      "\\midrule\n",
      "None & test &     0.008 &     0.119 &     0.067 &    0.129 &    0.100 &    0.119 \\\\\n",
      "{} & train &     0.008 &     0.127 &     0.098 &    0.147 &    0.110 &    0.124 \\\\\n",
      "tfchi2 & test &     0.008 &     0.085 &     0.010 &    0.105 &    0.102 &    0.085 \\\\\n",
      "{} & train &     0.010 &     0.094 &     0.013 &    0.113 &    0.103 &    0.088 \\\\\n",
      "tfgr & test &     0.007 &     0.096 &     0.013 &    0.111 &    0.096 &    0.102 \\\\\n",
      "{} & train &     0.007 &     0.098 &     0.013 &    0.112 &    0.101 &    0.095 \\\\\n",
      "tfidf & test &     0.008 &     0.108 &     0.078 &    0.131 &    0.103 &    0.117 \\\\\n",
      "{} & train &     0.011 &     0.125 &     0.103 &    0.153 &    0.115 &    0.135 \\\\\n",
      "tfig & test &     0.005 &     0.091 &     0.012 &    0.108 &    0.093 &    0.093 \\\\\n",
      "{} & train &     0.007 &     0.100 &     0.014 &    0.109 &    0.101 &    0.091 \\\\\n",
      "tfor & test &     0.008 &     0.083 &     0.076 &    0.133 &    0.102 &    0.126 \\\\\n",
      "{} & train &     0.006 &     0.159 &     0.152 &    0.179 &    0.117 &    0.138 \\\\\n",
      "tfrf & test &     0.006 &     0.074 &     0.057 &    0.120 &    0.095 &    0.113 \\\\\n",
      "{} & train &     0.007 &     0.117 &     0.105 &    0.148 &    0.102 &    0.122 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n",
      "\\caption[Accuracy improvements for LSA baseline with 200 dimensions on TREC datasets]{Accuracy improvements for LSA baseline with 200 dimensions on TREC datasets}\n",
      "\\label{tab:lsa:resuts:200:TREC}\n",
      "\\end{center}\n",
      "\\end{table}\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\\begin{table}[H]\n",
      "\\begin{center}\n",
      "\n",
      "\\begin{tabular}{llrrrrrr}\n",
      "\\toprule\n",
      "{} &  &  ABBR &  DESC &  ENTY &  HUM &  LOC &  NUM \\\\\n",
      "scheme &  &       &       &       &      &      &      \\\\\n",
      "\\midrule\n",
      "None & test &     0.008 &     0.124 &     0.073 &    0.134 &    0.107 &    0.124 \\\\\n",
      "{} & train &     0.009 &     0.140 &     0.110 &    0.160 &    0.113 &    0.132 \\\\\n",
      "tfchi2 & test &     0.010 &     0.103 &     0.012 &    0.111 &    0.098 &    0.091 \\\\\n",
      "{} & train &     0.008 &     0.098 &     0.015 &    0.112 &    0.103 &    0.087 \\\\\n",
      "tfgr & test &     0.008 &     0.088 &     0.012 &    0.102 &    0.098 &    0.096 \\\\\n",
      "{} & train &     0.007 &     0.098 &     0.012 &    0.113 &    0.103 &    0.093 \\\\\n",
      "tfidf & test &     0.009 &     0.121 &     0.085 &    0.142 &    0.105 &    0.123 \\\\\n",
      "{} & train &     0.011 &     0.142 &     0.121 &    0.170 &    0.124 &    0.144 \\\\\n",
      "tfig & test &     0.006 &     0.095 &     0.013 &    0.102 &    0.095 &    0.094 \\\\\n",
      "{} & train &     0.007 &     0.097 &     0.012 &    0.112 &    0.102 &    0.099 \\\\\n",
      "tfor & test &     0.006 &     0.084 &     0.083 &    0.140 &    0.109 &    0.127 \\\\\n",
      "{} & train &     0.007 &     0.162 &     0.156 &    0.182 &    0.119 &    0.140 \\\\\n",
      "tfrf & test &     0.006 &     0.084 &     0.058 &    0.127 &    0.096 &    0.119 \\\\\n",
      "{} & train &     0.007 &     0.124 &     0.113 &    0.151 &    0.103 &    0.122 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n",
      "\\caption[Accuracy improvements for LSA baseline with 300 dimensions on TREC datasets]{Accuracy improvements for LSA baseline with 300 dimensions on TREC datasets}\n",
      "\\label{tab:lsa:resuts:300:TREC}\n",
      "\\end{center}\n",
      "\\end{table}\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\\begin{table}[H]\n",
      "\\begin{center}\n",
      "\n",
      "\\begin{tabular}{llrrrrrr}\n",
      "\\toprule\n",
      "{} &  &  ABBR &  DESC &  ENTY &  HUM &  LOC &  NUM \\\\\n",
      "scheme &  &       &       &       &      &      &      \\\\\n",
      "\\midrule\n",
      "None & test &     0.008 &     0.132 &     0.091 &    0.136 &    0.097 &    0.125 \\\\\n",
      "{} & train &     0.010 &     0.150 &     0.126 &    0.165 &    0.118 &    0.137 \\\\\n",
      "tfchi2 & test &     0.009 &     0.097 &     0.010 &    0.111 &    0.099 &    0.084 \\\\\n",
      "{} & train &     0.009 &     0.097 &     0.013 &    0.114 &    0.102 &    0.086 \\\\\n",
      "tfgr & test &     0.003 &     0.094 &     0.012 &    0.107 &    0.099 &    0.094 \\\\\n",
      "{} & train &     0.007 &     0.099 &     0.013 &    0.113 &    0.102 &    0.098 \\\\\n",
      "tfidf & test &     0.010 &     0.124 &     0.085 &    0.140 &    0.107 &    0.127 \\\\\n",
      "{} & train &     0.011 &     0.151 &     0.135 &    0.181 &    0.134 &    0.155 \\\\\n",
      "tfig & test &     0.006 &     0.096 &     0.010 &    0.111 &    0.098 &    0.091 \\\\\n",
      "{} & train &     0.007 &     0.098 &     0.013 &    0.112 &    0.101 &    0.093 \\\\\n",
      "tfor & test &     0.009 &     0.086 &     0.073 &    0.144 &    0.103 &    0.129 \\\\\n",
      "{} & train &     0.007 &     0.165 &     0.159 &    0.182 &    0.119 &    0.139 \\\\\n",
      "tfrf & test &     0.007 &     0.082 &     0.058 &    0.134 &    0.098 &    0.115 \\\\\n",
      "{} & train &     0.007 &     0.124 &     0.113 &    0.152 &    0.104 &    0.124 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n",
      "\\caption[Accuracy improvements for LSA baseline with 400 dimensions on TREC datasets]{Accuracy improvements for LSA baseline with 400 dimensions on TREC datasets}\n",
      "\\label{tab:lsa:resuts:400:TREC}\n",
      "\\end{center}\n",
      "\\end{table}\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "all_res = df.loc[(slice(None), slice(None),['train','test']),utils.TREC]\n",
    "lsas = all_res.loc[['lsa200','lsa300','lsa400']]\n",
    "al = ''\n",
    "for dim in [200,300,400]:\n",
    "    tex = lsas.loc['lsa'+str(dim)].round(3).to_latex()\n",
    "    tex = multireplace(tex, [\n",
    "        ('Dataset',''),\n",
    "        ('TREC-',''),\n",
    "        ('   ',' '),\n",
    "        ('level\\_2',''),\n",
    "        ('lsa','')\n",
    "    ])\n",
    "    pom = tabular(tex, 'Accuracy for LSA baseline with {} dimensions on TREC datasets'.format(dim), \n",
    "                  'tab:lsa:resuts:abs:'+str(dim)+':TREC')\n",
    "    al += '\\n\\n\\n' + pom\n",
    "\n",
    "nobaseline = lsas.copy()\n",
    "subtract_baseline(nobaseline)\n",
    "for dim in [200,300,400]:\n",
    "    tex = nobaseline.loc['lsa'+str(dim)].round(3).to_latex()\n",
    "    tex = multireplace(tex, [\n",
    "        ('Dataset',''),\n",
    "        ('TREC-',''),\n",
    "        ('   ',' '),\n",
    "        ('level\\_2',''),\n",
    "        ('lsa','')\n",
    "    ])\n",
    "    pom = tabular(tex, 'Accuracy improvements for LSA baseline with {} dimensions on TREC datasets'.format(dim), \n",
    "                  'tab:lsa:resuts:'+str(dim)+':TREC')\n",
    "    al += '\\n\\n\\n' + pom  \n",
    "    \n",
    "pyperclip.copy(al)\n",
    "print(al)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "diplomka",
   "language": "python",
   "name": "diplomka"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}