Evaluate and Test models.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluate and Test Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-30T10:58:54.239502Z",
     "start_time": "2019-04-30T10:58:29.952855Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.0.1'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.nn.utils.rnn import pack_padded_sequence, pack_sequence, pad_packed_sequence, pad_sequence\n",
    "from torch.utils.data import DataLoader, Dataset, SequentialSampler\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "import time\n",
    "import random\n",
    "import os\n",
    "import copy\n",
    "import warnings\n",
    "from functools import partial\n",
    "\n",
    "from utils import pad_and_sort_batch, preprocess_for_batch, pad_or_truncate, pad_and_truncate_hierarchical, preprocess_for_batch_hierarchical\n",
    "\n",
    "warnings.filterwarnings('ignore')\n",
    "DEVICE = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "%matplotlib inline\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "torch.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-30T10:58:54.425496Z",
     "start_time": "2019-04-30T10:58:54.243512Z"
    }
   },
   "outputs": [],
   "source": [
    "# set random seeds to keep the results identical\n",
    "def setup_seed(seed):\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed_all(seed)\n",
    "    np.random.seed(seed)\n",
    "    random.seed(seed)\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "    \n",
    "def worker_init_fn(worker_id):\n",
    "    setup_seed(torch.initial_seed() + worker_id)\n",
    "    \n",
    "GLOBAL_SEED = 2019\n",
    "setup_seed(GLOBAL_SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-30T10:58:54.577900Z",
     "start_time": "2019-04-30T10:58:54.429005Z"
    }
   },
   "outputs": [],
   "source": [
    "base_dir = './'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load data for testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-30T10:58:54.738327Z",
     "start_time": "2019-04-30T10:58:54.581410Z"
    }
   },
   "outputs": [],
   "source": [
    "class CustomDataset(Dataset):\n",
    "    def __init__(self, features, labels):\n",
    "        self.features = features\n",
    "        self.labels = labels\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.features)\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        if self.features[idx].size(0) > 0:\n",
    "            features = self.features[idx]\n",
    "        else:\n",
    "            features = torch.LongTensor([1])\n",
    "        return features, self.labels[idx], max(len(self.features[idx]), 1)\n",
    "    \n",
    "def prepare_data_for_test(data_pth, batch_size=32, multiclass=False, truncate=False, maxlen=500):\n",
    "    X_test = pd.read_hdf(data_pth+'X_test.h5', key='s')\n",
    "    y_test = pd.read_hdf(data_pth+'y_test.h5', key='s')\n",
    "    if multiclass:\n",
    "        y_test = y_test-1\n",
    "    X_test_tensors = [torch.tensor(x, dtype=torch.int64) for x in X_test]\n",
    "    y_test_tensors = [torch.tensor(y, dtype=torch.int64) for y in y_test]\n",
    "    if truncate:\n",
    "        X_test_tensors = [pad_or_truncate(x, maxlen=maxlen, pad=False) for x in X_test_tensors] # just truncate no padding to save GPU memory\n",
    "    test_dataset = CustomDataset(X_test_tensors, y_test_tensors)\n",
    "    test_dataloader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, collate_fn=pad_and_sort_batch)\n",
    "    return test_dataloader\n",
    "\n",
    "\n",
    "class CustomDatasetHierarchical(Dataset):\n",
    "    def __init__(self, features, labels):\n",
    "        self.features = features\n",
    "        self.labels = labels\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.features)\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        return self.features[idx], self.labels[idx]\n",
    "    \n",
    "# def prepare_data_for_test_hierarchical(data_pth, batch_size=32, multiclass=False, max_sen_len=50, max_sen_num=50):\n",
    "#     X_test = pd.read_hdf(data_pth+'X_test.h5', key='s')\n",
    "#     y_test = pd.read_hdf(data_pth+'y_test.h5', key='s')\n",
    "#     if multiclass:\n",
    "#         y_test = y_test-1\n",
    "\n",
    "#     test_dataset = CustomDatasetHierarchical(X_test, y_test)\n",
    "\n",
    "#     test_dataloader = DataLoader(test_dataset, \n",
    "#                                   batch_size=batch_size, \n",
    "#                                   shuffle=False, \n",
    "#                                   collate_fn=partial(pad_and_truncate_hierarchical, MAX_SEN_LEN=max_sen_len,                                                     MAX_SEN_NUM=max_sen_num), \n",
    "#                                   )\n",
    "#     return test_dataloader\n",
    "\n",
    "\n",
    "def prepare_data_for_test_hierarchical(data_pth, batch_size=32, multiclass=False, max_sen_len=50, max_sen_num=50):\n",
    "    X_test = pd.read_hdf(data_pth+'X_test.h5', key='s')\n",
    "    y_test = pd.read_hdf(data_pth+'y_test.h5', key='s')\n",
    "    if multiclass:\n",
    "        y_test = y_test-1\n",
    "\n",
    "    X_test_sorted, y_test_sorted = preprocess_for_batch_hierarchical(X_test, y_test, batch_size)\n",
    "\n",
    "    test_dataset = CustomDatasetHierarchical(X_test_sorted, y_test_sorted)\n",
    "\n",
    "    test_dataloader = DataLoader(test_dataset, \n",
    "                                  batch_size=batch_size, \n",
    "                                  sampler=SequentialSampler(test_dataset), \n",
    "                                  shuffle=False, \n",
    "                                  collate_fn=partial(pad_and_truncate_hierarchical, MAX_SEN_LEN=max_sen_len,                                                     MAX_SEN_NUM=max_sen_num))\n",
    "    return test_dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-30T10:59:02.434938Z",
     "start_time": "2019-04-30T10:58:54.740833Z"
    }
   },
   "outputs": [],
   "source": [
    "yelp_polarity_data = prepare_data_for_test(base_dir + 'dataset/yelp_review_polarity_csv/',  batch_size=16, multiclass=False, truncate=False, maxlen=500)\n",
    "yelp_full_data = prepare_data_for_test(base_dir + 'dataset/yelp_review_full_csv/',  batch_size=16, multiclass=True, truncate=False, maxlen=500)\n",
    "imdb_data = prepare_data_for_test(base_dir + 'dataset/IMDB/',  batch_size=16, multiclass=False, truncate=False, maxlen=500)\n",
    "douban_data = prepare_data_for_test(base_dir + 'dataset/Douban/',  batch_size=16, multiclass=False, truncate=False, maxlen=500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-30T10:59:11.002937Z",
     "start_time": "2019-04-30T10:59:02.437946Z"
    }
   },
   "outputs": [],
   "source": [
    "yelp_polarity_hierarchical_data = prepare_data_for_test_hierarchical(base_dir + 'dataset_hierarchical/yelp_review_polarity_csv/',  batch_size=32, multiclass=False)\n",
    "\n",
    "yelp_full_hierarchical_data = prepare_data_for_test_hierarchical(base_dir + 'dataset_hierarchical/yelp_review_full_csv/',  batch_size=32, multiclass=True)\n",
    "imdb_hierarchical_data = prepare_data_for_test_hierarchical(base_dir + 'dataset_hierarchical/IMDB/',  batch_size=32, multiclass=False)\n",
    "douban_hierarchical_data = prepare_data_for_test_hierarchical(base_dir + 'dataset_hierarchical/Douban/',  batch_size=32, multiclass=False, max_sen_len=25, max_sen_num=25)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load trained models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-30T10:59:23.314854Z",
     "start_time": "2019-04-30T10:59:23.169939Z"
    }
   },
   "outputs": [],
   "source": [
    "from AttentionBiLSTM import AttnBiLSTM\n",
    "from AvgBiLSTM import AvgBiLSTM\n",
    "from SIAttentionBiLSTM import SIAttentionBiLSTM\n",
    "from HAN import HAN\n",
    "from Transformer import Transformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-30T10:59:26.793538Z",
     "start_time": "2019-04-30T10:59:26.645056Z"
    }
   },
   "outputs": [],
   "source": [
    "def load_AttnBiLSTM_model(path, output_size=1):\n",
    "    history = torch.load(path)\n",
    "    parameters = history['parameters']\n",
    "    model = AttnBiLSTM(output_size=output_size, hidden_size=parameters['hidden_size'], embedding=parameters['embedding'], method=parameters['att_method'], rnn_dropout=parameters['rnn_dropout'], embedding_dropout=parameters['embedding_dropout'], context_dropout=parameters['context_dropout'])\n",
    "    model.to(DEVICE)\n",
    "    model.load_state_dict(history['best_model'])\n",
    "    return model, history\n",
    "\n",
    "def load_Transformer_model(path, output_size=1):\n",
    "    history = torch.load(path)\n",
    "    parameters = history['parameters']\n",
    "    architecture = (parameters['n_layers'], parameters['d_model'], parameters['d_ff'], parameters['n_heads'], parameters['d_k'], parameters['d_v'])\n",
    "    model = Transformer(output_size=output_size, architecture=architecture, embedding=parameters['embedding'], method=parameters['final_att_method'], maxpos=500, embedding_dropout=parameters['embedding_dropout'], multihead_dropout=parameters['multihead_dropout'], att_dropout=parameters['att_dropout'], feedforward_dropout=parameters['feedforward_dropout'], final_dropout=parameters['final_dropout'])\n",
    "    model.to(DEVICE)\n",
    "    model.load_state_dict(history['best_model'])\n",
    "    return model, history\n",
    "\n",
    "def load_SIAttentionBiLSTM_model(path, output_size=1):\n",
    "    history = torch.load(path)\n",
    "    parameters = history['parameters']\n",
    "    model = SIAttentionBiLSTM(output_size=output_size, n_layers=parameters['n_layers'], d_model=parameters['d_model'], d_k=parameters['d_k'], d_v=parameters['d_v'], n_heads=parameters['n_heads'], hidden_size=parameters['hidden_size'], embedding=parameters['embedding'], method=parameters['att_method'], rnn_dropout=parameters['rnn_dropout'], embedding_dropout=parameters['embedding_dropout'], multihead_dropout=parameters['multihead_dropout'], context_dropout=parameters['context_dropout'], transformer_dropout=parameters['transformer_dropout'], inter_att_dropout=parameters['inter_att_dropout'], self_att_dropout=parameters['self_att_dropout'])\n",
    "    model.to(DEVICE)\n",
    "    model.load_state_dict(history['best_model'])\n",
    "    return model, history\n",
    "\n",
    "def load_HAN_model(path, output_size=1):\n",
    "    history = torch.load(path)\n",
    "    parameters = history['parameters']\n",
    "    model = HAN(output_size=output_size, hidden_size=parameters['hidden_size'], embedding=parameters['embedding'], method=parameters['att_method'], rnn_dropout=parameters['rnn_dropout'], embedding_dropout=parameters['embedding_dropout'], word_dropout=parameters['word_dropout'], sent_dropout=parameters['sent_dropout'])\n",
    "    model.to(DEVICE)\n",
    "    model.load_state_dict(history['best_model'], strict=False)\n",
    "    return model, history\n",
    "\n",
    "def load_AvgBiLSTM_model(path, output_size=1):\n",
    "    history = torch.load(path)\n",
    "    parameters = history['parameters']\n",
    "    model = AvgBiLSTM(output_size=output_size, hidden_size=parameters['hidden_size'], embedding=parameters['embedding'], rnn_dropout=parameters['rnn_dropout'], embedding_dropout=parameters['embedding_dropout'])\n",
    "    model.to(DEVICE)\n",
    "    model.load_state_dict(history['best_model'])\n",
    "    return model, history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-30T10:59:28.821441Z",
     "start_time": "2019-04-30T10:59:28.683073Z"
    }
   },
   "outputs": [],
   "source": [
    "from IPython.core.debugger import set_trace\n",
    "def predict(model, data_loader, input_len=True, multiclass=False, hierarchical=False):\n",
    "    model.eval()\n",
    "    y_true = []\n",
    "    y_pred = []\n",
    "    with torch.no_grad():\n",
    "        with tqdm(total=len(data_loader.dataset)) as pbar:\n",
    "            for idx, batch in enumerate(data_loader):\n",
    "                if hierarchical:\n",
    "                    input_batch, labels = batch\n",
    "                    if not multiclass:\n",
    "                        labels = labels.float()\n",
    "                    else:\n",
    "                        labels = labels.squeeze()\n",
    "                else:\n",
    "                    input_batch, labels, lengths = batch\n",
    "                if not multiclass and not hierarchical:\n",
    "                    labels = labels.float().unsqueeze(1) # polarity\n",
    "                if input_len:\n",
    "                    output = model(input_batch, lengths)\n",
    "                else:\n",
    "                    output = model(input_batch)\n",
    "                y_true.append(labels.cpu())\n",
    "                y_pred.append(output.cpu())\n",
    "                pbar.update(input_batch.size(0))\n",
    "    return torch.cat(y_pred), torch.cat(y_true)\n",
    "    return torch.tensor(np.array(y_pred).reshape(-1, y_pred[0].shape[-1])), torch.tensor(np.array(y_true).reshape(-1, y_pred[0].shape[-1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loss & Accuracy & Precision & Recall & F1 score & roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-30T10:59:29.813251Z",
     "start_time": "2019-04-30T10:59:29.676862Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import f1_score, precision_score, recall_score, roc_auc_score\n",
    "def evaluate(y_pred, y_true, multiclass=False, model_name='', dataset=''):\n",
    "    if multiclass:\n",
    "        criterion = nn.CrossEntropyLoss()\n",
    "        loss = criterion(y_pred, y_true).item()\n",
    "        _, preds = torch.max(y_pred, 1)\n",
    "        accuracy = (preds == y_true).float().mean().item()\n",
    "        return pd.Series([model_name, dataset, accuracy, loss], index=['model', 'dataset', 'accuracy', 'loss'])\n",
    "    else:\n",
    "        criterion = nn.BCEWithLogitsLoss()\n",
    "        loss = criterion(y_pred, y_true).item()\n",
    "        accuracy = (y_pred>0).eq(y_true>0).float().mean().item()\n",
    "        f1 = f1_score(y_true, y_pred>0)\n",
    "        precision = precision_score(y_true, y_pred>0)\n",
    "        recall = recall_score(y_true, y_pred>0)\n",
    "        auc = roc_auc_score(y_true, y_pred)\n",
    "        return pd.Series([model_name, dataset, accuracy, loss, f1, precision, recall, auc], index=['model', 'dataset', 'accuracy', 'loss', 'f1', 'precision', 'recall', 'auc'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate all the models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-30T10:59:33.933007Z",
     "start_time": "2019-04-30T10:59:33.800612Z"
    }
   },
   "outputs": [],
   "source": [
    "#evaluate_df = pd.DataFrame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-30T10:59:45.393577Z",
     "start_time": "2019-04-30T10:59:44.959201Z"
    }
   },
   "outputs": [],
   "source": [
    "evaluate_df = pd.read_csv('evaluate_df.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-09T04:04:31.360559Z",
     "start_time": "2019-04-09T04:04:31.245253Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Attention-c-query-BiLSTM-on-Douban.pth',\n",
       " 'Attention-c-query-BiLSTM-on-IMDB.pth',\n",
       " 'Attention-c-query-BiLSTM-on-Yelp-Full.pth',\n",
       " 'Attention-c-query-BiLSTM-on-Yelp-Polarity.pth',\n",
       " 'Attention-h-query-BiLSTM-on-Douban.pth',\n",
       " 'Attention-h-query-BiLSTM-on-IMDB.pth',\n",
       " 'Attention-h-query-BiLSTM-on-Yelp-Full.pth',\n",
       " 'Attention-h-query-BiLSTM-on-Yelp-Polarity.pth',\n",
       " 'Attention-independent-query-BiLSTM-on-Douban.pth',\n",
       " 'Attention-independent-query-BiLSTM-on-IMDB.pth',\n",
       " 'Attention-independent-query-BiLSTM-on-Yelp-Full.pth',\n",
       " 'Attention-independent-query-BiLSTM-on-Yelp-Polarity.pth',\n",
       " 'Attention-no-query-BiLSTM-on-Douban.pth',\n",
       " 'Attention-no-query-BiLSTM-on-IMDB.pth',\n",
       " 'Attention-no-query-BiLSTM-on-Yelp-Full.pth',\n",
       " 'Attention-no-query-BiLSTM-on-Yelp-Polarity.pth']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.listdir('./model save/Inter-Attention-BiLSTM')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-09T04:04:31.481908Z",
     "start_time": "2019-04-09T04:04:31.363567Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Transformer-on-Douban.pth',\n",
       " 'Transformer-on-IMDB.pth',\n",
       " 'Transformer-on-Yelp-Full.pth',\n",
       " 'Transformer-on-Yelp-Polarity.pth']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.listdir('./model save/Transformer')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-09T04:04:31.649363Z",
     "start_time": "2019-04-09T04:04:31.529007Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Self-Inter-Attention-BiLSTM-on-Douban.pth',\n",
       " 'Self-Inter-Attention-BiLSTM-on-IMDB.pth',\n",
       " 'Self-Inter-Attention-BiLSTM-on-Yelp-Full.pth',\n",
       " 'Self-Inter-Attention-BiLSTM-on-Yelp-Polarity.pth']"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.listdir('./model save/Self-Inter-Attention-BiLSTM')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-09T04:04:32.050227Z",
     "start_time": "2019-04-09T04:04:31.927930Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Hierarchical-Attention-Networks-on-Douban.pth',\n",
       " 'Hierarchical-Attention-Networks-on-IMDB.pth',\n",
       " 'Hierarchical-Attention-Networks-on-Yelp-Full.pth',\n",
       " 'Hierarchical-Attention-Networks-on-Yelp-Polarity.pth']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.listdir('./model save/HAN/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-09T04:04:32.711416Z",
     "start_time": "2019-04-09T04:04:32.597855Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Avg-BiLSTM-on-Douban.pth',\n",
       " 'Avg-BiLSTM-on-IMDB.pth',\n",
       " 'Avg-BiLSTM-on-Yelp-Full.pth',\n",
       " 'Avg-BiLSTM-on-Yelp-Polarity.pth']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.listdir('./model save/Avg-BiLSTM')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Avg-BiLSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-07T02:22:19.722476Z",
     "start_time": "2019-04-07T02:21:33.203228Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████| 100000/100000 [00:42<00:00, 2368.30it/s]\n"
     ]
    }
   ],
   "source": [
    "avg_bilstm_on_douban, avg_bilstm_on_douban_history = load_AvgBiLSTM_model(base_dir+'model save/Avg-BiLSTM/' + 'Avg-BiLSTM-on-Douban.pth')\n",
    "y_pred, y_true = predict(avg_bilstm_on_douban, douban_data)\n",
    "evaluate_df = evaluate_df.append(evaluate(y_pred, y_true, model_name='Avg-BiLSTM-on-Douban', dataset='Douban'), ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-07T02:22:29.791481Z",
     "start_time": "2019-04-07T02:22:19.724482Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████| 5000/5000 [00:09<00:00, 521.55it/s]\n"
     ]
    }
   ],
   "source": [
    "avg_bilstm_on_imdb, avg_bilstm_on_imdb_history = load_AvgBiLSTM_model(base_dir+'model save/Avg-BiLSTM/' + 'Avg-BiLSTM-on-IMDB.pth')\n",
    "y_pred, y_true = predict(avg_bilstm_on_imdb, imdb_data)\n",
    "evaluate_df = evaluate_df.append(evaluate(y_pred, y_true, model_name='Avg-BiLSTM-on-IMDB', dataset='IMDB'), ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-07T02:23:24.910251Z",
     "start_time": "2019-04-07T02:22:29.794507Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████| 38000/38000 [00:54<00:00, 699.41it/s]\n"
     ]
    }
   ],
   "source": [
    "avg_bilstm_on_yelp_polarity, avg_bilstm_on_yelp_polarity_history = load_AvgBiLSTM_model(base_dir+'model save/Avg-BiLSTM/' + 'Avg-BiLSTM-on-Yelp-Polarity.pth')\n",
    "y_pred, y_true = predict(avg_bilstm_on_yelp_polarity, yelp_polarity_data)\n",
    "evaluate_df = evaluate_df.append(evaluate(y_pred, y_true, model_name='Avg-BiLSTM-on-Yelp-Polarity', dataset='Yelp-Polarity'), ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-07T02:24:37.448882Z",
     "start_time": "2019-04-07T02:23:24.912257Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████| 50000/50000 [01:11<00:00, 697.27it/s]\n"
     ]
    }
   ],
   "source": [
    "avg_bilstm_on_yelp_full, avg_bilstm_on_yelp_full_history = load_AvgBiLSTM_model(base_dir+'model save/Avg-BiLSTM/' + 'Avg-BiLSTM-on-Yelp-Full.pth', output_size=5)\n",
    "y_pred, y_true = predict(avg_bilstm_on_yelp_full, yelp_full_data, multiclass=True)\n",
    "evaluate_df = evaluate_df.append(evaluate(y_pred, y_true, model_name='Avg-BiLSTM-on-Yelp-Full', dataset='Yelp-Full', multiclass=True), ignore_index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Inter-Attention-BiLSTM"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### c-query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-07T02:40:26.263297Z",
     "start_time": "2019-04-07T02:39:28.174406Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████| 100000/100000 [00:54<00:00, 1828.70it/s]\n"
     ]
    }
   ],
   "source": [
    "attn_bilstm_c_query_on_douban, attn_bilstm_c_query_on_douban_history = load_AttnBiLSTM_model(base_dir+'model save/Inter-Attention-BiLSTM/' + 'Attention-c-query-BiLSTM-on-Douban.pth')\n",
    "y_pred, y_true = predict(attn_bilstm_c_query_on_douban, douban_data)\n",
    "evaluate_df = evaluate_df.append(evaluate(y_pred, y_true, model_name='Inter-Attention-c-query-BiLSTM-on-Douban', dataset='Douban'), ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-07T02:41:25.525219Z",
     "start_time": "2019-04-07T02:41:11.126724Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████| 5000/5000 [00:13<00:00, 354.94it/s]\n"
     ]
    }
   ],
   "source": [
    "attn_bilstm_c_query_on_imdb, attn_bilstm_c_query_on_imdb_history = load_AttnBiLSTM_model(base_dir+'model save/Inter-Attention-BiLSTM/' + 'Attention-c-query-BiLSTM-on-IMDB.pth')\n",
    "y_pred, y_true = predict(attn_bilstm_c_query_on_imdb, imdb_data)\n",
    "evaluate_df = evaluate_df.append(evaluate(y_pred, y_true, model_name='Inter-Attention-c-query-BiLSTM-on-IMDB', dataset='IMDB'), ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-07T02:44:52.770773Z",
     "start_time": "2019-04-07T02:43:37.821201Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████| 38000/38000 [01:14<00:00, 456.00it/s]\n"
     ]
    }
   ],
   "source": [
    "attn_bilstm_c_query_on_yelp_polarity, attn_bilstm_c_query_on_yelp_polarity_history = load_AttnBiLSTM_model(base_dir+'model save/Inter-Attention-BiLSTM/' + 'Attention-c-query-BiLSTM-on-Yelp-Polarity.pth')\n",
    "y_pred, y_true = predict(attn_bilstm_c_query_on_yelp_polarity, yelp_polarity_data)\n",
    "evaluate_df = evaluate_df.append(evaluate(y_pred, y_true, model_name='Inter-Attention-c-query-BiLSTM-on-Yelp-Polarity', dataset='Yelp-Polarity'), ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-07T02:47:57.955412Z",
     "start_time": "2019-04-07T02:46:17.715865Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████| 50000/50000 [01:37<00:00, 542.17it/s]\n"
     ]
    }
   ],
   "source": [
    "attn_bilstm_c_query_on_yelp_full, attn_bilstm_c_query_on_yelp_full_history = load_AttnBiLSTM_model(base_dir+'model save/Inter-Attention-BiLSTM/' + 'Attention-c-query-BiLSTM-on-Yelp-Full.pth', output_size=5)\n",
    "y_pred, y_true = predict(attn_bilstm_c_query_on_yelp_full, yelp_full_data, multiclass=True)\n",
    "evaluate_df = evaluate_df.append(evaluate(y_pred, y_true, model_name='Inter-Attention-c-query-BiLSTM-on-Yelp-Full', dataset='Yelp-Full', multiclass=True), ignore_index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### h-query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-07T03:03:58.261613Z",
     "start_time": "2019-04-07T03:02:57.917823Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████| 100000/100000 [00:56<00:00, 1762.88it/s]\n"
     ]
    }
   ],
   "source": [
    "attn_bilstm_h_query_on_douban, attn_bilstm_h_query_on_douban_history = load_AttnBiLSTM_model(base_dir+'model save/Inter-Attention-BiLSTM/' + 'Attention-h-query-BiLSTM-on-Douban.pth')\n",
    "y_pred, y_true = predict(attn_bilstm_h_query_on_douban, douban_data)\n",
    "evaluate_df = evaluate_df.append(evaluate(y_pred, y_true, model_name='Inter-Attention-h-query-BiLSTM-on-Douban', dataset='Douban'), ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-07T03:04:15.790027Z",
     "start_time": "2019-04-07T03:04:01.080315Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████| 5000/5000 [00:13<00:00, 376.54it/s]\n"
     ]
    }
   ],
   "source": [
    "attn_bilstm_h_query_on_imdb, attn_bilstm_h_query_on_imdb_history = load_AttnBiLSTM_model(base_dir+'model save/Inter-Attention-BiLSTM/' + 'Attention-h-query-BiLSTM-on-IMDB.pth')\n",
    "y_pred, y_true = predict(attn_bilstm_h_query_on_imdb, imdb_data)\n",
    "evaluate_df = evaluate_df.append(evaluate(y_pred, y_true, model_name='Inter-Attention-h-query-BiLSTM-on-IMDB', dataset='IMDB'), ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-07T03:05:35.482415Z",
     "start_time": "2019-04-07T03:04:18.424863Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████| 38000/38000 [01:14<00:00, 509.71it/s]\n"
     ]
    }
   ],
   "source": [
    "attn_bilstm_h_query_on_yelp_polarity, attn_bilstm_h_query_on_yelp_polarity_history = load_AttnBiLSTM_model(base_dir+'model save/Inter-Attention-BiLSTM/' + 'Attention-h-query-BiLSTM-on-Yelp-Polarity.pth')\n",
    "y_pred, y_true = predict(attn_bilstm_h_query_on_yelp_polarity, yelp_polarity_data)\n",
    "evaluate_df = evaluate_df.append(evaluate(y_pred, y_true, model_name='Inter-Attention-h-query-BiLSTM-on-Yelp-Polarity', dataset='Yelp-Polarity'), ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-07T03:07:44.348155Z",
     "start_time": "2019-04-07T03:06:06.165969Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████| 50000/50000 [01:37<00:00, 513.61it/s]\n"
     ]
    }
   ],
   "source": [
    "attn_bilstm_h_query_on_yelp_full, attn_bilstm_h_query_on_yelp_full_history = load_AttnBiLSTM_model(base_dir+'model save/Inter-Attention-BiLSTM/' + 'Attention-h-query-BiLSTM-on-Yelp-Full.pth', output_size=5)\n",
    "y_pred, y_true = predict(attn_bilstm_h_query_on_yelp_full, yelp_full_data, multiclass=True)\n",
    "evaluate_df = evaluate_df.append(evaluate(y_pred, y_true, model_name='Inter-Attention-h-query-BiLSTM-on-Yelp-Full', dataset='Yelp-Full', multiclass=True), ignore_index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### independent parameters query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-07T03:09:01.240073Z",
     "start_time": "2019-04-07T03:08:02.130097Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████| 100000/100000 [00:55<00:00, 1792.79it/s]\n"
     ]
    }
   ],
   "source": [
    "attn_bilstm_independent_query_on_douban, attn_bilstm_independent_query_on_douban_history = load_AttnBiLSTM_model(base_dir+'model save/Inter-Attention-BiLSTM/' + 'Attention-independent-query-BiLSTM-on-Douban.pth')\n",
    "y_pred, y_true = predict(attn_bilstm_independent_query_on_douban, douban_data)\n",
    "evaluate_df = evaluate_df.append(evaluate(y_pred, y_true, model_name='Inter-Attention-independent-query-BiLSTM-on-Douban', dataset='Douban'), ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-07T03:22:28.327416Z",
     "start_time": "2019-04-07T03:22:11.639312Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████| 5000/5000 [00:13<00:00, 373.02it/s]\n"
     ]
    }
   ],
   "source": [
    "attn_bilstm_independent_query_on_imdb, attn_bilstm_independent_query_on_imdb_history = load_AttnBiLSTM_model(base_dir+'model save/Inter-Attention-BiLSTM/' + 'Attention-independent-query-BiLSTM-on-IMDB.pth')\n",
    "y_pred, y_true = predict(attn_bilstm_independent_query_on_imdb, imdb_data)\n",
    "evaluate_df = evaluate_df.append(evaluate(y_pred, y_true, model_name='Inter-Attention-independent-query-BiLSTM-on-IMDB', dataset='IMDB'), ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-07T03:23:45.108448Z",
     "start_time": "2019-04-07T03:22:28.481641Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████| 38000/38000 [01:14<00:00, 511.79it/s]\n"
     ]
    }
   ],
   "source": [
    "attn_bilstm_independent_query_on_yelp_polarity, attn_bilstm_independent_query_on_yelp_polarity_history = load_AttnBiLSTM_model(base_dir+'model save/Inter-Attention-BiLSTM/' + 'Attention-independent-query-BiLSTM-on-Yelp-Polarity.pth')\n",
    "y_pred, y_true = predict(attn_bilstm_independent_query_on_yelp_polarity, yelp_polarity_data)\n",
    "evaluate_df = evaluate_df.append(evaluate(y_pred, y_true, model_name='Inter-Attention-independent-query-BiLSTM-on-Yelp-Polarity', dataset='Yelp-Polarity'), ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-07T03:25:27.303582Z",
     "start_time": "2019-04-07T03:23:45.110454Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████| 50000/50000 [01:39<00:00, 501.79it/s]\n"
     ]
    }
   ],
   "source": [
    "attn_bilstm_independent_query_on_yelp_full, attn_bilstm_independent_query_on_yelp_full_history = load_AttnBiLSTM_model(base_dir+'model save/Inter-Attention-BiLSTM/' + 'Attention-independent-query-BiLSTM-on-Yelp-Full.pth', output_size=5)\n",
    "y_pred, y_true = predict(attn_bilstm_independent_query_on_yelp_full, yelp_full_data, multiclass=True)\n",
    "evaluate_df = evaluate_df.append(evaluate(y_pred, y_true, model_name='Inter-Attention-independent-query-BiLSTM-on-Yelp-Full', dataset='Yelp-Full', multiclass=True), ignore_index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Transformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-30T11:02:50.169638Z",
     "start_time": "2019-04-30T11:01:07.274925Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████| 100000/100000 [01:33<00:00, 1072.14it/s]\n"
     ]
    }
   ],
   "source": [
    "transformer_on_douban, transformer_on_douban_history = load_Transformer_model(base_dir+'model save/Transformer/' + 'Transformer-on-Douban.pth')\n",
    "y_pred, y_true = predict(transformer_on_douban, douban_data, input_len=False)\n",
    "evaluate_df = evaluate_df.append(evaluate(y_pred, y_true, model_name='Transformer-on-Douban', dataset='Douban'), ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-07T03:45:43.400045Z",
     "start_time": "2019-04-07T03:44:58.440198Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████| 5000/5000 [00:42<00:00, 116.43it/s]\n"
     ]
    }
   ],
   "source": [
    "transformer_on_imdb, transformer_on_imdb_history = load_Transformer_model(base_dir+'model save/Transformer/' + 'Transformer-on-IMDB.pth')\n",
    "y_pred, y_true = predict(transformer_on_imdb, imdb_data, input_len=False)\n",
    "evaluate_df = evaluate_df.append(evaluate(y_pred, y_true, model_name='Transformer-on-IMDB', dataset='IMDB'), ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-07T03:48:44.669987Z",
     "start_time": "2019-04-07T03:45:43.402086Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████| 38000/38000 [02:58<00:00, 213.16it/s]\n"
     ]
    }
   ],
   "source": [
    "transformer_on_yelp_polarity, transformer_on_yelp_polarity_history = load_Transformer_model(base_dir+'model save/Transformer/' + 'Transformer-on-Yelp-Polarity.pth')\n",
    "y_pred, y_true = predict(transformer_on_yelp_polarity, yelp_polarity_data, input_len=False)\n",
    "evaluate_df = evaluate_df.append(evaluate(y_pred, y_true, model_name='Transformer-on-Yelp-Polarity', dataset='Yelp-Polarity'), ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-07T03:52:44.095140Z",
     "start_time": "2019-04-07T03:48:44.671958Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████| 50000/50000 [03:56<00:00, 211.30it/s]\n"
     ]
    }
   ],
   "source": [
    "transformer_on_yelp_full, transformer_on_yelp_full_history = load_Transformer_model(base_dir+'model save/Transformer/' + 'Transformer-on-Yelp-Full.pth', output_size=5)\n",
    "y_pred, y_true = predict(transformer_on_yelp_full, yelp_full_data, multiclass=True, input_len=False)\n",
    "evaluate_df = evaluate_df.append(evaluate(y_pred, y_true, model_name='Transformer-on-Yelp-Full', dataset='Yelp-Full', multiclass=True), ignore_index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Self-Inter-Attention-BiLSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-07T02:25:35.975710Z",
     "start_time": "2019-04-07T02:24:37.450887Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████| 100000/100000 [00:57<00:00, 1745.89it/s]\n"
     ]
    }
   ],
   "source": [
    "si_attn_bilstm_on_douban, si_attn_bilstm_on_douban_history = load_SIAttentionBiLSTM_model(base_dir+'model save/Self-Inter-Attention-BiLSTM/' + 'Self-Inter-Attention-BiLSTM-on-Douban.pth')\n",
    "y_pred, y_true = predict(si_attn_bilstm_on_douban, douban_data)\n",
    "evaluate_df = evaluate_df.append(evaluate(y_pred, y_true, model_name='Self-Inter-Attention-BiLSTM-on-Douban', dataset='Douban'), ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-07T02:25:56.449675Z",
     "start_time": "2019-04-07T02:25:37.656257Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████| 5000/5000 [00:18<00:00, 243.38it/s]\n"
     ]
    }
   ],
   "source": [
    "si_attn_bilstm_on_imdb, si_attn_bilstm_on_imdb_history = load_SIAttentionBiLSTM_model(base_dir+'model save/Self-Inter-Attention-BiLSTM/' + 'Self-Inter-Attention-BiLSTM-on-IMDB.pth')\n",
    "y_pred, y_true = predict(si_attn_bilstm_on_imdb, imdb_data)\n",
    "evaluate_df = evaluate_df.append(evaluate(y_pred, y_true, model_name='Self-Inter-Attention-BiLSTM-on-IMDB', dataset='IMDB'), ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-07T02:27:50.985819Z",
     "start_time": "2019-04-07T02:26:13.671919Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████| 38000/38000 [01:34<00:00, 402.86it/s]\n"
     ]
    }
   ],
   "source": [
    "si_attn_bilstm_on_yelp_polarity, si_attn_bilstm_on_yelp_polarity_history = load_SIAttentionBiLSTM_model(base_dir+'model save/Self-Inter-Attention-BiLSTM/' + 'Self-Inter-Attention-BiLSTM-on-Yelp-Polarity.pth')\n",
    "y_pred, y_true = predict(si_attn_bilstm_on_yelp_polarity, yelp_polarity_data)\n",
    "evaluate_df = evaluate_df.append(evaluate(y_pred, y_true, model_name='Self-Inter-Attention-BiLSTM-on-Yelp-Polarity', dataset='Yelp-Polarity'), ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-07T02:31:42.966948Z",
     "start_time": "2019-04-07T02:29:35.290857Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████| 50000/50000 [02:04<00:00, 402.08it/s]\n"
     ]
    }
   ],
   "source": [
    "si_attn_bilstm_on_yelp_full, si_attn_bilstm_on_yelp_full_history = load_SIAttentionBiLSTM_model(base_dir+'model save/Self-Inter-Attention-BiLSTM/' + 'Self-Inter-Attention-BiLSTM-on-Yelp-Full.pth', output_size=5)\n",
    "y_pred, y_true = predict(si_attn_bilstm_on_yelp_full, yelp_full_data, multiclass=True)\n",
    "evaluate_df = evaluate_df.append(evaluate(y_pred, y_true, model_name='Self-Inter-Attention-BiLSTM-on-Yelp-Full', dataset='Yelp-Full', multiclass=True), ignore_index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Hierarchical-Attention-Networks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-09T03:55:51.689998Z",
     "start_time": "2019-04-09T03:55:51.570708Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Hierarchical-Attention-Networks-on-Douban.pth',\n",
       " 'Hierarchical-Attention-Networks-on-IMDB.pth',\n",
       " 'Hierarchical-Attention-Networks-on-Yelp-Full.pth',\n",
       " 'Hierarchical-Attention-Networks-on-Yelp-Polarity.pth']"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.listdir('./model save/HAN/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-09T03:59:11.241854Z",
     "start_time": "2019-04-09T03:57:31.979320Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████| 100000/100000 [01:38<00:00, 1019.43it/s]\n"
     ]
    }
   ],
   "source": [
    "hierarchical_on_douban, hierarchical_on_douban_history = load_HAN_model(base_dir+'model save/HAN/' + 'Hierarchical-Attention-Networks-on-Douban.pth')\n",
    "y_pred, y_true = predict(hierarchical_on_douban, douban_hierarchical_data, input_len=False, hierarchical=True)\n",
    "evaluate_df = evaluate_df.append(evaluate(y_pred, y_true, model_name='Hierarchical-on-Douban', dataset='Douban'), ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-09T04:02:01.002353Z",
     "start_time": "2019-04-09T04:01:39.300132Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████| 4992/4992 [00:21<00:00, 224.19it/s]\n"
     ]
    }
   ],
   "source": [
    "hierarchical_on_imdb, hierarchical_on_imdb_history = load_HAN_model(base_dir+'model save/HAN/' + 'Hierarchical-Attention-Networks-on-IMDB.pth')\n",
    "y_pred, y_true = predict(hierarchical_on_imdb, imdb_hierarchical_data, input_len=False, hierarchical=True)\n",
    "evaluate_df = evaluate_df.append(evaluate(y_pred, y_true, model_name='Hierarchical-on-IMDB', dataset='IMDB'), ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-09T04:05:34.426469Z",
     "start_time": "2019-04-09T04:04:46.054535Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████| 37984/37984 [00:44<00:00, 848.28it/s]\n"
     ]
    }
   ],
   "source": [
    "hierarchical_on_yelp_polarity, hierarchical_on_yelp_polarity_history = load_HAN_model(base_dir+'model save/HAN/' + 'Hierarchical-Attention-Networks-on-Yelp-Polarity.pth')\n",
    "y_pred, y_true = predict(hierarchical_on_yelp_polarity, yelp_polarity_hierarchical_data, input_len=False, hierarchical=True)\n",
    "evaluate_df = evaluate_df.append(evaluate(y_pred, y_true, model_name='Hierarchical-on-Yelp-Polarity', dataset='Yelp-Polarity'), ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-09T04:10:39.742153Z",
     "start_time": "2019-04-09T04:09:35.874762Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████| 49984/49984 [01:00<00:00, 862.84it/s]\n"
     ]
    }
   ],
   "source": [
    "hierarchical_on_yelp_full, hierarchical_on_yelp_full_history = load_HAN_model(base_dir+'model save/HAN/' + 'Hierarchical-Attention-Networks-on-Yelp-Full.pth', output_size=5)\n",
    "y_pred, y_true = predict(hierarchical_on_yelp_full, yelp_full_hierarchical_data, multiclass=True, input_len=False, hierarchical=True)\n",
    "evaluate_df = evaluate_df.append(evaluate(y_pred, y_true, model_name='Hierarchical-on-Yelp-Full', dataset='Yelp-Full', multiclass=True), ignore_index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-30T11:09:04.938852Z",
     "start_time": "2019-04-30T11:09:04.529128Z"
    }
   },
   "outputs": [],
   "source": [
    "evaluate_df.to_csv('evaluate_df.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-30T11:08:09.109573Z",
     "start_time": "2019-04-30T11:08:08.951180Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>accuracy</th>\n",
       "      <th>auc</th>\n",
       "      <th>f1</th>\n",
       "      <th>loss</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th>model</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">Douban</th>\n",
       "      <th>Avg-BiLSTM-on-Douban</th>\n",
       "      <td>0.839540</td>\n",
       "      <td>0.906459</td>\n",
       "      <td>0.839325</td>\n",
       "      <td>0.447697</td>\n",
       "      <td>0.840452</td>\n",
       "      <td>0.838200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Hierarchical-on-Douban</th>\n",
       "      <td>0.845100</td>\n",
       "      <td>0.926152</td>\n",
       "      <td>0.848342</td>\n",
       "      <td>0.355895</td>\n",
       "      <td>0.830949</td>\n",
       "      <td>0.866480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Inter-Attention-independent-query-BiLSTM-on-Douban</th>\n",
       "      <td>0.840280</td>\n",
       "      <td>0.922254</td>\n",
       "      <td>0.844658</td>\n",
       "      <td>0.372822</td>\n",
       "      <td>0.822125</td>\n",
       "      <td>0.868460</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Inter-Attention-h-query-BiLSTM-on-Douban</th>\n",
       "      <td>0.841990</td>\n",
       "      <td>0.922871</td>\n",
       "      <td>0.843121</td>\n",
       "      <td>0.369975</td>\n",
       "      <td>0.837129</td>\n",
       "      <td>0.849200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Inter-Attention-c-query-BiLSTM-on-Douban</th>\n",
       "      <td>0.839640</td>\n",
       "      <td>0.921216</td>\n",
       "      <td>0.841748</td>\n",
       "      <td>0.374190</td>\n",
       "      <td>0.830827</td>\n",
       "      <td>0.852960</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Transformer-on-Douban</th>\n",
       "      <td>0.802830</td>\n",
       "      <td>0.884197</td>\n",
       "      <td>0.805255</td>\n",
       "      <td>0.432439</td>\n",
       "      <td>0.795473</td>\n",
       "      <td>0.815280</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Self-Inter-Attention-BiLSTM-on-Douban</th>\n",
       "      <td>0.840970</td>\n",
       "      <td>0.920650</td>\n",
       "      <td>0.843737</td>\n",
       "      <td>0.388087</td>\n",
       "      <td>0.829306</td>\n",
       "      <td>0.858680</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">IMDB</th>\n",
       "      <th>Inter-Attention-h-query-BiLSTM-on-IMDB</th>\n",
       "      <td>0.920000</td>\n",
       "      <td>0.972595</td>\n",
       "      <td>0.921198</td>\n",
       "      <td>0.312038</td>\n",
       "      <td>0.907609</td>\n",
       "      <td>0.935200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Inter-Attention-c-query-BiLSTM-on-IMDB</th>\n",
       "      <td>0.921400</td>\n",
       "      <td>0.975055</td>\n",
       "      <td>0.922378</td>\n",
       "      <td>0.214704</td>\n",
       "      <td>0.911042</td>\n",
       "      <td>0.934000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Hierarchical-on-IMDB</th>\n",
       "      <td>0.924479</td>\n",
       "      <td>0.977709</td>\n",
       "      <td>0.925184</td>\n",
       "      <td>0.199881</td>\n",
       "      <td>0.917717</td>\n",
       "      <td>0.932773</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Self-Inter-Attention-BiLSTM-on-IMDB</th>\n",
       "      <td>0.919400</td>\n",
       "      <td>0.970322</td>\n",
       "      <td>0.918536</td>\n",
       "      <td>0.259638</td>\n",
       "      <td>0.928484</td>\n",
       "      <td>0.908800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Inter-Attention-independent-query-BiLSTM-on-IMDB</th>\n",
       "      <td>0.920200</td>\n",
       "      <td>0.972161</td>\n",
       "      <td>0.920943</td>\n",
       "      <td>0.294275</td>\n",
       "      <td>0.912446</td>\n",
       "      <td>0.929600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Avg-BiLSTM-on-IMDB</th>\n",
       "      <td>0.918600</td>\n",
       "      <td>0.969562</td>\n",
       "      <td>0.919581</td>\n",
       "      <td>0.287007</td>\n",
       "      <td>0.908629</td>\n",
       "      <td>0.930800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Transformer-on-IMDB</th>\n",
       "      <td>0.907200</td>\n",
       "      <td>0.967122</td>\n",
       "      <td>0.908733</td>\n",
       "      <td>0.242542</td>\n",
       "      <td>0.893963</td>\n",
       "      <td>0.924000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">Yelp-Full</th>\n",
       "      <th>Self-Inter-Attention-BiLSTM-on-Yelp-Full</th>\n",
       "      <td>0.662600</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.824653</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Inter-Attention-c-query-BiLSTM-on-Yelp-Full</th>\n",
       "      <td>0.659660</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.798322</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Avg-BiLSTM-on-Yelp-Full</th>\n",
       "      <td>0.666760</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.004622</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Hierarchical-on-Yelp-Full</th>\n",
       "      <td>0.673055</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.751090</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Inter-Attention-h-query-BiLSTM-on-Yelp-Full</th>\n",
       "      <td>0.678240</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.752670</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Inter-Attention-independent-query-BiLSTM-on-Yelp-Full</th>\n",
       "      <td>0.675160</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.762503</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Transformer-on-Yelp-Full</th>\n",
       "      <td>0.600840</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.920770</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">Yelp-Polarity</th>\n",
       "      <th>Hierarchical-on-Yelp-Polarity</th>\n",
       "      <td>0.967750</td>\n",
       "      <td>0.995059</td>\n",
       "      <td>0.967754</td>\n",
       "      <td>0.089756</td>\n",
       "      <td>0.967830</td>\n",
       "      <td>0.967677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Self-Inter-Attention-BiLSTM-on-Yelp-Polarity</th>\n",
       "      <td>0.965684</td>\n",
       "      <td>0.994038</td>\n",
       "      <td>0.965774</td>\n",
       "      <td>0.121675</td>\n",
       "      <td>0.963246</td>\n",
       "      <td>0.968316</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Inter-Attention-independent-query-BiLSTM-on-Yelp-Polarity</th>\n",
       "      <td>0.967658</td>\n",
       "      <td>0.994901</td>\n",
       "      <td>0.967604</td>\n",
       "      <td>0.095921</td>\n",
       "      <td>0.969214</td>\n",
       "      <td>0.966000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Inter-Attention-h-query-BiLSTM-on-Yelp-Polarity</th>\n",
       "      <td>0.966974</td>\n",
       "      <td>0.994831</td>\n",
       "      <td>0.967056</td>\n",
       "      <td>0.099442</td>\n",
       "      <td>0.964650</td>\n",
       "      <td>0.969474</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Inter-Attention-c-query-BiLSTM-on-Yelp-Polarity</th>\n",
       "      <td>0.964553</td>\n",
       "      <td>0.994387</td>\n",
       "      <td>0.964851</td>\n",
       "      <td>0.109183</td>\n",
       "      <td>0.956787</td>\n",
       "      <td>0.973053</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Transformer-on-Yelp-Polarity</th>\n",
       "      <td>0.935447</td>\n",
       "      <td>0.984157</td>\n",
       "      <td>0.935925</td>\n",
       "      <td>0.162424</td>\n",
       "      <td>0.929057</td>\n",
       "      <td>0.942895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Avg-BiLSTM-on-Yelp-Polarity</th>\n",
       "      <td>0.966711</td>\n",
       "      <td>0.987627</td>\n",
       "      <td>0.966780</td>\n",
       "      <td>0.226632</td>\n",
       "      <td>0.964778</td>\n",
       "      <td>0.968789</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                  accuracy  \\\n",
       "dataset       model                                                          \n",
       "Douban        Avg-BiLSTM-on-Douban                                0.839540   \n",
       "              Hierarchical-on-Douban                              0.845100   \n",
       "              Inter-Attention-independent-query-BiLSTM-on-Douban  0.840280   \n",
       "              Inter-Attention-h-query-BiLSTM-on-Douban            0.841990   \n",
       "              Inter-Attention-c-query-BiLSTM-on-Douban            0.839640   \n",
       "              Transformer-on-Douban                               0.802830   \n",
       "              Self-Inter-Attention-BiLSTM-on-Douban               0.840970   \n",
       "IMDB          Inter-Attention-h-query-BiLSTM-on-IMDB              0.920000   \n",
       "              Inter-Attention-c-query-BiLSTM-on-IMDB              0.921400   \n",
       "              Hierarchical-on-IMDB                                0.924479   \n",
       "              Self-Inter-Attention-BiLSTM-on-IMDB                 0.919400   \n",
       "              Inter-Attention-independent-query-BiLSTM-on-IMDB    0.920200   \n",
       "              Avg-BiLSTM-on-IMDB                                  0.918600   \n",
       "              Transformer-on-IMDB                                 0.907200   \n",
       "Yelp-Full     Self-Inter-Attention-BiLSTM-on-Yelp-Full            0.662600   \n",
       "              Inter-Attention-c-query-BiLSTM-on-Yelp-Full         0.659660   \n",
       "              Avg-BiLSTM-on-Yelp-Full                             0.666760   \n",
       "              Hierarchical-on-Yelp-Full                           0.673055   \n",
       "              Inter-Attention-h-query-BiLSTM-on-Yelp-Full         0.678240   \n",
       "              Inter-Attention-independent-query-BiLSTM-on-Yel...  0.675160   \n",
       "              Transformer-on-Yelp-Full                            0.600840   \n",
       "Yelp-Polarity Hierarchical-on-Yelp-Polarity                       0.967750   \n",
       "              Self-Inter-Attention-BiLSTM-on-Yelp-Polarity        0.965684   \n",
       "              Inter-Attention-independent-query-BiLSTM-on-Yel...  0.967658   \n",
       "              Inter-Attention-h-query-BiLSTM-on-Yelp-Polarity     0.966974   \n",
       "              Inter-Attention-c-query-BiLSTM-on-Yelp-Polarity     0.964553   \n",
       "              Transformer-on-Yelp-Polarity                        0.935447   \n",
       "              Avg-BiLSTM-on-Yelp-Polarity                         0.966711   \n",
       "\n",
       "                                                                       auc  \\\n",
       "dataset       model                                                          \n",
       "Douban        Avg-BiLSTM-on-Douban                                0.906459   \n",
       "              Hierarchical-on-Douban                              0.926152   \n",
       "              Inter-Attention-independent-query-BiLSTM-on-Douban  0.922254   \n",
       "              Inter-Attention-h-query-BiLSTM-on-Douban            0.922871   \n",
       "              Inter-Attention-c-query-BiLSTM-on-Douban            0.921216   \n",
       "              Transformer-on-Douban                               0.884197   \n",
       "              Self-Inter-Attention-BiLSTM-on-Douban               0.920650   \n",
       "IMDB          Inter-Attention-h-query-BiLSTM-on-IMDB              0.972595   \n",
       "              Inter-Attention-c-query-BiLSTM-on-IMDB              0.975055   \n",
       "              Hierarchical-on-IMDB                                0.977709   \n",
       "              Self-Inter-Attention-BiLSTM-on-IMDB                 0.970322   \n",
       "              Inter-Attention-independent-query-BiLSTM-on-IMDB    0.972161   \n",
       "              Avg-BiLSTM-on-IMDB                                  0.969562   \n",
       "              Transformer-on-IMDB                                 0.967122   \n",
       "Yelp-Full     Self-Inter-Attention-BiLSTM-on-Yelp-Full                 NaN   \n",
       "              Inter-Attention-c-query-BiLSTM-on-Yelp-Full              NaN   \n",
       "              Avg-BiLSTM-on-Yelp-Full                                  NaN   \n",
       "              Hierarchical-on-Yelp-Full                                NaN   \n",
       "              Inter-Attention-h-query-BiLSTM-on-Yelp-Full              NaN   \n",
       "              Inter-Attention-independent-query-BiLSTM-on-Yel...       NaN   \n",
       "              Transformer-on-Yelp-Full                                 NaN   \n",
       "Yelp-Polarity Hierarchical-on-Yelp-Polarity                       0.995059   \n",
       "              Self-Inter-Attention-BiLSTM-on-Yelp-Polarity        0.994038   \n",
       "              Inter-Attention-independent-query-BiLSTM-on-Yel...  0.994901   \n",
       "              Inter-Attention-h-query-BiLSTM-on-Yelp-Polarity     0.994831   \n",
       "              Inter-Attention-c-query-BiLSTM-on-Yelp-Polarity     0.994387   \n",
       "              Transformer-on-Yelp-Polarity                        0.984157   \n",
       "              Avg-BiLSTM-on-Yelp-Polarity                         0.987627   \n",
       "\n",
       "                                                                        f1  \\\n",
       "dataset       model                                                          \n",
       "Douban        Avg-BiLSTM-on-Douban                                0.839325   \n",
       "              Hierarchical-on-Douban                              0.848342   \n",
       "              Inter-Attention-independent-query-BiLSTM-on-Douban  0.844658   \n",
       "              Inter-Attention-h-query-BiLSTM-on-Douban            0.843121   \n",
       "              Inter-Attention-c-query-BiLSTM-on-Douban            0.841748   \n",
       "              Transformer-on-Douban                               0.805255   \n",
       "              Self-Inter-Attention-BiLSTM-on-Douban               0.843737   \n",
       "IMDB          Inter-Attention-h-query-BiLSTM-on-IMDB              0.921198   \n",
       "              Inter-Attention-c-query-BiLSTM-on-IMDB              0.922378   \n",
       "              Hierarchical-on-IMDB                                0.925184   \n",
       "              Self-Inter-Attention-BiLSTM-on-IMDB                 0.918536   \n",
       "              Inter-Attention-independent-query-BiLSTM-on-IMDB    0.920943   \n",
       "              Avg-BiLSTM-on-IMDB                                  0.919581   \n",
       "              Transformer-on-IMDB                                 0.908733   \n",
       "Yelp-Full     Self-Inter-Attention-BiLSTM-on-Yelp-Full                 NaN   \n",
       "              Inter-Attention-c-query-BiLSTM-on-Yelp-Full              NaN   \n",
       "              Avg-BiLSTM-on-Yelp-Full                                  NaN   \n",
       "              Hierarchical-on-Yelp-Full                                NaN   \n",
       "              Inter-Attention-h-query-BiLSTM-on-Yelp-Full              NaN   \n",
       "              Inter-Attention-independent-query-BiLSTM-on-Yel...       NaN   \n",
       "              Transformer-on-Yelp-Full                                 NaN   \n",
       "Yelp-Polarity Hierarchical-on-Yelp-Polarity                       0.967754   \n",
       "              Self-Inter-Attention-BiLSTM-on-Yelp-Polarity        0.965774   \n",
       "              Inter-Attention-independent-query-BiLSTM-on-Yel...  0.967604   \n",
       "              Inter-Attention-h-query-BiLSTM-on-Yelp-Polarity     0.967056   \n",
       "              Inter-Attention-c-query-BiLSTM-on-Yelp-Polarity     0.964851   \n",
       "              Transformer-on-Yelp-Polarity                        0.935925   \n",
       "              Avg-BiLSTM-on-Yelp-Polarity                         0.966780   \n",
       "\n",
       "                                                                      loss  \\\n",
       "dataset       model                                                          \n",
       "Douban        Avg-BiLSTM-on-Douban                                0.447697   \n",
       "              Hierarchical-on-Douban                              0.355895   \n",
       "              Inter-Attention-independent-query-BiLSTM-on-Douban  0.372822   \n",
       "              Inter-Attention-h-query-BiLSTM-on-Douban            0.369975   \n",
       "              Inter-Attention-c-query-BiLSTM-on-Douban            0.374190   \n",
       "              Transformer-on-Douban                               0.432439   \n",
       "              Self-Inter-Attention-BiLSTM-on-Douban               0.388087   \n",
       "IMDB          Inter-Attention-h-query-BiLSTM-on-IMDB              0.312038   \n",
       "              Inter-Attention-c-query-BiLSTM-on-IMDB              0.214704   \n",
       "              Hierarchical-on-IMDB                                0.199881   \n",
       "              Self-Inter-Attention-BiLSTM-on-IMDB                 0.259638   \n",
       "              Inter-Attention-independent-query-BiLSTM-on-IMDB    0.294275   \n",
       "              Avg-BiLSTM-on-IMDB                                  0.287007   \n",
       "              Transformer-on-IMDB                                 0.242542   \n",
       "Yelp-Full     Self-Inter-Attention-BiLSTM-on-Yelp-Full            0.824653   \n",
       "              Inter-Attention-c-query-BiLSTM-on-Yelp-Full         0.798322   \n",
       "              Avg-BiLSTM-on-Yelp-Full                             1.004622   \n",
       "              Hierarchical-on-Yelp-Full                           0.751090   \n",
       "              Inter-Attention-h-query-BiLSTM-on-Yelp-Full         0.752670   \n",
       "              Inter-Attention-independent-query-BiLSTM-on-Yel...  0.762503   \n",
       "              Transformer-on-Yelp-Full                            0.920770   \n",
       "Yelp-Polarity Hierarchical-on-Yelp-Polarity                       0.089756   \n",
       "              Self-Inter-Attention-BiLSTM-on-Yelp-Polarity        0.121675   \n",
       "              Inter-Attention-independent-query-BiLSTM-on-Yel...  0.095921   \n",
       "              Inter-Attention-h-query-BiLSTM-on-Yelp-Polarity     0.099442   \n",
       "              Inter-Attention-c-query-BiLSTM-on-Yelp-Polarity     0.109183   \n",
       "              Transformer-on-Yelp-Polarity                        0.162424   \n",
       "              Avg-BiLSTM-on-Yelp-Polarity                         0.226632   \n",
       "\n",
       "                                                                  precision  \\\n",
       "dataset       model                                                           \n",
       "Douban        Avg-BiLSTM-on-Douban                                 0.840452   \n",
       "              Hierarchical-on-Douban                               0.830949   \n",
       "              Inter-Attention-independent-query-BiLSTM-on-Douban   0.822125   \n",
       "              Inter-Attention-h-query-BiLSTM-on-Douban             0.837129   \n",
       "              Inter-Attention-c-query-BiLSTM-on-Douban             0.830827   \n",
       "              Transformer-on-Douban                                0.795473   \n",
       "              Self-Inter-Attention-BiLSTM-on-Douban                0.829306   \n",
       "IMDB          Inter-Attention-h-query-BiLSTM-on-IMDB               0.907609   \n",
       "              Inter-Attention-c-query-BiLSTM-on-IMDB               0.911042   \n",
       "              Hierarchical-on-IMDB                                 0.917717   \n",
       "              Self-Inter-Attention-BiLSTM-on-IMDB                  0.928484   \n",
       "              Inter-Attention-independent-query-BiLSTM-on-IMDB     0.912446   \n",
       "              Avg-BiLSTM-on-IMDB                                   0.908629   \n",
       "              Transformer-on-IMDB                                  0.893963   \n",
       "Yelp-Full     Self-Inter-Attention-BiLSTM-on-Yelp-Full                  NaN   \n",
       "              Inter-Attention-c-query-BiLSTM-on-Yelp-Full               NaN   \n",
       "              Avg-BiLSTM-on-Yelp-Full                                   NaN   \n",
       "              Hierarchical-on-Yelp-Full                                 NaN   \n",
       "              Inter-Attention-h-query-BiLSTM-on-Yelp-Full               NaN   \n",
       "              Inter-Attention-independent-query-BiLSTM-on-Yel...        NaN   \n",
       "              Transformer-on-Yelp-Full                                  NaN   \n",
       "Yelp-Polarity Hierarchical-on-Yelp-Polarity                        0.967830   \n",
       "              Self-Inter-Attention-BiLSTM-on-Yelp-Polarity         0.963246   \n",
       "              Inter-Attention-independent-query-BiLSTM-on-Yel...   0.969214   \n",
       "              Inter-Attention-h-query-BiLSTM-on-Yelp-Polarity      0.964650   \n",
       "              Inter-Attention-c-query-BiLSTM-on-Yelp-Polarity      0.956787   \n",
       "              Transformer-on-Yelp-Polarity                         0.929057   \n",
       "              Avg-BiLSTM-on-Yelp-Polarity                          0.964778   \n",
       "\n",
       "                                                                    recall  \n",
       "dataset       model                                                         \n",
       "Douban        Avg-BiLSTM-on-Douban                                0.838200  \n",
       "              Hierarchical-on-Douban                              0.866480  \n",
       "              Inter-Attention-independent-query-BiLSTM-on-Douban  0.868460  \n",
       "              Inter-Attention-h-query-BiLSTM-on-Douban            0.849200  \n",
       "              Inter-Attention-c-query-BiLSTM-on-Douban            0.852960  \n",
       "              Transformer-on-Douban                               0.815280  \n",
       "              Self-Inter-Attention-BiLSTM-on-Douban               0.858680  \n",
       "IMDB          Inter-Attention-h-query-BiLSTM-on-IMDB              0.935200  \n",
       "              Inter-Attention-c-query-BiLSTM-on-IMDB              0.934000  \n",
       "              Hierarchical-on-IMDB                                0.932773  \n",
       "              Self-Inter-Attention-BiLSTM-on-IMDB                 0.908800  \n",
       "              Inter-Attention-independent-query-BiLSTM-on-IMDB    0.929600  \n",
       "              Avg-BiLSTM-on-IMDB                                  0.930800  \n",
       "              Transformer-on-IMDB                                 0.924000  \n",
       "Yelp-Full     Self-Inter-Attention-BiLSTM-on-Yelp-Full                 NaN  \n",
       "              Inter-Attention-c-query-BiLSTM-on-Yelp-Full              NaN  \n",
       "              Avg-BiLSTM-on-Yelp-Full                                  NaN  \n",
       "              Hierarchical-on-Yelp-Full                                NaN  \n",
       "              Inter-Attention-h-query-BiLSTM-on-Yelp-Full              NaN  \n",
       "              Inter-Attention-independent-query-BiLSTM-on-Yel...       NaN  \n",
       "              Transformer-on-Yelp-Full                                 NaN  \n",
       "Yelp-Polarity Hierarchical-on-Yelp-Polarity                       0.967677  \n",
       "              Self-Inter-Attention-BiLSTM-on-Yelp-Polarity        0.968316  \n",
       "              Inter-Attention-independent-query-BiLSTM-on-Yel...  0.966000  \n",
       "              Inter-Attention-h-query-BiLSTM-on-Yelp-Polarity     0.969474  \n",
       "              Inter-Attention-c-query-BiLSTM-on-Yelp-Polarity     0.973053  \n",
       "              Transformer-on-Yelp-Polarity                        0.942895  \n",
       "              Avg-BiLSTM-on-Yelp-Polarity                         0.968789  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate_df.sort_values(by='dataset').set_index(['dataset', 'model'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-30T11:08:27.519284Z",
     "start_time": "2019-04-30T11:08:27.384066Z"
    }
   },
   "outputs": [],
   "source": [
    "df = evaluate_df.set_index('model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-30T11:08:27.955866Z",
     "start_time": "2019-04-30T11:08:27.818953Z"
    }
   },
   "outputs": [],
   "source": [
    "df.index = df.index.str.extract('(.*)-on')[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-30T11:08:29.834473Z",
     "start_time": "2019-04-30T11:08:29.698669Z"
    }
   },
   "outputs": [],
   "source": [
    "douban_accuracy = df[df['dataset']=='Douban']['accuracy']\n",
    "yelp_full_accuracy = df[df['dataset']=='Yelp-Full']['accuracy']\n",
    "yelp_polarity_accuracy = df[df['dataset']=='Yelp-Polarity']['accuracy']\n",
    "imdb_accuracy = df[df['dataset']=='IMDB']['accuracy']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-30T11:08:30.235556Z",
     "start_time": "2019-04-30T11:08:30.087670Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Douban</th>\n",
       "      <th>Yelp-Full</th>\n",
       "      <th>Yelp-Polarity</th>\n",
       "      <th>IMDB</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg-BiLSTM</th>\n",
       "      <td>0.83954</td>\n",
       "      <td>0.666760</td>\n",
       "      <td>0.966711</td>\n",
       "      <td>0.918600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Hierarchical</th>\n",
       "      <td>0.84510</td>\n",
       "      <td>0.673055</td>\n",
       "      <td>0.967750</td>\n",
       "      <td>0.924479</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Inter-Attention-c-query-BiLSTM</th>\n",
       "      <td>0.83964</td>\n",
       "      <td>0.659660</td>\n",
       "      <td>0.964553</td>\n",
       "      <td>0.921400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Inter-Attention-h-query-BiLSTM</th>\n",
       "      <td>0.84199</td>\n",
       "      <td>0.678240</td>\n",
       "      <td>0.966974</td>\n",
       "      <td>0.920000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Inter-Attention-independent-query-BiLSTM</th>\n",
       "      <td>0.84028</td>\n",
       "      <td>0.675160</td>\n",
       "      <td>0.967658</td>\n",
       "      <td>0.920200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Self-Inter-Attention-BiLSTM</th>\n",
       "      <td>0.84097</td>\n",
       "      <td>0.662600</td>\n",
       "      <td>0.965684</td>\n",
       "      <td>0.919400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Transformer</th>\n",
       "      <td>0.80283</td>\n",
       "      <td>0.600840</td>\n",
       "      <td>0.935447</td>\n",
       "      <td>0.907200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           Douban  Yelp-Full  Yelp-Polarity  \\\n",
       "Avg-BiLSTM                                0.83954   0.666760       0.966711   \n",
       "Hierarchical                              0.84510   0.673055       0.967750   \n",
       "Inter-Attention-c-query-BiLSTM            0.83964   0.659660       0.964553   \n",
       "Inter-Attention-h-query-BiLSTM            0.84199   0.678240       0.966974   \n",
       "Inter-Attention-independent-query-BiLSTM  0.84028   0.675160       0.967658   \n",
       "Self-Inter-Attention-BiLSTM               0.84097   0.662600       0.965684   \n",
       "Transformer                               0.80283   0.600840       0.935447   \n",
       "\n",
       "                                              IMDB  \n",
       "Avg-BiLSTM                                0.918600  \n",
       "Hierarchical                              0.924479  \n",
       "Inter-Attention-c-query-BiLSTM            0.921400  \n",
       "Inter-Attention-h-query-BiLSTM            0.920000  \n",
       "Inter-Attention-independent-query-BiLSTM  0.920200  \n",
       "Self-Inter-Attention-BiLSTM               0.919400  \n",
       "Transformer                               0.907200  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_df = pd.concat([douban_accuracy, yelp_full_accuracy, yelp_polarity_accuracy, imdb_accuracy], axis=1)\n",
    "accuracy_df.columns = ['Douban', 'Yelp-Full', 'Yelp-Polarity', 'IMDB']\n",
    "accuracy_df.index.name=None\n",
    "accuracy_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-30T11:08:36.802535Z",
     "start_time": "2019-04-30T11:08:36.664165Z"
    }
   },
   "outputs": [],
   "source": [
    "douban_loss= df[df['dataset']=='Douban']['loss']\n",
    "yelp_full_loss = df[df['dataset']=='Yelp-Full']['loss']\n",
    "yelp_polarity_loss = df[df['dataset']=='Yelp-Polarity']['loss']\n",
    "imdb_loss = df[df['dataset']=='IMDB']['loss']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-30T11:08:37.066491Z",
     "start_time": "2019-04-30T11:08:36.918598Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Douban</th>\n",
       "      <th>Yelp-Full</th>\n",
       "      <th>Yelp-Polarity</th>\n",
       "      <th>IMDB</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Avg-BiLSTM</th>\n",
       "      <td>0.447697</td>\n",
       "      <td>1.004622</td>\n",
       "      <td>0.226632</td>\n",
       "      <td>0.287007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Hierarchical</th>\n",
       "      <td>0.355895</td>\n",
       "      <td>0.751090</td>\n",
       "      <td>0.089756</td>\n",
       "      <td>0.199881</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Inter-Attention-c-query-BiLSTM</th>\n",
       "      <td>0.374190</td>\n",
       "      <td>0.798322</td>\n",
       "      <td>0.109183</td>\n",
       "      <td>0.214704</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Inter-Attention-h-query-BiLSTM</th>\n",
       "      <td>0.369975</td>\n",
       "      <td>0.752670</td>\n",
       "      <td>0.099442</td>\n",
       "      <td>0.312038</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Inter-Attention-independent-query-BiLSTM</th>\n",
       "      <td>0.372822</td>\n",
       "      <td>0.762503</td>\n",
       "      <td>0.095921</td>\n",
       "      <td>0.294275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Self-Inter-Attention-BiLSTM</th>\n",
       "      <td>0.388087</td>\n",
       "      <td>0.824653</td>\n",
       "      <td>0.121675</td>\n",
       "      <td>0.259638</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Transformer</th>\n",
       "      <td>0.432439</td>\n",
       "      <td>0.920770</td>\n",
       "      <td>0.162424</td>\n",
       "      <td>0.242542</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            Douban  Yelp-Full  Yelp-Polarity  \\\n",
       "Avg-BiLSTM                                0.447697   1.004622       0.226632   \n",
       "Hierarchical                              0.355895   0.751090       0.089756   \n",
       "Inter-Attention-c-query-BiLSTM            0.374190   0.798322       0.109183   \n",
       "Inter-Attention-h-query-BiLSTM            0.369975   0.752670       0.099442   \n",
       "Inter-Attention-independent-query-BiLSTM  0.372822   0.762503       0.095921   \n",
       "Self-Inter-Attention-BiLSTM               0.388087   0.824653       0.121675   \n",
       "Transformer                               0.432439   0.920770       0.162424   \n",
       "\n",
       "                                              IMDB  \n",
       "Avg-BiLSTM                                0.287007  \n",
       "Hierarchical                              0.199881  \n",
       "Inter-Attention-c-query-BiLSTM            0.214704  \n",
       "Inter-Attention-h-query-BiLSTM            0.312038  \n",
       "Inter-Attention-independent-query-BiLSTM  0.294275  \n",
       "Self-Inter-Attention-BiLSTM               0.259638  \n",
       "Transformer                               0.242542  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss_df = pd.concat([douban_loss, yelp_full_loss, yelp_polarity_loss, imdb_loss], axis=1)\n",
    "loss_df.columns = ['Douban', 'Yelp-Full', 'Yelp-Polarity', 'IMDB']\n",
    "loss_df.index.name=None\n",
    "loss_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ROC Curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-31T09:05:16.832613Z",
     "start_time": "2019-03-31T09:05:16.420516Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x2ca3d7bf400>"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XmcHHWd//HXZ+4zyWQm9x0IkIskMAQQl0WOiCwEXKOGYxdWhF01sv4UXfy5PzlU3FVXXBR0s4qweITDFYJyCXIjhAQSzEEOcpDJNbnnPnr68/ujasamM0dPMj09M/1+Ph79SFfVt6o+1T2pT3+/36r6mrsjIiICkJHqAEREpO9QUhARkTZKCiIi0kZJQURE2igpiIhIGyUFERFpo6Qg0gUzm2dmj6Q6joHKzE42s1dTHYcElBSkjZltNbN6M6sxs91mdq+ZFcWV+YCZ/dHMqs3ssJk9ZmbT4soMMrMfmNl74bY2hdNlHezXzOwGM1ttZrVmVmFmD5nZzGQebzfcDvxb7Iww5s1mtja+cPg5nh837xozezlmOsfMbjGzjeExbzWze8xsYk8GbmYTzew5M6szs3fi44orO8bMHjWzA+F38E9xyxeb2Xozi5rZNXHLcs3sDjPbaWYHzexuM8uOWb7IzJabWaOZ3Ru7rru/DRwys0t65KDlmCgpSLxL3L0ImA3MAb7ausDMzgSeBh4FRgOTgFXAK2Y2OSyTAzwLTAcuBAYBHwD2A3M72Od/Av8M3AAMBU4AHgH+prvBm1lWd9fpYnunAYPd/bW4RWcDw4HJYZnuehiYD1wBDAZmASuA844h3Pb8GngLKAW+BjxsZsM6KPsLYAswguCzv93MPhSzfBXwWeDNdta9CSgHZhB8f6cA/xqzfCfwTeCeDvb9S+AfEzgeSTZ310sv3B1gK3B+zPR3gN/HTL8E3N3Oek8A/xO+/zSwByhKcJ9TgBZgbidlngc+HTN9DfByzLQDnwM2EpzUfgJ8L24bjwJfDN+PBn4D7A3L39DJvr8O/LSd+fcQnMj+F/hRZ59jfMzA+UA9MC7J3+cJQCNQHPcd/lM7ZYvCz3FYzLzFwP3tlH0ZuCZu3nLg4zHTVwDb21n3m8C97cwfE34muan8P6CXq6Yg7TOzscBHgE3hdAHBL/6H2in+IHBB+P584El3r0lwV+cBFe6+7Ngi5jLgdGAa8Cvgk2ZmAGZWAswDlphZBvAYwa/eMeH+v2BmH+5guzOB9bEzws9iAUFS+CWwMKwhJep8YJm7b090BTP7nZkd6uD1uw5Wmw5sdvfqmHmrwvlH7CLu39b3MxINsZ11x5rZ4ERWdvcdQDNwYoL7kyRRUpB4j5hZNbAdqARuDucPJfh72dXOOruA1v6C0g7KdKS75TvybXc/4O71BL+GHfircNkC4E/uvhM4jeDX8G3u3uTum4H/BhZ2sN0hQHXcvL8l+AX+NPA7IIvuNXV1+5jd/WJ3H9LB6+IOVisCDsfNOwwUt7P9auAV4P+ZWZ6ZnQJ8DChIMMQngH82s2FmNpKgKZBurA/B5zykG+UlCZQUJN5l7l4MnAOcxF9O9geBKDCqnXVGAfvC9/s7KNOR7pbvSNuvbg/aI5YAl4ezriD4RQ8wARgd+0sb+L8E7ejtOciRJ9GrgQfdPeLujQRNSFfHLI8A2XHrZBP8EoaeO+au1BD06cQaxJFJrtWVBP1E24EfE3xmFQnu61sEfRcrgVcJ+oSaCX5YJKoYONSN8pIESgrSLnd/AbgX+F44XQv8Cfh4O8U/QdC5DPAM8GEzK0xwV88SNDOUd1Kmlvf/4hzZXshx078GFpjZBIJmpd+E87cDW+J+aRe7+0Ud7PttgrZ5oK1Z7VzgqvAKrd0ENZGLYq6ueg+YGLedScC28P0zwNxwWwkxsyfCK7naez3RwWprCDrCY5ParHD+Edx9W1gjGebupxPUaBJq1nP3endf5O5j3H0yQeJb4e4tCR7faCCHuKY6SYFUd2ro1XdeHNnRPIzghDw7nP5gOH0Dwa+6EoKOw0PAlLBMLvAG8CRBTSOD4OTyf4GLOtjvDwk6ic8hODHkETTn3BQu/xZBZ3MBcHxYNr6j+fh2trsW+APw25h5mQRX+fwLkB9OzwBO6yC2U4ANMdNfBdYRJKbY12bg82GZfyQ4uZ1E0LZeDuwGLozZztLwczqVoPmpGPgn4FM9/J2+RpDY84CPht/VsA7KTg3jyAGuIqj9xXY8t343rwDXhe8zwmVjCDrwDTiDIPnOi1k3Kyz/beD+8H1WzPIrgMdT/X9AL1dS0Osvr/ikEM77MfCbmOkPhifoGqAK+D0wI26dwcAPwhNDDfAu8H2gtIP9GsElqWuAOmAH8AAwPVxeRtB+39rufUuCSeH/hcs+Hjd/NEFNYjdB89Br8ccdV/4N4PTw/TutJ/+4Ml8BlofvMwgu0dwYfkZrgWvjyucAtxJ05NcS1CJ+Cozv4e90Yvh91RMkqtikfyWwJmb6CwRXZNUSXGFUHret58PPM/Z1Trjs7PDvpy7cz5Vx697Szrq3xCz/PTA/1f8H9HIs/EJEpANmNg/4rLtflupYBqLwJsXF7n5mqmMRlBREROQv1NEsIiJtlBRERKSNkoKIiLTp0YeH9YaysjKfOHFiqsMQEelXVqxYsc/dO3oYYpt+lxQmTpzI8uXLUx2GiEi/Ymbbui6l5iMREYmhpCAiIm2UFEREpI2SgoiItFFSEBGRNklLCuEg5JVmtrqD5WZmd4aDur8dDuohIiIplMyawr0EA7d35CME4/NOAa4neBqniIikUNLuU3D3F81sYidFLiUY7N2B18xsiJmNcveeGJpRRPo4d6cl6kSiTlNLlJYWp8WdaDivJepEPVweieIOjhONEpQLywblCKbdqaqPkJ+TQTTaOg8gtkyw72D9YF7rtvfVNDGkIDucBsIHhvpf3gaPmG47htZlHnNc4b/4X7bTVtbfV6Zte52Ui43jvKkjmDUuuSOWpvLmtTHEDKFIMOzfGNoZu9bMrieoTTB+/PheCU5kIIqGJ+DGSJTGSAv1TS3UN//l38ZIlMbmYFljc5TGliiNzS00tC6LRNm2v5aSgpy2bWzdV0dpUQ6RFqe5JUpz1Hlvfy3DinODedFouMyJRIPtR6JRmlv0hObuMIPhg/IGdFKwdua1+1fi7ouBxQDl5eX6S5K00RSJcri+mZrGCIfqmqhqiFDd0ExtY4SaxuBkvXVfLYW5Wby7t4bCnCwO1TdRWd1IXlYmW/cHy5oi0bYT+9Eyg9ysDLIzM6huiDBuaD45mRlkmLF5by1jhuSTk5VBYWYGwyaUsLe6kfEjC8nOMLIyjcyMDLIzjZzMDLKzMsL5GWRlGofrmxk5KI/MDAteZmSE/2ZlGhlm1DZGKCnMIcOMzAzIMAvfW/geMjOMqEOGQW5WJmZB3K1lMwws/Ld1XmwZgKyM8NRkYATLw0ksnLDw8wjeW1C2k3IWnu4s5qwXP7/D7ceu1AtSmRQqgHEx02OBnSmKRSRpmiJRDtQ2saeqgfrmFuqaIlQ3RKiqb+ZgXTMHapvaXgfrmqhqaGb7gXoyM4yWaOK/gUYNzqO6IcJxwwopzMkiPzuTc08aTlVDhEmlBeRlZ5KbnUledga5WZnkZmWQn51JQU4medmZ4fIMcrOCV152ZlAuO4O8rEyyM63XT1DS+1KZFJYCi8xsCcHA6ofVnyD9gbtzsK6ZioN17K9torYxwruVtTS3RNlf28S+mkb21TSyvyY40dc0RjrdXlFuFqVFOQwtzGF4cS5Thhdx5uRSDGNsST6DC7IpzsticH42g/OzKc7LpiAnk6LcLApysnSylh6VtKRgZr8mGIi9zMwqgJuBbAB3/wnwOHARwRi1dcA/JCsWkUREWqJUVjey63ADe6sb2VPVwL6aRiqrGtlf28SB2kb2VDWyt7qRppb2m2FKC3MoK8qlrDiHceOGMLQwh9LCHDIyghP8oPxsSgpyKM7LojgviyH5OeRk6XYh6TuSefXR5V0sd+Bzydq/SKxo1Nlb08jOQ/XsPtzA3ppGdhysZ/vBOnYdbqCyqpFdh+uJb63JMCgtyqW0MPglf9rEEkYMzmNEcR5jSvIZVpxLUW4WQwqyKS3MJTNDv9ilf+t3j84W6Yi7U1ndyNZ9tWzeV8uuQ/VsP1jPhj3VrN9dTSTujJ+TmcHYknxGDclj7qShjC3JZ8SgPMYMyWf4oFyGF+cxtDBHJ3pJK0oK0u+4O9v217Gq4hA7DtVTcbCejXuq2bCnhsP1zW3lMgxGDMpjyohi/v7MUkYOzmVsSQHjhxYwYpBO+CLtUVKQPq+yuoHXNx/gnd1VvF1xmNU7DnOw7i8n/yEF2Rw/rIiLZo5i6qhiJpQWMrmskFGD88jKVHu9SHcoKUif4e68d6COdbuqWLOzinW7qnhl037qm1uA4Br0E0cUc8G04K7OU8aXMH5oAYW5+jMW6Sn63yQpc6C2ibU7q1i/p5o33zvIW9sOsvNwAxAkgMllhZw7dThjh+TzNyeP4sSRxeRmZaY4apGBTUlBekVrLWDZlgMs23KAt7Yf4t29NW3PgBk1OI8544fw2ePKmDlmMCeOLCYvWwlApLcpKUjSVBysY/nWg7y8aR8vb9zH7qqgFlBSkM3scUO4dNZoTplQwpQRRQwvzktxtCICSgrSgw7VNfHSxn38afN+Xtm0j2376wAozsvi7BOGcfqkoZw+qZQpw4vI0FU/In2SkoIckwO1TTy/vpLHVu3khQ17iXrw2IbTJw3l78+cSPmEEmaOGawkINJPKClIt+2pauA3b1bw0oZ9LNt6gJaoM3JQHv/418fxoROHc+qEEl3/L9JPKSlIQhojLTy9Zg+PrtzJc+sraYk6U0cN4rq/msxFM0cyY7RqAyIDgZKCdGr7gTruf20bv1lRwf7aJkYMyuXTH5zEJ08bx+RhRakOT0R6mJKCHGHX4XqeXVfJyxv38Yd1ezDgnBOHcdUZEzh7yjDVCEQGMCUFAaAl6ry4cS8/fWkzr767H3cYXpzLp86ayNUfmMjYkoJUhygivUBJIc3VNEZ4cvVufvz8Jt7dW8uIQbn883lTuGTWaCaXFWrwFpE0o6SQptbsPMxPX9rC7/+8i6ZIlEllhdzxyVlcNHOUHiUhksaUFNKIu/Pa5gP86LmNvLJpP3nZGSw8bRzzZ43mlPEl6isQESWFdFDf1MLv/7yLX7y2jZXbD1FWlMuXP3wiV54+niEFOakOT0T6ECWFAay6oZm7n3+XJcve42BdM5OHFXLr/Ol8onwc+TlqIhKRIykpDEDuzpOrd3PLY2uorG5k3rQRXH3mRM48rlQdxyLSKSWFAWbz3hpuXrqGlzbuY3JZIf/7mVOZM74k1WGJSD+hpDBA1DVFuO/VbfzgmQ1kZhg3XzKNvztjgoajFJFuUVLo55pbovz8lS385IXNHKht4vypw/nGZTMYNTg/1aGJSD+kpNCPbdxTzWd++SabKms46/hSvnjBCZw6YWiqwxKRfkxJoR9qiTq/en0b33p8HQU5WfzkqlP48PSR6kQWkWOmpNDPvPXeQW5euoa3Kw5zxuShfP8Tsxk9RE1FItIzlBT6iWjU+fEL7/IfT6+nrCiXOz45i8tmj1HtQER6lJJCP7DzUD03PrSKV9/dz8Unj+L2v53JoLzsVIclIgOQkkIfFo06S97YzrcfX0eLO9+4bAZXnT5etQMRSRolhT5q+4E6vvTQKpZtOUD5hBK++/FZTCorTHVYIjLAKSn0QU+t2c2ND64i6s7tH53J5XPHqXYgIr0iqbe7mtmFZrbezDaZ2U3tLB9vZs+Z2Vtm9raZXZTMePqDn7+yhX+8fwUTygp48gtnc4Wai0SkFyWtpmBmmcBdwAVABfCGmS1197Uxxf4VeNDdf2xm04DHgYnJiqkvi0adO57ZwA//uIkLpo3gPxfOpiBHFTkR6V3JPOvMBTa5+2YAM1sCXArEJgUHBoXvBwM7kxhPn9UYaeHGh97msVU7+dgpY/n3j83UM4tEJCWSmRTGANtjpiuA0+PK3AI8bWafBwqB89vbkJldD1wPMH78+B4PNJWqG5q5/L9fY/WOKm6cdwKf+9Dxai4SkZRJ5s/R9s5sHjd9OXCvu48FLgLuN7MjYnL3xe5e7u7lw4YNS0KoqVHXFOELS1ayblc1P7x8DovOnaKEICIplcyaQgUwLmZ6LEc2D10LXAjg7n8yszygDKhMYlx9wp6qBq75+Ru8s7uKWy6ZziWzRqc6JBGRpNYU3gCmmNkkM8sBFgJL48q8B5wHYGZTgTxgbxJj6hNqGiNc9z/L2bKvhp9dXc7VH5iY6pBERIAkJgV3jwCLgKeAdQRXGa0xs9vMbH5Y7EvAdWa2Cvg1cI27xzcxDSh7qxv5+5+9zpqdVdy5cA7nnjQi1SGJiLRJ6jWP7v44wWWmsfO+HvN+LXBWMmPoSxojLVx//3LW7Kzih5fPYd70kakOSUTkfXTdYy9pikT53C/f5K33DvGdj53MRTNHpTokEZEjKCn0kn974h2eWVfJ1y+exmVzxqQ6HBGRdikp9IKHlm/nnle2cOXp4/nUByelOhwRkQ4pKSTZm+8d5GuPrOaMyUP5+iXTUh2OiEinlBSSqOJgHdf/z3JGDsrjh5efQm5WZqpDEhHplJ64liTNLVG++OAqGpqj/Pq6coYV56Y6JBGRLqmmkCS3PbaWZVsO8K9/M5UpI4pTHY6ISEKUFJLgqTW7uf+1bVz7wUksnDuwHuAnIgObkkIPe29/HTc+tIqTRhbzlQtPTHU4IiLdoqTQw76+dDXusPjvytWxLCL9jpJCD3pxw16eX7+XRecez/jSglSHIyLSbUoKPaS5JcrNS9cwbmg+1+ippyLSTykp9JD7Xt3Kln21/OvfTCMvW81GItI/KSn0gE2V1fz7k+9wzonDmDdNj8IWkf5LSeEYuTvffvwdcrMy+d7HZ2k4TRHp15QUjtErm/bz7DuVfOac4ygr0l3LItK/KSkcA3fn355cR1lRDtfq6aciMgAoKRyDR1buYPWOKr784RPVuSwiA4KSwlGqa4pw++PvMGvcED52ythUhyMi0iOUFI7Sj/64ib3Vjdx04UlkZepjFJGBQWezo/De/joWv7iZS2eP5szjSlMdjohIj1FSOAp3P7+JjAzjXy48KdWhiIj0KCWFbtpX08j/vrWDv50zhtFD8lMdjohIj1JS6Kb7Xt1KUyTKp3QJqogMQEoK3VDV0My9r25l3rQRnKDR1ERkAFJS6IaHl1dQ3RDhsx86PtWhiIgkhZJCgpoiUX7+6hZmjBnE7HFDUh2OiEhSZCVSyMzKgb8CRgP1wGrgGXc/kMTY+pRH3trB9gP13HbNjFSHIiKSNJ3WFMzsGjN7E/gqkA+sByqBDwJ/MLP7zGzAj0zf3BLlP5/dyEkjiznnxGGpDkdEJGm6qikUAme5e317C81sNjAFeK+nA+tLnl1XyY5D9Sz+u1P1aGwRGdA6rSm4+10dJYRw+Up3f7aj5WZ2oZmtN7NNZnZTB2U+YWZrzWyNmf0q8dB7z+//vIuSgmzOPWl4qkMREUmqTmsKZnZnZ8vd/YZO1s0E7gIuACqAN8xsqbuvjSkzhaBp6ix3P2hmfe6sW9sY4Zm1e7hszhg940hEBryumo9WHMO25wKb3H0zgJktAS4F1saUuQ64y90PArh75THsLyle2riX+uYW5s8anepQRESSrtOk4O73HcO2xwDbY6YrgNPjypwAYGavAJnALe7+ZPyGzOx64HqA8eN7t1/7D2srKczJ5NQJJb26XxGRVOiq+egxwDta7u7zO1u9vVXa2f8U4BxgLPCSmc1w90Nx+1kMLAYoLy/vMJ6eVtsY4ek1u/nIzFHkZKnpSEQGvq6aj753DNuuAMbFTI8FdrZT5jV3bwa2mNl6giTxxjHst8e8uGEv1Y0RDaIjImmjq+ajF45h228AU8xsErADWAhcEVfmEeBy4F4zKyNoTtp8DPvsUY+v3s2QgmzKJ6rpSETSQ0JtImY2xcweDi8d3dz66mwdd48Ai4CngHXAg+6+xsxuM7PWZqengP1mthZ4Dviyu+8/+sPpOftqGnly9S4umjmKbF11JCJpIqHHXAA/B24G7gA+BPwD7fcZvI+7Pw48Hjfv6zHvHfhi+OpTnvjzLppbnKtOn5DqUEREek2iP4Hzw5vUzN23ufstwLnJCyv1nl67h4mlBUwbPSjVoYiI9JpEawoNZpYBbDSzRQR9BH3uRrOe0tDcwutbDnDl6QP+sU4iIu+TaE3hC0ABcANwKnAVcHWygkq1VdsP0RSJctZxZakORUSkVyVUU3D31ktEawj6Ewa0P66vJDPDdMOaiKSdRK8++oOZDYmZLjGzp5IXVmr99s0dnDq+hJLCnFSHIiLSqxJtPiqLvcs4fFbRgOxTOFzXzIHaJo4fUZTqUEREel2iSSEaO5iOmU2gk8df9GevvLuPSNS5VA/AE5E0lOjVR18DXjaz1juczyZ8QN1A87u3d1JamMMp6k8QkTSUaEfzk2Z2CnAGwU1r/8fd9yU1shRojLTw4oZ9fGTGSN3FLCJpKdGOZgMuBE5x98eAAjObm9TIUuDNbYeoaYxw3tQRqQ5FRCQlEv05fDdwJsHD6wCqCUZVG1CeW19JdqZx1vGlqQ5FRCQlEu1TON3dTzGztyC4+sjMBtz1mn98p5LTJg6lOC871aGIiKREojWF5nDMZQcws2FANGlRpcDhumbe3VvDmZNVSxCR9JVoUrgT+C0w3My+BbwM3J60qFJg+bYDuMOc8brqSETSV6JXH/3SzFYA5xFcfXSZu69LamS9bNmWA+RkZujRFiKS1rpMCuHTUd929xnAO8kPKTVWbDvIjDGDyM/JTHUoIiIp02XzkbtHgVWxdzQPNO7Ohj3VGjtBRNJeolcfjQLWmNkyoLZ1prvP73iV/qOqIUJVQ4RxJQWpDkVEJKUSTQq3JjWKFNu8twaACaVKCiKS3jpNCmZmHnihqzI9H1rveeu94AGwM8YMTnEkIiKp1VWfwnNm9vn4/gQzyzGzc83sPgbACGzrd1dTWpjDWDUfiUia66r56ELgU8CvzWwScAjIAzKBp4E73H1lckNMvvV7qpmi8RNERDpPCu7eQPDco7vNLBsoA+pjB9zp76JRZ+X2Q1zzgYmpDkVEJOUS7WjG3ZuBXUmMJSW2HagDYGxJfoojERFJvbQfNOCt9w4C8IHjylIciYhI6qV9Ulizs4q87AxOGlmc6lBERFLuqJKCmWWa2ZU9HUwqbNtfx5gh+WRkWKpDERFJuU6TgpkNMrOvmtmPzGyeBT4PbAY+0TshJtfqHYeZPlr3J4iIQNcdzfcDB4E/AZ8GvgzkAJcOhEtRqxqa2V3VwIlqOhIRAbpOCpPdfSaAmf0U2AeMd/fqpEfWC9bvDg5j6iglBRER6LpPobn1jbu3AFsGSkIAWLuzCoATRigpiIhA10lhlplVmVm1mVUDJ8dMV3W1cTO70MzWm9kmM7upk3ILzMzNrLy7B3AsVm0/RFlRLmOG6B4FERHo+o7mox5xJhzT+S7gAqACeMPMlrr72rhyxcANwOtHu6+jtXZXFTPGDMJMVx6JiEDXVx/lmdkXwquPrjezhO+ABuYCm9x9s7s3AUuAS9sp9w3gO0BDN7Z9zNydHQfrGT9UD8ETEWnVVfPRfUA58GfgIuA/urHtMcD2mOmKcF4bM5sDjHP333W2oTAhLTez5Xv37u1GCB3bV9NEdWOESWWFPbI9EZGBoKtf/tNirj76GbCsG9tur02mbdyFcOznO4BrutqQuy8GFgOUl5f3yNgNW/cHA8hNLFVSEBFp1Z2rjyLd3HYFMC5meiywM2a6GJgBPG9mW4EzgKW91dncejnq8cP1yGwRkVZd1RRmx1xlZEB+OG2Au3tnI92/AUwJx2HYASwErmhd6O6HCR7FHWzc7HngRndf3u2jOApb99WSk5Whp6OKiMToKimscvc5R7Nhd4+Y2SLgKYJBee5x9zVmdhuw3N2XHs12e8qqikOMGZKvK49ERGJ0lRSOqf3e3R8HHo+b9/UOyp5zLPvqrqYWJztTCUFEJFZXSWG4mX2xo4Xu/v0ejqfX7DpUz1+fMCzVYYiI9CldJYVMoIj2ryTqt5pbolRWNzJKdzKLiLxPV0lhl7vf1iuR9KLK6kYAcrPSfowhEZH36eqsOKBqCK32VAU3T0/WjWsiIu/TVVI4r1ei6GWH64LbLwblZ6c4EhGRvqXTpODuB3orkN60LbybeYpuXBMReZ+0bFSvOFhPXnYGw4pzUx2KiEifkpZJYU91IyMG5enGNRGROOmZFKoaGK5agojIEdIyKVRWNTB8UF6qwxAR6XPSLim4O3uqGhmppCAicoS0Swr1zS3UN7dQVqTmIxGReGmXFPZUBXczlxXlpDgSEZG+J+2Sws5D9QCM0XOPRESOkHZJYdfh4BEXQ1VTEBE5Qtolhar64BEXxXl6xIWISLy0SwrVDcFQ08PU0SwicoQ0TArNFOZkkqPHZouIHCHtzoy1TREKcrsaRkJEJD2lXVI4WNvMED0yW0SkXWmXFA7UNVFSqCuPRETak3ZJobohwiBdeSQi0q60Swp1TREKcjJTHYaISJ+UdkmhtjFCoTqaRUTalXZJIWg+UlIQEWlPWiWFhuYWGiNRBunqIxGRdqVVUjhY1wRASYGuPhIRaU9aJYX9NUFSKNXD8ERE2pVWSeFAbZAUhuo+BRGRdqVlUtB9CiIi7UtqUjCzC81svZltMrOb2ln+RTNba2Zvm9mzZjYhmfHsD5OC7lMQEWlf0pKCmWUCdwEfAaYBl5vZtLhibwHl7n4y8DDwnWTFA+DuALr6SESkA8msKcwFNrn7ZndvApYAl8YWcPfn3L0unHwNGJvEeNi6vxaAQtUURETalcykMAbYHjNdEc7ryLXAE+0tMLPrzWy5mS3fu3fvUQdUkBPctJaVmVZdKSIiCUvm2dHameftFjS7CigHvtvecndf7O7l7l4+bNiwow6opjFCmS5HFRHpUDKf91ABjIuZHgvsjC9kZucDXwP+2t0bkxgP1Q0Rjc0sItKJZNYU3gDlRyEkAAAMQUlEQVSmmNkkM8sBFgJLYwuY2Rzgv4D57l6ZxFiA4DEXuRqGU0SkQ0k7Q7p7BFgEPAWsAx509zVmdpuZzQ+LfRcoAh4ys5VmtrSDzfWIhuYW8rLVySwi0pGkPi7U3R8HHo+b9/WY9+cnc//x6ptayFdSEBHpUFq1pdQ3t1CYq6QgItKRtEsKaj4SEelYWiWFukY1H4mIdCatkkJDpIV83c0sItKhtEoKVfXNuiRVRKQTaXOGdHeiDs0t7d5ULSIipFFSaE0GuvpIRKRjaZMUGiMtgMZnFhHpTNokhfrmICmoT0FEpGNpc4ZsikQBqGlsSXEkIiJ9V9okhUjYpzBiUG6KIxER6bvSJylEg5qCBtgREelY2pwhW68+ys5ob+wfERGBNEoKrc1HqimIiHQsbc6QVQ3NAERaoimORESk70qbpJAZNhsV5SV1CAkRkX4tbZJCNBr2Kaj5SESkQ2lzhmzxIClkqqNZRKRD6ZMUwppChikpiIh0JG2SQlQ1BRGRLqVNUmi96ChTNQURkQ6lzaU4bc1HaZMGRfqe5uZmKioqaGhoSHUoA1ZeXh5jx44lOzv7qNZPm6Sg5iOR1KuoqKC4uJiJEydiqrX3OHdn//79VFRUMGnSpKPaRtr8bm4O24+ylBREUqahoYHS0lIlhCQxM0pLS4+pJpY2SaG1+ShL7UciKaWEkFzH+vmmzRkyzAlqPhIR6UT6JIUwK+hHikh6y8zMZPbs2cyYMYNLLrmEQ4cOtS1bs2YN5557LieccAJTpkzhG9/4Bh72RwI88cQTlJeXM3XqVE466SRuvPHGdveRaLm+KG2Sgu5oFhGA/Px8Vq5cyerVqxk6dCh33XUXAPX19cyfP5+bbrqJDRs2sGrVKl599VXuvvtuAFavXs2iRYv4xS9+wbp161i9ejWTJ08+YvuJlutIS0tqR4dMv6uPVFUQ6RNufWwNa3dW9eg2p40exM2XTE+4/Jlnnsnbb78NwK9+9SvOOuss5s2bB0BBQQE/+tGPOOecc/jc5z7Hd77zHb72ta9x0kknAZCVlcVnP/vZI7bZWblrrrmGiy++mAULFgBQVFRETU0Nzz//PLfeeiujRo1i5cqVXHLJJUyYMKFtvVtuuYXi4mK+9KUv8d3vfpcHH3yQxsZGPvrRj3Lrrbce5afVvrSpKfyl+UhJQUSCX+TPPvss8+fPB4Kmo1NPPfV9ZY477jhqamqoqqpi9erVRyxvT6Ll4i1btoxvfetbrF27loULF/LAAw+0LXvwwQf5+Mc/ztNPP83GjRtZtmwZK1euZMWKFbz44ovd3ldn0qam0Hr1kZqPRPqG7vyi70n19fXMnj2brVu3cuqpp3LBBRcAwTX+Hf1o7I0fk3Pnzm27t2DOnDlUVlayc+dO9u7dS0lJCePHj+fOO+/k6aefZs6cOQDU1NSwceNGzj777B6LI6k1BTO70MzWm9kmM7upneW5ZvZAuPx1M5uYrFjarj5STUEkrbX2KWzbto2mpqa2PoXp06ezfPny95XdvHkzRUVFFBcXM336dFasWNHl9jsrl5WVRTQcL97daWpqaltWWFj4vrILFizg4Ycf5oEHHmDhwoVt63z1q19l5cqVrFy5kk2bNnHttdcmfvAJSFpSMLNM4C7gI8A04HIzmxZX7FrgoLsfD9wB/Huy4mntU7C0aTATkc4MHjyYO++8k+9973s0Nzdz5ZVX8vLLL/PMM88AQY3ihhtu4Ctf+QoAX/7yl7n99tvZsGEDANFolO9///tHbLezchMnTmxLGI8++ijNzc0dxrdw4UKWLFnCww8/3NYH8eEPf5h77rmHmpoaAHbs2EFlZWVPfBxtknmKnAtscvfN7t4ELAEujStzKXBf+P5h4DxLUj2trflINQURCc2ZM4dZs2axZMkS8vPzefTRR/nmN7/JiSeeyMyZMznttNNYtGgRACeffDI/+MEPuPzyy5k6dSozZsxg165dR2yzs3LXXXcdL7zwAnPnzuX1118/onYQa/r06VRXVzNmzBhGjRoFwLx587jiiis488wzmTlzJgsWLKC6urpHPxOLvQa3RzdstgC40N0/HU7/HXC6uy+KKbM6LFMRTr8bltkXt63rgesBxo8ff+q2bdu6Hc/Ta3bzyMod3PHJ2eRmZR7tYYnIMVi3bh1Tp05NdRgDXnufs5mtcPfyrtZNZkdzez/J4zNQImVw98XAYoDy8vKjymLzpo9k3vSRR7OqiEjaSGbzUQUwLmZ6LLCzozJmlgUMBg4kMSYREelEMpPCG8AUM5tkZjnAQmBpXJmlwNXh+wXAHz1Z7Vki0ifov3hyHevnm7Sk4O4RYBHwFLAOeNDd15jZbWY2Pyz2M6DUzDYBXwSOuGxVRAaOvLw89u/fr8SQJK3jKeTl5R31NpLW0Zws5eXlHn8tsYj0Dxp5Lfk6GnmtL3Q0i4i8T3Z29lGPCCa9Q7dyiYhIGyUFERFpo6QgIiJt+l1Hs5ntBbp/S3OgDNjXZamBRcecHnTM6eFYjnmCuw/rqlC/SwrHwsyWJ9L7PpDomNODjjk99MYxq/lIRETaKCmIiEibdEsKi1MdQAromNODjjk9JP2Y06pPQUREOpduNQUREemEkoKIiLQZkEnBzC40s/VmtsnMjnjyqpnlmtkD4fLXzWxi70fZsxI45i+a2Voze9vMnjWzCamIsyd1dcwx5RaYmZtZv798MZFjNrNPhN/1GjP7VW/H2NMS+Nseb2bPmdlb4d/3RamIs6eY2T1mVhmOTNnecjOzO8PP420zO6VHA3D3AfUCMoF3gclADrAKmBZX5rPAT8L3C4EHUh13Lxzzh4CC8P1n0uGYw3LFwIvAa0B5quPuhe95CvAWUBJOD0913L1wzIuBz4TvpwFbUx33MR7z2cApwOoOll8EPEEwcuUZwOs9uf+BWFOYC2xy983u3gQsAS6NK3MpcF/4/mHgPDNrb2jQ/qLLY3b359y9Lpx8jWAkvP4ske8Z4BvAd4CB8KzmRI75OuAudz8I4O6VvRxjT0vkmB0YFL4fzJEjPPYr7v4inY9AeSnwPx54DRhiZqN6av8DMSmMAbbHTFeE89ot48FgQIeB0l6JLjkSOeZY1xL80ujPujxmM5sDjHP33/VmYEmUyPd8AnCCmb1iZq+Z2YW9Fl1yJHLMtwBXmVkF8Djw+d4JLWW6+/+9WwbieArt/eKPv+42kTL9ScLHY2ZXAeXAXyc1ouTr9JjNLAO4A7imtwLqBYl8z1kETUjnENQGXzKzGe5+KMmxJUsix3w5cK+7/4eZnQncHx5zNPnhpURSz18DsaZQAYyLmR7LkdXJtjJmlkVQ5eysutbXJXLMmNn5wNeA+e7e2EuxJUtXx1wMzACeN7OtBG2vS/t5Z3Oif9uPunuzu28B1hMkif4qkWO+FngQwN3/BOQRPDhuoEro//vRGohJ4Q1giplNMrMcgo7kpXFllgJXh+8XAH/0sAenn+rymMOmlP8iSAj9vZ0Zujhmdz/s7mXuPtHdJxL0o8x39/48lmsif9uPEFxUgJmVETQnbe7VKHtWIsf8HnAegJlNJUgKe3s1yt61FPj78CqkM4DD7r6rpzY+4JqP3D1iZouApwiuXLjH3deY2W3AcndfCvyMoIq5iaCGsDB1ER+7BI/5u0AR8FDYp/6eu89PWdDHKMFjHlASPOangHlmthZoAb7s7vtTF/WxSfCYvwT8t5n9H4JmlGv68488M/s1QfNfWdhPcjOQDeDuPyHoN7kI2ATUAf/Qo/vvx5+diIj0sIHYfCQiIkdJSUFERNooKYiISBslBRERaaOkICIibZQURBJkZi1mtjLmNdHMzjGzw+ETOteZ2c1h2dj575jZ91Idv0giBtx9CiJJVO/us2NnhI9df8ndLzazQmClmbU+a6l1fj7wlpn91t1f6d2QRbpHNQWRHuLutcAK4Li4+fXASnrwoWUiyaKkIJK4/Jimo9/GLzSzUoJnLK2Jm19C8PyhF3snTJGjp+YjkcQd0XwU+iszewuIAv8WPobhnHD+28CJ4fzdvRiryFFRUhA5di+5+8UdzTezE4CXwz6Flb0dnEh3qPlIJMncfQPwbeBfUh2LSFeUFER6x0+As81sUqoDEemMnpIqIiJtVFMQEZE2SgoiItJGSUFERNooKYiISBslBRERaaOkICIibZQURESkzf8HZcZe1LgO9twAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.metrics import roc_curve, roc_auc_score\n",
    "fpr, tpr, thresholds = roc_curve(y_true, y_pred)\n",
    "score = roc_auc_score(y_true, y_pred)\n",
    "\n",
    "plt.plot(fpr, tpr, label=\"ROC Curve\")\n",
    "plt.xlabel(\"FPR\")\n",
    "plt.ylabel(\"TPR (recall)\")\n",
    "plt.title('ROC Curve (AUC = {:.4f})'.format(score))\n",
    "plt.legend(loc=4)\n",
    "#plt.savefig('t.png')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-31T09:03:44.067843Z",
     "start_time": "2019-03-31T09:03:43.928473Z"
    }
   },
   "source": [
    "## Confusion Matrix for Multiclass Classification (Yelp Full Dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "confusion = confusion_matrix(y_true, y_pred>0)\n",
    "print(\"Confusion matrix:\\n{}\".format(confusion))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plot train & validation loss and accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}