Transfer learning for NLP classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Transfer learning for NLP classification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Use a pretrained language model (trained on *wiki103*) to finetune on movie reviews to then finally classify movie reviews.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.text import *\n",
    "import html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH = Path('data/aclImdb/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "SOS = 'xbos' # Start/beginning of sentence\n",
    "FLD = 'xfld' # data field tag"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Standardize format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Classification path: sentiment analysis\n",
    "CLASS_PATH = Path('data/imdb_class/')\n",
    "CLASS_PATH.mkdir(exist_ok=True)\n",
    "\n",
    "# Language model path\n",
    "LM_PATH = Path('data/imdb_lm/')\n",
    "LM_PATH.mkdir(exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "CLASSES = ['neg', 'pos', 'unsup']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_texts(path):\n",
    "    texts, labels = [], []\n",
    "    for idx, label in enumerate(CLASSES):\n",
    "        for fname in (path/label).glob('*.*'):\n",
    "            texts.append(fname.open('r', encoding='utf-8').read())\n",
    "            labels.append(idx)\n",
    "    return np.array(texts), np.array(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "trn_texts, trn_labels = get_texts(PATH/'train')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_texts, val_labels = get_texts(PATH/'test')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(75000, 25000)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(trn_texts), len(val_texts)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Shuffle the dataset ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(42)\n",
    "trn_idx = np.random.permutation(len(trn_texts))\n",
    "val_idx = np.random.permutation(len(val_texts))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "trn_texts = trn_texts[trn_idx]\n",
    "val_texts = val_texts[val_idx]\n",
    "\n",
    "trn_labels = trn_labels[trn_idx]\n",
    "val_labels = val_labels[val_idx]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ... and create pandas dataframes to store data as csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "column_names = ['labels', 'text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_trn = pd.DataFrame({'text':trn_texts, 'labels':trn_labels}, columns=column_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_val = pd.DataFrame({'text':val_texts, 'labels':val_labels}, columns=column_names)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### csv files for classification\n",
    "Remove all texts with label `unsup`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_trn[df_trn['labels']!=2].to_csv(CLASS_PATH/'train.csv', header=False, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_val.to_csv(CLASS_PATH/'test.csv', header=False, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "(CLASS_PATH/'classes.txt').open('w', encoding='utf-8').writelines(f'{o}\\n' for o in CLASSES)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>labels</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>In the first and most memorable scene in this ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>Why is this film so bad? Well, if being so stu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>Ruth Gordon is one of the more sympathetic kil...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>I used to subscribe to Toyfare magazine and on...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>...and I don't mean nuclear!! This movie is pr...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   labels                                               text\n",
       "0       2  In the first and most memorable scene in this ...\n",
       "1       0  Why is this film so bad? Well, if being so stu...\n",
       "2       1  Ruth Gordon is one of the more sympathetic kil...\n",
       "3       2  I used to subscribe to Toyfare magazine and on...\n",
       "4       2  ...and I don't mean nuclear!! This movie is pr..."
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_trn.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### csv files for language model\n",
    "Labels can be ignored, so also `unsup` labeled texts can be used."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "trn_texts, val_texts = sklearn.model_selection.train_test_split(np.concatenate([trn_texts, val_texts]), test_size=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(90000, 10000)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(trn_texts), len(val_texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_trn = pd.DataFrame({'text':trn_texts, 'labels':[0]*len(trn_texts)}, columns=column_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_val = pd.DataFrame({'text':val_texts, 'labels':[0]*len(val_texts)}, columns=column_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>labels</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Hoo dawgies, what can I say about this movie t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>Back when cable TV started, there were tons of...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>Viewing this film makes you question, no pun i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>no, this is not supposed to be a high budget b...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>This is meant to be a thriller, but the audien...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   labels                                               text\n",
       "0       0  Hoo dawgies, what can I say about this movie t...\n",
       "1       0  Back when cable TV started, there were tons of...\n",
       "2       0  Viewing this film makes you question, no pun i...\n",
       "3       0  no, this is not supposed to be a high budget b...\n",
       "4       0  This is meant to be a thriller, but the audien..."
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_trn.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_trn.to_csv(LM_PATH/'train.csv', header=False, index=False)\n",
    "df_val.to_csv(LM_PATH/'test.csv', header=False, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tokenization\n",
    "Cleanup of text:\n",
    "\n",
    "1. Replace extra spaces, tab chars, new line chars and other chars with standard tokens\n",
    "1. Use *spacy* to tokenize the data. Fastai adds multicore functionality to spacy.\n",
    "\n",
    "*Chunksize:* `pd.read_csv` returns iterator so that we don't have to load the entire dataset into memory at once. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "chunksize = 24000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compile a regular expression pattern, returning a pattern object.\n",
    "new_re = re.compile(r' +')\n",
    "\n",
    "def cleanup(x):\n",
    "    '''Many nlp datasets contain strange characters. This function removes them.'''\n",
    "    x = x.replace('#39;', \"'\").replace('amp;', '&').replace('#146;', \"'\").replace(\n",
    "        'nbsp;', ' ').replace('#36;', '$').replace('\\\\n', \"\\n\").replace('quot;', \"'\").replace(\n",
    "        '<br />', \"\\n\").replace('\\\\\"', '\"').replace('<unk>','u_n').replace(' @.@ ','.').replace(\n",
    "        ' @-@ ','-').replace('\\\\', ' \\\\ ')\n",
    "    return new_re.sub(' ', html.unescape(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Start every training/testing example with SOS token so that language model can learn that a new text starts and it might need to forget its state. If there are several text columns, start each of them with {FLD} index so that the language model is able to distinguish between the columns.\n",
    "\n",
    "def get_texts(df, n_lbls=1):\n",
    "    labels = df.iloc[:, range(n_lbls)].values.astype(np.int64)\n",
    "    texts = f'\\n{SOS} {FLD} 1 ' + df[n_lbls].astype(str)\n",
    "\n",
    "    # here {FLD} 1 will be used twice if there is more than 1 text column i think. TODO: check this out\n",
    "    for i in range(n_lbls+1, len(df.columns)):\n",
    "        texts += f' {FLD} {i-nlbls} ' + df[i].astype(str)\n",
    "        \n",
    "    # cleanup disturbing chars\n",
    "    texts = list(texts.apply(cleanup).values)\n",
    "    \n",
    "    # tokenize using multicore functionality\n",
    "    tok = Tokenizer().proc_all_mp(partition_by_cores(texts))\n",
    "    \n",
    "    return tok, list(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_all(df, n_lbls):\n",
    "    tok, labels = [], []\n",
    "    # remeber that df is split into pieces of size chunksize\n",
    "    for i, r in enumerate(df):\n",
    "        print(i)\n",
    "        tok_, labels_ = get_texts(r, n_lbls)\n",
    "        tok += tok_\n",
    "        labels += labels_\n",
    "        \n",
    "    return tok, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_trn = pd.read_csv(LM_PATH/'train.csv', header=None, chunksize=chunksize)\n",
    "\n",
    "df_val = pd.read_csv(LM_PATH/'test.csv', header=None, chunksize=chunksize)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Attention:**\n",
    "If at this point of the notebook you do sth like this, you lose the first chunksize samples: \n",
    "`next(iter(df_trn)).head()`\n",
    "Don't do that!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "tok_trn, trn_labels = get_all(df_trn, 1)\n",
    "tok_val, val_labels = get_all(df_val, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "(LM_PATH/'tmp').mkdir(exist_ok=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Save tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(LM_PATH/'tmp'/'tok_trn.npy', tok_trn)\n",
    "np.save(LM_PATH/'tmp'/'tok_val.npy', tok_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "tok_trn = np.load(LM_PATH/'tmp'/'tok_trn.npy')\n",
    "tok_val = np.load(LM_PATH/'tmp'/'tok_val.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Counter counts hashable items\n",
    "frequency = Counter(tok for i in tok_trn for tok in i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('the', 1208409),\n",
       " ('.', 991990),\n",
       " (',', 985348),\n",
       " ('and', 588114),\n",
       " ('a', 583960),\n",
       " ('of', 525638),\n",
       " ('to', 485093),\n",
       " ('is', 393943),\n",
       " ('it', 341786),\n",
       " ('in', 337659),\n",
       " ('i', 307547),\n",
       " ('this', 270652),\n",
       " ('that', 261453),\n",
       " ('\"', 236846),\n",
       " (\"'s\", 221476),\n",
       " ('-', 188208),\n",
       " ('was', 180289),\n",
       " ('\\n\\n', 178920),\n",
       " ('as', 165777),\n",
       " ('with', 159249),\n",
       " ('for', 158941),\n",
       " ('movie', 157440),\n",
       " ('but', 150728),\n",
       " ('film', 144225),\n",
       " ('you', 124078)]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frequency.most_common(25)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Discard every token that appears less than 2 times. Vocabulary sizes <= 60,000 have been observed to yield good results.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_vocabs = 60000\n",
    "min_freq = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "itos = [tok for tok, count in frequency.most_common(max_vocabs) if count > min_freq]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "itos.insert(0, '_pad_')\n",
    "itos.insert(0, '_unk_')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Reverse mapping from string to index:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "stoi = collections.defaultdict(lambda:0, {val:key for key, val in enumerate(itos)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "60002"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(itos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "trn_lm = np.array([[stoi[string] for string in text] for text in tok_trn])\n",
    "val_lm = np.array([[stoi[string] for string in text] for text in tok_val])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n xbos xfld 1 hoo _unk_ , what can i say about this movie that has been said before and'"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "' '.join([itos[o] for o in trn_lm[0][:20]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Save:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(LM_PATH/'tmp'/'trn_ids.npy', trn_lm)\n",
    "np.save(LM_PATH/'tmp'/'val_ids.npy', val_lm)\n",
    "\n",
    "pickle.dump(itos, open(LM_PATH/'tmp'/'itos.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "trn_lm = np.load(LM_PATH/'tmp'/'trn_ids.npy')\n",
    "val_lm = np.load(LM_PATH/'tmp'/'val_ids.npy')\n",
    "\n",
    "itos = pickle.load(open(LM_PATH/'tmp'/'itos.pkl', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocab size: 60002, Number of training documents: 90000\n"
     ]
    }
   ],
   "source": [
    "voc_size = len(itos)\n",
    "print(\"Vocab size: {}, Number of training documents: {}\".format(voc_size, len(trn_lm)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Transfer learning for language models\n",
    "### Download weights for AWD LSTM pretrained on *wiki103*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ! wget -nH -r -np -P {PATH} http://files.fast.ai/models/wt103/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "emb_sz, nhidden, nlayers = 400, 1150, 3   # parameters of pretrained network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "PRE_PATH = PATH/'models'/'wt103'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "PRE_LM_PATH = PRE_PATH/'fwd_wt103.h5'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights = torch.load(PRE_LM_PATH, map_location=lambda storage, loc: storage)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder_weights = to_np(weights['0.encoder.weight'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**We will use the mean embedding to initialize the embedding for unkown tokens when we transfer the wikitext LM to the IMDB LM.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "mean_emb = encoder_weights.mean(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "itos_wiki = pickle.load((PRE_PATH/'itos_wt103.pkl').open('rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "stoi_wiki = collections.defaultdict(lambda: -1, {value:key for key, value in enumerate(itos_wiki)})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Map the IMDB vocab and the wikitext vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(238462, 400)"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoder_weights.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_encoder_weights = np.zeros((voc_size, emb_sz), dtype=np.float32)\n",
    "for imdb_idx, word in enumerate(itos):  # this is the IMDB vocab itos\n",
    "    wiki_idx = stoi_wiki[word]          # word from imdb to idx in wiki\n",
    "    new_encoder_weights[imdb_idx] = encoder_weights[wiki_idx] if wiki_idx >= 0 else mean_emb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60002, 400)"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_encoder_weights.shape  # vocab_size, emb_size"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Overwrite the weights in the weights dictionary\n",
    "The decoder module is loaded with the same weights due to *weight tying.*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights['0.encoder.weight'] = T(new_encoder_weights)\n",
    "weights['0.encoder_with_dropout.embed.weight'] = T(np.copy(new_encoder_weights))\n",
    "weights['1.decoder.weight'] = T(np.copy(new_encoder_weights))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create and train the IMDB language pytorch model\n",
    "Goal is to learn to predict the next word given a sequence of preceeding words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "wd = 1e-7\n",
    "bptt = 70    \n",
    "bs = 52"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**bptt:**\n",
    "\n",
    "`bptt` = \"ngrams\", in fastai perturbed around a mean of 70. The idea is that if you train for several epochs, the batches would always be the same if `bptt` would be constant. If `bptt` is varied, the batches differ every epoch and there is randomness that one might compare to image augmentation in cv."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt_fn = partial(optim.Adam, betas=(0.8, 0.99))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "trn_dl = LanguageModelLoader(np.concatenate(trn_lm), bs, bptt)\n",
    "val_dl = LanguageModelLoader(np.concatenate(val_lm), bs, bptt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The 1 is '_pad_' token\n",
    "modeldata = LanguageModelData(PATH, 1, voc_size, trn_dl, val_dl, bs=bs, bptt=bptt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "90000 documents containing a total of 25008684 tokens.\n"
     ]
    }
   ],
   "source": [
    "print(\"{} documents containing a total of {} tokens.\".format(len(trn_lm), len(np.concatenate(trn_lm))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Dropout:**\n",
    "These values have been determined through experimentation. You should only have to update the weighting factor (0.7).\n",
    "\n",
    "1. For smaller datasets increase the weighting factor to reduce overfitting\n",
    "1. For larger datasets you might be able to decrease it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "drops = np.array([0.25, 0.1, 0.2, 0.02, 0.15])*0.7"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### First step: finetune the embeddings (last layer)\n",
    "Some tokens are missing in the pretrained wiki model. These should be learned in this step."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```\n",
    "dropouth (float): dropout to apply to the activations going from one LSTM layer to another                                  \n",
    "dropouti (float): dropout to apply to the input layer.                                                                      \n",
    "dropoute (float): dropout to apply to the embedding layer.                                                                  \n",
    "wdrop (float): dropout used for a LSTM's internal (or hidden) recurrent weights.                                   ```      "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner= modeldata.get_model(opt_fn=opt_fn, emb_sz=emb_sz, n_hid=nhidden, n_layers=nlayers, \n",
    "    dropouti=drops[0], dropout=drops[1], wdrop=drops[2], dropoute=drops[3], dropouth=drops[4])\n",
    "learner.metrics = [accuracy]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.freeze_to(-1)  # freeze all but embedding layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.model.load_state_dict(weights)\n",
    "lr=1e-3\n",
    "lrs = lr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cf16522909aa475497d29a3f3c2d5fe6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  1%|          | 35/6869 [00:05<17:22,  6.56it/s, loss=5.25]\n",
      "epoch      trn_loss   val_loss   accuracy                     \n",
      "    0      4.677772   4.444184   0.257833  \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[4.444183904042081, 0.25783323926797375]"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.fit(lrs/2, 1, wds=wd, use_clr=(32,2), cycle_len=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "58af9c849b22464fb7ec7367a33e526a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                     \n",
      "    0      4.631245   4.400647   0.261047  \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[4.400647097685206, 0.26104733087806875]"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.fit(lrs/2, 1, wds=wd, use_clr=(32,2), cycle_len=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.save('lm_finetuned_emb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.load('lm_finetuned_emb')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Second step: unfreeze the earlier layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.unfreeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f80a6ae8478345cca6e9201cafdf4cd0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=15), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                     \n",
      "    0      4.350103   4.155856   0.285788  \n",
      "    1      4.24065    4.08416    0.29269                      \n",
      "    2      4.155276   4.037142   0.297476                     \n",
      "    3      4.110071   4.013286   0.300187                     \n",
      "    4      4.095423   3.994488   0.30194                      \n",
      "    5      4.051529   3.978413   0.303703                     \n",
      "    6      4.065727   3.965502   0.304916                     \n",
      "    7      4.016764   3.956877   0.306411                     \n",
      "    8      3.981582   3.949974   0.307226                     \n",
      "    9      3.994045   3.940068   0.308182                     \n",
      "    10     4.012587   3.929011   0.309132                     \n",
      "    11     3.975437   3.925294   0.309726                     \n",
      "    12     3.932798   3.921386   0.310328                     \n",
      "    13     3.898594   3.918957   0.310935                     \n",
      "    14     3.906602   3.913427   0.31162                      \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[3.913427404844229, 0.31161988681069824]"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.fit(lrs, 1, wds=wd, use_clr=(20,10), cycle_len=15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.save('lm_model_1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.load('lm_model_1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.save_encoder('lm_model_1_enc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xd4VFX6B/Dvm04ChEACBAKEjhB6pIOAgDSxK7jYWRQb6u4qrC4qNuxlcVcBFRXrTywsiIgQkA5BeocklACmQCAQSD2/P+bOZGYyLcnM3JnJ9/M8PN5y7tx3uPHl5txz3yNKKRARUWAJ0jsAIiJyPyZ3IqIAxORORBSAmNyJiAIQkzsRUQBiciciCkBM7kREAYjJnYgoADG5ExEFoBC9ThwbG6sSExP1Oj0RkV/aunVrjlIqzlk73ZJ7YmIiUlNT9To9EZFfEpGjrrRjtwwRUQBiciciCkBM7kREAYjJnYgoADG5ExEFICZ3IqIAxORORBSA/C65l5SW4dvU4ygt4/SARET2+F1yn78+A09+txNfbzmmdyhERD7L75J77sUiAEBeQbHOkRAR+S6/S+5lWneMUuyWISKyx++S+4e/pwEANmec1TkSIiLf5XfJ3Uj0DoCIyIf5bXLv27qB3iEQEfksv03u7HInIrLPb5M7ERHZ57fJvYy37kREdvltcudQSCIi+/w4uesdARGR73I5uYtIsIhsE5HFdvbfKiJ7RWSPiHzpvhAtRYYFa+fz1BmIiPxfZe7cpwLYZ2uHiLQFMB1Af6VUJwCPuSE2m+bf0wsA8MHqNE+dgojI77mU3EUkAcAYAPPsNPkrgPeVUmcBQCmV5Z7wKmrRIBIAcKGwxFOnICLye67eub8D4EkAZXb2twPQTkTWichGERnpluhsYKlfIiLnnCZ3ERkLIEsptdVBsxAAbQEMBjABwDwRqWfjsyaLSKqIpGZnZ1cp4NoRIVU6joioJnHlzr0/gHEikgHgawBDRWSBVZsTAH5SShUrpdIBHIAh2VtQSs1RSiUrpZLj4uKqFHDtMCZ3IiJnnCZ3pdR0pVSCUioRwHgAK5VSE62a/QhgCACISCwM3TQeeeIZFMRhMkREzlR5nLuIzBSRcdrqMgC5IrIXQAqAfyilct0RoC3hIYawn1u0x1OnICLya5VK7kqpVUqpsdryDKXUIm1ZKaWeUEp1VEp1Vkp97YlgjW7s0RSAYco9IiKqyC/fUN2UfkbvEIiIfJpfJve07It6h0BE5NP8MrkTEZFjTO5ERAGIyZ2IKAD5fXIvLCnVOwQiIp/j98n98W+26x0CEZHP8fvk/vOu03qHQETkc/wyuY9Kaqx3CEREPs0vk/tDQ9roHQIRkU/zy+RORESO+WVyb1qvlsW64mzZREQW/DK5x0SFWawPeDVFp0iIiHyTXyZ3a5l5lzj9HhGRGb9N7kdeHm2xfuJsgU6REBH5Hr9N7sFWMzJxvDsRUTm/Te7WsvML9Q6BiMhnBExy/3hdut4hEBH5jIBJ7kREVM6vk/uOZ0foHQIRkU/y6+QeXSvUYv1CYYlOkRAR+Ra/Tu4AcG3XJqbl0lKOdSciAgIguU+9uq1puYxlCIiIAARAcm/TsLZpufsLy7F01ykdoyEi8g1+n9ytffB7mt4hEBHpLuCS+8HT+XqHQESku4BL7peKOWE2EVFAJPdbkxP0DoGIyKcERHLPuVCkdwhERD4lIJL7yv1ZeodARORTAiK5h1iV/523hiNmiKhmC4jkfk1SY4v1F5fs0ykSIiLfEBDJ/eXrO+sdAhGRTwmI5B4dGYqMWWMstmXlX9YpGiIi/QVEcrclr6BY7xCIiHQTsMl9xNu/493fDukdBhGRLgI2uQPAh78f0TsEIiJdBFRyXzilr8V6kIidlkREgS2gknur2NoW6xcKSzBx3iZk5l3SKSIiIn0EVHKPiQpDy9goi21rD+dg8OspOkVERKQPl5O7iASLyDYRWeygzc0iokQk2T3hVd4bt3StsK1Ym37vy03HsPXoGW+HRETkdSGVaDsVwD4AdW3tFJE6AB4FsMkNcVVZcJD9fvZ//rALACqMiSciCjQu3bmLSAKAMQDmOWj2AoDXAOj69lDWedun7zbzVy9HQkSkH1e7Zd4B8CSAMls7RaQ7gGZKKbtdNlq7ySKSKiKp2dnZlYvURReLSmxu50tNRFSTOE3uIjIWQJZSaqud/UEA3gbwN2efpZSao5RKVkolx8XFVTpYV9QJD/XI5xIR+RNX7tz7AxgnIhkAvgYwVEQWmO2vAyAJwCqtTR8Ai/R6qJqcGKPHaYmIfIrT5K6Umq6USlBKJQIYD2ClUmqi2f5zSqlYpVSi1mYjgHFKqVRPBe1Ivcgwp23yCopw18eb8bT2gJWIKNBUeZy7iMwUkXHuDMZberywHKsPZuOLTccAAIt3nsSx3AKdoyIicp9KJXel1Cql1FhteYZSapGNNoP1ums3euvWimPdzZUpy/WHv9yG0e+t8WBERETeFVBvqBrd2CMBsbWdd8+Yu1Boe5QNEZE/CsjkDgAD27o2GqfdM0s9HAkRkfcFbHKfdZNrU+8VlZQP3c+9UOipcIiIvCpgk3t4SHClj+n54m/YeSLPA9EQEXlXwCb3qtp36rzeIRARVVuNSO7v394Ds2/vrncYREReU5mqkH5n6zPDEBkWglphhi6ah7/c5vQYpZw2ISLyeQF9596gdrgpsQNAx3ib1YotFJXarI1GRORXAjq5W/v+wX5O28z4aQ9O5l2C4i08EfmxGpXcI0KDkdTU+d17v1krMef3NC9ERETkGTUquQPAoocGuNTulaX7UWCnNjwRka+rcck9KEjwyNA2LrV97ZcDHo6GiMgzalxyB4AeLVyr+T5/fQbu+WSzh6MhInK/Gpncg8T+JNrWUg54ZjpAIiJPqpHJPdgquY+/splOkRAReUaNTO6t4qIs1sNCHP81DHtrNQ6czscvu09ZbC8pLUP+ZU68TUS+p0Ym9yb1auHAiyMBAE3r1UJ8dC2H7Q9nXcA17/yOBxb8YbH9qYW70Pm5Xz0WJxFRVQV0+QFHwkOCseXpYagVFowt6WdcPi5x2hIAQMasMVj4xwkAQFmZQlCQ6/34RESeViPv3I3i6oSjdngI2jSsbdo2546eLh176M9803Ip32YlIh9TY+/czTWrH4nY2mFo37gORnRq7NIxw9/+3bRcWqYQWvny8UREHsPkrkl9ZrhpeeGUfrjpv+tdPrbUesZtIiKd1ehuGXt6uviSk1EJkzsR+RgmdzfIyLlosf7C4r24m2+2EpGOmNzd4Lr311msf7Q2Hav4ZisR6YjJ3U1OnbukdwhERCZM7nY8cFXrSrXv+8pKJE5bgouFLBNMRPpjcrdj2qgOyJg1ptLHnThbfgffbeavWLbntDvDIiJyCZO7Ey/dkFSp9u/8dtC0nFdQjPs/3+rukIiInGJyd+IvvVtUqv3S3bxTJyL9MbkTEQUgJncveODzrfhi01G9wyCiGoTJ3QVxdcKrdfwve07j6R92uykaIiLnmNxdcGP3phbr9SJDdYqEiMg1TO4ueGpkB+ybORIvXNcJTaIjsOyxQQCAiX2aV+pzXlm6zxPhERFVIEqnWuTJyckqNTVVl3O7y/rDObh93qZKHVOVsfNEREYislUpleysHe/cq6Ffm1jc2bdyQyW/2XLMQ9EQEZVjcq+mmdcl4d7+LV1u/9TCXbhgp0RBZt4lJE5bgq1Hz7orPCKqoZjc3WDGtR0r1T4kSLDt2FkkTluCgqLyRL/uUA4A4OvNvLsnourhTEw66PCvX0zL3209gTv7JqKwpBRLd58CAHDuDyKqLpeTu4gEA0gFkKmUGmu17wkAkwCUAMgGcK9Sim/tuOBYbgHeTzmMn7Zn4uCfFwAAej3kJqLAUZlumakA7I3l2wYgWSnVBcB3AF6rbmD+7O3burrcdt7adLy+7IApsQOAdWovLVPYe/K8m6IjoprApeQuIgkAxgCYZ2u/UipFKVWgrW4EkOCe8PzHA1e1xuD2cZg+qgNu6F69r5+dX2ix/vbygxj93hocOJ1frc8loprD1W6ZdwA8CaCOC23vA7C0yhH5qWmjOtjcnvbyaLT658+V+qy1h3NwqagUxWVlqBMegtkphwEAfxw7i/aNXbkERFTTOb1zF5GxALKUUk4Lk4vIRADJAF63s3+yiKSKSGp2ds2YYzQoSLDn+WsqfVxmXgG6PPcr3k85jCvi6wIApn+/C4NeS3F3iEQUgFzplukPYJyIZAD4GsBQEVlg3UhEhgF4GsA4pVSh9X4AUErNUUolK6WS4+LiqhG271s6dSCma3fzUeGVH5S0+qBhWOQbvx5EbO0w0/ZjZwrsHUJEZOI06yilpgOYDgAiMhjA35VSE83biEh3AB8CGKmUyvJAnH7nivi6pjvuqnhh8V7T8hpt/DsRkauq/BKTiMwUkXHa6usAagP4PxHZLiKL3BId2bQxLRfz1qTpHQYR+bBK9RcopVYBWKUtzzDbPsytUZFD4+dsBABMGthK50iIyFex/ICXLH98EFrHRbn1M+evS0da9gWUOXil9cPVR7D6YM14eE1E5ZjcvaRtozpY8uhA3NzTfa8APPe/vRj65mr8d/URu21eWbofd3282W3nJCL/wOTuRRGhwXjjFtffXnXVH0fP4nJxKRKnLeGEIEQEgMldFzFunqYvLeciTuZdAgB8uLr8QWtqxhm3noeI/AeTuw7+NqK9Wz8vPeciZvy0x7R+Mu8SsvMLcfMHG+wes/14Hi4Xl7o1DiLyHSz5q4OmMbUqbNv/wkiEhwSh5fTKlSowWnu4fCx8v1krbbbZcCQXEaFBaFKvFq5/fx2u79YE74zvXqXzEZFv4527DsTGtojQYIgImteP9Mg5lVKYMHcjbvjPelzUZoL6cftJJE5bYndmKCLyX0zuOhjYNg4PDm6ND+/oCQC4b0D5NH2/PznEI+f8ZF2GaTlILP952ZLOvnmiQCN6TQyRnJysUlNTdTm3Lzl+pgAJMbUgZgn3XEExus781WPnfGhIa7yfUj588su/9kbt8BB0SajnsXMSkXuIyFalVLKzdrxz11mz+pEWiR0AoiNDcWVijMfOaZ7YAeC71BMYN3sdlu/902PnJCLvYnL3UUWl9n+jeutW946V/35bJgDgr585/01qU1ouPt+Q4dbzE5H7Mbn7qE5NDBUlB7SJxef39UKfVvUBAHPu6Ikbe3huoqvT5y473H/bnI34l9mwSyLyTRwK6aOevbYjJlzZHJ0TogEYHsJ6Q59XVmDhlH7o2SIGFwpLoJRC5+cM/f+7qzDpCBHpg8ndR4WHBJsSu7fd9N/1eOXGzpjx024Um3UPJT27zO4xz/y4CyFBQdicfgav3tRFt9iJyIDdMn7q03t7efTzp3+/yyKxWyssKUVhSSnOXCwCACzYeAzz12dg76nzePWX/R6NjYicY3L3U1e1i0P6K6N1O/+tH27ElAV/oMcLyyvsCw8JQl5BEYpLy5x+TklpGZbuOgW9huQSBSomdz8mIph3p9Phrh6x43geVu63PaNiRGgwus1cjie+3eH0c2anHMaUL/7gMEwiN2Ny93PDOjbCzudG6BrDn+ctR9goGO7C/7fjpNNjj58xVLPMu1Ts/sCIajAm9wBQN8J+CeGlUwd6/Py9X15hsf7zrtM22x0/U4Bnftxlsa20zNB1ExJkq+IOEVUVk3sAqhUajEkDWmJAm1hcEV9X11junb/FdGc/8LUULNh4DIf+zDftL9GmCAwJ5o8ikTvx/6gAs/iRAVj1j8F4ZmxHLJjUGwCw5Wnn85f/e4JnSv+u3J9V4c5+Q1ouHv9mO9KyL2DNIUOpYuOd++GsCwCAn3edwrjZa/mglaiKOM49QLxyY2c0iApDUtOK48vj6oQ7Pb57c88WDduUlmtaNk4s8oNW9gAAHvtmO6LCQ3DXx5vx1q1d8ff/24EyZXipqm+rBqw7T1RJvHMPEBN6NceITo3t7n/h+iQAQO+W9ZExawwWTulr2rfmySHw9A3ybXM2OtxfVFKGtGzDXfvOE+dM2/88X4gftzt/MEtElpjca4ge2p15iwaGyUB6tqhv2tfMQxOEVJax372wpBTW/9Z8tiEDH69N93pMRP6K3TI1RKcm0Zh3ZzL6t4m1uT/BxtR/3vavH3cDAL7afBxWVZBNXTn3mk1sYs8P205gcLuGiIkKc3uMRP6Cd+41yLCOjVArLNi0/tm9vbDq74MBoEJNeb1VtZvoaO5FPP7NDjzy1TaL7Rk5F5E4bQm2Hj3rhuiIfB+Tew02qF0cEmOjTOvbZwxH/zYNdIzIuTMXi/DJunTTKJrUjDMoLCnFf1cdweXiUizZdQoAcOrcJYvj7vt0CwDg0/UZXo2XSC/sliGTepFh+GJSHwBA4rQlNts8cFVrfLD6iM193mCsZdMyNgrhIcGYMHcjGtYJR1Z+Id5dcRCXiw0vRVnf+B/JvggAWLTjJM4WFOHz+3p7M2wir+OdO7nkwIsjMWVwazx6dRt8MLGn0/YjOjbyaDyPfrUNH2kPWLPyCwHAlNgBmLL7sLdW46vNxyyONY6td9X9n6fivvlbqh4skQ6Y3Mkl4SHBeGpkB0SGhWBkUmPMv+dKu20zZo3Bh3c4/wegOs5fLsFv++wXG0vLuYjcC4U4nHUB07/fVWH/2YtFmLcmzaVzLdvzJ1bYKZJG5KvYLUMO/fBgP2TmXaqwfXD7hhbrr97UGU8tLE+ivvCANtXBw9PuWvdOr5b10SWh4gtce06eQ1ydcDSsE+Gx+Ig8iXfuZFMDbRhh9+YxGNulidP2bRrW8XRIlXb/51udthk3ex0+XpuOI9oLVEZj3luLoW+srlDx0lzitCV48Avn5yDSA+/cyaZ104biYmGJS23fuKUreraI8XBEnjNz8V5ErwjFjmdHIHHaEvRuaXjB60JhSYW6OPmXi3G5uMxU0uHnXaexJeMMklvE+MRvK0RGvHMnmyJCg9GgtuOaNMFBgj6t6uPmngleispz8i+X15PflH7Gbrshb6zGlS/9ZrHtlg82YO6aNCROW4Jnf9rt9FxZ+Zfx+cajVQ+WyAW8c6cqO/Ky5TR/r9zYGduOlfdzp708Gu+tPIT6UWGmN0x9VZkC3l5+0Gm7nAuFNrcbZ6X6dMNRPH9dkml7WvYFZOZdwsC2caZtDy74A6lHz2JQ21i0aBBV4bOI3IF37uQ2E3o1x2s3dzWtBwUJHhvWDnf2TTTVtLE2V6dpAm15d8Uhh/u3H8+zuy8qzPZ90tA3V+OOjzZbbDtTYJhU3HqO2dwLhabiaUTVxeROXnF9t6YW69NHdUDGrDEY3rERts8YjoeGtHZ4/I3dmzrc7w3Xv7/OtFxilZgvFZealvO05G3LibMFSNNeqLIusTDkjVUY+uZqXCgscWlycXPrj+SgtIy176kckzt5RViI4Uftrr4tcE//RIsCYPUiw/CPazo4PP6t27p5NL7K6vL8rxbr64+U16uf+b+9+Gl7psX+g3/m4+ddpzDotRTTNutcfP6y4QF20rPLMPmzVJdjWX8kB7fP3YTZKw+7fAwFPva5k1fc278lsvML8eTI9oi004Vhz5JHB3goqqorKCq1u+/7bZn4flsmPlmXYdo24u3fK/X5KQey8cHqI2harxau7ep4KGrWecNzgLQcdulQOZfv3EUkWES2ichiG/vCReQbETksIptEJNGdQZL/qxUWjOfGdXI5sT85sj1eu7kLbuzeFJ2aGGaXalTX+YxSvsRRH71R/uVizP09DWU2ulRmLd1fobplZeRfLsa7vx1yW3fNybxLGPrmqgpF2cg3VeYWaiqAfQBszbh8H4CzSqk2IjIewKsAbnNDfFQDZcwaY1q+NbmZaXnZY4PQbeZyh8eKVL1csLe9u+IglAKW7j4NVaHUmevsHfvyz/vw1ebjSD16Bi1jozDTbBRPVXy1+RjSsi9izu9pePbaTtX6LPI8l+7cRSQBwBgA8+w0uQ7Ap9rydwCuFr7RQZXU0Mlcr/Uiw7Bjxgj88thAdGhc8Y3Yr/7aBxunX42vJ/dx6Xz3D2pVpTjd5eddp7F092kAwMs/77fbrrDEsgvo9Lnyt2YX7zyJx7/ZAQBYvtey1s65S4ax+2sO5eCzDdUfV1+kPeQ1724i3+Vqt8w7AJ4EYO8RflMAxwFAKVUC4BwA3y4MTj5n3bSh2PXcCIdtoiND0aFxXbxxS1eL7a3jotC3dQM0qhuBPq0aYOGUfk7P162ZZycFd5fuM5fjaK5hhM0vu0+jzysrsOZQNo7mXsTDX5Z320SFG34Rz71QiNIyhdwL9kftOLPhSC7Scy5abCsp9ZNfiQiAC90yIjIWQJZSaquIDLbXzMa2Cj8JIjIZwGQAaN68eSXCpJogNDgIocGu3W+0NJtkZPnjg9CmYW2L/c4S95eTevvNrEwFRaW46vVVFtsmf7bVYvglAPRKrI8zF4vQ88XfcP9VrSq8abv+cA5SDmTh6TEdnZ5zwlzDhObmXWSZZ9nX7k9c+T+pP4BxIpIB4GsAQ0VkgVWbEwCaAYCIhACIBlDhHW6l1BylVLJSKjkuLs56N5HLgrRev94t66NtozoV6roEB4mpLMLVHRoi9ZlhplE3kWHB6NcmFgf+zPdu0G5kndgBYMmuU6Yum2Vad4+52+dtwtw16TYf3rqCHa3+xWlyV0pNV0olKKUSAYwHsFIpNdGq2SIAd2nLN2tt+DsceUytsGAsnNIXc++y/4ar8W5+fK/miK0djqb1DJOAhwQZspRxPZA8/NUfACqOoTc3O8VyPLyr/6sGmWX3DWbj+sk3VfklJhGZKSLjtNWPADQQkcMAngAwzR3BETnSs0V91I0Itbv/rwNbYf49V2LYFYba88bJwW+70jACZ2KfFm6NZ9HD/d36eVVhfPv12JkCu23esqqh03L6z3huUXntH2P/vrXgoPLkPmHuRuw7dd7luFL2Z2H2yorlHXZnnsP+065/DrmuUm+TKKVWAVilLc8w234ZwC3uDIyouoKDxGJSkfCQYOx/YSTCtH79ZvUjkfL3wagfFYaw4CBcMeMXi+MHto3FmkM52Pav4Rjz3hqcPGe/tvt3D/S1OemHr9p14hw6xJePOJq/PgN7Tp7DFfF17Y6sMU/ugKEksqvu0aYpfHhoW4vtY/+9FoBl3z65B99QpRolIjTYYt38waw180m0v32gLwa8mmK3bXJi/eoH50XXzl6LpKZ1LapSbsk4iy0Zlg+Zf9yWiaYxtXDibAHqRlimi6O5Bbjlgw1Y/vggtG1UvclajF1DlRlBXVqmKvyDQ+WY3Ik0TaIjcOr8Zfz3Lz1hnTMSYipWtby9d3N8uelYhe3+YnfmeezOdNwl8tg32+3um/k/Q1fO4p2n8Pjw8uT+2YYM5FwowppD2RAA3z/ovLuq5fSf8a+xHXGfWc0hR9YcysYdH23Goof7+9VvTN7E5E6kWT/9aof7U58ZhuQXDRN1DO3QEC/f0Nmvk3t1lWhPbd9dcQhN6kWgab1IDGgbW6F2/+8Hs136vG+3HHc5uRvr529OP8PkbgerQhK5yDivLAB8fPeVVf6cWCczXBkdemlUlc/hDcZKnwDw1MJdmPjRJpvt7vzYsp797sxzNh+uAsCY99bg29TjLsdg3Y1z6twlzFuT5vLxgYx37kQuMiaSW8ymFVz5t6ss3tZ7d3w3iAg6NK5ToRLkZ/f2wqB2cVBKodfLK5Cdb3tWJwC4ql2cyy906SWvoLjCth+2nXB4zBPfbMeP2zNRpoCHhrSx2Gd87+DJ73Za1BQyUkrhUnEpIsNC7NYPuueTLdh/Oh+jOscH5FDXyvDtnx4iH7P/hZF49aYupvVWcbXROq787djrujXFuK5N0K5RHdzYw3KCEeNsVCKC9dOGOjzPi9dXr8iXXox1buz5flumaQx+sVU5A2e/0bzx6wF0nLHM4WQoxmkQdx7Pw64T51yIuFzK/iwcy7U/hNTf8M6dqBKsR9s48vINnXFLz2b4essx/LT9pEW549DgIDw4uDX+s+qIJ8L0C0VWs02Zz097LLcAZUohMTYKP23PxNSvyx/sTpi7yTTG3voFrBytns6ULwwvc1VmiOU987cgOEgqzA3sr3jnTuQhEaHB6Nu6AV69qQsWPzIAcVZVL6cOKx/zPfv27jjw4kg8OtTQVRFj1r/vTh3jbVXs1kfSs8vs7hv0egoGv7EKfxw7a5HYAVi8PFXm5O3aZ37chevMpkd0JpCmKmRyJ/KwiNBgJDWNrrA9PKT8t4CxXZogPCQYjw1rh30zR6K2VuHxmTFXmNqYP8C0tviRAfjHNe0dxvHDg/0wtENDh218zY3/We9w/5pDOSgsKcXA11bixNmKXSoLNh7DDhcmTVm042SVY/RVTO5EPiQoSExlEgBg0sBWyJg1BhmzxtisU5/cIgYD28YiqWk0Ojaxf1fev00DdG8eg7q1Aqsnds2hHDzx7Q4cP3PJ4Utm1t035sXTdmeew6N2ZrzKzi9E4rQlWLn/T5v7HUk5kOX0AbMnBdaVJvIzXRKisdPFB3/x0REAgJt7JuC7rSfw5i1dcZPZyJ0gG293zrmjJyZ/vtU08sZWG3+3/nCO0zazVx7GzsxzeOn6JJSUKfSbtRIPDWmN+OhapklNbNlz0nBt7p1vmLA87eXRCHLxrdh7PjGUXLihe4KTlp7B5E6kox8e7O9yVcb46FrY8vQwNIgKqzBZCQB0b17xZR5j904drcBaQkz58MCkpnXtvqH60JDWeD/FPx72nrUxJNPam1qxtOV7/0RosCE52/t+l4pKTb89WY+j33vqPP6z6jDevq2bRbeaL2JyJ9KRoTaK63fT1g9lzdWNCEWnJnWx5+R5PDG8HSYPaoXwkCBMH9UB4680TI5zTafGeP/2HujbugFOnbuEMe+ttfiMz+/rhUtFpRjRqTFGdorHtbPX2jqVX7Megmmt68xfsfmfVyPlQBYaRFn+fT/y1Tak51zErcm5FkXpjC4XlyI4SHziHQUmd6IANKR9Q9Owzfuvam3aLiIY0yUeAFDfxoicKxPrm47rnFDxIbC1Lyb1xl/m2X4z1V8VlZSZJmKYgi0kAAAK8ElEQVSf2Mdyxjjj1INP/7Ab68zeVUictsS0XDcixOE/wt6i/z8vROQ2L1yfhG7N6qFto9rOG1vp06p+pcbxr5s2FE0C/C3QBRtt1w7KzLuEt5cfNPXJmzt/uQRHsstr4hcUlSA7v7BS9e/dQfSaMCk5OVmlpqbqcm4iMkjPuYiI0CDERIYhJEgQYtWdsP5IDm6fW35nvn3GcNNdbcasMcjMu4T+s1a6LZ6beiRg4R/6jTDxhFZxUcg6X4gLhSVIf2V0pcoa2yIiW5VS9qcg07BbhqgGc1TP3lz35vUw/+5eiI4MxfLHB6G2Vtvd+HDSmRYNInHUyav9X0/ugz6tGgRcck8zu4t/89eD+LuT9xHchd0yRORUeEgQoiMNI27aNqqD+GhDd0y9WoZ++/sHtcKaJ4dUOO6DiT0AAHf0aYHHhrWtsN9c75b+NeFJVXy6PgPFVmUXPIXJnYjsMo4W6dDY9gtSYSFByJg1BtNHX4Fm9StOaDIyKR5fTuqNe/q3xIRezSvsv8KsHEJ1uyv8QX5hCdo+vdQr52JyJyK72jeug4VT+uKfo69w3hiGcgmxtcMskna/NrEIDhI0qhtRof2/J3SrcmwHXhxZ5WP15o0aNuxzJyKHerZwvbtk0sBWmDSwFS4Xl6KgqNRh20MvjbI5HrxjfF0UFJXgpRs62xxm+eL1SRjXrYlLLxHd1bcFPrUz4beeXl92ANNGdfDoOZjcicjtIkKDbQ6rXDp1IO7+ZDOmXt3OlNhX/O0qHDydb2rz89SBpuX7BrTER2vTLT5jYp8WLsfRt3WsTyb3dS6UTKguJnci8por4uti0z+HWWxrbTXhiTljPZ2QIEFJmcI7t1WuG+eaTo0wvGMjLN9b+cJfnmSrSqi7sc+diHyWcXamf1zTHgun9MX13S1nt/r+wX4Vjrm2axMAwKQBLSEiCHGx0FdlJvaorr02Xn5yN965E5HPuq5bE4QEC0YlxWt1eCz1aB5jWn50aBtc1b4hzl8qxv92nMQ1SY0BlBdPc8UHE3vggQV/2N3/4vVJqB8Vhge/sN/GFc7q27gDkzsR+SwRwdguTVxq+8SI8peDdj9/TfmEJ2M7onF0BP698jB6JdbH5owzFsdtnH41/jx/GYBh6KYjE/u0wN6T1S8j0MsLY/rZLUNEfq2hjSJd5nfr0bVC8bcR7bHq74PxyT1XAgDu7pdo2t84OgJdm1Usl2w0vGMji3VnU/sZXZkYY3dfVLjnywXzzp2I/NpPD/d3qShXolZqwdi33rx+pM076H6tG2D9kVzT+tw7ky2qPhaWOB7iafTO+O526+4Ee+GFLSZ3IvJr8dG1TOUQKuPeAS1tbr8lOQHrj+RiSPs4PD8uCQAwqF0cbtFmvSosdlw+YEyXeIzo2AhNHVTM9MbbuEzuRERmjFMR1okIRfMGhpIKn93by7TfuM2eSQNaorv2oHdI+zikHMiu0KZ/m1h3hWsXS/4SEZkpKCrBI19uw/PXdUJCjO1Efrm4FGVKQSmg07PLAADb/jUcpUqZhm8Chi6ceWvS8fqyA6ZtO2aMMBVhqwpXS/4yuRMRVcPR3IvIKyh2+FDWvM++uuPpWc+diMgLWjSIQosGrrX96aH+ng3GDJM7EZGHvXRDEjrG13V4d+9uTO5ERB72l96uFztzF77EREQUgJjciYgCEJM7EVEAYnInIgpATO5ERAHIaXIXkQgR2SwiO0Rkj4g8b6NNcxFJEZFtIrJTREZ7JlwiInKFK3fuhQCGKqW6AugGYKSI9LFq8wyAb5VS3QGMB/Af94ZJRESV4XScuzLUJ7igrYZqf6xrFigAdbXlaAAn3RUgERFVnku1ZUQkGMBWAG0AvK+UespqfzyAXwHEAIgCMEwptdXG50wGMFlbbQ/ggHUbF8UC8Pz04b6B3zVw1aTvy+/qPi2UUnHOGlWqcJiI1APwA4BHlFK7zbY/oX3WmyLSF8BHAJKUUo4LH1eRiKS6UjgnEPC7Bq6a9H35Xb2vUqNllFJ5AFYBGGm16z4A32ptNgCIgOFfLyIi0oEro2XitDt2iEgtAMMA7LdqdgzA1VqbK2BI7hUr1BMRkVe4UjgsHsCnWr97EAyjYhaLyEwAqUqpRQD+BmCuiDwOw8PVu5VnC8XP8eBn+xp+18BVk74vv6uX6TZZBxEReQ7fUCUiCkB+l9xFZKSIHBCRwyIyTe94XCEizbQ3ePdpb/lO1bbXF5HlInJI+2+Mtl1E5D3tO+4UkR5mn3WX1v6QiNxltr2niOzSjnlPvDG9ugMiEqy9sbxYW28pIpu0uL8RkTBte7i2fljbn2j2GdO17QdE5Bqz7T71MyAi9UTkOxHZr13jvoF6bUXkce1neLeIfKW9wR4Q11ZEPhaRLBExHwno8eto7xzVppTymz8AggEcAdAKQBiAHQA66h2XC3HHA+ihLdcBcBBARwCvAZimbZ8G4FVteTSApQAEQB8Am7Tt9QGkaf+N0ZZjtH2bAfTVjlkKYJTO3/kJAF8CWKytfwtgvLb8AYAp2vKDAD7QlscD+EZb7qhd33AALbXrHuyLPwMAPgUwSVsOA1AvEK8tgKYA0gHUMrumdwfKtQUwCEAPALvNtnn8Oto7R7W/j57/U1ThL78vgGVm69MBTNc7rip8j58ADIfhJa54bVs8gAPa8ocAJpi1P6DtnwDgQ7PtH2rb4gHsN9tu0U6H75cAYAWAoQAWaz/MOQBCrK8jgGUA+mrLIVo7sb62xna+9jMAw5vZ6dCeX1lfs0C6tjAk9+Na4grRru01gXRtASTCMrl7/DraO0d1//hbt4zxh8vohLbNb2i/mnYHsAlAI6XUKQDQ/ttQa2bvezrafsLGdr28A+BJAMaX2BoAyFNKlWjr5vGZvpO2/5zWvrJ/B3ppBcOw30+0bqh5IhKFALy2SqlMAG/AMPT5FAzXaisC99oC3rmO9s5RLf6W3G31NfrNcB8RqQ1gIYDHlFLnHTW1sU1VYbvXichYAFnKsvyEo/j89rtqQmD4Vf6/ylA47yIMv1rb47ffV+sLvg6GrpQmMJQaGWWjaaBcW0d8/rv5W3I/AaCZ2XoC/KRImYiEwpDYv1BKfa9t/lMMdXmM9XmytO32vqej7Qk2tuuhP4BxIpIB4GsYumbeAVBPRIzvVZjHZ/pO2v5oAGdQ+b8DvZwAcEIptUlb/w6GZB+I13YYgHSlVLZSqhjA9wD6IXCvLeCd62jvHNXib8l9C4C22tP5MBge0izSOSantKfiHwHYp5R6y2zXIgDGp+l3wdAXb9x+p/ZEvg+Ac9qva8sAjBCRGO0uagQMfZSnAOSLSB/tXHeafZZXKaWmK6USlFKJMFyflUqpvwBIAXCz1sz6uxr/Dm7W2itt+3htxEVLAG1heCDlUz8DSqnTAI6LSHtt09UA9iIAry0M3TF9RCRSi8X4XQPy2mq8cR3tnaN6vP1Qxg0PPEbDMNrkCICn9Y7HxZgHwPAr2E4A27U/o2Hof1wB4JD23/paewHwvvYddwFINvusewEc1v7cY7Y9GcBu7ZjZsHrAp9P3Hozy0TKtYPgf+DCA/wMQrm2P0NYPa/tbmR3/tPZ9DsBshIiv/QzAMM9BqnZ9f4RhlERAXlsAz8NQfmQ3gM9hGPESENcWwFcwPEsohuFO+z5vXEd756juH76hSkQUgPytW4aIiFzA5E5EFICY3ImIAhCTOxFRAGJyJyIKQEzuREQBiMmdiCgAMbkTEQWg/wdo4WzB5gfCzwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learner.sched.plot_loss()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set up data for classification\n",
    "This time we cannot use the training examples labeld `unsup`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_trn = pd.read_csv(CLASS_PATH/'train.csv', header=None, chunksize=chunksize)\n",
    "df_val = pd.read_csv(CLASS_PATH/'test.csv', header=None, chunksize=chunksize)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "0\n",
      "1\n"
     ]
    }
   ],
   "source": [
    "tok_trn, trn_labels = get_all(df_trn, 1)\n",
    "tok_val, val_labels = get_all(df_val, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [],
   "source": [
    "(CLASS_PATH/'tmp').mkdir(exist_ok=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Save tokens and labels:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(CLASS_PATH/'tmp'/'tok_trn.npy', tok_trn)\n",
    "np.save(CLASS_PATH/'tmp'/'tok_val.npy', tok_val)\n",
    "np.save(CLASS_PATH/'tmp'/'trn_labels.npy', trn_labels)\n",
    "np.save(CLASS_PATH/'tmp'/'val_labels.npy', val_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [],
   "source": [
    "tok_trn = np.load(CLASS_PATH/'tmp'/'tok_trn.npy')\n",
    "tok_val = np.load(CLASS_PATH/'tmp'/'tok_val.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [],
   "source": [
    "itos = pickle.load((LM_PATH/'tmp'/'itos.pkl').open('rb'))   # We obviously want to use the same numericalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [],
   "source": [
    "stoi = collections.defaultdict(lambda: 0, {string:idx for idx, string in enumerate(itos)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "60002"
      ]
     },
     "execution_count": 208,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(itos)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Numericalize:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [],
   "source": [
    "trn_class = np.array([[stoi[string] for string in text] for text in tok_trn])\n",
    "val_class = np.array([[stoi[string] for string in text] for text in tok_val])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Save and load:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(CLASS_PATH/'tmp'/'trn_numericalized.npy', trn_class)\n",
    "np.save(CLASS_PATH/'tmp'/'val_numericalized.npy', val_class)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [],
   "source": [
    "trn_class = np.load(CLASS_PATH/'tmp'/'trn_numericalized.npy')\n",
    "val_class = np.load(CLASS_PATH/'tmp'/'val_numericalized.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [],
   "source": [
    "trn_labels = np.squeeze(np.load(CLASS_PATH/'tmp'/'trn_labels.npy'))\n",
    "val_labels = np.squeeze(np.load(CLASS_PATH/'tmp'/'val_labels.npy'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Classification\n",
    "### Add a custom linear head ontop of our language model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [],
   "source": [
    "bptt = 70\n",
    "emb_sz = 400\n",
    "nhidden = 1150\n",
    "nlayers = 3\n",
    "bs = 48"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [],
   "source": [
    "voc_size = len(itos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt_fn = partial(optim.Adam, betas=(0.7, 0.99))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [],
   "source": [
    "min_label = trn_labels.min()\n",
    "trn_labels -= min_label\n",
    "val_labels -= min_label\n",
    "\n",
    "count = int(trn_labels.max()) + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 217,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Since we intend to do classification, we have to read an entire movie review. Since the reviews don't have the same length, we have to pad the shorter ones in a batch.**\n",
    "\n",
    "**To reduce padding and thus compute, we do the following things:**\n",
    "\n",
    "1. For validation we sort the reviews regarding their lenghts, then we group them into batches. This reduces padding because there won't be *really long* and *really short* reviews in the same batch.\n",
    "1. For training, we don't strictly sort, because we want more *randomness*. We use the *sortishSampler* which groups *larger* reviews together and *smaller* reviews together but maintains a degree of randomness while still reducing padding."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [],
   "source": [
    "trn_ds = TextDataset(trn_class, trn_labels)  # trn_class[i] numericalized version of review i/tok_trn[i]\n",
    "val_ds = TextDataset(val_class, val_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [],
   "source": [
    "trn_sampler = SortishSampler(trn_class, key=lambda x: len(trn_class[x]), bs=bs//2)\n",
    "val_sampler = SortSampler(val_class, key=lambda x: len(val_class[x]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [],
   "source": [
    "trn_dl = DataLoader(trn_ds, bs//2, transpose=True, num_workers=1, pad_idx=1, sampler=trn_sampler)\n",
    "val_dl = DataLoader(val_ds, bs, transpose=True, num_workers=1, pad_idx=1, sampler=val_sampler)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [],
   "source": [
    "modeldata = ModelData(PATH, trn_dl, val_dl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {},
   "outputs": [],
   "source": [
    "drops = np.array([0.4,0.5,0.05,0.3,0.4])#*0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = get_rnn_classifier(bptt=bptt, max_seq=20*70, n_class=count, n_tok=voc_size, emb_sz=emb_sz, n_hid=nhidden, n_layers=nlayers, pad_token=1, layers=[emb_sz*3, 50, count], drops=[drops[4], 0.1], dropouti=drops[0], wdrop=drops[1], dropoute=drops[2], dropouth=drops[3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner = RNN_Learner(modeldata, TextModel(to_gpu(model)), opt_fn=opt_fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.reg_fn = partial(seq2seq_reg, alpha=2, beta=1)  # Regularization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.clip = 0.25  # Gradient clipping\n",
    "learner.metrics = [accuracy]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {},
   "outputs": [],
   "source": [
    "warnings.filterwarnings(\"ignore\", category=UserWarning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 267,
   "metadata": {},
   "outputs": [],
   "source": [
    "wd = 1e-7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "metadata": {},
   "outputs": [],
   "source": [
    "lrs=np.array([1e-4,1e-4,1e-4,1e-3,1e-2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.load_encoder('lm_model_1_enc')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 1: Train only the last layer group"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.freeze_to(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aa7dd93a60734c0ba00381dbfbcd80aa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  2%|▏         | 23/1042 [00:03<02:35,  6.56it/s, loss=0.497]\n",
      "epoch      trn_loss   val_loss   accuracy                      \n",
      "    0      0.266736   0.178909   0.93108   \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.17890858514428137, 0.9310800185966491]"
      ]
     },
     "execution_count": 232,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.fit(lrs=lrs, n_cycle=1, wds=wd, cycle_len=1, use_clr=(8, 3))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 2: ... last two layer groups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.freeze_to(-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "19d65981783640d3adc0a16ba69fe6c6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                      \n",
      "    0      0.259551   0.161064   0.93888   \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.161064199603796, 0.9388800192832947]"
      ]
     },
     "execution_count": 234,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.fit(lrs=lrs, n_cycle=1, wds=wd, cycle_len=1, use_clr=(8, 3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.save('classifier_before_unfreezing_all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.load('classifier_before_unfreezing_all')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 3: Unfreeze entire network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.unfreeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 263,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7fc34ad41c6f496d99a621b7e8c72c46",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                      \n",
      "    0      0.221449   0.170925   0.93912   \n",
      "    1      0.176093   0.161105   0.9448                        \n",
      "    2      0.149651   0.157458   0.94648                       \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.15745827977597712, 0.9464800192451477]"
      ]
     },
     "execution_count": 263,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.fit(lrs=lrs, n_cycle=1, wds=wd, cycle_len=3, use_clr=(32,10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 264,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.save('classifier_after_unfreezing')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 265,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJztnXecVOX1/z/nzsx2OkvvXURAWBDBLirElthiS9RoNLHExMSE2Gs0Gr/5GWOimJjE3mINKAL2ArL0Jh0WpC2d7VOe3x/3PjPPvXPvzJ3Z2Wl73q8XL+7c+tyd3XPPPc85n0NCCDAMwzCtAy3TA2AYhmHSBxt9hmGYVgQbfYZhmFYEG32GYZhWBBt9hmGYVgQbfYZhmFYEG32GYZhWBBt9hmGYVgQbfYZhmFaEN9MDsNK5c2fRr1+/TA+DYRgmp1i4cOEeIUR5vP2yzuj369cPlZWVmR4GwzBMTkFEW9zsx+EdhmGYVgQbfYZhmFYEG32GYZhWBBt9hmGYVgQbfYZhmFYEG32GYZhWBBt9hmGYVgQb/SyjtjGAtxZvy/QwGIbJU7KuOKu1c+TdswAAfTqWYmzfDhkeDcMw+QZ7+lnEzOU7wsuHGvwZHAnDMPkKG/0sYmN1TXg5GBQZHAnDMPkKG/0s4nBDILwcCIUyOBKGYfIVNvpZRLd2ReHlBj8bfYZhUg8b/SyiV4eS8LLPY/5qdhysx/99uAZCcNiHYZjkYaOfRQSVkE5jIGja9stXluAvH63Hyu2H0j0shmHyCFdGn4imENEaIlpPRNNstt9CRKuIaBkRzSWivsq2PkT0IRGtNvbpl7rh5xdBJaJT7zcb/cVbDwAAQuzpMwzTDOIafSLyAHgSwFQAwwFcQkTDLbstBlAhhBgJ4A0AjyjbngPwqBDiCADjAexOxcDzkV+8sji8rMb099U2oSmgfw6xzWcYphm4Kc4aD2C9EGIjABDRKwDOBbBK7iCE+FjZfx6Ay419hwPwCiFmG/vVgHEkqFj0BsPTn/CHudh5qCG8/sv1ezC6d/u0j41hmPzATXinJ4CtyudtxjonrgbwvrE8BMABInqTiBYT0aPGmwNjYfOe2vAyUcToqwYfAB6dtSat42IYJr9wY/TJZp1tkIGILgdQAeBRY5UXwPEAfgNgHIABAK60Oe5aIqokosrq6moXQ8ovdh9uwEl/+iT8udjnQX1TEDOW7XA+iGEYJgncGP1tAHorn3sB2G7diYgmA7gdwDlCiEbl2MVCiI1CiACAtwGMsR4rhJguhKgQQlSUl8dt5p53/Pq1pabPRT4PGgJB3PDSogyNiGGYfMWN0V8AYDAR9SeiAgAXA3hX3YGIjgbwNHSDv9tybAcikpb8FChzAYxOTWPA9HlfbRNemFeVodEwDJPPxDX6hod+I4BZAFYDeE0IsZKI7iOic4zdHgVQBuB1IlpCRO8axwahh3bmEtFy6KGiZ1rgPnIajSIRtE9vPSlzA2EYJu9xJa0shJgJYKZl3V3K8uQYx84GMDLZAbY2OA2fYZiWhPX0swB1pjxW8dXxgztj6766lh8QwzB5C8swZAGVW/aHl0sKvLj+pIHwauakKY2Az9ftwea9ddhb02g9RUJ8sGInXq/cGn9HhmHyDvb0s4xu7YpQ4NUQsJTeejRCyNDYr65pRKeywqSv8bMXFgIALqzoHWdPhmHyjVbl6V/6zDz86J/zMz2MuBR6o+vX1Mneys37o7Ynw/7appSch2GY3KHVGH0hBL7asBefr9uDhVv2ZXo4MSnwRn8tqtH/dmdqlDaPvn92Ss7DMEzu0CrCO19t2IMteyMToG8u+g5j+3bM4Ijseev6iQDsjb5HifEfN6hz2sbEMEx+0So8/UufmY/fv7k8/LmuKRhj75bnYJ0f/abNwAcrdprWH92nAwDAp0UrXyiOvkmCmWEYJhFahdG3Ym1Qkm6qjLTLJz5aF3ffgeWlAHRP//GLRwMA/Gz1GYZJklZp9NUG5HbMWrkT9763ssWuX1yg/9jrjTeOir4dMLJXO9t9h3VrC0CP6Y8x3gRUo199uHnpmwzDtC5apdFvjNN0/LrnF+JfX25uwRHosRqZlqkRobQgenrlwrG9UN5GT83cV9sEr8d8XOXmfRj34JyE1TitNQAMw7QeWqXR79WxOKPXl1W3VfvqEAwJhISApnwTI3rqXv8JQ8rh80QMtNfYKWB4+su/OwgAWLA5sWwkteo3yK24GKZV0Sqyd6y0LfJl9PqBYMTQbj9Qrxt9ZaZ2RM92WHr36WhX7MPqHZH0TPkA8BvHS3tNCTruqp33B0PwaNzXhmFaC63S029yOREqklQ/E0LEnCy26uuEBEAWy92uWH8w1Sqyy16P4emHQqbxaQlafXV3tz8LhmHyg9Zp9APuDF2yBvGvH63H0Ds+wOEGv+12NaQihG68ncLsaoP0aE9fGn33YwuFhEnJ0+/yZ8EwTH7QKo3+Gwu3udrvrUXfAQDueHs5rn9xoevzv7lYP263Q2aN9NQBoCkYRNAS3lFRM3V8RkxfrpPPjkQ8feuDzB/kmD7DtCZapdEHIo3HY7GnphGvLdiKF+ZVYebyndhb04h+02bgf8uiukWaKDQqap2uoXrvjYEQQiFnw922ODL/oGkEjcxzAony7JebTJ85559hWhetwugXeKJv0y5Xf9ehBvSbNiP8+bXKbfjtf5eFP6/ecRgA4qZzFvr0idEGh9RQ9WHgDxrZOw7O+tXH9Td99no0+EPm7J1YGvxWNu+pNX3mmD7DtC5ahdFvUxSdpGQXb1+13SxkVmVpWCKNa7yK3iLD02908PRV77rBH4QQzp5+744lGNu3Q1h7x6cRAkGBQDAUzs9/5vNNtsfaMap3ewB6QZh1LAzD5D+tImUzaOMJ23n6Pps3ApX9dboUcbziriLp6Ts8HNTh3PPuyqg8fSv//fnE8LLXoyEQDEU/kEICmosZ3c6GDv+JQ8pRuWU//AGO6TNMa6JVePpqtsyvTxsCIL4Ugx03v7IEgB6Hj0WRT/+xOgm7qXny3+48jJAQUSmbTvg8BH9IRClxJpqGKo9vCmZWh4hhmPTS6oz+4K5lAIDqmoao/X756mJX54sb3jE8/QWb7Ctl7fL03Wbg+Dwa3l78HY7748fmMcV5+1CvBUQmm5vY02eYVkWrM/pd2hYBAA7URcf099S46yRVfbgxpuGXE8f/+XqL7Xar0d+0p9Yxp9+K10O2bxC/em2Jq+PlpeVkM8f0GaZ14croE9EUIlpDROuJaJrN9luIaBURLSOiuUTU17K9LRF9R0R/TdXAE0E1slKCoTma+iEBXPucc96+rKYd16+D7Xa7ZJtP1lS7urbPIfj/0be7XR0vfxbS02ejzzCti7hGn4g8AJ4EMBXAcACXENFwy26LAVQIIUYCeAPAI5bt9wP4tPnDTQ61yXhJgQdejUzyBsnw6VpnIy0vN6hLG4ftyYdUvJ7mKWTe9LIewipgo88wrRI3nv54AOuFEBuFEE0AXgFwrrqDEOJjIYRMJ5kHoJfcRkRjAXQF8GFqhpwYVtkBr0YoKfCYPP2mQAgPv/9twufeUF1jf03jggEHg9ocYUtvrDSfBJDN15u4IpdhWhVuLEhPAFuVz9uMdU5cDeB9ACAiDcBjAG5NdoDNJSpdk4B2JT7sq43E7/+7aBue+nRDwuc+9bFPMenhj6K8fjmHEHCw7s3x9H0WT7/Qpp+uE698UxV1nFvtnQWb92FvDTdsYZhcx43FsIsn2FotIrocQAWAR41V1wOYKYTYare/cty1RFRJRJXV1e5i226J0osXesxdnTh1K8Bmx3cH6nHFs9+YFDnlg8YpjdJOvfOS8X1cXc9aS3DdCQPcDhV/+nBNeNltTL/BH0SDP4gLn/oaFz39tWnbtv11uPCpr7C/1t0EOMMwmcdNcdY2AL2Vz70ARInPENFkALcDOFEIIV3CYwEcT0TXAygDUEBENUII02SwEGI6gOkAUFFRkbJ4gxAi3Hz8ogq9C1V5m0J4Nc3RC0+WL9bvwfGDywHoISUgsfBO+xJ3Gv/WmL5XeQhUH24Md9qyQ4Z0AIQrfOMZ/bH3z4bfGPCGarOEw/TPNmLB5v04+v7ZWHbP6RnvU8AwTHzcePoLAAwmov5EVADgYgDvqjsQ0dEAngZwjhAinEYihLhMCNFHCNEPwG8APGc1+C3JJ2uq8ctX9VTGIV3b4NYzhoGI4POQSbQs0SYkdqjhonB4xyFebhfeCbl8CFk9fXXo4x6cg6837HU8Vg0Fyct9uGpXzOvVNgVNb0LqW0ppYcRnuOXVpTHPwzBMdhDX6AshAgBuBDALwGoArwkhVhLRfUR0jrHbo9A9+deJaAkRvetwurRyoD5iiNXiJ93Tdx/SOaZ/x7j7/P2TyJyADO/4HWP6+v+lBRHP223bQmt/W6v0wgpDhM0OmZsPRN4YPl+3x9V1Je8s2Y7rnq8EoOsASTbusZ/UZhgmu3A1CyiEmCmEGCKEGCiEeNBYd5cQ4l1jebIQoqsQYrTx7xybc/xbCHFjaocfGzXTRQ2LeD1k0pGPN6/66nXHxr3WtzsPh5fjhXfkBadNHRY5xmW0yWPJ3pk0qLPp86odZtE4FSkPAQBHGX14E+WXry7BrJW7cLDebwotNTSj7oFhmPSR1xW5qlesevo+T2KePgBcY5E4joV8njiHd/T/jx3YWVnnzupXbjFLOxT5NPxoQqQW7i2jgYsdi6sOANANvvqzWVS133b/WO0i//7JBtODtN5FfwKGYTJPfht9B9VMr5Z4TH9ot0ih1VOXj425r/T0/Q4PFmngy8sik65uwztW+QghYBJfu3yCngX0wYqdjk1c/MGQSeBt2/562/1iCcs1BoKm6mA2+gyTG+S30Ve8WVWK2OfREq5EVZ1eTxwJY2nAF1cdsA3xhO07Af07l+rHJJm77w+GTB53p9JCLN16AD97YSGG3fmBrbdufcA4efSxMpw0ItN8glPDGIZhsou8NvqqcVZTIr0eChu0eRv3YubyHbbH//XSo/Hhr04AAAilNKFPx5KY11UN+DF/mBu1XRpZjYCfHq/n2bvN3rFDfbg1BUOm+oB/f7U5av8je7Q1fXZ6iAVjVOsW+zxRUhb/+Hwj7ntvFX73xjJMevgjN0NnGCbN5LXRVz3gMiW90KtpCAQF9tQ04uLp8zBvo70E8lkje2BIVz2so9rkod3a4I4zj3C8rmrA99oULoXCRp8gI1BuwztWjurZzjS52+gPmdI6731vVbgFZK8OxQCAh84baTqHk6xzrLePjqUFUUVtD8xYjWe/3IRXK7fiuwP2ISOGYTJLXnfOUo2Z2ifX5yH4gyFUbrY39nZIQ33JeL1OrXu7Ysd944VqpH3XiMLFVN3bFbkeCwD85yfjsf1APYgIH6yIvKk0BoKOFcYdSwswqEsZipVUUTkOO2JNdvuDoaRDUgzDZI689vTVjJjvHx2RC5LhHWv648e/OSkqD14iHxolBfpz0i4kIr31eDFzOS4i4OShXfD3y8bgplMHu7onOY4Th5SHpRvW7orkyDcGQrZa//2mzcCybQdtBducJrJjJTgFQsL0RtOzvfNDkGGY7CGvPf26xojxK1ILkzQN+2qb8NPnKk37d21baKRzRhvN7x/dE9v21+NaQ+tm2/7IxPCInm2x4rtDaAqEUOjVotIvm4IhkwSCUDx9IsLUo7q7vqeZNx+PJVsPOG5vDIRiagnZPdQemrkaL82vwm+nDMWRPSL5+7E8+aZAyPRwa1fs45AOw+QAee3p3/XOCtv1Tl2vPBrhkQtG2m7zeTT86rQhYekBtShqdO/2AID/m70GA26biR0Hza0YramP0kN20cc8ikFdynDB2F6mdXNuOSG83OAP2k7eSlRDLh8Am/fW4dO11bj9LfPPK9ZEbiBkDu/YFYU1Z3KaYZiWIa+N/vaD0X1wAeDlb+xFP32ahrNH9XB17r6dIhk8MuTzzOebAACb95iFyaz9a9WYfipQhc4aA6GY0gqzFa2dRXedBiBSnWt9CMXy9P1BEdeou23WzjBM+shroy+52RIvH++gpWPVsYlFsRIuUpeBaEmFGcu2W7ZHYvqpQM3WcSrIsqNtkQ+dywrCbz7WeYpgjKB+U0D39NWsqKh92OgzTNaR10Z/ZC/dgx1QXmpaf+3x7jXonVArWkss2TAATBLH97y3yrRNQDf4lCKr71MqcmNV0doe69HChVXW8cSy2YFQCMGQPldyUUUv232sbzgMw2SevDb654/RjdFxFlGy0hjeaTKU2JxPFTezIoRIWWgHMKejNiYoh+D1EOqa9CIr64tOrJTNF+ZVYXHVfng04Dijj4AV9vQZJvvIa6MvlCIolTZF0UZ6WDdzE/Ph3dtG7eNEqY2nX+SNXicJCZHUJK4TagvFBn8Qo3q1w4lD7A1x9LEaDjVIo09Yuf0g+k2bgSn/77OYKZuArizqIUKRQ8vGREJNDMOkh7xO2Qw6TJjaefp3nT08vLzsntNN3nM85ESuirUASiUkUhfaAcznavCHUFYk4NEIEwZ0dKw2lhR4tHCK51cb9uKqfy0AoBt01dPv0qYQuw9H98jVNDKlw6psP1CPgeVlCd8PwzAtR+vw9C13aTf5KOUWAH2C08mQ2VFaaOPpxzg+1Z6+SkMgiGBIf9C1sWlfOOMXx5k+Wx+IqmGXE843nDwQ/7McJ/HEMPr7LYqgDMNknrz19IUQeGDGagDxwzvXnTgAncuce8vGw87Tj2X0hUhduqaVBn8QwVAIXo3gsfTT/fr3p0TJR8RquiLlpycO7IwubexlIjxEjvMXB+vZ6DNMtpG3nv6WvZGKWauBtRrkZIuIZH77PhtRtc/WVjseFwqldiJXpcEfMiQmyNTO0KsRurVNTN8nELKfE1HRNDJVG2sE/PH8owAANQ0RFc6qvXUmjSCGYTJD3nr6KvHsa9cEjaHkhWuOweY9taZCLSeEEOHYe0iYG5qnmoamIDSNwk1kju7THm9dPynh81z2j/kAzGqlVjxGo3nJt/dPhUcj/O6/y029BKY+/hlqm4LY/PCZCY+DYZjUkbeevmro43nVVlkDt7Qr9mFU7/ZoX1IQd1+1IUlIiJQVZtlR2xSEhyIyC6l6q5h8RBeM6mXuratpZBJxK/Bq8GgEjWBqVFPLPXQZJivIX6Ov+NLxJk2d2io2l9u/F9HcVw2gECKh6t9EqWsKGJ6+fg1PM42+zO75xxXj8M6Nx6Gib4fwNq9GtplKIRGRpVBZuMW+Hy/DMOkhf41+HE9/qJKt44sRvmgOPz0hUvnrD6iefstN5AK6Lo6HIh54cy9lVe1UH1hFPs1UfaxS7w/i8TnrTMdvZyVOhskorow+EU0hojVEtJ6Iptlsv4WIVhHRMiKaS0R9jfWjiehrIlppbPthqm/ACVUrzM7ovXdTJAXRZ6Mxn2rU6tSWSNm8++zhmDiwU/izR4vE2mM9YD7/7clxz22VdlDfHJzOfYRR3PbnOWvx8jdV4fWJ9iZmGCa1xLV2ROQB8CSAqQCGA7iEiIZbdlsMoEIIMRLAGwAeMdbXAfixEOJIAFMA/D8iap+qwcdCVYi0K4QqUKpIUxFquXJiP9v1MpNFNXbLth3EnprojJ/mcNWk/rjm+P7hzx5lIjfWM82NJMXRfcxfmTqxa5WRlvTuEEkN3V8XuddUS2AwDJMYblzc8QDWCyE2CiGaALwC4Fx1ByHEx0IImSM5D0AvY/1aIcQ6Y3k7gN0A3OkDNJNke84mS5e25hDH9ScNBBBRwFSbiC//7mCLjEGVfvBo5Goi16oQauXN6ydGZTf98fxIz4FJFl0jiSoCd6g+EDUBnCwLt+zDTocHDcMw8XFj9HsCUAXotxnrnLgawPvWlUQ0HkABgA02264lokoiqqyuds5vTwRr96qWxmpYZXjju/16DPv2t+0buqSSQqVIasnWA7atEaOOcdDNkdjJUfRQWiM6vSQVKsftr2sK6/s0V4/n/L9/jQkPzcXuw2z4GSYZ3Bh9uz9rW4tKRJcDqADwqGV9dwDPA7hKCBEV1BVCTBdCVAghKsrLU/MikG5P35ohI7Xp6w0j982miAZOx9KCcLetVKIWSS3bdjAchon1ANQ0wu+mDMM7N9jn8cfK0QecJ4nV8FltYwCbjMYy9SlK3Xxi7vqUnIdhWhtujP42AL2Vz70AbLfuRESTAdwO4BwhRKOyvi2AGQDuEELMa95w3ZNuo2+dF5Afe9g0DO9Q4muRRuJWr11O5Mb7Wfz8pIHh3gNWfHHSWWVW0pfTTsFX004Jr1cfNEMVBdO6FBl9NxPCH6zYgeF3fcBqnwyj4MboLwAwmIj6E1EBgIsBvKvuQERHA3gausHfrawvAPAWgOeEEK+nbtjxSXN0JyrMIcM9PxynPy9VTX8hUjN5bMUqL1FrNIZ3Y2iJyOSdS5zURicM0LuPya5bPdsXmx5wn62NtGxUs38SbfKiospluHmo/+yFRahrCmJpjEbyKntrGlkviMl74hp9IUQAwI0AZgFYDeA1IcRKIrqPiM4xdnsUQBmA14loCRHJh8JFAE4AcKWxfgkRjU79bUQTq79rS2CN6cvPPo+Gkb3amWoBgi2ksql6+u2Kffj7J/r0ybJt7iaOC20MvFN45/uj9WkdJyNer3jXakhnT00jgiGBftNm4Ki7Z7kal0S9ViIvcvFCVID+5jD2gTkYde+HCY2JYXINVwnqQoiZQoghQoiBQogHjXV3CSHeNZYnCyG6CiFGG//OMda/IITwKetHCyGWtNztRHDjCY7o6b5RSjysnrvHInbmD5plGJpbJWtHkVIZK4SI2fnKjsNKhpHESX1UvlU4GX21sUy9Pxh+yP3zi03hcdldLxZvLIzkEyQyUe+md4H6+/L4nHUJjYthcom8rcjdWxPd8MPK29dPwtoHpqbkelbPXbUzPo9mLs4KpbaJikRNvxQCuHh8n2af0ymmL42+tVpXoraQrPcHTef506w1SY3lgKLPH0jA1XdT/aw+Q/48Z21C42KYXCJvjf61zy+Mu4/Xo9nGsZPBKXsH0A1nwFKR2xJyP6ph7VRWgPPHxMqsbR5SQ1/G9K3I/sSAHt5RPWk7TR43qP0B3lu63fGBI5HhLjeTvulO8WWYTJGXRl/N4X7ovKPScs3o7B3V6JvDO8EW1NOXvPjTCRjTp0P8HW2479wj4+5THMfT/9mJEd2hpkAoIc/cCWuaa7xJV1mc5ncxeWwdXYM/iHkb9yY0PobJBfLS6I9/cG54uSRGr9pU4jSRC+hvFP6geRKyJVU2AT2bJtkQkl3jeCtSWdMppm/u25ualEmylIyION65fNtqSsLT/9kLC3Hx9HnYbNQXMEy+kJdGX+XskT3Sch1ry0DViBQoRv+RD77FHhfzDZnEq2m486zh+NOFoxz3iRfTB4DXf3YserYvRoNDCChRrEY+XoaWDHfFCwMBwMh7zFk7n6zRK8P31aVWI4lhMk3eq1+1tEctmXJkN9x0yiB8tm4Plm49YApneD0U/jz9s40AgGCwZWLIK+49o9mZQV6NcPVx/WPuI/P3Y3nR4/p1xOCuZdi6r85xn0SwRogCcX6G0tP3N+Nn7SY0xDC5RN4Z/UVVmWnS4fVo+PXpQ7HCEFNTJ259Hg3+QAizV+0KG39/gumUbimzUbFMVPLhyB7xxdFkeOf4wbFlM7yahga/873uq21Cx9L4nceA6BCM0wPngr9/hQP1/rCn3xw55+Y8MBgmG8k7o3/e377K6PWlnLHq6fs8BH9I4KfPVYbXxfNSU8Xye053naH0xCVHY/3uGvRx0fO3a9sivH/z8RhYXhZzvwIvxYzpT/9sI6ZNHeZqfNafmF3YZtbKnag0unP16ajfh5uYvhMLNu/DcYPtlUQZJhfJO6OfaWTGiGrUvZqG6sPmOH6ihVPJ0qbI53rfs0clNv8hlURj4dU0U3WuFWGv3WeL1dO3evDBkMB1SqqurMSV+9U2BuAPhlz1NJY8PncdfnXaENf7M0y2k9cTudbJ1XQQ8fQjBun5eVui9mstYQOfJ2L07d44EkmPlxO5N5ys9yqwGn1rK0b5AJYyEKc89glG3zfb/QUNWMaZySfy2ugf079T/J1SzPdH697yyF6x4+iBVtI20OehsGG3a9jiJrNGIp+j7Yp9xrECT8xdh293HgIAfLJmt2n/tbtqAAAPzFiNpkAIuw4llzVV05CYXATDZDN5bfS9acrcUTn1iK7Y/PCZ6N+5NLzO2m4QSExGIJdRq4SP6N4mansiOfzyJyb7BtQ2BvDY7LW44O9fG+d3Djf93+yItMLzX29O6LotXUgH6AqiLAHNpIO8NvqeDBh9Ow7WRVeOpmsiN9OoCpeXT+gbtT2R+LqM6cuwXW2T7oHXGMJtsUT21u+uCS/f+c5KPJqk/k9Lcfe7KzHszg9M8tEM0xLk9USuG0nddGBXtZquidxMo3r6Xk3DqcO6YO63kTBMrw7um8nImL709GsUlc6j7p7l2MULMDdnBxA1sR6LdHxXrxsKooca/Ak9CBkmUfLc08+O2/PZPHxai0On3rvPQ5hoaaSeiMctf2ayGvhQfcToH24MhPvw2lFrkXGWhvx/y7bHlVpIx6S7rK84nMPzBwfr/Qk9TJnMkB1WsYXIREzfjtvPHB61Lt3tHDOF2pzdoxEmH9HFtD2RTlUyvFNaqBv9F2yyopyosRh9achvfGkxTn7sk/D639vUDKQjFCeL3VLVTjITTPjDXIx7cE6mh8HEIa+NfrbE9E8b3jVqXTyxsHxBDbl4NQ19O5XiqcvHJnUu+SMrNbzi7ywpmrHYtt+8r5o9pX4V54zugScvHWPa1031tPVNYsHmfdh1yH2qp8xssj6ccolY9RhM9pDXRj9R+YGWZN7vTzV9bh0mX++UJZFzLFNGdMPSu09P+FzS07eTmkiUQEjYPngDQRFVTxBPf2fexr048u5Z+HxddXjdhU99jVP+9Inr8UijX9eUu0afyQ3y2uiffmS0h50purUrwuaHzwx/vuyY5ne1ygXUGLUabpO59laen7cFlz4zz1aJVNpon0dzbNjulkBQ2M6rNAZCUUZfpteGQgIzl++Iqi1YXKU3Xv+NBAwDAAAgAElEQVRi3R7T+toEQjVynsL6xsAwqSavs3ey0Z1WDX9rQG0g43Uw1P5gKJzlc+fbKwAAFQ/MMf2shBDYV6tn4BD0GHhTfXOE1EJRWTltirwY0Lk0qgLXHwxBCIEBt80Mr1v34NTwmGUKaXPCGzKmX9vIIRKmZclrT7+VzJVmNWr83mli/YBNHYNk5faDWPHdQfztkw2473+rAOjFUqXNbI4TCImoyfS7zhoOTaNwm0U5J/TZ2j1Rv0uzV+0KL0svvTnFVTK8s+Og+3mKbCJXx90ayWujz2SeYUqVrHViXYa46mOEQc78yxc464kvTEaWKOIZJ8v+uqaoqmj5EJAVuPLzs19uinpAqLo/0tOPJSEdj73GW8yfPsy9puwfr9mNYx/6KNPDYFziyugT0RQiWkNE64loms32W4hoFREtI6K5RNRX2XYFEa0z/l2RysFbGXrH+6bPXdsWtuTlGBcUKfFxa73C8YZksVPGSr9pM8LL6gND0wib9zavMcuWvXVYsGmfad0xA3StJjvZBavRt9tHevrJZGZ9Y4zFmtKaC1z1rwWZHgKTAHGNPhF5ADwJYCqA4QAuISJr4vliABVCiJEA3gDwiHFsRwB3AzgGwHgAdxNRct264+APhqIqX5PtEcukjiJFZM1aLCdTL2tdZKyoLwmE2HUO39wWyZQa2cu5IczV/6k0fZZ6SbZG32LI1U9yzqLB+P1rTg1GSUHuT7OxlER248bTHw9gvRBioxCiCcArAM5VdxBCfCyEkK7XPAC9jOUzAMwWQuwTQuwHMBvAlNQM3czeGu5lmo2oRt8a05dyCtJDjmUs1Qd4PAE0NfRzyfjEs6Tks2lwF71BzJlHdY8aW1CZBP7bx+sBAPtqG3HH28vx+XpzFo8berQrApC6JvKZJJH6CSb9uDH6PQFsVT5vM9Y5cTUAGWdxdSwRXUtElURUWV1dbd3sim7GHw2TXahhGWvUQ06YyhTIxhgN1FV5Y/XZYRcOUfV+kvG62xTq6aSDu5ahTZEX5W0Ko86jpm3KUFNtYxAvzKuyDXccqGuKKactH2oNedCT9/hHPmZvP4txY/Tt3Crbb5SILgdQAeDRRI4VQkwXQlQIISrKy2P3XHWL9NKY7MEaIimwGH1/wNlQqNkhRIRbzxgKwL5Qy2t60CRufPp0KsG/rxqHRy4YhQKPhkAoFNPox6MpEMLo+2bjzndWmtZXH24Mj0+GJj9bW415G/cmPOZsY7XR44DJPtwY/W0AeiufewHYbt2JiCYDuB3AOUKIxkSObQl+ftLAdFyGSYAOJeaCrLDRNzzgWL1s9ytpnUSRtwRrBs77Nx8Pr0fDFcfquQTBkAiHThLhpKFdUFbohddD8AcEFlftN21X549kFpKT6oe8r5e/qQqv27ynFuMenBOuWFazgS6ePi/h8WYbi4yCNSb7cGP0FwAYTET9iagAwMUA3lV3IKKjATwN3eCr7YtmATidiDoYE7inG+tahCFdI9796Ud2a6nLMAny1bRTMP+2U6Mkg2VVrTSgbiWMfZqGeRv1bJcV3x0Mr//j+UeFG6lohgUOCuCfV46zPc8Ym+Y2UdfyaPCHQrjtrRWm9eoDSj52NlRHq3UebvAjqAi2yRDPJkPZ80sj/p/Im0Mu0LUNZ85lK3GNvhAiAOBG6MZ6NYDXhBArieg+IjrH2O1RAGUAXieiJUT0rnHsPgD3Q39wLABwn7GuRThlmC67cOXEfinRZ2FSQ4/2xejaNtrblt562Oi7VLP0eig8Waimbp4/pld4WU72hkLCsaPWoqoDGBVHn8nn0eAPCpw/Vp+KmmaocKpjDcYYd70/aHpA/NWY9JX37PNoWLPzcFQ174bqGuQyVfual1LLtByu8vSFEDOFEEOEEAOFEA8a6+4SQkjjPlkI0VUIMdr4d45y7LNCiEHGv3+1zG3oyNfrbJFUZmJjjenHCu+oeD2EO886AgAwsDzSllKdNPaEPf3YD5KlW2OHIWoaA1i69QB6dSgBAJw3pic0Mnvmsa7R0GSWe/jYaCAjJ619Xg1n/L/Poo67+t+5nfv+wIzVmR4C40BeVeTKP3QnjRcmu5Apm9KAPvfVZgDATacMiplq6dM0tC3S5wfUmL5dWmdz+xZUH27UvVbDsBPI8P4Vox/jGg2BoOmt4MQh5ag+3IibX1kCACh0+F3NJZnibjZvcUz2klfWUf7Rs6efG1g9/Y/X6Om6W/fV4aHzjkLnMvu2gZpG4fx/p5CQTOWcZOnUZcfEgZ3iTvbKrlwa6eNW30pCQjhqAdU3mcM7RQUebNsfCX34HIx+cyQd0k2olfSGyBfyMvCtsdHPCTwawaMRmoK6VyvjwPLhHcuWyPkAv0NIqKJfR9eKpi/9dELcfWRbRzk2tRgwJIAubYvCk7Mq9f6gqUCtoSlo6ibm80Z+Vwu9WjjWn0uevt2Ljsyeaik2VNdg7c7DmHpU9xa9Tj6SV56+zHn2sPxCzlDg0aIyV6RGTyz/UUo4nDjEXV3Ha9cdixtOHphQI3Y7NNJ7BLy7NJJ5LIRwbMre4A+aHkwNgRBUNYoCT+SB8OI1x4SXmwIhvKKkeGY3+jfVsTR9Dd1PfexT/PzFRWm7Xj6RV0Zfvmayo587FBje7d3vRFIirz1Br7GIFTboWFqAD391Ah467yi8f/PxePAHI2JeZ3z/jrj1jGEob2YqISn1hjL9Ugj7KkQg2ujXNwVNE84zl+8IL/ftVGo6dtqby6PON2/jXvSbNgNbsyg7JiSAH03oi0V3nhZeZ62fYLKHPDP6+v8c3skdCrwa5qzahf98HWly3rZI9+JlKf8zP66wPXZI1zbwejQc0b0tLjvGXTjhxpMHJTQ+67wCKX8xUh1UQDjqAdX7gyYDaA3b7FT66LYp8jq+MUj+t0x/w/jo292xd0wjISGiHK0X56f+LeWmlxeblFeB5k/Ut0byzOib9dCZ7KfQq+GwRVpZPrTln/NRPZ2VMhPl1CO6YsMfvud6f+tEq/qbdaheH3coBEdj/e6S7bhbkV9o8AdtJ5/bl/hQ5PNg00NnYlw/ZyHaNkbWUjY1UA+FRHiu4/GLR7fYdd5bGl3Mn29Fbekgr4y+jAawo587FHg1Ux9dIOLhnzZcL7Yr9nnCRnXOLSc0+5oejbDpIXeG/0eWCUnVoWgwcu1jefofr6nGqh0RHZoGf9A2bPWryUNsj9950Ny6MRsbqAtEHnrnjo6lxZh6Yon0MfbkldG3dj5ish+7BudykvaP54/E7F+dgHYlPiy64zR8c9upGNSlTUqu67bXwpmW7BD1MOllhoT5fJ//9mS8e+OkqHMVeDTU+4O2IQm1GfuCzRGdnwkPzTXt5/XI+gNXw08LQmTuby6XUluzhbwy+uHwDrv6OUOh1/wr+MvJg8NG3+fRMLirbuQ7lBagSwaKgAos41ONm/QyhRCmsE+vDsUY0SM6JOX1EJoC0YqdgDmMdPaoHo7jkTUoQZc6RXbc+vrSsOZPKrCL6bckqnIqe/qJk1dGv64x+XZ1TGZoW2xW3pRVuunghauPwcxfHB9zH7s3kVeu1fP6ZU69EDClYRKRreMxokc7NDoa/cj+I2PMYciHjlN2TG1jAB99u8t2G6C/Db++cBsu+8d8x30SJSSE6U3nqkn90KYFta/8QdXos6efKHll9F+t1Pu1fLjS+ZeeyS4WWyR416RRh/24wZ0xvIe9GJvE2r5QIwp7/43h8I4wpXI6UVbkRaM/ZKvVoz4IrL2EVSKevr3R/9kLC/GTf1diy97oQjEg9Z25GvxBNPhDprBXkc/TosZYrXBu5PBOwuSV0R9l9EPtwg3RcwZrFopLoc20UWyRV1C1/KXBEbBPHjjOIgFRWuhFY8A+pq/m8vu89n+W/mAI97y3CgCwfre9Cufn62JLNafa6E96+CMAwPYDkQnn7Qfq0RQM4WC93+mwhFF/Zv6AWuzG4Z1EySuj/8yPK9ClTSF+OXlwpofCJIkvy+djCEqbx2BkIhdEmPvrE00TuEU+859XsU+vPv7H55uiztukPO2cJkU3Knr9X22I3V1LhpdOfewT/PDpr8PrU+2B763V5Sj21TaG172zRE+tnLs6dW/co+/9MLzMnn7zyCuj36VtEb65fXLKMjyY9JNljj4AYOndp+PdGyfh1jOGwuvRwvMOjf4g5q7ehW827YVGwMDyMozsFdHntxpvWX386VpdWK690klM7Z9rNfm3vKYrcsYK+1iRnvGG6lrM3xRpYZGMp7+/tgk3vLQopueuvlkM66b//bW3dEprDmoth2roeSI3cfLK6DNMS9Cu2IeRvdrjBqOaV/X0r/5PJRr8IVvv3BrFKfTqse7+nXW5BdnnFwCO7uNckPXmou8AJFZ92hQI2TYnTybFcfrnGzFj2Q68MG+Lab06HvUN4rbv6b0OygpTZ/RV1KpmnshNnLxU2WRylykjsr/NpZzIXbcrEle388GtWWS6imYQlx3TBw/MWI2zRvZAmyIfKvp2QI/2ESE4p5T3RPRsmoIhzLYJr8gYeCJp9Z8ZbyZWRVO1QEz1vqWqaE1j6mL6Kmajz55+orCnz2SUir66h3tUz3ZYce8ZOCMHehvLnPp/G01fAPs4vLXytsCrt1580miZ6NEI54zqYTL4ABwzgdy2kwT0yc5aJSQiNWukcU5k5mTldj2j6omP1pvW1zfZG1/5UPzJvysTuIp71IcNF2clDht9JqPcedbw8HKu9DX22sXWbVbZhXcAYH+d7gE7SoDbrA6GBPbVNZnWxapH8QejpSGEELjkmXmOx8TDGl7yK59V49vSTYzUeYnGHOo7IBFCYFHV/vg7thBs9JmM0tmQOh7aLXcm331a9J+NnZ2Tnv5Tl4/FynvPiKo+tjkNAHsvfG9No8lzB6Lj2aa0xmAoKnvoQF0k3JIKcUq1IbyaOtnScfb6JnUiN/c8/ee+3oLz/vYVPl6TGaVUNvpMRunZvhj//flE3H9ubD38bMKu2tYuvCMd8dJCD0oLvSi0GGGvg9W30wUKieiYujUTR23AftW/F6Cuybx99+FGpBK1yOzOM1vujW3trsOmz7k+kbuhWp8L2mLTaS0dsNFnMs7Yvh2iiqByDbtIjYBZANAq6eAUBTlrZHdcVNHLJLH8l4/WhZupX3fCAADRBVjW8IvV6Du1lnSLtbeAqv9z/the4WX1rS0VxWCrd5irtFWjn+pis3Qgm+hkqtGMK6NPRFOIaA0RrSeiaTbbTyCiRUQUIKILLNseIaKVRLSaiP5CbuUNGSaHiKUyKTcV+qzVvfbHFPk8eOSCUbiwond43UtKUxLpSTcak7W7D+vVsFYjYs2rb66BtD5U5DPkb5eNcTymtjGAD1bsxMPvf9usa6s0NOW2py8TAfwZKj+Pa/SJyAPgSQBTAQwHcAkRDbfsVgXgSgAvWY6dCGASgJEARgAYB+DEZo+aYXKAh88biR9W9Ma4fh0BRCuKxuOiit545PyRUetLFKN/9hNfYPyDuvxy0GJErH0KapuaZ/StRkqGk+weeDKDJyR0PaCnPt2Q9HW7tNHVVX9xql5pX+eQNZQrpEIptTm4+S0cD2C9EGKjEKIJwCsAzlV3EEJsFkIsA2C9CwGgCEABgEIAPgCshsbkHXaGr3fHEvzxgpFhz84q0+yGchsdqVIjFPbe0u3YqMSF/RYjYjW0NQ3Na7xiTUGVl7PL1rnvnCP1MaVA+F/KLgzpWgbAuT4gV5C/Klnr6QPoCWCr8nmbsS4uQoivAXwMYIfxb5YQYrV1PyK6logqiaiyurrazakZJqtwEkBTSdTTdzpGevqPz11nWu9UsfvyT3Up6L21yU3kSiFDq3GXnr7HxujLycrH56yL2pYocu6iU6n+AHz6s43hbQ05GN558mP9YZyKB2IyuPkttC02dHNyIhoE4AgAvaA/KE4hoqh+d0KI6UKICiFERXl5uZtTM0xG6dupxPT5uwP1cY9JpleAndEvdZj0direKi3U93dqS7m3pjGsB7Rwy35c/+JCk4SDnIuwxs+l529n9KU4nJQ7B5I3cjKE07G0IHpbDk7kSjJVWObG6G8D0Fv53AtAdIdie34AYJ4QokYIUQPgfQATEhsiw2Qff/5h4g3Ak/P0ow18qU1KZCgkHD19GVaqt8T0Zdjkx89+gyue/QZNgRCue34hZi7fiT3KW4EsAmsMhEwFYfIhY2f0n/rRWADANcf1D6+zVvQCwJ6aRjz7xSYcrIsv5lbo1TDA0C2S5OJErqTeH8SmPbVpn5dw81u4AMBgIupPRAUALgbwrsvzVwE4kYi8ROSDPokbFd5hmFyj2JKJ0644vrhYMjF9u2NKbDz9pmDIMQVQpopaUzilwVyzU8+DbwgEw/Hm8Q/ODXvm6sNElTUOxvD05TzGjOU7wus22eSl//S5Stz3v1WoeHC27dgBxej7NNMchn4Puevpv/xNFU7+0yf4xcuL03rduL+FQogAgBsBzIJusF8TQqwkovuI6BwAIKJxRLQNwIUAniailcbhbwDYAGA5gKUAlgoh3muB+2CYtFJkMfpuFDDtjGM87N4O7Lx/pzaMQCQ8U+83h3dG3fshHpyxKhyrbfAHTbFcWcGrNkhRPWt5vVj3teNg5Ni6xuiJ5O1GWCzWpKZ80Ni1rswH7Z1Zae7056p0TggxE8BMy7q7lOUF0MM+1uOCAK5r5hgZJuuwevpuPM5YufxO2Hn6dusaA0F84lDWL43lnNXR25/5fFPYu29oMktEj3twDn47ZSh2HlKMvj+k5+MBmLNKN1ZuH2bRbxpB7DoUf3JZevr29537Rj/d5IbCFcNkGVZdGzfpd8nIE9h5t3bNVLYfaMBDDgVQ0lhWO8gwyDD9ywuqoqqEH521xvRZfbj952tdX99ROM7Ct0b/40MNfny2thqvVW5zdZycgLY+aK3jYdzBMgwMkwTW8I4bytsU4p0bJsXfUcFu0taOWNW2bieQdx9qjKoStj507Dxrt56+VBed+NBHuPGlxWGd/ngcavCjrNALr2UshV4tqTz98Q/Owb++jG5Z2ZK8NL8Ks1buTOs1nWCjzzBJYDWkw1yqhI7q3T7+Tgp2D5cOJTapizHCHNJwty2KfoAMKI9kwxR4tbipp3ZGNpG5ip+/sBA1NrF9ANiy116ALBgSYTnrK47tG15/4pDyhD19IQR2H27EvUaD+XRx21vLcd3zC7G3Jvpta/IRXdI6Fjb6DJMERIS3rp+I1392LABg0qDOro89pn9H9LQ0TkkEO+/fmo4pufWModA0gs9DJqEyyRHd24aXX/6mKmq7VbLfzsgmYvTfX+Hs7TqFn4IhEQ4hqXpEhT5Pwp5+Ii0nU8VuZU5k7ANzora3dZH5lUo4ps8wSSL72r51/USM6NnO9XGvXndsysdizcyR9OmoF5EVej22Hna8eHzQYvXt3ihi9HJJCCd7HBIiLGc9omc7lLcpRPXhRvg8lPBEbrqVLbfsrcWJj34Sc590yzGwp88wzeToPh3CeektwUvXHINfnDIo5j5qY5EbT47se9bI7gDMmS+r7jsjvLxL8ULtsHrGz1uao9vtkyxWbR9JIChMEhA/maQXfJW3KURT0L4BvBPpNvqvu5isTndVMRt9hslyJg7qjFtOHxpzn9veWh5eVvsMy4nZfbWRVotFSp7//E37XI1h0qBOAIAZy3ZEbXMy1l///hRX5453nqAwt378+UkDsfnhM9G+WJ/baEpA3uGjb9PbrcpNn4hExp8K2OgzTI7w5vUT8eI1xwAAphsyB3bY9vBVsOv8FY+Th0YmG5+wCL05xfQT1Rq69Jn5uP9/0ROsoZCwvYacTE+kT4C1+vXzddX4w8yWEwlw0y/Y2gynpWGjzzA5wpg+HcITxqcf2c1xMri5jcl/N2VY1Dq1AYzMz5c4ZS4lIzvxzy+iUymDwv7BIttPNqdA60f//AbTP9sY1X84VVjTTO1Id4EZG32GyVHuPtvay0jHoxEeu3AUXrj6mPC6RArDfn7SwKh1h5QuXNaHilMHsGQE5uwIhkK2Rl+GqZLV1FdF3pZuPZDc4OLAnj7DMCnj9CO7YcHtk6PWezUN54/theMGR9JInXLj3VK1ty683BgI4r8L409QNveNQ6KmbKpEPP3kJkI3K3UBhT4PhBAp17h3k87KRp9hGNfYhVA8cWL6yaCmbh6o9+PXry+NewwR4clLx+D9m4/H0K7uitdsrx2yn4eQnv6BemdZ5lhUH24MK5Z6NMLfPtmAwbe/H9VbuDnYSWZY4YlchmFcY9UAAgBfijxslfI2kbaNieTlnzmyO47o3tZVFovEOjEbEsL2raGHMaex24VoG2BuswgA1zxXGX6DaAqE8Ibx9rLHpmo2WTyavYl96vJIM/l0a+qz0WeYHKbQ60H3dkWmdbFCCnLyN9HS/4vH9UZF3w6JD9Cgq02vX8mMXxxn+vzADHMGz0ff7sby7w5GHdfGkJWwGnMnRtw9K3ql8aNqCoTCaqMiVdVmQJSAncT6MKhrZKPPMIxLurQ1G32vjXcpJ2fPH6sroE//UQXOOzq61fW5o3vYXkMjQj9L16pEePAHR+Giioj6+r1G43QA6N6uGL06RDKRNu+pgxtkaMYq2exErLqspmCkl0AKbb7jNa0Pg3T2y2WjzzA5jtWA2MX0ZSaN9GI1jaLmA44b1BmPXjDK9hpEwK9OG5L0GDuXFeIR5dxXTOwXDk0VejVcpGjqqEVaT34c3WJRIjWIag1PXwiB95ZuR1MghEAwhOe/3oxvdx5Cv2kz8NG39o1K5E+q0R/pJZBM0a4QAqt3HIpa71RwNqJnO1w5sR9OH95Vv34aJ3PZ6DNMjmNtzmKX6SJ1gkb2iqh8Xj4holj5v5uOwwvXHBN+EEwd0c10vEbULJE4Fdng/Pmrj8ENJw9EaaE3bLgBoFbx3K16/iqFXg1EEbG5T9ZW46aXF+PPc9bixflVuPOdlbj55SX6/dlUEgORdNOmYMToJyMr8d6yHZj6+Of4QBGUe2l+lW1jmyV3nYaubYtwzzlH4uxR+ptVIgVmzYWNPsPkOFYjb1eRe+KQcsz7/ak4zfAsAaBvpxLbZQB4/OKjTZ+TaPplyxe/Oxkf/fpEAMC4fh1x6xl6IZhslAIAB+uaoo5TJ5IjYyKUFnhRa8TDZYFV1d66cIqqzMRx0ueR99WoxPT9wRAa/EG8MG+La12fNUaDmPW7D4fX3fbWcsxcrj8E/nj+UeH17RVpbPkGlk5Pn1U2GSbHkcbqqcvHYEwM8bdulglfdT/r24I19JNMq0c7enUosV1/48mD8NJ8Xdp5eI+I3HOxz4N6f9C2gxiga9vIiVx5P42BUDjbRxgdgOMJrekTuRGv/8+z1+LpzzairNCL79vMfViJNw9w8jD7iXNZ6ZzO7B02+gyT40iDXFLgjZrUjYVq9OPZdLm9TZHX5JW/nWAnMCd6tC/G5ofPxPC7PkD3dpEwkuwB0LksunEMAJQWeMITuTLMI4QIyx/IUI2TTZY/uzveXoE2xhzBZc/MD+fOH7B567Aj3vuARoQ5t5yAPTXm80lPf9v+eozta3dk6uHwDsPkODJZJ1FnXE3tjFc5SsaUp9rJ65j+HTE6wU5g8Sj2edDgD4arVI8y+hTcrWT7mPYv8IY9/XvfWwkAWLz1QLgoKhyfd7DKavGUzzDAarHU/rrUFGppRBjUpQ0mDOhkWi8bx9z8ypKUXMfVWNzsRERTiGgNEa0nomk2208gokVEFCCiCyzb+hDRh0S0mohWEVG/1AydYRgAzco6kcRTxJTPBLXwq10LdHwq9Gp4cX4VhtzxPhoDQQzt1gY92xdjTB/7GoECr4bKLfvhD4bCBjoQDIXTVuU64WD11UxJVX5akmhfWycdohaol0uauEafiDwAngQwFcBwAJcQkVXpqQrAlQBesjnFcwAeFUIcAWA8gPQKWjNMnhM2+i3YIEQas/5KT914Es7JoKp5rttVg2BIwKGoFQCwbV8dDtT5cc+7K8PrzhndI6qC9/N1e2yPP9QQ25Mf24yCNBWnh8HpR+oT6z+akKbYDtx5+uMBrBdCbBRCNAF4BcC56g5CiM1CiGUATFPQxsPBK4SYbexXI4RwV3nBMIwrpH1zyglP5TWevDQiH+AkMdAcShS5hrveWaE3RY9xHZml8+L8SH9fjQhFFtkHdR7i91Mj0tHxxM5enF+Vkoepk6df6PXA5yGU2TStbyncfGs9AWxVPm8z1rlhCIADRPQmES0mokeNNweGYVJEKsI7djxxSSRtU8b025cUhMXTUqWiqbJKKXDasrdOV9iMcR07+eaV2w/F1B+6alJ/nGm0kXTDh6vsQzx7axqx42A9gOjsHWuuf6x78AcFFlftdz2e5uLG6NuN1u2vlxfA8QB+A2AcgAHQw0DmCxBdS0SVRFRZXV3t8tQMwwCR2Hqqwy2ycAgASLEU0ot2SqNsDqrx3FvbhEAoFLN5uxoOkizcst/k+ascN6gzCrwaTjuiq+12O2psdHECwRDGPjAHxz70ke0xhy1ho3gpr/M27os6pqVw806xDUBv5XMvANtdnn8bgMVCiI0AQERvA5gA4J/qTkKI6QCmA0BFRUV6OxczTI5z9zlHYmCXMpw4uDzhYx+/eLSrStuygoipkA1HkumMlSjBUGwv2cmjtxNoA4Av1uux/SKbh4UTdgJsz34Z3eFL5by/f2X67Cazan+tH22KUj85bsWN0V8AYDAR9QfwHYCLAVzq8vwLAHQgonIhRDWAUwBUJjVShmFsaVfsww0nD0rq2HNHx47U3nrGUPTpWGKrZ+9UBJZKgqFQzDeY7QcbbNcH4giYqVLPpwzrErNhup0XKlMtI/voe/mDIQRDAhura03byTZgYqYhTQVacb81IUQAwI0AZgFYDeA1IcRKIrqPiM4BACIaR0TbAFwI4GkiWmkcG4Qe2plLRMuhh4qeaeW+B4AAAAzGSURBVJlbYRgm1dxw8iBTmEclHUbqi/V7kqoG9seZ4FAnjGXfYSds49vWGH4wYvTrbXR03Mx/JNv2MVFcTRkLIWYCmGlZd5eyvAB62Mfu2NkARjZjjAzDZCEvza/CH35wVPwdm4E/aN9ARTJhQEfM27gvan1JgSdmZk6xEt6JV2+gzhtUH25EgUfDXiWnX22z6A8K24eE3ZuSlazx9BmGYVSk52/XQL25TBrUKWpdrJj+iB7tTJ+HddMzizqXOTdtAcwx/R/E0dZRUzbHPTgHYx+YjRE9I9dtDITCVbzTP9toai0JALd/74iY55ekS2mTjT7DMAkh6wFaoiL3sQtHR62LFdP/7ZRh+N9Nkc5bJwzRJ7Pr4zRWkeGdkgIPPBrFfLBY3xgCIRFurSiv1RSIGHr1IfGnC0fhpycMiDkW9TzpgI0+wzAJ0dtQykxGdz4eViVQAPhy/V7H/Qu8Go5UVDm9GsHnIdu4OgBcNakfgEh4R5r6WPdip4CpNkyp8wdNna/Ucy3bdsDxvJL//GQ8ADiOOdWw0WcYJiFk0k5Lyj4kAhGFi7Q8GqHQ63Hsm3vXWbqCjMzeOWlo/F7B8bTuaxsDJqOv/lg6lcYOMwGRkFRtmvrkstFnGCYhpo7Qq1lPTaDAqaWRRl8jQpFPQ4OSCaPWE0gNnCKfBx//5iQ8dpF9e0iVpjjpnzWNAazdFWmeosphOAm9qUR6/bpr8N5c2OgzDJMQI3q207XvlbBKpikwVEI1oijFUKmTb6V/59KYRVpvXj8RAPD5Wr2gy65ICwC27qvDBiUv/6Knvw4vH9E9/s+oxCh8Y0+fYZhWyYp7z8ALVx+T0DFSF9+jRevxxBNVU+lUGmnWIuWcv96ozyl8u/Ow7THWQq0te+vCYzrjyG52h5jwaIRin8fUJ7glYaPPMExWUVboNVXMdiiJnyUkq4M1jVBSaPbeG+OEZ1S6t3fuPOakYuqUamntMxyL0kJPuMdvS8NGn2GYrKNcybN3kyUkPX2NKCpHvykQwr+uGocZvzjO7lAAwL+uGgcAOPMovQZhZK92UfvYKXoCQK1DquWgLmVxxy3ZU9OEF+dXpSWuzz1yGYbJOvp0KsGb10/EeX/7ClNGxA+RSE/fQ2SqtpWcHCdL5+ShXbDoztPQvthnKjob168D6v1B1DQGHPsHOHnoboTsrHy787Bjl7BUwUafYZisZEyfDqi8Y7KrIjCZoaNphPdXmPXvf3HqYFfX61ga3Xy9fUkBFqzahRF3z8KFY81KMzedMghPfLTecQK21GECORbpqMrl8A7DMFlL57JCV2qe4Zi+pbB2/YNTcctpQ5K+vhrSeV2pwgUiQm0yJHP/90ckfR0JG32GYRgXSFE2q5xCLHkFN8RK6ZTnljH9gZ1LE+rIZUc6lDbZ6DMMk/MUKMVZ0380NrzeqSG5W5wmbwFdXtmrUThl06MRznWQoXZLOoqc2egzDJPzRMI7ZFLAbC6xPH1/MIT2JQVhHR6vh8IPn+MHx9bot3L32bo8RCDEnj7DMExc1OKssqLU5afsPGTfmQvQ5Rk6lponmWXfYH8CtQEAMNmQtEikkCxZOHuHYZicZ7cRYtl9qBGlBakza3ss1bYA8Lspw1C1rw7HDeqMDiWRjJ9Gfwhew+gHgonFaeQbwhqHqt9Uwp4+wzA5z+IqXcL4v4u2NXvyVsWuUflZI7vjofOOgs+joXObSCFYYyCErm31z6N6t0/oOvIN4R9fxG64ngrY02cYJm9IdXikbXG0iVSLrrq3jcg2EAF9O5Xig18ej4Hl7qtxAcAXY8I41bDRZxgmb4infZ8odqEitd9tB6Wg64TBeteuYd0SVx+NlSWUaji8wzBMznN0Hz2c8uvTh6b0vNbCsDaWSeJSRRjOTfNzt9dpSdjTZxgm5+nTsQSLqw5gYHlpSs/r85oN+S8nm6t7k5FacKLyjslpMf6urkBEU4hoDRGtJ6JpNttPIKJFRBQgogtstrclou+I6K+pGDTDMIzK/d8fgbvPHo7x/Tum9Lw+i8hae4sOUCqNfueywhZpNm8lrtEnIg+AJwFMBTAcwCVENNyyWxWAKwG85HCa+wF8mvwwGYZhnGlb5MNVk/qHK3Ar+qZGqVL1vB85fyR+cHRP0/aCNIZlUoWbEY8HsF4IsVEI0QTgFQDnqjsIITYLIZYBiJpFIaKxALoC+DAF42UYhonLq9cdi/UPTm32eX58bN/w8kXjekfF7b2e1KWHpgs3Rr8ngK3K523GurgQkQbgMQC3Jj40hmGY5PBoFC6Uag4dbOSWVaSnP2FAasNKLYmbn4rdo8xtudn1AGYKIbbG2omIriWiSiKqrK6udnlqhmGYzNKvsz5x7KYXbrbgZhZiG4DeyudeALa7PP+xAI4nousBlAEoIKIaIYRpMlgIMR3AdACoqKhIg84cwzBM8+nRvhgr7j3DlLqZ7bgx+gsADCai/gC+A3AxgEvdnFwIcZlcJqIrAVRYDT7DMEw289I1x2DHQWfhtbIUZvCkg7ijFUIEiOhGALMAeAA8K4RYSUT3AagUQrxLROMAvAWgA4CzieheIcSRLTpyhmGYNDBxUGIyydkOCZFd0ZSKigpRWVmZ6WEwDMPkFES0UAhREW+/3EsyZRiGYZKGjT7DMEwrgo0+wzBMK4KNPsMwTCuCjT7DMEwrgo0+wzBMK4KNPsMwTCsi6/L0iagawJYkD+8MYE8Kh5MJcv0ecn38AN9DtsD3kBh9hRDl8XbKOqPfHIio0k1xQjaT6/eQ6+MH+B6yBb6HloHDOwzDMK0INvoMwzCtiHwz+tMzPYAUkOv3kOvjB/gesgW+hxYgr2L6DMMwTGzyzdNnGIZhYpAXRp+IphDRGiJaT0RZ3aSFiDYT0XIiWkJElca6jkQ0m4jWGf93MNYTEf3FuK9lRDQmQ2N+loh2E9EKZV3CYyaiK4z91xHRFVlwD/cQ0XfGd7GEiL6nbPu9cQ9riOgMZX3GfteIqDcRfUxEq4loJRHdbKzPie8ixvhz5nsgoiIi+oaIlhr3cK+xvj8RzTd+nq8SUYGxvtD4vN7Y3i/evbU4Qoic/ge9scsGAAMAFABYCmB4pscVY7ybAXS2rHsEwDRjeRqAPxrL3wPwPvQ+xRMAzM/QmE8AMAbAimTHDKAjgI3G/x2M5Q4Zvod7APzGZt/hxu9RIYD+xu+XJ9O/awC6AxhjLLcBsNYYa058FzHGnzPfg/GzLDOWfQDmGz/b1wBcbKx/CsDPjeXrATxlLF8M4NVY95aOe8gHT388gPVCiI1CiCYArwA4N8NjSpRzAfzHWP4PgO8r658TOvMAtCei7ukenBDiMwD7LKsTHfMZAGYLIfYJIfYDmA1gSsuPXsfhHpw4F8ArQohGIcQmAOuh/55l9HdNCLFDCLHIWD4MYDWAnsiR7yLG+J3Iuu/B+FnWGB99xj8B4BQAbxjrrd+B/G7eAHAqERGc763FyQej3xPAVuXzNsT+Rco0AsCHRLSQiK411nUVQuwA9D8MAF2M9dl8b4mOOVvv5UYj9PGsDIsgB+7BCBMcDd3TzLnvwjJ+IIe+ByLyENESALuhPzA3ADgghAjYjCc8VmP7QQCdkMF7yAejTzbrsjklaZIQYgyAqQBuIKITYuyba/cGOI85G+/l7wAGAhgNYAeAx4z1WX0PRFQG4L8AfimEOBRrV5t1Gb8Pm/Hn1PcghAgKIUYD6AXdOz8ixniy7h7ywehvA9Bb+dwLwPYMjSUuQojtxv+7oTeTHw9glwzbGP/vNnbP5ntLdMxZdy9CiF3GH3AIwDOIvF5n7T0QkQ+6wXxRCPGmsTpnvgu78efi9wAAQogDAD6BHtNvT0Rem/GEx2psbwc9zJixe8gHo78AwGBj9rwA+mTJuxkeky1EVEpEbeQygNMBrIA+XplBcQWAd4zldwH82MjCmADgoHyNzwISHfMsAKcTUQfj9f10Y13GsMyP/AD6dwHo93CxkXnRH8BgAN8gw79rRiz4nwBWCyH+T9mUE9+F0/hz6XsgonIiam8sFwOYDH1u4mMAFxi7Wb8D+d1cAOAjoc/kOt1by5OO2eKW/gc9S2Et9Nja7ZkeT4xxDoA+Y78UwEo5VugxvrkA1hn/dxSRTIEnjftaDqAiQ+N+Gfprtx+6h3J1MmMG8BPoE1brAVyVBffwvDHGZdD/CLsr+99u3MMaAFOz4XcNwHHQQwDLACwx/n0vV76LGOPPme8BwEgAi42xrgBwl7F+AHSjvR7A6wAKjfVFxuf1xvYB8e6tpf9xRS7DMEwrIh/COwzDMIxL2OgzDMO0ItjoMwzDtCLY6DMMw7Qi2OgzDMO0ItjoMwzDtCLY6DMMw7Qi2OgzDMO0Iv4/+bbgkDKHHfoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learner.sched.plot_loss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 266,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEKCAYAAAA4t9PUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl8FPX9x/HXJwf3DUGRQ24UlEMjcoXWaisgigda8EKlnpy1ttVfL2tPa7UC4oHiWSsgXoCA2mIl3AQEuSEJp6JsuCSJHAnf3x87oTHmWJJsZjd5Px+PfWQzOzP5DBv2k5n5znvMOYeIiEhpxfhdgIiIRDc1EhERKRM1EhERKRM1EhERKRM1EhERKRM1EhERKRM1EhERKRM1EhERKRM1EhERKZM4vwuoCE2aNHGtW7f2uwwRkaixatWqDOdcQijzVolG0rp1a1JSUvwuQ0QkapjZzlDn1aEtEREpEzUSEREpEzUSEREpEzUSEREpEzUSEREpk7A2EjMbYGZbzCzVzB4s5PXqZjbde325mbX2pjc2s4/NLNPMniqwzIVmts5bZqKZWTi3QUREihe2RmJmscBkYCDQGRhuZp0LzDYSOOicaw/8A3jUm34U+A3wQCGrfga4C+jgPQaUf/UiIhKqcO6R9ARSnXPpzrnjwDRgSIF5hgCveM9nApeamTnnspxziwg2lFPMrBlQzzm31AXvEfwqcHUYt8EXuw9kM+ezLzh5UrdBFpHIF85G0hzYne/7Pd60QudxzuUAh4HGJaxzTwnrBMDM7jKzFDNLCQQCp1m6v/72wRZG/+tTRry0gn1Hjpa8gIiIj8LZSAo7d1HwT+xQ5inV/M65Kc65ROdcYkJCSFf5R4zPD2YDsHLHAQY+mcyCzV/5XJGISNHC2Uj2AC3zfd8C+KKoecwsDqgPHChhnS1KWGfUS8/I4saLWzF7dD8S6lbnjpdTeHjWBo6eyPW7NBGR7whnI1kJdDCzNmZWDRgGzCowzyxghPd8KLDAO/dRKOfcXuCImfXyRmvdCrxX/qX750DWcQ5ln6BdQh06nFGXd0f15fa+rXl5yQ6unryYbV8d8btEEZFvCVsj8c55jAY+ADYBM5xzG8zsETO7ypttKtDYzFKB+4FTQ4TNbAfwBHCbme3JN+LrXuAFIBVIA+aFaxv8kBbIBKBtQm0AasTH8rsru/DSbRcROHKMK59axOvLd1JMvxURqVBhTf91zs0F5haY9tt8z48C1xexbOsipqcA55VflZEl3Wsk7ZrU+db0S85pyrzxSfxsxlp+9c56Fm4N8Ndru9KwdjU/yhQROUVXtkeYtEAW1eJiaN6w5ndea1q3Bq/c3pNfX3EuCzbvY+CEZJam7fehShGR/1EjiTDpgUzaNK5NbEzhF+zHxBg/SWrLO/f1pVa1WG58YRmPfbCZE7knK7hSEZEgNZIIkx7Iol3T2iXOd17z+swe048bLmzJ5I/TuP7Zpezan10BFYqIfJsaSQQ5nnOSnQeyaVvg/EhRaleP49GhXXnqxh6kBTIZNDGZdz/9PMxVioh8mxpJBNl1IJvck+7UiK1QDe56FvPGJXHOmXUZP30N909fw5GjJ8JUpYjIt6mRRJBTI7YSQtsjya9Fw1pMu6sX4y/rwLtrPueKiYtYs/tQeZcoIvIdaiQRJC2QBXDaeyR54mJjGH9ZR2bc3Zvck46hzyzh6f+mKvxRRMJKjSSCpAcySahbnbo14su0nsTWjZg7LonLzzuTv83fws1Tl/PlYYU/ikh4qJFEkPSMLNqVcm+koPo143lqeA/+dl1XPt11iIETFvLhhi/LZd0iIvmpkUSQtEAmbUtxfqQoZsYNF7Vkzth+NG9Yk7teW8Vv3l2v8EcRKVdqJBEiL6yxbZPy2SPJr11CHd66tw93JrXhtWU7ueqpRWz+8uty/zkiUjWpkUSIUyO2mpbfHkl+1eNi+dUVnXn1jp4cyDrBVU8t5pUlOxT+KCJlpkYSIdKKCGssb/07JjB/fBJ92zXmd7M2cOerKRzIOh7WnykilZsaSYRILyassbw1qVOdF2+7iN9d2ZmFWzMY8ORCFm3LCPvPFZHKSY0kQqSVENZY3syM2/u24d1RfalXM55bXlzOX+Zt4niOwh9F5PSokUSI9EBWqS9ELIvOZ9Vj9uh+DO/Ziuc+SWfos0vYnpFV4XWISPRSI4kAJ3JPsutAti+NBKBmtVj+fM35PHvzBezcn80VE5OZuWqPTsSLSEjUSCLAzv3Z5Jx0pcrYKk8DzmvG/PFJnN+8Pg+8uZax09bwtcIfRaQEaiQRIP3Ufdr9bSQAzerX5F939uLnl3di7rq9DJqQzKqdB/0uS0QimBpJBEjPKFtYY3mLjTFGXdKeN+/pjRnc8NxSJv5nG7kKfxSRQqiRRIC0fcGwxnplDGssbxe0asj7Y5MY3LUZT3y0leFTlvH5oW/8LktEIowaSQRIz8gKSzRKeahXI54Jw3rwxA3d2PDFYQY+uZB56/b6XZaIRBA1kgiQHsgMWzRKebn2ghbMHZdEm4Q63Pv6ah56+zOyj+f4XZaIRAA1Ep8dyDrOwTCFNZa3sxvXZuY9vbn3++2YtnI3V05axIYvDvtdloj4TI3EZ2W5va4f4mNj+OWAc3h95MVkHsvhmslLmLpou645EanC1Eh8lu7dXjdaGkmePu2bMG9cf/p3TOAPczZy+8srCRw55ndZIuIDNRKfpQUyKyyssbw1ql2N52+9kD8M6cLStP0MnJDMJ1sDfpclIhVMjcRnaYEsWjeuVWFhjeXNzLild2tmje5H49rVGPHiCv44ZyPHcnQXRpGqQo3EZ+mBzKg7rFWYTmfW5b3Rfbm199m8sGg71z695NQ9VkSkclMj8ZHfYY3lrUZ8LI8MOY8Xbk3ki0PfMHjiIqat2KUT8SKVnBqJj3YdCIY1tg3zXREr2mWdz2D++P5ccHYDHnx7HaP+tZrD2Qp/FKmswtpIzGyAmW0xs1Qze7CQ16ub2XTv9eVm1jrfaw9507eY2eX5pv/UzDaY2Xoze8PMaoRzG8IpbV9479PupzPq1eC1Oy7mwYHn8OGGrxg4YSErth/wuywRCYOwNRIziwUmAwOBzsBwM+tcYLaRwEHnXHvgH8Cj3rKdgWFAF2AA8LSZxZpZc2AskOicOw+I9eaLSpEW1ljeYmKMe77Xjrfu7UO1uBiGTVnKEx9tJSdXd2EUqUzCuUfSE0h1zqU7544D04AhBeYZArziPZ8JXGpm5k2f5pw75pzbDqR66wOIA2qaWRxQC/gijNsQVumBTJrUibywxvLWrWUD5oxN4poeLZj4n238eMoydh/I9rssESkn4WwkzYHd+b7f400rdB7nXA5wGGhc1LLOuc+BvwO7gL3AYefch2GpvgKkBbJoV0n3RgqqUz2Ox2/oxoRh3dn65REGTUxm9tqo/RtARPIJZyMp7MKIgsN3ipqn0Olm1pDg3kob4CygtpndXOgPN7vLzFLMLCUQiMyL5NIDmRFxM6uKNKR7c+aOS6JD0zqMeeNTHnhzLVnHFP4oEs3C2Uj2AC3zfd+C7x6GOjWPd6iqPnCgmGUvA7Y75wLOuRPA20Cfwn64c26Kcy7ROZeYkJBQDptTvvLCGqvKHkl+LRvVYsbdvRn7g/a8vXoPgyct4rM9h/wuS0RKKZyNZCXQwczamFk1gifFZxWYZxYwwns+FFjgghcdzAKGeaO62gAdgBUED2n1MrNa3rmUS4FNYdyGsIm2sMbyFhcbw/0/6sQbd/bi6IlcrntmCc99ksZJ3YVRJOqErZF45zxGAx8Q/LCf4ZzbYGaPmNlV3mxTgcZmlgrcDzzoLbsBmAFsBOYDo5xzuc655QRPyq8G1nn1TwnXNoRTXlhjZR2xFaqL2zZm3rgkLj3nDP4ybzO3vriCfV8f9bssETkNVhWuOk5MTHQpKSl+l/Etf5m3iZcW7WDTHwZEbc5WeXLOMW3lbn4/ewO1qsXx2NCuXHruGX6XJVJlmdkq51xiKPPqynafpO3LonWT6A1rLG9mxvCerZgzJokz69Vg5CspPDxrA0dPKPxRJNKpkfgkPSOz0kWjlIf2Tevwzqg+3NG3DS8v2cHVkxez9asjfpclIsVQI/HBidyT7NqfTbumVfv8SFGqx8Xy2ys789LtFxE4cowrJy3in8t2KvxRJEKpkfigsoY1lrdLOjVl3vgkerZpxK/fXc/dr63iYNZxv8sSkQLUSHygEVuha1q3Bq/c3pNfX3EuH2/Zx8AJySxJy/C7LBHJR43EB3k3fKpqV7WXVkyM8ZOktrxzX19qVY/lpheW89gHmzmh8EeRiKBG4oO8sMb6NSt3WGN5O695feaM6ccNF7Zk8sdpXP/sUnbtV/ijiN/USHyQHsjSYa1SqlUtjkeHdmXyjReQHshk0MRk3vl0j99liVRpaiQ+SKsk92n30xVdmzFvfH/ObVaXn05fy0+nr+HIUd2FUcQPaiQV7GAVDmssb80b1OSNO3vx08s68t6az7li4iI+3XXQ77JEqhw1kgqWnpF3ol2NpDzExcYw7rIOzLi7N7knHdc/u5TJH6eSq/BHkQqjRlLB0vYFh/7q0Fb5SmzdiLnjkhhw3pk89sEWbn5hOV8eVvijSEVQI6lgaRmZVIuNoUXDWn6XUunUrxnPpOE9eGxoV9buOcSACQv5YMOXfpclUumpkVSw9EAWZzdWWGO4mBnXJ7Zkzph+tGxYi7tfW8Wv3lnHN8cV/igSLmokFUwjtipG24Q6vHVvH+7u35bXl+/iqqcWsWnv136XJVIpqZFUoLywRp1orxjV4mJ4aNC5vDayJ4e+OcGQyYt5efF2hT+KlDM1kgqUF9aoPZKKldQhgfnjkujXvgkPz97IyFdS2J95zO+yRCoNNZIKpLBG/zSuU52pIxJ5+MrOLErNYMCEZJK3BfwuS6RSUCOpQOkKa/SVmXFb3za8N6ovDWrGc8vUFfx57iaO5yj8UaQs1EgqUJrCGiPCuc3qMWt0P266uBVTFqZz3TNLTjV5ETl9aiQVSGGNkaNmtVj+dM35PHfLhew+mM3gSYuYkbJbJ+JFSkGNpAKlZ2QpYyvCXN7lTOaNS6Jri/r8YuZnjHnjUw5/o/BHkdMRciMxM30ClsHBrOMcyDquEVsRqFn9mrz+k178/PJOzFv/JYMmJLNq5wG/yxKJGiU2EjPrY2YbgU3e993M7OmwV1bJKKwxssXGGKMuac/Me3oTEwM3PLeMCf/epvBHkRCEskfyD+ByYD+Ac24t0D+cRVVGaXlDf5tojySS9WjVkLljk7iyazP+8e+tDJ+yjM8PfeN3WSIRLaRDW8653QUmKbjoNKUF8sIaa/pdipSgbo14nhzWg3/8uBsbvjjMwCcXMnfdXr/LEolYoTSS3WbWB3BmVs3MHsA7zCWhywtrjIvV+IZocU2PFswdl0SbhDrc9/pqHnzrM7KP5/hdlkjECeVT7R5gFNAc2AN0B+4LZ1GVUXogU+dHotDZjWsz857e3Pf9dkxP2c3gSYtY//lhv8sSiSihNJJOzrmbnHNnOOeaOuduBs4Nd2GVyYnck+zcn60RW1EqPjaGXww4h9dHXkzWsRyufXoJLySnc1In4kWA0BrJpBCnSRF2e2GNikaJbn3aN2H+uP58r1MCf3x/E7e/vJLAEYU/isQV9YKZ9Qb6AAlmdn++l+oBseEurDJJU1hjpdGwdjWm3HIhry/fxR/mbGTghIX8/fpufL9TU79LE/FNcXsk1YA6BJtN3XyPr4Gh4S+t8sjLcWqnob+Vgplxc6+zmT2mH03qVOe2l1byyOyNHMvRYEapmopsJM65T5xzvwd6Oed+n+/xhHNuWygrN7MBZrbFzFLN7MFCXq9uZtO915ebWet8rz3kTd9iZpfnm97AzGaa2WYz2+TtOUW09EAWTepUo34thTVWJh3PqMu7o/pyW5/WvLh4O1dPXkLqviN+lyVS4UI5R5JtZo+Z2VwzW5D3KGkhM4sFJgMDgc7AcDPrXGC2kcBB51x7ghc+Puot2xkYBnQBBgBPe+sDmADMd86dA3QjCoYipwUydX6kkqoRH8vDV3Vh6ohEvvr6KIMnLeKNFbsU/ihVSiiN5HVgM9AG+D2wA1gZwnI9gVTnXLpz7jgwDRhSYJ4hwCve85nApWZm3vRpzrljzrntQCrQ08zqEbyqfiqAc+64c+5QCLX4SmGNld+l557B/HFJJJ7diIfeXsd9r6/mUPZxv8sSqRChNJLGzrmpwAnvcNcdQK8QlmsO5L8ifo83rdB5nHM5wGGgcTHLtgUCwEtm9qmZvRDpYZJ5YY2KRqn8mtarwat39OShgefw0cavGDghmeXp+/0uSyTsQmkkeZnae83sCjPrAbQIYTkrZFrB/f2i5ilqehxwAfCMc64HkAV859wLgJndZWYpZpYSCPh3S9W8sMZ2TSO630k5iYkx7v5eO96+rw814mMZ/vwyHv9wCzm5ugujVF6hNJI/mll94GfAA8ALwE9DWG4P0DLf9y2AL4qax8zigPrAgWKW3QPscc4t96bPJNhYvsM5N8U5l+icS0xISAih3PBQWGPV1LVFA+aM6ce1F7Rg0oJUbnhuKbsPZPtdlkhYFNtIvBPcHZxzh51z651zlzjnLnTOzQph3SuBDmbWxsyqETx5XnC5WcAI7/lQYIELnqWcBQzzRnW1AToAK5xzXxLM/urkLXMpsDGUDfVLeiCL+FhTWGMVVLt6HH+/vhsTh/dg21eZDJqQzHtrPve7LJFyV2wjcc7lAleVZsXeOY/RwAcER1bNcM5tMLNHzCxvnVOBxmaWCtyPd5jKObcBmEGwScwHRnm1AIwBXjezzwjmfv25NPVVlLRAJq0b11ZYYxV2VbezmDsuiQ5n1GHctDX8bMZaMo8p/FEqDytpmKKZ/YngIafpBM9JAOCcWx3e0spPYmKiS0lJ8eVnX/r4f2nftA7P3ZLoy8+XyJGTe5KJ/9nGUx+n0qpRLSYO70HXFg38LkukUGa2yjkX0gdXKH8m9yF4PccjwOPe4++lL6/qOJF7kl0HsnUNiQAQFxvD/T/qxBt39uJ4zkmufXoJz36SpvBHiXpFZm3lcc5dUhGFVEa7D2RzItcp9Ve+5eK2jZk3rj8Pvv0Zf523mUXbMnjihm40rVfD79JESkUH7sMoXWGNUoT6teJ5+qYL+Ou157Nq50EGTEjm3xu/8rsskVJRIwmjNIU1SjHMjGE9WzF7TD/OrFeDn7yawu/eW8/REwp/lOiiRhJGCmuUULRvWod3RvVhZL82vLJ0J0OeWszWrxT+KNGjxHMkZnZtIZMPA+ucc/vKv6TKIz0jUxciSkiqx8Xym8GdSerQhAfeXMuVkxbx68GdufniVgTj50QiVyh7JCMJXs1+k/d4nuA1H4vN7JYw1hb10gJZOj8ip+X7nZoyb1x/erVtzG/eXc9dr63iQJbCHyWyhdJITgLnOueuc85dRzAS/hhwMfDLcBYXzQ5lB8MaNWJLTldC3eq8dNtF/GZwZz7ZEmDghIUsSc3wuyyRIoXSSFo75/IPJ9kHdHTOHeB/gY5SgG6vK2URE2OM7NeGt+/rQ+3qcdw0dTmPzt/MCYU/SgQKpZEkm9kcMxthZiOA94CFXnx7xN8LxC+nRmxpj0TK4Lzm9Zkzph/DLmrJM/9NY+gzS9i5P6vkBUUqUCiNZBTwMsFcqx7AqwSzr7J0sWLRFNYo5aVWtTj+cm1Xnr7pArZnZDFoQjJvr97jd1kip4RyZbsjGNc+M/zlVB7pgUzOVlijlKNB5zeje8sGjJ++hvtnrGXh1gB/uPo86tbQ8HLxV4mfcmZ2rZltM7PDZva1mR0xs68rorholhbI1O11pdyd1aAmb9zZi/t/2JHZn+1l0MRkVu866HdZUsWF8ufy34CrnHP1nXP1nHN1nXP1wl1YNMtRWKOEUWyMMfbSDsy4uxfOwfXPLuWpBdvIVfij+CSURvKVc25T2CupRHYf/IYTuY62TbRHIuFz4dmNmDsuiUHnN+PvH27lpheWsffwN36XJVVQKI0kxcymm9lw7zDXtUVc7S6etH1592nXHomEV70a8Uwc1p3Hhnblsz2HGfBkMvPXf+l3WVLFhNJI6gHZwI+AK73H4HAWFe3SMxTWKBXHzLg+sSXvj02iVaNa3PPPVfzfO+v45rjCH6VihDJq6/aKKKQySduXRePaCmuUitWmSW3eurcPj3+4hecWprNi+wEmDutB57N0SlPCq8hGYma/cM79zcwmAd85i+ecGxvWyqJYekamLkQUX1SLi+GhQeeS1CGBn85Yw9WTF/PQoHO4rU9rhT9K2BR3aCvvBHsKsKqQhxQhXWGN4rN+HZowf1wSSR2a8PvZGxn5Sgr7M4/5XZZUUkXukTjnZntfX6m4cqLfoezj7M86rkYivmtcpzovjEjk1aU7+dPcTQyYkMzj13ejf8cEv0uTSiaUCxI7mtkUM/vQzBbkPSqiuGiUF9aoQ1sSCcyMEX1a896ovjSoGc+tL67gz3M3cTxH4Y9Sfko82Q68CTxL8J4kGgZSgnQvrFEXI0okObdZPWaP6ccf39/IlIXpLEnLYOKwHvo9lXIRyvDfHOfcM865Fc65VXmPsFcWpdK8sMaWCmuUCFMjPpY/Xn0+U265kD0Hv2HwpEXMSNlNME5PpPRCaSSzzew+M2tmZo3yHmGvLEoprFEi3Y+6nMn8cf3p1qIBv5j5GaPf+JTD3+jWQlJ6oRzaGuF9/Xm+aQ5oW/7lRL/0jCxFo0jEO7N+Df75k4t5bmEaT3y4lTW7DjFhWHcSW+tvRDl9xf7ZbGYxwM3OuTYFHmoihcjJPcnO/Vk67ixRITbGuO/77Zl5bx9iY4wbnlvKk//eSo7uwiinqdhG4pw7Cfy9gmqJenlhjYqPl2jSvWUD3h/bj6u7N+fJf29j+PPL2HMw2++yJIqEciD/QzO7znRZbIk0YkuiVd0a8Tzx4+48+ePubNp7hIETknn/s71+lyVRIpRGcj/BIcDHdGOr4v3vPu3aI5HodHWP5swdm0S7hDqM+tdqfjFzLdnHc/wuSyJciY3Eu5FVjHOumm5sVbz0QDCssUGtan6XIlJqrRrX4s17ejP6kva8uWoPgycuYv3nh/0uSyJYSGNUzayhmfU0s/55j3AXFo3SApmKRpFKIT42hgcu78TrP7mY7OO5XPP0Yp5fmM5J3YVRChFKRMpPgIXAB8Dvva8Ph7JyMxtgZlvMLNXMHizk9ereTbNSzWy5mbXO99pD3vQtZnZ5geVizexTM5sTSh0VJT2QpWgUqVT6tGvCvHFJXNKpKX+au4kRL61g35GjfpclESaUPZJxwEXATufcJUAPIFDSQmYWC0wGBgKdgeFm1rnAbCOBg8659sA/gEe9ZTsDw4AuwADgaW99+WuKqNv/KqxRKquGtavx3C0X8serz2PF9gMMfDKZjzfv87ssiSChNJKjzrmjENyDcM5tBjqFsFxPINU5l+6cOw5MA4YUmGcIkJcuPBO41BsdNgSY5pw75pzbDqR668PMWgBXEMz+ihh5YY1tdVdEqYTMjJt7nc2cMf1IqFud219eye9nb+DoCcXvSWiNZI+ZNQDeBT4ys/eAL0JYrjmwO/96vGmFzuOcywEOA41LWPZJ4BdARF01lTf0V/dpl8qswxl1eXdUX27r05qXFu/gmqeXkLrviN9lic9CGbV1jXPukHPuYeA3wFTg6hDWXdh1JwXP1BU1T6HTzWwwsC+U0Egzu8vMUswsJRAo8UhcmaVnKKxRqoYa8bE8fFUXXrwtka++PsrgSYv41/JdCn+swkIdtdXPzG53zn0CLOW7exaF2QO0zPd9C767J3NqHjOLA+oDB4pZti9wlZntIHio7Adm9s/CfrhzbopzLtE5l5iQEP4b+aTty6RVo1oKa5Qq4wfnnMH8cUlc1LoR//fOOu7952oOZR/3uyzxQSijtn4H/BJ4yJsUDxT64V3ASqCDmbUxs2oET57PKjDPLP4XCjkUWOCCf9bMAoZ5o7raAB2AFc65h5xzLZxzrb31LXDO3RxCLWGXnqERW1L1NK1Xg1du78n/DTqH/2z+ioETklmWvt/vsqSChfLn8zXAVUAWgHPuC6BuSQt55zxGExwuvAmY4ZzbYGaPmNlV3mxTgcZmlkrwCvoHvWU3ADOAjcB8YJRzLmLP6imsUaqymBjjrv7tePvevtSIj2X488t4/MMtnFD4Y5URSoz8ceecMzMHYGYhj291zs0F5haY9tt8z48C1xex7J+APxWz7v8C/w21lnDKC2vU0F+pys5vUZ85Y/rx8KwNTFqQyqLU4F0YWzaq5XdpEmah7JHMMLPngAZmdifwb+D58JYVXU6N2NIeiVRxtavH8dj13Zg0vAep+zIZNCGZ99Z87ndZEmahjNr6O8FrPN4ieP3Ib51zk8JdWDRJ964hUVijSNCV3c5i7tgkOp5Zl3HT1nD/jDVkHlP4Y2UVyqEtnHMfAR+FuZaolRbIpJHCGkW+pWWjWky/qxeTFqQyacE2Vu08yMRhPejWsoHfpUk5K3KPJC8uvpCHYuQLCGZsaW9EpKC42Bh++sOOTLurNydyTnLdM0t45r9pCn+sZIpsJHlx8YU8FCNfQFogU9EoIsXo2aYR88b15/IuZ/Lo/M3cPHU5X32t8MfKQlfPldHh7BPszzpOu6baIxEpTv1a8Tx1Yw8eve58Pt11iAFPLuSjjV/5XZaUAzWSMkrL8G6vqz0SkRKZGT++qBVzxvbjrAY1ufPVFH7z7nqFP0Y5NZIyStuXd5927ZGIhKpdQh3evq8Pdya14bVlOxny1GK2fKnwx2ilRlJGp8IaddGVyGmpHhfLr67ozCt39GR/1nGufGoRry7dofDHKKRGUkbpgWBYY7zCGkVK5XsdE5g3Lok+7Rrz2/c2cOerKRzIUvhjNNGnXxmlBZSxJVJWCXWr8+KIi/jN4M4s3JrBgCcXsjg1w++yJERqJGWQF9aoaBSRsouJMUb2a8M7o/pQt0YcN09dzl/nbeZ4jsIfI50aSRnsUVijSLnrclZ9Zo/px7CLWvHsJ2kMfXYJOzKy/C5LiqFGUgZpp8Ia1UhEylNH3C3JAAASMElEQVStanH85drzeeamC9i5P5srJiYzc9UenYiPUGokZZAX1qhrSETCY+D5zZg3LokuzevzwJtrGTdtDV8fPeF3WVKAGkkZpGcEwxob1lZYo0i4nNWgJm/c2Yuf/bAj76/by6AJyazeddDvsiQfNZIySNuXRdsmOqwlEm6xMcaYSzsw4+7eAFz/7FKeWrCNXIU/RgQ1kjJIz8jUiC2RCnTh2Q2ZOy6JK85vxt8/3MqNzy/ji0Pf+F1WladGUkqHs0+QkXlcI7ZEKli9GvFMGNadx6/vxvrPDzNwQjLz1+/1u6wqTY2klE6FNWqPRKTCmRnXXdiC98cmcXbjWtzzz9U89PY6vjmu8Ec/qJGUkm6vK+K/1k1qM/OePtzzvXZMW7mLwZOS2fiF7rtX0dRISiktkElcjMIaRfxWLS6GBweewz9HXsyRozlcPXkxLy7armtOKpAaSSmlBzI5u7HCGkUiRd/2TZg/vj/9OzbhkTkbuf3llWRkHvO7rCpBn4KllK6wRpGI06h2NZ6/NZFHhnRhSdp+BjyZzCdbA36XVempkZRCTu5JduzP0ogtkQhkZtzauzWzR/ejUe14Rry4gj/O2cixHJ2IDxc1klLIC2vUNSQikavTmXWZNboft/Y+mxcWbefap5ecyseT8qVGUgrpGQprFIkGNeJjeWTIeTx/ayJfHPqGwRMXMX3lLp2IL2dqJKWQtk9hjSLR5Iedz2DeuP70aNWAX761jtH/+pTD2Qp/LC9qJKWgsEaR6HNm/Rq8NvJifjGgEx9s+JJBE5NZueOA32VVCmokpZAWUFijSDSKjTHu+357Zt7bh7hY48fPLeUfH20lJ1d3YSwLNZJSSA9kasSWSBTr3rIB749N4uoezZnwn20Mm7KMPQez/S4raqmRnKa8sEaN2BKJbnWqx/HEDd2ZMKw7m788wsAJycxe+4XfZUWlsDYSMxtgZlvMLNXMHizk9epmNt17fbmZtc732kPe9C1mdrk3raWZfWxmm8xsg5mNC2f9hVFYo0jlMqR7c+aOTaJdQh3GvPEpP39zLVnHcvwuK6qErZGYWSwwGRgIdAaGm1nnArONBA4659oD/wAe9ZbtDAwDugADgKe99eUAP3POnQv0AkYVss6wOnV7XR3aEqk0WjWuxZv39Gb0Je2ZuXoPgyctYv3nh/0uK2qEc4+kJ5DqnEt3zh0HpgFDCswzBHjFez4TuNTMzJs+zTl3zDm3HUgFejrn9jrnVgM4544Am4DmYdyG70j3whpbKaxRpFKJj43hgcs78cadvTh6Ipdrnl7M8wvTOam7MJYonI2kObA73/d7+O6H/ql5nHM5wGGgcSjLeofBegDLC/vhZnaXmaWYWUogUH5ZO2mBTFoprFGk0urVtjHzxiXxg3Oa8qe5mxjx0gr2HTnqd1kRLZyfhlbItIKtvah5il3WzOoAbwHjnXOF3nzAOTfFOZfonEtMSEgIseSSpQeydCGiSCXXoFY1nr35Qv58zfms3HGAgU8ms2DzV36XFbHC2Uj2AC3zfd8CKDgk4tQ8ZhYH1AcOFLesmcUTbCKvO+feDkvlRcjJPcnO/dm0a6rzIyKVnZlx48WtmD26Hwl1q3PHyyk8PGsDR08o/LGgcDaSlUAHM2tjZtUInjyfVWCeWcAI7/lQYIELhuDMAoZ5o7raAB2AFd75k6nAJufcE2GsvVB7Dn7D8dyTtNMeiUiV0eGMurw7qi+3923Ny0t2cPXkxWz76ojfZUWUsDUS75zHaOADgifFZzjnNpjZI2Z2lTfbVKCxmaUC9wMPestuAGYAG4H5wCjnXC7QF7gF+IGZrfEeg8K1DQWdCmvUHolIlVIjPpbfXdmFl267iMCRY1z51CJeX75T4Y8eqwr/EImJiS4lJaXM63khOZ0/vr+JT3/zQ+VsiVRR+44c5Wcz1pK8LYPLu5zBX6/tWik/D8xslXMuMZR5NfToNKQFMmlYK75S/tKISGia1q3BK7f35NdXnMuCzfsYOCGZpWn7/S7LV2okpyEtkKVoFBEhJsb4SVJb3rmvL7WqxXLjC8t47IPNnKii4Y9qJKcheJ92nR8RkaDzmtdn9ph+XH9hCyZ/nMb1zy5l1/6qF/6oRhKiw9+cICPzmDK2RORbaleP429Du/HUjT1IC2QyaGIy7376ud9lVSg1khClB/Jur6tGIiLfNbjrWcwbl8Q5Z9Zl/PQ13D99DUeOVo27MKqRhChNYY0iUoIWDWsx7a5ejL+sA++u+ZwrJi5ize5DfpcVdmokIVJYo4iEIi42hvGXdWT63b3JPekY+swSJn+cSm4lDn9UIwlReiBLYY0iErKLWjdi7tgkLu9yJo99sIWbX1jOl4crZ/ijPhVDlBbIVFijiJyW+rXieerGHvztuq6s2X2IARMW8uGGL/0uq9ypkYQg96QLhjXq/IiInCYz44aLWjJnbD9aNKzJXa+t4tfvrqtU4Y9qJCHYczA7GNaoEVsiUkrtEurw1r19uDOpDf9ctosrJy1i85eF3gUj6qiRhCAtkHefdu2RiEjpVY+L5VdXdObVO3pyMPsEVz21mFeW7Ij68Ec1khD87z7t2iMRkbLr3zGB+eOT6NuuMb+btYE7X03hQNZxv8sqNTWSEKQFsmhYK55GCmsUkXLSpE51XrztIn53ZWcWbs1gwJMLWbQtw++ySkWNJARpgUztjYhIuTMzbu/bhndH9aVezXhueXE5f5m3ieM50RX+qEYSguB92nV+RETCo/NZ9Zg9uh/De7biuU/SGfrsErZnZPldVsjUSEqQF9bYrqn2SEQkfGpWi+XP15zPszdfwM792VwxMZmZq/ZExYl4NZIS5IU1ao9ERCrCgPOaMX98Euc3r88Db65l7LQ1fB3h4Y9qJCXIG7GlPRIRqSjN6tfkX3f24ueXd2Luur0MmpDMqp0H/S6rSGokJUhTWKOI+CA2xhh1SXvevKc3ZnDDc0uZ+J9tERn+qEZSgvRAFq0aKaxRRPxxQauGvD82icFdm/HER1sZPmUZnx/6xu+yvkWfjiVIz9DQXxHxV70a8Tz54+48fn03NnxxmIFPLmTeur1+l3WKGkkxck86dmQorFFE/GdmXHdhC94fm0SbJrW59/XVPPT2Z2Qfz/G7NDWS4uSFNSpjS0QiResmtXnznj7c+/12TFu5mysnLWLDF4d9rUmNpBinRmzp0JaIRJBqcTH8csA5/HPkxRw5msM1k5fwQnI6J306Ea9GUoz/pf6qkYhI5Onbvgnzx/enf8cE/vj+Jm5/eSWBI8cqvA41kmKkBbJooLBGEYlgjWpX4/lbL+QPQ7qwLH0/Aycs5L9b9lVoDWokxUgPZOqwlohEPDPjlt6tmTW6H41rV+e2l1byhzkbOZZTMXdhVCMpRprCGkUkinQ6sy7vje7Lrb3PZuqi7VwzeQlZx8I/qisu7D8hSuXknqR/xyb0ad/Y71JEREJWIz6WR4acR/8OCazYcYDa1cP/MW/RkCxZVomJiS4lJcXvMkREooaZrXLOJYYyrw5tiYhImYS1kZjZADPbYmapZvZgIa9XN7Pp3uvLzax1vtce8qZvMbPLQ12niIhUrLA1EjOLBSYDA4HOwHAz61xgtpHAQedce+AfwKPesp2BYUAXYADwtJnFhrhOERGpQOHcI+kJpDrn0p1zx4FpwJAC8wwBXvGezwQuNTPzpk9zzh1zzm0HUr31hbJOERGpQOFsJM2B3fm+3+NNK3Qe51wOcBhoXMyyoawTADO7y8xSzCwlEAiUYTNERKQ44WwkVsi0gkPEiprndKd/d6JzU5xzic65xISEhGILFRGR0gtnI9kDtMz3fQvgi6LmMbM4oD5woJhlQ1mniIhUoHA2kpVABzNrY2bVCJ48n1VgnlnACO/5UGCBC17YMgsY5o3qagN0AFaEuE4REalAYbvk0TmXY2ajgQ+AWOBF59wGM3sESHHOzQKmAq+ZWSrBPZFh3rIbzGwGsBHIAUY553IBCltnSbWsWrUqw8x2lnJTmgAZpVw2EkR7/aBtiATRXj9oG07X2aHOWCWubC8LM0sJ9erOSBTt9YO2IRJEe/2gbQgnXdkuIiJlokYiIiJlokZSsil+F1BG0V4/aBsiQbTXD9qGsNE5EhERKRPtkYiISJmokRQhmlKGzWyHma0zszVmluJNa2RmH5nZNu9rQ2+6mdlEb7s+M7MLfKr5RTPbZ2br80077ZrNbIQ3/zYzG1HYz6rA+h82s8+992GNmQ3K91rEpVmbWUsz+9jMNpnZBjMb502PivehmPqj5n0wsxpmtsLM1nrb8HtvehsLJqJvs2BCejVv+mknplcI55weBR4Er1FJA9oC1YC1QGe/6yqm3h1AkwLT/gY86D1/EHjUez4ImEcwbqYXsNynmvsDFwDrS1sz0AhI97429J439LH+h4EHCpm3s/c7VB1o4/1uxfr9ewY0Ay7wntcFtnq1RsX7UEz9UfM+eP+Wdbzn8cBy7992BjDMm/4scK/3/D7gWe/5MGB6cdtWUb9L2iMpXGVIGc6frPwKcHW+6a+6oGVAAzNrVtHFOecWErwINb/Trfly4CPn3AHn3EHgI4K3HQi7IuovSkSmWTvn9jrnVnvPjwCbCIagRsX7UEz9RYm498H7t8z0vo33Hg74AcFEdPjue3A6iekVQo2kcCGnDEcIB3xoZqvM7C5v2hnOub0Q/A8HNPWmR/K2nW7Nkbgto73DPi/mHRIiCur3DpH0IPgXcdS9DwXqhyh6Hyx4r6U1wD6CTTgNOOSCiegF6zndxPQKoUZSuJBThiNEX+fcBQRv+DXKzPoXM2+0bRuUQxp0BXkGaAd0B/YCj3vTI7p+M6sDvAWMd859XdyshUzzfTsKqT+q3gfnXK5zrjvBENqewLnF1BOR26BGUrioShl2zn3hfd0HvEPwl/GrvENW3td93uyRvG2nW3NEbYtz7ivvQ+Ek8Dz/O7QQsfWbWTzBD+HXnXNve5Oj5n0orP5ofB8AnHOHgP8SPEfSwIKJ6AXrOd3E9AqhRlK4qEkZNrPaZlY37znwI2A9305WHgG85z2fBdzqjcDpBRzOO4wRAU635g+AH5lZQ+/wxY+8ab4ocK7pGoLvA0RomrV3bH0qsMk590S+l6LifSiq/mh6H8wswcwaeM9rApcRPNfzMcFEdPjue3A6iekVo6LO6kfbg+AIla0Ej1f+yu96iqmzLcHRGmuBDXm1Ejxu+h9gm/e1kTfdCN73Pg1YByT6VPcbBA87nCD419TI0tQM3EHwxGIqcLvP9b/m1fcZwf/YzfLN/yuv/i3AwEj4PQP6ETz88RmwxnsMipb3oZj6o+Z9ALoCn3q1rgd+601vS7ARpAJvAtW96TW871O919uWtG0V8dCV7SIiUiY6tCUiImWiRiIiImWiRiIiImWiRiIiImWiRiIiImWiRiJSAjNb4n1tbWY3lvO6/6+wnyUSTTT8VyREZvZ9gqmyg09jmVjnXG4xr2c65+qUR30iftEeiUgJzCwvnfWvQJJ3j4ufemF7j5nZSi8g8G5v/u9b8D4Z/yJ4YRxm9q4XqrkhL1jTzP4K1PTW93r+n+VdPf6Yma234L1mfpxv3f81s5lmttnMXveu8MbM/mpmG71a/l6R/0ZStcWVPIuIeB4k3x6J1xAOO+cuMrPqwGIz+9CbtydwngtGegPc4Zw74MVgrDSzt5xzD5rZaBcM7CvoWoKhg92AJt4yC73XegBdCGYpLQb6mtlGgnEg5zjnXF7shkhF0B6JSOn9iGD21BqC8eWNCWYcAazI10QAxprZWmAZwXC9DhSvH/CGC4YPfgV8AlyUb917XDCUcA3QGvgaOAq8YGbXAtll3jqREKmRiJSeAWOcc929RxvnXN4eSdapmYLnVi4DejvnuhHMVqoRwrqLcizf81wgzgXvTdGTYBLu1cD809oSkTJQIxEJ3RGCt3TN8wFwrxdljpl19BKYC6oPHHTOZZvZOQRjwvOcyFu+gIXAj73zMAkEb+1bZJqrd0+O+s65ucB4gofFRCqEzpGIhO4zIMc7RPUyMIHgYaXV3gnvAP+7JWp+84F7zOwzgsmsy/K9NgX4zMxWO+duyjf9HaA3wVRnB/zCOfel14gKUxd4z8xqENyb+WnpNlHk9Gn4r4iIlIkObYmISJmokYiISJmokYiISJmokYiISJmokYiISJmokYiISJmokYiISJmokYiISJn8P51YYXgfTr38AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learner.sched.plot_lr()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**The previous state of the art for this dataset was 94.1%, by using Transfer learning for language models as developed by fastai we were able to beat this :)**\n",
    "\n",
    "**TODO: Add a second language model that learns to classify reversed texts. Average the two predictions. This should give an even higher accuracy according to Jeremy!**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:deeplearning]",
   "language": "python",
   "name": "conda-env-deeplearning-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}