Resnet from scratch - CIFAR 10.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Resnet from scratch - CIFAR 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%reload_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.conv_learner import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH = \"data/cifar10/\"\n",
    "os.makedirs(PATH, exist_ok=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Calculate the mean and stdv of every channel for normalisation:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "folder_names = os.listdir(f'{PATH}train/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('ship',\n",
       " 'cat',\n",
       " 'truck',\n",
       " 'bird',\n",
       " 'horse',\n",
       " 'automobile',\n",
       " 'frog',\n",
       " 'dog',\n",
       " 'airplane',\n",
       " 'deer')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classes = tuple(folder_names)\n",
    "classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_names = [os.listdir(f'{PATH}train/' + folder) for folder in folder_names]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_names = [[folder_names[i] + \"/\" + file_name for file_name in file_names[i]] for i in range(len(folder_names))]\n",
    "file_names = list(itertools.chain(*file_names))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "images = np.array([np.array(Image.open(PATH + 'train/' + fname)) for fname in file_names])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(50000, 32, 32, 3)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "images.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "stats = (np.array([images[:,:,:,0].mean()/255, images[:,:,:,1].mean()/255, images[:,:,:,2].mean()/255]), np.array([images[:,:,:,0].std()/255, images[:,:,:,1].std()/255, images[:,:,:,2].std()/255]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0.4914 , 0.48216, 0.44653]), array([0.24703, 0.24349, 0.26159]))"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_data(sz,bs):\n",
    "    tfms = tfms_from_stats(stats, sz, aug_tfms=[RandomFlip()], pad=sz//8)\n",
    "    return ImageClassifierData.from_paths(PATH, val_name='test', tfms=tfms, bs=bs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs = 256"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Let's take a look at the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = get_data(32, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "x, y = next(iter(data.trn_dl))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7efe10a0b128>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAG4FJREFUeJztnVmMXOd5pt+vlt43djeb4r6JpizJIqVwNMrQyNhxYiuGZ2QHsWEDY+jCCIOZeDAGMheCB4g9c+UMxjZ85QE9FqIEjpfENqwLZ/EIHghJAMmULFGSqY1kc20uzWav7O5azjcXVUJa9P/+Xb1Vt/S/D0Cw+v/qr/PVqfPWqTpvfd9v7g4hRHrk1jsBIcT6IPELkSgSvxCJIvELkSgSvxCJIvELkSgSvxCJIvELkSgSvxCJUljJZDN7GMA3AOQB/B93/8oi99fPCcWqY8uY48uatRKad+i7e0NPzpb7814zywN4HcDvArgI4BcAPuPuv4rMkfjfRcSOsJyFzytZRAQeOTzyxrdWjBzDGZlXMf6h1zO+rVgsLvDqMmNLp1Hxr+Rj/4MA3nT3M+5eAvA9AI+s4PGEEE1kJeLfDuDCgr8v1seEEO8AVvKdP/TR4tc+95jZMQDHVrAdIcQasBLxXwSwc8HfOwBcvv1O7n4cwHFA3/mF2Eis5GP/LwAcMLO9ZtYC4NMAnlydtIQQa82yz/zuXjGzzwP4e9Ssvsfd/ZVVy0wIsaYs2+pb1sb0sf9dhay+X4tGYu8uq08I8Q5G4hciUSR+IRJF4hciUSR+IRJlRVV9Im1i17azXCU8J3Ih2iKHY94yGhvob6Ox/fs2B8evXBmnc86cm6axClppLOp/xIwuX92r/Y2iM78QiSLxC5EoEr8QiSLxC5EoEr8QiaKr/WIF8KvbOYSvwBvydE6WzdJYXzc/VI8e3UVjh+8dCI5Pjm+hc376t6/R2JmLPMeZUtjhAAAHdyvoblzjShid+YVIFIlfiESR+IVIFIlfiESR+IVIFIlfiERRGy8RJdoPKtIKq5jvDo7nM17EMthborGjR3fQ2Hve20VjbYWZ4HhPVy+dM3yB53HhCj+E3zjLi4XeHL5OY1OT5eB4TJtG2pO5u9p4CSHiSPxCJIrEL0SiSPxCJIrEL0SiSPxCJMqKqvrMbBjAFGpLjlTc/chqJCU2DlFv1nmlWkvrreD4+w700Tn/6tBWGtuzq4PGZud4z72sFM6xNMer84aGuCw2bd7EY738ubUWw9YnADx38nRwvFzmVYKrwWqU9H7Q3UdX4XGEEE1EH/uFSJSVit8B/IOZPWdmx1YjISFEc1jpx/6j7n7ZzIYA/MzMXnX3pxfeof6moDcGITYYKzrzu/vl+v/XAPwYwIOB+xx39yO6GCjExmLZ4jezTjPrfus2gA8DeHm1EhNCrC0r+di/BcCP69VFBQB/5e5/typZbVjYe2XsPZRXsVnESItZbJaLbS9sbeUjD9jZXqSxgU18KazOHl48tu/O8LyHDm2nc7ZxFw05m6ex8WluA45PdQbHz5y/Qed090ReM7tJY+U5Pm9TXwuNFfLhpqblUmQZr1UokF22+N39DIBDK85ACLEuyOoTIlEkfiESReIXIlEkfiESReIXIlG0Vt9SYPZKzHaJOTKRPou5HF/TjllDAJAvhCvB7tozROfc/z5eTTcwwCvLBrfw/Af7wk+8N89txZ4u/rxmIztytthDYzevh23AZ587R+fs4G4kDrwnvPYfANwRqTws9HLLtO2X4X1ixvfV3Hy4krEacQdvR2d+IRJF4hciUSR+IRJF4hciUSR+IRJFV/uXBLvaHyvAiDyctdJQnrfHQ77Mi1wevH9XcPzfffQeOqe7Y4zGKhmPdffy/DtL4avbrVXuHuRyPNbez6t+WtoGaezFH/0yOP7GqyN0ztCWbTS2ff9OGuve3E5jfnqKxu6+d09wvLXQT+e8/KtwAe2NUb5k2O3ozC9Eokj8QiSKxC9Eokj8QiSKxC9Eokj8QiSKrL4lkCNOn2fcz/N87P21TCM7hnghyPsfOEBj9x0OF56054bpnM4WnkdfP7eb2jt4rK+dxLIZOqda4LHJMrcVRy9wO3JTZ7hY6OBBvnzWbzz0Phob2rWPxq5N8oWrRiIWXP/mcC7m/Dnv2n1HcHxqku/D29GZX4hEkfiFSBSJX4hEkfiFSBSJX4hEkfiFSJRFrT4zexzAxwBcc/d762P9AL4PYA+AYQCfcne+jtG7hnBPtTwZBwADr8C7c294KSkA+Pcf2k1j7z8StnkAoFKeDI7PTvMywc29W2istWUzjfVuClcQAkC+pxQeb+GHXHmaW1unnr1EY92lPhr72EceCI7P5++nc/q37aCxsyPczjtznlfuzcxy69YtbLXOzU/QOS3k0LElnM4bueufA3j4trHHADzl7gcAPFX/WwjxDmJR8bv70wBu/xXFIwCeqN9+AsDHVzkvIcQas9zv/FvcfQQA6v/zvtBCiA3Jmv+818yOATi21tsRQiyN5Z75r5rZVgCo/3+N3dHdj7v7EXc/ssxtCSHWgOWK/0kAj9ZvPwrgJ6uTjhCiWTRi9X0XwAcADJrZRQBfAvAVAD8ws88BOA/gk2uZ5EYhIz5KLrKU1F3buZ33qY/spbH9e8JWGQBYxm0vnw/nMtTDl+RCiS935cVeGssXeMNK5MO2V2sHP+TOn7lFY2M3eOXktiFeobf7nnDDzenIsmFXx3mV48QMf63nZltozKvcxrR8+DFzRZ5HoTXcNNZiDWNvf4zF7uDunyGhDzW+GSHERkO/8BMiUSR+IRJF4hciUSR+IRJF4hciUdahgecy3m+ofcFtF3gktqxtAUB4LbmuLr4bP/J+Xp13365wc0kAyOXnaGx+lidZuRWuHsvyXXROlvGKs6kyb46ZdfEqtq3t4f1vETvs8iX+eFkrtxwH9kdszI6w1Xr5Aq+YqxQ7aKxnU7hBKgBcvsQLW2NHfUYKLp0cbwCQLzCrr/HjXmd+IRJF4hciUSR+IRJF4hciUSR+IRJF4hciUZps9RlgvJqKwt6ijFshVg1bIQAA7rBFaUf4MX/zIG/4+J4dvGIOVb5+W2cPt7bmpnjFX5aFra1sns8pEtsIALL8LI21t0eqAXNhS292glt9XW28ESe28aq4Yje35s5dCVumb0ZsxY4evj8mI2vh5Z2fS7MSt2fLpXCT13I13IwVAGC8IWuj6MwvRKJI/EIkisQvRKJI/EIkisQvRKI0vbAnh6Vfpcx7eE4+i1zRd/7UKpHltdi2AOCBA+Gijg//G758FuZGeGiC599e4EshVCd58UZXd7hIpyXHlw3rbOdX4Ac2c9ehvZvHZkmOVyNLWpVneM+6ge2DNDbFTROcPnMjOD5xixdOjY7zB5yd5u7H/DTPf26Wbw8edq0sopVqJRzzJRS06cwvRKJI/EIkisQvRKJI/EIkisQvRKJI/EIkSiPLdT0O4GMArrn7vfWxLwP4QwDX63f7orv/tJENZst4u8lVyXJGkfeuDNy+MucW26H93L565IPhpZ92DXEbrTrJYz0FXpBSLHE70ub4Y7Z2hWOtef5St7Xxopn+AV5sk0XyHyE98kYvcqtvYDtfdqslkv/VcW5v3SLLa81N8wKdWyVuy5XmeIFUpcStPrNIjPTqs4wXA83fIlbfEpz0RqT45wAeDox/3d0P1/81JHwhxMZhUfG7+9MAeAtXIcQ7kpV85/+8mZ00s8fNbNOqZSSEaArLFf83AewHcBjACICvsjua2TEzO2FmJ6J99oUQTWVZ4nf3q+5edfcMwLcAPBi573F3P+LuRxZZEUMI0USWJX4zW7hEyicAvLw66QghmkUjVt93AXwAwKCZXQTwJQAfMLPDqH2OHwbwR41srNiTx5aHuJXGmLoYtl7mRngPv9IMt8PuPcgrxB75wDYaOzAYzqO3K9w3DwC6e/fSWL7KfZlS5BtS2zy3jQqFsMXZ1s2tz55B/ppUIn3p5qZ5kuVb4Xndfbzv3/59/TR27Tq3CK9f4/bbJLFaZ6b445Ui1aKVSG/ISpVbhOUqtxYrFVIpGKlMvTUZ3laWNf7VelHxu/tnAsPfbngLQogNiX7hJ0SiSPxCJIrEL0SiSPxCJIrEL0SiNLWBZ/dQO/7tfz605Hmla2ErZOZV3mjRL3Eb8CP/+jdobFsWrkYDgPbKreB4SwevfPMqz6PYyW2Z2XK48SQAdLSHm3QCQN/g1uB4V08XneOt7TSWGbcxL1+4SWNGLMeBrbwSMF/g+2ry6nkau3KJW6a3ymFrMZdFXpcCl0W5wu28zHlzz1yBz2vPhdePm5vh68pNjYcfr1pRA08hxCJI/EIkisQvRKJI/EIkisQvRKJI/EIkSlOtvqpVMdHGq6kYW7eH7aEHB/fROfvA7bdWXgSG6evcKsn3h6sBS877FFjGbTkUuEWV6+SPuWUzt+3yLWFry9p4Vd/EFLeoJsd4bGyM7yvWsDJX4NbhtVF+bFw+e5XGZiZ5s9NKIXx+yxs/9HMRqy+f51V91hZ5PYv89SzPhOfdHOVW6s3R6eB4tRJZv/L2nBq+pxDiXYXEL0SiSPxCJIrEL0SiSPxCJEpTr/ZnXsFs+dqS57WUDgTH97SGxwGge26SxuZv8RyKHfyKrXWECy3KE7yn3sAQ7xdY7OC7v6WPX8HOtfOCj9m58BX4LMcfD3nuSFwfHeHbmuXPu0g2NzUVLo4CgItXRmns7DlexFVGxFHpDu+ragt3P9pa+f7d1Mq3VZnlV9onbvDCnksXw0Vcw2e5+zE9Hd7WUnr46cwvRKJI/EIkisQvRKJI/EIkisQvRKJI/EIkSiPLde0E8BcA7gCQATju7t8ws34A3wewB7Uluz7l7rwSAYAbUFmGuTg8fCk4/vPr3D4ZzPHqnXu2D9FYrhrpm3YpbL2Ur3BbsTDLe8VtvZMvDZYrcSsKVf6eXfTwPK/yHT92bozGSpHCnnyZ20qjI+FCnNHX+bJVc7P89Zya4M95/BbP38rh/ZF18f1bKfPn3NbGexDeuMxtzFdfuUJjl66En/fsVMS2Y68zIlVrt9HImb8C4E/c/b0AHgLwx2Z2N4DHADzl7gcAPFX/WwjxDmFR8bv7iLs/X789BeAUgO0AHgHwRP1uTwD4+FolKYRYfZb0nd/M9gC4H8AzALa4+whQe4MAwD9LCyE2HA2L38y6APwQwBfcnX/J/fV5x8zshJmdKE803mhACLG2NCR+MyuiJvzvuPuP6sNXzWxrPb4VQPAH8+5+3N2PuPuRYi+/mCaEaC6Lit/MDMC3AZxy968tCD0J4NH67UcB/GT10xNCrBWNGG9HAXwWwEtm9kJ97IsAvgLgB2b2OQDnAXxy0UeqFoCJ/iUnWe0J9z979cZZOudgF6+myw1torHySLg3GgBM3wxbStlUZM6b/BvSxE1eqdbVy3Ps7t9MY9Ya7u/XQpbPAoAbw7w/3tQVbqNNjPGqsytjYfvq6jhfDg3VeRrq641U7rXz/nhXL4W/ar78yqt0jrdxWRS7eB5T4/w4mJzkz80RLoE0Ml6bs3IWFb+7/yMAtnc/tAo5CCHWAf3CT4hEkfiFSBSJX4hEkfiFSBSJX4hEaWoDz/JoFRcfX/pyXf17w5VUA9vuoHNKVb6kVakwQGND+3bQWDYYtu2qM7yY8dql8zQ2GqliG5sIN3UEgPwMn9fTFbYB8xFvqFzllYczc7zCbfjiZRo7fT3c3HPa+bYG+yNLaHXxxqrtB3h1ZOt0+PzW5rz6bXSE27Mz1/nr4nPccsxbpCErMe6qiNiDRl5QNfAUQiyGxC9Eokj8QiSKxC9Eokj8QiSKxC9Eopj7atQHNbgxM7fC0t9vil1hC6h/aw+dUyhwa+joQ/fR2Cd//4M0ls2E15IbO/0anePTvIptoLOdxubmuRU1G2mcuWPvweB4sZXbaJUyX3Pv+WdfobF//qeXaWycVBFuf2AvnbP9Tm7PVjv4/vBdvTRW6g1bbFXulmLuHN+/k6/xicMvcuvz4mvXaWz6ZtjSK1cjzTir5PiuZHB37jkuQGd+IRJF4hciUSR+IRJF4hciUSR+IRKluVf7c+bLKSUqeniSZbwbcAm8gKSrj/ezO3R4D43t7g8XGNnlETqnJ8cLY+7cyQuMWjojS3L18D5yO+8KX03v6udX0nPFVho79Trv7/f3//cEz+PQ3cHxzvt4D8fpTbygptTOi1ymePqYaw87GZ0FXmgzlPEeiX1lnv/8GL/IfmWYF7RdOhvu5Xh+mBeFjbwZfl3GfzWO8kxZV/uFEByJX4hEkfiFSBSJX4hEkfiFSBSJX4hEWdTqM7OdAP4CwB0AMgDH3f0bZvZlAH8I4K2KhS+6+09jj5Uz88JyvL5cOMcs0pguA7cB85H3PC9zS6k9F3ZQ+tu4dbh3gPtQWwb4qsV9d/DH7Onnj3mQFLls2txN5xR6+mjs5mzEBnyD97PL9YWLlsZ2cwt2ejffH5UWXnyU5WPLWoWPg9ZI7UtLpHCqSBevAjq6O2msrZsXcVWJLT1f4vtj7mq4sOfH/+mfcP31iYasvkaUWAHwJ+7+vJl1A3jOzH5Wj33d3f9XIxsSQmwsGlmrbwTASP32lJmdArB9rRMTQqwtS/rOb2Z7ANwP4Jn60OfN7KSZPW5mfFlZIcSGo2Hxm1kXgB8C+IK7TwL4JoD9AA6j9sngq2TeMTM7YWYnmvdDYiHEYjQkfjMroib877j7jwDA3a+6e9XdMwDfAvBgaK67H3f3I+5+pKGrEEKIprCo+M3MAHwbwCl3/9qC8a0L7vYJALynkxBiw9HI1f6jAD4L4CUze6E+9kUAnzGzwwAcwDCAP1rsgRxARpYmipGRJYgcvE8fIrHMeVVcPsftmlmE7aaReb6t3Cy3tu46sI/HDvJLKOORJaOq5bC1NXJ5hs4ZG+axc9f5c7syym3Rnj1h+83v5hVzMz38XJSb5lZfS4XbuiiQz5t5PifjziFmS/w5l6Z5z72c816OpY7wc7MOLs8827+tMU28nUau9v8jEDQ3o56+EGJjo1/4CZEoEr8QiSLxC5EoEr8QiSLxC5EoyyixWxlV8EolDvFesth7V8Tqi1T8EVexRi5s223bxe2rhw7torH77uHLTLX5TRq7OR1eNgwAxvPh6r2ZEvevTrx0jcZOnuaNJ+cqfP/f07E7OL5pjlcXliK2aAF8mazZIrcBvTVszbU6Pw5bMi6LXOSQa3W+j22eH1gVCx+P1YznmBEdVbPGrT6d+YVIFIlfiESR+IVIFIlfiESR+IVIFIlfiERputW3PNh7VMyX45aHReY5qdwDgL6+cIXYx373Ljpn5yZeQTh5/SyNlY1Xj92a5bbXNWIpXbjM3+dfPHWLxm7MRGxR4/vxjV+EKw9bJ/nzKu7mOQ7u4g1NC7t4c8z8lvBjtnTxx0MxUtYXaRob6fsJeOR4JIdcjjSMBYAsRx5vCUWzOvMLkSgSvxCJIvELkSgSvxCJIvELkSgSvxCJ8g6x+li1V6SCKWJDuc/SWE8ft+Y++fF7guO/eZDPuXB6hMZKVf7eWyrySsEzN7kd+frZseD4pSt8f0zP8fX4qjleWebg+3FsZDw4no3w5qOkuA0AcKWTB20Tz7/rzq7g+PZ7ttA5/Xv469lCrEMAKA7QEFq7eP45YgNmZX58V3OkWWhkDcJf227D9xRCvKuQ+IVIFIlfiESR+IVIFIlfiERZ9Gq/mbUBeBpAa/3+f+PuXzKzvQC+B6AfwPMAPuvufL2iFUGuOEcubFok5s6vfA8M9NDYvr3bg+MTY7wH3ugNfsX2Gm+Ph0vj4avlAPDCK3zpp+kJ5gTwQpYsurQZL8QBKy4BUGoJvwAW6buYq/AXbXaSH6qVST5v5lx4J4/+M9+/hQF+Zb5jF3cCBt/bR2Ob9/B+jUM7+sPb6g07FQDQ2h3ej/lq4wZeI2f+eQC/7e6HUFuO+2EzewjAnwH4ursfAHATwOca3qoQYt1ZVPxeY7r+Z7H+zwH8NoC/qY8/AeDja5KhEGJNaOg7v5nl6yv0XgPwMwCnAYy7+1u/vrkIIPyZWAixIWlI/O5edffDAHYAeBDAe0N3C801s2NmdsLMTiw/TSHEarOkq/3uPg7g/wF4CECfmb11dWEHgMtkznF3P+LuR1aSqBBidVlU/Ga22cz66rfbAfwOgFMAfg7gD+p3exTAT9YqSSHE6mMx2wsAzOw+1C7o5VF7s/iBu/8PM9uHf7H6fgngP7jHfCHALFJtE8+CjPOHs4iLWch30FhbJ8/i/gfuCI7nq/xpn3+DL7s1ORPp0xcp6pgtRSpgWA/CaKETfzSPForwWD4fLsaySC1WJebd5vjrGbMPC/S58UQqkSXlrMhzzLXwPPJdfF7ftvASZgN3hC1AAOjdGbYOT/7wV5i+PtNQdc+ipqC7nwRwf2D8DGrf/4UQ70D0Cz8hEkXiFyJRJH4hEkXiFyJRJH4hEmVRq29VN2Z2HcC5+p+DAEabtnGO8ng7yuPtvNPy2O3uvAHkApoq/rdt2OzERvjVn/JQHqnmoY/9QiSKxC9Eoqyn+I+v47YXojzejvJ4O+/aPNbtO78QYn3Rx34hEmVdxG9mD5vZa2b2ppk9th451PMYNrOXzOyFZjYbMbPHzeyamb28YKzfzH5mZm/U/9+0Tnl82cwu1ffJC2b20SbksdPMfm5mp8zsFTP7L/Xxpu6TSB5N3Sdm1mZmz5rZi/U8/nt9fK+ZPVPfH983M96VtRHcvan/UCsNPg1gH2otZV8EcHez86jnMgxgcB22+1sAHgDw8oKx/wngsfrtxwD82Trl8WUA/7XJ+2MrgAfqt7sBvA7g7mbvk0geTd0nqNVKd9VvFwE8g1oDnR8A+HR9/H8D+I8r2c56nPkfBPCmu5/xWqvv7wF4ZB3yWDfc/WkAt6+o+QhqfROAJjVEJXk0HXcfcffn67enUGsWsx1N3ieRPJqK11jzprnrIf7tAC4s+Hs9m386gH8ws+fM7Ng65fAWW9x9BKgdhACG1jGXz5vZyfrXgjX/+rEQM9uDWv+IZ7CO++S2PIAm75NmNM1dD/GHuoysl+Vw1N0fAPB7AP7YzH5rnfLYSHwTwH7U1mgYAfDVZm3YzLoA/BDAF9x9slnbbSCPpu8TX0HT3EZZD/FfBLBzwd+0+eda4+6X6/9fA/BjrG9noqtmthUA6v/zZYDWEHe/Wj/wMgDfQpP2iZkVURPcd9z9R/Xhpu+TUB7rtU/q215y09xGWQ/x/wLAgfqVyxYAnwbwZLOTMLNOM+t+6zaADwN4OT5rTXkStUaowDo2RH1LbHU+gSbsEzMzAN8GcMrdv7Yg1NR9wvJo9j5pWtPcZl3BvO1q5kdRu5J6GsB/W6cc9qHmNLwI4JVm5gHgu6h9fCyj9knocwAGADwF4I36//3rlMdfAngJwEnUxLe1CXm8H7WPsCcBvFD/99Fm75NIHk3dJwDuQ60p7knU3mj+dMEx+yyANwH8NYDWlWxHv/ATIlH0Cz8hEkXiFyJRJH4hEkXiFyJRJH4hEkXiFyJRJH4hEkXiFyJR/j8MWQDDGZHU+wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(data.trn_ds.denorm(x)[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7efe1a537358>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAGF9JREFUeJztnWuMXdV1x//rPubhefj9GGyDH7gKCBoDE4eWBBIIyKGRALWJQBHiA4qjKEhFSj8gKhUq9QOpCogPLa0JVkhFeDSAsCrUhloUK2prGKhtDAYMjgHbE49fY3ve97H64R6rgzlrzb3n3nvuOPv/k0ZzZ6+7z15337Pm3Lv/Z60tqgpCSHhkWu0AIaQ1MPgJCRQGPyGBwuAnJFAY/IQECoOfkEBh8BMSKAx+QgKFwU9IoOTq6SwiGwE8BiAL4Geq+pD3/M6uLu2dN7+eIav3LZVRWoX36qw7Ns/vGVHzdSUjyQzOjNMzpRtpTw8PY3xstKo3O3Hwi0gWwN8DuBHAQQBvishWVX3P6tM7bz6+/6N7ah7LugVZkLX98w4oDQ4E93DJ3nVxD+rYjLlS5zWn+m/Bc93p5t+GXvscu8HvjOXZylpO1K+RPPdP/1D1c+v52L8BwEequl9VpwA8C+CWOo5HCEmReoJ/OYDPpv19MGojhJwH1BP8cZ+cvvDZRkQ2iciAiAyMj47WMRwhpJHUE/wHAayc9vcKAIfPfZKqblbVflXt7+zqqmM4QkgjqSf43wSwTkRWi0gbgNsBbG2MW4SQZpN4tV9ViyJyD4B/R0Xq26Kq787QCwp7RdQeyz6eaXGWc8VbeU2w9C3uQq5ntAfThotRtc970pHc4yU8YKNX+5OO5duc1f5GKxI1tsdRl86vqq8AeKWeYxBCWgPv8CMkUBj8hAQKg5+QQGHwExIoDH5CAoXBT0igMPgJCRQGPyGBwuAnJFAY/IQECoOfkECp697+2hFIkv83RuaMV+rq/C/jldBqlfFK+H++0SW+vKn3Zyq9Ml5Jx/JPg3RqENbyfvHKT0igMPgJCRQGPyGBwuAnJFAY/IQECoOfkEBJV+pTQEtJJA+rj1czzcHbvSaJDJhwlya/RpsnKTl15Kx2r/ac40ei+fDw5iph7TwPy/9yE8bydxzybIbRfc8sSbd6eOUnJFAY/IQECoOfkEBh8BMSKAx+QgKFwU9IoNQl9YnIAQBnAJQAFFW13+8AaCIpLb5PUhlthsGactRGjpREiGq4ZIfkkliaWD56Uqq31Vujs/M8mj1SI3T+b6rqsQYchxCSIvzYT0ig1Bv8CuDXIvKWiGxqhEOEkHSo92P/Nap6WESWAHhVRN5X1e3TnxD9U9gEAD1z59U5HCGkUdR15VfVw9HvIQAvAdgQ85zNqtqvqv2dXV31DEcIaSCJg19EukSk5+xjADcB2NMoxwghzaWej/1LAbwUSUg5AL9U1X9ze2hCecjo4slX7iiuD7WLbE2R0Rp+wHRluSTvc5pypJ/16RwvwVgz9bRedrOl1MTBr6r7AXy5gb4QQlKEUh8hgcLgJyRQGPyEBAqDn5BAYfATEigp79WnSJiT1mhHGoonyTRDvvp9JXHhzAZLYrMlW9GVshvgI6/8hAQKg5+QQGHwExIoDH5CAoXBT0igpLzanzC/JMUkkTRX7pOv2Dr9DB+boTkkTY6xD5hsS7FGp0Gpsw1cPmeHjPd2Foslp5+RfNRk1YFXfkIChcFPSKAw+AkJFAY/IYHC4CckUBj8hARKqlJfJa3HllEsxNSNkiU+NFoGbEoChtQ+T9GIRqvto1+1cLZsT5Vwey2Dctk5nvOerb1oqWk7OXzGtA0OnTRtSc5H28fq3y9e+QkJFAY/IYHC4CckUBj8hAQKg5+QQGHwExIoM0p9IrIFwHcADKnqZVHbAgDPAVgF4ACA76mqrWVMI1lWX82GpmBJMunXfEsiYybNmGssiev0JTymNRtlR13rmRg1bZd1FE3bjs5201Yu29JtNpuNbW/2+1LNlf/nADae03YfgG2qug7AtuhvQsh5xIzBr6rbAZw4p/kWAE9Fj58CcGuD/SKENJmk3/mXquogAES/lzTOJUJIGjR9wU9ENonIgIgMjI/Z36UIIemSNPiPiEgfAES/h6wnqupmVe1X1f7OOV0JhyOENJqkwb8VwF3R47sAvNwYdwghaVGN1PcMgG8AWCQiBwE8AOAhAM+LyN0APgXw3apHTCD1zJLdk1JF3Cw2Z0IMk5s31uRtoaqlGWOZx8zEy2sAcPqknZ136sOPTdtkb5/jh2mC+aY1ee5nDH5VvcMw3dBgXwghKcI7/AgJFAY/IYHC4CckUBj8hAQKg5+QQEl9r75kWHuZ2T3coppeVlyCoqBu/cWEm+S5hTNdk6X1OY6c53KeJ4tadTrLJTvL7o2jdnZeG6ZM29w1I6YN4lxnG7l/ZQ3H4pWfkEBh8BMSKAx+QgKFwU9IoDD4CQkUBj8hgdICqa92XcMUqTz1ypXzvH4JtDlPckyq9XmZe37H2NYkRS7rwZx/72UllUWd15Yx5nGqYA9WyHSatt9NTtpjTdjFPTOz8DI7C10ihKQBg5+QQGHwExIoDH5CAoXBT0igpLzar1C1EyrsbvErs15Ch5fY4ybvJKiP565SO8dzE5OSJO84zrh5Pc7RkpSe80mWsJTcjVJ8a9muJH3D5QtM28Tw70zb2KR9budy9nXWVCucE6QRCVK88hMSKAx+QgKFwU9IoDD4CQkUBj8hgcLgJyRQqtmuawuA7wAYUtXLorYHAfwAwNHoafer6ivVDOjLVIYPRnvG3WbKll3KVmE3mKpi5IeRNOPocm5ij6e/uYUBPRLUO3TmypNM/cwq65jJait6eOdBqRR/zANH7QSdQXsnLyzqtOv79Ypd30+dEyvNLdGmU82V/+cANsa0P6qq66OfqgKfEDJ7mDH4VXU7gBMp+EIISZF6vvPfIyK7RWSLiMxvmEeEkFRIGvyPA1gLYD2AQQAPW08UkU0iMiAiA+OjYwmHI4Q0mkTBr6pHVLWklVW1JwBscJ67WVX7VbW/s2tOUj8JIQ0mUfCLSN+0P28DsKcx7hBC0qIaqe8ZAN8AsEhEDgJ4AMA3RGQ9KrrSAQA/rGo0BbRce1afJc0VCo5E5fxfE0+ac9SrKcP3jLMVkzcWYPuf9WQ0ryCc+QK849nalni1/xyJ0JI/Pemw7M2jWyfRNo5MxB/zk+NO3b8Je317xTo7ZPI521YoxmcXVkiS1WcdqXrZcMbgV9U7YpqfrHoEQsishHf4ERIoDH5CAoXBT0igMPgJCRQGPyGBknoBT0+WsbDUwa45dobVyuUXmLa+ZUtNW8+cbtM2Pnkmtn1szM4QGx8vmLbJom0rFuytn4qFCfuYY/G2yWFbvipN2f5POQphIeMVSc3HtnsZeBnY81Fyzpupkn0af3AkfuutM2MjZp+bF5w2bZcvW2Tado/aPnbMsElcbKsnfZrt1WeD8spPSKAw+AkJFAY/IYHC4CckUBj8hAQKg5+QQElV6lNVlKZsOcfi4jUXxbbfdN01Zp8Vy/tMW1acbDRHKikeOBBveOe/zD452JJdedmFth9rLjZtWGTLmEUj/a30yce2H//xou3H/n2mrST26VPqjJf6pnL2HnnjGft4k85YxyZtie1LhrRYXmyfh99aZ2c57lphz728+4lpK2U92S7e/6wjb2aN11WLlM4rPyGBwuAnJFAY/IQECoOfkEBh8BMSKKmu9neI4pK28Zr7jU+Oxra//vrrZp+vf+2rpu2iC+1V9vf2HjBtv/xZ/Kr4+tPvm32uyx4xbfNgJ+i0LVxi2kau+1PTVvrq9bHtsmS52Uc3ft+05V7fatrad20zbZ1Fo97h3B6zz8QiW+EoOqfq2uyQabs2cyq2PVu0VZhdusK0Tandb83CDtN2etw+70vGVl7Fsq06mDvO1bDLG6/8hAQKg5+QQGHwExIoDH5CAoXBT0igMPgJCZRqtutaCeAXAJahsr/UZlV9TEQWAHgOwCpUtuz6nqqe9I7V3duLP77xxpqdXHrxZbHtI6N2rbVs1k5wGD513LS9/9F+0zbRFS9THZp/hdnng/FPTdsf9JomqLO7k/73dtvWPTe2vXChLaNNFafswb7+J6ZJVl9q2ha88Upse+epg2af08dsW6nDTggqZOKTiABAuxbEtrd3xNf2A4DP5q02bUMn46VDAMj3zjNtyxYsNG05I+lHy/ZJYG0R195mz8UXjlHFc4oAfqKqlwC4GsCPReRSAPcB2Kaq6wBsi/4mhJwnzBj8qjqoqm9Hj88A2AtgOYBbADwVPe0pALc2y0lCSOOp6Tu/iKwCcAWAHQCWquogUPkHAcC+JY0QMuuoOvhFpBvACwDuVVX7y/YX+20SkQERGRg+ZX9fIoSkS1XBLyJ5VAL/aVU9e4P7ERHpi+x9AGJvsFbVzarar6r98+bGL0YRQtJnxuAXEQHwJIC9qvrINNNWAHdFj+8C8HLj3SOENItqsvquAXAngHdEZGfUdj+AhwA8LyJ3A/gUwHdnOlBRyzg2YWeyWfSOHI1tLxWdbaYm7Bpt4/FJggCART32lMyT+PHajtmZe/NHPzRthUlb6yvn2kxbx/x4+QoAxo1EsELRnnd1pD53S7Fe+5Pc+NXfjm3Pn4h/LwFg1JG2xtSpTdduy3bjxiGHjtuqtKotl0l5zLSd+OwD01bO2j6WjJc2evKY3ScTv1XduJM9eC4zBr+q/gZ2ouANVY9ECJlV8A4/QgKFwU9IoDD4CQkUBj8hgcLgJyRQUi3gOTY6ircG/qfmfkOH4jPtuntsqaytzZbKMhn7f15b3pZ5/mhN/DHnvGtLfSVLawIwdPKEadP5doZYZu2XTNvxkXgd89h+OxNwfMzWPo8P2XLT5KgtK62fH5+Fd8ly+y7wfcfs+RjY/1vT1rfKzljMdsRnYr63z87ezHTNN225nF2kszzubEXnZdsZsq6225KupXxqDddzXvkJCRQGPyGBwuAnJFAY/IQECoOfkEBh8BMSKKlKfcVSCcdOOil1BqPjh2Lb53fbmVltziub48guuUz8HnMAgGz8QU9dbhfwLE/amYflsj1WtjM+awsA5nXOMW0nD8cXwXxv126zz8HP7MKZY5O2nDd8xpa2Fq5aHNt+64i9r2G+ZF+LjnfaGYRHS/abPX4qfm+9fM8ys4+33525Rx4AOD6WSvZcFSbi59iTnY2anxDu1UcImQkGPyGBwuAnJFAY/IQECoOfkEBJdbVfIMhla1iOjDg9Eb9ifvKMXUG8NGLbLszZPnQMxxYhBgAcKccXyCsuu8DsIx32FOedpeNiIX6VGgB6uwZN29y58clOXb3xCS4AsHD5ctO22Fn6PvO+nWzz9uBwbPvQEruWXTFr+9im9nXq6ElbkejujZ//XN5+XcWit1ea3a/k9Dtxxk6QGh2Nrwu4aGGf2cfysVhy1Kpz4JWfkEBh8BMSKAx+QgKFwU9IoDD4CQkUBj8hgTKj1CciKwH8AsAyAGUAm1X1MRF5EMAPAJzdf+l+VX3FO1YZBUyW7Hp3FtmskeDgJFmcGLPln/1DtpzXdcL272hPfF26XI9d1y2nti0zZUtDe3YfMG35jLEnF4Crrroktn3JYrsu3QXOtlsdeacWYpst243s2xvb/vpvbZnyEEZMW+Fiu/bfgrw9j/lcvIxWKNoJZhMF+9wpleyxppz3szBhJ6EB8VupjY7asp21LVepZCeSnUs1On8RwE9U9W0R6QHwloi8GtkeVdW/q3o0QsisoZq9+gYBDEaPz4jIXgD2XSGEkPOCmr7zi8gqAFcA2BE13SMiu0Vki4jYnysJIbOOqoNfRLoBvADgXlU9DeBxAGsBrEflk8HDRr9NIjIgIgPjY9V/HyGENJeqgl9E8qgE/tOq+iIAqOoRVS2pahnAEwA2xPVV1c2q2q+q/Z1z7Oo0hJB0mTH4RUQAPAlgr6o+Mq19etbBbQD2NN49QkizqGa1/xoAdwJ4R0R2Rm33A7hDRNajIrgdAPDDmQ6UgaAjZ8tUFmJ0aXfq3OWXLjJtJUOyA4Cey1abtrnd8ePlOmw5TBw9cnLSrus2vGyhaRsbnTJtUxPxstHieXbGXD5j14orlGz/Oy+ys84+Honfeuu1g/ZrnppjS45fWWaP1efM//BkvKSnTkalFOyvp1MZ2/92J2O1u9371Bt/DZ4oxL+XANDTFi8D5msIr2pW+3+D+JKGrqZPCJnd8A4/QgKFwU9IoDD4CQkUBj8hgcLgJyRQUi3gmc/l0bfILnZpEy+h5BzZcH6XnWGVydj/87JZO5PK217LQh2pr9Rh+/iVq+wtubwajVmjOOlUyZaNik4mWKFs+6hiOzJ3RbzUml/QbfbpdOQwzdiZdqNFW37LGWf4XLGzLTucAq8l9fbrss+rjHMelA3Zsez0qdxb90Xa8ztj2+N9IoQECYOfkEBh8BMSKAx+QgKFwU9IoDD4CQmUVKW+icIY3j+0u+Z+hqoBSwIEfInN2y3Qk/OsY3rijzuYgyXlAEDZlZsMH50uSecqI7bVtHh9xmzboVFHKvOmwzqk08d7zdmMc845+wmKc5lV481xTgHTx6minfF5LrzyExIoDH5CAoXBT0igMPgJCRQGPyGBwuAnJFBSlfomi1P4ZGh/7R0NWUZhayEi9kvzpBxP9rL7JLN5ZBJqhJakp15GojOU/9pqz3JMKrGJ0y+BF64fZSdbMZuxsy0zzrW0UIrfMxBwZFhHn7UU00mn+Oi58MpPSKAw+AkJFAY/IYHC4CckUBj8hATKjKv9ItIBYDuA9uj5v1LVB0RkNYBnASwA8DaAO1XVzSpQLWOqVH3iwVnMVVR32d6u6+b1E2dZWYwlVnUTjBy8ZBsvE8Q9pLFSnUmoO3ir8wlenLcyr0mToBybnwRl9IFdt7C9YNeNzOTs92ysbJ+PVmJPydkqLWMkGNWiVlVzhk0CuF5Vv4zKdtwbReRqAD8F8KiqrgNwEsDdVY9KCGk5Mwa/VhiJ/sxHPwrgegC/itqfAnBrUzwkhDSFqj5bikg22qF3CMCrAD4GMKyqxegpBwEsb46LhJBmUFXwq2pJVdcDWAFgA4BL4p4W11dENonIgIgMFCYT3YtFCGkCNa0qqeowgP8EcDWAefL/99CuAHDY6LNZVftVtT/fTnGBkNnCjNEoIotFZF70uBPAtwDsBfAagD+LnnYXgJeb5SQhpPFUk9jTB+ApEcmi8s/ieVX9VxF5D8CzIvI3AP4XwJNVjehs/2R2seQLpx6c92/Nq4/nFYTLGPKb1Q40XoaqHNNJaLKmytU3vbGS2UwZMEGyyoyDOS/ATpByagk672dRRu1+JbtfW4JbaibLtixunjs1nFMzBr+q7gZwRUz7flS+/xNCzkP4JZyQQGHwExIoDH5CAoXBT0igMPgJCRSxMoqaMpjIUQCfRH8uAnAstcFt6MfnoR+f53zz4yJVXVzNAVMN/s8NLDKgqv0tGZx+0A/6wY/9hIQKg5+QQGll8G9u4djToR+fh358nt9bP1r2nZ8Q0lr4sZ+QQGlJ8IvIRhH5QEQ+EpH7WuFD5McBEXlHRHaKyECK424RkSER2TOtbYGIvCoi+6Lf81vkx4Miciiak50icnMKfqwUkddEZK+IvCsifx61pzonjh+pzomIdIjIGyKyK/Ljr6P21SKyI5qP50Skra6BVDXVHwBZVMqArQHQBmAXgEvT9iPy5QCARS0Y91oAVwLYM63tbwHcFz2+D8BPW+THgwD+IuX56ANwZfS4B8CHAC5Ne04cP1KdE1Tyjbujx3kAO1ApoPM8gNuj9n8E8KN6xmnFlX8DgI9Udb9WSn0/C+CWFvjRMlR1O4AT5zTfgkohVCClgqiGH6mjqoOq+nb0+AwqxWKWI+U5cfxIFa3Q9KK5rQj+5QA+m/Z3K4t/KoBfi8hbIrKpRT6cZamqDgKVkxDAkhb6co+I7I6+FjT968d0RGQVKvUjdqCFc3KOH0DKc5JG0dxWBH9cCZVWSQ7XqOqVAL4N4Mcicm2L/JhNPA5gLSp7NAwCeDitgUWkG8ALAO5V1dNpjVuFH6nPidZRNLdaWhH8BwGsnPa3Wfyz2ajq4ej3EICX0NrKREdEpA8Aot9DrXBCVY9EJ14ZwBNIaU5EJI9KwD2tqi9GzanPSZwfrZqTaOyai+ZWSyuC/00A66KVyzYAtwPYmrYTItIlIj1nHwO4CcAev1dT2YpKIVSghQVRzwZbxG1IYU6ksg/akwD2quoj00ypzonlR9pzklrR3LRWMM9ZzbwZlZXUjwH8ZYt8WIOK0rALwLtp+gHgGVQ+PhZQ+SR0N4CFALYB2Bf9XtAiP/4ZwDsAdqMSfH0p+PE1VD7C7gawM/q5Oe05cfxIdU4A/CEqRXF3o/KP5q+mnbNvAPgIwL8AaK9nHN7hR0ig8A4/QgKFwU9IoDD4CQkUBj8hgcLgJyRQGPyEBAqDn5BAYfATEij/B6y4pOceY2AuAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(data.trn_ds.denorm(x)[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7efe1a290588>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFS1JREFUeJzt3V+sXFd1x/HvmutrUzt/7JDEOCYlgPIAQsWgqwgpFaLQohQhJWkLCg8oDxFGFZEaiT5EqVRSqQ9QFRAPFZVpIkJFMSl/RFRFLVFEFfESuKSJE3BbQhSIE8smjSFOTO6/WX2Yk/bambVmZs+ZM3b27yNZvvecOWev2TNrZu5Zs/c2d0dE6tObdwAiMh9KfpFKKflFKqXkF6mUkl+kUkp+kUop+UUqpeQXqZSSX6RSW6Y52MyuAb4ALAD/4O6fzm6/a9cuv+yyy6ZpcvzYsGynzMmru+uze9fNN2mffvppnjtxYqxuLk5+M1sA/g74A+AI8EMzu8fdfxIdc9lll3Hw4MGh+3o2+YcQTzo0O5/12n0KmpWdLzuq9JxtP8WyF9Gs/wm+Nt7r+NvkUT+W9m9JWwDZ1+hLvmIftXXdH//R2OeY5mP/VcDj7v6Eu68CB4FrpzifiHRomuTfCzy16fcjzTYROQdMk/zDPne84vOLme03s2UzWz5x4sQUzYlIm6ZJ/iPA5Zt+fz3wzJk3cvcD7r7k7ku7du2aojkRadM0yf9D4Eoze6OZbQVuAO5pJywRmbXiq/3uvm5mNwP/xqDUd6e7/3jEMWxsbAzd1+/1C4KId/WTy8rm8VXZomvAyVXetPiTXOXd6A/vp8GBBbtKJ23J7luyrxfsW8iaKiwEZnF0ebU/lfR/v+Cxifp3klNNVed393uBe6c5h4jMh77hJ1IpJb9IpZT8IpVS8otUSskvUiklv0illPwilVLyi1RKyS9SKSW/SKWU/CKVmuq7/ZMyMxYWhg/t0DRep9tqi0Xn1DRepzsXpvHqtTiN1yR3S+/8IpVS8otUSskvUiklv0illPwilVLyi1Sq01Kf9/us/OalofuiOckArDf8NSo7ph8cA+2XedI55FptaQxRCSg5JCs05SvNxPMuej84rnQuwUTJajhnU6mvqE+Ctvr98efC1Du/SKWU/CKVUvKLVErJL1IpJb9IpZT8IpWaqtRnZk8CJ4ENYN3dl7Lb991ZfSko9RWU5vLlojos9ZWOEkyX3SoriUWj8Ervcz5yLz9y4kNmoMtSX6mSSKJ+TEuKZ2ijzv977v5sC+cRkQ7pY79IpaZNfge+a2Y/MrP9bQQkIt2Y9mP/1e7+jJldCtxnZv/p7g9svkHzorAfYPfu3VM2JyJtmeqd392faf4/DnwbuGrIbQ64+5K7L+3cuXOa5kSkRcXJb2Y7zOz8l38G3g881lZgIjJb03zs3w18uymZbAH+yd3/NTvAHCwY7WX52LKhW/tZWeNcKPXNQnDfvOvxhUFz6aPcZYjZSMAkymxC0+KJXAvqn230VXHyu/sTwNunD0FE5kGlPpFKKflFKqXkF6mUkl+kUkp+kUp1OoEnJKPOiqaYzMouWQyFggPLX0FnUDY6S0arRY/YejLpZ5dD/vKmkjJg6fMqm9wzjWV29M4vUiklv0illPwilVLyi1RKyS9SqW6X68JZ21gfuq/n7b4OZdWDbH6/7JKtjb8S0v+3lQ0SyeYgTOY0TKsE4et5srRZMsglu4KdLk8VbF8srLWUrvIVxdjPruingSRtpceV9XFJHOPSO79IpZT8IpVS8otUSskvUiklv0illPwileq21OfOen9j6L6sJFail7yupVXFglKOEdcA02kG01JZWU3JgjvnyZ32ZLBNOuAqiT/cU3IMMGJts1hQMs3mjJxkyavTwyg9rkUTnEzv/CKVUvKLVErJL1IpJb9IpZT8IpVS8otUamSpz8zuBD4IHHf3tzXbLgK+DlwBPAl82N1PjNNgvz+8rFRaXolk58vbykpRwbJhyXpLlq3FlFXz+qWj36LX87Lly3qlcwIG9y1/mMv6Kq30Bfe7l64b1u2seiXzLkbP4UnyaJx3/i8D15yx7Vbgfne/Eri/+V1EziEjk9/dHwCeO2PztcBdzc93Ade1HJeIzFjp3/y73f0oQPP/pe2FJCJdmPkFPzPbb2bLZrb8q1//etbNiciYSpP/mJntAWj+Px7d0N0PuPuSuy/tvPDCwuZEpG2lyX8PcGPz843Ad9oJR0S6Mk6p72vAe4CLzewI8Cng08DdZnYT8AvgQ+M05u5sbASj+tIJK89uvpAtQVVYViysNkUj9LJRfVnf93pJGSotlw3fnE4WmjWVlfqSnb1gfa0tybpbaRlwFkqqqWGM4wc/Mvnd/SPBrveN3YqInHXO3bdbEZmKkl+kUkp+kUop+UUqpeQXqVSnE3hCXJaJSoAQj3pKJ8BMyj/RyMJRx0XteTaBZ7Ivnxwz3pUKSnr9ftYfcd+3XYL1JI7SCTCzxzPqY0/Wa1xIOj8flFgW/9k8qk9EXoWU/CKVUvKLVErJL1IpJb9IpZT8IpXqeK2+vKQ3qawMVTqBZ152GX7c6spvwiOyUXFbt24tjCMzPMZeukBhLKmIlZVa8+F5k58PsIISWz8Zulc6f2f5Y9aeSWLXO79IpZT8IpVS8otUSskvUiklv0ilur3aj7O+vj75gcFV1F4wXx2A9csqAdnyVKdefHHo9uNPHwmPec22+Ir+hRfuDPdt27Yt3JcNL+kHy3ytrsT9vpEM7MkqEgtJtSWq6qytrYXHpJP4JVZWVsJ9O7ZvH7r9tZdcHB7TT/p3FgN75kXv/CKVUvKLVErJL1IpJb9IpZT8IpVS8otUapzluu4EPggcd/e3NdtuBz4G/LK52W3ufu+oc7k7L60npZ44hqHbS+eXywZgrK6uhvsOPfLI0O3Hfv5UeMy2LXGpLCvnbdlSWIUN5rNbW4nv18ZGXAZcXFwM9/V6C8k5h5f6VtbiONLBWOEe0vLxvn37hm6/4KK4zLqWlJAz6fx5HVUBs+XQzjRO9nwZuGbI9s+7+77m38jEF5Gzy8jkd/cHgOc6iEVEOjTN3/w3m9khM7vTzHa1FpGIdKI0+b8IvBnYBxwFPhvd0Mz2m9mymS0///zzhc2JSNuKkt/dj7n7hg8Wg/8ScFVy2wPuvuTuSxdccEFpnCLSsqLkN7M9m369HnisnXBEpCvjlPq+BrwHuNjMjgCfAt5jZvsYFDCeBD4+TmNr6+scf+5/hu7LRtNZUNJLl7tKaivR+QDWk1FnL7w0fK6+dY/j6K/FI+ZW1l8K9/UWyv4iWwjKVL1k5F5WHtpIlsLKHrPojFkRLenG9DE7b/v54b6NoMVjzz4bHrPqydJx4Z4RZbaOSn1rE5TSRya/u39kyOY7JglIRM4++oafSKWU/CKVUvKLVErJL1IpJb9IpTqdwLO/scHJkyeH7stG2kWj90qXR8qOs2R5rTdc8Yah233vb4fH9JLiUG8hHhWXLfOVFZyiSU17Scku7Y+Wl6Dy9H6VyUZ3RvteODV8MlaAYA7UkdJRfR2ZJAa984tUSskvUiklv0illPwilVLyi1RKyS9SqbNmrb6spLQQlMRmMYGnJUPLwuMWk9Ft2ci3JPx+cYlt+HGWPNTZiLnS6lV4WHK30jJV0h/rwWShAFsKnyM1UM+IVErJL1IpJb9IpZT8IpVS8otUqtOr/Zl0qaZgX7QkFMxo0E+wr29ly0z1ksv9WSUjm7vQgr5asPh8C0lX9ZMBQSUDWUoflyyOrBLQD5YiywZcZSWO7HHJno+ZNvtRA3tEZCQlv0illPwilVLyi1RKyS9SKSW/SKXGWa7rcuArwOsYrLZ0wN2/YGYXAV8HrmCwZNeH3f1Edi73uBxSUgIqmfdv1HElJUeLp+JLl3eyYL49APpl5cNozavieemSfWk/BqU5T+7XQvaYZWN+4l1h2a60hFl6XOm+SFelvnXgk+7+FuBdwCfM7K3ArcD97n4lcH/zu4icI0Ymv7sfdfeHmp9PAoeBvcC1wF3Nze4CrptVkCLSvon+5jezK4B3AA8Cu939KAxeIIBL2w5ORGZn7OQ3s/OAbwK3uPvzExy338yWzWz51IvxXOki0q2xkt/MFhkk/lfd/VvN5mNmtqfZvwc4PuxYdz/g7kvuvrR9x442YhaRFoxMfhtcVrwDOOzun9u06x7gxubnG4HvtB+eiMzKOKP6rgY+CjxqZg83224DPg3cbWY3Ab8APjROg2G5LBuZFZRXsnJeVpIpHvEXbM9GxaVN5XW0MSIacljwep5VgDayvsoKacnSW1FJL323ycp56Ui7bPmy4dvT8XeFoxxLn1f50mzDtbEy2Mjkd/fvE3fH+6YPQUTmQd/wE6mUkl+kUkp+kUop+UUqpeQXqVTnE3i2MRppmmOmYQUlmaw01CssDWX3O+rfXuEko5btTZY28/7wYtpGVs4LlmWDUROrJqPwgpGTJX0I4FlbwX0ebfL3YM9GhM6sVRF5VVDyi1RKyS9SKSW/SKWU/CKVUvKLVOqcKPVFx5SPoopf8/I4hh/Xz8aIZSEulK37VtJXnk/7GZ+vsK/6Pjz+aGJPAFtIRu4law1uBOvxQRxjLykr9rPJU2dQXi47Z3SM1uoTkRGU/CKVUvKLVErJL1IpJb9IpTq/2p8NdImUXtUviSGdSzAYTJENmkljz5YbS69Gx/H3ozkS4yhaX0qqOXJ4HMngqKyvsupHOngqqFZkFYKsrS1b4pQprSLNi975RSql5BeplJJfpFJKfpFKKflFKqXkF6nUyFKfmV0OfAV4HdAHDrj7F8zsduBjwC+bm97m7vfmZ/N4oEVSJsn2hS0lpZW1tbW4rWQASS9Yl8vjqlw619rq6mq4byEp9aV9VVAWnUWJKoq/tK3SOfciWTmv1Pr65AOMIF5iLeNMPjfhmcap868Dn3T3h8zsfOBHZnZfs+/z7v63Y7cmImeNcdbqOwocbX4+aWaHgb2zDkxEZmuizxtmdgXwDuDBZtPNZnbIzO40s10txyYiMzR28pvZecA3gVvc/Xngi8CbgX0MPhl8Njhuv5ktm9nyqRdPtRCyiLRhrOQ3s0UGif9Vd/8WgLsfc/cNH1zR+hJw1bBj3f2Auy+5+9L2HdvbiltEpjQy+W1wKfUO4LC7f27T9j2bbnY98Fj74YnIrIxztf9q4KPAo2b2cLPtNuAjZraPwfCtJ4GPjz6VFZXtIukccslIr2xkVqYXjEjL7tLGRhxHVm7KypFbt24N9y1u2zZ0+/pqfL6sVJY9XlmpMjrntiC+UUpLhFk/RrIya3a+fORhVlosyYkOSn3u/n2GjwgdUdMXkbOZvuEnUiklv0illPwilVLyi1RKyS9SqU4n8DTikk1WNorKF1lJJtuXlbbanmixdHReVqrMYlyNSnrZpJ+FbWWi+1068i2TPZ5RWbf0PpePPIwf62xf3NbEh7yC3vlFKqXkF6mUkl+kUkp+kUop+UUqpeQXqVSnpb6+OysrKxMfF42kKp3UsbSk5MH6cxCP2IpGAg72ZeWfdte0s+Q+l5YcS+Lves26KP7iyTaTvs/LusnzsT/5BKQW9P0kk5nqnV+kUkp+kUop+UUqpeQXqZSSX6RSSn6RSnVa6gMPSy9Z2SgqX6Vr7hWu/ZeWSoJdvV5cGlpfj8uApaMSs1JULwgyKg2N2pcNH+un688Nt1BYwsz2ZeXINo+BaUaLJrEk6zlGej55efAV55j6DCJyTlLyi1RKyS9SKSW/SKWU/CKVGnm138xeAzwAbGtu/w13/5SZvRE4CFwEPAR81N3jifgGZ5to4MH/BRnMw9b2/HIwYpBLcFV2y5bsKm+2TNMMrm4H8ffDQUngydXmbMxJ9khaNJAlOSi7X21XAkqeh6Nk8ZdWF+LzDd8+SU6M886/ArzX3d/OYDnua8zsXcBngM+7+5XACeCmsVsVkbkbmfw+8ELz62Lzz4H3At9ott8FXDeTCEVkJsb6m9/MFpoVeo8D9wE/A37l7i8Pij4C7J1NiCIyC2Mlv7tvuPs+4PXAVcBbht1s2LFmtt/Mls1s+dSpF8sjFZFWTXS1391/Bfw78C5gp5m9fCXu9cAzwTEH3H3J3Ze2b98xTawi0qKRyW9ml5jZzubn3wJ+HzgMfA/4k+ZmNwLfmVWQItK+cQb27AHuMrMFBi8Wd7v7v5jZT4CDZvbXwH8Ad0wTSFaiiEp9pQN0Sudo628Mr6/ksS/G5ytcMqpk8FE6sCSJIxpUBaMGTwXl1MIp/PI44tJtpLTUV7pcVxZ/SSxRW5N078jkd/dDwDuGbH+Cwd//InIO0jf8RCql5BeplJJfpFJKfpFKKflFKmVdLp9kZr8Eft78ejHwbGeNxxTH6RTH6c61ON7g7peMc8JOk/+0hs2W3X1pLo0rDsWhOPSxX6RWSn6RSs0z+Q/Mse3NFMfpFMfpXrVxzO1vfhGZL33sF6nUXJLfzK4xs/8ys8fN7NZ5xNDE8aSZPWpmD5vZcoft3mlmx83ssU3bLjKz+8zsp83/u+YUx+1m9nTTJw+b2Qc6iONyM/uemR02sx+b2Z812zvtkySOTvvEzF5jZj8ws0eaOP6q2f5GM3uw6Y+vm9nWqRpy907/AQsMpgF7E7AVeAR4a9dxNLE8CVw8h3bfDbwTeGzTtr8Bbm1+vhX4zJziuB348477Yw/wzubn84H/Bt7adZ8kcXTaJwwGZp/X/LwIPMhgAp27gRua7X8P/Ok07czjnf8q4HF3f8IHU30fBK6dQxxz4+4PAM+dsflaBhOhQkcTogZxdM7dj7r7Q83PJxlMFrOXjvskiaNTPjDzSXPnkfx7gac2/T7PyT8d+K6Z/cjM9s8phpftdvejMHgSApfOMZabzexQ82fBzP/82MzMrmAwf8SDzLFPzogDOu6TLibNnUfyD5u2ZF4lh6vd/Z3AHwKfMLN3zymOs8kXgTczWKPhKPDZrho2s/OAbwK3uPvzXbU7Rhyd94lPMWnuuOaR/EeAyzf9Hk7+OWvu/kzz/3Hg28x3ZqJjZrYHoPn/+DyCcPdjzROvD3yJjvrEzBYZJNxX3f1bzebO+2RYHPPqk6btiSfNHdc8kv+HwJXNlcutwA3APV0HYWY7zOz8l38G3g88lh81U/cwmAgV5jgh6svJ1rieDvrEBpPY3QEcdvfPbdrVaZ9EcXTdJ51NmtvVFcwzrmZ+gMGV1J8BfzGnGN7EoNLwCPDjLuMAvsbg4+Mag09CNwGvBe4Hftr8f9Gc4vhH4FHgEIPk29NBHL/L4CPsIeDh5t8Huu6TJI5O+wT4HQaT4h5i8ELzl5uesz8AHgf+Gdg2TTv6hp9IpfQNP5FKKflFKqXkF6mUkl+kUkp+kUop+UUqpeQXqZSSX6RS/wui9iH97WwlGAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(data.trn_ds.denorm(x)[2])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fully connected model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = get_data(32, bs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SimpleNet(nn.Module):\n",
    "    def __init__(self, layers):\n",
    "        super().__init__()\n",
    "        self.layers = nn.ModuleList([nn.Linear(layers[i], layers[i + 1]) for i in range(len(layers) - 1)])\n",
    "        \n",
    "    def forward(self, x):\n",
    "        # minibatch size conserved, rest flattened\n",
    "        x = x.view(x.size(0), -1)\n",
    "        for l in self.layers:\n",
    "            x = F.relu(l(x))\n",
    "        return F.log_softmax(x, dim=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = ConvLearner.from_model_data(SimpleNet([32*32*3, 40, 10]), data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SimpleNet(\n",
       "  (layers): ModuleList(\n",
       "    (0): Linear(in_features=3072, out_features=40, bias=True)\n",
       "    (1): Linear(in_features=40, out_features=10, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[122880, 40, 400, 10]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[l.numel() for l in learn.model.parameters()]\n",
    "# weights first layer, bias first layer, weights output layer, bias output layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OrderedDict([('Linear-1',\n",
       "              OrderedDict([('input_shape', [-1, 3072]),\n",
       "                           ('output_shape', [-1, 40]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 122920)])),\n",
       "             ('Linear-2',\n",
       "              OrderedDict([('input_shape', [-1, 40]),\n",
       "                           ('output_shape', [-1, 10]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 410)]))])"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2776e85db9f2468593d1d1bc055dc105",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                   \n",
      "    0      2.244108   2.302585   0.1       \n",
      "\n"
     ]
    }
   ],
   "source": [
    "learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZQAAAEOCAYAAACuOOGFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xd4VFX6wPHvmw5JCARCgBR6CSpFQseCInYFe0NEFLGw9tV1XVd/7q666+rqrg0VsRcEERVsLC5SBEIvoQmhkwRCCYSS8v7+mIubjZOQhJncyeT9PM88uTn3nDvvyUDe3HvuPUdUFWOMMeZEhbgdgDHGmOBgCcUYY4xPWEIxxhjjE5ZQjDHG+IQlFGOMMT5hCcUYY4xPWEIxxhjjE35LKCKSIiIzRCRTRFaKyN1e6lwqIstEZImIZIjIgFL7hovIOuc1vFR5DxFZLiLrReRFERF/9cEYY0zlib8ebBSR5kBzVV0kIrHAQmCIqq4qVScGOKiqKiJdgE9UtZOIxAMZQDqgTtseqrpHROYDdwM/AVOBF1V1ml86YYwxptL8doaiqjtUdZGznQ9kAkll6hzQ/2a0aDzJA+Bc4DtVzVPVPcB3wHlOkmqgqnOddu8AQ/zVB2OMMZUXVhNvIiKtgO7APC/7hgJPAU2BC53iJGBLqWpbnbIkZ7tseYWaNGmirVq1qnrgxhhThy1cuHCXqiZUtr7fE4pzWWsicI+q7i+7X1U/Az4TkdOBJ4FBgLdxEa2g3Nv7jgJGAaSmppKRkVG9DhhjTB0lIpuqUt+vd3mJSDieZPK+qk6qqK6qzgTaikgTPGceKaV2JwPbnfJkL+XejjdWVdNVNT0hodIJ1hhjTDX58y4vAd4EMlX1uXLqtDt2l5aInApEALuBb4DBItJIRBoBg4FvVHUHkC8ifZx2NwKf+6sPxhhjKs+fl7z6A8OA5SKyxCl7BEgFUNVXgcuBG0WkEDgEXO0MtueJyJPAAqfd/6lqnrN9OzAeqAdMc17GGGNc5rfbhgNJenq62hiKMcZUjYgsVNX0yta3J+WNMcb4hCUUY4wxPlEjz6HUdcUlytKtezl4pIhG9SM4qUUDys4Yo6oUlSjhoZbjjTG1kyUUPygsLmHq8h18sXQ7+w8V8XPuAXYfPPrL/p6tGnFNz1S+XbWT/MNFtGxcn/kb89i0u4CreqbQuXkDpmdmk94qnltOa80XS3cwbfkOtuwpIDU+mit6JNGvXRMaRIW72EtjjPlfNih/gjbkHqBElaSG9akXEUpu/hGuHjuXDbkHSYmvR1LDejSPq8fATk1pHhdF5o79PP/dWvYUFJIQG0mLuCg27jpI5xYNSGlUn8lLtlFYrDRrEMXO/YeJiQzjwJEiWjWuT7umMSzduo/c/CMAtG4STXKjerRrGkPX5Ib0bhNP87h6fumnMabuqeqgvCWUKtpbcJT8w0UkN6rHuNlZPPmlZ67LqPAQxpzVnm9XZbN2Zz7PX92NwZ0TCQn59cP9ewuOsj7nAN1SGhJW5hLXjn2H2HeokI6JsXyzcifj52Rxba9ULunaAhGhqLiE+RvzWLhpD5k797N1zyHWZR/gUGExAB0TY7m0ewvO7NCU+OgIEhtE/urymjHGVIYlFC98kVC27ingprcWsD7nAABJDeuxbe8hzjupGeef0oxpy3fy9cqdiMBrN/Rg8EnNfBF6pRQVl7A2+wCz1+/i21U7WZC155d9nZrFMuas9px3cjNCvSQ3Y4wpjyUUL3yRUO58fxHTV2dz99kdqBcewsx1u+iQGMuD53b85Rf1D2tyKCxWzumc6Iuwq23T7oOs2LafnfsP8/68TWzIPUjbhGhuP7MdF3dtTmRYqKvxGWNqB0soXlQ3ocxat4tNeQdpHB3B6PcWcc+g9twzqIMfIvSf4hLl6xU7+ee/17F6Zz5NYiK5Mj2ZS7q2IK15A7fDM8YEsKomFLvLqwJfLd/Bh/M3A9AiLorbTm/rckRVFxoiXNilORec0oxZ63fx1uwsxs7cwCs//MwlXVvwxCUn0Sg6wu0wjTFBwM5QKqCqrM0+wPeZ2fRp05geLRv5Ibqat/vAEd6eu4mXZ6wnNiqMm/q15qqeyTRrEGUD+MaYX9glLy9sLi/vMnfs569fr2bGmlwAGkdH0LlFA7omN+SGPi1pFhflcoTGGDdZQvHCEkrF1mXnM+fn3azcvo9VO/aTuSOf0BAhrVksG3YdpGlsJH3aNObW09rQqkm02+EaY2qIJRQvLKFUzZa8Al75z89k7TpIm4Rotu89zNyfd1NUUsItp7Xh3kEdiAizKWKMCXY2KG9OWEp8ff4y9JT/KcvZf5i/frOGV374mfkb83jhmm4kN6rvUoTGmEBkZyimSr5Yup2HJi7jaFEJF3ZpTofEWBJiIunbtjEp8ZZgjAkmAXOGIiIpwDtAM6AEGKuqL5Spcz3wkPPtAeB2VV0qIh2Bj0tVbQM8pqr/EJHHgVuBXGffI6o61V/9MP/r4q4t6J7akDdnbeTTjK18vmT7L/tOa9+EF67pTrzdhmxMneS3MxQRaQ40V9VFIhILLASGqOqqUnX64Vlzfo+InA88rqq9yxwnFNgG9FbVTU5COaCqz1Y2FjtD8Z8jRcVsySvgm5XZvDB9HQkxkTx8fifO6ZxIVLg9kW9MbRYwZyiqugPY4Wzni0gmkASsKlVnTqkmPwHJXg51NvCzqm7yV6ym+iLDQmnXNJZ2TWMZ0K4JYz5czJgPF9MgKoxLurXgih4pdE2Os+dbjKkDauRWHRFpBXQH5lVQbSQwzUv5NcCHZcruEpFlIjJORILjacMg0DWlITMeOJP3RvbmrE5NmZCxlSEvzWbQc/9hzvpdbodnjPEzvw/Ki0gM8B/gz6o6qZw6A4GXgQGqurtUeQSwHThJVbOdskRgF6DAk3guq93s5ZijgFEAqampPTZtshOcmrb/cCFTl+3gtZkbyDt4lC/HDLCBe2Nqkape8vLrGYqIhAMTgfcrSCZdgDeAS0snE8f5wKJjyQRAVbNVtVhVS4DXgV7ejquqY1U1XVXTExISfNEdU0UNosK5plcq40f0pESVOz9YxMEjRW6HZYzxE78lFPFcNH8Tz6D7c+XUSQUmAcNUda2XKtdS5nKXM9h/zFBghW8iNv7SsnE0f7+yK8u37eO8F2Yy9+eyfzcYY4KBP+/yGgD8CCzHc9swwCNAKoCqvioibwCXA8euRxUdO70SkfrAFqCNqu4rddx3gW54LnllAbc5NwCUy+7yCgwLsvK475MlbMk7RHrLRvz+wjS6p9oQmDGByqZe8cISSuAoOFrER/O38PqPGyg4WsyUu/rTsrHND2ZMIAqoMRRjyqofEcbNA1rz8ai+iMCt72Sw/3Ch22EZY3zAEopxRWrj+rx03alsyD3I0Jdm83PuAbdDMsacIEsoxjX92zXh3ZG92VNQyGUvz2HjroNuh2SMOQGWUIyr+rZtzOQ7+hMiMOqdDA7YbcXG1FqWUIzrfrn8tesgI96ab2cqxtRSllBMQOjXrgl/v7Irq3fkc+4/ZjJ58Ta3QzLGVJElFBMwhnRP4vv7z6B7SkPu+2QJny+xpGJMbWIJxQSUxAZRvDWiJ71ax3Pvx0t4alomhwuL3Q7LGFMJllBMwKkfEca4m3pydc8UXvvPBi57eQ45+w+7HZYx5jgsoZiAVD8ijKcu68Kbw9PJ2n2Qy1+dQ5YN1hsT0CyhmIB2dloiH9zahwOHi7ju9Z/YklfgdkjGmHJYQjEBr1tKQ967pTcHjhRx3Rs/2eUvYwKUJRRTK5zUIo53RvZm94Gj3Pz2AltXxZgAZAnF1BrdUhry0nWnkrkjn7s+WERRccnxGxljaowlFFOrDOzUlP+79CRmrMnlj1NWUheWXzCmtghzOwBjqur63i3ZnFfAa//ZQEp8fUaf0dbtkIwx+HcJ4BQRmSEimSKyUkTu9lLnehFZ5rzmiEjXUvuyRGS5iCwRkYxS5fEi8p2IrHO+2pJ/ddBD53bioi7NeXraar5Yut3tcIwx+PeSVxFwv6qmAX2AO0Wkc5k6G4EzVLUL8CQwtsz+gararcyKYQ8D01W1PTDd+d7UMSEhwrNXdqVnq0bc/8lSfliT43ZIxtR5fksoqrpDVRc52/lAJpBUps4cVd3jfPsTkFyJQ18KvO1svw0M8U3EpraJCg9l7LB02jaNYeTbGUzI2OJ2SMbUaTUyKC8irYDuwLwKqo0EppX6XoFvRWShiIwqVZ6oqjvAk7SApr6N1tQmjaIj+OS2PvRt05gHP13Gl8vs8pcxbvF7QhGRGGAicI+q7i+nzkA8CeWhUsX9VfVU4Hw8l8tOr+L7jhKRDBHJyM3NrWb0pjaIjQrnjeHppLdsxH2fLCUjK8/tkIypk/yaUEQkHE8yeV9VJ5VTpwvwBnCpqu4+Vq6q252vOcBnQC9nV7aINHfaNge8XjxX1bGqmq6q6QkJCb7qkglQUeGhjL0xnaSG9bjprQXM+XmX2yEZU+f48y4vAd4EMlX1uXLqpAKTgGGqurZUebSIxB7bBgYDK5zdU4DhzvZw4HP/9MDUNvHREXx4ax+ax0Vx07gF3Pn+Il6fucEegDSmhvjzOZT+wDBguYgsccoeAVIBVPVV4DGgMfCyJ/9Q5NzRlQh85pSFAR+o6tfOMZ4GPhGRkcBm4Eo/9sHUMs3iopgwui+PT1nJ4i17+Wr5DopV7VkVY2qA1IUnjdPT0zUjI+P4FU1QUVVGv7eQGWtymXb3abRNiHE7JGNqFRFZWOaxjQrZ1CsmaIkIT156MvXCQ7nv4yUcOmorPxrjT5ZQTFBr2iCKZy7vwrJt+7j9/YUU2niKMX5jCcUEvfNObsafh5zCD2ty+fNXmW6HY0zQsoRi6oTreqcyon8rxs/JsluKjfETSyimzvjtuZ1o3SSa3366jN0HjrgdjjFBxxKKqTPqRYTy7JVdyck/wkX/nMWizXuO38gYU2mWUEyd0qNlIybd3o/QEOHKV+fy92/XcKTI7v4yxhcsoZg65+SkOL4acxpDuiXxz3+vZ8wHi23lRxN09hUU8rtJy9lXUFhj72kJxdRJcfXD+ftVXXn4/E58uyqbz5fYLMUmeKzLzufSl2bx6cItNXpp1xKKqdNuPa0Np6Y25I9TVpKTf9jtcIw5YTPW5DD05TkcOFLMh7f2YWCnmlvhwxKKqdNCQ4S/XtGVgqNFPPft2uM3MCaAjZ+9kZHjF5AaX58vxvQnvVV8jb6/JRRT57VrGsMNfVryScYW1mbnux2OMVVWVFzCHyav4PEvVnF2WiITRveleVy9Go/DEooxwJiz2hMdEcYz01a7HYoxVbLvUCEjxi/g3Z82cdsZbXjthh5ER/pzIvnyWUIxBs9aKrcPbMv01Tn8tGH38RsYEwC27T3EFa/MYe7Pu3nm8lP43flphISIa/FYQjHGcXP/1jSPi+KpqZl2G7EJeKt37ueyl2ezc/9h3hnZi6t7prodkiUUY46JCg/lvnM6sHTrPr5avsPtcIwp17wNu7ny1bkATBjdl35tm7gckYc/lwBOEZEZIpIpIitF5G4vda4XkWXOa46IdD1eWxF5XES2icgS53WBv/pg6p7LTk2mU7NY/vj5SrbkFbgdjjG/Mm35DoaNm0/T2Egm3t6PTs0auB3SL/x5hlIE3K+qaUAf4E4R6VymzkbgDFXtAjwJjK1k2+dVtZvzmurHPpg6JjRE+Nd1p1JYXMItb2eQf7jmnjI25nje/WkTd3ywiJNbNODT0f1IblTf7ZD+h98SiqruUNVFznY+kAkklakzR1WPPcb5E5Bc2bbG+Eu7pjG8ckMPfs49wJgPF1Nki3KZAPDSjPX8YfIKzurYlPdv6UOj6Ai3Q/qVGhlDEZFWQHdgXgXVRgLTKtn2Lucy2TgRaeSzQI1x9G/XhCeHnMwPa3L5ky3KZVykqjz7zRr+9s0ahnRrwWvDelAvItTtsLzye0IRkRhgInCPqu4vp85APAnloUq0fQVoC3QDdgB/L+eYo0QkQ0QycnNzfdIXU7dc2yuVWwa0ZvycLN6Zm+V2OKYOUlWe/DKTf81YzzU9U/j7Vd0ICw3ce6n8GpmIhONJCO+r6qRy6nQB3gAuVdXdx2urqtmqWqyqJcDrQC9vx1XVsaqarqrpCQkJvuuUqVN+d0Eag9Ka8sQXq/jPWvvDxNSckhLl95NXMG72Rm7q14qnLjuFUBefMakMf97lJcCbQKaqPldOnVRgEjBMVddWpq2INC/17VBgha9jN+aY0BDhhWu6075pDA9MWGprp5gaUVRcwgMTlvLBvM3ccWZb/nhxZzy/FgObP89Q+gPDgLNK3+IrIqNFZLRT5zGgMfCysz+jorbOvr+KyHIRWQYMBO71Yx+MIToyjN9fmEZu/hG+WmbPpxj/Ki5RHvx0GZMWb+P+czrw2/M61YpkAiB14Yng9PR0zcjIOH5FY8qhqgx+fiYRYSF8OWZArfkPbmqXkhLltxOX8enCrTx4bkfuHNjO1XhEZKGqple2fuCO7hgTQESEEf1bs3L7fhZk2Vr0xvc8YybL+XThVu4+u73ryaQ6LKEYU0lDuyfRqH44L/+w3u1QTJBRVR7/YiUfzt/CnQPbcs+g9m6HVC2WUIyppHoRoYw6vS0/rMmt0WVVTXA7dmvwO3M3Mer0NjwwuGOtvaRqCcWYKrixb0saR0fw/He2uqPxjee+W8u42RsZ0b8Vvzu/9gzAe2MJxZgqiI4MY/QZbflx3S7m/LzL7XBMLffW7I3889/ruTo9hccuqh23BlfEEooxVTSsb0uSG9XjiSmrKLR5vkw1TV68jSe+WMW5JyXy56En1/pkApZQjKmyqPBQ/nBRZ9Zk5/Pu3E1uh2NqoRlrcnhgwlL6tInnhWu6B/R0KlURHL0wpoYN7pzI6R0SeP67tWTvP+x2OKYWWbgpj9vfW0jHZrG8fmM6UeGBOdFjdVhCMaYaRIQnLjmJI8UlPPHFSrfDMbXE2ux8Rry1gGYNohg/ohexUeFuh+RTllCMqabWTaL5zVntmLp8J9Mzs90OxwS4nPzDjHhrAZHhobw7sjcJsZFuh+RzllCMOQGjTm9Lh8QYHvt8JQePFLkdjglQh44Wc8vbGeQdPMq44T1JiQ+slRZ9xRKKMScgIiyEvww9hW17D/GcPZtivCguUe7+aDHLt+3jxWu7c0pynNsh+Y0lFGNOUHqreK7tlcpbszcyf2Oe2+GYAPPU1Ey+XZXNYxd15pzOiW6H41eWUIzxgYfP70TLxtHcPH4BS7bsdTscEyDenZvFG7M8C2SN6N/a7XD8zhKKMT4QVy+cD27tTaPocIaPm8++Q4Vuh2Rc9sOaHP44ZSWD0pryh4s6ux1OjbCEYoyPNI+rx8vX9WDfoUI+XbjV7XCMizbkHmDMh4vp1KwBL1zTPeCX7vUVfy4BnCIiM0QkU0RWisjdXupcLyLLnNccEelaat95IrJGRNaLyMOlyluLyDwRWSciH4tIhL/6YExVnZIcR4+WjXh3bhYlJcG/eJ35tfzDhYx6dyHhoSGMvbEH0ZFhbodUY/x5hlIE3K+qaUAf4E4RKXvetxE4Q1W7AE8CYwFEJBR4CTgf6AxcW6rtM8Dzqtoe2AOM9GMfjKmyG/u2JGt3ATPX5bodiqlhJSXKvR8vYeOug7x03akkNwrO24PL47eEoqo7VHWRs50PZAJJZerMUdVjC0v8BCQ7272A9aq6QVWPAh8Bl4pn9rSzgE+dem8DQ/zVB2Oq4/yTm5MQG8mbsza6HYqpYf+Yvo7vM3P4w4Vp9G3b2O1walyNjKGISCugOzCvgmojgWnOdhKwpdS+rU5ZY2CvqhaVKff2nqNEJENEMnJz7S9FU3MiwkK47fQ2/LhuFzPX2r+9uuLrFTt4cfo6ruyRzPB+rdwOxxWVSigicreINBCPN0VkkYgMrmTbGGAicI+q7i+nzkA8CeWhY0VeqmkF5b8uVB2rqumqmp6QkFCZUI3xmWF9W5IaX5+/TM2k2MZSgt6anfnc98lSuqU05MkhwTEVfXVU9gzlZicZDAYSgBHA08drJCLheJLJ+6o6qZw6XYA3gEtVdbdTvBVIKVUtGdgO7AIaikhYmXJjAkpkWCgPndeJ1Tvz+XjBluM3MLXWgSNF3P7eQqIjw3j1hh5BNXtwVVU2oRxLtxcAb6nqUryfLfy3gSdFvwlkqupz5dRJBSYBw1S19LwVC4D2zh1dEcA1wBRVVWAGcIVTbzjweSX7YEyNuuCUZvRuHc8zX69m94Ejbodj/EBV+d2k5WTtPsg/r+1Os7got0NyVWUTykIR+RZPQvlGRGKB4y1V1x8YBpwlIkuc1wUiMlpERjt1HsMzLvKysz8DwBkjuQv4Bs9g/ieqemyO8IeA+0RkvdP2zUr2wZgaJSL8eejJFBwt4i9TV7sdjvGD9+Zt5oul27l/cEf6tKl7g/BlieeP/uNUEgkBugEbVHWviMQDyaq6zN8B+kJ6erpmZGS4HYapo/72zWpemvEz0+4+jbTmDdwOx/jIim37uOzlOfRr15hxw3sSEoQPL4rIQlVNr2z9yp6h9AXWOMnkBuBRYF91AjSmrrllQBvCQ8Weng8i+w8Xcsf7i2gcE8FzV3ULymRSHZVNKK8ABc6T7L8FNgHv+C0qY4JIo+gIBqUlMnnxNgqLj3el2AQ6VeW3E5axfe8h/nVdd+KjbbKOYyqbUIqcAfFLgRdU9QUg1n9hGRNcruiRzO6DR/lhjT2XUtuNn5PF1yt38tvzOtKjZbzb4QSUyiaUfBH5HZ5B9q+cqVGCazFkY/zo9A4JNImJ5JMMu4W4NsvcsZ+npq7m7E5NufW0Nm6HE3Aqm1CuBo7geR5lJ56n0//mt6iMCTLhoSFc1yuF71Zlk5Fli3DVRocLi7n7o8XE1Q/nr1d0qbMPL1akUgnFSSLvA3EichFwWFVtDMWYKhh9ZltaxEXx6OQVFNlYSq3z9LTVrM0+wN+u6ELjmEi3wwlIlZ165SpgPnAlcBUwT0SuqLiVMaa0+hFhPHZxZ1bvzOftuZvcDsdUwQ9rchg/J4ub+rXizI5N3Q4nYFV2ov7fAz1VNQdARBKA7/nvrL/GmEo496RmnNEhgee/W8tFXZqT2KBuP1ldG+w+cIQHJiyjY2IsD5/fye1wAlplx1BCjiUTx+4qtDXGOESEJy45iaPFJfxlaqbb4ZjjUFUemric/YcK+cc13er0PF2VUdmk8LWIfCMiN4nITcBXwFT/hWVM8GrVJJrRZ7Tl8yXbWbx5z/EbGNd8OH8L32dm89D5nWyWg0qo7KD8g3hWU+wCdAXGqupDFbcyxpTnttPbEBsZxttzstwOxZRj8+4C/vTVKk5r34QRdXR9k6qq9GUrVZ2oqvep6r2q+pk/gzIm2EVHhnF5j2SmLt/JLpuJOOCUlCgPTVxGiAjPXN7FplappAoTiojki8h+L698EfG6WJYxpnJu6JPK0eISe9gxAH0wfzNzN+zm0QvTaNGwntvh1BoVJhRVjVXVBl5esapqFxSNOQHtmsbSr21j3p27iQNHio7fwNSILXkFPDU1k9PaN+HqninHb2B+YXdqGeOie8/pQPb+wzwxZeXxKxu/U1UenrQMEeHpy+1p+KqyhGKMi3q2iueOM9sxYeFWpi7f4XY4dd6H87cwe/1uHrkgjSS71FVlfksoIpIiIjNEJFNEVorI3V7qdBKRuSJyREQeKFXesdQqj0uccZt7nH2Pi8i20qtA+qsPxtSEuwe1p0tyHI99vpL8w4Vuh1Nnbd1TwJ+/WkX/do25tpdd6qoOf56hFAH3q2oa0Ae4U0Q6l6mTB/wGeLZ0oaquUdVuqtoN6AEUAKXvLHv+2H5VtedhTK0WHhrCn4aczO6DR/jH9+vcDqdOOrY2vAJPX2aXuqrLbwlFVXeo6iJnOx/P2vBJZerkqOoCoKI/y84GflZVm/zIBK0uyQ25pmcq4+dksS473+1w6pzPl2znx3W7ePj8TqTE13c7nFqrRsZQRKQV0B2YV43m1wAflim7S0SWicg4EWlUznuOEpEMEcnIzbVFjUzge/DcjkSGhfDqfza4HUqdsq+gkD99tYquKQ25vndLt8Op1fyeUEQkBpgI3KOqVXp2RUQigEuACaWKXwHaAt2AHcDfvbVV1bGqmq6q6QkJCdWK3ZiaFB8dwRU9kvli6XZy8g+7HU6d8fTXq9lTUMhfhp5MqD3AeEL8mlBEJBxPMnlfVSdV4xDnA4tUNftYgapmq2qxqpYArwO9fBOtMe4b0b81hSUlvGfT29eIhZvy+HD+Zkb0a8VJLeLcDqfW8+ddXgK8CWSq6nPVPMy1lLncJSLNS307FFhRzWMbE3BaN4nm7E5NeW/eZg4XFrsdTlArLC7hkUkraBEXxb3ndHA7nKDgzzOU/njWoD+r9C2+IjJaREYDiEgzEdkK3Ac8KiJbRaSBs68+cA5Q9szmryKyXESWAQOBe/3YB2Nq3M0DWpN38CiTF29zO5Sg9uasjazJzufxS04iOrKyS0OZivjtp6iqs4AKL0g6Swsnl7OvAGjspXyYTwI0JkD1bdOYtOYNGDd7I1f3TLFbWP1gS14B//h+Led0TmTwSc3cDido2JPyxgQYEWHkgNaszT7Aj+t2uR1O0FFV/jhlJSHOYmfGdyyhGBOALu7anCYxkYydabcQ+9r0zBz+vTqHewd1sJmEfcwSijEBKDIslNFntGHW+l3MsrMUnzlSVMyTX62iXdMYburfyu1wgo4lFGMC1LC+LUlqWI+npmVSUqJuhxMUxs3KYtPuAh67qDPhofbrz9fsJ2pMgIoMC+WBczuwcvt+pq6wmYhPVM7+w/zr3+sYlJbI6R3sYWd/sIRiTAC7tGsSKfH1+HiBrep4op7+ejWFxcqjF6a5HUrQsoRiTAALCRGGdkti9vpd7Nxn07FU1+LNe5i0aBs3D2hNqybRbocTtCyhGBPghp6aTInC50vsQcfqKClRHp+ykqaxkdx1Vju3wwlqllCMCXCtm0TTPbUhn9mT89UycdFWlm7dx0PndSLGnoj3K0soxtQCl5+azOqd+bw0Yz2qdsdXZR04UsQzX6+hW0pDhnZPOn4Dc0IsXRtE9T2rAAAW50lEQVRTC1zdM4WMrDz+9s0aCo4W8eC5ndwOqVZ4a9ZGdh04wtgbexBiU9P7nSUUY2qB8NAQnruqG2GhIbz8w89c3LUFnZo1cDusgLa34ChjZ25gUFoip6Z6XYfP+Jhd8jKmlggJER69MI2YyDCe/WaN2+EEvNdmbuDA0SLuH2xT09cUSyjG1CIN60cw+oy2fJ+ZQ0ZWntvhBKztew/x1uyNXNylBWnN7UyuplhCMaaWGdG/FU1iInj5h5/dDiVg/fXr1ZQo/Pa8jm6HUqf4c8XGFBGZISKZIrJSRO72UqeTiMwVkSMi8kCZfVnOQlpLRCSjVHm8iHwnIuucr3Zx1NQp9SPCuK53S2asyWHz7gK3wwk4S7bsZfKS7dwyoDXJjeq7HU6d4s8zlCLgflVNA/oAd4pI5zJ18oDfAM+Wc4yBqtpNVdNLlT0MTFfV9sB053tj6pTre6cSKsK7P2W5HUrAeeH7tTSOjuCOgfYQY03zW0JR1R2qusjZzgcygaQydXJUdQFQWIVDXwq87Wy/DQzxQbjG1CqJDaI49+RmfLxgC4eO2trzx6zNzmfGmlxu6tfKHmJ0QY2MoYhIK6A7MK8KzRT4VkQWisioUuWJqroDPEkLaOqrOI2pTW7s05L9h4v4arnNRHzM6zM3UC88lBv6tHQ7lDrJ7wlFRGKAicA9qrq/Ck37q+qpwPl4LpedXsX3HSUiGSKSkZubW5WmxtQKvVrH07pJNBMybCZigOz9h5m8ZBtXpSfTKDrC7XDqJL8mFBEJx5NM3lfVSVVpq6rbna85wGdAL2dXtog0d47fHMgpp/1YVU1X1fSEBFv7wAQfEeGKHsnM25hng/PAP75fB8Atp7VxOZK6y593eQnwJpCpqs9VsW20iMQe2wYGAyuc3VOA4c72cOBz30RsTO1z2alJhAh8urBun6Wsz8nnk4wt3NCnJSnxdmeXW/w5atUfGAYsF5ElTtkjQCqAqr4qIs2ADKABUCIi9wCdgSbAZ56cRBjwgap+7RzjaeATERkJbAau9GMfjAlozePqMaB9Ap8u3MrdgzoQWkfnq3rm6zXUDw9lzFnt3Q6lTvNbQlHVWUCF/7pVdSeQ7GXXfqBrOW12A2efcIDGBIlre6Zw+/uLmJ6ZzeCTmrkdTo1bkJXHd6uyefDcjsTb2Imr7El5Y2q5czon0iIuirfnZrkdSo1TVf4yNZPEBpHc3L+12+HUeZZQjKnlwkJDuL5PS2av3836nHy3w6lR36zcyeLNe7nvnA7Uiwh1O5w6zxKKMUHgmp4pRISFMH5Oltuh1JjiEuXZb9fSrmkMl5/q7cq5qWmWUIwJAo1jIhnSrQWfLtxK3sGjbodTI75ctp31OQe4Z1B7wkLtV1kgsE/BmCAx6vQ2HC4s4Z25WW6H4nfFJcoL09fRMTGWC05u7nY4xmEJxZgg0a5pLGd3asrbc7LYf7gq0+PVPlOWbmND7kHuGdTelvYNIJZQjAkio89sy56CQno8+R23vL0gKCeOLCou4YXv15HWvAHn1sHbpAOZJRRjgkjPVvF8cltfbuzbiu8zc3hh+jq3Q/K5yUu2k7W7wM5OApDN72xMkOnVOp5erePJP1zI6z9u4JKuLejcIjiWwS0qLuHF6es4qUUDBndOdDscU4adoRgTpB65II1G9cP545QVx69cS0xbsZPNeQX85uz2OFMzmQBiCcWYINWwfgRjzmrPgqw9zN+Y53Y4J0xVGTtzA62bRHNOmp2dBCJLKMYEsavSU2gcHcFLM9a7HcoJ+2lDHsu37eOW01rb2EmAsoRiTBCrFxHKzQNa85+1uazYts/tcE7IuNkbaRwdYU/FBzBLKMYEuWF9WxITGcabsza6HUq17TtUyA9rchjSPYmocJuzK1BZQjEmyDWICueKHsl8uWw7OfmH3Q6nWr5blU1hsXJRF3sqPpBZQjGmDrixb0sKi5UP59XOlR2/WradpIb16JbS0O1QTAX8uQRwiojMEJFMEVkpInd7qdNJROaKyBEReaAybUXkcRHZJiJLnNcF/uqDMcGiTUIMZ3ZM4L15mzhaVOJ2OFWyr6CQH9ft4qIuze1W4QDnzzOUIuB+VU0D+gB3ikjnMnXygN8Az1ax7fOq2s15TfVT/MYElZv6tSI3/wjTVuxwO5Qq+XrlDopKlAvtclfA81tCUdUdqrrI2c4HMoGkMnVyVHUBUFjVtsaYqjm9fQJtmkTz1uwst0OpkgkZW2mTEM0pSXFuh2KOo0bGUESkFdAdmOejtneJyDIRGScijXwQojFBLyREuLFvS5Zs2cvizXvcDqdS1ufkk7FpD9f0TLHLXbWA3xOKiMQAE4F7VHW/D9q+ArQFugE7gL+X03aUiGSISEZubm614zcmmFzeI5mYyLBas7Ljxwu2EBYiXGbPntQKfk0oIhKOJyG8r6qTfNFWVbNVtVhVS4DXgV7e2qvqWFVNV9X0hISE6nfCmCASGxXOlenJfLVsB9v3HnI7nAodLSph4qJtDEpLpElMpNvhmErw511eArwJZKrqc75qKyKlR+aGAsEz850xNWDkgNYoBPyDjtMzs8k7eJSre6W4HYqpJH9OX98fGAYsF5ElTtkjQCqAqr4qIs2ADKABUCIi9wCdgS7e2jp3dP1VRLoBCmQBt/mxD8YEneRG9bm4S3M+nL+ZMWe1o2H9CLdD8uqjBVtoHhfF6e3tCkNt4beEoqqzgApH0VR1J+Dt4mi5bVV12IlHZ0zddtsZbZm8ZDvvz9vMnQPbuR3Or2zbe4iZ63IZc1Z7Qm0iyFrDnpQ3pg5Ka96A3q3jmbhoK6rqdji/MiHD80T/lT1sML42sYRiTB01tHsSG3IPsjzAZiEuLlEmZGxlQLsmpMTXdzscUwWWUIypo84/pTkRoSF8tnib26H8j9nrd7Ft7yGu7mmD8bWNJRRj6qi4euGcndaUL5Zup6g4cOb3+njBFhrVD+ccWzO+1rGEYkwdNrR7ErsOHOXDBYExC/HuA0f4dtVOhnZPJjLM1j2pbSyhGFOHDUpL5PQOCTz55Soyd1RpIgu/mLxkO4XFape7ailLKMbUYSEhwnNXdaVhvXDGfLiYkhJ37/iasTqHDokxdGwW62ocpnosoRhTxzWJieT3F6axPucAP67f5VochwuLmZ+Vx4B29iBjbWUJxRjDeSc3o1H9cD6av9m1GBZu2sPRohIGtG/sWgzmxFhCMcYQGRbK5acm892qbHLzj7gSw6z1uwgLEXq3toRSW1lCMcYAcE2vFIpKlImLtrry/rPX7+LU1EZER/pzikHjT5ZQjDEAtGsaS+/W8bw7d1ONP5eyt+Aoy7fto3+7JjX6vsa3LKEYY34xckBrtu09xLQVO2v0fZ+ethpVODutaY2+r/EtSyjGmF8MSkukVeP6vPHjhhqbNHJCxhY+WrCFO85sy8m2bnytZgnFGPOLkBBh5IDWLN26j+e+W8u+gkK/vdfs9bs47x8zefDTZfRqFc9953Tw23uZmmEJxRjzP65MT+Gczon889/rOfPZGWzJK/D5exSXKA9PWsaBI0U8fnFnxo3oSVio/Tqq7fy5BHCKiMwQkUwRWSkid3up00lE5orIERF5oMy+80RkjYisF5GHS5W3FpF5IrJORD4WkcBcbs6YWioqPJTXb0zni7sGUFSs3PvxEp8P0n+fmc2WvEP8/oI0burfmhi7syso+PNPgiLgflVNA/oAd4pI5zJ18oDfAM+WLhSRUOAl4Hw8SwJfW6rtM8Dzqtoe2AOM9F8XjKm7TkmO409DTyZj0x5enL7Op8ceN2sjSQ3r2YzCQcZvCUVVd6jqImc7H8gEksrUyVHVBUDZC7W9gPWqukFVjwIfAZeKiABnAZ869d4GhvirD8bUdZd2S+LKHsm8+O/1fOijp+gzsvKYtzGP4f1a2mWuIFMj55ki0groDsyrZJMkoPR82luB3kBjYK+qFpUqT8IY4zd/uewUdh04wiOfLSc2KoyLurSo9rHWZedz6zsZJDeqx9U9U30YpQkEfk8oIhIDTATuUdXKzo8tXsq0gnJv7zsKGAWQmmr/cI2prvDQEF6+vgc3jpvHvR8v4WhRCWt25hMTGcaYs9tX+jj7Dxcy7M35hIWG8P4tvYmrF+7HqI0b/Hq+KSLheJLJ+6o6qQpNtwKlF0RIBrYDu4CGIhJWpvxXVHWsqqaranpCgs1easyJqBcRyhvDe9K+aSz3fbKU12Zu4Lnv17J976FKH+Olf68nO/8wr9+YTsvG0X6M1rjFn3d5CfAmkKmqz1Wx+QKgvXNHVwRwDTBFPU9azQCucOoNBz73VczGmPLF1Qvn3ZG9ePTCND4a1QdVmLjQ+7xfqsqM1Tls3eO55XjT7oOMm72Ry09NpltKw5oM29Qgf17y6g8MA5aLyBKn7BEgFUBVXxWRZkAG0AAoEZF7gM6qul9E7gK+AUKBcaq60jnGQ8BHIvInYDGepGWMqQGNYyK55bQ2APRr25hPFm7hzoHtCAn536vR01bs5I73FwHQMTGWnPzDhIeG8OC5HWs8ZlNz/JZQVHUW3sc8StfZieeylbd9U4GpXso34LkLzBjjoqvSU7jn4yX8tHE3/dr+d1LHo0UlPPP1ato3jeGiLi1YtHkPJyfFcUm3FiQ2iHIxYuNv9jSRMaZazju5GQ2/COfRz1bw4ag+JDaIIv9wIW/O2sim3QW8NaInAzvaZI91iSUUY0y1RIWH8saN6QwfN58hL82mXngoG3YdBOD0Dgmc2cFuhqlrLKEYY6otvVU874zsxZ++yiQhJpKh3ZM4JTmOPm0a47kvx9QlllCMMSekR8t4Prujv9thmABg8x4YY4zxCUsoxhhjfMISijHGGJ+whGKMMcYnLKEYY4zxCUsoxhhjfMISijHGGJ+whGKMMcYnxDMjfHATkVxgUzWbxwH7TrCet31lyyr6vrztJnjWiKkuf/XNW3lV+3eifasotqrUC9TPrrzYqlLHPrtfl5XX17r62bVU1crPoaOq9qrgBYw90Xre9pUtq+j7CrYzArFvvujfifYt2D+7yvbPPruqlZXXV/vsKveyS17H94UP6nnbV7asou/L2z5R/uqbt/Jg6l8g9K2yx7PPrmpl5fXVPrtKqBOXvIKViGSoarrbcfhDMPcNgrt/wdw3CO7+nWjf7AyldhvrdgB+FMx9g+DuXzD3DYK7fyfUNztDMcYY4xN2hmKMMcYnLKEYY4zxCUsoxhhjfMISSpASkWgRWSgiF7kdi6+JSJqIvCoin4rI7W7H42siMkREXheRz0VksNvx+JKItBGRN0XkU7dj8QXn/9nbzud1vdvx+FpVPy9LKAFGRMaJSI6IrChTfp6IrBGR9SLycCUO9RDwiX+irD5f9E9VM1V1NHAVEFC3b/qof5NV9VbgJuBqP4ZbJT7q2wZVHenfSE9MFft5GfCp83ldUuPBVkNV+lfVz8sSSuAZD5xXukBEQoGXgPOBzsC1ItJZRE4RkS/LvJqKyCBgFZBd08FXwnhOsH9Om0uAWcD0mg3/uMbjg/45HnXaBYrx+K5vgWw8lewnkAxscaoV12CMJ2I8le9flYT5IjrjO6o6U0RalSnuBaxX1Q0AIvIRcKmqPgX86pKWiAwEovH8wzgkIlNVtcSvgVeSL/rnHGcKMEVEvgI+8F/EVeOjz0+Ap4FpqrrIvxFXnq8+u0BXlX4CW/EklSXUkj/Qq9i/VVU5dq34ARiS+O9fQeD5R5xUXmVV/b2q3oPnF+3rgZJMKlCl/onImSLyooi8Bkz1d3A+UKX+AWOAQcAVIjLan4H5QFU/u8Yi8irQXUR+5+/gfKi8fk4CLheRV/D99Cw1yWv/qvp52RlK7SBeyo77RKqqjvd9KH5Rpf6p6g/AD/4Kxg+q2r8XgRf9F45PVbVvu4FAT5LeeO2nqh4ERtR0MH5QXv+q9HnZGUrtsBVIKfV9MrDdpVj8wfpXewVz30oL9n76pH+WUGqHBUB7EWktIhHANcAUl2PyJetf7RXMfSst2Pvpk/5ZQgkwIvIhMBfoKCJbRWSkqhYBdwHfAJnAJ6q60s04q8v6V3v7F8x9Ky3Y++nP/tnkkMYYY3zCzlCMMcb4hCUUY4wxPmEJxRhjjE9YQjHGGOMTllCMMcb4hCUUY4wxPmEJxQQcETlQA+9xyfGmWvfDe54pIv2q0a67iLzhbN8kIv/yfXRVJyKtyk6B7qVOgoh8XVMxGXdZQjFBy5mS2ytVnaKqT/vhPSuaH+9MoMoJBXgE+Ge1AnKZquYCO0Skv9uxGP+zhGICmog8KCILRGSZiDxRqnyyeFakXCkio0qVHxCR/xOReUBfEckSkSdEZJGILBeRTk69X/7SF5HxzuzFc0Rkg4hc4ZSHiMjLznt8KSJTj+0rE+MPIvIXEfkPcLeIXCwi80RksYh8LyKJznTho4F7RWSJiJzm/PU+0enfAm+/dEUkFuiiqku97GspItOdn810EUl1ytuKyE/OMf/P2xmfeFYa/EpElorIChG52inv6fwclorIfBGJdc5EfnR+hou8nWWJSKiI/K3UZ3Vbqd2TgaBbzdB4oar2sldAvYADztfBwFg8M6GGAF8Cpzv74p2v9YAVQGPnewWuKnWsLGCMs30H8IazfRPwL2d7PDDBeY/OeNaFALgCz/T4IUAzYA9whZd4fwBeLvV9I/47C8UtwN+d7ceBB0rV+wAY4GynAplejj0QmFjq+9JxfwEMd7ZvBiY7218C1zrbo4/9PMsc93I8Sxsc+z4OiAA2AD2dsgZ4ZiSvD0Q5Ze2BDGe7FbDC2R4FPOpsRwIZQGvn+yRgudv/ruzl/5dNX28C2WDntdj5PgbPL7SZwG9EZKhTnuKU78azat7EMseZ5HxdiGfJVm8mq2fdmFUikuiUDQAmOOU7RWRGBbF+XGo7GfhYRJrj+SW9sZw2g4DOIr/MHN5ARGJVNb9UneZAbjnt+5bqz7vAX0uVD3G2PwCe9dJ2OfCsiDwDfKmqP4rIKcAOVV0AoKr7wXM2A/xLRLrh+fl28HK8wUCXUmdwcXg+k41ADtCinD6YIGIJxQQyAZ5S1df+p1DkTDy/jPuqaoGI/ABEObsPq2rZpViPOF+LKf/f/JFS21Lma2UcLLX9T+A5VZ3ixPp4OW1C8PThUAXHPcR/+3Y8lZ6YT1XXikgP4ALgKRH5Fs+lKW/HuBfPctJdnZgPe6kjeM4Ev/GyLwpPP0yQszEUE8i+AW4WkRgAEUkSz7rkccAeJ5l0Avr46f1n4VmNL8Q5azmzku3igG3O9vBS5flAbKnvv8UzwysAzhlAWZlAu3LeZw6eacbBM0Yxy9n+Cc8lLUrt/x8i0gIoUNX38JzBnAqsBlqISE+nTqxzk0EcnjOXEmAY4O1mh2+A20Uk3GnbwTmzAc8ZTYV3g5ngYAnFBCxV/RbPJZu5IrIc+BTPL+SvgTARWQY8iecXqD9MxLPw0ArgNWAesK8S7R4HJojIj8CuUuVfAEOPDcoDvwHSnUHsVXhZGU9VVwNxzuB8Wb8BRjg/h2HA3U75PcB9IjIfzyUzbzGfAswXkSXA74E/qepR4GrgnyKyFPgOz9nFy8BwEfkJT3I46OV4b+BZf3yRcyvxa/z3bHAg8JWXNibI2PT1xlRARGJU9YCINAbmA/1VdWcNx3AvkK+qb1Syfn3gkKqqiFyDZ4D+Ur8GWXE8M4FLVXWPWzGYmmFjKMZU7EsRaYhncP3Jmk4mjleAK6tQvweeQXQB9uK5A8wVIpKAZzzJkkkdYGcoxhhjfMLGUIwxxviEJRRjjDE+YQnFGGOMT1hCMcYY4xOWUIwxxviEJRRjjDE+8f+k05bJ85K2oAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.sched.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = 2e-2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9b5041136647432ea2d484146e8715aa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                   \n",
      "    0      1.783729   1.671426   0.4073    \n",
      "    1      1.690334   1.572153   0.4398                     \n",
      "\n",
      "CPU times: user 2min 29s, sys: 23.7 s, total: 2min 53s\n",
      "Wall time: 34.3 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([1.57215]), 0.4398]"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time learn.fit(lr, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fdf82e1d7adc4949922f80981eda1f7d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                   \n",
      "    0      1.590282   1.485381   0.4773    \n",
      "    1      1.569007   1.466308   0.4848                     \n",
      "\n",
      "CPU times: user 2min 26s, sys: 24.3 s, total: 2min 50s\n",
      "Wall time: 37.2 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([1.46631]), 0.4848]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time learn.fit(lr, 2, cycle_len=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ConvNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConvNet(nn.Module):\n",
    "    def __init__(self, layers, n_out):\n",
    "        super().__init__()\n",
    "        self.layers = nn.ModuleList([nn.Conv2d(in_channels=layers[i], out_channels=layers[i + 1], kernel_size=3, stride=2) for i in range(len(layers) - 1)])\n",
    "        self.pool = nn.AdaptiveMaxPool2d(output_size=1)\n",
    "        self.out = nn.Linear(layers[-1], n_out)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        for l in self.layers:\n",
    "            x = F.relu(l(x))\n",
    "        x = self.pool(x)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        return F.log_softmax(self.out(x), dim=-1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**AdaptiveMaxPool means that the output size is fixed while the input size can vary. One by one adaptive max pool usually used in the penultimate layer of fully convolutional architectures.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = ConvLearner.from_model_data(ConvNet([3, 20, 40, 80], 10), data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OrderedDict([('Conv2d-1',\n",
       "              OrderedDict([('input_shape', [-1, 3, 32, 32]),\n",
       "                           ('output_shape', [-1, 20, 15, 15]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 560)])),\n",
       "             ('Conv2d-2',\n",
       "              OrderedDict([('input_shape', [-1, 20, 15, 15]),\n",
       "                           ('output_shape', [-1, 40, 7, 7]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 7240)])),\n",
       "             ('Conv2d-3',\n",
       "              OrderedDict([('input_shape', [-1, 40, 7, 7]),\n",
       "                           ('output_shape', [-1, 80, 3, 3]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 28880)])),\n",
       "             ('AdaptiveMaxPool2d-4',\n",
       "              OrderedDict([('input_shape', [-1, 80, 3, 3]),\n",
       "                           ('output_shape', [-1, 80, 1, 1]),\n",
       "                           ('nb_params', 0)])),\n",
       "             ('Linear-5',\n",
       "              OrderedDict([('input_shape', [-1, 80]),\n",
       "                           ('output_shape', [-1, 10]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 810)]))])"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c47d3c79ae4b4a5382665a8254a9047b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 80%|███████▉  | 156/196 [00:12<00:03, 12.39it/s, loss=115] "
     ]
    }
   ],
   "source": [
    "learn.lr_find(end_lr=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEOCAYAAACaQSCZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xd4XOWZ///3rW71NpZsWbLcC8YFBDYYbAglLAEMKUCWkEbCkgps+iZXfsmSbDb1l4QsmxBI25BKSQIkQAi92CAX3HsvsmTJliXLklXu7x8zJsJI9sjW6IxGn9d1zeUz5zxn5p6HYT465znF3B0REZETSQq6ABERGRwUGCIiEhUFhoiIREWBISIiUVFgiIhIVBQYIiISFQWGiIhERYEhIiJRUWCIiEhUFBgiIhKVlKAL6E/FxcVeWVkZdBkiIoPG4sWL97l7KJq2CRUYlZWVVFdXB12GiMigYWbbom2rXVIiIhIVBYaIiERFgSEiIlFRYIiISFQUGCIiEhUFhoiIREWBATy+qoZNdc3odrUiIr1LqPMwTkZreyef+O1SjnR0UZqbwQWTQlw2rZRzxxWTlhLOU3enrrmNUHY6ZhZwxSIiwRjygZGeksQTt83jxU37eHHjPh5ZvoffvbqDgsxUrp5VxrhQNr9euI21NU0UZaVx9phCrq0qZ/7EEPsOtfH7V3Ywfng2/3L6iKA/ypDT2eVsrmtm5e5GOjqdSaU5lOUPo9Odw0c62dPYSn3zEUI56YwqGEZJbgbJSQp8kZNlibQbpqqqyk/1TO/W9k5e2LCPh5bu4onVNbR3OpNLc7hyxkg21TXzwoZ91Da1UVGYSU1jK0c6uwD40HljuOn8MTyxai/r9zYRykmnNDeDktwMhuemk52eQk5GKoVZaf3xUROau/PKlgZe2LiPrfUt1De3MTJ/GCPzh1Hf3Mb2hhZ27j/Mzv0ttHdG//1NTTZG5g9jVMEwRuVnMqEkm1kV+Zw2Mo+M1OQ3tK1vbmPp9gM8va6W5zfsoyArjTljCplYkkNhVhoj8jMYF8omNTkJd6ejy0lN1h5eGXzMbLG7V0XVVoHRu4ZDR9h94DCnjcx9fVdUe2cXjy7fwx+qdzA2lMV7z6nkN4u284uXtr6+Xm5GCgdbO3p8zXedOYqvX3P667u7BoOOzi5e2drA0u0H2F7fQkt7J289rYSLp5S86Ye2N02t7azf28SWfS10dnXR5XCorYODrR2s3XOQ5TsbcZzKoixqDrayrb6F5CRjZH4GRVnp7D5wmNqmNvIzU6kozKS8IJNRhcOYMDyH08vySE021u9toqaxleTkJDJSkhiRN4zCrDTqmtvYuf9oyBxmR0MLO/e3sK/5yOv1jcjLYHhuBm3tnRxoaafmYCsAWWnJnDOumMbDR1i248AbAiotJYmS3HT2NR3hcHsnhVlpjCoYxgUTQ1w1cySjCjKB8FasdmVKvFJgBOCxleGB80unljChJIcjHV3UNbdR09hKXVMrh9o6Wbm7kZ+/uJXzxhfz3WtnUJKbAYS3ato6usgblvqm191U10xZ/rCof5i7upylO/azrb6FvQfbGFUwjLMqCynJTaeto4ud+w+zZNt+lmwPP7bWtzC6MJNJpTnMmxDioinDaTzczms7D7C2polNtc28unU/jYfbAQjlpOMO+5rbyEhNIjUpifauLkbmD2PC8GxajnSyqbYZM2NsKIv0lGTW7T3IjobDvdZcWZTJzPJ8UpKT2LLvEJlpyVwzq4zLppWSmfbPvabtnV39+ld8bVMry7YfYM2eJrY1HKKuqY2M1GRy0lOYMiKX6aPymFmRT3pKuO9b2zupaWyloeUIOxpaWLX7IDWNrYRy0snJSKG2qY2Ne5t5dVsD3f+3KspKY1ZFPqOLskhOMoalJlNemMnookwqCjMZnqOxMQmOAiOO3b94J59/YDkdXU5pbgZpKUns2N+CO1QUZnLm6AIumVrC2FAW33psHU+traUwK40b54zmbdNHMC6U3eN++Oa2Du5buI37Fm1ne0PLCevIG5bKrIp8xhZns72hhZW7Gl//q/qo1GRjTHEWp5flc8nU4cwdX0xORiqdXc7Lm+p5el0t7pBksGN/Cxtrm8lKT2FscRYAG+uaaWvvYlJpDlNG5DK5NIdxoWzSUpIwg8y0FLLSkklJsF05ew+28sTqvRyMhOyWfYdYsn0/NY2tuENrR+cbAiUzLZm544u5dGoJM8rzqSjMjPoPBJFTpcCIc+tqmnh+Qx0rdzXS3uWMj/yIrtjZyCtbG2g4FN5VkpWWzIfOH8uq3Y08uaYWgGGpyUwoyaaiMJNxoWxOL8ujoeUI33l8HbVNbZw9ppAbZlcwfVQ+xdlpbNl36PUthPSUJELZ6ZwxOhwUSd2Cx91Ztfsgz66vozg7jZnlBYwLZSXcj3k8ONLRxa4Dh9ne0ML2+kOs29vEP9bUsqfxn4GdNyyV3GEpjMrPZGZFPjPL85lVns/wyFapSH9RYAxinV3Oq1sbWLGzkatmjnx9t9W2+kNUb93Pil2NbKprZntDCzsaWuiK/OebWZ7Pl6+cyhkVBQFWLyfL3Vmzp4kNtU1siwz0Hzjczua6Q6zZc5COyH/osvxhzCwPB8jMinxOL3vzgL1IXygwhoiWIx2s3n2QliOdnDe++A1bDJI4Wts7WbW7kaXbD7B0xwGWbT/ArgPhMaGUJGPyiJxIiBQwqyKfMUVZ+i5I1BQYIgnu6ID9sh3hx/KdjTS3hY/My81IYUZ5PmdVFjJvYojTy/J0/on0SoEhMsR0djmb6ppZun0/y3YcYOn2A6zb24Q7FGSmcv6EEPMnhpg3MUQoJz3ociWOxEVgmFk58CugFOgC7nb3HxzTZgFwR2R5B3Cbu78QWfY+4EuRpl9z91+e6D0VGCL/VN/cxgsb9/Hsujqe21D3+nknU0bkMmF4NmUFwzh/QjFzxhRpF9YQFi+BMQIY4e5LzCwHWAxc7e6ru7XJBg65u5vZdOAP7j7ZzAqBaqAK8Mi6Z7r7/uO9pwJDpGddXc7qPeGj4F7atI8dDYfZ03iY9k6nojCTq2eO5K3TSpk6IlfnhAwxfQmMmF1Lyt33AHsi001mtgYoA1Z3a9PcbZUswuEA8Fbg7+7eAGBmfwcuA34bq3pFEllSkjGtLI9pZXl87MLxQHgw/bGVNfyhegc/enojP3xqI5NLc/jUpZO4eMpwBYe8yYBcfNDMKoFZwKIell0DfAMYDrwtMrsM2NGt2c7IPBHpJxmpyVw9q4yrZ5Wxr7mNJ1bt5afPb+bDv6rmrMoCvnrVNKaOzA26TIkjMT8rK7Lb6QHC4xMHj13u7g+5+2TgasLjGQA9/WnT474zM7vZzKrNrLqurq6/yhYZUoqz0/nX2RU8cfs8vn7NNDbVHeLKH73AV/6yimU7DtDZlTgHx8jJi+lRUmaWCjwCPO7u34ui/RbgLOAS4AJ3/7fI/J8Az7j7cXdJaQxDpH8caDnCNx9by+9e3YE75GemcsPsCj503lgKdMXlhBIvg94G/BJocPfbemkzHtgUGfQ+A3gYGAUUEB7oPiPSdAnhQe+G472nAkOkf9U3t/Hipnr+unwPj6+uITM1mUumlnDBpOFcPLWE7PQhf0udQS8uBr2BucCNwAozWxaZ9x9ABYC7/xh4B/BeM2sHDgPXeTjBGszsDuDVyHr/eaKwEJH+V5SdzlUzRnLVjJGs39vET5/bzD/W1vKnZbspzk7nPy6fzDWzyjRAPkToxD0R6ZOuyPXO/utva3ltxwGqRhfw1QWncdrIvKBLk5PQly0MXYpURPokKcmYPbaIhz5yLt98x+ls3neIK+8MD5AfPtIZdHkSQwoMETkpSUnGdWdV8PSnLuCG2aP5xUtbueLO51m5qzHo0iRGFBgickryMlO54+pp3Peh2Rxq6+Tq/3mRu57ZqENxE5ACQ0T6xdzxxTx22/lceloJ33psHe/+6cLXL8MuiUGBISL9Jj8zjf/51zP4zrtmsGpXI5d9/zn+vGxX0GVJP1FgiEi/MjPeeeYo/nbrPCaW5HDr75bxhQdXcKSjK+jS5BQpMEQkJiqKMvn9zXP4yAXj+O0r23nPvYtev1+9DE4KDBGJmZTkJD532WR+cP1Mlu04wPV3v0x9c1vQZclJUmCISMwtmFnGLz5wFtsbWrjhHm1pDFYKDBEZEOeOK+ae957FlsiJfo+trCGRrjQxFCgwRGTAnDehmN98eA7Z6Snc8uvFfOTXSzQYPogoMERkQJ05uoBHPnken71sEo+tquHTf3yNLp3kNyjo2sQiMuBSk5P46AXhW8V+67F1FGen8+UrpwZclZyIAkNEAvOR+eOoPdjGz17cwozyPBbM1J2Y45l2SYlIYMyML71tClWjC/jiQyvZVn8o6JLkOBQYIhKolOQkvn/9TJIMPvHbpbR16BLp8UqBISKBG1WQybfeOZ3lOxv59B+XaxA8TikwRCQuXDZtBJ+7bDIPv7abr/91TdDlSA806C0iceOW+WPZe7CVe1/Ywmkjc3n7GaOCLkm60RaGiMQNM+PLV0xlZnk+3/jbWppa24MuSbpRYIhIXElKMr561WnUNbXxo6c2Bl2OdKPAEJG4M6M8n2urRvGzF7ewqa456HIkQoEhInHpM2+dTHpKMnf+Y0PQpUiEAkNE4lIoJ513VY3i0RV7qD3YGnQ5ggJDROLY+86ppKPL+fXCbUGXIigwRCSOVRZn8ZZJw7lv0XadAR4HFBgiEtc+MHcM9YeO8PBre4IuZchTYIhIXJs7vojJpTl8/8n1HGrrCLqcIU2BISJxzcz42tXT2HXgMN96bG3Q5QxpCgwRiXtVlYW875xKfvnyNl7Z0hB0OUOWAkNEBoXPXjaJ8sJhfP7B5bR36j7gQVBgiMigkJmWwv93xWlsrjvE717ZHnQ5Q5ICQ0QGjYumDGfO2EK+/+QGXZgwAAoMERk0zIwvXj6V+kNH+PGzm4IuZ8hRYIjIoHL6qDwWzBzJvS9s4aC2MgZUzALDzMrN7GkzW2Nmq8zs1h7a3GBmyyOPl8xsRrdlt0fWW2lmvzWzjFjVKiKDy/vPraS1vYvHVtQEXcqQEsstjA7gU+4+BZgDfMzMph7TZgsw392nA3cAdwOYWRnwSaDK3acBycD1MaxVRAaRmeX5jCnO4qGlu4IuZUiJWWC4+x53XxKZbgLWAGXHtHnJ3fdHni4Eut+PMQUYZmYpQCawO1a1isjgYmZcPbOMhVvq2X3gcNDlDBkDMoZhZpXALGDRcZrdBPwNwN13Ad8BtgN7gEZ3fyK2VYrIYHLNrDLc4c/L9LfkQIl5YJhZNvAAcJu7H+ylzYWEA+NzkecFwAJgDDASyDKz9/Sy7s1mVm1m1XV1dbH4CCIShyqKMjlzdAEPLd2JuwddzpAQ08Aws1TCYXGfuz/YS5vpwD3AAnevj8y+GNji7nXu3g48CJzb0/rufre7V7l7VSgU6v8PISJx65pZZazf28xrOxuDLmVIiOVRUgbcC6xx9+/10qaCcBjc6O7ruy3aDswxs8zI61xEeAxEROR1V88qIyc9hZ+/uCXoUoaEWG5hzAVuBN5iZssij8vN7BYzuyXS5stAEXBXZHk1gLsvAu4HlgArInXeHcNaRWQQyk5P4dqzynl0+R5qGnUb11izRNr3V1VV5dXV1UGXISIDaEdDC/O//TQfuWAcn3nr5KDLGXTMbLG7V0XTVmd6i8igVl6YyaVTS/nNou0cPqLbuMaSAkNEBr33z61kf0s7j63SbVxjSYEhIoPe2ZWFlOSm8/jKvUGXktAUGCIy6CUlGZdMLeHZ9XW0tmu3VKwoMEQkIbz1tFIOt3fy/IZ9QZeSsBQYIpIQZo8pIicjhcdX6Qq2saLAEJGEkJaSxEWTh/OPNXvp0D2/Y0KBISIJ49LTStnf0s6rW/efuLH0mQJDRBLG/IkhUpONZ9bVBl1KQlJgiEjCyEpPYWZ5Pgu3NARdSkJSYIhIQpkztoiVuxppbusIupSEo8AQkYQye0wRnV1O9VZtZfQ3BYaIJJQzRueTmmws3KzA6G8KDBFJKJlpKcwYlc/CzfUnbix9osAQkYQzZ2wRKzSO0e8UGCKScOaMDY9jLN6m8zH6kwJDRBLOGaPzSUkyXt6k3VL9SYEhIgknMy2FM0cX6AS+fqbAEJGEdMnUEtbWNLGjoSXoUhKGAkNEEtJFU0oAeHKNbqrUXxQYIpKQxhRnMX54tgKjHykwRCRhXTylhEWbG2g83B50KQlBgSEiCeuSqcPp6HKeXV8XdCkJQYEhIglrZnkBRVlp/H21dkv1BwWGiCSs5CRj/sQQL23ch7sHXc6gp8AQkYQ2e2wh9YeOsLG2OehSBj0FhogktNljigB0U6V+oMAQkYQ2uiiTktx0FunqtacsqsAws1vNLNfC7jWzJWZ2aayLExE5VWbG7DFFvLKlQeMYpyjaLYwPuvtB4FIgBHwA+O+YVSUi0o9mjy2ktqmNrfW6TMipiDYwLPLv5cDP3f21bvNEROLa7DGFANotdYqiDYzFZvYE4cB43MxygK7YlSUi0n/GhbIpzk5jkQa+T0lKlO1uAmYCm929xcwKCe+WEhGJe2bG2WMKWbi5HnfHTDtITka0WxjnAOvc/YCZvQf4EtAYu7JERPrXueOK2dPYyqa6Q0GXMmhFGxj/C7SY2Qzgs8A24FfHW8HMys3saTNbY2arzOzWHtrcYGbLI4+XIq9/dFm+md1vZmsjr3FOHz6XiMgbzJ8YAuA5XVfqpEUbGB0ePh5tAfADd/8BkHOidYBPufsUYA7wMTObekybLcB8d58O3AHc3W3ZD4DH3H0yMANYE2WtIiJvUl6YydjiLJ7boMA4WdEGRpOZfQG4EXjUzJKB1OOt4O573H1JZLqJ8A9+2TFtXnL3o3dpXwiMAjCzXGAecG+k3RF3PxBlrSIiPZo3McTCzfW0tncGXcqgFG1gXAe0ET4fo4bwD/+3o30TM6sEZgGLjtPsJuBvkemxQB3wczNbamb3mFlWtO8nItKT+RNDtLZ38epWHS11MqIKjEhI3AfkmdkVQKu7H3cM4ygzywYeAG6LnPzXU5sLCQfG5yKzUoAzgP9191nAIeDzvax7s5lVm1l1XZ02NUWkd7PHFpKWkqRxjJMU7aVBrgVeAd4FXAssMrN3RrFeKuGwuM/dH+ylzXTgHmCBux89q2YnsNPdj26R3E84QN7E3e929yp3rwqFQtF8HBEZojLTUji7sjDqGyr9edkuvvHXNXR16ZIiEP15GF8EznL3WgAzCwFPEv4h75GFD3S+F1jj7t/rpU0F8CBwo7uvPzrf3WvMbIeZTXL3dcBFwOooaxUR6dUFk0J87dE1vLq1gbMqC3tt98y6Wm7//TK6HIqy07h53rgBrDI+RTuGkXQ0LCLqo1h3LuFB8reY2bLI43Izu8XMbom0+TJQBNwVWV7dbf1PAPeZ2XLCJw3+V5S1ioj06vqzKxhdlMltv1vW672+19U08fHfLGVyaS6XTC3hm4+tY/E2jXtYNFdvNLNvA9OB30ZmXQcsd/fP9b7WwKuqqvLq6uoTNxSRIW3p9v2888cvc/npI/jh9TPfcOZ39dYGbvn1YpLM+PPH55KVnsIVP3yB9s4uHvroXErzMgKsvP+Z2WJ3r4qmbbSD3p8hfI7EdMLnRNwdb2EhIhKtWRUF3H7xBB5+bfcbri/14JKdvPunC8lOT+E3H57DiLxh5GakctcNZ3DwcDs33ruI/YeOBFh5sKK+gZK7P+Du/+7ut7v7Q7EsSkQk1j543hjSkpN4am14b3tzWweff3AFs8oL+NPH5jJ+ePbrbaeV5fHT91WxraGF9//8FbYP0cukHzcwzKzJzA728Ggysx4PkRURGQwy01I4e0whz64LHzH13Po6jnR08e+XTiQ/M+1N7c8dV8z//OsZrK1p4i3ffYYvPrSC5raOgS47UMcNDHfPcffcHh457p47UEWKiMTC/Ikh1u1tYveBwzyxqoaCzFSqRhf02v6SqSU8+5kLeffZFfz2le384Mn1vbZNRLqnt4gMWfMnhc/d+sfaWv6xtpaLppSQknz8n8XSvAzuuHoaC2aW8euF29nX3DYQpcYFBYaIDFkThmczIi+DHz21gabWDi6dWhL1uh9/y3haOzr56fObY1hhfFFgiMiQZWbMnxhi78E2MlKTOH9C9FeLGBfK5srpI/m/l7fRMESOnFJgiMiQdvQ+GfMmhBiWltyndT950XgOt3dy51MbYlFa3FFgiMiQdt6EYsYUZ3HdWeV9Xnf88BzefXYFv3xpK6/tSPw7MCgwRGRIy8lI5elPX8BFU6Ifv+ju8/8ymeLsdD7/4AraO7v6ubr4osAQETkFuRmp3HH1NNbsOci9L2wJupyYUmCIiJyit55WygWTQtzz/GbaOhL3bn4KDBGRfnDTeWPY13yER5fvCbqUmFFgiIj0g/PGFzMulMUvX9oadCkxo8AQEekHZsb7zq3ktZ2NLN2+P+hyYkKBISLST95+xiiy01MSditDgSEi0k+y01O4auZInli9NyEHvxUYIiL96C2ThtNypJPFWxNvt5QCQ0SkH50zrojUZOPZ9XVBl9LvFBgiIv0oKz2FsyoLFRgiInJi8yeGWFvTRE1ja9Cl9CsFhohIPzt6Y6bnEmwrQ4EhItLPJpXkUJKbnnC7pRQYIiL97OiNmZ7bUMeRjsS5gq0CQ0QkBi6bVkpTawdPr6sNupR+o8AQEYmBeRNCFGen88DinUGX0m8UGCIiMZCSnMTVM0fy9LrahLnntwJDRCRG3nHmKNo7nYdf2x10Kf1CgSEiEiNTRuQydUQuDyxJjN1SCgwRkRh6x5mjWL6zkbU1B4Mu5ZQpMEREYujts8pIT0niVy9vC7qUU6bAEBGJoYKsNBbMHMlDS3bReLg96HJOiQJDRCTG3ntOJYfbO7l/kB9iq8AQEYmxaWV5VI0u4P9e3kpXlwddzkmLWWCYWbmZPW1ma8xslZnd2kObG8xseeTxkpnNOGZ5spktNbNHYlWniMhAeO+5lWytb+G5DYP3+lKx3MLoAD7l7lOAOcDHzGzqMW22APPdfTpwB3D3MctvBdbEsEYRkQFx2WmlFGal8YfqHUGXctJiFhjuvsfdl0Smmwj/8Jcd0+Yldz96H8OFwKijy8xsFPA24J5Y1SgiMlDSUpK4ZlYZf1+9l/rmtqDLOSkDMoZhZpXALGDRcZrdBPyt2/PvA58FEudSjyIypF1bVU57p/PQ0l1Bl3JSYh4YZpYNPADc5u49nrliZhcSDozPRZ5fAdS6++IoXv9mM6s2s+q6usG7b1BEEt+k0hxmlOfzh+oduA++we+YBoaZpRIOi/vc/cFe2kwnvNtpgbvXR2bPBa4ys63A74C3mNmve1rf3e929yp3rwqFQv3+GURE+tN1VeWs39vMazsbgy6lz2J5lJQB9wJr3P17vbSpAB4EbnT39Ufnu/sX3H2Uu1cC1wNPuft7YlWriMhAuXLGCDLTkvnZC1uCLqXPYrmFMRe4kfDWwbLI43Izu8XMbom0+TJQBNwVWV4dw3pERAKXk5HKjXNG88jy3Wyqaw66nD6xwbgfrTdVVVVeXa3MEZH4VtfUxvnfeorLp43ge9fNDLQWM1vs7lXRtNWZ3iIiAyyUk84Ns0fzp2W72LrvUNDlRE2BISISgH+bN5bU5CTueWFz0KVETYEhIhKA4bkZXDhpOE+urh00h9gqMEREAnLh5BA1B1tZW9MUdClRUWCIiARk/sThADyzbnCcdKzAEBEJSGleBlNG5PLMutqgS4mKAkNEJEAXTApRvW0/B1vj/258CgwRkQBdOGk4nV3Oixv2BV3KCSkwREQCdEZFPjkZKYNiHEOBISISoJTkJM6fUMwz6+P/8FoFhohIwOZNCLH3YBsbauP72lIKDBGRgM2bGL41w3Pr43u3lAJDRCRgI/OHMX54Ns8qMERE5ETmTQjxypYGWts7gy6lVwoMEZE4cP7EYto6uli0pSHoUnqlwBARiQNzxhSRlpLE83G8W0qBISISB4alJXN2ZSHPbVBgiIjICcybWMz6vc3saTwcdCk9UmCIiMSJo4fXPr8+Pi8TosAQEYkTk0pyKMlN59k43S2lwBARiRNmxvkTQry4cR+dXfF3mRAFhohIHDl/QjEHWtpZsasx6FLeRIEhIhJHzp8Qwiw+LxOiwBARiSOFWWmcXpanwBARkRObNyHE0h0HaIqzu/ApMERE4syZlQV0djlr9jQFXcobKDBEROLM5NIcANbVHAy4kjdSYIiIxJnS3AzyhqWypkZbGCIichxmxqTSHNYpMERE5ESmRAIjnu7zrcAQEYlDk0pzaW7rYOf++LkQoQJDRCQOTR4RHvheG0e7pRQYIiJxaGJJ/B0ppcAQEYlD2ekpVBRmxtWRUgoMEZE4FW9HSsUsMMys3MyeNrM1ZrbKzG7toc0NZrY88njJzGZEu66ISKKbXJrDln2HaG3vDLoUILZbGB3Ap9x9CjAH+JiZTT2mzRZgvrtPB+4A7u7DuiIiCW1yaW7kEiHxMY4Rs8Bw9z3uviQy3QSsAcqOafOSu++PPF0IjIp2XRGRRDd7bCE5GSnc8chqOjq7gi5nYMYwzKwSmAUsOk6zm4C/neS6IiIJpzg7na9fczpLth/gzqc2Bl1O7APDzLKBB4Db3L3H7Sozu5BwYHzuJNa92cyqzay6ri7+rh8vInIqrpoxkrfPKuPOpzbw6taGQGuJaWCYWSrhH/z73P3BXtpMB+4BFrh7fV/WBXD3u929yt2rQqFQ/34AEZE48NUFp1FRmMlH71tC7cHWwOqI5VFSBtwLrHH37/XSpgJ4ELjR3df3ZV0RkaEiJyOVn9xYRXNrBx+5bwlHOoIZz4jlFsZc4EbgLWa2LPK43MxuMbNbIm2+DBQBd0WWVx9v3RjWKiIS1yaV5vDtd01n8bb9/P9Prj/xCjGQEqsXdvcXADtBmw8BHzqZdUVEhporpo/kqbW1/OyFLbz3nNGMyBs2oO+vM71FRAaR2y+eiDt8/+8bBvy9FRgiIoNIeWEmN8yp4I+Ld7CxtnlA31uBISIyyHz8wvEMS03m9t8vY/2/R5K8AAAI10lEQVTegbvWlAJDRGSQKcpO59vvmsGO/S38yw+e545HVg/I9aZiNugtIiKxc/npIzhnbBHfenwdi7ftJzU59n//KzBERAapgqw0vvH202nr6CQ5KfYHlmqXlIjIIJeekjwg76PAEBGRqCgwREQkKgoMERGJigJDRESiosAQEZGoKDBERCQqCgwREYmKuXvQNfQbM6sDtp3EqnlAYwzW6a1NX+YfO6/782OXFQP7TlBTX/W1b06lX4637Hj9cKLn8dAv0a7T177p63fm2Ofx0Df6zpzaOqf6nRkNfNDdHz5hNe4+5B/A3bFYp7c2fZl/7Lzuz3tYVh1035xKv/Slb/ryPB76JVZ909fvTDz2jb4z8fed6e2hXVJhJ07Wk1untzZ9mX/svIePsywW+voep9Ivx1t2vH6I5nl/G+jvTG/L+vqdibaOU6HvTM8G03emRwm1S2qoM7Nqd68Kuo54o37pnfqmZ+qXnmkLI7HcHXQBcUr90jv1Tc/ULz3QFoaIiERFWxgiIhIVBYaIiERFgSEiIlFRYAwRZpZlZovN7Iqga4knZjbFzH5sZveb2UeCrieemNnVZvZTM/uzmV0adD3xwszGmtm9ZnZ/0LUMNAVGnDOzn5lZrZmtPGb+ZWa2zsw2mtnno3ipzwF/iE2VweiPvnH3Ne5+C3AtkDCHUfZT3/zJ3T8MvB+4LoblDph+6pfN7n5TbCuNTzpKKs6Z2TygGfiVu0+LzEsG1gOXADuBV4F3A8nAN455iQ8C0wlf6iAD2OfujwxM9bHVH33j7rVmdhXweeBH7v6bgao/lvqrbyLrfRe4z92XDFD5MdPP/XK/u79zoGqPBylBFyDH5+7PmVnlMbPPBja6+2YAM/sdsMDdvwG8aZeTmV0IZAFTgcNm9ld374pp4QOgP/om8jp/Af5iZo8CCREY/fS9MeC/gb8lQlhA/31nhioFxuBUBuzo9nwnMLu3xu7+RQAzez/hLYxBHxbH0ae+MbMLgLcD6cBfY1pZ8PrUN8AngIuBPDMb7+4/jmVxAerrd6YI+Dowy8y+EAmWIUGBMThZD/NOuG/R3X/R/6XEnT71jbs/AzwTq2LiTF/75ofAD2NXTtzoa7/UA7fErpz4pUHvwWknUN7t+Shgd0C1xBv1Te/UNz1Tv0RJgTE4vQpMMLMxZpYGXA/8JeCa4oX6pnfqm56pX6KkwIhzZvZb4GVgkpntNLOb3L0D+DjwOLAG+IO7rwqyziCob3qnvumZ+uXU6LBaERGJirYwREQkKgoMERGJigJDRESiosAQEZGoKDBERCQqCgwREYmKAkMCY2bNA/AeV0V5+ff+fM8LzOzck1hvlpndE5l+v5n9qP+r6zszqzz2cuA9tAmZ2WMDVZMEQ4Ehg17k8tQ9cve/uPt/x+A9j3cdtguAPgcG8B/AnSdVUMDcvQ7YY2Zzg65FYkeBIXHBzD5jZq+a2XIz+2q3+X+K3ClwlZnd3G1+s5n9p5ktAs4xs61m9lUzW2JmK8xscqTd63+pm9kvzOyHZvaSmW02s3dG5ieZ2V2R93jEzP56dNkxNT5jZv9lZs8Ct5rZlWa2yMyWmtmTZlYSuXT2LcDtZrbMzM6P/PX9QOTzvdrTj6qZ5QDT3f21HpaNNrN/RPrmH2ZWEZk/zswWRl7zP3vaYrPwnRYfNbPXzGylmV0XmX9WpB9eM7NXzCwnsiXxfKQPl/S0lWRmyWb27W7/rf6t2+I/ATf0+B9YEoO766FHIA+gOfLvpcDdhK8amgQ8AsyLLCuM/DsMWAkURZ47cG2319oKfCIy/VHgnsj0+wnfGAngF8AfI+8xlfA9EADeSfjS5klAKbAfeGcP9T4D3NXteQH/vFrCh4DvRqa/Any6W7vfAOdFpiuANT289oXAA92ed6/7YeB9kekPAn+KTD8CvDsyfcvR/jzmdd8B/LTb8zwgDdgMnBWZl0v4ytWZQEZk3gSgOjJdCayMTN8MfCkynQ5UA2Miz8uAFUF/r/SI3UOXN5d4cGnksTTyPJvwD9ZzwCfN7JrI/PLI/HqgE3jgmNd5MPLvYsL3uOjJnzx8P5DVZlYSmXce8MfI/Boze/o4tf6+2/Qo4PdmNoLwj/CWXta5GJhq9vpVtHPNLMfdm7q1GQHU9bL+Od0+z/8B3+o2/+rI9G+A7/Sw7grgO2b2TeARd3/ezE4H9rj7qwDufhDCWyPAj8xsJuH+ndjD610KTO+2BZZH+L/JFqAWGNnLZ5AEoMCQeGDAN9z9J2+YGb650cXAOe7eYmbPEL7NLECru3ce8zptkX876f273dZt2o75NxqHuk3fCXzP3f8SqfUrvayTRPgzHD7O6x7mn5/tRKK+AJy7rzezM4HLgW+Y2ROEdx319Bq3A3uBGZGaW3toY4S35B7vYVkG4c8hCUpjGBIPHgc+aGbZAGZWZmbDCf/1uj8SFpOBOTF6/xeAd0TGMkoID1pHIw/YFZl+X7f5TUBOt+dPEL4aKgCRv+CPtQYY38v7vET4ktsQHiN4ITK9kPAuJ7otfwMzGwm0uPuvCW+BnAGsBUaa2VmRNjmRQfw8wlseXcCNhO9pfazHgY+YWWpk3YmRLRMIb5Ec92gqGdwUGBI4d3+C8C6Vl81sBXA/4R/cx4AUM1sO3EH4BzIWHiB8E52VwE+ARUBjFOt9BfijmT0P7Os2/2HgmqOD3sAngarIIPFqerhbm7uvJXwr1Jxjl0XW/0CkH24Ebo3Mvw34dzN7hfAurZ5qPh14xcyWAV8EvubuR4DrgDvN7DXg74S3Du4C3mdmCwn/+B/q4fXuAVYDSyKH2v6Ef27NXQg82sM6kiB0eXMRwMyy3b3ZwvdrfgWY6+41A1zD7UCTu98TZftM4LC7u5ldT3gAfEFMizx+Pc8BC9x9f1A1SGxpDEMk7BEzyyc8eH3HQIdFxP8C7+pD+zMJD1IbcIDwEVSBMLMQ4fEchUUC0xaGiIhERWMYIiISFQWGiIhERYEhIiJRUWCIiEhUFBgiIhIVBYaIiETl/wFGalCgrs60iwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.sched.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "16768c7ae08e43939fa2ec5271b33208",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 16%|█▌        | 31/196 [00:03<00:16,  9.80it/s, loss=2.16]\n",
      "epoch      trn_loss   val_loss   accuracy                   \n",
      "    0      1.696057   1.590924   0.4153    \n",
      "    1      1.500337   1.424076   0.4918                     \n",
      "\n",
      "CPU times: user 2min 32s, sys: 24 s, total: 2min 56s\n",
      "Wall time: 34.3 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([1.42408]), 0.4918]"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "% time learn.fit(1e-1, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "18f9c7d91ee443c7836955a71b82efbe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                   \n",
      "    0      1.337106   1.252252   0.5489    \n",
      "    1      1.292416   1.207744   0.5662                     \n",
      "    2      1.249593   1.170348   0.5799                     \n",
      "    3      1.213217   1.136477   0.5938                     \n",
      "\n",
      "CPU times: user 5min 6s, sys: 47.8 s, total: 5min 53s\n",
      "Wall time: 1min 9s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([1.13648]), 0.5938]"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "% time learn.fit(1e-1, 4, cycle_len=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Four times fewer parameters but 10% more accuracy."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Let's add padding and refactor the last model to shorten the code"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Padding=1 makes sure we reduce 32x32 to 16x16 instead of 15x15 as in the last model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Put defaults in layer definition\n",
    "\n",
    "class ConvLayer(nn.Module):\n",
    "    def __init__(self, f_in, f_out):\n",
    "        super().__init__()\n",
    "        self.conv = nn.Conv2d(f_in, f_out, kernel_size=3, stride=2, padding=1)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        return F.relu(self.conv(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConvNet2(nn.Module):\n",
    "    def __init__(self, layers, n_out):\n",
    "        super().__init__()\n",
    "        self.layers = nn.ModuleList([ConvLayer(layers[i], layers[i + 1]) for i in range(len(layers) - 1)])\n",
    "        self.out = nn.Linear(layers[-1], n_out)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        for l in self.layers:\n",
    "            x = l(x)\n",
    "        x = F.adaptive_max_pool2d(x, 1)\n",
    "        # preserve minibatch size, flatten rest\n",
    "        x = x.view(x.size(0), -1)\n",
    "        \n",
    "        return F.log_softmax(self.out(x), dim=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = ConvLearner.from_model_data(ConvNet2([3, 20, 40, 80], 10), data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OrderedDict([('Conv2d-1',\n",
       "              OrderedDict([('input_shape', [-1, 3, 32, 32]),\n",
       "                           ('output_shape', [-1, 20, 16, 16]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 560)])),\n",
       "             ('ConvLayer-2',\n",
       "              OrderedDict([('input_shape', [-1, 3, 32, 32]),\n",
       "                           ('output_shape', [-1, 20, 16, 16]),\n",
       "                           ('nb_params', 0)])),\n",
       "             ('Conv2d-3',\n",
       "              OrderedDict([('input_shape', [-1, 20, 16, 16]),\n",
       "                           ('output_shape', [-1, 40, 8, 8]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 7240)])),\n",
       "             ('ConvLayer-4',\n",
       "              OrderedDict([('input_shape', [-1, 20, 16, 16]),\n",
       "                           ('output_shape', [-1, 40, 8, 8]),\n",
       "                           ('nb_params', 0)])),\n",
       "             ('Conv2d-5',\n",
       "              OrderedDict([('input_shape', [-1, 40, 8, 8]),\n",
       "                           ('output_shape', [-1, 80, 4, 4]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 28880)])),\n",
       "             ('ConvLayer-6',\n",
       "              OrderedDict([('input_shape', [-1, 40, 8, 8]),\n",
       "                           ('output_shape', [-1, 80, 4, 4]),\n",
       "                           ('nb_params', 0)])),\n",
       "             ('Linear-7',\n",
       "              OrderedDict([('input_shape', [-1, 80]),\n",
       "                           ('output_shape', [-1, 10]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 810)]))])"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3afd38a302474f42bdcffbd3e61c6c04",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                   \n",
      "    0      1.702102   1.756182   0.3846    \n",
      "    1      1.510568   1.425706   0.4867                     \n",
      "\n",
      "CPU times: user 2min 33s, sys: 23.6 s, total: 2min 56s\n",
      "Wall time: 34.2 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([1.42571]), 0.4867]"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "% time learn.fit(1e-1, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "20561b844eac4593b77394a7302d5ac2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                   \n",
      "    0      1.300015   1.222579   0.5652    \n",
      "    1      1.253193   1.179739   0.5771                     \n",
      "    2      1.197974   1.141736   0.5916                     \n",
      "    3      1.173729   1.112343   0.6071                     \n",
      "\n",
      "CPU times: user 5min 8s, sys: 48.3 s, total: 5min 56s\n",
      "Wall time: 1min 9s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([1.11234]), 0.6071]"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time learn.fit(1e-1, 4, cycle_len=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Problem: when trying to add more layers, the model stops being resilient. Either the gradients explode, or the learning rate has to be so small, that it takes forever to train.\n",
    "\n",
    "Solution: batch normalization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Batch normalization\n",
    "Proved to be crucial for training deep nets. The following model is going to be much deeper than the previous and still resilient to train.\n",
    "\n",
    "Normalizes every layer and controls that the activations don't exponentially grow or vanish.\n",
    "\n",
    "Instead of scaling a layer up (increasing every weight), SGD can simply increase the parameter m. Instead of shifting the weights up or down, it can increase or decrease a.\n",
    "\n",
    "1. More resilient training\n",
    "1. Allows larger learning rates\n",
    "1. Allows deeper nets\n",
    "1. Regularizes: often less dropout or weight decay necessary! The regularization probably comes from the fact that every minibatch is different, so that the mean and std deviation is different as well which adds noise..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "# nn.BatchNorm2d. Here we do it from scratch\n",
    "# Actually a simplified version. Full BatchNorm \n",
    "# uses exponentially decaying averages of mean and std\n",
    "# of the filters\n",
    "class BatchNormLayer(nn.Module):\n",
    "    def __init__(self, f_in, f_out, stride=2, kernel_size=3, padding=1):\n",
    "        super().__init__()\n",
    "        self.conv = nn.Conv2d(f_in, f_out, kernel_size=kernel_size, stride=stride, bias=False, padding=padding)\n",
    "        self.a = nn.Parameter(torch.zeros(f_out, 1, 1))\n",
    "        self.m = nn.Parameter(torch.ones(f_out, 1, 1))\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.conv(x))\n",
    "        x_chan = x.transpose(0, 1).contiguous().view(x.size(1), -1)  # num of filters\n",
    "        if self.training:\n",
    "            # mean and std of each filter\n",
    "            self.means = x_chan.mean(1)[:, None, None]\n",
    "            self.stds  = x_chan.std(1)[:, None, None]\n",
    "        return (x-self.means) / self.stds * self.m + self.a\n",
    "# If this was really intended to be used one should probably divide by (self.stds + eps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConvBnNet(nn.Module):\n",
    "    def __init__(self, layers, n_out):\n",
    "        super().__init__()\n",
    "        self.conv1 = nn.Conv2d(3, layers[0], kernel_size=5, stride=1, padding=2)\n",
    "        self.layers = nn.ModuleList([BatchNormLayer(layers[i], layers[i + 1]) for i in range(len(layers) - 1)])\n",
    "        self.out = nn.Linear(layers[-1], n_out)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = self.conv1(x)\n",
    "        for l in self.layers:\n",
    "            x = l(x)\n",
    "        x = F.adaptive_max_pool2d(x, 1)\n",
    "        # preserve minibatch size, flatten rest\n",
    "        x = x.view(x.size(0), -1)\n",
    "        \n",
    "        return F.log_softmax(self.out(x), dim=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = ConvLearner.from_model_data(ConvBnNet([10, 20, 40, 80, 160], 10), data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OrderedDict([('Conv2d-1',\n",
       "              OrderedDict([('input_shape', [-1, 3, 32, 32]),\n",
       "                           ('output_shape', [-1, 10, 32, 32]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 760)])),\n",
       "             ('Conv2d-2',\n",
       "              OrderedDict([('input_shape', [-1, 10, 32, 32]),\n",
       "                           ('output_shape', [-1, 20, 16, 16]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 1800)])),\n",
       "             ('BatchNormLayer-3',\n",
       "              OrderedDict([('input_shape', [-1, 10, 32, 32]),\n",
       "                           ('output_shape', [-1, 20, 16, 16]),\n",
       "                           ('nb_params', 0)])),\n",
       "             ('Conv2d-4',\n",
       "              OrderedDict([('input_shape', [-1, 20, 16, 16]),\n",
       "                           ('output_shape', [-1, 40, 8, 8]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 7200)])),\n",
       "             ('BatchNormLayer-5',\n",
       "              OrderedDict([('input_shape', [-1, 20, 16, 16]),\n",
       "                           ('output_shape', [-1, 40, 8, 8]),\n",
       "                           ('nb_params', 0)])),\n",
       "             ('Conv2d-6',\n",
       "              OrderedDict([('input_shape', [-1, 40, 8, 8]),\n",
       "                           ('output_shape', [-1, 80, 4, 4]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 28800)])),\n",
       "             ('BatchNormLayer-7',\n",
       "              OrderedDict([('input_shape', [-1, 40, 8, 8]),\n",
       "                           ('output_shape', [-1, 80, 4, 4]),\n",
       "                           ('nb_params', 0)])),\n",
       "             ('Conv2d-8',\n",
       "              OrderedDict([('input_shape', [-1, 80, 4, 4]),\n",
       "                           ('output_shape', [-1, 160, 2, 2]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 115200)])),\n",
       "             ('BatchNormLayer-9',\n",
       "              OrderedDict([('input_shape', [-1, 80, 4, 4]),\n",
       "                           ('output_shape', [-1, 160, 2, 2]),\n",
       "                           ('nb_params', 0)])),\n",
       "             ('Linear-10',\n",
       "              OrderedDict([('input_shape', [-1, 160]),\n",
       "                           ('output_shape', [-1, 10]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 1610)]))])"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1c8b07eff8594fbc88508405c73105a4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 78%|███████▊  | 153/196 [00:11<00:03, 13.01it/s, loss=nan] "
     ]
    }
   ],
   "source": [
    "learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEOCAYAAABmVAtTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl4VOXd//H3N5OEhATCFvZA2BSRnbAoPlatWrUuKIqKouJCcan62PZn9+Vprdq6tC6ICK51qRX3BbUVVASBEHYCSEB2TAIkIZA99++PGdMYQwiYkzOTfF7XNRczZ+4z55vbMZ/c5z6LOecQEREBiPK7ABERCR8KBRERqaJQEBGRKgoFERGpolAQEZEqCgUREamiUBARkSoKBRERqaJQEBGRKgoFERGpEu13AUeqQ4cOLjU11e8yREQiytKlS3Odc8mHaxdxoZCamkp6errfZYiIRBQz21Kfdtp9JCIiVRQKIiJSRaEgIiJVFAoiIlJFoSAiIlUUCiIiUqXZhEJpeSWvZmxHtx8VETm0iDtP4WjNztjOL15dxZ7CUm44uXedbfMOlrJsax7llY6DpeXM/yKXBVl7aNMyhiEpbUiIDbC7oIToKGN0r3akpbala5t4osz4ZEMOH63LpqisgphAFENT2nDR8G60jG02XS0iEcwi7S/ntLQ0dzQnr1VWOm55MYP3Vu/m8StHUOngkblfEBuIYkzv9nRqHce+g6Ws2p7PJ1/kUFbx335pHRfN2L4d2F9czopteZRWVNIlKY7CkgpyC0uq2gWijIpKR+u4aNomxFJUWkH2/hLatIzh7IGd6ZOcyPFdkxjTux1m1iD9ISJSH2a21DmXdrh2zebP16go4/5LhrJj30J+9I+lOAd9khOIjY9ixiebKK8MhkC3NvFMHtuL0/p3JLFFNNEBo29yItGB4J62r0PUzHDOkZVTyMrt+XxVUEJBcRljerfnxD7tiQlE4Zwjfcs+Zn26mXdW7qKguByAoSltuP30fozu1Z742IA/HSIiUotmM1L4WnZBMb9+fTWn9u/IJSO6Ex2I4mBpOQdLK2gTH1P1y7+hOefIO1jGnDW7efg/X7Azvxgz6NmuJWcP6sIVo3vQvW1LT7YtIlLfkUKzC4VwUFJewdx1OazbXcCKbXl8vCEHgO8f14mrT0hlbN/22r0kIg1Ku4/CWIvoAGcN7MxZAzsDsCOviBcWbeGlxdv4cO1XJLaIpmPrFvTukMgPB3fmjAGdSWyh/1Qi4j2NFMJISXkF763azfJteWTvL2bFtnx25BURE7CqSerLRqUwMrWd36WKSITxffeRmaUAzwKdgUpghnPu77W0OwX4GxAD5DrnvlfX5zblUKipstKRsXUf/1mXTeauApZvyyPvYBknH5PMzaf0YVQvHcUkIvUTDqHQBejinMsws1bAUmCcc25ttTZtgAXAWc65rWbW0TmXXdfnNqdQqOlgaTn/+HwL0z/exN4DpRzftTVTTu7NuYO7EohSOIjIofkeCt/akNkbwCPOuQ+rLbsJ6Oqc+3V9P6c5h8LXikoreG3ZDp76bDNfZBfSr2MiN5zcmxN6t6d723iNHkTkW8IqFMwsFfgEGOicK6i2/OvdRscDrYC/O+eereuzFAr/VVnpeHf1Lh78cANZOQcAaNsyhh7tWtKzfQK3n96P3smJPlcpIuEgbI4+MrNEYDZwe/VAqLb9EcD3gXhgoZl97pzbUOMzpgBTAHr06OF1yREjKso4d3BXzh7YhXW7C8jYmsfanfls31fEvPXZfPJFDo9fOYLRvdv7XaqIRAhPQ8HMYggGwvPOuVdrabKd4OTyAeCAmX0CDAG+EQrOuRnADAiOFLysORIFoozjuyZxfNekqmVb9hxg8tNLuHLWIn5y5rFcc2IqcTE6e1pE6ubZVVItuGN7FpDpnHvgEM3eAP7HzKLNrCUwGsj0qqbmpGf7BF67cSzfOyaZe95bx/fv/5g3lu+gslKZKiKH5uWls8cCk4DTzGx56HGOmU01s6kAzrlMYA6wElgMzHTOrfawpmYlqWUMM68eyfPXjyYpPobbXlrOhdM+4/NNe/wuTUTClE5eayYqKx2vLdvBfR+sZ1d+MSf2ac8dZxxDmk6EE2kW6jvR3GxustPcRUUZ40d0Z+5PT+E35w5gw1eFXDx9Ib94dRX7i8v8Lk9EwoRGCs1UUWkFD/57AzM/3USn1nFcdUIqFw7rRuekOL9LExEPaKQgdYqPDfDLc47jX1NPpHvbeO6ds44T7/kPj83L8rs0EfGRLr3ZzI3o2ZZ/TT2RL3MP8Nf313PvnHWUlFdw2/f76cxokWZIoSAApHZI4KHLhxEfG+Bv//6CvQdK+eU5x+ncBpFmRqEgVQJRxl/GD6ZNfAwz529m8ea9PDJxGH07tvK7NBFpJJpTkG+IijJ+fe4Anrwmjez9JYx7dAGfbcz1uywRaSQKBanVaf078c6tJ9GtTTzXPLWY15Zt97skEWkECgU5pC5J8bw89QTSerbjjpdXMHupgkGkqVMoSJ2S4mN4avJIxvbpwM9eWcFbK3b6XZKIeEihIIcVFxNgxlUjSOvZjtv/uZxXMzRiEGmqFApSLy1jo3ly8kjG9A7uSnpsXhaRdja8iByeQkHqLbFFNE9dM4rzh3Tl3jnruP+DDYdfSUQiis5TkCMSGx3F3y4dSkKLAI/M3UhcTBS3nNbP77JEpIEoFOSIRUUZd40bRElZJfd9sIEDpRXcccYxxAQ08BSJdAoFOSpRUcZfLh5MbHQUj83L4rONuTx46VD6JCf6XZqIfAf6006OWnQginvGD+axK4azbe9BfvjQpzy78EtNQItEMIWCfGdnD+rC+7efzOhe7fntG2u4/pl0ikor/C5LRI6CQkEaRMfWcTw9eSS/O28AH63P5oZnFQwikUihIA3GzJg8thf3XTyEz7JyFQwiEUihIA1u/IjuCgaRCOVZKJhZipnNNbNMM1tjZrfV0uYUM8s3s+Whx2+9qkcaV81gKC5TMIhEAi8PSS0HfuKcyzCzVsBSM/vQObe2RrtPnXPneliH+GT8iO4A/PSVFVz/TDozr07TndxEwpxnIwXn3C7nXEbo+X4gE+jm1fYkPI0f0Z2/hkYM1z+jEYNIuGuUOQUzSwWGAYtqefsEM1thZu+Z2fGNUY80rosVDCIRw/NQMLNEYDZwu3OuoMbbGUBP59wQ4GHg9UN8xhQzSzez9JycHG8LFk8oGEQig6ehYGYxBAPheefcqzXfd84VOOcKQ8/fBWLMrEMt7WY459Kcc2nJycleliweqh4MmnwWCU9eHn1kwCwg0zn3wCHadA61w8xGherZ41VN4r+LR3TnL+MHM3+jgkEkHHl59NFYYBKwysyWh5b9EugB4JybDlwM3Ghm5UARcJnThXOavEvSUgD4f7NXcsOz6TxxlY5KEgkXnoWCc24+YIdp8wjwiFc1SPhSMIiEJ53RLL65JC1Fu5JEwoxCQXx1SVoK9yoYRMKGQkF8N0HBIBI2FAoSFqoHw9R/LKWkXMEg4geFgoSNCWkp3H3hIOatz+Hm5zMoLa/0uySRZkehIGHlslE9+OO4gfw7M5urnlxEbmGJ3yWJNCsKBQk7k8b05IEJQ1i2NY/zH57P6h35fpck0mwoFCQsXTS8O69MPRGAq55czJe5B3yuSKR5UChI2BrUPYl/XD8a5xzXPLWYvQdK/S5JpMlTKEhY652cyMyr09iZX8zkpxZrjkHEYwoFCXsjerZj2sThrNu9n4umLSArp9DvkkSaLIWCRITTB3TixSljOFBSzvjHFrB2Z81bc4hIQ1AoSMQY3qMtr950IvExAa6ctYgNX+33uySRJkehIBGlZ/sEXrhhDNFRxsQnPmfLHh2VJNKQFAoScXp1CAZDeaXj2qeXkF9U5ndJIk2GQkEiUt+OiUy/cgRb9x7klhcyKK/QJTFEGoJCQSLWmN7tuevCQXz6RS43PZ9BUakuoifyXSkUJKJNSEvhd+cN4MPMr5g483P26DwGke9EoSARb/LYXjx2xXDW7ixg3LTPWLdbh6uKHC2FgjQJZw3swj9/dAKl5ZVcNG0B763a5XdJIhFJoSBNxtCUNrx1y0kc27kVNz6fwQMfrKey0vldlkhE8SwUzCzFzOaaWaaZrTGz2+poO9LMKszsYq/qkeahY+s4Xpoyhglp3Xnoo41MeW4p+4t1yKpIfXk5UigHfuKcOw4YA9xsZgNqNjKzAHAv8L6HtUgz0iI6wL3jB/OH849n7vpsLpq2QJfeFqknz0LBObfLOZcRer4fyAS61dL0x8BsINurWqT5MTOuPjGV564dRW5hCec/Mp+lW/b5XZZI2GuUOQUzSwWGAYtqLO8GXAhMb4w6pPk5sW8H3rzlJNolxDJp1iIWZOX6XZJIWPM8FMwskeBI4HbnXM1jBf8G3Omcq/OsIzObYmbpZpaek5PjVanSRKW0a8nLPzqB7m3jmfzUEuau16BU5FDMOe+OzjCzGOBt4H3n3AO1vL8ZsNDLDsBBYIpz7vVDfWZaWppLT0/3olxp4vYeKOWqJxexfvd+Hr58GGcN7OJ3SSKNxsyWOufSDtfOy6OPDJgFZNYWCADOuV7OuVTnXCrwCnBTXYEg8l20S4jl+evHMKhbEje/sIx3dS6DyLd4uftoLDAJOM3Mloce55jZVDOb6uF2RQ4pKT6G564bzdCUNtzx8nJW78j3uySRsOLp7iMvaPeRNISc/SVc8Mh8zIw3bhlLh8QWfpck4infdx+JhLPkVi2YcVUauYUlXPdMOnkHS/0uSSQsKBSk2RrYLYlHJg4nc2cBl834nJz9usKqiEJBmrUzBnTiyWtGsmXPQS6dsVAjBmn2FArS7J3UrwNPTx7J9r1F3PiPDErLdRc3ab4UCiLA6N7tuWf8IBZu2sNvXl9NpB2AIdJQov0uQCRcXDS8O5tzD/DwRxsJBIw/XjCQQJQdfkWRJkShIFLNHWccQ6VzPDo3i/yiMh6cMJTYaA2opflQKIhUY2b87Af9aRMfy13vZlJR4Xhk4jCiAwoGaR70TRepxQ0n9+a35w5gzprd3PHyCip0BzdpJjRSEDmEa0/qRUl5JffOWUe7hFh+f/7xfpck4jmFgkgdbjylD7mFJcyav5kBXVszIS3F75JEPKXdRyKH8Yuz+3NS3w78+rXVZGzV3dukaVMoiBxGdCCKhy8fRuekOK57egkbvtrvd0kinlEoiNRD24RYnr12FDGBKCY+sYhNOYV+lyTiCYWCSD2ldkjghRtG45zjipmL2Lb3oN8liTQ4hYLIEejbsRXPXTeaAyXlTJz5Obvzi/0uSaRB1SsUzOw2M2ttQbPMLMPMzvS6OJFwNKBra569bjT7DpQxcaYuuS1NS31HCtc65wqAM4FkYDJwj2dViYS5oSlteGrySHblFTNp1iL2HdAlt6VpqG8ofH1VsHOAp5xzK6otE2mWRqa244mr0tiUe4CrnlysYJAmob6hsNTMPiAYCu+bWStAF52XZu+kfh2YfuVw1u/ez/mPzmfd7gK/SxL5TuobCtcBPwdGOucOAjEEdyGJNHun9e/ESz8aQ0lZJRdNW8CCjbl+lyRy1OobCicA651zeWZ2JfBrIL+uFcwsxczmmlmmma0xs9tqaXOBma00s+Vmlm5mJx35jyDiv+E92vLWj0+ie9t4bnw+gy9zD/hdkshRqW8oPAYcNLMhwP8DtgDPHmadcuAnzrnjgDHAzWY2oEab/wBDnHNDgWuBmfWuXCTMdGodx8yrRmIGNzybTmFJud8lSRPy2zdWM2f1Ls+3U99QKHfB+xNeAPzdOfd3oFVdKzjndjnnMkLP9wOZQLcabQrdf+97mADo+sQS0Xq0b8mjE4ezKfcAU59bSlFphd8lSRNwsLScZxdu4YuvvD+Tvr6hsN/MfgFMAt4xswDBeYV6MbNUYBiwqJb3LjSzdcA7BEcLIhFtbN8O3Dt+MAuycrn6qcUaMch3tiknuDuyb8dEz7dV31C4FCgheL7CboJ/8f+1PiuaWSIwG7g9dK7DNzjnXnPO9QfGAX88xGdMCc05pOfk5NSzZBH/XDyiO3+7bBhLt+xj0qxF5BeV+V2SRLCN2cERQp9wCYVQEDwPJJnZuUCxc+5wcwqYWQzBQHjeOffqYbbxCdDHzDrU8t4M51yacy4tOTm5PiWL+O78IV15dOJwVu/IZ+ITn7NX5zHIUdqYXUggykhtn+D5tup7mYsJwGLgEmACsMjMLj7MOgbMAjKdcw8cok3fUDvMbDgQC+ypf/ki4e2sgZ2ZcVUaG7MLmfjE5xSXaY5BjtzG7EJ6tmtJbLT3l6ur7xZ+RfAchaudc1cBo4DfHGadsQTnIE4LHXK63MzOMbOpZjY11GY8sNrMlgOPApdWm3gWaRJOPbYjj105nHW79zNt7ka/y5EIlJVT2Ci7jqD+t+OMcs5lV3u9h8MEinNuPoe5FIZz7l7g3nrWIBKxTuvfiXFDu/LYx1mcP7Rbo0wYStNQXlHJl3sOcPqATo2yvfqOFOaY2ftmdo2ZXUPwSKF3vStLpOn51Q8HEB8T4Devr0YDYqmvLXsPUlbh6JvcOH9I1Hei+WfADGAwMASY4Zy708vCRJqa5FYtuPPs/izctIf7PljvdzkSIb4+8qixRpf13X2Ec242wSOJROQoTRzVg9U7Cnh0bhZJ8TFMObmP3yVJmPs6FHone3/kERwmFMxsP7WfZWyAc8619qQqkSbKzPjTuIEUFJfx53fXER8bzaQxPf0uS8JYVnYhnVvH0Squ3ucLfyd1hoJzrs5LWYjIkQtEGQ9OGEpJWQW/eX01FRWVXDO2l99lSZjamFPYqAcm6B7NIj6IjY5i2hUjOHNAJ37/1lpeXLzV75IkDDnnyMpWKIg0C7HRUTx6xXC+d0wyv3tzDZm7dIMe+aZd+cUcKK1otHMUQKEg4quYQBQPTBhCUnwMt764TGc8yzes3J4HwIAujTd9q1AQ8Vn7xBbcf8kQvsgu5A9vrdE5DFIlY2sesdFRDOymUBBpVk4+JpmbTunDi4u3MeOTTX6XI2Fi6ZZ9DOqWRIvoQKNtU6EgEiZ+euaxnDu4C3e/t443V+z0uxzxWUl5Bau25zOiZ9tG3W69T14TEW9FRRn3XTKE7P0l/PTlFXRs1YIxvdv7XZb4ZPWOAkorKhneo3FDQSMFkTASFxNgxqQRpLSLZ8qz6WzM3u93SeKTjC37ABjes02jblehIBJm2rSM5enJo4iNDnD1k0vYmVfkd0nig4yt+0hpF0/HVnGNul2FgkgYSmnXkqeuGUlBURkTn/ic3fnFfpckjcg5R/qWfYxo5F1HoFAQCVuDuifx9LWjyNlfwsQnPidnf4nfJUkj2b6viJz9JQxv5ElmUCiIhLURPdvyzLWj2JlfxC0vZFBeUel3SdIIMraG5hM0UhCRmtJS23HXuEEs2ryX+z7Y4Hc50ghWbc+nRXQUx3Zu/GuSKhREIsD4Ed25fFQPpn+cxZzVu/wuRzy2cns+A7q2JibQ+L+iFQoiEeJ35w1gaEobbn1pOQuycv0uRzxSUelYvTOfwd2SfNm+QkEkQsTFBHjqmpGktm/JDc+ks3xbnt8liQc25RRysLSCQd0b9/yEr3kWCmaWYmZzzSzTzNaY2W21tLnCzFaGHgvMbIhX9Yg0BW0TYnnuutG0S4zluqeXsG3vQb9Lkga2cns+AIO7N72RQjnwE+fcccAY4GYzG1CjzWbge865wcAfgRke1iPSJHRqHcfTk0dRVlHJtU8voaC4zO+SpAGt2pFPy9gAfZIb7x4K1XkWCs65Xc65jNDz/UAm0K1GmwXOuX2hl58D3b2qR6Qp6ZOcyPRJI9ice4Ab/7GUg6XlfpckDWTF9jwGdk0iEGW+bL9R5hTMLBUYBiyqo9l1wHuNUY9IU3Binw7cO34wC7P2cMXMReQdLPW7JPmOyioqWbuzgEE+7TqCRggFM0sEZgO3O+dqvd+gmZ1KMBTuPMT7U8ws3czSc3JyvCtWJMKMH9GdaVcMZ83OAi6evlBzDBHui68KKSmv9G0+ATwOBTOLIRgIzzvnXj1Em8HATOAC59ye2to452Y459Kcc2nJycneFSwSgc4a2IVnQ5fDOP+R+SzaVOv/RhIBVu0IHlE2yKfDUcHbo48MmAVkOuceOESbHsCrwCTnnE7VFDlKY3q35/Wbx9I2IZYrZi5i7vpsv0uSo7B2ZwEJsQFS2yf4VoOXI4WxwCTgNDNbHnqcY2ZTzWxqqM1vgfbAtND76R7WI9Kk9eqQwGs3jeWYTq249YVluhdDBMrKOUCfjolE+TTJDB7eec05Nx+o8ydzzl0PXO9VDSLNTVJ8DE9cncYFj8zn+mfSef3msbRpGet3WVJPWTmFvt9tT2c0izQx3drE8/ikEezMK+b6Z9IpKq3wuySphwMl5ezKL6ZPsn+7jkChINIkjejZjr9dNpSMrfu48fmllJbrktvhbnPuAQB6+3TS2tcUCiJN1DmDuvDnCwcxb30Od7y8nIpK53dJUoesnEIA385k/ppncwoi4r/LRvUgv6iMu99bR1J8DH8aN5DggYESbrJyDhBl0LN9S1/rUCiINHE/+l4f9h0sY/rHWbRpGcPPftDf75KkFptyCunetiVxMQFf61AoiDQDd551LPlFpTw6N4s28bHccHJvv0uSGrJyDvg+yQwKBZFmwcz407hBFBSVc9e7mSTFxzBhZIrfZUlIZaVjc24hJ/bx93BUUCiINBuBKOPBS4dSUFzGz19dSfd28ZzYp4PfZQmwM7+I4rJK3yeZQUcfiTQrsdFRPHblCHonJ/LjF5axM6/I75KE4K4jICx2HykURJqZxBbRTL9yBCXlldz4fAbFZTq5zW+bQoej+n2OAigURJqlvh0TuX/CEFZsy+OWFzIoq9DJbX7KyimkdVw0HRL9vySJQkGkmfrB8Z3547iB/Dszm9v/qZPb/LR6RwH9OrUKi3NIFAoizdikMT351TnH8c7KXdzzXqbf5TRLu/OLWb4tj1OOCY97xejoI5Fm7oaTe7Nt30Ge+HQzo3q154wBnfwuqVl5f81uAM4e1NnnSoI0UhARfnnOcQzs1pqf/msF2/fplp6N6b3Vu+jbMZG+HVv5XQqgUBARIC4mwKMTh1NZ6bjh2aXsLy7zu6RmYU9hCYs37+XsgeExSgCFgoiE9GyfwMMTh7Hhq/3c9HyGLrfdCD5Y+xWVDs5SKIhIODrl2I7cc9EgPv0ilztnr6RSRyR56r3Vu+nRriUDurT2u5QqmmgWkW+4JC2FrwqKue+DDbSMDehy2x4pLClnYVYu15yYGlb9q1AQkW+5+dS+FJZUMP3jLOJjAvzqh8eF1S+upmD+F7mUVThO6x9eR3spFETkW8yMO886luKyCmbO30zL2AB3nHms32U1KXPXZdMqLpq01LZ+l/INns0pmFmKmc01s0wzW2Nmt9XSpr+ZLTSzEjP7qVe1iMiRMzN+e+4ALk1L4aGPNjJt3ka/S2oynHPMXZ/Nyf2SiQmE19SulyOFcuAnzrkMM2sFLDWzD51za6u12QvcCozzsA4ROUpRUcafLxpEcXkFf5mznviYAJPH9vK7rIi3ZmcB2ftLOOXY8DiLuTrPQsE5twvYFXq+38wygW7A2mptsoFsM/uhV3WIyHcTiDLuv2QIxWUV/OGttcTHBLhsVA+/y4po89ZnA8GjvcJNo4xbzCwVGAYsaoztiUjDig5E8dDlw/jeMcn84rVV/HPJVr9LimgfrctmSPckklu18LuUb/E8FMwsEZgN3O6cKzjKz5hiZulmlp6Tk9OwBYpIvbSIDvD4pBH8T79k7py9isfmZeGczmM4UnsPlLJsW15YjhLA41AwsxiCgfC8c+7Vo/0c59wM51yacy4tOTn89sGJNBdxMQFmXpXG+UO6cu+cdTz0H00+H6lPNuTgHJzWPzxDwbM5BQse1DwLyHTOPeDVdkSkccVGR/G3S4cSHTAe/PcGju2cyFkDu/hdVsT4aF02HRJjGdQtye9SauXl0UdjgUnAKjNbHlr2S6AHgHNuupl1BtKB1kClmd0ODDja3Uwi0jiiooy7LxrEppwD3PHyClI7JNC/c/hcqiFclVdU8vGGHE4/rhNRUeF5MqCXRx/NB+r8qZ1zu4HuXtUgIt75eo7hvIfnM/GJRTwwYUjY7icPF8u35ZFfVBa2u45AF8QTke+gU+s4Xpwyho6tWnDNU0t44IP1mnyuw0frsglEGSf16+B3KYekUBCR76RPciKv3zyWi0d056GPNvLkZ1/6XVLYmrs+h7SebUmKj/G7lENSKIjIdxYXE+Av4wfzg+M7cdc7a5kbOjlL/mtHXhGZuwrCetcRKBREpIFERRkPTBjKsZ1bc+sLy1i5Pc/vksJGbmEJNzyTTkzA+MHx4XNDndooFESkwSS0iGbW1Wm0SYjhipmLWLFNwbA7v5gJ0xeyKbeQmVePJLVDgt8l1UmhICINqmubeF6acgJtWsZw5cxFLNu6z++SfHX/B+vZlV/MP64bzfeOCf+TbxUKItLgurWJ559TTqBdYixXzVpMRjMNhoLiMt5auZNxw7qSltrO73LqRaEgIp4IjhjG0D4UDIs37/W7pEb3xvKdFJdVcnkEXVVWoSAinumSFNyV1LF1C66ctYi3V+70u6RG45zjxUVbGdClddhe0qI2CgUR8VTnpDhmTz2RId2TuOWFZTw5f7PfJTWKVTvyWburgMtHpUTU/a0VCiLiubYJsTx33WjOOr4z//f2Wl5a3PTvx/DMgi3ExURxwbBufpdyRBQKItIo4mICVTfq+eVrq3hn5S6/S/LM8m15vLpsO1eO7knruPA9e7k2CgURaTSx0VE8duVwhvVoyy0vZvCXOesoq6j0u6wGVVHp+M3rq0lObMFtp/fzu5wjplAQkUbVMjaa564bxaVpKUybl8WExxeSd7DU77IazAuLt7JqRz6/PncArSJslAAKBRHxQcvYaO4ZP5hHJg5jzY4CrnsmnaLSCr/L+s5yC0v465x1nNC7PecNjswbDykURMQ35w7uyt8vG8qyrfu48fmllJZH9q6ke99bx8HSCv447viIOuKoOoUl6/rWAAAM70lEQVSCiPjq7EFduOvCQcxbn8OU5yJ3xJD+5V7+tXQ71/1PL/p2bOV3OUdNoSAivrt8VA/uvmgQH2/I4eonF5NfVOZ3SUekotLxmzfW0CUpjltPi7zJ5eoUCiISFi4f1YOHLhtGxtZ9nPP3TyPqshizl24nc1cBv/rhcSS08Owux41CoSAiYeO8IV15eeoJBKKMy2Ys5E9vr6WgOLxHDUWlFTzw4QaGprThh4Mic3K5OoWCiISV4T3a8u5t/8Nlo3ow67PNnPrXeby0eCsVleF57+cnP9vM7oJifnF2/4idXK7Os1AwsxQzm2tmmWa2xsxuq6WNmdlDZrbRzFaa2XCv6hGRyJHYIpo/XziIt245id7JCfz81VWc9/D8sNultKewhOnzsjj9uE6M7t3e73IahJcjhXLgJ86544AxwM1mNqBGm7OBfqHHFOAxD+sRkQgzsFsSL//oBB6+fBh5B0u5bMbCsLrS6l3vZlJUVsHPzz7W71IajGeh4Jzb5ZzLCD3fD2QCNa8MdQHwrAv6HGhjZpG/U05EGoyZcd6Qrnx4x/dI69mO215azgdrdvtdFp9+kcOrGTu48ZQ+EX0Iak2NMqdgZqnAMGBRjbe6Aduqvd7Ot4NDRISEFtE8OXkkg7olcfMLGdz9Xib5B/2ZhC4sKedXr62md4cEbj61ry81eMXzY6fMLBGYDdzunCuo+XYtq3xrNsnMphDcvUSPHpFzByMRaViJLaJ55tpR/OHNNcz4ZBMvLtrKhLQULklL4djO3v+1Pm99Nk98uoklX+6jtLySF28YQ1xMwPPtNiZzzrsZfTOLAd4G3nfOPVDL+48D85xzL4ZerwdOcc4d8pq6aWlpLj093auSRSRCZO4q4KH/fMGHa7+ivNJxyYju3DN+MIEob44AmrN6N7e8kEHXNvGcMaATPzi+M6N6RcZ9lwHMbKlzLu1w7TwbKVjw2KxZQGZtgRDyJnCLmb0EjAby6woEEZGvHdelNY9dOYI9hSXM+GQTj3+yiQrn+OvFQxokGJxzvLRkG7vyi8E5ps3LYmC3JJ69blTE3SPhSHi5+2gsMAlYZWbLQ8t+CfQAcM5NB94FzgE2AgeByR7WIyJNUPvEFvzinOCZxA98uIGConJ+fvax33ny9/FPNnHPe+uqXo/o2ZanJo9s0oEAHoaCc24+tc8ZVG/jgJu9qkFEmo9bv9+PuJgoHvzwC8548CvOGdSFH5/Wl/6dW9e5XlFpBVk5hbRpGUOHxBbExQSYuy6be+es49zBXfj7ZcMoLCmndVx0kzg57XA8nVPwguYURKQuewpLmDV/M88s+JIDpRWcflxHxvbtwKBuSfTqkEC7hNiqX+7Lt+Vx64vL2Lr3YNX6CbEByiocfTsm8sqNJ9AyNrKvZfS1+s4pKBREpEnKO1jKk599yT+XbOWrgpKq5S1jA3RvG0+n1nEszNpDp9Zx3HHGMZRXVpJbWEpuYQllFZXcdEpfuraJ9/EnaFgKBRGRkN35xazekc/WvQfZvq+I7fuC//bv0orfnXs8SS2b9jwBhMHRRyIi4aJzUhydk+L8LiMi6CqpIiJSRaEgIiJVFAoiIlJFoSAiIlUUCiIiUkWhICIiVRQKIiJSRaEgIiJVIu6MZjPLAbYc5epJQL4H69TVprb36rOsrtcdgNzD1HQ0jrR/vOib2pYfrr9qvu9F/+i7U7dw/e7Utqy5fnf6OeeSDrtl51yzeQAzvFinrja1vVefZXW9BtLDoX+86Jv69EUt/VGzfYP3j747kfndqUd/NOvvTm2P5rb76C2P1qmrTW3v1WfZ4V574Ui34UXf1Lb8cP0Vjn1T33X03Tm6NvX97tS2TN+dOkTc7iMBM0t39biwVXOl/jk09U3d1D+aaI5UM/wuIMypfw5NfVO3Zt8/GimIiEgVjRRERKSKQkFERKooFEREpIpCoQkyswQzW2pm5/pdSzgxs+PMbLqZvWJmN/pdT7gxs3Fm9oSZvWFmZ/pdTzgxs95mNsvMXvG7Fq8pFMKImT1pZtlmtrrG8rPMbL2ZbTSzn9fjo+4EXvamSn80RN845zKdc1OBCUCTOuywgfrndefcDcA1wKUeltuoGqhvNjnnrvO20vCgo4/CiJmdDBQCzzrnBoaWBYANwBnAdmAJcDkQAO6u8RHXAoMJnqofB+Q6595unOq91RB945zLNrPzgZ8DjzjnXmis+r3WUP0TWu9+4HnnXEYjle+pBu6bV5xzFzdW7X6I9rsA+S/n3Cdmllpj8Shgo3NuE4CZvQRc4Jy7G/jW7iEzOxVIAAYARWb2rnOu0tPCG0FD9E3oc94E3jSzd4AmEwoN9N0x4B7gvaYSCNBw353mQqEQ/roB26q93g6MPlRj59yvAMzsGoIjhYgPhDocUd+Y2SnARUAL4F1PKwsPR9Q/wI+B04EkM+vrnJvuZXE+O9LvTnvgLmCYmf0iFB5NkkIh/Fktyw67z88593TDlxJ2jqhvnHPzgHleFROGjrR/HgIe8q6csHKkfbMHmOpdOeFDE83hbzuQUu11d2CnT7WEG/VN3dQ/h6a+OQSFQvhbAvQzs15mFgtcBrzpc03hQn1TN/XPoalvDkGhEEbM7EVgIXCsmW03s+ucc+XALcD7QCbwsnNujZ91+kF9Uzf1z6Gpb46MDkkVEZEqGimIiEgVhYKIiFRRKIiISBWFgoiIVFEoiIhIFYWCiIhUUSiI58yssBG2cX49LyvekNs8xcxOPIr1hpnZzNDza8zskYav7siZWWrNy0vX0ibZzOY0Vk3S+BQKEjFClzuulXPuTefcPR5ss67rg50CHHEoAL8EHj6qgnzmnMsBdpnZWL9rEW8oFKRRmdnPzGyJma00sz9UW/566G5xa8xsSrXlhWb2f2a2CDjBzL40sz+YWYaZrTKz/qF2VX9xm9nTZvaQmS0ws01mdnFoeZSZTQtt420ze/fr92rUOM/M/mxmHwO3mdl5ZrbIzJaZ2b/NrFPoUsxTgf81s+Vm9j+hv6Jnh36+JbX94jSzVsBg59yKWt7raWb/CfXNf8ysR2h5HzP7PPSZ/1fbyMuCd9t7x8xWmNlqM7s0tHxkqB9WmNliM2sVGhF8GurDjNpGO2YWMLO/Vvtv9aNqb78OXFHrf2CJfM45PfTw9AEUhv49E5hB8AqVUcDbwMmh99qF/o0HVgPtQ68dMKHaZ30J/Dj0/CZgZuj5NQRvnAPwNPCv0DYGELxuPsDFBC+ZHQV0BvYBF9dS7zxgWrXXbfnv2f/XA/eHnv8e+Gm1di8AJ4We9wAya/nsU4HZ1V5Xr/st4OrQ82uB10PP3wYuDz2f+nV/1vjc8cAT1V4nAbHAJmBkaFlrgldGbgnEhZb1A9JDz1OB1aHnU4Bfh563ANKBXqHX3YBVfn+v9PDmoUtnS2M6M/RYFnqdSPCX0ifArWZ2YWh5Smj5HqACmF3jc14N/buU4P0RavO6C95LYq2ZdQotOwn4V2j5bjObW0et/6z2vDvwTzPrQvAX7eZDrHM6MMCs6qrMrc2slXNuf7U2XYCcQ6x/QrWf5zngL9WWjws9fwG4r5Z1VwH3mdm9wNvOuU/NbBCwyzm3BMA5VwDBUQXwiJkNJdi/x9TyeWcCg6uNpJII/jfZDGQDXQ/xM0iEUyhIYzLgbufc499YGLz5zenACc65g2Y2j+DtRAGKnXMVNT6nJPRvBYf+DpdUe241/q2PA9WePww84Jx7M1Tr7w+xThTBn6Gojs8t4r8/2+HU+8JkzrkNZjYCOAe428w+ILibp7bP+F/gK2BIqObiWtoYwRHZ+7W8F0fw55AmSHMK0pjeB641s0QAM+tmZh0J/hW6LxQI/YExHm1/PjA+NLfQieBEcX0kATtCz6+utnw/0Kra6w8IXnkTgNBf4jVlAn0PsZ0FBC/hDMF99vNDzz8nuHuIau9/g5l1BQ465/5BcCQxHFgHdDWzkaE2rUIT50kERxCVwCSC9yWu6X3gRjOLCa17TGiEAcGRRZ1HKUnkUihIo3HOfUBw98dCM1sFvELwl+ocINrMVgJ/JPhL0AuzCd5cZTXwOLAIyK/Her8H/mVmnwK51Za/BVz49UQzcCuQFpqYXUstd+pyzq0jeLvLVjXfC60/OdQPk4DbQstvB+4ws8UEdz/VVvMgYLGZLQd+BfzJOVcKXAo8bGYrgA8J/pU/DbjazD4n+Av+QC2fNxNYC2SEDlN9nP+Oyk4F3qllHWkCdOlsaVbMLNE5V2jBe+4uBsY653Y3cg3/C+x3zs2sZ/uWQJFzzpnZZQQnnS/wtMi66/mE4E3u9/lVg3hHcwrS3LxtZm0IThj/sbEDIeQx4JIjaD+C4MSwAXkEj0zyhZklE5xfUSA0URopiIhIFc0piIhIFYWCiIhUUSiIiEgVhYKIiFRRKIiISBWFgoiIVPn/5JVXZmKJPWUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.sched.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9436e6071fc441f3ae0c1b430cca241e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 23%|██▎       | 45/196 [00:03<00:13, 11.26it/s, loss=1.87]\n",
      "epoch      trn_loss   val_loss   accuracy                   \n",
      "    0      1.496147   1.390415   0.5005    \n",
      "    1      1.281947   1.167531   0.5787                     \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([1.16753]), 0.5787]"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.fit(3e-2, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ecaba6b5884e4b239bd2eb66e3c56e3e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                   \n",
      "    0      1.164448   1.060637   0.6258    \n",
      "    1      1.054636   0.933383   0.6709                     \n",
      "    2      0.960631   0.875445   0.6872                      \n",
      "    3      0.900813   0.824036   0.7091                      \n",
      "\n",
      "CPU times: user 6min 2s, sys: 48.6 s, total: 6min 51s\n",
      "Wall time: 1min 6s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([0.82404]), 0.7091]"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time learn.fit(1e-1, 4, cycle_len=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8f6566aaf0e844d7933a86929d79b1b2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                    \n",
      "    0      0.848855   0.781855   0.7233    \n",
      "    1      0.807261   0.766067   0.7304                      \n",
      "    2      0.774059   0.756201   0.738                       \n",
      "    3      0.752608   0.735999   0.745                       \n",
      "\n",
      "CPU times: user 6min 2s, sys: 49.3 s, total: 6min 52s\n",
      "Wall time: 1min 6s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([0.736]), 0.745]"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time learn.fit(1e-1, 4, cycle_len=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Attention: make sure that your library does not update the BatchNorm parameters a and m when the all layers but the last are frozen. Happens in many libraries because they only check if in training mode but not if frozen. Fastai also allows to unfreeze earlier layers and keep the BatchNorm parameters frozen (bn_freeze). Works well when the data we finetune on is similiar to data the network was pretrained on."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Deep BatchNorm model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConvBnNet2(nn.Module):\n",
    "    def __init__(self, layers, n_out):\n",
    "        super().__init__()\n",
    "        \n",
    "        # first layer has a bigger filter:\n",
    "        # all modern architectures start with a larger\n",
    "        # filter like 5x5, 7x7 or even up to 11x11.\n",
    "        # stride is 1 and padding is kernel_size//2\n",
    "        # filter size will not change but more filters\n",
    "        #  -> richer input to the network\n",
    "        self.conv1 = nn.Conv2d(3, layers[0], kernel_size=5, stride=1, padding=2)\n",
    "        self.layers = nn.ModuleList([BatchNormLayer(layers[i], layers[i + 1]) for i in range(len(layers) - 1)])\n",
    "        self. layers2 = nn.ModuleList([BatchNormLayer(layers[i + 1], layers[i + 1], stride=1) for i in range(len(layers) - 1)])\n",
    "        # stride 1 => doesn't change the size (kernel_size=3, padding=1)\n",
    "        \n",
    "        self.out = nn.Linear(layers[-1], n_out)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = self.conv1(x)\n",
    "        for l, l2 in zip(self.layers, self.layers2):\n",
    "            x = l(x)\n",
    "            x = l2(x)\n",
    "        x = F.adaptive_max_pool2d(x, 1)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        return F.log_softmax(self.out(x), dim=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = ConvLearner.from_model_data(ConvBnNet2([10, 20, 40, 80, 160], 10), data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f97a5695562344cf88b4207cb671d349",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                   \n",
      "    0      1.511038   1.369083   0.5021    \n",
      "    1      1.293975   1.217385   0.5575                     \n",
      "\n",
      "CPU times: user 3min 13s, sys: 25.4 s, total: 3min 39s\n",
      "Wall time: 35 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([1.21739]), 0.5575]"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "% time learn.fit(1e-2, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c95ba40ff1ff4d56afa18a44179d5753",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                    \n",
      "    0      0.980867   0.921074   0.6721    \n",
      "    1      0.914814   0.903012   0.6806                      \n",
      "\n",
      "CPU times: user 3min 16s, sys: 25.2 s, total: 3min 42s\n",
      "Wall time: 35.7 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([0.90301]), 0.6806]"
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "% time learn.fit(1e-2, 2, cycle_len=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Did not really help! Too deep to handle even for batchnorm (12 layers). Solution: *resnet* blocks**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Resnet"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Instead of $a_{i+1} = f(a_{i})$ calculate $a_{i+1} =a_{i} + f(a_{i})$. \n",
    "\n",
    "Network learns how much it has to add ontop of the activations of the current layer to calculate the activations of the next layer.\n",
    "\n",
    "$f(a_i) = \\underbrace{a_{i+1} - a_{i}}_{\\text{residual}}$\n",
    "\n",
    "Resnet layer tries to learn a set of conv filters that estimate the residual/error/how much it was off by and adds that to the activations of the pref layer.\n",
    "\n",
    "(keyword *boosting*)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ResnetLayer(BatchNormLayer):\n",
    "    def forward(self, x):\n",
    "        return x + super().forward(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Full resnet block has two convolutions that get added to original input..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Resnet(nn.Module):\n",
    "    def __init__(self, layers, n_out):\n",
    "        super().__init__()\n",
    "        self.conv1 = nn.Conv2d(3, layers[0], kernel_size=5, stride=1, padding=2)\n",
    "        \n",
    "        # Bottleneck layer reduces the size\n",
    "        self.layers = nn.ModuleList([BatchNormLayer(layers[i], layers[i + 1]) for i in range(len(layers) - 1)])\n",
    "        \n",
    "        self.layers2 = nn.ModuleList([ResnetLayer(layers[i + 1], layers[i + 1], stride=1) for i in range(len(layers) - 1)])\n",
    "        \n",
    "        self.layers3 = nn.ModuleList([ResnetLayer(layers[i + 1], layers[i + 1], stride=1) for i in range(len(layers) - 1)])\n",
    "        \n",
    "        self.out = nn.Linear(layers[-1], n_out)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = self.conv1(x)\n",
    "        for l, l2, l3 in zip(self.layers, self.layers2, self.layers3):\n",
    "            x = l3(l2(l(x)))\n",
    "            \n",
    "        x = F.adaptive_max_pool2d(x, 1)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        return F.log_softmax(self.out(x), dim=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = ConvLearner.from_model_data(Resnet([10, 20, 40, 80, 160], 10), data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [],
   "source": [
    "wd = 1e-5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "69f08912ca54449c8a61f5922b770d4f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 45%|████▍     | 88/196 [00:08<00:10,  9.96it/s, loss=1.84]\n",
      "epoch      trn_loss   val_loss   accuracy                   \n",
      "    0      1.576525   1.448838   0.463     \n",
      "    1      1.361114   1.265466   0.5356                     \n",
      "\n",
      "CPU times: user 3min 36s, sys: 26.7 s, total: 4min 2s\n",
      "Wall time: 38.9 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([1.26547]), 0.5356]"
      ]
     },
     "execution_count": 181,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time learn.fit(1e-2, 2, wds=wd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fab9e252e3a14e0c992f105728acf3c7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=7), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                   \n",
      "    0      1.185055   1.101637   0.6028    \n",
      "    1      1.111434   1.06314    0.6245                     \n",
      "    2      0.998808   0.977068   0.6513                      \n",
      "    3      1.027849   0.975656   0.6556                     \n",
      "    4      0.906812   0.896426   0.6808                      \n",
      "    5      0.822771   0.825362   0.7067                      \n",
      "    6      0.796764   0.841008   0.7043                      \n",
      "\n",
      "CPU times: user 12min 32s, sys: 1min 33s, total: 14min 5s\n",
      "Wall time: 2min 15s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([0.84101]), 0.7043]"
      ]
     },
     "execution_count": 183,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time learn.fit(1e-2, 3, cycle_len=1, cycle_mult=2, wds=wd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e3d7b1a849e040498146112a0a964abb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=32), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                    \n",
      "    0      0.873156   0.840932   0.7053    \n",
      "    1      0.795489   0.774087   0.7249                      \n",
      "    2      0.717201   0.71873    0.7482                      \n",
      "    3      0.669033   0.707185   0.7537                      \n",
      "    4      0.775813   0.762757   0.7369                      \n",
      "    5      0.70549    0.739103   0.7456                      \n",
      "    6      0.621499   0.659228   0.7746                      \n",
      "    7      0.578924   0.669779   0.772                       \n",
      "    8      0.681729   0.715374   0.7505                      \n",
      "    9      0.6251     0.644874   0.7748                      \n",
      "    10     0.545657   0.624503   0.7876                      \n",
      "    11     0.508989   0.621296   0.7892                      \n",
      "    12     0.615765   0.686351   0.7705                      \n",
      "    13     0.550705   0.639152   0.7823                      \n",
      "    14     0.505529   0.590059   0.796                       \n",
      "    15     0.459851   0.585975   0.8005                      \n",
      "    16     0.563753   0.63697    0.7859                      \n",
      "    17     0.508115   0.597805   0.7984                      \n",
      "    18     0.454198   0.591694   0.8006                      \n",
      "    19     0.422986   0.555419   0.8159                      \n",
      "    20     0.540979   0.651008   0.7867                      \n",
      "    21     0.481044   0.585627   0.8012                      \n",
      "    22     0.424541   0.558195   0.8125                      \n",
      "    23     0.385262   0.556032   0.8141                      \n",
      "    24     0.495159   0.608694   0.7928                      \n",
      "    25     0.446958   0.571407   0.8102                      \n",
      "    26     0.388603   0.567979   0.8157                      \n",
      "    27     0.350226   0.567475   0.8131                      \n",
      "    28     0.453512   0.576033   0.8082                      \n",
      "    29     0.410017   0.564211   0.8138                      \n",
      "    30     0.360722   0.536591   0.8224                      \n",
      "    31     0.331976   0.532281   0.8235                      \n",
      "\n",
      "CPU times: user 57min 34s, sys: 7min 9s, total: 1h 4min 43s\n",
      "Wall time: 10min 20s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([0.53228]), 0.8235]"
      ]
     },
     "execution_count": 184,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time learn.fit(1e-2, 8, cycle_len=4, wds=wd)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Three times deeper than first BatchNorm model we created but trains beautifully! :)**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Let's tweak this a little more..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Resnet2(nn.Module):\n",
    "    def __init__(self, layers, n_out, p = 0.5):\n",
    "        super().__init__()\n",
    "        self.conv1 = BatchNormLayer(3, layers[0], stride=1, kernel_size=7)  # padding = 3\n",
    "        \n",
    "        self.layers = nn.ModuleList([BatchNormLayer(layers[i], layers[i + 1]) for i in range(len(layers) - 1)])\n",
    "        \n",
    "        self.layers2 = nn.ModuleList([ResnetLayer(layers[i + 1], layers[i + 1], stride=1) for i in range(len(layers) - 1)])\n",
    "        \n",
    "        self.layers3 = nn.ModuleList([ResnetLayer(layers[i + 1], layers[i + 1], stride=1) for i in range(len(layers) - 1)])\n",
    "        \n",
    "        self.out = nn.Linear(layers[-1], n_out)\n",
    "        self.drop = nn.Dropout(p)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = self.conv1(x)\n",
    "        for l, l2, l3 in zip(self.layers, self.layers2, self.layers3):\n",
    "            x = l3(l2(l(x)))\n",
    "            \n",
    "        x = F.adaptive_max_pool2d(x, 1)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        x = self.drop(x)\n",
    "        return F.log_softmax(self.out(x), dim=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = ConvLearner.from_model_data(Resnet2([16, 32, 65, 128, 256], 10, 0.2), data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [],
   "source": [
    "wd = 1e-6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6d2be10a7aab427b9d6609da473ff0b7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                   \n",
      "    0      1.627052   1.482385   0.4644    \n",
      "    1      1.399879   1.261052   0.5447                     \n",
      "\n",
      "CPU times: user 3min 47s, sys: 28.4 s, total: 4min 15s\n",
      "Wall time: 41.4 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([1.26105]), 0.5447]"
      ]
     },
     "execution_count": 192,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "% time learn.fit(lrs=1e-2, n_cycle=2, wds=wd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4adbb56d143a4885ab818d023a3f5564",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=7), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                   \n",
      "    0      1.192086   1.145631   0.5898    \n",
      "    1      1.138265   1.101608   0.6141                     \n",
      "    2      1.003317   0.957184   0.6625                     \n",
      "    3      1.061927   1.022754   0.6477                     \n",
      "    4      0.945411   0.916022   0.6819                      \n",
      "    5      0.838307   0.801131   0.7169                      \n",
      "    6      0.791739   0.804649   0.7185                      \n",
      "\n",
      "CPU times: user 13min 22s, sys: 1min 38s, total: 15min\n",
      "Wall time: 2min 24s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([0.80465]), 0.7185]"
      ]
     },
     "execution_count": 193,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "% time learn.fit(lrs=1e-2, n_cycle=3, cycle_len=1, cycle_mult=2, wds=wd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a14d6bb88b8846c0887389e529140a92",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=32), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                    \n",
      "    0      0.887833   0.882731   0.6931    \n",
      "    1      0.809342   0.788145   0.723                       \n",
      "    2      0.714211   0.707529   0.7515                      \n",
      "    3      0.673939   0.696332   0.7539                      \n",
      "    4      0.794014   0.754859   0.7393                      \n",
      "    5      0.710952   0.681661   0.7615                      \n",
      "    6      0.625848   0.672974   0.7707                      \n",
      "    7      0.585995   0.636027   0.78                        \n",
      "    8      0.71474    0.704283   0.7551                      \n",
      "    9      0.629979   0.672416   0.7711                      \n",
      "    10     0.567307   0.610239   0.7931                      \n",
      "    11     0.522102   0.598643   0.793                       \n",
      "    12     0.637208   0.629912   0.7825                      \n",
      "    13     0.577492   0.622961   0.7873                      \n",
      "    14     0.519527   0.556991   0.8105                      \n",
      "    15     0.474265   0.567799   0.8062                      \n",
      "    16     0.594252   0.620696   0.7878                      \n",
      "    17     0.532623   0.607602   0.7936                      \n",
      "    18     0.474768   0.547793   0.8159                      \n",
      "    19     0.43386    0.540294   0.8177                      \n",
      "    20     0.556208   0.587111   0.7972                      \n",
      "    21     0.495127   0.574156   0.8017                      \n",
      "    22     0.431497   0.537003   0.823                       \n",
      "    23     0.39906    0.510635   0.8245                      \n",
      "    24     0.513565   0.579479   0.8006                      \n",
      "    25     0.465707   0.540691   0.8201                      \n",
      "    26     0.400908   0.5318     0.8236                      \n",
      "    27     0.371208   0.510625   0.828                       \n",
      "    28     0.487277   0.56614    0.8102                      \n",
      "    29     0.439708   0.532742   0.8203                      \n",
      "    30     0.371234   0.509699   0.8275                      \n",
      "    31     0.340832   0.490735   0.8369                      \n",
      "\n",
      "CPU times: user 1h 1min 2s, sys: 7min 27s, total: 1h 8min 30s\n",
      "Wall time: 10min 57s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([0.49073]), 0.8369]"
      ]
     },
     "execution_count": 199,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "% time learn.fit(lrs=1e-2, n_cycle=8, cycle_len=4, cycle_mult=1, wds=wd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save('final')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                             \r"
     ]
    }
   ],
   "source": [
    "log_preds, y = learn.TTA()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5, 10000, 10)"
      ]
     },
     "execution_count": 202,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_preds.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = np.mean(np.exp(log_preds), axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.4413704039683223, 0.8488)"
      ]
     },
     "execution_count": 217,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics.log_loss(y, preds), (np.argmax(preds, axis=1) == y).mean()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:deeplearning_save_2]",
   "language": "python",
   "name": "conda-env-deeplearning_save_2-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}