collaborative_filtering.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# Collaborative Filtering algorithm\n",
    "\n",
    "Collaborative Filtering (CF) algorithm uses a sparse rating matrix $A \\in \\mathcal{R}^{m\\times n}$ with non-empty entries denoting ratings of items i (0<=i<n) provided by users j (0<=j<m).\n",
    "CF simultaneously optimizes feature vectors $\\u_j \\in R^k$ of every user and $\\v_i \\in R^k$ of every item from the training set so that the CF loss function is minimized:\n",
    "$$\n",
    "\\mathcal{L}^{CF}=\\sum_{i,j: \\exists A[j, i]} (u^t_j*v_i - A_{j,i})^2\n",
    "$$"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Load data"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "from tqdm.notebook import tqdm\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "VAL_SIZE = 0.05\n",
    "MAX_EPOCHS = 10\n",
    "LR = 0.001\n",
    "BATCH_SIZE = 10000\n",
    "EMB_DIM = 1024\n",
    "\n",
    "DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [],
   "source": [
    "!mkdir -p movielens"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [],
   "source": [
    "!test -e movielens/ml-1m.zip || (wget -O movielens/ml-1m.zip https://files.grouplens.org/datasets/movielens/ml-1m.zip && unzip movielens/ml-1m.zip -d movielens)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# CF for Movielens 1M dataset"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_15707/4245508241.py:3: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n",
      "  df_ratings_small = pd.read_csv(fname, names=['userId','movieId','rating','timestamp'], sep='::')\n"
     ]
    }
   ],
   "source": [
    "def load_ratings(fname):\n",
    "    if fname.endswith('dat'):\n",
    "        df_ratings_small = pd.read_csv(fname, names=['userId','movieId','rating','timestamp'], sep='::')\n",
    "    else:\n",
    "        df_ratings_small = pd.read_csv(fname)\n",
    "    movie_id_map = df_ratings_small.movieId.sort_values().unique()\n",
    "    movie_id_map = dict(zip(movie_id_map, range(len(movie_id_map))))\n",
    "    df_ratings_small['movie_emb_idx'] = df_ratings_small.movieId.map(movie_id_map)\n",
    "    df_ratings_small['user_emb_idx'] = df_ratings_small.userId - 1\n",
    "    df_ratings_small['rating_cent'] = df_ratings_small.rating - df_ratings_small.rating.median()\n",
    "    return df_ratings_small\n",
    "\n",
    "\n",
    "df_ratings = load_ratings('movielens/ml-1m/ratings.dat')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [],
   "source": [
    "def gen_datasets(df, train_size):\n",
    "    df_train, df_val = np.array_split(df.sample(frac=1., random_state=42), [int(len(df) * train_size)])\n",
    "    ds_train = torch.utils.data.dataset.TensorDataset(torch.as_tensor(df_train[['user_emb_idx', 'movie_emb_idx']].values).to(DEVICE),\n",
    "                                                      torch.as_tensor(df_train['rating_cent'].values).to(DEVICE))\n",
    "    ds_val = torch.utils.data.dataset.TensorDataset(torch.as_tensor(df_val[['user_emb_idx', 'movie_emb_idx']].values).to(DEVICE),\n",
    "                                                      torch.as_tensor(df_val['rating_cent'].values).to(DEVICE))\n",
    "    return ds_train, ds_val\n",
    "\n",
    "ds_train, ds_val = gen_datasets(df_ratings, train_size=1 - VAL_SIZE)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [
    {
     "data": {
      "text/plain": "tensor([0.7129], grad_fn=<SumBackward1>)"
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class CFModel(torch.nn.Module):\n",
    "    def __init__(self, movies_size, users_size, emb_dim):\n",
    "        super().__init__()\n",
    "        self.user_emb = torch.nn.Parameter(torch.randn(users_size, emb_dim) / np.sqrt(emb_dim))\n",
    "        self.movie_emb = torch.nn.Parameter(torch.randn(movies_size, emb_dim) / np.sqrt(emb_dim))\n",
    "        self.nn = torch.nn.Sequential(\n",
    "            torch.nn.Linear(emb_dim, emb_dim),\n",
    "            torch.nn.SiLU(),\n",
    "            torch.nn.Linear(emb_dim, emb_dim),\n",
    "            torch.nn.Tanh()\n",
    "        )\n",
    "\n",
    "    def forward(self, movie_ids, user_ids):\n",
    "        user_embs = self.user_emb[user_ids]\n",
    "        movie_embs = self.movie_emb[movie_ids]\n",
    "        return (user_embs * movie_embs).sum(dim=-1)\n",
    "\n",
    "CFModel(2, 2, 4).forward([1], [1])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [],
   "source": [
    "def evaluate(model, ds_val):\n",
    "    with torch.no_grad():\n",
    "        losses = []\n",
    "        predictions = []\n",
    "        for b, ratings in torch.utils.data.DataLoader(ds_val, batch_size=8192):\n",
    "            user_ids = b[:, 0]\n",
    "            movie_ids = b[:, 1]\n",
    "            pred = model.forward(movie_ids, user_ids)\n",
    "            loss = ((pred - ratings) ** 2).mean()\n",
    "            losses.append(loss.item())\n",
    "            predictions.append(pred)\n",
    "        loss = np.sqrt(np.mean(losses).item())\n",
    "        return loss, torch.concat(predictions)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [],
   "source": [
    "def train(ds_train, ds_val, max_epochs, lr, batch_size):\n",
    "    ds = torch.concat([ds_train.tensors[0], ds_val.tensors[0]])\n",
    "    max_user_id, max_movie_id = ds.max(dim=0).values\n",
    "    print(max_user_id, max_movie_id)\n",
    "    model = CFModel(\n",
    "        max_movie_id + 1,\n",
    "        max_user_id + 1,\n",
    "        EMB_DIM\n",
    "    ).to(DEVICE)\n",
    "    optim = torch.optim.Adam(model.parameters(), lr=lr)\n",
    "\n",
    "    progress = tqdm(range(max_epochs))\n",
    "    losses = []\n",
    "    for epoch in progress:\n",
    "        batches = torch.utils.data.DataLoader(ds_train, batch_size=batch_size, shuffle=True)\n",
    "        if len(ds_train) > batch_size:\n",
    "            batches = tqdm(batches)\n",
    "        for b, ratings in batches:\n",
    "            user_ids = b[:, 0]\n",
    "            movie_ids = b[:, 1]\n",
    "\n",
    "            optim.zero_grad()\n",
    "            pred = model.forward(movie_ids, user_ids)\n",
    "            loss = ((pred - ratings.to(DEVICE)) ** 2).mean()\n",
    "            loss.backward()\n",
    "            optim.step()\n",
    "\n",
    "        loss = evaluate(model, ds_val)[0]\n",
    "        losses.append(loss)\n",
    "        progress.set_description(f'Val loss:{loss:.03f}')\n",
    "\n",
    "    fig = plt.figure(figsize=(12, 5))\n",
    "    plt.plot(losses)\n",
    "    print(fig.axes)\n",
    "    ax = fig.axes[0]\n",
    "    ax.set_title(\"Validation RMSE loss per training epoch\")\n",
    "    avg = torch.std(ds_val.tensors[1]).cpu().item()\n",
    "    ax.axhline(avg, color='red')\n",
    "    ax.annotate('Average score baseline', xy=(0, avg))\n",
    "    return model\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(6039, device='cuda:0') tensor(3705, device='cuda:0')\n"
     ]
    },
    {
     "data": {
      "text/plain": "  0%|          | 0/10 [00:00<?, ?it/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "b9355c529f4049f4b19ff5592a3944ee"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": "  0%|          | 0/96 [00:00<?, ?it/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "cb046773a715420190dd8fdb2926683f"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": "  0%|          | 0/96 [00:00<?, ?it/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "dca39b592865473f84c1574829b8f2bc"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": "  0%|          | 0/96 [00:00<?, ?it/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "ed6ab716b3b24c549258f7ef655abc5d"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": "  0%|          | 0/96 [00:00<?, ?it/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "127e5c9e575548da922451b072199721"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": "  0%|          | 0/96 [00:00<?, ?it/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "a0fa90e97ca5468a8ae24dac90cb1af5"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": "  0%|          | 0/96 [00:00<?, ?it/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "4ba8cc664b2c4fc18e5cad95376fdb67"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": "  0%|          | 0/96 [00:00<?, ?it/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "661f9ccf4b344c019a3b19028e4914f9"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": "  0%|          | 0/96 [00:00<?, ?it/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "a4dbc8602b9244f399eaf12e9e7718fb"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": "  0%|          | 0/96 [00:00<?, ?it/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "7cda8f993eb547af871b3f2d6ccf3cad"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": "  0%|          | 0/96 [00:00<?, ?it/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "35067ff405f544418888b843cb65c0c4"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<AxesSubplot:>]\n"
     ]
    },
    {
     "data": {
      "text/plain": "<Figure size 864x360 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAskAAAE/CAYAAAC0Fl50AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAA8GklEQVR4nO3deXyU5b338e8vewKBAAlbElZRRJAtIi6ItlaxWsUF6lpte2o91W529alPF23Pscfz2NYutra1ti4oYGtxOS6tCi7YQ1BRUVRAlISdsCSEbDO/54+5E4chG5BwT5LP+/Wa18zc1z33/ZvJvOCbK9d13ebuAgAAAPCRlLALAAAAAJINIRkAAABIQEgGAAAAEhCSAQAAgASEZAAAACABIRkAAABIQEgG0G5m5mZ2RPD4t2b2f9uz70Gc5zIze+pg6+xqzGydmZ0edh3Jwsz+j5n9oaP3TXZmdqqZlYVdB4AYQjLQg5jZE2Z2UzPbzzOzTWaW1t5jufs17n5zB9Q0IgjUTed29/vc/YxDPXYz5zrVzKJmVmVmlWb2jpl9NmEfN7Mt8fWYWXqwzeO2HWNmT5lZhZntNLPlZvbJZs4Tfzuho99TsjGz58zs3w7lGO7+H+7ermMcyL4AcCAIyUDP8mdJl5uZJWy/QtJ97t4QQk2H2wZ37y2pj6SvS/q9mR2VsM8OSWfFPT8r2BbvEUlPSxosaaCkr0janXiehNvSjnwjYTOz1IN4Tbt/EQOAMBGSgZ7lYUkDJM1o3GBm/SSdI+kvZjbNzJYGPaMbzexXZpbR3IHM7G4z+3Hc828Fr9lgZp9L2PdsM3vVzHab2Xoz+2Fc85Lgfmdjb6uZXWVmL8S9/kQzW2Zmu4L7E+PanjOzm83sxaB3+Ckzy2/rg/CYxyVVSDo2ofkeSZ+Je/4ZSX+JO2e+pJGSfu/udcHtRXd/QYfIzDLN7OfB57gheJzZeF4zezT4+VSY2fNmlhK0fcfMyuN6yD/ewvHvDobKPB3su9jMhse1jw3aKoLjzE147R1m9riZ7ZF0WsKxf6LYd+tXwc/yV8F2N7Nrzew9Se8F234RfBd2B73w8d/JH5rZvcHjxr80XGlmH5rZNjP73kHum21mfzazHWb2tpl921oZ3tCOz6K1z7G172x/M/tT8PPdYWYPJ5z3Gxb7y8VGS/hLB4DDh5AM9CDuvlfSfO0bAOdKWuXuKyRFFOtdzZd0gqSPS/pSW8c1s1mSvinpE5LGSEocX7snOGeepLMl/buZzQ7aTgnu85rrbTWz/pIek3S7YgH/NkmPmdmAuN0ulfRZxXp0M4Ja2qo5xczODd7r6oTmhyWdYmZ5FvslYoakv8e1bw9ec6+ZzTazQW2d7wB8T9J0SZMkTZQ0TdKNQds3JJVJKpA0SNL/keQW6wm/TtJx7p4r6UxJ61o5x2WSblbsvb8m6T5JMrNeivWO36/YZ3mxpN+Y2bi4114q6SeSciXt80uBu39P0vOSrgt+ltfFNc+WdLykxmMtC95j/+B8C8wsq5WaT5Z0lGLfye+b2dEHse8PJI2QNEqx7+rlLR2gnZ9FS59jW9/ZeyTlSDomOPbP4o45WFJfSYWSPi/p18F3EMBhRkgGep4/S7ooLpB8Jtgmd1/u7i+7e4O7r5P0O0kz23HMuZL+5O5vuvseST+Mb3T359z9DXePuvvrkua187hSLFS/5+73BHXNk7RK0qfi9vmTu78b90vApFaON9TMdkraK+lvkq5391cT9qlRbDjFp4PbomBb4/txxXpR10n6f5I2mtkSMxuTeJ6EW692vN/LJN3k7lvcfaukHyk2HEaS6iUNkTTc3evd/fmgloikTEnjzCzd3de5+5pWzvGYuy9x91rFQvkJZlas2F8U1rn7n4LP+lVJD0maE/favwe95lF3r2nm2C35T3evCH5Gcvd73X17cJ7/F9SfOOwl3o/cfW/wy9wKxX6BONB950r6D3ff4e5lioXYlrTns2jpc2zxO2tmQxQbvnNNUEe9uy+OO2a9Yj//+uAvHVVtfC4AOgkhGehhgiEB2yTNNrPRivVU3i9JZnZk8Of8TWa2W9J/KNZL1pahktbHPf8gvtHMjjezZ81sq5ntknRNO4/beOwPErZ9oFhPW6NNcY+rJfVu5Xgb3D1PsTHJt0v6WAv7/UWxXyD2GWrRyN3L3P06dx8tabhiveXx+21w97yE255W6mqU+H4/CLZJ0q2K9WA/ZWZrzey7QS2rJX1NsV9OtpjZA2Y2VC1r+lm5e5ViQ06GBu/j+Phgr1hoH9zcaw/QPq8zs28GQx52Befpq9a/EwfyM25p38TvaWvv5YA+i4TPsbXvbLGkCndPHOPeaHvC3IC23iuATkJIBnqmxgB4uaQn3X1zsP0OxXq8xrh7H8X+nJ84ya85GxX7z7/RsIT2+xXrjS12976Sfht3XFfrNigWWOINk1TejrpaFPT+fUfShLihH/GeV6zXdpAShhU0c6z1kn4tafyh1BRIfL/Dgm1y90p3/4a7j5J0rqTrLRh77O73u/vJwWtd0k9bOUfTz8rMeis25GGDYqFvcUKw7+3u/x732rZ+Xi21x68MMkPStxXr2e0X/NKyS+37rh2KjZKK4p4Xt7Sj2vdZtPQ5tvadXS+pv5nlHfzbAHA4EJKBnukvio0b/oKCoRaBXMVWaKgys7GS/r2Z1zZnvqSrzGycmeUoNvYzXq5ivWc1ZjZNsXGtjbZKiio2TrQ5j0s60swuNbM0M/u0YuNaH21nbS1y9zrFhkt8v5k2V2xIx7nB4yZm1s/MfmRmRwRjm/MlfU7Sy4dak2JDUW40s4LguN+X1Dgx7ZzgnKZYqIxIiprZUWb2MYtN8KtRbChJtJVzfNLMTrbYpMybJb0cBP1HFfusr7DYsnfpZnZcG+N/E21Wyz/LRrmSGhT72aeZ2fcV69nvbPMl3RD8/AoVG8fdkvZ8Fi19ji1+Z919o6T/UWx8c7/guKcknhxA+AjJQA8UjDd+SVIvxXp4G31TsQBbKen3kh5s5/H+R9LPJT2j2HCAZxJ2+ZKkm8ysUrHQNz/utdWKTQR7Mfiz9vSEY29XbHzoNxSbMPdtSee4+7b21NYOd0kaZmafSmxw95XuvrKZ19QpNgHsH4r9UvGmpFpJV8XtM9T2Xyf5wnbU82NJpZJel/SGpFeCbVJsUuQ/FBunulTSb9z9WcXG896i2DCaTYpNBruhlXPcr9gvMhWSpiqYwObulZLOUGyS2obgWD8Njt9ev1BszPsOM2tpzO+Tkp6Q9K5iwxBqdPDDOA7ETYpNfHxfsc9xoWI/t/2087No6XNs6zt7hWJjj1dJ2qLYUBkAScYSOkgAAN2Ymd0tqczdb2xr3+7OzP5d0sXu3t5JpPGvvVt8jkC3Rk8yAKBHMLMhZnZSMETmKMV6ev8Wdl0AkhNXPgIA9BQZii1rOFLSTkkPSPpNmAUBSF4MtwAAAAASMNwCAAAASEBIBgAAABIk5Zjk/Px8HzFiRNhlAAAAoBtbvnz5NncvaK4tKUPyiBEjVFpaGnYZAAAA6MbMLPES8k0YbgEAAAAkICQDAAAACQjJAAAAQAJCMgAAAJCAkAwAAAAkICQDAAAACQjJAAAAQAJCMgAAAJCAkAwAAAAkICQHlq7ZrqdWbgq7DAAAACSBpLwsdRh++cx7KtuxV6cfPUgpKRZ2OQAAAAgRPcmBOSVF+rCiWv96vyLsUgAAABAyQnJg1jFDlJuZpgWl68MuBQAAACEjJAeyM1L1qUlD9fibG1VZUx92OQAAAAgRITnOnKlFqqmP6tHXN4ZdCgAAAEJESI4zqThPYwb21nyGXAAAAPRohOQ4Zqa5JcV69cOdWr2lMuxyAAAAEBJCcoLZkwuVmmJaUFoWdikAAAAICSE5QUFupj42dqAeeqVc9ZFo2OUAAAAgBITkZswtKda2qlotfmdr2KUAAAAgBITkZpx6VIHye2cygQ8AAKCHajMkm9ldZrbFzN5soX2smS01s1oz+2ZC2zoze8PMXjOz0o4qurOlp6bogimFembVFm2rqg27HAAAABxm7elJvlvSrFbaKyR9RdJ/t9B+mrtPcveSA6wtVHOmFqkh6nr41fKwSwEAAMBh1mZIdvcligXhltq3uPsySd3qMnVjBuVq8rA8Pbhsvdw97HIAAABwGHX2mGSX9JSZLTezqzv5XB1uztRivbelSivKdoVdCgAAAA6jzg7JJ7v7FElnSbrWzE5paUczu9rMSs2sdOvW5FhV4pyJQ5SVnqIFTOADAADoUTo1JLt7eXC/RdLfJE1rZd873b3E3UsKCgo6s6x265OVrk+OH6JFr23Q3rpI2OUAAADgMOm0kGxmvcwst/GxpDMkNbtCRjK7qKRIlbUNenLlprBLAQAAwGGS1tYOZjZP0qmS8s2sTNIPJKVLkrv/1swGSyqV1EdS1My+JmmcpHxJfzOzxvPc7+5PdMJ76FTTRw5Qcf9szS9dr9mTC8MuBwAAAIdBmyHZ3S9po32TpKJmmnZLmniQdSWNlBTTnKnFuu3pd7W+olrF/XPCLgkAAACdjCvutcOFU4tkJi1cXhZ2KQAAADgMCMntUJiXrZOPyNfC5WWKRlkzGQAAoLsjJLfTnJJile/cq6Vrt4ddCgAAADoZIbmdzhg3SH2y0jSfNZMBAAC6PUJyO2Wlp+q8SYV64s1N2rW3W12BGwAAAAkIyQdgbkmxahuiemTFhrBLAQAAQCciJB+A8YV9NHZwLpepBgAA6OYIyQfAzDSnpFgrynbpnU2VYZcDAACATkJIPkCzJw1VeqrRmwwAANCNEZIP0IDemfr42EH626vlqmuIhl0OAAAAOgEh+SDMPa5I2/fU6ZlVW8IuBQAAAJ2AkNyMhx9+WGamVatWNdt+ypgCDczN1MLlPXPIxbp16zR+/PhOO/5zzz2nc845R5K0aNEi3XLLLZ12LgAAgOYQkpsxb948nXzyyZo3b16z7WmpKbpgSpGefWertuyuafN4kUiko0vsFA0NDWGXsJ9zzz1X3/3ud8MuAwAA9DCE5ARVVVV64YUX9Mc//lEPPPCAJOmJJ57QnDlzmvZ57rnn9NTPvqZI1PXjOx/QCSecoClTpmjOnDmqqqqSJI0YMULf+c53NGXKFC1YsEC///3vddxxx2nixIm68MILVV1dLUlas2aNpk+frgkTJujGG29U7969m85z66236rjjjtOxxx6rH/zgB/vVGolEdNVVV2n8+PGaMGGCfvazn0mSVq9erdNPP10TJ07UlClTtGbNGrm7vvWtbzXt++CDDza9lxkzZujcc8/VuHHjFIlE9K1vfavpvL/73e+a/ZwaGhp02WWX6eijj9ZFF13U9H5uuukmHXfccRo/fryuvvpqubsk6fbbb9e4ceN07LHH6uKLL5Yk7dmzR5/73Oc0bdo0TZ48WX//+9/3O8/dd9+t6667TpJ01VVX6Stf+YpOPPFEjRo1SgsXLmz3ZwUAAHBA3D3pblOnTvWw3Hvvvf65z33O3d1POOEELy0t9fr6ei8uLvaqqip3d7/mmmv8nnvu8XP+6zHvO/JYr6ysdHf3W265xX/0ox+5u/vw4cP9pz/9adNxt23b1vT4e9/7nt9+++3u7n722Wf7/fff7+7ud9xxh/fq1cvd3Z988kn/whe+4NFo1CORiJ999tm+ePHifWotLS31008/ven5jh073N192rRp/te//tXd3ffu3et79uzxhQsX+umnn+4NDQ2+adMmLy4u9g0bNvizzz7rOTk5vnbtWnd3/93vfuc333yzu7vX1NT41KlTm9oavf/++y7JX3jhBXd3/+xnP+u33nqru7tv3769ab/LL7/cFy1a5O7uQ4YM8Zqamn3qvOGGG/yee+5p2jZmzBivqqryZ5991s8++2x3d//Tn/7k1157rbu7X3nllX7RRRd5JBLxlStX+ujRo9v9WQEAACSSVOot5FF6khPMmzevqafz4osv1rx585SWlqZZs2bpkUceUUNDgx577DGdd955Ojp1syo3rdPUaSdo0qRJ+vOf/6wPPvig6Vif/vSnmx6/+eabmjFjhiZMmKD77rtPK1eulCQtXbq0qZf60ksvbdr/qaee0lNPPaXJkydrypQpWrVqld577719ah01apTWrl2rL3/5y3riiSfUp08fVVZWqry8XOeff74kKSsrSzk5OXrhhRd0ySWXKDU1VYMGDdLMmTO1bNkySdK0adM0cuTIpvP+5S9/0aRJk3T88cdr+/bt+51XkoqLi3XSSSdJki6//HK98MILkqRnn31Wxx9/vCZMmKBnnnmm6X0ee+yxuuyyy3TvvfcqLS2t6Vy33HKLJk2apFNPPVU1NTX68MMPW/35zJ49WykpKRo3bpw2b97c7s8KAADgQKSFXUAyqaio0DPPPKM33nhDZqZIJCIz06233qqLL75Yv/rVr9S/f3+VlJQoNzdXJSP6qdfIyTr/ptt1y4XH7ne8Xr16NT2+6qqr9PDDD2vixIm6++679dxzz7Vai7vrhhtu0Be/+MUW9+nXr59WrFihJ598Ur/97W81f/58/eIXvzjg9x1fp7vrl7/8pc4888xWX2Nm+z2vqanRl770JZWWlqq4uFg//OEPVVMTG7P92GOPacmSJXrkkUf0k5/8RG+88YbcXQ899JCOOuqofY7VGH6bk5mZuU+tjfdtfVYAAAAHgp7kOAsXLtQVV1yhDz74QOvWrdP69es1cuRIPf/885o5c6ZeeeUV/f73v2/qaT5txknS5nf00HPLVV3XoD179ujdd99t9tiVlZUaMmSI6uvrdd999zVtnz59uh566CFJahoDLUlnnnmm7rrrrqYxzuXl5dqyZd8l57Zt26ZoNKoLL7xQP/7xj/XKK68oNzdXRUVFevjhhyVJtbW1qq6u1owZM/Tggw8qEolo69atWrJkiaZNm7ZfnWeeeabuuOMO1dfXS5Leffdd7dmzZ7/9PvzwQy1dulSSdP/99+vkk09uCsT5+fmqqqpqGjMcjUa1fv16nXbaafrpT3+qXbt2qaqqSmeeeaZ++ctfNoXdV199tbUfT4va81kBAAAcCGsMKMmkJDfXS6dOPeznPe211/SdYcM0q3//pm23l5Xp7epq3XHkkbruvfd096ZN2nLiicpJTZUkPbJ5m76zZq2UakpLNf14xAidm5+vES+/rNKpU5Wfni5JumPDBv3X+vUqSE/X8bm5qoxEdPfYsXqvulqXr1qlvdGoZvXrp/u2bFH5CSdIkn5RVqY/bNokSeqdkqJ7jz5ao7Ozm2pbUVWlz77zjqLBz/A/R47UWQMG6L3qan3xvfe0rb5e6WZaMG6cRmZl6dtr1+p/KipkZrpx2DB9euBAPbdzp/57/Xo9OmGCJCnqrhvff1+PVFTI3VWQnq6Hx49X37SP/uiwrqZGs15/XSW5uVpeVaVxOTm6Z+xY5aSm6sb339e8LVs0OCNDR2Zna3hWlr43bJhOW7FCuyIRubsuHzRI3x02THsjEX1tzRq9tHu3ou4amZWlRydM2KemuzdtUmllpX41ZoyuWrVK5wwYoIsKCmKfyfPPq2rGjHZ9VgAAAIls8eLl7l7SbBsh+dC4pBXrdyo9NUXHDO1zwK+vjkSUnZIiM9MDW7Zo3pYt+nsnrkEMAACAmNZCcnKOST7qKKmNMbvJwiS9+Oxq3frkO1r8rVM1fECvNl8Tb/nzz+u6666TuysvP193LVokHXFE5xQLAACAjyTMsYrHmOQOcMGUQqWYtHB52QG/dsaMGVqxYoVef/11LVmyREcQkAEAAEJHSO4AQ/pma8aYAi1cXqZINPmGrwAAAODAEJI7yNySYm3cVaMXV28LuxQAAAAcIkJyBzl93EDl5aRrfun6sEsBAADAISIkd5DMtFTNnlSop1Zu1s7qurDLAQAAwCEgJHegOSVFqotEtWjFhrBLAQAAwCEgJHegY4b21bghfRhyAQAA0MURkjvY3JIivVm+W29t2B12KQAAADhIhOQOdt6kQmWkpmjBcnqTAQAAuipCcgfr1ytDnxg3SA+/Wq7ahkjY5QAAAOAgEJI7wZySIu2ortc/394SdikAAAA4CG2GZDO7y8y2mNmbLbSPNbOlZlZrZt9MaJtlZu+Y2Woz+25HFZ3sZowp0OA+WVrABD4AAIAuqT09yXdLmtVKe4Wkr0j67/iNZpYq6deSzpI0TtIlZjbu4MrsWlJTTBdOLdTid7dq066asMsBAADAAWozJLv7EsWCcEvtW9x9maT6hKZpkla7+1p3r5P0gKTzDqXYrmTO1GJFXXrolbKwSwEAAMAB6swxyYWS4scblAXbeoQR+b00bWR/LVxeJncPuxwAAAAcgKSZuGdmV5tZqZmVbt26NexyOsScqUV6f9selX6wI+xSAAAAcAA6MySXSyqOe14UbGuWu9/p7iXuXlJQUNCJZR0+n5wwRL0yUjV/GRP4AAAAupLODMnLJI0xs5FmliHpYkmLOvF8SadXZprOOXaoHntjo/bUNoRdDgAAANqpPUvAzZO0VNJRZlZmZp83s2vM7JqgfbCZlUm6XtKNwT593L1B0nWSnpT0tqT57r6y895KcppTUqTquogee2Nj2KUAAACgndLa2sHdL2mjfZNiQymaa3tc0uMHV1r3MHV4P43K76UFpes1t6S47RcAAAAgdEkzca+7MjPNKSnWsnU7tHZrVdjlAAAAoB0IyYfBBVMKlWLSwuWsmQwAANAVEJIPg0F9snTqUQP10CtlikRZMxkAACDZEZIPk7klRdq8u1ZL3usea0ADAAB0Z4Tkw+RjYwepf68MLShlzWQAAIBkR0g+TDLSUjR7UqGefmuzKvbUhV0OAAAAWkFIPozmHlek+ojr76+1eOFBAAAAJAFC8mE0dnAfTSjsqweXrZc7E/gAAACSFSH5MJtbUqRVmyq1csPusEsBAABACwjJh9m5EwuVkZai+UzgAwAASFqE5MOsb066zjxmsP7+2gbV1EfCLgcAAADNICSHYG5JkXbtrdfTb20OuxQAAAA0g5AcghNH56swL5shFwAAAEmKkByC1BTThVMK9cLqbdqwc2/Y5QAAACABITkkF00tlrv00PKysEsBAABAAkJySIYNyNEJowZowfIyRaOsmQwAAJBMCMkhmlNSpA8rqvW/6yrCLgUAAABxCMkhOmv8EPXOTGMCHwAAQJIhJIcoOyNVn5o4VI+/sVGVNfVhlwMAAIAAITlkc0qKVFMf1WOvbwy7FAAAAAQIySGbXJynIwb2ZsgFAABAEiEkh8zMNLekSK98uFOrt1SGXQ4AAABESE4KsycXKjXFtIA1kwEAAJICITkJDMzN0mlHDdRDy8tVH4mGXQ4AAECPR0hOEnNLirStqlaL39kadikAAAA9HiE5SZw2dqDye2dowXIm8AEAAISNkJwk0lNTdP7kQv3z7S3aVlUbdjkAAAA9GiE5icwpKVZD1PXwq+VhlwIAANCjEZKTyJGDcjWxOE/zS9fL3cMuBwAAoMciJCeZuSVFendzlV4v2xV2KQAAAD0WITnJfGriUGWmpXAFPgAAgBARkpNMn6x0nTV+sBat2KCa+kjY5QAAAPRIbYZkM7vLzLaY2ZsttJuZ3W5mq83sdTObEtcWMbPXgtuijiy8O5tbUqzKmgY9uXJT2KUAAAD0SO3pSb5b0qxW2s+SNCa4XS3pjri2ve4+Kbide9BV9jDTRw1QUb9shlwAAACEpM2Q7O5LJFW0sst5kv7iMS9LyjOzIR1VYE+UkmK6aGqRXlqzXesrqsMuBwAAoMfpiDHJhZLiuzzLgm2SlGVmpWb2spnNbu0gZnZ1sG/p1q1cmvmiqUWSpIdeKQu5EgAAgJ6nsyfuDXf3EkmXSvq5mY1uaUd3v9PdS9y9pKCgoJPLSn5F/XJ00uh8LSgtUzTKmskAAACHU0eE5HJJxXHPi4JtcvfG+7WSnpM0uQPO12PMKSlS+c69ennt9rBLAQAA6FE6IiQvkvSZYJWL6ZJ2uftGM+tnZpmSZGb5kk6S9FYHnK/HOPOYwcrNSmMCHwAAwGGW1tYOZjZP0qmS8s2sTNIPJKVLkrv/VtLjkj4pabWkakmfDV56tKTfmVlUsTB+i7sTkg9AVnqqzps0VAtKy/SjvfXqm50edkkAAAA9Qpsh2d0vaaPdJV3bzPaXJE04+NIgSXOmFuvelz/Uo69v0GXHDw+7HAAAgB6BK+4luWOL+uqoQbmaX8oqFwAAAIcLITnJmZnmlBRpxfqdendzZdjlAAAA9AiE5C7g/MmFSksxLWACHwAAwGFBSO4CBvTO1MePHqi/vlKu+kg07HIAAAC6PUJyFzG3pFjb99TpmVVbwi4FAACg2yMkdxEzjyxQQW6mFjCBDwAAoNMRkruItNQUXTClUM++s0VbKmvCLgcAAKBbIyR3IXOmFisSdf3tlfKwSwEAAOjWCMldyBEDe2vq8H5asLxMsWu4AAAAoDMQkruYOVOLtHpLlV5dvzPsUgAAALotQnIXc/axQ5SdnsqayQAAAJ2IkNzF5Gal65MThuiRFRu1ty4SdjkAAADdEiG5C5pTUqSq2gb9z5sbwy4FAACgWyIkd0HHj+yv4QNyNJ8hFwAAAJ2CkNwFmZnmTC3Sy2sr9OH26rDLAQAA6HYIyV3UBVOKZCYtXE5vMgAAQEcjJHdRQ/OyNWNMgRYuL1MkyprJAAAAHYmQ3IXNLSnShl01emnNtrBLAQAA6FYIyV3Y6UcPUt/sdM0vLQu7FAAAgG6FkNyFZaWnavakoXpy5SbtrK4LuxwAAIBug5Dcxc0pKVZdQ1SLVmwIuxQAAIBug5DcxY0v7Kujh/TRAoZcAAAAdBhCcjcwt6RIb5Tv0lsbdoddCgAAQLdASO4GZk8qVEZqihawZjIAAECHICR3A/16Zej0cQP18KvlqmuIhl0OAABAl0dI7ibmlBRrR3W9/vn25rBLAQAA6PIIyd3EKWMKNLhPluaXMuQCAADgUBGSu4nUFNMFUwq1+N2t2ry7JuxyAAAAujRCcjcyp6RYUZceeoXl4AAAAA4FIbkbGZnfS9NG9NeC0jK5e9jlAAAAdFmE5G7mopIivb9tj5Z/sCPsUgAAALqsdoVkM7vLzLaY2ZsttJuZ3W5mq83sdTObEtd2pZm9F9yu7KjC0byzJwxRTkYqE/gAAAAOQXt7ku+WNKuV9rMkjQluV0u6Q5LMrL+kH0g6XtI0ST8ws34HWyza1iszTeccO0SPvr5Re2obwi4HAACgS2pXSHb3JZIqWtnlPEl/8ZiXJeWZ2RBJZ0p62t0r3H2HpKfVethGB5hTUqzquogef2Nj2KUAAAB0SR01JrlQUvzf98uCbS1tRycqGd5PI/N7aUEpq1wAAAAcjKSZuGdmV5tZqZmVbt26NexyujQz05ySIv3vugq9v21P2OUAAAB0OR0VksslFcc9Lwq2tbR9P+5+p7uXuHtJQUFBB5XVc104pUgpJi1czgQ+AACAA9VRIXmRpM8Eq1xMl7TL3TdKelLSGWbWL5iwd0awDZ1sUJ8szTyyQAuXlykSZc1kAACAA9HeJeDmSVoq6SgzKzOzz5vZNWZ2TbDL45LWSlot6feSviRJ7l4h6WZJy4LbTcE2HAZzS4q1eXetlrzH8BUAAIADkdaendz9kjbaXdK1LbTdJemuAy8Nh+rjRw9Sv5x0LSwt02lHDQy7HAAAgC4jaSbuoeNlpKVo9uRCPfXWJlXsqQu7HAAAgC6DkNzNzZlarPqI6++vNTtfEgAAAM0gJHdz44b20fjCPqyZDAAAcAAIyT3A3JJivbVxt94s3xV2KQAAAF0CIbkHOHfiUGWkpWhBKWsmAwAAtAchuQfIy8nQGeMG6eHXNqimPhJ2OQAAAEmPkNxDzC0p1q699frH25vDLgUAACDpEZJ7iJOOyNfQvlmazwQ+AACANhGSe4jUFNOFU4v0/HtbtWHn3rDLAQAASGqE5B7koqlFcpf++gq9yQAAAK0hJPcgwwf00vRR/bVgeZliVxIHAABAcwjJPcycqcX6YHu1/vf9irBLAQAASFqE5B7mrAmD1TszjQl8AAAArSAk9zA5GWn61MQhevyNjaqqbQi7HAAAgKRESO6BLpparL31ET32+oawSwEAAEhKhOQeaMqwPI0u6MWQCwAAgBYQknsgM9PckmIt/2CHVm+pCrscAACApENI7qHOn1Ko1BTTwuX0JgMAACQiJPdQA3OzdNpRBXrolTI1RKJhlwMAAJBUCMk92JySYm2trNXid7eGXQoAAEBSIST3YB8bO1ADemVoARP4AAAA9kFI7sHSU1N0/uRC/ePtzdpeVRt2OQAAAEmDkNzDzSkpVkPU9bdXy8MuBQAAIGkQknu4owbnamJRXy1cXiZ3D7scAACApEBIhuaUFGvVpkq9Ub4r7FIAAACSAiEZ+tTEocpMS9H80vVhlwIAAJAUCMlQ3+x0zRo/WIte26Ca+kjY5QAAAISOkAxJ0tySYu2uadCTKzeFXQoAAEDoCMmQJJ0waoAK87JZMxkAAECEZARSUkwXTS3Si2u2qWxHddjlAAAAhIqQjCYXTS2Su/TQctZMBgAAPRshGU2K++fopCMGaMHy9YpGWTMZAAD0XO0KyWY2y8zeMbPVZvbdZtqHm9k/zex1M3vOzIri2iJm9lpwW9SRxaPjzZlarLIde/Xy2u1hlwIAABCaNkOymaVK+rWksySNk3SJmY1L2O2/Jf3F3Y+VdJOk/4xr2+vuk4LbuR1UNzrJrPGDlZuVpgXLmcAHAAB6rvb0JE+TtNrd17p7naQHJJ2XsM84Sc8Ej59tph1dRFZ6qs6dOFSPv7FRu2vqwy4HAAAgFO0JyYWS4i/FVhZsi7dC0gXB4/Ml5ZrZgOB5lpmVmtnLZjb7UIrF4TG3pFi1DVFde98rWl/BShcAAKDn6aiJe9+UNNPMXpU0U1K5pMZLtw139xJJl0r6uZmNbu4AZnZ1EKZLt27d2kFl4WBMLM7TzbPHa/kHO3TGz5bojy+8rwgT+QAAQA/SnpBcLqk47nlRsK2Ju29w9wvcfbKk7wXbdgb35cH9WknPSZrc3Enc/U53L3H3koKCggN8G+hoV0wfrqevn6npo/rr5kff0gW/eVFvb9wddlkAAACHRXtC8jJJY8xspJllSLpY0j6rVJhZvpk1HusGSXcF2/uZWWbjPpJOkvRWRxWPzlWYl627rjpOt18yWWU79upTv3xBtz65SjX1kbZfDAAA0IW1GZLdvUHSdZKelPS2pPnuvtLMbjKzxtUqTpX0jpm9K2mQpJ8E24+WVGpmKxSb0HeLuxOSuxAz07kTh+of18/U7MmF+vWza3TWL55niTgAANCtmXvyjTUtKSnx0tLSsMtAM154b5tu+NvrWl+xV5dMK9Z3zzpafbPTwy4LAADggJnZ8mDu3H644h4OyMlj8vXU12bqi6eM0oPL1uv02xbriTc3hl0WAABAhyIk44BlZ6Tqhk8erb9fe7IG5mbqmntf0dV/KdWmXTVhlwYAANAhCMk4aBOK+urv156kG84aq8XvbtUnbluse1/+QFGWiwMAAF0cIRmHJC01RV+cOVpPfu0UTSjqqxsfflOfvnOpVm+pCrs0AACAg0ZIRocYkd9L9/3b8br1omP17uYqffIXz+v2f76nuoZo2KUBAAAcMEIyOoyZaU5Jsf5x/Uydccwg3fb0uzrnl8/rlQ93hF0aAADAASEko8MV5GbqV5dO0R+vLFFlTYMuvOMl/XDRSlXVNoRdGgAAQLsQktFpPn70ID19/Ux9Zvpw/XnpOp1x22I9s2pz2GUBAAC0iZCMTtU7M00/Om+8Fl5zonplpulzd5fqy/Ne1baq2rBLAwAAaBEhGYfF1OH99NhXZujrpx+pJ9/cpNNvW6wFpeuVjFd8BAAAICTjsMlIS9FXTx+jx796so4o6K1vLXxdl//xX/pg+56wSwMAANgHIRmH3REDczX/iyfo5tnjtWL9Lp358yW6c8kaNURYLg4AACQHQjJCkZJiumL6cD19/SmaMaZA//H4Ks3+zYt6s3xX2KUBAAAQkhGuIX2zdecVU3XHZVO0eXetzvv1i/rPx9/W3rpI2KUBAIAejJCM0JmZzpowRP/4+kzNmVqk3y1ZqzN/vkQvrt4WdmkAAKCHIiQjafTNSdctFx6reV+YrtQU02V/+Je+uWCFdlbXhV0aAADoYQjJSDonjB6g//nqDH3p1NF6+NVynX7bYi1asYHl4gAAwGFDSEZSykpP1bdnjdWi607W0LxsfWXeq/r8n0tVvnNv2KUBAIAegJCMpDZuaB/97Usn6cazj9bSNdt1xm2LdfeL7ysSpVcZAAB0HkIykl5qiunfZozSU18/RVNH9NcPH3lLF/32Jb27uTLs0gAAQDdFSEaXUdw/R3/+7HH62acnat22PTr79ud129PvqraB5eIAAEDHIiSjSzEznT+5SP+4fqbOOXaobv/ne/rkL57XsnUVYZcGAAC6EUIyuqQBvTP1s09P0p8/N0019VHN+e1S3fjwG9pdUx92aQAAoBsgJKNLm3lkgZ76+in6/Mkjdf+/PtQnblusp1ZuCrssAADQxRGS0eX1ykzT/z1nnP72pZPULydDV9+zXF+6b7m27K4JuzQAANBFEZLRbUwsztMjXz5Z3zrzKP3j7S36+G2L9cD/fshFSAAAwAEjJKNbSU9N0bWnHaEnvjpD44b00Xf/+oYuvvNlrd1aFXZpAACgCyEko1saVdBb874wXbdcMEFvbdytWb94Xr9+drXqI9GwSwMAAF0AIRndVkqK6eJpw/TP62fq9KMH6tYn39GnfvmCVqzfGXZpAAAgyRGS0e0N7JOl31w2VXdeMVU7qut0/m9e1E2PvKU9tQ1hlwYAAJIUIRk9xhnHDNbT18/UpccP010vvq8zfrZEz72zJeyyAABAEmpXSDazWWb2jpmtNrPvNtM+3Mz+aWavm9lzZlYU13almb0X3K7syOKBA9UnK10/nj1BC645QVnpKbrqT8v09QdfU8WeurBLAwAAScTaWh7LzFIlvSvpE5LKJC2TdIm7vxW3zwJJj7r7n83sY5I+6+5XmFl/SaWSSiS5pOWSprr7jtbOWVJS4qWlpYfwtoC21TZE9Otn1+iO51ard2aavv+pcZo9qVBmFnZpAADgMDCz5e5e0lxbe3qSp0la7e5r3b1O0gOSzkvYZ5ykZ4LHz8a1nynpaXevCILx05JmHegbADpDZlqqrv/EkXr0yzM0Ir+Xvv7gCl35p2VaX1EddmkAACBk7QnJhZLWxz0vC7bFWyHpguDx+ZJyzWxAO18LhOqowblaeM2J+tG5x2j5ugqd8bMl+sPzaxWJchESAAB6qo6auPdNSTPN7FVJMyWVS4ocyAHM7GozKzWz0q1bt3ZQWUD7pKaYrjxxhJ6+fqZOGD1AP37sbV3wmxf11obdYZcGAABC0J6QXC6pOO55UbCtibtvcPcL3H2ypO8F23a257Vxx7jT3UvcvaSgoKD97wDoQEPzsvXHK0v0y0smq3znXp37qxf0X0+sUk39Af3OBwAAurj2TNxLU2zi3scVC7jLJF3q7ivj9smXVOHuUTP7iaSIu38/mLi3XNKUYNdXFJu4V9HaOZm4h2Sws7pOP3nsbS1YXqYhfbN02tiBOnH0AE0fNUD5vTPDLg8AAByi1ibupbX1YndvMLPrJD0pKVXSXe6+0sxuklTq7osknSrpP83MJS2RdG3w2gozu1mxYC1JN7UVkIFkkZeToVvnTNTsyYW664X39chrG3T/vz6UJI0dnKsTRg/QiaPzdfyo/uqTlR5ytQAAoCO12ZMcBnqSkYwaIlG9Ub5LL63ZrqVrtmvZugrVNkSVYtKEojydOHqAThw9QCXD+ys7IzXscgEAQBta60kmJAMHqbYholc/3BmE5m169cOdaoi60lNNk4f1C0JzviYV5ykjjYtbAgCQbAjJwGGwp7ZBpR/s0Etrtmnpmu16o3yX3KXs9FQdN7J/U0/zMUP7KjWFC5YAABA2QjIQgl3V9Xr5/djQjJfWbNO7m6skSblZaZo+akBTT/ORg3pzlT8AAEJwSBP3ABycvjnpOvOYwTrzmMGSpC2VNXp5bYWWrtmml9Zs19NvbZYk5ffOCEJzvk46YoCG9c8hNAMAEDJ6koGQlO2obpoE+NKabdq8u1aSVJiXHaycMUAnjB6gIX2zQ64UAIDuieEWQJJzd63dtqdpEuDSNdu1o7pekjQqv1fTcnPTR/XXANZoBgCgQxCSgS4mGnWt2lTZNAnwX+9XqKq2QVJsjeaTjsjXiaMHaNrI/spljWYAAA4KIRno4hoiUb1evqtpaEbpuh2qbYgqNcU0obBv0yTAqcP7sUYzACBp1DVEtbumXrv31mt3TUNwX6/dexvittfrY2MH6mNjBx32+gjJQDdTUx9bo7lxEuBr62NrNGekpmjysDydODpfJx4xQBOLWKMZAHDwahsiCYG2paDboMpm9qmpj7Z6/LQUU5/sdH3p1NH6txmjDtO7+gghGejmqmobtGxdRVNP88oNu+Uu5WSk6rgR/Zt6mscN7cMazQDQg9TUR5oNtM0F3cqa/fepbWg75PbNTlef7HT1yUoL7tPVJzstuE9Xblbaftsan2enp4a6ohMhGehhdlbX7bPc3HtbYms094lfo/mIfI0ZyBrNAJCs3F21DdGmQLurlaBbWdN8+K1rI+SmpwYhNytduS0F3RbCb5+sdGWlp3Tp/0dYJxnoYfJyMjRr/GDNGh+s0by7RkvXNi43t11PNa3RnNl0JcATR+eruH92l/7HDgCSjbtrd02DdlXXa+feOu2srteO6rr9g27T4wZVxgXdukjrITcjNSUWWOPCa1G/7KYQm9sUcD8Kun3j9s1M69ohtzPRkwz0QOsrqpuGZry0Zru2VH60RnOsl3mAThiVr8F9s0KuFACSQzTqqqyNhd0d1XXaubdeO6vrtGtvvXbsiQXgWBCOtTc+3rW3XpFoy1krMy2lmaEKsee5+/Xc7t+jm5XOZO1DwXALAC1yd63ZuqdpaMbStdu1s3GN5oJeTb3M00cNUP9eGSFXCwCHJhp1VdY0NPXqNobdndX1wfPGx7Eg3BiKd+2tVytZV7mZaeqbk668nHT1y8lQ3+zY47zsjNh9TobyGrflxA9XIOSGiZAMoN2iUdfbm3brpdWxnub/fb9Ce+oikqRxQ/roxNEDNL6wr4bmZauwX7YG5WYqLZUVNAAcXo1hd/9e3cbnsV7cHUEA3hW3T1thN6/XR+G2b3Ys9DY+zsvJUL8g6PaN2yedfwe7JEIygINWH4nq9bJdTT3NpR/s2GciSGqKaXCfLBXmZWtoXpYK+2XHAnRetoqCxzkZTH8A0LxI1FVZU9/Uq9s0VKH6o7Db3ONde+vVWoTJzUprpSd3317dxud9CLs9DiEZQIepqY+obMdele/cqw0796o8eFwePN60u2a/8Xf9ctKbgnNhv+A+L7upN3pArwwmjgBdWOMFIyqDyWeNS4lVJqy8sLO6Tjsax+q2M+z2yUqLhdiEXt287HT1baZXNy87th9/4UJ7sLoFgA6TlZ6qIwb21hEDezfbHom6Nu+uaQrRZTuCML1zr97ftkcvrN6m6mD4RqPMtJSmAD20b/Z+vdGD+mRxURSgk7i7qmob4oJtYtD9aNWFfZ9/9LittXTNgmEMQajtm5Oh4f1zWu3VzcvJUJ+sNMIuQkNIBtChUlNMQ4Ne4ua4u3btrW/qeW7qkd65V+U7a/T2xi3aVlW7z2vMpEG5WcFwjpygJ3rfoR25WemH4+0BSaeuIarKmvgLQTT23LYccOMDcGVN62N0pdgvso0rLeQG6+YW5mU3rbDQuMxY40UjEvftlZGmFC5khC6GkAzgsDKz4E+nGTpmaN9m96mpj2jjrhqVB73QZXFDO1as36kn3tyo+si+/6vnBv9pF8UF58bhHEV52crvncl/0kg67q49dZGEYQn7h9v4IQuVCVdHa+uyv2ZS78y4MJuVrqF5WRqbldtmuG1sy0xjBQb0PIRkAEknKz1VI/N7aWR+r2bbo1HX1qra/Xujd8SGd/zr/QpV1jTs85qM1BQNyctqGs5RGDdGemhetob0zWIpJrSLu6suElV1bUR76hq0ty6iPXURVdc2xO7rGrSnNnb/Uaj9KADv3tugytrYfVVtQ6tr6EqNF4vYt8d2SN+sfUJvbtz6ufsE3+x09aYXFzgohGQAXU5KimlQnywN6pOlKcP6NbvP7pr6puD8UW90jcp3VOv597ZqS2XtfpOF8ntnNvU8D83L2qc3ujAvW32z05lg2MVEo6699bEw2xhqq+si2lO77311Qrj9KPQ2v39DW+MT4uRmpu0TXIf0zdKRWb336cHdtzd3395dfnkDwkFIBtAt9clKV5/B6Ro7uE+z7XUNUW3aVbPPyhyNY6Pf3rhb/3h7836TkXplpO4zDnpoEKaz0lKVmmJKSzWlpqQoLcWUYo3PTWkpjfcp+z4P2lMtaEuN39d6XCCva4gGvbINTYG1xXAbv702our65ntyEyeJtibFpJyMNOVkpKpXZnCfkab+vTJU3C9n3+1x7dkZqeqVmaqcjNjY25zMj7b3zkxTKr24QJdESAbQI2WkpWjYgBwNG5DTbLu7a1tVXVNwTlyp47X1O5uuTNhZUkz7BuvUWKCOf75f8I4L5CkpaiGYp+yzf8p+r9+3/aPwnqJUk1JTU5o9X2qKyd0/6pVtYQjC/r21Ee2ti6gu0vrY2ngZaSnqlREE08yP7vv1yohtz0xTTnrsvvF5c/v3igvFmWkpPe4XEwAtIyQDQDPMTAW5mSrIzdTE4rxm99lT26BNu2tUH4mqIeKKRF0N0cb7aNPz6D7bXZFoc/vHtkeiirVHE9ojvs/2SGJ73Pk+2t+1NxLZ75zNvW7/WrzNsbLt+xy1TxDNTo/1uublZKiwX2Pva8shtrFnN/E5F3wA0NkIyQBwkHplpml0QfPrRXcH7vuH52gbwbwpFAe9tFnp9M4C6JoIyQCAZlkwrprVvwD0RPy9CgAAAEhASAYAAAASEJIBAACABIRkAAAAIAEhGQAAAEjQrpBsZrPM7B0zW21m322mfZiZPWtmr5rZ62b2yWD7CDPba2avBbffdvQbAAAAADpam0vAmVmqpF9L+oSkMknLzGyRu78Vt9uNkua7+x1mNk7S45JGBG1r3H1Sh1YNAAAAdKL29CRPk7Ta3de6e52kBySdl7CPS+oTPO4raUPHlQgAAAAcXu0JyYWS1sc9Lwu2xfuhpMvNrEyxXuQvx7WNDIZhLDazGYdSLAAAAHA4dNTEvUsk3e3uRZI+KekeM0uRtFHSMHefLOl6SfebWZ/mDmBmV5tZqZmVbt26tYPKAgAAAA5cey5LXS6pOO55UbAt3uclzZIkd19qZlmS8t19i6TaYPtyM1sj6UhJpYkncfc7Jd0pSWa21cw+OMD30hHyJW0L4bxIfnw30Bq+H2gJ3w20hO9GchjeUkN7QvIySWPMbKRi4fhiSZcm7POhpI9LutvMjpaUJWmrmRVIqnD3iJmNkjRG0tq2TujuBe2oq8OZWam7l4RxbiQ3vhtoDd8PtITvBlrCdyP5tRmS3b3BzK6T9KSkVEl3uftKM7tJUqm7L5L0DUm/N7OvKzaJ7yp3dzM7RdJNZlYvKSrpGnev6LR3AwAAAHQAc/ewa0ga/FaHlvDdQGv4fqAlfDfQEr4byY8r7u3rzrALQNLiu4HW8P1AS/huoCV8N5IcPckAAABAAnqSAQAAgASE5ICZzTKzd8xstZl9N+x6kBzMrNjMnjWzt8xspZl9NeyakFzMLDW4YNKjYdeC5GFmeWa20MxWmdnbZnZC2DUheZjZ14P/U940s3nB0rlIMoRkxf6Tk/RrSWdJGifpEjMbF25VSBINkr7h7uMkTZd0Ld8NJPiqpLfDLgJJ5xeSnnD3sZImiu8IAmZWKOkrkkrcfbxiK4ddHG5VaA4hOWaapNXuvtbd6yQ9IOm8kGtCEnD3je7+SvC4UrH/6BIvy44eysyKJJ0t6Q9h14LkYWZ9JZ0i6Y+S5O517r4z1KKQbNIkZZtZmqQcSRtCrgfNICTHFEpaH/e8TAQhJDCzEZImS/pXyKUgefxc0rcVWwceaDRS0lZJfwqG4vzBzHqFXRSSg7uXS/pvxS7EtlHSLnd/Ktyq0BxCMtAOZtZb0kOSvubuu8OuB+Ezs3MkbXH35WHXgqSTJmmKpDvcfbKkPZKY6wJJkpn1U+yv1SMlDZXUy8wuD7cqNIeQHFMuqTjueVGwDZCZpSsWkO9z97+GXQ+SxkmSzjWzdYoN0fqYmd0bbklIEmWSyty98a9OCxULzYAknS7pfXff6u71kv4q6cSQa0IzCMkxyySNMbORZpah2AD6RSHXhCRgZqbYuMK33f22sOtB8nD3G9y9yN1HKPZvxjPuTm8Q5O6bJK03s6OCTR+X9FaIJSG5fChpupnlBP/HfFxM7ExKaWEXkAzcvcHMrpP0pGKzTO9y95Uhl4XkcJKkKyS9YWavBdv+j7s/Hl5JALqAL0u6L+h4WSvpsyHXgyTh7v8ys4WSXlFsBaVXxdX3khJX3AMAAAASMNwCAAAASEBIBgAAABIQkgEAAIAEhGQAAAAgASEZAAAASEBIBgAAABIQkgEAAIAEhGQAAAAgwf8HcDhSXVd7SX4AAAAASUVORK5CYII=\n"
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "testmodel = train(ds_train, ds_val, max_epochs=MAX_EPOCHS, lr=LR, batch_size=BATCH_SIZE)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}