Lookahead_radam_seed17.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Lookahead_radam_seed17.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "ehioQb4Kq5Ny",
        "colab_type": "code",
        "outputId": "a32687f2-c89f-46b1-ed4e-93244f6895bc",
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7Ci8vIE1heCBhbW91bnQgb2YgdGltZSB0byBibG9jayB3YWl0aW5nIGZvciB0aGUgdXNlci4KY29uc3QgRklMRV9DSEFOR0VfVElNRU9VVF9NUyA9IDMwICogMTAwMDsKCmZ1bmN0aW9uIF91cGxvYWRGaWxlcyhpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IHN0ZXBzID0gdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKTsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIC8vIENhY2hlIHN0ZXBzIG9uIHRoZSBvdXRwdXRFbGVtZW50IHRvIG1ha2UgaXQgYXZhaWxhYmxlIGZvciB0aGUgbmV4dCBjYWxsCiAgLy8gdG8gdXBsb2FkRmlsZXNDb250aW51ZSBmcm9tIFB5dGhvbi4KICBvdXRwdXRFbGVtZW50LnN0ZXBzID0gc3RlcHM7CgogIHJldHVybiBfdXBsb2FkRmlsZXNDb250aW51ZShvdXRwdXRJZCk7Cn0KCi8vIFRoaXMgaXMgcm91Z2hseSBhbiBhc3luYyBnZW5lcmF0b3IgKG5vdCBzdXBwb3J0ZWQgaW4gdGhlIGJyb3dzZXIgeWV0KSwKLy8gd2hlcmUgdGhlcmUgYXJlIG11bHRpcGxlIGFzeW5jaHJvbm91cyBzdGVwcyBhbmQgdGhlIFB5dGhvbiBzaWRlIGlzIGdvaW5nCi8vIHRvIHBvbGwgZm9yIGNvbXBsZXRpb24gb2YgZWFjaCBzdGVwLgovLyBUaGlzIHVzZXMgYSBQcm9taXNlIHRvIGJsb2NrIHRoZSBweXRob24gc2lkZSBvbiBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcCwKLy8gdGhlbiBwYXNzZXMgdGhlIHJlc3VsdCBvZiB0aGUgcHJldmlvdXMgc3RlcCBhcyB0aGUgaW5wdXQgdG8gdGhlIG5leHQgc3RlcC4KZnVuY3Rpb24gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpIHsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIGNvbnN0IHN0ZXBzID0gb3V0cHV0RWxlbWVudC5zdGVwczsKCiAgY29uc3QgbmV4dCA9IHN0ZXBzLm5leHQob3V0cHV0RWxlbWVudC5sYXN0UHJvbWlzZVZhbHVlKTsKICByZXR1cm4gUHJvbWlzZS5yZXNvbHZlKG5leHQudmFsdWUucHJvbWlzZSkudGhlbigodmFsdWUpID0+IHsKICAgIC8vIENhY2hlIHRoZSBsYXN0IHByb21pc2UgdmFsdWUgdG8gbWFrZSBpdCBhdmFpbGFibGUgdG8gdGhlIG5leHQKICAgIC8vIHN0ZXAgb2YgdGhlIGdlbmVyYXRvci4KICAgIG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSA9IHZhbHVlOwogICAgcmV0dXJuIG5leHQudmFsdWUucmVzcG9uc2U7CiAgfSk7Cn0KCi8qKgogKiBHZW5lcmF0b3IgZnVuY3Rpb24gd2hpY2ggaXMgY2FsbGVkIGJldHdlZW4gZWFjaCBhc3luYyBzdGVwIG9mIHRoZSB1cGxvYWQKICogcHJvY2Vzcy4KICogQHBhcmFtIHtzdHJpbmd9IGlucHV0SWQgRWxlbWVudCBJRCBvZiB0aGUgaW5wdXQgZmlsZSBwaWNrZXIgZWxlbWVudC4KICogQHBhcmFtIHtzdHJpbmd9IG91dHB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIG91dHB1dCBkaXNwbGF5LgogKiBAcmV0dXJuIHshSXRlcmFibGU8IU9iamVjdD59IEl0ZXJhYmxlIG9mIG5leHQgc3RlcHMuCiAqLwpmdW5jdGlvbiogdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKSB7CiAgY29uc3QgaW5wdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQoaW5wdXRJZCk7CiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gZmFsc2U7CgogIGNvbnN0IG91dHB1dEVsZW1lbnQgPSBkb2N1bWVudC5nZXRFbGVtZW50QnlJZChvdXRwdXRJZCk7CiAgb3V0cHV0RWxlbWVudC5pbm5lckhUTUwgPSAnJzsKCiAgY29uc3QgcGlja2VkUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBpbnB1dEVsZW1lbnQuYWRkRXZlbnRMaXN0ZW5lcignY2hhbmdlJywgKGUpID0+IHsKICAgICAgcmVzb2x2ZShlLnRhcmdldC5maWxlcyk7CiAgICB9KTsKICB9KTsKCiAgY29uc3QgY2FuY2VsID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnYnV0dG9uJyk7CiAgaW5wdXRFbGVtZW50LnBhcmVudEVsZW1lbnQuYXBwZW5kQ2hpbGQoY2FuY2VsKTsKICBjYW5jZWwudGV4dENvbnRlbnQgPSAnQ2FuY2VsIHVwbG9hZCc7CiAgY29uc3QgY2FuY2VsUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBjYW5jZWwub25jbGljayA9ICgpID0+IHsKICAgICAgcmVzb2x2ZShudWxsKTsKICAgIH07CiAgfSk7CgogIC8vIENhbmNlbCB1cGxvYWQgaWYgdXNlciBoYXNuJ3QgcGlja2VkIGFueXRoaW5nIGluIHRpbWVvdXQuCiAgY29uc3QgdGltZW91dFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgc2V0VGltZW91dCgoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9LCBGSUxFX0NIQU5HRV9USU1FT1VUX01TKTsKICB9KTsKCiAgLy8gV2FpdCBmb3IgdGhlIHVzZXIgdG8gcGljayB0aGUgZmlsZXMuCiAgY29uc3QgZmlsZXMgPSB5aWVsZCB7CiAgICBwcm9taXNlOiBQcm9taXNlLnJhY2UoW3BpY2tlZFByb21pc2UsIHRpbWVvdXRQcm9taXNlLCBjYW5jZWxQcm9taXNlXSksCiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdzdGFydGluZycsCiAgICB9CiAgfTsKCiAgaWYgKCFmaWxlcykgewogICAgcmV0dXJuIHsKICAgICAgcmVzcG9uc2U6IHsKICAgICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICAgIH0KICAgIH07CiAgfQoKICBjYW5jZWwucmVtb3ZlKCk7CgogIC8vIERpc2FibGUgdGhlIGlucHV0IGVsZW1lbnQgc2luY2UgZnVydGhlciBwaWNrcyBhcmUgbm90IGFsbG93ZWQuCiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gdHJ1ZTsKCiAgZm9yIChjb25zdCBmaWxlIG9mIGZpbGVzKSB7CiAgICBjb25zdCBsaSA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2xpJyk7CiAgICBsaS5hcHBlbmQoc3BhbihmaWxlLm5hbWUsIHtmb250V2VpZ2h0OiAnYm9sZCd9KSk7CiAgICBsaS5hcHBlbmQoc3BhbigKICAgICAgICBgKCR7ZmlsZS50eXBlIHx8ICduL2EnfSkgLSAke2ZpbGUuc2l6ZX0gYnl0ZXMsIGAgKwogICAgICAgIGBsYXN0IG1vZGlmaWVkOiAkewogICAgICAgICAgICBmaWxlLmxhc3RNb2RpZmllZERhdGUgPyBmaWxlLmxhc3RNb2RpZmllZERhdGUudG9Mb2NhbGVEYXRlU3RyaW5nKCkgOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnbi9hJ30gLSBgKSk7CiAgICBjb25zdCBwZXJjZW50ID0gc3BhbignMCUgZG9uZScpOwogICAgbGkuYXBwZW5kQ2hpbGQocGVyY2VudCk7CgogICAgb3V0cHV0RWxlbWVudC5hcHBlbmRDaGlsZChsaSk7CgogICAgY29uc3QgZmlsZURhdGFQcm9taXNlID0gbmV3IFByb21pc2UoKHJlc29sdmUpID0+IHsKICAgICAgY29uc3QgcmVhZGVyID0gbmV3IEZpbGVSZWFkZXIoKTsKICAgICAgcmVhZGVyLm9ubG9hZCA9IChlKSA9PiB7CiAgICAgICAgcmVzb2x2ZShlLnRhcmdldC5yZXN1bHQpOwogICAgICB9OwogICAgICByZWFkZXIucmVhZEFzQXJyYXlCdWZmZXIoZmlsZSk7CiAgICB9KTsKICAgIC8vIFdhaXQgZm9yIHRoZSBkYXRhIHRvIGJlIHJlYWR5LgogICAgbGV0IGZpbGVEYXRhID0geWllbGQgewogICAgICBwcm9taXNlOiBmaWxlRGF0YVByb21pc2UsCiAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgYWN0aW9uOiAnY29udGludWUnLAogICAgICB9CiAgICB9OwoKICAgIC8vIFVzZSBhIGNodW5rZWQgc2VuZGluZyB0byBhdm9pZCBtZXNzYWdlIHNpemUgbGltaXRzLiBTZWUgYi82MjExNTY2MC4KICAgIGxldCBwb3NpdGlvbiA9IDA7CiAgICB3aGlsZSAocG9zaXRpb24gPCBmaWxlRGF0YS5ieXRlTGVuZ3RoKSB7CiAgICAgIGNvbnN0IGxlbmd0aCA9IE1hdGgubWluKGZpbGVEYXRhLmJ5dGVMZW5ndGggLSBwb3NpdGlvbiwgTUFYX1BBWUxPQURfU0laRSk7CiAgICAgIGNvbnN0IGNodW5rID0gbmV3IFVpbnQ4QXJyYXkoZmlsZURhdGEsIHBvc2l0aW9uLCBsZW5ndGgpOwogICAgICBwb3NpdGlvbiArPSBsZW5ndGg7CgogICAgICBjb25zdCBiYXNlNjQgPSBidG9hKFN0cmluZy5mcm9tQ2hhckNvZGUuYXBwbHkobnVsbCwgY2h1bmspKTsKICAgICAgeWllbGQgewogICAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgICBhY3Rpb246ICdhcHBlbmQnLAogICAgICAgICAgZmlsZTogZmlsZS5uYW1lLAogICAgICAgICAgZGF0YTogYmFzZTY0LAogICAgICAgIH0sCiAgICAgIH07CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPQogICAgICAgICAgYCR7TWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCl9JSBkb25lYDsKICAgIH0KICB9CgogIC8vIEFsbCBkb25lLgogIHlpZWxkIHsKICAgIHJlc3BvbnNlOiB7CiAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgIH0KICB9Owp9CgpzY29wZS5nb29nbGUgPSBzY29wZS5nb29nbGUgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYiA9IHNjb3BlLmdvb2dsZS5jb2xhYiB8fCB7fTsKc2NvcGUuZ29vZ2xlLmNvbGFiLl9maWxlcyA9IHsKICBfdXBsb2FkRmlsZXMsCiAgX3VwbG9hZEZpbGVzQ29udGludWUsCn07Cn0pKHNlbGYpOwo=",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 366
        }
      },
      "source": [
        "from google.colab import files\n",
        "files.upload()"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-a7834f34-617a-4c27-a3be-aca9e2a1ccd5\" name=\"files[]\" multiple disabled />\n",
              "     <output id=\"result-a7834f34-617a-4c27-a3be-aca9e2a1ccd5\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Saving SGDW.py to SGDW.py\n",
            "Saving RAdam.py to RAdam.py\n",
            "Saving OneCycle.py to OneCycle.py\n",
            "Saving Lookahead.py to Lookahead.py\n",
            "Saving CLR.py to CLR.py\n",
            "Saving AdamW.py to AdamW.py\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'AdamW.py': b'import math\\nimport torch\\nfrom torch.optim.optimizer import Optimizer\\n\\nclass AdamW(Optimizer):\\n    r\"\"\"Implements Adam algorithm with decoupled weight decay.\\n\\n    It has been proposed in `DECOUPLED WEIGHT DECAY REGULARIZATION`_.\\n    Below is modification of adam code from pytorch repository(https://github.com/pytorch/pytorch)\\n\\n    Arguments:\\n        params (iterable): iterable of parameters to optimize or dicts defining\\n            parameter groups\\n        lr (float, optional): learning rate (default: 1e-3)\\n        betas (Tuple[float, float], optional): coefficients used for computing\\n            running averages of gradient and its square (default: (0.9, 0.999))\\n        eps (float, optional): term added to the denominator to improve\\n            numerical stability (default: 1e-8)\\n        weight_decay (float, optional): weight decay (default: 0)\\n        amsgrad (boolean, optional): whether to use the AMSGrad variant of this\\n            algorithm from the paper `On the Convergence of Adam and Beyond`_\\n            (default: False)\\n           \\n    .. _DECOUPLED WEIGHT DECAY REGULARIZATION:\\n        https://arxiv.org/pdf/1711.05101.pdf\\n    \"\"\"\\n\\n    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8,\\n                 weight_decay=0, amsgrad=False):\\n        if not 0.0 <= lr:\\n            raise ValueError(\"Invalid learning rate: {}\".format(lr))\\n        if not 0.0 <= eps:\\n            raise ValueError(\"Invalid epsilon value: {}\".format(eps))\\n        if not 0.0 <= betas[0] < 1.0:\\n            raise ValueError(\"Invalid beta parameter at index 0: {}\".format(betas[0]))\\n        if not 0.0 <= betas[1] < 1.0:\\n            raise ValueError(\"Invalid beta parameter at index 1: {}\".format(betas[1]))\\n        defaults = dict(lr=lr, betas=betas, eps=eps,\\n                        weight_decay=weight_decay, amsgrad=amsgrad)\\n        super(AdamW, self).__init__(params, defaults)\\n\\n    def __setstate__(self, state):\\n        super(AdamW, self).__setstate__(state)\\n        for group in self.param_groups:\\n            group.setdefault(\\'amsgrad\\', False)\\n\\n    def step(self, closure=None):\\n        \"\"\"Performs a single optimization step.\\n\\n        Arguments:\\n            closure (callable, optional): A closure that reevaluates the model\\n                and returns the loss.\\n        \"\"\"\\n        loss = None\\n        if closure is not None:\\n            loss = closure()\\n\\n        for group in self.param_groups:\\n            for p in group[\\'params\\']:\\n                if p.grad is None:\\n                    continue\\n                grad = p.grad.data\\n                if grad.is_sparse:\\n                    raise RuntimeError(\\'Adam does not support sparse gradients, please consider SparseAdam instead\\')\\n                amsgrad = group[\\'amsgrad\\']\\n\\n                state = self.state[p]\\n\\n                # State initialization\\n                if len(state) == 0:\\n                    state[\\'step\\'] = 0\\n                    # Exponential moving average of gradient values\\n                    state[\\'exp_avg\\'] = torch.zeros_like(p.data)\\n                    # Exponential moving average of squared gradient values\\n                    state[\\'exp_avg_sq\\'] = torch.zeros_like(p.data)\\n                    if amsgrad:\\n                        # Maintains max of all exp. moving avg. of sq. grad. values\\n                        state[\\'max_exp_avg_sq\\'] = torch.zeros_like(p.data)\\n\\n                exp_avg, exp_avg_sq = state[\\'exp_avg\\'], state[\\'exp_avg_sq\\']\\n                if amsgrad:\\n                    max_exp_avg_sq = state[\\'max_exp_avg_sq\\']\\n                beta1, beta2 = group[\\'betas\\']\\n\\n                state[\\'step\\'] += 1\\n\\n                old = p.data.float()\\n\\n                # Decay the first and second moment running average coefficient\\n                exp_avg.mul_(beta1).add_(1 - beta1, grad)\\n                exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad)\\n                if amsgrad:\\n                    # Maintains the maximum of all 2nd moment running avg. till now\\n                    torch.max(max_exp_avg_sq, exp_avg_sq, out=max_exp_avg_sq)\\n                    # Use the max. for normalizing running avg. of gradient\\n                    denom = max_exp_avg_sq.sqrt().add_(group[\\'eps\\'])\\n                else:\\n                    denom = exp_avg_sq.sqrt().add_(group[\\'eps\\'])\\n\\n                bias_correction1 = 1 - beta1 ** state[\\'step\\']\\n                bias_correction2 = 1 - beta2 ** state[\\'step\\']\\n                \\n                step_size = group[\\'lr\\'] * math.sqrt(bias_correction2) / bias_correction1\\n                    \\n                p.data.addcdiv_(-step_size, exp_avg, denom)\\n                \\n                if group[\\'weight_decay\\'] != 0:\\n                    p.data.add_(-group[\\'weight_decay\\'] * group[\\'lr\\'], old)\\n                \\n        return loss',\n",
              " 'CLR.py': b'import math\\nimport matplotlib.pyplot as plt\\n\\n\\nclass CLR(object):\\n    \"\"\"\\n    The method is described in paper : https://arxiv.org/abs/1506.01186 to find out optimum \\n    learning rate. The learning rate is increased from lower value to higher per iteration \\n    for some iterations till loss starts exploding.The learning rate one power lower than \\n    the one where loss is minimum is chosen as optimum learning rate for training.\\n\\n    Args:\\n        optim   Optimizer used in training.\\n\\n        bn      Total number of iterations used for this test run.\\n                The learning rate increasing factor is calculated based on this \\n                iteration number.\\n\\n        base_lr The lower boundary for learning rate which will be used as\\n                initial learning rate during test run. It is adviced to start from\\n                small learning rate value like 1e-4.\\n                Default value is 1e-5\\n\\n        max_lr  The upper boundary for learning rate. This value defines amplitude\\n                for learning rate increase(max_lr-base_lr). max_lr value may not be \\n                reached in test run as loss may explode before reaching max_lr.\\n                It is adviced to use higher value like 10, 100.\\n                Default value is 100.\\n\\n    \"\"\"\\n    def __init__(self, optim, bn, base_lr=1e-7, max_lr=100):\\n        self.base_lr = base_lr\\n        self.max_lr = max_lr\\n        self.optim = optim\\n        self.bn = bn - 1\\n        ratio = self.max_lr/self.base_lr\\n        self.mult = ratio ** (1/self.bn)\\n        self.best_loss = 1e9\\n        self.iteration = 0\\n        self.lrs = []\\n        self.losses = []\\n        \\n    def calc_lr(self, loss):\\n        self.iteration +=1\\n        if math.isnan(loss) or loss > 4 * self.best_loss:\\n            return -1\\n        if loss < self.best_loss and self.iteration > 1:\\n            self.best_loss = loss\\n            \\n        mult = self.mult ** self.iteration\\n        lr = self.base_lr * mult\\n        \\n        self.lrs.append(lr)\\n        self.losses.append(loss)\\n        \\n        return lr\\n        \\n    def plot(self, start=10, end=-5):\\n        plt.xlabel(\"Learning Rate\")\\n        plt.ylabel(\"Losses\")\\n        plt.plot(self.lrs[start:end], self.losses[start:end])\\n        plt.xscale(\\'log\\')\\n        \\n        \\n    def plot_lr(self):\\n        plt.xlabel(\"Iterations\")\\n        plt.ylabel(\"Learning Rate\")\\n        plt.plot(self.lrs)\\n        plt.yscale(\\'log\\')',\n",
              " 'Lookahead.py': b\"import torch\\nfrom torch.optim import Optimizer\\nfrom collections import defaultdict\\n\\nclass Lookahead(Optimizer):\\n    r'''Implements Lookahead optimizer.\\n\\n    It's been proposed in paper: Lookahead Optimizer: k steps forward, 1 step back\\n    (https://arxiv.org/pdf/1907.08610.pdf)\\n\\n    Args:\\n        optimizer: The optimizer object used in inner loop for fast weight updates.\\n        alpha:     The learning rate for slow weight update.\\n                   Default: 0.5\\n        k:         Number of iterations of fast weights updates before updating slow\\n                   weights.\\n                   Default: 5\\n\\n    Example:\\n        > optim = Lookahead(optimizer)\\n        > optim = Lookahead(optimizer, alpha=0.6, k=10)\\n    '''\\n    def __init__(self, optimizer, alpha=0.5, k=5):\\n        assert(0.0 <= alpha <= 1.0)\\n        assert(k >= 1)\\n        self.optimizer = optimizer\\n        self.alpha = alpha\\n        self.k = k\\n        self.k_counter = 0\\n        self.param_groups = self.optimizer.param_groups\\n        self.state = defaultdict(dict)\\n        self.slow_weights = [[param.clone().detach() for param in group['params']] for group in self.param_groups]\\n    \\n    def step(self, closure=None):\\n        loss = self.optimizer.step(closure)\\n        self.k_counter += 1\\n        if self.k_counter >= self.k:\\n            for group, slow_weight in zip(self.param_groups, self.slow_weights):\\n                for param, weight in zip(group['params'], slow_weight):\\n                    weight.data.add_(self.alpha, (param.data - weight.data))\\n                    param.data.copy_(weight.data)\\n            self.k_counter = 0\\n        return loss\\n\\n    def __getstate__(self):\\n        return {\\n            'state': self.state,\\n            'optimizer': self.optimizer,\\n            'alpha': self.alpha,\\n            'k': self.k,\\n            'k_counter': self.k_counter\\n        }\\n\\n    def state_dict(self):\\n        return self.optimizer.state_dict()\\n\\n    def load_state_dict(self, state_dict):\\n        self.optimizer.load_state_dict(state_dict)\",\n",
              " 'OneCycle.py': b'import math\\n\\nclass OneCycle(object):\\n    \"\"\"\\n    In paper (https://arxiv.org/pdf/1803.09820.pdf), author suggests to do one cycle during \\n    whole run with 2 steps of equal length. During first step, increase the learning rate \\n    from lower learning rate to higher learning rate. And in second step, decrease it from \\n    higher to lower learning rate. This is Cyclic learning rate policy. Author suggests one \\n    addition to this. - During last few hundred/thousand iterations of cycle reduce the \\n    learning rate to 1/100th or 1/1000th of the lower learning rate.\\n\\n    Also, Author suggests that reducing momentum when learning rate is increasing. So, we make \\n    one cycle of momentum also with learning rate - Decrease momentum when learning rate is \\n    increasing and increase momentum when learning rate is decreasing.\\n\\n    Args:\\n        nb              Total number of iterations including all epochs\\n\\n        max_lr          The optimum learning rate. This learning rate will be used as highest \\n                        learning rate. The learning rate will fluctuate between max_lr to\\n                        max_lr/div and then (max_lr/div)/div.\\n\\n        momentum_vals   The maximum and minimum momentum values between which momentum will\\n                        fluctuate during cycle.\\n                        Default values are (0.95, 0.85)\\n\\n        prcnt           The percentage of cycle length for which we annihilate learning rate\\n                        way below the lower learnig rate.\\n                        The default value is 10\\n\\n        div             The division factor used to get lower boundary of learning rate. This\\n                        will be used with max_lr value to decide lower learning rate boundary.\\n                        This value is also used to decide how much we annihilate the learning \\n                        rate below lower learning rate.\\n                        The default value is 10.\\n\\n        use_cosine      Use cosine annealation instead of linear to change learning rate and \\n                        momentum. \\n                        The default value is False\\n    \"\"\"\\n    def __init__(self, nb, max_lr, momentum_vals=(0.95, 0.85), prcnt= 10, div=10, use_cosine=False):\\n        self.nb = nb\\n        self.div = div\\n        self.high_lr = max_lr\\n        self.low_mom = momentum_vals[1]\\n        self.high_mom = momentum_vals[0]\\n        self.use_cosine = use_cosine\\n        if self.use_cosine:\\n            self.prcnt = 0\\n        else:\\n            self.prcnt = prcnt\\n        self.iteration = 0\\n        self.lrs = []\\n        self.moms = []\\n        if self.use_cosine:\\n            self.step_len =  int(self.nb / 4)\\n        else:\\n            self.step_len =  int(self.nb * (1- prcnt/100)/2)\\n        \\n    def calc(self):\\n        if self.use_cosine:\\n            lr = self.calc_lr_cosine()\\n            mom = self.calc_mom_cosine()\\n        else:\\n            lr = self.calc_lr()\\n            mom = self.calc_mom()\\n        self.iteration += 1\\n        return (lr, mom)\\n        \\n    def calc_lr(self):\\n        if self.iteration ==  0:\\n            self.lrs.append(self.high_lr/self.div)\\n            return self.high_lr/self.div\\n        elif self.iteration == self.nb:\\n            self.iteration = 0\\n            self.lrs.append(self.high_lr/self.div)\\n            return self.high_lr/self.div\\n        elif self.iteration > 2 * self.step_len:\\n            ratio = (self.iteration - 2 * self.step_len) / (self.nb - 2 * self.step_len)\\n            #lr = self.high_lr * ( 1 - 0.99 * ratio)/self.div\\n            lr = (self.high_lr / self.div) * (1- ratio * (1 - 1/self.div))\\n        elif self.iteration > self.step_len:\\n            ratio = 1- (self.iteration -self.step_len)/self.step_len\\n            lr = self.high_lr * (1 + ratio * (self.div - 1)) / self.div\\n        else :\\n            ratio = self.iteration/self.step_len\\n            lr = self.high_lr * (1 + ratio * (self.div - 1)) / self.div\\n        self.lrs.append(lr)\\n        return lr\\n\\n    def calc_mom(self):\\n        if self.iteration == 0:\\n            self.moms.append(self.high_mom)\\n            return self.high_mom\\n        elif self.iteration == self.nb:\\n            self.iteration = 0\\n            self.moms.append(self.high_mom)\\n            return self.high_mom\\n        elif self.iteration > 2 * self.step_len:\\n            mom = self.high_mom\\n        elif self.iteration > self.step_len:\\n            ratio = (self.iteration -self.step_len)/self.step_len\\n            mom = self.low_mom + ratio * (self.high_mom - self.low_mom)\\n        else :\\n            ratio = self.iteration/self.step_len\\n            mom = self.high_mom - ratio * (self.high_mom - self.low_mom)\\n        self.moms.append(mom)\\n        return mom\\n\\n    def calc_lr_cosine(self):\\n        if self.iteration ==  0:\\n            self.lrs.append(self.high_lr/self.div)\\n            return self.high_lr/self.div\\n        elif self.iteration == self.nb:\\n            self.iteration = 0\\n            self.lrs.append(self.high_lr/self.div)\\n            return self.high_lr/self.div\\n        elif self.iteration > self.step_len:\\n            ratio = (self.iteration -self.step_len)/(self.nb - self.step_len)\\n            lr = (self.high_lr/self.div) + 0.5 * (self.high_lr - self.high_lr/self.div) * (1 + math.cos(math.pi * ratio))\\n        else :\\n            ratio = self.iteration/self.step_len\\n            lr = self.high_lr - 0.5 * (self.high_lr - self.high_lr/self.div) * (1 + math.cos(math.pi * ratio))\\n        self.lrs.append(lr)\\n        return lr\\n\\n    def calc_mom_cosine(self):\\n        if self.iteration == 0:\\n            self.moms.append(self.high_mom)\\n            return self.high_mom\\n        elif self.iteration == self.nb:\\n            self.iteration = 0\\n            self.moms.append(self.high_mom)\\n            return self.high_mom\\n        elif self.iteration > self.step_len:\\n            ratio = (self.iteration -self.step_len)/(self.nb - self.step_len)\\n            mom = self.high_mom - 0.5 * (self.high_mom - self.low_mom) * (1 + math.cos(math.pi * ratio))\\n        else :\\n            ratio = self.iteration/self.step_len\\n            mom = self.low_mom + 0.5 * (self.high_mom - self.low_mom) * (1 + math.cos(math.pi * ratio))\\n        self.moms.append(mom)\\n        return mom\\n',\n",
              " 'RAdam.py': b'import math\\nimport torch\\nfrom torch.optim import Optimizer\\n\\n\\nclass RAdam(Optimizer):\\n    r\"\"\"Implements RAdam algorithm.\\n    It has been proposed in `ON THE VARIANCE OF THE ADAPTIVE LEARNING\\n    RATE AND BEYOND(https://arxiv.org/pdf/1908.03265.pdf)`_.\\n    \\n    Arguments:\\n        params (iterable):      iterable of parameters to optimize or dicts defining\\n                                parameter groups\\n        lr (float, optional):   learning rate (default: 1e-3)\\n        betas (Tuple[float, float], optional):  coefficients used for computing\\n                                                running averages of gradient and \\n                                                its square (default: (0.9, 0.999))\\n        eps (float, optional):  term added to the denominator to improve\\n                                numerical stability (default: 1e-8)\\n        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)\\n        amsgrad (boolean, optional):    whether to use the AMSGrad variant of this\\n                                        algorithm from the paper `On the Convergence \\n                                        of Adam and Beyond`_(default: False)\\n        \\n        sma_thresh:             simple moving average threshold.\\n                                Length till where the variance of adaptive lr is intracable.\\n                                Default: 4 (as per paper)\\n    \"\"\"\\n    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8,\\n                 weight_decay=0, amsgrad=False, sma_thresh=4):\\n        if not 0.0 <= lr:\\n            raise ValueError(\"Invalid learning rate: {}\".format(lr))\\n        if not 0.0 <= eps:\\n            raise ValueError(\"Invalid epsilon value: {}\".format(eps))\\n        if not 0.0 <= betas[0] < 1.0:\\n            raise ValueError(\"Invalid beta parameter at index 0: {}\".format(betas[0]))\\n        if not 0.0 <= betas[1] < 1.0:\\n            raise ValueError(\"Invalid beta parameter at index 1: {}\".format(betas[1]))\\n        defaults = dict(lr=lr, betas=betas, eps=eps,\\n                        weight_decay=weight_decay, amsgrad=amsgrad)\\n        super(RAdam, self).__init__(params, defaults)\\n\\n        self.radam_buffer = [[None, None, None] for ind in range(10)]\\n        self.sma_thresh = sma_thresh\\n\\n    def __setstate__(self, state):\\n        super(RAdam, self).__setstate__(state)\\n        for group in self.param_groups:\\n            group.setdefault(\\'amsgrad\\', False)\\n\\n    def step(self, closure=None):\\n        loss = None\\n        if closure is not None:\\n            loss = closure()\\n\\n        for group in self.param_groups:\\n            for p in group[\\'params\\']:\\n                if p.grad is None:\\n                    continue\\n\\n                # Perform optimization step\\n                grad = p.grad.data\\n                if grad.is_sparse:\\n                    raise RuntimeError(\\'Adam does not support sparse gradients, please consider SparseAdam instead\\')\\n                amsgrad = group[\\'amsgrad\\']\\n\\n                state = self.state[p]\\n\\n                # State initialization\\n                if len(state) == 0:\\n                    state[\\'step\\'] = 0\\n                    # Exponential moving average of gradient values\\n                    state[\\'exp_avg\\'] = torch.zeros_like(p.data)\\n                    # Exponential moving average of squared gradient values\\n                    state[\\'exp_avg_sq\\'] = torch.zeros_like(p.data)\\n                    if amsgrad:\\n                        # Maintains max of all exp. moving avg. of sq. grad. values\\n                        state[\\'max_exp_avg_sq\\'] = torch.zeros_like(p.data)\\n\\n                exp_avg, exp_avg_sq = state[\\'exp_avg\\'], state[\\'exp_avg_sq\\']\\n                if amsgrad:\\n                    max_exp_avg_sq = state[\\'max_exp_avg_sq\\']\\n                beta1, beta2 = group[\\'betas\\']\\n\\n                state[\\'step\\'] += 1\\n                old = p.data.float()\\n\\n                # Decay the first and second moment running average coefficient\\n                exp_avg.mul_(beta1).add_(1 - beta1, grad)\\n                exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad)\\n\\n                buffer = self.radam_buffer[int(state[\\'step\\']%10)]\\n\\n                if buffer[0] == state[\\'step\\']:\\n                    sma_t, step_size = buffer[1], buffer[2]\\n                else:                 \\n                    sma_max_len = 2/(1-beta2) - 1  \\n                    beta2_t = beta2 ** state[\\'step\\']\\n                    sma_t = sma_max_len - 2 * state[\\'step\\'] * beta2_t /(1 - beta2_t)\\n                    buffer[0] = state[\\'step\\']\\n                    buffer[1] = sma_t\\n\\n                    if sma_t > self.sma_thresh :\\n                        rt = math.sqrt(((sma_t - 4) * (sma_t - 2) * sma_max_len)/((sma_max_len -4) * (sma_max_len - 2) * sma_t))\\n                        step_size = group[\\'lr\\'] * rt * math.sqrt((1 - beta2_t)) / (1 - beta1 ** state[\\'step\\'])                      \\n                    else:\\n                        step_size = group[\\'lr\\'] / (1 - beta1 ** state[\\'step\\'])                        \\n                    buffer[2] = step_size\\n\\n                if group[\\'weight_decay\\'] != 0:\\n                    p.data.add_(-group[\\'weight_decay\\'] * group[\\'lr\\'], old)\\n\\n                if sma_t > self.sma_thresh :\\n                    denom = exp_avg_sq.sqrt().add_(group[\\'eps\\'])\\n                    p.data.addcdiv_(-step_size, exp_avg, denom)\\n                else:\\n                    p.data.add_(-step_size, exp_avg)\\n\\n        return loss\\n',\n",
              " 'SGDW.py': b'import torch\\nfrom torch.optim.optimizer import Optimizer, required\\n\\n\\nclass SGDW(Optimizer):\\n    r\"\"\"Implements stochastic gradient descent with decoupled weight decay(optionally with momentum).\\n\\n    It has been proposed in `DECOUPLED WEIGHT DECAY REGULARIZATION`_.\\n    Below is modification of adam code from pytorch repository(https://github.com/pytorch/pytorch).\\n\\n    Args:\\n        params (iterable): iterable of parameters to optimize or dicts defining\\n            parameter groups\\n        lr (float): learning rate\\n        momentum (float, optional): momentum factor (default: 0)\\n        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)\\n        dampening (float, optional): dampening for momentum (default: 0)\\n        nesterov (bool, optional): enables Nesterov momentum (default: False)\\n\\n    Example:\\n        >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9)\\n        >>> optimizer.zero_grad()\\n        >>> loss_fn(model(input), target).backward()\\n        >>> optimizer.step()\\n\\n    __ http://www.cs.toronto.edu/%7Ehinton/absps/momentum.pdf\\n\\n    .. note::\\n        The implementation of SGD with Momentum/Nesterov subtly differs from\\n        Sutskever et. al. and implementations in some other frameworks.\\n\\n        Considering the specific case of Momentum, the update can be written as\\n\\n        .. math::\\n                  v = \\\\rho * v + g \\\\\\\\\\n                  p = p - lr * v\\n\\n        where p, g, v and :math:`\\\\rho` denote the parameters, gradient,\\n        velocity, and momentum respectively.\\n\\n        This is in contrast to Sutskever et. al. and\\n        other frameworks which employ an update of the form\\n\\n        .. math::\\n             v = \\\\rho * v + lr * g \\\\\\\\\\n             p = p - v\\n\\n        .. _DECOUPLED WEIGHT DECAY REGULARIZATION:\\n        https://arxiv.org/pdf/1711.05101.pdf\\n    \"\"\"\\n\\n    def __init__(self, params, lr=required, momentum=0, dampening=0,\\n                 weight_decay=0, nesterov=False):\\n        if lr is not required and lr < 0.0:\\n            raise ValueError(\"Invalid learning rate: {}\".format(lr))\\n        if momentum < 0.0:\\n            raise ValueError(\"Invalid momentum value: {}\".format(momentum))\\n        if weight_decay < 0.0:\\n            raise ValueError(\"Invalid weight_decay value: {}\".format(weight_decay))\\n\\n        defaults = dict(lr=lr, momentum=momentum, dampening=dampening,\\n                        weight_decay=weight_decay, nesterov=nesterov)\\n        if nesterov and (momentum <= 0 or dampening != 0):\\n            raise ValueError(\"Nesterov momentum requires a momentum and zero dampening\")\\n        super(SGDW, self).__init__(params, defaults)\\n\\n    def __setstate__(self, state):\\n        super(SGDW, self).__setstate__(state)\\n        for group in self.param_groups:\\n            group.setdefault(\\'nesterov\\', False)\\n\\n    def step(self, closure=None):\\n        \"\"\"Performs a single optimization step.\\n\\n        Arguments:\\n            closure (callable, optional): A closure that reevaluates the model\\n                and returns the loss.\\n        \"\"\"\\n        loss = None\\n        if closure is not None:\\n            loss = closure()\\n\\n        for group in self.param_groups:\\n            weight_decay = group[\\'weight_decay\\']\\n            momentum = group[\\'momentum\\']\\n            dampening = group[\\'dampening\\']\\n            nesterov = group[\\'nesterov\\']\\n\\n            for p in group[\\'params\\']:\\n                if p.grad is None:\\n                    continue\\n                d_p = p.grad.data\\n                \\n                old = torch.clone(p.data).detach()\\n                #if weight_decay != 0:\\n                #    d_p.add_(weight_decay, p.data)\\n                if momentum != 0:\\n                    param_state = self.state[p]\\n                    if \\'momentum_buffer\\' not in param_state:\\n                        buf = param_state[\\'momentum_buffer\\'] = torch.zeros_like(p.data)\\n                        buf.mul_(momentum).add_(d_p)\\n                    else:\\n                        buf = param_state[\\'momentum_buffer\\']\\n                        buf.mul_(momentum).add_(1 - dampening, d_p)\\n                    if nesterov:\\n                        d_p = d_p.add(momentum, buf)\\n                    else:\\n                        d_p = buf\\n\\n                p.data.add_(-group[\\'lr\\'], d_p)\\n                \\n                if weight_decay != 0:\\n                    p.data.add_(-weight_decay, old)\\n\\n        return loss\\n'}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 1
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o7Dp2pUUqgDN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%matplotlib inline\n",
        "%reload_ext autoreload\n",
        "%autoreload 2\n",
        "%config InlineBackend.figure_format = 'svg'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K7aNg3F9q3zR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import numpy as np\n",
        "from torch.utils.data.dataset import Dataset\n",
        "from torch.utils.data import DataLoader\n",
        "from torchvision import transforms\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "from torch.autograd import Variable\n",
        "from torchvision.datasets import CIFAR10\n",
        "import torchvision.models as models\n",
        "import matplotlib.pyplot as plt\n",
        "import time\n",
        "import math\n",
        "import tqdm as tqdm\n",
        "from Lookahead import Lookahead\n",
        "from AdamW import AdamW\n",
        "from RAdam import RAdam"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gLSYwdcSrXvb",
        "colab_type": "code",
        "outputId": "c76ee199-3464-4546-d6a3-d067581bd64e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "device"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "device(type='cuda')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "anHwr0nYuVC0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "np.random.seed(17)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wzJWgAPIrhwO",
        "colab_type": "text"
      },
      "source": [
        "# Load Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hYVOcHGXrf6s",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "mean = [0.4914, 0.4822, 0.4465]\n",
        "std = [0.2023, 0.1994, 0.2010]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dhPxnE1vrtHT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "normalize = transforms.Normalize(mean=mean, std=std)\n",
        "train_transforms = transforms.Compose([\n",
        "                    transforms.RandomCrop(32, padding=4, padding_mode='reflect'),\n",
        "                    transforms.RandomHorizontalFlip(),\n",
        "                    transforms.ToTensor(),\n",
        "                    normalize\n",
        "                ])\n",
        "valid_transforms = transforms.Compose([\n",
        "                    transforms.ToTensor(),\n",
        "                    normalize\n",
        "                ])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W6rlE2XJrwRV",
        "colab_type": "code",
        "outputId": "109f0883-d592-49a2-8964-b0277c4577cd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "source": [
        "trainset = CIFAR10('./data', train=True, download=True, transform=train_transforms)\n",
        "testset = CIFAR10('./data', train=False, download=True, transform=valid_transforms)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ./data/cifar-10-python.tar.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 99%|█████████▊| 167944192/170498071 [00:11<00:00, 17875864.66it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting ./data/cifar-10-python.tar.gz to ./data\n",
            "Files already downloaded and verified\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KTnDa9Pqr0Cn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "bs=128\n",
        "epochs=90\n",
        "classes = ('plane', 'car', 'bird', 'cat',\n",
        "           'deer', 'dog', 'frog', 'horse', 'ship', 'truck')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5btvC80or6bE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_loader = DataLoader(trainset, batch_size=bs, shuffle=True, num_workers=4)\n",
        "test_loader = DataLoader(testset, batch_size=bs, shuffle=False, num_workers=4)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NiRuMuZEsInt",
        "colab_type": "text"
      },
      "source": [
        "# Plot Images"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L5FF-b2YsHTA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Plot few of the images\n",
        "# This can be used to print predictions too\n",
        "def plot_img(preds=None, is_pred=False):        \n",
        "    fig = plt.figure(figsize=(8,8))\n",
        "    columns = 4\n",
        "    rows = 5\n",
        "\n",
        "    for i in range(1, columns*rows +1):\n",
        "        fig.add_subplot(rows, columns, i)\n",
        "        \n",
        "        if is_pred:\n",
        "            img_xy = np.random.randint(len(testset));\n",
        "            img = testset[img_xy][0].numpy()\n",
        "            #img = img * std + mean\n",
        "        else:\n",
        "            img_xy = np.random.randint(len(trainset));\n",
        "            img = trainset[img_xy][0].numpy()\n",
        "            #img = img * std + mean\n",
        "        \n",
        "        if is_pred:\n",
        "            plt.title(classes[int(preds[img_xy])] + \"/\" + classes[testset[img_xy][1]])\n",
        "        else:\n",
        "            plt.title(classes[trainset[img_xy][1]])\n",
        "        plt.axis('off')\n",
        "        img = np.clip(img, 0, 1)\n",
        "        plt.imshow(np.transpose(img , (1,2,0)))\n",
        "    plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7VXJ9KCzsTX5",
        "colab_type": "code",
        "outputId": "bf8a448f-cd91-4722-e722-ba55db4ba13d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 640
        }
      },
      "source": [
        "plot_img()"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 576x576 with 20 Axes>"
            ],
            "image/svg+xml": "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n  \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<!-- Created with matplotlib (https://matplotlib.org/) -->\n<svg height=\"464.398125pt\" version=\"1.1\" viewBox=\"0 0 438.735832 464.398125\" width=\"438.735832pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n <defs>\n  <style type=\"text/css\">\n*{stroke-linecap:butt;stroke-linejoin:round;}\n  </style>\n </defs>\n <g id=\"figure_1\">\n  <g id=\"patch_1\">\n   <path d=\"M 0 464.398125 \nL 438.735832 464.398125 \nL 438.735832 0 \nL 0 0 \nz\n\" style=\"fill:none;\"/>\n  </g>\n  <g id=\"axes_1\">\n   <g clip-path=\"url(#p304f25154f)\">\n    <image height=\"75\" id=\"image3f885fd8ce\" transform=\"scale(1 -1)translate(0 -75)\" width=\"75\" x=\"7.2\" xlink:href=\"data:image/png;base64,\niVBORw0KGgoAAAANSUhEUgAAAEsAAABLCAYAAAA4TnrqAAAABHNCSVQICAgIfAhkiAAAB4ZJREFUeJztXF1sVEUU/q7cNbjRLJIl6T5Qk24TSkIhFAMSEUsTjGkxKOEnBQMSCIkReOBHK7FCQCJGqw/8KRGxBEEoGhosmhKBmBKyJELCNrEQVkNBF8JK3CJLw0rWh3PuzMC92+70Z1uS+V7m7jkzc+9Ov5l75puztTKZTAZZEOfyVFTaEu1Uno/dAwD4g4/L+knqKpWmz2UllvD52Hag4QIA4FL9p8qd6rM9wqDCYwP9AI8SbMvaAgC4k6kRRj+XIS6rS2WDxjCVrQliVCx6RfjSyQRdpJIAgHhMNvQHRgAACvzUe2zSeOG7H2njq0hPv0deYJilATNYGrAAH63K084I47kTEwAA4z2bEHi9xt52aQtyWcjzNxmXvkgLlY3NNG3PNO+WzvgRvlDeJIMQhlkasAVHTj4rjGVjfwYA1F+oAAAs9GjISzlOt0lbkrtaUkhlZaH0lc+n8m7wGQBAOlUtfK0RWvQ720+ypVnrS+QLhlkasD2tUQoYP6ojZp2aJF3OOuRPUdkWvSd8aT+FE1GuXxmEC0UcepSWjhY2X5p4esbpPGmY9cjDDJYGvKehbwkA4MfV9HFzi3Qda6BXf3kpLdThkNwblkyhcrbH9HNCjRRP34BSJ+VsJp0dwCCFYZYGbGAaXQWrhPHKzdcAAM6b/4spskEqRYzyMUNmvyp9RVym4cZpLq8zedpjd4TvasyJbLlTKDEHlKh3gGGYpQF73CcnAADfrpbGwiyVAeCdl6iMMH1GKT5HpTjO5SkPX5rbJZIp4fOJBcy5c6ybxx4YGGZpwAyWBuyFVd1XUjHGKX3Z68z0sDlbyChPw5KyEcKXSJFIeCN6ngxpPwYjDLM0YJeW0MXdPN3Qz4z0K+Qp4wC3I1UOALh2RJEyBpHUbJilAesyH4WpO5RADg2dF7+6ugiNi84rEFW2Sc5RWEfS3VeBE1cw67bV/SB8l5peyeFp8gPDLA2YwdKA/TdfeE29dp46n9WdELZz52iXdzZyjtoFQ8JXECJlr4Oj89QDm0SaYz4flSMLZewf4Ag+Hqc5+oTn7nLgYZilAbuAL64qxtYWkooPrFkGADgekbkIfzzUQaciCtzo8lakNQ8NUwCaSkjtKhAcCQBIJIhR6VRHLs+edxhmacB29vmq0vB5y1oAQCMzqmvG5ApSEgJp0s/GlDwvPEmWT+PJiwCAzvjgzKoxzNKAGSwNiAOLNqnF4VgDTYOUq3pvQJvQG+30RhhZOFF4wiXku9rO09CIf48+bCfR5a5Co0CQgssAKPC83Qc3GjdnBQAgFOC+/XI3GouSshAIUMDaNy+UvodhlgbMYGnAbuVAepSi0VTPXwUAKA+dBQA01u8UvqMa+7b9f8lE6POcpxZpvgwA+KluvVKzAQAwJKgcQuYBQ0LyfvfjjsjofD+fUo92H4ZZGrC31v0GAFi1VMbwo0poJCsnzQEAFCoa8NGdW7rtdN2hfwEA1VKQwES+/njuXra4o/RwmCpd6u+UhwDtIu4/IHk6jHL0F6lSOvUMszRgAcgAwLqa74Vxcw3lOniJXMuLKQntcIwOFVSVuJPLp8LEyI7Lh9ztORNye4Xl8kmoYnXfhsYPQqG++D0Jn+CgzVXPMEsDZrA0IKahiuiiowCAMV/PyNrwgwk0HX+PSbrueejkZnHtUXH91Ubq6xh/rrK6moYDCWdqxl0ewywNeDLLQfPGgwCA6bVzs3awdUaFuF7ZdDJrvcnzawEAJWFKwdxT1ySdLCPPW0qC4MEv1bNCj4PGfoXzcnG/WAyzNNAlsxysXblLXI/kZIUVH74BADi85l3hm1PXfcAqIbcT+3fQAclETpQrLi5W6uVb2zLM6hOYwdJATtNwuF+mK2+oIUViRS1F+cmWC8I37IVxvXqY9av3AQBiCRnB76uf1as+9WGmYZ/Adv5upxWjI+tOZXk56ZOL8cU2TmUEMSswZazwTeXylx4+TNH8BQCA5WXSNr2KFIxFc5/sYa+a8LESkXbn3xtmacCaymuWLhuGBGhu//fPn8KWjlCgWfDc6wCAW17tuJyJsLB9l7nc7f0OK7HpnGH9uFVirQtJd4BtmKUBM1gasJ0X5VuKsZHLa100vM8Ja5b1tLAtm0Zq4dscadS0uJphA0u07+Uw9VQk1S1imPaZiG3S6iMXDGUJvdNjS2qYpQE7wK9pX0iK9+UJYsjpCO3LHk5gy4ZdJ7tXCFo5pzm2U2Ykh98krctRxkY/vkA2SO/P8e79D8MsDViLV1dlAOD4EakvlfPvd9tjxKzryr/+KOBANcHZuVcV3+0eJs4O5y3GLXFC4lYp84XJVbQenmlyr4eGWRowg6UBa14ZRfAhZYFP9HCBzwXzuNy8Qx5mDKYFfmiIflPYGW9y+QyzNGC9zHvDsGLMJSj1ghOUFqUphPAKSjeJoPSmVt+7FRFgacX7dNEfQalhVt/AdvTA7ZoNc1EdvLCBg9LzljyUyEV1CKh5F/3AKAedqey5FYZZGjCDpQFrFi/wXcvKMtFtyjSK7rd9s9HV2Yucv9BTWbn+Vzo7qVRk5WMN9G9Y8icr83c1snLv8D8s8QX0Exa//gAAAABJRU5ErkJggg==\" y=\"-22.297435\"/>\n   </g>\n   <g id=\"text_1\">\n    <!-- dog -->\n    <defs>\n     <path d=\"M 45.40625 46.390625 \nL 45.40625 75.984375 \nL 54.390625 75.984375 \nL 54.390625 0 \nL 45.40625 0 \nL 45.40625 8.203125 \nQ 42.578125 3.328125 38.25 0.953125 \nQ 33.9375 -1.421875 27.875 -1.421875 \nQ 17.96875 -1.421875 11.734375 6.484375 \nQ 5.515625 14.40625 5.515625 27.296875 \nQ 5.515625 40.1875 11.734375 48.09375 \nQ 17.96875 56 27.875 56 \nQ 33.9375 56 38.25 53.625 \nQ 42.578125 51.265625 45.40625 46.390625 \nz\nM 14.796875 27.296875 \nQ 14.796875 17.390625 18.875 11.75 \nQ 22.953125 6.109375 30.078125 6.109375 \nQ 37.203125 6.109375 41.296875 11.75 \nQ 45.40625 17.390625 45.40625 27.296875 \nQ 45.40625 37.203125 41.296875 42.84375 \nQ 37.203125 48.484375 30.078125 48.484375 \nQ 22.953125 48.484375 18.875 42.84375 \nQ 14.796875 37.203125 14.796875 27.296875 \nz\n\" id=\"DejaVuSans-100\"/>\n     <path d=\"M 30.609375 48.390625 \nQ 23.390625 48.390625 19.1875 42.75 \nQ 14.984375 37.109375 14.984375 27.296875 \nQ 14.984375 17.484375 19.15625 11.84375 \nQ 23.34375 6.203125 30.609375 6.203125 \nQ 37.796875 6.203125 41.984375 11.859375 \nQ 46.1875 17.53125 46.1875 27.296875 \nQ 46.1875 37.015625 41.984375 42.703125 \nQ 37.796875 48.390625 30.609375 48.390625 \nz\nM 30.609375 56 \nQ 42.328125 56 49.015625 48.375 \nQ 55.71875 40.765625 55.71875 27.296875 \nQ 55.71875 13.875 49.015625 6.21875 \nQ 42.328125 -1.421875 30.609375 -1.421875 \nQ 18.84375 -1.421875 12.171875 6.21875 \nQ 5.515625 13.875 5.515625 27.296875 \nQ 5.515625 40.765625 12.171875 48.375 \nQ 18.84375 56 30.609375 56 \nz\n\" id=\"DejaVuSans-111\"/>\n     <path d=\"M 45.40625 27.984375 \nQ 45.40625 37.75 41.375 43.109375 \nQ 37.359375 48.484375 30.078125 48.484375 \nQ 22.859375 48.484375 18.828125 43.109375 \nQ 14.796875 37.75 14.796875 27.984375 \nQ 14.796875 18.265625 18.828125 12.890625 \nQ 22.859375 7.515625 30.078125 7.515625 \nQ 37.359375 7.515625 41.375 12.890625 \nQ 45.40625 18.265625 45.40625 27.984375 \nz\nM 54.390625 6.78125 \nQ 54.390625 -7.171875 48.1875 -13.984375 \nQ 42 -20.796875 29.203125 -20.796875 \nQ 24.46875 -20.796875 20.265625 -20.09375 \nQ 16.0625 -19.390625 12.109375 -17.921875 \nL 12.109375 -9.1875 \nQ 16.0625 -11.328125 19.921875 -12.34375 \nQ 23.78125 -13.375 27.78125 -13.375 \nQ 36.625 -13.375 41.015625 -8.765625 \nQ 45.40625 -4.15625 45.40625 5.171875 \nL 45.40625 9.625 \nQ 42.625 4.78125 38.28125 2.390625 \nQ 33.9375 0 27.875 0 \nQ 17.828125 0 11.671875 7.65625 \nQ 5.515625 15.328125 5.515625 27.984375 \nQ 5.515625 40.671875 11.671875 48.328125 \nQ 17.828125 56 27.875 56 \nQ 33.9375 56 38.28125 53.609375 \nQ 42.625 51.21875 45.40625 46.390625 \nL 45.40625 54.6875 \nL 54.390625 54.6875 \nz\n\" id=\"DejaVuSans-103\"/>\n    </defs>\n    <g transform=\"translate(33.40028 16.318125)scale(0.12 -0.12)\">\n     <use xlink:href=\"#DejaVuSans-100\"/>\n     <use x=\"63.476562\" xlink:href=\"#DejaVuSans-111\"/>\n     <use x=\"124.658203\" xlink:href=\"#DejaVuSans-103\"/>\n    </g>\n   </g>\n  </g>\n  <g id=\"axes_2\">\n   <g clip-path=\"url(#pb58017fcb0)\">\n    <image height=\"75\" id=\"image531b4b9dfc\" transform=\"scale(1 -1)translate(0 -75)\" width=\"75\" x=\"123.652174\" xlink:href=\"data:image/png;base64,\niVBORw0KGgoAAAANSUhEUgAAAEsAAABLCAYAAAA4TnrqAAAABHNCSVQICAgIfAhkiAAACuRJREFUeJztXH1sU9cV/2Xym8AVszbe1LoCV8RT46G4KKkUC5FNCSjZlExKKkEZ0QTbkpWxjmgaZaNIIApdi4BUK91Gx9ryMRTEgjqy1UgNYkYQhII6I+YMOdNMNINmqhokF+kVzdvYH+e8e26wiXxN2P7Y+/3zbu657yPv/e65554P16C7/y4A4MRelCDiBwB8o3mT6jry1tYpQzZvb1Vtx98IAMiNzQcAnBw6rmRFFAEAd2DRef0/VrKunq8BAL6+YycAYDL+YumzVIpO99np8OVAoxKd3ZoEACx2h/jltMy6KDWsED9wVsnC+1IAgE9V/1T/f/CVZZSLtAMAqF+3RHXNCdAnu11IAwCK8YKS1ffTuECYzivGmpXMHwywrI7GRqJKli8QuSfjg9X9Fzr4sgvbNwAAxjNxkYXpcCHDR0c7byDFjRTuxVk+eswygPeyDOCrZNCuwaOqXd/cDgC4EKdpmMkXlczKDwMAUhmamuMpkXVEuwAAq3tIA58cHFGyukCQW3mjh1cIaW17BQAgEGoAAEykT4jMnfmZ6m7jMcsANQDuAsCy7jWq8/SJQ/c/I8Rrs0PL8IKQpiUtYtTkWOlpX23eAgD43bntAICTJ/6hZF3dnwYAzF36AgDgVmKgoof/1h/6AQAH9mqLlLverOmjYyopslSOjiO5iq5/LzxmGUDprOnY9LlYn2rX2WTJXYjTktwQEBMg0k8GYCrEOisuOiscIUP1UpIYdWlUdJZTJD14K2FmOhxYyozSdVY/6azFzS0AgItOQon+lamOUS48ZhnAe1kGqMh0+FHPKtX+ydbnp8jCtqXatTaZBxZb8FZ0XMkK+ZsAgMODNH2XRGTPFgi417D5aDhdslo7P0T3y9LcDFjyfLdKjXMjeMwygA/dtPxO53UYT51XXe6e0IXVGVDt8RyNy2VImZ8aG1Wye70OAd3rEH0CALCgswfAA3odmD1XimR+TPE6sDHqeR3+C/CJntDAXbOCxILheKlZ8WSItigta15SfakxYp3D25zbGC05D6AvXcjLl7PYwmiIkEdiMl5yUuVwb+nqJ3+yZEhvkChltcZUX6FIerLgJ1nAEbOnpZt8dh6zDOC9LAP4dv6W9mybntHcxbz5vxMgS/xOrnDveejoI0u5IfQl1VdnUfuXQ69zjyj/R/3kCKxlk6EtJoq3gWeDndA0brVwb8mq5FrpLMS1Ipk2cxvlfpkkqZDLrEoWhWV8oZH+8JhlAN/H7DRYueeK6jw2sJAaZbZqy1rJ67C67xUAgC16EDlm5GNBUv4rG2VPGWptAwAULfr0tj8oJ7p2Y7GUwcZw1w12jhSnGKx0eMl1myVkJVkUZpYl+IXEhHX9PM5jlgF8rzxSQy1bggsb9+wHAOxO0tZmWUY8C73rlgMAGoKPlF6NybL+9WcBAIUXZKsRjlKgo6WbDNBLh/5ecno2e7Wa/2Eq3FAY25bXNefrHD7eLuMxvZxhRvFMuTyiRzMIHrMM4L0sAyi3MkKicBPvfQAAcMMUvdHHlSw7dI6GN5OZ0KTpaXfXGHE7NOWfT3wEACgEPg8A+CR/V8nqO0kV1HxhKXVkxGFnijkULkQLK+zff0+m00o+th2kQdsO7VOy64nSaediXitdy2OWAcSflRUf0tuHNgIAmtYcAADM107IBOgLbB4gZtkhMTxtXiR6eyg3okFjlh0iRp0f+xsAoKv1idKnqZpR8gxOgNqnUtmSUY6fFpzzRRpzPXN/Nulwx3nMMkBZT+nxAbJGG8JkJlzMyJcLREgjtXWuBgCMp99WsqYI50TwknzqrfeVrO3VrwAAuiKljHIeLI6ART1bVLuxdTYAID1AZs9NbVzbqz8FAFwK3KCOUvKVB4/zmGUA72UZwKd0o7YtW87W76p1zwAArmpW8HwevyTyHQDA+YI40EJFMo3jAwcBAGf2valk7jQshzcG37+vrBJcHhHnZKydnucmpzisDUpQMdZNbuvhpz9b1X08ZhnAt5i3bxGtc9d7/wYgDmdH8zxPJH8FADg+ehgAkE8K7daHycU8MUTmbF4LaeX2/REAEGx9uuSG2wZlkagKeYlxZbJkr8zlv4NBefhah0yAq1Um63jMMoD3sgzg87OV3WSLIrSjNVMGnc+dU+03x54DAFxgv9kszUrv9RO/7QDtpWIQ3+wNjubcTJLMSYr75k5y6EH+hyk4vZV2qPP471xB/Mpn2KCbrPLaHrMM4LvKjrAbK36oOg+PHAQArG7/JgBgSVCCEhOd5Bi8OUIMW26J9XwqeQoAMGrRuv3ddnEr57Jkm3R0fxEA8P19v5ip/6EsCkzc6B7J4R8e2fhA1/SYZYAarCB/1t7fiH/JSf8JANAVfgoA8M/MR0r22rPLAAB1Nhl/wWZx7K95i/IlFnLRxZ83XFay/CgpN3sFmQ41j0/VizONORxp69KMbXeLe6FMeKwSeMwygPeyDOADU/JnW8V18u0NVIAxwUqyKSe8tVNkLdvt5E8OrehSso02rRbbYj+nDv9Tct46OqaTkqX8MDCLVUADu4mOVOqGqQAeswzgcx1bf9khn+DjCFmchRh/pqCUye3qfxcAkBmjvWHY6Vaylhjn0kc4pqjvwZiluxJlkuRnEL283nT0EZWdlHgkPhiqzI18P3jMMoBvYTuFcK/EJe5/dOdJAMDaVqLDcb/s6tdvomhzOMV+rJSkTRajFMriQgtkNGaFeeeTKsxAPsM0KICeq8Oi2p3XtNqdRy0u7eMt2i3Da3vMMoD3sgzg291He8JObRpOpqgd23OEOvxakpkbgQ5SlPrUIaF5fZim4fCJvwIAJjKSedGUpZKTm+mJmXv6MrBt8ioeHX0DAHAxJQ7IOo5xOn463kqb2RUeswzgyybO3Fe4+QS5e3+9aYPqS/OHupanlKGX41JxsbxIUTq/1QEAyGjMGo7/AICw9mHB4mqN41lalJok5IlIlJT/mBuo1C2J6UjGrj6PWQaQLJpp8Pz2Y6rdFqHE26OjiwAAF8fErFjbSArtYpoM1XcTkqUicD2yM7gP0bCgka5fn6XrN4a0yAh7gxMgw/hsWjNjPGbNLLyXZYCKpqGOzVtoSuaSlBpWqyWzRQu0G+gecq36cmXubPmHO6Urx+OdaSpZY1rbDZKoy4tps7DgTBneFJZc2TMZ8p4cQ3V7RI9ZBjBmlosd/ZQ3sKpTAh1wSBO+M0LVRiMpyT6+lqWv+mHWDXtpMbRKsEFr35OfMc+SqHNvlhagzwwSSyMRoeQwM2u/xdPBMavW9JhlgIrKfsthy176cluGpEzuuf7dAIA6rnW5kZOizg9Hq/xRHpcFWdE9C9iF5rDumQ/x1s5mv5llcXV/UJRqA1d8LONS4NOHPGY9NHgvywBVT0OFnJi++19cOc1AA/i1HKcYh8w1S3y2Uw8AqON0olBAptongy8DkHyLvKRUoBiqJZnFpoYtZXzITxNM5HEeswzw4Mx6GGiUwipE6as+yfXTABAJk4kStolRtX5xLdxge+IMJ7gFHGFdXYjOK+bZ3x3SKjCnY1bIK840RtkE3P8ZwsyosOineSGq76gNSf5YOOgyio52Vh7+zBhVabhGS52ms1xmOXn2Z2lmxbTgcR6zDOC9LAP4wBWmSGs/m1JlNm/VcH9ch6fJLFumx2PcDml987kdLtIvuiW1Wudsnqx616+Qt8S0cTgnNMD3mZOWvevtaR5vDhdPecwywH8AmnoCjarBNecAAAAASUVORK5CYII=\" y=\"-22.297435\"/>\n   </g>\n   <g id=\"text_2\">\n    <!-- bird -->\n    <defs>\n     <path d=\"M 48.6875 27.296875 \nQ 48.6875 37.203125 44.609375 42.84375 \nQ 40.53125 48.484375 33.40625 48.484375 \nQ 26.265625 48.484375 22.1875 42.84375 \nQ 18.109375 37.203125 18.109375 27.296875 \nQ 18.109375 17.390625 22.1875 11.75 \nQ 26.265625 6.109375 33.40625 6.109375 \nQ 40.53125 6.109375 44.609375 11.75 \nQ 48.6875 17.390625 48.6875 27.296875 \nz\nM 18.109375 46.390625 \nQ 20.953125 51.265625 25.265625 53.625 \nQ 29.59375 56 35.59375 56 \nQ 45.5625 56 51.78125 48.09375 \nQ 58.015625 40.1875 58.015625 27.296875 \nQ 58.015625 14.40625 51.78125 6.484375 \nQ 45.5625 -1.421875 35.59375 -1.421875 \nQ 29.59375 -1.421875 25.265625 0.953125 \nQ 20.953125 3.328125 18.109375 8.203125 \nL 18.109375 0 \nL 9.078125 0 \nL 9.078125 75.984375 \nL 18.109375 75.984375 \nz\n\" id=\"DejaVuSans-98\"/>\n     <path d=\"M 9.421875 54.6875 \nL 18.40625 54.6875 \nL 18.40625 0 \nL 9.421875 0 \nz\nM 9.421875 75.984375 \nL 18.40625 75.984375 \nL 18.40625 64.59375 \nL 9.421875 64.59375 \nz\n\" id=\"DejaVuSans-105\"/>\n     <path d=\"M 41.109375 46.296875 \nQ 39.59375 47.171875 37.8125 47.578125 \nQ 36.03125 48 33.890625 48 \nQ 26.265625 48 22.1875 43.046875 \nQ 18.109375 38.09375 18.109375 28.8125 \nL 18.109375 0 \nL 9.078125 0 \nL 9.078125 54.6875 \nL 18.109375 54.6875 \nL 18.109375 46.1875 \nQ 20.953125 51.171875 25.484375 53.578125 \nQ 30.03125 56 36.53125 56 \nQ 37.453125 56 38.578125 55.875 \nQ 39.703125 55.765625 41.0625 55.515625 \nz\n\" id=\"DejaVuSans-114\"/>\n    </defs>\n    <g transform=\"translate(149.391204 16.318125)scale(0.12 -0.12)\">\n     <use xlink:href=\"#DejaVuSans-98\"/>\n     <use x=\"63.476562\" xlink:href=\"#DejaVuSans-105\"/>\n     <use x=\"91.259766\" xlink:href=\"#DejaVuSans-114\"/>\n     <use x=\"132.357422\" xlink:href=\"#DejaVuSans-100\"/>\n    </g>\n   </g>\n  </g>\n  <g id=\"axes_3\">\n   <g clip-path=\"url(#p89866d4cbd)\">\n    <image height=\"75\" id=\"imageb11c6952fa\" transform=\"scale(1 -1)translate(0 -75)\" width=\"75\" x=\"240.104348\" xlink:href=\"data:image/png;base64,\niVBORw0KGgoAAAANSUhEUgAAAEsAAABLCAYAAAA4TnrqAAAABHNCSVQICAgIfAhkiAAACMVJREFUeJztXFFsXEcVvQvZqFpR9gNXdJHqH3/USLVSttBVxVKUQvzRLdSpmki4UBfVkUhRLYqTyKFN1dISLFKjJhINohZKSmRBUypTkn4sH1bajZRUxWrYfLgCf2BFbKv6x4RaESv0+Ljnvjm7+/zw1GuBxNyfGc+dmff83rl3zr0zbzMiEslmST/VF/5z9+ifwyIi8lQ2G7c9fddJreyijss2/4CWu+tOV0GZQ3l6nfdqYuMq1NbQ4mOeU/1fS0Y2E1kDVDeUpbzp6K/bRUTkrd5i3FbaPaWVHHVcQtlE2SDdCsplWVsMNfPUZnP0tN0vtQVkeUh4WB7SVTOMRgZ10pPVjzT+B3Dij708FLfdUK1pZSfZ1epHmt6JrR9FaltEOVHQcq7R0T8gy0O2dHW2ZfW4k3DsE/WUvglyGM7/5gdmXaOhYDDv2hrw4hc978/msoVnkXTDKHepdcjqSacr6OoSkOUhm0IdojPqfDL3rI8Rfhrl+zb+xVKsy+xJgQ9AIL4uspzQthfcxIjuIsHuovrLgCwPCQ/LQ7rq4L9ilZI64/sKTvdqo6N7LN9A+TzK5lLPWl1bxcyP2f16aIUx9yFqa2IByfZpOU3mj+sEZHmIc/BZam0m9pWWfgl9okMIqH74Cy1n97sL7VzbUUfjOi4z1Zma+Hy/XvDthbSbWqcYAg19xEbirIb59TnSAWwBWR7ikEX+RVL8S9wvoU9k0fyZP6Pi0JTJfHPNKaM/AonFEvpuS7mB/54EZHlIeFgessWcV0uclCZmfgnjDpzV8ifLz2plxdnqfs3ryRFznI6ky82NV0RE5F3ZIyIiU+XeWDdeW5L/FQnI8pANx4Yfp/rDbbq7iVvee2FCRERunZkUEZFLTFXGR0VEJMq+2DH/FzIZERF5m5f5lY5uGxfMfx3ixmtnnWo/dAFZHrJhZDEh+OWgQmlrVaP0r5HutehV1M6LiMhNx6dj3ZVHAJUeZY3RBx+6gSs7RUTky8drcdMbffBpjyBuSducWKf8DRmMzyRkMKJdIZ/lLeFheciGsw4czT02CD5RTbCLmZe0HNYcw+Onp2LVXqssI2hr7HPjCrpsnOtxqeZPzun8V20BOUPZvIMwV47t1pCqYyhSqLXqojEX0rw+rRQoIMtDnIMnkpi6EVBq7XM9qa6eAPM8ra91mpzl/XiL+b9oeP9SxqWcR1Baevm9MZr0qP3B5PSwiIhsOXiHiIj8q0gcpYnsxAMJZLYtro0oHj6PtlsA0vxRd7gic5vea0CWhzifRfYbZxIthcQE0voBWVd5thkYPtD3hz73xh+Gn5FF7fPgBZfefPAkfNVeTD7wBE1qyafbqe2zIiLyXkFhcMNuYpCjmNdQs5eGHdMi3qGj8wxftNt5Eziv8UEIlYAsDwkPy0OcGXKiH3463hDYTrq0DYEq7BZnCH6z4ijEr41174bjfZnGvWDO1KJL3oF4HeXPqe0tERHpaer8z5Dm0HTbDR531ftwO7dYkpJjzAmrnBARkVeWtkq7BGR5SHJsaGcB6m1/c1uaGBJp0bizoF7/3CRWBgbPh+ZpH0V52elmAA2mM6NYQepIpjWJAzR14mexk32IhkWWg7P/hxm1ZTUu/ENERG7c84lY9T7C2IAsDwkPy0O6ezDE+Jgd3+kh8xhQr//MlHKiJ9icj6Icw8owT8oGFg02Q+N/u3DB4jgp70Z/pLafpDDC3EKSK7H5wcueGh+MVU/v04gkIMtDuossW5Lt+GGTPPxka6zWclEDoC3fHDGYM+bhBhZD8OAbpLRrgtUv/s6p7qm2zpVCgw6ccJmMI3mdMyDLQ7p7TNJyQhYS1tfexrqV6u8YOTQUsX8yHR+WtYRFvNPGPMTqgGuVfJZRBTs9Q6cx26X2ECW4XlCUBWR5SHhYHrJxM2RnbETclmaOvczvwkQvcQbETGw7KECBl3uY1SLZpu2Cx6eQOI19G0rY9IijADKJec2xj9IwM0nc8/N0wumrK9oYkOUhG0dW0hkzc/CcUKzgTc/VO8Zl4EsjI5T9z9HAN7WYvdM1GWJz7Q0sCASzpGunDMPU3dB9UIvbadiBXv2HArI8pLvUoX2jg5E1iHXeEDVEzm5RG79+7HsiIvLa2J9o4Je0mCDfk4PvibPW71L/D1CCX0wm7L4YatjfVjD/vs4t6b/PhXDHW8LD8pDummGaYPmNd47y5OER6f++R/t8q3ZXrPpVGanm7M9c/3GL94wK0O7OKi6wAi+edGjELs3JvxIaezt1R0Lyz182B1mWou2jthm8OXOu/BFFHzxtv3KBU0W3MpzCWfrq0EjctkPuRw1p6Jw7by9N8IO+e7UcIoc9g9LIc4VuMA9qU1Fq89OFTkgGZHnI5nx9b8hK4oqW1OQXZ/7hx4Bbmfbe8CHm9b0ObZ9bUBZ7rt/8GO+oGB8w5knImvq2lhXM3/99p5vVUz6PP6mU43BCNjUgy0PCw/KQzf0RDP7Exb4U7UNjk+zQaMUY0gA1ogIlMOssOeMFfL+8qiZ3vOiCvO/EH+RZIEg0fRmUI3uT3kLtfKza+hAYf8r51IAsD9mcrbBigs6AUQTCioQUyx6UsKbXybvOI3l1mkiszW+Uo8/NdR0Wh2sgp9ty7lzR5XmNEwvYSLlyLIFWpEhAlod0l5TaC076zK63rS+jpwyoNOAwcrQBcRaIIjcW10c745ZrZdxETs9DXJqhXwC4qDd0JYv+ZUK3fW2fckQ0IMtDwsPykO6aoW0k5BN0tpJbTDhArNtOGBegrJP9VuDEV9fp4GHv11bVnrYNu6Th5f4UB7+On2oJyPKQ7iLLXr69JSalFTtFjMYl2q02Uloqtf4tIjIMwjniSUpzCaQUGw8xKR12JrC1GkhpVyVkHUxC1qG7Eh6Wh2zuhgWfnzezsx8843MGZZhO3hwwMfh5Nbnf0s9x7ug/hZqZH6WVV+DY8zu0nPuu002inMX3dc/RIlPWk3g/wte4n6o7P2GeIyDLQ/4N86E8YwAoVacAAAAASUVORK5CYII=\" y=\"-22.297435\"/>\n   </g>\n   <g id=\"text_3\">\n    <!-- deer -->\n    <defs>\n     <path d=\"M 56.203125 29.59375 \nL 56.203125 25.203125 \nL 14.890625 25.203125 \nQ 15.484375 15.921875 20.484375 11.0625 \nQ 25.484375 6.203125 34.421875 6.203125 \nQ 39.59375 6.203125 44.453125 7.46875 \nQ 49.3125 8.734375 54.109375 11.28125 \nL 54.109375 2.78125 \nQ 49.265625 0.734375 44.1875 -0.34375 \nQ 39.109375 -1.421875 33.890625 -1.421875 \nQ 20.796875 -1.421875 13.15625 6.1875 \nQ 5.515625 13.8125 5.515625 26.8125 \nQ 5.515625 40.234375 12.765625 48.109375 \nQ 20.015625 56 32.328125 56 \nQ 43.359375 56 49.78125 48.890625 \nQ 56.203125 41.796875 56.203125 29.59375 \nz\nM 47.21875 32.234375 \nQ 47.125 39.59375 43.09375 43.984375 \nQ 39.0625 48.390625 32.421875 48.390625 \nQ 24.90625 48.390625 20.390625 44.140625 \nQ 15.875 39.890625 15.1875 32.171875 \nz\n\" id=\"DejaVuSans-101\"/>\n    </defs>\n    <g transform=\"translate(263.934628 16.318125)scale(0.12 -0.12)\">\n     <use xlink:href=\"#DejaVuSans-100\"/>\n     <use x=\"63.476562\" xlink:href=\"#DejaVuSans-101\"/>\n     <use x=\"125\" xlink:href=\"#DejaVuSans-101\"/>\n     <use x=\"186.523438\" xlink:href=\"#DejaVuSans-114\"/>\n    </g>\n   </g>\n  </g>\n  <g id=\"axes_4\">\n   <g clip-path=\"url(#p885007983c)\">\n    <image height=\"75\" id=\"imageb01aba5ca7\" transform=\"scale(1 -1)translate(0 -75)\" width=\"75\" x=\"356.556522\" xlink:href=\"data:image/png;base64,\niVBORw0KGgoAAAANSUhEUgAAAEsAAABLCAYAAAA4TnrqAAAABHNCSVQICAgIfAhkiAAAB2tJREFUeJztW2tsVEUUPqO7xmDSNWEJFLUkgKZEFtJKrAIhFEPji9CEiFpeDeCjiBAElUiQ+sBWAQFF6wMjRQENMYKGGCFKVKqBBIjamFUpamssxsa4kBTDaq4/5jt3zt693XZoCzTO9+fMzjkzd3rvN2fOnJkq70nyiIjocfIxFPIn6nu4EnIQ5Emh+yFge5MoN0KOhPwkpO+Luje0/xciFEOp3FQeT6MQh9wT0rJNi6dDVImols1pU5eCPA25Vdj/2rWxdogHRHlTQeYD2/sZ3YutWk7F+ArndNxnst6Ud+PvcMyygHtZFlBeAg7+W1F7G2QzpNSNhyyFLC4xuoIm1LHRMNEQnRzZq+VBoeJya0ASUQrN2kMGn89uIi4q4R6oJFu3rz6zqqhMtJsPuVmLo3uzu3TMsoDyNoJZ+0Wtc/DOwXcXygWlGi4o7WG4l2UB5ZVhGh4wlWOxTrNvnCgarDwXo+oGLobkjclIofs8R7uBkL/nsHHMsoAaQ5pZx0XlnwGjS0X57xyd8Vfl1TtP6L4O2D4W0p4jjXGi7tZ8LYeKQJVDjTsh34mJBhy9IuxpTxnVNOgewu/JBUaXRABeiLp9zUb3PKRjlgUUgVm1ovIo5LuQconlJXUC5Aah4+U6H0GfzGSs2Kklf6XT843O305ht0SCDYQtiRKBMYcHLdx/Utgzy0L6egTUnY3f0p8F0SjKHEA7ZlnAvSwLKM+bo0MHsdXfqvSWm7dOx0SD4ZBHIIvuEEp2ikz9fKFDAwXdFKH6gB/E2Yeo0R3CFCsR+0yPCzwNd4nOKiBDpuEO9HU3hYD3wW3Zqh2QjlkWiBBtyapsI5XxO55lYZIURdIZs3Pl5VsEuuyoOYPxoVClsMOPFaMiYXSzZC4tOCCmt0ybsbMPyY35hOWQQYQHWYwSYUUUdo5ZFlCe58EF/OZX5qkriIjoFH57paKBzHsRkVcT0it/+Vh2XfUkLZ8QKg5QV3twgO0mTlCXaZoOFPYnPHZWSGc2iTwqMp3MlFYx3ufgx9Yzy+WUYZYxowTTluAxjlkWcC/LAhFKIXZfZ2L4UwGjfYGpJ9Gw3ZSvx7SLgt6tIgS4DxG8dOyMZyCnLtAee/+e7OMJGW03TtLzKZnUdu3CvBkLzgn8Fv7dn2ljMa1CvIQ/++S6xVGLY5YF1GjEeC2iMph16Co46/Bvt4bUOXhB4PS39NP5cND9ERCvF0duHGmsAaWuEswvQCfNoFaLCIIfBs0csyygvMMJHTrsMdFfEocXI87HiLoAP1xZzuu8oEibzu8+O0D/PcvD2n+DgnRo7Pc4PSy2amqUlo5ZFnAvywIR4v2Y2Md9f16G0nWk1mkZK4cXjpkgoHVGx9PPR3DKEZlYoV/ARsAxywIRSmPDJBxa2Eb/QsItWN6/vAEeer7x1INzBNCjuRCWu4oGpIhHuJ1jlgXcy7KA8mqQpZUXu66D0rIzPnVZDVm83LjZRG1tln13cTPkR+I+nToYakpEJnszz0O2UN6J8neAmIcFJnZ7Q2lX5ZhlAeWVg1kyNQtn33+Zlrn2iutEmU96/YOE93/0dYPU1UREdOIXffKohsjTSDv4Z5b4+EWHl/q6haP0iF4Kaeex2doqLbfXGSUTiQlWUWV0y7SdY5YF1JlSzayoXEaxbCaR/F8i9lB8Ws0x22yjomrIVKGWjSXFvi4V16cQs5feT0REeYNv9HXB/FlnuAZyLo+h3PiXFvwdJQfSFIT3MZxbq/6D8irNicV7kNMgT24RJxb5eqo5ZlnAvSwLKG+tOeD1EQvIuAjvS3BJvk7nk9tECjj+6Ve60KSXWjV8ptVgxkB+tsgcJ/WrwflxU4MxbIJ/SPGtO7HJK8Qd/DTvG8Xh4LB5Wu7SfS2cYcKZ1Sv1CrfiKT32TdvE7rJcX4JyzLKA8s5s0MyK/iOq8VVS2DemRXItPhEF/ppyZagkIqJXKm8nIqKq+rAL9J2jVvjWR4/hiCMqKjOOE4gyjx54rHw0LceHSxV1mvFqwU5fsx+LUimaeS+LSxxVbxORY5YVIhRdrEtypeWPEwuadw3DyqbqwlkyqyEjG8D/B3S5ZS85kuKlOuBZRYZZExPaZ61KNmXYaFxCRI5ZVnAvywIRpWxzC0ApwvwmMWfY5yezrK0weZFMCmtfkKI/RM2A7j2gUKdYqqsqTN0c7Tqq47szbCQcsyzg31buUeRI+odhSoVOB7xas4aIiPILstkuj/g4RJ5bV0lERG8uqA+aE9+kumu7CTwP7dWB9PHXcQ87aoLZWYvvJSKitza+pivSZvBD79E37h2zLCAus9khjctvURrcoc21I6b75e+SOzu0O8sh+JB+94UvNGseHD89y66B/iIionGWYQi3c8yygHtZFjjradgVNIoNf2JIptPevO1nvzyvYkhvDaFH4ZhlgV5lVsaDAsHvOXpsj8IxywKR3uw8LCadEFLXV+CYZQH3sixw3hy8RF9x9o5ZFuhVB58LM8uWdm50gcExywLnjFl9xS/lgmOWBdzLsoB7WRZwL8sC/wEcguNt6cQ22QAAAABJRU5ErkJggg==\" y=\"-22.297435\"/>\n   </g>\n   <g id=\"text_4\">\n    <!-- horse -->\n    <defs>\n     <path d=\"M 54.890625 33.015625 \nL 54.890625 0 \nL 45.90625 0 \nL 45.90625 32.71875 \nQ 45.90625 40.484375 42.875 44.328125 \nQ 39.84375 48.1875 33.796875 48.1875 \nQ 26.515625 48.1875 22.3125 43.546875 \nQ 18.109375 38.921875 18.109375 30.90625 \nL 18.109375 0 \nL 9.078125 0 \nL 9.078125 75.984375 \nL 18.109375 75.984375 \nL 18.109375 46.1875 \nQ 21.34375 51.125 25.703125 53.5625 \nQ 30.078125 56 35.796875 56 \nQ 45.21875 56 50.046875 50.171875 \nQ 54.890625 44.34375 54.890625 33.015625 \nz\n\" id=\"DejaVuSans-104\"/>\n     <path d=\"M 44.28125 53.078125 \nL 44.28125 44.578125 \nQ 40.484375 46.53125 36.375 47.5 \nQ 32.28125 48.484375 27.875 48.484375 \nQ 21.1875 48.484375 17.84375 46.4375 \nQ 14.5 44.390625 14.5 40.28125 \nQ 14.5 37.15625 16.890625 35.375 \nQ 19.28125 33.59375 26.515625 31.984375 \nL 29.59375 31.296875 \nQ 39.15625 29.25 43.1875 25.515625 \nQ 47.21875 21.78125 47.21875 15.09375 \nQ 47.21875 7.46875 41.1875 3.015625 \nQ 35.15625 -1.421875 24.609375 -1.421875 \nQ 20.21875 -1.421875 15.453125 -0.5625 \nQ 10.6875 0.296875 5.421875 2 \nL 5.421875 11.28125 \nQ 10.40625 8.6875 15.234375 7.390625 \nQ 20.0625 6.109375 24.8125 6.109375 \nQ 31.15625 6.109375 34.5625 8.28125 \nQ 37.984375 10.453125 37.984375 14.40625 \nQ 37.984375 18.0625 35.515625 20.015625 \nQ 33.0625 21.96875 24.703125 23.78125 \nL 21.578125 24.515625 \nQ 13.234375 26.265625 9.515625 29.90625 \nQ 5.8125 33.546875 5.8125 39.890625 \nQ 5.8125 47.609375 11.28125 51.796875 \nQ 16.75 56 26.8125 56 \nQ 31.78125 56 36.171875 55.265625 \nQ 40.578125 54.546875 44.28125 53.078125 \nz\n\" id=\"DejaVuSans-115\"/>\n    </defs>\n    <g transform=\"translate(377.288364 16.318125)scale(0.12 -0.12)\">\n     <use xlink:href=\"#DejaVuSans-104\"/>\n     <use x=\"63.378906\" xlink:href=\"#DejaVuSans-111\"/>\n     <use x=\"124.560547\" xlink:href=\"#DejaVuSans-114\"/>\n     <use x=\"165.673828\" xlink:href=\"#DejaVuSans-115\"/>\n     <use x=\"217.773438\" xlink:href=\"#DejaVuSans-101\"/>\n    </g>\n   </g>\n  </g>\n  <g id=\"axes_5\">\n   <g clip-path=\"url(#p712ee46120)\">\n    <image height=\"75\" id=\"image27d34a061e\" transform=\"scale(1 -1)translate(0 -75)\" width=\"75\" x=\"7.2\" xlink:href=\"data:image/png;base64,\niVBORw0KGgoAAAANSUhEUgAAAEsAAABLCAYAAAA4TnrqAAAABHNCSVQICAgIfAhkiAAADWpJREFUeJztXGtoHNcV/lTvlGTAGogG4gnRQLwQLUbbskuwCFqKlGA1WD/kFrmp3BAntQlOoaHNow2kaUNeOE4caofi4gep3Maua+FIJDJGoki0UsOKsiKsaFeGVejadGQ6DoxdxoGxUX+cM/dcS7LxhKSUMufP7N4zM/fO3O887jnnTtNDHd1LAJC184ip7lcBAHNeBQBgWobiRQEdbw8dAMBV31S89ZYFAAgCuv4jeFhOD+ZLAIA/VqdU2xrQdVnYAIAWR/ozLWqLpBv48AEAVwwaTCOSfq75/IPHucEtKd7VMKIfBh1bbOlnoU5jvuCFK8YMfjVfWclJ6UaUCUFvOTB81Ri3NZuEHsuUGQh8mrKsTbx8R4dcF/h8DsHAajQUbzIklMaIeri7R/FmyjUAQMSwDTQYWSb9jqDNeETjMwwalxXK+Z8yMtYYhNZznjyXyci/4HNbTcZ3UyLQpchKQunLSkAZEwzlyFaNJivmSyEdrxqaguefdZ9487UxxUui4E9MyHXLFbyl9xeG8aCEmB+LZmBqIkraAdcCEulWRwxXrOAtmw1JzlG8VMF/wZRxXFLQ2mTCMaktYiXphXXFa7Zoii+xovc19LRZNFM/3f46AGDL/scV75HidgDASOXkikFc46PPhqXV0lDOCt4LRVFfiRhR7B9opyPk5+gsdRMvFPSYBt0r5HvZwoLLyB2JygCAzyJtgFk6pMhKQBmD9URg1lSjEeQAADk2yQsVQY9tkQA7tgsAaDTEufRZh/TtewwAsPTMY4r3zv73AADVKs3cx5H0F3uQn/K/ec2it7DjGaMJAJodGrMRI8WXC2xWbiY7pRZEZArFAgBgujJJPNNSvCikC1y+Z0s+p3iL7PSmyEpA6ctKQBmTxch3A9VoNUgBWnUy762a9zwfTAAQUf0MmocM97qb1+v/Ur/7tm4BACw25gAAH5/cvcpwaAznQxnL+VXOWlun8RQ6yA2ZrohIZx0SrViZR9q9ZsqT9Fw28SqViuKda5AR81ncYzcIAAxekaTISkAZ3yNnrB7JDMCnt1oL6C0bKy6DhjWxv8V8JwDg6Ct/BgBEWjRgx1PfAQBUajKbSWit9ruf15XvThxbcV6DpaIZhLZSXpzSWoPavAY960Vt3ehF1z/rmeEJxXso3wsgRVYianowX1oCgMASx9MKyAsz2cwbEPk9pXQUtW2AzNzBfScAAEeGyfGcnDiqeCHo/hegIfgW6DY+vrBV+pn3yGP83VRtlSuI1vCxTYuYXOLlzioLGuW23IxSZCWg9GUloKb78j1LAGBAIB2B3IJ4oT9fFRG9oNaCBOmH8zsVr6/UBQDYduDhzzWYe/j4ySq8Ndrva8t4G7Lispxr0PguRxGWUyzS8XPp4vjZ8ntCFpzzrDpSZCWgpvtzpOBz2jqpxo7cTJ0QdS1aGZeK6V6zqH6fDT+fW7CG59rluV4NWf8LlCIrAWVcl5zKdldMc9jgSGeNXIfropTL7G6goekOPvYVaWljF+XkNw+P4Ub0Ujc5fZMT5HKkyPo/oPRlJaBMyGvChUA867jtviK5EPPayv1yQIG2O0My14VAFHzI3v2efb8CANglMb++vwEA8O4wGY07tUFc5XvGZz+h8Q4mfqQvj1JkJaCMmyUFP7B9l2o8PvgqAMDjTLGhra/WsodhcpZ2HhJWbuMIhJ2PMfJVxXPjqD+vETdlJZNdrRKy4pIKO6tYeMige56p3th9+W9RiqwE1DT1/ge03DFkyRBFNNMRN60rylRvfIB0j8lexaZQeHGJwkt/eBMAkOv9luJ9/2sPgPohpGzu6VK8yeHjAIBGjWNIWgDNY5B2bBHX5peDDOvVwgdfIqXISkDpy0pAmdODIwAAry7ugZMlLd7+Yj8AoKAlIkxezK+Lm7RyH8ciV2OxRso41/FPxSuUKLng2pS7M0NR2JZB8uSy8Who8cH4tKAikY9vd1DnpyZusWToC6IUWQmo6Q6YSwDQrlWbzHESw+4hpISBxLrOlwkFsQvRaguy7ICU964tWwEAm3s7FW/k5GkAgGWQxvYaWhibq2AC7jfUUHeJkyZlzXPIlaifBicZZqpa6uxLVPopshJQ+rISUGbzTioFsiIJ/rkGrfdCizA9eXJlUO8yI/9vmrNzb0SiVa+S2A5potbfQx77PGeBfU/LzHDVcDZLPlt7XupNI67/9PZLqVIne/9tJslmjbNQgISJbiVbc49ehc0V0udrN5bjFFkJKBOxgva08kCD23I5mulTB25yBy0vsI6T01c8LnRryD2tbZsAAK0GoaGuKXEjYCPBdQpOR5fclIvt2ifEkGRzZDhcl+omzGOCrFtBVEyfBNrgg5UJjuWUIisBZQz2Mjt7N6nGmdo4AGD45GFquJk51nihT3JfDeJaeUGDPzUNAIg3T7iOVkjGk1qdoAiGrdU9Fg5RjQRsKS6bqlwCAPQXuwAAOVf02QJHJ2KEDX74luK9sJ8qd86PkR78hlb086db8G9TZCWg9GUloKbRv5+hwpCGtgXEJdn6xbNUbfzX0VsLvN3J9eV9WcrurNeyQiU28xuz1Oj7olDZQ8H4FIVoqloY+2f/+DcA4Mizb6u2PYP7AQD9OXJxIogMNXNt+4tcMnSbFu6JK5Dv5v960ups/COOOOl1eTycFFkJKONYNDvT5VHV2M/FWwffOgMAGM1L6dBvT+4FAHjsJlyWejBccOiPYdKs5utS6BawJTjKtQiOZhg2MULizVPTNUFW48CPiOeK0j/LiZHXeXfH17UHMmvk7MYJkQureAQDvbQOHpqqahfyMe5Gv44fI0VWAsoYvIuiXJ5VjQNPbgMAjI4S2hZqIt2vvUdoGwqeAwCcGNVmh1VbNaBlTr8h18Xh5Om4TFJD1nid+pHFkUY+tXbtk6RYmbfqnT5Mrs2Pn5Hl0XHeE/QO51H00rku1kOmTZ23y75N9JUIbR471BYkXG7zzpEUWQkofVkJKFOpEF6zefGo4zabt31MRiOKVwvWAQBO7Gfxk2UZQAEMzHBA4aghgvV0kTA/4JDxqI+JQZlkZRpnICXHDYwfI2OxI/eGaiu4JCJDLMpWUeRp17YBOnLG6Li2VW+WPYwql1LleiV3aVoko16d1YQtGr7Fov5SZCWgzGx1EgBgGqIK47Z2tx0AkCuKC3DOWwQAPL6T1lzlaFjxzJCnbgvNSrks93x1bBAAsIl3K2g+Kdr4GEfNOjVHMt4pMTMsme+NvYSelrjhsGZkDu2gY0Br0YGe7Yo1wLuf3h6jMc9XNLGIwclL0JovMbyA17wpshJQZuQABavac7Kqn2PHDt00K+sd8f1nOU/l8279blfTMGH8DQZSJvWizFx1iq4b4P02lhbdfIM3ei7wfycn/bUXyaQvBtrihMdjZEn/NY1qGz7vot/389+/lLTreLfFLoee9bUxidaWs1zz30P3jgTIiPhZU2QloPRlJaCMX+eEgCFi4df5wxgRmXenW0zzrE/QbbBBsDQR3VwkUxyapOgDXxSv1U2uycbePrqn8aji/fqbjwAA4ixjV1FymMjSWIaOiagN76b776nrfsv19BEfvzslS4XCFJ1/hQ3IhGZk4p8Of3qlVBS3IqindfCJqWn31o4lAJiNJHxQ4KzxD3ufBgDs2fuc4rVtJZTNxd9U0PYud/JHctwsGfUG5hTP40+tWLxJ6dFiv4yiRm3Tg+RIthoSn8rGhXHaB4JeYjcidlo+1h5oeSpsg8aLV5Cn+XhW491Ny2EUeskBzWmuceSnyEpMTWuBJQB4+nlx3t7eTQ5kF8eZdj2/VfEstxkA0AguAgC8QAtocTVbMUtupolFxSrk6Tr4lGxAvVkuY9REPp0/r8XWmhmJuW07pJ869X1kL0VMs3mJEHg+6aURvk4WccBGToS8w2PWEXk3A6nQQ/hzLdGbcaI3RVYCSl9WAmp64qntSwBQ0PYSz1bJ5HO1D35/6DeKF4J21I8Pk3r1WRwBILJuBwBkuQJ6vRbhsw0SMYM/xGMabYqHZRlpryqByJkquzbat65CrqWIP3gBU5T/LNeltnDE5KL+QSLeZlNlP+EDGQH2vvwkjc/ZCABYDGTsXmMeQIqsRJTZ9QzVv48Oj6vGuC3gzPJPfv4DxWvlEOvFOs30Ou37BwMvk6thGXcBAGZr7yuexcrVctkj1LbsgUO54HqLeW23h2dTrNq3xZBc5aqbDv68VKUscbO2PKXh3CzH4rR4Visj/vaA7tnQMnxRRLyI3dNIM07gtW6KrATUNPjhQUqy+lrFi339xthIcwjHGYEBfxMh70q0wmSd05pfDwDo1Ex6jvcBqel0tG0U8VqDEXZ0akixjoy9wv3I+S/sJAQbHGeCVj5md9CnoKIq6a7Z6ozihRw9KNdJ550ztHoLjkRE6kNmguT4c1kpshJQ+rISUMbjMPFFTQxD9sQdh9Z4lith5WyRXN11Jnngjqaoh46R+IQsokFtQfGmJybpejYIBUeU8hyvvRocQg5M8e5L3aSw12nlS07ue3Rd5RAAoFn/xBoXyXks2y2sEgAgMloBAAaPIdTckcDz+NnpPTQC4TmcDU+RlYD+A0LWkzt1OM/AAAAAAElFTkSuQmCC\" y=\"-112.272608\"/>\n   </g>\n   <g id=\"text_5\">\n    <!-- frog -->\n    <defs>\n     <path d=\"M 37.109375 75.984375 \nL 37.109375 68.5 \nL 28.515625 68.5 \nQ 23.6875 68.5 21.796875 66.546875 \nQ 19.921875 64.59375 19.921875 59.515625 \nL 19.921875 54.6875 \nL 34.71875 54.6875 \nL 34.71875 47.703125 \nL 19.921875 47.703125 \nL 19.921875 0 \nL 10.890625 0 \nL 10.890625 47.703125 \nL 2.296875 47.703125 \nL 2.296875 54.6875 \nL 10.890625 54.6875 \nL 10.890625 58.5 \nQ 10.890625 67.625 15.140625 71.796875 \nQ 19.390625 75.984375 28.609375 75.984375 \nz\n\" id=\"DejaVuSans-102\"/>\n    </defs>\n    <g transform=\"translate(32.632468 106.293297)scale(0.12 -0.12)\">\n     <use xlink:href=\"#DejaVuSans-102\"/>\n     <use x=\"35.205078\" xlink:href=\"#DejaVuSans-114\"/>\n     <use x=\"76.287109\" xlink:href=\"#DejaVuSans-111\"/>\n     <use x=\"137.46875\" xlink:href=\"#DejaVuSans-103\"/>\n    </g>\n   </g>\n  </g>\n  <g id=\"axes_6\">\n   <g clip-path=\"url(#p4edf2f0782)\">\n    <image height=\"75\" id=\"image372625ce22\" transform=\"scale(1 -1)translate(0 -75)\" width=\"75\" x=\"123.652174\" xlink:href=\"data:image/png;base64,\niVBORw0KGgoAAAANSUhEUgAAAEsAAABLCAYAAAA4TnrqAAAABHNCSVQICAgIfAhkiAAABOVJREFUeJztm09oXFUUxr+xM6IDMgvHxZPypI6lU0mEjpCxtEhCaZAGaWItwWzqxkVijVIFV7qwK0VcpLSbujALFbGWdqHQwTihRHSCJOCEMqUMxVE6i04X08UUHGRcnHPfvem8TOY0f8Xz25w75757352T79533r2TCIAWlK54aKsH8F9CgyVAgyVAgyVAgyVAgyVAgyVAgyVAgyVAgyVAgyVAgyVAgyVAgyVAgyVAgyVAgyVAgyVAgyVAgyVAgyVAgyVAgyVAgyVAgyVAgyVAgyVAgyVAgyUgOpqKAQAaTeusNsn3W7WxFWPatqiyBGiwBEQPDA8DAOZLtcBZr3C5WtyKMW1bVFkCIkCcfyb5/1nM3xzwAQDlSiXwpXzync1XQtsAqiwRGiwBEXT4tfKL6SwA4GqpsFnjEfEYW3cB+WcD76fKEhB9NkmFO/Vk4Dz5wQ0AwGL+JXKUuutsV4Lso3Wy19ZjhB242zJ3iDtekwLRYGZf3x3UDEx333f+hC33f0HxUGUJiF4LclGblCabJKWL+ZXXqlZr5X/MmJ0aAQAMvH1pzQPsjJH8iON7atkVfRnPfpiudt3zsnZ4BoAqS4QGS0A0zDl+ev+aOu0ZOkKFDZ6G5dxlAEBq0J0yZq+JEopGvf5Afbvt4rgCQJUlomNS+kqa7MWQ1KF1bogK7q5hg/4atRw9GJ6Ya2+3k+1fwoHuDPH1s93jZA5JFtl1U9dr6zx+7RteWPk+lzJkq771XefNF1WWgI7K6sRptvccn9n9SoRcb3wxto4eg3LYgz3ODTznL/10muTSSJKkvLi9o+/TdEg2qLcez65nPy9MAQA+4z6/G7N9HvuK7CkezIHMZFC3VKW+VFkCNFgCHngaric72Jodg3cyds7l+Z216WUC34/n3wcA3GnSA6U0Z7e/LxT+BACkcnkAwNG6Xc3PJOj6Bc4KDqbsGObKZDM8o9+q26l9OUH3VmUJ2BbKMjzC9pNsOvBNFlbf8tgFe/3N+7ZIDnm27tUTtJXwdYlW8aWafaT0cM7xWppW/wvTdotipkp9qrIEhL7ubBX9qYMAgETCTT5WV9b9anJJ+Xafbj5Hr1+1+CAA4PDg4aCumFvka75vazfDAlRlCdBgCYiCE9zRcZux9vWSPN8deXlzRxOjjNxPO8/03Nq6rDXtccbiAqURN/nzqS8/Cuq++fBjAMA90HvtvoxNVQyqLAHR27coc0iGVB7hreO9kcimDKbRpEe6u8DvT1KC+ktt5ZPiTpTKdvGvxWgajZ6bAQCknPfN0fPk+2Fid1s7gypLQKTV6eTBXLRJyjLscMrre2hKyvr271sAAD9mayq823D84SfZ074HosoSoMESsC2noRT3eCXL9nHzeTIb1PV9+isAIMHTz31kmLW+ztNx/r0XgrrCFKUTqiwBXSnLZfbznwAAZ944BMAeGgBm+bTWxaSGJkXZN+a8/40f5wKfPMScx3aRs9KYkyQWWRMm4fScvnxOaMfOsuO5sK8h5HcAqiwRYmW1MbHXluNmu5FXgGWnEqyaAT5CKzvnUUYhaW5XKtu6ApeHss713HGD+6w5yurtYzvR9VfoFlWWAA2WgK6m4ZJTNjOrh62TBIOP6hAPW+HxB1s+LWg409AcDpqfPRXv2rrkHrKesxOB51cb8oagyhLwL3vgKSLNDkOBAAAAAElFTkSuQmCC\" y=\"-112.272608\"/>\n   </g>\n   <g id=\"text_6\">\n    <!-- truck -->\n    <defs>\n     <path d=\"M 18.3125 70.21875 \nL 18.3125 54.6875 \nL 36.8125 54.6875 \nL 36.8125 47.703125 \nL 18.3125 47.703125 \nL 18.3125 18.015625 \nQ 18.3125 11.328125 20.140625 9.421875 \nQ 21.96875 7.515625 27.59375 7.515625 \nL 36.8125 7.515625 \nL 36.8125 0 \nL 27.59375 0 \nQ 17.1875 0 13.234375 3.875 \nQ 9.28125 7.765625 9.28125 18.015625 \nL 9.28125 47.703125 \nL 2.6875 47.703125 \nL 2.6875 54.6875 \nL 9.28125 54.6875 \nL 9.28125 70.21875 \nz\n\" id=\"DejaVuSans-116\"/>\n     <path d=\"M 8.5 21.578125 \nL 8.5 54.6875 \nL 17.484375 54.6875 \nL 17.484375 21.921875 \nQ 17.484375 14.15625 20.5 10.265625 \nQ 23.53125 6.390625 29.59375 6.390625 \nQ 36.859375 6.390625 41.078125 11.03125 \nQ 45.3125 15.671875 45.3125 23.6875 \nL 45.3125 54.6875 \nL 54.296875 54.6875 \nL 54.296875 0 \nL 45.3125 0 \nL 45.3125 8.40625 \nQ 42.046875 3.421875 37.71875 1 \nQ 33.40625 -1.421875 27.6875 -1.421875 \nQ 18.265625 -1.421875 13.375 4.4375 \nQ 8.5 10.296875 8.5 21.578125 \nz\nM 31.109375 56 \nz\n\" id=\"DejaVuSans-117\"/>\n     <path d=\"M 48.78125 52.59375 \nL 48.78125 44.1875 \nQ 44.96875 46.296875 41.140625 47.34375 \nQ 37.3125 48.390625 33.40625 48.390625 \nQ 24.65625 48.390625 19.8125 42.84375 \nQ 14.984375 37.3125 14.984375 27.296875 \nQ 14.984375 17.28125 19.8125 11.734375 \nQ 24.65625 6.203125 33.40625 6.203125 \nQ 37.3125 6.203125 41.140625 7.25 \nQ 44.96875 8.296875 48.78125 10.40625 \nL 48.78125 2.09375 \nQ 45.015625 0.34375 40.984375 -0.53125 \nQ 36.96875 -1.421875 32.421875 -1.421875 \nQ 20.0625 -1.421875 12.78125 6.34375 \nQ 5.515625 14.109375 5.515625 27.296875 \nQ 5.515625 40.671875 12.859375 48.328125 \nQ 20.21875 56 33.015625 56 \nQ 37.15625 56 41.109375 55.140625 \nQ 45.0625 54.296875 48.78125 52.59375 \nz\n\" id=\"DejaVuSans-99\"/>\n     <path d=\"M 9.078125 75.984375 \nL 18.109375 75.984375 \nL 18.109375 31.109375 \nL 44.921875 54.6875 \nL 56.390625 54.6875 \nL 27.390625 29.109375 \nL 57.625 0 \nL 45.90625 0 \nL 18.109375 26.703125 \nL 18.109375 0 \nL 9.078125 0 \nz\n\" id=\"DejaVuSans-107\"/>\n    </defs>\n    <g transform=\"translate(145.747142 106.293297)scale(0.12 -0.12)\">\n     <use xlink:href=\"#DejaVuSans-116\"/>\n     <use x=\"39.208984\" xlink:href=\"#DejaVuSans-114\"/>\n     <use x=\"80.322266\" xlink:href=\"#DejaVuSans-117\"/>\n     <use x=\"143.701172\" xlink:href=\"#DejaVuSans-99\"/>\n     <use x=\"198.681641\" xlink:href=\"#DejaVuSans-107\"/>\n    </g>\n   </g>\n  </g>\n  <g id=\"axes_7\">\n   <g clip-path=\"url(#p9d84ca2e97)\">\n    <image height=\"75\" id=\"image53223339b4\" transform=\"scale(1 -1)translate(0 -75)\" width=\"75\" x=\"240.104348\" xlink:href=\"data:image/png;base64,\niVBORw0KGgoAAAANSUhEUgAAAEsAAABLCAYAAAA4TnrqAAAABHNCSVQICAgIfAhkiAAABgRJREFUeJztm39olVUYxz83diXuqAtmbKOY4QRneI2tcNXuP7pctFkqOFHJtX5S5I+gIglc4vphkf9YiFkYGxrqhBb+IBc5qUHbWAu7kVPYwAluoyVeyVt0idsf57znHpt7d8/73jsHne8/z3mf85wfHL7veZ9znucNACmyjILq7QAMntyqdKEs9t/SIaY8KzgGQE30znE2if5TADw2v0rpvvM57i0+2/+vkJeLTkfbGwHIz29Turd27wdg21PzPfXZr5XDQfE0L1I8of0gCQCCusmQp6EVLLMMYBfLAAFysMG7owKAqu17laa8YiEAS6PieYH2NUhKqX8gklJZFBzfe4JrAIzFxau6beXjqu7zjmHv08Yyywg3gVkuCJYC0Nb1tVJVls8GYJZxZ1cAeGbJvUpjmTWFyInr4BlJsc/093UqVSgYB2BOcYnSlYTzM+hMbGixmD826bDMMoBdLANMr9dQbvCl5VGlKot43eCFfxGJFClNr93gpw43gVlmTmlCyjGth8yc0oFsTPY6WGYZILfMCpWrYjZvHc51nwVggbx10F2JEKJ8dVjsT0MD1nW4KbCLZYCcvIa5uFYu1co9SfF0Lia2/ZLoeI9+jhwx6fPCT4dllgGye+tQvA6A1IUDWevSDT/JvfsOfQph4TrEB8Qn4eGFD6i6X32OZ5llAN971szqD1W55+SrfrszQlnReN03MeGp9ncIp9Qvm3RYZhnALpYBPLyG4vz/+oHfAKhfl645Ig9wIXlm2xj2MzVvGA7OAKClc2wSyxvjPinPaLq7pbTMMoAxs5rPSkZJLzEQWKLVdlxn+0FdrypfPHy/+ew8oFLOq6U46m4I3KqVnZDbn1I+qdWdltIyywABwuXCKY33TWwVTt8epK78CMAjTRcA+LbxnowGenr3HwDseymTYIN/HJHHnLrZD2ra7knbOWzbqOmcE5NllgHsYhkgUFC7JQUwenzHhEYFtVtUeeTYe6JhIGA4lHC3U6lLhu38YU3DKVU+1FzlYikwU8oyTVcvoyWWWQbIC88qBGDUxcixAYh7HSlY67WlL5QEzQIXl6U8renWSv/WMssAdrEMkHc+NjipkW6z5rVjngbq+/vTSW2SrcJ3GwilX531y8Sm3KvZFUi5vLwagPpN76q6wog4KXy8azcALc0ve5rvP1r5opSWWQbIYyA2uZVmE6quybjzh+rSUecyF7sNM+YC0JkUjDrjYgvpj9HevnYhG9oznpMXOAlQllkGCECRDFi4RW7T97eOU/l2q5BbV9+l6qpqDwFw8NhqIPPMF3MHd2rhnBctswxgF8sAxnHD7kvCfNENIismeGLlalU+2tbqr7MpgmWWAYyZVRAREYqRn/1FnQvzZ6jyaCLpYjl9YJllAM+5Dm3XRLPlHtNjTjQ9r8q1jZ8ZtXWOOxukLNHqTki539u0XGGZZQC7WAbwnBiyYu5KAFKXvvTUvia6SHsyew1flPJZeUQIalvBmAzFOPcWP3iZ3ASwzDKA95SjYfH/854daWbVPCfYVpzJobB0cUbDOD/ARTTdPCnjMkl+REtr+F1KJ0X+Nq2dk1Ov31WZwDLLADn5ObOzS0SfKysyiz7/0iZuX2OtewAID6Ujx0MyG0b3UJygyYiUekjCiR4P/sdW7+My3mCZZQC7WAbI6T/SqVQWut65GYD+znTiysE2cdHraPRrSye4cKM4qHOJ95fHqVhmGSCnzGo9nP6LflXdo/46S6aneaLhFQC++qIZgJi2jTsbezaZZa+VPWD671lukM7o+8salWpfdxMA5+Wzngq5VEonLzjTmwmbgOsBdrEMkJNf6HZs2jq5UTYgz6BvdG1XqsU7VwEQ6xCngUhxQtUtqpYnzJiIsMcbm1XdUZdhbpfSMssAWd3gX6ioA+CTrsPZ6jI3OC7Ooh9tTmfYbBu4/i/OhFaulNIyywC+96w3oytU+Z3vpzmjHNQuA6CmvUepRlp3ARAPCceiU2Oac5dmmWUAu1gG8PwaOvniy9fVZ2kqU4+SurWqvD4sLqKvBkV0PLizSdUttXnw5vgXm5ZpQLl6s5AAAAAASUVORK5CYII=\" y=\"-112.272608\"/>\n   </g>\n   <g id=\"text_7\">\n    <!-- dog -->\n    <g transform=\"translate(266.304628 106.293297)scale(0.12 -0.12)\">\n     <use xlink:href=\"#DejaVuSans-100\"/>\n     <use x=\"63.476562\" xlink:href=\"#DejaVuSans-111\"/>\n     <use x=\"124.658203\" xlink:href=\"#DejaVuSans-103\"/>\n    </g>\n   </g>\n  </g>\n  <g id=\"axes_8\">\n   <g clip-path=\"url(#p95ac4061cc)\">\n    <image height=\"75\" id=\"image5664591ae8\" transform=\"scale(1 -1)translate(0 -75)\" width=\"75\" x=\"356.556522\" xlink:href=\"data:image/png;base64,\niVBORw0KGgoAAAANSUhEUgAAAEsAAABLCAYAAAA4TnrqAAAABHNCSVQICAgIfAhkiAAABrlJREFUeJztXG2IVFUYfm6NZANp0ErNj6aPAV3IJXYTt3ARRXYSl2gNllrbXBKRVbAgRfsgP/CPygqyoJtWWKFWZqSkKxm0ErvILjX+2EXW2rGahNHYfozRFCwx/Tjve847n92zzgybnOfPe+55zzn3cOc5733Oe+8dryPalgEAYBKMeOwsACA9oep+hcGipgYAwJcDMdggdqYOAFDfEgIAnDx1XvvaVpboGCabFnVUXnZQ2aZO40rStA4/YTU9X7ij/EPevvAWR5oyAPBtfCDPeQ/ZeTNM3YEzRwEA8RHFt1Wb35jSiXu/bzQHoSQAYDieAAAMzjauH5jwE6ZuILoKALAIxwAAoxPHtK9uTseU5uMHjlkWcBfLAt4jwXAGAH5KJ3x1eLm1RRVSKQBAX79ZvjMQoZJaM9eQ8jXm/W3K7j3RBABYhDrtu4o4ACAhInw7ugAAb20bBQDs37Xb13luFY5ZFvBmAhkA+LuMg65rVPLi/SEjL/ge8SjZy2U8X7XgmGUBD8SsauF5shdE3Y1qTuAW4JhlAXexLBCYSYVqBfgRsv+XpSfhmGUBJ0ot4JhlAZd1sIBjlgXcxbKA59LK/uGYZQGvIxrNAEC41gTcsYFTAIDTMSUh/5ni4HeKsp8xFncrG1pr6sIU7JOiXZwWQZqUSbDG+CbJN6Nf2YtP+5xsCTz1lbKOWRYIxJIqdiQm47qyBuqnCtLxH1Mc3C8j19Gdv10pAnwtfCyVI6IuRFImSYyqF77r5JuMKnuxVThP+ZwQAIh+jTSWY5YF3MWygJfJvJ0BgMdW7tKVl23o6hMdtUoybH13KQDgPI5rX0htCfEsVDRPiD3lIbLNYiwO9ifJ1gofp625/SfCt9PzP9/tIiX6AlnHLAt4mcyPGQBIY4+uHKRImCT117nBKMLFdFtfrbQpQuJ3TWEMADDMx/3mRF21Shc8GOqnNme1j7eeC8mKbSBGyc4XdX+R5RGEctBj8YxfsWBTMfQQyxyzLBDY8uEyAEBzp8lnNev7poodqYP9wqd+x1oSFnIXktS/seJGemlY+xbqG7xi7TzR73qOlQI0RFYkIvA7WWaR3I4xA3eUMe7upbEcsyzgLpYFAkMxtfy2dMowyQtBqfrVwhPUJbXU+sRCZJVdj8MAgFqRyQC2k1VJxkLLivMYV4RvDS3EoJATKTonz+WSaH832Ws9KBt4LMcsC2hRCtwnqq+SVSH0dGqf9rTeq+zucWUjYtOWoO3la5GfqUY+sFhGNp1jDZgh22W32VkGgJEYjFFR5qB/oAySIReOWRZwF8sCgR3blIZKYUhXzg+pwNy8VpGfl57ERxT0ZjeYuq0NlMvQz533iB4cjos/S2QltlOsuTGycdEuV6UHhW9/GZJ9xeCYZYHAwGdKJqQT5jYfo59s7YbiLLhJ6qJRxOnmOk5Ns0wYET2kjCiN+iJlxmmyvIcclpL/PCoGxywLeAuC6mW27/Lv5IWh0lGYS4dXvtmvXX14EwBQT9FEPMhGjY4shU7ELSd163yYXARLBY5nbRWQCYXgmGUBd7EsEPC9/BiUrZlFsXxw5GPtStZlD7Yi64j1QDrnGMiXExPIhXwhahbZk3mtKgvHLAsEbDss36Tsue7fAABpPKx9LBSWFOzJ93dmVCFZkn8T4NJN0Yp7flqlwM5wzLKAL2YdHzcPJdojh7N8QSE26YmWji/hrHa5pULMYh416Zog5b/kRJum9v7cLcMxywLuYlmg5DLs/1O9qbEk2C5q3yHLS9MsQ34abJZcISXOyNb32RhBLi7Ig+q8nJwHxywLFGRWzxeKNUuCR6jmGeHlvJdkhkJQc4pzzZIh3J4ZKdmUzyQG3yzWV1kmFIJjlgUCj9MPHX3dVG5sVV8wDOMuAEDji8Y3Ti+eRfSbYZIhzCiZ12RwbGNxWiqemaj3kGe7H6scHLMs4C6WBbznulXy7/NN4skDBWPPG8rrsPyEsufa6IOnrLc9ORxzblcuodw9YZ3wqfOtoU/ujjzpc/ZVhmOWBQIpve0zX0yUerSwsY1LHKhL7fEkVNC+RO0bVgq5UIHXMisBxywLBC7Ql3MTLabyek6jBVFTXgH+/OE9H8MbCfDAZsXEG/uKtZ3+cMyygLtYFtB/gtEr3vvuIsuvANVnvWk+hv/GegDASz29uuboq1Oe47SBY5YFNLPmig8cBz9QgbmmoAQojj6yLdMgQ1AJOGZZoPQf99Bjr0y33JpwtkBpjh0TRsLunFPeyU03OGZZwF0sC/j6/6x1v5jyoTCnmtXnTJ7Xm9/hNoVjlgX+BWSRxkXs3NsUAAAAAElFTkSuQmCC\" y=\"-112.272608\"/>\n   </g>\n   <g id=\"text_8\">\n    <!-- frog -->\n    <g transform=\"translate(381.988989 106.293297)scale(0.12 -0.12)\">\n     <use xlink:href=\"#DejaVuSans-102\"/>\n     <use x=\"35.205078\" xlink:href=\"#DejaVuSans-114\"/>\n     <use x=\"76.287109\" xlink:href=\"#DejaVuSans-111\"/>\n     <use x=\"137.46875\" xlink:href=\"#DejaVuSans-103\"/>\n    </g>\n   </g>\n  </g>\n  <g id=\"axes_9\">\n   <g clip-path=\"url(#pa1e2343e73)\">\n    <image height=\"75\" id=\"imagea499662f68\" transform=\"scale(1 -1)translate(0 -75)\" width=\"75\" x=\"7.2\" xlink:href=\"data:image/png;base64,\niVBORw0KGgoAAAANSUhEUgAAAEsAAABLCAYAAAA4TnrqAAAABHNCSVQICAgIfAhkiAAABopJREFUeJztW11sVEUU/lZ7TXsTco2ukWvgmrgPraYbwmooTZekGwJiq9AH+oImJopoIITEEo02Gl7E+MCDoBTQB2kUHiAGY1AsAg2hiTWhhGwftsaLcUtcYxaTS+NC3GB9OGd+Spe2U7pQzXwvM50zd2Z65jtnzvxs7OfP/HEAGA4LEOg/R+nSDKUXzkgRWtOUNiZ8AMBIVn03MMSytEeyMJKyOipC3HcAACWUpSxoorbcgCqVHSULB0MAwLpkUtVPpjjnUlIM1QCLRUoL1EbJdaUoX6bxjERUx4l7UuY4NK6TZ3JUV2uSP8M9sJgxai6ViBn16QZZ6KVKAAAnHgcABMmilC3kmboU0gz051VjWSbEnh6aijH1mQaqtDKjSloDGsNoROmQ1mbAbbZlfK2NBCURjaF0pE9KHJcYUmDWOV5cyuLMHpRuSgF4iQAA4GeofpRUViFYZ5llAKssA8QGvyMH39enHPUVskIsbSIqXxgsSdmD7C9XryaziMqKyvkSCQ+zkzylLQzITez4Oc0MffYAEVtaWVkOWlLk2J2CMotMQGbYmFrPJaNaP7Q69bOHDgvq/wrYNOMOpWFe+Qkugu+w01fdIWLXYZllgNiWLowDwFfHVeFlZsECnukxNTlYxCxY105ppDnxzw9ObHyBWrXh8YQt5O9HFVlR5PwN0ZbW30ywpV3lP3q/EwDwTZbimFxBrRaJFHWed6ijclkNPsgTe1pd+qfjCbXgwbUO3hgxgJj1X0ZzQuXXc+x6lUOOZVrE0fbJx5Qp9AMAhqFYV4wov6yhCQDgQg9VLLOMYZVlgJq7PYDpUMvW0JJWZWL719FOK0g5r8KX/btpzX+jk8ramrQYBWRiiP8OAFhcUvFBgUOgfcePAQCuaHvDV9vJ2VtmGUAya6VWKFZ8sfd3NJlY8U9VcVALtFX79Q5KWzMqDvGYbUuTFDMcPqQYshO0TyxGPPqNO7WW+T/h0GHEU6cbLR6dZKwKXuQSLX7JZwFYZhmhZglnPK1Q6LuF0x81magnvrtYhUE9plF56BCl9Y6KYls2CuoRG9zS4KQ2RnPOhDqEDwAAvceOAABODqmgdB0zdxWfuBRySuaW7HbHGFZZBqgRZqTt2/HnTZW+1vIP3KLOXOJiVuWvcTqgzvdQLtPm1Q96AAAdm7XTQsbCOIcJZfVh77tvAQCOclFe25/6Lv2RK5JwWLPsBvY9llkGiC1xaW8YaFuhkvBtQvPa6YHLZ015XlkvarNjgmbN747yKcdlbuthrZ7ouk4bg08nwAh4LL6nLjNyQ+SMkw20YVyWUatFLweciSQ1VuerGOVoH51SDHMU4mor3hgz3TLLALGnAmJWSgsE6wOajdGQZmlxQs3OCJ/7DDEbhjV3cb2aI62AWk6vjf8jy/p3fQoAWLt9EwBlHABw4zb7s8wygFWWASoe/tWy1V0vT/xbLxO4V8s3silf5QXil4r3hnOHJzhNN62WZQcG+ypXngNYZhlg3h8rCye+v7NDlgVxovraHtrjjd2hsVhmGWDen5SKcKTgqqi5u4e2OXeKUQKWWQawyjLAvHfw8wEiPLLMMoBllgEsswxglWUAqywDWGUZoOaF02cBAMVQvWM8sY0OziodK6/58AAA9dirNb1CyjbeF6veSOcBLLMMEBsfH58UOjyz6wsAwIlddIO7putNKfu26/lpGy0wI32NkbmIunl8Bb+qyGrPVDD5Kms+wjLLAFZZBpBmOKAViiuzxZzqt9XicU8Lqgf9NLo3/AsAsH/3Pln200F+Fh1pV9eMt8//AQB4L/XQtP10h3/L/M7l9NRl0+kvAQA57d392aebAVhmGaGigxeoEDncNezR1oCX+UZajCv25KNSduT8rwCA9Zge+t2Lc8taCpZZBqh4rCz8175DPwAAXtuwXMqq6aumwtZA5QXjRRi9qKNTymZiBYc53bB5ryzL7t0MAGic4jvLLANYZRlgSge/h9Ots209/E3lE4/MtpVJEG4i/QqZzsqONin7vv3ZOevnZlhmGWDKe8NVt9u6N32V2UAuMiHtL9uqwKaXtB9nDpyjwNgyywBT+qzZons7+ZId76hfNzje/XPdjdx69Wpls/avM4BllgGssgxw2w9D9BOCHdsoIq4fpJe/1TA9HWL90E2v0o+z5gqWWQaoioP/v8IyywBWWQawyjKAVZYBrLIMYJVlAKssA1hlGcAqywBWWQawyjKAVZYBrLIM8C/De7IRZiH79gAAAABJRU5ErkJggg==\" y=\"-202.24778\"/>\n   </g>\n   <g id=\"text_9\">\n    <!-- bird -->\n    <g transform=\"translate(32.93903 196.26847)scale(0.12 -0.12)\">\n     <use xlink:href=\"#DejaVuSans-98\"/>\n     <use x=\"63.476562\" xlink:href=\"#DejaVuSans-105\"/>\n     <use x=\"91.259766\" xlink:href=\"#DejaVuSans-114\"/>\n     <use x=\"132.357422\" xlink:href=\"#DejaVuSans-100\"/>\n    </g>\n   </g>\n  </g>\n  <g id=\"axes_10\">\n   <g clip-path=\"url(#p80e32c39f8)\">\n    <image height=\"75\" id=\"image159f4a2b40\" transform=\"scale(1 -1)translate(0 -75)\" width=\"75\" x=\"123.652174\" xlink:href=\"data:image/png;base64,\niVBORw0KGgoAAAANSUhEUgAAAEsAAABLCAYAAAA4TnrqAAAABHNCSVQICAgIfAhkiAAABW9JREFUeJztXF1oXFUQ/q7dQLuiCzZQFtoVu2JWaQrJgwHTh6ggUsFEUQpRiCg+1VIf1aagbbUvvlgCgrZQxUQ0Ii2oFItaRDEtmKArNoGupRvpKt0Kq/RaXGV9ODP3TjY3253Nbrql873M2Zl7z5mcfmfOnJ9br1KpVHDN4QIAwD8yCQCIb+kLTdPfAADy0zMAgPlcLjAlEgkAQDabBQAUCoXANjdbdu85gUwirLKnNwkAuKFZ7l8PiKnfmJ5y0i86WSqHthIXfCcynaGtTDq/w8lEMrSl6J+xSHVmp4WNniuLdogR8fJF93tWPD8756ovubo6O+OBqVh07/m5vJMhsdCfdrKL/oY/QxNSVIcxSwHrLAW8+gJ8+Ejx2W0AgLGDLrhmxVMcE2kwIS9sTOubq34DwFmSq0iKwYuBTjdsk/FwOCUSrryhww3NTCoV2Hzf6cq+G0/5XOgFD+R5ctAXI7uHIvovBfdeQdiKFEGMWQrUySwB/0cAwE87nwQAPHww5NbZyBdaj9sidGtISgZ3kWTS/BbxHica/0XYjFkK6JlVjeyHYWWbty3XnyWxSpQ5Nv7RstaiYcxSwDpLAX0GXw05x7YQ/46/Hv4YutvJz13m/u7E0cA0MvlVy3wwZimw7ADveV6zfKmJI8P3BuXB8S8BABup7ZVKWYxZCnjraC3zu1CuI9lDMpHuCGyD6W4AQI7ixe4mOnMTyb9q2ABgB8nXmth2PTBmKWCdpYAHuaWgAGfUUWuoRnEXyZ+bWGczYcxSoCazeKXXI3THSWZIDggbTwO8uyz2/PEcyV/VLrYPjFkKxCaoMCyUrBskWRK2GZLMmseiamWjePEUyZWe7psJY5YC1lkKeAcowPcKJZ/CccDOCBtvu9IePrqFjY8UDpGUBxbHludnW8CYpUDsWyqkhJJ1zCKZAvBzHLtfFTa5vmw2bhFl3k5eTfJyC9sFgEdJGrMU8CYpZu0TylGSzCh5kMrHR8yst5QNTpJ8XPmexHqSu6ok0NpDDGOWAtZZCsQ4PZgXStbxCW5O2LjM9xFWC1s9gZZu9qAiZo17aEx/R7/3iuc/IvmD0D1T5YNcu35Rhw+NwpilQIwDtVz/cZmuigWpBACcILmRpEwr6mHWSySPlZZ+5qIoM4vE1begzBOPTJp57dqKQG/MUsA6S4Egg+8QStbxOe9xYeOTF14HarP2etaIR0W5v0oCoa+8Po0L21aSvC4tYjE4V9QOVWOWAjFmjQzOrOPLvHJq5uA6QFIGeN5t+LpBZ3j9J0+YudwldMwe9kXetuB5g1l3Sth4Qvg74j2esF7Z4v6iwbc/Do2Z+wAYs1RQH4V9OuzSyq3jZ5Z85rMD7i7CQzvvVzmTveRc6b5x8f2JB0WZT6TfJ5kWtg0kOY7Jy7ybaPMtuecFVxjar/LPmKWAdZYC6mG4Puk+Kpo/P6VqaNeLhwEAT+9/CgBw+4KrSq7OM5WpCNti1LpAcp7yiuQ/fK59p8rPWjBmKaC+JtkvP1dTYG3aTcknIq0nAQAvv1kfW5lR2V4Xxjd9f6khn7QwZilQV8zavvu9oDy254mGGlrjbQYAXF6wSR2NO0R5ro2+HTVmKWCdpcAVhqHLfyuVC7paI+4craV0oNZKf2+f28YbnTqta2+FYMxSoGbq8PwbHzRWa/YTAMDMyUOBaowuUyT63H32rqEdgS39wCONtbPCMGYpUDNmXZP/5UMLYcxSwDpLgRhSI66UfyfUss6wAMYsBYJP6PZNnAuUo8O3XjWH2hnGLAWW//X9dQRjlgLWWQpYZylgnaWAdZYC1lkKBPtZlpReGcYsBayzFPCQGnEZfMSuQ+Xc4avgUvvCmKXA/yTTRBVeUlBrAAAAAElFTkSuQmCC\" y=\"-202.24778\"/>\n   </g>\n   <g id=\"text_10\">\n    <!-- truck -->\n    <g transform=\"translate(145.747142 196.26847)scale(0.12 -0.12)\">\n     <use xlink:href=\"#DejaVuSans-116\"/>\n     <use x=\"39.208984\" xlink:href=\"#DejaVuSans-114\"/>\n     <use x=\"80.322266\" xlink:href=\"#DejaVuSans-117\"/>\n     <use x=\"143.701172\" xlink:href=\"#DejaVuSans-99\"/>\n     <use x=\"198.681641\" xlink:href=\"#DejaVuSans-107\"/>\n    </g>\n   </g>\n  </g>\n  <g id=\"axes_11\">\n   <g clip-path=\"url(#pe1b9c972d5)\">\n    <image height=\"75\" id=\"image18c3119317\" transform=\"scale(1 -1)translate(0 -75)\" width=\"75\" x=\"240.104348\" xlink:href=\"data:image/png;base64,\niVBORw0KGgoAAAANSUhEUgAAAEsAAABLCAYAAAA4TnrqAAAABHNCSVQICAgIfAhkiAAABKZJREFUeJztmj9oE1Ecx3/RHOhBDdJAyZKhWYqYpYMuLiIEoQguIrhUsA5twaWLLg5OLo7qqouLo05F6NKlDnVoQR3aIRUOIQhp4SocUof3/d39krtc89M0nvD7QHiXe39y+b3v+73fe+9K197TERGRV6GYTteltZpLq1WR13FpECBP1ItQLwp72yEi2ufrCEmU5Pm+S8/6vWWIiHxu36MUNeTN+Om8HU7bojz+B/9Xfk4iog4/J38PkrwfyDyV/hljEGXuRNnTFSgqQm9Gouf4HpcJhXr6y3ii3iSrE79zKH5vuo4L1AtEr4bo/X2hgnNoq4s22kI9rMSPaOObaGuP28Dv/PiSfnZiJXfSWaYsBWYsBWX2m55woOzc2MF7Ygh4UW+ZHgffdyEd6H7QlyeG4Vfk5Tl4XzpxtFtBXr1OKS4hb1LUSzl4Uc8c/Igp0ZwLHc7MJjd99MB0w6XN2SR22Np0Zt7F3BwKR/hzExd8TzrQjIlAxc3k8gKetQLV1K8keY0Zl56D+vaEQrq4x5OGL8MlzsN3qXyuZ8pSUCJyyspF9Bytn9izjIamS5afuDQSvjjEdQjVyICVy1U5bBJNsoJNWQrMWAqGG4Y1cR0MLFUoplouvdhK7tXh/HmtK9euhKHGQ7Qr8ngiMGUpKNEClCV2FtjKrKgJoawDVhanIvCMQ4YtpBujeswRc9Ult+eTWxyUB/gPMgiuIXg1ZSkoXf/slCVXE21ekuC7DNC4B3hGrsupGeknKOv7Y9Fo1gr/BJhAejDKRuHrTFkKzFgKSvTCDcNlEaXzVOn1b/cS0SHGGu8oyM2/5xzd76DCW+H9RbTsEItR1nm8gEwaffnqHRERLd49n/c/YlYbzi+0dqJjSuoxZSko0WLziIho/UWyuXMR3jhED3vC/Udw4z72X7djVRBdWYJ8Xm7ROJkS1x8uu8Vhc2N0z/AaqSlLQflGyykqEuuY7chdB4FTUShcDwdrtZq7GYkzNG7r3R8ry/mxiYf34jsHT5ePrfVdXN8eoaL6MWUpMGMpKNE81oYZp7oxchbOOBmO4eH6OqfMf4wpS8Fw+1kGEZmyVJTH/5M4X+tZ/7DD/NvzspPFlKXAjKVgjMOwPzaR8UjO8OPT8M2Md4DGjClLQfFDhxU+SRHK8nGCUkHeI7EezNvGquCkossnKeHAolmYshSYsRT8gzhLSRNDRZ5BXnVbSM/mFomIaMUXx+RLORMBD9sQb480ko1LqmLzc30VN9KTjilLQbay+KWAds5h3zBlsuDOHLYa9/iMqIBFwC5BBcGQYUX8rJgF5A6KP+nSClTXTR+nm7IUFD904BOzy+IexHb6gdvS/vVG+Jf7msZFoFy95VJ+uT5aE+Wcck1ZCsxYCoo/DBt3XBq+Se5xpLCAVPriPz7c4SE5OKo3ZSkoflA6zDsLIzkqPH6daMpSUHxl5Z29rWoj3L/DlKXAjKWg+Mbyqu6TReuW+4yJ4hurQBQ/KOWX1mktt9Q4MGUp+A+UVRxMWQrMWArMWArMWArMWArMWArMWArMWArMWArMWArMWArMWArMWArMWArMWArMWArMWArMWAp+A0irIQF5eNNsAAAAAElFTkSuQmCC\" y=\"-202.24778\"/>\n   </g>\n   <g id=\"text_11\">\n    <!-- truck -->\n    <g transform=\"translate(262.199315 196.26847)scale(0.12 -0.12)\">\n     <use xlink:href=\"#DejaVuSans-116\"/>\n     <use x=\"39.208984\" xlink:href=\"#DejaVuSans-114\"/>\n     <use x=\"80.322266\" xlink:href=\"#DejaVuSans-117\"/>\n     <use x=\"143.701172\" xlink:href=\"#DejaVuSans-99\"/>\n     <use x=\"198.681641\" xlink:href=\"#DejaVuSans-107\"/>\n    </g>\n   </g>\n  </g>\n  <g id=\"axes_12\">\n   <g clip-path=\"url(#pe8589aeb86)\">\n    <image height=\"75\" id=\"imageeb3f1fa1e9\" transform=\"scale(1 -1)translate(0 -75)\" width=\"75\" x=\"356.556522\" xlink:href=\"data:image/png;base64,\niVBORw0KGgoAAAANSUhEUgAAAEsAAABLCAYAAAA4TnrqAAAABHNCSVQICAgIfAhkiAAACPpJREFUeJztXE2IHEUUfqMzYAZ0QCfIiLuHncOMwUV2kQ2yQTYXDwm6OUTRCBtiPJhDAhqIgaAHQzTmYMQcFKMJGzCrZg3+BTUeNgcX1EM8RGQVEmFX2AijMBvoBEcZD/O9rm9q3jRbOde7vJ561dXV3V+9+urV68m1X5O2iIhcEieDnl4kmx4PQ18mW0v6SqvR0b8mHX1XxdkKKCvXUbCFTlwPfcYVPfZER6NJeXGjs20ahW2ho680qQ/o3wDuq1RwtkTr4GCFziug3i0SZdWSl4pRqm9YkcJ1St1lrTlnWlzu6Ov4XRuUHmnh/KTkysp6oCj9jk5QZD3uik7ghLWA1hbqwxxQMFbs6GLZ2RRILaAmIWS1PG1JRFaAxIcVILn293DwNCwEwyn1oATldEiqAzztTDPvdfQFOMnnNztbHUNyGec1l52thPpNjIEi9WVQJ5JR6gOGz/tPd/SzZHoA+pNtHX21QUZcZwj3UKAx10KbCcpaxniMyAqQfOpAz1OpPlV1gAnZlDo8Ak1T+s+odxG/f6K3WgdCKnC8FUayCk48QA77JI4fpWqfH+/onUD8LF3na+hZtLWZEKmOPUUu3ZciSmkCIytSh5uQfHrE6NFj9VU87oteCzT9DkEr85j5wdnqaGvkYxSQTRFVAcp3LjjTSaDhC6reeBndA5048baz3QN9FG1s3eBsA0DUMihKme6l4N+XIRFZARIfVoC4Ycj0YLG3ollPxDl6EdkK5l3C0HmH1ptr9DyF+6fUhk4SmNJrZDoMvZ/KvgXteAp0pEJDbQf6cBK/52k1sAlryH/hvJvkesrq9OFWrhK1WUH9iKwAyXetw1RKnrYWTHoeOf8S3vR4o8ck9e04UDLLRiW6cPTPkEkBeTuVjWt9nebpHi5KtyR0HSWhQ9WOXqSIyRLq3eEhTCRSh5uSXPsDLHd46tRjfcoU20lpReLVEXEIPOe1IyKifkVpAS2T0reItl4npMx7TYuITEI/B46yhqjGDWhF4hHysZPwrxVFJjWqpLSB+2KfpWiLyAqQ+LACJJ9CsWlYE6PMr1cw6migbtmw6RqUhk7q4OFkd5NpCZoJ/wz0Lm7Dk2vQV8jBa7AvwX0VyU3osXqVFbqv62gjIitA8ilSLIRYa8M0sm/U96MUTG41LqWvjp3/LmjAp0hktok2lqj6n8al+8kxOh7T/qAPVZ9gi0hT75VG1ZpIHcIln0k8LVH0FD3Nom+H/Zv6L32b28imW1+ozyTzM+hrcnNyg47ngdgC+lAd7ame9qFJhLUGihKRFSDxYQVIPnVk1nCyqIPvxHli0CE9aNi0vtKEUcO2F6fRqqCIQB8Pw4egdc541+imJXNwDzV1OXx/GJoa9r5KbkkfTURWgOTax7E2tKhDlli71foKmp4WcbvNGnOmbbI0VKA5C0Q2j93X0Xuo+hlccwLQGqDNFnbovtwK/SGiDuO0Y16pdtdNiFArYY3ICpB8igamDjr1FwybH/HMOo/9YNXTWdEKanMTrreHiPEEYmNlXGeSkPWR9Jf/oBfQ1siwsy1iFJTQZ2sDIyIrQOLDCpC8mWujjrns/RbpHj7+eT6dYNs2z8aikFfHThsQVd0xP+fKDh3ubj4j+GDKEdxPhdauE5h4WsbzaKB+RFaAOAffyKznRJ+4FT3w15eFPse+qKPVHAeu+2VHrcu5oje90++l49ugsyiEEtw3aHdjWFMnNa5FEYnLQGBEVoDEhxUgbkfa2t2x1o0+/7ASSrQOs3uNCyvP4sCbTiA6tHm3Gm2Rz5dfvC78Qcd3QmcNQ6udZQw1zaIukkspRQYfLvlMq5/UllVHxCFD11WUn57m2Svb5uAf0itTtPF5QN2Q9Je76fgA9D7o1SBMROQUnP0h7C02aMLT3PiIrADJm+Hk1SSFW3V0Fa+ZNUxgs6hJ1avD+fMaZ6KiOSBvI6jGFNn0koqoB8n2FdaUY9Md/TvZzkK/gFFRor6vRFIaLvFhBYhj8LzH54eTrXXjoGFTx75eekVtVcOm6UgHoTlvaEP3ZUVEJhCS3jHX3bRfT8QtDkREytif/BGByLVGutUsJqKj9PmLsvmIrAAxcx30czczJ9wK+qloG/rGmAJ4aUWWNPBWuzaKcb1hKvwREYh9QPAJSoTQOeJh6J3cFrKayxgVb5FJw9anoKdopI0AyRFZAZK3do81y0RpfkI+rOSnOTJh9VHKgaayV8brF7TVxHlL9MpHgERGd/rdT9LdFRG30prC9cYZpuoLce3dlDy8B2T5b/zmHH69dERWgMSHFSB5i1kXvaQP80sNaxj6gUT+eMrPgJ4mW9LdVMKTB6Z5LhpBCFjTg7h/mkgyoCcwd6CPskSki8asg9ZIBHuQMbQRkRUguX/2d3ak+UPF9DX6cS0Rl3sOzXkJPXuJfJ4fhmb2CMqwABTVeW2oGwn0qgtetvE8OeMNnE8pIpcop6J2sesWur7yn0EMbRa/JwmuU990dERWgOTae5HrYMWlrC0tP62Sp2YfUexM/JgY/85K6rWWR+oTjUS8CyCsGznaCrkIxCYgnPfTsqykBNoiz7BFZAVIfFgBkjc/OfEdNQ8n6zMUFW3DWjf6Dp6dv5YVPO3Xkz716HoTcNrTWJ9uJ2o0iuF3TlcPdJ2mF4EocY5T/IQuXHLtV+DgrS8sLBSoGJ+apY7XQqT/8RSjz0eIlY6UVd9CPvpy4KAzvSrdMkdRkRomqiVQlBGaWJZRFpEVILn2SwZ18Dc9rSway79oPYs6+KixttesJDjrvyV8sZCIyESL+v4kohlnpVfOIzJa0z8Wojz4v0CaI7ICJD6sAHHDkMX/mMmKLIhh8wOJFr1YTZu8KtB67Aqy0p58+mKseWtIhvvN6N40nH6V8+DjJ3ThYiMry1GvJk3ST7PkNnQjwMqA7ncNsZmDOUlY9EPFQ13usFEHcrrujusxiyZc7CwafwrPyqKxCKGKRUesNn2fxajQuJnls6zP+PxRkeGzLFGfNWjk90dkBUh8WAHihqHlqC0GbzF332YNNW2/bNiy2rZYfb/zRHoS6iwGb1GGTAYfh2G4/A/d9FtKH2A14gAAAABJRU5ErkJggg==\" y=\"-202.24778\"/>\n   </g>\n   <g id=\"text_12\">\n    <!-- frog -->\n    <g transform=\"translate(381.988989 196.26847)scale(0.12 -0.12)\">\n     <use xlink:href=\"#DejaVuSans-102\"/>\n     <use x=\"35.205078\" xlink:href=\"#DejaVuSans-114\"/>\n     <use x=\"76.287109\" xlink:href=\"#DejaVuSans-111\"/>\n     <use x=\"137.46875\" xlink:href=\"#DejaVuSans-103\"/>\n    </g>\n   </g>\n  </g>\n  <g id=\"axes_13\">\n   <g clip-path=\"url(#pea86b63c85)\">\n    <image height=\"75\" id=\"image1e631d8586\" transform=\"scale(1 -1)translate(0 -75)\" width=\"75\" x=\"7.2\" xlink:href=\"data:image/png;base64,\niVBORw0KGgoAAAANSUhEUgAAAEsAAABLCAYAAAA4TnrqAAAABHNCSVQICAgIfAhkiAAACVZJREFUeJztnG2IXUcZx39r7xFyhL1ijiS3dC80K7iELJqrdJGkNVEalLUkQrbSEohiFOtLorZCvsRCo0nwjdaXT1a0ghs0oSbUBUnArnVFA/HWujWkHzaFm+BVOBHvQs9Kz+r64Zm3u+fc653dBP0w/w97ZueZMzP3zP955pln5pyhA4cfXQGIogiNPM/pDSWLimXcOqSezBHmXbevB7qdsn6u7sOgMtux3mXeMEjnAgSVLMsKmX2JFUn5voNkRqdYty+1ylhUzHM7k3Xl9e+nHwKzPBAelgcqcRwDPgZelRvIwLtCbeDXphdu3f0NfDxQHUUEA39LUckyeaKxMyB6xPS1a0QMQ4qjurp8l1HuUbYcLot6l+/HsNK+94Guol/xwCwPVKrVoo5rO5YrPc4zhyGxnpJ767ge1TiuurkAaFcljnvbljK4DEnTFICX5+e72gPY3mgAkCTJmuvvhcAsD4SH5YFKp90Cuu112hFVeeXqAgB/b80PVluiVEvd7zrrdyv12DW5E4AsT61QaUCMcmPczqhkXLNq9cNTT/fswo2m/J5PPnFYbu/2X1aVHmyS0QjM8kDl2enpW1dbWrYWFLzabEqiXgNgx+66keWKZbky/okzqjovSvyc2VRNSrWSSaav12JQdF8CszwwtHVS4llXZr75P+vExw4fkoRiT9yxDK2q5Uunakf60rzY0svTM73rPCp1li2TLMPK7FmZMyvpwCwPhIflgaH3n3xhBWCi3jGZPz5zGoAb5/oY/3i3XJXBBtikVgN/u9R7au+Hd0/tA2BifMw2o6IUaWb7V6vL5HDiyGOFOj5z7BEAcopByhyz0JSrq2mrFyRdGhrU0BuV384Ii8amdpnM/ZP7AXjywpxkZC17RyKj+sAjewAYrQ0bUdp6BYDTbZmu/9WybBgEl8+ck+s5O+Tv2DcFQGPUso2s0rMOG23Q/zsyRZe03Qbs2hJgy+goANcWZPLYNj5uZElNtCcwywNDjDVWAFhom8wPnDwBQE2NxKWZi0Y2dWgvADv2ybIldra70tZVqerqywBkDrPOKpa+dOHSujt917jYyxvzzxdkB449CkBVuRoLaskG8Munz6yr3cAsD4SH5YEhYKWXcNNRofRXpx4yeYky8FHtrQC4K68s/QtgVXNzbtXwekvU4WLrGgDfumC9739Oz621/wXcMSEq+tDDYi5mp88b2Y1LRbX1QWCWB/oyS2PTyUMm/bVDXwcgXRAHtF57o5FpIull3LZRW0ccvQZAlkmE4Wb7qpGdfX4WgBOfPqUqGvwH9IaOf6V9S/kgMMsDAzHLxZf+9FMA9o8/CMDchdeNLI6FZVVlyJzgJiMqXTNR0decWsVteXFB2Hbg+HeM5MozF3y6d1sRmOWB8LA80HuR1ROR8xeupdY9SKriTkRK5VLHUEfa0dcqypuMLOZtAGwfleuff/QhI/vCvp8A8OSHD/h3tQDt6OjOJCUy/XvciUEms8AsDwxk4O842DDp7aOyJrx8XDmVyYSRvefQ5wBoNMRnGFGOK4DeINbhL3cjfERdbWSsiBfT35h04877JLFWFyPSEQynAr2xYeJmbiBMJp7ALA+Eh+WB/gZeMXPv2E6T9awOc2h33fHEfzdzVqV2SJExG0BbymRNmeXKF3N0Lqt2NUfZkZHtyb0mvfL6vwF47yc+AsALbuilLCy8Cg9MielwD4/oAJXulj58AvDcdFBDb1hmTdin/JYJMehb1GbBeO3tRtaZkjBvpsc/G8FCyiexsC7rWNa11ZSc5TJ2He5cdRe0VMLdfC7fiB4C4Nff/xkA56dOGclXvi2bJZdnFoq3KVRVX+oOs2Ldtj6rlxZD4oFZHrCugzOCW49OAjBZlyk2yrcYWcaSSQGQW2bpw6+5cegi5z6xSJnyWOPEbkBsGRWWjSuyumvKzcqQue5jv1MPrc4vAPjgkccBuPJMs1Dm7ro0dP+eSZMXK3cna6q4mxNve7UlvzUwywPhYXnAGnhnqq0qazes3OzF1pKR6bOk9kypc8xIHw+KllSV9r5YN6A38tq2wevqvsVEVGGLMuAAOn7YceLX2rWo8w+VerOR1auyrtTry3fW32dkLx2XsLJWq+a83WmaGK935ekyLgKzPGCZ5Yzc2JgY38WOYljX2xQbAFjOhykg6i5TZoirZmKwu9wd5QA2laObYuPRWSasyR0nVs/4i7l0en7u90a2d7c4wrGKavzxiV8Z2fnGlwH47GPHAbgZWcczz9pdeXc5IfHvfuMYEJjlBeM6bD25x2TuUTu+KL3dmLsbXkLG5b4TuB8yVVcrFvpEkW0vUeuizTU71KOjYtO07bo4YyMSeUfOL3zqoCxptlXfVWjvYvMpAH4wfdpm6nP5yn5+/GG7/Xd/4wgQmOWF8LA8MEQkavj5n580mZk6dVzNhJrDjjFfvg2dME6IPibkzDZpXubxi2rqPcvYsQhpKrtG1xbEA78nsS7A/p0f7Wq3hVXf2blZAHbt3CV1cy+rEZjlgaH7Dh9cAdg5YUeuo5nFRgCi0giTX0y3ok6XLZceQtd5i/JftNH2hVrXFSBSIWB9yGykbp3SRBXrZLKf2WmeM7IvTsoxzAi7iz4IcqSuwCwPVPY37gHgevu6yTRMMqdXy9yE/86sSslraf0YVjFRTiurqj202NmaSjvivLbVNJ+l1q24OSosGxkT9jR2P2hkq3bjBkammBiY5YHwsDxQoSORgSizKqODeJF509418L1fZlo7pJ0l/QpdZB2UDWq9OOwcjBtWHv5fcymftmwIOVXhiVyFC2PH+LfVz/BVQ72ZEZjlgUpHGckNDnuWCjtkt+R02QDYUMiJVNuxExvTvUkixTYnKpKmwoNsQZjVwe6YL1VVnKzEE5pt/gGAXY3iWnJJKVNglgcqy/HGQqbzaqS6unbq1rCs1K0oKZercp2SV9oqyp1IHKboeNmwPrPgRDzb6ozD8Jh4roupjanNNWflfhUsG07sy6Ptq8LWwCwPhIflgaHHn/qe15lSH5Sp2u1GcWXgWnOlvmpCWMbdddYTiEwyFcfBWM7DK3TeWMMxyf9vFNls/1/O9TVXZd1iJpjWVcatIzDLA7eVWcvO4FTWab7ckV6rLbR9UJEPV6bb6dNGYJYHwsPyQEXTrez7Uf2/nlbEoF9BWy/82unzkVdXFq3OCx/BWBf+AyPjzEUuBVQUAAAAAElFTkSuQmCC\" y=\"-292.222953\"/>\n   </g>\n   <g id=\"text_13\">\n    <!-- plane -->\n    <defs>\n     <path d=\"M 18.109375 8.203125 \nL 18.109375 -20.796875 \nL 9.078125 -20.796875 \nL 9.078125 54.6875 \nL 18.109375 54.6875 \nL 18.109375 46.390625 \nQ 20.953125 51.265625 25.265625 53.625 \nQ 29.59375 56 35.59375 56 \nQ 45.5625 56 51.78125 48.09375 \nQ 58.015625 40.1875 58.015625 27.296875 \nQ 58.015625 14.40625 51.78125 6.484375 \nQ 45.5625 -1.421875 35.59375 -1.421875 \nQ 29.59375 -1.421875 25.265625 0.953125 \nQ 20.953125 3.328125 18.109375 8.203125 \nz\nM 48.6875 27.296875 \nQ 48.6875 37.203125 44.609375 42.84375 \nQ 40.53125 48.484375 33.40625 48.484375 \nQ 26.265625 48.484375 22.1875 42.84375 \nQ 18.109375 37.203125 18.109375 27.296875 \nQ 18.109375 17.390625 22.1875 11.75 \nQ 26.265625 6.109375 33.40625 6.109375 \nQ 40.53125 6.109375 44.609375 11.75 \nQ 48.6875 17.390625 48.6875 27.296875 \nz\n\" id=\"DejaVuSans-112\"/>\n     <path d=\"M 9.421875 75.984375 \nL 18.40625 75.984375 \nL 18.40625 0 \nL 9.421875 0 \nz\n\" id=\"DejaVuSans-108\"/>\n     <path d=\"M 34.28125 27.484375 \nQ 23.390625 27.484375 19.1875 25 \nQ 14.984375 22.515625 14.984375 16.5 \nQ 14.984375 11.71875 18.140625 8.90625 \nQ 21.296875 6.109375 26.703125 6.109375 \nQ 34.1875 6.109375 38.703125 11.40625 \nQ 43.21875 16.703125 43.21875 25.484375 \nL 43.21875 27.484375 \nz\nM 52.203125 31.203125 \nL 52.203125 0 \nL 43.21875 0 \nL 43.21875 8.296875 \nQ 40.140625 3.328125 35.546875 0.953125 \nQ 30.953125 -1.421875 24.3125 -1.421875 \nQ 15.921875 -1.421875 10.953125 3.296875 \nQ 6 8.015625 6 15.921875 \nQ 6 25.140625 12.171875 29.828125 \nQ 18.359375 34.515625 30.609375 34.515625 \nL 43.21875 34.515625 \nL 43.21875 35.40625 \nQ 43.21875 41.609375 39.140625 45 \nQ 35.0625 48.390625 27.6875 48.390625 \nQ 23 48.390625 18.546875 47.265625 \nQ 14.109375 46.140625 10.015625 43.890625 \nL 10.015625 52.203125 \nQ 14.9375 54.109375 19.578125 55.046875 \nQ 24.21875 56 28.609375 56 \nQ 40.484375 56 46.34375 49.84375 \nQ 52.203125 43.703125 52.203125 31.203125 \nz\n\" id=\"DejaVuSans-97\"/>\n     <path d=\"M 54.890625 33.015625 \nL 54.890625 0 \nL 45.90625 0 \nL 45.90625 32.71875 \nQ 45.90625 40.484375 42.875 44.328125 \nQ 39.84375 48.1875 33.796875 48.1875 \nQ 26.515625 48.1875 22.3125 43.546875 \nQ 18.109375 38.921875 18.109375 30.90625 \nL 18.109375 0 \nL 9.078125 0 \nL 9.078125 54.6875 \nL 18.109375 54.6875 \nL 18.109375 46.1875 \nQ 21.34375 51.125 25.703125 53.5625 \nQ 30.078125 56 35.796875 56 \nQ 45.21875 56 50.046875 50.171875 \nQ 54.890625 44.34375 54.890625 33.015625 \nz\n\" id=\"DejaVuSans-110\"/>\n    </defs>\n    <g transform=\"translate(28.042468 286.243642)scale(0.12 -0.12)\">\n     <use xlink:href=\"#DejaVuSans-112\"/>\n     <use x=\"63.476562\" xlink:href=\"#DejaVuSans-108\"/>\n     <use x=\"91.259766\" xlink:href=\"#DejaVuSans-97\"/>\n     <use x=\"152.539062\" xlink:href=\"#DejaVuSans-110\"/>\n     <use x=\"215.917969\" xlink:href=\"#DejaVuSans-101\"/>\n    </g>\n   </g>\n  </g>\n  <g id=\"axes_14\">\n   <g clip-path=\"url(#pb321a3bd37)\">\n    <image height=\"75\" id=\"imagecfab282603\" transform=\"scale(1 -1)translate(0 -75)\" width=\"75\" x=\"123.652174\" xlink:href=\"data:image/png;base64,\niVBORw0KGgoAAAANSUhEUgAAAEsAAABLCAYAAAA4TnrqAAAABHNCSVQICAgIfAhkiAAAByRJREFUeJztXGGIVFUU/l7ukxihkRqpCZ2oiRrFMRzDqdyi1VYhLbXcH67S/GijtnIJXMi0FdLCpXYjjFDTULNWIgsjlFhQg6ZCoRHbsP3RBq7R9GMIJ2iIBpl+nHPfPey8HedOO+4I9/tz7p7z7p3nm++ed865Z3RKP54rAQAicXhwWWazJKO3ogyZkwCAr195yVMVwyEAQGvXa6RIPFg+D//y2kWtCk/zua7xcN1k38C1hCZ8eopG0VGtzZwlGQgAAIrFgmf6diQDAPgikwYAFIKuZ3tzx1s0SCQqfORUEuGptd/1JMEyywD2YRnAAVAaz3gPaBtGo1FPtybVDgCIRGYBAOYm9IshGJ9Xj3tsGFhmGcAp9bxDzBLsqdnBH9gPAAgmmllzbYQE1cIyywCODUqrh2WWAezDMoBT2tRB27D9aa0N0XZCkbdKOKxtbpAHahtpB49hcvrI5nmeeGkEeF6A1w6JCD7LW7PBo3rLLAM4l1wKSoOxiNbGObdLMduWrvCZ+jeJ7IhWBSnUQEAxUThxMLOYdHAdbQqouFjoGhCWWQZoushffm5IVx2ief76XTaqvwGgbR0P+HUfNkxxhn8gmVwglJPPqC3dewEAb/Q9M+41llkGsA/LAE4mECoBQLGQ85Q3sIwpRWqVntH5PsnkjPrfXZ2QHyL5wGKdYZzPUdgzJ0R57Xcnv/FsQU5uLLMM4KQD4RIALGrWueHo4CAA4A/+e6GISdH1HslNz5t9Uv4CD/ilwRUNAIDrk3vWAcf7KZ9d3r3EaN6xvhMALLOM4BzlSunKjo1aG6RvP9+/EwCQFRNiCQ4u+z4i2eIXsFbA4F7+DBEEJ9k7ZqhWhsRqszUFihlicD5HNbgZy+bUvNZYWGYZwD4sAzibeRtuCWuHG+jppEHsDpLrXvBsOd6TIVVsaFuqV+vhc8NYhag+e4Bk+qLWtT3HAy5fp4e1LcpbtKBL214FI0wvhjPd3Z5p976jAID9eZGzThAsswzgLGFmiWoW1rbwN7eth2Rwrjb2Hic5sHP8VeMca3R2aV17K6+1oPx6/Exi8AgA4N1lWz3LhlQLz39KXx7jMCdHL6Kz993vmdZzZHJ+/LurGZZZBmg6wYOkUJ49RVWG+bv6SZESYcW2NSQ7FpE8dlzb0qdJDrHPGdiubQl2ckk/Zs0mwdMelaa+r0jKyip+JzHyId2nKJvVg1EKllkGsA/LAF6vw0GRqnHw6zn9YEqcKXZsIBljxxsQhxIjvI8yR3iiWHSVWm36le+qKNovsnwzrsgjjr1OcutB+kfIFKOOsMwygMesu4SSXbiXE+4RNncH17NiHCzmMtoY+oukcrjxVm1TvRTeUVqFY6/hA3o8cJhketBTfcD9dx3jr1AXWGYZwD4sA/g2s6l4WGVvMu7xtuTHHKGNnNHGItdro1x+cW/RthA7+yBv1cRCbXPHNJDsEicsvftIigOmJ3n8+dgbrzMsswzQ5Kf8nuUTLA8Lm8r9D/VupkFKZJVp/sqzHMknW8RM5dipvRKuCDnGIrJIj7PMrBZ9/Wed1NLkrF4//hp1gGWWASo24PrhZpYvsnw1KgLWZs4wXWZYVJSOo3eTzLHP6hBVBLfCsVruF5KhO8tM+enk66bn01e+8QmAZZYB7MMygPE2VLid5bNC9/Ly5TRIcORe+FUbVZqY5/C+WTj4IL9nlrazosrTbn7bvD1N96RuRGGci/8/LLMMUDOzbmQ5S+j4uAKt7Zy1Nc/XxlE+3x5V7Uzih1WqkU6VjiN+BcIKOKWHzuL6tS9ZZhnANyitBn+yFO232KwGAxRItubET+la2njAPmVYd+14PfiqN15EHFVBxL6lni8BAM72xwwXuTIsswxgH5YBanbwflBOX50yysxQhRjhCG/cgNhr7SmSq7jsGJ9d+YMGuS+1hV8EbvklZxwKJ5ITGEpYZhlgQpmlMIWl7IHjzBCq0Cx/Re0Vn7v4BDu1VhsLiokhrVO1sdNcavY6qAWGiH2PzLvXU50ov8oIllkGqAuz/DCT5U0sRa3CO3J7OMbsUWkTABQ4nBgSXTGb+Cd6imHy1/6qZ7+fA5miPo5z+nfVcOcallkGsA/LAFdtG47FTDFWRxfKra+RFyZ5i50W55Nqu+79hGTzYs/00zQqEsYLtG17xVJrQxSu3JYTeWkVuMDzLLMM0BDMUr/oUKHGSmFTLPOJOxGK8CHIhUtaeZQa45zV5V3Kqv1uN8tK7UlytmritMwywKQxyw/q6C0odN5/BCB0qpVCMXFPTAcibh/5MWfFxPW/K1hmGcA+LAM01DZU8NtAlZzxQ2Ksjkh+q+Jzpojx5TG6yyiHZZYBGpJZ1ULVz2TF6p86fp5llgFqPrC42pD9ModYqqTlcWE7V+P617OsxEzLLAPYh2WAa8bBy1zyYoIPO0ZoIzr58uvrAcssA/wHXViQv/PG3E8AAAAASUVORK5CYII=\" y=\"-292.222953\"/>\n   </g>\n   <g id=\"text_14\">\n    <!-- frog -->\n    <g transform=\"translate(149.084642 286.243642)scale(0.12 -0.12)\">\n     <use xlink:href=\"#DejaVuSans-102\"/>\n     <use x=\"35.205078\" xlink:href=\"#DejaVuSans-114\"/>\n     <use x=\"76.287109\" xlink:href=\"#DejaVuSans-111\"/>\n     <use x=\"137.46875\" xlink:href=\"#DejaVuSans-103\"/>\n    </g>\n   </g>\n  </g>\n  <g id=\"axes_15\">\n   <g clip-path=\"url(#p34415d6a3e)\">\n    <image height=\"75\" id=\"image5e93e7f9e5\" transform=\"scale(1 -1)translate(0 -75)\" width=\"75\" x=\"240.104348\" xlink:href=\"data:image/png;base64,\niVBORw0KGgoAAAANSUhEUgAAAEsAAABLCAYAAAA4TnrqAAAABHNCSVQICAgIfAhkiAAABgtJREFUeJztmn9olVUYxz9r9wW7oRfshl5xi3YFV8xkl+ECRzSjJS1ogWbTSMIFm+UKgjTS/aEUEViyP1RISAOTfl8hJfSPLX9Ailz/2KitugvvguvqKtzCa/g21h/Pec/7ut3mjj9yW+fzz/vyPO897+G533Pe5zznlHS2t40AEEYz0NUFQEV9vRgKBHDE5IqxuyerPYW82I71dDEdueN2d2AqEXLdvNwVfGll+/oAKKutE4PrS2swK0rakZye6hkPqywDbLAMCLlqiLlq4gbIqJFZ5boAOIFhWMjnrtnobNXWRdyb1c9JgVWWAaGIo1KBgq8CJSzCyhYO+6pzr0ojijPdFOVhlWVACTByuzsxVbDKMsAGy4DrDlZNZS01lbU3sy+THqssA0LFjDPV9c9xftjYtBKAwU4/SR0qpG9WvyYlVlkGXHfqUKqWNMPTNAEthlWWATZYBvzPMniZOu4rjwNQES/XHtcVX2+ffKQu5gPTiyOFUassA4qmDpORmU5E35fFRRkXclIfyeX89GXzli0AhJxZAGzpeEn7Vq3eBMDSRBUAp0+d1r4Vq58CIBKLAVARX6B9727/ALDKMuI2zFnyzy1OLNKW/tQpAKrVBkmdtwUHdHWdAKC1ZZ22tbz4nLrLMxaZX+aUSxtDmUPX1ctzl/ywHE5+B1hlGWGDZUDoBXWzMGA8oK7N6tof8H14gy98uFaGX2t7q7b1p2TIHDgiQ64v7deuz6Rk8t6954C2bd/5JQCvrX90TPvzKxsBCIflgzCUCby7aSsAx5Id4/RQhvGbb2zXlkxa9kqtsgwo2ZOIjQDsTvlnFv5Q11nq2pqIad/L6rm/JtJ6eYO+Xay+/JWVkghW1S7VvvNZUZKXB0YjUe17f+sz/9q8+sNZMO9ebTty/HsAUj09AGxav0r7du3/BoC2NQ+IwfH7hys77DUNqwGIhP2E1UtmrbIMCA2o7a7BgNE79eDZBgLbZBNSlEfGPw9xOSLSWhITFSUSm7XPDcsbHZV4hidYyYgrwXfuDcykjprvVJ9rapu0qzclc+KMSN3VzwJOOAFANCqNRpy7ta/gXgassoywwTJAZ/BzAkZvpeVNs0PjNDAjcD92iPofhlVNMpk+ltwHwLqff/Mfi98DQHfqEgCFyF2+T3XiCX9pOCEKaoT1p3/Xtqw6LtWXlnyiv89PinJ5WQ1Eo2UA5ANVh0JBPnlWWQaEStVNMfV4ttKAbXjUM8Un/EoA3vn6jLY0N4paPlpfDcDZnD+5VksRgbkJeeZooKWTKqk0VRbuFQAOJj/WpmfXtkhbDQ8CkM4+qX0HDx0HYCBdbNPlTsAqywgbLANCo4dVMYLPzFfXQkx2o1tbNmrf29tU6aTpEwA2NgYmasXGna8AcDJg8z4o3sphRcC3IZhIG/Be5y4APkse1LaySlmXxmLLABgITP4ejjqCFQ4HrbasbIxxWflXdV1eL6v75pantc+J/gRAWcM8AM4GflftPaOuVQFfdNQ1myvinCDpjBTt4otE+V+sbdO+v9WmRH+PpCjBcvSFnKQVmex56UOgE94BPqssA4yVNUON30W1SwDoyVzxG3Pk3+nelwJgIOofr6TpcQBe7fgKgG/3+4ocTW/6nL7v75Pk8JE6fwPh9Q6pNbkFSSTdwGZGRbwCAO9k54CqPgCovJOsUtRgxk8Tuk51y7tT8vzwVec2pA9WWQbYYBnwH+7utKurt9syseNJy+ulxNLcskHb1q7xdneyRX5x67DKMmDKnHXQBTugtf15AHZs83ab/QpBqfoADTP2wP5s5YtEpBryS35i6i61ZWVzbpGyvDpWZKzNSydi/jpmtjrVMjcipdx4uZ+JOko1K1b6GxdHk59KEzF57q22ZTfc44UPrQHgx1NelcKvxZWWSxXFKssAGywDSkZGRkYAegPrsRMnfgCgru5+AKoM12dTmcOy+ODz5F5tW5iQj4tVlgFaWZZrY5VlgA2WATZYBthgGWCDZYANlgE2WAbYYBlgg2WADZYBNlgG2GAZYINlgA2WATZYBthgGWCDZYANlgE2WAbYYBnwD1opjpGo0AvCAAAAAElFTkSuQmCC\" y=\"-292.222953\"/>\n   </g>\n   <g id=\"text_15\">\n    <!-- ship -->\n    <g transform=\"translate(265.18994 286.243642)scale(0.12 -0.12)\">\n     <use xlink:href=\"#DejaVuSans-115\"/>\n     <use x=\"52.099609\" xlink:href=\"#DejaVuSans-104\"/>\n     <use x=\"115.478516\" xlink:href=\"#DejaVuSans-105\"/>\n     <use x=\"143.261719\" xlink:href=\"#DejaVuSans-112\"/>\n    </g>\n   </g>\n  </g>\n  <g id=\"axes_16\">\n   <g clip-path=\"url(#p9a38c3e85d)\">\n    <image height=\"75\" id=\"image3c789e8b0b\" transform=\"scale(1 -1)translate(0 -75)\" width=\"75\" x=\"356.556522\" xlink:href=\"data:image/png;base64,\niVBORw0KGgoAAAANSUhEUgAAAEsAAABLCAYAAAA4TnrqAAAABHNCSVQICAgIfAhkiAAACOtJREFUeJztXEGMHMUVfZOdjuxG9ih2o+xEYhJYFGzEYrGRGCXZQ/awG0KQYgRICZEwwgYJA4ZDiJIcfCEHcuCQ+OAD5ECkRDlEwRFJJLAUI7GXjWARLDIbxGAxRrQR7ZXbKxqU9mo5/P+rfneXh63Iwhup3qVq6lf/qu5+9evXr+pp3fLc+jpqyDNKzxaUbu1Y2Sc5pTtjSjtJ/Wog4+vPZB+bsvfTIWWGKaULx+0FJTeUsFKpCwBRRGlc2rKkU2l8x+RVRtSJewCA8Q5dVyIysrhD1xUl6Xrl2N+szqUFSgvuS6RuLF8GAHwJARtGmx828twWJl1O+XehXmpcY1KpZKIr4Trx8AojSzN6w2vCIt1gQW8OWdrsYXeSlceqkHWAdJSpZU/UpQ5FzLAkscMiYh2DjNuOd1qVCbeTDaopACAwyxvhYXmgLfZMUkANLWZ3ZFmOTOqVTVlUq18dojQEVoxx1sNKxn2PO5NZkdSPVP0uj/OShtNqZutHne0AgK1yeayGocwV/HsssTZlLT1d1a2RkY7ALA+0xc4qEth8iQZKR5lAdMVxs24u/oiwoKv8EcmnbOhzxSzDjLJZxg1sm7zGiJJkAoB1IbTrEEmeZ6y1Uk0yUi1ilvd6SkYTT2CWB9ryvjQLJC9FF5Q9a4tdcmnjwpzrny+sU7pWb2hp2V7X61YbjLtKNsG6VYsdmvK3dMgZvUqxNGFHtSiINUVqbyyKykoz22KrczViHXKvqXYdqF5glgfCw/JA2xDRMa6MzVOzdt3AV1wHTo2Bj60HP8bCNblgclezwYI9+Fx58kNpXDXUo058mtJQOx1ZA3+hpPrj7IZ0ErU25I4NB6R/VS9NxNjLzXYnrCwNBt4bzrWhgcMpNSgrVQDYtWHdOSUZGd6VhBtaVA0Kk4QFOpQhzqumd0e8XkpXB+/Wu4ytXD2BcgGk02zYxyI7MdgJiPuSKveF3Z3ALA+0XQ5knUl6aFdWKbXrRJeLWbn4E7Liz5U/IrZq6Ig6CKMS1RDbKqHyNrVsSZg1xpUttONJuqTlNbVMMoVSVok6LAEIzPJCeFgeaLsMu8TgfMPKossVVl6T8SrDqqOUxhx4S/qU6rByKcsCR1iZB1vUHbfVZU3INiBV9xfzWDPWoThrhdkSl/FNd5TrwG0HZnmgLfb2vHoDZUl7GDmHgIvMvtWYDWjJzt/ZomVk2yUYwL8rpGXHztAuVWtDiTBJZ/RblQhDpCYEmfIjWiOuFHYmWZEMuxdFaWekYsDxL4lqLCkGG1vfqReYOwnM8kBbZl0V10cUEVtK0HLF5VbIu6wsfyRcxCSIM8uGVblg8U3OLKgLay5DfpG8YGBa51THnoiVp+R3op1SaW5eFDmUXxyBWR4ID8sDrcffoB3pwhH8c/2ue/f6t8TSxJPXzrM47PML/wUAnDz8kBUWz3h3/HIgMMsDrSfeImZ9othTeDBLRWaxtWb8XdEKKTp0basp3OQIzPKAiTpEjr0ws9fqcA+EIdsdm6yuDVgpO+MILPy/IDDLA+FheaD17Dky8NkI12EU9FBL6kE/racWGPznU69a0fD3AIDhwlEAwPPKud/B6YotwjdZx9se/bwUCMzyQOtXrxKzSh0Clh0pfnNqiWdOMsqeQUdtHstEMM5llYNvtRMsesX2g5+9TLKUYkpF9h8jW108AQDYMWWjBysv6nVlFWOcdrkP102osw4xrRsj9pafn7ezzdXiA3G8LVWBvptmpgAEZnkhPCwPtIADfFpZx0Jk/CyiKZNYzhSnmUPGiPQBDw79TtAA/OqkHYgfPvUAq+TwMg9HyrMN6KkxPXT162JQfeAY+PXc9sl5uyv+2NP3kOolOsH8198dbmgKzPJACzjUOAcvp3NHvzlhkePMgrlOb0rwGxZ7mx9V9Zk1svuRq3CvYa5m8OdjS4+YPzs3Z8r27yP2fG96d713DRxfes/k5278BoDALC+07Vas8g9QP7kySpY1ZfJ1gvYd5Chjzgdd9WaB1C9dfRGoTQwMazLrHozx6Rdh1J13/cTIZplRtU11J2Ynv94oC8zyQHhYHmjv+ePTAIDX5z+wpSfIaxaDu2Xv943o02MvUEb23mZmjGzP9NdI18JHVKCiiFf3SXbqfkfQz8SJ+HepT8+5hmZ9Uaijk9J10tGbsK7DRobfS8vnAAA/vv2WhiwwywOtZ9dpbajNp7wBeZfqYzfMXqQOYHfhcofsF1cc5EJxGbRbwQ6qnIPQxySNFs0LOXBPrLl+rm8ke39IPbzvwE8BVO9rI5CWX1t4x5T12JEOzPJAa4mZpVlQn7gd0WGD2JFvc3rH0XNGdvLgV6oXTt1tslv4HHuXQxjf3WU/trx5iljT7++2ZY7Pa/4XOKLlIxGY5YHwsDzQdhXWKelL1zOcnjz4bYeUIgt7+rOmZG5mGgBwz13XAgBu2EAblwLfefINk3/l8BHKdHmy0RuiRTit7A1j4C9F7F/ehdBVHUI0K0iJYo0rmYo4faE4ok4cFTw7nU8pCHNisXkcKTDLA613NsCsCyrvNHIMYZa4m3qGry9aXC7HF40/qIDJLu7sqDhLYJYHwsPyQOtjhwcvGDU0XS5EXEsvJx7+5RMAgFm1GvjRvnsBAG/yjT3+6JNG1t+7HwCwM7kSQPXe5ch+YJYHWuvrzT/uuVwQdh/5h90smJmm8O7No3YXNohlNuh//tO/AABt9Xcu8QTFJ2ZmaC16Tc/G3SSKEpjlgU3FLMFvXrTf/JxZpjPr6dBuUqRD2vSQz/Lkc14AiBL66mJqipZQtz74LSOTBdZfWNVLx54zsp8fuh1A9QxGHYFZHggPywObchhq/JpPF03oz2V41A0GFFyM1ffTt05/GQDw7wHdVpFap2j/NH1eI467dpfM8HvmLUoP7EYdgVke2PTMGgVhyGlVdtOI+uJoSjzhOH/tAQB39omR3Uf/TgW/va1xfWCWB0YFEQxcK/DNgKSWfh5kiXaaXYdHmE0VTPebZYzALA+Eh+UBE3Vw/TPbqONjQn0dfagff3fJNrLhsVkRmOWBzwDTnqU1pMYP4AAAAABJRU5ErkJggg==\" y=\"-292.222953\"/>\n   </g>\n   <g id=\"text_16\">\n    <!-- ship -->\n    <g transform=\"translate(381.642114 286.243642)scale(0.12 -0.12)\">\n     <use xlink:href=\"#DejaVuSans-115\"/>\n     <use x=\"52.099609\" xlink:href=\"#DejaVuSans-104\"/>\n     <use x=\"115.478516\" xlink:href=\"#DejaVuSans-105\"/>\n     <use x=\"143.261719\" xlink:href=\"#DejaVuSans-112\"/>\n    </g>\n   </g>\n  </g>\n  <g id=\"axes_17\">\n   <g clip-path=\"url(#pc7263cf74d)\">\n    <image height=\"75\" id=\"imageb3da9c82c6\" transform=\"scale(1 -1)translate(0 -75)\" width=\"75\" x=\"7.2\" xlink:href=\"data:image/png;base64,\niVBORw0KGgoAAAANSUhEUgAAAEsAAABLCAYAAAA4TnrqAAAABHNCSVQICAgIfAhkiAAACFBJREFUeJztXH9sXMUR/g7uEFyEDiWHyCFh1F6QjUQixYANyg/ZhRoSo6SRkkgBNaEhKSBoQQoKRpRUkNImiNASoUaFUBq3dSQcEZvmB3KgtoQDJCqOwEAMygWwUC4Ih/aEcCsd6PhjZt7Ovdu7eF8cqUL7/TP7Znbf23vv293Z2d2LTQVKALD/XQRYMIuk6KZljG3OpSSnZ0m+sNvYpteRLBZZkTC2RJIkC6SMCQcOkWy7juT8BmN7+XWS40WjO57je6VJXqBulkqXPzttTKo6L7BUP2z8HgDAaA/d/OQhY2p6+t8AgHPgMWHEv+SEfHkAEF39TJKaBZ/zF354DedRLFC3cEJvT/n1wnaTFqboOoiuwHVJKAanVbo6hFE3GVWiFQDw+DZi1uigMe1vvR+AZ5YT/MtyQFwS9dlKY6pShXNZruRmGLXplWG8/LJ1We3syUS5dEdlwUVXbwcA/GO4MndsyQ4AnllOiJ/PCc0Q0ckXL6hhu54/SspGu4iYHWb1pNC1FhoqNDZGheGZ5YC4jSCiO7yXpHZKM5PIKEEi1IUUC5P/jHJcGqmUZ5YD/MtyQHzcohTdT5eTXNymbGOTX4nwPfNqXoa5k/+8qPDMckDcNkqL7iOWO/vObiVOjpRfzz4LgwjB4nk7wDPLAfHTZwE+U+lzq+aKjsvS5denhkzacEHTTbifd3xSs2P+cnhmOcC/LAfEp1mUovvcYvtWEpbQcVRkQtfJYtmVpYRr8xOMnD5LDXhmOSBuI8ZEyNJJ4R+svPvMKxHmzlXNNuskTBj7aeTIbd8EAPjrsM0lrw7PLAeoqIP5vqlw6NKC9/o5YWWWzauszoxD3SSDOJrVKS3alG7gOs/oeihScc8sB/iX5YB4XdD6zABel+Ql3xqt8SluOk+Uaan99G6lJrdZsf0w30tcjwtVqa9YXi+KnL7nmYY51PCRd+vQw/DMckDcEEp9zky5ysYCYUhBrdz+/iFi1KNKVw1fWXQyBHQqRq7skJR2MFwYoso18GaMiM6pZ5YD4vWWiXigY2bNUbZXQnmb5pn0R4gGYa4Q+nllWxmkZiqtDqWeDirUuo62/OzYsQEAsGp4o8N9PLOc4F+WA2InjqVLAJDJGrrmc9RDL5hBw/ZPVAH2GPDBBG5+iUrXs5Tu9riyTWf5Y5YDyvbqMU5kVymtTB8k+tCobNJEpS9pVbbfldVvQSwWpMPdiw2eWQ6IZ7LyBcwQK7o20JJ0iyogsa5nWOpOfSpLGRD08oCU+8ZiE2bVWWzBKJ/VroNMHmW+mKm0sR9y44xNgeW1MUr/gPO4urueWQ7wL8sBanVnVKmJpg/zlQ7iysKLNICpyraYpTRD3TjCARbtgydDOh183LWF5NJ2HbcphkqYBpXfTu1v0Vrq6P+FSnwcMZDomeWAuOlO9eIdfamUbFfsNhYZ8uXrt6hSMvTbmCUzTymnmXaKZT6UBwDuZS9haVm1ZfOFLJUb1yEFcnuEUUefM3su711LtXgNamHSAZ5ZDoibvmqFUm8m0U6sS3WbiMTjLIWHmiGiq0MlwhsTdbn/hsp1Kpssxy2KGRfg5ZK4xFLn6wJbMilsI5+jYY2ZXT6DDwEAp7ro+jLF4eMpenprj2pGIXhmOcC/LAfEbv5lGx10WmdCIAu20Nbd/TO54+zrMiW6qdlK4x1QNwsI3/EHkkk13A8fITlIHXAhbzpZGfhl8LhT3fNjS6VLb/M8tvFvrDHnWabF6OiIHKkpld5RJfkEF/5JYuOvjWnZHQCA2JU/szyR4JnlgBgSc0sAsCljHLuOPHXVpd2PkqJBOXEbniLZxbP7RhVce/JZkknuqkdVuWUXkxz8D8kf3RyYDhfpXvv4eq+qoM2pFMxneVDpJNy9MU11+NUXnyrr1wCAgXlXAAB2Dhp3+0+fvgkAuPbyG/i5laFrzywHxI4990kJALJFM6vPJeitZtdcXr1kHzOk7aIK0+GLbwMANDW2GGU7x5XkwE1fv7GNkVu6uf8BAECvutdJlra+qxZKpVJV264ZtwMAcjkTnn6wdBQAUNhBp0Evun1+YLsBFEvzzHKAf1kOiJWEr13qkPSts6pkV5D+z7bXrOsEAKDYZZqanJTd138AAFBXZ/z8q95+jBKDVJXfzDPfcCdLHYb+X41qbeVVoF+c4N8T3ikHAHQiDiN7XwxUDX+hTf/FB/4IALhx2z2B7QhHYTyzHGCY1f200S6777QFj2ykrzKmjsUXufOek6TPmRo1tl17yTFYn6MA1XQ1g3xjD8/12qeQ3Ho0sK2+jzraYUsQ+D2WP1S69/dwx94ezl0boz20MjKw5BoAQK+Keb3E0jPLAYZZjtg8ay0A4MOc2TdQSBGzmhqo33iw47eBbf0tPwcAdBYpKpBUO9bWr6J8d63jw0KaRBLK6H4rUOWHh/jZFEXYN2yOgDxRMqyMgtWx8wAAIyoukkzQtM8zywH+ZTlgQsdRbJjb3AIASOpjqHXUtLLNTXTdNiUw1bdT0Lm+h30O9a8b6SyHBvXejzBmmgBfhoN94hW0ONW8NlY8Qn+9cnBwINBNa6YneWY5IHIHjy6azR8ZNvOrApOs5bHlFdnH2OFcvZw6+kTadPB/3v0kACB1ZifcJhUH+14P0sUCxfc8sxwQnVlDX1fqGqdU6kLoZGc2oY7xr1hzU7Xs/yeg3+qZ5QD/shwQvRkKbJsWvqfwzHJAZKfUd/AeNRGdWSMUGihzSnmts5ZTumsbRUq1U7qwlZjlndLvEfzLckDkZniwn/YsDA2p9WCOOoxvokniwo4lgal3C60SFPIc2RszPscBXixZ+sgEFkrOMg5s+DsAH3U4Y3wH7C0DVLfUlnEAAAAASUVORK5CYII=\" y=\"-382.198125\"/>\n   </g>\n   <g id=\"text_17\">\n    <!-- horse -->\n    <g transform=\"translate(27.931843 376.218815)scale(0.12 -0.12)\">\n     <use xlink:href=\"#DejaVuSans-104\"/>\n     <use x=\"63.378906\" xlink:href=\"#DejaVuSans-111\"/>\n     <use x=\"124.560547\" xlink:href=\"#DejaVuSans-114\"/>\n     <use x=\"165.673828\" xlink:href=\"#DejaVuSans-115\"/>\n     <use x=\"217.773438\" xlink:href=\"#DejaVuSans-101\"/>\n    </g>\n   </g>\n  </g>\n  <g id=\"axes_18\">\n   <g clip-path=\"url(#p7f85553f70)\">\n    <image height=\"75\" id=\"imagec78ea22e5a\" transform=\"scale(1 -1)translate(0 -75)\" width=\"75\" x=\"123.652174\" xlink:href=\"data:image/png;base64,\niVBORw0KGgoAAAANSUhEUgAAAEsAAABLCAYAAAA4TnrqAAAABHNCSVQICAgIfAhkiAAACv9JREFUeJztXF1oHNcV/rbeKfUUe4o9IE3wTmtvibZGW6PFaAnyg1VjFawGq2BT7Ae7AYc6tFELiakp+KENhTQkkAT60NpN6tLGD3aLApEfZBKbVmpZYyRcCWcVWLtdm2xFRyYj0UlhMOrDOXfuWVs2vY4sv9zvZX7OzJ07M98999xz7rk5FItLAIAoRoagCABYE/oAAMfVIsdxAACLtRoAYF21msm6SiUAwNTkJADgTizKdEUhABBF2e66Ij1vsdEAAGwqD2Syt06+CgD4wQ9fys4Vwo0AgMuTl+i++rQui+uw2EzoRJJksg0su91q0Ql+HgAgDHgnVW+qZU26/nOw+L+Rg+8zsyJx2qNNtcyH+it3BPQHFMPSNM1kc+pP8XZNpZLJepmB8/ycZrOpn+bR8+amiSFbd+zPZK5HLJpv/Ss7t953uCwqo9XSZd1JuD7jVNaun/00k70/er7t5aEYBmBDhd7V96kukWhptyepLMssA9iPZYAcdvRTM5QKrUYKGqWQtolooqrZseLcduRIJro6MtJ+TUM3D5RIiYOb8Trfz0SLSglzh7C1OpjJXLcTAHDl9G+yc0+fOAoAmBi/AAC4LRW1QsIdiuvpc5PUKZ368BoA4JWXf5GJWtwkF6MG16+YyQKus2WWAfId/Me7SvpL/jlmxdes33uHw3/Kpa999WJNy/gPbzv+AsneGdGyeqNtu6iJBci+BcA1wZQNrOw3qM4GQIVNgDi6SfUVihpjFwEAawaHAABhGGaiwuAOAMCZs+/QcVjIZHn1bGZWIQgyWcDXWWYZID83Sl95rq6/JAJlDnC7L/VrWcKUmOa/GWhGokp67+o4/d02a/ZuRPcXoaWFTkrddk9FG79xRIxPYrpuW6WUyebZuLw1Qu91w9X3reeq32zQffsG9+oyW8TSXUOkL+OmNh2KbJxbZhnAfiwD5BCyBR8I00FZ7NPc9Ve1RQ2Xm90INzVfNLXmxZWplejtt+4nRT0rzJAwIKW9ls2PcqU3k7VY6UcxKepqVcumJ68DAKZq1LTv1HWZ24foHZez4K+MnAVgmWWEHIoBMashul/VrSs9K7rR7SdeBwA4CbHvby9+RxSX4jNBPUZUBUp5+4JuMRuxHrF6nTA8F3nM+dQQKXZXdDIp3+Z69KCmeGdlYvj+egBAFC1kMjWOtcwyQA7VEussYSUqP1SshjbijjJ7EkZGaZsKn9WjRFWbB1vZg3FtmoZlm8TQqcD7VTZi25jF277+PQAAx9GMvDxJ5kjK7yNlvcxuyywD2I9lgLxqfh0l7aibU2OtlE0I4eCDw/ur1fwUanqcusB13sR1OTCgrfTuUhcAYGr8MgBgz/59ugxuWg67uFNHm0ud3IFMjE8BAPp27MxknkvXWWYZII9xUpJzLa3FNw3Q+OiW+pviDyDRwYHHhhbVa98QjVnThvZ8vDt2GgCwd5D8bN2B9jok7LPz3S8DAKbif2ayNCU7qatYaDsGACe1Y0Nj5KEc/Mo7CuCW0lVseErHPuJlvJKrjIJDf72QkrE4MaaZ9fwwtYquSg8AwHO/msnc9GMAwO9GfwsA8IWva2+Vwm8BcgCAFpYy2dQ0DeMsswxgP5YB8h3DwwCAuZdf02cna3dd5mOlsI63chT5X8My1BByoEz1mjmtZd//Lo0sfn2SLPigrB2XjvMEAGBPP7mXlWscABxufqorc/kYAHrLZJpYZhkgh0qZNJkYC6GmzIOVNzyf5G2f8Ea/bdhnPMP2s+L7Gd03YTe/xlsfvk87rmgV3tfbymni39m+y6X5zKhIKPiE3S+WWQbIoRiyP0sERNkoRcC/yREMOzW6Ig/eriNb+PEw6ZX9z5p5Wn/EKueQjsmik50TN0sUjusdPCTuYD8Wh/iCUFfCwZfu+5wUnwCwzDKC/VgGyG0+cnAJaJ8CdEd5GVoqivygIN/jx5Nifx83w5///U+043x7mTs+5q2IlQpT4V6QprLMMkBexfS7irovj3jC2UJAxthHL765+jUzwEdif+cQ7ziDy13KeMLwCcQ6yywD2I9lgLzy6zXFNJ+Bgd0AgN0cDW68oCdQDOd2rV7tBDaIwPcWNsqVN/ncWS07x6babnx+xetgmWWAfKtB48Aw1Aq+2ZgBAEQx9cOBJ36rumyVfYC3RezyNls5V3jbIa57lGEUyywD5DnKA98VQQn2NkWcWQBH/9bNzKwbj9+7nKEgPBh7Dx98ZM+xzDKA/VgGyHusvFPh6O3bQRPAnJRkrnAMfu8gxeOOj51arTreg58MU70cnvUWx1qFeF6w7D0rAcssA+SOnTyxBABJojvd3mofAB0uLJe0k6zLp+t+9dIzAIBGQ89B+ON7/6BCc195VPUFAGxmore4yoeGNLPOjFALWVhauvu2zwzLLAPkeyvdAIB6fSY7mXCGRblEUVo3FSEjtiJeeeP3AIC4KaIFCLHS+AJvZbisk5l1g5l1YVTr279+8PqK10HBMssA9mMZIH9pjNLQtgjnn8fzMF2o+ZX6hlmOjCxw2lsc61m9jcljAIAO7g/mDGcnPcXxwKNH9Fi0yFOGeio6suyGfbxHNzRq4/r66rNmDzWAZZYB8nFEDJmNNQ0CzkrwS8SslpDVI8pSOMc5yHWR+X7o4GEAQC/PLN53XE+9jDkpabZOzPzlmyJOyXj+OGU5HNj/tjj7Rd5+Is4pI5ncvcXq1+7/hisIyywD5F77w6tLADArZs50FclUcAPSHb6YI+/7lA0fRfN0jbs2k126eIlknA2/t19PjK3znPUDBw8AAKYmpzKZym7oKe/kM3LIooxlmY63Oky6G5ZZBrAfywB5x6Hs4EJRWOlsKlznIIYnkoyUiVHgpiObU52Vd8ixyFikofX3UxzP5SlA0juw1qWmHXOWv+fq1BPdJB8uiarZfC/bD8NvPVQZCpZZBsgde+O5JQBwxAI3SsE7nHjpcVoZAHgeMaNWowwGubyKWnLlwtgYACAQ1mzACUidzMg9HG4DgNnGLADA5VS4nVUdetPM0vVTc9Sb7PYuFr/xgFf8j9hP2rZpouvnuPefCpomdjKbMfIuByqUdxQAEjYLHI8Y1VPWJsBlNjHUAjyxWPaphxfqqfAiOKlgq2LdPPvNUhkg4VyaadaRO6vSTCCjNIq08atScr0sRCfZo3DXCgJUi7Y6OzK/OFEGOLE1EDpVtR7LLAPYj2WA/BZW5lE8n51s8ngv5GVMZuqyCdB1nWwebBRZpBPjNPpXtI1EGovrqs6CqH+erX1AjxcDn+d8xnrcmPKEuk9Fk+6ufLPtJeL4L9m+au6qb3Ec7ZBsNuk9FlSGbqqb6Fpew0uph6ZYk+tTbraWWQbIPf1cZQkAjh7Wazd0l7YAAOb5b14c14anYo3azosV3d7lJaHUqUKo/5wyCxzpHGOUy5wNwWztLfZlstAjI3ijMIyTVDGW6heGWhlHsTJs6frZug6dq45B5T97IsNiPZtEE3Uaw/aJZNWF2JoOxsjP8iyaiZpu2y6n07KrCzPTWmcpRiXMOskUtZpkF3fRgUiMPD9K8+c3sgejKGbt7B7Yw8+hoEkiRjbdnO0fx1r/Xa/TvtK37WRtZ35LrBXYyOb6u1wHfVfMa0Wo97opJiSrMKFllgHsxzJAvqtIyrVPLM66pUgK3mNKdpd1RPpBCv4yL/SqFbxWroWwfcnOKNLN6sIYLYmpFLw07mfYaSgVvDJbVJNJU2GJQ5kOShV0aomzlrf3V/BNHgcWRJarVfAPgf8BVa5sc1yr+gMAAAAASUVORK5CYII=\" y=\"-382.198125\"/>\n   </g>\n   <g id=\"text_18\">\n    <!-- frog -->\n    <g transform=\"translate(149.084642 376.218815)scale(0.12 -0.12)\">\n     <use xlink:href=\"#DejaVuSans-102\"/>\n     <use x=\"35.205078\" xlink:href=\"#DejaVuSans-114\"/>\n     <use x=\"76.287109\" xlink:href=\"#DejaVuSans-111\"/>\n     <use x=\"137.46875\" xlink:href=\"#DejaVuSans-103\"/>\n    </g>\n   </g>\n  </g>\n  <g id=\"axes_19\">\n   <g clip-path=\"url(#p4dfbfac83f)\">\n    <image height=\"75\" id=\"image48e3e530fb\" transform=\"scale(1 -1)translate(0 -75)\" width=\"75\" x=\"240.104348\" xlink:href=\"data:image/png;base64,\niVBORw0KGgoAAAANSUhEUgAAAEsAAABLCAYAAAA4TnrqAAAABHNCSVQICAgIfAhkiAAABVRJREFUeJztm11sVEUUx3/XsCjFUBMWpSTWRHgoDZCwJmwTIzEaChFfqlBjY/wGQ0zaaCGC8qGAWhJidK0lAfxArUQqWqIYvmJJgAgGSrSNrYmYuFW3iZumfXBB+7A+zNwPuLtsz2XbcpP5vdzZOXPPnT39z+yZuVMrm81mMYyIG8a7A2HCBEuACZYAEywBE4rp7FSX+q24e65VTLejRsZTvnNWDQCRSASAvp59vvZGWQICK+vomX8AqK66OW+bNZu/dsrTy8oAGB4e9rXTf0zHdjHjtplSWgJAJuPqwLbn8oX2xbB9cduURJSv+bEYAMsX35q375Z1i1POZgcBoywRljQpPX9BXWOz/PPSkuWbATjUttFnW1n/OQCZTC5lRfJ+TqaSAESjUW8DAIaGM772ES2tMq3kUo9tSolS1utbtgEwkPrMsVXGGwEoKVH9O9uRcGx2iIyyBJhgCRjRMHxho/sz+vaWRwI96MgJ9Zg1q9UQjUanOrbmlgYA9CihP+XeF59XBcDK+tecuq6kmgu+7zgKwE16yAFc6j2pS9oZHmfM1NcOUd+fqt8JGGWJGJGyLKt4SeamN38E3FQC4IGl0wAoLVWfPVnCZWWbviF1/bj9NACP1lU5tkXl+j79uQQ/zzUcBmBnYolTVxlbC0B/uheAgWS77z6jLAFXTUqHhM6eX/spAO81PZa3zSQ9MZ3v7HTqotFqAA60q7/mgnjcsS2qngFAX9L1cbyzB4AP170MQDr9omN7X/s/3nEcgMiQm6pk0mkABlK7ff36ubMpb59tjLIEmGAJGPMJPswYZQm46gRvTbxnrPoRiC0tKg1Zv2pe3jbeHyl7WVqql4uThSPGKEvABHs+OnL6X6eyuurGQM42bT8EwKuNi322rQmVQK6vr/LZiomdKPyid0eSntWOvQFhX38fdKfr5h1/ATA1plKVx2PufTOmqRgZZQkwwRJgAaP2RvpMj+t6ToW65lqrFZO0Hoe/qSUeQ54ZftjZauaKAkTsjum1ZerkoGNbNFM5McoScFVl2flqyrPynzO7FoCBZFugB942sw6A/l9bA91fCPsH6+Q51Xd7J8OLvZNx4YJbl0prm1ZYhee+ZUvV1ShLgEdZFZ5qNeBH8i5jb/t/TrmuJljKYdN2+E8AlumdhiBYlkpQ9+z7CQDPthkL9GaGnZQm066tOaG+x/TYRAC+bfvOsR1rvQ8wyhJhgiXAamo5mwV4adVdTuW2Hee4si4IrzS5Un5j3f0BvaiZ1n4rfC3s1TvFdTWTdY1/z3rhE2oV8slH7ipEZxNGWRLEb6SLxaRpDzrlS+mDgXzk6nq33n6ee0eNrvG/eJD6tDHKEmCCJaCoh9kkXPz7G1/d1sQJADY0LAzs991Ejy7Jhh/ECrYwyhIwbhO8lNtnP+mU/+jdA8BDz6pjTPt31Tq2t3arr9O4olLXeF445kgVbEYSBqMsAaFRVi7Oa9HML3fr7B2SU/owzZBnW/mLNnXG4YfOA+q+uLtwPPbVhoLPM8oSYIIlIJTD8OnV6nDdB9trC7TMjWWpN0zZ7GnRfUZZAsYtKb0WysvLCjfKgb11vLv1y0D3G2UJCKWyppdFC7axz+sfPNjj1M2tUAdwn6kLtm1tlCXABEtAKFOHh1e8A8D+XQ1j+lyjLAGhnOAXxAvvPY0GRlkCQjlndeudhDnBctPAGGUJMMESEJphmPYcPItG8rcbTYyyBIQmdejuco823auPBY01RlkCQqOszGX/pWmUdd1jgiUgNKmD98R02Wgfps+DUZaA0CjresAoS4AJlgATLAEmWAJMsASYYAn4H1nQfncsL4gjAAAAAElFTkSuQmCC\" y=\"-382.198125\"/>\n   </g>\n   <g id=\"text_19\">\n    <!-- truck -->\n    <g transform=\"translate(262.199315 376.218815)scale(0.12 -0.12)\">\n     <use xlink:href=\"#DejaVuSans-116\"/>\n     <use x=\"39.208984\" xlink:href=\"#DejaVuSans-114\"/>\n     <use x=\"80.322266\" xlink:href=\"#DejaVuSans-117\"/>\n     <use x=\"143.701172\" xlink:href=\"#DejaVuSans-99\"/>\n     <use x=\"198.681641\" xlink:href=\"#DejaVuSans-107\"/>\n    </g>\n   </g>\n  </g>\n  <g id=\"axes_20\">\n   <g clip-path=\"url(#p9259087acf)\">\n    <image height=\"75\" id=\"imagea2753246e0\" transform=\"scale(1 -1)translate(0 -75)\" width=\"75\" x=\"356.556522\" xlink:href=\"data:image/png;base64,\niVBORw0KGgoAAAANSUhEUgAAAEsAAABLCAYAAAA4TnrqAAAABHNCSVQICAgIfAhkiAAAChxJREFUeJztXG+IXUcV/z333bq9xb6SPKzvQyLkia6QIF2qC6ZEghqhQcwHF6nV+iVRQ6loC8U/KKhVKrRiWzFtbfAf1mqCJkiDRiS1rJRF2aAJYSl9Cb6UbiJ3pTfR6+rNsn4458z83r13tzum/Ta/L2fenLkzs3N/c+bMmbnbwkP/XAEA5AUckkRkWYrspCAlAGA8uQYA0O15zfUdTWhViwtedzGz1N9ELJROdzOOAgCeemAIAOgnvtJHZgcAgNPDx13eqbIFADibSF1F2XW6Ipd6l7W98e4NTldm/5I+Q5R5981OtzR8WQvlIlNfJ/LrAACvQ8S60R5PZNSWyqKu1ZdvZRhLCy8BAHJiQVLIGy+VUUxIxywjVOHb232LPNBP7IFNTnfXlDDr4yeedHl5ertUoQ2k8A11OtKHF7OXtU++D4kyvyyl/FLmdShuGO1fPiDlNgCRWUGIgxWA9tLwpXpupvzsCG8byzh+t+g5EYXKjWQjN6S6juiK0Bl6A7+9b6kZlT+ihkSZFlSZrj9FIdPp8pBWEl2cxqwQ/utUhS5Yy7oIgKbomM3ktKPt5b7KwXEAkVlBaKNcQ9uo09eRjAgAQLYgb3FTKmXKhVNOtykRg/mPgfkXE17XM4Nu5f1bNeN608SbXM6sEmnZGJVRL5QRy4l0Pp3YTHWJrlDqb+94ap09/jMAwIszXwEAXEYdkVkB8MxKmSM6gc05LRoolmoZUhW63F6ByGuLw0432ZsHAFwod0rZzDPrmSPnAQAT92yzmqihWwEAeXba5VxS53JcbSqKjtMtFUNwxy4O5p3urT35e9qz9wEAnp05Uv+7FDsofV5lZFYA4mAFoG10HU+ZyoXmqadbsKtrll10ebHiNP2+1LGxlKnw4V3es75rp7gMm/adAABcPrXd6c7blHaGnY3ySW3OG/gP7ZIdxeF56Uu+cI3vu+06utKXbtf34fkH79RmxBVgB+WOPZrYpXKWlFI8MisEbUuUZd2IN+WZx3ljLuzJMOVU5me+KxFjfPdOb6i3XCfG9KLL+qDTfXNO5G36Vrfu2QMPYc/de/fVurJdiTjtbTjG1NhvnZG95Om5z9aeW5nWxD2U+bTK+0WcHHrVt1RGZgWghb1ndB9CjqAxylyHjrdnUGfvRsjrvJhOe91QXIZ3DN4LAPjLHL2eq8QL3/+PS/e70q/5QwcAAG8vvBuCI8KDDWpovMMB9H6oiUkrS0qpCofVz72XVOf6QsHIrADEwQpA2025oiGsbHmpX37HerKsZx2l/oAWAY36NU2/MZXL/2dH37Lv9UHlF806fI8yrVtPqHzYq36s8hsqz/U/75Xq+UdmBcC5DujRyUPVwJMLsWz7xFQdwfKv/rm5gwCAN0zoHq/0DLs84EjCq4snKX3bLzSx2/pEygOjD7yHVNa75yd/IwneKvdl4YrMCoBnVkZbmrVcB7c9kp3/0swXvWoont2n98trHRTeYd02fRMA4Kv33/lq9BsAsGK7ol9Rpk0QY9TXSafblmsb6urs+bkkhjobZn3ExAJokVkBiIMVgBamn6t78DbtLI+noXoR4/kzoprzPL93SsodPCS0PUMNzf327wCAyQ+8MaiDG1ReoLzEpp3Ncg6KWGTajPmMV9HRiqBzi0/nM1VtDZFZAWiPOKOrocFhTYbyJrZMeSN+LJMllhllCGHUOKUXLQDBzqV1x07AmBRfEHFSy7x7rYbWwSZGZFYA2s5loC2Ncx1Msj1Tb63QSOlzQ3outQMHcxPZmKyOj6n8iUUDvkbKnSopZgW7hmAxKAp5HlXJEbGrh8yeyKwAxMEKQAsTPxDXob959VIDiiL01UVO5CABxUavW7DDDJ2+c59cVydWPqqJnza1rZIPECpuwX2k+vK6WlwHks/4dE/MUGRWAPzekK9JZrom8x1IB9sv2v0Eut1S6HO2p+zt97oFocPN+vNPXKUVsy7wunBC5YOUp8b+U/rzcbwGSDkKI39jZFYA4mAFoI1sUZMUHs4ujJbKLvl0oeUnzRcjP0tDzujqfJr1HrKZ+sce0MQ2eFgVZsSPk06n3zx170sqf4nXELxr6cgiFpkVgLa790223l8z1tEtyYNP1VUYqhXm68B243koF8NAl9ke26sJcxPoZqPzzo1R5JGb58DneL92ET6NGnSIdm63cQJXhZKDoXLPIjIrAG3nQOYXVi/FwXu7q55vEZmd97pSjnjHCjE+Vyhc5O4VDCoS8IzSLeVBUilH8fuRDmllm98msjhKOuvsNOowtqyHdeQS6RBFZgUgDlYAfIimy564feyj4WQ++eltFZnoiXT5KD0n0++SVcVXeszW23pwiHQaavmc/vxOY1d3UVrPZ+y28mbytie1f/kVLcOGmj/YeiWwKyXuUmRWANrOII5cOdK0G1yy8FZOWYTiCad6QYmYPqQZfOXBqldGDbxX4ZxMb6ZHjoNV8tnl+dFyHX+FEvb1xELDCXhXZ0NmEYWH62Ucylo6MisAbe948sGF3XHPRn8DQEff5lA2MCNf2dhWxh4j9phb8Kg28wipzrgLt7bck19hdrMkZrl+aV845GzHdn11bebPep2xzp5LKSpSWCV2MY6ZfD2AyKwgxMEKQNsbsk6DuvLRMwCcEv/aTOMdfIxiUQMz7BQ9+LbK76o8R7ecvVtghv39XlVYxIMNbmUaljRtM5u29kEWmxfTqW9T1i/d+Xa4vX8DiMwKQguAfiJBAXq3L7I36JfYj6h8ysjA5xzGLDXs7yTVn13KXAF2MqsM3tLQ1UVKV88j2U2wOmxRImfbMTeplAV8GCStlPF5kVkBaO1QZj07MpJ2JCwfKu6g+fsHY5JFOmnaW/jKvso5N9KUBbKabIIxi/uASh5vVdwn/A3lK+4B6hfxajZvpK4GO414YBGMOFgBaK1MyTRs8YkvRsPKK7zK5yMqHKRpaOd4/q77pFe6lWA9H2Xz9FADndBK4u7pVzpTS69Sl7MdPLWtruoi4MtFZgXAnVIcoIHcrw7dgSZ7qy/lffoCR8O99oBZ//Uu2/ZWzfDySbjtXYnC7h/1NBl/g9XV5GyvBesL9z1GHYLRPmZhqQYW2bs/RvbMogWeUexcGiPMwWP7ZLVZQ02sa0LTMm/1dlFH1SYyg21blDaULRryRuuMzApAHKwA0N5wvbDpYJc96StSN2WM7rxnq16WY6Nc/edar3SwYEa7aTq6f4DVUFd138d7zGp5fk6maGRWANrAJzTJHwzbm7PR5qDVrSp/11BddY/HxrJ6gNBkSKv/34HLNe0Nm3RNRr/6XF6RQH2Rqd+0jswKAF2d2U3ZZnP4soLhjyobDjOcXarGw4D6VqZYQ1c/hhpFNWbVxJBeg87STZGPKuq2LjIrAHGwAkDT8GnKNkNtZ2ls4C20V50KQH2PxzSvGtAmg900DZtQ9cB5/2dpvi1X7UNTZGGtvWt0HYLxPzKg9oXruqzDAAAAAElFTkSuQmCC\" y=\"-382.198125\"/>\n   </g>\n   <g id=\"text_20\">\n    <!-- plane -->\n    <g transform=\"translate(377.398989 376.218815)scale(0.12 -0.12)\">\n     <use xlink:href=\"#DejaVuSans-112\"/>\n     <use x=\"63.476562\" xlink:href=\"#DejaVuSans-108\"/>\n     <use x=\"91.259766\" xlink:href=\"#DejaVuSans-97\"/>\n     <use x=\"152.539062\" xlink:href=\"#DejaVuSans-110\"/>\n     <use x=\"215.917969\" xlink:href=\"#DejaVuSans-101\"/>\n    </g>\n   </g>\n  </g>\n </g>\n <defs>\n  <clipPath id=\"p304f25154f\">\n   <rect height=\"74.97931\" width=\"74.97931\" x=\"7.2\" y=\"22.318125\"/>\n  </clipPath>\n  <clipPath id=\"pb58017fcb0\">\n   <rect height=\"74.97931\" width=\"74.97931\" x=\"123.652174\" y=\"22.318125\"/>\n  </clipPath>\n  <clipPath id=\"p89866d4cbd\">\n   <rect height=\"74.97931\" width=\"74.97931\" x=\"240.104348\" y=\"22.318125\"/>\n  </clipPath>\n  <clipPath id=\"p885007983c\">\n   <rect height=\"74.97931\" width=\"74.97931\" x=\"356.556522\" y=\"22.318125\"/>\n  </clipPath>\n  <clipPath id=\"p712ee46120\">\n   <rect height=\"74.97931\" width=\"74.97931\" x=\"7.2\" y=\"112.293297\"/>\n  </clipPath>\n  <clipPath id=\"p4edf2f0782\">\n   <rect height=\"74.97931\" width=\"74.97931\" x=\"123.652174\" y=\"112.293297\"/>\n  </clipPath>\n  <clipPath id=\"p9d84ca2e97\">\n   <rect height=\"74.97931\" width=\"74.97931\" x=\"240.104348\" y=\"112.293297\"/>\n  </clipPath>\n  <clipPath id=\"p95ac4061cc\">\n   <rect height=\"74.97931\" width=\"74.97931\" x=\"356.556522\" y=\"112.293297\"/>\n  </clipPath>\n  <clipPath id=\"pa1e2343e73\">\n   <rect height=\"74.97931\" width=\"74.97931\" x=\"7.2\" y=\"202.26847\"/>\n  </clipPath>\n  <clipPath id=\"p80e32c39f8\">\n   <rect height=\"74.97931\" width=\"74.97931\" x=\"123.652174\" y=\"202.26847\"/>\n  </clipPath>\n  <clipPath id=\"pe1b9c972d5\">\n   <rect height=\"74.97931\" width=\"74.97931\" x=\"240.104348\" y=\"202.26847\"/>\n  </clipPath>\n  <clipPath id=\"pe8589aeb86\">\n   <rect height=\"74.97931\" width=\"74.97931\" x=\"356.556522\" y=\"202.26847\"/>\n  </clipPath>\n  <clipPath id=\"pea86b63c85\">\n   <rect height=\"74.97931\" width=\"74.97931\" x=\"7.2\" y=\"292.243642\"/>\n  </clipPath>\n  <clipPath id=\"pb321a3bd37\">\n   <rect height=\"74.97931\" width=\"74.97931\" x=\"123.652174\" y=\"292.243642\"/>\n  </clipPath>\n  <clipPath id=\"p34415d6a3e\">\n   <rect height=\"74.97931\" width=\"74.97931\" x=\"240.104348\" y=\"292.243642\"/>\n  </clipPath>\n  <clipPath id=\"p9a38c3e85d\">\n   <rect height=\"74.97931\" width=\"74.97931\" x=\"356.556522\" y=\"292.243642\"/>\n  </clipPath>\n  <clipPath id=\"pc7263cf74d\">\n   <rect height=\"74.97931\" width=\"74.97931\" x=\"7.2\" y=\"382.218815\"/>\n  </clipPath>\n  <clipPath id=\"p7f85553f70\">\n   <rect height=\"74.97931\" width=\"74.97931\" x=\"123.652174\" y=\"382.218815\"/>\n  </clipPath>\n  <clipPath id=\"p4dfbfac83f\">\n   <rect height=\"74.97931\" width=\"74.97931\" x=\"240.104348\" y=\"382.218815\"/>\n  </clipPath>\n  <clipPath id=\"p9259087acf\">\n   <rect height=\"74.97931\" width=\"74.97931\" x=\"356.556522\" y=\"382.218815\"/>\n  </clipPath>\n </defs>\n</svg>\n"
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qlQ0W-q7snrR",
        "colab_type": "text"
      },
      "source": [
        "# save and load checkpoint"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eGyG96ysskV1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def save_checkpoint(model, is_best, filename='data/checkpoint.pth.tar'):\n",
        "    \"\"\"Save checkpoint if a new best is achieved\"\"\"\n",
        "    if is_best:\n",
        "        torch.save(model.state_dict(), filename)  # save checkpoint\n",
        "    else:\n",
        "        print (\"=> Validation Accuracy did not improve\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ODkWdaKUsw8Q",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def load_checkpoint(model, filename = 'data/checkpoint.pth.tar'):\n",
        "    sd = torch.load(filename, map_location=lambda storage, loc: storage)\n",
        "    names = set(model.state_dict().keys())\n",
        "    for n in list(sd.keys()):\n",
        "        if n not in names and n+'_raw' in names:\n",
        "            if n+'_raw' not in sd: sd[n+'_raw'] = sd[n]\n",
        "            del sd[n]\n",
        "    model.load_state_dict(sd)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BMqnHHlLs1nL",
        "colab_type": "text"
      },
      "source": [
        "# Global variables"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6oEDCaDYs0qk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_loss = 0.0\n",
        "test_loss = 0.0\n",
        "best_acc = 0.0\n",
        "trn_losses = []\n",
        "trn_accs = []\n",
        "val_losses = []\n",
        "val_accs = []"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "po5STdeZs9ai",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "total = 0\n",
        "correct = 0"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ogaK2kjKtB1O",
        "colab_type": "text"
      },
      "source": [
        "# Helper functions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xNVROs82tBDb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def accuracy(output, target, is_test=False):\n",
        "    global total\n",
        "    global correct\n",
        "    batch_size = target.size(0)\n",
        "    total += batch_size    \n",
        "    _, pred = output.max(dim=1)\n",
        "    if is_test:\n",
        "        preds.extend(pred)\n",
        "    correct += torch.sum(pred == target.data)\n",
        "    return (correct.float() / total) * 100"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OdwknXrXtPWK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def reset():\n",
        "    global total, correct\n",
        "    global train_loss, test_loss, best_acc\n",
        "    global trn_accs, trn_losses, val_accs, val_losses\n",
        "    total, correct = 0, 0\n",
        "    train_loss, test_loss, best_acc = 0.0, 0.0, 0.0\n",
        "    trn_losses, trn_accs, val_accs, val_losses = [], [], [], []"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U_QEGWOBtSoR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class AvgStats(object):\n",
        "    def __init__(self):\n",
        "        self.reset()\n",
        "        \n",
        "    def reset(self):\n",
        "        self.losses =[]\n",
        "        self.precs =[]\n",
        "        self.its = []\n",
        "        \n",
        "    def append(self, loss, prec, it):\n",
        "        self.losses.append(loss)\n",
        "        self.precs.append(prec)\n",
        "        self.its.append(it)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TiptBnxLtaHx",
        "colab_type": "text"
      },
      "source": [
        "# Train and Test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BrJGCUsTtZHN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def train(epoch=0, model=None, optimizer=None):\n",
        "    model.train()\n",
        "    global best_acc\n",
        "    global trn_accs, trn_losses\n",
        "    is_improving = True\n",
        "    counter = 0\n",
        "    running_loss = 0.\n",
        "    avg_beta = 0.98\n",
        "\n",
        "    for i, (input, target) in enumerate(train_loader):\n",
        "        bt_start = time.time()\n",
        "        input, target = input.to(device), target.to(device)\n",
        "        var_ip, var_tg = Variable(input), Variable(target)\n",
        "                                    \n",
        "        output = model(var_ip)\n",
        "        loss = criterion(output, var_tg)\n",
        "            \n",
        "        running_loss = avg_beta * running_loss + (1-avg_beta) *loss.item()\n",
        "        smoothed_loss = running_loss / (1 - avg_beta**(i+1))\n",
        "        \n",
        "        trn_losses.append(smoothed_loss)\n",
        "            \n",
        "        # measure accuracy and record loss\n",
        "        prec = accuracy(output.data, target)\n",
        "        trn_accs.append(prec)\n",
        "\n",
        "        train_stats.append(smoothed_loss, prec, time.time()-bt_start)\n",
        "        if prec > best_acc :\n",
        "            best_acc = prec\n",
        "            save_checkpoint(model, True)\n",
        "\n",
        "        # compute gradient and do SGD step\n",
        "        optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "        optimizer.step()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gOZQwwhXtiUv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def test(model=None):\n",
        "    with torch.no_grad():\n",
        "        model.eval()\n",
        "        global val_accs, val_losses\n",
        "        running_loss = 0.\n",
        "        avg_beta = 0.98\n",
        "        for i, (input, target) in enumerate(test_loader):\n",
        "            bt_start = time.time()\n",
        "            input, target = input.to(device), target.to(device)\n",
        "            var_ip, var_tg = Variable(input), Variable(target)\n",
        "            output = model(var_ip)\n",
        "            loss = criterion(output, var_tg)\n",
        "        \n",
        "            running_loss = avg_beta * running_loss + (1-avg_beta) *loss.item()\n",
        "            smoothed_loss = running_loss / (1 - avg_beta**(i+1))\n",
        "\n",
        "            # measure accuracy and record loss\n",
        "            prec = accuracy(output.data, target, is_test=True)\n",
        "            test_stats.append(loss.item(), prec, time.time()-bt_start)\n",
        "        \n",
        "            val_losses.append(smoothed_loss)\n",
        "            val_accs.append(prec)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6tp2g4a3tw8p",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def fit(model=None, optimizer=None, sched=None):\n",
        "    print(\"Epoch\\tTrn_loss\\tVal_loss\\tTrn_acc\\t\\tVal_acc\")\n",
        "    for j in range(epochs):\n",
        "        train(j, model, optimizer)\n",
        "        test(model)\n",
        "        if sched:\n",
        "          sched.step(j)\n",
        "        print(\"{}\\t{:06.8f}\\t{:06.8f}\\t{:06.8f}\\t{:06.8f}\"\n",
        "              .format(j+1, trn_losses[-1], val_losses[-1], trn_accs[-1], val_accs[-1]))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-vBIKBdbt4f2",
        "colab_type": "text"
      },
      "source": [
        "# Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VGMxBRimt2xJ",
        "colab_type": "code",
        "outputId": "4e3d40e6-2829-4894-a90f-7b33d4beca0e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        }
      },
      "source": [
        "model = models.resnet18(pretrained=True)"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading: \"https://download.pytorch.org/models/resnet18-5c106cde.pth\" to /root/.cache/torch/checkpoints/resnet18-5c106cde.pth\n",
            "\n",
            "  0%|          | 0.00/44.7M [00:00<?, ?B/s]\u001b[A\n",
            "  6%|▋         | 2.83M/44.7M [00:00<00:01, 29.6MB/s]\u001b[A\n",
            " 52%|█████▏    | 23.2M/44.7M [00:00<00:00, 39.9MB/s]\u001b[A\n",
            " 97%|█████████▋| 43.2M/44.7M [00:00<00:00, 52.7MB/s]\u001b[A\n",
            "100%|██████████| 44.7M/44.7M [00:00<00:00, 150MB/s] \u001b[A"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_pX5ab6Hkh_l",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model.fc = nn.Linear(in_features=model.fc.in_features, out_features=10, bias=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xSY_R_Wxt986",
        "colab_type": "code",
        "outputId": "34515d60-0dc4-481d-9067-1ee4dc9d6b44",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "model = model.to(device)\n",
        "model"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "ResNet(\n",
              "  (conv1): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n",
              "  (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "  (relu): ReLU(inplace=True)\n",
              "  (maxpool): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n",
              "  (layer1): Sequential(\n",
              "    (0): BasicBlock(\n",
              "      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "    )\n",
              "    (1): BasicBlock(\n",
              "      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "    )\n",
              "  )\n",
              "  (layer2): Sequential(\n",
              "    (0): BasicBlock(\n",
              "      (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (downsample): Sequential(\n",
              "        (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
              "        (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "    )\n",
              "    (1): BasicBlock(\n",
              "      (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "    )\n",
              "  )\n",
              "  (layer3): Sequential(\n",
              "    (0): BasicBlock(\n",
              "      (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (downsample): Sequential(\n",
              "        (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
              "        (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "    )\n",
              "    (1): BasicBlock(\n",
              "      (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "    )\n",
              "  )\n",
              "  (layer4): Sequential(\n",
              "    (0): BasicBlock(\n",
              "      (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (downsample): Sequential(\n",
              "        (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
              "        (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "    )\n",
              "    (1): BasicBlock(\n",
              "      (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "    )\n",
              "  )\n",
              "  (avgpool): AdaptiveAvgPool2d(output_size=(1, 1))\n",
              "  (fc): Linear(in_features=512, out_features=10, bias=True)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z-2R9xnsuCIr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "for param in model.parameters():\n",
        "  param.requires_grad = True"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-wFs4jrvuGAM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "criterion = nn.CrossEntropyLoss()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Nxvk1OPTuJhG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "optimizer = optim.SGD(model.parameters(), lr=1e-2, weight_decay=1e-4, momentum=0.9)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pkBtb_RGiKrt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sched = torch.optim.lr_scheduler.StepLR(optimizer, step_size=30, gamma=0.2)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vY-VnxO5ckNS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "save_checkpoint(model, True, filename='before_sgd_resnet18.pth.tar')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q82CU_THuNY2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "preds = []\n",
        "train_stats = AvgStats()\n",
        "test_stats = AvgStats()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W-kJTqs_k_4y",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "reset()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bH-IxGTelOg5",
        "colab_type": "code",
        "outputId": "e05c991f-be24-46aa-a746-4a82d7aca5a6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "fit(model, optimizer, sched)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch\tTrn_loss\tVal_loss\tTrn_acc\t\tVal_acc\n",
            "1\t0.79313249\t0.74132563\t66.39599609\t67.74333191\n",
            "2\t0.64829487\t0.69601622\t71.70090485\t72.15666962\n",
            "3\t0.57526764\t0.55291760\t74.33882141\t74.75055695\n",
            "4\t0.54393889\t0.54045472\t76.20870209\t76.45500183\n",
            "5\t0.49835263\t0.50361715\t77.51206970\t77.70300293\n",
            "6\t0.46420780\t0.49422138\t78.58942413\t78.72583008\n",
            "7\t0.45107309\t0.49169979\t79.45463562\t79.55451965\n",
            "8\t0.41838781\t0.46016596\t80.17276764\t80.25228882\n",
            "9\t0.39948578\t0.47960735\t80.82415009\t80.87444305\n",
            "10\t0.38481720\t0.46406015\t81.36254120\t81.41949463\n",
            "11\t0.37888035\t0.46209894\t81.86215973\t81.90515900\n",
            "12\t0.35764240\t0.49648709\t82.33126068\t82.34652710\n",
            "13\t0.34535255\t0.44940313\t82.72687531\t82.74948883\n",
            "14\t0.34110329\t0.44997972\t83.10205078\t83.13071442\n",
            "15\t0.32506131\t0.50102261\t83.45652008\t83.46256256\n",
            "16\t0.31396908\t0.47125938\t83.77588654\t83.78469086\n",
            "17\t0.29703725\t0.44831329\t84.07327271\t84.08509827\n",
            "18\t0.28847559\t0.46605055\t84.36186981\t84.36953735\n",
            "19\t0.26968112\t0.46469901\t84.62699127\t84.63517761\n",
            "20\t0.27772126\t0.47915473\t84.88411713\t84.88533783\n",
            "21\t0.25286054\t0.46115444\t85.13272095\t85.13761902\n",
            "22\t0.26605012\t0.47559459\t85.36656189\t85.36371613\n",
            "23\t0.24831418\t0.49757756\t85.59007263\t85.58673859\n",
            "24\t0.25059786\t0.48623496\t85.79328918\t85.79034424\n",
            "25\t0.23600223\t0.47068544\t85.99718475\t85.99359894\n",
            "26\t0.22494007\t0.50869533\t86.19309235\t86.18724060\n",
            "27\t0.23348828\t0.48087260\t86.37037659\t86.36710358\n",
            "28\t0.22676802\t0.47215305\t86.54916382\t86.54654694\n",
            "29\t0.21785564\t0.46354608\t86.72693634\t86.72379303\n",
            "30\t0.21162462\t0.53274215\t86.89458466\t86.88027954\n",
            "31\t0.21166761\t0.53869241\t87.04238129\t87.03113556\n",
            "32\t0.12901613\t0.45956419\t87.24330139\t87.24255371\n",
            "33\t0.10548838\t0.47356927\t87.46573639\t87.46464539\n",
            "34\t0.10196704\t0.47425541\t87.68570709\t87.68515015\n",
            "35\t0.09636748\t0.49221567\t87.90023804\t87.89894867\n",
            "36\t0.08946390\t0.49302650\t88.11074829\t88.10971832\n",
            "37\t0.08915682\t0.50736658\t88.31027222\t88.30720520\n",
            "38\t0.08167949\t0.51957551\t88.50184631\t88.49829102\n",
            "39\t0.07785407\t0.51736467\t88.68635559\t88.68264771\n",
            "40\t0.07487367\t0.51906090\t88.86427307\t88.85987091\n",
            "41\t0.07801018\t0.53166702\t89.03677368\t89.03207397\n",
            "42\t0.07089360\t0.53714359\t89.20402527\t89.19841003\n",
            "43\t0.06642022\t0.53923886\t89.36606598\t89.36000824\n",
            "44\t0.06520043\t0.54707644\t89.51817322\t89.51212311\n",
            "45\t0.06288845\t0.55625769\t89.66650391\t89.65874481\n",
            "46\t0.06024813\t0.56420227\t89.80803680\t89.80083466\n",
            "47\t0.05890661\t0.57606644\t89.94541168\t89.93719482\n",
            "48\t0.05916005\t0.57991717\t90.07871246\t90.07065582\n",
            "49\t0.05510516\t0.56493656\t90.20498657\t90.19751740\n",
            "50\t0.05324254\t0.57187006\t90.33040619\t90.32279968\n",
            "51\t0.05592724\t0.57877875\t90.45095825\t90.44166565\n",
            "52\t0.05205885\t0.58473457\t90.56552887\t90.55656433\n",
            "53\t0.05795166\t0.58477140\t90.67735291\t90.66786194\n",
            "54\t0.05167289\t0.58932762\t90.78689575\t90.77799225\n",
            "55\t0.05348674\t0.58989571\t90.89057159\t90.88117981\n",
            "56\t0.04920138\t0.60327963\t90.99250793\t90.98291016\n",
            "57\t0.04874067\t0.59635470\t91.08953857\t91.08005524\n",
            "58\t0.05661839\t0.59648271\t91.18529510\t91.17563629\n",
            "59\t0.04860071\t0.59933345\t91.27849579\t91.26858521\n",
            "60\t0.04954080\t0.61841550\t91.36839294\t91.35786438\n",
            "61\t0.04683500\t0.62086906\t91.45500946\t91.44398499\n",
            "62\t0.03361535\t0.60653936\t91.54169464\t91.53148651\n",
            "63\t0.03629625\t0.61012442\t91.62700653\t91.61732483\n",
            "64\t0.03661882\t0.61889784\t91.70989227\t91.69974518\n",
            "65\t0.03158181\t0.61121627\t91.79228973\t91.78195190\n",
            "66\t0.03198171\t0.60985318\t91.87187195\t91.86165619\n",
            "67\t0.03294961\t0.62393221\t91.94958496\t91.93979645\n",
            "68\t0.03185345\t0.62945219\t92.02545929\t92.01586151\n",
            "69\t0.03056886\t0.63330169\t92.10055542\t92.09095001\n",
            "70\t0.02982614\t0.62788526\t92.17372894\t92.16364288\n",
            "71\t0.02658203\t0.63505867\t92.24443054\t92.23478699\n",
            "72\t0.02896749\t0.64647386\t92.31350708\t92.30355835\n",
            "73\t0.02909809\t0.65142596\t92.37981415\t92.36981201\n",
            "74\t0.03035102\t0.65025293\t92.44519043\t92.43528748\n",
            "75\t0.02877921\t0.65210523\t92.50855255\t92.49853516\n",
            "76\t0.03093172\t0.65117952\t92.57009125\t92.55984497\n",
            "77\t0.02432454\t0.64902322\t92.63073730\t92.62064362\n",
            "78\t0.02246516\t0.66204643\t92.69081116\t92.68065643\n",
            "79\t0.02524504\t0.66263886\t92.74877167\t92.73876190\n",
            "80\t0.02615849\t0.65878375\t92.80528259\t92.79504395\n",
            "81\t0.02918135\t0.66602669\t92.86038971\t92.85063934\n",
            "82\t0.02597109\t0.66621750\t92.91503143\t92.90481567\n",
            "83\t0.02587199\t0.66403043\t92.96772766\t92.95747375\n",
            "84\t0.02510108\t0.65672521\t93.01892853\t93.00936127\n",
            "85\t0.02427394\t0.65371345\t93.06957245\t93.05979919\n",
            "86\t0.02851331\t0.65958244\t93.11873627\t93.10866547\n",
            "87\t0.02575279\t0.66556215\t93.16689301\t93.15695190\n",
            "88\t0.02143758\t0.65576641\t93.21432495\t93.20447540\n",
            "89\t0.02345233\t0.65387320\t93.26042938\t93.25056458\n",
            "90\t0.02382656\t0.65687062\t93.30610657\t93.29640961\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0OipEoMoaHt-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "save_checkpoint(model, True, 'sgd_resnet18_9330.pth.tar')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "upexYQmyabrD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ep_sgd_losses = []\n",
        "for i in range(0, len(train_stats.losses), len(train_loader)):\n",
        "    if i != 0 :\n",
        "        ep_sgd_losses.append(train_stats.losses[i])\n",
        "        \n",
        "ep_sgd_lossesv = []\n",
        "for i in range(0, len(test_stats.losses), len(test_loader)):\n",
        "    if(i != 0):\n",
        "        ep_sgd_lossesv.append(test_stats.losses[i])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ojQ4G1KWb0zR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ep_sgd_accs = []\n",
        "for i in range(0, len(train_stats.losses), len(train_loader)):\n",
        "    if i != 0 :\n",
        "        ep_sgd_accs.append(train_stats.precs[i])\n",
        "        \n",
        "ep_sgd_accsv = []\n",
        "for i in range(0, len(test_stats.losses), len(test_loader)):\n",
        "    if(i != 0):\n",
        "        ep_sgd_accsv.append(test_stats.precs[i])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3QTgQbtgb1Hi",
        "colab_type": "code",
        "outputId": "8a465f34-baf7-432e-d9b7-6d803736841e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 371
        }
      },
      "source": [
        "plt.xlabel(\"Epochs\")\n",
        "plt.ylabel(\"Accuracy\")\n",
        "plt.plot(ep_sgd_accs, 'r', label='Train')\n",
        "plt.plot(ep_sgd_accsv, 'b', label='Val')\n",
        "plt.legend()\n",
        "plt.savefig('sgd_resnet18_9330_acc.jpg')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/svg+xml": "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n  \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<!-- Created with matplotlib (https://matplotlib.org/) -->\n<svg height=\"262.19625pt\" version=\"1.1\" viewBox=\"0 0 382.603125 262.19625\" width=\"382.603125pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n <defs>\n  <style type=\"text/css\">\n*{stroke-linecap:butt;stroke-linejoin:round;}\n  </style>\n </defs>\n <g id=\"figure_1\">\n  <g id=\"patch_1\">\n   <path d=\"M 0 262.19625 \nL 382.603125 262.19625 \nL 382.603125 0 \nL 0 0 \nz\n\" style=\"fill:none;\"/>\n  </g>\n  <g id=\"axes_1\">\n   <g id=\"patch_2\">\n    <path d=\"M 40.603125 224.64 \nL 375.403125 224.64 \nL 375.403125 7.2 \nL 40.603125 7.2 \nz\n\" style=\"fill:#ffffff;\"/>\n   </g>\n   <g id=\"matplotlib.axis_1\">\n    <g id=\"xtick_1\">\n     <g id=\"line2d_1\">\n      <defs>\n       <path d=\"M 0 0 \nL 0 3.5 \n\" id=\"mac5e7df0cf\" style=\"stroke:#000000;stroke-width:0.8;\"/>\n      </defs>\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"55.821307\" xlink:href=\"#mac5e7df0cf\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_1\">\n      <!-- 0 -->\n      <defs>\n       <path d=\"M 31.78125 66.40625 \nQ 24.171875 66.40625 20.328125 58.90625 \nQ 16.5 51.421875 16.5 36.375 \nQ 16.5 21.390625 20.328125 13.890625 \nQ 24.171875 6.390625 31.78125 6.390625 \nQ 39.453125 6.390625 43.28125 13.890625 \nQ 47.125 21.390625 47.125 36.375 \nQ 47.125 51.421875 43.28125 58.90625 \nQ 39.453125 66.40625 31.78125 66.40625 \nz\nM 31.78125 74.21875 \nQ 44.046875 74.21875 50.515625 64.515625 \nQ 56.984375 54.828125 56.984375 36.375 \nQ 56.984375 17.96875 50.515625 8.265625 \nQ 44.046875 -1.421875 31.78125 -1.421875 \nQ 19.53125 -1.421875 13.0625 8.265625 \nQ 6.59375 17.96875 6.59375 36.375 \nQ 6.59375 54.828125 13.0625 64.515625 \nQ 19.53125 74.21875 31.78125 74.21875 \nz\n\" id=\"DejaVuSans-48\"/>\n      </defs>\n      <g transform=\"translate(52.640057 239.238438)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_2\">\n     <g id=\"line2d_2\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"124.994861\" xlink:href=\"#mac5e7df0cf\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_2\">\n      <!-- 20 -->\n      <defs>\n       <path d=\"M 19.1875 8.296875 \nL 53.609375 8.296875 \nL 53.609375 0 \nL 7.328125 0 \nL 7.328125 8.296875 \nQ 12.9375 14.109375 22.625 23.890625 \nQ 32.328125 33.6875 34.8125 36.53125 \nQ 39.546875 41.84375 41.421875 45.53125 \nQ 43.3125 49.21875 43.3125 52.78125 \nQ 43.3125 58.59375 39.234375 62.25 \nQ 35.15625 65.921875 28.609375 65.921875 \nQ 23.96875 65.921875 18.8125 64.3125 \nQ 13.671875 62.703125 7.8125 59.421875 \nL 7.8125 69.390625 \nQ 13.765625 71.78125 18.9375 73 \nQ 24.125 74.21875 28.421875 74.21875 \nQ 39.75 74.21875 46.484375 68.546875 \nQ 53.21875 62.890625 53.21875 53.421875 \nQ 53.21875 48.921875 51.53125 44.890625 \nQ 49.859375 40.875 45.40625 35.40625 \nQ 44.1875 33.984375 37.640625 27.21875 \nQ 31.109375 20.453125 19.1875 8.296875 \nz\n\" id=\"DejaVuSans-50\"/>\n      </defs>\n      <g transform=\"translate(118.632361 239.238438)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-50\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_3\">\n     <g id=\"line2d_3\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"194.168414\" xlink:href=\"#mac5e7df0cf\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_3\">\n      <!-- 40 -->\n      <defs>\n       <path d=\"M 37.796875 64.3125 \nL 12.890625 25.390625 \nL 37.796875 25.390625 \nz\nM 35.203125 72.90625 \nL 47.609375 72.90625 \nL 47.609375 25.390625 \nL 58.015625 25.390625 \nL 58.015625 17.1875 \nL 47.609375 17.1875 \nL 47.609375 0 \nL 37.796875 0 \nL 37.796875 17.1875 \nL 4.890625 17.1875 \nL 4.890625 26.703125 \nz\n\" id=\"DejaVuSans-52\"/>\n      </defs>\n      <g transform=\"translate(187.805914 239.238438)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-52\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_4\">\n     <g id=\"line2d_4\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"263.341968\" xlink:href=\"#mac5e7df0cf\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_4\">\n      <!-- 60 -->\n      <defs>\n       <path d=\"M 33.015625 40.375 \nQ 26.375 40.375 22.484375 35.828125 \nQ 18.609375 31.296875 18.609375 23.390625 \nQ 18.609375 15.53125 22.484375 10.953125 \nQ 26.375 6.390625 33.015625 6.390625 \nQ 39.65625 6.390625 43.53125 10.953125 \nQ 47.40625 15.53125 47.40625 23.390625 \nQ 47.40625 31.296875 43.53125 35.828125 \nQ 39.65625 40.375 33.015625 40.375 \nz\nM 52.59375 71.296875 \nL 52.59375 62.3125 \nQ 48.875 64.0625 45.09375 64.984375 \nQ 41.3125 65.921875 37.59375 65.921875 \nQ 27.828125 65.921875 22.671875 59.328125 \nQ 17.53125 52.734375 16.796875 39.40625 \nQ 19.671875 43.65625 24.015625 45.921875 \nQ 28.375 48.1875 33.59375 48.1875 \nQ 44.578125 48.1875 50.953125 41.515625 \nQ 57.328125 34.859375 57.328125 23.390625 \nQ 57.328125 12.15625 50.6875 5.359375 \nQ 44.046875 -1.421875 33.015625 -1.421875 \nQ 20.359375 -1.421875 13.671875 8.265625 \nQ 6.984375 17.96875 6.984375 36.375 \nQ 6.984375 53.65625 15.1875 63.9375 \nQ 23.390625 74.21875 37.203125 74.21875 \nQ 40.921875 74.21875 44.703125 73.484375 \nQ 48.484375 72.75 52.59375 71.296875 \nz\n\" id=\"DejaVuSans-54\"/>\n      </defs>\n      <g transform=\"translate(256.979468 239.238438)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-54\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_5\">\n     <g id=\"line2d_5\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"332.515522\" xlink:href=\"#mac5e7df0cf\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_5\">\n      <!-- 80 -->\n      <defs>\n       <path d=\"M 31.78125 34.625 \nQ 24.75 34.625 20.71875 30.859375 \nQ 16.703125 27.09375 16.703125 20.515625 \nQ 16.703125 13.921875 20.71875 10.15625 \nQ 24.75 6.390625 31.78125 6.390625 \nQ 38.8125 6.390625 42.859375 10.171875 \nQ 46.921875 13.96875 46.921875 20.515625 \nQ 46.921875 27.09375 42.890625 30.859375 \nQ 38.875 34.625 31.78125 34.625 \nz\nM 21.921875 38.8125 \nQ 15.578125 40.375 12.03125 44.71875 \nQ 8.5 49.078125 8.5 55.328125 \nQ 8.5 64.0625 14.71875 69.140625 \nQ 20.953125 74.21875 31.78125 74.21875 \nQ 42.671875 74.21875 48.875 69.140625 \nQ 55.078125 64.0625 55.078125 55.328125 \nQ 55.078125 49.078125 51.53125 44.71875 \nQ 48 40.375 41.703125 38.8125 \nQ 48.828125 37.15625 52.796875 32.3125 \nQ 56.78125 27.484375 56.78125 20.515625 \nQ 56.78125 9.90625 50.3125 4.234375 \nQ 43.84375 -1.421875 31.78125 -1.421875 \nQ 19.734375 -1.421875 13.25 4.234375 \nQ 6.78125 9.90625 6.78125 20.515625 \nQ 6.78125 27.484375 10.78125 32.3125 \nQ 14.796875 37.15625 21.921875 38.8125 \nz\nM 18.3125 54.390625 \nQ 18.3125 48.734375 21.84375 45.5625 \nQ 25.390625 42.390625 31.78125 42.390625 \nQ 38.140625 42.390625 41.71875 45.5625 \nQ 45.3125 48.734375 45.3125 54.390625 \nQ 45.3125 60.0625 41.71875 63.234375 \nQ 38.140625 66.40625 31.78125 66.40625 \nQ 25.390625 66.40625 21.84375 63.234375 \nQ 18.3125 60.0625 18.3125 54.390625 \nz\n\" id=\"DejaVuSans-56\"/>\n      </defs>\n      <g transform=\"translate(326.153022 239.238438)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-56\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"text_6\">\n     <!-- Epochs -->\n     <defs>\n      <path d=\"M 9.8125 72.90625 \nL 55.90625 72.90625 \nL 55.90625 64.59375 \nL 19.671875 64.59375 \nL 19.671875 43.015625 \nL 54.390625 43.015625 \nL 54.390625 34.71875 \nL 19.671875 34.71875 \nL 19.671875 8.296875 \nL 56.78125 8.296875 \nL 56.78125 0 \nL 9.8125 0 \nz\n\" id=\"DejaVuSans-69\"/>\n      <path d=\"M 18.109375 8.203125 \nL 18.109375 -20.796875 \nL 9.078125 -20.796875 \nL 9.078125 54.6875 \nL 18.109375 54.6875 \nL 18.109375 46.390625 \nQ 20.953125 51.265625 25.265625 53.625 \nQ 29.59375 56 35.59375 56 \nQ 45.5625 56 51.78125 48.09375 \nQ 58.015625 40.1875 58.015625 27.296875 \nQ 58.015625 14.40625 51.78125 6.484375 \nQ 45.5625 -1.421875 35.59375 -1.421875 \nQ 29.59375 -1.421875 25.265625 0.953125 \nQ 20.953125 3.328125 18.109375 8.203125 \nz\nM 48.6875 27.296875 \nQ 48.6875 37.203125 44.609375 42.84375 \nQ 40.53125 48.484375 33.40625 48.484375 \nQ 26.265625 48.484375 22.1875 42.84375 \nQ 18.109375 37.203125 18.109375 27.296875 \nQ 18.109375 17.390625 22.1875 11.75 \nQ 26.265625 6.109375 33.40625 6.109375 \nQ 40.53125 6.109375 44.609375 11.75 \nQ 48.6875 17.390625 48.6875 27.296875 \nz\n\" id=\"DejaVuSans-112\"/>\n      <path d=\"M 30.609375 48.390625 \nQ 23.390625 48.390625 19.1875 42.75 \nQ 14.984375 37.109375 14.984375 27.296875 \nQ 14.984375 17.484375 19.15625 11.84375 \nQ 23.34375 6.203125 30.609375 6.203125 \nQ 37.796875 6.203125 41.984375 11.859375 \nQ 46.1875 17.53125 46.1875 27.296875 \nQ 46.1875 37.015625 41.984375 42.703125 \nQ 37.796875 48.390625 30.609375 48.390625 \nz\nM 30.609375 56 \nQ 42.328125 56 49.015625 48.375 \nQ 55.71875 40.765625 55.71875 27.296875 \nQ 55.71875 13.875 49.015625 6.21875 \nQ 42.328125 -1.421875 30.609375 -1.421875 \nQ 18.84375 -1.421875 12.171875 6.21875 \nQ 5.515625 13.875 5.515625 27.296875 \nQ 5.515625 40.765625 12.171875 48.375 \nQ 18.84375 56 30.609375 56 \nz\n\" id=\"DejaVuSans-111\"/>\n      <path d=\"M 48.78125 52.59375 \nL 48.78125 44.1875 \nQ 44.96875 46.296875 41.140625 47.34375 \nQ 37.3125 48.390625 33.40625 48.390625 \nQ 24.65625 48.390625 19.8125 42.84375 \nQ 14.984375 37.3125 14.984375 27.296875 \nQ 14.984375 17.28125 19.8125 11.734375 \nQ 24.65625 6.203125 33.40625 6.203125 \nQ 37.3125 6.203125 41.140625 7.25 \nQ 44.96875 8.296875 48.78125 10.40625 \nL 48.78125 2.09375 \nQ 45.015625 0.34375 40.984375 -0.53125 \nQ 36.96875 -1.421875 32.421875 -1.421875 \nQ 20.0625 -1.421875 12.78125 6.34375 \nQ 5.515625 14.109375 5.515625 27.296875 \nQ 5.515625 40.671875 12.859375 48.328125 \nQ 20.21875 56 33.015625 56 \nQ 37.15625 56 41.109375 55.140625 \nQ 45.0625 54.296875 48.78125 52.59375 \nz\n\" id=\"DejaVuSans-99\"/>\n      <path d=\"M 54.890625 33.015625 \nL 54.890625 0 \nL 45.90625 0 \nL 45.90625 32.71875 \nQ 45.90625 40.484375 42.875 44.328125 \nQ 39.84375 48.1875 33.796875 48.1875 \nQ 26.515625 48.1875 22.3125 43.546875 \nQ 18.109375 38.921875 18.109375 30.90625 \nL 18.109375 0 \nL 9.078125 0 \nL 9.078125 75.984375 \nL 18.109375 75.984375 \nL 18.109375 46.1875 \nQ 21.34375 51.125 25.703125 53.5625 \nQ 30.078125 56 35.796875 56 \nQ 45.21875 56 50.046875 50.171875 \nQ 54.890625 44.34375 54.890625 33.015625 \nz\n\" id=\"DejaVuSans-104\"/>\n      <path d=\"M 44.28125 53.078125 \nL 44.28125 44.578125 \nQ 40.484375 46.53125 36.375 47.5 \nQ 32.28125 48.484375 27.875 48.484375 \nQ 21.1875 48.484375 17.84375 46.4375 \nQ 14.5 44.390625 14.5 40.28125 \nQ 14.5 37.15625 16.890625 35.375 \nQ 19.28125 33.59375 26.515625 31.984375 \nL 29.59375 31.296875 \nQ 39.15625 29.25 43.1875 25.515625 \nQ 47.21875 21.78125 47.21875 15.09375 \nQ 47.21875 7.46875 41.1875 3.015625 \nQ 35.15625 -1.421875 24.609375 -1.421875 \nQ 20.21875 -1.421875 15.453125 -0.5625 \nQ 10.6875 0.296875 5.421875 2 \nL 5.421875 11.28125 \nQ 10.40625 8.6875 15.234375 7.390625 \nQ 20.0625 6.109375 24.8125 6.109375 \nQ 31.15625 6.109375 34.5625 8.28125 \nQ 37.984375 10.453125 37.984375 14.40625 \nQ 37.984375 18.0625 35.515625 20.015625 \nQ 33.0625 21.96875 24.703125 23.78125 \nL 21.578125 24.515625 \nQ 13.234375 26.265625 9.515625 29.90625 \nQ 5.8125 33.546875 5.8125 39.890625 \nQ 5.8125 47.609375 11.28125 51.796875 \nQ 16.75 56 26.8125 56 \nQ 31.78125 56 36.171875 55.265625 \nQ 40.578125 54.546875 44.28125 53.078125 \nz\n\" id=\"DejaVuSans-115\"/>\n     </defs>\n     <g transform=\"translate(190.0875 252.916563)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-69\"/>\n      <use x=\"63.183594\" xlink:href=\"#DejaVuSans-112\"/>\n      <use x=\"126.660156\" xlink:href=\"#DejaVuSans-111\"/>\n      <use x=\"187.841797\" xlink:href=\"#DejaVuSans-99\"/>\n      <use x=\"242.822266\" xlink:href=\"#DejaVuSans-104\"/>\n      <use x=\"306.201172\" xlink:href=\"#DejaVuSans-115\"/>\n     </g>\n    </g>\n   </g>\n   <g id=\"matplotlib.axis_2\">\n    <g id=\"ytick_1\">\n     <g id=\"line2d_6\">\n      <defs>\n       <path d=\"M 0 0 \nL -3.5 0 \n\" id=\"m3c2b475771\" style=\"stroke:#000000;stroke-width:0.8;\"/>\n      </defs>\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"40.603125\" xlink:href=\"#m3c2b475771\" y=\"197.49707\"/>\n      </g>\n     </g>\n     <g id=\"text_7\">\n      <!-- 70 -->\n      <defs>\n       <path d=\"M 8.203125 72.90625 \nL 55.078125 72.90625 \nL 55.078125 68.703125 \nL 28.609375 0 \nL 18.3125 0 \nL 43.21875 64.59375 \nL 8.203125 64.59375 \nz\n\" id=\"DejaVuSans-55\"/>\n      </defs>\n      <g transform=\"translate(20.878125 201.296289)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-55\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_2\">\n     <g id=\"line2d_7\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"40.603125\" xlink:href=\"#m3c2b475771\" y=\"158.791746\"/>\n      </g>\n     </g>\n     <g id=\"text_8\">\n      <!-- 75 -->\n      <defs>\n       <path d=\"M 10.796875 72.90625 \nL 49.515625 72.90625 \nL 49.515625 64.59375 \nL 19.828125 64.59375 \nL 19.828125 46.734375 \nQ 21.96875 47.46875 24.109375 47.828125 \nQ 26.265625 48.1875 28.421875 48.1875 \nQ 40.625 48.1875 47.75 41.5 \nQ 54.890625 34.8125 54.890625 23.390625 \nQ 54.890625 11.625 47.5625 5.09375 \nQ 40.234375 -1.421875 26.90625 -1.421875 \nQ 22.3125 -1.421875 17.546875 -0.640625 \nQ 12.796875 0.140625 7.71875 1.703125 \nL 7.71875 11.625 \nQ 12.109375 9.234375 16.796875 8.0625 \nQ 21.484375 6.890625 26.703125 6.890625 \nQ 35.15625 6.890625 40.078125 11.328125 \nQ 45.015625 15.765625 45.015625 23.390625 \nQ 45.015625 31 40.078125 35.4375 \nQ 35.15625 39.890625 26.703125 39.890625 \nQ 22.75 39.890625 18.8125 39.015625 \nQ 14.890625 38.140625 10.796875 36.28125 \nz\n\" id=\"DejaVuSans-53\"/>\n      </defs>\n      <g transform=\"translate(20.878125 162.590964)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-55\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-53\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_3\">\n     <g id=\"line2d_8\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"40.603125\" xlink:href=\"#m3c2b475771\" y=\"120.086421\"/>\n      </g>\n     </g>\n     <g id=\"text_9\">\n      <!-- 80 -->\n      <g transform=\"translate(20.878125 123.88564)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-56\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_4\">\n     <g id=\"line2d_9\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"40.603125\" xlink:href=\"#m3c2b475771\" y=\"81.381097\"/>\n      </g>\n     </g>\n     <g id=\"text_10\">\n      <!-- 85 -->\n      <g transform=\"translate(20.878125 85.180315)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-56\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-53\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_5\">\n     <g id=\"line2d_10\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"40.603125\" xlink:href=\"#m3c2b475771\" y=\"42.675772\"/>\n      </g>\n     </g>\n     <g id=\"text_11\">\n      <!-- 90 -->\n      <defs>\n       <path d=\"M 10.984375 1.515625 \nL 10.984375 10.5 \nQ 14.703125 8.734375 18.5 7.8125 \nQ 22.3125 6.890625 25.984375 6.890625 \nQ 35.75 6.890625 40.890625 13.453125 \nQ 46.046875 20.015625 46.78125 33.40625 \nQ 43.953125 29.203125 39.59375 26.953125 \nQ 35.25 24.703125 29.984375 24.703125 \nQ 19.046875 24.703125 12.671875 31.3125 \nQ 6.296875 37.9375 6.296875 49.421875 \nQ 6.296875 60.640625 12.9375 67.421875 \nQ 19.578125 74.21875 30.609375 74.21875 \nQ 43.265625 74.21875 49.921875 64.515625 \nQ 56.59375 54.828125 56.59375 36.375 \nQ 56.59375 19.140625 48.40625 8.859375 \nQ 40.234375 -1.421875 26.421875 -1.421875 \nQ 22.703125 -1.421875 18.890625 -0.6875 \nQ 15.09375 0.046875 10.984375 1.515625 \nz\nM 30.609375 32.421875 \nQ 37.25 32.421875 41.125 36.953125 \nQ 45.015625 41.5 45.015625 49.421875 \nQ 45.015625 57.28125 41.125 61.84375 \nQ 37.25 66.40625 30.609375 66.40625 \nQ 23.96875 66.40625 20.09375 61.84375 \nQ 16.21875 57.28125 16.21875 49.421875 \nQ 16.21875 41.5 20.09375 36.953125 \nQ 23.96875 32.421875 30.609375 32.421875 \nz\n\" id=\"DejaVuSans-57\"/>\n      </defs>\n      <g transform=\"translate(20.878125 46.474991)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-57\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"text_12\">\n     <!-- Accuracy -->\n     <defs>\n      <path d=\"M 34.1875 63.1875 \nL 20.796875 26.90625 \nL 47.609375 26.90625 \nz\nM 28.609375 72.90625 \nL 39.796875 72.90625 \nL 67.578125 0 \nL 57.328125 0 \nL 50.6875 18.703125 \nL 17.828125 18.703125 \nL 11.1875 0 \nL 0.78125 0 \nz\n\" id=\"DejaVuSans-65\"/>\n      <path d=\"M 8.5 21.578125 \nL 8.5 54.6875 \nL 17.484375 54.6875 \nL 17.484375 21.921875 \nQ 17.484375 14.15625 20.5 10.265625 \nQ 23.53125 6.390625 29.59375 6.390625 \nQ 36.859375 6.390625 41.078125 11.03125 \nQ 45.3125 15.671875 45.3125 23.6875 \nL 45.3125 54.6875 \nL 54.296875 54.6875 \nL 54.296875 0 \nL 45.3125 0 \nL 45.3125 8.40625 \nQ 42.046875 3.421875 37.71875 1 \nQ 33.40625 -1.421875 27.6875 -1.421875 \nQ 18.265625 -1.421875 13.375 4.4375 \nQ 8.5 10.296875 8.5 21.578125 \nz\nM 31.109375 56 \nz\n\" id=\"DejaVuSans-117\"/>\n      <path d=\"M 41.109375 46.296875 \nQ 39.59375 47.171875 37.8125 47.578125 \nQ 36.03125 48 33.890625 48 \nQ 26.265625 48 22.1875 43.046875 \nQ 18.109375 38.09375 18.109375 28.8125 \nL 18.109375 0 \nL 9.078125 0 \nL 9.078125 54.6875 \nL 18.109375 54.6875 \nL 18.109375 46.1875 \nQ 20.953125 51.171875 25.484375 53.578125 \nQ 30.03125 56 36.53125 56 \nQ 37.453125 56 38.578125 55.875 \nQ 39.703125 55.765625 41.0625 55.515625 \nz\n\" id=\"DejaVuSans-114\"/>\n      <path d=\"M 34.28125 27.484375 \nQ 23.390625 27.484375 19.1875 25 \nQ 14.984375 22.515625 14.984375 16.5 \nQ 14.984375 11.71875 18.140625 8.90625 \nQ 21.296875 6.109375 26.703125 6.109375 \nQ 34.1875 6.109375 38.703125 11.40625 \nQ 43.21875 16.703125 43.21875 25.484375 \nL 43.21875 27.484375 \nz\nM 52.203125 31.203125 \nL 52.203125 0 \nL 43.21875 0 \nL 43.21875 8.296875 \nQ 40.140625 3.328125 35.546875 0.953125 \nQ 30.953125 -1.421875 24.3125 -1.421875 \nQ 15.921875 -1.421875 10.953125 3.296875 \nQ 6 8.015625 6 15.921875 \nQ 6 25.140625 12.171875 29.828125 \nQ 18.359375 34.515625 30.609375 34.515625 \nL 43.21875 34.515625 \nL 43.21875 35.40625 \nQ 43.21875 41.609375 39.140625 45 \nQ 35.0625 48.390625 27.6875 48.390625 \nQ 23 48.390625 18.546875 47.265625 \nQ 14.109375 46.140625 10.015625 43.890625 \nL 10.015625 52.203125 \nQ 14.9375 54.109375 19.578125 55.046875 \nQ 24.21875 56 28.609375 56 \nQ 40.484375 56 46.34375 49.84375 \nQ 52.203125 43.703125 52.203125 31.203125 \nz\n\" id=\"DejaVuSans-97\"/>\n      <path d=\"M 32.171875 -5.078125 \nQ 28.375 -14.84375 24.75 -17.8125 \nQ 21.140625 -20.796875 15.09375 -20.796875 \nL 7.90625 -20.796875 \nL 7.90625 -13.28125 \nL 13.1875 -13.28125 \nQ 16.890625 -13.28125 18.9375 -11.515625 \nQ 21 -9.765625 23.484375 -3.21875 \nL 25.09375 0.875 \nL 2.984375 54.6875 \nL 12.5 54.6875 \nL 29.59375 11.921875 \nL 46.6875 54.6875 \nL 56.203125 54.6875 \nz\n\" id=\"DejaVuSans-121\"/>\n     </defs>\n     <g transform=\"translate(14.798438 138.834844)rotate(-90)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-65\"/>\n      <use x=\"68.392578\" xlink:href=\"#DejaVuSans-99\"/>\n      <use x=\"123.373047\" xlink:href=\"#DejaVuSans-99\"/>\n      <use x=\"178.353516\" xlink:href=\"#DejaVuSans-117\"/>\n      <use x=\"241.732422\" xlink:href=\"#DejaVuSans-114\"/>\n      <use x=\"282.845703\" xlink:href=\"#DejaVuSans-97\"/>\n      <use x=\"344.125\" xlink:href=\"#DejaVuSans-99\"/>\n      <use x=\"399.105469\" xlink:href=\"#DejaVuSans-121\"/>\n     </g>\n    </g>\n   </g>\n   <g id=\"line2d_11\">\n    <path clip-path=\"url(#pa4072b92cc)\" d=\"M 55.821307 214.756364 \nL 59.279985 180.727145 \nL 62.738662 160.66116 \nL 66.19734 147.511886 \nL 69.656018 137.83272 \nL 73.114695 129.947137 \nL 76.573373 123.519794 \nL 80.032051 118.115247 \nL 83.490728 113.313757 \nL 86.949406 109.084083 \nL 90.408084 105.32665 \nL 93.866761 101.915779 \nL 97.325439 98.791465 \nL 100.784117 95.845275 \nL 104.242794 93.273818 \nL 107.701472 90.786698 \nL 111.16015 88.4578 \nL 114.618827 86.254403 \nL 118.077505 84.201372 \nL 121.536183 82.26144 \nL 124.994861 80.308987 \nL 128.453538 78.558696 \nL 131.912216 76.831556 \nL 135.370894 75.259093 \nL 138.829571 73.683441 \nL 142.288249 72.187224 \nL 145.746927 70.794716 \nL 149.205604 69.40581 \nL 152.664282 68.033559 \nL 156.12296 66.821065 \nL 159.581637 65.653988 \nL 163.040315 64.018036 \nL 166.498993 62.297392 \nL 169.95767 60.591041 \nL 173.416348 58.935127 \nL 176.875026 57.303427 \nL 180.333704 55.775968 \nL 183.792381 54.296583 \nL 187.251059 52.871356 \nL 190.709737 51.497924 \nL 194.168414 50.164475 \nL 197.627092 48.877625 \nL 201.08577 47.6281 \nL 204.544447 46.449683 \nL 208.003125 45.313671 \nL 211.461803 44.214985 \nL 214.92048 43.159234 \nL 218.379158 42.125632 \nL 221.837836 41.143765 \nL 225.296513 40.174774 \nL 228.755191 39.254447 \nL 232.213869 38.364301 \nL 235.672546 37.502857 \nL 239.131224 36.650685 \nL 242.589902 35.852494 \nL 246.04858 35.064816 \nL 249.507257 34.313105 \nL 252.965935 33.572615 \nL 256.424613 32.853091 \nL 259.88329 32.162979 \nL 263.341968 31.495487 \nL 266.800646 30.819077 \nL 270.259323 30.154125 \nL 273.718001 29.516458 \nL 277.176679 28.8795 \nL 280.635356 28.262681 \nL 284.094034 27.657851 \nL 287.552712 27.069499 \nL 291.011389 26.487703 \nL 294.470067 25.925219 \nL 297.928745 25.374487 \nL 301.387423 24.841947 \nL 304.8461 24.329132 \nL 308.304778 23.822518 \nL 311.763456 23.332973 \nL 315.222133 22.85837 \nL 318.680811 22.387723 \nL 322.139489 21.923042 \nL 325.598166 21.473362 \nL 329.056844 21.037856 \nL 332.515522 20.607312 \nL 335.974199 20.187929 \nL 339.432877 19.780831 \nL 342.891555 19.378812 \nL 346.350232 18.988428 \nL 349.80891 18.610446 \nL 353.267588 18.236244 \nL 356.726265 17.86848 \nL 360.184943 17.511701 \n\" style=\"fill:none;stroke:#ff0000;stroke-linecap:square;stroke-width:1.5;\"/>\n   </g>\n   <g id=\"line2d_12\">\n    <path clip-path=\"url(#pa4072b92cc)\" d=\"M 55.821307 184.272377 \nL 59.279985 163.83786 \nL 62.738662 149.389865 \nL 66.19734 139.303542 \nL 69.656018 130.989362 \nL 73.114695 124.296251 \nL 76.573373 118.731947 \nL 80.032051 113.698531 \nL 83.490728 109.528566 \nL 86.949406 105.662817 \nL 90.408084 102.038209 \nL 93.866761 98.970297 \nL 97.325439 96.066158 \nL 100.784117 93.325731 \nL 104.242794 90.855562 \nL 107.701472 88.548633 \nL 111.16015 86.315116 \nL 114.618827 84.266751 \nL 118.077505 82.277268 \nL 121.536183 80.353046 \nL 124.994861 78.540682 \nL 128.453538 76.81307 \nL 131.912216 75.239012 \nL 135.370894 73.66088 \nL 138.829571 72.145941 \nL 142.288249 70.772273 \nL 145.746927 69.387915 \nL 149.205604 68.012357 \nL 152.664282 66.714285 \nL 156.12296 65.571895 \nL 159.581637 64.014256 \nL 163.040315 62.29249 \nL 166.498993 60.589801 \nL 169.95767 58.929634 \nL 173.416348 57.300179 \nL 176.875026 55.755061 \nL 180.333704 54.272132 \nL 183.792381 52.844011 \nL 187.251059 51.466799 \nL 190.709737 50.13152 \nL 194.168414 48.837228 \nL 197.627092 47.583156 \nL 201.08577 46.405506 \nL 204.544447 45.257919 \nL 208.003125 44.162008 \nL 211.461803 43.098935 \nL 214.92048 42.066572 \nL 218.379158 41.089371 \nL 221.837836 40.117722 \nL 225.296513 39.185584 \nL 228.755191 38.29774 \nL 232.213869 37.432103 \nL 235.672546 36.584597 \nL 239.131224 35.782036 \nL 242.589902 34.992822 \nL 246.04858 34.24182 \nL 249.507257 33.500444 \nL 252.965935 32.778794 \nL 256.424613 32.083899 \nL 259.88329 31.41245 \nL 263.341968 30.741414 \nL 266.800646 30.081068 \nL 270.259323 29.439444 \nL 273.718001 28.801836 \nL 277.176679 28.186022 \nL 280.635356 27.584263 \nL 284.094034 26.996915 \nL 287.552712 26.416004 \nL 291.011389 25.849209 \nL 294.470067 25.30208 \nL 297.928745 24.767177 \nL 301.387423 24.254008 \nL 304.8461 23.748162 \nL 308.304778 23.257849 \nL 311.763456 22.781297 \nL 315.222133 22.311832 \nL 318.680811 21.846442 \nL 322.139489 21.397943 \nL 325.598166 20.960193 \nL 329.056844 20.533723 \nL 332.515522 20.110974 \nL 335.974199 19.702872 \nL 339.432877 19.3067 \nL 342.891555 18.915017 \nL 346.350232 18.5342 \nL 349.80891 18.161002 \nL 353.267588 17.794065 \nL 356.726265 17.437226 \nL 360.184943 17.083636 \n\" style=\"fill:none;stroke:#0000ff;stroke-linecap:square;stroke-width:1.5;\"/>\n   </g>\n   <g id=\"patch_3\">\n    <path d=\"M 40.603125 224.64 \nL 40.603125 7.2 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_4\">\n    <path d=\"M 375.403125 224.64 \nL 375.403125 7.2 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_5\">\n    <path d=\"M 40.603125 224.64 \nL 375.403125 224.64 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_6\">\n    <path d=\"M 40.603125 7.2 \nL 375.403125 7.2 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"legend_1\">\n    <g id=\"patch_7\">\n     <path d=\"M 47.603125 44.55625 \nL 105.04375 44.55625 \nQ 107.04375 44.55625 107.04375 42.55625 \nL 107.04375 14.2 \nQ 107.04375 12.2 105.04375 12.2 \nL 47.603125 12.2 \nQ 45.603125 12.2 45.603125 14.2 \nL 45.603125 42.55625 \nQ 45.603125 44.55625 47.603125 44.55625 \nz\n\" style=\"fill:#ffffff;opacity:0.8;stroke:#cccccc;stroke-linejoin:miter;\"/>\n    </g>\n    <g id=\"line2d_13\">\n     <path d=\"M 49.603125 20.298438 \nL 69.603125 20.298438 \n\" style=\"fill:none;stroke:#ff0000;stroke-linecap:square;stroke-width:1.5;\"/>\n    </g>\n    <g id=\"line2d_14\"/>\n    <g id=\"text_13\">\n     <!-- Train -->\n     <defs>\n      <path d=\"M -0.296875 72.90625 \nL 61.375 72.90625 \nL 61.375 64.59375 \nL 35.5 64.59375 \nL 35.5 0 \nL 25.59375 0 \nL 25.59375 64.59375 \nL -0.296875 64.59375 \nz\n\" id=\"DejaVuSans-84\"/>\n      <path d=\"M 9.421875 54.6875 \nL 18.40625 54.6875 \nL 18.40625 0 \nL 9.421875 0 \nz\nM 9.421875 75.984375 \nL 18.40625 75.984375 \nL 18.40625 64.59375 \nL 9.421875 64.59375 \nz\n\" id=\"DejaVuSans-105\"/>\n      <path d=\"M 54.890625 33.015625 \nL 54.890625 0 \nL 45.90625 0 \nL 45.90625 32.71875 \nQ 45.90625 40.484375 42.875 44.328125 \nQ 39.84375 48.1875 33.796875 48.1875 \nQ 26.515625 48.1875 22.3125 43.546875 \nQ 18.109375 38.921875 18.109375 30.90625 \nL 18.109375 0 \nL 9.078125 0 \nL 9.078125 54.6875 \nL 18.109375 54.6875 \nL 18.109375 46.1875 \nQ 21.34375 51.125 25.703125 53.5625 \nQ 30.078125 56 35.796875 56 \nQ 45.21875 56 50.046875 50.171875 \nQ 54.890625 44.34375 54.890625 33.015625 \nz\n\" id=\"DejaVuSans-110\"/>\n     </defs>\n     <g transform=\"translate(77.603125 23.798438)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-84\"/>\n      <use x=\"60.865234\" xlink:href=\"#DejaVuSans-114\"/>\n      <use x=\"101.978516\" xlink:href=\"#DejaVuSans-97\"/>\n      <use x=\"163.257812\" xlink:href=\"#DejaVuSans-105\"/>\n      <use x=\"191.041016\" xlink:href=\"#DejaVuSans-110\"/>\n     </g>\n    </g>\n    <g id=\"line2d_15\">\n     <path d=\"M 49.603125 34.976563 \nL 69.603125 34.976563 \n\" style=\"fill:none;stroke:#0000ff;stroke-linecap:square;stroke-width:1.5;\"/>\n    </g>\n    <g id=\"line2d_16\"/>\n    <g id=\"text_14\">\n     <!-- Val -->\n     <defs>\n      <path d=\"M 28.609375 0 \nL 0.78125 72.90625 \nL 11.078125 72.90625 \nL 34.1875 11.53125 \nL 57.328125 72.90625 \nL 67.578125 72.90625 \nL 39.796875 0 \nz\n\" id=\"DejaVuSans-86\"/>\n      <path d=\"M 9.421875 75.984375 \nL 18.40625 75.984375 \nL 18.40625 0 \nL 9.421875 0 \nz\n\" id=\"DejaVuSans-108\"/>\n     </defs>\n     <g transform=\"translate(77.603125 38.476563)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-86\"/>\n      <use x=\"68.298828\" xlink:href=\"#DejaVuSans-97\"/>\n      <use x=\"129.578125\" xlink:href=\"#DejaVuSans-108\"/>\n     </g>\n    </g>\n   </g>\n  </g>\n </g>\n <defs>\n  <clipPath id=\"pa4072b92cc\">\n   <rect height=\"217.44\" width=\"334.8\" x=\"40.603125\" y=\"7.2\"/>\n  </clipPath>\n </defs>\n</svg>\n"
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MffrvIMVb1sB",
        "colab_type": "code",
        "outputId": "60aa9d1c-28c0-4666-bf06-d4619faa07b8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 371
        }
      },
      "source": [
        "plt.xlabel(\"Epochs\")\n",
        "plt.ylabel(\"Losses\")\n",
        "plt.plot(ep_sgd_losses, 'r', label='Train')\n",
        "plt.plot(ep_sgd_lossesv, 'b', label='Val')\n",
        "plt.legend()\n",
        "plt.savefig('sgd_resnet18_9330_loss.jpg')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/svg+xml": "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n  \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<!-- Created with matplotlib (https://matplotlib.org/) -->\n<svg height=\"262.19625pt\" version=\"1.1\" viewBox=\"0 0 385.78125 262.19625\" width=\"385.78125pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n <defs>\n  <style type=\"text/css\">\n*{stroke-linecap:butt;stroke-linejoin:round;}\n  </style>\n </defs>\n <g id=\"figure_1\">\n  <g id=\"patch_1\">\n   <path d=\"M 0 262.19625 \nL 385.78125 262.19625 \nL 385.78125 0 \nL 0 0 \nz\n\" style=\"fill:none;\"/>\n  </g>\n  <g id=\"axes_1\">\n   <g id=\"patch_2\">\n    <path d=\"M 43.78125 224.64 \nL 378.58125 224.64 \nL 378.58125 7.2 \nL 43.78125 7.2 \nz\n\" style=\"fill:#ffffff;\"/>\n   </g>\n   <g id=\"matplotlib.axis_1\">\n    <g id=\"xtick_1\">\n     <g id=\"line2d_1\">\n      <defs>\n       <path d=\"M 0 0 \nL 0 3.5 \n\" id=\"ma5d1099511\" style=\"stroke:#000000;stroke-width:0.8;\"/>\n      </defs>\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"58.999432\" xlink:href=\"#ma5d1099511\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_1\">\n      <!-- 0 -->\n      <defs>\n       <path d=\"M 31.78125 66.40625 \nQ 24.171875 66.40625 20.328125 58.90625 \nQ 16.5 51.421875 16.5 36.375 \nQ 16.5 21.390625 20.328125 13.890625 \nQ 24.171875 6.390625 31.78125 6.390625 \nQ 39.453125 6.390625 43.28125 13.890625 \nQ 47.125 21.390625 47.125 36.375 \nQ 47.125 51.421875 43.28125 58.90625 \nQ 39.453125 66.40625 31.78125 66.40625 \nz\nM 31.78125 74.21875 \nQ 44.046875 74.21875 50.515625 64.515625 \nQ 56.984375 54.828125 56.984375 36.375 \nQ 56.984375 17.96875 50.515625 8.265625 \nQ 44.046875 -1.421875 31.78125 -1.421875 \nQ 19.53125 -1.421875 13.0625 8.265625 \nQ 6.59375 17.96875 6.59375 36.375 \nQ 6.59375 54.828125 13.0625 64.515625 \nQ 19.53125 74.21875 31.78125 74.21875 \nz\n\" id=\"DejaVuSans-48\"/>\n      </defs>\n      <g transform=\"translate(55.818182 239.238437)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_2\">\n     <g id=\"line2d_2\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"128.172986\" xlink:href=\"#ma5d1099511\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_2\">\n      <!-- 20 -->\n      <defs>\n       <path d=\"M 19.1875 8.296875 \nL 53.609375 8.296875 \nL 53.609375 0 \nL 7.328125 0 \nL 7.328125 8.296875 \nQ 12.9375 14.109375 22.625 23.890625 \nQ 32.328125 33.6875 34.8125 36.53125 \nQ 39.546875 41.84375 41.421875 45.53125 \nQ 43.3125 49.21875 43.3125 52.78125 \nQ 43.3125 58.59375 39.234375 62.25 \nQ 35.15625 65.921875 28.609375 65.921875 \nQ 23.96875 65.921875 18.8125 64.3125 \nQ 13.671875 62.703125 7.8125 59.421875 \nL 7.8125 69.390625 \nQ 13.765625 71.78125 18.9375 73 \nQ 24.125 74.21875 28.421875 74.21875 \nQ 39.75 74.21875 46.484375 68.546875 \nQ 53.21875 62.890625 53.21875 53.421875 \nQ 53.21875 48.921875 51.53125 44.890625 \nQ 49.859375 40.875 45.40625 35.40625 \nQ 44.1875 33.984375 37.640625 27.21875 \nQ 31.109375 20.453125 19.1875 8.296875 \nz\n\" id=\"DejaVuSans-50\"/>\n      </defs>\n      <g transform=\"translate(121.810486 239.238437)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-50\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_3\">\n     <g id=\"line2d_3\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"197.346539\" xlink:href=\"#ma5d1099511\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_3\">\n      <!-- 40 -->\n      <defs>\n       <path d=\"M 37.796875 64.3125 \nL 12.890625 25.390625 \nL 37.796875 25.390625 \nz\nM 35.203125 72.90625 \nL 47.609375 72.90625 \nL 47.609375 25.390625 \nL 58.015625 25.390625 \nL 58.015625 17.1875 \nL 47.609375 17.1875 \nL 47.609375 0 \nL 37.796875 0 \nL 37.796875 17.1875 \nL 4.890625 17.1875 \nL 4.890625 26.703125 \nz\n\" id=\"DejaVuSans-52\"/>\n      </defs>\n      <g transform=\"translate(190.984039 239.238437)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-52\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_4\">\n     <g id=\"line2d_4\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"266.520093\" xlink:href=\"#ma5d1099511\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_4\">\n      <!-- 60 -->\n      <defs>\n       <path d=\"M 33.015625 40.375 \nQ 26.375 40.375 22.484375 35.828125 \nQ 18.609375 31.296875 18.609375 23.390625 \nQ 18.609375 15.53125 22.484375 10.953125 \nQ 26.375 6.390625 33.015625 6.390625 \nQ 39.65625 6.390625 43.53125 10.953125 \nQ 47.40625 15.53125 47.40625 23.390625 \nQ 47.40625 31.296875 43.53125 35.828125 \nQ 39.65625 40.375 33.015625 40.375 \nz\nM 52.59375 71.296875 \nL 52.59375 62.3125 \nQ 48.875 64.0625 45.09375 64.984375 \nQ 41.3125 65.921875 37.59375 65.921875 \nQ 27.828125 65.921875 22.671875 59.328125 \nQ 17.53125 52.734375 16.796875 39.40625 \nQ 19.671875 43.65625 24.015625 45.921875 \nQ 28.375 48.1875 33.59375 48.1875 \nQ 44.578125 48.1875 50.953125 41.515625 \nQ 57.328125 34.859375 57.328125 23.390625 \nQ 57.328125 12.15625 50.6875 5.359375 \nQ 44.046875 -1.421875 33.015625 -1.421875 \nQ 20.359375 -1.421875 13.671875 8.265625 \nQ 6.984375 17.96875 6.984375 36.375 \nQ 6.984375 53.65625 15.1875 63.9375 \nQ 23.390625 74.21875 37.203125 74.21875 \nQ 40.921875 74.21875 44.703125 73.484375 \nQ 48.484375 72.75 52.59375 71.296875 \nz\n\" id=\"DejaVuSans-54\"/>\n      </defs>\n      <g transform=\"translate(260.157593 239.238437)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-54\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_5\">\n     <g id=\"line2d_5\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"335.693647\" xlink:href=\"#ma5d1099511\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_5\">\n      <!-- 80 -->\n      <defs>\n       <path d=\"M 31.78125 34.625 \nQ 24.75 34.625 20.71875 30.859375 \nQ 16.703125 27.09375 16.703125 20.515625 \nQ 16.703125 13.921875 20.71875 10.15625 \nQ 24.75 6.390625 31.78125 6.390625 \nQ 38.8125 6.390625 42.859375 10.171875 \nQ 46.921875 13.96875 46.921875 20.515625 \nQ 46.921875 27.09375 42.890625 30.859375 \nQ 38.875 34.625 31.78125 34.625 \nz\nM 21.921875 38.8125 \nQ 15.578125 40.375 12.03125 44.71875 \nQ 8.5 49.078125 8.5 55.328125 \nQ 8.5 64.0625 14.71875 69.140625 \nQ 20.953125 74.21875 31.78125 74.21875 \nQ 42.671875 74.21875 48.875 69.140625 \nQ 55.078125 64.0625 55.078125 55.328125 \nQ 55.078125 49.078125 51.53125 44.71875 \nQ 48 40.375 41.703125 38.8125 \nQ 48.828125 37.15625 52.796875 32.3125 \nQ 56.78125 27.484375 56.78125 20.515625 \nQ 56.78125 9.90625 50.3125 4.234375 \nQ 43.84375 -1.421875 31.78125 -1.421875 \nQ 19.734375 -1.421875 13.25 4.234375 \nQ 6.78125 9.90625 6.78125 20.515625 \nQ 6.78125 27.484375 10.78125 32.3125 \nQ 14.796875 37.15625 21.921875 38.8125 \nz\nM 18.3125 54.390625 \nQ 18.3125 48.734375 21.84375 45.5625 \nQ 25.390625 42.390625 31.78125 42.390625 \nQ 38.140625 42.390625 41.71875 45.5625 \nQ 45.3125 48.734375 45.3125 54.390625 \nQ 45.3125 60.0625 41.71875 63.234375 \nQ 38.140625 66.40625 31.78125 66.40625 \nQ 25.390625 66.40625 21.84375 63.234375 \nQ 18.3125 60.0625 18.3125 54.390625 \nz\n\" id=\"DejaVuSans-56\"/>\n      </defs>\n      <g transform=\"translate(329.331147 239.238437)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-56\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"text_6\">\n     <!-- Epochs -->\n     <defs>\n      <path d=\"M 9.8125 72.90625 \nL 55.90625 72.90625 \nL 55.90625 64.59375 \nL 19.671875 64.59375 \nL 19.671875 43.015625 \nL 54.390625 43.015625 \nL 54.390625 34.71875 \nL 19.671875 34.71875 \nL 19.671875 8.296875 \nL 56.78125 8.296875 \nL 56.78125 0 \nL 9.8125 0 \nz\n\" id=\"DejaVuSans-69\"/>\n      <path d=\"M 18.109375 8.203125 \nL 18.109375 -20.796875 \nL 9.078125 -20.796875 \nL 9.078125 54.6875 \nL 18.109375 54.6875 \nL 18.109375 46.390625 \nQ 20.953125 51.265625 25.265625 53.625 \nQ 29.59375 56 35.59375 56 \nQ 45.5625 56 51.78125 48.09375 \nQ 58.015625 40.1875 58.015625 27.296875 \nQ 58.015625 14.40625 51.78125 6.484375 \nQ 45.5625 -1.421875 35.59375 -1.421875 \nQ 29.59375 -1.421875 25.265625 0.953125 \nQ 20.953125 3.328125 18.109375 8.203125 \nz\nM 48.6875 27.296875 \nQ 48.6875 37.203125 44.609375 42.84375 \nQ 40.53125 48.484375 33.40625 48.484375 \nQ 26.265625 48.484375 22.1875 42.84375 \nQ 18.109375 37.203125 18.109375 27.296875 \nQ 18.109375 17.390625 22.1875 11.75 \nQ 26.265625 6.109375 33.40625 6.109375 \nQ 40.53125 6.109375 44.609375 11.75 \nQ 48.6875 17.390625 48.6875 27.296875 \nz\n\" id=\"DejaVuSans-112\"/>\n      <path d=\"M 30.609375 48.390625 \nQ 23.390625 48.390625 19.1875 42.75 \nQ 14.984375 37.109375 14.984375 27.296875 \nQ 14.984375 17.484375 19.15625 11.84375 \nQ 23.34375 6.203125 30.609375 6.203125 \nQ 37.796875 6.203125 41.984375 11.859375 \nQ 46.1875 17.53125 46.1875 27.296875 \nQ 46.1875 37.015625 41.984375 42.703125 \nQ 37.796875 48.390625 30.609375 48.390625 \nz\nM 30.609375 56 \nQ 42.328125 56 49.015625 48.375 \nQ 55.71875 40.765625 55.71875 27.296875 \nQ 55.71875 13.875 49.015625 6.21875 \nQ 42.328125 -1.421875 30.609375 -1.421875 \nQ 18.84375 -1.421875 12.171875 6.21875 \nQ 5.515625 13.875 5.515625 27.296875 \nQ 5.515625 40.765625 12.171875 48.375 \nQ 18.84375 56 30.609375 56 \nz\n\" id=\"DejaVuSans-111\"/>\n      <path d=\"M 48.78125 52.59375 \nL 48.78125 44.1875 \nQ 44.96875 46.296875 41.140625 47.34375 \nQ 37.3125 48.390625 33.40625 48.390625 \nQ 24.65625 48.390625 19.8125 42.84375 \nQ 14.984375 37.3125 14.984375 27.296875 \nQ 14.984375 17.28125 19.8125 11.734375 \nQ 24.65625 6.203125 33.40625 6.203125 \nQ 37.3125 6.203125 41.140625 7.25 \nQ 44.96875 8.296875 48.78125 10.40625 \nL 48.78125 2.09375 \nQ 45.015625 0.34375 40.984375 -0.53125 \nQ 36.96875 -1.421875 32.421875 -1.421875 \nQ 20.0625 -1.421875 12.78125 6.34375 \nQ 5.515625 14.109375 5.515625 27.296875 \nQ 5.515625 40.671875 12.859375 48.328125 \nQ 20.21875 56 33.015625 56 \nQ 37.15625 56 41.109375 55.140625 \nQ 45.0625 54.296875 48.78125 52.59375 \nz\n\" id=\"DejaVuSans-99\"/>\n      <path d=\"M 54.890625 33.015625 \nL 54.890625 0 \nL 45.90625 0 \nL 45.90625 32.71875 \nQ 45.90625 40.484375 42.875 44.328125 \nQ 39.84375 48.1875 33.796875 48.1875 \nQ 26.515625 48.1875 22.3125 43.546875 \nQ 18.109375 38.921875 18.109375 30.90625 \nL 18.109375 0 \nL 9.078125 0 \nL 9.078125 75.984375 \nL 18.109375 75.984375 \nL 18.109375 46.1875 \nQ 21.34375 51.125 25.703125 53.5625 \nQ 30.078125 56 35.796875 56 \nQ 45.21875 56 50.046875 50.171875 \nQ 54.890625 44.34375 54.890625 33.015625 \nz\n\" id=\"DejaVuSans-104\"/>\n      <path d=\"M 44.28125 53.078125 \nL 44.28125 44.578125 \nQ 40.484375 46.53125 36.375 47.5 \nQ 32.28125 48.484375 27.875 48.484375 \nQ 21.1875 48.484375 17.84375 46.4375 \nQ 14.5 44.390625 14.5 40.28125 \nQ 14.5 37.15625 16.890625 35.375 \nQ 19.28125 33.59375 26.515625 31.984375 \nL 29.59375 31.296875 \nQ 39.15625 29.25 43.1875 25.515625 \nQ 47.21875 21.78125 47.21875 15.09375 \nQ 47.21875 7.46875 41.1875 3.015625 \nQ 35.15625 -1.421875 24.609375 -1.421875 \nQ 20.21875 -1.421875 15.453125 -0.5625 \nQ 10.6875 0.296875 5.421875 2 \nL 5.421875 11.28125 \nQ 10.40625 8.6875 15.234375 7.390625 \nQ 20.0625 6.109375 24.8125 6.109375 \nQ 31.15625 6.109375 34.5625 8.28125 \nQ 37.984375 10.453125 37.984375 14.40625 \nQ 37.984375 18.0625 35.515625 20.015625 \nQ 33.0625 21.96875 24.703125 23.78125 \nL 21.578125 24.515625 \nQ 13.234375 26.265625 9.515625 29.90625 \nQ 5.8125 33.546875 5.8125 39.890625 \nQ 5.8125 47.609375 11.28125 51.796875 \nQ 16.75 56 26.8125 56 \nQ 31.78125 56 36.171875 55.265625 \nQ 40.578125 54.546875 44.28125 53.078125 \nz\n\" id=\"DejaVuSans-115\"/>\n     </defs>\n     <g transform=\"translate(193.265625 252.916562)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-69\"/>\n      <use x=\"63.183594\" xlink:href=\"#DejaVuSans-112\"/>\n      <use x=\"126.660156\" xlink:href=\"#DejaVuSans-111\"/>\n      <use x=\"187.841797\" xlink:href=\"#DejaVuSans-99\"/>\n      <use x=\"242.822266\" xlink:href=\"#DejaVuSans-104\"/>\n      <use x=\"306.201172\" xlink:href=\"#DejaVuSans-115\"/>\n     </g>\n    </g>\n   </g>\n   <g id=\"matplotlib.axis_2\">\n    <g id=\"ytick_1\">\n     <g id=\"line2d_6\">\n      <defs>\n       <path d=\"M 0 0 \nL -3.5 0 \n\" id=\"m90bf01fccf\" style=\"stroke:#000000;stroke-width:0.8;\"/>\n      </defs>\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#m90bf01fccf\" y=\"215.799688\"/>\n      </g>\n     </g>\n     <g id=\"text_7\">\n      <!-- 0.0 -->\n      <defs>\n       <path d=\"M 10.6875 12.40625 \nL 21 12.40625 \nL 21 0 \nL 10.6875 0 \nz\n\" id=\"DejaVuSans-46\"/>\n      </defs>\n      <g transform=\"translate(20.878125 219.598907)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_2\">\n     <g id=\"line2d_7\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#m90bf01fccf\" y=\"185.059623\"/>\n      </g>\n     </g>\n     <g id=\"text_8\">\n      <!-- 0.1 -->\n      <defs>\n       <path d=\"M 12.40625 8.296875 \nL 28.515625 8.296875 \nL 28.515625 63.921875 \nL 10.984375 60.40625 \nL 10.984375 69.390625 \nL 28.421875 72.90625 \nL 38.28125 72.90625 \nL 38.28125 8.296875 \nL 54.390625 8.296875 \nL 54.390625 0 \nL 12.40625 0 \nz\n\" id=\"DejaVuSans-49\"/>\n      </defs>\n      <g transform=\"translate(20.878125 188.858842)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-49\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_3\">\n     <g id=\"line2d_8\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#m90bf01fccf\" y=\"154.319557\"/>\n      </g>\n     </g>\n     <g id=\"text_9\">\n      <!-- 0.2 -->\n      <g transform=\"translate(20.878125 158.118776)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-50\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_4\">\n     <g id=\"line2d_9\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#m90bf01fccf\" y=\"123.579492\"/>\n      </g>\n     </g>\n     <g id=\"text_10\">\n      <!-- 0.3 -->\n      <defs>\n       <path d=\"M 40.578125 39.3125 \nQ 47.65625 37.796875 51.625 33 \nQ 55.609375 28.21875 55.609375 21.1875 \nQ 55.609375 10.40625 48.1875 4.484375 \nQ 40.765625 -1.421875 27.09375 -1.421875 \nQ 22.515625 -1.421875 17.65625 -0.515625 \nQ 12.796875 0.390625 7.625 2.203125 \nL 7.625 11.71875 \nQ 11.71875 9.328125 16.59375 8.109375 \nQ 21.484375 6.890625 26.8125 6.890625 \nQ 36.078125 6.890625 40.9375 10.546875 \nQ 45.796875 14.203125 45.796875 21.1875 \nQ 45.796875 27.640625 41.28125 31.265625 \nQ 36.765625 34.90625 28.71875 34.90625 \nL 20.21875 34.90625 \nL 20.21875 43.015625 \nL 29.109375 43.015625 \nQ 36.375 43.015625 40.234375 45.921875 \nQ 44.09375 48.828125 44.09375 54.296875 \nQ 44.09375 59.90625 40.109375 62.90625 \nQ 36.140625 65.921875 28.71875 65.921875 \nQ 24.65625 65.921875 20.015625 65.03125 \nQ 15.375 64.15625 9.8125 62.3125 \nL 9.8125 71.09375 \nQ 15.4375 72.65625 20.34375 73.4375 \nQ 25.25 74.21875 29.59375 74.21875 \nQ 40.828125 74.21875 47.359375 69.109375 \nQ 53.90625 64.015625 53.90625 55.328125 \nQ 53.90625 49.265625 50.4375 45.09375 \nQ 46.96875 40.921875 40.578125 39.3125 \nz\n\" id=\"DejaVuSans-51\"/>\n      </defs>\n      <g transform=\"translate(20.878125 127.378711)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-51\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_5\">\n     <g id=\"line2d_10\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#m90bf01fccf\" y=\"92.839426\"/>\n      </g>\n     </g>\n     <g id=\"text_11\">\n      <!-- 0.4 -->\n      <g transform=\"translate(20.878125 96.638645)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-52\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_6\">\n     <g id=\"line2d_11\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#m90bf01fccf\" y=\"62.099361\"/>\n      </g>\n     </g>\n     <g id=\"text_12\">\n      <!-- 0.5 -->\n      <defs>\n       <path d=\"M 10.796875 72.90625 \nL 49.515625 72.90625 \nL 49.515625 64.59375 \nL 19.828125 64.59375 \nL 19.828125 46.734375 \nQ 21.96875 47.46875 24.109375 47.828125 \nQ 26.265625 48.1875 28.421875 48.1875 \nQ 40.625 48.1875 47.75 41.5 \nQ 54.890625 34.8125 54.890625 23.390625 \nQ 54.890625 11.625 47.5625 5.09375 \nQ 40.234375 -1.421875 26.90625 -1.421875 \nQ 22.3125 -1.421875 17.546875 -0.640625 \nQ 12.796875 0.140625 7.71875 1.703125 \nL 7.71875 11.625 \nQ 12.109375 9.234375 16.796875 8.0625 \nQ 21.484375 6.890625 26.703125 6.890625 \nQ 35.15625 6.890625 40.078125 11.328125 \nQ 45.015625 15.765625 45.015625 23.390625 \nQ 45.015625 31 40.078125 35.4375 \nQ 35.15625 39.890625 26.703125 39.890625 \nQ 22.75 39.890625 18.8125 39.015625 \nQ 14.890625 38.140625 10.796875 36.28125 \nz\n\" id=\"DejaVuSans-53\"/>\n      </defs>\n      <g transform=\"translate(20.878125 65.898579)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-53\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_7\">\n     <g id=\"line2d_12\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#m90bf01fccf\" y=\"31.359295\"/>\n      </g>\n     </g>\n     <g id=\"text_13\">\n      <!-- 0.6 -->\n      <g transform=\"translate(20.878125 35.158514)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-54\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"text_14\">\n     <!-- Losses -->\n     <defs>\n      <path d=\"M 9.8125 72.90625 \nL 19.671875 72.90625 \nL 19.671875 8.296875 \nL 55.171875 8.296875 \nL 55.171875 0 \nL 9.8125 0 \nz\n\" id=\"DejaVuSans-76\"/>\n      <path d=\"M 56.203125 29.59375 \nL 56.203125 25.203125 \nL 14.890625 25.203125 \nQ 15.484375 15.921875 20.484375 11.0625 \nQ 25.484375 6.203125 34.421875 6.203125 \nQ 39.59375 6.203125 44.453125 7.46875 \nQ 49.3125 8.734375 54.109375 11.28125 \nL 54.109375 2.78125 \nQ 49.265625 0.734375 44.1875 -0.34375 \nQ 39.109375 -1.421875 33.890625 -1.421875 \nQ 20.796875 -1.421875 13.15625 6.1875 \nQ 5.515625 13.8125 5.515625 26.8125 \nQ 5.515625 40.234375 12.765625 48.109375 \nQ 20.015625 56 32.328125 56 \nQ 43.359375 56 49.78125 48.890625 \nQ 56.203125 41.796875 56.203125 29.59375 \nz\nM 47.21875 32.234375 \nQ 47.125 39.59375 43.09375 43.984375 \nQ 39.0625 48.390625 32.421875 48.390625 \nQ 24.90625 48.390625 20.390625 44.140625 \nQ 15.875 39.890625 15.1875 32.171875 \nz\n\" id=\"DejaVuSans-101\"/>\n     </defs>\n     <g transform=\"translate(14.798438 132.655156)rotate(-90)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-76\"/>\n      <use x=\"55.697266\" xlink:href=\"#DejaVuSans-111\"/>\n      <use x=\"116.878906\" xlink:href=\"#DejaVuSans-115\"/>\n      <use x=\"168.978516\" xlink:href=\"#DejaVuSans-115\"/>\n      <use x=\"221.078125\" xlink:href=\"#DejaVuSans-101\"/>\n      <use x=\"282.601562\" xlink:href=\"#DejaVuSans-115\"/>\n     </g>\n    </g>\n   </g>\n   <g id=\"line2d_13\">\n    <path clip-path=\"url(#p9517cfd558)\" d=\"M 58.999432 32.061993 \nL 62.45811 51.584569 \nL 65.916787 82.358282 \nL 69.375465 62.914987 \nL 72.834143 109.272963 \nL 76.29282 66.965736 \nL 79.751498 109.390795 \nL 83.210176 95.956486 \nL 86.668853 107.60447 \nL 90.127531 121.356134 \nL 93.586209 128.618553 \nL 97.044886 117.003128 \nL 100.503564 119.405365 \nL 103.962242 120.982593 \nL 107.420919 139.921552 \nL 110.879597 106.311487 \nL 114.338275 134.957499 \nL 117.796952 157.68197 \nL 121.25563 121.211469 \nL 124.714308 161.673643 \nL 128.172986 151.50226 \nL 131.631663 170.620555 \nL 135.090341 176.620539 \nL 138.549019 149.452329 \nL 142.007696 176.794667 \nL 145.466374 151.863644 \nL 148.925052 151.047505 \nL 152.383729 147.152025 \nL 155.842407 172.92024 \nL 159.301085 174.996977 \nL 162.759762 165.448006 \nL 166.21844 178.096761 \nL 169.677118 170.473956 \nL 173.135795 185.933033 \nL 176.594473 194.297404 \nL 180.053151 196.011667 \nL 183.511829 169.584083 \nL 186.970506 193.479268 \nL 190.429184 178.979576 \nL 193.887862 189.83666 \nL 197.346539 201.996521 \nL 200.805217 185.739978 \nL 204.263895 172.966504 \nL 207.722572 189.781802 \nL 211.18125 211.432068 \nL 214.639928 190.291635 \nL 218.098605 198.004575 \nL 221.557283 208.816446 \nL 225.015961 202.890748 \nL 228.474638 201.055659 \nL 231.933316 205.166061 \nL 235.391994 212.322022 \nL 238.850671 212.365641 \nL 242.309349 210.659836 \nL 245.768027 191.346801 \nL 249.226705 193.527646 \nL 252.685382 196.040518 \nL 256.14406 210.828758 \nL 259.602738 212.592431 \nL 263.061415 197.69501 \nL 266.520093 212.395682 \nL 269.978771 197.953887 \nL 273.437448 204.367108 \nL 276.896126 197.359784 \nL 280.354804 209.25242 \nL 283.813481 210.273426 \nL 287.272159 211.956958 \nL 290.730837 200.822645 \nL 294.189514 212.167123 \nL 297.648192 211.845962 \nL 301.10687 206.335009 \nL 304.565548 212.801045 \nL 308.024225 212.744529 \nL 311.482903 208.48084 \nL 314.941581 210.940162 \nL 318.400258 207.708657 \nL 321.858936 208.428733 \nL 325.317614 213.600058 \nL 328.776291 214.009018 \nL 332.234969 207.615993 \nL 335.693647 211.931331 \nL 339.152324 213.334502 \nL 342.611002 173.997091 \nL 346.06968 210.220226 \nL 349.528357 210.919363 \nL 352.987035 207.610899 \nL 356.445713 214.029104 \nL 359.90439 214.756364 \nL 363.363068 211.80086 \n\" style=\"fill:none;stroke:#ff0000;stroke-linecap:square;stroke-width:1.5;\"/>\n   </g>\n   <g id=\"line2d_14\">\n    <path clip-path=\"url(#p9517cfd558)\" d=\"M 58.999432 17.083636 \nL 62.45811 82.978334 \nL 65.916787 71.184374 \nL 69.375465 102.936161 \nL 72.834143 66.11122 \nL 76.29282 70.255377 \nL 79.751498 95.719374 \nL 83.210176 88.661664 \nL 86.668853 107.143375 \nL 90.127531 105.795535 \nL 93.586209 55.714663 \nL 97.044886 109.407295 \nL 100.503564 100.049395 \nL 103.962242 108.849429 \nL 107.420919 95.901711 \nL 110.879597 93.333949 \nL 114.338275 111.986151 \nL 117.796952 88.760844 \nL 121.25563 86.086948 \nL 124.714308 84.133659 \nL 128.172986 101.767817 \nL 131.631663 70.447388 \nL 135.090341 88.150182 \nL 138.549019 98.751126 \nL 142.007696 98.18836 \nL 145.466374 79.888609 \nL 148.925052 95.119404 \nL 152.383729 77.142624 \nL 155.842407 79.198483 \nL 159.301085 55.185308 \nL 162.759762 87.889022 \nL 166.21844 93.963089 \nL 169.677118 93.241759 \nL 173.135795 90.356926 \nL 176.594473 92.047058 \nL 180.053151 103.173694 \nL 183.511829 89.694 \nL 186.970506 95.445948 \nL 190.429184 85.95422 \nL 193.887862 67.569069 \nL 197.346539 67.564332 \nL 200.805217 81.21962 \nL 204.263895 56.211405 \nL 207.722572 54.783898 \nL 211.18125 57.114888 \nL 214.639928 58.666621 \nL 218.098605 45.476669 \nL 221.557283 52.565556 \nL 225.015961 71.248833 \nL 228.474638 66.34791 \nL 231.933316 76.877791 \nL 235.391994 52.604619 \nL 238.850671 56.765074 \nL 242.309349 57.335472 \nL 245.768027 49.109968 \nL 249.226705 78.412768 \nL 252.685382 78.637915 \nL 256.14406 81.953492 \nL 259.602738 47.6047 \nL 263.061415 59.054673 \nL 266.520093 71.749779 \nL 269.978771 71.08196 \nL 273.437448 75.100993 \nL 276.896126 61.075829 \nL 280.354804 65.17571 \nL 283.813481 68.453771 \nL 287.272159 71.844195 \nL 290.730837 69.129771 \nL 294.189514 59.719377 \nL 297.648192 61.212899 \nL 301.10687 60.111021 \nL 304.565548 55.220688 \nL 308.024225 47.357255 \nL 311.482903 38.347987 \nL 314.941581 52.562368 \nL 318.400258 50.991762 \nL 321.858936 50.556566 \nL 325.317614 47.082161 \nL 328.776291 42.492629 \nL 332.234969 48.183343 \nL 335.693647 48.388904 \nL 339.152324 36.199764 \nL 342.611002 46.385209 \nL 346.06968 43.542765 \nL 349.528357 45.190527 \nL 352.987035 50.985313 \nL 356.445713 46.119185 \nL 359.90439 36.130102 \nL 363.363068 40.136355 \n\" style=\"fill:none;stroke:#0000ff;stroke-linecap:square;stroke-width:1.5;\"/>\n   </g>\n   <g id=\"patch_3\">\n    <path d=\"M 43.78125 224.64 \nL 43.78125 7.2 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_4\">\n    <path d=\"M 378.58125 224.64 \nL 378.58125 7.2 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_5\">\n    <path d=\"M 43.78125 224.64 \nL 378.58125 224.64 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_6\">\n    <path d=\"M 43.78125 7.2 \nL 378.58125 7.2 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"legend_1\">\n    <g id=\"patch_7\">\n     <path d=\"M 50.78125 219.64 \nL 108.221875 219.64 \nQ 110.221875 219.64 110.221875 217.64 \nL 110.221875 189.28375 \nQ 110.221875 187.28375 108.221875 187.28375 \nL 50.78125 187.28375 \nQ 48.78125 187.28375 48.78125 189.28375 \nL 48.78125 217.64 \nQ 48.78125 219.64 50.78125 219.64 \nz\n\" style=\"fill:#ffffff;opacity:0.8;stroke:#cccccc;stroke-linejoin:miter;\"/>\n    </g>\n    <g id=\"line2d_15\">\n     <path d=\"M 52.78125 195.382187 \nL 72.78125 195.382187 \n\" style=\"fill:none;stroke:#ff0000;stroke-linecap:square;stroke-width:1.5;\"/>\n    </g>\n    <g id=\"line2d_16\"/>\n    <g id=\"text_15\">\n     <!-- Train -->\n     <defs>\n      <path d=\"M -0.296875 72.90625 \nL 61.375 72.90625 \nL 61.375 64.59375 \nL 35.5 64.59375 \nL 35.5 0 \nL 25.59375 0 \nL 25.59375 64.59375 \nL -0.296875 64.59375 \nz\n\" id=\"DejaVuSans-84\"/>\n      <path d=\"M 41.109375 46.296875 \nQ 39.59375 47.171875 37.8125 47.578125 \nQ 36.03125 48 33.890625 48 \nQ 26.265625 48 22.1875 43.046875 \nQ 18.109375 38.09375 18.109375 28.8125 \nL 18.109375 0 \nL 9.078125 0 \nL 9.078125 54.6875 \nL 18.109375 54.6875 \nL 18.109375 46.1875 \nQ 20.953125 51.171875 25.484375 53.578125 \nQ 30.03125 56 36.53125 56 \nQ 37.453125 56 38.578125 55.875 \nQ 39.703125 55.765625 41.0625 55.515625 \nz\n\" id=\"DejaVuSans-114\"/>\n      <path d=\"M 34.28125 27.484375 \nQ 23.390625 27.484375 19.1875 25 \nQ 14.984375 22.515625 14.984375 16.5 \nQ 14.984375 11.71875 18.140625 8.90625 \nQ 21.296875 6.109375 26.703125 6.109375 \nQ 34.1875 6.109375 38.703125 11.40625 \nQ 43.21875 16.703125 43.21875 25.484375 \nL 43.21875 27.484375 \nz\nM 52.203125 31.203125 \nL 52.203125 0 \nL 43.21875 0 \nL 43.21875 8.296875 \nQ 40.140625 3.328125 35.546875 0.953125 \nQ 30.953125 -1.421875 24.3125 -1.421875 \nQ 15.921875 -1.421875 10.953125 3.296875 \nQ 6 8.015625 6 15.921875 \nQ 6 25.140625 12.171875 29.828125 \nQ 18.359375 34.515625 30.609375 34.515625 \nL 43.21875 34.515625 \nL 43.21875 35.40625 \nQ 43.21875 41.609375 39.140625 45 \nQ 35.0625 48.390625 27.6875 48.390625 \nQ 23 48.390625 18.546875 47.265625 \nQ 14.109375 46.140625 10.015625 43.890625 \nL 10.015625 52.203125 \nQ 14.9375 54.109375 19.578125 55.046875 \nQ 24.21875 56 28.609375 56 \nQ 40.484375 56 46.34375 49.84375 \nQ 52.203125 43.703125 52.203125 31.203125 \nz\n\" id=\"DejaVuSans-97\"/>\n      <path d=\"M 9.421875 54.6875 \nL 18.40625 54.6875 \nL 18.40625 0 \nL 9.421875 0 \nz\nM 9.421875 75.984375 \nL 18.40625 75.984375 \nL 18.40625 64.59375 \nL 9.421875 64.59375 \nz\n\" id=\"DejaVuSans-105\"/>\n      <path d=\"M 54.890625 33.015625 \nL 54.890625 0 \nL 45.90625 0 \nL 45.90625 32.71875 \nQ 45.90625 40.484375 42.875 44.328125 \nQ 39.84375 48.1875 33.796875 48.1875 \nQ 26.515625 48.1875 22.3125 43.546875 \nQ 18.109375 38.921875 18.109375 30.90625 \nL 18.109375 0 \nL 9.078125 0 \nL 9.078125 54.6875 \nL 18.109375 54.6875 \nL 18.109375 46.1875 \nQ 21.34375 51.125 25.703125 53.5625 \nQ 30.078125 56 35.796875 56 \nQ 45.21875 56 50.046875 50.171875 \nQ 54.890625 44.34375 54.890625 33.015625 \nz\n\" id=\"DejaVuSans-110\"/>\n     </defs>\n     <g transform=\"translate(80.78125 198.882187)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-84\"/>\n      <use x=\"60.865234\" xlink:href=\"#DejaVuSans-114\"/>\n      <use x=\"101.978516\" xlink:href=\"#DejaVuSans-97\"/>\n      <use x=\"163.257812\" xlink:href=\"#DejaVuSans-105\"/>\n      <use x=\"191.041016\" xlink:href=\"#DejaVuSans-110\"/>\n     </g>\n    </g>\n    <g id=\"line2d_17\">\n     <path d=\"M 52.78125 210.060312 \nL 72.78125 210.060312 \n\" style=\"fill:none;stroke:#0000ff;stroke-linecap:square;stroke-width:1.5;\"/>\n    </g>\n    <g id=\"line2d_18\"/>\n    <g id=\"text_16\">\n     <!-- Val -->\n     <defs>\n      <path d=\"M 28.609375 0 \nL 0.78125 72.90625 \nL 11.078125 72.90625 \nL 34.1875 11.53125 \nL 57.328125 72.90625 \nL 67.578125 72.90625 \nL 39.796875 0 \nz\n\" id=\"DejaVuSans-86\"/>\n      <path d=\"M 9.421875 75.984375 \nL 18.40625 75.984375 \nL 18.40625 0 \nL 9.421875 0 \nz\n\" id=\"DejaVuSans-108\"/>\n     </defs>\n     <g transform=\"translate(80.78125 213.560312)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-86\"/>\n      <use x=\"68.298828\" xlink:href=\"#DejaVuSans-97\"/>\n      <use x=\"129.578125\" xlink:href=\"#DejaVuSans-108\"/>\n     </g>\n    </g>\n   </g>\n  </g>\n </g>\n <defs>\n  <clipPath id=\"p9517cfd558\">\n   <rect height=\"217.44\" width=\"334.8\" x=\"43.78125\" y=\"7.2\"/>\n  </clipPath>\n </defs>\n</svg>\n"
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AqWhvdXob2XE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import files\n",
        "files.download('./sgd_resnet18_9330.pth.tar')\n",
        "files.download('./sgd_resnet18_9330_acc.jpg')\n",
        "files.download('./sgd_resnet18_9330_loss.jpg')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eCf3vbmn6kbX",
        "colab_type": "text"
      },
      "source": [
        "# ADAMW"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SDKSoqqr6i20",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "load_checkpoint(model, filename='before_sgd_resnet18.pth.tar')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LpAxh9gAARoM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "preds = []\n",
        "train_stats = AvgStats()\n",
        "test_stats = AvgStats()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Pf0p8aYCAWCZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "reset()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ctd5bSF0AW8x",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "optimizer = AdamW(model.parameters(), lr=1e-4, weight_decay=0.1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "09lIUkbeAcTr",
        "colab_type": "code",
        "outputId": "2b857ee7-0360-4023-b879-0ef415353dcc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "fit(model, optimizer, sched)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch\tTrn_loss\tVal_loss\tTrn_acc\t\tVal_acc\n",
            "1\t0.85013218\t0.73820112\t61.88399887\t63.94333267\n",
            "2\t0.68763721\t0.60742203\t69.11545563\t69.92750549\n",
            "3\t0.59119868\t0.56625939\t72.55647278\t73.00778198\n",
            "4\t0.54223408\t0.52267484\t74.78478241\t75.10124969\n",
            "5\t0.49131328\t0.50490741\t76.43138123\t76.65533447\n",
            "6\t0.45364076\t0.50395180\t77.73428345\t77.88972473\n",
            "7\t0.41434392\t0.49108689\t78.77463531\t78.89309692\n",
            "8\t0.39298269\t0.49514570\t79.67872620\t79.76708221\n",
            "9\t0.37476659\t0.48024492\t80.44981384\t80.52166748\n",
            "10\t0.35891538\t0.47812114\t81.15677643\t81.21416473\n",
            "11\t0.32745018\t0.47881951\t81.76569366\t81.80788422\n",
            "12\t0.31069746\t0.48954680\t82.32915497\t82.35777283\n",
            "13\t0.30159125\t0.47937265\t82.83116913\t82.85474396\n",
            "14\t0.28103011\t0.48615615\t83.29578400\t83.31571198\n",
            "15\t0.27771957\t0.47482704\t83.74247742\t83.76033020\n",
            "16\t0.25743926\t0.47146901\t84.16210175\t84.17302704\n",
            "17\t0.25000483\t0.49392118\t84.53821564\t84.54343414\n",
            "18\t0.24719775\t0.50725957\t84.89345551\t84.89398193\n",
            "19\t0.22206496\t0.49611363\t85.22557831\t85.22579193\n",
            "20\t0.21006189\t0.51358514\t85.53176880\t85.53182983\n",
            "21\t0.20532295\t0.49769041\t85.83472443\t85.83134460\n",
            "22\t0.19467919\t0.52070518\t86.11854553\t86.11061096\n",
            "23\t0.18837937\t0.51286097\t86.38446045\t86.37398529\n",
            "24\t0.17922431\t0.54872005\t86.64384460\t86.63062286\n",
            "25\t0.17295205\t0.52026344\t86.87812042\t86.86586761\n",
            "26\t0.16404083\t0.55050876\t87.10541534\t87.08974457\n",
            "27\t0.16027646\t0.55272451\t87.32366943\t87.31055450\n",
            "28\t0.15356673\t0.54807038\t87.52982330\t87.51731873\n",
            "29\t0.15205272\t0.53336038\t87.73260498\t87.72052002\n",
            "30\t0.15144035\t0.56158244\t87.93235016\t87.91694641\n",
            "31\t0.14505456\t0.54905042\t88.11567688\t88.10371399\n",
            "32\t0.14543191\t0.54311195\t88.28979492\t88.27671814\n",
            "33\t0.13382972\t0.56356190\t88.45989990\t88.44535065\n",
            "34\t0.13199268\t0.56716411\t88.62275696\t88.60671997\n",
            "35\t0.13232779\t0.57398035\t88.78043365\t88.76419067\n",
            "36\t0.12782316\t0.57685430\t88.92543793\t88.90869904\n",
            "37\t0.12822344\t0.56419943\t89.06433868\t89.04936981\n",
            "38\t0.10977605\t0.57902198\t89.20356750\t89.18741608\n",
            "39\t0.11047226\t0.58134795\t89.33596802\t89.32128143\n",
            "40\t0.11691693\t0.57395895\t89.46289062\t89.44662476\n",
            "41\t0.11518477\t0.58564208\t89.58624268\t89.57032776\n",
            "42\t0.10476859\t0.58431447\t89.70518494\t89.68797302\n",
            "43\t0.10681815\t0.57714300\t89.81918335\t89.80387878\n",
            "44\t0.10405797\t0.57081244\t89.93372345\t89.91893768\n",
            "45\t0.11100372\t0.58945164\t90.04327393\t90.02544403\n",
            "46\t0.10360213\t0.57951752\t90.14457703\t90.12749481\n",
            "47\t0.09576851\t0.59679878\t90.24370575\t90.22673798\n",
            "48\t0.10843114\t0.60764928\t90.33815002\t90.32086945\n",
            "49\t0.09958194\t0.59652640\t90.43030548\t90.41319275\n",
            "50\t0.08899974\t0.57427175\t90.52482605\t90.50899506\n",
            "51\t0.08770413\t0.60381258\t90.61550903\t90.59993744\n",
            "52\t0.09244790\t0.60828647\t90.70163727\t90.68537903\n",
            "53\t0.09165288\t0.61691286\t90.78527069\t90.76898956\n",
            "54\t0.09048818\t0.61422930\t90.86754608\t90.84993744\n",
            "55\t0.08632776\t0.60942170\t90.94504547\t90.92827606\n",
            "56\t0.09271458\t0.59936090\t91.01675415\t91.00119019\n",
            "57\t0.08240274\t0.57893817\t91.09223175\t91.07666779\n",
            "58\t0.08602332\t0.59577386\t91.16552734\t91.14994049\n",
            "59\t0.07999472\t0.62391376\t91.23795319\t91.22239685\n",
            "60\t0.07713582\t0.60471990\t91.30668640\t91.29164124\n",
            "61\t0.07669106\t0.59094734\t91.37605286\t91.36089325\n",
            "62\t0.08484927\t0.59978940\t91.44064331\t91.42623901\n",
            "63\t0.07930319\t0.59985073\t91.50509644\t91.49076843\n",
            "64\t0.07905199\t0.58912809\t91.56806183\t91.55375671\n",
            "65\t0.08243416\t0.61980205\t91.62799835\t91.61325836\n",
            "66\t0.07474981\t0.61083096\t91.68936920\t91.67393494\n",
            "67\t0.08015342\t0.62589693\t91.74710846\t91.73168182\n",
            "68\t0.07637443\t0.60969181\t91.80252838\t91.78831482\n",
            "69\t0.07530810\t0.61084137\t91.85839844\t91.84454346\n",
            "70\t0.07122711\t0.62127142\t91.91267395\t91.89756775\n",
            "71\t0.07065166\t0.63328891\t91.96578979\t91.95103455\n",
            "72\t0.07358607\t0.63613563\t92.01596069\t92.00051117\n",
            "73\t0.06907463\t0.62414871\t92.06487274\t92.05065918\n",
            "74\t0.07498297\t0.61461439\t92.11464691\t92.10116577\n",
            "75\t0.07507715\t0.62186696\t92.16166687\t92.14695740\n",
            "76\t0.06458644\t0.61684685\t92.20909882\t92.19445038\n",
            "77\t0.06491736\t0.62819734\t92.25495148\t92.24099731\n",
            "78\t0.06908488\t0.61722458\t92.29904175\t92.28540039\n",
            "79\t0.06238256\t0.63074326\t92.34407806\t92.33025360\n",
            "80\t0.07432589\t0.64589174\t92.38593292\t92.37145996\n",
            "81\t0.06678547\t0.62850324\t92.42791748\t92.41388702\n",
            "82\t0.06749087\t0.62236502\t92.46869659\t92.45542145\n",
            "83\t0.06876000\t0.60769736\t92.50794983\t92.49413300\n",
            "84\t0.06721120\t0.65127670\t92.54838562\t92.53456116\n",
            "85\t0.05977475\t0.61746488\t92.58808136\t92.57525635\n",
            "86\t0.06953103\t0.66695803\t92.62594604\t92.61186218\n",
            "87\t0.06519690\t0.62830518\t92.66376495\t92.65005493\n",
            "88\t0.06582824\t0.61767410\t92.69947052\t92.68657684\n",
            "89\t0.06600655\t0.62057328\t92.73598480\t92.72368622\n",
            "90\t0.05915999\t0.60422787\t92.77219391\t92.75981903\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "soIYin9sAfXw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "save_checkpoint(model, True, 'adamw_resnet18_9277.pth.tar')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vF5f3PqoBAKn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ep_adamw_losses = []\n",
        "for i in range(0, len(train_stats.losses), len(train_loader)):\n",
        "    if i != 0 :\n",
        "        ep_adamw_losses.append(train_stats.losses[i])\n",
        "        \n",
        "ep_adamw_lossesv = []\n",
        "for i in range(0, len(test_stats.losses), len(test_loader)):\n",
        "    if(i != 0):\n",
        "        ep_adamw_lossesv.append(test_stats.losses[i])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5VbJIT8mj0Tu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ep_adamw_accs = []\n",
        "for i in range(0, len(train_stats.losses), len(train_loader)):\n",
        "    if i != 0 :\n",
        "        ep_adamw_accs.append(train_stats.precs[i])\n",
        "        \n",
        "ep_adamw_accsv = []\n",
        "for i in range(0, len(test_stats.losses), len(test_loader)):\n",
        "    if(i != 0):\n",
        "        ep_adamw_accsv.append(test_stats.precs[i])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "93Ax_UHsj0ep",
        "colab_type": "code",
        "outputId": "5ce3826e-3c49-4af9-ca38-9209abc1cd7a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 371
        }
      },
      "source": [
        "plt.xlabel(\"Epochs\")\n",
        "plt.ylabel(\"Accuracy\")\n",
        "plt.plot(ep_adamw_accs, 'r', label='Train')\n",
        "plt.plot(ep_adamw_accsv, 'b', label='Val')\n",
        "plt.legend()\n",
        "plt.savefig('adamw_resnet18_9277_acc.jpg')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/svg+xml": "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n  \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<!-- Created with matplotlib (https://matplotlib.org/) -->\n<svg height=\"262.19625pt\" version=\"1.1\" viewBox=\"0 0 382.603125 262.19625\" width=\"382.603125pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n <defs>\n  <style type=\"text/css\">\n*{stroke-linecap:butt;stroke-linejoin:round;}\n  </style>\n </defs>\n <g id=\"figure_1\">\n  <g id=\"patch_1\">\n   <path d=\"M 0 262.19625 \nL 382.603125 262.19625 \nL 382.603125 0 \nL 0 0 \nz\n\" style=\"fill:none;\"/>\n  </g>\n  <g id=\"axes_1\">\n   <g id=\"patch_2\">\n    <path d=\"M 40.603125 224.64 \nL 375.403125 224.64 \nL 375.403125 7.2 \nL 40.603125 7.2 \nz\n\" style=\"fill:#ffffff;\"/>\n   </g>\n   <g id=\"matplotlib.axis_1\">\n    <g id=\"xtick_1\">\n     <g id=\"line2d_1\">\n      <defs>\n       <path d=\"M 0 0 \nL 0 3.5 \n\" id=\"m697eb5f3a4\" style=\"stroke:#000000;stroke-width:0.8;\"/>\n      </defs>\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"55.821307\" xlink:href=\"#m697eb5f3a4\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_1\">\n      <!-- 0 -->\n      <defs>\n       <path d=\"M 31.78125 66.40625 \nQ 24.171875 66.40625 20.328125 58.90625 \nQ 16.5 51.421875 16.5 36.375 \nQ 16.5 21.390625 20.328125 13.890625 \nQ 24.171875 6.390625 31.78125 6.390625 \nQ 39.453125 6.390625 43.28125 13.890625 \nQ 47.125 21.390625 47.125 36.375 \nQ 47.125 51.421875 43.28125 58.90625 \nQ 39.453125 66.40625 31.78125 66.40625 \nz\nM 31.78125 74.21875 \nQ 44.046875 74.21875 50.515625 64.515625 \nQ 56.984375 54.828125 56.984375 36.375 \nQ 56.984375 17.96875 50.515625 8.265625 \nQ 44.046875 -1.421875 31.78125 -1.421875 \nQ 19.53125 -1.421875 13.0625 8.265625 \nQ 6.59375 17.96875 6.59375 36.375 \nQ 6.59375 54.828125 13.0625 64.515625 \nQ 19.53125 74.21875 31.78125 74.21875 \nz\n\" id=\"DejaVuSans-48\"/>\n      </defs>\n      <g transform=\"translate(52.640057 239.238437)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_2\">\n     <g id=\"line2d_2\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"124.994861\" xlink:href=\"#m697eb5f3a4\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_2\">\n      <!-- 20 -->\n      <defs>\n       <path d=\"M 19.1875 8.296875 \nL 53.609375 8.296875 \nL 53.609375 0 \nL 7.328125 0 \nL 7.328125 8.296875 \nQ 12.9375 14.109375 22.625 23.890625 \nQ 32.328125 33.6875 34.8125 36.53125 \nQ 39.546875 41.84375 41.421875 45.53125 \nQ 43.3125 49.21875 43.3125 52.78125 \nQ 43.3125 58.59375 39.234375 62.25 \nQ 35.15625 65.921875 28.609375 65.921875 \nQ 23.96875 65.921875 18.8125 64.3125 \nQ 13.671875 62.703125 7.8125 59.421875 \nL 7.8125 69.390625 \nQ 13.765625 71.78125 18.9375 73 \nQ 24.125 74.21875 28.421875 74.21875 \nQ 39.75 74.21875 46.484375 68.546875 \nQ 53.21875 62.890625 53.21875 53.421875 \nQ 53.21875 48.921875 51.53125 44.890625 \nQ 49.859375 40.875 45.40625 35.40625 \nQ 44.1875 33.984375 37.640625 27.21875 \nQ 31.109375 20.453125 19.1875 8.296875 \nz\n\" id=\"DejaVuSans-50\"/>\n      </defs>\n      <g transform=\"translate(118.632361 239.238437)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-50\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_3\">\n     <g id=\"line2d_3\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"194.168414\" xlink:href=\"#m697eb5f3a4\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_3\">\n      <!-- 40 -->\n      <defs>\n       <path d=\"M 37.796875 64.3125 \nL 12.890625 25.390625 \nL 37.796875 25.390625 \nz\nM 35.203125 72.90625 \nL 47.609375 72.90625 \nL 47.609375 25.390625 \nL 58.015625 25.390625 \nL 58.015625 17.1875 \nL 47.609375 17.1875 \nL 47.609375 0 \nL 37.796875 0 \nL 37.796875 17.1875 \nL 4.890625 17.1875 \nL 4.890625 26.703125 \nz\n\" id=\"DejaVuSans-52\"/>\n      </defs>\n      <g transform=\"translate(187.805914 239.238437)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-52\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_4\">\n     <g id=\"line2d_4\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"263.341968\" xlink:href=\"#m697eb5f3a4\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_4\">\n      <!-- 60 -->\n      <defs>\n       <path d=\"M 33.015625 40.375 \nQ 26.375 40.375 22.484375 35.828125 \nQ 18.609375 31.296875 18.609375 23.390625 \nQ 18.609375 15.53125 22.484375 10.953125 \nQ 26.375 6.390625 33.015625 6.390625 \nQ 39.65625 6.390625 43.53125 10.953125 \nQ 47.40625 15.53125 47.40625 23.390625 \nQ 47.40625 31.296875 43.53125 35.828125 \nQ 39.65625 40.375 33.015625 40.375 \nz\nM 52.59375 71.296875 \nL 52.59375 62.3125 \nQ 48.875 64.0625 45.09375 64.984375 \nQ 41.3125 65.921875 37.59375 65.921875 \nQ 27.828125 65.921875 22.671875 59.328125 \nQ 17.53125 52.734375 16.796875 39.40625 \nQ 19.671875 43.65625 24.015625 45.921875 \nQ 28.375 48.1875 33.59375 48.1875 \nQ 44.578125 48.1875 50.953125 41.515625 \nQ 57.328125 34.859375 57.328125 23.390625 \nQ 57.328125 12.15625 50.6875 5.359375 \nQ 44.046875 -1.421875 33.015625 -1.421875 \nQ 20.359375 -1.421875 13.671875 8.265625 \nQ 6.984375 17.96875 6.984375 36.375 \nQ 6.984375 53.65625 15.1875 63.9375 \nQ 23.390625 74.21875 37.203125 74.21875 \nQ 40.921875 74.21875 44.703125 73.484375 \nQ 48.484375 72.75 52.59375 71.296875 \nz\n\" id=\"DejaVuSans-54\"/>\n      </defs>\n      <g transform=\"translate(256.979468 239.238437)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-54\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_5\">\n     <g id=\"line2d_5\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"332.515522\" xlink:href=\"#m697eb5f3a4\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_5\">\n      <!-- 80 -->\n      <defs>\n       <path d=\"M 31.78125 34.625 \nQ 24.75 34.625 20.71875 30.859375 \nQ 16.703125 27.09375 16.703125 20.515625 \nQ 16.703125 13.921875 20.71875 10.15625 \nQ 24.75 6.390625 31.78125 6.390625 \nQ 38.8125 6.390625 42.859375 10.171875 \nQ 46.921875 13.96875 46.921875 20.515625 \nQ 46.921875 27.09375 42.890625 30.859375 \nQ 38.875 34.625 31.78125 34.625 \nz\nM 21.921875 38.8125 \nQ 15.578125 40.375 12.03125 44.71875 \nQ 8.5 49.078125 8.5 55.328125 \nQ 8.5 64.0625 14.71875 69.140625 \nQ 20.953125 74.21875 31.78125 74.21875 \nQ 42.671875 74.21875 48.875 69.140625 \nQ 55.078125 64.0625 55.078125 55.328125 \nQ 55.078125 49.078125 51.53125 44.71875 \nQ 48 40.375 41.703125 38.8125 \nQ 48.828125 37.15625 52.796875 32.3125 \nQ 56.78125 27.484375 56.78125 20.515625 \nQ 56.78125 9.90625 50.3125 4.234375 \nQ 43.84375 -1.421875 31.78125 -1.421875 \nQ 19.734375 -1.421875 13.25 4.234375 \nQ 6.78125 9.90625 6.78125 20.515625 \nQ 6.78125 27.484375 10.78125 32.3125 \nQ 14.796875 37.15625 21.921875 38.8125 \nz\nM 18.3125 54.390625 \nQ 18.3125 48.734375 21.84375 45.5625 \nQ 25.390625 42.390625 31.78125 42.390625 \nQ 38.140625 42.390625 41.71875 45.5625 \nQ 45.3125 48.734375 45.3125 54.390625 \nQ 45.3125 60.0625 41.71875 63.234375 \nQ 38.140625 66.40625 31.78125 66.40625 \nQ 25.390625 66.40625 21.84375 63.234375 \nQ 18.3125 60.0625 18.3125 54.390625 \nz\n\" id=\"DejaVuSans-56\"/>\n      </defs>\n      <g transform=\"translate(326.153022 239.238437)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-56\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"text_6\">\n     <!-- Epochs -->\n     <defs>\n      <path d=\"M 9.8125 72.90625 \nL 55.90625 72.90625 \nL 55.90625 64.59375 \nL 19.671875 64.59375 \nL 19.671875 43.015625 \nL 54.390625 43.015625 \nL 54.390625 34.71875 \nL 19.671875 34.71875 \nL 19.671875 8.296875 \nL 56.78125 8.296875 \nL 56.78125 0 \nL 9.8125 0 \nz\n\" id=\"DejaVuSans-69\"/>\n      <path d=\"M 18.109375 8.203125 \nL 18.109375 -20.796875 \nL 9.078125 -20.796875 \nL 9.078125 54.6875 \nL 18.109375 54.6875 \nL 18.109375 46.390625 \nQ 20.953125 51.265625 25.265625 53.625 \nQ 29.59375 56 35.59375 56 \nQ 45.5625 56 51.78125 48.09375 \nQ 58.015625 40.1875 58.015625 27.296875 \nQ 58.015625 14.40625 51.78125 6.484375 \nQ 45.5625 -1.421875 35.59375 -1.421875 \nQ 29.59375 -1.421875 25.265625 0.953125 \nQ 20.953125 3.328125 18.109375 8.203125 \nz\nM 48.6875 27.296875 \nQ 48.6875 37.203125 44.609375 42.84375 \nQ 40.53125 48.484375 33.40625 48.484375 \nQ 26.265625 48.484375 22.1875 42.84375 \nQ 18.109375 37.203125 18.109375 27.296875 \nQ 18.109375 17.390625 22.1875 11.75 \nQ 26.265625 6.109375 33.40625 6.109375 \nQ 40.53125 6.109375 44.609375 11.75 \nQ 48.6875 17.390625 48.6875 27.296875 \nz\n\" id=\"DejaVuSans-112\"/>\n      <path d=\"M 30.609375 48.390625 \nQ 23.390625 48.390625 19.1875 42.75 \nQ 14.984375 37.109375 14.984375 27.296875 \nQ 14.984375 17.484375 19.15625 11.84375 \nQ 23.34375 6.203125 30.609375 6.203125 \nQ 37.796875 6.203125 41.984375 11.859375 \nQ 46.1875 17.53125 46.1875 27.296875 \nQ 46.1875 37.015625 41.984375 42.703125 \nQ 37.796875 48.390625 30.609375 48.390625 \nz\nM 30.609375 56 \nQ 42.328125 56 49.015625 48.375 \nQ 55.71875 40.765625 55.71875 27.296875 \nQ 55.71875 13.875 49.015625 6.21875 \nQ 42.328125 -1.421875 30.609375 -1.421875 \nQ 18.84375 -1.421875 12.171875 6.21875 \nQ 5.515625 13.875 5.515625 27.296875 \nQ 5.515625 40.765625 12.171875 48.375 \nQ 18.84375 56 30.609375 56 \nz\n\" id=\"DejaVuSans-111\"/>\n      <path d=\"M 48.78125 52.59375 \nL 48.78125 44.1875 \nQ 44.96875 46.296875 41.140625 47.34375 \nQ 37.3125 48.390625 33.40625 48.390625 \nQ 24.65625 48.390625 19.8125 42.84375 \nQ 14.984375 37.3125 14.984375 27.296875 \nQ 14.984375 17.28125 19.8125 11.734375 \nQ 24.65625 6.203125 33.40625 6.203125 \nQ 37.3125 6.203125 41.140625 7.25 \nQ 44.96875 8.296875 48.78125 10.40625 \nL 48.78125 2.09375 \nQ 45.015625 0.34375 40.984375 -0.53125 \nQ 36.96875 -1.421875 32.421875 -1.421875 \nQ 20.0625 -1.421875 12.78125 6.34375 \nQ 5.515625 14.109375 5.515625 27.296875 \nQ 5.515625 40.671875 12.859375 48.328125 \nQ 20.21875 56 33.015625 56 \nQ 37.15625 56 41.109375 55.140625 \nQ 45.0625 54.296875 48.78125 52.59375 \nz\n\" id=\"DejaVuSans-99\"/>\n      <path d=\"M 54.890625 33.015625 \nL 54.890625 0 \nL 45.90625 0 \nL 45.90625 32.71875 \nQ 45.90625 40.484375 42.875 44.328125 \nQ 39.84375 48.1875 33.796875 48.1875 \nQ 26.515625 48.1875 22.3125 43.546875 \nQ 18.109375 38.921875 18.109375 30.90625 \nL 18.109375 0 \nL 9.078125 0 \nL 9.078125 75.984375 \nL 18.109375 75.984375 \nL 18.109375 46.1875 \nQ 21.34375 51.125 25.703125 53.5625 \nQ 30.078125 56 35.796875 56 \nQ 45.21875 56 50.046875 50.171875 \nQ 54.890625 44.34375 54.890625 33.015625 \nz\n\" id=\"DejaVuSans-104\"/>\n      <path d=\"M 44.28125 53.078125 \nL 44.28125 44.578125 \nQ 40.484375 46.53125 36.375 47.5 \nQ 32.28125 48.484375 27.875 48.484375 \nQ 21.1875 48.484375 17.84375 46.4375 \nQ 14.5 44.390625 14.5 40.28125 \nQ 14.5 37.15625 16.890625 35.375 \nQ 19.28125 33.59375 26.515625 31.984375 \nL 29.59375 31.296875 \nQ 39.15625 29.25 43.1875 25.515625 \nQ 47.21875 21.78125 47.21875 15.09375 \nQ 47.21875 7.46875 41.1875 3.015625 \nQ 35.15625 -1.421875 24.609375 -1.421875 \nQ 20.21875 -1.421875 15.453125 -0.5625 \nQ 10.6875 0.296875 5.421875 2 \nL 5.421875 11.28125 \nQ 10.40625 8.6875 15.234375 7.390625 \nQ 20.0625 6.109375 24.8125 6.109375 \nQ 31.15625 6.109375 34.5625 8.28125 \nQ 37.984375 10.453125 37.984375 14.40625 \nQ 37.984375 18.0625 35.515625 20.015625 \nQ 33.0625 21.96875 24.703125 23.78125 \nL 21.578125 24.515625 \nQ 13.234375 26.265625 9.515625 29.90625 \nQ 5.8125 33.546875 5.8125 39.890625 \nQ 5.8125 47.609375 11.28125 51.796875 \nQ 16.75 56 26.8125 56 \nQ 31.78125 56 36.171875 55.265625 \nQ 40.578125 54.546875 44.28125 53.078125 \nz\n\" id=\"DejaVuSans-115\"/>\n     </defs>\n     <g transform=\"translate(190.0875 252.916562)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-69\"/>\n      <use x=\"63.183594\" xlink:href=\"#DejaVuSans-112\"/>\n      <use x=\"126.660156\" xlink:href=\"#DejaVuSans-111\"/>\n      <use x=\"187.841797\" xlink:href=\"#DejaVuSans-99\"/>\n      <use x=\"242.822266\" xlink:href=\"#DejaVuSans-104\"/>\n      <use x=\"306.201172\" xlink:href=\"#DejaVuSans-115\"/>\n     </g>\n    </g>\n   </g>\n   <g id=\"matplotlib.axis_2\">\n    <g id=\"ytick_1\">\n     <g id=\"line2d_6\">\n      <defs>\n       <path d=\"M 0 0 \nL -3.5 0 \n\" id=\"me0cd80a575\" style=\"stroke:#000000;stroke-width:0.8;\"/>\n      </defs>\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"40.603125\" xlink:href=\"#me0cd80a575\" y=\"207.710693\"/>\n      </g>\n     </g>\n     <g id=\"text_7\">\n      <!-- 65 -->\n      <defs>\n       <path d=\"M 10.796875 72.90625 \nL 49.515625 72.90625 \nL 49.515625 64.59375 \nL 19.828125 64.59375 \nL 19.828125 46.734375 \nQ 21.96875 47.46875 24.109375 47.828125 \nQ 26.265625 48.1875 28.421875 48.1875 \nQ 40.625 48.1875 47.75 41.5 \nQ 54.890625 34.8125 54.890625 23.390625 \nQ 54.890625 11.625 47.5625 5.09375 \nQ 40.234375 -1.421875 26.90625 -1.421875 \nQ 22.3125 -1.421875 17.546875 -0.640625 \nQ 12.796875 0.140625 7.71875 1.703125 \nL 7.71875 11.625 \nQ 12.109375 9.234375 16.796875 8.0625 \nQ 21.484375 6.890625 26.703125 6.890625 \nQ 35.15625 6.890625 40.078125 11.328125 \nQ 45.015625 15.765625 45.015625 23.390625 \nQ 45.015625 31 40.078125 35.4375 \nQ 35.15625 39.890625 26.703125 39.890625 \nQ 22.75 39.890625 18.8125 39.015625 \nQ 14.890625 38.140625 10.796875 36.28125 \nz\n\" id=\"DejaVuSans-53\"/>\n      </defs>\n      <g transform=\"translate(20.878125 211.509911)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-54\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-53\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_2\">\n     <g id=\"line2d_7\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"40.603125\" xlink:href=\"#me0cd80a575\" y=\"173.390792\"/>\n      </g>\n     </g>\n     <g id=\"text_8\">\n      <!-- 70 -->\n      <defs>\n       <path d=\"M 8.203125 72.90625 \nL 55.078125 72.90625 \nL 55.078125 68.703125 \nL 28.609375 0 \nL 18.3125 0 \nL 43.21875 64.59375 \nL 8.203125 64.59375 \nz\n\" id=\"DejaVuSans-55\"/>\n      </defs>\n      <g transform=\"translate(20.878125 177.19001)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-55\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_3\">\n     <g id=\"line2d_8\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"40.603125\" xlink:href=\"#me0cd80a575\" y=\"139.07089\"/>\n      </g>\n     </g>\n     <g id=\"text_9\">\n      <!-- 75 -->\n      <g transform=\"translate(20.878125 142.870109)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-55\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-53\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_4\">\n     <g id=\"line2d_9\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"40.603125\" xlink:href=\"#me0cd80a575\" y=\"104.750989\"/>\n      </g>\n     </g>\n     <g id=\"text_10\">\n      <!-- 80 -->\n      <g transform=\"translate(20.878125 108.550208)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-56\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_5\">\n     <g id=\"line2d_10\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"40.603125\" xlink:href=\"#me0cd80a575\" y=\"70.431088\"/>\n      </g>\n     </g>\n     <g id=\"text_11\">\n      <!-- 85 -->\n      <g transform=\"translate(20.878125 74.230307)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-56\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-53\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_6\">\n     <g id=\"line2d_11\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"40.603125\" xlink:href=\"#me0cd80a575\" y=\"36.111187\"/>\n      </g>\n     </g>\n     <g id=\"text_12\">\n      <!-- 90 -->\n      <defs>\n       <path d=\"M 10.984375 1.515625 \nL 10.984375 10.5 \nQ 14.703125 8.734375 18.5 7.8125 \nQ 22.3125 6.890625 25.984375 6.890625 \nQ 35.75 6.890625 40.890625 13.453125 \nQ 46.046875 20.015625 46.78125 33.40625 \nQ 43.953125 29.203125 39.59375 26.953125 \nQ 35.25 24.703125 29.984375 24.703125 \nQ 19.046875 24.703125 12.671875 31.3125 \nQ 6.296875 37.9375 6.296875 49.421875 \nQ 6.296875 60.640625 12.9375 67.421875 \nQ 19.578125 74.21875 30.609375 74.21875 \nQ 43.265625 74.21875 49.921875 64.515625 \nQ 56.59375 54.828125 56.59375 36.375 \nQ 56.59375 19.140625 48.40625 8.859375 \nQ 40.234375 -1.421875 26.421875 -1.421875 \nQ 22.703125 -1.421875 18.890625 -0.6875 \nQ 15.09375 0.046875 10.984375 1.515625 \nz\nM 30.609375 32.421875 \nQ 37.25 32.421875 41.125 36.953125 \nQ 45.015625 41.5 45.015625 49.421875 \nQ 45.015625 57.28125 41.125 61.84375 \nQ 37.25 66.40625 30.609375 66.40625 \nQ 23.96875 66.40625 20.09375 61.84375 \nQ 16.21875 57.28125 16.21875 49.421875 \nQ 16.21875 41.5 20.09375 36.953125 \nQ 23.96875 32.421875 30.609375 32.421875 \nz\n\" id=\"DejaVuSans-57\"/>\n      </defs>\n      <g transform=\"translate(20.878125 39.910406)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-57\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"text_13\">\n     <!-- Accuracy -->\n     <defs>\n      <path d=\"M 34.1875 63.1875 \nL 20.796875 26.90625 \nL 47.609375 26.90625 \nz\nM 28.609375 72.90625 \nL 39.796875 72.90625 \nL 67.578125 0 \nL 57.328125 0 \nL 50.6875 18.703125 \nL 17.828125 18.703125 \nL 11.1875 0 \nL 0.78125 0 \nz\n\" id=\"DejaVuSans-65\"/>\n      <path d=\"M 8.5 21.578125 \nL 8.5 54.6875 \nL 17.484375 54.6875 \nL 17.484375 21.921875 \nQ 17.484375 14.15625 20.5 10.265625 \nQ 23.53125 6.390625 29.59375 6.390625 \nQ 36.859375 6.390625 41.078125 11.03125 \nQ 45.3125 15.671875 45.3125 23.6875 \nL 45.3125 54.6875 \nL 54.296875 54.6875 \nL 54.296875 0 \nL 45.3125 0 \nL 45.3125 8.40625 \nQ 42.046875 3.421875 37.71875 1 \nQ 33.40625 -1.421875 27.6875 -1.421875 \nQ 18.265625 -1.421875 13.375 4.4375 \nQ 8.5 10.296875 8.5 21.578125 \nz\nM 31.109375 56 \nz\n\" id=\"DejaVuSans-117\"/>\n      <path d=\"M 41.109375 46.296875 \nQ 39.59375 47.171875 37.8125 47.578125 \nQ 36.03125 48 33.890625 48 \nQ 26.265625 48 22.1875 43.046875 \nQ 18.109375 38.09375 18.109375 28.8125 \nL 18.109375 0 \nL 9.078125 0 \nL 9.078125 54.6875 \nL 18.109375 54.6875 \nL 18.109375 46.1875 \nQ 20.953125 51.171875 25.484375 53.578125 \nQ 30.03125 56 36.53125 56 \nQ 37.453125 56 38.578125 55.875 \nQ 39.703125 55.765625 41.0625 55.515625 \nz\n\" id=\"DejaVuSans-114\"/>\n      <path d=\"M 34.28125 27.484375 \nQ 23.390625 27.484375 19.1875 25 \nQ 14.984375 22.515625 14.984375 16.5 \nQ 14.984375 11.71875 18.140625 8.90625 \nQ 21.296875 6.109375 26.703125 6.109375 \nQ 34.1875 6.109375 38.703125 11.40625 \nQ 43.21875 16.703125 43.21875 25.484375 \nL 43.21875 27.484375 \nz\nM 52.203125 31.203125 \nL 52.203125 0 \nL 43.21875 0 \nL 43.21875 8.296875 \nQ 40.140625 3.328125 35.546875 0.953125 \nQ 30.953125 -1.421875 24.3125 -1.421875 \nQ 15.921875 -1.421875 10.953125 3.296875 \nQ 6 8.015625 6 15.921875 \nQ 6 25.140625 12.171875 29.828125 \nQ 18.359375 34.515625 30.609375 34.515625 \nL 43.21875 34.515625 \nL 43.21875 35.40625 \nQ 43.21875 41.609375 39.140625 45 \nQ 35.0625 48.390625 27.6875 48.390625 \nQ 23 48.390625 18.546875 47.265625 \nQ 14.109375 46.140625 10.015625 43.890625 \nL 10.015625 52.203125 \nQ 14.9375 54.109375 19.578125 55.046875 \nQ 24.21875 56 28.609375 56 \nQ 40.484375 56 46.34375 49.84375 \nQ 52.203125 43.703125 52.203125 31.203125 \nz\n\" id=\"DejaVuSans-97\"/>\n      <path d=\"M 32.171875 -5.078125 \nQ 28.375 -14.84375 24.75 -17.8125 \nQ 21.140625 -20.796875 15.09375 -20.796875 \nL 7.90625 -20.796875 \nL 7.90625 -13.28125 \nL 13.1875 -13.28125 \nQ 16.890625 -13.28125 18.9375 -11.515625 \nQ 21 -9.765625 23.484375 -3.21875 \nL 25.09375 0.875 \nL 2.984375 54.6875 \nL 12.5 54.6875 \nL 29.59375 11.921875 \nL 46.6875 54.6875 \nL 56.203125 54.6875 \nz\n\" id=\"DejaVuSans-121\"/>\n     </defs>\n     <g transform=\"translate(14.798438 138.834844)rotate(-90)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-65\"/>\n      <use x=\"68.392578\" xlink:href=\"#DejaVuSans-99\"/>\n      <use x=\"123.373047\" xlink:href=\"#DejaVuSans-99\"/>\n      <use x=\"178.353516\" xlink:href=\"#DejaVuSans-117\"/>\n      <use x=\"241.732422\" xlink:href=\"#DejaVuSans-114\"/>\n      <use x=\"282.845703\" xlink:href=\"#DejaVuSans-97\"/>\n      <use x=\"344.125\" xlink:href=\"#DejaVuSans-99\"/>\n      <use x=\"399.105469\" xlink:href=\"#DejaVuSans-121\"/>\n     </g>\n    </g>\n   </g>\n   <g id=\"line2d_12\">\n    <path clip-path=\"url(#p3dad255986)\" d=\"M 55.821307 214.756364 \nL 59.279985 173.817015 \nL 62.738662 152.712863 \nL 66.19734 138.353396 \nL 69.656018 127.670112 \nL 73.114695 119.210531 \nL 76.573373 112.325865 \nL 80.032051 106.335593 \nL 83.490728 101.152574 \nL 86.949406 96.404314 \nL 90.408084 92.331077 \nL 93.866761 88.555291 \nL 97.325439 85.146448 \nL 100.784117 81.97944 \nL 104.242794 78.934921 \nL 107.701472 76.100868 \nL 111.16015 73.555626 \nL 114.618827 71.153505 \nL 118.077505 68.875287 \nL 121.536183 66.778629 \nL 124.994861 64.720828 \nL 128.453538 62.802326 \nL 131.912216 60.993377 \nL 135.370894 59.236064 \nL 138.829571 57.618416 \nL 142.288249 56.083405 \nL 145.746927 54.567979 \nL 149.205604 53.148178 \nL 152.664282 51.754455 \nL 156.12296 50.405979 \nL 159.581637 49.123853 \nL 163.040315 47.935832 \nL 166.498993 46.779547 \nL 169.95767 45.671387 \nL 173.416348 44.590354 \nL 176.875026 43.598295 \nL 180.333704 42.632838 \nL 183.792381 41.686705 \nL 187.251059 40.76707 \nL 190.709737 39.907659 \nL 194.168414 39.057307 \nL 197.627092 38.250683 \nL 201.08577 37.454165 \nL 204.544447 36.665241 \nL 208.003125 35.935074 \nL 211.461803 35.23439 \nL 214.92048 34.55282 \nL 218.379158 33.906023 \nL 221.837836 33.273103 \nL 225.296513 32.61578 \nL 228.755191 31.992129 \nL 232.213869 31.404769 \nL 235.672546 30.830711 \nL 239.131224 30.276029 \nL 242.589902 29.737372 \nL 246.04858 29.238357 \nL 249.507257 28.719076 \nL 252.965935 28.215924 \nL 256.424613 27.719213 \nL 259.88329 27.243502 \nL 263.341968 26.768315 \nL 266.800646 26.319888 \nL 270.259323 25.876959 \nL 273.718001 25.445132 \nL 277.176679 25.036295 \nL 280.635356 24.619969 \nL 284.094034 24.223072 \nL 287.552712 23.834973 \nL 291.011389 23.448707 \nL 294.470067 23.085115 \nL 297.928745 22.71833 \nL 301.387423 22.37888 \nL 304.8461 22.034351 \nL 308.304778 21.687518 \nL 311.763456 21.373153 \nL 315.222133 21.047633 \nL 318.680811 20.727665 \nL 322.139489 20.423511 \nL 325.598166 20.114959 \nL 329.056844 19.832433 \nL 332.515522 19.541058 \nL 335.974199 19.256123 \nL 339.432877 18.990408 \nL 342.891555 18.712648 \nL 346.350232 18.433788 \nL 349.80891 18.182736 \nL 353.267588 17.920372 \nL 356.726265 17.669634 \nL 360.184943 17.414655 \n\" style=\"fill:none;stroke:#ff0000;stroke-linecap:square;stroke-width:1.5;\"/>\n   </g>\n   <g id=\"line2d_13\">\n    <path clip-path=\"url(#p3dad255986)\" d=\"M 55.821307 179.384154 \nL 59.279985 155.794301 \nL 62.738662 140.511482 \nL 66.19734 129.221881 \nL 69.656018 120.28617 \nL 73.114695 113.144849 \nL 76.573373 106.943062 \nL 80.032051 101.660858 \nL 83.490728 96.800268 \nL 86.949406 92.622505 \nL 90.408084 88.756332 \nL 93.866761 85.311669 \nL 97.325439 82.123557 \nL 100.784117 79.0583 \nL 104.242794 76.179997 \nL 107.701472 73.596159 \nL 111.16015 71.160261 \nL 114.618827 68.880995 \nL 118.077505 66.778996 \nL 121.536183 64.700404 \nL 124.994861 62.750848 \nL 128.453538 60.928022 \nL 131.912216 59.146724 \nL 135.370894 57.538869 \nL 138.829571 55.977988 \nL 142.288249 54.480158 \nL 145.746927 53.067688 \nL 149.205604 51.673913 \nL 152.664282 50.303757 \nL 156.12296 49.044725 \nL 159.581637 47.851153 \nL 163.040315 46.682875 \nL 166.498993 45.566023 \nL 169.95767 44.482214 \nL 173.416348 43.486541 \nL 176.875026 42.534753 \nL 180.333704 41.579769 \nL 183.792381 40.669823 \nL 187.251059 39.798943 \nL 190.709737 38.951367 \nL 194.168414 38.136154 \nL 197.627092 37.353409 \nL 201.08577 36.566108 \nL 204.544447 35.814208 \nL 208.003125 35.120856 \nL 211.461803 34.439496 \nL 214.92048 33.790761 \nL 218.379158 33.159098 \nL 221.837836 32.508582 \nL 225.296513 31.887707 \nL 228.755191 31.296053 \nL 232.213869 30.722257 \nL 235.672546 30.158253 \nL 239.131224 29.625933 \nL 242.589902 29.133202 \nL 246.04858 28.614863 \nL 249.507257 28.112706 \nL 252.965935 27.614582 \nL 256.424613 27.14306 \nL 259.88329 26.666354 \nL 263.341968 26.222954 \nL 266.800646 25.781544 \nL 270.259323 25.348932 \nL 273.718001 24.937948 \nL 277.176679 24.516019 \nL 280.635356 24.120483 \nL 284.094034 23.739611 \nL 287.552712 23.35612 \nL 291.011389 22.98326 \nL 294.470067 22.618831 \nL 297.928745 22.274249 \nL 301.387423 21.938989 \nL 304.8461 21.596869 \nL 308.304778 21.274596 \nL 311.763456 20.948396 \nL 315.222133 20.633769 \nL 318.680811 20.331239 \nL 322.139489 20.022425 \nL 325.598166 19.734819 \nL 329.056844 19.447005 \nL 332.515522 19.166888 \nL 335.974199 18.897874 \nL 339.432877 18.619433 \nL 342.891555 18.347015 \nL 346.350232 18.087636 \nL 349.80891 17.827943 \nL 353.267588 17.582651 \nL 356.726265 17.332018 \nL 360.184943 17.083636 \n\" style=\"fill:none;stroke:#0000ff;stroke-linecap:square;stroke-width:1.5;\"/>\n   </g>\n   <g id=\"patch_3\">\n    <path d=\"M 40.603125 224.64 \nL 40.603125 7.2 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_4\">\n    <path d=\"M 375.403125 224.64 \nL 375.403125 7.2 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_5\">\n    <path d=\"M 40.603125 224.64 \nL 375.403125 224.64 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_6\">\n    <path d=\"M 40.603125 7.2 \nL 375.403125 7.2 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"legend_1\">\n    <g id=\"patch_7\">\n     <path d=\"M 47.603125 44.55625 \nL 105.04375 44.55625 \nQ 107.04375 44.55625 107.04375 42.55625 \nL 107.04375 14.2 \nQ 107.04375 12.2 105.04375 12.2 \nL 47.603125 12.2 \nQ 45.603125 12.2 45.603125 14.2 \nL 45.603125 42.55625 \nQ 45.603125 44.55625 47.603125 44.55625 \nz\n\" style=\"fill:#ffffff;opacity:0.8;stroke:#cccccc;stroke-linejoin:miter;\"/>\n    </g>\n    <g id=\"line2d_14\">\n     <path d=\"M 49.603125 20.298437 \nL 69.603125 20.298437 \n\" style=\"fill:none;stroke:#ff0000;stroke-linecap:square;stroke-width:1.5;\"/>\n    </g>\n    <g id=\"line2d_15\"/>\n    <g id=\"text_14\">\n     <!-- Train -->\n     <defs>\n      <path d=\"M -0.296875 72.90625 \nL 61.375 72.90625 \nL 61.375 64.59375 \nL 35.5 64.59375 \nL 35.5 0 \nL 25.59375 0 \nL 25.59375 64.59375 \nL -0.296875 64.59375 \nz\n\" id=\"DejaVuSans-84\"/>\n      <path d=\"M 9.421875 54.6875 \nL 18.40625 54.6875 \nL 18.40625 0 \nL 9.421875 0 \nz\nM 9.421875 75.984375 \nL 18.40625 75.984375 \nL 18.40625 64.59375 \nL 9.421875 64.59375 \nz\n\" id=\"DejaVuSans-105\"/>\n      <path d=\"M 54.890625 33.015625 \nL 54.890625 0 \nL 45.90625 0 \nL 45.90625 32.71875 \nQ 45.90625 40.484375 42.875 44.328125 \nQ 39.84375 48.1875 33.796875 48.1875 \nQ 26.515625 48.1875 22.3125 43.546875 \nQ 18.109375 38.921875 18.109375 30.90625 \nL 18.109375 0 \nL 9.078125 0 \nL 9.078125 54.6875 \nL 18.109375 54.6875 \nL 18.109375 46.1875 \nQ 21.34375 51.125 25.703125 53.5625 \nQ 30.078125 56 35.796875 56 \nQ 45.21875 56 50.046875 50.171875 \nQ 54.890625 44.34375 54.890625 33.015625 \nz\n\" id=\"DejaVuSans-110\"/>\n     </defs>\n     <g transform=\"translate(77.603125 23.798437)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-84\"/>\n      <use x=\"60.865234\" xlink:href=\"#DejaVuSans-114\"/>\n      <use x=\"101.978516\" xlink:href=\"#DejaVuSans-97\"/>\n      <use x=\"163.257812\" xlink:href=\"#DejaVuSans-105\"/>\n      <use x=\"191.041016\" xlink:href=\"#DejaVuSans-110\"/>\n     </g>\n    </g>\n    <g id=\"line2d_16\">\n     <path d=\"M 49.603125 34.976562 \nL 69.603125 34.976562 \n\" style=\"fill:none;stroke:#0000ff;stroke-linecap:square;stroke-width:1.5;\"/>\n    </g>\n    <g id=\"line2d_17\"/>\n    <g id=\"text_15\">\n     <!-- Val -->\n     <defs>\n      <path d=\"M 28.609375 0 \nL 0.78125 72.90625 \nL 11.078125 72.90625 \nL 34.1875 11.53125 \nL 57.328125 72.90625 \nL 67.578125 72.90625 \nL 39.796875 0 \nz\n\" id=\"DejaVuSans-86\"/>\n      <path d=\"M 9.421875 75.984375 \nL 18.40625 75.984375 \nL 18.40625 0 \nL 9.421875 0 \nz\n\" id=\"DejaVuSans-108\"/>\n     </defs>\n     <g transform=\"translate(77.603125 38.476562)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-86\"/>\n      <use x=\"68.298828\" xlink:href=\"#DejaVuSans-97\"/>\n      <use x=\"129.578125\" xlink:href=\"#DejaVuSans-108\"/>\n     </g>\n    </g>\n   </g>\n  </g>\n </g>\n <defs>\n  <clipPath id=\"p3dad255986\">\n   <rect height=\"217.44\" width=\"334.8\" x=\"40.603125\" y=\"7.2\"/>\n  </clipPath>\n </defs>\n</svg>\n"
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SxoQon-Yj0po",
        "colab_type": "code",
        "outputId": "31fd165d-71cf-4699-ad31-f86a8c649c99",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 371
        }
      },
      "source": [
        "plt.xlabel(\"Epochs\")\n",
        "plt.ylabel(\"Losses\")\n",
        "plt.plot(ep_adamw_losses, 'r', label='Train')\n",
        "plt.plot(ep_adamw_lossesv, 'b', label='Val')\n",
        "plt.legend()\n",
        "plt.savefig('adamw_resnet18_9277_loss.jpg')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/svg+xml": "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n  \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<!-- Created with matplotlib (https://matplotlib.org/) -->\n<svg height=\"262.19625pt\" version=\"1.1\" viewBox=\"0 0 385.78125 262.19625\" width=\"385.78125pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n <defs>\n  <style type=\"text/css\">\n*{stroke-linecap:butt;stroke-linejoin:round;}\n  </style>\n </defs>\n <g id=\"figure_1\">\n  <g id=\"patch_1\">\n   <path d=\"M 0 262.19625 \nL 385.78125 262.19625 \nL 385.78125 0 \nL 0 0 \nz\n\" style=\"fill:none;\"/>\n  </g>\n  <g id=\"axes_1\">\n   <g id=\"patch_2\">\n    <path d=\"M 43.78125 224.64 \nL 378.58125 224.64 \nL 378.58125 7.2 \nL 43.78125 7.2 \nz\n\" style=\"fill:#ffffff;\"/>\n   </g>\n   <g id=\"matplotlib.axis_1\">\n    <g id=\"xtick_1\">\n     <g id=\"line2d_1\">\n      <defs>\n       <path d=\"M 0 0 \nL 0 3.5 \n\" id=\"m92dc2154a4\" style=\"stroke:#000000;stroke-width:0.8;\"/>\n      </defs>\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"58.999432\" xlink:href=\"#m92dc2154a4\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_1\">\n      <!-- 0 -->\n      <defs>\n       <path d=\"M 31.78125 66.40625 \nQ 24.171875 66.40625 20.328125 58.90625 \nQ 16.5 51.421875 16.5 36.375 \nQ 16.5 21.390625 20.328125 13.890625 \nQ 24.171875 6.390625 31.78125 6.390625 \nQ 39.453125 6.390625 43.28125 13.890625 \nQ 47.125 21.390625 47.125 36.375 \nQ 47.125 51.421875 43.28125 58.90625 \nQ 39.453125 66.40625 31.78125 66.40625 \nz\nM 31.78125 74.21875 \nQ 44.046875 74.21875 50.515625 64.515625 \nQ 56.984375 54.828125 56.984375 36.375 \nQ 56.984375 17.96875 50.515625 8.265625 \nQ 44.046875 -1.421875 31.78125 -1.421875 \nQ 19.53125 -1.421875 13.0625 8.265625 \nQ 6.59375 17.96875 6.59375 36.375 \nQ 6.59375 54.828125 13.0625 64.515625 \nQ 19.53125 74.21875 31.78125 74.21875 \nz\n\" id=\"DejaVuSans-48\"/>\n      </defs>\n      <g transform=\"translate(55.818182 239.238437)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_2\">\n     <g id=\"line2d_2\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"128.172986\" xlink:href=\"#m92dc2154a4\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_2\">\n      <!-- 20 -->\n      <defs>\n       <path d=\"M 19.1875 8.296875 \nL 53.609375 8.296875 \nL 53.609375 0 \nL 7.328125 0 \nL 7.328125 8.296875 \nQ 12.9375 14.109375 22.625 23.890625 \nQ 32.328125 33.6875 34.8125 36.53125 \nQ 39.546875 41.84375 41.421875 45.53125 \nQ 43.3125 49.21875 43.3125 52.78125 \nQ 43.3125 58.59375 39.234375 62.25 \nQ 35.15625 65.921875 28.609375 65.921875 \nQ 23.96875 65.921875 18.8125 64.3125 \nQ 13.671875 62.703125 7.8125 59.421875 \nL 7.8125 69.390625 \nQ 13.765625 71.78125 18.9375 73 \nQ 24.125 74.21875 28.421875 74.21875 \nQ 39.75 74.21875 46.484375 68.546875 \nQ 53.21875 62.890625 53.21875 53.421875 \nQ 53.21875 48.921875 51.53125 44.890625 \nQ 49.859375 40.875 45.40625 35.40625 \nQ 44.1875 33.984375 37.640625 27.21875 \nQ 31.109375 20.453125 19.1875 8.296875 \nz\n\" id=\"DejaVuSans-50\"/>\n      </defs>\n      <g transform=\"translate(121.810486 239.238437)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-50\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_3\">\n     <g id=\"line2d_3\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"197.346539\" xlink:href=\"#m92dc2154a4\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_3\">\n      <!-- 40 -->\n      <defs>\n       <path d=\"M 37.796875 64.3125 \nL 12.890625 25.390625 \nL 37.796875 25.390625 \nz\nM 35.203125 72.90625 \nL 47.609375 72.90625 \nL 47.609375 25.390625 \nL 58.015625 25.390625 \nL 58.015625 17.1875 \nL 47.609375 17.1875 \nL 47.609375 0 \nL 37.796875 0 \nL 37.796875 17.1875 \nL 4.890625 17.1875 \nL 4.890625 26.703125 \nz\n\" id=\"DejaVuSans-52\"/>\n      </defs>\n      <g transform=\"translate(190.984039 239.238437)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-52\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_4\">\n     <g id=\"line2d_4\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"266.520093\" xlink:href=\"#m92dc2154a4\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_4\">\n      <!-- 60 -->\n      <defs>\n       <path d=\"M 33.015625 40.375 \nQ 26.375 40.375 22.484375 35.828125 \nQ 18.609375 31.296875 18.609375 23.390625 \nQ 18.609375 15.53125 22.484375 10.953125 \nQ 26.375 6.390625 33.015625 6.390625 \nQ 39.65625 6.390625 43.53125 10.953125 \nQ 47.40625 15.53125 47.40625 23.390625 \nQ 47.40625 31.296875 43.53125 35.828125 \nQ 39.65625 40.375 33.015625 40.375 \nz\nM 52.59375 71.296875 \nL 52.59375 62.3125 \nQ 48.875 64.0625 45.09375 64.984375 \nQ 41.3125 65.921875 37.59375 65.921875 \nQ 27.828125 65.921875 22.671875 59.328125 \nQ 17.53125 52.734375 16.796875 39.40625 \nQ 19.671875 43.65625 24.015625 45.921875 \nQ 28.375 48.1875 33.59375 48.1875 \nQ 44.578125 48.1875 50.953125 41.515625 \nQ 57.328125 34.859375 57.328125 23.390625 \nQ 57.328125 12.15625 50.6875 5.359375 \nQ 44.046875 -1.421875 33.015625 -1.421875 \nQ 20.359375 -1.421875 13.671875 8.265625 \nQ 6.984375 17.96875 6.984375 36.375 \nQ 6.984375 53.65625 15.1875 63.9375 \nQ 23.390625 74.21875 37.203125 74.21875 \nQ 40.921875 74.21875 44.703125 73.484375 \nQ 48.484375 72.75 52.59375 71.296875 \nz\n\" id=\"DejaVuSans-54\"/>\n      </defs>\n      <g transform=\"translate(260.157593 239.238437)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-54\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_5\">\n     <g id=\"line2d_5\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"335.693647\" xlink:href=\"#m92dc2154a4\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_5\">\n      <!-- 80 -->\n      <defs>\n       <path d=\"M 31.78125 34.625 \nQ 24.75 34.625 20.71875 30.859375 \nQ 16.703125 27.09375 16.703125 20.515625 \nQ 16.703125 13.921875 20.71875 10.15625 \nQ 24.75 6.390625 31.78125 6.390625 \nQ 38.8125 6.390625 42.859375 10.171875 \nQ 46.921875 13.96875 46.921875 20.515625 \nQ 46.921875 27.09375 42.890625 30.859375 \nQ 38.875 34.625 31.78125 34.625 \nz\nM 21.921875 38.8125 \nQ 15.578125 40.375 12.03125 44.71875 \nQ 8.5 49.078125 8.5 55.328125 \nQ 8.5 64.0625 14.71875 69.140625 \nQ 20.953125 74.21875 31.78125 74.21875 \nQ 42.671875 74.21875 48.875 69.140625 \nQ 55.078125 64.0625 55.078125 55.328125 \nQ 55.078125 49.078125 51.53125 44.71875 \nQ 48 40.375 41.703125 38.8125 \nQ 48.828125 37.15625 52.796875 32.3125 \nQ 56.78125 27.484375 56.78125 20.515625 \nQ 56.78125 9.90625 50.3125 4.234375 \nQ 43.84375 -1.421875 31.78125 -1.421875 \nQ 19.734375 -1.421875 13.25 4.234375 \nQ 6.78125 9.90625 6.78125 20.515625 \nQ 6.78125 27.484375 10.78125 32.3125 \nQ 14.796875 37.15625 21.921875 38.8125 \nz\nM 18.3125 54.390625 \nQ 18.3125 48.734375 21.84375 45.5625 \nQ 25.390625 42.390625 31.78125 42.390625 \nQ 38.140625 42.390625 41.71875 45.5625 \nQ 45.3125 48.734375 45.3125 54.390625 \nQ 45.3125 60.0625 41.71875 63.234375 \nQ 38.140625 66.40625 31.78125 66.40625 \nQ 25.390625 66.40625 21.84375 63.234375 \nQ 18.3125 60.0625 18.3125 54.390625 \nz\n\" id=\"DejaVuSans-56\"/>\n      </defs>\n      <g transform=\"translate(329.331147 239.238437)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-56\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"text_6\">\n     <!-- Epochs -->\n     <defs>\n      <path d=\"M 9.8125 72.90625 \nL 55.90625 72.90625 \nL 55.90625 64.59375 \nL 19.671875 64.59375 \nL 19.671875 43.015625 \nL 54.390625 43.015625 \nL 54.390625 34.71875 \nL 19.671875 34.71875 \nL 19.671875 8.296875 \nL 56.78125 8.296875 \nL 56.78125 0 \nL 9.8125 0 \nz\n\" id=\"DejaVuSans-69\"/>\n      <path d=\"M 18.109375 8.203125 \nL 18.109375 -20.796875 \nL 9.078125 -20.796875 \nL 9.078125 54.6875 \nL 18.109375 54.6875 \nL 18.109375 46.390625 \nQ 20.953125 51.265625 25.265625 53.625 \nQ 29.59375 56 35.59375 56 \nQ 45.5625 56 51.78125 48.09375 \nQ 58.015625 40.1875 58.015625 27.296875 \nQ 58.015625 14.40625 51.78125 6.484375 \nQ 45.5625 -1.421875 35.59375 -1.421875 \nQ 29.59375 -1.421875 25.265625 0.953125 \nQ 20.953125 3.328125 18.109375 8.203125 \nz\nM 48.6875 27.296875 \nQ 48.6875 37.203125 44.609375 42.84375 \nQ 40.53125 48.484375 33.40625 48.484375 \nQ 26.265625 48.484375 22.1875 42.84375 \nQ 18.109375 37.203125 18.109375 27.296875 \nQ 18.109375 17.390625 22.1875 11.75 \nQ 26.265625 6.109375 33.40625 6.109375 \nQ 40.53125 6.109375 44.609375 11.75 \nQ 48.6875 17.390625 48.6875 27.296875 \nz\n\" id=\"DejaVuSans-112\"/>\n      <path d=\"M 30.609375 48.390625 \nQ 23.390625 48.390625 19.1875 42.75 \nQ 14.984375 37.109375 14.984375 27.296875 \nQ 14.984375 17.484375 19.15625 11.84375 \nQ 23.34375 6.203125 30.609375 6.203125 \nQ 37.796875 6.203125 41.984375 11.859375 \nQ 46.1875 17.53125 46.1875 27.296875 \nQ 46.1875 37.015625 41.984375 42.703125 \nQ 37.796875 48.390625 30.609375 48.390625 \nz\nM 30.609375 56 \nQ 42.328125 56 49.015625 48.375 \nQ 55.71875 40.765625 55.71875 27.296875 \nQ 55.71875 13.875 49.015625 6.21875 \nQ 42.328125 -1.421875 30.609375 -1.421875 \nQ 18.84375 -1.421875 12.171875 6.21875 \nQ 5.515625 13.875 5.515625 27.296875 \nQ 5.515625 40.765625 12.171875 48.375 \nQ 18.84375 56 30.609375 56 \nz\n\" id=\"DejaVuSans-111\"/>\n      <path d=\"M 48.78125 52.59375 \nL 48.78125 44.1875 \nQ 44.96875 46.296875 41.140625 47.34375 \nQ 37.3125 48.390625 33.40625 48.390625 \nQ 24.65625 48.390625 19.8125 42.84375 \nQ 14.984375 37.3125 14.984375 27.296875 \nQ 14.984375 17.28125 19.8125 11.734375 \nQ 24.65625 6.203125 33.40625 6.203125 \nQ 37.3125 6.203125 41.140625 7.25 \nQ 44.96875 8.296875 48.78125 10.40625 \nL 48.78125 2.09375 \nQ 45.015625 0.34375 40.984375 -0.53125 \nQ 36.96875 -1.421875 32.421875 -1.421875 \nQ 20.0625 -1.421875 12.78125 6.34375 \nQ 5.515625 14.109375 5.515625 27.296875 \nQ 5.515625 40.671875 12.859375 48.328125 \nQ 20.21875 56 33.015625 56 \nQ 37.15625 56 41.109375 55.140625 \nQ 45.0625 54.296875 48.78125 52.59375 \nz\n\" id=\"DejaVuSans-99\"/>\n      <path d=\"M 54.890625 33.015625 \nL 54.890625 0 \nL 45.90625 0 \nL 45.90625 32.71875 \nQ 45.90625 40.484375 42.875 44.328125 \nQ 39.84375 48.1875 33.796875 48.1875 \nQ 26.515625 48.1875 22.3125 43.546875 \nQ 18.109375 38.921875 18.109375 30.90625 \nL 18.109375 0 \nL 9.078125 0 \nL 9.078125 75.984375 \nL 18.109375 75.984375 \nL 18.109375 46.1875 \nQ 21.34375 51.125 25.703125 53.5625 \nQ 30.078125 56 35.796875 56 \nQ 45.21875 56 50.046875 50.171875 \nQ 54.890625 44.34375 54.890625 33.015625 \nz\n\" id=\"DejaVuSans-104\"/>\n      <path d=\"M 44.28125 53.078125 \nL 44.28125 44.578125 \nQ 40.484375 46.53125 36.375 47.5 \nQ 32.28125 48.484375 27.875 48.484375 \nQ 21.1875 48.484375 17.84375 46.4375 \nQ 14.5 44.390625 14.5 40.28125 \nQ 14.5 37.15625 16.890625 35.375 \nQ 19.28125 33.59375 26.515625 31.984375 \nL 29.59375 31.296875 \nQ 39.15625 29.25 43.1875 25.515625 \nQ 47.21875 21.78125 47.21875 15.09375 \nQ 47.21875 7.46875 41.1875 3.015625 \nQ 35.15625 -1.421875 24.609375 -1.421875 \nQ 20.21875 -1.421875 15.453125 -0.5625 \nQ 10.6875 0.296875 5.421875 2 \nL 5.421875 11.28125 \nQ 10.40625 8.6875 15.234375 7.390625 \nQ 20.0625 6.109375 24.8125 6.109375 \nQ 31.15625 6.109375 34.5625 8.28125 \nQ 37.984375 10.453125 37.984375 14.40625 \nQ 37.984375 18.0625 35.515625 20.015625 \nQ 33.0625 21.96875 24.703125 23.78125 \nL 21.578125 24.515625 \nQ 13.234375 26.265625 9.515625 29.90625 \nQ 5.8125 33.546875 5.8125 39.890625 \nQ 5.8125 47.609375 11.28125 51.796875 \nQ 16.75 56 26.8125 56 \nQ 31.78125 56 36.171875 55.265625 \nQ 40.578125 54.546875 44.28125 53.078125 \nz\n\" id=\"DejaVuSans-115\"/>\n     </defs>\n     <g transform=\"translate(193.265625 252.916562)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-69\"/>\n      <use x=\"63.183594\" xlink:href=\"#DejaVuSans-112\"/>\n      <use x=\"126.660156\" xlink:href=\"#DejaVuSans-111\"/>\n      <use x=\"187.841797\" xlink:href=\"#DejaVuSans-99\"/>\n      <use x=\"242.822266\" xlink:href=\"#DejaVuSans-104\"/>\n      <use x=\"306.201172\" xlink:href=\"#DejaVuSans-115\"/>\n     </g>\n    </g>\n   </g>\n   <g id=\"matplotlib.axis_2\">\n    <g id=\"ytick_1\">\n     <g id=\"line2d_6\">\n      <defs>\n       <path d=\"M 0 0 \nL -3.5 0 \n\" id=\"m5821419d61\" style=\"stroke:#000000;stroke-width:0.8;\"/>\n      </defs>\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#m5821419d61\" y=\"218.754751\"/>\n      </g>\n     </g>\n     <g id=\"text_7\">\n      <!-- 0.0 -->\n      <defs>\n       <path d=\"M 10.6875 12.40625 \nL 21 12.40625 \nL 21 0 \nL 10.6875 0 \nz\n\" id=\"DejaVuSans-46\"/>\n      </defs>\n      <g transform=\"translate(20.878125 222.55397)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_2\">\n     <g id=\"line2d_7\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#m5821419d61\" y=\"190.469556\"/>\n      </g>\n     </g>\n     <g id=\"text_8\">\n      <!-- 0.1 -->\n      <defs>\n       <path d=\"M 12.40625 8.296875 \nL 28.515625 8.296875 \nL 28.515625 63.921875 \nL 10.984375 60.40625 \nL 10.984375 69.390625 \nL 28.421875 72.90625 \nL 38.28125 72.90625 \nL 38.28125 8.296875 \nL 54.390625 8.296875 \nL 54.390625 0 \nL 12.40625 0 \nz\n\" id=\"DejaVuSans-49\"/>\n      </defs>\n      <g transform=\"translate(20.878125 194.268775)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-49\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_3\">\n     <g id=\"line2d_8\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#m5821419d61\" y=\"162.184361\"/>\n      </g>\n     </g>\n     <g id=\"text_9\">\n      <!-- 0.2 -->\n      <g transform=\"translate(20.878125 165.98358)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-50\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_4\">\n     <g id=\"line2d_9\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#m5821419d61\" y=\"133.899166\"/>\n      </g>\n     </g>\n     <g id=\"text_10\">\n      <!-- 0.3 -->\n      <defs>\n       <path d=\"M 40.578125 39.3125 \nQ 47.65625 37.796875 51.625 33 \nQ 55.609375 28.21875 55.609375 21.1875 \nQ 55.609375 10.40625 48.1875 4.484375 \nQ 40.765625 -1.421875 27.09375 -1.421875 \nQ 22.515625 -1.421875 17.65625 -0.515625 \nQ 12.796875 0.390625 7.625 2.203125 \nL 7.625 11.71875 \nQ 11.71875 9.328125 16.59375 8.109375 \nQ 21.484375 6.890625 26.8125 6.890625 \nQ 36.078125 6.890625 40.9375 10.546875 \nQ 45.796875 14.203125 45.796875 21.1875 \nQ 45.796875 27.640625 41.28125 31.265625 \nQ 36.765625 34.90625 28.71875 34.90625 \nL 20.21875 34.90625 \nL 20.21875 43.015625 \nL 29.109375 43.015625 \nQ 36.375 43.015625 40.234375 45.921875 \nQ 44.09375 48.828125 44.09375 54.296875 \nQ 44.09375 59.90625 40.109375 62.90625 \nQ 36.140625 65.921875 28.71875 65.921875 \nQ 24.65625 65.921875 20.015625 65.03125 \nQ 15.375 64.15625 9.8125 62.3125 \nL 9.8125 71.09375 \nQ 15.4375 72.65625 20.34375 73.4375 \nQ 25.25 74.21875 29.59375 74.21875 \nQ 40.828125 74.21875 47.359375 69.109375 \nQ 53.90625 64.015625 53.90625 55.328125 \nQ 53.90625 49.265625 50.4375 45.09375 \nQ 46.96875 40.921875 40.578125 39.3125 \nz\n\" id=\"DejaVuSans-51\"/>\n      </defs>\n      <g transform=\"translate(20.878125 137.698385)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-51\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_5\">\n     <g id=\"line2d_10\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#m5821419d61\" y=\"105.613971\"/>\n      </g>\n     </g>\n     <g id=\"text_11\">\n      <!-- 0.4 -->\n      <g transform=\"translate(20.878125 109.413189)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-52\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_6\">\n     <g id=\"line2d_11\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#m5821419d61\" y=\"77.328775\"/>\n      </g>\n     </g>\n     <g id=\"text_12\">\n      <!-- 0.5 -->\n      <defs>\n       <path d=\"M 10.796875 72.90625 \nL 49.515625 72.90625 \nL 49.515625 64.59375 \nL 19.828125 64.59375 \nL 19.828125 46.734375 \nQ 21.96875 47.46875 24.109375 47.828125 \nQ 26.265625 48.1875 28.421875 48.1875 \nQ 40.625 48.1875 47.75 41.5 \nQ 54.890625 34.8125 54.890625 23.390625 \nQ 54.890625 11.625 47.5625 5.09375 \nQ 40.234375 -1.421875 26.90625 -1.421875 \nQ 22.3125 -1.421875 17.546875 -0.640625 \nQ 12.796875 0.140625 7.71875 1.703125 \nL 7.71875 11.625 \nQ 12.109375 9.234375 16.796875 8.0625 \nQ 21.484375 6.890625 26.703125 6.890625 \nQ 35.15625 6.890625 40.078125 11.328125 \nQ 45.015625 15.765625 45.015625 23.390625 \nQ 45.015625 31 40.078125 35.4375 \nQ 35.15625 39.890625 26.703125 39.890625 \nQ 22.75 39.890625 18.8125 39.015625 \nQ 14.890625 38.140625 10.796875 36.28125 \nz\n\" id=\"DejaVuSans-53\"/>\n      </defs>\n      <g transform=\"translate(20.878125 81.127994)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-53\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_7\">\n     <g id=\"line2d_12\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#m5821419d61\" y=\"49.04358\"/>\n      </g>\n     </g>\n     <g id=\"text_13\">\n      <!-- 0.6 -->\n      <g transform=\"translate(20.878125 52.842799)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-54\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_8\">\n     <g id=\"line2d_13\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#m5821419d61\" y=\"20.758385\"/>\n      </g>\n     </g>\n     <g id=\"text_14\">\n      <!-- 0.7 -->\n      <defs>\n       <path d=\"M 8.203125 72.90625 \nL 55.078125 72.90625 \nL 55.078125 68.703125 \nL 28.609375 0 \nL 18.3125 0 \nL 43.21875 64.59375 \nL 8.203125 64.59375 \nz\n\" id=\"DejaVuSans-55\"/>\n      </defs>\n      <g transform=\"translate(20.878125 24.557604)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-55\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"text_15\">\n     <!-- Losses -->\n     <defs>\n      <path d=\"M 9.8125 72.90625 \nL 19.671875 72.90625 \nL 19.671875 8.296875 \nL 55.171875 8.296875 \nL 55.171875 0 \nL 9.8125 0 \nz\n\" id=\"DejaVuSans-76\"/>\n      <path d=\"M 56.203125 29.59375 \nL 56.203125 25.203125 \nL 14.890625 25.203125 \nQ 15.484375 15.921875 20.484375 11.0625 \nQ 25.484375 6.203125 34.421875 6.203125 \nQ 39.59375 6.203125 44.453125 7.46875 \nQ 49.3125 8.734375 54.109375 11.28125 \nL 54.109375 2.78125 \nQ 49.265625 0.734375 44.1875 -0.34375 \nQ 39.109375 -1.421875 33.890625 -1.421875 \nQ 20.796875 -1.421875 13.15625 6.1875 \nQ 5.515625 13.8125 5.515625 26.8125 \nQ 5.515625 40.234375 12.765625 48.109375 \nQ 20.015625 56 32.328125 56 \nQ 43.359375 56 49.78125 48.890625 \nQ 56.203125 41.796875 56.203125 29.59375 \nz\nM 47.21875 32.234375 \nQ 47.125 39.59375 43.09375 43.984375 \nQ 39.0625 48.390625 32.421875 48.390625 \nQ 24.90625 48.390625 20.390625 44.140625 \nQ 15.875 39.890625 15.1875 32.171875 \nz\n\" id=\"DejaVuSans-101\"/>\n     </defs>\n     <g transform=\"translate(14.798438 132.655156)rotate(-90)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-76\"/>\n      <use x=\"55.697266\" xlink:href=\"#DejaVuSans-111\"/>\n      <use x=\"116.878906\" xlink:href=\"#DejaVuSans-115\"/>\n      <use x=\"168.978516\" xlink:href=\"#DejaVuSans-115\"/>\n      <use x=\"221.078125\" xlink:href=\"#DejaVuSans-101\"/>\n      <use x=\"282.601562\" xlink:href=\"#DejaVuSans-115\"/>\n     </g>\n    </g>\n   </g>\n   <g id=\"line2d_14\">\n    <path clip-path=\"url(#p00025a4031)\" d=\"M 58.999432 17.083636 \nL 62.45811 51.31484 \nL 65.916787 43.568098 \nL 69.375465 57.54514 \nL 72.834143 111.755092 \nL 76.29282 103.679998 \nL 79.751498 114.939685 \nL 83.210176 92.215866 \nL 86.668853 153.565263 \nL 90.127531 117.713241 \nL 93.586209 125.86362 \nL 97.044886 154.643705 \nL 100.503564 153.010377 \nL 103.962242 157.782522 \nL 107.420919 148.208097 \nL 110.879597 144.683646 \nL 114.338275 184.818728 \nL 117.796952 154.937706 \nL 121.25563 172.247898 \nL 124.714308 136.978486 \nL 128.172986 154.725233 \nL 131.631663 182.275867 \nL 135.090341 183.531218 \nL 138.549019 158.415908 \nL 142.007696 189.145096 \nL 145.466374 173.384859 \nL 148.925052 165.754896 \nL 152.383729 183.046041 \nL 155.842407 172.653035 \nL 159.301085 181.207388 \nL 162.759762 182.895749 \nL 166.21844 185.249818 \nL 169.677118 180.269409 \nL 173.135795 191.242034 \nL 176.594473 186.802479 \nL 180.053151 190.962542 \nL 183.511829 196.838677 \nL 186.970506 161.943416 \nL 190.429184 182.378443 \nL 193.887862 187.097721 \nL 197.346539 201.296039 \nL 200.805217 177.907642 \nL 204.263895 209.393831 \nL 207.722572 198.012918 \nL 211.18125 173.956844 \nL 214.639928 185.523105 \nL 218.098605 192.185897 \nL 221.557283 211.013213 \nL 225.015961 191.120459 \nL 228.474638 189.14293 \nL 231.933316 185.243169 \nL 235.391994 202.332551 \nL 238.850671 202.862191 \nL 242.309349 195.944916 \nL 245.768027 207.286426 \nL 249.226705 175.161643 \nL 252.685382 202.606529 \nL 256.14406 209.385186 \nL 259.602738 200.793003 \nL 263.061415 207.705467 \nL 266.520093 204.414906 \nL 269.978771 206.372552 \nL 273.437448 207.521099 \nL 276.896126 187.799851 \nL 280.354804 207.875164 \nL 283.813481 201.399574 \nL 287.272159 214.119553 \nL 290.730837 200.797057 \nL 294.189514 211.523556 \nL 297.648192 202.806194 \nL 301.10687 203.726574 \nL 304.565548 199.926262 \nL 308.024225 190.749055 \nL 311.482903 206.08404 \nL 314.941581 203.745109 \nL 318.400258 187.327068 \nL 321.858936 212.590518 \nL 325.317614 190.255706 \nL 328.776291 210.805195 \nL 332.234969 203.447296 \nL 335.693647 206.20507 \nL 339.152324 198.160732 \nL 342.611002 206.013598 \nL 346.06968 214.756364 \nL 349.528357 195.159256 \nL 352.987035 199.784037 \nL 356.445713 199.607902 \nL 359.90439 206.008669 \nL 363.363068 213.471013 \n\" style=\"fill:none;stroke:#ff0000;stroke-linecap:square;stroke-width:1.5;\"/>\n   </g>\n   <g id=\"line2d_15\">\n    <path clip-path=\"url(#p00025a4031)\" d=\"M 58.999432 56.705615 \nL 62.45811 73.639524 \nL 65.916787 94.317199 \nL 69.375465 85.357675 \nL 72.834143 86.71151 \nL 76.29282 93.002933 \nL 79.751498 86.640718 \nL 83.210176 66.899298 \nL 86.668853 101.589631 \nL 90.127531 97.646167 \nL 93.586209 108.978148 \nL 97.044886 106.421394 \nL 100.503564 94.744506 \nL 103.962242 101.712695 \nL 107.420919 99.775007 \nL 110.879597 95.151995 \nL 114.338275 81.767464 \nL 117.796952 104.46645 \nL 121.25563 99.053969 \nL 124.714308 67.142696 \nL 128.172986 105.402781 \nL 131.631663 99.481596 \nL 135.090341 93.873901 \nL 138.549019 119.421222 \nL 142.007696 110.497675 \nL 145.466374 103.59297 \nL 148.925052 97.480643 \nL 152.383729 126.30106 \nL 155.842407 103.629656 \nL 159.301085 116.448103 \nL 162.759762 83.495946 \nL 166.21844 109.79584 \nL 169.677118 95.70799 \nL 173.135795 103.453822 \nL 176.594473 131.638466 \nL 180.053151 92.258275 \nL 183.511829 67.400727 \nL 186.970506 80.297932 \nL 190.429184 92.520833 \nL 193.887862 118.318565 \nL 197.346539 80.934227 \nL 200.805217 76.321787 \nL 204.263895 104.245534 \nL 207.722572 72.951598 \nL 211.18125 44.006625 \nL 214.639928 84.644831 \nL 218.098605 94.707011 \nL 221.557283 74.318279 \nL 225.015961 111.307393 \nL 228.474638 66.380116 \nL 231.933316 69.163214 \nL 235.391994 74.75684 \nL 238.850671 59.013011 \nL 242.309349 50.885114 \nL 245.768027 55.841914 \nL 249.226705 80.05091 \nL 252.685382 72.54453 \nL 256.14406 88.517308 \nL 259.602738 70.674144 \nL 263.061415 59.510849 \nL 266.520093 85.220828 \nL 269.978771 62.598982 \nL 273.437448 54.639121 \nL 276.896126 55.621377 \nL 280.354804 77.27904 \nL 283.813481 67.446955 \nL 287.272159 60.972971 \nL 290.730837 62.173403 \nL 294.189514 81.022056 \nL 297.648192 62.336028 \nL 301.10687 66.630207 \nL 304.565548 81.289276 \nL 308.024225 83.938089 \nL 311.482903 58.418569 \nL 314.941581 64.180417 \nL 318.400258 101.723334 \nL 321.858936 90.137562 \nL 325.317614 78.813025 \nL 328.776291 80.227208 \nL 332.234969 40.432708 \nL 335.693647 76.996664 \nL 339.152324 55.999717 \nL 342.611002 78.632218 \nL 346.06968 108.841393 \nL 349.528357 104.007531 \nL 352.987035 68.955845 \nL 356.445713 79.888513 \nL 359.90439 58.549128 \nL 363.363068 51.192728 \n\" style=\"fill:none;stroke:#0000ff;stroke-linecap:square;stroke-width:1.5;\"/>\n   </g>\n   <g id=\"patch_3\">\n    <path d=\"M 43.78125 224.64 \nL 43.78125 7.2 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_4\">\n    <path d=\"M 378.58125 224.64 \nL 378.58125 7.2 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_5\">\n    <path d=\"M 43.78125 224.64 \nL 378.58125 224.64 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_6\">\n    <path d=\"M 43.78125 7.2 \nL 378.58125 7.2 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"legend_1\">\n    <g id=\"patch_7\">\n     <path d=\"M 50.78125 219.64 \nL 108.221875 219.64 \nQ 110.221875 219.64 110.221875 217.64 \nL 110.221875 189.28375 \nQ 110.221875 187.28375 108.221875 187.28375 \nL 50.78125 187.28375 \nQ 48.78125 187.28375 48.78125 189.28375 \nL 48.78125 217.64 \nQ 48.78125 219.64 50.78125 219.64 \nz\n\" style=\"fill:#ffffff;opacity:0.8;stroke:#cccccc;stroke-linejoin:miter;\"/>\n    </g>\n    <g id=\"line2d_16\">\n     <path d=\"M 52.78125 195.382187 \nL 72.78125 195.382187 \n\" style=\"fill:none;stroke:#ff0000;stroke-linecap:square;stroke-width:1.5;\"/>\n    </g>\n    <g id=\"line2d_17\"/>\n    <g id=\"text_16\">\n     <!-- Train -->\n     <defs>\n      <path d=\"M -0.296875 72.90625 \nL 61.375 72.90625 \nL 61.375 64.59375 \nL 35.5 64.59375 \nL 35.5 0 \nL 25.59375 0 \nL 25.59375 64.59375 \nL -0.296875 64.59375 \nz\n\" id=\"DejaVuSans-84\"/>\n      <path d=\"M 41.109375 46.296875 \nQ 39.59375 47.171875 37.8125 47.578125 \nQ 36.03125 48 33.890625 48 \nQ 26.265625 48 22.1875 43.046875 \nQ 18.109375 38.09375 18.109375 28.8125 \nL 18.109375 0 \nL 9.078125 0 \nL 9.078125 54.6875 \nL 18.109375 54.6875 \nL 18.109375 46.1875 \nQ 20.953125 51.171875 25.484375 53.578125 \nQ 30.03125 56 36.53125 56 \nQ 37.453125 56 38.578125 55.875 \nQ 39.703125 55.765625 41.0625 55.515625 \nz\n\" id=\"DejaVuSans-114\"/>\n      <path d=\"M 34.28125 27.484375 \nQ 23.390625 27.484375 19.1875 25 \nQ 14.984375 22.515625 14.984375 16.5 \nQ 14.984375 11.71875 18.140625 8.90625 \nQ 21.296875 6.109375 26.703125 6.109375 \nQ 34.1875 6.109375 38.703125 11.40625 \nQ 43.21875 16.703125 43.21875 25.484375 \nL 43.21875 27.484375 \nz\nM 52.203125 31.203125 \nL 52.203125 0 \nL 43.21875 0 \nL 43.21875 8.296875 \nQ 40.140625 3.328125 35.546875 0.953125 \nQ 30.953125 -1.421875 24.3125 -1.421875 \nQ 15.921875 -1.421875 10.953125 3.296875 \nQ 6 8.015625 6 15.921875 \nQ 6 25.140625 12.171875 29.828125 \nQ 18.359375 34.515625 30.609375 34.515625 \nL 43.21875 34.515625 \nL 43.21875 35.40625 \nQ 43.21875 41.609375 39.140625 45 \nQ 35.0625 48.390625 27.6875 48.390625 \nQ 23 48.390625 18.546875 47.265625 \nQ 14.109375 46.140625 10.015625 43.890625 \nL 10.015625 52.203125 \nQ 14.9375 54.109375 19.578125 55.046875 \nQ 24.21875 56 28.609375 56 \nQ 40.484375 56 46.34375 49.84375 \nQ 52.203125 43.703125 52.203125 31.203125 \nz\n\" id=\"DejaVuSans-97\"/>\n      <path d=\"M 9.421875 54.6875 \nL 18.40625 54.6875 \nL 18.40625 0 \nL 9.421875 0 \nz\nM 9.421875 75.984375 \nL 18.40625 75.984375 \nL 18.40625 64.59375 \nL 9.421875 64.59375 \nz\n\" id=\"DejaVuSans-105\"/>\n      <path d=\"M 54.890625 33.015625 \nL 54.890625 0 \nL 45.90625 0 \nL 45.90625 32.71875 \nQ 45.90625 40.484375 42.875 44.328125 \nQ 39.84375 48.1875 33.796875 48.1875 \nQ 26.515625 48.1875 22.3125 43.546875 \nQ 18.109375 38.921875 18.109375 30.90625 \nL 18.109375 0 \nL 9.078125 0 \nL 9.078125 54.6875 \nL 18.109375 54.6875 \nL 18.109375 46.1875 \nQ 21.34375 51.125 25.703125 53.5625 \nQ 30.078125 56 35.796875 56 \nQ 45.21875 56 50.046875 50.171875 \nQ 54.890625 44.34375 54.890625 33.015625 \nz\n\" id=\"DejaVuSans-110\"/>\n     </defs>\n     <g transform=\"translate(80.78125 198.882187)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-84\"/>\n      <use x=\"60.865234\" xlink:href=\"#DejaVuSans-114\"/>\n      <use x=\"101.978516\" xlink:href=\"#DejaVuSans-97\"/>\n      <use x=\"163.257812\" xlink:href=\"#DejaVuSans-105\"/>\n      <use x=\"191.041016\" xlink:href=\"#DejaVuSans-110\"/>\n     </g>\n    </g>\n    <g id=\"line2d_18\">\n     <path d=\"M 52.78125 210.060312 \nL 72.78125 210.060312 \n\" style=\"fill:none;stroke:#0000ff;stroke-linecap:square;stroke-width:1.5;\"/>\n    </g>\n    <g id=\"line2d_19\"/>\n    <g id=\"text_17\">\n     <!-- Val -->\n     <defs>\n      <path d=\"M 28.609375 0 \nL 0.78125 72.90625 \nL 11.078125 72.90625 \nL 34.1875 11.53125 \nL 57.328125 72.90625 \nL 67.578125 72.90625 \nL 39.796875 0 \nz\n\" id=\"DejaVuSans-86\"/>\n      <path d=\"M 9.421875 75.984375 \nL 18.40625 75.984375 \nL 18.40625 0 \nL 9.421875 0 \nz\n\" id=\"DejaVuSans-108\"/>\n     </defs>\n     <g transform=\"translate(80.78125 213.560312)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-86\"/>\n      <use x=\"68.298828\" xlink:href=\"#DejaVuSans-97\"/>\n      <use x=\"129.578125\" xlink:href=\"#DejaVuSans-108\"/>\n     </g>\n    </g>\n   </g>\n  </g>\n </g>\n <defs>\n  <clipPath id=\"p00025a4031\">\n   <rect height=\"217.44\" width=\"334.8\" x=\"43.78125\" y=\"7.2\"/>\n  </clipPath>\n </defs>\n</svg>\n"
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bdYQCyJEj00H",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import files\n",
        "files.download('./adamw_resnet18_9277.pth.tar')\n",
        "files.download('./adamw_resnet18_9277_acc.jpg')\n",
        "files.download('./adamw_resnet18_9277_loss.jpg')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n0Z4WML9lYr1",
        "colab_type": "text"
      },
      "source": [
        "# Lookahead with SGD"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OWY9RM02ljjd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "load_checkpoint(model, filename='before_sgd_resnet18.pth.tar')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t0Hm9c06Bazq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "reset()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eeygLVC1Bcs3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "preds = []\n",
        "train_stats = AvgStats()\n",
        "test_stats = AvgStats()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4QHsZryCBfLK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "optims = torch.optim.SGD(model.parameters(), lr=1e-2, momentum=0.9, weight_decay=1e-4)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OqPU5Im6yHO1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sched_la = torch.optim.lr_scheduler.StepLR(optims, step_size=30, gamma=0.2)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eVTRIpgiF3Wq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "la_optim = Lookahead(optims, alpha=0.5, k=10)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f1nY9hntBkqz",
        "colab_type": "code",
        "outputId": "a735d87c-0256-40a7-edec-720b51e26b51",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "fit(model, la_optim, sched_la)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch\tTrn_loss\tVal_loss\tTrn_acc\t\tVal_acc\n",
            "1\t0.80412821\t0.67540877\t65.55999756\t67.44499969\n",
            "2\t0.66217067\t0.58627998\t71.49272919\t72.21833801\n",
            "3\t0.57036327\t0.54968659\t74.33705902\t74.73944092\n",
            "4\t0.52546535\t0.52136333\t76.20521545\t76.47208405\n",
            "5\t0.48797978\t0.52578297\t77.59517670\t77.74566650\n",
            "6\t0.45466794\t0.52268913\t78.67485809\t78.78138733\n",
            "7\t0.44103390\t0.51987436\t79.53073120\t79.59832764\n",
            "8\t0.43288209\t0.54712344\t80.27595520\t80.31375122\n",
            "9\t0.39000792\t0.51206695\t80.90849304\t80.95500183\n",
            "10\t0.38323111\t0.46767378\t81.46084595\t81.52149963\n",
            "11\t0.35768960\t0.46104806\t82.00984955\t82.06697083\n",
            "12\t0.34646576\t0.46351539\t82.48830414\t82.52639008\n",
            "13\t0.31617146\t0.46825647\t82.93545532\t82.95871735\n",
            "14\t0.30741357\t0.47725594\t83.32891846\t83.34714508\n",
            "15\t0.29908946\t0.51705303\t83.69853973\t83.70611572\n",
            "16\t0.29979351\t0.50682819\t84.03672791\t84.03812408\n",
            "17\t0.26345370\t0.53779406\t84.35950470\t84.35304260\n",
            "18\t0.26686016\t0.52753349\t84.64569855\t84.64037323\n",
            "19\t0.25963379\t0.49933115\t84.91690063\t84.91438293\n",
            "20\t0.24406346\t0.46621495\t85.18235779\t85.18791962\n",
            "21\t0.24605640\t0.47870038\t85.44288635\t85.44761658\n",
            "22\t0.22933037\t0.50044199\t85.69519806\t85.69530487\n",
            "23\t0.23551696\t0.49359190\t85.92956543\t85.92500305\n",
            "24\t0.23585711\t0.52966879\t86.14810944\t86.13971710\n",
            "25\t0.21761365\t0.49815269\t86.35328674\t86.34759521\n",
            "26\t0.21179014\t0.53948659\t86.55838013\t86.54878235\n",
            "27\t0.20755552\t0.51999784\t86.75137329\t86.74227905\n",
            "28\t0.19100767\t0.53901067\t86.94616699\t86.93297577\n",
            "29\t0.19442678\t0.54001485\t87.11370087\t87.09976959\n",
            "30\t0.18433813\t0.49711203\t87.28150940\t87.27478027\n",
            "31\t0.18271810\t0.49971102\t87.45205688\t87.44414520\n",
            "32\t0.11150168\t0.48057817\t87.65753937\t87.65380096\n",
            "33\t0.10004535\t0.51005509\t87.87847900\t87.87373352\n",
            "34\t0.09094289\t0.51303850\t88.09093475\t88.08725739\n",
            "35\t0.08855261\t0.52339033\t88.29808807\t88.29152679\n",
            "36\t0.08384666\t0.52753442\t88.49758148\t88.49102020\n",
            "37\t0.08035230\t0.55532337\t88.68872833\t88.68184662\n",
            "38\t0.08057527\t0.54815901\t88.87017059\t88.86311340\n",
            "39\t0.07443073\t0.55170693\t89.04639435\t89.03888702\n",
            "40\t0.07610609\t0.55354919\t89.21665192\t89.20867157\n",
            "41\t0.06954104\t0.57187951\t89.38162994\t89.37174988\n",
            "42\t0.06392259\t0.58057678\t89.53936005\t89.52944183\n",
            "43\t0.06306377\t0.58074945\t89.68738556\t89.67752075\n",
            "44\t0.06489520\t0.58621885\t89.83003235\t89.82007599\n",
            "45\t0.06309309\t0.59237101\t89.97041321\t89.96077728\n",
            "46\t0.06328040\t0.59522055\t90.10250854\t90.09152222\n",
            "47\t0.05882112\t0.59528857\t90.23135376\t90.22035217\n",
            "48\t0.05964288\t0.59417139\t90.35522461\t90.34399414\n",
            "49\t0.06110721\t0.59049225\t90.47299957\t90.46149445\n",
            "50\t0.05143825\t0.60327020\t90.59090424\t90.57903290\n",
            "51\t0.05354226\t0.60306064\t90.70390320\t90.69284058\n",
            "52\t0.04812535\t0.62138870\t90.81552887\t90.80252838\n",
            "53\t0.05467139\t0.63696844\t90.91958618\t90.90748596\n",
            "54\t0.05082451\t0.61953466\t91.02101898\t91.00833893\n",
            "55\t0.05256863\t0.62551275\t91.11945343\t91.10733032\n",
            "56\t0.04988101\t0.63677380\t91.21441650\t91.20184326\n",
            "57\t0.05231185\t0.64034698\t91.30677795\t91.29418182\n",
            "58\t0.05042453\t0.64983543\t91.39613342\t91.38301849\n",
            "59\t0.04404119\t0.63960657\t91.48181152\t91.47025299\n",
            "60\t0.04589654\t0.65390717\t91.56880188\t91.55625153\n",
            "61\t0.04143657\t0.64633957\t91.65120697\t91.63874054\n",
            "62\t0.04036147\t0.64688914\t91.73285675\t91.72075653\n",
            "63\t0.04010691\t0.65050345\t91.81507111\t91.80296326\n",
            "64\t0.03624383\t0.64728400\t91.89315796\t91.88190460\n",
            "65\t0.03755778\t0.64531217\t91.97084808\t91.95899963\n",
            "66\t0.03619455\t0.65630949\t92.04547119\t92.03426361\n",
            "67\t0.03535636\t0.64257300\t92.11853027\t92.10681152\n",
            "68\t0.03872749\t0.65361976\t92.18923950\t92.17730713\n",
            "69\t0.03296895\t0.65504545\t92.25817871\t92.24620819\n",
            "70\t0.03415921\t0.64708675\t92.32486725\t92.31345367\n",
            "71\t0.03340222\t0.65284292\t92.39089203\t92.37896729\n",
            "72\t0.03095217\t0.65443883\t92.45480347\t92.44323730\n",
            "73\t0.03101914\t0.65985176\t92.51807404\t92.50611877\n",
            "74\t0.02971222\t0.66103503\t92.57879639\t92.56673431\n",
            "75\t0.02889135\t0.65834207\t92.63822174\t92.62639618\n",
            "76\t0.03377232\t0.65812392\t92.69593048\t92.68372345\n",
            "77\t0.03025321\t0.66263856\t92.75095367\t92.73917389\n",
            "78\t0.03474191\t0.65134035\t92.80541992\t92.79318237\n",
            "79\t0.03108282\t0.66148431\t92.85810089\t92.84664917\n",
            "80\t0.02860017\t0.65932600\t92.91125488\t92.89920807\n",
            "81\t0.03487168\t0.66459786\t92.96166992\t92.94941711\n",
            "82\t0.03214167\t0.66780273\t93.01110077\t92.99926758\n",
            "83\t0.03210842\t0.66008967\t93.05912018\t93.04723358\n",
            "84\t0.03174598\t0.67214058\t93.10560608\t93.09384918\n",
            "85\t0.02925161\t0.66452971\t93.15216827\t93.14088440\n",
            "86\t0.02940951\t0.67107120\t93.19792175\t93.18641663\n",
            "87\t0.02793779\t0.66929708\t93.24238586\t93.23089600\n",
            "88\t0.02731009\t0.66999438\t93.28641510\t93.27483368\n",
            "89\t0.02877710\t0.67171198\t93.32862854\t93.31737518\n",
            "90\t0.02957945\t0.67844489\t93.36963654\t93.35842896\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YFEcAgEhPzGv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "save_checkpoint(model, True, 'lookahead_sgd_resnet18_9336.pth.tar')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4fah7QvhBmOG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ep_lookahead_losses = []\n",
        "for i in range(0, len(train_stats.losses), len(train_loader)):\n",
        "    if i != 0 :\n",
        "        ep_lookahead_losses.append(train_stats.losses[i])\n",
        "        \n",
        "ep_lookahead_lossesv = []\n",
        "for i in range(0, len(test_stats.losses), len(test_loader)):\n",
        "    if(i != 0):\n",
        "        ep_lookahead_lossesv.append(test_stats.losses[i])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dsZWOzOv51tO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ep_lookahead_accs = []\n",
        "for i in range(0, len(train_stats.losses), len(train_loader)):\n",
        "    if i != 0 :\n",
        "        ep_lookahead_accs.append(train_stats.precs[i])\n",
        "        \n",
        "ep_lookahead_accsv = []\n",
        "for i in range(0, len(test_stats.losses), len(test_loader)):\n",
        "    if(i != 0):\n",
        "        ep_lookahead_accsv.append(test_stats.precs[i])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "llTa7tns6uz_",
        "colab_type": "code",
        "outputId": "953efd88-bdd2-48ca-8aad-974384694e00",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 371
        }
      },
      "source": [
        "plt.xlabel(\"Epochs\")\n",
        "plt.ylabel(\"Accuracy\")\n",
        "plt.plot(ep_lookahead_accs, 'r', label='Train')\n",
        "plt.plot(ep_lookahead_accsv, 'b', label='Val')\n",
        "plt.legend()\n",
        "plt.savefig('lookahead_sgd_resnet18_9336_acc.jpg')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/svg+xml": "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n  \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<!-- Created with matplotlib (https://matplotlib.org/) -->\n<svg height=\"262.19625pt\" version=\"1.1\" viewBox=\"0 0 382.603125 262.19625\" width=\"382.603125pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n <defs>\n  <style type=\"text/css\">\n*{stroke-linecap:butt;stroke-linejoin:round;}\n  </style>\n </defs>\n <g id=\"figure_1\">\n  <g id=\"patch_1\">\n   <path d=\"M 0 262.19625 \nL 382.603125 262.19625 \nL 382.603125 0 \nL 0 0 \nz\n\" style=\"fill:none;\"/>\n  </g>\n  <g id=\"axes_1\">\n   <g id=\"patch_2\">\n    <path d=\"M 40.603125 224.64 \nL 375.403125 224.64 \nL 375.403125 7.2 \nL 40.603125 7.2 \nz\n\" style=\"fill:#ffffff;\"/>\n   </g>\n   <g id=\"matplotlib.axis_1\">\n    <g id=\"xtick_1\">\n     <g id=\"line2d_1\">\n      <defs>\n       <path d=\"M 0 0 \nL 0 3.5 \n\" id=\"m06b1f7d893\" style=\"stroke:#000000;stroke-width:0.8;\"/>\n      </defs>\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"55.821307\" xlink:href=\"#m06b1f7d893\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_1\">\n      <!-- 0 -->\n      <defs>\n       <path d=\"M 31.78125 66.40625 \nQ 24.171875 66.40625 20.328125 58.90625 \nQ 16.5 51.421875 16.5 36.375 \nQ 16.5 21.390625 20.328125 13.890625 \nQ 24.171875 6.390625 31.78125 6.390625 \nQ 39.453125 6.390625 43.28125 13.890625 \nQ 47.125 21.390625 47.125 36.375 \nQ 47.125 51.421875 43.28125 58.90625 \nQ 39.453125 66.40625 31.78125 66.40625 \nz\nM 31.78125 74.21875 \nQ 44.046875 74.21875 50.515625 64.515625 \nQ 56.984375 54.828125 56.984375 36.375 \nQ 56.984375 17.96875 50.515625 8.265625 \nQ 44.046875 -1.421875 31.78125 -1.421875 \nQ 19.53125 -1.421875 13.0625 8.265625 \nQ 6.59375 17.96875 6.59375 36.375 \nQ 6.59375 54.828125 13.0625 64.515625 \nQ 19.53125 74.21875 31.78125 74.21875 \nz\n\" id=\"DejaVuSans-48\"/>\n      </defs>\n      <g transform=\"translate(52.640057 239.238438)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_2\">\n     <g id=\"line2d_2\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"124.994861\" xlink:href=\"#m06b1f7d893\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_2\">\n      <!-- 20 -->\n      <defs>\n       <path d=\"M 19.1875 8.296875 \nL 53.609375 8.296875 \nL 53.609375 0 \nL 7.328125 0 \nL 7.328125 8.296875 \nQ 12.9375 14.109375 22.625 23.890625 \nQ 32.328125 33.6875 34.8125 36.53125 \nQ 39.546875 41.84375 41.421875 45.53125 \nQ 43.3125 49.21875 43.3125 52.78125 \nQ 43.3125 58.59375 39.234375 62.25 \nQ 35.15625 65.921875 28.609375 65.921875 \nQ 23.96875 65.921875 18.8125 64.3125 \nQ 13.671875 62.703125 7.8125 59.421875 \nL 7.8125 69.390625 \nQ 13.765625 71.78125 18.9375 73 \nQ 24.125 74.21875 28.421875 74.21875 \nQ 39.75 74.21875 46.484375 68.546875 \nQ 53.21875 62.890625 53.21875 53.421875 \nQ 53.21875 48.921875 51.53125 44.890625 \nQ 49.859375 40.875 45.40625 35.40625 \nQ 44.1875 33.984375 37.640625 27.21875 \nQ 31.109375 20.453125 19.1875 8.296875 \nz\n\" id=\"DejaVuSans-50\"/>\n      </defs>\n      <g transform=\"translate(118.632361 239.238438)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-50\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_3\">\n     <g id=\"line2d_3\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"194.168414\" xlink:href=\"#m06b1f7d893\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_3\">\n      <!-- 40 -->\n      <defs>\n       <path d=\"M 37.796875 64.3125 \nL 12.890625 25.390625 \nL 37.796875 25.390625 \nz\nM 35.203125 72.90625 \nL 47.609375 72.90625 \nL 47.609375 25.390625 \nL 58.015625 25.390625 \nL 58.015625 17.1875 \nL 47.609375 17.1875 \nL 47.609375 0 \nL 37.796875 0 \nL 37.796875 17.1875 \nL 4.890625 17.1875 \nL 4.890625 26.703125 \nz\n\" id=\"DejaVuSans-52\"/>\n      </defs>\n      <g transform=\"translate(187.805914 239.238438)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-52\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_4\">\n     <g id=\"line2d_4\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"263.341968\" xlink:href=\"#m06b1f7d893\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_4\">\n      <!-- 60 -->\n      <defs>\n       <path d=\"M 33.015625 40.375 \nQ 26.375 40.375 22.484375 35.828125 \nQ 18.609375 31.296875 18.609375 23.390625 \nQ 18.609375 15.53125 22.484375 10.953125 \nQ 26.375 6.390625 33.015625 6.390625 \nQ 39.65625 6.390625 43.53125 10.953125 \nQ 47.40625 15.53125 47.40625 23.390625 \nQ 47.40625 31.296875 43.53125 35.828125 \nQ 39.65625 40.375 33.015625 40.375 \nz\nM 52.59375 71.296875 \nL 52.59375 62.3125 \nQ 48.875 64.0625 45.09375 64.984375 \nQ 41.3125 65.921875 37.59375 65.921875 \nQ 27.828125 65.921875 22.671875 59.328125 \nQ 17.53125 52.734375 16.796875 39.40625 \nQ 19.671875 43.65625 24.015625 45.921875 \nQ 28.375 48.1875 33.59375 48.1875 \nQ 44.578125 48.1875 50.953125 41.515625 \nQ 57.328125 34.859375 57.328125 23.390625 \nQ 57.328125 12.15625 50.6875 5.359375 \nQ 44.046875 -1.421875 33.015625 -1.421875 \nQ 20.359375 -1.421875 13.671875 8.265625 \nQ 6.984375 17.96875 6.984375 36.375 \nQ 6.984375 53.65625 15.1875 63.9375 \nQ 23.390625 74.21875 37.203125 74.21875 \nQ 40.921875 74.21875 44.703125 73.484375 \nQ 48.484375 72.75 52.59375 71.296875 \nz\n\" id=\"DejaVuSans-54\"/>\n      </defs>\n      <g transform=\"translate(256.979468 239.238438)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-54\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_5\">\n     <g id=\"line2d_5\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"332.515522\" xlink:href=\"#m06b1f7d893\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_5\">\n      <!-- 80 -->\n      <defs>\n       <path d=\"M 31.78125 34.625 \nQ 24.75 34.625 20.71875 30.859375 \nQ 16.703125 27.09375 16.703125 20.515625 \nQ 16.703125 13.921875 20.71875 10.15625 \nQ 24.75 6.390625 31.78125 6.390625 \nQ 38.8125 6.390625 42.859375 10.171875 \nQ 46.921875 13.96875 46.921875 20.515625 \nQ 46.921875 27.09375 42.890625 30.859375 \nQ 38.875 34.625 31.78125 34.625 \nz\nM 21.921875 38.8125 \nQ 15.578125 40.375 12.03125 44.71875 \nQ 8.5 49.078125 8.5 55.328125 \nQ 8.5 64.0625 14.71875 69.140625 \nQ 20.953125 74.21875 31.78125 74.21875 \nQ 42.671875 74.21875 48.875 69.140625 \nQ 55.078125 64.0625 55.078125 55.328125 \nQ 55.078125 49.078125 51.53125 44.71875 \nQ 48 40.375 41.703125 38.8125 \nQ 48.828125 37.15625 52.796875 32.3125 \nQ 56.78125 27.484375 56.78125 20.515625 \nQ 56.78125 9.90625 50.3125 4.234375 \nQ 43.84375 -1.421875 31.78125 -1.421875 \nQ 19.734375 -1.421875 13.25 4.234375 \nQ 6.78125 9.90625 6.78125 20.515625 \nQ 6.78125 27.484375 10.78125 32.3125 \nQ 14.796875 37.15625 21.921875 38.8125 \nz\nM 18.3125 54.390625 \nQ 18.3125 48.734375 21.84375 45.5625 \nQ 25.390625 42.390625 31.78125 42.390625 \nQ 38.140625 42.390625 41.71875 45.5625 \nQ 45.3125 48.734375 45.3125 54.390625 \nQ 45.3125 60.0625 41.71875 63.234375 \nQ 38.140625 66.40625 31.78125 66.40625 \nQ 25.390625 66.40625 21.84375 63.234375 \nQ 18.3125 60.0625 18.3125 54.390625 \nz\n\" id=\"DejaVuSans-56\"/>\n      </defs>\n      <g transform=\"translate(326.153022 239.238438)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-56\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"text_6\">\n     <!-- Epochs -->\n     <defs>\n      <path d=\"M 9.8125 72.90625 \nL 55.90625 72.90625 \nL 55.90625 64.59375 \nL 19.671875 64.59375 \nL 19.671875 43.015625 \nL 54.390625 43.015625 \nL 54.390625 34.71875 \nL 19.671875 34.71875 \nL 19.671875 8.296875 \nL 56.78125 8.296875 \nL 56.78125 0 \nL 9.8125 0 \nz\n\" id=\"DejaVuSans-69\"/>\n      <path d=\"M 18.109375 8.203125 \nL 18.109375 -20.796875 \nL 9.078125 -20.796875 \nL 9.078125 54.6875 \nL 18.109375 54.6875 \nL 18.109375 46.390625 \nQ 20.953125 51.265625 25.265625 53.625 \nQ 29.59375 56 35.59375 56 \nQ 45.5625 56 51.78125 48.09375 \nQ 58.015625 40.1875 58.015625 27.296875 \nQ 58.015625 14.40625 51.78125 6.484375 \nQ 45.5625 -1.421875 35.59375 -1.421875 \nQ 29.59375 -1.421875 25.265625 0.953125 \nQ 20.953125 3.328125 18.109375 8.203125 \nz\nM 48.6875 27.296875 \nQ 48.6875 37.203125 44.609375 42.84375 \nQ 40.53125 48.484375 33.40625 48.484375 \nQ 26.265625 48.484375 22.1875 42.84375 \nQ 18.109375 37.203125 18.109375 27.296875 \nQ 18.109375 17.390625 22.1875 11.75 \nQ 26.265625 6.109375 33.40625 6.109375 \nQ 40.53125 6.109375 44.609375 11.75 \nQ 48.6875 17.390625 48.6875 27.296875 \nz\n\" id=\"DejaVuSans-112\"/>\n      <path d=\"M 30.609375 48.390625 \nQ 23.390625 48.390625 19.1875 42.75 \nQ 14.984375 37.109375 14.984375 27.296875 \nQ 14.984375 17.484375 19.15625 11.84375 \nQ 23.34375 6.203125 30.609375 6.203125 \nQ 37.796875 6.203125 41.984375 11.859375 \nQ 46.1875 17.53125 46.1875 27.296875 \nQ 46.1875 37.015625 41.984375 42.703125 \nQ 37.796875 48.390625 30.609375 48.390625 \nz\nM 30.609375 56 \nQ 42.328125 56 49.015625 48.375 \nQ 55.71875 40.765625 55.71875 27.296875 \nQ 55.71875 13.875 49.015625 6.21875 \nQ 42.328125 -1.421875 30.609375 -1.421875 \nQ 18.84375 -1.421875 12.171875 6.21875 \nQ 5.515625 13.875 5.515625 27.296875 \nQ 5.515625 40.765625 12.171875 48.375 \nQ 18.84375 56 30.609375 56 \nz\n\" id=\"DejaVuSans-111\"/>\n      <path d=\"M 48.78125 52.59375 \nL 48.78125 44.1875 \nQ 44.96875 46.296875 41.140625 47.34375 \nQ 37.3125 48.390625 33.40625 48.390625 \nQ 24.65625 48.390625 19.8125 42.84375 \nQ 14.984375 37.3125 14.984375 27.296875 \nQ 14.984375 17.28125 19.8125 11.734375 \nQ 24.65625 6.203125 33.40625 6.203125 \nQ 37.3125 6.203125 41.140625 7.25 \nQ 44.96875 8.296875 48.78125 10.40625 \nL 48.78125 2.09375 \nQ 45.015625 0.34375 40.984375 -0.53125 \nQ 36.96875 -1.421875 32.421875 -1.421875 \nQ 20.0625 -1.421875 12.78125 6.34375 \nQ 5.515625 14.109375 5.515625 27.296875 \nQ 5.515625 40.671875 12.859375 48.328125 \nQ 20.21875 56 33.015625 56 \nQ 37.15625 56 41.109375 55.140625 \nQ 45.0625 54.296875 48.78125 52.59375 \nz\n\" id=\"DejaVuSans-99\"/>\n      <path d=\"M 54.890625 33.015625 \nL 54.890625 0 \nL 45.90625 0 \nL 45.90625 32.71875 \nQ 45.90625 40.484375 42.875 44.328125 \nQ 39.84375 48.1875 33.796875 48.1875 \nQ 26.515625 48.1875 22.3125 43.546875 \nQ 18.109375 38.921875 18.109375 30.90625 \nL 18.109375 0 \nL 9.078125 0 \nL 9.078125 75.984375 \nL 18.109375 75.984375 \nL 18.109375 46.1875 \nQ 21.34375 51.125 25.703125 53.5625 \nQ 30.078125 56 35.796875 56 \nQ 45.21875 56 50.046875 50.171875 \nQ 54.890625 44.34375 54.890625 33.015625 \nz\n\" id=\"DejaVuSans-104\"/>\n      <path d=\"M 44.28125 53.078125 \nL 44.28125 44.578125 \nQ 40.484375 46.53125 36.375 47.5 \nQ 32.28125 48.484375 27.875 48.484375 \nQ 21.1875 48.484375 17.84375 46.4375 \nQ 14.5 44.390625 14.5 40.28125 \nQ 14.5 37.15625 16.890625 35.375 \nQ 19.28125 33.59375 26.515625 31.984375 \nL 29.59375 31.296875 \nQ 39.15625 29.25 43.1875 25.515625 \nQ 47.21875 21.78125 47.21875 15.09375 \nQ 47.21875 7.46875 41.1875 3.015625 \nQ 35.15625 -1.421875 24.609375 -1.421875 \nQ 20.21875 -1.421875 15.453125 -0.5625 \nQ 10.6875 0.296875 5.421875 2 \nL 5.421875 11.28125 \nQ 10.40625 8.6875 15.234375 7.390625 \nQ 20.0625 6.109375 24.8125 6.109375 \nQ 31.15625 6.109375 34.5625 8.28125 \nQ 37.984375 10.453125 37.984375 14.40625 \nQ 37.984375 18.0625 35.515625 20.015625 \nQ 33.0625 21.96875 24.703125 23.78125 \nL 21.578125 24.515625 \nQ 13.234375 26.265625 9.515625 29.90625 \nQ 5.8125 33.546875 5.8125 39.890625 \nQ 5.8125 47.609375 11.28125 51.796875 \nQ 16.75 56 26.8125 56 \nQ 31.78125 56 36.171875 55.265625 \nQ 40.578125 54.546875 44.28125 53.078125 \nz\n\" id=\"DejaVuSans-115\"/>\n     </defs>\n     <g transform=\"translate(190.0875 252.916563)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-69\"/>\n      <use x=\"63.183594\" xlink:href=\"#DejaVuSans-112\"/>\n      <use x=\"126.660156\" xlink:href=\"#DejaVuSans-111\"/>\n      <use x=\"187.841797\" xlink:href=\"#DejaVuSans-99\"/>\n      <use x=\"242.822266\" xlink:href=\"#DejaVuSans-104\"/>\n      <use x=\"306.201172\" xlink:href=\"#DejaVuSans-115\"/>\n     </g>\n    </g>\n   </g>\n   <g id=\"matplotlib.axis_2\">\n    <g id=\"ytick_1\">\n     <g id=\"line2d_6\">\n      <defs>\n       <path d=\"M 0 0 \nL -3.5 0 \n\" id=\"m2709e29ccb\" style=\"stroke:#000000;stroke-width:0.8;\"/>\n      </defs>\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"40.603125\" xlink:href=\"#m2709e29ccb\" y=\"195.419623\"/>\n      </g>\n     </g>\n     <g id=\"text_7\">\n      <!-- 70 -->\n      <defs>\n       <path d=\"M 8.203125 72.90625 \nL 55.078125 72.90625 \nL 55.078125 68.703125 \nL 28.609375 0 \nL 18.3125 0 \nL 43.21875 64.59375 \nL 8.203125 64.59375 \nz\n\" id=\"DejaVuSans-55\"/>\n      </defs>\n      <g transform=\"translate(20.878125 199.218842)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-55\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_2\">\n     <g id=\"line2d_7\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"40.603125\" xlink:href=\"#m2709e29ccb\" y=\"157.263959\"/>\n      </g>\n     </g>\n     <g id=\"text_8\">\n      <!-- 75 -->\n      <defs>\n       <path d=\"M 10.796875 72.90625 \nL 49.515625 72.90625 \nL 49.515625 64.59375 \nL 19.828125 64.59375 \nL 19.828125 46.734375 \nQ 21.96875 47.46875 24.109375 47.828125 \nQ 26.265625 48.1875 28.421875 48.1875 \nQ 40.625 48.1875 47.75 41.5 \nQ 54.890625 34.8125 54.890625 23.390625 \nQ 54.890625 11.625 47.5625 5.09375 \nQ 40.234375 -1.421875 26.90625 -1.421875 \nQ 22.3125 -1.421875 17.546875 -0.640625 \nQ 12.796875 0.140625 7.71875 1.703125 \nL 7.71875 11.625 \nQ 12.109375 9.234375 16.796875 8.0625 \nQ 21.484375 6.890625 26.703125 6.890625 \nQ 35.15625 6.890625 40.078125 11.328125 \nQ 45.015625 15.765625 45.015625 23.390625 \nQ 45.015625 31 40.078125 35.4375 \nQ 35.15625 39.890625 26.703125 39.890625 \nQ 22.75 39.890625 18.8125 39.015625 \nQ 14.890625 38.140625 10.796875 36.28125 \nz\n\" id=\"DejaVuSans-53\"/>\n      </defs>\n      <g transform=\"translate(20.878125 161.063177)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-55\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-53\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_3\">\n     <g id=\"line2d_8\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"40.603125\" xlink:href=\"#m2709e29ccb\" y=\"119.108294\"/>\n      </g>\n     </g>\n     <g id=\"text_9\">\n      <!-- 80 -->\n      <g transform=\"translate(20.878125 122.907513)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-56\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_4\">\n     <g id=\"line2d_9\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"40.603125\" xlink:href=\"#m2709e29ccb\" y=\"80.95263\"/>\n      </g>\n     </g>\n     <g id=\"text_10\">\n      <!-- 85 -->\n      <g transform=\"translate(20.878125 84.751849)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-56\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-53\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_5\">\n     <g id=\"line2d_10\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"40.603125\" xlink:href=\"#m2709e29ccb\" y=\"42.796965\"/>\n      </g>\n     </g>\n     <g id=\"text_11\">\n      <!-- 90 -->\n      <defs>\n       <path d=\"M 10.984375 1.515625 \nL 10.984375 10.5 \nQ 14.703125 8.734375 18.5 7.8125 \nQ 22.3125 6.890625 25.984375 6.890625 \nQ 35.75 6.890625 40.890625 13.453125 \nQ 46.046875 20.015625 46.78125 33.40625 \nQ 43.953125 29.203125 39.59375 26.953125 \nQ 35.25 24.703125 29.984375 24.703125 \nQ 19.046875 24.703125 12.671875 31.3125 \nQ 6.296875 37.9375 6.296875 49.421875 \nQ 6.296875 60.640625 12.9375 67.421875 \nQ 19.578125 74.21875 30.609375 74.21875 \nQ 43.265625 74.21875 49.921875 64.515625 \nQ 56.59375 54.828125 56.59375 36.375 \nQ 56.59375 19.140625 48.40625 8.859375 \nQ 40.234375 -1.421875 26.421875 -1.421875 \nQ 22.703125 -1.421875 18.890625 -0.6875 \nQ 15.09375 0.046875 10.984375 1.515625 \nz\nM 30.609375 32.421875 \nQ 37.25 32.421875 41.125 36.953125 \nQ 45.015625 41.5 45.015625 49.421875 \nQ 45.015625 57.28125 41.125 61.84375 \nQ 37.25 66.40625 30.609375 66.40625 \nQ 23.96875 66.40625 20.09375 61.84375 \nQ 16.21875 57.28125 16.21875 49.421875 \nQ 16.21875 41.5 20.09375 36.953125 \nQ 23.96875 32.421875 30.609375 32.421875 \nz\n\" id=\"DejaVuSans-57\"/>\n      </defs>\n      <g transform=\"translate(20.878125 46.596184)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-57\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"text_12\">\n     <!-- Accuracy -->\n     <defs>\n      <path d=\"M 34.1875 63.1875 \nL 20.796875 26.90625 \nL 47.609375 26.90625 \nz\nM 28.609375 72.90625 \nL 39.796875 72.90625 \nL 67.578125 0 \nL 57.328125 0 \nL 50.6875 18.703125 \nL 17.828125 18.703125 \nL 11.1875 0 \nL 0.78125 0 \nz\n\" id=\"DejaVuSans-65\"/>\n      <path d=\"M 8.5 21.578125 \nL 8.5 54.6875 \nL 17.484375 54.6875 \nL 17.484375 21.921875 \nQ 17.484375 14.15625 20.5 10.265625 \nQ 23.53125 6.390625 29.59375 6.390625 \nQ 36.859375 6.390625 41.078125 11.03125 \nQ 45.3125 15.671875 45.3125 23.6875 \nL 45.3125 54.6875 \nL 54.296875 54.6875 \nL 54.296875 0 \nL 45.3125 0 \nL 45.3125 8.40625 \nQ 42.046875 3.421875 37.71875 1 \nQ 33.40625 -1.421875 27.6875 -1.421875 \nQ 18.265625 -1.421875 13.375 4.4375 \nQ 8.5 10.296875 8.5 21.578125 \nz\nM 31.109375 56 \nz\n\" id=\"DejaVuSans-117\"/>\n      <path d=\"M 41.109375 46.296875 \nQ 39.59375 47.171875 37.8125 47.578125 \nQ 36.03125 48 33.890625 48 \nQ 26.265625 48 22.1875 43.046875 \nQ 18.109375 38.09375 18.109375 28.8125 \nL 18.109375 0 \nL 9.078125 0 \nL 9.078125 54.6875 \nL 18.109375 54.6875 \nL 18.109375 46.1875 \nQ 20.953125 51.171875 25.484375 53.578125 \nQ 30.03125 56 36.53125 56 \nQ 37.453125 56 38.578125 55.875 \nQ 39.703125 55.765625 41.0625 55.515625 \nz\n\" id=\"DejaVuSans-114\"/>\n      <path d=\"M 34.28125 27.484375 \nQ 23.390625 27.484375 19.1875 25 \nQ 14.984375 22.515625 14.984375 16.5 \nQ 14.984375 11.71875 18.140625 8.90625 \nQ 21.296875 6.109375 26.703125 6.109375 \nQ 34.1875 6.109375 38.703125 11.40625 \nQ 43.21875 16.703125 43.21875 25.484375 \nL 43.21875 27.484375 \nz\nM 52.203125 31.203125 \nL 52.203125 0 \nL 43.21875 0 \nL 43.21875 8.296875 \nQ 40.140625 3.328125 35.546875 0.953125 \nQ 30.953125 -1.421875 24.3125 -1.421875 \nQ 15.921875 -1.421875 10.953125 3.296875 \nQ 6 8.015625 6 15.921875 \nQ 6 25.140625 12.171875 29.828125 \nQ 18.359375 34.515625 30.609375 34.515625 \nL 43.21875 34.515625 \nL 43.21875 35.40625 \nQ 43.21875 41.609375 39.140625 45 \nQ 35.0625 48.390625 27.6875 48.390625 \nQ 23 48.390625 18.546875 47.265625 \nQ 14.109375 46.140625 10.015625 43.890625 \nL 10.015625 52.203125 \nQ 14.9375 54.109375 19.578125 55.046875 \nQ 24.21875 56 28.609375 56 \nQ 40.484375 56 46.34375 49.84375 \nQ 52.203125 43.703125 52.203125 31.203125 \nz\n\" id=\"DejaVuSans-97\"/>\n      <path d=\"M 32.171875 -5.078125 \nQ 28.375 -14.84375 24.75 -17.8125 \nQ 21.140625 -20.796875 15.09375 -20.796875 \nL 7.90625 -20.796875 \nL 7.90625 -13.28125 \nL 13.1875 -13.28125 \nQ 16.890625 -13.28125 18.9375 -11.515625 \nQ 21 -9.765625 23.484375 -3.21875 \nL 25.09375 0.875 \nL 2.984375 54.6875 \nL 12.5 54.6875 \nL 29.59375 11.921875 \nL 46.6875 54.6875 \nL 56.203125 54.6875 \nz\n\" id=\"DejaVuSans-121\"/>\n     </defs>\n     <g transform=\"translate(14.798438 138.834844)rotate(-90)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-65\"/>\n      <use x=\"68.392578\" xlink:href=\"#DejaVuSans-99\"/>\n      <use x=\"123.373047\" xlink:href=\"#DejaVuSans-99\"/>\n      <use x=\"178.353516\" xlink:href=\"#DejaVuSans-117\"/>\n      <use x=\"241.732422\" xlink:href=\"#DejaVuSans-114\"/>\n      <use x=\"282.845703\" xlink:href=\"#DejaVuSans-97\"/>\n      <use x=\"344.125\" xlink:href=\"#DejaVuSans-99\"/>\n      <use x=\"399.105469\" xlink:href=\"#DejaVuSans-121\"/>\n     </g>\n    </g>\n   </g>\n   <g id=\"line2d_11\">\n    <path clip-path=\"url(#pa2845dd77c)\" d=\"M 55.821307 214.756364 \nL 59.279985 178.455851 \nL 62.738662 159.208479 \nL 66.19734 145.99495 \nL 69.656018 136.307628 \nL 73.114695 128.390398 \nL 76.573373 122.169609 \nL 80.032051 116.694688 \nL 83.490728 111.811582 \nL 86.949406 107.486525 \nL 90.408084 103.32466 \nL 93.866761 99.817082 \nL 97.325439 96.521312 \nL 100.784117 93.559148 \nL 104.242794 90.818107 \nL 107.701472 88.28538 \nL 111.16015 85.885863 \nL 114.618827 83.690178 \nL 118.077505 81.602434 \nL 121.536183 79.509158 \nL 124.994861 77.531568 \nL 128.453538 75.640727 \nL 131.912216 73.887754 \nL 135.370894 72.250057 \nL 138.829571 70.665633 \nL 142.288249 69.131104 \nL 145.746927 67.651419 \nL 149.205604 66.199215 \nL 152.664282 64.92441 \nL 156.12296 63.58952 \nL 159.581637 62.296899 \nL 163.040315 60.696814 \nL 166.498993 59.018712 \nL 169.95767 57.389167 \nL 173.416348 55.830534 \nL 176.875026 54.308057 \nL 180.333704 52.852709 \nL 183.792381 51.468914 \nL 187.251059 50.127737 \nL 190.709737 48.831681 \nL 194.168414 47.587965 \nL 197.627092 46.386227 \nL 201.08577 45.254529 \nL 204.544447 44.166787 \nL 208.003125 43.093776 \nL 211.461803 42.096451 \nL 214.92048 41.112576 \nL 218.379158 40.169164 \nL 221.837836 39.27262 \nL 225.296513 38.376192 \nL 228.755191 37.507595 \nL 232.213869 36.670552 \nL 235.672546 35.869491 \nL 239.131224 35.099985 \nL 242.589902 34.344626 \nL 246.04858 33.623444 \nL 249.507257 32.918621 \nL 252.965935 32.240988 \nL 256.424613 31.575523 \nL 259.88329 30.91914 \nL 263.341968 30.289714 \nL 266.800646 29.663722 \nL 270.259323 29.036974 \nL 273.718001 28.434271 \nL 277.176679 27.846356 \nL 280.635356 27.27125 \nL 284.094034 26.717802 \nL 287.552712 26.180365 \nL 291.011389 25.654397 \nL 294.470067 25.14147 \nL 297.928745 24.641178 \nL 301.387423 24.15055 \nL 304.8461 23.671101 \nL 308.304778 23.208419 \nL 311.763456 22.753132 \nL 315.222133 22.315893 \nL 318.680811 21.892394 \nL 322.139489 21.480597 \nL 325.598166 21.072352 \nL 329.056844 20.671443 \nL 332.515522 20.288291 \nL 335.974199 19.907817 \nL 339.432877 19.541782 \nL 342.891555 19.186518 \nL 346.350232 18.827178 \nL 349.80891 18.479891 \nL 353.267588 18.14023 \nL 356.726265 17.805168 \nL 360.184943 17.480412 \n\" style=\"fill:none;stroke:#ff0000;stroke-linecap:square;stroke-width:1.5;\"/>\n   </g>\n   <g id=\"line2d_12\">\n    <path clip-path=\"url(#pa2845dd77c)\" d=\"M 55.821307 183.934905 \nL 59.279985 162.256286 \nL 62.738662 148.025462 \nL 66.19734 137.434378 \nL 69.656018 129.20258 \nL 73.114695 122.68335 \nL 76.573373 116.997145 \nL 80.032051 112.166206 \nL 83.490728 107.952932 \nL 86.949406 103.763821 \nL 90.408084 100.11168 \nL 93.866761 96.701622 \nL 97.325439 93.703652 \nL 100.784117 90.879239 \nL 104.242794 88.30471 \nL 107.701472 85.832824 \nL 111.16015 83.655129 \nL 114.618827 81.585841 \nL 118.077505 79.560451 \nL 121.536183 77.572556 \nL 124.994861 75.6462 \nL 128.453538 73.85841 \nL 131.912216 72.191895 \nL 135.370894 70.6268 \nL 138.829571 69.058794 \nL 142.288249 67.589181 \nL 145.746927 66.101753 \nL 149.205604 64.823804 \nL 152.664282 63.541197 \nL 156.12296 62.24025 \nL 159.581637 60.671138 \nL 163.040315 58.985643 \nL 166.498993 57.364423 \nL 169.95767 55.78419 \nL 173.416348 54.261888 \nL 176.875026 52.803279 \nL 180.333704 51.417388 \nL 183.792381 50.073708 \nL 187.251059 48.774217 \nL 190.709737 47.516586 \nL 194.168414 46.312054 \nL 197.627092 45.182451 \nL 201.08577 44.093662 \nL 204.544447 43.02228 \nL 208.003125 42.014476 \nL 211.461803 41.031591 \nL 214.92048 40.086898 \nL 218.379158 39.18785 \nL 221.837836 38.288454 \nL 225.296513 37.425619 \nL 228.755191 36.574138 \nL 232.213869 35.779539 \nL 235.672546 35.006249 \nL 239.131224 34.255083 \nL 242.589902 33.530232 \nL 246.04858 32.825468 \nL 249.507257 32.143584 \nL 252.965935 31.489938 \nL 256.424613 30.826569 \nL 259.88329 30.197725 \nL 263.341968 29.574179 \nL 266.800646 28.946848 \nL 270.259323 28.350957 \nL 273.718001 27.758093 \nL 277.176679 27.188343 \nL 280.635356 26.630878 \nL 284.094034 26.091345 \nL 287.552712 25.56526 \nL 291.011389 25.056584 \nL 294.470067 24.552682 \nL 297.928745 24.064966 \nL 301.387423 23.582314 \nL 304.8461 23.118876 \nL 308.304778 22.665335 \nL 311.763456 22.224893 \nL 315.222133 21.805353 \nL 318.680811 21.389539 \nL 322.139489 20.987233 \nL 325.598166 20.581899 \nL 329.056844 20.197 \nL 332.515522 19.819845 \nL 335.974199 19.453403 \nL 339.432877 19.098721 \nL 342.891555 18.74334 \nL 346.350232 18.394015 \nL 349.80891 18.054878 \nL 353.267588 17.71871 \nL 356.726265 17.396516 \nL 360.184943 17.083636 \n\" style=\"fill:none;stroke:#0000ff;stroke-linecap:square;stroke-width:1.5;\"/>\n   </g>\n   <g id=\"patch_3\">\n    <path d=\"M 40.603125 224.64 \nL 40.603125 7.2 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_4\">\n    <path d=\"M 375.403125 224.64 \nL 375.403125 7.2 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_5\">\n    <path d=\"M 40.603125 224.64 \nL 375.403125 224.64 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_6\">\n    <path d=\"M 40.603125 7.2 \nL 375.403125 7.2 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"legend_1\">\n    <g id=\"patch_7\">\n     <path d=\"M 47.603125 44.55625 \nL 105.04375 44.55625 \nQ 107.04375 44.55625 107.04375 42.55625 \nL 107.04375 14.2 \nQ 107.04375 12.2 105.04375 12.2 \nL 47.603125 12.2 \nQ 45.603125 12.2 45.603125 14.2 \nL 45.603125 42.55625 \nQ 45.603125 44.55625 47.603125 44.55625 \nz\n\" style=\"fill:#ffffff;opacity:0.8;stroke:#cccccc;stroke-linejoin:miter;\"/>\n    </g>\n    <g id=\"line2d_13\">\n     <path d=\"M 49.603125 20.298438 \nL 69.603125 20.298438 \n\" style=\"fill:none;stroke:#ff0000;stroke-linecap:square;stroke-width:1.5;\"/>\n    </g>\n    <g id=\"line2d_14\"/>\n    <g id=\"text_13\">\n     <!-- Train -->\n     <defs>\n      <path d=\"M -0.296875 72.90625 \nL 61.375 72.90625 \nL 61.375 64.59375 \nL 35.5 64.59375 \nL 35.5 0 \nL 25.59375 0 \nL 25.59375 64.59375 \nL -0.296875 64.59375 \nz\n\" id=\"DejaVuSans-84\"/>\n      <path d=\"M 9.421875 54.6875 \nL 18.40625 54.6875 \nL 18.40625 0 \nL 9.421875 0 \nz\nM 9.421875 75.984375 \nL 18.40625 75.984375 \nL 18.40625 64.59375 \nL 9.421875 64.59375 \nz\n\" id=\"DejaVuSans-105\"/>\n      <path d=\"M 54.890625 33.015625 \nL 54.890625 0 \nL 45.90625 0 \nL 45.90625 32.71875 \nQ 45.90625 40.484375 42.875 44.328125 \nQ 39.84375 48.1875 33.796875 48.1875 \nQ 26.515625 48.1875 22.3125 43.546875 \nQ 18.109375 38.921875 18.109375 30.90625 \nL 18.109375 0 \nL 9.078125 0 \nL 9.078125 54.6875 \nL 18.109375 54.6875 \nL 18.109375 46.1875 \nQ 21.34375 51.125 25.703125 53.5625 \nQ 30.078125 56 35.796875 56 \nQ 45.21875 56 50.046875 50.171875 \nQ 54.890625 44.34375 54.890625 33.015625 \nz\n\" id=\"DejaVuSans-110\"/>\n     </defs>\n     <g transform=\"translate(77.603125 23.798438)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-84\"/>\n      <use x=\"60.865234\" xlink:href=\"#DejaVuSans-114\"/>\n      <use x=\"101.978516\" xlink:href=\"#DejaVuSans-97\"/>\n      <use x=\"163.257812\" xlink:href=\"#DejaVuSans-105\"/>\n      <use x=\"191.041016\" xlink:href=\"#DejaVuSans-110\"/>\n     </g>\n    </g>\n    <g id=\"line2d_15\">\n     <path d=\"M 49.603125 34.976563 \nL 69.603125 34.976563 \n\" style=\"fill:none;stroke:#0000ff;stroke-linecap:square;stroke-width:1.5;\"/>\n    </g>\n    <g id=\"line2d_16\"/>\n    <g id=\"text_14\">\n     <!-- Val -->\n     <defs>\n      <path d=\"M 28.609375 0 \nL 0.78125 72.90625 \nL 11.078125 72.90625 \nL 34.1875 11.53125 \nL 57.328125 72.90625 \nL 67.578125 72.90625 \nL 39.796875 0 \nz\n\" id=\"DejaVuSans-86\"/>\n      <path d=\"M 9.421875 75.984375 \nL 18.40625 75.984375 \nL 18.40625 0 \nL 9.421875 0 \nz\n\" id=\"DejaVuSans-108\"/>\n     </defs>\n     <g transform=\"translate(77.603125 38.476563)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-86\"/>\n      <use x=\"68.298828\" xlink:href=\"#DejaVuSans-97\"/>\n      <use x=\"129.578125\" xlink:href=\"#DejaVuSans-108\"/>\n     </g>\n    </g>\n   </g>\n  </g>\n </g>\n <defs>\n  <clipPath id=\"pa2845dd77c\">\n   <rect height=\"217.44\" width=\"334.8\" x=\"40.603125\" y=\"7.2\"/>\n  </clipPath>\n </defs>\n</svg>\n"
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RDDR6rDi61eB",
        "colab_type": "code",
        "outputId": "837014cc-cc75-44ac-c8f0-26fc25fddbcc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 371
        }
      },
      "source": [
        "plt.xlabel(\"Epochs\")\n",
        "plt.ylabel(\"Losses\")\n",
        "plt.plot(ep_lookahead_losses, 'r', label='Train')\n",
        "plt.plot(ep_lookahead_lossesv, 'b', label='Val')\n",
        "plt.legend()\n",
        "plt.savefig('lookahead_sgd_resnet18_9336_loss.jpg')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/svg+xml": "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n  \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<!-- Created with matplotlib (https://matplotlib.org/) -->\n<svg height=\"262.19625pt\" version=\"1.1\" viewBox=\"0 0 385.78125 262.19625\" width=\"385.78125pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n <defs>\n  <style type=\"text/css\">\n*{stroke-linecap:butt;stroke-linejoin:round;}\n  </style>\n </defs>\n <g id=\"figure_1\">\n  <g id=\"patch_1\">\n   <path d=\"M 0 262.19625 \nL 385.78125 262.19625 \nL 385.78125 0 \nL 0 0 \nz\n\" style=\"fill:none;\"/>\n  </g>\n  <g id=\"axes_1\">\n   <g id=\"patch_2\">\n    <path d=\"M 43.78125 224.64 \nL 378.58125 224.64 \nL 378.58125 7.2 \nL 43.78125 7.2 \nz\n\" style=\"fill:#ffffff;\"/>\n   </g>\n   <g id=\"matplotlib.axis_1\">\n    <g id=\"xtick_1\">\n     <g id=\"line2d_1\">\n      <defs>\n       <path d=\"M 0 0 \nL 0 3.5 \n\" id=\"ma2b315a799\" style=\"stroke:#000000;stroke-width:0.8;\"/>\n      </defs>\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"58.999432\" xlink:href=\"#ma2b315a799\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_1\">\n      <!-- 0 -->\n      <defs>\n       <path d=\"M 31.78125 66.40625 \nQ 24.171875 66.40625 20.328125 58.90625 \nQ 16.5 51.421875 16.5 36.375 \nQ 16.5 21.390625 20.328125 13.890625 \nQ 24.171875 6.390625 31.78125 6.390625 \nQ 39.453125 6.390625 43.28125 13.890625 \nQ 47.125 21.390625 47.125 36.375 \nQ 47.125 51.421875 43.28125 58.90625 \nQ 39.453125 66.40625 31.78125 66.40625 \nz\nM 31.78125 74.21875 \nQ 44.046875 74.21875 50.515625 64.515625 \nQ 56.984375 54.828125 56.984375 36.375 \nQ 56.984375 17.96875 50.515625 8.265625 \nQ 44.046875 -1.421875 31.78125 -1.421875 \nQ 19.53125 -1.421875 13.0625 8.265625 \nQ 6.59375 17.96875 6.59375 36.375 \nQ 6.59375 54.828125 13.0625 64.515625 \nQ 19.53125 74.21875 31.78125 74.21875 \nz\n\" id=\"DejaVuSans-48\"/>\n      </defs>\n      <g transform=\"translate(55.818182 239.238437)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_2\">\n     <g id=\"line2d_2\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"128.172986\" xlink:href=\"#ma2b315a799\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_2\">\n      <!-- 20 -->\n      <defs>\n       <path d=\"M 19.1875 8.296875 \nL 53.609375 8.296875 \nL 53.609375 0 \nL 7.328125 0 \nL 7.328125 8.296875 \nQ 12.9375 14.109375 22.625 23.890625 \nQ 32.328125 33.6875 34.8125 36.53125 \nQ 39.546875 41.84375 41.421875 45.53125 \nQ 43.3125 49.21875 43.3125 52.78125 \nQ 43.3125 58.59375 39.234375 62.25 \nQ 35.15625 65.921875 28.609375 65.921875 \nQ 23.96875 65.921875 18.8125 64.3125 \nQ 13.671875 62.703125 7.8125 59.421875 \nL 7.8125 69.390625 \nQ 13.765625 71.78125 18.9375 73 \nQ 24.125 74.21875 28.421875 74.21875 \nQ 39.75 74.21875 46.484375 68.546875 \nQ 53.21875 62.890625 53.21875 53.421875 \nQ 53.21875 48.921875 51.53125 44.890625 \nQ 49.859375 40.875 45.40625 35.40625 \nQ 44.1875 33.984375 37.640625 27.21875 \nQ 31.109375 20.453125 19.1875 8.296875 \nz\n\" id=\"DejaVuSans-50\"/>\n      </defs>\n      <g transform=\"translate(121.810486 239.238437)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-50\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_3\">\n     <g id=\"line2d_3\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"197.346539\" xlink:href=\"#ma2b315a799\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_3\">\n      <!-- 40 -->\n      <defs>\n       <path d=\"M 37.796875 64.3125 \nL 12.890625 25.390625 \nL 37.796875 25.390625 \nz\nM 35.203125 72.90625 \nL 47.609375 72.90625 \nL 47.609375 25.390625 \nL 58.015625 25.390625 \nL 58.015625 17.1875 \nL 47.609375 17.1875 \nL 47.609375 0 \nL 37.796875 0 \nL 37.796875 17.1875 \nL 4.890625 17.1875 \nL 4.890625 26.703125 \nz\n\" id=\"DejaVuSans-52\"/>\n      </defs>\n      <g transform=\"translate(190.984039 239.238437)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-52\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_4\">\n     <g id=\"line2d_4\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"266.520093\" xlink:href=\"#ma2b315a799\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_4\">\n      <!-- 60 -->\n      <defs>\n       <path d=\"M 33.015625 40.375 \nQ 26.375 40.375 22.484375 35.828125 \nQ 18.609375 31.296875 18.609375 23.390625 \nQ 18.609375 15.53125 22.484375 10.953125 \nQ 26.375 6.390625 33.015625 6.390625 \nQ 39.65625 6.390625 43.53125 10.953125 \nQ 47.40625 15.53125 47.40625 23.390625 \nQ 47.40625 31.296875 43.53125 35.828125 \nQ 39.65625 40.375 33.015625 40.375 \nz\nM 52.59375 71.296875 \nL 52.59375 62.3125 \nQ 48.875 64.0625 45.09375 64.984375 \nQ 41.3125 65.921875 37.59375 65.921875 \nQ 27.828125 65.921875 22.671875 59.328125 \nQ 17.53125 52.734375 16.796875 39.40625 \nQ 19.671875 43.65625 24.015625 45.921875 \nQ 28.375 48.1875 33.59375 48.1875 \nQ 44.578125 48.1875 50.953125 41.515625 \nQ 57.328125 34.859375 57.328125 23.390625 \nQ 57.328125 12.15625 50.6875 5.359375 \nQ 44.046875 -1.421875 33.015625 -1.421875 \nQ 20.359375 -1.421875 13.671875 8.265625 \nQ 6.984375 17.96875 6.984375 36.375 \nQ 6.984375 53.65625 15.1875 63.9375 \nQ 23.390625 74.21875 37.203125 74.21875 \nQ 40.921875 74.21875 44.703125 73.484375 \nQ 48.484375 72.75 52.59375 71.296875 \nz\n\" id=\"DejaVuSans-54\"/>\n      </defs>\n      <g transform=\"translate(260.157593 239.238437)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-54\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_5\">\n     <g id=\"line2d_5\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"335.693647\" xlink:href=\"#ma2b315a799\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_5\">\n      <!-- 80 -->\n      <defs>\n       <path d=\"M 31.78125 34.625 \nQ 24.75 34.625 20.71875 30.859375 \nQ 16.703125 27.09375 16.703125 20.515625 \nQ 16.703125 13.921875 20.71875 10.15625 \nQ 24.75 6.390625 31.78125 6.390625 \nQ 38.8125 6.390625 42.859375 10.171875 \nQ 46.921875 13.96875 46.921875 20.515625 \nQ 46.921875 27.09375 42.890625 30.859375 \nQ 38.875 34.625 31.78125 34.625 \nz\nM 21.921875 38.8125 \nQ 15.578125 40.375 12.03125 44.71875 \nQ 8.5 49.078125 8.5 55.328125 \nQ 8.5 64.0625 14.71875 69.140625 \nQ 20.953125 74.21875 31.78125 74.21875 \nQ 42.671875 74.21875 48.875 69.140625 \nQ 55.078125 64.0625 55.078125 55.328125 \nQ 55.078125 49.078125 51.53125 44.71875 \nQ 48 40.375 41.703125 38.8125 \nQ 48.828125 37.15625 52.796875 32.3125 \nQ 56.78125 27.484375 56.78125 20.515625 \nQ 56.78125 9.90625 50.3125 4.234375 \nQ 43.84375 -1.421875 31.78125 -1.421875 \nQ 19.734375 -1.421875 13.25 4.234375 \nQ 6.78125 9.90625 6.78125 20.515625 \nQ 6.78125 27.484375 10.78125 32.3125 \nQ 14.796875 37.15625 21.921875 38.8125 \nz\nM 18.3125 54.390625 \nQ 18.3125 48.734375 21.84375 45.5625 \nQ 25.390625 42.390625 31.78125 42.390625 \nQ 38.140625 42.390625 41.71875 45.5625 \nQ 45.3125 48.734375 45.3125 54.390625 \nQ 45.3125 60.0625 41.71875 63.234375 \nQ 38.140625 66.40625 31.78125 66.40625 \nQ 25.390625 66.40625 21.84375 63.234375 \nQ 18.3125 60.0625 18.3125 54.390625 \nz\n\" id=\"DejaVuSans-56\"/>\n      </defs>\n      <g transform=\"translate(329.331147 239.238437)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-56\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"text_6\">\n     <!-- Epochs -->\n     <defs>\n      <path d=\"M 9.8125 72.90625 \nL 55.90625 72.90625 \nL 55.90625 64.59375 \nL 19.671875 64.59375 \nL 19.671875 43.015625 \nL 54.390625 43.015625 \nL 54.390625 34.71875 \nL 19.671875 34.71875 \nL 19.671875 8.296875 \nL 56.78125 8.296875 \nL 56.78125 0 \nL 9.8125 0 \nz\n\" id=\"DejaVuSans-69\"/>\n      <path d=\"M 18.109375 8.203125 \nL 18.109375 -20.796875 \nL 9.078125 -20.796875 \nL 9.078125 54.6875 \nL 18.109375 54.6875 \nL 18.109375 46.390625 \nQ 20.953125 51.265625 25.265625 53.625 \nQ 29.59375 56 35.59375 56 \nQ 45.5625 56 51.78125 48.09375 \nQ 58.015625 40.1875 58.015625 27.296875 \nQ 58.015625 14.40625 51.78125 6.484375 \nQ 45.5625 -1.421875 35.59375 -1.421875 \nQ 29.59375 -1.421875 25.265625 0.953125 \nQ 20.953125 3.328125 18.109375 8.203125 \nz\nM 48.6875 27.296875 \nQ 48.6875 37.203125 44.609375 42.84375 \nQ 40.53125 48.484375 33.40625 48.484375 \nQ 26.265625 48.484375 22.1875 42.84375 \nQ 18.109375 37.203125 18.109375 27.296875 \nQ 18.109375 17.390625 22.1875 11.75 \nQ 26.265625 6.109375 33.40625 6.109375 \nQ 40.53125 6.109375 44.609375 11.75 \nQ 48.6875 17.390625 48.6875 27.296875 \nz\n\" id=\"DejaVuSans-112\"/>\n      <path d=\"M 30.609375 48.390625 \nQ 23.390625 48.390625 19.1875 42.75 \nQ 14.984375 37.109375 14.984375 27.296875 \nQ 14.984375 17.484375 19.15625 11.84375 \nQ 23.34375 6.203125 30.609375 6.203125 \nQ 37.796875 6.203125 41.984375 11.859375 \nQ 46.1875 17.53125 46.1875 27.296875 \nQ 46.1875 37.015625 41.984375 42.703125 \nQ 37.796875 48.390625 30.609375 48.390625 \nz\nM 30.609375 56 \nQ 42.328125 56 49.015625 48.375 \nQ 55.71875 40.765625 55.71875 27.296875 \nQ 55.71875 13.875 49.015625 6.21875 \nQ 42.328125 -1.421875 30.609375 -1.421875 \nQ 18.84375 -1.421875 12.171875 6.21875 \nQ 5.515625 13.875 5.515625 27.296875 \nQ 5.515625 40.765625 12.171875 48.375 \nQ 18.84375 56 30.609375 56 \nz\n\" id=\"DejaVuSans-111\"/>\n      <path d=\"M 48.78125 52.59375 \nL 48.78125 44.1875 \nQ 44.96875 46.296875 41.140625 47.34375 \nQ 37.3125 48.390625 33.40625 48.390625 \nQ 24.65625 48.390625 19.8125 42.84375 \nQ 14.984375 37.3125 14.984375 27.296875 \nQ 14.984375 17.28125 19.8125 11.734375 \nQ 24.65625 6.203125 33.40625 6.203125 \nQ 37.3125 6.203125 41.140625 7.25 \nQ 44.96875 8.296875 48.78125 10.40625 \nL 48.78125 2.09375 \nQ 45.015625 0.34375 40.984375 -0.53125 \nQ 36.96875 -1.421875 32.421875 -1.421875 \nQ 20.0625 -1.421875 12.78125 6.34375 \nQ 5.515625 14.109375 5.515625 27.296875 \nQ 5.515625 40.671875 12.859375 48.328125 \nQ 20.21875 56 33.015625 56 \nQ 37.15625 56 41.109375 55.140625 \nQ 45.0625 54.296875 48.78125 52.59375 \nz\n\" id=\"DejaVuSans-99\"/>\n      <path d=\"M 54.890625 33.015625 \nL 54.890625 0 \nL 45.90625 0 \nL 45.90625 32.71875 \nQ 45.90625 40.484375 42.875 44.328125 \nQ 39.84375 48.1875 33.796875 48.1875 \nQ 26.515625 48.1875 22.3125 43.546875 \nQ 18.109375 38.921875 18.109375 30.90625 \nL 18.109375 0 \nL 9.078125 0 \nL 9.078125 75.984375 \nL 18.109375 75.984375 \nL 18.109375 46.1875 \nQ 21.34375 51.125 25.703125 53.5625 \nQ 30.078125 56 35.796875 56 \nQ 45.21875 56 50.046875 50.171875 \nQ 54.890625 44.34375 54.890625 33.015625 \nz\n\" id=\"DejaVuSans-104\"/>\n      <path d=\"M 44.28125 53.078125 \nL 44.28125 44.578125 \nQ 40.484375 46.53125 36.375 47.5 \nQ 32.28125 48.484375 27.875 48.484375 \nQ 21.1875 48.484375 17.84375 46.4375 \nQ 14.5 44.390625 14.5 40.28125 \nQ 14.5 37.15625 16.890625 35.375 \nQ 19.28125 33.59375 26.515625 31.984375 \nL 29.59375 31.296875 \nQ 39.15625 29.25 43.1875 25.515625 \nQ 47.21875 21.78125 47.21875 15.09375 \nQ 47.21875 7.46875 41.1875 3.015625 \nQ 35.15625 -1.421875 24.609375 -1.421875 \nQ 20.21875 -1.421875 15.453125 -0.5625 \nQ 10.6875 0.296875 5.421875 2 \nL 5.421875 11.28125 \nQ 10.40625 8.6875 15.234375 7.390625 \nQ 20.0625 6.109375 24.8125 6.109375 \nQ 31.15625 6.109375 34.5625 8.28125 \nQ 37.984375 10.453125 37.984375 14.40625 \nQ 37.984375 18.0625 35.515625 20.015625 \nQ 33.0625 21.96875 24.703125 23.78125 \nL 21.578125 24.515625 \nQ 13.234375 26.265625 9.515625 29.90625 \nQ 5.8125 33.546875 5.8125 39.890625 \nQ 5.8125 47.609375 11.28125 51.796875 \nQ 16.75 56 26.8125 56 \nQ 31.78125 56 36.171875 55.265625 \nQ 40.578125 54.546875 44.28125 53.078125 \nz\n\" id=\"DejaVuSans-115\"/>\n     </defs>\n     <g transform=\"translate(193.265625 252.916562)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-69\"/>\n      <use x=\"63.183594\" xlink:href=\"#DejaVuSans-112\"/>\n      <use x=\"126.660156\" xlink:href=\"#DejaVuSans-111\"/>\n      <use x=\"187.841797\" xlink:href=\"#DejaVuSans-99\"/>\n      <use x=\"242.822266\" xlink:href=\"#DejaVuSans-104\"/>\n      <use x=\"306.201172\" xlink:href=\"#DejaVuSans-115\"/>\n     </g>\n    </g>\n   </g>\n   <g id=\"matplotlib.axis_2\">\n    <g id=\"ytick_1\">\n     <g id=\"line2d_6\">\n      <defs>\n       <path d=\"M 0 0 \nL -3.5 0 \n\" id=\"mdb0e0febe7\" style=\"stroke:#000000;stroke-width:0.8;\"/>\n      </defs>\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#mdb0e0febe7\" y=\"215.838125\"/>\n      </g>\n     </g>\n     <g id=\"text_7\">\n      <!-- 0.0 -->\n      <defs>\n       <path d=\"M 10.6875 12.40625 \nL 21 12.40625 \nL 21 0 \nL 10.6875 0 \nz\n\" id=\"DejaVuSans-46\"/>\n      </defs>\n      <g transform=\"translate(20.878125 219.637344)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_2\">\n     <g id=\"line2d_7\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#mdb0e0febe7\" y=\"186.745172\"/>\n      </g>\n     </g>\n     <g id=\"text_8\">\n      <!-- 0.1 -->\n      <defs>\n       <path d=\"M 12.40625 8.296875 \nL 28.515625 8.296875 \nL 28.515625 63.921875 \nL 10.984375 60.40625 \nL 10.984375 69.390625 \nL 28.421875 72.90625 \nL 38.28125 72.90625 \nL 38.28125 8.296875 \nL 54.390625 8.296875 \nL 54.390625 0 \nL 12.40625 0 \nz\n\" id=\"DejaVuSans-49\"/>\n      </defs>\n      <g transform=\"translate(20.878125 190.544391)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-49\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_3\">\n     <g id=\"line2d_8\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#mdb0e0febe7\" y=\"157.652219\"/>\n      </g>\n     </g>\n     <g id=\"text_9\">\n      <!-- 0.2 -->\n      <g transform=\"translate(20.878125 161.451438)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-50\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_4\">\n     <g id=\"line2d_9\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#mdb0e0febe7\" y=\"128.559266\"/>\n      </g>\n     </g>\n     <g id=\"text_10\">\n      <!-- 0.3 -->\n      <defs>\n       <path d=\"M 40.578125 39.3125 \nQ 47.65625 37.796875 51.625 33 \nQ 55.609375 28.21875 55.609375 21.1875 \nQ 55.609375 10.40625 48.1875 4.484375 \nQ 40.765625 -1.421875 27.09375 -1.421875 \nQ 22.515625 -1.421875 17.65625 -0.515625 \nQ 12.796875 0.390625 7.625 2.203125 \nL 7.625 11.71875 \nQ 11.71875 9.328125 16.59375 8.109375 \nQ 21.484375 6.890625 26.8125 6.890625 \nQ 36.078125 6.890625 40.9375 10.546875 \nQ 45.796875 14.203125 45.796875 21.1875 \nQ 45.796875 27.640625 41.28125 31.265625 \nQ 36.765625 34.90625 28.71875 34.90625 \nL 20.21875 34.90625 \nL 20.21875 43.015625 \nL 29.109375 43.015625 \nQ 36.375 43.015625 40.234375 45.921875 \nQ 44.09375 48.828125 44.09375 54.296875 \nQ 44.09375 59.90625 40.109375 62.90625 \nQ 36.140625 65.921875 28.71875 65.921875 \nQ 24.65625 65.921875 20.015625 65.03125 \nQ 15.375 64.15625 9.8125 62.3125 \nL 9.8125 71.09375 \nQ 15.4375 72.65625 20.34375 73.4375 \nQ 25.25 74.21875 29.59375 74.21875 \nQ 40.828125 74.21875 47.359375 69.109375 \nQ 53.90625 64.015625 53.90625 55.328125 \nQ 53.90625 49.265625 50.4375 45.09375 \nQ 46.96875 40.921875 40.578125 39.3125 \nz\n\" id=\"DejaVuSans-51\"/>\n      </defs>\n      <g transform=\"translate(20.878125 132.358485)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-51\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_5\">\n     <g id=\"line2d_10\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#mdb0e0febe7\" y=\"99.466313\"/>\n      </g>\n     </g>\n     <g id=\"text_11\">\n      <!-- 0.4 -->\n      <g transform=\"translate(20.878125 103.265532)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-52\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_6\">\n     <g id=\"line2d_11\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#mdb0e0febe7\" y=\"70.37336\"/>\n      </g>\n     </g>\n     <g id=\"text_12\">\n      <!-- 0.5 -->\n      <defs>\n       <path d=\"M 10.796875 72.90625 \nL 49.515625 72.90625 \nL 49.515625 64.59375 \nL 19.828125 64.59375 \nL 19.828125 46.734375 \nQ 21.96875 47.46875 24.109375 47.828125 \nQ 26.265625 48.1875 28.421875 48.1875 \nQ 40.625 48.1875 47.75 41.5 \nQ 54.890625 34.8125 54.890625 23.390625 \nQ 54.890625 11.625 47.5625 5.09375 \nQ 40.234375 -1.421875 26.90625 -1.421875 \nQ 22.3125 -1.421875 17.546875 -0.640625 \nQ 12.796875 0.140625 7.71875 1.703125 \nL 7.71875 11.625 \nQ 12.109375 9.234375 16.796875 8.0625 \nQ 21.484375 6.890625 26.703125 6.890625 \nQ 35.15625 6.890625 40.078125 11.328125 \nQ 45.015625 15.765625 45.015625 23.390625 \nQ 45.015625 31 40.078125 35.4375 \nQ 35.15625 39.890625 26.703125 39.890625 \nQ 22.75 39.890625 18.8125 39.015625 \nQ 14.890625 38.140625 10.796875 36.28125 \nz\n\" id=\"DejaVuSans-53\"/>\n      </defs>\n      <g transform=\"translate(20.878125 74.172579)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-53\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_7\">\n     <g id=\"line2d_12\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#mdb0e0febe7\" y=\"41.280407\"/>\n      </g>\n     </g>\n     <g id=\"text_13\">\n      <!-- 0.6 -->\n      <g transform=\"translate(20.878125 45.079626)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-54\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_8\">\n     <g id=\"line2d_13\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#mdb0e0febe7\" y=\"12.187454\"/>\n      </g>\n     </g>\n     <g id=\"text_14\">\n      <!-- 0.7 -->\n      <defs>\n       <path d=\"M 8.203125 72.90625 \nL 55.078125 72.90625 \nL 55.078125 68.703125 \nL 28.609375 0 \nL 18.3125 0 \nL 43.21875 64.59375 \nL 8.203125 64.59375 \nz\n\" id=\"DejaVuSans-55\"/>\n      </defs>\n      <g transform=\"translate(20.878125 15.986673)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-55\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"text_15\">\n     <!-- Losses -->\n     <defs>\n      <path d=\"M 9.8125 72.90625 \nL 19.671875 72.90625 \nL 19.671875 8.296875 \nL 55.171875 8.296875 \nL 55.171875 0 \nL 9.8125 0 \nz\n\" id=\"DejaVuSans-76\"/>\n      <path d=\"M 56.203125 29.59375 \nL 56.203125 25.203125 \nL 14.890625 25.203125 \nQ 15.484375 15.921875 20.484375 11.0625 \nQ 25.484375 6.203125 34.421875 6.203125 \nQ 39.59375 6.203125 44.453125 7.46875 \nQ 49.3125 8.734375 54.109375 11.28125 \nL 54.109375 2.78125 \nQ 49.265625 0.734375 44.1875 -0.34375 \nQ 39.109375 -1.421875 33.890625 -1.421875 \nQ 20.796875 -1.421875 13.15625 6.1875 \nQ 5.515625 13.8125 5.515625 26.8125 \nQ 5.515625 40.234375 12.765625 48.109375 \nQ 20.015625 56 32.328125 56 \nQ 43.359375 56 49.78125 48.890625 \nQ 56.203125 41.796875 56.203125 29.59375 \nz\nM 47.21875 32.234375 \nQ 47.125 39.59375 43.09375 43.984375 \nQ 39.0625 48.390625 32.421875 48.390625 \nQ 24.90625 48.390625 20.390625 44.140625 \nQ 15.875 39.890625 15.1875 32.171875 \nz\n\" id=\"DejaVuSans-101\"/>\n     </defs>\n     <g transform=\"translate(14.798438 132.655156)rotate(-90)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-76\"/>\n      <use x=\"55.697266\" xlink:href=\"#DejaVuSans-111\"/>\n      <use x=\"116.878906\" xlink:href=\"#DejaVuSans-115\"/>\n      <use x=\"168.978516\" xlink:href=\"#DejaVuSans-115\"/>\n      <use x=\"221.078125\" xlink:href=\"#DejaVuSans-101\"/>\n      <use x=\"282.601562\" xlink:href=\"#DejaVuSans-115\"/>\n     </g>\n    </g>\n   </g>\n   <g id=\"line2d_14\">\n    <path clip-path=\"url(#pc4ccb6b665)\" d=\"M 58.999432 48.219961 \nL 62.45811 17.083636 \nL 65.916787 67.407121 \nL 69.375465 85.581754 \nL 72.834143 49.605002 \nL 76.29282 88.859529 \nL 79.751498 69.21042 \nL 83.210176 123.29859 \nL 86.668853 113.783529 \nL 90.127531 128.712096 \nL 93.586209 128.100531 \nL 97.044886 134.208446 \nL 100.503564 142.041168 \nL 103.962242 140.852789 \nL 107.420919 152.821454 \nL 110.879597 145.246094 \nL 114.338275 130.767287 \nL 117.796952 138.880131 \nL 121.25563 142.715463 \nL 124.714308 191.995628 \nL 128.172986 129.487375 \nL 131.631663 162.466475 \nL 135.090341 158.290956 \nL 138.549019 157.768354 \nL 142.007696 140.973429 \nL 145.466374 146.050974 \nL 148.925052 180.782955 \nL 152.383729 147.223616 \nL 155.842407 177.587502 \nL 159.301085 163.69854 \nL 162.759762 172.55926 \nL 166.21844 187.141115 \nL 169.677118 193.794241 \nL 173.135795 197.718494 \nL 176.594473 177.15821 \nL 180.053151 200.393238 \nL 183.511829 184.199805 \nL 186.970506 199.669725 \nL 190.429184 198.271683 \nL 193.887862 206.706026 \nL 197.346539 198.160119 \nL 200.805217 172.512566 \nL 204.263895 198.960502 \nL 207.722572 201.60925 \nL 211.18125 191.912302 \nL 214.639928 194.724263 \nL 218.098605 200.872428 \nL 221.557283 202.719651 \nL 225.015961 190.603543 \nL 228.474638 195.909843 \nL 231.933316 200.077772 \nL 235.391994 198.222429 \nL 238.850671 201.718773 \nL 242.309349 203.091945 \nL 245.768027 206.747116 \nL 249.226705 207.505508 \nL 252.685382 202.267753 \nL 256.14406 194.084753 \nL 259.602738 193.67927 \nL 263.061415 196.326934 \nL 266.520093 199.240046 \nL 269.978771 209.414817 \nL 273.437448 190.331365 \nL 276.896126 205.992863 \nL 280.354804 199.097413 \nL 283.813481 213.484027 \nL 287.272159 204.237264 \nL 290.730837 203.626104 \nL 294.189514 208.087452 \nL 297.648192 198.48493 \nL 301.10687 207.133843 \nL 304.565548 213.25672 \nL 308.024225 209.486859 \nL 311.482903 205.713968 \nL 314.941581 211.657723 \nL 318.400258 196.724302 \nL 321.858936 208.082146 \nL 325.317614 202.77805 \nL 328.776291 213.744492 \nL 332.234969 211.523347 \nL 335.693647 210.828276 \nL 339.152324 211.09145 \nL 342.611002 211.825227 \nL 346.06968 191.064092 \nL 349.528357 212.296871 \nL 352.987035 203.211338 \nL 356.445713 214.296655 \nL 359.90439 199.875829 \nL 363.363068 214.756364 \n\" style=\"fill:none;stroke:#ff0000;stroke-linecap:square;stroke-width:1.5;\"/>\n   </g>\n   <g id=\"line2d_15\">\n    <path clip-path=\"url(#pc4ccb6b665)\" d=\"M 58.999432 70.354338 \nL 62.45811 82.77557 \nL 65.916787 90.965763 \nL 69.375465 95.454061 \nL 72.834143 75.69489 \nL 76.29282 82.010695 \nL 79.751498 62.682096 \nL 83.210176 78.858095 \nL 86.668853 92.602149 \nL 90.127531 97.912606 \nL 93.586209 96.60103 \nL 97.044886 99.268003 \nL 100.503564 75.867162 \nL 103.962242 106.538988 \nL 107.420919 77.827213 \nL 110.879597 105.131414 \nL 114.338275 84.069814 \nL 117.796952 83.976001 \nL 121.25563 98.173116 \nL 124.714308 80.0538 \nL 128.172986 98.312571 \nL 131.631663 95.88412 \nL 135.090341 82.274335 \nL 138.549019 56.616769 \nL 142.007696 90.111063 \nL 145.466374 88.878699 \nL 148.925052 95.393273 \nL 152.383729 69.459208 \nL 155.842407 111.386908 \nL 159.301085 118.605299 \nL 162.759762 136.114698 \nL 166.21844 126.114174 \nL 169.677118 121.524978 \nL 173.135795 117.105593 \nL 176.594473 96.668642 \nL 180.053151 95.429723 \nL 183.511829 99.098688 \nL 186.970506 105.260784 \nL 190.429184 102.091346 \nL 193.887862 97.956062 \nL 197.346539 100.242484 \nL 200.805217 105.123922 \nL 204.263895 99.919391 \nL 207.722572 96.937536 \nL 211.18125 105.8548 \nL 214.639928 108.936781 \nL 218.098605 85.77982 \nL 221.557283 86.349551 \nL 225.015961 89.268459 \nL 228.474638 96.583628 \nL 231.933316 79.888352 \nL 235.391994 86.928125 \nL 238.850671 74.75105 \nL 242.309349 53.469353 \nL 245.768027 71.425164 \nL 249.226705 85.164345 \nL 252.685382 64.022692 \nL 256.14406 59.444664 \nL 259.602738 61.369453 \nL 263.061415 65.511829 \nL 266.520093 68.051867 \nL 269.978771 77.000605 \nL 273.437448 63.441187 \nL 276.896126 67.18646 \nL 280.354804 71.890243 \nL 283.813481 70.14011 \nL 287.272159 68.020845 \nL 290.730837 65.08679 \nL 294.189514 71.543575 \nL 297.648192 63.624288 \nL 301.10687 66.566493 \nL 304.565548 61.476792 \nL 308.024225 64.232324 \nL 311.482903 69.534502 \nL 314.941581 63.20759 \nL 318.400258 64.208845 \nL 321.858936 66.17043 \nL 325.317614 68.000417 \nL 328.776291 69.015163 \nL 332.234969 61.538837 \nL 335.693647 65.35897 \nL 339.152324 71.638767 \nL 342.611002 75.263426 \nL 346.06968 70.946342 \nL 349.528357 73.278482 \nL 352.987035 74.388325 \nL 356.445713 73.383125 \nL 359.90439 76.562812 \nL 363.363068 71.276207 \n\" style=\"fill:none;stroke:#0000ff;stroke-linecap:square;stroke-width:1.5;\"/>\n   </g>\n   <g id=\"patch_3\">\n    <path d=\"M 43.78125 224.64 \nL 43.78125 7.2 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_4\">\n    <path d=\"M 378.58125 224.64 \nL 378.58125 7.2 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_5\">\n    <path d=\"M 43.78125 224.64 \nL 378.58125 224.64 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_6\">\n    <path d=\"M 43.78125 7.2 \nL 378.58125 7.2 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"legend_1\">\n    <g id=\"patch_7\">\n     <path d=\"M 314.140625 44.55625 \nL 371.58125 44.55625 \nQ 373.58125 44.55625 373.58125 42.55625 \nL 373.58125 14.2 \nQ 373.58125 12.2 371.58125 12.2 \nL 314.140625 12.2 \nQ 312.140625 12.2 312.140625 14.2 \nL 312.140625 42.55625 \nQ 312.140625 44.55625 314.140625 44.55625 \nz\n\" style=\"fill:#ffffff;opacity:0.8;stroke:#cccccc;stroke-linejoin:miter;\"/>\n    </g>\n    <g id=\"line2d_16\">\n     <path d=\"M 316.140625 20.298437 \nL 336.140625 20.298437 \n\" style=\"fill:none;stroke:#ff0000;stroke-linecap:square;stroke-width:1.5;\"/>\n    </g>\n    <g id=\"line2d_17\"/>\n    <g id=\"text_16\">\n     <!-- Train -->\n     <defs>\n      <path d=\"M -0.296875 72.90625 \nL 61.375 72.90625 \nL 61.375 64.59375 \nL 35.5 64.59375 \nL 35.5 0 \nL 25.59375 0 \nL 25.59375 64.59375 \nL -0.296875 64.59375 \nz\n\" id=\"DejaVuSans-84\"/>\n      <path d=\"M 41.109375 46.296875 \nQ 39.59375 47.171875 37.8125 47.578125 \nQ 36.03125 48 33.890625 48 \nQ 26.265625 48 22.1875 43.046875 \nQ 18.109375 38.09375 18.109375 28.8125 \nL 18.109375 0 \nL 9.078125 0 \nL 9.078125 54.6875 \nL 18.109375 54.6875 \nL 18.109375 46.1875 \nQ 20.953125 51.171875 25.484375 53.578125 \nQ 30.03125 56 36.53125 56 \nQ 37.453125 56 38.578125 55.875 \nQ 39.703125 55.765625 41.0625 55.515625 \nz\n\" id=\"DejaVuSans-114\"/>\n      <path d=\"M 34.28125 27.484375 \nQ 23.390625 27.484375 19.1875 25 \nQ 14.984375 22.515625 14.984375 16.5 \nQ 14.984375 11.71875 18.140625 8.90625 \nQ 21.296875 6.109375 26.703125 6.109375 \nQ 34.1875 6.109375 38.703125 11.40625 \nQ 43.21875 16.703125 43.21875 25.484375 \nL 43.21875 27.484375 \nz\nM 52.203125 31.203125 \nL 52.203125 0 \nL 43.21875 0 \nL 43.21875 8.296875 \nQ 40.140625 3.328125 35.546875 0.953125 \nQ 30.953125 -1.421875 24.3125 -1.421875 \nQ 15.921875 -1.421875 10.953125 3.296875 \nQ 6 8.015625 6 15.921875 \nQ 6 25.140625 12.171875 29.828125 \nQ 18.359375 34.515625 30.609375 34.515625 \nL 43.21875 34.515625 \nL 43.21875 35.40625 \nQ 43.21875 41.609375 39.140625 45 \nQ 35.0625 48.390625 27.6875 48.390625 \nQ 23 48.390625 18.546875 47.265625 \nQ 14.109375 46.140625 10.015625 43.890625 \nL 10.015625 52.203125 \nQ 14.9375 54.109375 19.578125 55.046875 \nQ 24.21875 56 28.609375 56 \nQ 40.484375 56 46.34375 49.84375 \nQ 52.203125 43.703125 52.203125 31.203125 \nz\n\" id=\"DejaVuSans-97\"/>\n      <path d=\"M 9.421875 54.6875 \nL 18.40625 54.6875 \nL 18.40625 0 \nL 9.421875 0 \nz\nM 9.421875 75.984375 \nL 18.40625 75.984375 \nL 18.40625 64.59375 \nL 9.421875 64.59375 \nz\n\" id=\"DejaVuSans-105\"/>\n      <path d=\"M 54.890625 33.015625 \nL 54.890625 0 \nL 45.90625 0 \nL 45.90625 32.71875 \nQ 45.90625 40.484375 42.875 44.328125 \nQ 39.84375 48.1875 33.796875 48.1875 \nQ 26.515625 48.1875 22.3125 43.546875 \nQ 18.109375 38.921875 18.109375 30.90625 \nL 18.109375 0 \nL 9.078125 0 \nL 9.078125 54.6875 \nL 18.109375 54.6875 \nL 18.109375 46.1875 \nQ 21.34375 51.125 25.703125 53.5625 \nQ 30.078125 56 35.796875 56 \nQ 45.21875 56 50.046875 50.171875 \nQ 54.890625 44.34375 54.890625 33.015625 \nz\n\" id=\"DejaVuSans-110\"/>\n     </defs>\n     <g transform=\"translate(344.140625 23.798437)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-84\"/>\n      <use x=\"60.865234\" xlink:href=\"#DejaVuSans-114\"/>\n      <use x=\"101.978516\" xlink:href=\"#DejaVuSans-97\"/>\n      <use x=\"163.257812\" xlink:href=\"#DejaVuSans-105\"/>\n      <use x=\"191.041016\" xlink:href=\"#DejaVuSans-110\"/>\n     </g>\n    </g>\n    <g id=\"line2d_18\">\n     <path d=\"M 316.140625 34.976562 \nL 336.140625 34.976562 \n\" style=\"fill:none;stroke:#0000ff;stroke-linecap:square;stroke-width:1.5;\"/>\n    </g>\n    <g id=\"line2d_19\"/>\n    <g id=\"text_17\">\n     <!-- Val -->\n     <defs>\n      <path d=\"M 28.609375 0 \nL 0.78125 72.90625 \nL 11.078125 72.90625 \nL 34.1875 11.53125 \nL 57.328125 72.90625 \nL 67.578125 72.90625 \nL 39.796875 0 \nz\n\" id=\"DejaVuSans-86\"/>\n      <path d=\"M 9.421875 75.984375 \nL 18.40625 75.984375 \nL 18.40625 0 \nL 9.421875 0 \nz\n\" id=\"DejaVuSans-108\"/>\n     </defs>\n     <g transform=\"translate(344.140625 38.476562)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-86\"/>\n      <use x=\"68.298828\" xlink:href=\"#DejaVuSans-97\"/>\n      <use x=\"129.578125\" xlink:href=\"#DejaVuSans-108\"/>\n     </g>\n    </g>\n   </g>\n  </g>\n </g>\n <defs>\n  <clipPath id=\"pc4ccb6b665\">\n   <rect height=\"217.44\" width=\"334.8\" x=\"43.78125\" y=\"7.2\"/>\n  </clipPath>\n </defs>\n</svg>\n"
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "unBgkhOEQUzH",
        "colab_type": "code",
        "outputId": "617a5fe5-1f21-4404-8fcd-4d7a7d2f20fa",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 442
        }
      },
      "source": [
        "from google.colab import files\n",
        "files.download('./lookahead_sgd_resnet18_9336.pth.tar')\n",
        "files.download('./lookahead_sgd_resnet18_9336_acc.jpg')\n",
        "files.download('./lookahead_sgd_resnet18_9336_loss.jpg')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "----------------------------------------\n",
            "Exception happened during processing of request from ('::ffff:127.0.0.1', 35828, 0, 0)\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/lib/python3.6/socketserver.py\", line 320, in _handle_request_noblock\n",
            "    self.process_request(request, client_address)\n",
            "  File \"/usr/lib/python3.6/socketserver.py\", line 351, in process_request\n",
            "    self.finish_request(request, client_address)\n",
            "  File \"/usr/lib/python3.6/socketserver.py\", line 364, in finish_request\n",
            "    self.RequestHandlerClass(request, client_address, self)\n",
            "  File \"/usr/lib/python3.6/socketserver.py\", line 724, in __init__\n",
            "    self.handle()\n",
            "  File \"/usr/lib/python3.6/http/server.py\", line 418, in handle\n",
            "    self.handle_one_request()\n",
            "  File \"/usr/lib/python3.6/http/server.py\", line 406, in handle_one_request\n",
            "    method()\n",
            "  File \"/usr/lib/python3.6/http/server.py\", line 639, in do_GET\n",
            "    self.copyfile(f, self.wfile)\n",
            "  File \"/usr/lib/python3.6/http/server.py\", line 800, in copyfile\n",
            "    shutil.copyfileobj(source, outputfile)\n",
            "  File \"/usr/lib/python3.6/shutil.py\", line 82, in copyfileobj\n",
            "    fdst.write(buf)\n",
            "  File \"/usr/lib/python3.6/socketserver.py\", line 803, in write\n",
            "    self._sock.sendall(b)\n",
            "ConnectionResetError: [Errno 104] Connection reset by peer\n",
            "----------------------------------------\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YUY6Jar0-pHs",
        "colab_type": "text"
      },
      "source": [
        "# RAdam"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JhuMIoYH-r2m",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "load_checkpoint(model, filename='before_sgd_resnet18.pth.tar')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CHH8il-C-uqQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "reset()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fv4k7LHH-yVj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "preds = []\n",
        "train_stats = AvgStats()\n",
        "test_stats = AvgStats()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HUaNclvt-1kM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "optimizer = RAdam( model.parameters(), lr=1e-4, weight_decay=1e-1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KsWliJjKRSzP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sched = torch.optim.lr_scheduler.StepLR(optimizer, step_size=30, gamma=0.2)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7_jUbtha-9V6",
        "colab_type": "code",
        "outputId": "31f8e7fe-8df3-408e-dce2-ede2fecb0f17",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "fit(model, optimizer, sched)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch\tTrn_loss\tVal_loss\tTrn_acc\t\tVal_acc\n",
            "1\t1.17103652\t1.04456021\t41.75599670\t45.49499893\n",
            "2\t0.86184681\t0.77707684\t55.59727097\t57.11083221\n",
            "3\t0.72119151\t0.67261847\t61.94058609\t62.78555298\n",
            "4\t0.64092174\t0.59949302\t65.96261597\t66.53375244\n",
            "5\t0.56243319\t0.55623573\t68.81655121\t69.21533203\n",
            "6\t0.52930620\t0.54810457\t70.96142578\t71.24583435\n",
            "7\t0.48743979\t0.51196633\t72.68366241\t72.92761993\n",
            "8\t0.46908906\t0.50594005\t74.11425781\t74.30020905\n",
            "9\t0.44485717\t0.49418168\t75.33792114\t75.49111176\n",
            "10\t0.39754846\t0.48649221\t76.39373016\t76.52466583\n",
            "11\t0.38853284\t0.49979419\t77.33507538\t77.43742371\n",
            "12\t0.35277096\t0.47928442\t78.17619324\t78.25888824\n",
            "13\t0.34195214\t0.47881888\t78.91259003\t78.98653412\n",
            "14\t0.32298536\t0.48684639\t79.59421539\t79.65309143\n",
            "15\t0.29404765\t0.48961397\t80.21977997\t80.27278137\n",
            "16\t0.29296536\t0.49782645\t80.78904724\t80.82979584\n",
            "17\t0.27651679\t0.48747552\t81.31484985\t81.35372925\n",
            "18\t0.27250822\t0.48827815\t81.80316925\t81.83573914\n",
            "19\t0.24975174\t0.49880514\t82.27380371\t82.29859161\n",
            "20\t0.23640815\t0.48936872\t82.70151520\t82.72391510\n",
            "21\t0.22828422\t0.50030361\t83.10680389\t83.12237549\n",
            "22\t0.21155675\t0.51053697\t83.48580933\t83.49916840\n",
            "23\t0.20569640\t0.50606565\t83.84000397\t83.85239410\n",
            "24\t0.19419527\t0.50092564\t84.18720245\t84.20055389\n",
            "25\t0.18973599\t0.50126533\t84.51992798\t84.52919769\n",
            "26\t0.19002261\t0.52019892\t84.82567596\t84.83179474\n",
            "27\t0.18157091\t0.53948772\t85.11292267\t85.11579895\n",
            "28\t0.17043613\t0.55356344\t85.38957977\t85.38833618\n",
            "29\t0.16801454\t0.54111792\t85.64612579\t85.64431000\n",
            "30\t0.15436218\t0.53231354\t85.89245605\t85.88994598\n",
            "31\t0.15716540\t0.54954993\t86.13465118\t86.13150787\n",
            "32\t0.10828137\t0.51330214\t86.39388275\t86.39500427\n",
            "33\t0.09549051\t0.50689746\t86.66375732\t86.66292572\n",
            "34\t0.08576781\t0.51947838\t86.92240906\t86.92166901\n",
            "35\t0.08192715\t0.52379053\t87.16899872\t87.16457367\n",
            "36\t0.07370163\t0.53708441\t87.40380859\t87.39935303\n",
            "37\t0.07160849\t0.54029767\t87.62908936\t87.62414551\n",
            "38\t0.07528697\t0.55027998\t87.84576416\t87.84004211\n",
            "39\t0.06386290\t0.55347697\t88.05764008\t88.05166626\n",
            "40\t0.06206949\t0.57190780\t88.25803375\t88.24979401\n",
            "41\t0.06141426\t0.56972918\t88.44804382\t88.44023895\n",
            "42\t0.05812667\t0.56708702\t88.63290405\t88.62460327\n",
            "43\t0.06013964\t0.57575901\t88.80805206\t88.80050659\n",
            "44\t0.05896430\t0.58334152\t88.97577667\t88.96685791\n",
            "45\t0.04975346\t0.57674738\t89.13929749\t89.13011169\n",
            "46\t0.05506935\t0.59012260\t89.29410553\t89.28413391\n",
            "47\t0.05412992\t0.59084280\t89.44541168\t89.43595886\n",
            "48\t0.04747208\t0.60526176\t89.59275818\t89.58298492\n",
            "49\t0.04825117\t0.59825636\t89.73416138\t89.72339630\n",
            "50\t0.04693762\t0.60680299\t89.86953735\t89.85783386\n",
            "51\t0.04757245\t0.61661935\t89.99829865\t89.98693085\n",
            "52\t0.04833268\t0.61154946\t90.12354279\t90.11230469\n",
            "53\t0.05088182\t0.60487623\t90.24602509\t90.23433685\n",
            "54\t0.04572169\t0.60574174\t90.36373901\t90.35259247\n",
            "55\t0.04608624\t0.59230305\t90.47732544\t90.46569824\n",
            "56\t0.04077914\t0.60429984\t90.58776855\t90.57633972\n",
            "57\t0.04117016\t0.60729643\t90.69604492\t90.68461609\n",
            "58\t0.04271014\t0.60987374\t90.79890442\t90.78655243\n",
            "59\t0.04113921\t0.60698710\t90.89727783\t90.88562012\n",
            "60\t0.03975525\t0.60819876\t90.99523926\t90.98325348\n",
            "61\t0.03613241\t0.62205672\t91.08980560\t91.07720947\n",
            "62\t0.03910494\t0.60857356\t91.18013000\t91.16819763\n",
            "63\t0.03293076\t0.61171188\t91.27146149\t91.25984192\n",
            "64\t0.02992569\t0.61503542\t91.36036682\t91.34822845\n",
            "65\t0.03319902\t0.61391380\t91.44699860\t91.43536377\n",
            "66\t0.03407719\t0.60969466\t91.53124237\t91.51939392\n",
            "67\t0.02873874\t0.60664875\t91.61337280\t91.60176849\n",
            "68\t0.03243608\t0.62139138\t91.69332123\t91.68182373\n",
            "69\t0.02977537\t0.61900564\t91.77111053\t91.75925446\n",
            "70\t0.03121793\t0.61635309\t91.84719086\t91.83566284\n",
            "71\t0.02850157\t0.61612645\t91.92218781\t91.91053772\n",
            "72\t0.02832584\t0.61509050\t91.99380493\t91.98254395\n",
            "73\t0.03022717\t0.61702809\t92.06368256\t92.05189514\n",
            "74\t0.02933962\t0.62793543\t92.13157654\t92.11979675\n",
            "75\t0.02608274\t0.61966107\t92.19850922\t92.18680573\n",
            "76\t0.03054024\t0.62939939\t92.26296234\t92.25129700\n",
            "77\t0.02434677\t0.62585152\t92.32666016\t92.31480408\n",
            "78\t0.02559175\t0.62552071\t92.38857269\t92.37686157\n",
            "79\t0.02738662\t0.62365657\t92.44833374\t92.43624878\n",
            "80\t0.02445083\t0.63489407\t92.50732422\t92.49555969\n",
            "81\t0.02448168\t0.63066293\t92.56404114\t92.55227661\n",
            "82\t0.02466886\t0.63117877\t92.61979675\t92.60814667\n",
            "83\t0.02344810\t0.63419166\t92.67517090\t92.66353607\n",
            "84\t0.02272245\t0.62703315\t92.72927856\t92.71779633\n",
            "85\t0.02819029\t0.63377669\t92.78128052\t92.76992035\n",
            "86\t0.02765000\t0.63412934\t92.83215332\t92.82140350\n",
            "87\t0.02402053\t0.64031695\t92.88307190\t92.87166595\n",
            "88\t0.02050567\t0.63955865\t92.93282318\t92.92127228\n",
            "89\t0.02202749\t0.63165819\t92.98099518\t92.96963501\n",
            "90\t0.02399244\t0.63640087\t93.02825928\t93.01690674\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cFGTZvRw_Bl8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "save_checkpoint(model, True, 'radam_resnet18_9302.pth.tar')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kCJQ2vCwHWNp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ep_lookahead_losses = []\n",
        "for i in range(0, len(train_stats.losses), len(train_loader)):\n",
        "    if i != 0 :\n",
        "        ep_lookahead_losses.append(train_stats.losses[i])\n",
        "        \n",
        "ep_lookahead_lossesv = []\n",
        "for i in range(0, len(test_stats.losses), len(test_loader)):\n",
        "    if(i != 0):\n",
        "        ep_lookahead_lossesv.append(test_stats.losses[i])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iVM5wkzAHbhi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ep_lookahead_accs = []\n",
        "for i in range(0, len(train_stats.losses), len(train_loader)):\n",
        "    if i != 0 :\n",
        "        ep_lookahead_accs.append(train_stats.precs[i])\n",
        "        \n",
        "ep_lookahead_accsv = []\n",
        "for i in range(0, len(test_stats.losses), len(test_loader)):\n",
        "    if(i != 0):\n",
        "        ep_lookahead_accsv.append(test_stats.precs[i])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ss4EKzWCHfXl",
        "colab_type": "code",
        "outputId": "2d95225f-fdba-4883-e90c-5fa50b222666",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 371
        }
      },
      "source": [
        "plt.xlabel(\"Epochs\")\n",
        "plt.ylabel(\"Accuracy\")\n",
        "plt.plot(ep_lookahead_accs, 'r', label='Train')\n",
        "plt.plot(ep_lookahead_accsv, 'b', label='Val')\n",
        "plt.legend()\n",
        "plt.savefig('radam_resnet18_9302_acc.jpg')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/svg+xml": "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n  \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<!-- Created with matplotlib (https://matplotlib.org/) -->\n<svg height=\"262.19625pt\" version=\"1.1\" viewBox=\"0 0 382.603125 262.19625\" width=\"382.603125pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n <defs>\n  <style type=\"text/css\">\n*{stroke-linecap:butt;stroke-linejoin:round;}\n  </style>\n </defs>\n <g id=\"figure_1\">\n  <g id=\"patch_1\">\n   <path d=\"M 0 262.19625 \nL 382.603125 262.19625 \nL 382.603125 0 \nL 0 0 \nz\n\" style=\"fill:none;\"/>\n  </g>\n  <g id=\"axes_1\">\n   <g id=\"patch_2\">\n    <path d=\"M 40.603125 224.64 \nL 375.403125 224.64 \nL 375.403125 7.2 \nL 40.603125 7.2 \nz\n\" style=\"fill:#ffffff;\"/>\n   </g>\n   <g id=\"matplotlib.axis_1\">\n    <g id=\"xtick_1\">\n     <g id=\"line2d_1\">\n      <defs>\n       <path d=\"M 0 0 \nL 0 3.5 \n\" id=\"m8fb91fd3d6\" style=\"stroke:#000000;stroke-width:0.8;\"/>\n      </defs>\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"55.821307\" xlink:href=\"#m8fb91fd3d6\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_1\">\n      <!-- 0 -->\n      <defs>\n       <path d=\"M 31.78125 66.40625 \nQ 24.171875 66.40625 20.328125 58.90625 \nQ 16.5 51.421875 16.5 36.375 \nQ 16.5 21.390625 20.328125 13.890625 \nQ 24.171875 6.390625 31.78125 6.390625 \nQ 39.453125 6.390625 43.28125 13.890625 \nQ 47.125 21.390625 47.125 36.375 \nQ 47.125 51.421875 43.28125 58.90625 \nQ 39.453125 66.40625 31.78125 66.40625 \nz\nM 31.78125 74.21875 \nQ 44.046875 74.21875 50.515625 64.515625 \nQ 56.984375 54.828125 56.984375 36.375 \nQ 56.984375 17.96875 50.515625 8.265625 \nQ 44.046875 -1.421875 31.78125 -1.421875 \nQ 19.53125 -1.421875 13.0625 8.265625 \nQ 6.59375 17.96875 6.59375 36.375 \nQ 6.59375 54.828125 13.0625 64.515625 \nQ 19.53125 74.21875 31.78125 74.21875 \nz\n\" id=\"DejaVuSans-48\"/>\n      </defs>\n      <g transform=\"translate(52.640057 239.238438)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_2\">\n     <g id=\"line2d_2\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"124.994861\" xlink:href=\"#m8fb91fd3d6\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_2\">\n      <!-- 20 -->\n      <defs>\n       <path d=\"M 19.1875 8.296875 \nL 53.609375 8.296875 \nL 53.609375 0 \nL 7.328125 0 \nL 7.328125 8.296875 \nQ 12.9375 14.109375 22.625 23.890625 \nQ 32.328125 33.6875 34.8125 36.53125 \nQ 39.546875 41.84375 41.421875 45.53125 \nQ 43.3125 49.21875 43.3125 52.78125 \nQ 43.3125 58.59375 39.234375 62.25 \nQ 35.15625 65.921875 28.609375 65.921875 \nQ 23.96875 65.921875 18.8125 64.3125 \nQ 13.671875 62.703125 7.8125 59.421875 \nL 7.8125 69.390625 \nQ 13.765625 71.78125 18.9375 73 \nQ 24.125 74.21875 28.421875 74.21875 \nQ 39.75 74.21875 46.484375 68.546875 \nQ 53.21875 62.890625 53.21875 53.421875 \nQ 53.21875 48.921875 51.53125 44.890625 \nQ 49.859375 40.875 45.40625 35.40625 \nQ 44.1875 33.984375 37.640625 27.21875 \nQ 31.109375 20.453125 19.1875 8.296875 \nz\n\" id=\"DejaVuSans-50\"/>\n      </defs>\n      <g transform=\"translate(118.632361 239.238438)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-50\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_3\">\n     <g id=\"line2d_3\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"194.168414\" xlink:href=\"#m8fb91fd3d6\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_3\">\n      <!-- 40 -->\n      <defs>\n       <path d=\"M 37.796875 64.3125 \nL 12.890625 25.390625 \nL 37.796875 25.390625 \nz\nM 35.203125 72.90625 \nL 47.609375 72.90625 \nL 47.609375 25.390625 \nL 58.015625 25.390625 \nL 58.015625 17.1875 \nL 47.609375 17.1875 \nL 47.609375 0 \nL 37.796875 0 \nL 37.796875 17.1875 \nL 4.890625 17.1875 \nL 4.890625 26.703125 \nz\n\" id=\"DejaVuSans-52\"/>\n      </defs>\n      <g transform=\"translate(187.805914 239.238438)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-52\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_4\">\n     <g id=\"line2d_4\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"263.341968\" xlink:href=\"#m8fb91fd3d6\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_4\">\n      <!-- 60 -->\n      <defs>\n       <path d=\"M 33.015625 40.375 \nQ 26.375 40.375 22.484375 35.828125 \nQ 18.609375 31.296875 18.609375 23.390625 \nQ 18.609375 15.53125 22.484375 10.953125 \nQ 26.375 6.390625 33.015625 6.390625 \nQ 39.65625 6.390625 43.53125 10.953125 \nQ 47.40625 15.53125 47.40625 23.390625 \nQ 47.40625 31.296875 43.53125 35.828125 \nQ 39.65625 40.375 33.015625 40.375 \nz\nM 52.59375 71.296875 \nL 52.59375 62.3125 \nQ 48.875 64.0625 45.09375 64.984375 \nQ 41.3125 65.921875 37.59375 65.921875 \nQ 27.828125 65.921875 22.671875 59.328125 \nQ 17.53125 52.734375 16.796875 39.40625 \nQ 19.671875 43.65625 24.015625 45.921875 \nQ 28.375 48.1875 33.59375 48.1875 \nQ 44.578125 48.1875 50.953125 41.515625 \nQ 57.328125 34.859375 57.328125 23.390625 \nQ 57.328125 12.15625 50.6875 5.359375 \nQ 44.046875 -1.421875 33.015625 -1.421875 \nQ 20.359375 -1.421875 13.671875 8.265625 \nQ 6.984375 17.96875 6.984375 36.375 \nQ 6.984375 53.65625 15.1875 63.9375 \nQ 23.390625 74.21875 37.203125 74.21875 \nQ 40.921875 74.21875 44.703125 73.484375 \nQ 48.484375 72.75 52.59375 71.296875 \nz\n\" id=\"DejaVuSans-54\"/>\n      </defs>\n      <g transform=\"translate(256.979468 239.238438)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-54\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_5\">\n     <g id=\"line2d_5\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"332.515522\" xlink:href=\"#m8fb91fd3d6\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_5\">\n      <!-- 80 -->\n      <defs>\n       <path d=\"M 31.78125 34.625 \nQ 24.75 34.625 20.71875 30.859375 \nQ 16.703125 27.09375 16.703125 20.515625 \nQ 16.703125 13.921875 20.71875 10.15625 \nQ 24.75 6.390625 31.78125 6.390625 \nQ 38.8125 6.390625 42.859375 10.171875 \nQ 46.921875 13.96875 46.921875 20.515625 \nQ 46.921875 27.09375 42.890625 30.859375 \nQ 38.875 34.625 31.78125 34.625 \nz\nM 21.921875 38.8125 \nQ 15.578125 40.375 12.03125 44.71875 \nQ 8.5 49.078125 8.5 55.328125 \nQ 8.5 64.0625 14.71875 69.140625 \nQ 20.953125 74.21875 31.78125 74.21875 \nQ 42.671875 74.21875 48.875 69.140625 \nQ 55.078125 64.0625 55.078125 55.328125 \nQ 55.078125 49.078125 51.53125 44.71875 \nQ 48 40.375 41.703125 38.8125 \nQ 48.828125 37.15625 52.796875 32.3125 \nQ 56.78125 27.484375 56.78125 20.515625 \nQ 56.78125 9.90625 50.3125 4.234375 \nQ 43.84375 -1.421875 31.78125 -1.421875 \nQ 19.734375 -1.421875 13.25 4.234375 \nQ 6.78125 9.90625 6.78125 20.515625 \nQ 6.78125 27.484375 10.78125 32.3125 \nQ 14.796875 37.15625 21.921875 38.8125 \nz\nM 18.3125 54.390625 \nQ 18.3125 48.734375 21.84375 45.5625 \nQ 25.390625 42.390625 31.78125 42.390625 \nQ 38.140625 42.390625 41.71875 45.5625 \nQ 45.3125 48.734375 45.3125 54.390625 \nQ 45.3125 60.0625 41.71875 63.234375 \nQ 38.140625 66.40625 31.78125 66.40625 \nQ 25.390625 66.40625 21.84375 63.234375 \nQ 18.3125 60.0625 18.3125 54.390625 \nz\n\" id=\"DejaVuSans-56\"/>\n      </defs>\n      <g transform=\"translate(326.153022 239.238438)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-56\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"text_6\">\n     <!-- Epochs -->\n     <defs>\n      <path d=\"M 9.8125 72.90625 \nL 55.90625 72.90625 \nL 55.90625 64.59375 \nL 19.671875 64.59375 \nL 19.671875 43.015625 \nL 54.390625 43.015625 \nL 54.390625 34.71875 \nL 19.671875 34.71875 \nL 19.671875 8.296875 \nL 56.78125 8.296875 \nL 56.78125 0 \nL 9.8125 0 \nz\n\" id=\"DejaVuSans-69\"/>\n      <path d=\"M 18.109375 8.203125 \nL 18.109375 -20.796875 \nL 9.078125 -20.796875 \nL 9.078125 54.6875 \nL 18.109375 54.6875 \nL 18.109375 46.390625 \nQ 20.953125 51.265625 25.265625 53.625 \nQ 29.59375 56 35.59375 56 \nQ 45.5625 56 51.78125 48.09375 \nQ 58.015625 40.1875 58.015625 27.296875 \nQ 58.015625 14.40625 51.78125 6.484375 \nQ 45.5625 -1.421875 35.59375 -1.421875 \nQ 29.59375 -1.421875 25.265625 0.953125 \nQ 20.953125 3.328125 18.109375 8.203125 \nz\nM 48.6875 27.296875 \nQ 48.6875 37.203125 44.609375 42.84375 \nQ 40.53125 48.484375 33.40625 48.484375 \nQ 26.265625 48.484375 22.1875 42.84375 \nQ 18.109375 37.203125 18.109375 27.296875 \nQ 18.109375 17.390625 22.1875 11.75 \nQ 26.265625 6.109375 33.40625 6.109375 \nQ 40.53125 6.109375 44.609375 11.75 \nQ 48.6875 17.390625 48.6875 27.296875 \nz\n\" id=\"DejaVuSans-112\"/>\n      <path d=\"M 30.609375 48.390625 \nQ 23.390625 48.390625 19.1875 42.75 \nQ 14.984375 37.109375 14.984375 27.296875 \nQ 14.984375 17.484375 19.15625 11.84375 \nQ 23.34375 6.203125 30.609375 6.203125 \nQ 37.796875 6.203125 41.984375 11.859375 \nQ 46.1875 17.53125 46.1875 27.296875 \nQ 46.1875 37.015625 41.984375 42.703125 \nQ 37.796875 48.390625 30.609375 48.390625 \nz\nM 30.609375 56 \nQ 42.328125 56 49.015625 48.375 \nQ 55.71875 40.765625 55.71875 27.296875 \nQ 55.71875 13.875 49.015625 6.21875 \nQ 42.328125 -1.421875 30.609375 -1.421875 \nQ 18.84375 -1.421875 12.171875 6.21875 \nQ 5.515625 13.875 5.515625 27.296875 \nQ 5.515625 40.765625 12.171875 48.375 \nQ 18.84375 56 30.609375 56 \nz\n\" id=\"DejaVuSans-111\"/>\n      <path d=\"M 48.78125 52.59375 \nL 48.78125 44.1875 \nQ 44.96875 46.296875 41.140625 47.34375 \nQ 37.3125 48.390625 33.40625 48.390625 \nQ 24.65625 48.390625 19.8125 42.84375 \nQ 14.984375 37.3125 14.984375 27.296875 \nQ 14.984375 17.28125 19.8125 11.734375 \nQ 24.65625 6.203125 33.40625 6.203125 \nQ 37.3125 6.203125 41.140625 7.25 \nQ 44.96875 8.296875 48.78125 10.40625 \nL 48.78125 2.09375 \nQ 45.015625 0.34375 40.984375 -0.53125 \nQ 36.96875 -1.421875 32.421875 -1.421875 \nQ 20.0625 -1.421875 12.78125 6.34375 \nQ 5.515625 14.109375 5.515625 27.296875 \nQ 5.515625 40.671875 12.859375 48.328125 \nQ 20.21875 56 33.015625 56 \nQ 37.15625 56 41.109375 55.140625 \nQ 45.0625 54.296875 48.78125 52.59375 \nz\n\" id=\"DejaVuSans-99\"/>\n      <path d=\"M 54.890625 33.015625 \nL 54.890625 0 \nL 45.90625 0 \nL 45.90625 32.71875 \nQ 45.90625 40.484375 42.875 44.328125 \nQ 39.84375 48.1875 33.796875 48.1875 \nQ 26.515625 48.1875 22.3125 43.546875 \nQ 18.109375 38.921875 18.109375 30.90625 \nL 18.109375 0 \nL 9.078125 0 \nL 9.078125 75.984375 \nL 18.109375 75.984375 \nL 18.109375 46.1875 \nQ 21.34375 51.125 25.703125 53.5625 \nQ 30.078125 56 35.796875 56 \nQ 45.21875 56 50.046875 50.171875 \nQ 54.890625 44.34375 54.890625 33.015625 \nz\n\" id=\"DejaVuSans-104\"/>\n      <path d=\"M 44.28125 53.078125 \nL 44.28125 44.578125 \nQ 40.484375 46.53125 36.375 47.5 \nQ 32.28125 48.484375 27.875 48.484375 \nQ 21.1875 48.484375 17.84375 46.4375 \nQ 14.5 44.390625 14.5 40.28125 \nQ 14.5 37.15625 16.890625 35.375 \nQ 19.28125 33.59375 26.515625 31.984375 \nL 29.59375 31.296875 \nQ 39.15625 29.25 43.1875 25.515625 \nQ 47.21875 21.78125 47.21875 15.09375 \nQ 47.21875 7.46875 41.1875 3.015625 \nQ 35.15625 -1.421875 24.609375 -1.421875 \nQ 20.21875 -1.421875 15.453125 -0.5625 \nQ 10.6875 0.296875 5.421875 2 \nL 5.421875 11.28125 \nQ 10.40625 8.6875 15.234375 7.390625 \nQ 20.0625 6.109375 24.8125 6.109375 \nQ 31.15625 6.109375 34.5625 8.28125 \nQ 37.984375 10.453125 37.984375 14.40625 \nQ 37.984375 18.0625 35.515625 20.015625 \nQ 33.0625 21.96875 24.703125 23.78125 \nL 21.578125 24.515625 \nQ 13.234375 26.265625 9.515625 29.90625 \nQ 5.8125 33.546875 5.8125 39.890625 \nQ 5.8125 47.609375 11.28125 51.796875 \nQ 16.75 56 26.8125 56 \nQ 31.78125 56 36.171875 55.265625 \nQ 40.578125 54.546875 44.28125 53.078125 \nz\n\" id=\"DejaVuSans-115\"/>\n     </defs>\n     <g transform=\"translate(190.0875 252.916563)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-69\"/>\n      <use x=\"63.183594\" xlink:href=\"#DejaVuSans-112\"/>\n      <use x=\"126.660156\" xlink:href=\"#DejaVuSans-111\"/>\n      <use x=\"187.841797\" xlink:href=\"#DejaVuSans-99\"/>\n      <use x=\"242.822266\" xlink:href=\"#DejaVuSans-104\"/>\n      <use x=\"306.201172\" xlink:href=\"#DejaVuSans-115\"/>\n     </g>\n    </g>\n   </g>\n   <g id=\"matplotlib.axis_2\">\n    <g id=\"ytick_1\">\n     <g id=\"line2d_6\">\n      <defs>\n       <path d=\"M 0 0 \nL -3.5 0 \n\" id=\"mab2ffbe999\" style=\"stroke:#000000;stroke-width:0.8;\"/>\n      </defs>\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"40.603125\" xlink:href=\"#mab2ffbe999\" y=\"196.183214\"/>\n      </g>\n     </g>\n     <g id=\"text_7\">\n      <!-- 50 -->\n      <defs>\n       <path d=\"M 10.796875 72.90625 \nL 49.515625 72.90625 \nL 49.515625 64.59375 \nL 19.828125 64.59375 \nL 19.828125 46.734375 \nQ 21.96875 47.46875 24.109375 47.828125 \nQ 26.265625 48.1875 28.421875 48.1875 \nQ 40.625 48.1875 47.75 41.5 \nQ 54.890625 34.8125 54.890625 23.390625 \nQ 54.890625 11.625 47.5625 5.09375 \nQ 40.234375 -1.421875 26.90625 -1.421875 \nQ 22.3125 -1.421875 17.546875 -0.640625 \nQ 12.796875 0.140625 7.71875 1.703125 \nL 7.71875 11.625 \nQ 12.109375 9.234375 16.796875 8.0625 \nQ 21.484375 6.890625 26.703125 6.890625 \nQ 35.15625 6.890625 40.078125 11.328125 \nQ 45.015625 15.765625 45.015625 23.390625 \nQ 45.015625 31 40.078125 35.4375 \nQ 35.15625 39.890625 26.703125 39.890625 \nQ 22.75 39.890625 18.8125 39.015625 \nQ 14.890625 38.140625 10.796875 36.28125 \nz\n\" id=\"DejaVuSans-53\"/>\n      </defs>\n      <g transform=\"translate(20.878125 199.982433)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-53\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_2\">\n     <g id=\"line2d_7\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"40.603125\" xlink:href=\"#mab2ffbe999\" y=\"154.559416\"/>\n      </g>\n     </g>\n     <g id=\"text_8\">\n      <!-- 60 -->\n      <g transform=\"translate(20.878125 158.358634)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-54\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_3\">\n     <g id=\"line2d_8\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"40.603125\" xlink:href=\"#mab2ffbe999\" y=\"112.935617\"/>\n      </g>\n     </g>\n     <g id=\"text_9\">\n      <!-- 70 -->\n      <defs>\n       <path d=\"M 8.203125 72.90625 \nL 55.078125 72.90625 \nL 55.078125 68.703125 \nL 28.609375 0 \nL 18.3125 0 \nL 43.21875 64.59375 \nL 8.203125 64.59375 \nz\n\" id=\"DejaVuSans-55\"/>\n      </defs>\n      <g transform=\"translate(20.878125 116.734836)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-55\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_4\">\n     <g id=\"line2d_9\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"40.603125\" xlink:href=\"#mab2ffbe999\" y=\"71.311819\"/>\n      </g>\n     </g>\n     <g id=\"text_10\">\n      <!-- 80 -->\n      <g transform=\"translate(20.878125 75.111038)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-56\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_5\">\n     <g id=\"line2d_10\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"40.603125\" xlink:href=\"#mab2ffbe999\" y=\"29.688021\"/>\n      </g>\n     </g>\n     <g id=\"text_11\">\n      <!-- 90 -->\n      <defs>\n       <path d=\"M 10.984375 1.515625 \nL 10.984375 10.5 \nQ 14.703125 8.734375 18.5 7.8125 \nQ 22.3125 6.890625 25.984375 6.890625 \nQ 35.75 6.890625 40.890625 13.453125 \nQ 46.046875 20.015625 46.78125 33.40625 \nQ 43.953125 29.203125 39.59375 26.953125 \nQ 35.25 24.703125 29.984375 24.703125 \nQ 19.046875 24.703125 12.671875 31.3125 \nQ 6.296875 37.9375 6.296875 49.421875 \nQ 6.296875 60.640625 12.9375 67.421875 \nQ 19.578125 74.21875 30.609375 74.21875 \nQ 43.265625 74.21875 49.921875 64.515625 \nQ 56.59375 54.828125 56.59375 36.375 \nQ 56.59375 19.140625 48.40625 8.859375 \nQ 40.234375 -1.421875 26.421875 -1.421875 \nQ 22.703125 -1.421875 18.890625 -0.6875 \nQ 15.09375 0.046875 10.984375 1.515625 \nz\nM 30.609375 32.421875 \nQ 37.25 32.421875 41.125 36.953125 \nQ 45.015625 41.5 45.015625 49.421875 \nQ 45.015625 57.28125 41.125 61.84375 \nQ 37.25 66.40625 30.609375 66.40625 \nQ 23.96875 66.40625 20.09375 61.84375 \nQ 16.21875 57.28125 16.21875 49.421875 \nQ 16.21875 41.5 20.09375 36.953125 \nQ 23.96875 32.421875 30.609375 32.421875 \nz\n\" id=\"DejaVuSans-57\"/>\n      </defs>\n      <g transform=\"translate(20.878125 33.487239)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-57\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"text_12\">\n     <!-- Accuracy -->\n     <defs>\n      <path d=\"M 34.1875 63.1875 \nL 20.796875 26.90625 \nL 47.609375 26.90625 \nz\nM 28.609375 72.90625 \nL 39.796875 72.90625 \nL 67.578125 0 \nL 57.328125 0 \nL 50.6875 18.703125 \nL 17.828125 18.703125 \nL 11.1875 0 \nL 0.78125 0 \nz\n\" id=\"DejaVuSans-65\"/>\n      <path d=\"M 8.5 21.578125 \nL 8.5 54.6875 \nL 17.484375 54.6875 \nL 17.484375 21.921875 \nQ 17.484375 14.15625 20.5 10.265625 \nQ 23.53125 6.390625 29.59375 6.390625 \nQ 36.859375 6.390625 41.078125 11.03125 \nQ 45.3125 15.671875 45.3125 23.6875 \nL 45.3125 54.6875 \nL 54.296875 54.6875 \nL 54.296875 0 \nL 45.3125 0 \nL 45.3125 8.40625 \nQ 42.046875 3.421875 37.71875 1 \nQ 33.40625 -1.421875 27.6875 -1.421875 \nQ 18.265625 -1.421875 13.375 4.4375 \nQ 8.5 10.296875 8.5 21.578125 \nz\nM 31.109375 56 \nz\n\" id=\"DejaVuSans-117\"/>\n      <path d=\"M 41.109375 46.296875 \nQ 39.59375 47.171875 37.8125 47.578125 \nQ 36.03125 48 33.890625 48 \nQ 26.265625 48 22.1875 43.046875 \nQ 18.109375 38.09375 18.109375 28.8125 \nL 18.109375 0 \nL 9.078125 0 \nL 9.078125 54.6875 \nL 18.109375 54.6875 \nL 18.109375 46.1875 \nQ 20.953125 51.171875 25.484375 53.578125 \nQ 30.03125 56 36.53125 56 \nQ 37.453125 56 38.578125 55.875 \nQ 39.703125 55.765625 41.0625 55.515625 \nz\n\" id=\"DejaVuSans-114\"/>\n      <path d=\"M 34.28125 27.484375 \nQ 23.390625 27.484375 19.1875 25 \nQ 14.984375 22.515625 14.984375 16.5 \nQ 14.984375 11.71875 18.140625 8.90625 \nQ 21.296875 6.109375 26.703125 6.109375 \nQ 34.1875 6.109375 38.703125 11.40625 \nQ 43.21875 16.703125 43.21875 25.484375 \nL 43.21875 27.484375 \nz\nM 52.203125 31.203125 \nL 52.203125 0 \nL 43.21875 0 \nL 43.21875 8.296875 \nQ 40.140625 3.328125 35.546875 0.953125 \nQ 30.953125 -1.421875 24.3125 -1.421875 \nQ 15.921875 -1.421875 10.953125 3.296875 \nQ 6 8.015625 6 15.921875 \nQ 6 25.140625 12.171875 29.828125 \nQ 18.359375 34.515625 30.609375 34.515625 \nL 43.21875 34.515625 \nL 43.21875 35.40625 \nQ 43.21875 41.609375 39.140625 45 \nQ 35.0625 48.390625 27.6875 48.390625 \nQ 23 48.390625 18.546875 47.265625 \nQ 14.109375 46.140625 10.015625 43.890625 \nL 10.015625 52.203125 \nQ 14.9375 54.109375 19.578125 55.046875 \nQ 24.21875 56 28.609375 56 \nQ 40.484375 56 46.34375 49.84375 \nQ 52.203125 43.703125 52.203125 31.203125 \nz\n\" id=\"DejaVuSans-97\"/>\n      <path d=\"M 32.171875 -5.078125 \nQ 28.375 -14.84375 24.75 -17.8125 \nQ 21.140625 -20.796875 15.09375 -20.796875 \nL 7.90625 -20.796875 \nL 7.90625 -13.28125 \nL 13.1875 -13.28125 \nQ 16.890625 -13.28125 18.9375 -11.515625 \nQ 21 -9.765625 23.484375 -3.21875 \nL 25.09375 0.875 \nL 2.984375 54.6875 \nL 12.5 54.6875 \nL 29.59375 11.921875 \nL 46.6875 54.6875 \nL 56.203125 54.6875 \nz\n\" id=\"DejaVuSans-121\"/>\n     </defs>\n     <g transform=\"translate(14.798438 138.834844)rotate(-90)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-65\"/>\n      <use x=\"68.392578\" xlink:href=\"#DejaVuSans-99\"/>\n      <use x=\"123.373047\" xlink:href=\"#DejaVuSans-99\"/>\n      <use x=\"178.353516\" xlink:href=\"#DejaVuSans-117\"/>\n      <use x=\"241.732422\" xlink:href=\"#DejaVuSans-114\"/>\n      <use x=\"282.845703\" xlink:href=\"#DejaVuSans-97\"/>\n      <use x=\"344.125\" xlink:href=\"#DejaVuSans-99\"/>\n      <use x=\"399.105469\" xlink:href=\"#DejaVuSans-121\"/>\n     </g>\n    </g>\n   </g>\n   <g id=\"line2d_11\">\n    <path clip-path=\"url(#p0cb82f06c5)\" d=\"M 55.821307 214.756364 \nL 59.279985 166.537071 \nL 62.738662 142.928731 \nL 66.19734 127.325603 \nL 69.656018 116.173377 \nL 73.114695 107.732865 \nL 76.573373 100.739176 \nL 80.032051 95.023589 \nL 83.490728 90.072316 \nL 86.949406 85.761475 \nL 90.408084 81.972646 \nL 93.866761 78.55216 \nL 97.325439 75.52069 \nL 100.784117 72.749306 \nL 104.242794 70.168905 \nL 107.701472 67.851637 \nL 111.16015 65.670224 \nL 114.618827 63.666425 \nL 118.077505 61.740651 \nL 121.536183 59.968959 \nL 124.994861 58.311781 \nL 128.453538 56.742823 \nL 131.912216 55.273484 \nL 135.370894 53.823136 \nL 138.829571 52.457418 \nL 142.288249 51.196466 \nL 145.746927 50.014808 \nL 149.205604 48.878881 \nL 152.664282 47.816278 \nL 156.12296 46.793054 \nL 159.581637 45.787963 \nL 163.040315 44.691127 \nL 166.498993 43.575651 \nL 169.95767 42.498599 \nL 173.416348 41.487093 \nL 176.875026 40.50998 \nL 180.333704 39.574625 \nL 183.792381 38.676299 \nL 187.251059 37.795535 \nL 190.709737 36.970788 \nL 194.168414 36.178147 \nL 197.627092 35.41088 \nL 201.08577 34.678958 \nL 204.544447 33.986572 \nL 208.003125 33.306857 \nL 211.461803 32.665822 \nL 214.92048 32.03425 \nL 218.379158 31.422144 \nL 221.837836 30.837762 \nL 225.296513 30.278119 \nL 228.755191 29.740832 \nL 232.213869 29.218978 \nL 235.672546 28.711224 \nL 239.131224 28.21919 \nL 242.589902 27.748083 \nL 246.04858 27.287837 \nL 249.507257 26.837182 \nL 252.965935 26.412916 \nL 256.424613 26.00059 \nL 259.88329 25.594044 \nL 263.341968 25.203186 \nL 266.800646 24.824268 \nL 270.259323 24.443191 \nL 273.718001 24.075102 \nL 277.176679 23.712348 \nL 280.635356 23.36271 \nL 284.094034 23.019836 \nL 287.552712 22.686806 \nL 291.011389 22.364383 \nL 294.470067 22.046247 \nL 297.928745 21.734843 \nL 301.387423 21.434999 \nL 304.8461 21.146364 \nL 308.304778 20.8637 \nL 311.763456 20.584911 \nL 315.222133 20.316569 \nL 318.680811 20.052133 \nL 322.139489 19.793858 \nL 325.598166 19.546697 \nL 329.056844 19.299696 \nL 332.515522 19.063714 \nL 335.974199 18.831193 \nL 339.432877 18.600578 \nL 342.891555 18.374758 \nL 346.350232 18.157957 \nL 349.80891 17.943664 \nL 353.267588 17.734326 \nL 356.726265 17.527877 \nL 360.184943 17.3267 \n\" style=\"fill:none;stroke:#ff0000;stroke-linecap:square;stroke-width:1.5;\"/>\n   </g>\n   <g id=\"line2d_12\">\n    <path clip-path=\"url(#p0cb82f06c5)\" d=\"M 55.821307 172.791374 \nL 59.279985 146.433705 \nL 62.738662 129.708987 \nL 66.19734 117.834461 \nL 69.656018 108.911029 \nL 73.114695 101.747983 \nL 76.573373 95.798002 \nL 80.032051 90.705698 \nL 83.490728 86.312449 \nL 86.949406 82.397833 \nL 90.408084 78.898527 \nL 93.866761 75.832602 \nL 97.325439 72.998276 \nL 100.784117 70.39644 \nL 104.242794 68.025028 \nL 107.701472 65.836914 \nL 111.16015 63.804629 \nL 114.618827 61.8464 \nL 118.077505 60.066324 \nL 121.536183 58.378946 \nL 124.994861 56.801572 \nL 128.453538 55.327152 \nL 131.912216 53.882997 \nL 135.370894 52.497654 \nL 138.829571 51.224888 \nL 142.288249 50.029893 \nL 145.746927 48.877642 \nL 149.205604 47.809705 \nL 152.664282 46.785147 \nL 156.12296 45.776912 \nL 159.581637 44.697352 \nL 163.040315 43.573904 \nL 166.498993 42.497551 \nL 169.95767 41.471469 \nL 173.416348 40.493943 \nL 176.875026 39.55608 \nL 180.333704 38.654832 \nL 183.792381 37.772829 \nL 187.251059 36.938746 \nL 190.709737 36.148233 \nL 194.168414 35.378965 \nL 197.627092 34.649488 \nL 201.08577 33.951545 \nL 204.544447 33.270941 \nL 208.003125 32.62638 \nL 211.461803 31.996936 \nL 214.92048 31.383243 \nL 218.379158 30.794478 \nL 221.837836 30.2315 \nL 225.296513 29.695388 \nL 228.755191 29.173884 \nL 232.213869 28.664542 \nL 235.672546 28.174445 \nL 239.131224 27.701814 \nL 242.589902 27.241917 \nL 246.04858 26.791103 \nL 249.507257 26.36309 \nL 252.965935 25.953464 \nL 256.424613 25.545774 \nL 259.88329 25.152344 \nL 263.341968 24.776157 \nL 266.800646 24.396128 \nL 270.259323 24.025975 \nL 273.718001 23.665412 \nL 277.176679 23.314758 \nL 280.635356 22.9729 \nL 284.094034 22.640251 \nL 287.552712 22.316335 \nL 291.011389 21.999692 \nL 294.470067 21.687685 \nL 297.928745 21.389301 \nL 301.387423 21.098539 \nL 304.8461 20.815843 \nL 308.304778 20.537403 \nL 311.763456 20.268871 \nL 315.222133 20.003768 \nL 318.680811 19.746191 \nL 322.139489 19.497443 \nL 325.598166 19.251871 \nL 329.056844 19.015793 \nL 332.515522 18.783749 \nL 335.974199 18.553324 \nL 339.432877 18.328012 \nL 342.891555 18.111592 \nL 346.350232 17.899904 \nL 349.80891 17.687961 \nL 353.267588 17.481005 \nL 356.726265 17.280209 \nL 360.184943 17.083636 \n\" style=\"fill:none;stroke:#0000ff;stroke-linecap:square;stroke-width:1.5;\"/>\n   </g>\n   <g id=\"patch_3\">\n    <path d=\"M 40.603125 224.64 \nL 40.603125 7.2 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_4\">\n    <path d=\"M 375.403125 224.64 \nL 375.403125 7.2 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_5\">\n    <path d=\"M 40.603125 224.64 \nL 375.403125 224.64 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_6\">\n    <path d=\"M 40.603125 7.2 \nL 375.403125 7.2 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"legend_1\">\n    <g id=\"patch_7\">\n     <path d=\"M 47.603125 44.55625 \nL 105.04375 44.55625 \nQ 107.04375 44.55625 107.04375 42.55625 \nL 107.04375 14.2 \nQ 107.04375 12.2 105.04375 12.2 \nL 47.603125 12.2 \nQ 45.603125 12.2 45.603125 14.2 \nL 45.603125 42.55625 \nQ 45.603125 44.55625 47.603125 44.55625 \nz\n\" style=\"fill:#ffffff;opacity:0.8;stroke:#cccccc;stroke-linejoin:miter;\"/>\n    </g>\n    <g id=\"line2d_13\">\n     <path d=\"M 49.603125 20.298438 \nL 69.603125 20.298438 \n\" style=\"fill:none;stroke:#ff0000;stroke-linecap:square;stroke-width:1.5;\"/>\n    </g>\n    <g id=\"line2d_14\"/>\n    <g id=\"text_13\">\n     <!-- Train -->\n     <defs>\n      <path d=\"M -0.296875 72.90625 \nL 61.375 72.90625 \nL 61.375 64.59375 \nL 35.5 64.59375 \nL 35.5 0 \nL 25.59375 0 \nL 25.59375 64.59375 \nL -0.296875 64.59375 \nz\n\" id=\"DejaVuSans-84\"/>\n      <path d=\"M 9.421875 54.6875 \nL 18.40625 54.6875 \nL 18.40625 0 \nL 9.421875 0 \nz\nM 9.421875 75.984375 \nL 18.40625 75.984375 \nL 18.40625 64.59375 \nL 9.421875 64.59375 \nz\n\" id=\"DejaVuSans-105\"/>\n      <path d=\"M 54.890625 33.015625 \nL 54.890625 0 \nL 45.90625 0 \nL 45.90625 32.71875 \nQ 45.90625 40.484375 42.875 44.328125 \nQ 39.84375 48.1875 33.796875 48.1875 \nQ 26.515625 48.1875 22.3125 43.546875 \nQ 18.109375 38.921875 18.109375 30.90625 \nL 18.109375 0 \nL 9.078125 0 \nL 9.078125 54.6875 \nL 18.109375 54.6875 \nL 18.109375 46.1875 \nQ 21.34375 51.125 25.703125 53.5625 \nQ 30.078125 56 35.796875 56 \nQ 45.21875 56 50.046875 50.171875 \nQ 54.890625 44.34375 54.890625 33.015625 \nz\n\" id=\"DejaVuSans-110\"/>\n     </defs>\n     <g transform=\"translate(77.603125 23.798438)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-84\"/>\n      <use x=\"60.865234\" xlink:href=\"#DejaVuSans-114\"/>\n      <use x=\"101.978516\" xlink:href=\"#DejaVuSans-97\"/>\n      <use x=\"163.257812\" xlink:href=\"#DejaVuSans-105\"/>\n      <use x=\"191.041016\" xlink:href=\"#DejaVuSans-110\"/>\n     </g>\n    </g>\n    <g id=\"line2d_15\">\n     <path d=\"M 49.603125 34.976562 \nL 69.603125 34.976562 \n\" style=\"fill:none;stroke:#0000ff;stroke-linecap:square;stroke-width:1.5;\"/>\n    </g>\n    <g id=\"line2d_16\"/>\n    <g id=\"text_14\">\n     <!-- Val -->\n     <defs>\n      <path d=\"M 28.609375 0 \nL 0.78125 72.90625 \nL 11.078125 72.90625 \nL 34.1875 11.53125 \nL 57.328125 72.90625 \nL 67.578125 72.90625 \nL 39.796875 0 \nz\n\" id=\"DejaVuSans-86\"/>\n      <path d=\"M 9.421875 75.984375 \nL 18.40625 75.984375 \nL 18.40625 0 \nL 9.421875 0 \nz\n\" id=\"DejaVuSans-108\"/>\n     </defs>\n     <g transform=\"translate(77.603125 38.476562)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-86\"/>\n      <use x=\"68.298828\" xlink:href=\"#DejaVuSans-97\"/>\n      <use x=\"129.578125\" xlink:href=\"#DejaVuSans-108\"/>\n     </g>\n    </g>\n   </g>\n  </g>\n </g>\n <defs>\n  <clipPath id=\"p0cb82f06c5\">\n   <rect height=\"217.44\" width=\"334.8\" x=\"40.603125\" y=\"7.2\"/>\n  </clipPath>\n </defs>\n</svg>\n"
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uDO1w_zSIX08",
        "colab_type": "code",
        "outputId": "6dd5725a-7110-4e3e-a148-740f1e94769f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 371
        }
      },
      "source": [
        "plt.xlabel(\"Epochs\")\n",
        "plt.ylabel(\"Losses\")\n",
        "plt.plot(ep_lookahead_losses, 'r', label='Train')\n",
        "plt.plot(ep_lookahead_lossesv, 'b', label='Val')\n",
        "plt.legend()\n",
        "plt.savefig('radam_resnet18_9302_loss.jpg')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/svg+xml": "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n  \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<!-- Created with matplotlib (https://matplotlib.org/) -->\n<svg height=\"262.19625pt\" version=\"1.1\" viewBox=\"0 0 385.78125 262.19625\" width=\"385.78125pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n <defs>\n  <style type=\"text/css\">\n*{stroke-linecap:butt;stroke-linejoin:round;}\n  </style>\n </defs>\n <g id=\"figure_1\">\n  <g id=\"patch_1\">\n   <path d=\"M 0 262.19625 \nL 385.78125 262.19625 \nL 385.78125 0 \nL 0 0 \nz\n\" style=\"fill:none;\"/>\n  </g>\n  <g id=\"axes_1\">\n   <g id=\"patch_2\">\n    <path d=\"M 43.78125 224.64 \nL 378.58125 224.64 \nL 378.58125 7.2 \nL 43.78125 7.2 \nz\n\" style=\"fill:#ffffff;\"/>\n   </g>\n   <g id=\"matplotlib.axis_1\">\n    <g id=\"xtick_1\">\n     <g id=\"line2d_1\">\n      <defs>\n       <path d=\"M 0 0 \nL 0 3.5 \n\" id=\"m8a71ce57d2\" style=\"stroke:#000000;stroke-width:0.8;\"/>\n      </defs>\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"58.999432\" xlink:href=\"#m8a71ce57d2\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_1\">\n      <!-- 0 -->\n      <defs>\n       <path d=\"M 31.78125 66.40625 \nQ 24.171875 66.40625 20.328125 58.90625 \nQ 16.5 51.421875 16.5 36.375 \nQ 16.5 21.390625 20.328125 13.890625 \nQ 24.171875 6.390625 31.78125 6.390625 \nQ 39.453125 6.390625 43.28125 13.890625 \nQ 47.125 21.390625 47.125 36.375 \nQ 47.125 51.421875 43.28125 58.90625 \nQ 39.453125 66.40625 31.78125 66.40625 \nz\nM 31.78125 74.21875 \nQ 44.046875 74.21875 50.515625 64.515625 \nQ 56.984375 54.828125 56.984375 36.375 \nQ 56.984375 17.96875 50.515625 8.265625 \nQ 44.046875 -1.421875 31.78125 -1.421875 \nQ 19.53125 -1.421875 13.0625 8.265625 \nQ 6.59375 17.96875 6.59375 36.375 \nQ 6.59375 54.828125 13.0625 64.515625 \nQ 19.53125 74.21875 31.78125 74.21875 \nz\n\" id=\"DejaVuSans-48\"/>\n      </defs>\n      <g transform=\"translate(55.818182 239.238438)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_2\">\n     <g id=\"line2d_2\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"128.172986\" xlink:href=\"#m8a71ce57d2\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_2\">\n      <!-- 20 -->\n      <defs>\n       <path d=\"M 19.1875 8.296875 \nL 53.609375 8.296875 \nL 53.609375 0 \nL 7.328125 0 \nL 7.328125 8.296875 \nQ 12.9375 14.109375 22.625 23.890625 \nQ 32.328125 33.6875 34.8125 36.53125 \nQ 39.546875 41.84375 41.421875 45.53125 \nQ 43.3125 49.21875 43.3125 52.78125 \nQ 43.3125 58.59375 39.234375 62.25 \nQ 35.15625 65.921875 28.609375 65.921875 \nQ 23.96875 65.921875 18.8125 64.3125 \nQ 13.671875 62.703125 7.8125 59.421875 \nL 7.8125 69.390625 \nQ 13.765625 71.78125 18.9375 73 \nQ 24.125 74.21875 28.421875 74.21875 \nQ 39.75 74.21875 46.484375 68.546875 \nQ 53.21875 62.890625 53.21875 53.421875 \nQ 53.21875 48.921875 51.53125 44.890625 \nQ 49.859375 40.875 45.40625 35.40625 \nQ 44.1875 33.984375 37.640625 27.21875 \nQ 31.109375 20.453125 19.1875 8.296875 \nz\n\" id=\"DejaVuSans-50\"/>\n      </defs>\n      <g transform=\"translate(121.810486 239.238438)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-50\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_3\">\n     <g id=\"line2d_3\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"197.346539\" xlink:href=\"#m8a71ce57d2\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_3\">\n      <!-- 40 -->\n      <defs>\n       <path d=\"M 37.796875 64.3125 \nL 12.890625 25.390625 \nL 37.796875 25.390625 \nz\nM 35.203125 72.90625 \nL 47.609375 72.90625 \nL 47.609375 25.390625 \nL 58.015625 25.390625 \nL 58.015625 17.1875 \nL 47.609375 17.1875 \nL 47.609375 0 \nL 37.796875 0 \nL 37.796875 17.1875 \nL 4.890625 17.1875 \nL 4.890625 26.703125 \nz\n\" id=\"DejaVuSans-52\"/>\n      </defs>\n      <g transform=\"translate(190.984039 239.238438)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-52\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_4\">\n     <g id=\"line2d_4\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"266.520093\" xlink:href=\"#m8a71ce57d2\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_4\">\n      <!-- 60 -->\n      <defs>\n       <path d=\"M 33.015625 40.375 \nQ 26.375 40.375 22.484375 35.828125 \nQ 18.609375 31.296875 18.609375 23.390625 \nQ 18.609375 15.53125 22.484375 10.953125 \nQ 26.375 6.390625 33.015625 6.390625 \nQ 39.65625 6.390625 43.53125 10.953125 \nQ 47.40625 15.53125 47.40625 23.390625 \nQ 47.40625 31.296875 43.53125 35.828125 \nQ 39.65625 40.375 33.015625 40.375 \nz\nM 52.59375 71.296875 \nL 52.59375 62.3125 \nQ 48.875 64.0625 45.09375 64.984375 \nQ 41.3125 65.921875 37.59375 65.921875 \nQ 27.828125 65.921875 22.671875 59.328125 \nQ 17.53125 52.734375 16.796875 39.40625 \nQ 19.671875 43.65625 24.015625 45.921875 \nQ 28.375 48.1875 33.59375 48.1875 \nQ 44.578125 48.1875 50.953125 41.515625 \nQ 57.328125 34.859375 57.328125 23.390625 \nQ 57.328125 12.15625 50.6875 5.359375 \nQ 44.046875 -1.421875 33.015625 -1.421875 \nQ 20.359375 -1.421875 13.671875 8.265625 \nQ 6.984375 17.96875 6.984375 36.375 \nQ 6.984375 53.65625 15.1875 63.9375 \nQ 23.390625 74.21875 37.203125 74.21875 \nQ 40.921875 74.21875 44.703125 73.484375 \nQ 48.484375 72.75 52.59375 71.296875 \nz\n\" id=\"DejaVuSans-54\"/>\n      </defs>\n      <g transform=\"translate(260.157593 239.238438)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-54\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_5\">\n     <g id=\"line2d_5\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"335.693647\" xlink:href=\"#m8a71ce57d2\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_5\">\n      <!-- 80 -->\n      <defs>\n       <path d=\"M 31.78125 34.625 \nQ 24.75 34.625 20.71875 30.859375 \nQ 16.703125 27.09375 16.703125 20.515625 \nQ 16.703125 13.921875 20.71875 10.15625 \nQ 24.75 6.390625 31.78125 6.390625 \nQ 38.8125 6.390625 42.859375 10.171875 \nQ 46.921875 13.96875 46.921875 20.515625 \nQ 46.921875 27.09375 42.890625 30.859375 \nQ 38.875 34.625 31.78125 34.625 \nz\nM 21.921875 38.8125 \nQ 15.578125 40.375 12.03125 44.71875 \nQ 8.5 49.078125 8.5 55.328125 \nQ 8.5 64.0625 14.71875 69.140625 \nQ 20.953125 74.21875 31.78125 74.21875 \nQ 42.671875 74.21875 48.875 69.140625 \nQ 55.078125 64.0625 55.078125 55.328125 \nQ 55.078125 49.078125 51.53125 44.71875 \nQ 48 40.375 41.703125 38.8125 \nQ 48.828125 37.15625 52.796875 32.3125 \nQ 56.78125 27.484375 56.78125 20.515625 \nQ 56.78125 9.90625 50.3125 4.234375 \nQ 43.84375 -1.421875 31.78125 -1.421875 \nQ 19.734375 -1.421875 13.25 4.234375 \nQ 6.78125 9.90625 6.78125 20.515625 \nQ 6.78125 27.484375 10.78125 32.3125 \nQ 14.796875 37.15625 21.921875 38.8125 \nz\nM 18.3125 54.390625 \nQ 18.3125 48.734375 21.84375 45.5625 \nQ 25.390625 42.390625 31.78125 42.390625 \nQ 38.140625 42.390625 41.71875 45.5625 \nQ 45.3125 48.734375 45.3125 54.390625 \nQ 45.3125 60.0625 41.71875 63.234375 \nQ 38.140625 66.40625 31.78125 66.40625 \nQ 25.390625 66.40625 21.84375 63.234375 \nQ 18.3125 60.0625 18.3125 54.390625 \nz\n\" id=\"DejaVuSans-56\"/>\n      </defs>\n      <g transform=\"translate(329.331147 239.238438)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-56\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"text_6\">\n     <!-- Epochs -->\n     <defs>\n      <path d=\"M 9.8125 72.90625 \nL 55.90625 72.90625 \nL 55.90625 64.59375 \nL 19.671875 64.59375 \nL 19.671875 43.015625 \nL 54.390625 43.015625 \nL 54.390625 34.71875 \nL 19.671875 34.71875 \nL 19.671875 8.296875 \nL 56.78125 8.296875 \nL 56.78125 0 \nL 9.8125 0 \nz\n\" id=\"DejaVuSans-69\"/>\n      <path d=\"M 18.109375 8.203125 \nL 18.109375 -20.796875 \nL 9.078125 -20.796875 \nL 9.078125 54.6875 \nL 18.109375 54.6875 \nL 18.109375 46.390625 \nQ 20.953125 51.265625 25.265625 53.625 \nQ 29.59375 56 35.59375 56 \nQ 45.5625 56 51.78125 48.09375 \nQ 58.015625 40.1875 58.015625 27.296875 \nQ 58.015625 14.40625 51.78125 6.484375 \nQ 45.5625 -1.421875 35.59375 -1.421875 \nQ 29.59375 -1.421875 25.265625 0.953125 \nQ 20.953125 3.328125 18.109375 8.203125 \nz\nM 48.6875 27.296875 \nQ 48.6875 37.203125 44.609375 42.84375 \nQ 40.53125 48.484375 33.40625 48.484375 \nQ 26.265625 48.484375 22.1875 42.84375 \nQ 18.109375 37.203125 18.109375 27.296875 \nQ 18.109375 17.390625 22.1875 11.75 \nQ 26.265625 6.109375 33.40625 6.109375 \nQ 40.53125 6.109375 44.609375 11.75 \nQ 48.6875 17.390625 48.6875 27.296875 \nz\n\" id=\"DejaVuSans-112\"/>\n      <path d=\"M 30.609375 48.390625 \nQ 23.390625 48.390625 19.1875 42.75 \nQ 14.984375 37.109375 14.984375 27.296875 \nQ 14.984375 17.484375 19.15625 11.84375 \nQ 23.34375 6.203125 30.609375 6.203125 \nQ 37.796875 6.203125 41.984375 11.859375 \nQ 46.1875 17.53125 46.1875 27.296875 \nQ 46.1875 37.015625 41.984375 42.703125 \nQ 37.796875 48.390625 30.609375 48.390625 \nz\nM 30.609375 56 \nQ 42.328125 56 49.015625 48.375 \nQ 55.71875 40.765625 55.71875 27.296875 \nQ 55.71875 13.875 49.015625 6.21875 \nQ 42.328125 -1.421875 30.609375 -1.421875 \nQ 18.84375 -1.421875 12.171875 6.21875 \nQ 5.515625 13.875 5.515625 27.296875 \nQ 5.515625 40.765625 12.171875 48.375 \nQ 18.84375 56 30.609375 56 \nz\n\" id=\"DejaVuSans-111\"/>\n      <path d=\"M 48.78125 52.59375 \nL 48.78125 44.1875 \nQ 44.96875 46.296875 41.140625 47.34375 \nQ 37.3125 48.390625 33.40625 48.390625 \nQ 24.65625 48.390625 19.8125 42.84375 \nQ 14.984375 37.3125 14.984375 27.296875 \nQ 14.984375 17.28125 19.8125 11.734375 \nQ 24.65625 6.203125 33.40625 6.203125 \nQ 37.3125 6.203125 41.140625 7.25 \nQ 44.96875 8.296875 48.78125 10.40625 \nL 48.78125 2.09375 \nQ 45.015625 0.34375 40.984375 -0.53125 \nQ 36.96875 -1.421875 32.421875 -1.421875 \nQ 20.0625 -1.421875 12.78125 6.34375 \nQ 5.515625 14.109375 5.515625 27.296875 \nQ 5.515625 40.671875 12.859375 48.328125 \nQ 20.21875 56 33.015625 56 \nQ 37.15625 56 41.109375 55.140625 \nQ 45.0625 54.296875 48.78125 52.59375 \nz\n\" id=\"DejaVuSans-99\"/>\n      <path d=\"M 54.890625 33.015625 \nL 54.890625 0 \nL 45.90625 0 \nL 45.90625 32.71875 \nQ 45.90625 40.484375 42.875 44.328125 \nQ 39.84375 48.1875 33.796875 48.1875 \nQ 26.515625 48.1875 22.3125 43.546875 \nQ 18.109375 38.921875 18.109375 30.90625 \nL 18.109375 0 \nL 9.078125 0 \nL 9.078125 75.984375 \nL 18.109375 75.984375 \nL 18.109375 46.1875 \nQ 21.34375 51.125 25.703125 53.5625 \nQ 30.078125 56 35.796875 56 \nQ 45.21875 56 50.046875 50.171875 \nQ 54.890625 44.34375 54.890625 33.015625 \nz\n\" id=\"DejaVuSans-104\"/>\n      <path d=\"M 44.28125 53.078125 \nL 44.28125 44.578125 \nQ 40.484375 46.53125 36.375 47.5 \nQ 32.28125 48.484375 27.875 48.484375 \nQ 21.1875 48.484375 17.84375 46.4375 \nQ 14.5 44.390625 14.5 40.28125 \nQ 14.5 37.15625 16.890625 35.375 \nQ 19.28125 33.59375 26.515625 31.984375 \nL 29.59375 31.296875 \nQ 39.15625 29.25 43.1875 25.515625 \nQ 47.21875 21.78125 47.21875 15.09375 \nQ 47.21875 7.46875 41.1875 3.015625 \nQ 35.15625 -1.421875 24.609375 -1.421875 \nQ 20.21875 -1.421875 15.453125 -0.5625 \nQ 10.6875 0.296875 5.421875 2 \nL 5.421875 11.28125 \nQ 10.40625 8.6875 15.234375 7.390625 \nQ 20.0625 6.109375 24.8125 6.109375 \nQ 31.15625 6.109375 34.5625 8.28125 \nQ 37.984375 10.453125 37.984375 14.40625 \nQ 37.984375 18.0625 35.515625 20.015625 \nQ 33.0625 21.96875 24.703125 23.78125 \nL 21.578125 24.515625 \nQ 13.234375 26.265625 9.515625 29.90625 \nQ 5.8125 33.546875 5.8125 39.890625 \nQ 5.8125 47.609375 11.28125 51.796875 \nQ 16.75 56 26.8125 56 \nQ 31.78125 56 36.171875 55.265625 \nQ 40.578125 54.546875 44.28125 53.078125 \nz\n\" id=\"DejaVuSans-115\"/>\n     </defs>\n     <g transform=\"translate(193.265625 252.916563)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-69\"/>\n      <use x=\"63.183594\" xlink:href=\"#DejaVuSans-112\"/>\n      <use x=\"126.660156\" xlink:href=\"#DejaVuSans-111\"/>\n      <use x=\"187.841797\" xlink:href=\"#DejaVuSans-99\"/>\n      <use x=\"242.822266\" xlink:href=\"#DejaVuSans-104\"/>\n      <use x=\"306.201172\" xlink:href=\"#DejaVuSans-115\"/>\n     </g>\n    </g>\n   </g>\n   <g id=\"matplotlib.axis_2\">\n    <g id=\"ytick_1\">\n     <g id=\"line2d_6\">\n      <defs>\n       <path d=\"M 0 0 \nL -3.5 0 \n\" id=\"me066859c92\" style=\"stroke:#000000;stroke-width:0.8;\"/>\n      </defs>\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#me066859c92\" y=\"215.377314\"/>\n      </g>\n     </g>\n     <g id=\"text_7\">\n      <!-- 0.0 -->\n      <defs>\n       <path d=\"M 10.6875 12.40625 \nL 21 12.40625 \nL 21 0 \nL 10.6875 0 \nz\n\" id=\"DejaVuSans-46\"/>\n      </defs>\n      <g transform=\"translate(20.878125 219.176532)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_2\">\n     <g id=\"line2d_7\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#me066859c92\" y=\"176.262969\"/>\n      </g>\n     </g>\n     <g id=\"text_8\">\n      <!-- 0.2 -->\n      <g transform=\"translate(20.878125 180.062187)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-50\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_3\">\n     <g id=\"line2d_8\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#me066859c92\" y=\"137.148624\"/>\n      </g>\n     </g>\n     <g id=\"text_9\">\n      <!-- 0.4 -->\n      <g transform=\"translate(20.878125 140.947842)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-52\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_4\">\n     <g id=\"line2d_9\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#me066859c92\" y=\"98.034279\"/>\n      </g>\n     </g>\n     <g id=\"text_10\">\n      <!-- 0.6 -->\n      <g transform=\"translate(20.878125 101.833497)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-54\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_5\">\n     <g id=\"line2d_10\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#me066859c92\" y=\"58.919934\"/>\n      </g>\n     </g>\n     <g id=\"text_11\">\n      <!-- 0.8 -->\n      <g transform=\"translate(20.878125 62.719152)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-56\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_6\">\n     <g id=\"line2d_11\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#me066859c92\" y=\"19.805589\"/>\n      </g>\n     </g>\n     <g id=\"text_12\">\n      <!-- 1.0 -->\n      <defs>\n       <path d=\"M 12.40625 8.296875 \nL 28.515625 8.296875 \nL 28.515625 63.921875 \nL 10.984375 60.40625 \nL 10.984375 69.390625 \nL 28.421875 72.90625 \nL 38.28125 72.90625 \nL 38.28125 8.296875 \nL 54.390625 8.296875 \nL 54.390625 0 \nL 12.40625 0 \nz\n\" id=\"DejaVuSans-49\"/>\n      </defs>\n      <g transform=\"translate(20.878125 23.604807)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-49\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"text_13\">\n     <!-- Losses -->\n     <defs>\n      <path d=\"M 9.8125 72.90625 \nL 19.671875 72.90625 \nL 19.671875 8.296875 \nL 55.171875 8.296875 \nL 55.171875 0 \nL 9.8125 0 \nz\n\" id=\"DejaVuSans-76\"/>\n      <path d=\"M 56.203125 29.59375 \nL 56.203125 25.203125 \nL 14.890625 25.203125 \nQ 15.484375 15.921875 20.484375 11.0625 \nQ 25.484375 6.203125 34.421875 6.203125 \nQ 39.59375 6.203125 44.453125 7.46875 \nQ 49.3125 8.734375 54.109375 11.28125 \nL 54.109375 2.78125 \nQ 49.265625 0.734375 44.1875 -0.34375 \nQ 39.109375 -1.421875 33.890625 -1.421875 \nQ 20.796875 -1.421875 13.15625 6.1875 \nQ 5.515625 13.8125 5.515625 26.8125 \nQ 5.515625 40.234375 12.765625 48.109375 \nQ 20.015625 56 32.328125 56 \nQ 43.359375 56 49.78125 48.890625 \nQ 56.203125 41.796875 56.203125 29.59375 \nz\nM 47.21875 32.234375 \nQ 47.125 39.59375 43.09375 43.984375 \nQ 39.0625 48.390625 32.421875 48.390625 \nQ 24.90625 48.390625 20.390625 44.140625 \nQ 15.875 39.890625 15.1875 32.171875 \nz\n\" id=\"DejaVuSans-101\"/>\n     </defs>\n     <g transform=\"translate(14.798438 132.655156)rotate(-90)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-76\"/>\n      <use x=\"55.697266\" xlink:href=\"#DejaVuSans-111\"/>\n      <use x=\"116.878906\" xlink:href=\"#DejaVuSans-115\"/>\n      <use x=\"168.978516\" xlink:href=\"#DejaVuSans-115\"/>\n      <use x=\"221.078125\" xlink:href=\"#DejaVuSans-101\"/>\n      <use x=\"282.601562\" xlink:href=\"#DejaVuSans-115\"/>\n     </g>\n    </g>\n   </g>\n   <g id=\"line2d_12\">\n    <path clip-path=\"url(#p736468c2e5)\" d=\"M 58.999432 17.083636 \nL 62.45811 53.169658 \nL 65.916787 68.963984 \nL 69.375465 129.767502 \nL 72.834143 132.373392 \nL 76.29282 111.789841 \nL 79.751498 122.950009 \nL 83.210176 133.355586 \nL 86.668853 125.588923 \nL 90.127531 167.989666 \nL 93.586209 137.253547 \nL 97.044886 152.955824 \nL 100.503564 173.16444 \nL 103.962242 160.019186 \nL 107.420919 163.791188 \nL 110.879597 157.892346 \nL 114.338275 186.064882 \nL 117.796952 158.718162 \nL 121.25563 155.347697 \nL 124.714308 179.126136 \nL 128.172986 170.305069 \nL 131.631663 180.876003 \nL 135.090341 161.8586 \nL 138.549019 185.421918 \nL 142.007696 165.506128 \nL 145.466374 182.108377 \nL 148.925052 185.334794 \nL 152.383729 203.342217 \nL 155.842407 170.88174 \nL 159.301085 194.285238 \nL 162.759762 187.00862 \nL 166.21844 185.377455 \nL 169.677118 188.79778 \nL 173.135795 201.938115 \nL 176.594473 208.139884 \nL 180.053151 208.288889 \nL 183.511829 203.078745 \nL 186.970506 203.328699 \nL 190.429184 201.934345 \nL 193.887862 201.907419 \nL 197.346539 201.743026 \nL 200.805217 209.542172 \nL 204.263895 206.236966 \nL 207.722572 203.232745 \nL 211.18125 205.373492 \nL 214.639928 207.922674 \nL 218.098605 205.762619 \nL 221.557283 206.545185 \nL 225.015961 202.004957 \nL 228.474638 212.154038 \nL 231.933316 211.186294 \nL 235.391994 209.769803 \nL 238.850671 206.347616 \nL 242.309349 202.511675 \nL 245.768027 212.322047 \nL 249.226705 206.633715 \nL 252.685382 207.406258 \nL 256.14406 208.558287 \nL 259.602738 209.386377 \nL 263.061415 212.794059 \nL 266.520093 211.29006 \nL 269.978771 214.120453 \nL 273.437448 207.42535 \nL 276.896126 210.792167 \nL 280.354804 212.455238 \nL 283.813481 204.006282 \nL 287.272159 207.748576 \nL 290.730837 206.080145 \nL 294.189514 213.253978 \nL 297.648192 213.306819 \nL 301.10687 206.97416 \nL 304.565548 210.457177 \nL 308.024225 210.819677 \nL 311.482903 212.110858 \nL 314.941581 209.274761 \nL 318.400258 205.599773 \nL 321.858936 213.066154 \nL 325.317614 212.607098 \nL 328.776291 212.89477 \nL 332.234969 213.051452 \nL 335.693647 208.30069 \nL 339.152324 211.142563 \nL 342.611002 213.223126 \nL 346.06968 212.612776 \nL 349.528357 207.020231 \nL 352.987035 212.763032 \nL 356.445713 214.756364 \nL 359.90439 213.966588 \nL 363.363068 209.768064 \n\" style=\"fill:none;stroke:#ff0000;stroke-linecap:square;stroke-width:1.5;\"/>\n   </g>\n   <g id=\"line2d_13\">\n    <path clip-path=\"url(#p736468c2e5)\" d=\"M 58.999432 83.43846 \nL 62.45811 104.010634 \nL 65.916787 113.832948 \nL 69.375465 129.737229 \nL 72.834143 134.753654 \nL 76.29282 135.320458 \nL 79.751498 127.93696 \nL 83.210176 132.544697 \nL 86.668853 139.322597 \nL 90.127531 126.523999 \nL 93.586209 136.932974 \nL 97.044886 138.52846 \nL 100.503564 128.671064 \nL 103.962242 124.416352 \nL 107.420919 128.668272 \nL 110.879597 137.521506 \nL 114.338275 125.805375 \nL 117.796952 137.120203 \nL 121.25563 123.417203 \nL 124.714308 113.353601 \nL 128.172986 128.516481 \nL 131.631663 137.748159 \nL 135.090341 131.654838 \nL 138.549019 122.007728 \nL 142.007696 135.22536 \nL 145.466374 123.850878 \nL 148.925052 137.770633 \nL 152.383729 139.915163 \nL 155.842407 137.549139 \nL 159.301085 124.833235 \nL 162.759762 129.521231 \nL 166.21844 129.787989 \nL 169.677118 124.785418 \nL 173.135795 124.021997 \nL 176.594473 132.31337 \nL 180.053151 126.431256 \nL 183.511829 119.619545 \nL 186.970506 124.058232 \nL 190.429184 114.421894 \nL 193.887862 120.493859 \nL 197.346539 116.361033 \nL 200.805217 118.351376 \nL 204.263895 133.264498 \nL 207.722572 114.904377 \nL 211.18125 120.900181 \nL 214.639928 115.222787 \nL 218.098605 119.295982 \nL 221.557283 127.463779 \nL 225.015961 113.277796 \nL 228.474638 118.604099 \nL 231.933316 114.363796 \nL 235.391994 112.999789 \nL 238.850671 115.910969 \nL 242.309349 124.374119 \nL 245.768027 125.034458 \nL 249.226705 118.946156 \nL 252.685382 121.089892 \nL 256.14406 124.417687 \nL 259.602738 111.472794 \nL 263.061415 114.649916 \nL 266.520093 113.031694 \nL 269.978771 119.521859 \nL 273.437448 122.041638 \nL 276.896126 121.777631 \nL 280.354804 120.504793 \nL 283.813481 118.443542 \nL 287.272159 120.528521 \nL 290.730837 118.799027 \nL 294.189514 121.280886 \nL 297.648192 121.724673 \nL 301.10687 124.546025 \nL 304.565548 123.473419 \nL 308.024225 120.770736 \nL 311.482903 120.962482 \nL 314.941581 119.96247 \nL 318.400258 118.240046 \nL 321.858936 117.664628 \nL 325.317614 125.869658 \nL 328.776291 122.849112 \nL 332.234969 119.505901 \nL 335.693647 118.312779 \nL 339.152324 118.919147 \nL 342.611002 117.836166 \nL 346.06968 121.180759 \nL 349.528357 118.914542 \nL 352.987035 119.687651 \nL 356.445713 117.56274 \nL 359.90439 120.589377 \nL 363.363068 121.561662 \n\" style=\"fill:none;stroke:#0000ff;stroke-linecap:square;stroke-width:1.5;\"/>\n   </g>\n   <g id=\"patch_3\">\n    <path d=\"M 43.78125 224.64 \nL 43.78125 7.2 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_4\">\n    <path d=\"M 378.58125 224.64 \nL 378.58125 7.2 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_5\">\n    <path d=\"M 43.78125 224.64 \nL 378.58125 224.64 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_6\">\n    <path d=\"M 43.78125 7.2 \nL 378.58125 7.2 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"legend_1\">\n    <g id=\"patch_7\">\n     <path d=\"M 314.140625 44.55625 \nL 371.58125 44.55625 \nQ 373.58125 44.55625 373.58125 42.55625 \nL 373.58125 14.2 \nQ 373.58125 12.2 371.58125 12.2 \nL 314.140625 12.2 \nQ 312.140625 12.2 312.140625 14.2 \nL 312.140625 42.55625 \nQ 312.140625 44.55625 314.140625 44.55625 \nz\n\" style=\"fill:#ffffff;opacity:0.8;stroke:#cccccc;stroke-linejoin:miter;\"/>\n    </g>\n    <g id=\"line2d_14\">\n     <path d=\"M 316.140625 20.298438 \nL 336.140625 20.298438 \n\" style=\"fill:none;stroke:#ff0000;stroke-linecap:square;stroke-width:1.5;\"/>\n    </g>\n    <g id=\"line2d_15\"/>\n    <g id=\"text_14\">\n     <!-- Train -->\n     <defs>\n      <path d=\"M -0.296875 72.90625 \nL 61.375 72.90625 \nL 61.375 64.59375 \nL 35.5 64.59375 \nL 35.5 0 \nL 25.59375 0 \nL 25.59375 64.59375 \nL -0.296875 64.59375 \nz\n\" id=\"DejaVuSans-84\"/>\n      <path d=\"M 41.109375 46.296875 \nQ 39.59375 47.171875 37.8125 47.578125 \nQ 36.03125 48 33.890625 48 \nQ 26.265625 48 22.1875 43.046875 \nQ 18.109375 38.09375 18.109375 28.8125 \nL 18.109375 0 \nL 9.078125 0 \nL 9.078125 54.6875 \nL 18.109375 54.6875 \nL 18.109375 46.1875 \nQ 20.953125 51.171875 25.484375 53.578125 \nQ 30.03125 56 36.53125 56 \nQ 37.453125 56 38.578125 55.875 \nQ 39.703125 55.765625 41.0625 55.515625 \nz\n\" id=\"DejaVuSans-114\"/>\n      <path d=\"M 34.28125 27.484375 \nQ 23.390625 27.484375 19.1875 25 \nQ 14.984375 22.515625 14.984375 16.5 \nQ 14.984375 11.71875 18.140625 8.90625 \nQ 21.296875 6.109375 26.703125 6.109375 \nQ 34.1875 6.109375 38.703125 11.40625 \nQ 43.21875 16.703125 43.21875 25.484375 \nL 43.21875 27.484375 \nz\nM 52.203125 31.203125 \nL 52.203125 0 \nL 43.21875 0 \nL 43.21875 8.296875 \nQ 40.140625 3.328125 35.546875 0.953125 \nQ 30.953125 -1.421875 24.3125 -1.421875 \nQ 15.921875 -1.421875 10.953125 3.296875 \nQ 6 8.015625 6 15.921875 \nQ 6 25.140625 12.171875 29.828125 \nQ 18.359375 34.515625 30.609375 34.515625 \nL 43.21875 34.515625 \nL 43.21875 35.40625 \nQ 43.21875 41.609375 39.140625 45 \nQ 35.0625 48.390625 27.6875 48.390625 \nQ 23 48.390625 18.546875 47.265625 \nQ 14.109375 46.140625 10.015625 43.890625 \nL 10.015625 52.203125 \nQ 14.9375 54.109375 19.578125 55.046875 \nQ 24.21875 56 28.609375 56 \nQ 40.484375 56 46.34375 49.84375 \nQ 52.203125 43.703125 52.203125 31.203125 \nz\n\" id=\"DejaVuSans-97\"/>\n      <path d=\"M 9.421875 54.6875 \nL 18.40625 54.6875 \nL 18.40625 0 \nL 9.421875 0 \nz\nM 9.421875 75.984375 \nL 18.40625 75.984375 \nL 18.40625 64.59375 \nL 9.421875 64.59375 \nz\n\" id=\"DejaVuSans-105\"/>\n      <path d=\"M 54.890625 33.015625 \nL 54.890625 0 \nL 45.90625 0 \nL 45.90625 32.71875 \nQ 45.90625 40.484375 42.875 44.328125 \nQ 39.84375 48.1875 33.796875 48.1875 \nQ 26.515625 48.1875 22.3125 43.546875 \nQ 18.109375 38.921875 18.109375 30.90625 \nL 18.109375 0 \nL 9.078125 0 \nL 9.078125 54.6875 \nL 18.109375 54.6875 \nL 18.109375 46.1875 \nQ 21.34375 51.125 25.703125 53.5625 \nQ 30.078125 56 35.796875 56 \nQ 45.21875 56 50.046875 50.171875 \nQ 54.890625 44.34375 54.890625 33.015625 \nz\n\" id=\"DejaVuSans-110\"/>\n     </defs>\n     <g transform=\"translate(344.140625 23.798438)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-84\"/>\n      <use x=\"60.865234\" xlink:href=\"#DejaVuSans-114\"/>\n      <use x=\"101.978516\" xlink:href=\"#DejaVuSans-97\"/>\n      <use x=\"163.257812\" xlink:href=\"#DejaVuSans-105\"/>\n      <use x=\"191.041016\" xlink:href=\"#DejaVuSans-110\"/>\n     </g>\n    </g>\n    <g id=\"line2d_16\">\n     <path d=\"M 316.140625 34.976562 \nL 336.140625 34.976562 \n\" style=\"fill:none;stroke:#0000ff;stroke-linecap:square;stroke-width:1.5;\"/>\n    </g>\n    <g id=\"line2d_17\"/>\n    <g id=\"text_15\">\n     <!-- Val -->\n     <defs>\n      <path d=\"M 28.609375 0 \nL 0.78125 72.90625 \nL 11.078125 72.90625 \nL 34.1875 11.53125 \nL 57.328125 72.90625 \nL 67.578125 72.90625 \nL 39.796875 0 \nz\n\" id=\"DejaVuSans-86\"/>\n      <path d=\"M 9.421875 75.984375 \nL 18.40625 75.984375 \nL 18.40625 0 \nL 9.421875 0 \nz\n\" id=\"DejaVuSans-108\"/>\n     </defs>\n     <g transform=\"translate(344.140625 38.476562)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-86\"/>\n      <use x=\"68.298828\" xlink:href=\"#DejaVuSans-97\"/>\n      <use x=\"129.578125\" xlink:href=\"#DejaVuSans-108\"/>\n     </g>\n    </g>\n   </g>\n  </g>\n </g>\n <defs>\n  <clipPath id=\"p736468c2e5\">\n   <rect height=\"217.44\" width=\"334.8\" x=\"43.78125\" y=\"7.2\"/>\n  </clipPath>\n </defs>\n</svg>\n"
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qf2xc-E0Jj4g",
        "colab_type": "code",
        "outputId": "4db611ef-4f0c-4ac7-890c-06b49d501299",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 442
        }
      },
      "source": [
        "from google.colab import files\n",
        "files.download('./radam_resnet18_9302.pth.tar')\n",
        "files.download('./radam_resnet18_9302_acc.jpg')\n",
        "files.download('./radam_resnet18_9302_loss.jpg')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "----------------------------------------\n",
            "Exception happened during processing of request from ('::ffff:127.0.0.1', 59682, 0, 0)\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/lib/python3.6/socketserver.py\", line 320, in _handle_request_noblock\n",
            "    self.process_request(request, client_address)\n",
            "  File \"/usr/lib/python3.6/socketserver.py\", line 351, in process_request\n",
            "    self.finish_request(request, client_address)\n",
            "  File \"/usr/lib/python3.6/socketserver.py\", line 364, in finish_request\n",
            "    self.RequestHandlerClass(request, client_address, self)\n",
            "  File \"/usr/lib/python3.6/socketserver.py\", line 724, in __init__\n",
            "    self.handle()\n",
            "  File \"/usr/lib/python3.6/http/server.py\", line 418, in handle\n",
            "    self.handle_one_request()\n",
            "  File \"/usr/lib/python3.6/http/server.py\", line 406, in handle_one_request\n",
            "    method()\n",
            "  File \"/usr/lib/python3.6/http/server.py\", line 639, in do_GET\n",
            "    self.copyfile(f, self.wfile)\n",
            "  File \"/usr/lib/python3.6/http/server.py\", line 800, in copyfile\n",
            "    shutil.copyfileobj(source, outputfile)\n",
            "  File \"/usr/lib/python3.6/shutil.py\", line 82, in copyfileobj\n",
            "    fdst.write(buf)\n",
            "  File \"/usr/lib/python3.6/socketserver.py\", line 803, in write\n",
            "    self._sock.sendall(b)\n",
            "ConnectionResetError: [Errno 104] Connection reset by peer\n",
            "----------------------------------------\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZIXHO2jpKxte",
        "colab_type": "text"
      },
      "source": [
        "# Lookahead with radam"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NwfdntdgJ5AD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "load_checkpoint(model, filename='before_sgd_resnet18.pth.tar')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0PUungkDK84g",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "reset()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TPPSktmSLA0d",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "preds = []\n",
        "train_stats = AvgStats()\n",
        "test_stats = AvgStats()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4_9RymEyLD6k",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "optim = RAdam( model.parameters(), lr=1e-3, weight_decay=1e-4)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WhPiSEpQWvog",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sched = torch.optim.lr_scheduler.StepLR(optim, step_size=30, gamma=0.2)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5AwTxvz0L4qa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "optimizer = Lookahead(optim, alpha=0.5, k=10)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DYXjaVT7MPRK",
        "colab_type": "code",
        "outputId": "92dd20e4-ddd2-4926-a2fb-28ea5094ee8c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "fit(model, optimizer, sched)"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch\tTrn_loss\tVal_loss\tTrn_acc\t\tVal_acc\n",
            "1\t0.84970677\t0.71778688\t58.85800171\t61.55999756\n",
            "2\t0.69664995\t0.60272739\t67.87818146\t68.87333679\n",
            "3\t0.59945265\t0.55649941\t71.80059052\t72.31111145\n",
            "4\t0.56007283\t0.54820792\t74.09957123\t74.39791870\n",
            "5\t0.52714975\t0.56799161\t75.69138336\t75.85766602\n",
            "6\t0.49470810\t0.58072263\t76.85285950\t76.97444916\n",
            "7\t0.47242748\t0.55495768\t77.79829407\t77.88975525\n",
            "8\t0.44865313\t0.57384148\t78.60936737\t78.64521027\n",
            "9\t0.45181180\t0.60109896\t79.24471283\t79.24462891\n",
            "10\t0.41533526\t0.44392665\t79.80440521\t79.89366913\n",
            "11\t0.40102104\t0.45208262\t80.38661194\t80.45803070\n",
            "12\t0.39814142\t0.46110680\t80.90126801\t80.95444489\n",
            "13\t0.38534148\t0.46938068\t81.35987091\t81.40948486\n",
            "14\t0.36481857\t0.54749257\t81.78047943\t81.78952026\n",
            "15\t0.33921807\t0.47814232\t82.15618134\t82.18477631\n",
            "16\t0.33558479\t0.48380847\t82.52516174\t82.54125214\n",
            "17\t0.32106344\t0.50848424\t82.85960388\t82.87235260\n",
            "18\t0.31863529\t0.50099569\t83.17149353\t83.18379211\n",
            "19\t0.30196240\t0.50486462\t83.47628021\t83.48508453\n",
            "20\t0.29221481\t0.42451117\t83.76496124\t83.78949738\n",
            "21\t0.28168410\t0.43603403\t84.05416107\t84.07357025\n",
            "22\t0.27588260\t0.45164326\t84.33801270\t84.35220337\n",
            "23\t0.25528114\t0.48580634\t84.60496521\t84.61196136\n",
            "24\t0.25256926\t0.46760467\t84.85349274\t84.85784912\n",
            "25\t0.24503219\t0.49049659\t85.08623505\t85.08766937\n",
            "26\t0.24271293\t0.51308975\t85.30838776\t85.30679321\n",
            "27\t0.22329401\t0.50162684\t85.52136993\t85.51833344\n",
            "28\t0.22141811\t0.49873747\t85.72569275\t85.71821594\n",
            "29\t0.21780259\t0.48181448\t85.92225647\t85.91793060\n",
            "30\t0.21352108\t0.45899147\t86.11038971\t86.11495209\n",
            "31\t0.20607073\t0.47418155\t86.30291748\t86.30360413\n",
            "32\t0.12094477\t0.44388409\t86.54424286\t86.55301666\n",
            "33\t0.10322880\t0.44828811\t86.79898834\t86.80651093\n",
            "34\t0.10390436\t0.46402556\t87.04428101\t87.04990387\n",
            "35\t0.09327614\t0.48122609\t87.28162384\t87.28447723\n",
            "36\t0.08450223\t0.48441526\t87.50772095\t87.51087952\n",
            "37\t0.08340293\t0.48408016\t87.72872925\t87.73031616\n",
            "38\t0.07087400\t0.49858418\t87.94259644\t87.94324493\n",
            "39\t0.07634135\t0.50809918\t88.14892578\t88.14781952\n",
            "40\t0.07046859\t0.50239113\t88.34201050\t88.34045410\n",
            "41\t0.06643582\t0.52629625\t88.53359222\t88.53118134\n",
            "42\t0.06261581\t0.53421072\t88.71669006\t88.71349335\n",
            "43\t0.06429337\t0.54383477\t88.89205933\t88.88887787\n",
            "44\t0.05425642\t0.55993160\t89.06095123\t89.05700684\n",
            "45\t0.06337544\t0.54051597\t89.22441864\t89.21992493\n",
            "46\t0.06207630\t0.55115843\t89.38123322\t89.37634277\n",
            "47\t0.05530623\t0.55349363\t89.53238678\t89.52695465\n",
            "48\t0.04990523\t0.55685220\t89.67905426\t89.67302704\n",
            "49\t0.04994980\t0.56487831\t89.81965637\t89.81282043\n",
            "50\t0.04673949\t0.57186450\t89.95561981\t89.94856262\n",
            "51\t0.05370835\t0.57523431\t90.08676147\t90.07990265\n",
            "52\t0.05204609\t0.58732738\t90.21414948\t90.20647430\n",
            "53\t0.04337433\t0.57431231\t90.33646393\t90.32893372\n",
            "54\t0.04904619\t0.59616666\t90.45429993\t90.44555664\n",
            "55\t0.04301035\t0.58807094\t90.56741333\t90.55893707\n",
            "56\t0.04606398\t0.59267329\t90.67764282\t90.66954803\n",
            "57\t0.04411003\t0.58817260\t90.78533936\t90.77658081\n",
            "58\t0.03894026\t0.60049689\t90.88936615\t90.88080597\n",
            "59\t0.03850748\t0.61618188\t90.99065399\t90.98177338\n",
            "60\t0.04050585\t0.60924376\t91.08838654\t91.07962036\n",
            "61\t0.04184041\t0.61992990\t91.18328857\t91.17499542\n",
            "62\t0.03132458\t0.60261124\t91.27991486\t91.27215576\n",
            "63\t0.02968304\t0.59718353\t91.37382507\t91.36618805\n",
            "64\t0.02508351\t0.60683254\t91.46691895\t91.45844269\n",
            "65\t0.02655629\t0.60463456\t91.55686188\t91.54874420\n",
            "66\t0.02602059\t0.61911426\t91.64397430\t91.63560486\n",
            "67\t0.02364243\t0.61833171\t91.73045349\t91.72206116\n",
            "68\t0.02558348\t0.63085697\t91.81336975\t91.80507660\n",
            "69\t0.02068682\t0.61918321\t91.89505768\t91.88636017\n",
            "70\t0.02075773\t0.63228027\t91.97334290\t91.96502686\n",
            "71\t0.02456797\t0.62677694\t92.05020905\t92.04129791\n",
            "72\t0.02123541\t0.63256397\t92.12515259\t92.11676025\n",
            "73\t0.02124867\t0.63912366\t92.19844055\t92.18994904\n",
            "74\t0.02594392\t0.63301797\t92.26895905\t92.26024628\n",
            "75\t0.02214816\t0.63777815\t92.33854675\t92.32991028\n",
            "76\t0.02072974\t0.64895447\t92.40679169\t92.39773560\n",
            "77\t0.02093583\t0.64185537\t92.47257996\t92.46342468\n",
            "78\t0.02133939\t0.64329816\t92.53681183\t92.52796936\n",
            "79\t0.02069614\t0.64752653\t92.59919739\t92.59008789\n",
            "80\t0.01955293\t0.64954962\t92.65981293\t92.65083313\n",
            "81\t0.01852710\t0.64885134\t92.72048950\t92.71147919\n",
            "82\t0.01933916\t0.65929629\t92.77902222\t92.77004242\n",
            "83\t0.01842161\t0.64786327\t92.83609009\t92.82711029\n",
            "84\t0.01846824\t0.66677865\t92.89176941\t92.88257599\n",
            "85\t0.01939010\t0.65927191\t92.94603729\t92.93699646\n",
            "86\t0.02049377\t0.65824198\t92.99945831\t92.99015808\n",
            "87\t0.01713984\t0.65712373\t93.05178833\t93.04273987\n",
            "88\t0.01780189\t0.66754142\t93.10295868\t93.09383392\n",
            "89\t0.02036871\t0.66587020\t93.15317535\t93.14417267\n",
            "90\t0.01868475\t0.66410169\t93.20250702\t93.19377899\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2z8fDSpsMT9U",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "save_checkpoint(model, True, 'radam_lookahead_resnet18_9320.pth.tar')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4wbix-lBWov4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ep_lookahead_losses = []\n",
        "for i in range(0, len(train_stats.losses), len(train_loader)):\n",
        "    if i != 0 :\n",
        "        ep_lookahead_losses.append(train_stats.losses[i])\n",
        "        \n",
        "ep_lookahead_lossesv = []\n",
        "for i in range(0, len(test_stats.losses), len(test_loader)):\n",
        "    if(i != 0):\n",
        "        ep_lookahead_lossesv.append(test_stats.losses[i])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ek4v7sy_WvKr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ep_lookahead_accs = []\n",
        "for i in range(0, len(train_stats.losses), len(train_loader)):\n",
        "    if i != 0 :\n",
        "        ep_lookahead_accs.append(train_stats.precs[i])\n",
        "        \n",
        "ep_lookahead_accsv = []\n",
        "for i in range(0, len(test_stats.losses), len(test_loader)):\n",
        "    if(i != 0):\n",
        "        ep_lookahead_accsv.append(test_stats.precs[i])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ltn4TITVWzpW",
        "colab_type": "code",
        "outputId": "f09cb212-5a96-4a96-e1b3-271b9852cca3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 371
        }
      },
      "source": [
        "plt.xlabel(\"Epochs\")\n",
        "plt.ylabel(\"Accuracy\")\n",
        "plt.plot(ep_lookahead_accs, 'r', label='Train')\n",
        "plt.plot(ep_lookahead_accsv, 'b', label='Val')\n",
        "plt.legend()\n",
        "plt.savefig('radam_lookahead_resnet18_9320_acc.jpg')"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/svg+xml": "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n  \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<!-- Created with matplotlib (https://matplotlib.org/) -->\n<svg height=\"262.19625pt\" version=\"1.1\" viewBox=\"0 0 382.603125 262.19625\" width=\"382.603125pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n <defs>\n  <style type=\"text/css\">\n*{stroke-linecap:butt;stroke-linejoin:round;}\n  </style>\n </defs>\n <g id=\"figure_1\">\n  <g id=\"patch_1\">\n   <path d=\"M 0 262.19625 \nL 382.603125 262.19625 \nL 382.603125 0 \nL 0 0 \nz\n\" style=\"fill:none;\"/>\n  </g>\n  <g id=\"axes_1\">\n   <g id=\"patch_2\">\n    <path d=\"M 40.603125 224.64 \nL 375.403125 224.64 \nL 375.403125 7.2 \nL 40.603125 7.2 \nz\n\" style=\"fill:#ffffff;\"/>\n   </g>\n   <g id=\"matplotlib.axis_1\">\n    <g id=\"xtick_1\">\n     <g id=\"line2d_1\">\n      <defs>\n       <path d=\"M 0 0 \nL 0 3.5 \n\" id=\"m19142fac47\" style=\"stroke:#000000;stroke-width:0.8;\"/>\n      </defs>\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"55.821307\" xlink:href=\"#m19142fac47\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_1\">\n      <!-- 0 -->\n      <defs>\n       <path d=\"M 31.78125 66.40625 \nQ 24.171875 66.40625 20.328125 58.90625 \nQ 16.5 51.421875 16.5 36.375 \nQ 16.5 21.390625 20.328125 13.890625 \nQ 24.171875 6.390625 31.78125 6.390625 \nQ 39.453125 6.390625 43.28125 13.890625 \nQ 47.125 21.390625 47.125 36.375 \nQ 47.125 51.421875 43.28125 58.90625 \nQ 39.453125 66.40625 31.78125 66.40625 \nz\nM 31.78125 74.21875 \nQ 44.046875 74.21875 50.515625 64.515625 \nQ 56.984375 54.828125 56.984375 36.375 \nQ 56.984375 17.96875 50.515625 8.265625 \nQ 44.046875 -1.421875 31.78125 -1.421875 \nQ 19.53125 -1.421875 13.0625 8.265625 \nQ 6.59375 17.96875 6.59375 36.375 \nQ 6.59375 54.828125 13.0625 64.515625 \nQ 19.53125 74.21875 31.78125 74.21875 \nz\n\" id=\"DejaVuSans-48\"/>\n      </defs>\n      <g transform=\"translate(52.640057 239.238437)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_2\">\n     <g id=\"line2d_2\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"124.994861\" xlink:href=\"#m19142fac47\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_2\">\n      <!-- 20 -->\n      <defs>\n       <path d=\"M 19.1875 8.296875 \nL 53.609375 8.296875 \nL 53.609375 0 \nL 7.328125 0 \nL 7.328125 8.296875 \nQ 12.9375 14.109375 22.625 23.890625 \nQ 32.328125 33.6875 34.8125 36.53125 \nQ 39.546875 41.84375 41.421875 45.53125 \nQ 43.3125 49.21875 43.3125 52.78125 \nQ 43.3125 58.59375 39.234375 62.25 \nQ 35.15625 65.921875 28.609375 65.921875 \nQ 23.96875 65.921875 18.8125 64.3125 \nQ 13.671875 62.703125 7.8125 59.421875 \nL 7.8125 69.390625 \nQ 13.765625 71.78125 18.9375 73 \nQ 24.125 74.21875 28.421875 74.21875 \nQ 39.75 74.21875 46.484375 68.546875 \nQ 53.21875 62.890625 53.21875 53.421875 \nQ 53.21875 48.921875 51.53125 44.890625 \nQ 49.859375 40.875 45.40625 35.40625 \nQ 44.1875 33.984375 37.640625 27.21875 \nQ 31.109375 20.453125 19.1875 8.296875 \nz\n\" id=\"DejaVuSans-50\"/>\n      </defs>\n      <g transform=\"translate(118.632361 239.238437)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-50\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_3\">\n     <g id=\"line2d_3\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"194.168414\" xlink:href=\"#m19142fac47\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_3\">\n      <!-- 40 -->\n      <defs>\n       <path d=\"M 37.796875 64.3125 \nL 12.890625 25.390625 \nL 37.796875 25.390625 \nz\nM 35.203125 72.90625 \nL 47.609375 72.90625 \nL 47.609375 25.390625 \nL 58.015625 25.390625 \nL 58.015625 17.1875 \nL 47.609375 17.1875 \nL 47.609375 0 \nL 37.796875 0 \nL 37.796875 17.1875 \nL 4.890625 17.1875 \nL 4.890625 26.703125 \nz\n\" id=\"DejaVuSans-52\"/>\n      </defs>\n      <g transform=\"translate(187.805914 239.238437)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-52\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_4\">\n     <g id=\"line2d_4\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"263.341968\" xlink:href=\"#m19142fac47\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_4\">\n      <!-- 60 -->\n      <defs>\n       <path d=\"M 33.015625 40.375 \nQ 26.375 40.375 22.484375 35.828125 \nQ 18.609375 31.296875 18.609375 23.390625 \nQ 18.609375 15.53125 22.484375 10.953125 \nQ 26.375 6.390625 33.015625 6.390625 \nQ 39.65625 6.390625 43.53125 10.953125 \nQ 47.40625 15.53125 47.40625 23.390625 \nQ 47.40625 31.296875 43.53125 35.828125 \nQ 39.65625 40.375 33.015625 40.375 \nz\nM 52.59375 71.296875 \nL 52.59375 62.3125 \nQ 48.875 64.0625 45.09375 64.984375 \nQ 41.3125 65.921875 37.59375 65.921875 \nQ 27.828125 65.921875 22.671875 59.328125 \nQ 17.53125 52.734375 16.796875 39.40625 \nQ 19.671875 43.65625 24.015625 45.921875 \nQ 28.375 48.1875 33.59375 48.1875 \nQ 44.578125 48.1875 50.953125 41.515625 \nQ 57.328125 34.859375 57.328125 23.390625 \nQ 57.328125 12.15625 50.6875 5.359375 \nQ 44.046875 -1.421875 33.015625 -1.421875 \nQ 20.359375 -1.421875 13.671875 8.265625 \nQ 6.984375 17.96875 6.984375 36.375 \nQ 6.984375 53.65625 15.1875 63.9375 \nQ 23.390625 74.21875 37.203125 74.21875 \nQ 40.921875 74.21875 44.703125 73.484375 \nQ 48.484375 72.75 52.59375 71.296875 \nz\n\" id=\"DejaVuSans-54\"/>\n      </defs>\n      <g transform=\"translate(256.979468 239.238437)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-54\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_5\">\n     <g id=\"line2d_5\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"332.515522\" xlink:href=\"#m19142fac47\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_5\">\n      <!-- 80 -->\n      <defs>\n       <path d=\"M 31.78125 34.625 \nQ 24.75 34.625 20.71875 30.859375 \nQ 16.703125 27.09375 16.703125 20.515625 \nQ 16.703125 13.921875 20.71875 10.15625 \nQ 24.75 6.390625 31.78125 6.390625 \nQ 38.8125 6.390625 42.859375 10.171875 \nQ 46.921875 13.96875 46.921875 20.515625 \nQ 46.921875 27.09375 42.890625 30.859375 \nQ 38.875 34.625 31.78125 34.625 \nz\nM 21.921875 38.8125 \nQ 15.578125 40.375 12.03125 44.71875 \nQ 8.5 49.078125 8.5 55.328125 \nQ 8.5 64.0625 14.71875 69.140625 \nQ 20.953125 74.21875 31.78125 74.21875 \nQ 42.671875 74.21875 48.875 69.140625 \nQ 55.078125 64.0625 55.078125 55.328125 \nQ 55.078125 49.078125 51.53125 44.71875 \nQ 48 40.375 41.703125 38.8125 \nQ 48.828125 37.15625 52.796875 32.3125 \nQ 56.78125 27.484375 56.78125 20.515625 \nQ 56.78125 9.90625 50.3125 4.234375 \nQ 43.84375 -1.421875 31.78125 -1.421875 \nQ 19.734375 -1.421875 13.25 4.234375 \nQ 6.78125 9.90625 6.78125 20.515625 \nQ 6.78125 27.484375 10.78125 32.3125 \nQ 14.796875 37.15625 21.921875 38.8125 \nz\nM 18.3125 54.390625 \nQ 18.3125 48.734375 21.84375 45.5625 \nQ 25.390625 42.390625 31.78125 42.390625 \nQ 38.140625 42.390625 41.71875 45.5625 \nQ 45.3125 48.734375 45.3125 54.390625 \nQ 45.3125 60.0625 41.71875 63.234375 \nQ 38.140625 66.40625 31.78125 66.40625 \nQ 25.390625 66.40625 21.84375 63.234375 \nQ 18.3125 60.0625 18.3125 54.390625 \nz\n\" id=\"DejaVuSans-56\"/>\n      </defs>\n      <g transform=\"translate(326.153022 239.238437)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-56\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"text_6\">\n     <!-- Epochs -->\n     <defs>\n      <path d=\"M 9.8125 72.90625 \nL 55.90625 72.90625 \nL 55.90625 64.59375 \nL 19.671875 64.59375 \nL 19.671875 43.015625 \nL 54.390625 43.015625 \nL 54.390625 34.71875 \nL 19.671875 34.71875 \nL 19.671875 8.296875 \nL 56.78125 8.296875 \nL 56.78125 0 \nL 9.8125 0 \nz\n\" id=\"DejaVuSans-69\"/>\n      <path d=\"M 18.109375 8.203125 \nL 18.109375 -20.796875 \nL 9.078125 -20.796875 \nL 9.078125 54.6875 \nL 18.109375 54.6875 \nL 18.109375 46.390625 \nQ 20.953125 51.265625 25.265625 53.625 \nQ 29.59375 56 35.59375 56 \nQ 45.5625 56 51.78125 48.09375 \nQ 58.015625 40.1875 58.015625 27.296875 \nQ 58.015625 14.40625 51.78125 6.484375 \nQ 45.5625 -1.421875 35.59375 -1.421875 \nQ 29.59375 -1.421875 25.265625 0.953125 \nQ 20.953125 3.328125 18.109375 8.203125 \nz\nM 48.6875 27.296875 \nQ 48.6875 37.203125 44.609375 42.84375 \nQ 40.53125 48.484375 33.40625 48.484375 \nQ 26.265625 48.484375 22.1875 42.84375 \nQ 18.109375 37.203125 18.109375 27.296875 \nQ 18.109375 17.390625 22.1875 11.75 \nQ 26.265625 6.109375 33.40625 6.109375 \nQ 40.53125 6.109375 44.609375 11.75 \nQ 48.6875 17.390625 48.6875 27.296875 \nz\n\" id=\"DejaVuSans-112\"/>\n      <path d=\"M 30.609375 48.390625 \nQ 23.390625 48.390625 19.1875 42.75 \nQ 14.984375 37.109375 14.984375 27.296875 \nQ 14.984375 17.484375 19.15625 11.84375 \nQ 23.34375 6.203125 30.609375 6.203125 \nQ 37.796875 6.203125 41.984375 11.859375 \nQ 46.1875 17.53125 46.1875 27.296875 \nQ 46.1875 37.015625 41.984375 42.703125 \nQ 37.796875 48.390625 30.609375 48.390625 \nz\nM 30.609375 56 \nQ 42.328125 56 49.015625 48.375 \nQ 55.71875 40.765625 55.71875 27.296875 \nQ 55.71875 13.875 49.015625 6.21875 \nQ 42.328125 -1.421875 30.609375 -1.421875 \nQ 18.84375 -1.421875 12.171875 6.21875 \nQ 5.515625 13.875 5.515625 27.296875 \nQ 5.515625 40.765625 12.171875 48.375 \nQ 18.84375 56 30.609375 56 \nz\n\" id=\"DejaVuSans-111\"/>\n      <path d=\"M 48.78125 52.59375 \nL 48.78125 44.1875 \nQ 44.96875 46.296875 41.140625 47.34375 \nQ 37.3125 48.390625 33.40625 48.390625 \nQ 24.65625 48.390625 19.8125 42.84375 \nQ 14.984375 37.3125 14.984375 27.296875 \nQ 14.984375 17.28125 19.8125 11.734375 \nQ 24.65625 6.203125 33.40625 6.203125 \nQ 37.3125 6.203125 41.140625 7.25 \nQ 44.96875 8.296875 48.78125 10.40625 \nL 48.78125 2.09375 \nQ 45.015625 0.34375 40.984375 -0.53125 \nQ 36.96875 -1.421875 32.421875 -1.421875 \nQ 20.0625 -1.421875 12.78125 6.34375 \nQ 5.515625 14.109375 5.515625 27.296875 \nQ 5.515625 40.671875 12.859375 48.328125 \nQ 20.21875 56 33.015625 56 \nQ 37.15625 56 41.109375 55.140625 \nQ 45.0625 54.296875 48.78125 52.59375 \nz\n\" id=\"DejaVuSans-99\"/>\n      <path d=\"M 54.890625 33.015625 \nL 54.890625 0 \nL 45.90625 0 \nL 45.90625 32.71875 \nQ 45.90625 40.484375 42.875 44.328125 \nQ 39.84375 48.1875 33.796875 48.1875 \nQ 26.515625 48.1875 22.3125 43.546875 \nQ 18.109375 38.921875 18.109375 30.90625 \nL 18.109375 0 \nL 9.078125 0 \nL 9.078125 75.984375 \nL 18.109375 75.984375 \nL 18.109375 46.1875 \nQ 21.34375 51.125 25.703125 53.5625 \nQ 30.078125 56 35.796875 56 \nQ 45.21875 56 50.046875 50.171875 \nQ 54.890625 44.34375 54.890625 33.015625 \nz\n\" id=\"DejaVuSans-104\"/>\n      <path d=\"M 44.28125 53.078125 \nL 44.28125 44.578125 \nQ 40.484375 46.53125 36.375 47.5 \nQ 32.28125 48.484375 27.875 48.484375 \nQ 21.1875 48.484375 17.84375 46.4375 \nQ 14.5 44.390625 14.5 40.28125 \nQ 14.5 37.15625 16.890625 35.375 \nQ 19.28125 33.59375 26.515625 31.984375 \nL 29.59375 31.296875 \nQ 39.15625 29.25 43.1875 25.515625 \nQ 47.21875 21.78125 47.21875 15.09375 \nQ 47.21875 7.46875 41.1875 3.015625 \nQ 35.15625 -1.421875 24.609375 -1.421875 \nQ 20.21875 -1.421875 15.453125 -0.5625 \nQ 10.6875 0.296875 5.421875 2 \nL 5.421875 11.28125 \nQ 10.40625 8.6875 15.234375 7.390625 \nQ 20.0625 6.109375 24.8125 6.109375 \nQ 31.15625 6.109375 34.5625 8.28125 \nQ 37.984375 10.453125 37.984375 14.40625 \nQ 37.984375 18.0625 35.515625 20.015625 \nQ 33.0625 21.96875 24.703125 23.78125 \nL 21.578125 24.515625 \nQ 13.234375 26.265625 9.515625 29.90625 \nQ 5.8125 33.546875 5.8125 39.890625 \nQ 5.8125 47.609375 11.28125 51.796875 \nQ 16.75 56 26.8125 56 \nQ 31.78125 56 36.171875 55.265625 \nQ 40.578125 54.546875 44.28125 53.078125 \nz\n\" id=\"DejaVuSans-115\"/>\n     </defs>\n     <g transform=\"translate(190.0875 252.916562)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-69\"/>\n      <use x=\"63.183594\" xlink:href=\"#DejaVuSans-112\"/>\n      <use x=\"126.660156\" xlink:href=\"#DejaVuSans-111\"/>\n      <use x=\"187.841797\" xlink:href=\"#DejaVuSans-99\"/>\n      <use x=\"242.822266\" xlink:href=\"#DejaVuSans-104\"/>\n      <use x=\"306.201172\" xlink:href=\"#DejaVuSans-115\"/>\n     </g>\n    </g>\n   </g>\n   <g id=\"matplotlib.axis_2\">\n    <g id=\"ytick_1\">\n     <g id=\"line2d_6\">\n      <defs>\n       <path d=\"M 0 0 \nL -3.5 0 \n\" id=\"ma3675f8a22\" style=\"stroke:#000000;stroke-width:0.8;\"/>\n      </defs>\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"40.603125\" xlink:href=\"#ma3675f8a22\" y=\"193.398107\"/>\n      </g>\n     </g>\n     <g id=\"text_7\">\n      <!-- 65 -->\n      <defs>\n       <path d=\"M 10.796875 72.90625 \nL 49.515625 72.90625 \nL 49.515625 64.59375 \nL 19.828125 64.59375 \nL 19.828125 46.734375 \nQ 21.96875 47.46875 24.109375 47.828125 \nQ 26.265625 48.1875 28.421875 48.1875 \nQ 40.625 48.1875 47.75 41.5 \nQ 54.890625 34.8125 54.890625 23.390625 \nQ 54.890625 11.625 47.5625 5.09375 \nQ 40.234375 -1.421875 26.90625 -1.421875 \nQ 22.3125 -1.421875 17.546875 -0.640625 \nQ 12.796875 0.140625 7.71875 1.703125 \nL 7.71875 11.625 \nQ 12.109375 9.234375 16.796875 8.0625 \nQ 21.484375 6.890625 26.703125 6.890625 \nQ 35.15625 6.890625 40.078125 11.328125 \nQ 45.015625 15.765625 45.015625 23.390625 \nQ 45.015625 31 40.078125 35.4375 \nQ 35.15625 39.890625 26.703125 39.890625 \nQ 22.75 39.890625 18.8125 39.015625 \nQ 14.890625 38.140625 10.796875 36.28125 \nz\n\" id=\"DejaVuSans-53\"/>\n      </defs>\n      <g transform=\"translate(20.878125 197.197326)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-54\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-53\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_2\">\n     <g id=\"line2d_7\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"40.603125\" xlink:href=\"#ma3675f8a22\" y=\"162.139447\"/>\n      </g>\n     </g>\n     <g id=\"text_8\">\n      <!-- 70 -->\n      <defs>\n       <path d=\"M 8.203125 72.90625 \nL 55.078125 72.90625 \nL 55.078125 68.703125 \nL 28.609375 0 \nL 18.3125 0 \nL 43.21875 64.59375 \nL 8.203125 64.59375 \nz\n\" id=\"DejaVuSans-55\"/>\n      </defs>\n      <g transform=\"translate(20.878125 165.938666)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-55\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_3\">\n     <g id=\"line2d_8\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"40.603125\" xlink:href=\"#ma3675f8a22\" y=\"130.880786\"/>\n      </g>\n     </g>\n     <g id=\"text_9\">\n      <!-- 75 -->\n      <g transform=\"translate(20.878125 134.680005)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-55\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-53\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_4\">\n     <g id=\"line2d_9\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"40.603125\" xlink:href=\"#ma3675f8a22\" y=\"99.622126\"/>\n      </g>\n     </g>\n     <g id=\"text_10\">\n      <!-- 80 -->\n      <g transform=\"translate(20.878125 103.421344)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-56\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_5\">\n     <g id=\"line2d_10\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"40.603125\" xlink:href=\"#ma3675f8a22\" y=\"68.363465\"/>\n      </g>\n     </g>\n     <g id=\"text_11\">\n      <!-- 85 -->\n      <g transform=\"translate(20.878125 72.162684)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-56\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-53\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_6\">\n     <g id=\"line2d_11\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"40.603125\" xlink:href=\"#ma3675f8a22\" y=\"37.104805\"/>\n      </g>\n     </g>\n     <g id=\"text_12\">\n      <!-- 90 -->\n      <defs>\n       <path d=\"M 10.984375 1.515625 \nL 10.984375 10.5 \nQ 14.703125 8.734375 18.5 7.8125 \nQ 22.3125 6.890625 25.984375 6.890625 \nQ 35.75 6.890625 40.890625 13.453125 \nQ 46.046875 20.015625 46.78125 33.40625 \nQ 43.953125 29.203125 39.59375 26.953125 \nQ 35.25 24.703125 29.984375 24.703125 \nQ 19.046875 24.703125 12.671875 31.3125 \nQ 6.296875 37.9375 6.296875 49.421875 \nQ 6.296875 60.640625 12.9375 67.421875 \nQ 19.578125 74.21875 30.609375 74.21875 \nQ 43.265625 74.21875 49.921875 64.515625 \nQ 56.59375 54.828125 56.59375 36.375 \nQ 56.59375 19.140625 48.40625 8.859375 \nQ 40.234375 -1.421875 26.421875 -1.421875 \nQ 22.703125 -1.421875 18.890625 -0.6875 \nQ 15.09375 0.046875 10.984375 1.515625 \nz\nM 30.609375 32.421875 \nQ 37.25 32.421875 41.125 36.953125 \nQ 45.015625 41.5 45.015625 49.421875 \nQ 45.015625 57.28125 41.125 61.84375 \nQ 37.25 66.40625 30.609375 66.40625 \nQ 23.96875 66.40625 20.09375 61.84375 \nQ 16.21875 57.28125 16.21875 49.421875 \nQ 16.21875 41.5 20.09375 36.953125 \nQ 23.96875 32.421875 30.609375 32.421875 \nz\n\" id=\"DejaVuSans-57\"/>\n      </defs>\n      <g transform=\"translate(20.878125 40.904023)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-57\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"text_13\">\n     <!-- Accuracy -->\n     <defs>\n      <path d=\"M 34.1875 63.1875 \nL 20.796875 26.90625 \nL 47.609375 26.90625 \nz\nM 28.609375 72.90625 \nL 39.796875 72.90625 \nL 67.578125 0 \nL 57.328125 0 \nL 50.6875 18.703125 \nL 17.828125 18.703125 \nL 11.1875 0 \nL 0.78125 0 \nz\n\" id=\"DejaVuSans-65\"/>\n      <path d=\"M 8.5 21.578125 \nL 8.5 54.6875 \nL 17.484375 54.6875 \nL 17.484375 21.921875 \nQ 17.484375 14.15625 20.5 10.265625 \nQ 23.53125 6.390625 29.59375 6.390625 \nQ 36.859375 6.390625 41.078125 11.03125 \nQ 45.3125 15.671875 45.3125 23.6875 \nL 45.3125 54.6875 \nL 54.296875 54.6875 \nL 54.296875 0 \nL 45.3125 0 \nL 45.3125 8.40625 \nQ 42.046875 3.421875 37.71875 1 \nQ 33.40625 -1.421875 27.6875 -1.421875 \nQ 18.265625 -1.421875 13.375 4.4375 \nQ 8.5 10.296875 8.5 21.578125 \nz\nM 31.109375 56 \nz\n\" id=\"DejaVuSans-117\"/>\n      <path d=\"M 41.109375 46.296875 \nQ 39.59375 47.171875 37.8125 47.578125 \nQ 36.03125 48 33.890625 48 \nQ 26.265625 48 22.1875 43.046875 \nQ 18.109375 38.09375 18.109375 28.8125 \nL 18.109375 0 \nL 9.078125 0 \nL 9.078125 54.6875 \nL 18.109375 54.6875 \nL 18.109375 46.1875 \nQ 20.953125 51.171875 25.484375 53.578125 \nQ 30.03125 56 36.53125 56 \nQ 37.453125 56 38.578125 55.875 \nQ 39.703125 55.765625 41.0625 55.515625 \nz\n\" id=\"DejaVuSans-114\"/>\n      <path d=\"M 34.28125 27.484375 \nQ 23.390625 27.484375 19.1875 25 \nQ 14.984375 22.515625 14.984375 16.5 \nQ 14.984375 11.71875 18.140625 8.90625 \nQ 21.296875 6.109375 26.703125 6.109375 \nQ 34.1875 6.109375 38.703125 11.40625 \nQ 43.21875 16.703125 43.21875 25.484375 \nL 43.21875 27.484375 \nz\nM 52.203125 31.203125 \nL 52.203125 0 \nL 43.21875 0 \nL 43.21875 8.296875 \nQ 40.140625 3.328125 35.546875 0.953125 \nQ 30.953125 -1.421875 24.3125 -1.421875 \nQ 15.921875 -1.421875 10.953125 3.296875 \nQ 6 8.015625 6 15.921875 \nQ 6 25.140625 12.171875 29.828125 \nQ 18.359375 34.515625 30.609375 34.515625 \nL 43.21875 34.515625 \nL 43.21875 35.40625 \nQ 43.21875 41.609375 39.140625 45 \nQ 35.0625 48.390625 27.6875 48.390625 \nQ 23 48.390625 18.546875 47.265625 \nQ 14.109375 46.140625 10.015625 43.890625 \nL 10.015625 52.203125 \nQ 14.9375 54.109375 19.578125 55.046875 \nQ 24.21875 56 28.609375 56 \nQ 40.484375 56 46.34375 49.84375 \nQ 52.203125 43.703125 52.203125 31.203125 \nz\n\" id=\"DejaVuSans-97\"/>\n      <path d=\"M 32.171875 -5.078125 \nQ 28.375 -14.84375 24.75 -17.8125 \nQ 21.140625 -20.796875 15.09375 -20.796875 \nL 7.90625 -20.796875 \nL 7.90625 -13.28125 \nL 13.1875 -13.28125 \nQ 16.890625 -13.28125 18.9375 -11.515625 \nQ 21 -9.765625 23.484375 -3.21875 \nL 25.09375 0.875 \nL 2.984375 54.6875 \nL 12.5 54.6875 \nL 29.59375 11.921875 \nL 46.6875 54.6875 \nL 56.203125 54.6875 \nz\n\" id=\"DejaVuSans-121\"/>\n     </defs>\n     <g transform=\"translate(14.798438 138.834844)rotate(-90)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-65\"/>\n      <use x=\"68.392578\" xlink:href=\"#DejaVuSans-99\"/>\n      <use x=\"123.373047\" xlink:href=\"#DejaVuSans-99\"/>\n      <use x=\"178.353516\" xlink:href=\"#DejaVuSans-117\"/>\n      <use x=\"241.732422\" xlink:href=\"#DejaVuSans-114\"/>\n      <use x=\"282.845703\" xlink:href=\"#DejaVuSans-97\"/>\n      <use x=\"344.125\" xlink:href=\"#DejaVuSans-99\"/>\n      <use x=\"399.105469\" xlink:href=\"#DejaVuSans-121\"/>\n     </g>\n    </g>\n   </g>\n   <g id=\"line2d_12\">\n    <path clip-path=\"url(#pbaf296b6ce)\" d=\"M 55.821307 214.756364 \nL 59.279985 169.084978 \nL 62.738662 147.658229 \nL 66.19734 134.608969 \nL 69.656018 125.508681 \nL 73.114695 118.52585 \nL 76.573373 112.799452 \nL 80.032051 108.07717 \nL 83.490728 104.332245 \nL 86.949406 100.273618 \nL 90.408084 96.746335 \nL 93.866761 93.64446 \nL 97.325439 90.804156 \nL 100.784117 88.429087 \nL 104.242794 85.956669 \nL 107.701472 83.731416 \nL 111.16015 81.660702 \nL 114.618827 79.714095 \nL 118.077505 77.827633 \nL 121.536183 75.925623 \nL 124.994861 74.150629 \nL 128.453538 72.408165 \nL 131.912216 70.783607 \nL 135.370894 69.247671 \nL 138.829571 67.812041 \nL 142.288249 66.442328 \nL 145.746927 65.118548 \nL 149.205604 63.868793 \nL 152.664282 62.622996 \nL 156.12296 61.389029 \nL 159.581637 60.209818 \nL 163.040315 58.650748 \nL 166.498993 57.065875 \nL 169.95767 55.545678 \nL 173.416348 54.077567 \nL 176.875026 52.662351 \nL 180.333704 51.290349 \nL 183.792381 49.959652 \nL 187.251059 48.680611 \nL 190.709737 47.476693 \nL 194.168414 46.28427 \nL 197.627092 45.144408 \nL 201.08577 44.048524 \nL 204.544447 42.99776 \nL 208.003125 41.978906 \nL 211.461803 41.000881 \nL 214.92048 40.060345 \nL 218.379158 39.147235 \nL 221.837836 38.272473 \nL 225.296513 37.42404 \nL 228.755191 36.603462 \nL 232.213869 35.811693 \nL 235.672546 35.046539 \nL 239.131224 34.317348 \nL 242.589902 33.608572 \nL 246.04858 32.916919 \nL 249.507257 32.248589 \nL 252.965935 31.596477 \nL 256.424613 30.964922 \nL 259.88329 30.354258 \nL 263.341968 29.757426 \nL 266.800646 29.149815 \nL 270.259323 28.562094 \nL 273.718001 27.985819 \nL 277.176679 27.42104 \nL 280.635356 26.878058 \nL 284.094034 26.337223 \nL 287.552712 25.818518 \nL 291.011389 25.310403 \nL 294.470067 24.818648 \nL 297.928745 24.342012 \nL 301.387423 23.869908 \nL 304.8461 23.412542 \nL 308.304778 22.97311 \nL 311.763456 22.53759 \nL 315.222133 22.113564 \nL 318.680811 21.702989 \nL 322.139489 21.299377 \nL 325.598166 20.911029 \nL 329.056844 20.531314 \nL 332.515522 20.152123 \nL 335.974199 19.786192 \nL 339.432877 19.429562 \nL 342.891555 19.082567 \nL 346.350232 18.742488 \nL 349.80891 18.410183 \nL 353.267588 18.081456 \nL 356.726265 17.762078 \nL 360.184943 17.447182 \n\" style=\"fill:none;stroke:#ff0000;stroke-linecap:square;stroke-width:1.5;\"/>\n   </g>\n   <g id=\"line2d_13\">\n    <path clip-path=\"url(#pbaf296b6ce)\" d=\"M 55.821307 175.301653 \nL 59.279985 150.827166 \nL 62.738662 136.482457 \nL 66.19734 126.534499 \nL 69.656018 119.26935 \nL 73.114695 113.370718 \nL 76.573373 108.320853 \nL 80.032051 104.337349 \nL 83.490728 100.837634 \nL 86.949406 97.198311 \nL 90.408084 93.98287 \nL 93.866761 91.115807 \nL 97.325439 88.488517 \nL 100.784117 86.138203 \nL 104.242794 83.831961 \nL 107.701472 81.739736 \nL 111.16015 79.791554 \nL 114.618827 77.888161 \nL 118.077505 76.082593 \nL 121.536183 74.27488 \nL 124.994861 72.4996 \nL 128.453538 70.831877 \nL 131.912216 69.279628 \nL 135.370894 67.82511 \nL 138.829571 66.434792 \nL 142.288249 65.106146 \nL 145.746927 63.824673 \nL 149.205604 62.595237 \nL 152.664282 61.422035 \nL 156.12296 60.216782 \nL 159.581637 58.706935 \nL 163.040315 57.114192 \nL 166.498993 55.581117 \nL 169.95767 54.09779 \nL 173.416348 52.684149 \nL 176.875026 51.303132 \nL 180.333704 49.965376 \nL 183.792381 48.676414 \nL 187.251059 47.469348 \nL 190.709737 46.270676 \nL 194.168414 45.126379 \nL 197.627092 44.030065 \nL 201.08577 42.974294 \nL 204.544447 41.95191 \nL 208.003125 40.971833 \nL 211.461803 40.027243 \nL 214.92048 39.110317 \nL 218.379158 38.231597 \nL 221.837836 37.381876 \nL 225.296513 36.562061 \nL 228.755191 35.765713 \nL 232.213869 35.000845 \nL 235.672546 34.263975 \nL 239.131224 33.557059 \nL 242.589902 32.867839 \nL 246.04858 32.194406 \nL 249.507257 31.544821 \nL 252.965935 30.911215 \nL 256.424613 30.301171 \nL 259.88329 29.70682 \nL 263.341968 29.102595 \nL 266.800646 28.515541 \nL 270.259323 27.933543 \nL 273.718001 27.371101 \nL 277.176679 26.826545 \nL 280.635356 26.285948 \nL 284.094034 25.767578 \nL 287.552712 25.257221 \nL 291.011389 24.767803 \nL 294.470067 24.287256 \nL 297.928745 23.818777 \nL 301.387423 23.360409 \nL 304.8461 22.919737 \nL 308.304778 22.484551 \nL 311.763456 22.058092 \nL 315.222133 21.646802 \nL 318.680811 21.245384 \nL 322.139489 20.855414 \nL 325.598166 20.476319 \nL 329.056844 20.097081 \nL 332.515522 19.730959 \nL 335.974199 19.374281 \nL 339.432877 19.026332 \nL 342.891555 18.686969 \nL 346.350232 18.353138 \nL 349.80891 18.025842 \nL 353.267588 17.705795 \nL 356.726265 17.392045 \nL 360.184943 17.083636 \n\" style=\"fill:none;stroke:#0000ff;stroke-linecap:square;stroke-width:1.5;\"/>\n   </g>\n   <g id=\"patch_3\">\n    <path d=\"M 40.603125 224.64 \nL 40.603125 7.2 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_4\">\n    <path d=\"M 375.403125 224.64 \nL 375.403125 7.2 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_5\">\n    <path d=\"M 40.603125 224.64 \nL 375.403125 224.64 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_6\">\n    <path d=\"M 40.603125 7.2 \nL 375.403125 7.2 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"legend_1\">\n    <g id=\"patch_7\">\n     <path d=\"M 47.603125 44.55625 \nL 105.04375 44.55625 \nQ 107.04375 44.55625 107.04375 42.55625 \nL 107.04375 14.2 \nQ 107.04375 12.2 105.04375 12.2 \nL 47.603125 12.2 \nQ 45.603125 12.2 45.603125 14.2 \nL 45.603125 42.55625 \nQ 45.603125 44.55625 47.603125 44.55625 \nz\n\" style=\"fill:#ffffff;opacity:0.8;stroke:#cccccc;stroke-linejoin:miter;\"/>\n    </g>\n    <g id=\"line2d_14\">\n     <path d=\"M 49.603125 20.298437 \nL 69.603125 20.298437 \n\" style=\"fill:none;stroke:#ff0000;stroke-linecap:square;stroke-width:1.5;\"/>\n    </g>\n    <g id=\"line2d_15\"/>\n    <g id=\"text_14\">\n     <!-- Train -->\n     <defs>\n      <path d=\"M -0.296875 72.90625 \nL 61.375 72.90625 \nL 61.375 64.59375 \nL 35.5 64.59375 \nL 35.5 0 \nL 25.59375 0 \nL 25.59375 64.59375 \nL -0.296875 64.59375 \nz\n\" id=\"DejaVuSans-84\"/>\n      <path d=\"M 9.421875 54.6875 \nL 18.40625 54.6875 \nL 18.40625 0 \nL 9.421875 0 \nz\nM 9.421875 75.984375 \nL 18.40625 75.984375 \nL 18.40625 64.59375 \nL 9.421875 64.59375 \nz\n\" id=\"DejaVuSans-105\"/>\n      <path d=\"M 54.890625 33.015625 \nL 54.890625 0 \nL 45.90625 0 \nL 45.90625 32.71875 \nQ 45.90625 40.484375 42.875 44.328125 \nQ 39.84375 48.1875 33.796875 48.1875 \nQ 26.515625 48.1875 22.3125 43.546875 \nQ 18.109375 38.921875 18.109375 30.90625 \nL 18.109375 0 \nL 9.078125 0 \nL 9.078125 54.6875 \nL 18.109375 54.6875 \nL 18.109375 46.1875 \nQ 21.34375 51.125 25.703125 53.5625 \nQ 30.078125 56 35.796875 56 \nQ 45.21875 56 50.046875 50.171875 \nQ 54.890625 44.34375 54.890625 33.015625 \nz\n\" id=\"DejaVuSans-110\"/>\n     </defs>\n     <g transform=\"translate(77.603125 23.798437)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-84\"/>\n      <use x=\"60.865234\" xlink:href=\"#DejaVuSans-114\"/>\n      <use x=\"101.978516\" xlink:href=\"#DejaVuSans-97\"/>\n      <use x=\"163.257812\" xlink:href=\"#DejaVuSans-105\"/>\n      <use x=\"191.041016\" xlink:href=\"#DejaVuSans-110\"/>\n     </g>\n    </g>\n    <g id=\"line2d_16\">\n     <path d=\"M 49.603125 34.976562 \nL 69.603125 34.976562 \n\" style=\"fill:none;stroke:#0000ff;stroke-linecap:square;stroke-width:1.5;\"/>\n    </g>\n    <g id=\"line2d_17\"/>\n    <g id=\"text_15\">\n     <!-- Val -->\n     <defs>\n      <path d=\"M 28.609375 0 \nL 0.78125 72.90625 \nL 11.078125 72.90625 \nL 34.1875 11.53125 \nL 57.328125 72.90625 \nL 67.578125 72.90625 \nL 39.796875 0 \nz\n\" id=\"DejaVuSans-86\"/>\n      <path d=\"M 9.421875 75.984375 \nL 18.40625 75.984375 \nL 18.40625 0 \nL 9.421875 0 \nz\n\" id=\"DejaVuSans-108\"/>\n     </defs>\n     <g transform=\"translate(77.603125 38.476562)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-86\"/>\n      <use x=\"68.298828\" xlink:href=\"#DejaVuSans-97\"/>\n      <use x=\"129.578125\" xlink:href=\"#DejaVuSans-108\"/>\n     </g>\n    </g>\n   </g>\n  </g>\n </g>\n <defs>\n  <clipPath id=\"pbaf296b6ce\">\n   <rect height=\"217.44\" width=\"334.8\" x=\"40.603125\" y=\"7.2\"/>\n  </clipPath>\n </defs>\n</svg>\n"
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "87N8H050W7yp",
        "colab_type": "code",
        "outputId": "d6132098-8300-474c-ec11-0276fdd71382",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 371
        }
      },
      "source": [
        "plt.xlabel(\"Epochs\")\n",
        "plt.ylabel(\"Losses\")\n",
        "plt.plot(ep_lookahead_losses, 'r', label='Train')\n",
        "plt.plot(ep_lookahead_lossesv, 'b', label='Val')\n",
        "plt.legend()\n",
        "plt.savefig('radam_lookahead_resnet18_9320_loss.jpg')"
      ],
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/svg+xml": "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n  \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<!-- Created with matplotlib (https://matplotlib.org/) -->\n<svg height=\"262.19625pt\" version=\"1.1\" viewBox=\"0 0 385.78125 262.19625\" width=\"385.78125pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n <defs>\n  <style type=\"text/css\">\n*{stroke-linecap:butt;stroke-linejoin:round;}\n  </style>\n </defs>\n <g id=\"figure_1\">\n  <g id=\"patch_1\">\n   <path d=\"M 0 262.19625 \nL 385.78125 262.19625 \nL 385.78125 0 \nL 0 0 \nz\n\" style=\"fill:none;\"/>\n  </g>\n  <g id=\"axes_1\">\n   <g id=\"patch_2\">\n    <path d=\"M 43.78125 224.64 \nL 378.58125 224.64 \nL 378.58125 7.2 \nL 43.78125 7.2 \nz\n\" style=\"fill:#ffffff;\"/>\n   </g>\n   <g id=\"matplotlib.axis_1\">\n    <g id=\"xtick_1\">\n     <g id=\"line2d_1\">\n      <defs>\n       <path d=\"M 0 0 \nL 0 3.5 \n\" id=\"m6544ae3bd0\" style=\"stroke:#000000;stroke-width:0.8;\"/>\n      </defs>\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"58.999432\" xlink:href=\"#m6544ae3bd0\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_1\">\n      <!-- 0 -->\n      <defs>\n       <path d=\"M 31.78125 66.40625 \nQ 24.171875 66.40625 20.328125 58.90625 \nQ 16.5 51.421875 16.5 36.375 \nQ 16.5 21.390625 20.328125 13.890625 \nQ 24.171875 6.390625 31.78125 6.390625 \nQ 39.453125 6.390625 43.28125 13.890625 \nQ 47.125 21.390625 47.125 36.375 \nQ 47.125 51.421875 43.28125 58.90625 \nQ 39.453125 66.40625 31.78125 66.40625 \nz\nM 31.78125 74.21875 \nQ 44.046875 74.21875 50.515625 64.515625 \nQ 56.984375 54.828125 56.984375 36.375 \nQ 56.984375 17.96875 50.515625 8.265625 \nQ 44.046875 -1.421875 31.78125 -1.421875 \nQ 19.53125 -1.421875 13.0625 8.265625 \nQ 6.59375 17.96875 6.59375 36.375 \nQ 6.59375 54.828125 13.0625 64.515625 \nQ 19.53125 74.21875 31.78125 74.21875 \nz\n\" id=\"DejaVuSans-48\"/>\n      </defs>\n      <g transform=\"translate(55.818182 239.238437)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_2\">\n     <g id=\"line2d_2\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"128.172986\" xlink:href=\"#m6544ae3bd0\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_2\">\n      <!-- 20 -->\n      <defs>\n       <path d=\"M 19.1875 8.296875 \nL 53.609375 8.296875 \nL 53.609375 0 \nL 7.328125 0 \nL 7.328125 8.296875 \nQ 12.9375 14.109375 22.625 23.890625 \nQ 32.328125 33.6875 34.8125 36.53125 \nQ 39.546875 41.84375 41.421875 45.53125 \nQ 43.3125 49.21875 43.3125 52.78125 \nQ 43.3125 58.59375 39.234375 62.25 \nQ 35.15625 65.921875 28.609375 65.921875 \nQ 23.96875 65.921875 18.8125 64.3125 \nQ 13.671875 62.703125 7.8125 59.421875 \nL 7.8125 69.390625 \nQ 13.765625 71.78125 18.9375 73 \nQ 24.125 74.21875 28.421875 74.21875 \nQ 39.75 74.21875 46.484375 68.546875 \nQ 53.21875 62.890625 53.21875 53.421875 \nQ 53.21875 48.921875 51.53125 44.890625 \nQ 49.859375 40.875 45.40625 35.40625 \nQ 44.1875 33.984375 37.640625 27.21875 \nQ 31.109375 20.453125 19.1875 8.296875 \nz\n\" id=\"DejaVuSans-50\"/>\n      </defs>\n      <g transform=\"translate(121.810486 239.238437)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-50\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_3\">\n     <g id=\"line2d_3\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"197.346539\" xlink:href=\"#m6544ae3bd0\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_3\">\n      <!-- 40 -->\n      <defs>\n       <path d=\"M 37.796875 64.3125 \nL 12.890625 25.390625 \nL 37.796875 25.390625 \nz\nM 35.203125 72.90625 \nL 47.609375 72.90625 \nL 47.609375 25.390625 \nL 58.015625 25.390625 \nL 58.015625 17.1875 \nL 47.609375 17.1875 \nL 47.609375 0 \nL 37.796875 0 \nL 37.796875 17.1875 \nL 4.890625 17.1875 \nL 4.890625 26.703125 \nz\n\" id=\"DejaVuSans-52\"/>\n      </defs>\n      <g transform=\"translate(190.984039 239.238437)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-52\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_4\">\n     <g id=\"line2d_4\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"266.520093\" xlink:href=\"#m6544ae3bd0\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_4\">\n      <!-- 60 -->\n      <defs>\n       <path d=\"M 33.015625 40.375 \nQ 26.375 40.375 22.484375 35.828125 \nQ 18.609375 31.296875 18.609375 23.390625 \nQ 18.609375 15.53125 22.484375 10.953125 \nQ 26.375 6.390625 33.015625 6.390625 \nQ 39.65625 6.390625 43.53125 10.953125 \nQ 47.40625 15.53125 47.40625 23.390625 \nQ 47.40625 31.296875 43.53125 35.828125 \nQ 39.65625 40.375 33.015625 40.375 \nz\nM 52.59375 71.296875 \nL 52.59375 62.3125 \nQ 48.875 64.0625 45.09375 64.984375 \nQ 41.3125 65.921875 37.59375 65.921875 \nQ 27.828125 65.921875 22.671875 59.328125 \nQ 17.53125 52.734375 16.796875 39.40625 \nQ 19.671875 43.65625 24.015625 45.921875 \nQ 28.375 48.1875 33.59375 48.1875 \nQ 44.578125 48.1875 50.953125 41.515625 \nQ 57.328125 34.859375 57.328125 23.390625 \nQ 57.328125 12.15625 50.6875 5.359375 \nQ 44.046875 -1.421875 33.015625 -1.421875 \nQ 20.359375 -1.421875 13.671875 8.265625 \nQ 6.984375 17.96875 6.984375 36.375 \nQ 6.984375 53.65625 15.1875 63.9375 \nQ 23.390625 74.21875 37.203125 74.21875 \nQ 40.921875 74.21875 44.703125 73.484375 \nQ 48.484375 72.75 52.59375 71.296875 \nz\n\" id=\"DejaVuSans-54\"/>\n      </defs>\n      <g transform=\"translate(260.157593 239.238437)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-54\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_5\">\n     <g id=\"line2d_5\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"335.693647\" xlink:href=\"#m6544ae3bd0\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_5\">\n      <!-- 80 -->\n      <defs>\n       <path d=\"M 31.78125 34.625 \nQ 24.75 34.625 20.71875 30.859375 \nQ 16.703125 27.09375 16.703125 20.515625 \nQ 16.703125 13.921875 20.71875 10.15625 \nQ 24.75 6.390625 31.78125 6.390625 \nQ 38.8125 6.390625 42.859375 10.171875 \nQ 46.921875 13.96875 46.921875 20.515625 \nQ 46.921875 27.09375 42.890625 30.859375 \nQ 38.875 34.625 31.78125 34.625 \nz\nM 21.921875 38.8125 \nQ 15.578125 40.375 12.03125 44.71875 \nQ 8.5 49.078125 8.5 55.328125 \nQ 8.5 64.0625 14.71875 69.140625 \nQ 20.953125 74.21875 31.78125 74.21875 \nQ 42.671875 74.21875 48.875 69.140625 \nQ 55.078125 64.0625 55.078125 55.328125 \nQ 55.078125 49.078125 51.53125 44.71875 \nQ 48 40.375 41.703125 38.8125 \nQ 48.828125 37.15625 52.796875 32.3125 \nQ 56.78125 27.484375 56.78125 20.515625 \nQ 56.78125 9.90625 50.3125 4.234375 \nQ 43.84375 -1.421875 31.78125 -1.421875 \nQ 19.734375 -1.421875 13.25 4.234375 \nQ 6.78125 9.90625 6.78125 20.515625 \nQ 6.78125 27.484375 10.78125 32.3125 \nQ 14.796875 37.15625 21.921875 38.8125 \nz\nM 18.3125 54.390625 \nQ 18.3125 48.734375 21.84375 45.5625 \nQ 25.390625 42.390625 31.78125 42.390625 \nQ 38.140625 42.390625 41.71875 45.5625 \nQ 45.3125 48.734375 45.3125 54.390625 \nQ 45.3125 60.0625 41.71875 63.234375 \nQ 38.140625 66.40625 31.78125 66.40625 \nQ 25.390625 66.40625 21.84375 63.234375 \nQ 18.3125 60.0625 18.3125 54.390625 \nz\n\" id=\"DejaVuSans-56\"/>\n      </defs>\n      <g transform=\"translate(329.331147 239.238437)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-56\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"text_6\">\n     <!-- Epochs -->\n     <defs>\n      <path d=\"M 9.8125 72.90625 \nL 55.90625 72.90625 \nL 55.90625 64.59375 \nL 19.671875 64.59375 \nL 19.671875 43.015625 \nL 54.390625 43.015625 \nL 54.390625 34.71875 \nL 19.671875 34.71875 \nL 19.671875 8.296875 \nL 56.78125 8.296875 \nL 56.78125 0 \nL 9.8125 0 \nz\n\" id=\"DejaVuSans-69\"/>\n      <path d=\"M 18.109375 8.203125 \nL 18.109375 -20.796875 \nL 9.078125 -20.796875 \nL 9.078125 54.6875 \nL 18.109375 54.6875 \nL 18.109375 46.390625 \nQ 20.953125 51.265625 25.265625 53.625 \nQ 29.59375 56 35.59375 56 \nQ 45.5625 56 51.78125 48.09375 \nQ 58.015625 40.1875 58.015625 27.296875 \nQ 58.015625 14.40625 51.78125 6.484375 \nQ 45.5625 -1.421875 35.59375 -1.421875 \nQ 29.59375 -1.421875 25.265625 0.953125 \nQ 20.953125 3.328125 18.109375 8.203125 \nz\nM 48.6875 27.296875 \nQ 48.6875 37.203125 44.609375 42.84375 \nQ 40.53125 48.484375 33.40625 48.484375 \nQ 26.265625 48.484375 22.1875 42.84375 \nQ 18.109375 37.203125 18.109375 27.296875 \nQ 18.109375 17.390625 22.1875 11.75 \nQ 26.265625 6.109375 33.40625 6.109375 \nQ 40.53125 6.109375 44.609375 11.75 \nQ 48.6875 17.390625 48.6875 27.296875 \nz\n\" id=\"DejaVuSans-112\"/>\n      <path d=\"M 30.609375 48.390625 \nQ 23.390625 48.390625 19.1875 42.75 \nQ 14.984375 37.109375 14.984375 27.296875 \nQ 14.984375 17.484375 19.15625 11.84375 \nQ 23.34375 6.203125 30.609375 6.203125 \nQ 37.796875 6.203125 41.984375 11.859375 \nQ 46.1875 17.53125 46.1875 27.296875 \nQ 46.1875 37.015625 41.984375 42.703125 \nQ 37.796875 48.390625 30.609375 48.390625 \nz\nM 30.609375 56 \nQ 42.328125 56 49.015625 48.375 \nQ 55.71875 40.765625 55.71875 27.296875 \nQ 55.71875 13.875 49.015625 6.21875 \nQ 42.328125 -1.421875 30.609375 -1.421875 \nQ 18.84375 -1.421875 12.171875 6.21875 \nQ 5.515625 13.875 5.515625 27.296875 \nQ 5.515625 40.765625 12.171875 48.375 \nQ 18.84375 56 30.609375 56 \nz\n\" id=\"DejaVuSans-111\"/>\n      <path d=\"M 48.78125 52.59375 \nL 48.78125 44.1875 \nQ 44.96875 46.296875 41.140625 47.34375 \nQ 37.3125 48.390625 33.40625 48.390625 \nQ 24.65625 48.390625 19.8125 42.84375 \nQ 14.984375 37.3125 14.984375 27.296875 \nQ 14.984375 17.28125 19.8125 11.734375 \nQ 24.65625 6.203125 33.40625 6.203125 \nQ 37.3125 6.203125 41.140625 7.25 \nQ 44.96875 8.296875 48.78125 10.40625 \nL 48.78125 2.09375 \nQ 45.015625 0.34375 40.984375 -0.53125 \nQ 36.96875 -1.421875 32.421875 -1.421875 \nQ 20.0625 -1.421875 12.78125 6.34375 \nQ 5.515625 14.109375 5.515625 27.296875 \nQ 5.515625 40.671875 12.859375 48.328125 \nQ 20.21875 56 33.015625 56 \nQ 37.15625 56 41.109375 55.140625 \nQ 45.0625 54.296875 48.78125 52.59375 \nz\n\" id=\"DejaVuSans-99\"/>\n      <path d=\"M 54.890625 33.015625 \nL 54.890625 0 \nL 45.90625 0 \nL 45.90625 32.71875 \nQ 45.90625 40.484375 42.875 44.328125 \nQ 39.84375 48.1875 33.796875 48.1875 \nQ 26.515625 48.1875 22.3125 43.546875 \nQ 18.109375 38.921875 18.109375 30.90625 \nL 18.109375 0 \nL 9.078125 0 \nL 9.078125 75.984375 \nL 18.109375 75.984375 \nL 18.109375 46.1875 \nQ 21.34375 51.125 25.703125 53.5625 \nQ 30.078125 56 35.796875 56 \nQ 45.21875 56 50.046875 50.171875 \nQ 54.890625 44.34375 54.890625 33.015625 \nz\n\" id=\"DejaVuSans-104\"/>\n      <path d=\"M 44.28125 53.078125 \nL 44.28125 44.578125 \nQ 40.484375 46.53125 36.375 47.5 \nQ 32.28125 48.484375 27.875 48.484375 \nQ 21.1875 48.484375 17.84375 46.4375 \nQ 14.5 44.390625 14.5 40.28125 \nQ 14.5 37.15625 16.890625 35.375 \nQ 19.28125 33.59375 26.515625 31.984375 \nL 29.59375 31.296875 \nQ 39.15625 29.25 43.1875 25.515625 \nQ 47.21875 21.78125 47.21875 15.09375 \nQ 47.21875 7.46875 41.1875 3.015625 \nQ 35.15625 -1.421875 24.609375 -1.421875 \nQ 20.21875 -1.421875 15.453125 -0.5625 \nQ 10.6875 0.296875 5.421875 2 \nL 5.421875 11.28125 \nQ 10.40625 8.6875 15.234375 7.390625 \nQ 20.0625 6.109375 24.8125 6.109375 \nQ 31.15625 6.109375 34.5625 8.28125 \nQ 37.984375 10.453125 37.984375 14.40625 \nQ 37.984375 18.0625 35.515625 20.015625 \nQ 33.0625 21.96875 24.703125 23.78125 \nL 21.578125 24.515625 \nQ 13.234375 26.265625 9.515625 29.90625 \nQ 5.8125 33.546875 5.8125 39.890625 \nQ 5.8125 47.609375 11.28125 51.796875 \nQ 16.75 56 26.8125 56 \nQ 31.78125 56 36.171875 55.265625 \nQ 40.578125 54.546875 44.28125 53.078125 \nz\n\" id=\"DejaVuSans-115\"/>\n     </defs>\n     <g transform=\"translate(193.265625 252.916562)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-69\"/>\n      <use x=\"63.183594\" xlink:href=\"#DejaVuSans-112\"/>\n      <use x=\"126.660156\" xlink:href=\"#DejaVuSans-111\"/>\n      <use x=\"187.841797\" xlink:href=\"#DejaVuSans-99\"/>\n      <use x=\"242.822266\" xlink:href=\"#DejaVuSans-104\"/>\n      <use x=\"306.201172\" xlink:href=\"#DejaVuSans-115\"/>\n     </g>\n    </g>\n   </g>\n   <g id=\"matplotlib.axis_2\">\n    <g id=\"ytick_1\">\n     <g id=\"line2d_6\">\n      <defs>\n       <path d=\"M 0 0 \nL -3.5 0 \n\" id=\"mb65e906bb8\" style=\"stroke:#000000;stroke-width:0.8;\"/>\n      </defs>\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#mb65e906bb8\" y=\"215.76133\"/>\n      </g>\n     </g>\n     <g id=\"text_7\">\n      <!-- 0.0 -->\n      <defs>\n       <path d=\"M 10.6875 12.40625 \nL 21 12.40625 \nL 21 0 \nL 10.6875 0 \nz\n\" id=\"DejaVuSans-46\"/>\n      </defs>\n      <g transform=\"translate(20.878125 219.560549)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_2\">\n     <g id=\"line2d_7\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#mb65e906bb8\" y=\"191.123647\"/>\n      </g>\n     </g>\n     <g id=\"text_8\">\n      <!-- 0.1 -->\n      <defs>\n       <path d=\"M 12.40625 8.296875 \nL 28.515625 8.296875 \nL 28.515625 63.921875 \nL 10.984375 60.40625 \nL 10.984375 69.390625 \nL 28.421875 72.90625 \nL 38.28125 72.90625 \nL 38.28125 8.296875 \nL 54.390625 8.296875 \nL 54.390625 0 \nL 12.40625 0 \nz\n\" id=\"DejaVuSans-49\"/>\n      </defs>\n      <g transform=\"translate(20.878125 194.922865)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-49\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_3\">\n     <g id=\"line2d_8\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#mb65e906bb8\" y=\"166.485963\"/>\n      </g>\n     </g>\n     <g id=\"text_9\">\n      <!-- 0.2 -->\n      <g transform=\"translate(20.878125 170.285182)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-50\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_4\">\n     <g id=\"line2d_9\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#mb65e906bb8\" y=\"141.848279\"/>\n      </g>\n     </g>\n     <g id=\"text_10\">\n      <!-- 0.3 -->\n      <defs>\n       <path d=\"M 40.578125 39.3125 \nQ 47.65625 37.796875 51.625 33 \nQ 55.609375 28.21875 55.609375 21.1875 \nQ 55.609375 10.40625 48.1875 4.484375 \nQ 40.765625 -1.421875 27.09375 -1.421875 \nQ 22.515625 -1.421875 17.65625 -0.515625 \nQ 12.796875 0.390625 7.625 2.203125 \nL 7.625 11.71875 \nQ 11.71875 9.328125 16.59375 8.109375 \nQ 21.484375 6.890625 26.8125 6.890625 \nQ 36.078125 6.890625 40.9375 10.546875 \nQ 45.796875 14.203125 45.796875 21.1875 \nQ 45.796875 27.640625 41.28125 31.265625 \nQ 36.765625 34.90625 28.71875 34.90625 \nL 20.21875 34.90625 \nL 20.21875 43.015625 \nL 29.109375 43.015625 \nQ 36.375 43.015625 40.234375 45.921875 \nQ 44.09375 48.828125 44.09375 54.296875 \nQ 44.09375 59.90625 40.109375 62.90625 \nQ 36.140625 65.921875 28.71875 65.921875 \nQ 24.65625 65.921875 20.015625 65.03125 \nQ 15.375 64.15625 9.8125 62.3125 \nL 9.8125 71.09375 \nQ 15.4375 72.65625 20.34375 73.4375 \nQ 25.25 74.21875 29.59375 74.21875 \nQ 40.828125 74.21875 47.359375 69.109375 \nQ 53.90625 64.015625 53.90625 55.328125 \nQ 53.90625 49.265625 50.4375 45.09375 \nQ 46.96875 40.921875 40.578125 39.3125 \nz\n\" id=\"DejaVuSans-51\"/>\n      </defs>\n      <g transform=\"translate(20.878125 145.647498)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-51\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_5\">\n     <g id=\"line2d_10\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#mb65e906bb8\" y=\"117.210596\"/>\n      </g>\n     </g>\n     <g id=\"text_11\">\n      <!-- 0.4 -->\n      <g transform=\"translate(20.878125 121.009814)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-52\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_6\">\n     <g id=\"line2d_11\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#mb65e906bb8\" y=\"92.572912\"/>\n      </g>\n     </g>\n     <g id=\"text_12\">\n      <!-- 0.5 -->\n      <defs>\n       <path d=\"M 10.796875 72.90625 \nL 49.515625 72.90625 \nL 49.515625 64.59375 \nL 19.828125 64.59375 \nL 19.828125 46.734375 \nQ 21.96875 47.46875 24.109375 47.828125 \nQ 26.265625 48.1875 28.421875 48.1875 \nQ 40.625 48.1875 47.75 41.5 \nQ 54.890625 34.8125 54.890625 23.390625 \nQ 54.890625 11.625 47.5625 5.09375 \nQ 40.234375 -1.421875 26.90625 -1.421875 \nQ 22.3125 -1.421875 17.546875 -0.640625 \nQ 12.796875 0.140625 7.71875 1.703125 \nL 7.71875 11.625 \nQ 12.109375 9.234375 16.796875 8.0625 \nQ 21.484375 6.890625 26.703125 6.890625 \nQ 35.15625 6.890625 40.078125 11.328125 \nQ 45.015625 15.765625 45.015625 23.390625 \nQ 45.015625 31 40.078125 35.4375 \nQ 35.15625 39.890625 26.703125 39.890625 \nQ 22.75 39.890625 18.8125 39.015625 \nQ 14.890625 38.140625 10.796875 36.28125 \nz\n\" id=\"DejaVuSans-53\"/>\n      </defs>\n      <g transform=\"translate(20.878125 96.372131)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-53\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_7\">\n     <g id=\"line2d_12\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#mb65e906bb8\" y=\"67.935228\"/>\n      </g>\n     </g>\n     <g id=\"text_13\">\n      <!-- 0.6 -->\n      <g transform=\"translate(20.878125 71.734447)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-54\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_8\">\n     <g id=\"line2d_13\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#mb65e906bb8\" y=\"43.297545\"/>\n      </g>\n     </g>\n     <g id=\"text_14\">\n      <!-- 0.7 -->\n      <defs>\n       <path d=\"M 8.203125 72.90625 \nL 55.078125 72.90625 \nL 55.078125 68.703125 \nL 28.609375 0 \nL 18.3125 0 \nL 43.21875 64.59375 \nL 8.203125 64.59375 \nz\n\" id=\"DejaVuSans-55\"/>\n      </defs>\n      <g transform=\"translate(20.878125 47.096763)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-55\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_9\">\n     <g id=\"line2d_14\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#mb65e906bb8\" y=\"18.659861\"/>\n      </g>\n     </g>\n     <g id=\"text_15\">\n      <!-- 0.8 -->\n      <g transform=\"translate(20.878125 22.45908)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-56\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"text_16\">\n     <!-- Losses -->\n     <defs>\n      <path d=\"M 9.8125 72.90625 \nL 19.671875 72.90625 \nL 19.671875 8.296875 \nL 55.171875 8.296875 \nL 55.171875 0 \nL 9.8125 0 \nz\n\" id=\"DejaVuSans-76\"/>\n      <path d=\"M 56.203125 29.59375 \nL 56.203125 25.203125 \nL 14.890625 25.203125 \nQ 15.484375 15.921875 20.484375 11.0625 \nQ 25.484375 6.203125 34.421875 6.203125 \nQ 39.59375 6.203125 44.453125 7.46875 \nQ 49.3125 8.734375 54.109375 11.28125 \nL 54.109375 2.78125 \nQ 49.265625 0.734375 44.1875 -0.34375 \nQ 39.109375 -1.421875 33.890625 -1.421875 \nQ 20.796875 -1.421875 13.15625 6.1875 \nQ 5.515625 13.8125 5.515625 26.8125 \nQ 5.515625 40.234375 12.765625 48.109375 \nQ 20.015625 56 32.328125 56 \nQ 43.359375 56 49.78125 48.890625 \nQ 56.203125 41.796875 56.203125 29.59375 \nz\nM 47.21875 32.234375 \nQ 47.125 39.59375 43.09375 43.984375 \nQ 39.0625 48.390625 32.421875 48.390625 \nQ 24.90625 48.390625 20.390625 44.140625 \nQ 15.875 39.890625 15.1875 32.171875 \nz\n\" id=\"DejaVuSans-101\"/>\n     </defs>\n     <g transform=\"translate(14.798438 132.655156)rotate(-90)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-76\"/>\n      <use x=\"55.697266\" xlink:href=\"#DejaVuSans-111\"/>\n      <use x=\"116.878906\" xlink:href=\"#DejaVuSans-115\"/>\n      <use x=\"168.978516\" xlink:href=\"#DejaVuSans-115\"/>\n      <use x=\"221.078125\" xlink:href=\"#DejaVuSans-101\"/>\n      <use x=\"282.601562\" xlink:href=\"#DejaVuSans-115\"/>\n     </g>\n    </g>\n   </g>\n   <g id=\"line2d_15\">\n    <path clip-path=\"url(#p1692245912)\" d=\"M 58.999432 17.083636 \nL 62.45811 94.313065 \nL 65.916787 73.703776 \nL 69.375465 90.800937 \nL 72.834143 75.754741 \nL 76.29282 96.485278 \nL 79.751498 121.652193 \nL 83.210176 115.983043 \nL 86.668853 134.115716 \nL 90.127531 145.239898 \nL 93.586209 151.527412 \nL 97.044886 139.0672 \nL 100.503564 125.808921 \nL 103.962242 137.973791 \nL 107.420919 124.183415 \nL 110.879597 125.964099 \nL 114.338275 141.249216 \nL 117.796952 135.844113 \nL 121.25563 160.982517 \nL 124.714308 157.12445 \nL 128.172986 151.504194 \nL 131.631663 170.18323 \nL 135.090341 175.145552 \nL 138.549019 167.062661 \nL 142.007696 161.026147 \nL 145.466374 163.539541 \nL 148.925052 160.354967 \nL 152.383729 170.440897 \nL 155.842407 142.767864 \nL 159.301085 178.541747 \nL 162.759762 175.680985 \nL 166.21844 178.946134 \nL 169.677118 188.823154 \nL 173.135795 186.863977 \nL 176.594473 198.655509 \nL 180.053151 199.855072 \nL 183.511829 204.134573 \nL 186.970506 203.396317 \nL 190.429184 201.17001 \nL 193.887862 201.817149 \nL 197.346539 208.64538 \nL 200.805217 205.288806 \nL 204.263895 197.193878 \nL 207.722572 181.172587 \nL 211.18125 202.625444 \nL 214.639928 208.279477 \nL 218.098605 198.209876 \nL 221.557283 185.859832 \nL 225.015961 211.868382 \nL 228.474638 204.343696 \nL 231.933316 197.88355 \nL 235.391994 209.384974 \nL 238.850671 209.495534 \nL 242.309349 206.929544 \nL 245.768027 211.0584 \nL 249.226705 208.828556 \nL 252.685382 195.80513 \nL 256.14406 206.607554 \nL 259.602738 211.455206 \nL 263.061415 204.918166 \nL 266.520093 206.63611 \nL 269.978771 212.955956 \nL 273.437448 212.902696 \nL 276.896126 195.644576 \nL 280.354804 201.686602 \nL 283.813481 204.113947 \nL 287.272159 212.967388 \nL 290.730837 210.963131 \nL 294.189514 209.957074 \nL 297.648192 211.081695 \nL 301.10687 201.731323 \nL 304.565548 211.717271 \nL 308.024225 211.471951 \nL 311.482903 208.368487 \nL 314.941581 208.148928 \nL 318.400258 210.504816 \nL 321.858936 212.764182 \nL 325.317614 214.128266 \nL 328.776291 211.556998 \nL 332.234969 212.973153 \nL 335.693647 214.655952 \nL 339.152324 212.666767 \nL 342.611002 205.852692 \nL 346.06968 214.756364 \nL 349.528357 212.892987 \nL 352.987035 210.540427 \nL 356.445713 208.142238 \nL 359.90439 211.405917 \nL 363.363068 214.727806 \n\" style=\"fill:none;stroke:#ff0000;stroke-linecap:square;stroke-width:1.5;\"/>\n   </g>\n   <g id=\"line2d_16\">\n    <path clip-path=\"url(#p1692245912)\" d=\"M 58.999432 81.318172 \nL 62.45811 91.043654 \nL 65.916787 88.328301 \nL 69.375465 94.105379 \nL 72.834143 110.927823 \nL 76.29282 98.823096 \nL 79.751498 27.622693 \nL 83.210176 85.220486 \nL 86.668853 102.060156 \nL 90.127531 121.863234 \nL 93.586209 107.266163 \nL 97.044886 116.357913 \nL 100.503564 96.627857 \nL 103.962242 111.509659 \nL 107.420919 112.44142 \nL 110.879597 119.763375 \nL 114.338275 115.244752 \nL 117.796952 118.458286 \nL 121.25563 122.560436 \nL 124.714308 92.655678 \nL 128.172986 121.057897 \nL 131.631663 108.161733 \nL 135.090341 89.408955 \nL 138.549019 115.92431 \nL 142.007696 112.18401 \nL 145.466374 75.87629 \nL 148.925052 117.276545 \nL 152.383729 118.872262 \nL 155.842407 117.744306 \nL 159.301085 133.21102 \nL 162.759762 152.573086 \nL 166.21844 151.023452 \nL 169.677118 140.696178 \nL 173.135795 137.42104 \nL 176.594473 138.401733 \nL 180.053151 131.270495 \nL 183.511829 145.409571 \nL 186.970506 135.341424 \nL 190.429184 134.465173 \nL 193.887862 135.88985 \nL 197.346539 130.167489 \nL 200.805217 136.698234 \nL 204.263895 135.598004 \nL 207.722572 141.830706 \nL 211.18125 136.612267 \nL 214.639928 152.158413 \nL 218.098605 118.713074 \nL 221.557283 145.941336 \nL 225.015961 139.750099 \nL 228.474638 141.683031 \nL 231.933316 139.636347 \nL 235.391994 118.40514 \nL 238.850671 125.070108 \nL 242.309349 129.092143 \nL 245.768027 122.578653 \nL 249.226705 136.947927 \nL 252.685382 126.905032 \nL 256.14406 136.402371 \nL 259.602738 123.386031 \nL 263.061415 129.618167 \nL 266.520093 125.653808 \nL 269.978771 123.721338 \nL 273.437448 124.265667 \nL 276.896126 128.477457 \nL 280.354804 123.924537 \nL 283.813481 126.686927 \nL 287.272159 126.570062 \nL 290.730837 126.787447 \nL 294.189514 121.740936 \nL 297.648192 132.16779 \nL 301.10687 125.975429 \nL 304.565548 128.637276 \nL 308.024225 132.41202 \nL 311.482903 125.962675 \nL 314.941581 122.458836 \nL 318.400258 115.796475 \nL 321.858936 121.828056 \nL 325.317614 126.666536 \nL 328.776291 118.308152 \nL 332.234969 119.794412 \nL 335.693647 121.548009 \nL 339.152324 127.583583 \nL 342.611002 127.614143 \nL 346.06968 126.732921 \nL 349.528357 120.231517 \nL 352.987035 126.541543 \nL 356.445713 122.873635 \nL 359.90439 121.952197 \nL 363.363068 122.9787 \n\" style=\"fill:none;stroke:#0000ff;stroke-linecap:square;stroke-width:1.5;\"/>\n   </g>\n   <g id=\"patch_3\">\n    <path d=\"M 43.78125 224.64 \nL 43.78125 7.2 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_4\">\n    <path d=\"M 378.58125 224.64 \nL 378.58125 7.2 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_5\">\n    <path d=\"M 43.78125 224.64 \nL 378.58125 224.64 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_6\">\n    <path d=\"M 43.78125 7.2 \nL 378.58125 7.2 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"legend_1\">\n    <g id=\"patch_7\">\n     <path d=\"M 314.140625 44.55625 \nL 371.58125 44.55625 \nQ 373.58125 44.55625 373.58125 42.55625 \nL 373.58125 14.2 \nQ 373.58125 12.2 371.58125 12.2 \nL 314.140625 12.2 \nQ 312.140625 12.2 312.140625 14.2 \nL 312.140625 42.55625 \nQ 312.140625 44.55625 314.140625 44.55625 \nz\n\" style=\"fill:#ffffff;opacity:0.8;stroke:#cccccc;stroke-linejoin:miter;\"/>\n    </g>\n    <g id=\"line2d_17\">\n     <path d=\"M 316.140625 20.298437 \nL 336.140625 20.298437 \n\" style=\"fill:none;stroke:#ff0000;stroke-linecap:square;stroke-width:1.5;\"/>\n    </g>\n    <g id=\"line2d_18\"/>\n    <g id=\"text_17\">\n     <!-- Train -->\n     <defs>\n      <path d=\"M -0.296875 72.90625 \nL 61.375 72.90625 \nL 61.375 64.59375 \nL 35.5 64.59375 \nL 35.5 0 \nL 25.59375 0 \nL 25.59375 64.59375 \nL -0.296875 64.59375 \nz\n\" id=\"DejaVuSans-84\"/>\n      <path d=\"M 41.109375 46.296875 \nQ 39.59375 47.171875 37.8125 47.578125 \nQ 36.03125 48 33.890625 48 \nQ 26.265625 48 22.1875 43.046875 \nQ 18.109375 38.09375 18.109375 28.8125 \nL 18.109375 0 \nL 9.078125 0 \nL 9.078125 54.6875 \nL 18.109375 54.6875 \nL 18.109375 46.1875 \nQ 20.953125 51.171875 25.484375 53.578125 \nQ 30.03125 56 36.53125 56 \nQ 37.453125 56 38.578125 55.875 \nQ 39.703125 55.765625 41.0625 55.515625 \nz\n\" id=\"DejaVuSans-114\"/>\n      <path d=\"M 34.28125 27.484375 \nQ 23.390625 27.484375 19.1875 25 \nQ 14.984375 22.515625 14.984375 16.5 \nQ 14.984375 11.71875 18.140625 8.90625 \nQ 21.296875 6.109375 26.703125 6.109375 \nQ 34.1875 6.109375 38.703125 11.40625 \nQ 43.21875 16.703125 43.21875 25.484375 \nL 43.21875 27.484375 \nz\nM 52.203125 31.203125 \nL 52.203125 0 \nL 43.21875 0 \nL 43.21875 8.296875 \nQ 40.140625 3.328125 35.546875 0.953125 \nQ 30.953125 -1.421875 24.3125 -1.421875 \nQ 15.921875 -1.421875 10.953125 3.296875 \nQ 6 8.015625 6 15.921875 \nQ 6 25.140625 12.171875 29.828125 \nQ 18.359375 34.515625 30.609375 34.515625 \nL 43.21875 34.515625 \nL 43.21875 35.40625 \nQ 43.21875 41.609375 39.140625 45 \nQ 35.0625 48.390625 27.6875 48.390625 \nQ 23 48.390625 18.546875 47.265625 \nQ 14.109375 46.140625 10.015625 43.890625 \nL 10.015625 52.203125 \nQ 14.9375 54.109375 19.578125 55.046875 \nQ 24.21875 56 28.609375 56 \nQ 40.484375 56 46.34375 49.84375 \nQ 52.203125 43.703125 52.203125 31.203125 \nz\n\" id=\"DejaVuSans-97\"/>\n      <path d=\"M 9.421875 54.6875 \nL 18.40625 54.6875 \nL 18.40625 0 \nL 9.421875 0 \nz\nM 9.421875 75.984375 \nL 18.40625 75.984375 \nL 18.40625 64.59375 \nL 9.421875 64.59375 \nz\n\" id=\"DejaVuSans-105\"/>\n      <path d=\"M 54.890625 33.015625 \nL 54.890625 0 \nL 45.90625 0 \nL 45.90625 32.71875 \nQ 45.90625 40.484375 42.875 44.328125 \nQ 39.84375 48.1875 33.796875 48.1875 \nQ 26.515625 48.1875 22.3125 43.546875 \nQ 18.109375 38.921875 18.109375 30.90625 \nL 18.109375 0 \nL 9.078125 0 \nL 9.078125 54.6875 \nL 18.109375 54.6875 \nL 18.109375 46.1875 \nQ 21.34375 51.125 25.703125 53.5625 \nQ 30.078125 56 35.796875 56 \nQ 45.21875 56 50.046875 50.171875 \nQ 54.890625 44.34375 54.890625 33.015625 \nz\n\" id=\"DejaVuSans-110\"/>\n     </defs>\n     <g transform=\"translate(344.140625 23.798437)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-84\"/>\n      <use x=\"60.865234\" xlink:href=\"#DejaVuSans-114\"/>\n      <use x=\"101.978516\" xlink:href=\"#DejaVuSans-97\"/>\n      <use x=\"163.257812\" xlink:href=\"#DejaVuSans-105\"/>\n      <use x=\"191.041016\" xlink:href=\"#DejaVuSans-110\"/>\n     </g>\n    </g>\n    <g id=\"line2d_19\">\n     <path d=\"M 316.140625 34.976562 \nL 336.140625 34.976562 \n\" style=\"fill:none;stroke:#0000ff;stroke-linecap:square;stroke-width:1.5;\"/>\n    </g>\n    <g id=\"line2d_20\"/>\n    <g id=\"text_18\">\n     <!-- Val -->\n     <defs>\n      <path d=\"M 28.609375 0 \nL 0.78125 72.90625 \nL 11.078125 72.90625 \nL 34.1875 11.53125 \nL 57.328125 72.90625 \nL 67.578125 72.90625 \nL 39.796875 0 \nz\n\" id=\"DejaVuSans-86\"/>\n      <path d=\"M 9.421875 75.984375 \nL 18.40625 75.984375 \nL 18.40625 0 \nL 9.421875 0 \nz\n\" id=\"DejaVuSans-108\"/>\n     </defs>\n     <g transform=\"translate(344.140625 38.476562)scale(0.1 -0.1)\">\n      <use xlink:href=\"#DejaVuSans-86\"/>\n      <use x=\"68.298828\" xlink:href=\"#DejaVuSans-97\"/>\n      <use x=\"129.578125\" xlink:href=\"#DejaVuSans-108\"/>\n     </g>\n    </g>\n   </g>\n  </g>\n </g>\n <defs>\n  <clipPath id=\"p1692245912\">\n   <rect height=\"217.44\" width=\"334.8\" x=\"43.78125\" y=\"7.2\"/>\n  </clipPath>\n </defs>\n</svg>\n"
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SV5OcSRgXEBs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import files\n",
        "files.download('./radam_lookahead_resnet18_9320.pth.tar')\n",
        "files.download('./radam_lookahead_resnet18_9320_acc.jpg')\n",
        "files.download('./radam_lookahead_resnet18_9320_loss.jpg')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "um6lklwAFTUo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}