04-패션_아이템을_구분하는_DNN/overfitting_and_regularization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4.3 오버피팅과 정규화 (Overfitting and Regularization)\n",
    "\n",
    "머신러닝 모델\n",
    "과적합(Overfitting)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "from torchvision import transforms, datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "USE_CUDA = torch.cuda.is_available()\n",
    "DEVICE = torch.device(\"cuda\" if USE_CUDA else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "EPOCHS = 50\n",
    "BATCH_SIZE = 64"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 데이터셋에 노이즈 추가하기\n",
    "\n",
    "![original.png](./assets/original.png)\n",
    "\n",
    "![horizontalflip.png](./assets/horizontalflip.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-images-idx3-ubyte.gz to ./.data/FashionMNIST/raw/train-images-idx3-ubyte.gz\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1aeda6e7f2f949f3b40910d4bbe9f643",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=1.0, bar_style='info', layout=Layout(width='20px'), max=1.0…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ./.data/FashionMNIST/raw/train-images-idx3-ubyte.gz to ./.data/FashionMNIST/raw\n",
      "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-labels-idx1-ubyte.gz to ./.data/FashionMNIST/raw/train-labels-idx1-ubyte.gz\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "32d0e96e1df2498aa0cd05bfef74a0cb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=1.0, bar_style='info', layout=Layout(width='20px'), max=1.0…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ./.data/FashionMNIST/raw/train-labels-idx1-ubyte.gz to ./.data/FashionMNIST/raw\n",
      "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-images-idx3-ubyte.gz to ./.data/FashionMNIST/raw/t10k-images-idx3-ubyte.gz\n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9ba3ba91a8114a3cb6c93dfdfb0716ea",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=1.0, bar_style='info', layout=Layout(width='20px'), max=1.0…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ./.data/FashionMNIST/raw/t10k-images-idx3-ubyte.gz to ./.data/FashionMNIST/raw\n",
      "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-labels-idx1-ubyte.gz to ./.data/FashionMNIST/raw/t10k-labels-idx1-ubyte.gz\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b22aa7f5b217481da36362df3baf5082",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=1.0, bar_style='info', layout=Layout(width='20px'), max=1.0…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ./.data/FashionMNIST/raw/t10k-labels-idx1-ubyte.gz to ./.data/FashionMNIST/raw\n",
      "Processing...\n",
      "Done!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/keon/projects/3-min-pytorch/env/lib/python3.8/site-packages/torchvision/datasets/mnist.py:469: UserWarning: The given NumPy array is not writeable, and PyTorch does not support non-writeable tensors. This means you can write to the underlying (supposedly non-writeable) NumPy array using the tensor. You may want to copy the array to protect its data or make it writeable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at  ../torch/csrc/utils/tensor_numpy.cpp:141.)\n",
      "  return torch.from_numpy(parsed.astype(m[2], copy=False)).view(*s)\n"
     ]
    }
   ],
   "source": [
    "train_loader = torch.utils.data.DataLoader(\n",
    "    datasets.FashionMNIST('./.data',\n",
    "                   train=True,\n",
    "                   download=True,\n",
    "                   transform=transforms.Compose([\n",
    "                       transforms.RandomHorizontalFlip(),\n",
    "                       transforms.ToTensor(),\n",
    "                       transforms.Normalize((0.1307,), (0.3081,))\n",
    "                   ])),\n",
    "    batch_size=BATCH_SIZE, shuffle=True)\n",
    "test_loader = torch.utils.data.DataLoader(\n",
    "    datasets.FashionMNIST('./.data',\n",
    "                   train=False, \n",
    "                   transform=transforms.Compose([\n",
    "                       transforms.ToTensor(),\n",
    "                       transforms.Normalize((0.1307,), (0.3081,))\n",
    "                   ])),\n",
    "    batch_size=BATCH_SIZE, shuffle=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 뉴럴넷으로 Fashion MNIST 학습하기\n",
    "\n",
    "입력 `x` 는 `[배치크기, 색, 높이, 넓이]`로 이루어져 있습니다.\n",
    "`x.size()`를 해보면 `[64, 1, 28, 28]`이라고 표시되는 것을 보실 수 있습니다.\n",
    "Fashion MNIST에서 이미지의 크기는 28 x 28, 색은 흑백으로 1 가지 입니다.\n",
    "그러므로 입력 x의 총 특성값 갯수는 28 x 28 x 1, 즉 784개 입니다.\n",
    "\n",
    "우리가 사용할 모델은 3개의 레이어를 가진 뉴럴네트워크 입니다. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "    def __init__(self, dropout_p=0.2):\n",
    "        super(Net, self).__init__()\n",
    "        self.fc1 = nn.Linear(784, 256)\n",
    "        self.fc2 = nn.Linear(256, 128)\n",
    "        self.fc3 = nn.Linear(128, 10)\n",
    "        # 드롭아웃 확률\n",
    "        self.dropout_p = dropout_p\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = x.view(-1, 784)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        # 드롭아웃 추가\n",
    "        x = F.dropout(x, training=self.training,\n",
    "                      p=self.dropout_p)\n",
    "        x = F.relu(self.fc2(x))\n",
    "        # 드롭아웃 추가\n",
    "        x = F.dropout(x, training=self.training,\n",
    "                      p=self.dropout_p)\n",
    "        x = self.fc3(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 모델 준비하기 \n",
    "\n",
    "`to()` 함수는 모델의 파라미터들을 지정한 곳으로 보내는 역할을 합니다.\n",
    "일반적으로 CPU 1개만 사용할 경우 필요는 없지만,\n",
    "GPU를 사용하고자 하는 경우 `to(\"cuda\")`로 지정하여 GPU로 보내야 합니다.\n",
    "지정하지 않을 경우 계속 CPU에 남아 있게 되며 빠른 훈련의 이점을 누리실 수 없습니다.\n",
    "\n",
    "최적화 알고리즘으로 파이토치에 내장되어 있는 `optim.SGD`를 사용하겠습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "model        = Net(dropout_p=0.2).to(DEVICE)\n",
    "optimizer    = optim.SGD(model.parameters(), lr=0.01)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 학습하기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, train_loader, optimizer):\n",
    "    model.train()\n",
    "    for batch_idx, (data, target) in enumerate(train_loader):\n",
    "        data, target = data.to(DEVICE), target.to(DEVICE)\n",
    "        optimizer.zero_grad()\n",
    "        output = model(data)\n",
    "        loss = F.cross_entropy(output, target)\n",
    "        loss.backward()\n",
    "        optimizer.step()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 테스트하기\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "def evaluate(model, test_loader):\n",
    "    model.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    with torch.no_grad():\n",
    "        for data, target in test_loader:\n",
    "            data, target = data.to(DEVICE), target.to(DEVICE)\n",
    "            output = model(data)\n",
    "            test_loss += F.cross_entropy(output, target,\n",
    "                                         reduction='sum').item()\n",
    "            \n",
    "            # 맞춘 갯수 계산\n",
    "            pred = output.max(1, keepdim=True)[1]\n",
    "            correct += pred.eq(target.view_as(pred)).sum().item()\n",
    "\n",
    "    test_loss /= len(test_loader.dataset)\n",
    "    test_accuracy = 100. * correct / len(test_loader.dataset)\n",
    "    return test_loss, test_accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 코드 돌려보기\n",
    "\n",
    "자, 이제 모든 준비가 끝났습니다. 코드를 돌려서 실제로 훈련이 되는지 확인해봅시다!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] Test Loss: 0.6536, Accuracy: 76.92%\n",
      "[2] Test Loss: 0.5359, Accuracy: 80.72%\n",
      "[3] Test Loss: 0.4977, Accuracy: 82.18%\n",
      "[4] Test Loss: 0.4584, Accuracy: 83.49%\n",
      "[5] Test Loss: 0.4421, Accuracy: 84.22%\n",
      "[6] Test Loss: 0.4245, Accuracy: 84.91%\n",
      "[7] Test Loss: 0.4205, Accuracy: 84.82%\n",
      "[8] Test Loss: 0.4067, Accuracy: 85.32%\n",
      "[9] Test Loss: 0.4024, Accuracy: 85.54%\n",
      "[10] Test Loss: 0.3923, Accuracy: 85.82%\n",
      "[11] Test Loss: 0.3875, Accuracy: 86.22%\n",
      "[12] Test Loss: 0.3772, Accuracy: 86.19%\n",
      "[13] Test Loss: 0.3807, Accuracy: 86.11%\n",
      "[14] Test Loss: 0.3762, Accuracy: 86.33%\n",
      "[15] Test Loss: 0.3636, Accuracy: 86.70%\n",
      "[16] Test Loss: 0.3699, Accuracy: 86.29%\n",
      "[17] Test Loss: 0.3625, Accuracy: 86.92%\n",
      "[18] Test Loss: 0.3567, Accuracy: 87.06%\n",
      "[19] Test Loss: 0.3515, Accuracy: 87.41%\n",
      "[20] Test Loss: 0.3490, Accuracy: 87.50%\n",
      "[21] Test Loss: 0.3572, Accuracy: 86.86%\n",
      "[22] Test Loss: 0.3504, Accuracy: 87.10%\n",
      "[23] Test Loss: 0.3465, Accuracy: 87.48%\n",
      "[24] Test Loss: 0.3383, Accuracy: 87.60%\n",
      "[25] Test Loss: 0.3347, Accuracy: 87.91%\n",
      "[26] Test Loss: 0.3370, Accuracy: 87.76%\n",
      "[27] Test Loss: 0.3330, Accuracy: 88.07%\n",
      "[28] Test Loss: 0.3338, Accuracy: 88.12%\n",
      "[29] Test Loss: 0.3316, Accuracy: 87.98%\n",
      "[30] Test Loss: 0.3280, Accuracy: 88.29%\n",
      "[31] Test Loss: 0.3241, Accuracy: 88.31%\n",
      "[32] Test Loss: 0.3279, Accuracy: 88.23%\n",
      "[33] Test Loss: 0.3261, Accuracy: 88.05%\n",
      "[34] Test Loss: 0.3262, Accuracy: 88.13%\n",
      "[35] Test Loss: 0.3207, Accuracy: 88.47%\n",
      "[36] Test Loss: 0.3287, Accuracy: 88.15%\n",
      "[37] Test Loss: 0.3221, Accuracy: 88.43%\n",
      "[38] Test Loss: 0.3215, Accuracy: 88.31%\n",
      "[39] Test Loss: 0.3165, Accuracy: 88.60%\n",
      "[40] Test Loss: 0.3178, Accuracy: 88.66%\n",
      "[41] Test Loss: 0.3270, Accuracy: 88.00%\n",
      "[42] Test Loss: 0.3130, Accuracy: 88.84%\n",
      "[43] Test Loss: 0.3118, Accuracy: 89.00%\n",
      "[44] Test Loss: 0.3142, Accuracy: 88.57%\n",
      "[45] Test Loss: 0.3131, Accuracy: 88.70%\n",
      "[46] Test Loss: 0.3202, Accuracy: 88.44%\n",
      "[47] Test Loss: 0.3144, Accuracy: 88.76%\n",
      "[48] Test Loss: 0.3121, Accuracy: 89.03%\n",
      "[49] Test Loss: 0.3177, Accuracy: 88.58%\n",
      "[50] Test Loss: 0.3127, Accuracy: 88.94%\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(1, EPOCHS + 1):\n",
    "    train(model, train_loader, optimizer)\n",
    "    test_loss, test_accuracy = evaluate(model, test_loader)\n",
    "    \n",
    "    print('[{}] Test Loss: {:.4f}, Accuracy: {:.2f}%'.format(\n",
    "          epoch, test_loss, test_accuracy))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}