fashion-mnist.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/pradeepsingh/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:34: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "from keras.datasets import mnist\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Flatten, Lambda\n",
    "from keras.layers import Conv2D, MaxPooling2D, BatchNormalization\n",
    "from keras.optimizers import Adam\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.image as mpimg\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "np.random.seed(12345)\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://raw.githubusercontent.com/zalandoresearch/fashion-mnist/master/data/fashion/train-images-idx3-ubyte.gz\n",
      "26427392/26421880 [==============================] - 1s 0us/step\n",
      "Downloading data from https://raw.githubusercontent.com/zalandoresearch/fashion-mnist/master/data/fashion/train-labels-idx1-ubyte.gz\n",
      "32768/29515 [=================================] - 0s 0us/step\n",
      "Downloading data from https://raw.githubusercontent.com/zalandoresearch/fashion-mnist/master/data/fashion/t10k-images-idx3-ubyte.gz\n",
      "4423680/4422102 [==============================] - 0s 0us/step\n",
      "Downloading data from https://raw.githubusercontent.com/zalandoresearch/fashion-mnist/master/data/fashion/t10k-labels-idx1-ubyte.gz\n",
      "8192/5148 [===============================================] - 0s 0us/step\n"
     ]
    }
   ],
   "source": [
    "train_images_path = keras.utils.get_file('train-images-idx3-ubyte.gz', 'https://raw.githubusercontent.com/zalandoresearch/fashion-mnist/master/data/fashion/train-images-idx3-ubyte.gz')\n",
    "train_labels_path = keras.utils.get_file('train-labels-idx1-ubyte.gz', 'https://raw.githubusercontent.com/zalandoresearch/fashion-mnist/master/data/fashion/train-labels-idx1-ubyte.gz')\n",
    "test_images_path = keras.utils.get_file('t10k-images-idx3-ubyte.gz', 'https://raw.githubusercontent.com/zalandoresearch/fashion-mnist/master/data/fashion/t10k-images-idx3-ubyte.gz')\n",
    "test_labels_path = keras.utils.get_file('t10k-labels-idx1-ubyte.gz', 'https://raw.githubusercontent.com/zalandoresearch/fashion-mnist/master/data/fashion/t10k-labels-idx1-ubyte.gz')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batch_size = 512"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def load_mnist(images_path, labels_path):\n",
    "    import os\n",
    "    import gzip\n",
    "    import numpy as np\n",
    "\n",
    "    with gzip.open(labels_path, 'rb') as lbpath:\n",
    "        labels = np.frombuffer(lbpath.read(), dtype=np.uint8,\n",
    "                               offset=8)\n",
    "\n",
    "    with gzip.open(images_path, 'rb') as imgpath:\n",
    "        images = np.frombuffer(imgpath.read(), dtype=np.uint8,\n",
    "                               offset=16).reshape(len(labels), 784)\n",
    "\n",
    "    return images, labels\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train_orig, y_train_orig = load_mnist(train_images_path, train_labels_path)\n",
    "X_test, y_test = load_mnist(test_images_path, test_labels_path)\n",
    "X_train_orig = X_train_orig.astype('float32')\n",
    "X_test = X_test.astype('float32')\n",
    "X_train_orig /= 255\n",
    "X_test /= 255\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60000, 784)\n",
      "(60000,)\n",
      "(10000, 784)\n",
      "(10000,)\n"
     ]
    }
   ],
   "source": [
    "print(X_train_orig.shape)\n",
    "print(y_train_orig.shape)\n",
    "print(X_test.shape)\n",
    "print(y_test.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_val, y_train, y_val = train_test_split(X_train_orig, y_train_orig, test_size=0.2, random_state=12345)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(48000, 784)\n",
      "(48000,)\n",
      "(12000, 784)\n",
      "(12000,)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape)\n",
    "print(y_train.shape)\n",
    "print(X_val.shape)\n",
    "print(y_val.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x1a32c55e80>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAFKlJREFUeJzt3W1sXOWVB/D/mfHYThwbx4mTmMQE\nCNmWACKAN7yku5sqhUJFBa1URFai6VI1SAtSkfiwLNUK9sOqaLW0y25XbNOSElaFtlLLgipogXRV\nFtoFDAqv4SWAIcZJnMQkceyMPS9nP/imMuB7HjN3Zu6E8/9JKPYc35nH1/w9Mz73eR5RVRCRP5m0\nB0BE6WD4iZxi+ImcYviJnGL4iZxi+ImcYviJnGL4iZxi+ImcaqrngzVLi7airZ4P2RgkUK/hRZbF\nhfb57lg0ZtbzJft/kSYpf+IxHTP5WuXHzop13j+lF7bmMYZJnQj9HwcgYfhF5FIAdwLIAvixqt5u\nfX0r2nC+rE/ykMclabJPs5ZK9h0kuAT7wFcvNOvr//aPZn3H4SVmfUGL/csjY6Rs8IIj5rFJWee9\nluc8TU/rtll/bcUv+0UkC+A/AFwGYBWADSKyqtL7I6L6SvKefw2Anar6tqpOAvgZgCuqMywiqrUk\n4V8KYNe0zwej2z5ERDaJSL+I9BcwkeDhiKiakoR/pj8qfOyNkqpuVtU+Ve3LoSXBwxFRNSUJ/yCA\n3mmfLwMwlGw4RFQvScL/LICVInKKiDQDuBrAQ9UZFhHVWsWtPlUtisgNAH6LqVbfFlV9pWojazSZ\nbGxJsvE1ANDCZLVH8yHZ7u7YWv8/3mUe++Jk3qznO+3vLRtomJ/X0hxbO+2+b5jHrvjr7WY9RIvF\nio+VFvstqk4c/3+/StTnV9WHATxcpbEQUR3x8l4ipxh+IqcYfiKnGH4ipxh+IqcYfiKnpJ479nRI\nl6Y2pdfo0wMANDC3vIbnac+NF5n1z1/zjFm/dsGTFT92Xu3z0gz7vLSIPTU2K/HnraT2tPOhUrtZ\nv3PwYrM++t3e2Frzb541jw2SwJT5lKYEP63bcFhHZjWfn8/8RE4x/EROMfxETjH8RE4x/EROMfxE\nTvlp9dXQGz/8c7P+ky/cbdZXNY+a9QMlu3OztzQvttYqBfPYw+VWs/6fQ+vM+kltH5j1a7r+EFs7\nWJ5jHhvS23TYrOeM6ca3Dn3JPHb/tYvNemnHm2ZdcvFTmYHaTfNmq4+Ighh+IqcYfiKnGH4ipxh+\nIqcYfiKnGH4ip/z0+RNOwXxjS19sbfsl/24e+8jYiWa9FPgd3J45atYtC7L2TrhnNttLUL9TsMeW\nC2zRvcxYH/rdov0zeXWiJ/DY9nTikrFH91nNu81jfztm7zn7yBmdZj0t7PMTURDDT+QUw0/kFMNP\n5BTDT+QUw0/kFMNP5FSiPr+IDAAYBVACUFTV+GY4ju/5/Je9cjC2tqTpkHnsZGB57NaMPec+X84l\nOt4S2mK7LWNfB9CZGTfr7xfnx9YKgfPSHZivH7KnGN+L31s4wTz2r9peM+t/84MbzXrPHfHrGNTS\nJ+nzJ9qiO/J5Vd1fhfshojriy34ip5KGXwE8KiLPicimagyIiOoj6cv+tao6JCKLADwmIq+p6hPT\nvyD6pbAJAFoxN+HDEVG1JHrmV9Wh6N9hAA8AWDPD12xW1T5V7cuhJcnDEVEVVRx+EWkTkfZjHwO4\nBMDL1RoYEdVWkpf9iwE8IFNTZZsA3Keqv6nKqIio5ioOv6q+DeDsKo4lVU29y8z6uXO2x9aGCvG9\nbADoyObNemi+fj5j9/kLGv9jDF1jkFf7vvMlu76naPfL5xrXCWQCawEcLLWZ9QPGfgWAfY3CvMDP\nJHT9w/h59vUNxwO2+oicYviJnGL4iZxi+ImcYviJnGL4iZyqxqy+T4X963rN+pLsWGxtX7HDPLa3\nacSs/9/RFWY9NK22PRPftgq1rJoD9z1Wtq/KLKn9/NFsLK8dakOOle1trpfm7PN6oBjfCuw0fp4A\n0BJYFrx7vr2t+vGAz/xETjH8RE4x/EROMfxETjH8RE4x/EROMfxETrHPHxm+0O7r5ox+ubUVNAD0\nNtlLa78VWKI6NLXV6ofn1e6VN0vRrIe2wc4Fjj9Yil+6LXQNQavY5+2F8eVm/dSW4dha6NqJoVK7\nWV/e8YFZt6uNgc/8RE4x/EROMfxETjH8RE4x/EROMfxETjH8RE6xzx9pP9Gen10I9PIt88Re/vqM\n5j1mfcDY5hqw562H+vgZ2Mtnh7bRDrHuvxxYCyCXscdu9fEB4KyWodjaLmP7biC8pPmXF75g1u+F\nvT5EI+AzP5FTDD+RUww/kVMMP5FTDD+RUww/kVMMP5FTwT6/iGwBcDmAYVU9M7qtC8DPAZwMYADA\nVap6PExhjnV+z3tmPW/0u8cD89In1O5Xd2XsXvtoxl5jfiwT//jWuvlAeE59oZzs+aFkPL9Y23cD\nQFtmMlC3j7eE9gywtj0HgMvbBs36p6XPfw+ASz9y280AtqnqSgDbos+J6DgSDL+qPgHgo1ujXAFg\na/TxVgBXVnlcRFRjlb6mW6yquwEg+ndR9YZERPVQ82v7RWQTgE0A0Ir49dyIqL4qfebfKyI9ABD9\nGzvDQlU3q2qfqvblYP9xiYjqp9LwPwRgY/TxRgAPVmc4RFQvwfCLyP0A/gjgMyIyKCLfBHA7gItF\n5E0AF0efE9FxJPieX1U3xJTWV3ksqVrf+apZt/r8oTnxY2rXJ+K3BAAADAfWkM8aewqE+tmhXnno\nOoBs4HvPSny9FJjPf6AUv04BAAwV7HUOrOsIQnsChK7dOCEzx6xnWlvNejmfN+v1wCv8iJxi+Imc\nYviJnGL4iZxi+ImcYviJnOLS3ZFVLbvN+ng5finn0LTZroy9Tfb+sj11NdSWstppoampIR2Zo2Y9\n1AoMtfOSOCFrT3XuzR6JrQ0FLjXfV+qoaEzHTF50hllv+t1zie6/GvjMT+QUw0/kFMNP5BTDT+QU\nw0/kFMNP5BTDT+QU+/yR7sB20G+X43v1rRm7D/9GwZ6ze1rO/jFYfXzA3kY7NN04JBe4hiE0Jdia\nVjsSmLKbg/0zOVRqM+t3DMfPOr9p0Tbz2LcmK9+SHQDGF9tbfCe7iqA6+MxP5BTDT+QUw0/kFMNP\n5BTDT+QUw0/kFMNP5JSbPn+2w+6s/v6ovaWy1e++su2geexpj1xv1v9+7cNmfe2ct8z6K5NLYmtJ\ntrEGgHLgOoES7H64dY1ClzHfHgDyavfK9xftZckPF+PXGjglF1gWvBS7CRUAYMekvfR2fr79vMo+\nPxGlhuEncorhJ3KK4SdyiuEncorhJ3KK4SdyKtjnF5EtAC4HMKyqZ0a33QbgWwD2RV92i6razeqU\n7f+qvY56d9NTZn1XYUFsLSv279Cex+3TPHGh3c9enLV77QNGLz+0rn5Hxu5Xjxr7FQDhXvxIMb6f\nHloHoT0wtsW5Q2b9tZHzYmuFk+x1Cl6ZsK/7uHjuG2Z9bFlg3/UGMJtn/nsAXDrD7d9X1dXRfw0d\nfCL6uGD4VfUJACN1GAsR1VGS9/w3iMiLIrJFROZXbUREVBeVhv8uACsArAawG8AdcV8oIptEpF9E\n+gtIdp05EVVPReFX1b2qWlLVMoAfAVhjfO1mVe1T1b4c7D8+EVH9VBR+EemZ9ulXALxcneEQUb3M\nptV3P4B1ABaKyCCAWwGsE5HVABTAAIDrajhGIqqBYPhVdcMMN99dg7HU1NFue955aH36yQT73Dcf\ntu/73DnvmPWdhVazvjQb3+9+q9xtHhtirbsPAJMle069dXxW7F54aL+Clbk9Zn1kR/y1Gblz7HGH\ndGbsF82lXvsahUbAK/yInGL4iZxi+ImcYviJnGL4iZxi+ImccrN093hPsq2qkyjn7DZjX4vdCvzJ\noZPN+gVz3o6t5cTe5jq0hXdeA23G3AdmvTNzNLYWWvb71YmlZr2vxW719T4ef16PXG234kJLnu8r\n223K5lb7vDcCPvMTOcXwEznF8BM5xfATOcXwEznF8BM5xfATOeWmz589cdysjweWuG6VydjaU3m7\nV547YvfxW8Re/jo0trwx3bhV7OWxC7CntmZh97P3FDvN+gGJX7o7H1gWfHCyy6wv67C30c5OxJ/3\n1wv281539rBZHyjY33cmk951JbPFZ34ipxh+IqcYfiKnGH4ipxh+IqcYfiKnGH4ip9z0+Tvb4+eV\nz8aiptHY2g/3rjOPbXn2zUSPHeqHD5faY2uheemh+y6ofR1A6Pg84uuh++7JHTTrIS3v7I+tPX7E\n3rL9c22vm/XQ1uedbcn+f6sHPvMTOcXwEznF8BM5xfATOcXwEznF8BM5xfATORXs84tIL4B7ASwB\nUAawWVXvFJEuAD8HcDKAAQBXqaq9iHuK2prj5+MDQF7tfnVv00hs7amdK8xjV/6ZPac+5NQWe956\nwZjPP1q2f7/PlcB1AIG1Bloz9vdm9cN7mw+Yxx4stZn1oIn4n/lLo/aeAF+c97JZP6Dx6xQAQCGw\ndXkjmM0zfxHATap6OoALAFwvIqsA3Axgm6quBLAt+pyIjhPB8KvqblV9Pvp4FMAOAEsBXAFga/Rl\nWwFcWatBElH1faL3/CJyMoBzADwNYLGq7gamfkEAWFTtwRFR7cw6/CIyD8AvAdyoqvYCZx8+bpOI\n9ItIfwH2+0siqp9ZhV9EcpgK/k9V9VfRzXtFpCeq9wCY8a9SqrpZVftUtS8HezIEEdVPMPwiIgDu\nBrBDVb83rfQQgI3RxxsBPFj94RFRrcxmSu9aANcAeElEtke33QLgdgC/EJFvAngPwNdqM8TqaMvZ\nrb6Q81qaY2vL/8tu6xz8bLJXPKXA7+gc4peoDm3BHVJW+7FDU3oXNcW/Q7TGDQCjJXt78JBDFy2P\nre3cZn9f7V//tVm32qsAoGpvP94IguFX1SeB2I3U11d3OERUL7zCj8gphp/IKYafyCmGn8gphp/I\nKYafyCk3S3e3ZpNNq7XkHu0364f/4aJE918K9IxzmWLF9z2u9jUIGbGvE1ias2dxd2WPxNZ2FRaY\nxyZ1ZGn89RcnPW4vrT13o1lGa8a+buTQEfsaBXvz8frgMz+RUww/kVMMP5FTDD+RUww/kVMMP5FT\nDD+RU276/OPF+Pn4tXb0VHv5ssFifC8cANoyHRU/dmje+ZImexvsPcVOsx5a8vzVifglskNrDWRE\nzXrIxPz4WvaI3afPSeDaisBaBPpuwmXH64DP/EROMfxETjH8RE4x/EROMfxETjH8RE4x/EROuenz\nDxywZ1B3L7d3IBsvVz5nfmH3qH3fgfn62cCc+rZM/HUErWJfQ5CF3UsfLdvz0tszebPemR2PrR0s\nzTWPDXlmwl6jodwc/71lh+ztwQtqn5cC7L0aFr6Q7BqFeuAzP5FTDD+RUww/kVMMP5FTDD+RUww/\nkVMMP5FTwT6/iPQCuBfAEgBlAJtV9U4RuQ3AtwDsi770FlV9uFYDTeroB3PMemje+7ff/wujOmYe\n+53P2KdlbmDe+qTaPWVrTv1o2f6+Q/cdsq/YbtbnGtcg5MSeEz9WtvcU6DbuGwAml8RfB1Dcvcc8\ndm/JXqcg9P9L+7v29Q+NYDYX+RQB3KSqz4tIO4DnROSxqPZ9Vf2X2g2PiGolGH5V3Q1gd/TxqIjs\nABC/PAsRHRc+0Xt+ETkZwDkAno5uukFEXhSRLSIy46JJIrJJRPpFpL8A+2UaEdXPrMMvIvMA/BLA\njap6GMBdAFYAWI2pVwZ3zHScqm5W1T5V7cvBfg9HRPUzq/CLSA5Twf+pqv4KAFR1r6qWVLUM4EcA\n1tRumERUbcHwi4gAuBvADlX93rTbe6Z92VcAvFz94RFRrczmr/1rAVwD4CUR2R7ddguADSKyGoAC\nGABwXU1GWCXZufaU3LOb7amv63qfiq19EavNY7+z5etm/bvX3mPWV+T2mfVWo2WWCUzZPb052bTa\nkOcm4pfIDk0X7szaLdRTcvPM+mf/Lf748pqzzGPPan7erJ+QsVuFk512q7AR3gDP5q/9TwKYacJ5\nw/b0iSiMV/gROcXwEznF8BM5xfATOcXwEznF8BM5JRpYoriaOqRLz5f1dXu86bId9jbXo1843axn\nJuPPU+uvn6loTLOlF51t1kdWxffqj3bby4JPLLSXBbeWvwaA3CH7+aNpLP7x23fZj73g94NmvbjL\nriex/7oLzXo5Z5/XRT/4QzWHM2tP6zYc1hF7cBE+8xM5xfATOcXwEznF8BM5xfATOcXwEznF8BM5\nVdc+v4jsA/DutJsWAthftwF8Mo06tkYdF8CxVaqaY1uuqt2z+cK6hv9jDy7Sr6p9qQ3A0Khja9Rx\nARxbpdIaG1/2EznF8BM5lXb4N6f8+JZGHVujjgvg2CqVythSfc9PROlJ+5mfiFKSSvhF5FIReV1E\ndorIzWmMIY6IDIjISyKyXUT6Ux7LFhEZFpGXp93WJSKPicib0b8zbpOW0thuE5H3o3O3XUS+lNLY\nekXkf0Rkh4i8IiLfjm5P9dwZ40rlvNX9Zb+IZAG8AeBiAIMAngWwQVVfretAYojIAIA+VU29Jywi\nfwngCIB7VfXM6LZ/BjCiqrdHvzjnq+rfNcjYbgNwJO2dm6MNZXqm7ywN4EoA30CK584Y11VI4byl\n8cy/BsBOVX1bVScB/AzAFSmMo+Gp6hMARj5y8xUAtkYfb8XU/zx1FzO2hqCqu1X1+ejjUQDHdpZO\n9dwZ40pFGuFfCmDXtM8H0VhbfiuAR0XkORHZlPZgZrA42jb92Pbpi1Iez0cFd26up4/sLN0w566S\nHa+rLY3wz7TEUCO1HNaq6rkALgNwffTylmZnVjs318sMO0s3hEp3vK62NMI/CKB32ufLAAylMI4Z\nqepQ9O8wgAfQeLsP7z22SWr073DK4/mTRtq5eaadpdEA566RdrxOI/zPAlgpIqeISDOAqwE8lMI4\nPkZE2qI/xEBE2gBcgsbbffghABujjzcCeDDFsXxIo+zcHLezNFI+d42243UqF/lErYx/BZAFsEVV\n/6nug5iBiJyKqWd7YGoT0/vSHJuI3A9gHaZmfe0FcCuA/wbwCwAnAXgPwNdUte5/eIsZ2zpMvXT9\n087Nx95j13lsnwPwvwBeAnBsieBbMPX+OrVzZ4xrA1I4b7zCj8gpXuFH5BTDT+QUw0/kFMNP5BTD\nT+QUw0/kFMNP5BTDT+TU/wOP4iVkOh85ugAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1a327f36a0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(X_train[1, :].reshape((28, 28)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 2 layer NN\n",
    "model = Sequential([\n",
    "    Dense(512, input_shape=(784,), activation='relu'),\n",
    "    Dense(128, activation = 'relu'),\n",
    "    Dense(10, activation='softmax')\n",
    "])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_1 (Dense)              (None, 512)               401920    \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 128)               65664     \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 10)                1290      \n",
      "=================================================================\n",
      "Total params: 468,874\n",
      "Trainable params: 468,874\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.compile(optimizer=Adam(lr=0.001),\n",
    "              loss='sparse_categorical_crossentropy',\n",
    "              metrics=['accuracy'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/20\n",
      "48000/48000 [==============================] - 4s 74us/step - loss: 0.6274 - acc: 0.7810 - val_loss: 0.4209 - val_acc: 0.8573\n",
      "Epoch 2/20\n",
      "48000/48000 [==============================] - 3s 68us/step - loss: 0.4142 - acc: 0.8551 - val_loss: 0.3942 - val_acc: 0.8651\n",
      "Epoch 3/20\n",
      "48000/48000 [==============================] - 3s 70us/step - loss: 0.3754 - acc: 0.8667 - val_loss: 0.3562 - val_acc: 0.8790\n",
      "Epoch 4/20\n",
      "48000/48000 [==============================] - 3s 69us/step - loss: 0.3390 - acc: 0.8775 - val_loss: 0.3260 - val_acc: 0.8876\n",
      "Epoch 5/20\n",
      "48000/48000 [==============================] - 4s 74us/step - loss: 0.3192 - acc: 0.8854 - val_loss: 0.3126 - val_acc: 0.8895\n",
      "Epoch 6/20\n",
      "48000/48000 [==============================] - 3s 65us/step - loss: 0.2993 - acc: 0.8919 - val_loss: 0.3252 - val_acc: 0.8857\n",
      "Epoch 7/20\n",
      "48000/48000 [==============================] - 3s 67us/step - loss: 0.2874 - acc: 0.8946 - val_loss: 0.3199 - val_acc: 0.8872\n",
      "Epoch 8/20\n",
      "48000/48000 [==============================] - 3s 66us/step - loss: 0.2735 - acc: 0.9000 - val_loss: 0.2899 - val_acc: 0.8987\n",
      "Epoch 9/20\n",
      "48000/48000 [==============================] - 3s 64us/step - loss: 0.2621 - acc: 0.9045 - val_loss: 0.3053 - val_acc: 0.8896\n",
      "Epoch 10/20\n",
      "48000/48000 [==============================] - 3s 70us/step - loss: 0.2494 - acc: 0.9076 - val_loss: 0.2904 - val_acc: 0.8953\n",
      "Epoch 11/20\n",
      "48000/48000 [==============================] - 3s 63us/step - loss: 0.2420 - acc: 0.9100 - val_loss: 0.2891 - val_acc: 0.8946\n",
      "Epoch 12/20\n",
      "48000/48000 [==============================] - 3s 63us/step - loss: 0.2297 - acc: 0.9153 - val_loss: 0.2973 - val_acc: 0.8966\n",
      "Epoch 13/20\n",
      "48000/48000 [==============================] - 3s 64us/step - loss: 0.2254 - acc: 0.9176 - val_loss: 0.2869 - val_acc: 0.8952\n",
      "Epoch 14/20\n",
      "48000/48000 [==============================] - 3s 64us/step - loss: 0.2138 - acc: 0.9209 - val_loss: 0.2970 - val_acc: 0.8954\n",
      "Epoch 15/20\n",
      "48000/48000 [==============================] - 3s 64us/step - loss: 0.2007 - acc: 0.9257 - val_loss: 0.2805 - val_acc: 0.9024\n",
      "Epoch 16/20\n",
      "48000/48000 [==============================] - 3s 64us/step - loss: 0.1996 - acc: 0.9257 - val_loss: 0.2960 - val_acc: 0.8970\n",
      "Epoch 17/20\n",
      "48000/48000 [==============================] - 3s 64us/step - loss: 0.1878 - acc: 0.9307 - val_loss: 0.2849 - val_acc: 0.9023\n",
      "Epoch 18/20\n",
      "48000/48000 [==============================] - 3s 64us/step - loss: 0.1835 - acc: 0.9319 - val_loss: 0.2984 - val_acc: 0.8971\n",
      "Epoch 19/20\n",
      "48000/48000 [==============================] - 3s 64us/step - loss: 0.1816 - acc: 0.9330 - val_loss: 0.2831 - val_acc: 0.9024\n",
      "Epoch 20/20\n",
      "48000/48000 [==============================] - 3s 65us/step - loss: 0.1686 - acc: 0.9385 - val_loss: 0.2991 - val_acc: 0.8984\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(X_train, y_train,\n",
    "                    batch_size=batch_size,\n",
    "                    epochs=20,\n",
    "                    verbose=1,\n",
    "                    validation_data=(X_val, y_val))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test loss: 0.3530888082802296\n",
      "Test accuracy: 0.8827\n"
     ]
    }
   ],
   "source": [
    "score = model.evaluate(X_test, y_test, verbose=0)\n",
    "print('Test loss:', score[0])\n",
    "print('Test accuracy:', score[1])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## CNN with 1 Convolutional Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "img_rows = 28\n",
    "img_cols = 28\n",
    "input_shape = (img_rows, img_cols, 1)\n",
    "X_train = X_train.reshape(X_train.shape[0], img_rows, img_cols, 1)\n",
    "X_test = X_test.reshape(X_test.shape[0], img_rows, img_cols, 1)\n",
    "X_val = X_val.reshape(X_val.shape[0], img_rows, img_cols, 1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cnn1 = Sequential([\n",
    "    Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=input_shape),\n",
    "    MaxPooling2D(pool_size=(2, 2)),\n",
    "    Dropout(0.2),\n",
    "    Flatten(),\n",
    "    Dense(128, activation='relu'),\n",
    "    Dense(10, activation='softmax')\n",
    "])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cnn1.compile(loss='sparse_categorical_crossentropy',\n",
    "              optimizer=Adam(lr=0.001),\n",
    "              metrics=['accuracy'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/10\n",
      "48000/48000 [==============================] - 22s 468us/step - loss: 0.6472 - acc: 0.7787 - val_loss: 0.4267 - val_acc: 0.8533\n",
      "Epoch 2/10\n",
      "48000/48000 [==============================] - 23s 473us/step - loss: 0.4020 - acc: 0.8589 - val_loss: 0.3467 - val_acc: 0.8817\n",
      "Epoch 3/10\n",
      "48000/48000 [==============================] - 20s 426us/step - loss: 0.3478 - acc: 0.8765 - val_loss: 0.3205 - val_acc: 0.8900\n",
      "Epoch 4/10\n",
      "48000/48000 [==============================] - 20s 409us/step - loss: 0.3227 - acc: 0.8843 - val_loss: 0.2994 - val_acc: 0.8951\n",
      "Epoch 5/10\n",
      "48000/48000 [==============================] - 20s 410us/step - loss: 0.2987 - acc: 0.8941 - val_loss: 0.2777 - val_acc: 0.9037\n",
      "Epoch 6/10\n",
      "48000/48000 [==============================] - 19s 399us/step - loss: 0.2848 - acc: 0.8979 - val_loss: 0.2703 - val_acc: 0.9055\n",
      "Epoch 7/10\n",
      "48000/48000 [==============================] - 20s 407us/step - loss: 0.2708 - acc: 0.9030 - val_loss: 0.2681 - val_acc: 0.9065\n",
      "Epoch 8/10\n",
      "48000/48000 [==============================] - 19s 406us/step - loss: 0.2634 - acc: 0.9044 - val_loss: 0.2512 - val_acc: 0.9124\n",
      "Epoch 9/10\n",
      "48000/48000 [==============================] - 19s 401us/step - loss: 0.2506 - acc: 0.9088 - val_loss: 0.2518 - val_acc: 0.9089\n",
      "Epoch 10/10\n",
      "48000/48000 [==============================] - 19s 402us/step - loss: 0.2446 - acc: 0.9114 - val_loss: 0.2788 - val_acc: 0.8992\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1a32b5d080>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cnn1.fit(X_train, y_train,\n",
    "          batch_size=batch_size,\n",
    "          epochs=10,\n",
    "          verbose=1,\n",
    "          validation_data=(X_val, y_val))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cnn1.optimizer.lr = 0.0001\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/10\n",
      "48000/48000 [==============================] - 22s 463us/step - loss: 0.2323 - acc: 0.9152 - val_loss: 0.2408 - val_acc: 0.9156\n",
      "Epoch 2/10\n",
      "48000/48000 [==============================] - 21s 432us/step - loss: 0.2212 - acc: 0.9201 - val_loss: 0.2418 - val_acc: 0.9145\n",
      "Epoch 3/10\n",
      "48000/48000 [==============================] - 19s 402us/step - loss: 0.2162 - acc: 0.9201 - val_loss: 0.2360 - val_acc: 0.9182\n",
      "Epoch 4/10\n",
      "48000/48000 [==============================] - 19s 402us/step - loss: 0.2069 - acc: 0.9237 - val_loss: 0.2343 - val_acc: 0.9177\n",
      "Epoch 5/10\n",
      "48000/48000 [==============================] - 19s 404us/step - loss: 0.2019 - acc: 0.9258 - val_loss: 0.2240 - val_acc: 0.9201\n",
      "Epoch 6/10\n",
      "48000/48000 [==============================] - 20s 409us/step - loss: 0.1923 - acc: 0.9296 - val_loss: 0.2254 - val_acc: 0.9217\n",
      "Epoch 7/10\n",
      "48000/48000 [==============================] - 22s 449us/step - loss: 0.1868 - acc: 0.9322 - val_loss: 0.2323 - val_acc: 0.9172\n",
      "Epoch 8/10\n",
      "48000/48000 [==============================] - 19s 404us/step - loss: 0.1807 - acc: 0.9330 - val_loss: 0.2287 - val_acc: 0.9193\n",
      "Epoch 9/10\n",
      "48000/48000 [==============================] - 19s 397us/step - loss: 0.1761 - acc: 0.9359 - val_loss: 0.2217 - val_acc: 0.9227\n",
      "Epoch 10/10\n",
      "48000/48000 [==============================] - 20s 422us/step - loss: 0.1687 - acc: 0.9371 - val_loss: 0.2153 - val_acc: 0.9246\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1a34161630>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cnn1.fit(X_train, y_train,\n",
    "          batch_size=batch_size,\n",
    "          epochs=10,\n",
    "          verbose=1,\n",
    "          validation_data=(X_val, y_val))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test loss: 0.24918004976511002\n",
      "Test accuracy: 0.9131\n"
     ]
    }
   ],
   "source": [
    "score = cnn1.evaluate(X_test, y_test, verbose=0)\n",
    "print('Test loss:', score[0])\n",
    "print('Test accuracy:', score[1])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Data Augmentation\n",
    "gen = ImageDataGenerator(rotation_range=8, width_shift_range=0.08, shear_range=0.3,\n",
    "                               height_shift_range=0.08, zoom_range=0.08)\n",
    "batches = gen.flow(X_train, y_train, batch_size=batch_size)\n",
    "val_batches = gen.flow(X_val, y_val, batch_size=batch_size)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n",
      "93/93 [==============================] - 25s 272ms/step - loss: 0.4696 - acc: 0.8262 - val_loss: 0.4127 - val_acc: 0.8491\n",
      "Epoch 2/50\n",
      "93/93 [==============================] - 26s 284ms/step - loss: 0.4191 - acc: 0.8431 - val_loss: 0.3868 - val_acc: 0.8538\n",
      "Epoch 3/50\n",
      "93/93 [==============================] - 25s 268ms/step - loss: 0.3933 - acc: 0.8524 - val_loss: 0.3630 - val_acc: 0.8670\n",
      "Epoch 4/50\n",
      "93/93 [==============================] - 24s 258ms/step - loss: 0.3816 - acc: 0.8565 - val_loss: 0.3569 - val_acc: 0.8686\n",
      "Epoch 5/50\n",
      "93/93 [==============================] - 23s 243ms/step - loss: 0.3699 - acc: 0.8619 - val_loss: 0.3450 - val_acc: 0.8734\n",
      "Epoch 6/50\n",
      "93/93 [==============================] - 22s 236ms/step - loss: 0.3646 - acc: 0.8628 - val_loss: 0.3365 - val_acc: 0.8766\n",
      "Epoch 7/50\n",
      "93/93 [==============================] - 22s 234ms/step - loss: 0.3552 - acc: 0.8670 - val_loss: 0.3410 - val_acc: 0.8770\n",
      "Epoch 8/50\n",
      "93/93 [==============================] - 25s 269ms/step - loss: 0.3479 - acc: 0.8707 - val_loss: 0.3311 - val_acc: 0.8787\n",
      "Epoch 9/50\n",
      "93/93 [==============================] - 25s 270ms/step - loss: 0.3465 - acc: 0.8722 - val_loss: 0.3306 - val_acc: 0.8794\n",
      "Epoch 10/50\n",
      "93/93 [==============================] - 27s 290ms/step - loss: 0.3390 - acc: 0.8742 - val_loss: 0.3268 - val_acc: 0.8808\n",
      "Epoch 11/50\n",
      "93/93 [==============================] - 25s 263ms/step - loss: 0.3349 - acc: 0.8765 - val_loss: 0.3137 - val_acc: 0.8837\n",
      "Epoch 12/50\n",
      "93/93 [==============================] - 22s 235ms/step - loss: 0.3293 - acc: 0.8782 - val_loss: 0.3143 - val_acc: 0.8855\n",
      "Epoch 13/50\n",
      "93/93 [==============================] - 22s 234ms/step - loss: 0.3263 - acc: 0.8777 - val_loss: 0.3100 - val_acc: 0.8854\n",
      "Epoch 14/50\n",
      "93/93 [==============================] - 22s 234ms/step - loss: 0.3223 - acc: 0.8806 - val_loss: 0.3221 - val_acc: 0.8827\n",
      "Epoch 15/50\n",
      "93/93 [==============================] - 22s 234ms/step - loss: 0.3230 - acc: 0.8796 - val_loss: 0.3074 - val_acc: 0.8899\n",
      "Epoch 16/50\n",
      "93/93 [==============================] - 22s 235ms/step - loss: 0.3172 - acc: 0.8820 - val_loss: 0.3051 - val_acc: 0.8880\n",
      "Epoch 17/50\n",
      "93/93 [==============================] - 22s 239ms/step - loss: 0.3147 - acc: 0.8831 - val_loss: 0.3035 - val_acc: 0.8875\n",
      "Epoch 18/50\n",
      "93/93 [==============================] - 22s 236ms/step - loss: 0.3131 - acc: 0.8848 - val_loss: 0.2906 - val_acc: 0.8917\n",
      "Epoch 19/50\n",
      "93/93 [==============================] - 22s 236ms/step - loss: 0.3094 - acc: 0.8838 - val_loss: 0.2968 - val_acc: 0.8885\n",
      "Epoch 20/50\n",
      "93/93 [==============================] - 22s 240ms/step - loss: 0.3051 - acc: 0.8868 - val_loss: 0.2884 - val_acc: 0.8939\n",
      "Epoch 21/50\n",
      "93/93 [==============================] - 22s 235ms/step - loss: 0.3016 - acc: 0.8892 - val_loss: 0.2975 - val_acc: 0.8943\n",
      "Epoch 22/50\n",
      "93/93 [==============================] - 22s 235ms/step - loss: 0.3023 - acc: 0.8887 - val_loss: 0.2847 - val_acc: 0.8952\n",
      "Epoch 23/50\n",
      "93/93 [==============================] - 22s 236ms/step - loss: 0.3035 - acc: 0.8864 - val_loss: 0.2855 - val_acc: 0.8934\n",
      "Epoch 24/50\n",
      "93/93 [==============================] - 22s 237ms/step - loss: 0.2986 - acc: 0.8886 - val_loss: 0.3017 - val_acc: 0.8894\n",
      "Epoch 25/50\n",
      "93/93 [==============================] - 22s 237ms/step - loss: 0.2898 - acc: 0.8907 - val_loss: 0.2864 - val_acc: 0.8962\n",
      "Epoch 26/50\n",
      "93/93 [==============================] - 23s 243ms/step - loss: 0.2934 - acc: 0.8910 - val_loss: 0.2884 - val_acc: 0.8952\n",
      "Epoch 27/50\n",
      "93/93 [==============================] - 24s 254ms/step - loss: 0.2950 - acc: 0.8899 - val_loss: 0.2921 - val_acc: 0.8939\n",
      "Epoch 28/50\n",
      "93/93 [==============================] - 22s 238ms/step - loss: 0.2908 - acc: 0.8921 - val_loss: 0.2819 - val_acc: 0.8976\n",
      "Epoch 29/50\n",
      "93/93 [==============================] - 22s 237ms/step - loss: 0.2876 - acc: 0.8933 - val_loss: 0.2770 - val_acc: 0.9004\n",
      "Epoch 30/50\n",
      "93/93 [==============================] - 22s 238ms/step - loss: 0.2902 - acc: 0.8908 - val_loss: 0.2849 - val_acc: 0.8939\n",
      "Epoch 31/50\n",
      "93/93 [==============================] - 22s 237ms/step - loss: 0.2863 - acc: 0.8920 - val_loss: 0.2862 - val_acc: 0.8944\n",
      "Epoch 32/50\n",
      "93/93 [==============================] - 22s 237ms/step - loss: 0.2818 - acc: 0.8951 - val_loss: 0.2712 - val_acc: 0.8984\n",
      "Epoch 33/50\n",
      "93/93 [==============================] - 22s 240ms/step - loss: 0.2782 - acc: 0.8958 - val_loss: 0.2712 - val_acc: 0.9012\n",
      "Epoch 34/50\n",
      "93/93 [==============================] - 22s 236ms/step - loss: 0.2764 - acc: 0.8966 - val_loss: 0.2811 - val_acc: 0.8968\n",
      "Epoch 35/50\n",
      "93/93 [==============================] - 22s 238ms/step - loss: 0.2757 - acc: 0.8981 - val_loss: 0.2808 - val_acc: 0.8983\n",
      "Epoch 36/50\n",
      "93/93 [==============================] - 26s 278ms/step - loss: 0.2816 - acc: 0.8939 - val_loss: 0.2769 - val_acc: 0.9001\n",
      "Epoch 37/50\n",
      "93/93 [==============================] - 22s 234ms/step - loss: 0.2737 - acc: 0.8968 - val_loss: 0.2698 - val_acc: 0.9027\n",
      "Epoch 38/50\n",
      "93/93 [==============================] - 24s 263ms/step - loss: 0.2749 - acc: 0.8973 - val_loss: 0.2707 - val_acc: 0.9006\n",
      "Epoch 39/50\n",
      "93/93 [==============================] - 22s 240ms/step - loss: 0.2707 - acc: 0.8997 - val_loss: 0.2652 - val_acc: 0.9018\n",
      "Epoch 40/50\n",
      "93/93 [==============================] - 23s 248ms/step - loss: 0.2688 - acc: 0.8995 - val_loss: 0.2690 - val_acc: 0.9058\n",
      "Epoch 41/50\n",
      "93/93 [==============================] - 24s 258ms/step - loss: 0.2675 - acc: 0.9013 - val_loss: 0.2696 - val_acc: 0.9006\n",
      "Epoch 42/50\n",
      "93/93 [==============================] - 23s 252ms/step - loss: 0.2711 - acc: 0.8983 - val_loss: 0.2694 - val_acc: 0.9038\n",
      "Epoch 43/50\n",
      "93/93 [==============================] - 23s 252ms/step - loss: 0.2704 - acc: 0.8999 - val_loss: 0.2660 - val_acc: 0.9041\n",
      "Epoch 44/50\n",
      "93/93 [==============================] - 25s 268ms/step - loss: 0.2641 - acc: 0.9004 - val_loss: 0.2638 - val_acc: 0.9045\n",
      "Epoch 45/50\n",
      "93/93 [==============================] - 24s 261ms/step - loss: 0.2603 - acc: 0.9037 - val_loss: 0.2805 - val_acc: 0.8969\n",
      "Epoch 46/50\n",
      "93/93 [==============================] - 24s 261ms/step - loss: 0.2605 - acc: 0.9036 - val_loss: 0.2693 - val_acc: 0.8979\n",
      "Epoch 47/50\n",
      "93/93 [==============================] - 23s 250ms/step - loss: 0.2662 - acc: 0.9013 - val_loss: 0.2616 - val_acc: 0.9079\n",
      "Epoch 48/50\n",
      "93/93 [==============================] - 22s 241ms/step - loss: 0.2558 - acc: 0.9046 - val_loss: 0.2591 - val_acc: 0.9086\n",
      "Epoch 49/50\n",
      "93/93 [==============================] - 23s 249ms/step - loss: 0.2580 - acc: 0.9032 - val_loss: 0.2600 - val_acc: 0.9089\n",
      "Epoch 50/50\n",
      "93/93 [==============================] - 23s 250ms/step - loss: 0.2600 - acc: 0.9028 - val_loss: 0.2737 - val_acc: 0.8983\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1a34161898>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cnn1.fit_generator(batches, steps_per_epoch=48000//batch_size, epochs=50,\n",
    "                    validation_data=val_batches, validation_steps=12000//batch_size, use_multiprocessing=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test loss: 0.2702846100747585\n",
      "Test accuracy: 0.9063\n"
     ]
    }
   ],
   "source": [
    "score = cnn1.evaluate(X_test, y_test, verbose=0)\n",
    "print('Test loss:', score[0])\n",
    "print('Test accuracy:', score[1])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### CNN with 3 Convolutional Layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cnn2 = Sequential([\n",
    "    Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=input_shape),\n",
    "    MaxPooling2D(pool_size=(2, 2)),\n",
    "    Dropout(0.2),\n",
    "\n",
    "    Conv2D(64, kernel_size=(3, 3), activation='relu'),\n",
    "    MaxPooling2D(pool_size=(2, 2)),\n",
    "    Dropout(0.2),\n",
    "\n",
    "    Conv2D(128, kernel_size=(3, 3), activation='relu'),\n",
    "    Dropout(0.2),\n",
    "\n",
    "    Flatten(),\n",
    "\n",
    "    Dense(128, activation='relu'),\n",
    "    Dropout(0.2),\n",
    "    Dense(10, activation='softmax')\n",
    "])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cnn2.compile(loss='sparse_categorical_crossentropy',\n",
    "              optimizer=Adam(lr=0.001),\n",
    "              metrics=['accuracy'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/10\n",
      "48000/48000 [==============================] - 41s 861us/step - loss: 0.9826 - acc: 0.6359 - val_loss: 0.5913 - val_acc: 0.7714\n",
      "Epoch 2/10\n",
      "48000/48000 [==============================] - 43s 905us/step - loss: 0.5765 - acc: 0.7822 - val_loss: 0.4627 - val_acc: 0.8335\n",
      "Epoch 3/10\n",
      "48000/48000 [==============================] - 43s 906us/step - loss: 0.4976 - acc: 0.8127 - val_loss: 0.4068 - val_acc: 0.8531\n",
      "Epoch 4/10\n",
      "48000/48000 [==============================] - 41s 851us/step - loss: 0.4484 - acc: 0.8326 - val_loss: 0.3710 - val_acc: 0.8653\n",
      "Epoch 5/10\n",
      "48000/48000 [==============================] - 40s 843us/step - loss: 0.4142 - acc: 0.8457 - val_loss: 0.3402 - val_acc: 0.8771\n",
      "Epoch 6/10\n",
      "48000/48000 [==============================] - 40s 840us/step - loss: 0.3912 - acc: 0.8555 - val_loss: 0.3248 - val_acc: 0.8795\n",
      "Epoch 7/10\n",
      "48000/48000 [==============================] - 40s 841us/step - loss: 0.3683 - acc: 0.8635 - val_loss: 0.3191 - val_acc: 0.8839\n",
      "Epoch 8/10\n",
      "48000/48000 [==============================] - 41s 845us/step - loss: 0.3505 - acc: 0.8704 - val_loss: 0.2951 - val_acc: 0.8933\n",
      "Epoch 9/10\n",
      "48000/48000 [==============================] - 45s 944us/step - loss: 0.3330 - acc: 0.8770 - val_loss: 0.2882 - val_acc: 0.8934\n",
      "Epoch 10/10\n",
      "48000/48000 [==============================] - 41s 858us/step - loss: 0.3199 - acc: 0.8829 - val_loss: 0.2763 - val_acc: 0.8982\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1a33ca07b8>"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cnn2.fit(X_train, y_train,\n",
    "          batch_size=batch_size,\n",
    "          epochs=10,\n",
    "          verbose=1,\n",
    "          validation_data=(X_val, y_val))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cnn2.optimizer.lr = 0.0001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/10\n",
      "48000/48000 [==============================] - 41s 857us/step - loss: 0.3096 - acc: 0.8869 - val_loss: 0.2675 - val_acc: 0.9027\n",
      "Epoch 2/10\n",
      "48000/48000 [==============================] - 43s 901us/step - loss: 0.2998 - acc: 0.8897 - val_loss: 0.2555 - val_acc: 0.9064\n",
      "Epoch 3/10\n",
      "48000/48000 [==============================] - 42s 867us/step - loss: 0.2914 - acc: 0.8931 - val_loss: 0.2545 - val_acc: 0.9051\n",
      "Epoch 4/10\n",
      "48000/48000 [==============================] - 41s 851us/step - loss: 0.2873 - acc: 0.8937 - val_loss: 0.2550 - val_acc: 0.9051\n",
      "Epoch 5/10\n",
      "48000/48000 [==============================] - 41s 845us/step - loss: 0.2743 - acc: 0.8983 - val_loss: 0.2411 - val_acc: 0.9124\n",
      "Epoch 6/10\n",
      "48000/48000 [==============================] - 41s 845us/step - loss: 0.2712 - acc: 0.9009 - val_loss: 0.2487 - val_acc: 0.9075\n",
      "Epoch 7/10\n",
      "48000/48000 [==============================] - 41s 849us/step - loss: 0.2632 - acc: 0.9035 - val_loss: 0.2370 - val_acc: 0.9111\n",
      "Epoch 8/10\n",
      "48000/48000 [==============================] - 41s 849us/step - loss: 0.2599 - acc: 0.9046 - val_loss: 0.2419 - val_acc: 0.9091\n",
      "Epoch 9/10\n",
      "48000/48000 [==============================] - 41s 856us/step - loss: 0.2534 - acc: 0.9066 - val_loss: 0.2442 - val_acc: 0.9073\n",
      "Epoch 10/10\n",
      "48000/48000 [==============================] - 41s 850us/step - loss: 0.2451 - acc: 0.9090 - val_loss: 0.2364 - val_acc: 0.9108\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1a341617b8>"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cnn2.fit(X_train, y_train,\n",
    "          batch_size=batch_size,\n",
    "          epochs=10,\n",
    "          verbose=1,\n",
    "          validation_data=(X_val, y_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test loss: 0.26379832727909086\n",
      "Test accuracy: 0.9052\n"
     ]
    }
   ],
   "source": [
    "score = cnn2.evaluate(X_test, y_test, verbose=0)\n",
    "print('Test loss:', score[0])\n",
    "print('Test accuracy:', score[1])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}