10_training_deep_nets_ex9.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![book](https://raw.githubusercontent.com/ageron/tensorflow-safari-course/master/images/intro_to_tf_course.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Try not to peek at the solutions when you go through the exercises. ;-)**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First let's make sure this notebook works well in both Python 2 and Python 3:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import, division, print_function, unicode_literals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "tf.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"tmp/data/\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Techniques for Training Deep Nets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using He initialization and the ELU activation function (with the help of a `partial()`):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from functools import partial\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 100\n",
    "n_hidden2 = 100\n",
    "n_outputs = 10\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "    with tf.name_scope(\"inputs\"):\n",
    "        X = tf.placeholder(tf.float32, shape=[None, n_inputs], name=\"X\")\n",
    "        y = tf.placeholder(tf.int32, shape=[None], name=\"y\")\n",
    "\n",
    "    he_init = tf.contrib.layers.variance_scaling_initializer()\n",
    "    \n",
    "    dense_layer = partial(tf.layers.dense,\n",
    "                          kernel_initializer=he_init,\n",
    "                          activation=tf.nn.elu)\n",
    "    hidden1 = dense_layer(X, n_hidden1, name=\"hidden1\")\n",
    "    hidden2 = dense_layer(hidden1, n_hidden2, name=\"hidden2\")\n",
    "    logits = dense_layer(hidden2, n_outputs, activation=None, name=\"output\")\n",
    "    Y_proba = tf.nn.softmax(logits)\n",
    "    \n",
    "    with tf.name_scope(\"train\"):\n",
    "        xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=y)\n",
    "        loss = tf.reduce_mean(xentropy)\n",
    "        optimizer = tf.train.AdamOptimizer()\n",
    "        training_op = optimizer.minimize(loss)\n",
    "\n",
    "    with tf.name_scope(\"eval\"):\n",
    "        correct = tf.nn.in_top_k(logits, y, 1)\n",
    "        accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "\n",
    "    with tf.name_scope(\"init_and_save\"):\n",
    "        init = tf.global_variables_initializer()\n",
    "        saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 20\n",
    "batch_size = 50\n",
    "\n",
    "with tf.Session(graph=graph) as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(mnist.train.num_examples // batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_train = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_val = accuracy.eval(feed_dict={X: mnist.validation.images, y: mnist.validation.labels})\n",
    "        print(epoch, \"Train accuracy:\", acc_train, \"Validation accuracy:\", acc_val)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_mnist_model\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exercise 9"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![Exercise](https://c1.staticflickr.com/9/8101/8553474140_c50cf08708_b.jpg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this exercise, you will add a 50% dropout rate to the following neural network model below."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "9.1) Add a `training` placeholder, of type `tf.bool`.\n",
    "\n",
    "Tip: you can use `tf.placeholder_with_default()` to make this `False` by default."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "9.2) Add a dropout layer between the input layer and the first hidden layer, using `tf.layers.dropout()`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 100\n",
    "n_hidden2 = 100\n",
    "n_outputs = 10\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "    with tf.name_scope(\"inputs\"):\n",
    "        X = tf.placeholder(tf.float32, shape=[None, n_inputs], name=\"X\")\n",
    "        y = tf.placeholder(tf.int32, shape=[None], name=\"y\")\n",
    "\n",
    "    he_init = tf.contrib.layers.variance_scaling_initializer()\n",
    "\n",
    "    dense_layer = partial(tf.layers.dense,\n",
    "                          kernel_initializer=he_init,\n",
    "                          activation=tf.nn.elu)\n",
    "    hidden1 = dense_layer(X, n_hidden1, name=\"hidden1\")\n",
    "    hidden2 = dense_layer(hidden1, n_hidden2, name=\"hidden2\")\n",
    "    logits = dense_layer(hidden2, n_outputs, activation=None, name=\"output\")\n",
    "    Y_proba = tf.nn.softmax(logits)\n",
    "    \n",
    "    with tf.name_scope(\"train\"):\n",
    "        xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=y)\n",
    "        loss = tf.reduce_mean(xentropy)\n",
    "        optimizer = tf.train.AdamOptimizer()\n",
    "        training_op = optimizer.minimize(loss)\n",
    "\n",
    "    with tf.name_scope(\"eval\"):\n",
    "        correct = tf.nn.in_top_k(logits, y, 1)\n",
    "        accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "\n",
    "    with tf.name_scope(\"init_and_save\"):\n",
    "        init = tf.global_variables_initializer()\n",
    "        saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "9.3) Update the following training code to feed the value of the `training` placeholder, where appropriate, then run the code and see if the model performs better than without dropout."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 20\n",
    "batch_size = 50\n",
    "\n",
    "with tf.Session(graph=graph) as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(mnist.train.num_examples // batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_train = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_val = accuracy.eval(feed_dict={X: mnist.validation.images, y: mnist.validation.labels})\n",
    "        print(epoch, \"Train accuracy:\", acc_train, \"Validation accuracy:\", acc_val)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_mnist_model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Try not to peek at the solution below before you have done the exercise! :)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![thinking](https://upload.wikimedia.org/wikipedia/commons/0/06/Filos_segundo_logo_%28flipped%29.jpg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exercise 9 - Solution"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "9.1-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 100\n",
    "n_hidden2 = 100\n",
    "n_outputs = 10\n",
    "\n",
    "dropout_rate = 0.5                                                               # <= CHANGED\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "    with tf.name_scope(\"inputs\"):\n",
    "        X = tf.placeholder(tf.float32, shape=[None, n_inputs], name=\"X\")\n",
    "        y = tf.placeholder(tf.int32, shape=[None], name=\"y\")\n",
    "        training = tf.placeholder_with_default(False, shape=[], name='training') # <= CHANGED\n",
    "        X_drop = tf.layers.dropout(X, dropout_rate, training=training)           # <= CHANGED\n",
    "\n",
    "    he_init = tf.contrib.layers.variance_scaling_initializer()\n",
    "\n",
    "    dense_layer = partial(tf.layers.dense,\n",
    "                          kernel_initializer=he_init,\n",
    "                          activation=tf.nn.elu)\n",
    "    hidden1 = dense_layer(X_drop, n_hidden1, name=\"hidden1\")                     # <= CHANGED\n",
    "    hidden2 = dense_layer(hidden1, n_hidden2, name=\"hidden2\")\n",
    "    logits = dense_layer(hidden2, n_outputs, activation=None, name=\"output\")\n",
    "    Y_proba = tf.nn.softmax(logits)\n",
    "    \n",
    "    with tf.name_scope(\"train\"):\n",
    "        xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=y)\n",
    "        loss = tf.reduce_mean(xentropy)\n",
    "        optimizer = tf.train.AdamOptimizer()\n",
    "        training_op = optimizer.minimize(loss)\n",
    "\n",
    "    with tf.name_scope(\"eval\"):\n",
    "        correct = tf.nn.in_top_k(logits, y, 1)\n",
    "        accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "\n",
    "    with tf.name_scope(\"init_and_save\"):\n",
    "        init = tf.global_variables_initializer()\n",
    "        saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "9.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 20\n",
    "batch_size = 50\n",
    "\n",
    "with tf.Session(graph=graph) as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(mnist.train.num_examples // batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch, training: True}) # <= CHANGED\n",
    "        acc_train = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_val = accuracy.eval(feed_dict={X: mnist.validation.images, y: mnist.validation.labels})\n",
    "        print(epoch, \"Train accuracy:\", acc_train, \"Validation accuracy:\", acc_val)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_mnist_model\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Early Stopping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 1000\n",
    "batch_size = 50\n",
    "\n",
    "best_acc_val = 0\n",
    "check_interval = 100\n",
    "checks_since_last_progress = 0\n",
    "max_checks_without_progress = 100\n",
    "\n",
    "with tf.Session(graph=graph) as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(mnist.train.num_examples // batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch, training: True})\n",
    "            if iteration % check_interval == 0:\n",
    "                acc_val = accuracy.eval(feed_dict={X: mnist.validation.images[:2000], y: mnist.validation.labels[:2000]})\n",
    "                if acc_val > best_acc_val:\n",
    "                    best_acc_val = acc_val\n",
    "                    checks_since_last_progress = 0\n",
    "                    saver.save(sess, \"./my_best_model_so_far\")\n",
    "                else:\n",
    "                    checks_since_last_progress += 1                    \n",
    "        acc_train = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_val = accuracy.eval(feed_dict={X: mnist.validation.images[2000:], y: mnist.validation.labels[2000:]})\n",
    "        print(epoch, \"Train accuracy:\", acc_train, \"Validation accuracy:\", acc_val, \"Best validation accuracy:\", best_acc_val)\n",
    "        if checks_since_last_progress > max_checks_without_progress:\n",
    "            print(\"Early stopping!\")\n",
    "            saver.restore(sess, \"./my_best_model_so_far\")\n",
    "            break\n",
    "\n",
    "    acc_test = accuracy.eval(feed_dict={X: mnist.test.images[2000:], y: mnist.test.labels[2000:]})\n",
    "    print(\"Final accuracy on test set:\", acc_test)\n",
    "    save_path = saver.save(sess, \"./my_mnist_model\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Saving the model to disk so often slows down training. Let's save to RAM instead:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_model_params():\n",
    "    gvars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)\n",
    "    return {gvar.op.name: value for gvar, value in zip(gvars, tf.get_default_session().run(gvars))}\n",
    "\n",
    "def restore_model_params(model_params):\n",
    "    gvar_names = list(model_params.keys())\n",
    "    assign_ops = {gvar_name: tf.get_default_graph().get_operation_by_name(gvar_name + \"/Assign\")\n",
    "                  for gvar_name in gvar_names}\n",
    "    init_values = {gvar_name: assign_op.inputs[1] for gvar_name, assign_op in assign_ops.items()}\n",
    "    feed_dict = {init_values[gvar_name]: model_params[gvar_name] for gvar_name in gvar_names}\n",
    "    tf.get_default_session().run(assign_ops, feed_dict=feed_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 1000\n",
    "batch_size = 50\n",
    "\n",
    "best_acc_val = 0\n",
    "check_interval = 100\n",
    "checks_since_last_progress = 0\n",
    "max_checks_without_progress = 100\n",
    "best_model_params = None\n",
    "\n",
    "with tf.Session(graph=graph) as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(mnist.train.num_examples // batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch, training: True})\n",
    "            if iteration % check_interval == 0:\n",
    "                acc_val = accuracy.eval(feed_dict={X: mnist.validation.images[:2000], y: mnist.validation.labels[:2000]})\n",
    "                if acc_val > best_acc_val:\n",
    "                    best_acc_val = acc_val\n",
    "                    checks_since_last_progress = 0\n",
    "                    best_model_params = get_model_params()\n",
    "                else:\n",
    "                    checks_since_last_progress += 1\n",
    "        acc_train = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_val = accuracy.eval(feed_dict={X: mnist.validation.images[2000:], y: mnist.validation.labels[2000:]})\n",
    "        print(epoch, \"Train accuracy:\", acc_train, \"Validation accuracy:\", acc_val, \"Best validation accuracy:\", best_acc_val)\n",
    "        if checks_since_last_progress > max_checks_without_progress:\n",
    "            print(\"Early stopping!\")\n",
    "            break\n",
    "\n",
    "    if best_model_params:\n",
    "        restore_model_params(best_model_params)\n",
    "    acc_test = accuracy.eval(feed_dict={X: mnist.test.images[2000:], y: mnist.test.labels[2000:]})\n",
    "    print(\"Final accuracy on test set:\", acc_test)\n",
    "    save_path = saver.save(sess, \"./my_mnist_model\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}