experiments.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Adapted from Victor Zhou's CNN published at [this blog post](https://victorzhou.com/blog/keras-cnn-tutorial/)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import mnist\n",
    "import numpy as np\n",
    "import time\n",
    "\n",
    "# Using tf.keras for Ghost Batch Norm capability.\n",
    "import tensorflow.keras.backend as K\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Conv2D, MaxPooling2D, Dense, Flatten, BatchNormalization\n",
    "from tensorflow.keras.optimizers import SGD\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from tensorflow.keras.callbacks import LearningRateScheduler, Callback\n",
    "from tensorflow.keras.optimizers import Nadam\n",
    "\n",
    "import tensorflow as tf\n",
    "#print(tf.version)\n",
    "# Disable INFO and WARNING messages from TensorFlow.\n",
    "# Our Keras version (2.2.4 / TF 1.15.0) throws deprecation warnings.\n",
    "tf.logging.set_verbosity(tf.logging.ERROR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_images = mnist.train_images() \n",
    "train_labels = mnist.train_labels()\n",
    "test_images = mnist.test_images()\n",
    "test_labels = mnist.test_labels()\n",
    "\n",
    "# Normalize the images.\n",
    "train_images = (train_images / 255) - 0.5\n",
    "test_images = (test_images / 255) - 0.5\n",
    "\n",
    "# Reshape the images.\n",
    "train_images = np.expand_dims(train_images, axis=3)\n",
    "test_images = np.expand_dims(test_images, axis=3)\n",
    "\n",
    "num_filters = 8\n",
    "filter_size = 3\n",
    "pool_size = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build the model.\n",
    "batch_size = 100       # TODO: make this less arbitrary.\n",
    "ghost_batch_size = 10  # TODO: make this less arbitrary.\n",
    "\n",
    "model = Sequential([\n",
    "    # The authors required ghost batch normalization in their experiments.\n",
    "    # The papers they reference use normalization preceding each convolutional layer.\n",
    "    BatchNormalization(virtual_batch_size=ghost_batch_size),\n",
    "    Conv2D(filters=num_filters, kernel_size=filter_size, input_shape=(28, 28, 1)),\n",
    "    MaxPooling2D(pool_size=pool_size),\n",
    "    Flatten(),\n",
    "    Dense(units=10, activation='softmax'),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adapted from https://stackoverflow.com/questions/43178668/record-the-computation-time-for-each-epoch-in-keras-during-model-fit\n",
    "class TimeHistory(Callback):\n",
    "    def on_train_begin(self, logs={}):\n",
    "        self.time = 0\n",
    "\n",
    "    def on_epoch_begin(self, epoch, logs={}):\n",
    "        self.epoch_time_start = time.time()\n",
    "\n",
    "    def on_epoch_end(self, epoch, logs={}):\n",
    "        self.time += time.time() - self.epoch_time_start"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adapted from https://machinelearningmastery.com/using-learning-rate-schedules-deep-learning-models-python-keras/\n",
    "def step_lr_decay(epoch):\n",
    "    # This starts with a learning rate higher than the default learning rate in\n",
    "    # Vanilla SGD. It drops by a factor of 5 each step, and I chose to drop after\n",
    "    # every other epoch to make training not take too long on a laptop.\n",
    "    # Think of this as a half-life equation.\n",
    "    initial_lrate=0.15\n",
    "    drop=0.2\n",
    "    epochs_drop=2.0\n",
    "    current_rate = initial_lrate * math.pow(drop, math.floor(epoch / epochs_drop))\n",
    "    print(\"Epoch %d learning rate: %f\" % (epoch + 1, current_rate))\n",
    "    return current_rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compile the model.\n",
    "epochs = 6            # Run enough epochs to test the training schedule.\n",
    "learning_rate = 0.00  # Will be overrided by the scheduler.\n",
    "decay_rate = 0.00     # Default decay rate.\n",
    "momentum = 0.0        # Default momentum.\n",
    "\n",
    "sgd = SGD(lr=learning_rate, momentum=momentum, decay=decay_rate, nesterov=False)\n",
    "\n",
    "model.compile(\n",
    "    optimizer=sgd,  # Vanilla SGD experiment.\n",
    "    loss='categorical_crossentropy',\n",
    "    metrics=['accuracy'],\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Experiment 1: Vanilla SGD, decaying learning rate."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train the model.\n",
    "decay_lr_scheduler = LearningRateScheduler(step_lr_decay)\n",
    "time_callback = TimeHistory()\n",
    "\n",
    "exp1_results = model.fit(\n",
    "    train_images,\n",
    "    to_categorical(train_labels),\n",
    "    epochs=epochs,\n",
    "    batch_size=batch_size,\n",
    "    validation_data=(test_images, to_categorical(test_labels)),\n",
    "    callbacks=[decay_lr_scheduler, time_callback]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(exp1_results.history)\n",
    "print(\"\")\n",
    "print(time_callback.time)\n",
    "times = []\n",
    "times.append(time_callback.time)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Experiment 2: Vanilla SGD, increasing batch size."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Keras has no built in for this. \n",
    "# Code is from https://www.codementor.io/nitinsurya/how-to-re-initialize-keras-model-weights-et41zre2g\n",
    "def reset_weights(model):\n",
    "    session = K.get_session()\n",
    "    for layer in model.layers: \n",
    "        if hasattr(layer, 'kernel_initializer'):\n",
    "            layer.kernel.initializer.run(session=session)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_weights(model)\n",
    "learning_rate = 0.07  # Set a new constant.\n",
    "sgd = SGD(lr=learning_rate, momentum=momentum, decay=decay_rate, nesterov=False)\n",
    "\n",
    "model.compile(\n",
    "    optimizer=sgd,  # Vanilla SGD experiment.\n",
    "    loss='categorical_crossentropy',\n",
    "    metrics=['accuracy'],\n",
    ")\n",
    "\n",
    "# Just run each step independently. Keras has no callback for this.\n",
    "exp2_results = model.fit(\n",
    "    train_images,\n",
    "    to_categorical(train_labels),\n",
    "    epochs=2,\n",
    "    batch_size=batch_size,\n",
    "    validation_data=(test_images, to_categorical(test_labels)),\n",
    "    callbacks=[time_callback]\n",
    ")\n",
    "time_tmp = time_callback.time\n",
    "print(time_tmp)\n",
    "tmp_results = model.fit(\n",
    "    train_images,\n",
    "    to_categorical(train_labels),\n",
    "    epochs=2,\n",
    "    batch_size=batch_size * 5,\n",
    "    validation_data=(test_images, to_categorical(test_labels)),\n",
    "    callbacks=[time_callback]\n",
    ")\n",
    "time_tmp += time_callback.time\n",
    "print(time_tmp)\n",
    "\n",
    "for i in tmp_results.history.keys():\n",
    "    exp2_results.history[i].extend(tmp_results.history[i])\n",
    "\n",
    "tmp_results = model.fit(\n",
    "    train_images,\n",
    "    to_categorical(train_labels),\n",
    "    epochs=2,\n",
    "    batch_size=batch_size * 25,\n",
    "    validation_data=(test_images, to_categorical(test_labels)),\n",
    "    callbacks=[time_callback]\n",
    ")\n",
    "time_tmp += time_callback.time\n",
    "print(time_tmp)\n",
    "\n",
    "for i in tmp_results.history.keys():\n",
    "    exp2_results.history[i].extend(tmp_results.history[i])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(exp2_results.history)\n",
    "times.append(time_tmp)\n",
    "print(times)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Experiment 3: Vanilla SGD, hybrid."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def hybrid_lr_decay(epoch):\n",
    "    # Start off from the constant learning rate from the hybrid's initialization.\n",
    "    initial_lrate=0.05\n",
    "    drop=0.2\n",
    "    epochs_drop=2.0\n",
    "    current_rate = initial_lrate * math.pow(drop, math.floor(epoch / epochs_drop))\n",
    "    print(\"Epoch %d learning rate: %f\" % (epoch + 1, current_rate))\n",
    "    return current_rate\n",
    "\n",
    "hybrid_lr_scheduler = LearningRateScheduler(hybrid_lr_decay)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_weights(model)\n",
    "learning_rate = 0.05  # Set a new constant.\n",
    "sgd = SGD(lr=learning_rate, momentum=momentum, decay=decay_rate, nesterov=False)\n",
    "\n",
    "model.compile(\n",
    "    optimizer=sgd,  # Vanilla SGD experiment.\n",
    "    loss='categorical_crossentropy',\n",
    "    metrics=['accuracy'],\n",
    ")\n",
    "\n",
    "# Run a step, and then increase the batch size manually. The decay scheduler\n",
    "# will decay the learning rate in the next step.\n",
    "exp3_results = model.fit(\n",
    "    train_images,\n",
    "    to_categorical(train_labels),\n",
    "    epochs=2,\n",
    "    batch_size=batch_size,\n",
    "    validation_data=(test_images, to_categorical(test_labels)),\n",
    "    callbacks=[time_callback]\n",
    ")\n",
    "time_tmp = time_callback.time\n",
    "\n",
    "tmp_results = model.fit(\n",
    "    train_images,\n",
    "    to_categorical(train_labels),\n",
    "    epochs=4,\n",
    "    batch_size=batch_size * 5,\n",
    "    validation_data=(test_images, to_categorical(test_labels)),\n",
    "    callbacks=[hybrid_lr_scheduler, time_callback]\n",
    ")\n",
    "time_tmp += time_callback.time\n",
    "\n",
    "for i in tmp_results.history.keys():\n",
    "    if i == 'lr':\n",
    "        continue\n",
    "    exp3_results.history[i].extend(tmp_results.history[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(exp3_results.history)\n",
    "times.append(time_tmp)\n",
    "print(times)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Experiment 4: Adam, decaying learning rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "# Adapted from https://machinelearningmastery.com/using-learning-rate-schedules-deep-learning-models-python-keras/\n",
    "def step_lr_decay_adam(epoch):\n",
    "    # Same learning-rate decaying algorithm as Vanilla SGD, except setting the initial rate to 0.001\n",
    "    initial_lrate=0.001\n",
    "    drop=0.2\n",
    "    epochs_drop=2.0\n",
    "    current_rate = initial_lrate * math.pow(drop, math.floor(epoch / epochs_drop))\n",
    "    print(\"Epoch %d learning rate: %f\" % (epoch + 1, current_rate))\n",
    "    return current_rate\n",
    "'''\n",
    "reset_weights(model)\n",
    "nadam = Nadam() #The paper uses the default settings for Adam\n",
    "\n",
    "model.compile(\n",
    "    optimizer=nadam,  # Adam experiment.\n",
    "    loss='categorical_crossentropy',\n",
    "    metrics=['accuracy'],\n",
    ")\n",
    "\n",
    "# Train the model.\n",
    "decay_lr_scheduler = LearningRateScheduler(step_lr_decay)\n",
    "\n",
    "exp4_results = model.fit(\n",
    "    train_images,\n",
    "    to_categorical(train_labels),\n",
    "    epochs=epochs,\n",
    "    batch_size=batch_size,\n",
    "    validation_data=(test_images, to_categorical(test_labels)),\n",
    "    callbacks=[decay_lr_scheduler, time_callback]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(exp4_results.history)\n",
    "times.append(time_callback.time)\n",
    "print(times)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Experiment 5: Adam, increasing batch size."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_weights(model)\n",
    "nadam = Nadam() #The paper uses the default settings for Adam\n",
    "\n",
    "model.compile(\n",
    "    optimizer=nadam,  # Adam experiment.\n",
    "    loss='categorical_crossentropy',\n",
    "    metrics=['accuracy'],\n",
    ")\n",
    "\n",
    "# Just run each step independently. Keras has no callback for this.\n",
    "exp5_results = model.fit(\n",
    "    train_images,\n",
    "    to_categorical(train_labels),\n",
    "    epochs=2,\n",
    "    batch_size=batch_size,\n",
    "    validation_data=(test_images, to_categorical(test_labels)),\n",
    "    callbacks=[time_callback]\n",
    ")\n",
    "time_tmp = time_callback.time\n",
    "tmp_results = model.fit(\n",
    "    train_images,\n",
    "    to_categorical(train_labels),\n",
    "    epochs=2,\n",
    "    batch_size=batch_size * 5,\n",
    "    validation_data=(test_images, to_categorical(test_labels)),\n",
    "    callbacks=[time_callback]\n",
    ")\n",
    "time_tmp += time_callback.time\n",
    "\n",
    "for i in tmp_results.history.keys():\n",
    "    exp5_results.history[i].extend(tmp_results.history[i])\n",
    "\n",
    "tmp_results = model.fit(\n",
    "    train_images,\n",
    "    to_categorical(train_labels),\n",
    "    epochs=2,\n",
    "    batch_size=batch_size * 25,\n",
    "    validation_data=(test_images, to_categorical(test_labels)),\n",
    "    callbacks=[time_callback]\n",
    ")\n",
    "time_tmp += time_callback.time\n",
    "\n",
    "for i in tmp_results.history.keys():\n",
    "    exp5_results.history[i].extend(tmp_results.history[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(exp5_results.history)\n",
    "times.append(time_tmp)\n",
    "print(times)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Experiment 6: Adam, hybrid."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def hybrid_lr_decay_adam(epoch):\n",
    "    # Start off from the constant learning rate from the hybrid's initialization.\n",
    "    initial_lrate=0.001 #initial learning rate for Adam's Keras's default = 0.001\n",
    "    drop=0.2\n",
    "    epochs_drop=2.0\n",
    "    current_rate = initial_lrate * math.pow(drop, math.floor(epoch / epochs_drop))\n",
    "    print(\"Epoch %d learning rate: %f\" % (epoch + 1, current_rate))\n",
    "    return current_rate\n",
    "\n",
    "hybrid_lr_scheduler_adam = LearningRateScheduler(hybrid_lr_decay_adam)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_weights(model)\n",
    "nadam = Nadam() #The paper uses the default settings for Adam\n",
    "\n",
    "model.compile(\n",
    "    optimizer=nadam,  # Adam experiment.\n",
    "    loss='categorical_crossentropy',\n",
    "    metrics=['accuracy'],\n",
    ")\n",
    "\n",
    "# Run a step, and then increase the batch size manually. The decay scheduler\n",
    "# will decay the learning rate in the next step.\n",
    "exp6_results = model.fit(\n",
    "    train_images,\n",
    "    to_categorical(train_labels),\n",
    "    epochs=2,\n",
    "    batch_size=batch_size,\n",
    "    validation_data=(test_images, to_categorical(test_labels)),\n",
    "    callbacks=[time_callback]\n",
    ")\n",
    "time_tmp = time_callback.time\n",
    "\n",
    "tmp_results = model.fit(\n",
    "    train_images,\n",
    "    to_categorical(train_labels),\n",
    "    epochs=4,\n",
    "    batch_size=batch_size * 5,\n",
    "    validation_data=(test_images, to_categorical(test_labels)),\n",
    "    callbacks=[hybrid_lr_scheduler_adam, time_callback]\n",
    ")\n",
    "time_tmp += time_callback.time\n",
    "\n",
    "for i in tmp_results.history.keys():\n",
    "    if i == 'lr':\n",
    "        continue\n",
    "    exp6_results.history[i].extend(tmp_results.history[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(exp6_results.history)\n",
    "times.append(time_tmp)\n",
    "print(times)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Experiment 7: SGD with momentum, decaying learning rate."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_weights(model)\n",
    "learning_rate = 0.07  # Set a new constant.\n",
    "momentum = 0.1\n",
    "sgd = SGD(lr=learning_rate, momentum=momentum, decay=decay_rate, nesterov=True)\n",
    "\n",
    "model.compile(\n",
    "    optimizer=sgd,  # Vanilla SGD experiment.\n",
    "    loss='categorical_crossentropy',\n",
    "    metrics=['accuracy'],\n",
    ")\n",
    "\n",
    "# Train the model.\n",
    "decay_lr_scheduler = LearningRateScheduler(step_lr_decay)\n",
    "\n",
    "exp7_results = model.fit(\n",
    "    train_images,\n",
    "    to_categorical(train_labels),\n",
    "    epochs=epochs,\n",
    "    batch_size=batch_size,\n",
    "    validation_data=(test_images, to_categorical(test_labels)),\n",
    "    callbacks=[decay_lr_scheduler, time_callback]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(exp7_results.history)\n",
    "times.append(time_callback.time)\n",
    "print(times)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Experiment 8: SGD with momentum, increasing batch size."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_weights(model)\n",
    "momentum = 0.1\n",
    "sgd = SGD(lr=learning_rate, momentum=momentum, decay=decay_rate, nesterov=False)\n",
    "\n",
    "model.compile(\n",
    "    optimizer=sgd,  # Vanilla SGD experiment.\n",
    "    loss='categorical_crossentropy',\n",
    "    metrics=['accuracy'],\n",
    ")\n",
    "\n",
    "# Just run each step independently. Keras has no callback for this.\n",
    "exp8_results = model.fit(\n",
    "    train_images,\n",
    "    to_categorical(train_labels),\n",
    "    epochs=2,\n",
    "    batch_size=batch_size,\n",
    "    validation_data=(test_images, to_categorical(test_labels)),\n",
    "    callbacks=[time_callback]\n",
    ")\n",
    "time_tmp = time_callback.time\n",
    "\n",
    "tmp_results = model.fit(\n",
    "    train_images,\n",
    "    to_categorical(train_labels),\n",
    "    epochs=2,\n",
    "    batch_size=batch_size * 5,\n",
    "    validation_data=(test_images, to_categorical(test_labels)),\n",
    "    callbacks=[time_callback]\n",
    ")\n",
    "time_tmp += time_callback.time\n",
    "\n",
    "for i in tmp_results.history.keys():\n",
    "    exp8_results.history[i].extend(tmp_results.history[i])\n",
    "\n",
    "tmp_results = model.fit(\n",
    "    train_images,\n",
    "    to_categorical(train_labels),\n",
    "    epochs=2,\n",
    "    batch_size=batch_size * 25,\n",
    "    validation_data=(test_images, to_categorical(test_labels)),\n",
    "    callbacks=[time_callback]\n",
    ")\n",
    "time_tmp += time_callback.time\n",
    "\n",
    "for i in tmp_results.history.keys():\n",
    "    exp8_results.history[i].extend(tmp_results.history[i])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "print(exp8_results.history)\n",
    "times.append(time_tmp)\n",
    "print(times)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Experiment 9: SGD with momentum, hybrid."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_weights(model)\n",
    "learning_rate = 0.05  # Set a new constant.\n",
    "momentum = 0.1\n",
    "sgd = SGD(lr=learning_rate, momentum=momentum, decay=decay_rate, nesterov=False)\n",
    "\n",
    "model.compile(\n",
    "    optimizer=sgd,  # Vanilla SGD experiment.\n",
    "    loss='categorical_crossentropy',\n",
    "    metrics=['accuracy'],\n",
    ")\n",
    "\n",
    "# Run a step, and then increase the batch size manually. The decay scheduler\n",
    "# will decay the learning rate in the next step.\n",
    "exp9_results = model.fit(\n",
    "    train_images,\n",
    "    to_categorical(train_labels),\n",
    "    epochs=2,\n",
    "    batch_size=batch_size,\n",
    "    validation_data=(test_images, to_categorical(test_labels)),\n",
    "    callbacks=[time_callback]\n",
    ")\n",
    "time_tmp = time_callback.time\n",
    "\n",
    "tmp_results = model.fit(\n",
    "    train_images,\n",
    "    to_categorical(train_labels),\n",
    "    epochs=4,\n",
    "    batch_size=batch_size * 5,\n",
    "    validation_data=(test_images, to_categorical(test_labels)),\n",
    "    callbacks=[hybrid_lr_scheduler, time_callback]\n",
    ")\n",
    "time_tmp += time_callback.time\n",
    "\n",
    "for i in tmp_results.history.keys():\n",
    "    if i == 'lr':\n",
    "        continue\n",
    "    exp9_results.history[i].extend(tmp_results.history[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(exp9_results.history)\n",
    "times.append(time_tmp)\n",
    "print(times)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Experiment 1-3 Plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f_vanillasgd, a_vanillasgd = plt.subplots()\n",
    "a_vanillasgd.plot(range(1,7), exp1_results.history[\"val_acc\"], label=\"Decaying L-rate\")\n",
    "a_vanillasgd.plot(range(1,7), exp3_results.history[\"val_acc\"], label=\"Hybrid\")\n",
    "a_vanillasgd.plot(range(1,7), exp2_results.history[\"val_acc\"], label=\"Increasing Batch\")\n",
    "\n",
    "legend = a_vanillasgd.legend(loc='lower right', shadow=True, fontsize='medium')\n",
    "a_vanillasgd.set(xlabel='epochs', ylabel='val acc', title='Vanilla SGD')\n",
    "a_vanillasgd.grid()\n",
    "\n",
    "f_vanillasgd.savefig(\"vanilla_sgd.png\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Experiment 4-6 Plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f_adam, a_adam = plt.subplots()\n",
    "a_adam.plot(range(1,7), exp4_results.history[\"val_acc\"], label=\"Decaying L-rate\")\n",
    "a_adam.plot(range(1,7), exp6_results.history[\"val_acc\"], label=\"Hybrid\")\n",
    "a_adam.plot(range(1,7), exp5_results.history[\"val_acc\"], label=\"Increasing Batch\")\n",
    "\n",
    "legend = a_adam.legend(loc='lower right', shadow=True, fontsize='medium')\n",
    "a_adam.set(xlabel='epochs', ylabel='val acc', title='Adam')\n",
    "a_adam.grid()\n",
    "\n",
    "f_adam.savefig(\"adam.png\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Experiment 7-9 Plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f_sgdm, a_sgdm = plt.subplots()\n",
    "a_sgdm.plot(range(1,7), exp7_results.history[\"val_acc\"], label=\"Decaying L-rate\")\n",
    "a_sgdm.plot(range(1,7), exp9_results.history[\"val_acc\"], label=\"Hybrid\")\n",
    "a_sgdm.plot(range(1,7), exp8_results.history[\"val_acc\"], label=\"Increasing Batch\")\n",
    "\n",
    "legend = a_sgdm.legend(loc='lower right', shadow=True, fontsize='medium')\n",
    "a_sgdm.set(xlabel='epochs', ylabel='val acc', title='SGD with momentum')\n",
    "a_sgdm.grid()\n",
    "\n",
    "f_sgdm.savefig(\"sgd_with_momentum.png\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Time comparison (1-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "objects = ('Decay L_rate', 'Inc. Batch', 'Hybrid')\n",
    "y_pos = np.arange(len(objects))\n",
    "performance = times[0:3]\n",
    "\n",
    "plt.bar(y_pos, performance, align='center', alpha=0.5)\n",
    "plt.xticks(y_pos, objects)\n",
    "plt.ylabel('Time (sec)')\n",
    "plt.title('Vanilla SGD Training Time')\n",
    "\n",
    "plt.show()\n",
    "#plt.savefig(\"vanillaSGD_time.png\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Time comparison (4-6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "objects = ('Decay L_rate', 'Inc. Batch', 'Hybrid')\n",
    "y_pos = np.arange(len(objects))\n",
    "performance = times[3:6]\n",
    "\n",
    "plt.bar(y_pos, performance, align='center', alpha=0.5)\n",
    "plt.xticks(y_pos, objects)\n",
    "plt.ylabel('Time (sec)')\n",
    "plt.title('Adam Training Time')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "objects = ('Decay L_rate', 'Inc. Batch', 'Hybrid')\n",
    "y_pos = np.arange(len(objects))\n",
    "performance = times[6:9]\n",
    "\n",
    "plt.bar(y_pos, performance, align='center', alpha=0.5)\n",
    "plt.xticks(y_pos, objects)\n",
    "plt.ylabel('Time (sec)')\n",
    "plt.title('SGD with Momentum Training Time')\n",
    "\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}