Test_Keras_WGAN_3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%run 'Lib.ipynb'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mkdir: cannot create directory ‘out/3b’: File exists\r\n"
     ]
    }
   ],
   "source": [
    "%mkdir 'out/3b'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from PIL import Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import keras.backend as K\n",
    "from keras.datasets import mnist\n",
    "from keras.layers import *\n",
    "from keras.layers.convolutional import *\n",
    "from keras.layers.noise import GaussianNoise\n",
    "from keras.layers.advanced_activations import LeakyReLU\n",
    "from keras.models import Sequential, Model\n",
    "from keras.optimizers import *\n",
    "from keras.backend.common import _EPSILON\n",
    "from keras.utils.generic_utils import Progbar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Z_SIZE = 100\n",
    "\n",
    "# batch and latent size taken from the paper\n",
    "EPOCHS = 50\n",
    "BATCH_SIZE = 100\n",
    "\n",
    "# Adam parameters suggested in https://arxiv.org/abs/1511.06434\n",
    "ADAM_LR = 0.0002\n",
    "ADAM_BETA_1 = 0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.random.seed(777)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "K.set_image_dim_ordering('tf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# ???\n",
    "def modified_binary_crossentropy(y_true, y_pred):\n",
    "#     output = K.clip(output, _EPSILON, 1.0 - _EPSILON)\n",
    "#     return -(target * output + (1.0 - target) * (1.0 - output))\n",
    "    return K.mean(y_true * y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def create_D():\n",
    "    input_image = Input(shape=(28, 28, 1), name='input_image')\n",
    "\n",
    "    x = Conv2D(32, (3, 3), padding='same', name='conv_1')(input_image)\n",
    "    x = LeakyReLU()(x)\n",
    "    x = MaxPool2D(pool_size=2)(x)\n",
    "    x = Dropout(0.3)(x)\n",
    "\n",
    "    x = Conv2D(64, (3, 3), padding='same', name='conv_2')(x)\n",
    "    x = MaxPool2D(pool_size=1)(x)\n",
    "    x = LeakyReLU()(x)\n",
    "    x = Dropout(0.3)(x)\n",
    "\n",
    "    x = Conv2D(128, (3, 3), padding='same', name='conv_3')(x)\n",
    "    x = MaxPool2D(pool_size=2)(x)\n",
    "    x = LeakyReLU()(x)\n",
    "    x = Dropout(0.3)(x)\n",
    "\n",
    "    x = Conv2D(256, (3, 3), padding='same', name='coonv_4')(x)\n",
    "    x = MaxPool2D(pool_size=1)(x)\n",
    "    x = LeakyReLU()(x)\n",
    "    x = Dropout(0.3)(x)\n",
    "\n",
    "    features = Flatten()(x)\n",
    "\n",
    "    output_is_fake = Dense(\n",
    "        1, activation='linear', name='output_is_fake')(features)\n",
    "    output_class = Dense(\n",
    "        10, activation='softmax', name='output_class')(features)\n",
    "\n",
    "    return Model(inputs=[input_image], outputs=[output_is_fake, output_class], name='D')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def create_G(Z_SIZE=Z_SIZE):\n",
    "    DICT_LEN = 10\n",
    "    EMBEDDING_LEN = Z_SIZE\n",
    "\n",
    "    # class#\n",
    "    input_class = Input(shape=(1, ), dtype='int32', name='input_class')\n",
    "    # encode class# to the same size as Z to use hadamard multiplication later on\n",
    "    e = Embedding(\n",
    "        DICT_LEN, EMBEDDING_LEN,\n",
    "        embeddings_initializer='glorot_uniform')(input_class)\n",
    "    embedded_class = Flatten(name='embedded_class')(e)\n",
    "\n",
    "    # latent var\n",
    "    input_z = Input(shape=(Z_SIZE, ), name='input_z')\n",
    "\n",
    "    # hadamard product\n",
    "    h = multiply([input_z, embedded_class], name='h')\n",
    "\n",
    "    # cnn part\n",
    "    x = Dense(1024)(h)\n",
    "    x = LeakyReLU()(x)\n",
    "\n",
    "    x = Dense(128 * 7 * 7)(x)\n",
    "    x = LeakyReLU()(x)\n",
    "    x = Reshape((7, 7, 128))(x)\n",
    "\n",
    "    x = UpSampling2D(size=(2, 2))(x)\n",
    "    x = Conv2D(256, (5, 5), padding='same')(x)\n",
    "    x = LeakyReLU()(x)\n",
    "\n",
    "    x = UpSampling2D(size=(2, 2))(x)\n",
    "    x = Conv2D(128, (5, 5), padding='same')(x)\n",
    "    x = LeakyReLU()(x)\n",
    "\n",
    "    x = Conv2D(1, (2, 2), padding='same', activation='tanh', name='output_generated_image')(x)\n",
    "\n",
    "    return Model(inputs=[input_z, input_class], outputs=x, name='G')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "D = create_D()\n",
    "\n",
    "D.compile(\n",
    "    optimizer=SGD(clipvalue=0.01),\n",
    "    loss=[modified_binary_crossentropy, 'sparse_categorical_crossentropy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "G = create_G()\n",
    "\n",
    "G.compile(\n",
    "    optimizer=Adam(lr=ADAM_LR, beta_1=ADAM_BETA_1), loss='binary_crossentropy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "input_z = Input(shape=(Z_SIZE, ), name='input_z_')\n",
    "input_class = Input(shape=(1, ),name='input_class_', dtype='int32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# create combined D(G) model\n",
    "# D.trainable = False\n",
    "output_is_fake, output_class = D(G(inputs=[input_z, input_class]))\n",
    "combined = Model(inputs=[input_z, input_class], outputs=[output_is_fake, output_class])\n",
    "combined.get_layer('D').trainable = False # freeze D in generator training faze\n",
    "\n",
    "combined.compile(\n",
    "    optimizer='RMSprop',\n",
    "    loss=[modified_binary_crossentropy, 'sparse_categorical_crossentropy']\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7fcf0446bf60>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfoAAAH0CAYAAADVH+85AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAWJQAAFiUBSVIk8AAAIABJREFUeJzt3Xtw3Xd55/HPI1myfJNkyRdZjhPZSZyUS660SRM25DKw\npAwQSLLLH4UMAx3KlqWhYWGnEGqgO8MfO9xCGzqFbWaSmQ2dMEmHbRpYSEgCIWWSkHjBSZzg+92y\nrJutu7/7x/kpCEey7O9zrJ/86P2a0RzrnPOc5+uffjof/aRzfo+llAQAAGKqKXsBAADg9CHoAQAI\njKAHACAwgh4AgMAIegAAAiPoAQAIjKAHACAwgh4AgMAIegAAAiPoAQAIjKAHACAwgh4AgMAIegAA\nAiPoAQAIjKAHACAwgh4AgMDmlb2A08HMtkpqlLSt5KUAAJCrQ1JvSmmt50FCBr2kxrq6upa2traW\n3AcYGRnJbl5bW5tdK0ljY2Ol1EpSXV1ddu3o6GhpvT21knTs2LHs2nnzfN9Gnt6eWkmqr68vrbd3\nf/HU19T4fpnp+R73fo969jfP85okDQ8PZ9eamau3R5m9Pftpd3e3e3+RSg56MztL0pckvVNSq6S9\nkh6S9MWU0mHHQ29ra2tr+cxnPpP9APv378+ubWxszK6VpJ6enuzavr4+V+/ly5dn13Z1dbl6r1y5\nMru2vb3d1dvzBNbc3OzqPTQ0lF07MDDg6u3ZboODg67e3v3F8z3a0NDg6t3Skn0Mod7eXlfvZcuW\nZdfu3bvX1Xv79u3Ztd4fiD0/nHmD3lN/+HB+lD344IM6dOjQtuwHKJQW9GZ2rqSnJK2Q9C+SXpL0\nR5L+UtI7zezqlNKhstYHAEAEZb4Y7+9VCflPppRuSin995TS9ZK+JukCSf+jxLUBABBCKUFfHM2/\nQ5UXy/3dcTf/jaQjkj5oZotmeGkAAIRS1hH9dcXlj1JKv/eKnpRSn6SfS1oo6cqZXhgAAJGU9Tf6\nC4rLzVPc/ooqR/zrJf1kqgcxs2enuOnC/KUBABBHWUf0TcXlVC8vH7/e93JmAADmuDP6ffQppcsn\nu7440r9shpcDAMCsU9YR/fgRe9MUt49f3z0DawEAIKyygv7l4nL9FLefX1xO9Td8AABwEsoK+seK\ny3eY2e+twcyWSLpa0lFJT8/0wgAAiKSUoE8p/VbSj1Q5Yf9fHHfzFyUtknRvSunIDC8NAIBQynwx\n3n9R5RS43zSzGyS9KOkKVd5jv1nS50pcGwAAIZR2CtziqP4tku5RJeDvkHSupG9IupLz3AMA4Ffq\n2+tSSjslfbjMNQAAENkZ/T76EzEzLViwILt+/vz52bVr1qzJrpV8Y0u3bdvm6u0Zc7tjxw5X77Vr\n12bXLlmyxNW7ra0tu7a/v9/Ve/fu3dm13v/3yy+/PP2dprBw4UJXb882l6Tf/OY32bWekciS7/kh\npeTq7RlTe+zYsenvdAKeMbeeEdiS7/vEM4Za8n3NqjFP3qvM6XUAAOA0I+gBAAiMoAcAIDCCHgCA\nwAh6AAACI+gBAAiMoAcAIDCCHgCAwAh6AAACI+gBAAiMoAcAIDCCHgCAwAh6AAACI+gBAAgs7Jja\nkZER7dy5M7veMxKxo6Mju1aSWltbs2vr6+tdvWtra7Nru7u7Xb09Y4W941pXrFiRXdvY2Ojq7Rlz\n297e7urd2dmZXdvb2+vqffDgQVe953vU29vzfbZp0yZX7+eeey671vP9Lfm226JFi1y9583Lj6ua\nGt8x7ZEjR7JrPf9v77pfe5yqPAoAAJiVCHoAAAIj6AEACIygBwAgMIIeAIDACHoAAAIj6AEACIyg\nBwAgMIIeAIDACHoAAAIj6AEACIygBwAgMIIeAIDACHoAAAIj6AEACCzsPHozU0NDQ3Z9XV1ddu22\nbduyayXfbPWuri5Xb8/843Xr1rl6e+ZNe2bZS9Lw8HB2rZm5env208WLF7t6v/DCC9m13tnm3u02\nMjKSXeuZJy9Jhw4dyq7t7u529fbMhPfONx8YGMiu9axb8n3NvF/vw4cPZ9e2trZm1zKPHgAATIug\nBwAgMIIeAIDACHoAAAIj6AEACIygBwAgMIIeAIDACHoAAAIj6AEACIygBwAgMIIeAIDACHoAAAIj\n6AEACIygBwAgsLBjakdHR3XgwIGyl5Fl5cqV2bWXXXaZq/fOnTuza71jRz3jOz3jViXpnHPOya5t\na2tz9X766add9R59fX3Ztd4Rmp7xnZLU2NiYXesd7+v5vy9btszV27P2VatWuXp7xkF7Rn9LvhG5\ne/bscfXu6enJrl2xYoWrdzVwRA8AQGAEPQAAgRH0AAAERtADABAYQQ8AQGAEPQAAgRH0AAAERtAD\nABAYQQ8AQGAEPQAAgRH0AAAERtADABAYQQ8AQGAEPQAAgRH0AAAEFnYevZlp3rz8/55n3nRXV1d2\nrSTV1tZm17a0tLh6e/7fCxcudPX2bLfBwUFX7+eeey679g1veIOr9+joaHbt9u3bXb09hoeHXfVm\n5qp/85vfnF27b98+V2/P12z16tWu3iMjI9m1a9ascfWur6/PrvU8t0i+5wfPNpOklFJ27ZIlS7Jr\nvdvstcepyqNkMLNtZpam+PB9FwIAAEnlH9H3SPr6JNf3z/RCAACIqOyg704pbSh5DQAAhMWL8QAA\nCKzsI/r5Zvanks6WdETSRklPpJTGyl0WAAAxlB30bZLuPe66rWb24ZTS49MVm9mzU9x0oXtlAAAE\nUOav7v9J0g2qhP0iSW+W9A+SOiT9m5ldXN7SAACIobQj+pTSF4+76teS/tzM+iXdIWmDpPdN8xiX\nT3Z9caR/WRWWCQDAGW02vhjv28XlNaWuAgCAAGZj0B8sLheVugoAAAKYjUF/ZXG5pdRVAAAQQClB\nb2Z/YGavO2I3sw5J3yo+vW8m1wQAQERlvRjvP0u6w8yekLRdUp+kcyW9S1KDpIcl/c+S1gYAQBhl\nBf1jki6QdKmkq1X5e3y3pJ+p8r76e5NnXBAAAJBUUtAXJ8OZ9oQ4HjU1Na7xgJ6fM7yjQ/fs2eOq\n99i0aVN27VlnneXq3dvbm117+PBhV++lS5dm1z7zzDOu3suXL8+uXbBggav3kSNHsmu9I5EHBgZc\n9WNj+SfQ9Hy9JamzszO71vs1W7VqVXatZ8ys5Pua9/X1uXp3d3dn13rH1K5YsSK7trm5ObvWM7J8\notn4YjwAAFAlBD0AAIER9AAABEbQAwAQGEEPAEBgBD0AAIER9AAABEbQAwAQGEEPAEBgBD0AAIER\n9AAABEbQAwAQGEEPAEBgBD0AAIER9AAABFbKPPqZMDY2pp6enuz6gwcPZtd6Zx8fPXo0u3bt2rWu\n3uvWrcuuXb16tav3b3/72+xazyx7SWpvb8+uPXTokKt3XV1ddu2iRYtcvUdHR7NrPfPgJenIkSOu\nes92X7lypau3Zyb8kiVLXL2XLVuWXWtmrt5DQ0PZtZ7nVEkaHBzMrvVsM8n3nFxTk3887f16vbaG\nqjwKAACYlQh6AAACI+gBAAiMoAcAIDCCHgCAwAh6AAACI+gBAAiMoAcAIDCCHgCAwAh6AAACI+gB\nAAiMoAcAIDCCHgCAwAh6AAACCzumtqamRvX19dn1zc3N2bXe0YI7d+7Mrh0YGHD19myzN77xja7e\nZY4G7ujoyK694IILXL09o2b7+vpcvT3jWg8fPuzqfezYMVf9ihUrsms932OSb0Tu/PnzXb1feuml\n7NqLLrrI1Xv9+vXZtbt373b19uyrnrHCkvTiiy+66svGET0AAIER9AAABEbQAwAQGEEPAEBgBD0A\nAIER9AAABEbQAwAQGEEPAEBgBD0AAIER9AAABEbQAwAQGEEPAEBgBD0AAIER9AAABEbQAwAQWNh5\n9CMjI9q3b192fXd3d3ZtXV1ddq3km1ftWbckHThwILt27dq1rt6emfKeme6S1NLSkl3rmYvu1d7e\n7qofHBys0kpOnXcefWdnZ3Ztf3+/q3dtbW0ptZLU3NycXVtfX+/q3dHRkV37/PPPu3rX1OQfl3qf\nk3ft2pVd29bWll1rZtm1E3FEDwBAYAQ9AACBEfQAAARG0AMAEBhBDwBAYAQ9AACBEfQAAARG0AMA\nEBhBDwBAYAQ9AACBEfQAAARG0AMAEBhBDwBAYAQ9AACBhR1TW1tbq8bGxux6z4jb4eHh7FrJN2rW\nM7pTks4555zsWu/4Tc+4V+/YUY8jR4646pcuXZpd6x07unXr1uzavr4+V2/v6FDP2NL169e7eo+N\njWXXekdJe8b77ty509XbU+/9Hm1oaMiuXb58uav32WefnV3rGTs+q8bUmtktZnaXmT1pZr1mlszs\nvmlqrjKzh82sy8wGzGyjmd1uZr60AAAAr6nWEf3nJV0sqV/SLkkXnujOZvZeSd+XNCjpe5K6JL1b\n0tckXS3p1iqtCwCAOa1af6P/lKT1kholffxEdzSzRkn/KGlM0rUppY+klP6bpEsk/ULSLWb2gSqt\nCwCAOa0qQZ9Seiyl9EpKKZ3E3W+RtFzS/SmlZyY8xqAqvxmQpvlhAQAAnJwyXnV/fXH5yCS3PSHp\nqKSrzCz/FQwAAEBSOUF/QXG5+fgbUkqjkraq8tqBdTO5KAAAIirj7XVNxWXPFLePX9883QOZ2bNT\n3HTCFwMCADBXcMIcAAACK+OIfvyIvWmK28evn/asEimlyye7vjjSv+zUlwYAQCxlHNG/XFy+7tRU\nZjZP0lpJo5K2zOSiAACIqIygf7S4fOckt10jaaGkp1JKQzO3JAAAYioj6B+Q1CnpA2b2lvErzaxB\n0t8Wn95dwroAAAinKn+jN7ObJN1UfNpWXP6xmd1T/LszpfRpSUop9ZrZn6kS+D81s/tVOQXue1R5\n690DqpwWFwAAOFXrxXiXSLrtuOvW6Xfvhd8u6dPjN6SUHjKzt0n6nKSbJTVIelXSX0n65kmeYQ8A\nAEyjKkGfUtogacMp1vxc0p9Uoz8AAJhc2Hn0KSXXzOhly5a5ent45nw3Nja6eg8ODmbXbtnie6OE\np3dbW9v0dzqBgYGB7FrvfPHdu3dn1x48eNDV+5VXXsmura+vd/VetWqVq97T3/s1q6ury671PC9J\nUk/PVOcam968eb6n/D179mTXemerX3TRRdm1NTW+l6M1N097/rYpeb7e1frlNifMAQAgMIIeAIDA\nCHoAAAIj6AEACIygBwAgMIIeAIDACHoAAAIj6AEACIygBwAgMIIeAIDACHoAAAIj6AEACIygBwAg\nMIIeAIDAwo6pHR4e1o4dO7LrPaMgW1pasmsl3yjJNWvWuHp7xrXu3LnT1Xv16tXZtf39/a7ey5cv\nz64dHh529faM7/SOJS5TV1eXq94z3vess85y9fZ8zffv3+/q3dHRkV3rGa8rSYsXL86uPe+881y9\nPdvN87wm+b5HPfsKY2oBAMC0CHoAAAIj6AEACIygBwAgMIIeAIDACHoAAAIj6AEACIygBwAgMIIe\nAIDACHoAAAIj6AEACIygBwAgMIIeAIDACHoAAAIj6AEACCzsPHovz/xh76xrz+zk1tZWV+/6+vrs\nWu9c9qVLl2bXer5eknTw4MHs2scff9zVu7+/P7v22LFjrt6e+eSjo6Ou3meffbarfnBwMLt206ZN\nrt49PT3ZtQsWLHD1Hhsby671fo9u3bo1u9b73OTZV5uamly9PXPh+/r6smu932PjOKIHACAwgh4A\ngMAIegAAAiPoAQAIjKAHACAwgh4AgMAIegAAAiPoAQAIjKAHACAwgh4AgMAIegAAAiPoAQAIjKAH\nACAwgh4AgMDCjqk1M9fI1fPPPz+79vLLL8+ulaTly5dn13rHlj733HPZtV1dXa7emzdvzq6dP3++\nq/euXbuyaz3bTJJGRkaya73/b89+vnjxYlfvmhrfcUZjY2N27b59+1y9V69eXVrvAwcOZNd2dna6\nenvq3/rWt7p6e8ZYe8fULlmyJLv20KFD2bXe5/NxHNEDABAYQQ8AQGAEPQAAgRH0AAAERtADABAY\nQQ8AQGAEPQAAgRH0AAAERtADABAYQQ8AQGAEPQAAgRH0AAAERtADABAYQQ8AQGAEPQAAgYWdR19f\nX6+zzjoru76npye79uDBg9m1krRy5crsWu/84paWFle9x549e7Jre3t7Xb0986rb29tdvT0zvt/1\nrne5ere2tmbXemayS/559H19fdm1ixYtcvX2fM2efvppV2/P9/jg4KCrt+c59cYbb3T1HhgYyK71\nPq+Njo5m1+7fvz+71vs98trjVONBzOwWM7vLzJ40s14zS2Z23xT37Shun+rj/mqsCQAAVO+I/vOS\nLpbUL2mXpAtPouYFSQ9Ncv2vq7QmAADmvGoF/adUCfhXJb1N0mMnUfN8SmlDlfoDAIBJVCXoU0qv\nBbuZVeMhAQBAFZT5Yrx2M/uYpFZJhyT9IqW0scT1AAAQTplB//bi4zVm9lNJt6WUdpzMA5jZs1Pc\ndDKvEQAAILwy3kd/VNKXJV0uaWnxMf53/Wsl/cTMfO99AQAAkko4ok8pHZD0heOufsLM3iHpZ5Ku\nkPRRSd84ice6fLLriyP9y5xLBQDgjDdrzoyXUhqV9J3i02vKXAsAAFHMmqAvjJ9Sjl/dAwBQBbMt\n6K8sLreUugoAAIKY8aA3s8vM7HV9zewGVU68I0mTnj4XAACcmqq8GM/MbpJ0U/FpW3H5x2Z2T/Hv\nzpTSp4t/f1XS+Wb2lCpn05OkiyRdX/z7zpTSU9VYFwAAc121XnV/iaTbjrtuXfEhSdsljQf9vZLe\nJ+kPJd0oqU7Sfkn/LOlbKaUnq7QmAADmvGqdAneDpA0ned/vSvpuNfoCAIATCzuPvra2Vo2Njdn1\nnjnd3pnPW7duza4dGRlx9R4eHs6uXbNmjat3c3Nzdu3u3btdvXfsOKmTMU5q27Ztrt67du2a/k5T\nWLVqlav3tddem13rmYsuSXV1da76sbGx7NqjR4+6end0dGTXbt++3dX7N7/5TXZtX1+fq3dDQ0N2\nbU9Pj6v32rVrs2s98+QlafPmzdm1KSVX72qYba+6BwAAVUTQAwAQGEEPAEBgBD0AAIER9AAABEbQ\nAwAQGEEPAEBgBD0AAIER9AAABEbQAwAQGEEPAEBgBD0AAIER9AAABEbQAwAQWNgxtZJkZtm1Q0ND\n2bXz58/PrpV8IzS9Yyg9oz/b29tdvWtra7NrvWNHPSNT9+7d6+q9cuXK7NqmpiZXb8/+4h2J3NbW\n5qpfvnx5du3FF1/s6u35Hj98+LCrt2df3bNnj6t3S0tLdq13pHJNTf5x6bJly1y9zz///OzajRs3\nunpXA0f0AAAERtADABAYQQ8AQGAEPQAAgRH0AAAERtADABAYQQ8AQGAEPQAAgRH0AAAERtADABAY\nQQ8AQGAEPQAAgRH0AAAERtADABAYQQ8AQGBh59GbmebNy//v1dfXZ9deeuml2bVSZe25du/e7eq9\nf//+7FrPvGhJWrduXXbtOeec4+o9MDCQXeuZ0S1Jv/rVr7Jrx8bGXL0PHjyYXVtbW+vq3dra6qpf\nv359dq1nlr0kpZSya2+++WZX71/+8pfZtdu2bXP1XrhwYXatZ5tJ0rFjx7Jrvc+LfX192bWeHPJk\nwUQc0QMAEBhBDwBAYAQ9AACBEfQAAARG0AMAEBhBDwBAYAQ9AACBEfQAAARG0AMAEBhBDwBAYAQ9\nAACBEfQAAARG0AMAEBhBDwBAYGHH1I6Ojqqrqyu7ftWqVdm1w8PD2bWSNDIykl3rGSPprd+3b5+r\nt2eMZX9/v6v3li1bsms7OztdvZcsWZJd++qrr7p6e0ZoescSb9q0yVXf1NSUXbto0SJXb89zi5fn\n+2zjxo2u3s3Nzdm1PT09rt6rV6/Orm1oaHD19oyDHhoayq71jOadiCN6AAACI+gBAAiMoAcAIDCC\nHgCAwAh6AAACI+gBAAiMoAcAIDCCHgCAwAh6AAACI+gBAAiMoAcAIDCCHgCAwAh6AAACI+gBAAiM\noAcAILCw8+jHxsbU3d2dXe+Ztd3R0ZFd62Vmrvr6+vrs2hdffNHVu6yZz5K0d+/e7Nply5a5ere0\ntGTXemaTS1JtbW12bW9vr6u3d6Z7Z2dndu3FF1/s6n3o0KHs2rq6Oldvz77e19fn6t3W1pZdu3z5\nclfvpqam7Npzzz3X1XtgYCC79pFHHsmunTXz6M2s1cw+amYPmtmrZjZgZj1m9jMz+4iZTdrDzK4y\ns4fNrKuo2Whmt5tZ/jMPAAD4PdU4or9V0t2S9kp6TNIOSSslvV/SdyTdaGa3ppTSeIGZvVfS9yUN\nSvqepC5J75b0NUlXF48JAACcqhH0myW9R9K/ppRe+z2Dmf21pF9KulmV0P9+cX2jpH+UNCbp2pTS\nM8X1d0p6VNItZvaBlNL9VVgbAABzmvtX9ymlR1NKP5gY8sX1+yR9u/j02gk33SJpuaT7x0O+uP+g\npM8Xn37cuy4AAHD6X3U/UlyOTrju+uJyslcoPCHpqKSrzGz+6VwYAABzwWl71b2ZzZP0oeLTiaF+\nQXG5+fialNKomW2V9EZJ6ySd8GXcZvbsFDddeGqrBQAgptN5RP8VSW+S9HBK6YcTrh9/j0TPFHXj\n1zefroUBADBXnJYjejP7pKQ7JL0k6YOno4ckpZQun6L/s5IuO119AQA4U1T9iN7MPiHpG5I2Sbou\npXT8WTHGj9inOvvB+PX5Z7sBAACSqhz0Zna7pLsk/VqVkJ/stF0vF5frJ6mfJ2mtKi/e21LNtQEA\nMBdVLejN7LOqnPDmeVVC/sAUd320uHznJLddI2mhpKdSSr5zmgIAgOoEfXGym69IelbSDSmlE52E\n+gFJnZI+YGZvmfAYDZL+tvj07mqsCwCAuc79Yjwzu03Sl1Q5092Tkj45yWCVbSmleyQppdRrZn+m\nSuD/1MzuV+UUuO9R5a13D6hyWlwAAOBUjVfdry0uayXdPsV9Hpd0z/gnKaWHzOxtkj6nyilyGyS9\nKumvJH1z4nnxAQBAPnfQp5Q2SNqQUfdzSX/i7T+VsbEx1xhNz7jW3bt3Z9dKvtGES5YscfX2jA5t\naGhw9R4eHs6u9Y6p3bVrV2m9e3qmOqXE9DZt2uTq7Rkd2t/f7+rt/T7xjPf1jESWfCN6L7nkEldv\nz5jbSy+91NV78eLFpdR66+fP951o9fDhw9m1nudkzxjpiU73KXABAECJCHoAAAIj6AEACIygBwAg\nMIIeAIDACHoAAAIj6AEACIygBwAgMIIeAIDACHoAAAIj6AEACIygBwAgMIIeAIDACHoAAAIj6AEA\nCMw9j362GhkZ0d69e7PrPTPCd+zYkV0r+WaEX3jhha7enrnNjY2Nrt6er9fIyIirt2e2umfetCQN\nDg5m13pmk0u+edee/VSSzjvvPFf9kSNHXPUe+/fvz6798Y9/7Ord3d2dXTs0NOTqvXTp0uzadevW\nuXp3dnZm1/b19bl6Dw8PZ9d6ZtmPjo5m107EET0AAIER9AAABEbQAwAQGEEPAEBgBD0AAIER9AAA\nBEbQAwAQGEEPAEBgBD0AAIER9AAABEbQAwAQGEEPAEBgBD0AAIER9AAABBZ2TG1dXZ3a29uz6xsa\nGrJrveMYPWMoyxyZumzZMldvz9fLM1ZY8q3dzFy9PV9vz5hZSWpubs6uPXr0qKt3W1ubq76lpSW7\n1juu9Yorrsiu9YxblaQtW7aU1ttT7x1L3Nvbm13r/Xo3NTVl13pGaKeUsmsn4ogeAIDACHoAAAIj\n6AEACIygBwAgMIIeAIDACHoAAAIj6AEACIygBwAgMIIeAIDACHoAAAIj6AEACIygBwAgMIIeAIDA\nCHoAAAIj6AEACCzsPHpJGhsby67dvXt3dq13Jvz8+fOza3fu3Onq7dlmHR0drt51dXXZtatWrXL1\n9ti7d6+rfvHixdm13n3NMyvb+/X2zJOXpOHh4eza/fv3u3qbWXat93vUo7m52VW/ffv27FrPLHtJ\nOnDgQHatd657a2trdm19fX12rWc/m4gjegAAAiPoAQAIjKAHACAwgh4AgMAIegAAAiPoAQAIjKAH\nACAwgh4AgMAIegAAAiPoAQAIjKAHACAwgh4AgMAIegAAAiPoAQAILOyYWjNTQ0NDdn1NTf7PQAMD\nA9m1km9MbVNTk6t3f39/du2uXbtcvT1Wrlzpqh8aGsquXbZsmau3Z4TmwoULXb2PHDmSXevZTyX/\nqFgPzzhmyTeu1TO2VPKt3Tuu9Zxzzsmu9W7zo0ePZtd6skCS9u3bl107b175Mes+ojezVjP7qJk9\naGavmtmAmfWY2c/M7CNmVnPc/TvMLJ3g437vmgAAQEU1ftS4VdLdkvZKekzSDkkrJb1f0nck3Whm\nt6bX/yj5gqSHJnm8X1dhTQAAQNUJ+s2S3iPpX1NKx8avNLO/lvRLSTerEvrfP67u+ZTShir0BwAA\nU3D/6j6l9GhK6QcTQ764fp+kbxefXuvtAwAATt3pfpXASHE5Oslt7Wb2MUmtkg5J+kVKaeNpXg8A\nAHPKaQt6M5sn6UPFp49Mcpe3Fx8Ta34q6baU0o7TtS4AAOaS03lE/xVJb5L0cErphxOuPyrpy6q8\nEG9Lcd1FkjZIuk7ST8zskpTStO/7MbNnp7jpwtxFAwAQyWk5YY6ZfVLSHZJekvTBibellA6klL6Q\nUnoupdRdfDwh6R2S/l3SeZI+ejrWBQDAXFP1I3oz+4Skb0jaJOmGlFLXydSllEbN7DuSrpB0TfEY\n09VcPsUanpV02UkvGgCAoKp6RG9mt0u6S5X3wl9XvPL+VBwsLhdVc10AAMxVVQt6M/uspK9Jel6V\nkD+Q8TBXFpdbTngvAABwUqoS9GZ2pyovvntWlV/Xd57gvpcdf1rc4vobJH2q+PS+aqwLAIC5zv03\nejO7TdKXJI1JelLSJ83s+LttSyndU/z7q5LON7OnJI1PQblI0vXFv+9MKT3lXRcAAKjOi/HWFpe1\nkm6f4j6PS7qn+Pe9kt4n6Q8l3SipTtJ+Sf8s6VsppSersCYAAKAqBH1xvvoNp3D/70r6rrcvAACY\nXvmDck//d6HpAAAIm0lEQVST+vp6nX322dn1bW1t2bXe2cdLly7Nru3r63P19vDOXW5qasqu9c66\nXrBgQXatdyb84cOHs2sbGxtdvT2Gh4dd9T09Pa76Sf5EeNIWLfK9scfzNVu8eLGrt+f5xfPcIknN\nzc3ZtYODg67enufzkZGR6e90Anv27MmuXbJkSXZtTU11Xi9/Wk6YAwAAZgeCHgCAwAh6AAACI+gB\nAAiMoAcAIDCCHgCAwAh6AAACI+gBAAiMoAcAIDCCHgCAwAh6AAACI+gBAAiMoAcAIDCCHgCAwMKO\nqR0bG3ONwaytrc2uPXbsWHat5BuZ6h0V6xlj6R39uXXr1uzauro6V2/PNt+7d6+rt2ebd3Z2unp7\nvkfmz5/v6l1fX++q94z/9Iy4lXwjV7u6uly9PWNuvaNiPeNavV9vT733+aG9vT271pMH3v10HEf0\nAAAERtADABAYQQ8AQGAEPQAAgRH0AAAERtADABAYQQ8AQGAEPQAAgRH0AAAERtADABAYQQ8AQGAE\nPQAAgRH0AAAERtADABCYpZTKXkPVmdmh2traFs/4T894QO9oQc+IXC9P75oa38+NQ0ND2bXebe5Z\n+9jYmKt3mSORPWv3fr29zz1lfp94tpv3a+b5f3u/Tzxr9+4vZT4ne/ZVT21nZ6dGR0e7Ukqt2Q+i\nuPPoe8fGxtTZ2bltitsvLC5fmqH1RMA2y8N2y8N2O3Vsszyzebt1SOr1PkjII/rpmNmzkpRSurzs\ntZwp2GZ52G552G6njm2WZy5sN/5GDwBAYAQ9AACBEfQAAARG0AMAEBhBDwBAYHPyVfcAAMwVHNED\nABAYQQ8AQGAEPQAAgRH0AAAERtADABAYQQ8AQGAEPQAAgc2poDezs8zsf5nZHjMbMrNtZvZ1M8sf\nXB9csY3SFB/7yl5fWczsFjO7y8yeNLPeYnvcN03NVWb2sJl1mdmAmW00s9vNrLzB6jPsVLabmXWc\nYN9LZnb/TK+/DGbWamYfNbMHzezVYt/pMbOfmdlHzGzS5/G5vr+d6naLvL9FnUf/OmZ2rqSnJK2Q\n9C+qzB7+I0l/KemdZnZ1SulQiUuczXokfX2S6/tneiGzyOclXazKNtil3820npSZvVfS9yUNSvqe\npC5J75b0NUlXS7r1dC52Fjml7VZ4QdJDk1z/6yquaza7VdLdkvZKekzSDkkrJb1f0nck3Whmt6YJ\nZz9jf5OUsd0K8fa3lNKc+JD0Q0lJ0n897vqvFtd/u+w1zsYPSdskbSt7HbPtQ9J1ks6XZJKuLfah\n+6a4b6OkA5KGJL1lwvUNqvzwmSR9oOz/0yzcbh3F7feUve6St9n1qoR0zXHXt6kSXknSzROuZ3/L\n225h97c58av74mj+HaqE1t8dd/PfSDoi6YNmtmiGl4YzVErpsZTSK6l4hpjGLZKWS7o/pfTMhMcY\nVOUIV5I+fhqWOeuc4naDpJTSoymlH6SUjh13/T5J3y4+vXbCTexvytpuYc2VX91fV1z+aJIvep+Z\n/VyVHwSulPSTmV7cGWC+mf2ppLNV+aFoo6QnUkpj5S7rjHF9cfnIJLc9IemopKvMbH5KaWjmlnXG\naDezj0lqlXRI0i9SShtLXtNsMVJcjk64jv1tepNtt3Hh9re5EvQXFJebp7j9FVWCfr0I+sm0Sbr3\nuOu2mtmHU0qPl7GgM8yU+19KadTMtkp6o6R1kl6cyYWdId5efLzGzH4q6baU0o5SVjQLmNk8SR8q\nPp0Y6uxvJ3CC7TYu3P42J351L6mpuOyZ4vbx65tnYC1nmn+SdIMqYb9I0psl/YMqf8/6NzO7uLyl\nnTHY//IclfRlSZdLWlp8vE2VF1ZdK+knc/zPbV+R9CZJD6eUfjjheva3E5tqu4Xd3+ZK0CNTSumL\nxd+69qeUjqaUfp1S+nNVXsS4QNKGcleIqFJKB1JKX0gpPZdS6i4+nlDlt2//Luk8SR8td5XlMLNP\nSrpDlXcPfbDk5ZwxTrTdIu9vcyXox3+CbZri9vHru2dgLVGMv5jlmlJXcWZg/6uilNKoKm+Pkubg\n/mdmn5D0DUmbJF2XUuo67i7sb5M4ie02qQj721wJ+peLy/VT3H5+cTnV3/DxegeLyzPyV1kzbMr9\nr/h74VpVXhS0ZSYXdYabk/ufmd0u6S5V3tN9XfEK8uOxvx3nJLfbiZzR+9tcCfrHist3THI2pCWq\nnEDiqKSnZ3phZ7Ari8s582Th8Ghx+c5JbrtG0kJJT83hV0DnmHP7n5l9VpUT3jyvSlgdmOKu7G8T\nnMJ2O5Ezen+bE0GfUvqtpB+p8gKyvzju5i+q8lPavSmlIzO8tFnNzP5gshefmFmHpG8Vn57wtK+Q\nJD0gqVPSB8zsLeNXmlmDpL8tPr27jIXNZmZ22WSndzWzGyR9qvh0Tux/ZnanKi8ie1bSDSmlzhPc\nnf2tcCrbLfL+ZnPlvBWTnAL3RUlXqPIe+82SrkqcAvf3mNkGVV648oSk7ZL6JJ0r6V2qnGXrYUnv\nSykNl7XGspjZTZJuKj5tk/QfVflp/8nius6U0qePu/8DqpyS9H5VTkn6HlXeCvWApP80F04icyrb\nrXhL0/mqfN/uKm6/SL97n/idKaXx4ArLzG6TdI+kMVV+/TzZq+m3pZTumVAz5/e3U91uofe3sk/N\nN5Mfktao8naxvZKGVQmvr0taWvbaZuOHKm8t+d+qvEK1W5WTTByU9H9VeR+qlb3GErfNBlVOlznV\nx7ZJaq5W5Yejw5IGJP0/VY4Uasv+/8zG7SbpI5L+jypntOxX5ZSuO1Q5d/t/KPv/Mou2WZL0U/Y3\n33aLvL/NmSN6AADmojnxN3oAAOYqgh4AgMAIegAAAiPoAQAIjKAHACAwgh4AgMAIegAAAiPoAQAI\njKAHACAwgh4AgMAIegAAAiPoAQAIjKAHACAwgh4AgMAIegAAAiPoAQAIjKAHACCw/w8RsMkrOkdp\nqwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fcf08ebf7f0>"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 250,
       "width": 253
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# try generating image\n",
    "\n",
    "z = np.random.randn(100).reshape(1, 100)\n",
    "img = G.predict([z, np.array([8])])\n",
    "plt.imshow(img.reshape((28, 28)), cmap=plt.cm.gray)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# get our mnist data, and force it to be of shape (..., 28, 28, 1) with\n",
    "# range [-1., 1.]\n",
    "(X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
    "X_train = (X_train.astype(np.float32) - 127.5) / 127.5\n",
    "X_train = np.expand_dims(X_train, axis=3)\n",
    "\n",
    "X_test = (X_test.astype(np.float32) - 127.5) / 127.5\n",
    "X_test = np.expand_dims(X_test, axis=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7fcf03dfdf60>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfoAAAH0CAYAAADVH+85AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAWJQAAFiUBSVIk8AAAHU1JREFUeJzt3X2sZXV5L/DvU8aCEHm5WiWmWNALUtsqglYKVXlJFdpU\nQeBekgrEaKNe0GLFYlvtHam3aNJWBK9iqi0JJhcbrLZViiiggFgbMcClIi/FAU1FRS7vL4L+7h97\nTZ2O58zLXnvOPud3Pp9kZ5291n7275k1a+Z71t7rpVprAQD69DPzbgAA2HYEPQB0TNADQMcEPQB0\nTNADQMcEPQB0TNADQMcEPQB0TNADQMcEPQB0TNADQMcEPQB0TNADQMcEPQB0TNADQMcEPQB0bM28\nG9gWquqbSXZOsm7OrQDAtPZMcl9rba8xb9Jl0GcS8v9leADAqjXXj+6r6uer6q+r6t+r6tGqWldV\nZ1XVbiPfet0s+gOAOVs39g3mtkdfVc9KcnWSpyb5+yTfSPKrSX4vyRFVdXBr7Qfz6g8AejDPPfoP\nZhLyb26tHdVae3tr7bAk70vy7CT/a469AUAXqrW29INO9uZvzeQjiWe11n68wbInJflOkkry1Nba\ng1O8/zVJ9p9NtwAwN19rrR0w5g3mtUd/6DC9ZMOQT5LW2v1JvpRkxyQHLnVjANCTeX1H/+xhevMi\ny29J8rIk+yS5dLE3GfbcF7Lv9K0BQD/mtUe/yzC9d5Hl6+fvugS9AEC3VvR59It9b+E7egCYmNce\n/fo99l0WWb5+/j1L0AsAdGteQX/TMN1nkeV7D9PFvsMHALbAvIL+8mH6sqr6Tz0Mp9cdnOShJP+8\n1I0BQE/mEvSttX9LckkmF+w/eaPF70qyU5LzpzmHHgD4iXkejPc/MrkE7tlVdXiSG5O8KJNz7G9O\n8sdz7A0AujC3S+AOe/UvSHJeJgH/1iTPSvL+JAe6zj0AjDfX0+taa99K8pp59gAAPZvrbWoBgG1L\n0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANA\nxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9\nAHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM\n0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANA\nxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAx9bMuwFgZdpxxx1H\n1T//+c8fVf+a17xm6tpjjjlm1Ni77LLLqPp5qapR9Q8//PDUtaeffvqosc8555xR9avZ3Pboq2pd\nVbVFHnfOqy8A6Mm89+jvTXLWAvMfWOpGAKBH8w76e1pra+fcAwB0y8F4ANCxee/Rb19Vr07yjCQP\nJrk+yRWttR/Nty0A6MO8g373JOdvNO+bVfWa1toXN1dcVdcssmjf0Z0BQAfm+dH93yQ5PJOw3ynJ\nryT5cJI9k/xTVT1vfq0BQB/mtkffWnvXRrNuSPKGqnogyVuTrE1y9Gbe44CF5g97+vvPoE0AWNGW\n48F45w7Tl8y1CwDowHIM+u8P053m2gUAdGA5Bv2Bw/S2uXYBAB2YS9BX1S9W1U/tsVfVnkk+MDz9\n2FL2BAA9mtfBeP89yVur6ooktye5P8mzkvxWkh2SXJTkz+fUGwB0Y15Bf3mSZyd5fpKDM/k+/p4k\nV2VyXv35rbU2p94AoBtzCfrhYjibvSAOsGm77bbbqPozzjhj6tojjzxy1Nh77bXXqPoxt1wdux8x\nz/2QRx55ZOraJz7xiaPG3n777aeufcpTnjJqbKa3HA/GAwBmRNADQMcEPQB0TNADQMcEPQB0TNAD\nQMcEPQB0TNADQMcEPQB0TNADQMcEPQB0TNADQMcEPQB0TNADQMcEPQB0bC73o4ee/MIv/MKo+pNP\nPnnq2rH3hH/Oc54zqn6luuWWW0bVX3zxxVPXXnbZZaPGvuOOO6aufeELXzhq7Icffnjq2s9//vOj\nxmZ69ugBoGOCHgA6JugBoGOCHgA6JugBoGOCHgA6JugBoGOCHgA6JugBoGOCHgA6JugBoGOCHgA6\nJugBoGOCHgA65ja1kOSpT33q1LUf/OAHR4095lazrbVRY48x9lav//AP/zCq/u/+7u+mrr3++utH\njf3QQw+Nqp+Xa6+9dlT9mFsy33nnnaPGZnr26AGgY4IeADom6AGgY4IeADom6AGgY4IeADom6AGg\nY4IeADom6AGgY4IeADom6AGgY4IeADom6AGgY4IeADom6AGgY+5HD0lOOOGEqWuPOOKIGXaydR55\n5JFR9e9973unrj3zzDNHjf3YY4+Nqmfr/eEf/uGo+j/4gz+Yuvbcc88dNfbY3lcze/QA0DFBDwAd\nE/QA0DFBDwAdE/QA0DFBDwAdE/QA0DFBDwAdE/QA0DFBDwAdE/QA0DFBDwAdE/QA0DFBDwAdc5ta\nGKmq5lY/duyvfOUrU9e6zex0jjvuuFH1H//4x6euHbu9/PCHP5y69jOf+cyosZneTPboq+rYqjqn\nqq6sqvuqqlXVxzZTc1BVXVRVd1fVw1V1fVWdWlXbzaInAGB2e/TvSPK8JA8k+XaSfTf14qp6ZZJP\nJHkkyceT3J3kt5O8L8nBScb9ygsAJJndd/RvSbJPkp2TvHFTL6yqnZP8VZIfJTmktfba1trbkuyX\n5MtJjq2q42fUFwCsajMJ+tba5a21W1prbQtefmySn0tyQWvtqxu8xyOZfDKQbOaXBQBgy8zjqPvD\nhunFCyy7IslDSQ6qqu2XriUA6NM8gv7Zw/TmjRe01h5P8s1Mjh145lI2BQA9msfpdbsM03sXWb5+\n/q6be6OqumaRRZs8GBAAVgsXzAGAjs1jj379HvsuiyxfP/+ezb1Ra+2AheYPe/r7b31rANCXeezR\n3zRM99l4QVWtSbJXkseT3LaUTQFAj+YR9JcN0yMWWPaSJDsmubq19ujStQQAfZpH0F+Y5K4kx1fV\nC9bPrKodkrx7ePqhOfQFAN2ZyXf0VXVUkqOGp7sP01+rqvOGn+9qrZ2WJK21+6rqdzMJ/C9U1QWZ\nXAL3FZmcendhJpfFBQBGmtXBePslOWmjec/MT86Fvz3JaesXtNY+VVUvTfLHSY5JskOSW5P8fpKz\nt/AKewDAZswk6Ftra5Os3cqaLyX5zVmMDwAsrHrceXZ6HVvrZ3/2Z6euPfnkk0eN/Rd/8RdT1479\n9/vAAw9MXXvssceOGvtzn/vcqPoxdtxxx1H1p5xyytS17373uzf/ok3Ybrvp7+T9+OOPjxr7iCMW\nOoZ6y1x++eWjxl7FvrbYqeRbygVzAKBjgh4AOiboAaBjgh4AOiboAaBjgh4AOiboAaBjgh4AOibo\nAaBjgh4AOiboAaBjgh4AOiboAaBjgh4AOuY2tTBnp59++tS1f/ZnfzbDTrbOmFvcJslpp502qn7v\nvfeeuvbII48cNfZznvOcUfVj3H777VPXnnjiiaPGvuqqq0bVMxW3qQUAFifoAaBjgh4AOiboAaBj\ngh4AOiboAaBjgh4AOiboAaBjgh4AOiboAaBjgh4AOiboAaBjgh4AOiboAaBjgh4AOuZ+9DBnT3jC\nE6auPeuss0aN/YY3vGFU/TxV1dS1Y//fu//++6eu/frXvz5q7Fe/+tVT1952222jxmYu3I8eAFic\noAeAjgl6AOiYoAeAjgl6AOiYoAeAjgl6AOiYoAeAjgl6AOiYoAeAjgl6AOiYoAeAjgl6AOiYoAeA\njrlNLaxgBx100Kj6K6+8ckadLL0xt6m97rrrRo19+umnT117ySWXjBqbVcdtagGAxQl6AOiYoAeA\njgl6AOiYoAeAjgl6AOiYoAeAjgl6AOiYoAeAjgl6AOiYoAeAjgl6AOiYoAeAjgl6AOiYoAeAjq2Z\ndwOw2h188MFT1376058eNfaYe7rP25jejz/++FFj33TTTaPqYSnNZI++qo6tqnOq6sqquq+qWlV9\nbJHX7jksX+xxwSx6AgBmt0f/jiTPS/JAkm8n2XcLaq5L8qkF5t8wo54AYNWbVdC/JZOAvzXJS5Nc\nvgU117bW1s5ofABgATMJ+tbafwT7Sv7ODwB6M8+D8Z5eVa9P8uQkP0jy5dba9XPsBwC6M8+g/43h\n8R+q6gtJTmqt3bElb1BV1yyyaEuOEQCA7s3jPPqHkvxpkgOS7DY81n+vf0iSS6tqpzn0BQDdWfI9\n+tba95L8yUazr6iqlyW5KsmLkrwuyfu34L0OWGj+sKe//8hWAWDFWzZXxmutPZ7kI8PTl8yzFwDo\nxbIJ+sH3h6mP7gFgBpZb0B84TG+baxcA0IklD/qq2r+qfmrcqjo8kwvvJMmCl88FALbOTA7Gq6qj\nkhw1PN19mP5aVZ03/HxXa+204ee/TLJ3VV2dydX0kuS5SQ4bfn5na+3qWfQFAKvdrI663y/JSRvN\ne+bwSJLbk6wP+vOTHJ3khUmOTPKEJN9N8rdJPtBau3JGPQHAqjerS+CuTbJ2C1/70SQfncW4AMCm\nuR89JFmzZvp/Cr/zO78zauyzzjpr6tqdd9551NiPPvro1LUPPvjgqLF32223UfVjvOpVrxpVf+aZ\nZ86oE9j2lttR9wDADAl6AOiYoAeAjgl6AOiYoAeAjgl6AOiYoAeAjgl6AOiYoAeAjgl6AOiYoAeA\njgl6AOiYoAeAjgl6AOiY29RCkte//vVT15599tmjxq6qqWvH3GY2SU444YSpa2+44YZRY//rv/7r\nqPoxxt7eF1YSe/QA0DFBDwAdE/QA0DFBDwAdE/QA0DFBDwAdE/QA0DFBDwAdE/QA0DFBDwAdE/QA\n0DFBDwAdE/QA0DFBDwAdE/QA0DH3o6cLBx100Kj6M844Y0adbL3rrrtu6tqjjz561Njr1q2bunb3\n3XcfNfZ3v/vdUfVjx4fVwh49AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM\n0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAx9ymli687W1vG1W/6667zqiTrfemN71p6toxt5kda+3ataPq\nn/a0p82mEWCT7NEDQMcEPQB0TNADQMcEPQB0TNADQMcEPQB0TNADQMcEPQB0TNADQMcEPQB0TNAD\nQMcEPQB0TNADQMcEPQB0TNADQMfcj55l4ylPecrUta985Stn2MnW+ehHPzqq/qqrrppRJ1tvv/32\nm7r2xBNPHDV2VY2qv//++6euvfTSS0eNDSvJ6D36qnpyVb2uqj5ZVbdW1cNVdW9VXVVVr62qBceo\nqoOq6qKqunuoub6qTq2q7cb2BABMzGKP/rgkH0rynSSXJ7kjydOSvCrJR5IcWVXHtdba+oKqemWS\nTyR5JMnHk9yd5LeTvC/JwcN7AgAjzSLob07yiiSfaa39eP3MqvqjJP+S5JhMQv8Tw/ydk/xVkh8l\nOaS19tVh/juTXJbk2Ko6vrV2wQx6A4BVbfRH9621y1pr/7hhyA/z70xy7vD0kA0WHZvk55JcsD7k\nh9c/kuQdw9M3ju0LANj2R90/Nkwf32DeYcP04gVef0WSh5IcVFXbb8vGAGA12GZH3VfVmiTrD8vd\nMNSfPUxv3rimtfZ4VX0zyS8leWaSGzczxjWLLNp367oFgD5tyz369yT55SQXtdY+u8H8XYbpvYvU\nrZ+/67ZqDABWi22yR19Vb07y1iTfSHLCthgjSVprBywy/jVJ9t9W4wLASjHzPfqqOiXJ+5N8Pcmh\nrbW7N3rJ+j32XbKw9fPvmXVvALDazDToq+rUJOckuSGTkL9zgZfdNEz3WaB+TZK9Mjl477ZZ9gYA\nq9HMgr6qTs/kgjfXZhLy31vkpZcN0yMWWPaSJDsmubq19uisegOA1WomQT9c7OY9Sa5Jcnhr7a5N\nvPzCJHclOb6qXrDBe+yQ5N3D0w/Noi8AWO1GH4xXVSclOSOTK91dmeTNC9ysYl1r7bwkaa3dV1W/\nm0ngf6GqLsjkErivyOTUuwszuSwuADDSLI6632uYbpfk1EVe88Uk561/0lr7VFW9NMkfZ3KJ3B2S\n3Jrk95OcveF18QGA6Y0O+tba2iRrp6j7UpLfHDs+JMk8fzfcfvtxF3F80pOeNHXtr//6r48a+8Mf\n/vDUtWP/3GP/zt7+9rdPXfv5z39+1NiwkmzrS+ACAHMk6AGgY4IeADom6AGgY4IeADom6AGgY4Ie\nADom6AGgY4IeADom6AGgY4IeADom6AGgY4IeADom6AGgY4IeADpW87yP97ZSVdck2X/efbB1dt55\n56lrb7zxxlFj77777qPqx/jWt741de0ee+wxw06W1th7wr/85S+fUSewrH2ttXbAmDewRw8AHRP0\nANAxQQ8AHRP0ANAxQQ8AHRP0ANAxQQ8AHRP0ANAxQQ8AHRP0ANAxQQ8AHRP0ANAxQQ8AHRP0ANCx\nNfNuANa77777pq495ZRTRo199tlnT1379Kc/fdTYz3jGM6aunedtpr/61a+Oqn/jG984o06ATbFH\nDwAdE/QA0DFBDwAdE/QA0DFBDwAdE/QA0DFBDwAdE/QA0DFBDwAdE/QA0DFBDwAdE/QA0DFBDwAd\nE/QA0DFBDwAdcz96uvDJT35yVP1tt902de0pp5wyauwXv/jFU9fuscceo8Z+73vfO3XtmWeeOWrs\nxx57bFQ9sGXs0QNAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANA\nxwQ9AHRM0ANAxwQ9AHSsWmvz7mHmquqaJPvPuw8AGOlrrbUDxrzB6D36qnpyVb2uqj5ZVbdW1cNV\ndW9VXVVVr62qn9no9XtWVdvE44KxPQEAE2tm8B7HJflQku8kuTzJHUmeluRVST6S5MiqOq799EcH\n1yX51ALvd8MMegIAMpugvznJK5J8prX24/Uzq+qPkvxLkmMyCf1PbFR3bWtt7QzGBwAWMfqj+9ba\nZa21f9ww5If5dyY5d3h6yNhxAICtN4s9+k15bJg+vsCyp1fV65M8OckPkny5tXb9Nu4HAFaVbRb0\nVbUmyYnD04sXeMlvDI8Na76Q5KTW2h3bqi8AWE225R79e5L8cpKLWmuf3WD+Q0n+NJMD8W4b5j03\nydokhya5tKr2a609uLkBhtPoFrLvtE0DQE+2yXn0VfXmJO9P8o0kB7fW7t6CmjVJrkryoiSnttbe\nvwU1mwr6Hbe8YwBYlkafRz/zPfqqOiWTkP96ksO3JOSTpLX2eFV9JJOgf8nwHpurWfAP74I5ADAx\n00vgVtWpSc7J5Fz4Q4cj77fG94fpTrPsCwBWq5kFfVWdnuR9Sa7NJOS/N8XbHDhMb9vkqwCALTKT\noK+qd2Zy8N01mXxcf9cmXrv/xpfFHeYfnuQtw9OPzaIvAFjtRn9HX1UnJTkjyY+SXJnkzVW18cvW\ntdbOG37+yyR7V9XVSb49zHtuksOGn9/ZWrt6bF8AwGwOxttrmG6X5NRFXvPFJOcNP5+f5OgkL0xy\nZJInJPlukr9N8oHW2pUz6AkAiNvUAsByNv/b1AIAy5egB4COCXoA6JigB4COCXoA6JigB4COCXoA\n6JigB4COCXoA6JigB4COCXoA6JigB4COCXoA6JigB4COCXoA6JigB4COCXoA6JigB4COCXoA6Jig\nB4COCXoA6JigB4COCXoA6JigB4COCXoA6JigB4COCXoA6JigB4COCXoA6JigB4CO9Rr0e867AQCY\ngT3HvsGaGTSxHN03TNctsnzfYfqNbd9KN6yz6Vhv07Hetp51Np3lvN72zE/ybGrVWhvfygpTVdck\nSWvtgHn3slJYZ9Ox3qZjvW0962w6q2G99frRPQAQQQ8AXRP0ANAxQQ8AHRP0ANCxVXnUPQCsFvbo\nAaBjgh4AOiboAaBjgh4AOiboAaBjgh4AOiboAaBjqyroq+rnq+qvq+rfq+rRqlpXVWdV1W7z7m25\nGtZRW+Rx57z7m5eqOraqzqmqK6vqvmF9fGwzNQdV1UVVdXdVPVxV11fVqVW13VL1PW9bs96qas9N\nbHutqi5Y6v7noaqeXFWvq6pPVtWtw7Zzb1VdVVWvraoF/x9f7dvb1q63nre3Xu9H/1Oq6llJrk7y\n1CR/n8m9h381ye8lOaKqDm6t/WCOLS5n9yY5a4H5Dyx1I8vIO5I8L5N18O385J7WC6qqVyb5RJJH\nknw8yd1JfjvJ+5IcnOS4bdnsMrJV621wXZJPLTD/hhn2tZwdl+RDSb6T5PIkdyR5WpJXJflIkiOr\n6ri2wdXPbG9Jplhvg/62t9baqngk+WySluRNG83/y2H+ufPucTk+kqxLsm7efSy3R5JDk+ydpJIc\nMmxDH1vktTsn+V6SR5O8YIP5O2Tyy2dLcvy8/0zLcL3tOSw/b959z3mdHZZJSP/MRvN3zyS8WpJj\nNphve5tuvXW7va2Kj+6HvfmXZRJa/3ujxf8zyYNJTqiqnZa4NVao1trlrbVb2vA/xGYcm+TnklzQ\nWvvqBu/xSCZ7uEnyxm3Q5rKzleuNJK21y1pr/9ha+/FG8+9Mcu7w9JANFtneMtV669Zq+ej+0GF6\nyQJ/6fdX1Zcy+UXgwCSXLnVzK8D2VfXqJM/I5Jei65Nc0Vr70XzbWjEOG6YXL7DsiiQPJTmoqrZv\nrT26dG2tGE+vqtcneXKSHyT5cmvt+jn3tFw8Nkwf32Ce7W3zFlpv63W3va2WoH/2ML15keW3ZBL0\n+0TQL2T3JOdvNO+bVfWa1toX59HQCrPo9tdae7yqvpnkl5I8M8mNS9nYCvEbw+M/VNUXkpzUWrtj\nLh0tA1W1JsmJw9MNQ932tgmbWG/rdbe9rYqP7pPsMkzvXWT5+vm7LkEvK83fJDk8k7DfKcmvJPlw\nJt9n/VNVPW9+ra0Ytr/pPJTkT5MckGS34fHSTA6sOiTJpav867b3JPnlJBe11j67wXzb26Yttt66\n3d5WS9Azpdbau4bvur7bWnuotXZDa+0NmRzE+MQka+fbIb1qrX2vtfYnrbWvtdbuGR5XZPLp21eS\n/Nckr5tvl/NRVW9O8tZMzh46Yc7trBibWm89b2+rJejX/wa7yyLL18+/Zwl66cX6g1leMtcuVgbb\n3wy11h7P5PSoZBVuf1V1SpL3J/l6kkNba3dv9BLb2wK2YL0tqIftbbUE/U3DdJ9Flu89TBf7Dp+f\n9v1huiI/ylpii25/w/eFe2VyUNBtS9nUCrcqt7+qOjXJOZmc033ocAT5xmxvG9nC9bYpK3p7Wy1B\nf/kwfdkCV0N6UiYXkHgoyT8vdWMr2IHDdNX8ZzHCZcP0iAWWvSTJjkmuXsVHQE9j1W1/VXV6Jhe8\nuTaTsPreIi+1vW1gK9bbpqzo7W1VBH1r7d+SXJLJAWQnb7T4XZn8lnZ+a+3BJW5tWauqX1zo4JOq\n2jPJB4anm7zsK0mSC5PcleT4qnrB+plVtUOSdw9PPzSPxpazqtp/ocu7VtXhSd4yPF0V219VvTOT\ng8iuSXJ4a+2uTbzc9jbYmvXW8/ZWq+W6FQtcAvfGJC/K5Bz7m5Mc1FwC9z+pqrWZHLhyRZLbk9yf\n5FlJfiuTq2xdlOTo1toP59XjvFTVUUmOGp7unuTlmfy2f+Uw767W2mkbvf7CTC5JekEmlyR9RSan\nQl2Y5L+thovIbM16G05p2juTf7ffHpY/Nz85T/ydrbX1wdWtqjopyXlJfpTJx88LHU2/rrV23gY1\nq35729r11vX2Nu9L8y3lI8kemZwu9p0kP8wkvM5Kstu8e1uOj0xOLfk/mRyhek8mF5n4fpLPZXIe\nas27xzmum7WZXC5zsce6BWoOzuSXo/+X5OEk/zeTPYXt5v3nWY7rLclrk3w6kytaPpDJJV3vyOTa\n7S+e959lGa2zluQLtrdx663n7W3V7NEDwGq0Kr6jB4DVStADQMcEPQB0TNADQMcEPQB0TNADQMcE\nPQB0TNADQMcEPQB0TNADQMcEPQB0TNADQMcEPQB0TNADQMcEPQB0TNADQMcEPQB07P8DBRP/vtg3\nRjEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fcf0449d668>"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 250,
       "width": 253
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(X_train[777].reshape((28, 28)), cmap=plt.cm.gray)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_num_samples, test_num_samples = X_train.shape[0], X_test.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_history = defaultdict(list)\n",
    "test_history = defaultdict(list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def save_generated_samples(epoch, G):\n",
    "\n",
    "    # new batch of z's\n",
    "    zz = np.random.normal(0., 1., (100, Z_SIZE))\n",
    "\n",
    "    # new batch of classes [0..9]\n",
    "    generated_classes = np.array(list(range(0, 10)) * 10, dtype=np.int32)\n",
    "\n",
    "    generated_images = G.predict([zz, generated_classes.reshape(-1, 1)])\n",
    "\n",
    "    rr = []\n",
    "    for c in range(10):\n",
    "        rr.append(\n",
    "            np.concatenate(generated_images[c * 10:(1 + c) * 10]).reshape(\n",
    "                280, 28))\n",
    "    img = np.hstack(rr)\n",
    "\n",
    "    plt.imsave('out/3b/generated-e%d.png' % epoch, img, cmap=plt.cm.gray)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 1 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 2.0345 - D(G)_loss: 2.1246\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[1.4295319318771362, 0.007126178592443466, 1.4224066734313965]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[1.1916042566299438, -0.14951910078525543, 1.341122031211853]\n",
      "\n",
      "Epoch 2 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 0.3197 - D(G)_loss: -0.9730\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[0.2895541489124298, 0.02179291844367981, 0.26776131987571716]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[-0.9964799880981445, -1.0358372926712036, 0.03935704007744789]\n",
      "\n",
      "Epoch 3 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 0.2420 - D(G)_loss: -1.1303\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[0.2277207374572754, 0.02510138787329197, 0.20261940360069275]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[-1.2031484842300415, -1.2249082326889038, 0.02175978571176529]\n",
      "\n",
      "Epoch 4 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 0.1985 - D(G)_loss: -1.3284\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[0.1903405636548996, 0.022692296653985977, 0.1676482856273651]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[-1.351906180381775, -1.369633674621582, 0.01772717945277691]\n",
      "\n",
      "Epoch 5 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 0.1744 - D(G)_loss: -1.4190\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[0.16386793553829193, 0.02202387899160385, 0.14184384047985077]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[-1.4457952976226807, -1.4603002071380615, 0.014505265280604362]\n",
      "\n",
      "Epoch 6 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 0.1497 - D(G)_loss: -1.4317\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[0.14311623573303223, 0.023205004632472992, 0.11991121619939804]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[-1.4254313707351685, -1.4386833906173706, 0.01325148344039917]\n",
      "\n",
      "Epoch 7 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 0.1314 - D(G)_loss: -1.3282\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[0.12416395545005798, 0.021082395687699318, 0.1030815914273262]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[-1.3116395473480225, -1.324163556098938, 0.01252397708594799]\n",
      "\n",
      "Epoch 8 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 0.1163 - D(G)_loss: -1.3642\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[0.11080408841371536, 0.01979309692978859, 0.09101099520921707]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[-1.362634539604187, -1.3742790222167969, 0.011644350364804268]\n",
      "\n",
      "Epoch 9 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 0.1089 - D(G)_loss: -1.4112\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[0.1047215610742569, 0.021085171028971672, 0.08363644033670425]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[-1.3947144746780396, -1.4052934646606445, 0.010579311288893223]\n",
      "\n",
      "Epoch 10 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 0.1002 - D(G)_loss: -1.4052\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[0.09708385914564133, 0.019626904278993607, 0.07745696604251862]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[-1.423093557357788, -1.4335596561431885, 0.010466351173818111]\n",
      "\n",
      "Epoch 11 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 0.0897 - D(G)_loss: -1.4525\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[0.09015941619873047, 0.018024856224656105, 0.07213465869426727]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[-1.4390769004821777, -1.4497100114822388, 0.010632652789354324]\n",
      "\n",
      "Epoch 12 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 0.0867 - D(G)_loss: -1.4937\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[0.08631152659654617, 0.018165769055485725, 0.06814570724964142]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[-1.491133213043213, -1.501743197441101, 0.010610190220177174]\n",
      "\n",
      "Epoch 13 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 0.0819 - D(G)_loss: -1.4956\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[0.08112640678882599, 0.015959858894348145, 0.06516655534505844]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[-1.526281714439392, -1.5361688137054443, 0.009886905550956726]\n",
      "\n",
      "Epoch 14 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 0.0824 - D(G)_loss: -1.5684\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[0.07946400344371796, 0.016786841675639153, 0.06267714500427246]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[-1.6114895343780518, -1.6210700273513794, 0.009580316953361034]\n",
      "\n",
      "Epoch 15 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 0.0771 - D(G)_loss: -1.6606\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[0.07647522538900375, 0.01725499890744686, 0.05922023952007294]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[-1.672945261001587, -1.6826616525650024, 0.009717349894344807]\n",
      "\n",
      "Epoch 16 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 0.0670 - D(G)_loss: -2.0327\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[0.06483405828475952, 0.01636074110865593, 0.04847333952784538]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[-2.102656602859497, -2.1122970581054688, 0.009639644995331764]\n",
      "\n",
      "Epoch 23 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 0.0557 - D(G)_loss: -2.4991\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[0.055296674370765686, 0.014965793117880821, 0.040330883115530014]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[-2.5729787349700928, -2.5807578563690186, 0.0077809556387364864]\n",
      "\n",
      "Epoch 35 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 0.0529 - D(G)_loss: -2.6307\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[0.05384095385670662, 0.013897421769797802, 0.039943497627973557]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[-2.710481882095337, -2.718869924545288, 0.008388523943722248]\n",
      "\n",
      "Epoch 36 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 0.0585 - D(G)_loss: -2.7697\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[0.0560920387506485, 0.016524039208889008, 0.03956795856356621]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[-2.787222385406494, -2.7955551147460938, 0.008332562632858753]\n",
      "\n",
      "Epoch 37 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 0.0543 - D(G)_loss: -2.7803\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[0.05340272933244705, 0.014148402027785778, 0.039254333823919296]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[-2.7528059482574463, -2.7616002559661865, 0.008795344270765781]\n",
      "\n",
      "Epoch 38 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 0.0524 - D(G)_loss: -2.8259\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[0.05302724614739418, 0.014955083839595318, 0.038072146475315094]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[-2.8587515354156494, -2.8669991493225098, 0.008245622739195824]\n",
      "\n",
      "Epoch 39 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 0.0507 - D(G)_loss: -2.7792\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[0.04981032758951187, 0.01220142375677824, 0.03760889917612076]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[-2.7829856872558594, -2.790847063064575, 0.007861158810555935]\n",
      "\n",
      "Epoch 40 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 0.0524 - D(G)_loss: -2.8386\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[0.05366689711809158, 0.016249196603894234, 0.037417683750391006]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[-2.828339099884033, -2.836601734161377, 0.008265404962003231]\n",
      "\n",
      "Epoch 41 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 0.0525 - D(G)_loss: -2.7590\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[0.052686214447021484, 0.015898875892162323, 0.03678734600543976]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[-2.784984827041626, -2.793124198913574, 0.008140106685459614]\n",
      "\n",
      "Epoch 42 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 0.0502 - D(G)_loss: -2.8257\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[0.05159994587302208, 0.014780217781662941, 0.036819733679294586]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[-2.850041627883911, -2.8573296070098877, 0.0072892215102910995]\n",
      "\n",
      "Epoch 43 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 0.0502 - D(G)_loss: -2.8971\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[0.04876039922237396, 0.012949571013450623, 0.03581084683537483]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[-2.895810842514038, -2.903877019882202, 0.008066510781645775]\n",
      "\n",
      "Epoch 44 / 50\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "599/600 [============================>.] - ETA: 0s - D_loss: 0.0534 - D(G)_loss: -2.9033\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[0.05069008097052574, 0.014183834195137024, 0.036506250500679016]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[-2.953293561935425, -2.960998058319092, 0.007704651448875666]\n",
      "\n",
      "Epoch 45 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 0.0485 - D(G)_loss: -2.8524\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[0.048887740820646286, 0.01393364928662777, 0.03495410457253456]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[-2.836778163909912, -2.844243049621582, 0.007463573478162289]\n",
      "\n",
      "Epoch 46 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 0.0533 - D(G)_loss: -2.8249\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[0.05112990364432335, 0.015551096759736538, 0.03557882085442543]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[-2.920527696609497, -2.9281723499298096, 0.007643483579158783]\n",
      "\n",
      "Epoch 47 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 0.0493 - D(G)_loss: -2.9619\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[0.04827269911766052, 0.013274604454636574, 0.03499805927276611]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[-2.9625837802886963, -2.9704604148864746, 0.007877292111515999]\n",
      "\n",
      "Epoch 48 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 0.0493 - D(G)_loss: -3.0037\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[0.048888325691223145, 0.014086763374507427, 0.03480154275894165]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[-3.0110976696014404, -3.018357276916504, 0.007258192636072636]\n",
      "\n",
      "Epoch 49 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 0.0529 - D(G)_loss: -2.8143\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[0.05034257099032402, 0.015547797083854675, 0.034794822335243225]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[-2.8083715438842773, -2.8161604404449463, 0.007790946867316961]\n",
      "\n",
      "Epoch 50 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 0.0462 - D(G)_loss: -2.9388\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[0.046343132853507996, 0.01200808584690094, 0.034335024654865265]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[-2.976073741912842, -2.9840409755706787, 0.007966369390487671]\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(EPOCHS):\n",
    "\n",
    "    print('\\nEpoch {} / {}'.format(epoch + 1, EPOCHS))\n",
    "\n",
    "    batches_num = int(train_num_samples / BATCH_SIZE)\n",
    "    progress_bar = Progbar(target=batches_num)\n",
    "\n",
    "    epoch_combined_losses = []\n",
    "    epoch_D_losses = []\n",
    "\n",
    "    for batch_i in range(batches_num):\n",
    "\n",
    "        if len(epoch_D_losses) + len(epoch_combined_losses) > 1:\n",
    "            progress_bar.update(\n",
    "                batch_i,\n",
    "                values=[\n",
    "                        ('D_loss', np.mean(np.array(epoch_D_losses), axis=0)[0]),\n",
    "                        ('D(G)_loss', np.mean(np.array(epoch_combined_losses),axis=0)[0])\n",
    "                ]\n",
    "            )\n",
    "        else:\n",
    "            progress_bar.update(batch_i)\n",
    "            \n",
    "        # 1: train D on real+generated images\n",
    "            \n",
    "        # create batches of images and labels\n",
    "        real_images = X_train[batch_i * BATCH_SIZE:(batch_i + 1) * BATCH_SIZE]\n",
    "        real_images_classes = y_train[batch_i * BATCH_SIZE:(batch_i + 1) * BATCH_SIZE]\n",
    "        \n",
    "        # new batch of z's\n",
    "        zz = np.random.normal(0., 1., (BATCH_SIZE, Z_SIZE))\n",
    "        \n",
    "        # new batch of classes [0..9]\n",
    "        generated_classes = np.random.randint(0, 10, BATCH_SIZE)\n",
    "        \n",
    "        generated_images = G.predict([zz, generated_classes.reshape(-1, 1)])\n",
    "        \n",
    "        X = np.concatenate((real_images, generated_images))\n",
    "        y_is_fake = np.array([-1] * BATCH_SIZE + [1] * BATCH_SIZE) # [is_fake=-1xBATCH_SIZE, 1xBATCH_SIZE]\n",
    "        y_classes = np.concatenate((real_images_classes, generated_classes))\n",
    "        \n",
    "        # train \n",
    "        D_losses = D.train_on_batch(X, [y_is_fake, y_classes])\n",
    "        epoch_D_losses.append(D_losses)\n",
    "        \n",
    "        \n",
    "        # 2: train D(G) (only G is trainable)\n",
    "        \n",
    "        # 2x so G is trained for the same number of images\n",
    "        # as D above (which is trained on real+generated)\n",
    "        zz = np.random.normal(0., 1., (2 * BATCH_SIZE, Z_SIZE)) \n",
    "        generated_classes = np.random.randint(0, 10, 2 * BATCH_SIZE)\n",
    "\n",
    "        y_is_fake = -np.ones(2 * BATCH_SIZE) # all not-fake\n",
    "\n",
    "        combined_losses = combined.train_on_batch(\n",
    "            [zz, generated_classes.reshape((-1, 1))],\n",
    "            [y_is_fake, generated_classes])\n",
    "        \n",
    "        epoch_combined_losses.append(combined_losses)\n",
    "        \n",
    "    print('\\nD:', D.metrics_names)\n",
    "    print(np.mean(np.array(epoch_D_losses), axis=0).tolist())\n",
    "    \n",
    "    print('D(G):', combined.metrics_names)\n",
    "    print(np.mean(np.array(epoch_combined_losses), axis=0).tolist())\n",
    "    \n",
    "    save_generated_samples(epoch, G)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "save_generated_samples(101, G)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "gist": {
   "data": {
    "description": "Keras  WACGAN",
    "public": false
   },
   "id": ""
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}