training_caption_generator.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1075ccf200ab40c4ac1da1802189df6e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import string\n",
    "import numpy as np\n",
    "from PIL import Image\n",
    "import os\n",
    "from pickle import dump, load\n",
    "import numpy as np\n",
    "\n",
    "from keras.applications.xception import Xception, preprocess_input\n",
    "from keras.preprocessing.image import load_img, img_to_array\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.utils import to_categorical\n",
    "from keras.layers.merge import add\n",
    "from keras.models import Model, load_model\n",
    "from keras.layers import Input, Dense, LSTM, Embedding, Dropout\n",
    "\n",
    "# small library for seeing the progress of loops.\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "tqdm().pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loading a text file into memory\n",
    "def load_doc(filename):\n",
    "    # Opening the file as read only\n",
    "    file = open(filename, 'r')\n",
    "    text = file.read()\n",
    "    file.close()\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get all imgs with their captions\n",
    "def all_img_captions(filename):\n",
    "    file = load_doc(filename)\n",
    "    captions = file.split('\\n')\n",
    "    descriptions ={}\n",
    "    for caption in captions[:-1]:\n",
    "        img, caption = caption.split('\\t')\n",
    "        if img[:-2] not in descriptions:\n",
    "            descriptions[img[:-2]] = [caption]\n",
    "        else:\n",
    "            descriptions[img[:-2]].append(caption)\n",
    "    return descriptions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##Data cleaning- lower casing, removing puntuations and words containing numbers\n",
    "def cleaning_text(captions):\n",
    "    table = str.maketrans('','',string.punctuation)\n",
    "    for img,caps in captions.items():\n",
    "        for i,img_caption in enumerate(caps):\n",
    "\n",
    "            img_caption.replace(\"-\",\" \")\n",
    "            desc = img_caption.split()\n",
    "\n",
    "            #converts to lower case\n",
    "            desc = [word.lower() for word in desc]\n",
    "            #remove punctuation from each token\n",
    "            desc = [word.translate(table) for word in desc]\n",
    "            #remove hanging 's and a \n",
    "            desc = [word for word in desc if(len(word)>1)]\n",
    "            #remove tokens with numbers in them\n",
    "            desc = [word for word in desc if(word.isalpha())]\n",
    "            #convert back to string\n",
    "\n",
    "            img_caption = ' '.join(desc)\n",
    "            captions[img][i]= img_caption\n",
    "    return captions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def text_vocabulary(descriptions):\n",
    "    # build vocabulary of all unique words\n",
    "    vocab = set()\n",
    "    \n",
    "    for key in descriptions.keys():\n",
    "        [vocab.update(d.split()) for d in descriptions[key]]\n",
    "    \n",
    "    return vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "#All descriptions in one file \n",
    "def save_descriptions(descriptions, filename):\n",
    "    lines = list()\n",
    "    for key, desc_list in descriptions.items():\n",
    "        for desc in desc_list:\n",
    "            lines.append(key + '\\t' + desc )\n",
    "    data = \"\\n\".join(lines)\n",
    "    file = open(filename,\"w\")\n",
    "    file.write(data)\n",
    "    file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# all_train_captions = []\n",
    "# for key, val in descriptions.items():\n",
    "#     for cap in val:\n",
    "#         all_train_captions.append(cap)\n",
    "\n",
    "# # Consider only words which occur at least 8 times in the corpus\n",
    "# word_count_threshold = 8\n",
    "# word_counts = {}\n",
    "# nsents = 0\n",
    "# for sent in all_train_captions:\n",
    "#     nsents += 1\n",
    "#     for w in sent.split(' '):\n",
    "#         word_counts[w] = word_counts.get(w, 0) + 1\n",
    "\n",
    "# vocab = [w for w in word_counts if word_counts[w] >= word_count_threshold]\n",
    "\n",
    "# print('preprocessed words %d ' % len(vocab))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_text = \"D:\\dataflair projects\\Project - Image Caption Generator\\Flickr_8k_text\"\n",
    "dataset_images = \"D:\\dataflair projects\\Project - Image Caption Generator\\Flicker8k_Dataset\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Length of descriptions = 8092\n",
      "Length of vocabulary =  8763\n"
     ]
    }
   ],
   "source": [
    "#we prepare our text data\n",
    "filename = dataset_text + \"/\" + \"Flickr8k.token.txt\"\n",
    "#loading the file that contains all data\n",
    "#mapping them into descriptions dictionary img to 5 captions\n",
    "descriptions = all_img_captions(filename)\n",
    "print(\"Length of descriptions =\" ,len(descriptions))\n",
    "\n",
    "#cleaning the descriptions\n",
    "clean_descriptions = cleaning_text(descriptions)\n",
    "\n",
    "#building vocabulary \n",
    "vocabulary = text_vocabulary(clean_descriptions)\n",
    "print(\"Length of vocabulary = \", len(vocabulary))\n",
    "\n",
    "#saving each description to file \n",
    "save_descriptions(clean_descriptions, \"descriptions.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def extract_features(directory):\n",
    "        model = Xception( include_top=False, pooling='avg' )\n",
    "        features = {}\n",
    "        for img in tqdm(os.listdir(directory)):\n",
    "            filename = directory + \"/\" + img\n",
    "            image = Image.open(filename)\n",
    "            image = image.resize((299,299))\n",
    "            image = np.expand_dims(image, axis=0)\n",
    "            #image = preprocess_input(image)\n",
    "            image = image/127.5\n",
    "            image = image - 1.0\n",
    "            \n",
    "            feature = model.predict(image)\n",
    "            features[img] = feature\n",
    "        return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7105a40519eb4b6b9a55a8b0cdd67f39",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=8091), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-29-c323c1b51dbd>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m#2048 feature vector\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mfeatures\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mextract_features\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdataset_images\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[0mdump\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"features.p\"\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m\"wb\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-28-8ced593b6a79>\u001b[0m in \u001b[0;36mextract_features\u001b[1;34m(directory)\u001b[0m\n\u001b[0;32m     11\u001b[0m             \u001b[0mimage\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mimage\u001b[0m \u001b[1;33m-\u001b[0m \u001b[1;36m1.0\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     12\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 13\u001b[1;33m             \u001b[0mfeature\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mimage\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     14\u001b[0m             \u001b[0mfeatures\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mimg\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfeature\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     15\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mfeatures\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\asus4\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mpredict\u001b[1;34m(self, x, batch_size, verbose, steps, callbacks, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[0;32m   1460\u001b[0m                                             \u001b[0mverbose\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mverbose\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1461\u001b[0m                                             \u001b[0msteps\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msteps\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1462\u001b[1;33m                                             callbacks=callbacks)\n\u001b[0m\u001b[0;32m   1463\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1464\u001b[0m     def train_on_batch(self, x, y,\n",
      "\u001b[1;32mc:\\users\\asus4\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\keras\\engine\\training_arrays.py\u001b[0m in \u001b[0;36mpredict_loop\u001b[1;34m(model, f, ins, batch_size, verbose, steps, callbacks)\u001b[0m\n\u001b[0;32m    322\u001b[0m             \u001b[0mbatch_logs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m{\u001b[0m\u001b[1;34m'batch'\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mbatch_index\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'size'\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch_ids\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    323\u001b[0m             \u001b[0mcallbacks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_call_batch_hook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'predict'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'begin'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_index\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_logs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 324\u001b[1;33m             \u001b[0mbatch_outs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    325\u001b[0m             \u001b[0mbatch_outs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mto_list\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    326\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mbatch_index\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\asus4\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\tensorflow_core\\python\\keras\\backend.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, inputs)\u001b[0m\n\u001b[0;32m   3738\u001b[0m         \u001b[0mvalue\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmath_ops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcast\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtensor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3739\u001b[0m       \u001b[0mconverted_inputs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3740\u001b[1;33m     \u001b[0moutputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_graph_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mconverted_inputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   3741\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3742\u001b[0m     \u001b[1;31m# EagerTensor.numpy() will often make a copy to ensure memory safety.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\asus4\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\tensorflow_core\\python\\eager\\function.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1079\u001b[0m       \u001b[0mTypeError\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mFor\u001b[0m \u001b[0minvalid\u001b[0m \u001b[0mpositional\u001b[0m\u001b[1;33m/\u001b[0m\u001b[0mkeyword\u001b[0m \u001b[0margument\u001b[0m \u001b[0mcombinations\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1080\u001b[0m     \"\"\"\n\u001b[1;32m-> 1081\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_call_impl\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1082\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1083\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_call_impl\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcancellation_manager\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\asus4\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\tensorflow_core\\python\\eager\\function.py\u001b[0m in \u001b[0;36m_call_impl\u001b[1;34m(self, args, kwargs, cancellation_manager)\u001b[0m\n\u001b[0;32m   1119\u001b[0m       raise TypeError(\"Keyword arguments {} unknown. Expected {}.\".format(\n\u001b[0;32m   1120\u001b[0m           list(kwargs.keys()), list(self._arg_keywords)))\n\u001b[1;32m-> 1121\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_call_flat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcaptured_inputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcancellation_manager\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1122\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1123\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_filtered_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\asus4\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\tensorflow_core\\python\\eager\\function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[1;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[0;32m   1222\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mexecuting_eagerly\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1223\u001b[0m       flat_outputs = forward_function.call(\n\u001b[1;32m-> 1224\u001b[1;33m           ctx, args, cancellation_manager=cancellation_manager)\n\u001b[0m\u001b[0;32m   1225\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1226\u001b[0m       \u001b[0mgradient_name\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_delayed_rewrite_functions\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mregister\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\asus4\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\tensorflow_core\\python\\eager\\function.py\u001b[0m in \u001b[0;36mcall\u001b[1;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[0;32m    509\u001b[0m               \u001b[0minputs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    510\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"executor_type\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mexecutor_type\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"config_proto\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mconfig\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 511\u001b[1;33m               ctx=ctx)\n\u001b[0m\u001b[0;32m    512\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    513\u001b[0m           outputs = execute.execute_with_cancellation(\n",
      "\u001b[1;32mc:\\users\\asus4\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\tensorflow_core\\python\\eager\\execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[1;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[0;32m     59\u001b[0m     tensors = pywrap_tensorflow.TFE_Py_Execute(ctx._handle, device_name,\n\u001b[0;32m     60\u001b[0m                                                \u001b[0mop_name\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mattrs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 61\u001b[1;33m                                                num_outputs)\n\u001b[0m\u001b[0;32m     62\u001b[0m   \u001b[1;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     63\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "#2048 feature vector\n",
    "features = extract_features(dataset_images)\n",
    "dump(features, open(\"features.p\",\"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "features = load(open(\"features.p\",\"rb\"))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#load the data \n",
    "def load_photos(filename):\n",
    "    file = load_doc(filename)\n",
    "    photos = file.split(\"\\n\")[:-1]\n",
    "    return photos\n",
    "\n",
    "\n",
    "def load_clean_descriptions(filename, photos):   \n",
    "    #loading clean_descriptions\n",
    "    file = load_doc(filename)\n",
    "    descriptions = {}\n",
    "    for line in file.split(\"\\n\"):\n",
    "        \n",
    "        words = line.split()\n",
    "        if len(words)<1 :\n",
    "            continue\n",
    "    \n",
    "        image, image_caption = words[0], words[1:]\n",
    "        \n",
    "        if image in photos:\n",
    "            if image not in descriptions:\n",
    "                descriptions[image] = []\n",
    "            desc = '<start> ' + \" \".join(image_caption) + ' <end>'\n",
    "            descriptions[image].append(desc)\n",
    "\n",
    "    return descriptions\n",
    "\n",
    "\n",
    "def load_features(photos):\n",
    "    #loading all features\n",
    "    all_features = load(open(\"features.p\",\"rb\"))\n",
    "    #selecting only needed features\n",
    "    features = {k:all_features[k] for k in photos}\n",
    "    return features\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = dataset_text + \"/\" + \"Flickr_8k.trainImages.txt\"\n",
    "\n",
    "#train = loading_data(filename)\n",
    "train_imgs = load_photos(filename)\n",
    "train_descriptions = load_clean_descriptions(\"descriptions.txt\", train_imgs)\n",
    "train_features = load_features(train_imgs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "#converting dictionary to clean list of descriptions\n",
    "def dict_to_list(descriptions):\n",
    "    all_desc = []\n",
    "    for key in descriptions.keys():\n",
    "        [all_desc.append(d) for d in descriptions[key]]\n",
    "    return all_desc\n",
    "\n",
    "#creating tokenizer class \n",
    "#this will vectorise text corpus\n",
    "#each integer will represent token in dictionary \n",
    "\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "\n",
    "def create_tokenizer(descriptions):\n",
    "    desc_list = dict_to_list(descriptions)\n",
    "    tokenizer = Tokenizer()\n",
    "    tokenizer.fit_on_texts(desc_list)\n",
    "    return tokenizer\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7577"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# give each word a index, and store that into tokenizer.p pickle file\n",
    "tokenizer = create_tokenizer(train_descriptions)\n",
    "dump(tokenizer, open('tokenizer.p', 'wb'))\n",
    "vocab_size = len(tokenizer.word_index) + 1\n",
    "vocab_size "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "32"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#calculate maximum length of descriptions\n",
    "def max_length(descriptions):\n",
    "    desc_list = dict_to_list(descriptions)\n",
    "    return max(len(d.split()) for d in desc_list)\n",
    "\n",
    "max_length = max_length(descriptions)\n",
    "max_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.36452794, 0.12713662, 0.0013574 , ..., 0.221817  , 0.01178991,\n",
       "       0.24176797], dtype=float32)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features['1000268201_693b08cb0e.jpg'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the model\n",
    "\n",
    "#1 Photo feature extractor - we extracted features from pretrained model Xception. \n",
    "#2 Sequence processor - word embedding layer that handles text, followed by LSTM \n",
    "#3 Decoder - Both 1 and 2 model produce fixed length vector. They are merged together and processed by dense layer to make final prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "#create input-output sequence pairs from the image description.\n",
    "\n",
    "#data generator, used by model.fit_generator()\n",
    "def data_generator(descriptions, features, tokenizer, max_length):\n",
    "    while 1:\n",
    "        for key, description_list in descriptions.items():\n",
    "            #retrieve photo features\n",
    "            feature = features[key][0]\n",
    "            input_image, input_sequence, output_word = create_sequences(tokenizer, max_length, description_list, feature)\n",
    "            yield [[input_image, input_sequence], output_word]         \n",
    "\n",
    "def create_sequences(tokenizer, max_length, desc_list, feature):\n",
    "    X1, X2, y = list(), list(), list()\n",
    "    # walk through each description for the image\n",
    "    for desc in desc_list:\n",
    "        # encode the sequence\n",
    "        seq = tokenizer.texts_to_sequences([desc])[0]\n",
    "        # split one sequence into multiple X,y pairs\n",
    "        for i in range(1, len(seq)):\n",
    "            # split into input and output pair\n",
    "            in_seq, out_seq = seq[:i], seq[i]\n",
    "            # pad input sequence\n",
    "            in_seq = pad_sequences([in_seq], maxlen=max_length)[0]\n",
    "            # encode output sequence\n",
    "            out_seq = to_categorical([out_seq], num_classes=vocab_size)[0]\n",
    "            # store\n",
    "            X1.append(feature)\n",
    "            X2.append(in_seq)\n",
    "            y.append(out_seq)\n",
    "    return np.array(X1), np.array(X2), np.array(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((47, 2048), (47, 32), (47, 7577))"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[a,b],c = next(data_generator(train_descriptions, features, tokenizer, max_length))\n",
    "a.shape, b.shape, c.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.utils import plot_model\n",
    "\n",
    "# define the captioning model\n",
    "def define_model(vocab_size, max_length):\n",
    "    \n",
    "    # features from the CNN model squeezed from 2048 to 256 nodes\n",
    "    inputs1 = Input(shape=(2048,))\n",
    "    fe1 = Dropout(0.5)(inputs1)\n",
    "    fe2 = Dense(256, activation='relu')(fe1)\n",
    "\n",
    "    # LSTM sequence model\n",
    "    inputs2 = Input(shape=(max_length,))\n",
    "    se1 = Embedding(vocab_size, 256, mask_zero=True)(inputs2)\n",
    "    se2 = Dropout(0.5)(se1)\n",
    "    se3 = LSTM(256)(se2)\n",
    "\n",
    "    # Merging both models\n",
    "    decoder1 = add([fe2, se3])\n",
    "    decoder2 = Dense(256, activation='relu')(decoder1)\n",
    "    outputs = Dense(vocab_size, activation='softmax')(decoder2)\n",
    "    \n",
    "    # tie it together [image, seq] [word]\n",
    "    model = Model(inputs=[inputs1, inputs2], outputs=outputs)\n",
    "    model.compile(loss='categorical_crossentropy', optimizer='adam')\n",
    "    \n",
    "    # summarize model\n",
    "    print(model.summary())\n",
    "    plot_model(model, to_file='model.png', show_shapes=True)\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset:  6000\n",
      "Descriptions: train= 6000\n",
      "Photos: train= 6000\n",
      "Vocabulary Size: 7577\n",
      "Description Length:  32\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_2 (InputLayer)            (None, 32)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_1 (InputLayer)            (None, 2048)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, 32, 256)      1939712     input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 2048)         0           input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)             (None, 32, 256)      0           embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 256)          524544      dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lstm_1 (LSTM)                   (None, 256)          525312      dropout_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_1 (Add)                     (None, 256)          0           dense_1[0][0]                    \n",
      "                                                                 lstm_1[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 256)          65792       add_1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 7577)         1947289     dense_2[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 5,002,649\n",
      "Trainable params: 5,002,649\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "# train our model\n",
    "print('Dataset: ', len(train_imgs))\n",
    "print('Descriptions: train=', len(train_descriptions))\n",
    "print('Photos: train=', len(train_features))\n",
    "print('Vocabulary Size:', vocab_size)\n",
    "print('Description Length: ', max_length)\n",
    "\n",
    "model = define_model(vocab_size, max_length)\n",
    "epochs = 10\n",
    "steps = len(train_descriptions)\n",
    "# making a directory models to save our models\n",
    "os.mkdir(\"models\")\n",
    "for i in range(epochs):\n",
    "    generator = data_generator(train_descriptions, train_features, tokenizer, max_length)\n",
    "    model.fit_generator(generator, epochs=1, steps_per_epoch= steps, verbose=1)\n",
    "    model.save(\"models/model_\" + str(i) + \".h5\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}