prepare_flickr8k_model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare model for Flickr8k\n",
    "\n",
    "Here we will prepare model to run caption generation for the paper \"Show Attend and Tell\". \n",
    "\n",
    "This is for the Flickr8k dataset. \n",
    "\n",
    "### Prerequisite \n",
    " - The data is available from \n",
    "     - Images http://nlp.cs.illinois.edu/HockenmaierGroup/Framing_Image_Description/Flickr8k_Dataset.zip\n",
    "     - Texts http://nlp.cs.illinois.edu/HockenmaierGroup/Framing_Image_Description/Flickr8k_text.zip"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load and Preprocess images\n",
    "\n",
    "Download data and downsize so that large side is 256.  And center crop by 224 by 224.  \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Initial Environment setup \n",
    "\n",
    "originalImagesPath = '/home/intuinno/project/data/Flickr8k/originalImages'\n",
    "preprocessedImagesPath = '/home/intuinno/project/data/Flickr8k/preprocessedImage'\n",
    "\n",
    "caffe_root = '/home/intuinno/codegit/caffe/'\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "import sys\n",
    "sys.path.insert(0, caffe_root + 'python')\n",
    "\n",
    "import caffe\n",
    "\n",
    "plt.rcParams['figure.figsize'] = (10, 10)\n",
    "plt.rcParams['image.interpolation'] = 'nearest'\n",
    "plt.rcParams['image.cmap'] = 'gray'\n",
    "\n",
    "import os\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "import sys\n",
    "sys.path.insert(0, caffe_root + 'python')\n",
    "\n",
    "import caffe\n",
    "\n",
    "plt.rcParams['figure.figsize'] = (10, 10)\n",
    "plt.rcParams['image.interpolation'] = 'nearest'\n",
    "plt.rcParams['image.cmap'] = 'gray'\n",
    "\n",
    "import os\n",
    "# if not os.path.isfile(caffe_root + 'models/bvlc_reference_caffenet/bvlc_reference_caffenet.caffemodel'):\n",
    "#     print(\"Downloading pre-trained CaffeNet model...\")\n",
    "#     !../scripts/download_model_binary.py ../models/bvlc_reference_caffenet"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Set caffe to use GPU.  And we will use vgg_il"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "caffe.set_device(0)\n",
    "caffe.set_mode_gpu()\n",
    "\n",
    "net = caffe.Net(caffe_root + 'models/vgg_ilsvrc_19/VGG_ILSVRC_19_layers_deploy.prototxt',\n",
    "                caffe_root + 'models/vgg_ilsvrc_19/VGG_ILSVRC_19_layers.caffemodel',\n",
    "                caffe.TEST)\n",
    "\n",
    "# input preprocessing: 'data' is the name of the input blob == net.inputs[0]\n",
    "transformer = caffe.io.Transformer({'data': net.blobs['data'].data.shape})\n",
    "transformer.set_transpose('data', (2,0,1))\n",
    "# transformer.set_mean('data', np.load(caffe_root + 'python/caffe/imagenet/ilsvrc_2012_mean.npy').mean(1).mean(1)) # mean pixel\n",
    "# transformer.set_raw_scale('data', 255)  # the reference model operates on images in [0,255] range instead of [0,1]\n",
    "# transformer.set_channel_swap('data', (2,1,0))  # the reference model has channels in BGR order instead of RGB"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's start with a simple classification. We'll set a batch of 50 to demonstrate batch processing, even though we'll only be classifying one image. (Note that the batch size can also be changed on-the-fly.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# set net to batch size of 50\n",
    "# net.blobs['data'].reshape(5,3,224,224)\n",
    "\n",
    "# resultingFeatures = \n",
    "\n",
    "# files = ['test', 'train', 'dev']\n",
    "files = [ 'train', 'dev']\n",
    "import pdb\n",
    "from sys import stdout\n",
    "import scipy\n",
    "import  pickle\n",
    "\n",
    "for fname in files:\n",
    "    print fname \n",
    "    f = open('/home/intuinno/project/pointTeach/data/Flicker8k/Flickr_8k.' + fname + 'Images.txt')\n",
    "    counter = 0\n",
    "    \n",
    "    imageList = [i for i in f]\n",
    "    numImage = len(imageList)\n",
    "    pdb.set_trace()\n",
    "\n",
    "    beginning = True\n",
    "\n",
    "    for i in f:\n",
    "        i = i.rstrip()\n",
    "        net.blobs['data'].data[...] = transformer.preprocess('data', caffe.io.load_image( '/home/intuinno/project/pointTeach/data/Flicker8k/preprocessedImages/' +  i))\n",
    "        out = net.forward()\n",
    "        feat = net.blobs['conv5_4'].data\n",
    "    #     print feat.shape\n",
    "        reshapeFeat = np.swapaxes(feat, 1,3)\n",
    "        reshapeFeat2 = np.reshape(reshapeFeat,(1,-1))\n",
    "    #     print reshapeFeat2.shape\n",
    "    #     print i\n",
    "        counter += 1\n",
    "        stdout.write(\"\\r%d\" % counter)\n",
    "        stdout.flush()\n",
    "        if beginning:\n",
    "            result = reshapeFeat2\n",
    "            beginning = False\n",
    "        else:\n",
    "            result = np.vstack((result, reshapeFeat2))\n",
    "\n",
    "    print result.shape\n",
    "    \n",
    "    resultSave = scipy.sparse.csr_matrix(result)\n",
    "    pickle.dump(resultSave, open('flicker_8k_feature.' + fname + '.pkl','wb'))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pickle as pkl\n",
    "print pkl.HIGHEST_PROTOCOL\n",
    "pkl.dump(resultSave, open('flicker_8k_feature.' + fname + '.pkl','wb'),protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "\n",
    "def show_sizeof(x, level=0):\n",
    "\n",
    "    print \"\\t\" * level, x.__class__, sys.getsizeof(x), x\n",
    "\n",
    "    if hasattr(x, '__iter__'):\n",
    "        if hasattr(x, 'items'):\n",
    "            for xx in x.items():\n",
    "                show_sizeof(xx, level + 1)\n",
    "        else:\n",
    "            for xx in x:\n",
    "                show_sizeof(xx, level + 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "show_sizeof(None)\n",
    "show_sizeof(3)\n",
    "show_sizeof(2**63)\n",
    "show_sizeof(102947298469128649161972364837164)\n",
    "show_sizeof(918659326943756134897561304875610348756384756193485761304875613948576297485698417)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "capFile = open('/home/intuinno/project/pointTeach/data/Flicker8k/Flickr8k.token.txt')\n",
    "\n",
    "capDict = {}\n",
    "import re\n",
    "for line in capFile:\n",
    "    match = re.search(r'^([\\w]+\\.jpg)#(\\d)\\s([\\w\\W.\\s-]+)$', line)\n",
    "#     print line\n",
    "    if not match:\n",
    "        print line\n",
    "    else:\n",
    "        if match.group(2) == '0':\n",
    "            capDict[match.group(1)] = [match.group(3)]\n",
    "        else:\n",
    "            capDict[match.group(1)].append(match.group(3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import cPickle as pickle\n",
    "files = ['test', 'train', 'dev']\n",
    "\n",
    "for name in files:\n",
    "    counter = 0\n",
    "    feat = pickle.load(open('flicker_8k_feature.' + name +'.pkl','rb'))\n",
    "    filenames = open('/home/intuinno/project/pointTeach/data/Flicker8k/Flickr_8k.' + name + 'Images.txt')\n",
    "    cap = []\n",
    "    for imageFile in filenames:\n",
    "        imageFile = imageFile.rstrip()\n",
    "        for sen in capDict[imageFile]:\n",
    "            cap.append([sen.rstrip(), counter])\n",
    "        counter += 1\n",
    "    saveFile = open('flicker_8k_feature.' + name + '.pkl', 'wb')\n",
    "    pickle.dump(cap, saveFile, protocol=cPickle.HIGHEST_PROTOCOL)\n",
    "    pickle.dump(feat, saveFile, protocol=cPickle.HIGHEST_PROTOCOL)\n",
    "    saveFile.close()\n",
    "            \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "annotation_path = '/home/intuinno/project/pointTeach/data/Flicker8k/Flickr8k.token.txt'\n",
    "annotations = pd.read_table(annotation_path, sep='\\t', header=None, names=['image', 'caption'])\n",
    "\n",
    "captions = annotations['caption'].values\n",
    "\n",
    "vectorizer = CountVectorizer(analyzer=str.split, lowercase=False).fit(captions)\n",
    "dictionary = vectorizer.vocabulary_\n",
    "dictionary_series = pd.Series(dictionary.values(), index=dictionary.keys()) + 2\n",
    "dictionary = dictionary_series.to_dict()\n",
    "\n",
    "# Sort dictionary in descending order\n",
    "from collections import OrderedDict\n",
    "dictionary = OrderedDict(sorted(dictionary.items(), key=lambda x:x[1], reverse=True))\n",
    "\n",
    "with open('dictionary.pkl', 'wb') as f:\n",
    "    pickle.dump(dictionary, f)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dictionary['People']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dictionary['people']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "saveFile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "description": "Instant recognition with a pre-trained model and a tour of the net interface for visualizing features and parameters layer-by-layer.",
  "example_name": "Image Classification and Filter Visualization",
  "include_in_docs": true,
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  },
  "priority": 1
 },
 "nbformat": 4,
 "nbformat_minor": 0
}