Tweets airlines Sentiment Analysis-Copy1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Learning with Keras \n",
    "----------------------------------------------------\n",
    "This is a demo of a Binary classification, I apply all encoding techniches que you already know, we are no looking to do a bag of words instead we are trying to figureout the sequence of words, order matter in a sentences and in particular in Sentiment analysis"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "#########################################################################\n",
    "######## @angelo337 Angelo Rodriguez\n",
    "######## @version: 0.1 \n",
    "######## @ https://bigdatacolombia.slack.com/\n",
    "######## @todo: enhance the code with a embeding could be Glove or Word2Vec\n",
    "######## @see: check http://keras.io for mor references\n",
    "######## ################################################################"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "#Copyright 2015 Creangel Ltda. All Rights Reserved.\n",
    "#Licensed under the Apache License, Version 2.0 (the \"License\");\n",
    "#you may not use this file except in compliance with the License.\n",
    "#You may obtain a copy of the License at\n",
    "#    http://www.apache.org/licenses/LICENSE-2.0\n",
    "#Unless required by applicable law or agreed to in writing, software\n",
    "#distributed under the License is distributed on an \"AS IS\" BASIS,\n",
    "#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
    "#See the License for the specific language governing permissions and\n",
    "#limitations under the License.\n",
    "#=============================================================================="
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading tools "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 263,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#this is to force to be compatible con Python 3.xx\n",
    "#from __future__ import absolute_import\n",
    "#from __future__ import print_function\n",
    "%matplotlib inline \n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    " \n",
    "import seaborn as sns\n",
    "import nltk\n",
    "import csv\n",
    "import itertools\n",
    "import operator\n",
    "import unicodedata\n",
    "import nltk\n",
    "import sys\n",
    "import os\n",
    "import time, re\n",
    "from datetime import datetime\n",
    "import collections\n",
    "from bs4 import BeautifulSoup \n",
    "from nltk.stem import SnowballStemmer\n",
    "from nltk.corpus import stopwords\n",
    "stemmer = SnowballStemmer(\"english\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading utilities tools to process text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 264,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn import metrics\n",
    "from sklearn import preprocessing\n",
    "from sklearn.preprocessing import StandardScaler, LabelEncoder\n",
    "from sklearn.metrics import mean_squared_error, classification_report\n",
    "from sklearn.cross_validation import train_test_split"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading actual Tools and Layers of Deep Learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 265,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "from keras.preprocessing import sequence\n",
    "from keras.utils import np_utils\n",
    "from keras.models import Sequential\n",
    "from keras.layers.core import Dense, Dropout, Activation, Flatten\n",
    "from keras.layers.normalization import BatchNormalization #https://github.com/fchollet/keras/blob/master/examples/kaggle_otto_nn.py\n",
    "from keras.layers.advanced_activations import PReLU, LeakyReLU, ELU, ParametricSoftplus, ThresholdedLinear, ThresholdedReLU\n",
    "from keras.layers.convolutional import Convolution1D, MaxPooling1D\n",
    "from keras.optimizers import SGD\n",
    "from keras.callbacks import History, EarlyStopping\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "\n",
    "# this is to be sure that al results in development are comparable\n",
    "np.random.seed(1337)  # for reproducibility"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## actual code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 266,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"data/Tweets_airline.csv\")\n",
    "vocabulary_size = 0\n",
    "unknown_token = \"UNKNOWN_TOKEN\"\n",
    "sentence_start_token = \"SENTENCE_START\"\n",
    "sentence_end_token = \"SENTENCE_END\"\n",
    "\n",
    "#print (df.shape)\n",
    "#print (df.info())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Some small utilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 267,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "An example of a single message: 0                     virginamerica what dhepburn said\n",
      "1    virginamerica plus you ve added commercials to...\n",
      "2    virginamerica i didn t today must mean i need ...\n",
      "3    virginamerica it s really aggressive to blast ...\n",
      "4    virginamerica and it s a really big bad thing ...\n",
      "Name: text, dtype: object\n"
     ]
    }
   ],
   "source": [
    "def cleanword(w):        \n",
    "    return re.sub('[^a-zA-Z0-9,]' , ' ' , w)  \n",
    "\n",
    "def cleantext(review):\n",
    "    review = BeautifulSoup(review ,\"lxml\").get_text()\n",
    "    review_words = cleanword(review.lower()).split()    \n",
    "#    stop = stopwords.words('english')\n",
    "#    stemmed_words = [stemmer.stem(w) for w in review_words if w not in stop]\n",
    "#    return \" \".join(stemmed_words)\n",
    "    return \" \".join(review_words)\n",
    "\n",
    "\n",
    "df[\"text\"]  = df[\"text\"].apply(cleantext)\n",
    "print \"An example of a single message:\", df.head(5)['text']\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tokenize all sentences\n",
    "it is a good Idea to tokenize with Keras, instead of bag of words\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 268,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 15100 uniq words tokens.\n",
      "Example of a tokenize sentence [u'virginamerica', u'awaiting', u'my', u'return', u'phone', u'call', u',', u'just', u'would', u'prefer', u'to', u'use', u'your', u'online', u'self', u'service', u'option']\n"
     ]
    }
   ],
   "source": [
    "################# ML ##############################################3\n",
    "\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.cross_validation import train_test_split \n",
    "\n",
    "X_clean = df[\"text\"] \n",
    "\n",
    "sentences = X_clean\n",
    "tokenized_sentences = [nltk.word_tokenize(sent) for sent in sentences]\n",
    "word_freq = nltk.FreqDist(itertools.chain(*tokenized_sentences))\n",
    "print \"Found %d uniq words tokens.\" % len(word_freq.items())\n",
    "w_tokens = len(word_freq.items())\n",
    "vocabulary_size = w_tokens +1\n",
    "print \"Example of a tokenize sentence\", tokenized_sentences[33]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 269,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "using vocabulary size 15101\n",
      "Example of a sequence sentence [2271, 12, 489, 98, 82, 6, 45, 76, 2265, 0, 262, 22, 205, 1953, 46, 490]\n"
     ]
    }
   ],
   "source": [
    "max_words= len(word_freq.items())+1\n",
    "vocab = word_freq.most_common(vocabulary_size-1)\n",
    "index_to_word = [x[0] for x in vocab]\n",
    "index_to_word.append(unknown_token)\n",
    "word_to_index = dict([w,i] for i,w in enumerate(index_to_word))\n",
    "print \"using vocabulary size %d\" % vocabulary_size\n",
    "for i, sent in enumerate (tokenized_sentences):\n",
    "    tokenized_sentences[i] = [w if w in word_to_index else unknown_token for w in sent]\n",
    "X_clean = np.asarray([[word_to_index[w] for w in sent[1:]] for sent in tokenized_sentences])\n",
    "X_clean = np.asarray(X_clean)\n",
    "print \"Example of a sequence sentence\", X_clean[33]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 270,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Example of a sentiment sentence:  negative\n",
      "Example of a sentiment After Encoder:  0\n",
      "Convert class vector to binary class matrix (for use with categorical_crossentropy)\n",
      "(3, 'classes')\n",
      "Example of a sentiment After categorical Encoder:  [ 1.  0.  0.]\n"
     ]
    }
   ],
   "source": [
    "# LabelEncoder is a utility class to help normalize labels such \n",
    "# that they contain only values between 0 and n_classes-1. \n",
    "le = preprocessing.LabelEncoder()\n",
    "le.fit(df[\"airline_sentiment\"])\n",
    "print \"Example of a sentiment sentence: \", df[\"airline_sentiment\"][33]\n",
    "y = le.transform(df[\"airline_sentiment\"])\n",
    "y = np.asarray(y)\n",
    "print \"Example of a sentiment After Encoder: \",y[33]\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_clean, y, test_size=0.2) #, random_state=42\n",
    "\n",
    "print(\"Convert class vector to binary class matrix (for use with categorical_crossentropy)\")\n",
    "y_train = np.array(y_train, dtype = int)\n",
    "y_test = np.array(y_test, dtype = int)\n",
    "nb_classes = np.max(y_train)+1\n",
    "nb_classes = int(nb_classes)\n",
    "print(nb_classes, 'classes')\n",
    "\n",
    "Y_train = np_utils.to_categorical(y_train, nb_classes)\n",
    "Y_test = np_utils.to_categorical(y_test, nb_classes)\n",
    "print \"Example of a sentiment After categorical Encoder: \",Y_train[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 271,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vectorizing sequence data...\n",
      "[ 1.  1.  1. ...,  0.  0.  0.]\n"
     ]
    }
   ],
   "source": [
    "print(\"Vectorizing sequence data...\")\n",
    "tokenizer = Tokenizer(nb_words=max_words)\n",
    "X_train = tokenizer.sequences_to_matrix(X_train, mode=\"binary\")\n",
    "X_test = tokenizer.sequences_to_matrix(X_test, mode=\"binary\")\n",
    "\n",
    "#X_train = tokenizer.sequences_to_matrix(X_train, mode=\"count\")\n",
    "#X_test = tokenizer.sequences_to_matrix(X_test, mode=\"count\")\n",
    "print X_train[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Some parameters \n",
    "That make the model perform better or worse depending on the selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "max_features = 350\n",
    "maxlen = 350  # cut texts after this number of words (among top max_features most common words)\n",
    "batch_size = 64\n",
    "max_words = w_tokens+1 # you should run it for the first time anc copy the value of mismatch dimension [1] (32,15101)x(11501,64)->(32,64)\n",
    "nb_epoch = 300\n",
    "value_dense = 64\n",
    "drop_layer = 0.5\n",
    "##\n",
    "# BatchNormalization\n",
    "# Normalize the activations of the previous layer at each batch, \n",
    "# i.e. applies a transformation that maintains the mean activation \n",
    "# close to 0 and the activation standard deviation close to 1.\n",
    "#\n",
    "##\n",
    "b_norm = 1\n",
    "extra_layers = 5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![alt_text](http://scs.ryerson.ca/~aharley/neural-networks/images/fcn_visualization7_big.png \"_\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Building the Actual Model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 273,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Building model...\n"
     ]
    }
   ],
   "source": [
    "print(\"Building model...\")\n",
    "model = Sequential()\n",
    "model.add(Dense(value_dense, input_shape=(max_words,)))\n",
    "model.add(Activation('relu'))\n",
    "if(b_norm):\n",
    "    model.add(BatchNormalization())\n",
    "model.add(Dropout(drop_layer))\n",
    "    \n",
    "for i in range(extra_layers):\n",
    "    model.add(Dense((value_dense), input_shape=(max_words,)))\n",
    "    model.add(Activation('relu'))\n",
    "    if(b_norm):\n",
    "        model.add(BatchNormalization())\n",
    "    model.add(Dropout((drop_layer)))\n",
    "    \n",
    "model.add(Dense(nb_classes))\n",
    "model.add(Activation('softmax')) #softmax\n",
    "# this is to use Stochastic Gradiend Decendant\n",
    "sgd = SGD(lr=0.1, decay=1e-6, momentum=0.95, nesterov=True)\n",
    "model.compile(loss='categorical_crossentropy', optimizer='Adadelta') #Adadelta outperform SGD"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![alt text](http://scs.ryerson.ca/~aharley/neural-networks/images/one_target_graddescent.gif \"http://scs.ryerson.ca/~aharley/neural-networks/\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate image of the model that we are using "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 274,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fontconfig warning: ignoring UTF-8: not a valid region tag\n",
      "\n",
      "Fontconfig warning: ignoring UTF-8: not a valid region tag\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/svg+xml": [
       "<svg height=\"1896pt\" viewBox=\"0.00 0.00 292.00 1896.00\" width=\"292pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g class=\"graph\" id=\"graph0\" transform=\"scale(1 1) rotate(0) translate(4 1892)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"white\" points=\"-4,4 -4,-1892 288,-1892 288,4 -4,4\" stroke=\"none\"/>\n",
       "<!-- layer0 -->\n",
       "<g class=\"node\" id=\"node1\"><title>layer0</title>\n",
       "<polygon fill=\"none\" points=\"53,-1 53,-37 231,-37 231,-1 53,-1\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"142\" y=\"-15.3\">(keras.layers.core.Activation)</text>\n",
       "</g>\n",
       "<!-- layer1 -->\n",
       "<g class=\"node\" id=\"node2\"><title>layer1</title>\n",
       "<polygon fill=\"none\" points=\"64.5,-75 64.5,-111 219.5,-111 219.5,-75 64.5,-75\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"142\" y=\"-89.3\">(keras.layers.core.Dense)</text>\n",
       "</g>\n",
       "<!-- layer1&#45;&gt;layer0 -->\n",
       "<g class=\"edge\" id=\"edge1\"><title>layer1-&gt;layer0</title>\n",
       "<path d=\"M142,-74.937C142,-66.8072 142,-56.8761 142,-47.7047\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"145.5,-47.4406 142,-37.4407 138.5,-47.4407 145.5,-47.4406\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- layer2 -->\n",
       "<g class=\"node\" id=\"node3\"><title>layer2</title>\n",
       "<polygon fill=\"none\" points=\"59,-149 59,-185 225,-185 225,-149 59,-149\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"142\" y=\"-163.3\">(keras.layers.core.Dropout)</text>\n",
       "</g>\n",
       "<!-- layer2&#45;&gt;layer1 -->\n",
       "<g class=\"edge\" id=\"edge2\"><title>layer2-&gt;layer1</title>\n",
       "<path d=\"M142,-148.937C142,-140.807 142,-130.876 142,-121.705\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"145.5,-121.441 142,-111.441 138.5,-121.441 145.5,-121.441\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- layer3 -->\n",
       "<g class=\"node\" id=\"node4\"><title>layer3</title>\n",
       "<polygon fill=\"none\" points=\"0,-223 0,-259 284,-259 284,-223 0,-223\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"142\" y=\"-237.3\">(keras.layers.normalization.BatchNormalization)</text>\n",
       "</g>\n",
       "<!-- layer3&#45;&gt;layer2 -->\n",
       "<g class=\"edge\" id=\"edge3\"><title>layer3-&gt;layer2</title>\n",
       "<path d=\"M142,-222.937C142,-214.807 142,-204.876 142,-195.705\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"145.5,-195.441 142,-185.441 138.5,-195.441 145.5,-195.441\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- layer4 -->\n",
       "<g class=\"node\" id=\"node5\"><title>layer4</title>\n",
       "<polygon fill=\"none\" points=\"53,-297 53,-333 231,-333 231,-297 53,-297\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"142\" y=\"-311.3\">(keras.layers.core.Activation)</text>\n",
       "</g>\n",
       "<!-- layer4&#45;&gt;layer3 -->\n",
       "<g class=\"edge\" id=\"edge4\"><title>layer4-&gt;layer3</title>\n",
       "<path d=\"M142,-296.937C142,-288.807 142,-278.876 142,-269.705\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"145.5,-269.441 142,-259.441 138.5,-269.441 145.5,-269.441\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- layer5 -->\n",
       "<g class=\"node\" id=\"node6\"><title>layer5</title>\n",
       "<polygon fill=\"none\" points=\"64.5,-371 64.5,-407 219.5,-407 219.5,-371 64.5,-371\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"142\" y=\"-385.3\">(keras.layers.core.Dense)</text>\n",
       "</g>\n",
       "<!-- layer5&#45;&gt;layer4 -->\n",
       "<g class=\"edge\" id=\"edge5\"><title>layer5-&gt;layer4</title>\n",
       "<path d=\"M142,-370.937C142,-362.807 142,-352.876 142,-343.705\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"145.5,-343.441 142,-333.441 138.5,-343.441 145.5,-343.441\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- layer6 -->\n",
       "<g class=\"node\" id=\"node7\"><title>layer6</title>\n",
       "<polygon fill=\"none\" points=\"59,-445 59,-481 225,-481 225,-445 59,-445\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"142\" y=\"-459.3\">(keras.layers.core.Dropout)</text>\n",
       "</g>\n",
       "<!-- layer6&#45;&gt;layer5 -->\n",
       "<g class=\"edge\" id=\"edge6\"><title>layer6-&gt;layer5</title>\n",
       "<path d=\"M142,-444.937C142,-436.807 142,-426.876 142,-417.705\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"145.5,-417.441 142,-407.441 138.5,-417.441 145.5,-417.441\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- layer7 -->\n",
       "<g class=\"node\" id=\"node8\"><title>layer7</title>\n",
       "<polygon fill=\"none\" points=\"0,-519 0,-555 284,-555 284,-519 0,-519\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"142\" y=\"-533.3\">(keras.layers.normalization.BatchNormalization)</text>\n",
       "</g>\n",
       "<!-- layer7&#45;&gt;layer6 -->\n",
       "<g class=\"edge\" id=\"edge7\"><title>layer7-&gt;layer6</title>\n",
       "<path d=\"M142,-518.937C142,-510.807 142,-500.876 142,-491.705\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"145.5,-491.441 142,-481.441 138.5,-491.441 145.5,-491.441\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- layer8 -->\n",
       "<g class=\"node\" id=\"node9\"><title>layer8</title>\n",
       "<polygon fill=\"none\" points=\"53,-593 53,-629 231,-629 231,-593 53,-593\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"142\" y=\"-607.3\">(keras.layers.core.Activation)</text>\n",
       "</g>\n",
       "<!-- layer8&#45;&gt;layer7 -->\n",
       "<g class=\"edge\" id=\"edge8\"><title>layer8-&gt;layer7</title>\n",
       "<path d=\"M142,-592.937C142,-584.807 142,-574.876 142,-565.705\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"145.5,-565.441 142,-555.441 138.5,-565.441 145.5,-565.441\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- layer9 -->\n",
       "<g class=\"node\" id=\"node10\"><title>layer9</title>\n",
       "<polygon fill=\"none\" points=\"64.5,-667 64.5,-703 219.5,-703 219.5,-667 64.5,-667\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"142\" y=\"-681.3\">(keras.layers.core.Dense)</text>\n",
       "</g>\n",
       "<!-- layer9&#45;&gt;layer8 -->\n",
       "<g class=\"edge\" id=\"edge9\"><title>layer9-&gt;layer8</title>\n",
       "<path d=\"M142,-666.937C142,-658.807 142,-648.876 142,-639.705\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"145.5,-639.441 142,-629.441 138.5,-639.441 145.5,-639.441\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- layer10 -->\n",
       "<g class=\"node\" id=\"node11\"><title>layer10</title>\n",
       "<polygon fill=\"none\" points=\"59,-741 59,-777 225,-777 225,-741 59,-741\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"142\" y=\"-755.3\">(keras.layers.core.Dropout)</text>\n",
       "</g>\n",
       "<!-- layer10&#45;&gt;layer9 -->\n",
       "<g class=\"edge\" id=\"edge10\"><title>layer10-&gt;layer9</title>\n",
       "<path d=\"M142,-740.937C142,-732.807 142,-722.876 142,-713.705\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"145.5,-713.441 142,-703.441 138.5,-713.441 145.5,-713.441\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- layer11 -->\n",
       "<g class=\"node\" id=\"node12\"><title>layer11</title>\n",
       "<polygon fill=\"none\" points=\"0,-815 0,-851 284,-851 284,-815 0,-815\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"142\" y=\"-829.3\">(keras.layers.normalization.BatchNormalization)</text>\n",
       "</g>\n",
       "<!-- layer11&#45;&gt;layer10 -->\n",
       "<g class=\"edge\" id=\"edge11\"><title>layer11-&gt;layer10</title>\n",
       "<path d=\"M142,-814.937C142,-806.807 142,-796.876 142,-787.705\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"145.5,-787.441 142,-777.441 138.5,-787.441 145.5,-787.441\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- layer12 -->\n",
       "<g class=\"node\" id=\"node13\"><title>layer12</title>\n",
       "<polygon fill=\"none\" points=\"53,-889 53,-925 231,-925 231,-889 53,-889\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"142\" y=\"-903.3\">(keras.layers.core.Activation)</text>\n",
       "</g>\n",
       "<!-- layer12&#45;&gt;layer11 -->\n",
       "<g class=\"edge\" id=\"edge12\"><title>layer12-&gt;layer11</title>\n",
       "<path d=\"M142,-888.937C142,-880.807 142,-870.876 142,-861.705\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"145.5,-861.441 142,-851.441 138.5,-861.441 145.5,-861.441\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- layer13 -->\n",
       "<g class=\"node\" id=\"node14\"><title>layer13</title>\n",
       "<polygon fill=\"none\" points=\"64.5,-963 64.5,-999 219.5,-999 219.5,-963 64.5,-963\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"142\" y=\"-977.3\">(keras.layers.core.Dense)</text>\n",
       "</g>\n",
       "<!-- layer13&#45;&gt;layer12 -->\n",
       "<g class=\"edge\" id=\"edge13\"><title>layer13-&gt;layer12</title>\n",
       "<path d=\"M142,-962.937C142,-954.807 142,-944.876 142,-935.705\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"145.5,-935.441 142,-925.441 138.5,-935.441 145.5,-935.441\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- layer14 -->\n",
       "<g class=\"node\" id=\"node15\"><title>layer14</title>\n",
       "<polygon fill=\"none\" points=\"59,-1037 59,-1073 225,-1073 225,-1037 59,-1037\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"142\" y=\"-1051.3\">(keras.layers.core.Dropout)</text>\n",
       "</g>\n",
       "<!-- layer14&#45;&gt;layer13 -->\n",
       "<g class=\"edge\" id=\"edge14\"><title>layer14-&gt;layer13</title>\n",
       "<path d=\"M142,-1036.94C142,-1028.81 142,-1018.88 142,-1009.7\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"145.5,-1009.44 142,-999.441 138.5,-1009.44 145.5,-1009.44\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- layer15 -->\n",
       "<g class=\"node\" id=\"node16\"><title>layer15</title>\n",
       "<polygon fill=\"none\" points=\"0,-1111 0,-1147 284,-1147 284,-1111 0,-1111\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"142\" y=\"-1125.3\">(keras.layers.normalization.BatchNormalization)</text>\n",
       "</g>\n",
       "<!-- layer15&#45;&gt;layer14 -->\n",
       "<g class=\"edge\" id=\"edge15\"><title>layer15-&gt;layer14</title>\n",
       "<path d=\"M142,-1110.94C142,-1102.81 142,-1092.88 142,-1083.7\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"145.5,-1083.44 142,-1073.44 138.5,-1083.44 145.5,-1083.44\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- layer16 -->\n",
       "<g class=\"node\" id=\"node17\"><title>layer16</title>\n",
       "<polygon fill=\"none\" points=\"53,-1185 53,-1221 231,-1221 231,-1185 53,-1185\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"142\" y=\"-1199.3\">(keras.layers.core.Activation)</text>\n",
       "</g>\n",
       "<!-- layer16&#45;&gt;layer15 -->\n",
       "<g class=\"edge\" id=\"edge16\"><title>layer16-&gt;layer15</title>\n",
       "<path d=\"M142,-1184.94C142,-1176.81 142,-1166.88 142,-1157.7\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"145.5,-1157.44 142,-1147.44 138.5,-1157.44 145.5,-1157.44\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- layer17 -->\n",
       "<g class=\"node\" id=\"node18\"><title>layer17</title>\n",
       "<polygon fill=\"none\" points=\"64.5,-1259 64.5,-1295 219.5,-1295 219.5,-1259 64.5,-1259\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"142\" y=\"-1273.3\">(keras.layers.core.Dense)</text>\n",
       "</g>\n",
       "<!-- layer17&#45;&gt;layer16 -->\n",
       "<g class=\"edge\" id=\"edge17\"><title>layer17-&gt;layer16</title>\n",
       "<path d=\"M142,-1258.94C142,-1250.81 142,-1240.88 142,-1231.7\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"145.5,-1231.44 142,-1221.44 138.5,-1231.44 145.5,-1231.44\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- layer18 -->\n",
       "<g class=\"node\" id=\"node19\"><title>layer18</title>\n",
       "<polygon fill=\"none\" points=\"59,-1333 59,-1369 225,-1369 225,-1333 59,-1333\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"142\" y=\"-1347.3\">(keras.layers.core.Dropout)</text>\n",
       "</g>\n",
       "<!-- layer18&#45;&gt;layer17 -->\n",
       "<g class=\"edge\" id=\"edge18\"><title>layer18-&gt;layer17</title>\n",
       "<path d=\"M142,-1332.94C142,-1324.81 142,-1314.88 142,-1305.7\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"145.5,-1305.44 142,-1295.44 138.5,-1305.44 145.5,-1305.44\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- layer19 -->\n",
       "<g class=\"node\" id=\"node20\"><title>layer19</title>\n",
       "<polygon fill=\"none\" points=\"0,-1407 0,-1443 284,-1443 284,-1407 0,-1407\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"142\" y=\"-1421.3\">(keras.layers.normalization.BatchNormalization)</text>\n",
       "</g>\n",
       "<!-- layer19&#45;&gt;layer18 -->\n",
       "<g class=\"edge\" id=\"edge19\"><title>layer19-&gt;layer18</title>\n",
       "<path d=\"M142,-1406.94C142,-1398.81 142,-1388.88 142,-1379.7\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"145.5,-1379.44 142,-1369.44 138.5,-1379.44 145.5,-1379.44\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- layer20 -->\n",
       "<g class=\"node\" id=\"node21\"><title>layer20</title>\n",
       "<polygon fill=\"none\" points=\"53,-1481 53,-1517 231,-1517 231,-1481 53,-1481\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"142\" y=\"-1495.3\">(keras.layers.core.Activation)</text>\n",
       "</g>\n",
       "<!-- layer20&#45;&gt;layer19 -->\n",
       "<g class=\"edge\" id=\"edge20\"><title>layer20-&gt;layer19</title>\n",
       "<path d=\"M142,-1480.94C142,-1472.81 142,-1462.88 142,-1453.7\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"145.5,-1453.44 142,-1443.44 138.5,-1453.44 145.5,-1453.44\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- layer21 -->\n",
       "<g class=\"node\" id=\"node22\"><title>layer21</title>\n",
       "<polygon fill=\"none\" points=\"64.5,-1555 64.5,-1591 219.5,-1591 219.5,-1555 64.5,-1555\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"142\" y=\"-1569.3\">(keras.layers.core.Dense)</text>\n",
       "</g>\n",
       "<!-- layer21&#45;&gt;layer20 -->\n",
       "<g class=\"edge\" id=\"edge21\"><title>layer21-&gt;layer20</title>\n",
       "<path d=\"M142,-1554.94C142,-1546.81 142,-1536.88 142,-1527.7\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"145.5,-1527.44 142,-1517.44 138.5,-1527.44 145.5,-1527.44\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- layer22 -->\n",
       "<g class=\"node\" id=\"node23\"><title>layer22</title>\n",
       "<polygon fill=\"none\" points=\"59,-1629 59,-1665 225,-1665 225,-1629 59,-1629\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"142\" y=\"-1643.3\">(keras.layers.core.Dropout)</text>\n",
       "</g>\n",
       "<!-- layer22&#45;&gt;layer21 -->\n",
       "<g class=\"edge\" id=\"edge22\"><title>layer22-&gt;layer21</title>\n",
       "<path d=\"M142,-1628.94C142,-1620.81 142,-1610.88 142,-1601.7\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"145.5,-1601.44 142,-1591.44 138.5,-1601.44 145.5,-1601.44\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- layer23 -->\n",
       "<g class=\"node\" id=\"node24\"><title>layer23</title>\n",
       "<polygon fill=\"none\" points=\"0,-1703 0,-1739 284,-1739 284,-1703 0,-1703\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"142\" y=\"-1717.3\">(keras.layers.normalization.BatchNormalization)</text>\n",
       "</g>\n",
       "<!-- layer23&#45;&gt;layer22 -->\n",
       "<g class=\"edge\" id=\"edge23\"><title>layer23-&gt;layer22</title>\n",
       "<path d=\"M142,-1702.94C142,-1694.81 142,-1684.88 142,-1675.7\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"145.5,-1675.44 142,-1665.44 138.5,-1675.44 145.5,-1675.44\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- layer24 -->\n",
       "<g class=\"node\" id=\"node25\"><title>layer24</title>\n",
       "<polygon fill=\"none\" points=\"53,-1777 53,-1813 231,-1813 231,-1777 53,-1777\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"142\" y=\"-1791.3\">(keras.layers.core.Activation)</text>\n",
       "</g>\n",
       "<!-- layer24&#45;&gt;layer23 -->\n",
       "<g class=\"edge\" id=\"edge24\"><title>layer24-&gt;layer23</title>\n",
       "<path d=\"M142,-1776.94C142,-1768.81 142,-1758.88 142,-1749.7\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"145.5,-1749.44 142,-1739.44 138.5,-1749.44 145.5,-1749.44\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- layer25 -->\n",
       "<g class=\"node\" id=\"node26\"><title>layer25</title>\n",
       "<polygon fill=\"none\" points=\"64.5,-1851 64.5,-1887 219.5,-1887 219.5,-1851 64.5,-1851\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"142\" y=\"-1865.3\">(keras.layers.core.Dense)</text>\n",
       "</g>\n",
       "<!-- layer25&#45;&gt;layer24 -->\n",
       "<g class=\"edge\" id=\"edge25\"><title>layer25-&gt;layer24</title>\n",
       "<path d=\"M142,-1850.94C142,-1842.81 142,-1832.88 142,-1823.7\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"145.5,-1823.44 142,-1813.44 138.5,-1823.44 145.5,-1823.44\" stroke=\"black\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>"
      ],
      "text/plain": [
       "<IPython.core.display.SVG object>"
      ]
     },
     "execution_count": 274,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from keras.utils.visualize_util import plot\n",
    "plot(model, to_file='model.png')\n",
    "from IPython.display import SVG\n",
    "from keras.utils.visualize_util import to_graph\n",
    "SVG(to_graph(model).create(prog='dot', format='svg'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 275,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 10540 samples, validate on 1172 samples\n",
      "Epoch 1/300\n",
      "10540/10540 [==============================] - 2s - loss: 0.9223 - acc: 0.6252 - val_loss: 0.8101 - val_acc: 0.6109\n",
      "Epoch 2/300\n",
      "10540/10540 [==============================] - 2s - loss: 0.7667 - acc: 0.6589 - val_loss: 0.6405 - val_acc: 0.7253\n",
      "Epoch 3/300\n",
      "10540/10540 [==============================] - 2s - loss: 0.6682 - acc: 0.7071 - val_loss: 0.5829 - val_acc: 0.7449\n",
      "Epoch 4/300\n",
      "10540/10540 [==============================] - 2s - loss: 0.6160 - acc: 0.7344 - val_loss: 0.5696 - val_acc: 0.7534\n",
      "Epoch 5/300\n",
      "10540/10540 [==============================] - 2s - loss: 0.5795 - acc: 0.7582 - val_loss: 0.5889 - val_acc: 0.7833\n",
      "Epoch 6/300\n",
      "10540/10540 [==============================] - 2s - loss: 0.5455 - acc: 0.7778 - val_loss: 0.5636 - val_acc: 0.7560\n",
      "Epoch 7/300\n",
      "10540/10540 [==============================] - 2s - loss: 0.5126 - acc: 0.8014 - val_loss: 0.5700 - val_acc: 0.7944\n",
      "Epoch 8/300\n",
      "10540/10540 [==============================] - 2s - loss: 0.4877 - acc: 0.8111 - val_loss: 0.5538 - val_acc: 0.8072\n",
      "Epoch 9/300\n",
      "10540/10540 [==============================] - 2s - loss: 0.4575 - acc: 0.8275 - val_loss: 0.6279 - val_acc: 0.7910\n",
      "Epoch 10/300\n",
      "10540/10540 [==============================] - 2s - loss: 0.4242 - acc: 0.8492 - val_loss: 0.5694 - val_acc: 0.7952\n",
      "Epoch 11/300\n",
      "10540/10540 [==============================] - 2s - loss: 0.4098 - acc: 0.8528 - val_loss: 0.5750 - val_acc: 0.8020\n",
      "Epoch 12/300\n",
      "10540/10540 [==============================] - 2s - loss: 0.3792 - acc: 0.8703 - val_loss: 0.5736 - val_acc: 0.8046\n",
      "Epoch 13/300\n",
      "10540/10540 [==============================] - 2s - loss: 0.3576 - acc: 0.8748 - val_loss: 0.5795 - val_acc: 0.8072\n",
      "Epoch 14/300\n",
      "10496/10540 [============================>.] - ETA: 0s - loss: 0.3408 - acc: 0.8828Epoch 00013: early stopping\n",
      "10540/10540 [==============================] - 2s - loss: 0.3408 - acc: 0.8829 - val_loss: 0.5774 - val_acc: 0.8029\n",
      "2928/2928 [==============================] - 0s     \n",
      "('Test score:', 0.63682733826298532)\n",
      "('Test accuracy: 78.79', '%')\n",
      "2928/2928 [==============================] - 0s     \n",
      "2928/2928 [==============================] - 0s     \n"
     ]
    }
   ],
   "source": [
    "history = History()\n",
    "early_stopping = EarlyStopping(patience=5, verbose=1)\n",
    "model.fit(X_train, Y_train, nb_epoch=nb_epoch, batch_size=batch_size, verbose=1, \n",
    "          show_accuracy=True, validation_split=0.1, callbacks=[history,early_stopping])\n",
    "score = model.evaluate(X_test, Y_test, batch_size=batch_size, verbose=1, show_accuracy=True)\n",
    "print('Test score:', score[0])\n",
    "print('Test accuracy: %.2f' % (score[1]*100),\"%\")\n",
    "#print \"history.history\",history.totals\n",
    "classes = model.predict_classes(X_test, batch_size=32)\n",
    "proba = model.predict_proba(X_test, batch_size=32)\n",
    "pred = model.predict(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate your Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 276,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('mean_squared_error', 0.11063677490305002)\n",
      "[[1612  180   58]\n",
      " [ 164  368   65]\n",
      " [  77   77  327]]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "print(\"mean_squared_error\",mean_squared_error(Y_test, pred)) \n",
    "cm = metrics.confusion_matrix(y_test, classes) \n",
    "print(cm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 277,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAU4AAAEbCAYAAAC1EG0tAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl4FFXWwOFfBxK2IKssghvqEVGURUQEAUE2xQURZURH\ngVHG5ROXmREd92UYdEDBHRQ3XHBEHVB0cEARURBUFNmOCygimxIgoElIUt8fVR06TdLpDp2u7uS8\n8/Qz3VXVdU8qeHJv3Vv3gjHGGGOMMcYYY4wxxhhjjDHGGGOMMcbERcDvAEz8icgNwHAgHagO/Be4\nWVV37sc5pwHdgZGq+l6M3z0JuFtV+5e3/LDzPQv8AWimqlkh27sBHwLDVfW5Ms5xITBbVbNL2PcP\n4AdVfTIe8ZrKp7rfAZj4EpFxuAmur6puFJHawETgLW97eQ0FjlLVtbF+UVU/BeKSNENsAgYDT4Vs\nGwqsB5wovn8n8BFQLHGKSEBVb4lTjKaSshpnJSIiDYGfgHaqqiHbawCnA7OBGsBDQE+g0Nv2N1Ut\nFJF1wD+AkcDBwEuq+hcR+QA36X4LjAYeB4ap6kLv/OuAi4DFwJNAN6Aa8BVwGdARmKKqR4lIzVjL\nL+HnfAbIxU3kvb1t1YDVwBLgv6r6nIgcDTwNNMStfd+mqq+IyFQvrm9xa+aXA7961+g+4AzgG9ya\n+gygjaruFpFbvGt7QXS/EVNZpfkdgImrk4GfQpMmgKrmqurbquoA1wEtgDZAB+BU3GYvuDW1U73z\ndAT+T0QOUtWe3v6eqvqOd1xorS74vj9wmKq2VtWjgGVAl7AYYy6/lJ91EXC4iDTzPp+GmzRzQ+L5\nF25zvA0wAnhaRKqp6oiQn2eh974X0ElVXw3+fKq6FHgDuEVEWgBXAv9XSjymCrHEWbk0BDaXccwZ\nwGRVLVTVHOBFoG/I/pdU1VHVjd65Do6h/C1AGxEZJCJ1VPUeVZ0TdsyZcSrfAV4DLvQ+DwWme++D\nLalzgfu99wuBmkDzUs41V1XzQrYFz/F3YAgwFfc+bVnX11QBljgrl19wa3ORHAhkhXzeDjQJ+bwj\n5H0BbpM7Kqq6BLdG9n/ARhF5UUTqhR3WOI7lvwxcJCLVcWuM73jbgzXOvsB8EVkDrMBNhqX9m88q\naaOq7gb+jXv74cUIsZgqxBJn5bIIaCoi7UM3iki6iNwrIrVwa3GNQ3Y3wu1oiUV4QmsQfKOqM1S1\nF3AoUBv4K8Wb9fEoH9ym9BdAPdz7lPNDa4wiko6b8O5R1aOBdkTXaVSMd6vgD8BLuB1KxljirExU\ndTtu0/R5ETkCwOtVn4zbqfE7bu/6SBFJE5E6wMXA2zEWtRE3EQWH9dQEAiJymYjc6sWSBazB7QAK\nFY/yYW9T+hXgDvY204Pq4Cbuz7zPo4E8oK73OZ+9CT+8kzTA3iQ7CRgHXA9cKCInlCNWU8lY4qxk\nVPUu3EQ5U0RWA0txE9153iEP4w7ZWYHbmTJLVV+L4tShtbV7gBtEZDnQ2juXA/wH6CgiKiIrvX0T\nKJ6Iylt+afG8gjtSoNjY0pA/Il+IyOe4PehvArO8PyavAh+LyBBK6ewSkTOAQ1X1SVXdBdwCTBER\nG41SxSX0H4DXfHoWOAS3uTc8fFygiOzBHV8X1FtVw2stxhjjm0QPgL8I2Kaqw0SkDzAWtzc01HZV\nPS3BcRljTNQSnTh7AcFH4ebiDvEwxpj9JiLH4467naCqj3ot3OeAI3CfEDtfVbeLyDDce96FuEPj\npkbTGg6V6HuczYCtAF7z2/GGkoSq6Q1j+UhErk9wfMaYFOTdtx6P+7RX0OXAZlXtjNt5eKrXIXkb\n0Bv36bXrRaQBe1vDp+I+PTY2UnkVVuMUkZHAn8I2dw77XNI91huBad77D0XkQ1X9rITjjDEmKBcY\nCIxhb0ffQOB2AFWdAiAivYAlwcldRGQh0JUYW8MVljhV9Wnc54SLeM8YNweWe1XjgKrmh31vcsjx\nc4G27B1Sso/fc/KcmjXS4xm6MSYKOflQKz2wXx3MNdtdHfXY2pxlj5ZalqoWAAUiErr5MOAMEXkA\nd6zwVYS0ej1bcHNSM9wHSPDmTXBEpHp4fgpK9D3OObiPr80BzgLmhe70JmUYhzt0JgCcgjuIuVQ1\na6RTq/01FRJsef3+xSNJF9PaDybQrF4Gm3bklX1wAiVrTDkl/ufin5rVSbqY4iJQoXcLA8BqVb1b\nRP4O3Ax8UcIxpX23VIm+xzkdqCYiC3AnTLgZQERuEpGTVXUN7gw3n+IOSZrtTbRgjKmMAoHoX7Hb\nDMz33v8XOBb4Gbd2GdTC21a0vbTWcKiE1ji9DqERJWwfF/J+TCJjMsb4KP41zgB7a4vvAANwe8tP\nxK2ULQae8uZQKMC9vzkaOIAIreFwNpGxMcY/aVHPIRORiJwMTMGdMCZfREbhTnP4kNdRnQ1cqqo5\nIjIGtwbqAHeqaraITAf6eK3hHNz5WktlidMY45/961sqoqqLcDuSw+0z6bSqzsCdoDp0W4mt4dJU\nycTpFOZD/u84hQUVcn5VpTBne4Wcu7y++1bZWTeDrdnJ1RETj5hq1qxFw0aNqVWrVpyiMglTsZ1D\nFabKJE6nsICWmb9xaofD6NG5DUcf2YpGDcOnioyf5f+5vcLOvT8OrJvhdwj72N+Ydu/eyarPvyFr\nZzbVM2pxcrce1KhRI07RmQoVpxpnolWJxOkU5nNBl/o89q8HyMzM9DscUwHaezOQ7tq1i2kvTaf/\nOUOoWbOmv0GZsqVojTM1o46B4zj0PS6DqY/ca0mzCsjMzGTk8D/y7sx/U1hok2olvYodjlRhKn3i\nPDBjJ89MuoPq1atE5doA6enpXDB4EJ98NL/sg42/AmnRv5JIpc8m7Y5uRqNGjfwOwyRYw4YNyflt\nl99hmLLEaThSolXqxOk4Du1at/Q7DOOT9GoBHMchkGTNPBMiyWqS0UrNqKNVkEfH46Xs40yldGDj\nRuzcsaPsA41/0gLRv5JIpU6cTkEuLQ9qUvaBplJqUL8eO3da4kxqKXqPM7miiTenkAPq1i1x15Il\nSzj++OP3ebVp04ZbbrmF/Px8HnzwQfr06UP79u3p06cPEyZMYM+ePSWe78033yzxfMcccwyPPvoo\nALm5uUycOJF+/frRrl07BgwYwMyZM4ud5+uvv+ayyy6jc+fOnHrqqdx4441kZe1d8nvz5s3cdNNN\n9OjRg/bt23PRRRfx9ddfF+3fs2dPmXHPnTuXIUOG0KFDBzp06MAVV1zBN998UyyODz74gHPPPZcT\nTjiB7t278+CDDxbrpV64cCFDhw6lU6dO9OrVizvuuIOcnJxi55gyZQq9evXihBNO4Mwzz2TWrFnF\n9i9btoyhQ4fSrl07unTpUuwcP/30ExdccAHt27dn5MiR7Ny5s9h3ly5dytlnn13q7wOgVq1a5OXl\nlrrfJIEU7VVPeY7jODXbXV3iK6P1MGfNmjVOtLZu3ep07tzZWbJkifPggw86Xbt2dVatWuUUFhY6\nq1atcrp27epMmjQp6vOtXr3a6dixo7Nu3TrHcRznnnvucXr27OmsWrXK2bNnjzNnzhznuOOOcz76\n6CPHcRwnKyvL6dy5szN+/HgnOzvb+eWXX5w//elPziWXXOI4juMUFBQ455xzjjNixAhny5Ytzu7d\nu50HH3zQ6dSpk/Prr786juOUGffnn3/uHHvssc6sWbOcgoICJysry7n66qud7t27F8W9ZMkSp337\n9s7bb7/t5ObmOitXrnQGDRrkzJs3z3Ecx1m7dq1z3HHHOdOmTXNycnKc9evXO4MGDXLGjBlTdI4n\nn3zS6dWrl7N8+XInNzfXeffdd50BAwY4mzZtchzHcX744QenXbt2RedYt26dc/HFFzuvvPKK4ziO\nc+ONNzp33nmns2fPHmfUqFHFrvvu3bud/v37OytXrox4/desWeN8tHS5s3F7bkwvx3Gc3/ck1ysZ\nY/p9jxPzOvXhavYe60T7ike+iJfKXeOM0e23384ZZ5zBiSeeyNdff81JJ51E69atCQQCtG7dmk6d\nOrF8+fKozpWfn8/NN9/MlVdeyaGHHgrAO++8wx/+8Adat25N9erV6dOnD3379uX5558H4K233iIQ\nCHDdddeRmZlJo0aNuPHGG/n0009Zs2YNa9euZfXq1Vx77bUceOCB1K5dm9GjR1OrVi3eeOMNgDLj\nrlOnDg888AADBw4kLS2N+vXrc+6557J582a2bdsGwBNPPMG5557LGWecQUZGBscccwyvv/46p53m\nrqE3ffp0jjzySIYNG0aNGjVo2bIlV199NbNmzWL79u3k5eXx1FNP8de//pXjjjuOjIwM+vXrx+zZ\ns2natCkATz/9NCeddFLROQ499FBeeOEFLrzwQgBWrFjBaaedRvXq1enevTtffvll0bUdN24cAwcO\n5JhjjtnfX7nxW4rWOC1xeubNm8cXX3zBjTfeCMCAAQNYtGgRy5cvp6CggNWrV7NkyRL69+8f1fle\nfvll8vLyGDGi+LwB4YOyGzRoUJTUli1bRps2bUhL2/trOfroo6lRo0axxFFQsPcZ+0AgQL169Yqa\n62XFLSIMGDAAcEcdrF+/nmnTptGjRw8aNmxIYWEhS5cupWnTplx++eWceOKJ9O/fn+eee66ozGXL\nltG2bfH5FNq2bUt+fj4rVqxgxYoV7Ny5k7y8PAYNGkTHjh05//zz+fjjj4uOX7x4Ma1ateKGG26g\nU6dO9O7dm4ceeoj8/Pyinyv05w1ek4ULF7JixQratGnD4MGDOe+88/a53WFSSFq16F9JpFIPR4pW\nYWEhEyZMYNSoUdSpUweAwYMH8+OPP3LBBXsnVxk+fDjnnXdemefbtWsXjz/+OHfddVexBNC3b19e\nfvllunbtSuvWrVm6dCnvvfce27e7E4JkZWVRv379YucKJsZff/2Vww8/nKOOOoqJEycybtw46tev\nz4wZM1i/fj0NGjSIKe4lS5YwfPhw8vPz6d+/P/fdd19RDDk5OUyfPp3x48dz7LHHMnfuXP76179S\nv359zjnnnBLjDJb/66+/ssPryX799dd5+OGHqV+/Pk888QSjRo1i9uzZHHzwwWzcuJHXX3+dcePG\nMW7cOJYuXcq1115LRkYGV111Fe3atWPevHl07tyZDz74gJNOOons7GzuvvtuJk6cyGWXXcZjjz1G\ns2bNOPPMM+nRowf16lXc3AOmgiRZp0+0UjPqOJszZw5btmzhoosuKtr29NNPM2vWLKZPn85XX33F\nq6++ypw5c4o6eiKZPn06DRo0oE+fPsW2/+1vf6NPnz5cc801dOvWjddff50hQ4ZE9VRTIBAgLS2N\nxx9/nNq1a3POOecwYMAAtm7dyqmnnlp0jmjj7tSpE19//TVz5swhPz+fiy++mLy8PBzvttW5555L\n+/btycjIYMCAAfTu3bvodkBZcQb9+c9/pmXLlmRmZnLDDTdQr1493nrrraL9PXv2pGfPnqSnp9Ol\nSxeGDBlSVMbo0aP57rvv6NatGzVr1mTYsGHce++9DB06lEDAHZ/ZoUMHDjroIA4//HCWLVtWZmwm\nCVlTPXXNnDmT3r17k5Gxd5aeqVOnMmzYMI4//njS09Np27YtF198MdOmTYtwpr3nCzaHQ9WqVYtb\nb72V+fPns3jxYh544AF2795N8+bNAWjUqFGxHnRwm9M7duygcePGALRs2ZLHH3+cxYsX8/7773Pd\nddexYcMGWrRoUa64DznkEP7xj3+watUqPvroIxo2bEj16tX3qVEefPDBbN68GYDGjRvvE2fwc+PG\njWnSxB0CFnqOtLQ0WrRowaZNmwBo0qRJxDKaNm3KCy+8wJIlS3j00UdZtGgRGzZsYPjw4WRnZ1O7\ndu1i1zW8192kCBuOlJp27drFggUL6N27d7HthYWFxe4lgjvUxymjI3Ht2rWsWbOG008/fZ99n332\nGZ988kmxbR9++CFdunQBoEOHDqxYsaLoPh/A8uXLyc3NpWPHjgC8++67fP/990X7t2zZwurVqzn5\n5JOjinv8+PFcfvnlxfbn5rpDdqpXr05aWhpHHXUUX331VbFjfvzxR1q2dJ/Cat++/T41vM8++4yM\njAzatm3LEUccQfXq1Yt1pBUUFPDTTz8VnePoo4+OWEaobdu2cf/99zN2rLvUdWZmZrFEmZWVZRO4\npCqrcaamVatWsWfPHtq0aVNse9++fXnllVdYuXJlUSfLq6++yplnnll0zKWXXlrUIx705ZdfUr16\ndY466qh9yvr888/5y1/+wnfffUdeXh4PPfQQGzduZOTIkQAMHDiQ9PR0xo8fz65du9i0aRP3338/\nPXv25PDDDwfc+4Z33HEH27dvJysri1tuuYUjjzySfv36RRV3t27d+Oijj3jppZfIzc0lKyuLcePG\n0bRpUzp06ADAyJEjeffdd5k9ezZ5eXm89957zJ07l2HDhgEwdOhQfvrpJ5599llycnL4/vvvefjh\nh7ngggvIzMykQYMGnHfeeTzyyCOsXLmSnJwcJk6cSE5ODoMGDQLc+65ffvklzz33HLm5uSxZsoTX\nXnutqIxQd911FyNGjODggw8GoFWrVlSvXp358+ezevVqfvrpJ0444YQYf/MmKaRojbPKdw5t2bKF\nQCCwz0QgY8aMoW7dulx33XVs3ryZunXrMmjQIK65Zu+yv+vXr9+nybplyxbq1atHtWr79gKOHDmS\nbdu28cc//pHdu3dzzDHH8MwzzxQlhMzMTJ555hnuvfdeunXrRo0aNejduzd///vfi87xz3/+k9tu\nu40+ffrgOA7dunVj7NixReWVFXfnzp157LHHmDRpEuPGjaN27dq0a9eOqVOnFtXaBg4cyK5du3jo\noYe46aabOOiggxg3bhw9e/YEoEWLFkyZMoX777+fCRMmcMABB3DWWWcVjUgAuPXWW6lRowaXX345\n2dnZtGnThueff77olkPHjh2ZNGkSEydO5F//+heNGjXimmuu2SdxvvXWW+zatYuhQ4cWbcvIyGDs\n2LHccccd5Ofnc/fdd9OwYcMoftsm6SRZQoxWctV/y8FxHKe0NcwLc7az/D+3E7ZIvaki1qxZwy+7\n9nDEkbH9/m1d9ejVSt+/NnTNs5+IemB7zsw/J02+qtQ1zkBaGlnbrdOgqvrtt9+oUaPkR25Nkojj\nvUsROR54A5igqo+GbO8HvKOqad7nYbhLAhcCk1V1qreW+rPAIbjLBg9X1bWllZWa9eRoVavJd2t/\n9DsK45MtW3+hYUObizWpxekep4jUBsbjLvsbur0mcDPws/e5DnAb0BvoCVwvIg2Ai4BtqnoqcB8w\nNlJ5lTpxBqpl8PHnq/0Ow/gka8dOansPNJgkFb9e9VxgILA5bPstwMNAcDaYzsASVc1W1RxgIdAV\n6IVbWwWY620rVaVOnACfLFtra89UQYWFhewpSKp5IUwJAoFA1K9IVLVAVYtNhSVu50Ybbx31oGbA\n1pDPW4Dm3vZfvHMVAo6IlHors9InztWb4N4HHvM7DJNgr73+Bu07dfE7DFOGeCXOMMG/mOOBGyMd\nSOkd5BELrPSJszCtBuOnf8Hd4x6xmmcVUFhYyKuvzeDwo4+nceMD/Q7HlCGQFoj6FQsROQhoDbwi\nIp8AzUXkfWADbu0yqAXu/c+fg9u9jqKAqpY6jsGXXnUReRD3XoMDjFbVpSH7Tse9OVsAzFbVe/e3\nvDynFuNe+pL/zBvFySccTteOrWl12ME0qH+ArUeT4hzH4bfffmPL1l/I2rGTPQUO7Tt1saSZIirg\nv78AbtL7GSh6CkVE1qrqaSJSC3hKROrh5piuuD3sBwBDgDnAWcC8SIUkPHGKSA/gSFU9RURaA1OB\nU0IOmQj0xf0LMF9EZqjqqv0ttzCtBiu3wMr3fubpd3+AghycwoKyv1gOy/9zO23PubtCzl1e86aN\n4cC6GWzNzvM7lGLiEVONGnU56rjDrCMoBcUrcYrIycAUoAmQLyKjgJ6qus07xAFQ1d9FZAxu77sD\n3Kmq2SIyHegjIguAHOCySOX5UeMs6r1S1dUi0kBEMlV1l4i0wh0SsAFARGbjDhvY78QZKlAtHaql\nV9jofxEhrWb9sg9MoCOOFJrVy+CAHcmVOJMxJpM48UqcqroIaBthf6uQ9zOAGWH7C4ER4d8rjR/3\nOIt6rzxb2XvPobQeL2NMZRSI4ZVEkuHJoUiXJMkulzEmnlK1j8GPxFnUe+U5CNjovQ/v8WrpbYvo\n9y8eiVtw8ZKMMYHbNE42yRhTzWSoUoRJtpji8ey8Jc7ozQHuAiaLSAdgg6ruBlDVH0TkABE5FDdh\nnon7KFREpU3y4Zffv3gk6WJa+8EEmtXLYFOS3U9M1piSbUKNZJ3kY3+Frq+VShKeOFX1ExH5TEQW\n4g4HuFpELgV2qOqbwJXAy97hr6jqt4mO0RiTIKlZ4fTnHqeq3hy2aXnIvgUUH55kjKmkrKlujDEx\nssRpjDExssRpjDGxSs28aYnTGOMfq3EaY0yMbDiSMcbEyGqcxhgTq9TMm5Y4jTH+sRqnMcbEyBKn\nMcbEyBKnMcbEKjXzpiVOY4x/bDiSMcbEyJrqxhgTo3gmThE5Hnc9swmq+qiIHAw8g5vn9gAXq+pm\nERmGu7JlITBZVad6SwI/CxyCO93lcFVdW1pZqVlPNsZUDnFac0hEagPj2bt6JcA9uImxJ25CvcE7\n7jbcRSB7AteLSAPcCdO3qeqpuMuTj41UniVOY4xvAoFA1K8y5AIDgc3sTbNXs3c1y1+ARkBnYImq\nZqtqDrAQd231otV3gbnetlJZ4jTG+CZeiVNVC1Q1N2zbblUtEJFqwFXAi5S+km7R6rveUsGOiJR6\nK9MSpzHGN4FA9K/y8JLmC8BcVX2/pBBKCy3SeS1xGmN8k5YWiPpVTs8Aa1T1Hu9z+Cq7LbxtRdu9\njqKAqpa6PJ71qhtjfFMBw5GKTuj1nueq6l0h+z8FnhKReri9511xe9gPAIbgrsJ7FjAvUiGWOI0x\nvolX3hSRk4EpQBMgX0T+DFQDfheRYBN9hapeIyJj2Nv7fqeqZovIdKCPiCwAcoDLIpVnidMY45v9\naIIXo6qLgLZRHjuDvb3twW2FwIhoy7PEaYzxTYo+OGSJ0xjjH3vk0hhjYhSvpnqiWeI0xvjGapzG\nGBOjFM2bljiNMf6xGqcxxsQoRfOmP4lTRB7EnaXEAUar6tKQfeuAH3FH9QMMU9WfEx6kMabCWY0z\nSiLSAzhSVU8RkdbAVOCUkEMcoL+q/pbo2IwxiZWiedOXST6K5r1T1dVAAxHJDDsmRS+nMSYWCZjk\no0L4kTiL5r3zbMWdDy/UEyKyQEQizsJsjEltcZzIOKGSoXMowN6p7sGd1v5dIAt4U0QGe8+Wlur3\nLx6pwPDKJxljAmhWL8PvEPaRjDHVTIb/MsIkW0y/73HKPqgMSZYPo+bHryJ8PryDgI3BD6o6Lfhe\nRGbjPrgfMXFuyd4T5xD3T5O66UkX089ZObQ7pC7Lfsz2O5RikjWmeCSFeKqVHki6mOIh2WqS0fKj\nqT4HOB9ARDoAG1R1t/e5nojMF5Fa3rHdgeU+xGiMSYCKngG+oiS8xqmqn4jIZyKyEHfI0dUicimw\nQ1XfFJEZwMcisgv4oqxmujEmdaVqjdOXuyaqenPYpuUh+yYBkxIbkTHGDymaN5Oic8gYU0WlpaXm\nsmeWOI0xvrEapzHGxMjucRpjTIxSNG9a4jTG+CeeNU4ROR73ce4JqvqoiBwMvIA77HIjcImq5nnL\nBo8GCoHJqjrVW0v9WeAQ3NE+w1V1bWllpeadWWNMpRCvcZwiUhsYz95lfwHuBh5W1e7At8AIEamD\n+3Rib6AncL2INAAuArap6qnAfUDEx70tcRpjfJMWCET9KkMuMBDYHLKtBzDTez8LOB04CViiqtmq\nmgMsBLoSMvkQMNfbVnrcsfyQxhgTT/GaHUlVC1Q1N2xzHVUNPvscnEyomfc+aEvI9l+8cxUCjoiU\neivT7nEaY3yTwNniSisp1u1AhMQpIhFro15WNsaYcqvg4Ui7RKSGVxNtgTvBUPgkQy2ARSHbv/I6\nigKqml/aiSMlx/wIr+Sa+scYk5IqYJKPAHtri//Dm1AIGAy8AywGOnkTCmXi3sv8EHfyoSHesWcB\n8yIVUmqNU1Xt/qcxpkIF4rTYg4icDEwBmgD5IjIK6A88671fBzynqgUiMoa9ve93qmq2iEwH+ojI\nAiAHuCxSeWXe4xSRhsAtQDNVvVhEzgY+UdWtZXzVGGMiitc9TlVdhDt3b7i+JRw7g7A5fr1bjyOi\nLS+aWuVTwHqglfe5BvBctAUYY0xpUnXpjGgS54GqOhF3nBSq+m+gToVGZYypEqqlBaJ+JZNohiM5\nXi8TACLSFKhdcSEZY6qKJKtIRi2axPkIsARoLiKzcEfej67QqIwxVUKyNcGjVWbiVNVXReQT4GTc\n5voVqrqxjK8ZY0yZUjRvln2P0xvrdDZwGtAPONt7oN4YY/ZLHJ9VT6homuqv4T7P+TFuou2O+zD9\nWRUYlzGmCkiudBi9aBJnXVXtH/L5MRH5sKICMsZUHcnWWx6taIYjfS8iBwU/iEhz3LntjDFmv6Tq\nOM5Ik3ws8N7WAL4TkdW4Mya3Bj5PQGzGmEouyfJh1CI11W+LsM+JsM8YY6KSbDXJaEWa5OOD4Huv\nZ72h97Em8CLQqUIjM8ZUeil6izOqST7+hjvJR00gG/epoRcrOC5jTBWQqjXOaDqHzgea4s6IdCDw\nB2B1hUZljKkSAjG8kkk0iXO3N4NyBoCqzsTGcBpj4qAyT/KxTUQuBVaIyDPAKtwaaLmFr38ctu90\n3OU5C4DZqnrv/pRljElelbmp/kdgPnA97vjNFrjN9XIJW/+4JBOB83CntO8rIseUtyxjTHKrgKUz\nEiLSOM5WYZuaAq947/dnOFJw/eMxpZS5TVU3eJ9n4y4cv2o/yjPGJKlkewY9WpGa6vOInCAPL0+B\nqloAFIhISbtLWvP4iPKUY4xJfimaNyOO4zwsgXEEhSfqqC5rk7rpZR+UYMkWUzCedofU9TmSfSVj\nTLXSk++yiIu2AAAR0ElEQVS/6GSL6fc9+/8cTKre44ymcyiRwtc8bglsKOtLW7KTa7XiJnXTky6m\nn7NyaHdIXZb9mO13KMUka0zxSArxVCs9kHQxxUO8ltL1HtJ5HqiP+5j4Xbi3+F7witkIXKKqeSIy\nDHcy9kJgsqpO9Svu8tjnT42q/gAcICKHikh14Ezc9Y6NMZVQHIcjXQasVtVeuGPPJ+Emz4dVtTtu\nx/YIEamD+zh5b6AncL2INIg17oTXOEtY//jPwDPA96r6JnAl8LJ3+CuqajMxGVNJxXF45mb2Lg/c\nELevpCcwyts2C/gLsAZYoqrZACKyEHcEz1uxFBbNI5eHAf8CGqtqTxG5HPhAVb+JpaCgCOsfB/cv\nAE4pz7mNMaklXvc4VfXfIjJcRL4B6uG2Vt9S1eA9s61Ac0rugG4ea3nRNNWnsPc+AbgZe3KsBRlj\nTLi0QPSvSETkYuBHVT0KOB14lOKdzaWdoVyZO5rEma6q/8F9kgdV/bC8hRljTKg4DoA/Ba8/RFW/\nwu1Y3i0iNb39LXA7n8vVAR0umsTpiEj94AcRORZ3piRjjNkvcVys7VugM4CIHArsAt4DBnv7BwPv\nAIuBTiJSz+uJPwVYsO/pIoumc+huYBHuuurLgcbAxbEWZIwx4eI4rOdJYKqIfICb167AncXteREZ\nBawDnlPVAhEZg/vItwPcGewoikU066q/LyLtgeNwH5dUVc2JtSBjjAkXr1mPVHU3cGEJu/qWcOwM\nYMb+lBdNr/o9uJk5+BM6IoKq3r4/BRtjTIo+OBRVTbnAe+UD1YBeuN39xhizX+LVq55o0TTV7wz9\nLCLVgNcrKiBjTNVRGWdHKk0GcGS8AzHGVD0pmjejusf5E8UHkjYEnq2ogIwxVUeyNcGjFU2Nsysh\nHUPATlXNqriQjDFVRSBFn6WJmDhFJIC7LtDgSMcZY0x5VPdzfrb9EDFxqqojIt+IyAjgYyAvZN/3\nFR2cMaZyq8wTGQ+l5CU0yrV0hjHGBFW6e5wicrGqTvNpCQ1jTBWQohXOiAPgRyYsCmNMlRTHST4S\nKtnWHDLGVCGVrqkOdBGR9aXsc1T1kIoIyBhTdVRLsppktCIlzi9wO4ZS8yczxiS9FM2bERNnjrfq\npDHGVIjK2FT/NGFRGGOqpGTr9IlWqYlTVW9KZCDGmKonRfOm9aobY/xT6WqcxhhT0VI0b1riNMb4\npzIORzLGmAoVz7QpIsOAv+Iu83M7sBx4AfcJyY3AJaqa5x03GigEJqvq1FjLStFJnYwxlUG8HrkU\nkUa4ybIrMBA4B7gLeFhVu+Ouuz5CROoAtwG9gZ7A9SLSINa4rcZpjPFNHGucpwP/85YJ3g2MEpHv\ngVHe/lnAX4A1wJLgWuoishA32b4VS2G+JE4ROR54A3eS5EfD9q0DfsRdWRNgmKr+nNgIjTGJEMdb\nnIcCtUXkP0AD3NpmHVXd4+3fCjQHmnnvg7Z422OS8MQpIrWB8cB/SznEAfqr6m+Ji8oY44c4TmSc\nhrse2iDgMOCD8KJKC6G8hSVaLu49iM0RjknNrjZjTEzSYniVYRPwiaoWeqtTZAPZIlLT298C+Nl7\nNQv5XktgQ3niTihVLVDV3DIOe0JEFojI2IQEZYzxRRzn45wD9BKRgNdRVAf4HxBcL20w8A6wGOgk\nIvVEJBM4BVgQa9zJ2Dl0G/AukAW8KSKDVXVGpC80qZuekMBikWwxBeNpd0hdnyPZVzLGVCs9+Ro9\nyRbTig279vsc8Wqqq+rPIvIasMjbdA2wFHheREYB64DnVLVARMbg3ip0gDuDHUWxSLrEqarTgu9F\nZDbQFoiYOHfnlbQkkn/qZASSLiZIzriSNaZ4JIV4OrZFZtLFFA/xbPKq6mRgctjmviUcN4MyckpZ\n/Eyc+/ypEZF6wEzczqHfge7Aa4kOzBiTGJV5lcu4EpGTgSlAEyBfRP4MPAN8r6pvisgM4GMR2QV8\nUVYz3RiTulIzbfqQOFV1EW7zu7T9k4BJiYvIGOOXFK1wJt89TmNM1ZGWonVOS5zGGN/YfJzGGBOj\nFM2bljiNMf6xproxxsTIapzGGBMjS5zGGBOjgDXVjTEmNmmpmTctcRpj/GPDkYwxJkbWVDfGmBhZ\nU90YY2JkNU5jjIlRit7itMRpjPFPiuZNS5zGGP9US9EqpyVOY4x/UjNvWuI0xvgn3p1DIlIL+Bq4\nG5gHvIC7tNFG4BJVzRORYcBooBCYrKpTYy3Hj3XVjTEGcDuHon1F6VbgF+/93cDDqtod+BYYISJ1\ncFfS7Q30BK4XkQaxxm2J0xjjm0AMr7KISGugNfC2t6kH7uKPALOA04GTgCWqmq2qOcBCoGuscVvi\nNMb4J56ZEx4Arg85uo6q7vHebwWaA82890FbvO0xscRpjPFNIIb/RSIifwQ+VNUfi04dXlRpIZSD\ndQ4ZY3wTx0cuzwBaich5QEsgF8gWkZpek7wF8LP3ahbyvZbAJ7EWZonTGOOfOCVOVR0afC8idwDr\ngFOAwcCL3v+/AywGnhKRekCBd8y1sZZnTXVjjG/i1VQvgQPcAVwqIh8C9YHnvNrnGOC/wHvAnaqa\nHevJrcZpjPFNRTw4pKp3hXzsW8L+GcCM/SnDEqcxxjcp+uCQJU5jjI9SNHNa4jTG+Mbm44yBiNwP\ndPPKH6uqb4TsOx24D7fHa7aq3utHjMaYipeqM8AnvFddRE4DjlXVU4D+wENhh0wEzsN9DKqviByT\n4BCNMYkS3yeHEsaP4UgfAhd473cAdUQkACAirYBtqrpBVR1gNu7D+MaYSqgChyNVqIQ31VW1ANjt\nfRwJvO0lSSj5OdIjEhieMSaBUnQeY/86h0TkHGAE0CdksxN2WIpeVmNMNFL1P3C/Oof6ATcD/cNG\n7Zf0HOmGss5XJyP5Ln8yxgTJGVcyxnRsi0y/Q9hHssW0YsOu/T9J8v3qo5LwxOk9I/oA0EtVt4fu\nU9UfROQAETkUN2GeCVxU1jl354VXVP1VJyOQdDFBcsaVrDHFJSnE0bEtMpMupnhItnuX0fKjxnkh\n0Aj4t4gEt80Dlqvqm8CVwMve9ldU9dvEh2iMSYRUHY7kR+fQZGByhP0LcGcsMcZUdpY4jTEmNtZU\nN8aYGNlwJGOMiVGK5k1LnMYYH6Vo5rTEaYzxjd3jNMaYGNlwJGOMiZUlTmOMiY011Y0xJkbxHI4U\nPkE6sBR4AXf6zI3AJaqaJyLDgNFAITBZVafGWpYtD2yM8U285jEuYYL0icBdwMOq2h34FhghInWA\n23Dn+e0JXC8iDWKN2xKnMcY3gUD0rzLsM0E60AOY6W2bBZwOnAQsUdVsb431hbirTcTEmurGGB/F\np61e0gTpQD9V3eNt2wo0p+TJ0pvHWp4lTmOMb+I9HMmbIH040A/4JmRXaSWVKwJrqhtjfBPHpnpw\ngvRbgAGquhPYJSI1vN0tcCdKL9dk6eEscRpjfBOvxdpCJkg/M2SC9P8B53vvBwPvAIuBTiJST0Qy\ncaewXBBr3NZUN8b4J35N9fAJ0h3gMuApERkFrAOeU9UCERkD/Nc75s6w5XuikpqjT0M4juMk49IL\nyRYTJGdcyRpTsi1TkaxLZxzXsu5+5ZBNO6L/5TerlzyLU1mN0xjjG5uP0xhjYhRI0cxpidMY45vU\nTJuWOI0xPkrRCqclTmOMf2x2JGOMiVGq1jhtALwxxsTIapzGGN+kao3TEqcxxjdpKZo5LXEaY3yT\nmmnTEqcxxk8pmjl9SZzha4Oo6hsh+9YBPwIF3qZhqvpzwoM0xlQ4G44UpdC1QUSkIfAF8EbIIQ7Q\nX1V/S3RsxpjEStFbnL4MR9pnbRARCb98KXo5jTGxiNdibYmW8BpnSWuDqGr41FJPiMhhwEeqenMi\n4zPGJFCyZcQo+TYA3lsbZARwTdiu24DrcZfuPE5EBic4NGNMgqQFAlG/qjwR6Scii0SkfhnHXSki\ndyYoLGOMiUrCa5wha4MMDFkbpGifiMwXkVrepu7A8kTHaIwxkfgxHCl8bRCAecByVX1TRGYAH4vI\nLuALVZ3hQ4zGGGOMMcYYY4wxxhhjjDGVUsoNjhKRdOBZ4BDc59mHq+rasGP2AB+FbOqtqoUVFM+D\nQGfcR0VHq+rSkH2nA/d5cc5W1XsrIoYYY1qHD3MBiMjxuI/WTlDVR8P2+XKdoohrHf5cq0hzOfj1\nb8rmlwiRirMjXQRsU9VhItIHGAsMDTtmu6qeVtGBiEgP4EjvufvWwFTglJBDJgJ9gZ+B+SIyQ1VX\n+RxTwucCEJHawHjgv6UckvDrFGVcflyrsuZy8OPflM0vESYVl87oxd5f2lygazLEoqqrgQYikgkg\nIq1wE/wG75HS2UBvP2MKkeiWRi4wENgcvsPH6xQxrhCJvlalzuXg47Wy+SXCpGLibAZsBfCa346I\nhNeca4rIiyLykYhcX8Gx/BLyeau3rVicni1A8wqMJVJM4eU+ISILRGRsAuJBVQtUNbeU3X5dp7Li\nCvLjWpU2l4Mv16qMmIISep38ltRNdREZCfwpbHPnsM8l/aW7EZjmvf9QRD5U1c/iHV8JIv3V9esv\ncgC3KRV0G/AukAW8KSKDfX7IIPw/wGSqufh2rULmcugTstnXa1VKTJB8/6YqXFInTlV9Gng6dJuI\nPIP7V3a511EUUNX8sO9NDjl+LtAWqIjE+TN7a5gABwEbvfcbwva19LZVtEgxoarBPyiIyGzca+Pn\nP/LweBN1ncrk17USkX7Azbj3DbNDdvl2rSLElIz/pipcKjbV5wBDvPdn4T6uWUREjhaRN0UkTUSq\n4XaMfF2BsZzvldsB2BBs0qjqD8ABInKodyvhTO/4ilZqTEkwF8A+NSQfr1PEuPy6VpHmcvDrWtn8\nEvtKpmZRVEQkDXgKOArIAS5T1Q0ichMwX1UXicg/gdOBPcBMVa2w+y7ePZ3uuEMxrgY6ADu85+5P\nBcZ5h76mqhMqKo4YYroWGA4E5wK4NgHxnAxMAZoA+cA24Bnge5+vU1lx+XGtrgDuADRkc+hcDgm/\nVlHElPDrZIwxxhhjjDHGGGOMMcYYY4wxxhhjjDGVRMqN4zT7x1uvfg3wsbcpHfgBuEpVd5TznH8C\nuqrqcBF5GbhBVTeWcmwXYFP4VIARzl0dyFPVtLDtdwLVVPW2CN9dB/RS1e+jLOtZYIH3xJoxpUrF\nJ4fM/tuiqqd5r264j+3dGnpACbPfREVV/1Ba0vSMAFqV59xhwp/bLu2YWH4OJ8rzmiouqZ9VNwmz\nALgCimppr+A+mTVYRC4ArsFNQFuBP6nqNhG5CrgSWI/7DLUT8v1ewDpgEtDRK2M87tM55wMnerNW\nfQ88CtQGMoFbVHWuiByNO0nLbuCDsoIXkSuBP+I+SZYLXBhSe75CRE7EfTroGlWdLyKHlFSud7y1\nwkyZrMZZxXnP85+HO+ciuAlQVXWwiBwM3II7g/6pwHzgFu/Z5buB7qp6BtA45JTBGtswoImqdgH6\nA5cBM4FlwI2q+gHwODBeVXsD5wBPefHcATylqj2Br6L4MTKAAd7k1euAi0P2bfHOPxr4l7ettHKN\niYrVOKumA0Xkfe99Gm7SfDBkf/D+ZxfcmajmiAhADdxa4hHAOlXN8o57Hzgh5PsB4CRvO17tbyCA\nd56g04BMEQkua5KHWzM8Dnd5CAibxKUU24GZ3nkOw60BB73n/f8nwLFllGtMVCxxVk1by1haJM/7\n/xzgU1U9K3Sn1/QNXcOppH9HDlBWLS4HGKSq28LOHwg5f8RziEhL3Jl72qjqLyLyQAlxgJvMg+cs\nrdwywjXGZU11E8lS4CQRaQogIkNE5GzgW6CVN6VYgJKXb/gYt4kenHpskTd/aiFu0xrcBfUu9I5p\nLO4icwAr2btO0ullxHgg8IuXNBsC/YCa3r7Q2Lqyd7qz0so1JiqWOKumqHqOvZUKRwNvich83KnD\nPvHmZLwPt1PpTSB8aJEDvAqsFZGFuHNGjlfVPbhN5ydF5FzgWmCQiHwIvI27hhS490+vEpF3AcGd\nHrC0n2MZ8I2ILMa9d3kHcJmIdPX2NxCRWbidU3/xvldauVFfG2OMMcYYY4wxxhhjjDHGGGOMMcYY\nY4wxxhhjjDHGmKT3/zchy5j5OPk0AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f984e38a5d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.87      0.87      0.87      1850\n",
      "          1       0.59      0.62      0.60       597\n",
      "          2       0.73      0.68      0.70       481\n",
      "\n",
      "avg / total       0.79      0.79      0.79      2928\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "from mpl_toolkits.axes_grid.anchored_artists import AnchoredText\n",
    "\n",
    "def plot_confusion_matrix(cm, title='Confusion matrix', cmap=plt.cm.Blues):\n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    plt.title(title)\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(len(classes))\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')\n",
    "\n",
    "fig = plt.figure()\n",
    "ax = plt.subplot()\n",
    "Accuracy_ = score[1]*100\n",
    "at = AnchoredText(str(Accuracy_)+\"%\",\n",
    "                      prop=dict(size=15), frameon=True,\n",
    "                      loc=2,)\n",
    "at.patch.set_boxstyle(\"round,pad=0.,rounding_size=0.5\")\n",
    "ax.add_artist(at)\n",
    "plt.ylabel('True Label')\n",
    "plt.xlabel('Predicated Label')\n",
    "plot_confusion_matrix(cm, title='Confusion Matrix')\n",
    "plt.show()\n",
    "print(metrics.classification_report(y_test, classes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}