HAN3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "tf.logging.set_verbosity(tf.logging.WARN)\n",
    "import pickle\n",
    "import numpy as np\n",
    "import os\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import accuracy_score\n",
    "import os\n",
    "import math\n",
    "# from tensorflow.python.client import device_lib601"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from numpy import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('embedding_p2', 'rb') as f:\n",
    "    word_embedding = pickle.load(f)\n",
    "\n",
    "with open('vocab_p2', 'rb') as f:\n",
    "    vocab = pickle.load(f)\n",
    "\n",
    "\n",
    "word2id = dict((w, i) for i,w in enumerate(vocab))\n",
    "id2word = dict((i, w) for i,w in enumerate(vocab))\n",
    "\n",
    "unknown_token = \"UNKNOWN_TOKEN\"\n",
    "\n",
    "# with open(\"processed_data/train_final\", 'rb') as f:\n",
    "#     full_data = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# f = open('vocab_p2','wb')\n",
    "# pkl.dump(vocab, f, protocol=2)\n",
    "# f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# f = open('embedding_p2','wb')\n",
    "# pkl.dump(word_embedding, f, protocol=2)\n",
    "# f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# f = open('fulldata_p2','wb')\n",
    "# pkl.dump(full_data, f, protocol=2)\n",
    "# f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# random.shuffle(full_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# full_data[2][2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Parameters\n",
    "mode = 'train'\n",
    "num_classes = 2\n",
    "batch_size = 100\n",
    "vocab_size = len(vocab)\n",
    "unk_vocab_size = 2\n",
    "word_emb_size = len(word_embedding[0])\n",
    "avg_no_sent = 10\n",
    "avg_sent_size = 23\n",
    "max_title_size = 22\n",
    "hidden_size = 100\n",
    "hidden_size2 = 50\n",
    "fc_layer_hidden_size = 100\n",
    "keep_prob = 0.5\n",
    "l2_lambda = 0.001\n",
    "init_lr = 0.001\n",
    "decay_steps = 5000\n",
    "decay_rate = 0.96\n",
    "clip_norm = 1\n",
    "clipping = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def word_emb_maker(mode, x, word_emb_mat, shape_arr):\n",
    "    with tf.name_scope(\"word_embedding_layer\"):\n",
    "        if mode == 'train':\n",
    "            unk_word_emb_mat = tf.get_variable(\"word_emb_mat\", dtype='float', shape=[unk_vocab_size, word_emb_size], initializer=tf.contrib.layers.xavier_initializer(uniform=True, seed=0, dtype=tf.float32))\n",
    "        else:\n",
    "            unk_word_emb_mat = tf.get_variable(\"word_emb_mat\", shape=[unk_vocab_size, word_emb_size], dtype='float')\n",
    "\n",
    "        final_word_emb_mat = tf.concat([word_emb_mat, unk_word_emb_mat], 0)\n",
    "        Wx = tf.nn.embedding_lookup(final_word_emb_mat, x)\n",
    "        \n",
    "        return Wx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def word_emb_maker_title(mode, x, word_emb_mat, shape_arr):\n",
    "    with tf.name_scope(\"word_embedding_layer_top\"):\n",
    "        if mode == 'train':\n",
    "            unk_word_emb_mat_t = tf.get_variable(\"word_emb_mat_t\", dtype='float', shape=[unk_vocab_size, word_emb_size], initializer=tf.contrib.layers.xavier_initializer(uniform=True, seed=0, dtype=tf.float32))\n",
    "        else:\n",
    "            unk_word_emb_mat_t = tf.get_variable(\"word_emb_mat_t\", shape=[unk_vocab_size, word_emb_size], dtype='float')\n",
    "\n",
    "        final_word_emb_mat_t = tf.concat([word_emb_mat, unk_word_emb_mat_t], 0)\n",
    "        Wx_t = tf.nn.embedding_lookup(final_word_emb_mat_t, x)\n",
    "        \n",
    "        return Wx_t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = tf.placeholder('int32', [None, avg_no_sent, avg_sent_size], name=\"x\")\n",
    "y = tf.placeholder('int32', [None], name=\"y\")\n",
    "word_emb_mat = tf.placeholder('float', [None, word_emb_size], name='emb_mat')\n",
    "\n",
    "x_mask = tf.placeholder('int32', [None, avg_no_sent, avg_sent_size], name=\"x_mask\")    \n",
    "x_len = tf.reduce_sum(tf.cast(x_mask, 'int32'), 2)    \n",
    "\n",
    " #title\n",
    "x_title = tf.placeholder('int32', [None, max_title_size], name=\"x_title\")\n",
    "x_title_mask  = tf.placeholder('bool', [None, max_title_size], name='x_title_mask')\n",
    "x_title_len = tf.reduce_sum(tf.cast(x_title_mask, 'int32'), 1)\n",
    "\n",
    "x_title_total = tf.placeholder('int32', [None, max_title_size+1], name=\"x_title_total\")\n",
    "x_title_total_mask  = tf.placeholder('bool', [None, max_title_size+1], name='x_title_total_mask')\n",
    "x_title_total_len = tf.reduce_sum(tf.cast(x_title_total_mask, 'int32'), 1)\n",
    "\n",
    "sent_mask = tf.placeholder('int32', [None, avg_no_sent], name=\"sent_mask\")    \n",
    "sent_len = tf.reduce_sum(tf.cast(sent_mask, 'int32'), 1) \n",
    "\n",
    "is_train = tf.placeholder('bool', [], name='is_train')\n",
    "input_keep_prob = tf.cond(is_train,lambda:keep_prob, lambda:tf.constant(1.0))\n",
    "\n",
    "\n",
    "Wx = word_emb_maker(mode='train', x=x, word_emb_mat=word_emb_mat, shape_arr=[unk_vocab_size, word_emb_size])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Wx_title= word_emb_maker_title(mode='train',x=x_title, word_emb_mat=word_emb_mat, shape_arr=[unk_vocab_size, word_emb_size])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<tf.Tensor 'Sum:0' shape=(?, 10) dtype=int32>,\n",
       " <tf.Tensor 'x_mask:0' shape=(?, 10, 23) dtype=int32>,\n",
       " <tf.Tensor 'Reshape:0' shape=(?, 23, 100) dtype=float32>,\n",
       " <tf.Tensor 'word_embedding_layer/embedding_lookup:0' shape=(?, 10, 23, 100) dtype=float32>,\n",
       " <tf.Tensor 'Reshape_1:0' shape=(?,) dtype=int32>)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "flat_wx = tf.reshape(Wx, [-1, avg_sent_size, word_emb_size])\n",
    "flat_xlen = tf.reshape(x_len, [-1])\n",
    "x_len, x_mask, flat_wx ,Wx ,flat_xlen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.variable_scope(\"lstm_layer\"):\n",
    "    cell_fw = tf.contrib.rnn.BasicLSTMCell(hidden_size,state_is_tuple=True)\n",
    "    cell_bw = tf.contrib.rnn.BasicLSTMCell(hidden_size,state_is_tuple=True)\n",
    "\n",
    "    d_cell_fw = tf.contrib.rnn.DropoutWrapper(cell_fw, input_keep_prob=input_keep_prob)\n",
    "    d_cell_bw = tf.contrib.rnn.DropoutWrapper(cell_bw, input_keep_prob=input_keep_prob)\n",
    "    \n",
    "    (fw_h, bw_h), _ = tf.nn.bidirectional_dynamic_rnn(d_cell_fw, d_cell_bw, flat_wx, sequence_length=flat_xlen, dtype='float', scope='lstm')\n",
    "    h = tf.concat([fw_h, bw_h], 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'lstm_layer/concat:0' shape=(?, 23, 200) dtype=float32>"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "h"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "h_4d = tf.reshape(h, [-1, avg_no_sent, avg_sent_size, 2*hidden_size])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.variable_scope(\"W_att_local\"):\n",
    "    W_att_local = tf.get_variable(\"W_att_local\", shape=[2*hidden_size, 1], initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1, seed=0))\n",
    "    flat_h = tf.reshape(h, [-1, 2*hidden_size])\n",
    "    h_tanh = tf.tanh(flat_h)\n",
    "    u_flat = tf.matmul(h_tanh, W_att_local)\n",
    "    u_local = tf.reshape(u_flat, [-1, avg_no_sent, avg_sent_size])\n",
    "    final_u = (tf.cast(x_mask, 'float') -1)*(1e7) + u_local\n",
    "\n",
    "    c = tf.reduce_sum(tf.multiply(h_4d, tf.expand_dims(tf.nn.softmax(final_u, dim=-1), 3)), 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'W_att_local/add:0' shape=(?, 10, 23) dtype=float32>"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_u"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<tf.Tensor 'W_att_local/MatMul:0' shape=(?, 1) dtype=float32>,\n",
       " <tf.Tensor 'W_att_local/Reshape_1:0' shape=(?, 10, 23) dtype=float32>,\n",
       " <tf.Tensor 'W_att_local/add:0' shape=(?, 10, 23) dtype=float32>,\n",
       " <tf.Tensor 'W_att_local/Sum:0' shape=(?, 10, 200) dtype=float32>,\n",
       " <tf.Tensor 'Sum_3:0' shape=(?,) dtype=int32>)"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "u_flat,u_local, final_u, c, sent_len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.variable_scope(\"lstm_layer_2\"):\n",
    "    cell_fw_2 = tf.contrib.rnn.BasicLSTMCell(hidden_size2,state_is_tuple=True)\n",
    "    cell_bw_2 = tf.contrib.rnn.BasicLSTMCell(hidden_size2,state_is_tuple=True)\n",
    "\n",
    "    d_cell_fw_2 = tf.contrib.rnn.DropoutWrapper(cell_fw_2, input_keep_prob=input_keep_prob)\n",
    "    d_cell_bw_2 = tf.contrib.rnn.DropoutWrapper(cell_bw_2, input_keep_prob=input_keep_prob)\n",
    "    \n",
    "    (fw_h_2, bw_h_2), _ = tf.nn.bidirectional_dynamic_rnn(d_cell_fw_2, d_cell_bw_2, c, sequence_length=sent_len, dtype='float', scope='lstm2')\n",
    "    h2 = tf.concat([fw_h_2, bw_h_2], 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "with tf.variable_scope(\"W_att_local_2\"):\n",
    "    W_att_local_2 = tf.get_variable(\"W_att_local_2\", shape=[2*hidden_size2, 1], initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1, seed=0))\n",
    "    flat_h_2 = tf.reshape(h2, [-1, 2*hidden_size2])\n",
    "    h_tanh_2 = tf.tanh(flat_h_2)\n",
    "    u_flat_2 = tf.matmul(h_tanh_2, W_att_local_2)\n",
    "\n",
    "    u_local_2 = tf.reshape(u_flat_2, [-1, avg_no_sent])\n",
    "    final_u_2 = (tf.cast(sent_mask, 'float') -1)*(1e7) + u_local_2\n",
    "\n",
    "    c2 = tf.reduce_sum(tf.multiply(h2, tf.expand_dims(tf.nn.softmax(final_u_2, dim=-1), 2)), 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'W_att_local_2/add:0' shape=(?, 10) dtype=float32>"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_u_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[None, 10]"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_u_2.get_shape().as_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<tf.Tensor 'W_att_local_2/MatMul:0' shape=(?, 1) dtype=float32>,\n",
       " <tf.Tensor 'W_att_local_2/Reshape_1:0' shape=(?, 10) dtype=float32>,\n",
       " <tf.Tensor 'W_att_local_2/add:0' shape=(?, 10) dtype=float32>,\n",
       " <tf.Tensor 'W_att_local_2/Sum:0' shape=(?, 100) dtype=float32>,\n",
       " <tf.Tensor 'word_embedding_layer_top/embedding_lookup:0' shape=(?, 22, 100) dtype=float32>)"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "u_flat_2,u_local_2, final_u_2, c2, Wx_title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Wx_final = tf.concat([tf.expand_dims(c2,1), Wx_title],1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'concat:0' shape=(?, 23, 100) dtype=float32>"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Wx_final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.variable_scope(\"lstm_layer_top\"):\n",
    "    cell_fw_t = tf.contrib.rnn.BasicLSTMCell(hidden_size,state_is_tuple=True)\n",
    "    cell_bw_t = tf.contrib.rnn.BasicLSTMCell(hidden_size,state_is_tuple=True)\n",
    "\n",
    "    d_cell_fw_t = tf.contrib.rnn.DropoutWrapper(cell_fw_t, input_keep_prob=input_keep_prob)\n",
    "    d_cell_bw_t = tf.contrib.rnn.DropoutWrapper(cell_bw_t, input_keep_prob=input_keep_prob)\n",
    "    \n",
    "    (fw_h_t, bw_h_t), _ = tf.nn.bidirectional_dynamic_rnn(d_cell_fw_t, d_cell_bw_t, Wx_final, sequence_length=x_title_len, dtype='float', scope='lstm_top')\n",
    "    ht = tf.concat([fw_h_t, bw_h_t], 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'x_title_total_mask:0' shape=(?, 23) dtype=bool>"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_title_total_mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.variable_scope(\"W_att_local_top\"):\n",
    "    W_att_local_t = tf.get_variable(\"W_att_local_t\", shape=[2*hidden_size, 1], initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1, seed=0))\n",
    "    flat_h_t = tf.reshape(ht, [-1, 2*hidden_size])\n",
    "    h_tanh_t = tf.tanh(flat_h_t)\n",
    "    u_flat_t = tf.matmul(h_tanh_t, W_att_local_t)\n",
    "\n",
    "    u_local_t = tf.reshape(u_flat_t, [-1, max_title_size+1])\n",
    "    final_u_t = (tf.cast(x_title_total_mask, 'float')-1)*(1e7) + u_local_t\n",
    "\n",
    "    ct = tf.reduce_sum(tf.multiply(ht, tf.expand_dims(tf.nn.softmax(final_u_t, dim=-1), 2)), 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'W_att_local_top/add:0' shape=(?, 23) dtype=float32>"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_u_t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[None, 23]"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_u_t.get_shape().as_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.variable_scope(\"fc_layer\"):\n",
    "    W = tf.Variable(tf.truncated_normal([2*hidden_size, num_classes], mean=0.0, stddev=0.1, seed=0), name=\"W\")\n",
    "    b = tf.Variable(tf.zeros([num_classes]), name=\"b\")\n",
    "    drop_ct = tf.nn.dropout(ct, input_keep_prob)\n",
    "    logits = tf.matmul(drop_ct, W) + b\n",
    "    predictions = tf.argmax(logits, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=y))\n",
    "global_step = tf.Variable(0, trainable=False, name=\"global_step\")\n",
    "\n",
    "learning_rate = tf.train.exponential_decay(init_lr, global_step, decay_steps, decay_rate, staircase=True)\n",
    "\n",
    "tv_all = tf.trainable_variables()\n",
    "tv_regu =[]\n",
    "for t in tv_all:\n",
    "    if t.name.find('b:')==-1:\n",
    "        tv_regu.append(t)\n",
    "        \n",
    "# l2 Loss\n",
    "l2_loss = l2_lambda * tf.reduce_sum([ tf.nn.l2_loss(v) for v in tv_regu ])\n",
    "\n",
    "total_loss = loss + l2_loss\n",
    "\n",
    "# Optimizer for loss\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "\n",
    "# Gradients and Variables for Loss\n",
    "grads_vars = optimizer.compute_gradients(total_loss)\n",
    "\n",
    "# Clipping of Gradients\n",
    "clipped_grads = grads_vars\n",
    "if(clipping == True):\n",
    "    clipped_grads = [(tf.clip_by_norm(grad, clip_norm), var) for grad, var in clipped_grads]\n",
    "\n",
    "# Training Optimizer for Total Loss\n",
    "train_op = optimizer.apply_gradients(clipped_grads, global_step=global_step)\n",
    "\n",
    "# Summaries\n",
    "var_summaries = []\n",
    "for v in tv_all:\n",
    "    var_summary = tf.summary.histogram(\"{}/var\".format(v.name), v)\n",
    "    var_summaries.append(var_summary)\n",
    "\n",
    "var_summaries_merged = tf.summary.merge(var_summaries)\n",
    "\n",
    "loss_summary = tf.summary.scalar(\"loss\", loss)\n",
    "total_loss_summary = tf.summary.scalar(\"total_loss\", total_loss)\n",
    "summary = tf.summary.merge_all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Model Description\n",
    "model_name = '3HAN_1.0'\n",
    "model_dir = 'output/' + model_name\n",
    "save_dir = os.path.join(model_dir, \"save/\")\n",
    "log_dir = os.path.join(model_dir, \"log\")\n",
    "\n",
    "if not os.path.exists(model_dir):\n",
    "    print(1)\n",
    "    os.mkdir(model_dir)\n",
    "if not os.path.exists(save_dir):\n",
    "    os.mkdir(save_dir)\n",
    "if not os.path.exists(log_dir):\n",
    "    os.mkdir(log_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\"   # see issue #152\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"2,3\"\n",
    "# print (device_lib.list_local_devices())\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "config.allow_soft_placement = True\n",
    "sess = tf.Session(config=config)\n",
    "sess.run(tf.global_variables_initializer())                          # For initializing all the variables\n",
    "saver = tf.train.Saver()                                             # For Saving the model\n",
    "summary_writer = tf.summary.FileWriter(log_dir, sess.graph)          # For writing Summaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# #Splitting\n",
    "# data_x = [data[0:2] for data in full_data]\n",
    "# data_y = [data[2] for data in full_data]\n",
    "# x_train, x_test, y_train, y_test = train_test_split(data_x, data_y, train_size=0.8, test_size=0.2, shuffle=True, stratify=data_y, random_state=0)\n",
    "\n",
    "# x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, train_size=0.9, test_size=0.1, shuffle=True, stratify=y_train, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def data_prepare(x):\n",
    "    #x=[news[1] for news in x_]\n",
    "    num_examples = len(x)\n",
    "\n",
    "    xx = np.zeros([num_examples, avg_no_sent, avg_sent_size], dtype=int)\n",
    "    xx_mask = np.zeros([num_examples, avg_no_sent, avg_sent_size], dtype=bool)\n",
    "    \n",
    "    xx_title = np.zeros([num_examples, max_title_size], dtype=int)\n",
    "    xx_title_mask = np.zeros([num_examples, max_title_size], dtype=bool)\n",
    "    \n",
    "    xx_total_title = np.zeros([num_examples, max_title_size+1], dtype=int)\n",
    "    xx_total_title_mask = np.zeros([num_examples, max_title_size+1], dtype=bool)\n",
    "    \n",
    "    ss_mask = np.zeros([num_examples, avg_no_sent], dtype=bool)    \n",
    "\n",
    "    for j in range(num_examples):\n",
    "        for i in range(avg_no_sent):\n",
    "            for k in range(avg_sent_size):\n",
    "                w = x[j][1][i][k]\n",
    "                if w == 'PAD':\n",
    "                    xx[j][i][k] = word2id[w]\n",
    "                elif w in word2id:\n",
    "                    xx[j][i][k] = word2id[w]\n",
    "                    xx_mask[j][i][k] = True\n",
    "                else:\n",
    "                    xx[j][i][k] = word2id['UNKNOWN_TOKEN']\n",
    "                \n",
    "    \n",
    "    for j in range(num_examples):\n",
    "        for i in range(max_title_size):\n",
    "            w = x[j][0][i]\n",
    "            if w == 'PAD':\n",
    "                xx_title[j][i] = word2id[w]\n",
    "            elif w in word2id:\n",
    "                xx_title[j][i] = word2id[w]\n",
    "                xx_title_mask[j][i] = True\n",
    "            else:\n",
    "                xx_title[j][i] = word2id['UNKNOWN_TOKEN']\n",
    "    \n",
    "    for j in range(num_examples):\n",
    "        xx_total_title_mask[j][0] = True\n",
    "        for i in range(max_title_size):\n",
    "            w = x[j][0][i]\n",
    "            if w == 'PAD':\n",
    "                pass\n",
    "            elif w in word2id:\n",
    "                xx_total_title_mask[j][i+1] = True\n",
    "                \n",
    "    \n",
    "    for j in range(num_examples):\n",
    "        for i in range(avg_no_sent):\n",
    "            if all(np.array(x[j][1][i])=='PAD'):\n",
    "                pass\n",
    "            else:\n",
    "                ss_mask[j][i]=True\n",
    "                \n",
    "            \n",
    "    return xx, xx_mask ,xx_title,xx_title_mask,xx_total_title,xx_total_title_mask,ss_mask\n",
    "\n",
    "def eval_score(yy, pred):\n",
    "    num_batches = int(len(yy)/batch_size)\n",
    "    f1 = f1_score(yy[:batch_size*num_batches], pred, average='macro')\n",
    "    accu = accuracy_score(yy[:batch_size*num_batches], pred)\n",
    "    return f1*100, accu*100\n",
    "\n",
    "def model(xx, yy, xx_mask, xx_title, xx_title_mask, xx_title_total_mask, ss_mask, train_cond=True):\n",
    "    num_batches = int(len(xx)/batch_size)\n",
    "    losses = 0\n",
    "    preds = []\n",
    "    for j in range(num_batches): \n",
    "        \n",
    "        s = j * batch_size\n",
    "        e = (j+1) * batch_size\n",
    "        \n",
    "        feed_dict = {word_emb_mat:word_embedding,\n",
    "                     input_keep_prob:keep_prob,\n",
    "                     is_train:train_cond,\n",
    "                     x:xx[s:e], y:yy[s:e], x_mask:xx_mask[s:e],x_title:xx_title[s:e],\n",
    "                     x_title_mask:xx_title_mask[s:e],\n",
    "                     x_title_total_mask:xx_title_total_mask[s:e],\n",
    "                     sent_mask:ss_mask[s:e]}\n",
    "        \n",
    "        \n",
    "        if(train_cond==True):\n",
    "            _, _loss, step, _summary = sess.run([train_op, total_loss, global_step, summary], feed_dict)\n",
    "            summary_writer.add_summary(_summary, step)  \n",
    "#             print(\"Steps:{}\".format(step), \", Loss: {}\".format(_loss))\n",
    "\n",
    "        else:\n",
    "            _loss, pred = sess.run([total_loss, predictions], feed_dict)\n",
    "            preds.append(pred)\n",
    "            \n",
    "        losses +=_loss\n",
    "\n",
    "    if(train_cond==False):\n",
    "        y_pred = []\n",
    "        for i in range(num_batches):\n",
    "            for pred in preds[i]:\n",
    "                y_pred.append(pred)\n",
    "        return losses/num_batches, y_pred\n",
    "    \n",
    "    return losses/num_batches, step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def predict(xx, yy, xx_mask, xx_title, xx_title_mask, xx_title_total_mask, ss_mask, train_cond=True):\n",
    "    num_batches = int(len(xx)/batch_size)\n",
    "    losses = 0\n",
    "    preds = []\n",
    "    for j in range(num_batches): \n",
    "        \n",
    "        s = j * batch_size\n",
    "        e = (j+1) * batch_size\n",
    "        \n",
    "        feed_dict = {word_emb_mat:word_embedding,\n",
    "                     input_keep_prob:keep_prob,\n",
    "                     is_train:train_cond,\n",
    "                     x:xx[s:e], y:yy[s:e], x_mask:xx_mask[s:e],x_title:xx_title[s:e],\n",
    "                     x_title_mask:xx_title_mask[s:e],\n",
    "                     x_title_total_mask:xx_title_total_mask[s:e],\n",
    "                     sent_mask:ss_mask[s:e]}\n",
    "        \n",
    "        \n",
    "        if(train_cond==True):\n",
    "            _, _loss, step, _summary = sess.run([train_op, total_loss, global_step, summary], feed_dict)\n",
    "            summary_writer.add_summary(_summary, step)  \n",
    "#             print(\"Steps:{}\".format(step), \", Loss: {}\".format(_loss))\n",
    "\n",
    "        else:\n",
    "            _loss, pred = sess.run([total_loss, predictions], feed_dict)\n",
    "            preds.append(pred)\n",
    "            \n",
    "        losses +=_loss\n",
    "\n",
    "    if(train_cond==False):\n",
    "        y_pred = []\n",
    "        for i in range(num_batches):\n",
    "            for pred in preds[i]:\n",
    "                y_pred.append(pred)\n",
    "        return  y_pred\n",
    "    \n",
    "    return  step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "f = open('headline.txt','r')\n",
    "str1 = f.read()\n",
    "f.close()\n",
    "\n",
    "f = open('article.txt', 'r')\n",
    "str2 = f.read()\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Donald Trump secretly worried that he may end up becoming President\\n'"
      ]
     },
     "execution_count": 156,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "str1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "str1 = \"Donald Trump secretly worried that he may end up becoming President\"\n",
    "str2 = \"Speaking to faking News, a close aide of Mr. Trump said, Well he entered the race because he thought it would be fun. He thought he will get a stage to insult politicians and journalists, something we all want to do. And boy did he do that handsomely, every politician from Republicans to the Democrats was rubbed the wrong way by the Donald. The plan was to enjoy insulting these guys for a couple of months and then drop out but thanks to the incompetent opponents, he kept growing in popularity.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/praphul/anaconda3/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: compiletime version 3.5 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.6\n",
      "  return f(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import jsonlines\n",
    "import numpy as np\n",
    "import random\n",
    "import pickle as pkl\n",
    "import re\n",
    "import sklearn \n",
    "import pickle as pkl\n",
    "import glob\n",
    "from nltk import word_tokenize   \n",
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def filter(line, whitelist):\n",
    "    ##############################################################\n",
    "    #   Filters out all characters which are not in whitelist    #\n",
    "    ##############################################################\n",
    "\n",
    "    return ''.join([ch for ch in line if ch in whitelist])\n",
    "\n",
    "\n",
    "WHITELIST1 = \"0123456789abcdefghijklmnopqrstuvwxyz'?.! \"\n",
    "WHITELIST2 = \"0123456789abcdefghijklmnopqrstuvwxyz' \""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def preprocess(string): \n",
    "    string = re.sub(\"\\.\\.\\.\", \" \\. \", string)\n",
    "    string = re.sub(r\"won't\", \"will not\", string)\n",
    "    string = re.sub(r\"n\\'t\", \" not\", string)\n",
    "    string = re.sub(r\"n't\", \" not \", string)\n",
    "    string = re.sub(r\"\\'re\", \" are\", string)\n",
    "    string = re.sub(r\"\\'s\", \" is\", string)\n",
    "    string = re.sub(r\"\\'d\", \" would\", string)\n",
    "    string = re.sub(r\"\\'ll\", \" will\", string)\n",
    "    string = re.sub(r\"\\'t\", \" not\", string)\n",
    "    string = re.sub(r\"\\'ve\", \" have\", string)\n",
    "    string = re.sub(r\"\\'m\", \" am\", string)\n",
    "    string = re.sub(r\"\\'d've\", \" would have\", string)\n",
    "    string = re.sub(r\"\\'d'y\", \" do you\", string)\n",
    "    string = re.sub(r\"'\", \" \", string)\n",
    "    return string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def process(str1, str2):\n",
    "    str1 = filter(str1.lower(), WHITELIST2)\n",
    "    str2 = filter(str2.lower(), WHITELIST1)\n",
    "    str1 = preprocess(str1)\n",
    "    str2 = preprocess(str2)\n",
    "    str1 = str1.split()\n",
    "    str2 = re.split(\"[\\.?!]\", str2)\n",
    "    for k in range(len(str2)):\n",
    "        str2[k] = str2[k].split()\n",
    "    test = []\n",
    "    temp1 = []\n",
    "    temp2 = []\n",
    "    for i in range(22):\n",
    "        if(i<len(str1)):\n",
    "            temp1.append(str1[i])\n",
    "        else:\n",
    "            temp1.append('PAD')\n",
    "            \n",
    "    pad_sent = []\n",
    "    for i in range(23):\n",
    "        pad_sent.append('PAD')\n",
    "    for i in range(10):\n",
    "        if(i<len(str2)):\n",
    "            temp3 = []\n",
    "            for j in range(23):\n",
    "                if(j<len(str2[i])):\n",
    "                    temp3.append(str2[i][j])\n",
    "                else:\n",
    "                    temp3.append('PAD')\n",
    "            temp2.append(temp3)\n",
    "        else:\n",
    "            temp2.append(pad_sent)\n",
    "            \n",
    "    test.append(np.array([temp1, temp2]))\n",
    "    return test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = process(str1, str2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "t_test, t_mask_test ,t_title_test,t_title_mask_test,t_total_title_test,t_total_title_mask_test,tt_mask_test = data_prepare(test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "t_test_yy = np.zeros([len(t_test), 1])\n",
    "t_test_yy = np.array(t_test_yy).reshape(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "saver.restore(sess, save_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "feed_dict = {word_emb_mat:word_embedding,\n",
    "             input_keep_prob:keep_prob,\n",
    "             is_train:False,\n",
    "             x:t_test, y:t_test_yy, x_mask:t_mask_test,x_title:t_title_test,\n",
    "             x_title_mask:t_title_mask_test,\n",
    "             x_title_total_mask:t_total_title_mask_test,\n",
    "             sent_mask:tt_mask_test}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.])"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t_test_yy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pred = sess.run([predictions], feed_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([1])]"
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# xx_train, xx_mask_train ,xx_title_train,xx_title_mask_train,xx_total_title_train,xx_total_title_mask_train,ss_mask_train = data_prepare(x_train)\n",
    "# xx_val, xx_mask_val ,xx_title_val,xx_title_mask_val,xx_total_title_val,xx_total_title_mask_val,ss_mask_val = data_prepare(x_val)\n",
    "# xx_test, xx_mask_test ,xx_title_test,xx_title_mask_test,xx_total_title_test,xx_total_title_mask_test,ss_mask_test = data_prepare(x_test)\n",
    "# yy_train = np.array(y_train).reshape(-1)\n",
    "# yy_val = np.array(y_val).reshape(-1)\n",
    "# yy_test = np.array(y_test).reshape(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((23759, 10, 23), (2640, 10, 23), (6600, 10, 23))"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# xx_train.shape, xx_val.shape, xx_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((5000, 10, 23), (2640, 10, 23), (3000, 10, 23))"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# xx_train.shape, xx_val.shape, xx_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1 Step: 237 loss: 0.6621203081768776 Epoch_Time: 2567.780829668045\n",
      "Saved Model Complete\n"
     ]
    }
   ],
   "source": [
    "# num_epochs = 5\n",
    "# for i in range(num_epochs):\n",
    "#     start_time = time.time()\n",
    "#     random = np.random.choice(len(yy_train), size=(len(yy_train)), replace=False)\n",
    "#     xx_train = xx_train[random]\n",
    "#     yy_train = yy_train[random]\n",
    "#     xx_mask_train = xx_mask_train[random]\n",
    "#     xx_title_train = xx_title_train[random]\n",
    "#     xx_title_mask_train = xx_title_mask_train[random]                           \n",
    "#     xx_total_title_train = xx_total_title_train[random]\n",
    "#     xx_total_title_mask_train = xx_total_title_mask_train[random]\n",
    "#     ss_mask_train = ss_mask_train[random]\n",
    "            \n",
    "#     losses, step = model(xx_train, yy_train, xx_mask_train ,xx_title_train,xx_title_mask_train\n",
    "#                         ,xx_total_title_mask_train,ss_mask_train, train_cond=True)\n",
    "    \n",
    "#     print(\"Epoch:\", i+1,\"Step:\", step, \"loss:\",losses, \"Epoch_Time:\",time.time() - start_time)\n",
    "#     saver.save(sess, save_path=save_dir)                         \n",
    "#     print(\"Saved Model Complete\")\n",
    "    \n",
    "#     if((i+1)%1==0):\n",
    "#         train_loss, train_pred = model(xx_train, yy_train, xx_mask_train ,xx_title_train,xx_title_mask_train,\n",
    "#                                        xx_total_title_mask_train,ss_mask_train, train_cond=False)\n",
    "        \n",
    "#         f1_, accu_ = eval_score(y_train, train_pred)\n",
    "#         print(\"Train: F1 Score: \",  f1_, \"Accuracy: \", accu_, \"Loss: \", train_loss)\n",
    "        \n",
    "#         val_loss, val_pred = model(xx_val, yy_val, xx_mask_val ,xx_title_val,xx_title_mask_val,\n",
    "#                                    xx_total_title_mask_val,ss_mask_val, train_cond=False)\n",
    "        \n",
    "#         f1_, accu_ = eval_score(y_val, val_pred)\n",
    "#         print(\"Val: F1 Score: \",  f1_, \"Accuracy: \", accu_, \"Loss: \", val_loss)\n",
    "        \n",
    "# test_loss, test_pred = model(xx_test, yy_test, xx_mask_test ,xx_title_test,xx_title_mask_test,xx_total_title_mask_test,ss_mask_test, train_cond=False)\n",
    "\n",
    "# f1_, accu_ = eval_score(y_test, test_pred)\n",
    "# print(\"Test: F1 Score: \",  f1_, \"Accuracy: \", accu_, \"Loss: \", test_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "saver.restore(sess, save_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val: F1 Score:  80.55031910300636 Accuracy:  81.15384615384616 Loss:  0.4871215568139003\n"
     ]
    }
   ],
   "source": [
    "# val_loss, val_pred = model(xx_val, yy_val, xx_mask_val ,xx_title_val,xx_title_mask_val,\n",
    "#                            xx_total_title_mask_val,ss_mask_val, train_cond=False)\n",
    "\n",
    "# f1_, accu_ = eval_score(y_val, val_pred)\n",
    "# print(\"Val: F1 Score: \",  f1_, \"Accuracy: \", accu_, \"Loss: \", val_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1 Step: 474 loss: 0.4549066013173212 Epoch_Time: 1839.4972031116486\n",
      "Saved Model Complete\n",
      "Train: F1 Score:  49.72728413611226 Accuracy:  53.06751054852321 Loss:  0.4262467407727543\n",
      "Val: F1 Score:  82.322433748656 Accuracy:  83.46153846153847 Loss:  0.42888274788856506\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# num_epochs = 5\n",
    "\n",
    "# for i in range(num_epochs):\n",
    "#     start_time = time.time()\n",
    "#     random = np.random.choice(len(yy_train), size=(len(yy_train)), replace=False)\n",
    "#     xx_train = xx_train[random]\n",
    "#     yy_train = yy_train[random]\n",
    "#     xx_mask_train = xx_mask_train[random]\n",
    "#     xx_title_train = xx_title_train[random]\n",
    "#     xx_title_mask_train = xx_title_mask_train[random]                           \n",
    "#     xx_total_title_train = xx_total_title_train[random]\n",
    "#     xx_total_title_mask_train = xx_total_title_mask_train[random]\n",
    "#     ss_mask_train = ss_mask_train[random]\n",
    "            \n",
    "#     losses, step = model(xx_train, yy_train, xx_mask_train ,xx_title_train,xx_title_mask_train\n",
    "#                         ,xx_total_title_mask_train,ss_mask_train, train_cond=True)\n",
    "    \n",
    "#     print(\"Epoch:\", i+1,\"Step:\", step, \"loss:\",losses, \"Epoch_Time:\",time.time() - start_time)\n",
    "#     saver.save(sess, save_path=save_dir)                         \n",
    "#     print(\"Saved Model Complete\")\n",
    "    \n",
    "#     if((i+1)%1==0):\n",
    "#         train_loss, train_pred = model(xx_train, yy_train, xx_mask_train ,xx_title_train,xx_title_mask_train,\n",
    "#                                        xx_total_title_mask_train,ss_mask_train, train_cond=False)\n",
    "        \n",
    "#         f1_, accu_ = eval_score(y_train, train_pred)\n",
    "#         print(\"Train: F1 Score: \",  f1_, \"Accuracy: \", accu_, \"Loss: \", train_loss)\n",
    "        \n",
    "#         val_loss, val_pred = model(xx_val, yy_val, xx_mask_val ,xx_title_val,xx_title_mask_val,\n",
    "#                                    xx_total_title_mask_val,ss_mask_val, train_cond=False)\n",
    "        \n",
    "#         f1_, accu_ = eval_score(y_val, val_pred)\n",
    "#         print(\"Val: F1 Score: \",  f1_, \"Accuracy: \", accu_, \"Loss: \", val_loss)\n",
    "        \n",
    "# test_loss, test_pred = model(xx_test, yy_test, xx_mask_test ,xx_title_test,xx_title_mask_test,xx_total_title_mask_test,ss_mask_test, train_cond=False)\n",
    "\n",
    "# f1_, accu_ = eval_score(y_test, test_pred)\n",
    "# print(\"Test: F1 Score: \",  f1_, \"Accuracy: \", accu_, \"Loss: \", test_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1 Step: 711 loss: 0.4213623989231979 Epoch_Time: 1972.3159937858582\n",
      "Saved Model Complete\n",
      "Val: F1 Score:  81.23202327673324 Accuracy:  83.0 Loss:  0.4333428774888699\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# num_epochs = 5\n",
    "\n",
    "# for i in range(num_epochs):\n",
    "#     start_time = time.time()\n",
    "#     random = np.random.choice(len(yy_train), size=(len(yy_train)), replace=False)\n",
    "#     xx_train = xx_train[random]\n",
    "#     yy_train = yy_train[random]\n",
    "#     xx_mask_train = xx_mask_train[random]\n",
    "#     xx_title_train = xx_title_train[random]\n",
    "#     xx_title_mask_train = xx_title_mask_train[random]                           \n",
    "#     xx_total_title_train = xx_total_title_train[random]\n",
    "#     xx_total_title_mask_train = xx_total_title_mask_train[random]\n",
    "#     ss_mask_train = ss_mask_train[random]\n",
    "            \n",
    "#     losses, step = model(xx_train, yy_train, xx_mask_train ,xx_title_train,xx_title_mask_train\n",
    "#                         ,xx_total_title_mask_train,ss_mask_train, train_cond=True)\n",
    "    \n",
    "#     print(\"Epoch:\", i+1,\"Step:\", step, \"loss:\",losses, \"Epoch_Time:\",time.time() - start_time)\n",
    "#     saver.save(sess, save_path=save_dir)                         \n",
    "#     print(\"Saved Model Complete\")\n",
    "    \n",
    "#     if((i+1)%1==0):\n",
    "# #         train_loss, train_pred = model(xx_train, yy_train, xx_mask_train ,xx_title_train,xx_title_mask_train,\n",
    "# #                                        xx_total_title_mask_train,ss_mask_train, train_cond=False)\n",
    "        \n",
    "# #         f1_, accu_ = eval_score(y_train, train_pred)\n",
    "# #         print(\"Train: F1 Score: \",  f1_, \"Accuracy: \", accu_, \"Loss: \", train_loss)\n",
    "        \n",
    "#         val_loss, val_pred = model(xx_val, yy_val, xx_mask_val ,xx_title_val,xx_title_mask_val,\n",
    "#                                    xx_total_title_mask_val,ss_mask_val, train_cond=False)\n",
    "        \n",
    "#         f1_, accu_ = eval_score(y_val, val_pred)\n",
    "#         print(\"Val: F1 Score: \",  f1_, \"Accuracy: \", accu_, \"Loss: \", val_loss)\n",
    "        \n",
    "# test_loss, test_pred = model(xx_test, yy_test, xx_mask_test ,xx_title_test,xx_title_mask_test,xx_total_title_mask_test,ss_mask_test, train_cond=False)\n",
    "\n",
    "# f1_, accu_ = eval_score(y_test, test_pred)\n",
    "# print(\"Test: F1 Score: \",  f1_, \"Accuracy: \", accu_, \"Loss: \", test_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1 Step: 948 loss: 0.39857222561091815 Epoch_Time: 1392.0765545368195\n",
      "Saved Model Complete\n",
      "Val: F1 Score:  84.99530779794797 Accuracy:  85.42307692307692 Loss:  0.4068056448147847\n",
      "Epoch: 2 Step: 1185 loss: 0.3807800622177526 Epoch_Time: 1123.9129800796509\n",
      "Saved Model Complete\n",
      "Val: F1 Score:  84.52966009755932 Accuracy:  85.96153846153847 Loss:  0.39526177713504207\n",
      "Epoch: 3 Step: 1422 loss: 0.3722822886856296 Epoch_Time: 1074.4930148124695\n",
      "Saved Model Complete\n",
      "Val: F1 Score:  86.92716475954583 Accuracy:  87.65384615384615 Loss:  0.359285810819039\n",
      "Epoch: 4 Step: 1659 loss: 0.36547351631685654 Epoch_Time: 1307.4297301769257\n",
      "Saved Model Complete\n",
      "Val: F1 Score:  86.30352162319552 Accuracy:  87.3076923076923 Loss:  0.35827719133633834\n",
      "Epoch: 5 Step: 1896 loss: 0.35762700262451974 Epoch_Time: 1752.1305372714996\n",
      "Saved Model Complete\n",
      "Val: F1 Score:  87.16160584994097 Accuracy:  87.96153846153845 Loss:  0.36131645624454206\n",
      "Test: F1 Score:  87.016968604988 Accuracy:  87.83333333333333 Loss:  0.35453890444654407\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# num_epochs = 5\n",
    "\n",
    "# for i in range(num_epochs):\n",
    "#     start_time = time.time()\n",
    "#     random = np.random.choice(len(yy_train), size=(len(yy_train)), replace=False)\n",
    "#     xx_train = xx_train[random]\n",
    "#     yy_train = yy_train[random]\n",
    "#     xx_mask_train = xx_mask_train[random]\n",
    "#     xx_title_train = xx_title_train[random]\n",
    "#     xx_title_mask_train = xx_title_mask_train[random]                           \n",
    "#     xx_total_title_train = xx_total_title_train[random]\n",
    "#     xx_total_title_mask_train = xx_total_title_mask_train[random]\n",
    "#     ss_mask_train = ss_mask_train[random]\n",
    "            \n",
    "#     losses, step = model(xx_train, yy_train, xx_mask_train ,xx_title_train,xx_title_mask_train\n",
    "#                         ,xx_total_title_mask_train,ss_mask_train, train_cond=True)\n",
    "    \n",
    "#     print(\"Epoch:\", i+1,\"Step:\", step, \"loss:\",losses, \"Epoch_Time:\",time.time() - start_time)\n",
    "#     saver.save(sess, save_path=save_dir)                         \n",
    "#     print(\"Saved Model Complete\")\n",
    "    \n",
    "#     if((i+1)%1==0):\n",
    "# #         train_loss, train_pred = model(xx_train, yy_train, xx_mask_train ,xx_title_train,xx_title_mask_train,\n",
    "# #                                        xx_total_title_mask_train,ss_mask_train, train_cond=False)\n",
    "        \n",
    "# #         f1_, accu_ = eval_score(y_train, train_pred)\n",
    "# #         print(\"Train: F1 Score: \",  f1_, \"Accuracy: \", accu_, \"Loss: \", train_loss)\n",
    "        \n",
    "#         val_loss, val_pred = model(xx_val, yy_val, xx_mask_val ,xx_title_val,xx_title_mask_val,\n",
    "#                                    xx_total_title_mask_val,ss_mask_val, train_cond=False)\n",
    "        \n",
    "#         f1_, accu_ = eval_score(y_val, val_pred)\n",
    "#         print(\"Val: F1 Score: \",  f1_, \"Accuracy: \", accu_, \"Loss: \", val_loss)\n",
    "        \n",
    "# test_loss, test_pred = model(xx_test, yy_test, xx_mask_test ,xx_title_test,xx_title_mask_test,xx_total_title_mask_test,ss_mask_test, train_cond=False)\n",
    "\n",
    "# f1_, accu_ = eval_score(y_test, test_pred)\n",
    "# print(\"Test: F1 Score: \",  f1_, \"Accuracy: \", accu_, \"Loss: \", test_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "ename": "InternalError",
     "evalue": "Dst tensor is not initialized.\n\t [[Node: _arg_emb_mat_0_1/_195 = _Recv[client_terminated=false, recv_device=\"/job:localhost/replica:0/task:0/gpu:0\", send_device=\"/job:localhost/replica:0/task:0/cpu:0\", send_device_incarnation=1, tensor_name=\"edge_7416__arg_emb_mat_0_1\", tensor_type=DT_FLOAT, _device=\"/job:localhost/replica:0/task:0/gpu:0\"]()]]\n\t [[Node: lstm_layer_top/lstm_top/bw/bw/All/_317 = _Recv[client_terminated=false, recv_device=\"/job:localhost/replica:0/task:0/cpu:0\", send_device=\"/job:localhost/replica:0/task:0/gpu:0\", send_device_incarnation=1, tensor_name=\"edge_3725_lstm_layer_top/lstm_top/bw/bw/All\", tensor_type=DT_BOOL, _device=\"/job:localhost/replica:0/task:0/cpu:0\"]()]]",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mInternalError\u001b[0m                             Traceback (most recent call last)",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1138\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1139\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1140\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1120\u001b[0m                                  \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1121\u001b[0;31m                                  status, run_metadata)\n\u001b[0m\u001b[1;32m   1122\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.5/contextlib.py\u001b[0m in \u001b[0;36m__exit__\u001b[0;34m(self, type, value, traceback)\u001b[0m\n\u001b[1;32m     65\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 66\u001b[0;31m                 \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgen\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     67\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/tensorflow/python/framework/errors_impl.py\u001b[0m in \u001b[0;36mraise_exception_on_not_ok_status\u001b[0;34m()\u001b[0m\n\u001b[1;32m    465\u001b[0m           \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpywrap_tensorflow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_Message\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstatus\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 466\u001b[0;31m           pywrap_tensorflow.TF_GetCode(status))\n\u001b[0m\u001b[1;32m    467\u001b[0m   \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mInternalError\u001b[0m: Dst tensor is not initialized.\n\t [[Node: _arg_emb_mat_0_1/_195 = _Recv[client_terminated=false, recv_device=\"/job:localhost/replica:0/task:0/gpu:0\", send_device=\"/job:localhost/replica:0/task:0/cpu:0\", send_device_incarnation=1, tensor_name=\"edge_7416__arg_emb_mat_0_1\", tensor_type=DT_FLOAT, _device=\"/job:localhost/replica:0/task:0/gpu:0\"]()]]\n\t [[Node: lstm_layer_top/lstm_top/bw/bw/All/_317 = _Recv[client_terminated=false, recv_device=\"/job:localhost/replica:0/task:0/cpu:0\", send_device=\"/job:localhost/replica:0/task:0/gpu:0\", send_device_incarnation=1, tensor_name=\"edge_3725_lstm_layer_top/lstm_top/bw/bw/All\", tensor_type=DT_BOOL, _device=\"/job:localhost/replica:0/task:0/cpu:0\"]()]]",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mInternalError\u001b[0m                             Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-43-92e60d843587>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m     losses, step = model(xx_train, yy_train, xx_mask_train ,xx_title_train,xx_title_mask_train\n\u001b[0;32m---> 17\u001b[0;31m                         ,xx_total_title_mask_train,ss_mask_train, train_cond=True)\n\u001b[0m\u001b[1;32m     18\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Epoch:\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"Step:\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstep\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"loss:\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mlosses\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"Epoch_Time:\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mstart_time\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-35-03a0c11ea38a>\u001b[0m in \u001b[0;36mmodel\u001b[0;34m(xx, yy, xx_mask, xx_title, xx_title_mask, xx_title_total_mask, ss_mask, train_cond)\u001b[0m\n\u001b[1;32m     83\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     84\u001b[0m         \u001b[0;32mif\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_cond\u001b[0m\u001b[0;34m==\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 85\u001b[0;31m             \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_loss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstep\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_summary\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtrain_op\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtotal_loss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mglobal_step\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msummary\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     86\u001b[0m             \u001b[0msummary_writer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_summary\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_summary\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     87\u001b[0m \u001b[0;31m#             print(\"Steps:{}\".format(step), \", Loss: {}\".format(_loss))\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    787\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    788\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 789\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    790\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    791\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    995\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    996\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m--> 997\u001b[0;31m                              feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[1;32m    998\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    999\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1130\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1131\u001b[0m       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[0;32m-> 1132\u001b[0;31m                            target_list, options, run_metadata)\n\u001b[0m\u001b[1;32m   1133\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1134\u001b[0m       return self._do_call(_prun_fn, self._session, handle, feed_dict,\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1150\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1151\u001b[0m           \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1152\u001b[0;31m       \u001b[0;32mraise\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode_def\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1153\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1154\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mInternalError\u001b[0m: Dst tensor is not initialized.\n\t [[Node: _arg_emb_mat_0_1/_195 = _Recv[client_terminated=false, recv_device=\"/job:localhost/replica:0/task:0/gpu:0\", send_device=\"/job:localhost/replica:0/task:0/cpu:0\", send_device_incarnation=1, tensor_name=\"edge_7416__arg_emb_mat_0_1\", tensor_type=DT_FLOAT, _device=\"/job:localhost/replica:0/task:0/gpu:0\"]()]]\n\t [[Node: lstm_layer_top/lstm_top/bw/bw/All/_317 = _Recv[client_terminated=false, recv_device=\"/job:localhost/replica:0/task:0/cpu:0\", send_device=\"/job:localhost/replica:0/task:0/gpu:0\", send_device_incarnation=1, tensor_name=\"edge_3725_lstm_layer_top/lstm_top/bw/bw/All\", tensor_type=DT_BOOL, _device=\"/job:localhost/replica:0/task:0/cpu:0\"]()]]"
     ]
    }
   ],
   "source": [
    "\n",
    "# num_epochs = 5\n",
    "\n",
    "# for i in range(num_epochs):\n",
    "#     start_time = time.time()\n",
    "#     random = np.random.choice(len(yy_train), size=(len(yy_train)), replace=False)\n",
    "#     xx_train = xx_train[random]\n",
    "#     yy_train = yy_train[random]\n",
    "#     xx_mask_train = xx_mask_train[random]\n",
    "#     xx_title_train = xx_title_train[random]\n",
    "#     xx_title_mask_train = xx_title_mask_train[random]                           \n",
    "#     xx_total_title_train = xx_total_title_train[random]\n",
    "#     xx_total_title_mask_train = xx_total_title_mask_train[random]\n",
    "#     ss_mask_train = ss_mask_train[random]\n",
    "            \n",
    "#     losses, step = model(xx_train, yy_train, xx_mask_train ,xx_title_train,xx_title_mask_train\n",
    "#                         ,xx_total_title_mask_train,ss_mask_train, train_cond=True)\n",
    "    \n",
    "#     print(\"Epoch:\", i+1,\"Step:\", step, \"loss:\",losses, \"Epoch_Time:\",time.time() - start_time)\n",
    "#     saver.save(sess, save_path=save_dir)                         \n",
    "#     print(\"Saved Model Complete\")\n",
    "    \n",
    "#     if((i+1)%1==0):\n",
    "#         train_loss, train_pred = model(xx_train, yy_train, xx_mask_train ,xx_title_train,xx_title_mask_train,\n",
    "#                                        xx_total_title_mask_train,ss_mask_train, train_cond=False)\n",
    "        \n",
    "#         f1_, accu_ = eval_score(y_train, train_pred)\n",
    "#         print(\"Train: F1 Score: \",  f1_, \"Accuracy: \", accu_, \"Loss: \", train_loss)\n",
    "        \n",
    "#         val_loss, val_pred = model(xx_val, yy_val, xx_mask_val ,xx_title_val,xx_title_mask_val,\n",
    "#                                    xx_total_title_mask_val,ss_mask_val, train_cond=False)\n",
    "        \n",
    "#         f1_, accu_ = eval_score(y_val, val_pred)\n",
    "#         print(\"Val: F1 Score: \",  f1_, \"Accuracy: \", accu_, \"Loss: \", val_loss)\n",
    "        \n",
    "# test_loss, test_pred = model(xx_test, yy_test, xx_mask_test ,xx_title_test,xx_title_mask_test,xx_total_title_mask_test,ss_mask_test, train_cond=False)\n",
    "\n",
    "# f1_, accu_ = eval_score(y_test, test_pred)\n",
    "# print(\"Test: F1 Score: \",  f1_, \"Accuracy: \", accu_, \"Loss: \", test_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train: F1 Score:  87.11688604666425 Accuracy:  87.51476793248946 Loss:  0.35891008037555067\n",
      "Val: F1 Score:  87.83927056343241 Accuracy:  88.1923076923077 Loss:  0.3618659835595351\n",
      "Test: F1 Score:  87.2067696728499 Accuracy:  87.60606060606061 Loss:  0.35782169889320026\n"
     ]
    }
   ],
   "source": [
    "#  train_loss, train_pred = model(xx_train, yy_train, xx_mask_train ,xx_title_train,xx_title_mask_train,\n",
    "#                                        xx_total_title_mask_train,ss_mask_train, train_cond=False)\n",
    "        \n",
    "# f1_, accu_ = eval_score(y_train, train_pred)\n",
    "# print(\"Train: F1 Score: \",  f1_, \"Accuracy: \", accu_, \"Loss: \", train_loss)\n",
    "\n",
    "# val_loss, val_pred = model(xx_val, yy_val, xx_mask_val ,xx_title_val,xx_title_mask_val,\n",
    "#                            xx_total_title_mask_val,ss_mask_val, train_cond=False)\n",
    "\n",
    "# f1_, accu_ = eval_score(y_val, val_pred)\n",
    "# print(\"Val: F1 Score: \",  f1_, \"Accuracy: \", accu_, \"Loss: \", val_loss)\n",
    "\n",
    "# test_loss, test_pred = model(xx_test, yy_test, xx_mask_test ,xx_title_test,xx_title_mask_test,xx_total_title_mask_test,ss_mask_test, train_cond=False)\n",
    "\n",
    "# f1_, accu_ = eval_score(y_test, test_pred)\n",
    "# print(\"Test: F1 Score: \",  f1_, \"Accuracy: \", accu_, \"Loss: \", test_loss)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# int(len(y_train)/batch_size) ,len(train_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# saver.restore(sess, save_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}