news_predict2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import pandas as pd\n",
    "pd.set_option(\"display.max_colwidth\", 200)\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Headline</th>\n",
       "      <th>SentimentTitle</th>\n",
       "      <th>SentimentHeadline</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Obama Lays Wreath at Arlington National Cemetery</td>\n",
       "      <td>Obama Lays Wreath at Arlington National Cemetery. President Barack Obama has laid a wreath at the Tomb of the Unknowns to honor</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.053300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A Look at the Health of the Chinese Economy</td>\n",
       "      <td>Tim Haywood, investment director business-unit head for fixed income at Gam, discusses the China beige book and the state of the economy.</td>\n",
       "      <td>0.208333</td>\n",
       "      <td>-0.156386</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Nouriel Roubini: Global Economy Not Back to 2008</td>\n",
       "      <td>Nouriel Roubini, NYU professor and chairman at Roubini Global Economics, explains why the global economy isn't facing the same conditions</td>\n",
       "      <td>-0.425210</td>\n",
       "      <td>0.139754</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Finland GDP Expands In Q4</td>\n",
       "      <td>Finland's economy expanded marginally in the three months ended December, after contracting in the previous quarter, preliminary figures from Statistics Finland showed Monday.</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.026064</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Tourism, govt spending buoys Thai economy in January</td>\n",
       "      <td>Tourism and public spending continued to boost the economy in January, in light of contraction in private consumption and exports, according to the Bank of Thailand data.</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.141084</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  Title  \\\n",
       "0      Obama Lays Wreath at Arlington National Cemetery   \n",
       "1           A Look at the Health of the Chinese Economy   \n",
       "2      Nouriel Roubini: Global Economy Not Back to 2008   \n",
       "3                             Finland GDP Expands In Q4   \n",
       "4  Tourism, govt spending buoys Thai economy in January   \n",
       "\n",
       "                                                                                                                                                                           Headline  \\\n",
       "0                                                   Obama Lays Wreath at Arlington National Cemetery. President Barack Obama has laid a wreath at the Tomb of the Unknowns to honor   \n",
       "1                                         Tim Haywood, investment director business-unit head for fixed income at Gam, discusses the China beige book and the state of the economy.   \n",
       "2                                         Nouriel Roubini, NYU professor and chairman at Roubini Global Economics, explains why the global economy isn't facing the same conditions   \n",
       "3  Finland's economy expanded marginally in the three months ended December, after contracting in the previous quarter, preliminary figures from Statistics Finland showed Monday.    \n",
       "4       Tourism and public spending continued to boost the economy in January, in light of contraction in private consumption and exports, according to the Bank of Thailand data.    \n",
       "\n",
       "   SentimentTitle  SentimentHeadline  \n",
       "0        0.000000          -0.053300  \n",
       "1        0.208333          -0.156386  \n",
       "2       -0.425210           0.139754  \n",
       "3        0.000000           0.026064  \n",
       "4        0.000000           0.141084  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv(\"train_file.csv\")\n",
    "train = train.drop(['IDLink', 'Source', 'PublishDate', 'Topic', 'Facebook', 'GooglePlus', 'LinkedIn'], axis=1)\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>IDLink</th>\n",
       "      <th>Title</th>\n",
       "      <th>Headline</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>tFrqIR6Chj</td>\n",
       "      <td>Sliding Economy: FG fights back with N3trn TSA funds</td>\n",
       "      <td>With the 2016 budget now passed by the National Assembly and a N3trillion war chest, the government of President Muhammadu Buhari says</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DVAaGErjlF</td>\n",
       "      <td>Microsoft shows how HoloLens can bring distant family members ...</td>\n",
       "      <td>A recent Microsoft Research video shows how the $3000 augmented reality system can be used to transmit 3D models of people anywhere in</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>OT9UIZm5M2</td>\n",
       "      <td>Microsoft’s Twitter Robot Praises Hitler, Trump &amp; Recites Racism</td>\n",
       "      <td>* Microsoft teamed with Bing to create TayTweets, an account for a robot that was designed to learn about “conversational understanding,” by having automated discussions with Twitter users, and mi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>lflGp3q2Fj</td>\n",
       "      <td>Flood of Central Bank Moves Can't Get World Economy Out of Rut</td>\n",
       "      <td>Central bankers have managed to steer the world economy clear of a recession while leaving it stuck in the same rut that led to its troubles in the first place.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>zDYG0SoovZ</td>\n",
       "      <td>USD/JPY: bears lining up on mixed U.S. economy outlook</td>\n",
       "      <td>However, this streak of seven-day gains might end here as markets take a step back and ponder in respect of the US economy and its inflation</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       IDLink  \\\n",
       "0  tFrqIR6Chj   \n",
       "1  DVAaGErjlF   \n",
       "2  OT9UIZm5M2   \n",
       "3  lflGp3q2Fj   \n",
       "4  zDYG0SoovZ   \n",
       "\n",
       "                                                               Title  \\\n",
       "0               Sliding Economy: FG fights back with N3trn TSA funds   \n",
       "1  Microsoft shows how HoloLens can bring distant family members ...   \n",
       "2   Microsoft’s Twitter Robot Praises Hitler, Trump & Recites Racism   \n",
       "3     Flood of Central Bank Moves Can't Get World Economy Out of Rut   \n",
       "4             USD/JPY: bears lining up on mixed U.S. economy outlook   \n",
       "\n",
       "                                                                                                                                                                                                  Headline  \n",
       "0                                                                   With the 2016 budget now passed by the National Assembly and a N3trillion war chest, the government of President Muhammadu Buhari says  \n",
       "1                                                                   A recent Microsoft Research video shows how the $3000 augmented reality system can be used to transmit 3D models of people anywhere in  \n",
       "2  * Microsoft teamed with Bing to create TayTweets, an account for a robot that was designed to learn about “conversational understanding,” by having automated discussions with Twitter users, and mi...  \n",
       "3                                        Central bankers have managed to steer the world economy clear of a recession while leaving it stuck in the same rut that led to its troubles in the first place.   \n",
       "4                                                             However, this streak of seven-day gains might end here as markets take a step back and ponder in respect of the US economy and its inflation  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test = pd.read_csv(\"test_file.csv\")\n",
    "test = test.drop(['Source', 'PublishDate', 'Topic', 'Facebook', 'GooglePlus', 'LinkedIn'], axis=1)\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((55932, 4), (37288, 3))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.shape, test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = train['Title'].values\n",
    "Y = train['SentimentTitle'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "tk = Tokenizer(lower = True)\n",
    "tk.fit_on_texts(X)\n",
    "X_seq = tk.texts_to_sequences(X)\n",
    "X_pad = pad_sequences(X_seq, maxlen = 100, padding = 'post')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X_pad, Y, test_size = 0.2, random_state = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 100 word vectors.\n"
     ]
    }
   ],
   "source": [
    "f = open('glove.6B.100d.sample.txt')\n",
    "embedd_index = {}\n",
    "for line in f:\n",
    "    val = line.split()\n",
    "    word = val[0]\n",
    "    coff = np.asarray(val[1:],dtype = 'float')\n",
    "    embedd_index[word] = coff\n",
    "\n",
    "f.close()\n",
    "print('Found %s word vectors.' % len(embedd_index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "index_of_words = tk.word_index\n",
    "embed_num_dims= 100\n",
    "embedding_matrix = np.zeros((len(index_of_words) + 1, embed_num_dims))\n",
    "\n",
    "tokens = []\n",
    "labels = []\n",
    "\n",
    "for word,i in index_of_words.items():\n",
    "    temp = embedd_index.get(word)\n",
    "    if temp is not None:\n",
    "        embedding_matrix[i] = temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import Dense , Flatten ,Embedding,Input\n",
    "#Embedding layer before the actual BLSTM \n",
    "embedd_layer = Embedding(len(index_of_words) + 1 , embed_num_dims , input_length = 100 , weights = [embedding_matrix])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 64\n",
    "X_train1 = X_train[batch_size:]\n",
    "y_train1 = y_train[batch_size:]\n",
    "X_valid = X_train[:batch_size]\n",
    "y_valid = y_train[:batch_size]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Embedding, LSTM, Dense, Dropout\n",
    "from keras.layers import Bidirectional, GlobalMaxPooling1D\n",
    "vocabulary_size = len(tk.word_counts.keys())+1\n",
    "max_words = 100\n",
    "embedding_size = 32\n",
    "model = Sequential()\n",
    "model.add(embedd_layer)\n",
    "model.add(Bidirectional(LSTM(30 , return_sequences = True , dropout = 0.1 , recurrent_dropout = 0.1)))\n",
    "model.add(GlobalMaxPooling1D())\n",
    "model.add(Dense(30,activation = 'relu'))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(1,activation = 'sigmoid'))\n",
    "\n",
    "from keras import metrics\n",
    "\n",
    "model.compile(loss = 'mean_squared_error',\n",
    "              optimizer = 'sgd',\n",
    "              metrics=[metrics.mae])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 44681 samples, validate on 64 samples\n",
      "Epoch 1/10\n",
      "44681/44681 [==============================] - 173s 4ms/step - loss: 0.0762 - mean_absolute_error: 0.2283 - val_loss: 0.0202 - val_mean_absolute_error: 0.1173\n",
      "Epoch 2/10\n",
      "44681/44681 [==============================] - 172s 4ms/step - loss: 0.0258 - mean_absolute_error: 0.1237 - val_loss: 0.0158 - val_mean_absolute_error: 0.1023\n",
      "Epoch 3/10\n",
      "44681/44681 [==============================] - 168s 4ms/step - loss: 0.0222 - mean_absolute_error: 0.1113 - val_loss: 0.0150 - val_mean_absolute_error: 0.0984\n",
      "Epoch 4/10\n",
      "44681/44681 [==============================] - 171s 4ms/step - loss: 0.0211 - mean_absolute_error: 0.1071 - val_loss: 0.0147 - val_mean_absolute_error: 0.0970\n",
      "Epoch 5/10\n",
      "44681/44681 [==============================] - 170s 4ms/step - loss: 0.0205 - mean_absolute_error: 0.1049 - val_loss: 0.0146 - val_mean_absolute_error: 0.0961\n",
      "Epoch 6/10\n",
      "44681/44681 [==============================] - 168s 4ms/step - loss: 0.0201 - mean_absolute_error: 0.1033 - val_loss: 0.0145 - val_mean_absolute_error: 0.0955\n",
      "Epoch 7/10\n",
      "44681/44681 [==============================] - 170s 4ms/step - loss: 0.0199 - mean_absolute_error: 0.1024 - val_loss: 0.0145 - val_mean_absolute_error: 0.0952\n",
      "Epoch 8/10\n",
      "44681/44681 [==============================] - 169s 4ms/step - loss: 0.0198 - mean_absolute_error: 0.1015 - val_loss: 0.0145 - val_mean_absolute_error: 0.0949\n",
      "Epoch 9/10\n",
      "44681/44681 [==============================] - 169s 4ms/step - loss: 0.0197 - mean_absolute_error: 0.1011 - val_loss: 0.0145 - val_mean_absolute_error: 0.0946\n",
      "Epoch 10/10\n",
      "44681/44681 [==============================] - 169s 4ms/step - loss: 0.0196 - mean_absolute_error: 0.1007 - val_loss: 0.0145 - val_mean_absolute_error: 0.0944\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1341439b0>"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train1, y_train1, validation_data = (X_valid, y_valid), batch_size = batch_size, epochs = 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores = model.evaluate(X_test, y_test, verbose = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = test['Title'].values\n",
    "tk = Tokenizer(lower = True)\n",
    "tk.fit_on_texts(X)\n",
    "X_seq = tk.texts_to_sequences(X)\n",
    "X_pad = pad_sequences(X_seq, maxlen = 100, padding = 'post')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.00554429]\n",
      " [0.00196987]\n",
      " [0.01750842]\n",
      " ...\n",
      " [0.00491612]\n",
      " [0.00194016]\n",
      " [0.00328234]]\n"
     ]
    }
   ],
   "source": [
    "title_predict = model.predict(X_pad, verbose = 0)\n",
    "print(title_predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "final = pd.DataFrame()\n",
    "final['IDLink'] = test['IDLink']\n",
    "final['SentimentTitle'] = title_predict\n",
    "final.to_csv('result.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}