disaster_tweets_NLP.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c4707b9f",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext sql\n",
    "import pandas as pd\n",
    "import os\n",
    "import numpy as np\n",
    "from sklearn import feature_extraction, linear_model, model_selection, preprocessing\n",
    "from config import db_host, db_name, username, db_pw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "20f9c39f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Connected: disaster@disaster_db'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "host = \"localhost\"\n",
    "database = \"disaster_db\"\n",
    "user = (username)\n",
    "password = (db_pw)\n",
    "connection_string = f\"postgresql://{user}:{password}@{host}/{database}\"\n",
    "%sql $connection_string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d7f64d41",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " * postgresql://disaster:***@localhost/disaster_db\n",
      "10 rows affected.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "    <thead>\n",
       "        <tr>\n",
       "            <th>id</th>\n",
       "            <th>keyword</th>\n",
       "            <th>location</th>\n",
       "            <th>text</th>\n",
       "        </tr>\n",
       "    </thead>\n",
       "    <tbody>\n",
       "        <tr>\n",
       "            <td>0</td>\n",
       "            <td></td>\n",
       "            <td></td>\n",
       "            <td>Just happened a terrible car crash</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>2</td>\n",
       "            <td></td>\n",
       "            <td></td>\n",
       "            <td>&quot;Heard about #earthquake is different cities, stay safe everyone.&quot;</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>3</td>\n",
       "            <td></td>\n",
       "            <td></td>\n",
       "            <td>&quot;there is a forest fire at spot pond, geese are fleeing across the street, I cannot save them all&quot;</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>9</td>\n",
       "            <td></td>\n",
       "            <td></td>\n",
       "            <td>Apocalypse lighting. #Spokane #wildfires</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>11</td>\n",
       "            <td></td>\n",
       "            <td></td>\n",
       "            <td>Typhoon Soudelor kills 28 in China and Taiwan</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>12</td>\n",
       "            <td></td>\n",
       "            <td></td>\n",
       "            <td>We&#x27;re shaking...It&#x27;s an earthquake</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>21</td>\n",
       "            <td></td>\n",
       "            <td></td>\n",
       "            <td>&quot;They&#x27;d probably still show more life than Arsenal did yesterday, eh? EH?&quot;</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>22</td>\n",
       "            <td></td>\n",
       "            <td></td>\n",
       "            <td>Hey! How are you?</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>27</td>\n",
       "            <td></td>\n",
       "            <td></td>\n",
       "            <td>What a nice hat?</td>\n",
       "        </tr>\n",
       "        <tr>\n",
       "            <td>29</td>\n",
       "            <td></td>\n",
       "            <td></td>\n",
       "            <td>Fuck off!</td>\n",
       "        </tr>\n",
       "    </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "[(0, '', '', 'Just happened a terrible car crash'),\n",
       " (2, '', '', '\"Heard about #earthquake is different cities, stay safe everyone.\"'),\n",
       " (3, '', '', '\"there is a forest fire at spot pond, geese are fleeing across the street, I cannot save them all\"'),\n",
       " (9, '', '', 'Apocalypse lighting. #Spokane #wildfires'),\n",
       " (11, '', '', 'Typhoon Soudelor kills 28 in China and Taiwan'),\n",
       " (12, '', '', \"We're shaking...It's an earthquake\"),\n",
       " (21, '', '', '\"They\\'d probably still show more life than Arsenal did yesterday, eh? EH?\"'),\n",
       " (22, '', '', 'Hey! How are you?'),\n",
       " (27, '', '', 'What a nice hat?'),\n",
       " (29, '', '', 'Fuck off!')]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%sql\n",
    "\n",
    "SELECT * from test LIMIT 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "1f502e8f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>keyword</th>\n",
       "      <th>location</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>Just happened a terrible car crash</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>\"Heard about #earthquake is different cities, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>\"there is a forest fire at spot pond, geese ar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>Apocalypse lighting. #Spokane #wildfires</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>Typhoon Soudelor kills 28 in China and Taiwan</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id keyword location                                               text\n",
       "0   0                                  Just happened a terrible car crash\n",
       "1   2                   \"Heard about #earthquake is different cities, ...\n",
       "2   3                   \"there is a forest fire at spot pond, geese ar...\n",
       "3   9                            Apocalypse lighting. #Spokane #wildfires\n",
       "4  11                       Typhoon Soudelor kills 28 in China and Taiwan"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df = pd.read_sql_query('''SELECT * FROM test;''', connection_string)\n",
    "test_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "c3ec9d88",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>keywork</th>\n",
       "      <th>location</th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>Our Deeds are the Reason of this #earthquake M...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>Forest fire near La Ronge Sask. Canada</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>All residents asked to 'shelter in place' are ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>\"13,000 people receive #wildfires evacuation o...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>Just got sent this photo from Ruby #Alaska as ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id keywork location                                               text  \\\n",
       "0   1                   Our Deeds are the Reason of this #earthquake M...   \n",
       "1   4                              Forest fire near La Ronge Sask. Canada   \n",
       "2   5                   All residents asked to 'shelter in place' are ...   \n",
       "3   6                   \"13,000 people receive #wildfires evacuation o...   \n",
       "4   7                   Just got sent this photo from Ruby #Alaska as ...   \n",
       "\n",
       "   target  \n",
       "0       1  \n",
       "1       1  \n",
       "2       1  \n",
       "3       1  \n",
       "4       1  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df = pd.read_sql_query('''SELECT * FROM train;''', connection_string)\n",
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "f729a7a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "count_vectorizer = feature_extraction.text.CountVectorizer()\n",
    "\n",
    "# counts for the first 5 tweets in the train_df\n",
    "example_train_vectors = count_vectorizer.fit_transform(train_df[\"text\"][0:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "faa5d439",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 54)\n",
      "[[0 0 0 1 1 1 0 0 0 0 0 0 1 1 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 1 0 0 0 0 1 0\n",
      "  0 0 0 1 0 0 0 0 0 0 0 0 0 1 1 0 1 0]]\n"
     ]
    }
   ],
   "source": [
    "print(example_train_vectors[0].todense().shape)\n",
    "print(example_train_vectors[0].todense())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "0943696e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create vectors for all tweets\n",
    "train_vectors = count_vectorizer.fit_transform(train_df[\"text\"])\n",
    "# ensure vectors used in test are the same vectors used in train\n",
    "test_vectors = count_vectorizer.transform(test_df[\"text\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "9dd641b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# utilize ridge regression to push model weights toward 0\n",
    "clf = linear_model.RidgeClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "16e83471",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.5958132 , 0.56681351, 0.6481665 ])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores = model_selection.cross_val_score(clf, train_vectors, train_df[\"target\"], cv=3, scoring = \"f1\")\n",
    "scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "e0a3c37f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RidgeClassifier()"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.fit(train_vectors, train_df[\"target\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "94c4f0c4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  target\n",
       "0   0     NaN\n",
       "1   2     NaN\n",
       "2   3     NaN\n",
       "3   9     NaN\n",
       "4  11     NaN"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_submission = pd.read_csv(\"kaggle/input/nlp-getting-started/sample_submission.csv\")\n",
    "sample_submission.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "0d48a210",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  target\n",
       "0   0       0\n",
       "1   2       1\n",
       "2   3       1\n",
       "3   9       0\n",
       "4  11       1"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_submission[\"target\"] = clf.predict(test_vectors)\n",
    "sample_submission.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "94a8d072",
   "metadata": {},
   "outputs": [],
   "source": [
    "# sample_submission.to_csv(\"submission.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6c1f4637",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "mlenv",
   "language": "python",
   "name": "mlenv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}