Multi class classification using Machine Learning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords\n",
    "import re\n",
    "import pandas as pd\n",
    "df = pd.read_csv('cuisine_data.csv')\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cuisine_description</th>\n",
       "      <th>cuisine</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>romaine lettuce black olives grape tomatoes ga...</td>\n",
       "      <td>greek</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>plain flour ground pepper salt tomatoes ground...</td>\n",
       "      <td>southern_us</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>eggs pepper salt mayonaise cooking oil green c...</td>\n",
       "      <td>filipino</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>water vegetable oil wheat salt</td>\n",
       "      <td>indian</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>black pepper shallots cornflour cayenne pepper...</td>\n",
       "      <td>indian</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 cuisine_description      cuisine\n",
       "0  romaine lettuce black olives grape tomatoes ga...        greek\n",
       "1  plain flour ground pepper salt tomatoes ground...  southern_us\n",
       "2  eggs pepper salt mayonaise cooking oil green c...     filipino\n",
       "3                     water vegetable oil wheat salt       indian\n",
       "4  black pepper shallots cornflour cayenne pepper...       indian"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df.cuisine.value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['greek', 'southern_us', 'filipino', 'indian', 'jamaican',\n",
       "       'spanish', 'italian', 'mexican', 'chinese', 'british', 'thai',\n",
       "       'vietnamese', 'cajun_creole', 'brazilian', 'french', 'japanese',\n",
       "       'irish', 'korean', 'moroccan', 'russian'], dtype=object)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.cuisine.unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Checking for NULLs and removing them"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "cuisine_description    0\n",
       "cuisine                0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dropping duplicates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(39774, 2)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.drop_duplicates(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(39677, 2)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocessing the text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "806112\n"
     ]
    }
   ],
   "source": [
    "print(df['cuisine_description'].apply(lambda x: len(x.split(' '))).sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "special_character_remover = re.compile('[/(){}\\[\\]\\|@,;]')\n",
    "extra_symbol_remover = re.compile('[^0-9a-z #+_]')\n",
    "STOPWORDS = set(stopwords.words('english'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_text(text):\n",
    "    text = text.lower()\n",
    "    text = special_character_remover.sub(' ', text)\n",
    "    text = extra_symbol_remover.sub('', text)\n",
    "    text = ' '.join(word for word in text.split() if word not in STOPWORDS)\n",
    "    return text\n",
    "    \n",
    "df['cuisine_description'] = df['cuisine_description'].apply(clean_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "803337\n"
     ]
    }
   ],
   "source": [
    "print(df['cuisine_description'].apply(lambda x: len(x.split(' '))).sum())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train Test split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X = df.cuisine_description\n",
    "y = df.cuisine\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state = 42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((27773,), (11904,), (27773,), (11904,))"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape,X_test.shape,y_train.shape,y_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Applying Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/akshay/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy is : 0.7819220430107527\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "\n",
    "lr = Pipeline([('vect', CountVectorizer()),\n",
    "               ('tfidf', TfidfTransformer()),\n",
    "               ('clf', LogisticRegression()),\n",
    "              ])\n",
    "\n",
    "lr.fit(X_train,y_train)\n",
    "y_pred1 = lr.predict(X_test)\n",
    "\n",
    "print(f\"Accuracy is : {accuracy_score(y_pred1,y_test)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Applying Naive Bayes Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy 0.6682627688172043\n"
     ]
    }
   ],
   "source": [
    "from sklearn.naive_bayes import MultinomialNB\n",
    "\n",
    "\n",
    "naivebayes = Pipeline([('vect', CountVectorizer()),\n",
    "               ('tfidf', TfidfTransformer()),\n",
    "               ('clf', MultinomialNB()),\n",
    "              ])\n",
    "naivebayes.fit(X_train, y_train)\n",
    "\n",
    "y_pred = naivebayes.predict(X_test)\n",
    "\n",
    "print(f'accuracy {accuracy_score(y_pred,y_test)}')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Applying Xgboost Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/akshay/anaconda3/lib/python3.7/site-packages/dask/dataframe/utils.py:14: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.\n",
      "  import pandas.util.testing as tm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy 0.778309811827957\n"
     ]
    }
   ],
   "source": [
    "from xgboost import XGBClassifier\n",
    "\n",
    "xgboost = Pipeline([('vect', CountVectorizer()),\n",
    "               ('tfidf', TfidfTransformer()),\n",
    "               ('clf', XGBClassifier()),\n",
    "              ])\n",
    "xgboost.fit(X_train, y_train)\n",
    "\n",
    "y_pred = xgboost.predict(X_test)\n",
    "\n",
    "print(f'accuracy {accuracy_score(y_pred,y_test)}')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Applying BERT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "from transformers import BertTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import BertForSequenceClassification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import TensorDataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'greek': 0,\n",
       " 'southern_us': 1,\n",
       " 'filipino': 2,\n",
       " 'indian': 3,\n",
       " 'jamaican': 4,\n",
       " 'spanish': 5,\n",
       " 'italian': 6,\n",
       " 'mexican': 7,\n",
       " 'chinese': 8,\n",
       " 'british': 9,\n",
       " 'thai': 10,\n",
       " 'vietnamese': 11,\n",
       " 'cajun_creole': 12,\n",
       " 'brazilian': 13,\n",
       " 'french': 14,\n",
       " 'japanese': 15,\n",
       " 'irish': 16,\n",
       " 'korean': 17,\n",
       " 'moroccan': 18,\n",
       " 'russian': 19}"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "possible_labels = df.cuisine.unique()\n",
    "\n",
    "label_dict = {}\n",
    "for index, possible_label in enumerate(possible_labels):\n",
    "    label_dict[possible_label] = index\n",
    "label_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['label'] = df.cuisine.replace(label_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cuisine_description</th>\n",
       "      <th>cuisine</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>romaine lettuce black olives grape tomatoes ga...</td>\n",
       "      <td>greek</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>plain flour ground pepper salt tomatoes ground...</td>\n",
       "      <td>southern_us</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>eggs pepper salt mayonaise cooking oil green c...</td>\n",
       "      <td>filipino</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>water vegetable oil wheat salt</td>\n",
       "      <td>indian</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>black pepper shallots cornflour cayenne pepper...</td>\n",
       "      <td>indian</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 cuisine_description      cuisine  label\n",
       "0  romaine lettuce black olives grape tomatoes ga...        greek      0\n",
       "1  plain flour ground pepper salt tomatoes ground...  southern_us      1\n",
       "2  eggs pepper salt mayonaise cooking oil green c...     filipino      2\n",
       "3                     water vegetable oil wheat salt       indian      3\n",
       "4  black pepper shallots cornflour cayenne pepper...       indian      3"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>cuisine_description</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cuisine</th>\n",
       "      <th>label</th>\n",
       "      <th>data_type</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">brazilian</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">13</th>\n",
       "      <th>train</th>\n",
       "      <td>394</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>val</th>\n",
       "      <td>69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">british</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">9</th>\n",
       "      <th>train</th>\n",
       "      <td>682</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>val</th>\n",
       "      <td>121</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">cajun_creole</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">12</th>\n",
       "      <th>train</th>\n",
       "      <td>1310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>val</th>\n",
       "      <td>231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">chinese</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">8</th>\n",
       "      <th>train</th>\n",
       "      <td>2266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>val</th>\n",
       "      <td>400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">filipino</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2</th>\n",
       "      <th>train</th>\n",
       "      <td>642</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>val</th>\n",
       "      <td>113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">french</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">14</th>\n",
       "      <th>train</th>\n",
       "      <td>2241</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>val</th>\n",
       "      <td>396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">greek</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">0</th>\n",
       "      <th>train</th>\n",
       "      <td>996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>val</th>\n",
       "      <td>176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">indian</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">3</th>\n",
       "      <th>train</th>\n",
       "      <td>2547</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>val</th>\n",
       "      <td>450</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">irish</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">16</th>\n",
       "      <th>train</th>\n",
       "      <td>567</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>val</th>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">italian</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">6</th>\n",
       "      <th>train</th>\n",
       "      <td>6656</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>val</th>\n",
       "      <td>1175</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">jamaican</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">4</th>\n",
       "      <th>train</th>\n",
       "      <td>444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>val</th>\n",
       "      <td>78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">japanese</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">15</th>\n",
       "      <th>train</th>\n",
       "      <td>1204</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>val</th>\n",
       "      <td>213</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">korean</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">17</th>\n",
       "      <th>train</th>\n",
       "      <td>703</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>val</th>\n",
       "      <td>124</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">mexican</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">7</th>\n",
       "      <th>train</th>\n",
       "      <td>5465</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>val</th>\n",
       "      <td>964</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">moroccan</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">18</th>\n",
       "      <th>train</th>\n",
       "      <td>695</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>val</th>\n",
       "      <td>123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">russian</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">19</th>\n",
       "      <th>train</th>\n",
       "      <td>416</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>val</th>\n",
       "      <td>73</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">southern_us</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">1</th>\n",
       "      <th>train</th>\n",
       "      <td>3654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>val</th>\n",
       "      <td>645</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">spanish</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">5</th>\n",
       "      <th>train</th>\n",
       "      <td>839</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>val</th>\n",
       "      <td>148</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">thai</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">10</th>\n",
       "      <th>train</th>\n",
       "      <td>1306</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>val</th>\n",
       "      <td>230</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">vietnamese</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">11</th>\n",
       "      <th>train</th>\n",
       "      <td>698</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>val</th>\n",
       "      <td>123</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              cuisine_description\n",
       "cuisine      label data_type                     \n",
       "brazilian    13    train                      394\n",
       "                   val                         69\n",
       "british      9     train                      682\n",
       "                   val                        121\n",
       "cajun_creole 12    train                     1310\n",
       "                   val                        231\n",
       "chinese      8     train                     2266\n",
       "                   val                        400\n",
       "filipino     2     train                      642\n",
       "                   val                        113\n",
       "french       14    train                     2241\n",
       "                   val                        396\n",
       "greek        0     train                      996\n",
       "                   val                        176\n",
       "indian       3     train                     2547\n",
       "                   val                        450\n",
       "irish        16    train                      567\n",
       "                   val                        100\n",
       "italian      6     train                     6656\n",
       "                   val                       1175\n",
       "jamaican     4     train                      444\n",
       "                   val                         78\n",
       "japanese     15    train                     1204\n",
       "                   val                        213\n",
       "korean       17    train                      703\n",
       "                   val                        124\n",
       "mexican      7     train                     5465\n",
       "                   val                        964\n",
       "moroccan     18    train                      695\n",
       "                   val                        123\n",
       "russian      19    train                      416\n",
       "                   val                         73\n",
       "southern_us  1     train                     3654\n",
       "                   val                        645\n",
       "spanish      5     train                      839\n",
       "                   val                        148\n",
       "thai         10    train                     1306\n",
       "                   val                        230\n",
       "vietnamese   11    train                      698\n",
       "                   val                        123"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train, X_val, y_train, y_val = train_test_split(df.index.values, \n",
    "                                                  df.label.values, \n",
    "                                                  test_size=0.15, \n",
    "                                                  random_state=42, \n",
    "                                                  stratify=df.label.values)\n",
    "\n",
    "df['data_type'] = ['not_set']*df.shape[0]\n",
    "\n",
    "df.loc[X_train, 'data_type'] = 'train'\n",
    "df.loc[X_val, 'data_type'] = 'val'\n",
    "\n",
    "df.groupby(['cuisine', 'label', 'data_type']).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cuisine_description</th>\n",
       "      <th>cuisine</th>\n",
       "      <th>label</th>\n",
       "      <th>data_type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>romaine lettuce black olives grape tomatoes ga...</td>\n",
       "      <td>greek</td>\n",
       "      <td>0</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>plain flour ground pepper salt tomatoes ground...</td>\n",
       "      <td>southern_us</td>\n",
       "      <td>1</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>eggs pepper salt mayonaise cooking oil green c...</td>\n",
       "      <td>filipino</td>\n",
       "      <td>2</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>water vegetable oil wheat salt</td>\n",
       "      <td>indian</td>\n",
       "      <td>3</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>black pepper shallots cornflour cayenne pepper...</td>\n",
       "      <td>indian</td>\n",
       "      <td>3</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 cuisine_description      cuisine  label  \\\n",
       "0  romaine lettuce black olives grape tomatoes ga...        greek      0   \n",
       "1  plain flour ground pepper salt tomatoes ground...  southern_us      1   \n",
       "2  eggs pepper salt mayonaise cooking oil green c...     filipino      2   \n",
       "3                     water vegetable oil wheat salt       indian      3   \n",
       "4  black pepper shallots cornflour cayenne pepper...       indian      3   \n",
       "\n",
       "  data_type  \n",
       "0     train  \n",
       "1     train  \n",
       "2     train  \n",
       "3     train  \n",
       "4     train  "
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cuisine_description</th>\n",
       "      <th>cuisine</th>\n",
       "      <th>label</th>\n",
       "      <th>data_type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>romaine lettuce black olives grape tomatoes ga...</td>\n",
       "      <td>greek</td>\n",
       "      <td>0</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>plain flour ground pepper salt tomatoes ground...</td>\n",
       "      <td>southern_us</td>\n",
       "      <td>1</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>eggs pepper salt mayonaise cooking oil green c...</td>\n",
       "      <td>filipino</td>\n",
       "      <td>2</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>water vegetable oil wheat salt</td>\n",
       "      <td>indian</td>\n",
       "      <td>3</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>black pepper shallots cornflour cayenne pepper...</td>\n",
       "      <td>indian</td>\n",
       "      <td>3</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 cuisine_description      cuisine  label  \\\n",
       "0  romaine lettuce black olives grape tomatoes ga...        greek      0   \n",
       "1  plain flour ground pepper salt tomatoes ground...  southern_us      1   \n",
       "2  eggs pepper salt mayonaise cooking oil green c...     filipino      2   \n",
       "3                     water vegetable oil wheat salt       indian      3   \n",
       "4  black pepper shallots cornflour cayenne pepper...       indian      3   \n",
       "\n",
       "  data_type  \n",
       "0     train  \n",
       "1     train  \n",
       "2     train  \n",
       "3     train  \n",
       "4     train  "
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "82f76a03f2574245b52cd79cbfbb1361",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', \n",
    "                                          do_lower_case=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
      "/home/akshay/anaconda3/envs/NLP/lib/python3.6/site-packages/transformers/tokenization_utils_base.py:2179: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
      "  FutureWarning,\n"
     ]
    }
   ],
   "source": [
    "encoded_data_train = tokenizer.batch_encode_plus(\n",
    "    df[df.data_type=='train'].cuisine_description.values, \n",
    "    add_special_tokens=True, \n",
    "    return_attention_mask=True, \n",
    "    pad_to_max_length=True, \n",
    "    max_length=256, \n",
    "    return_tensors='pt'\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoded_data_val = tokenizer.batch_encode_plus(\n",
    "    df[df.data_type=='val'].cuisine_description.values, \n",
    "    add_special_tokens=True, \n",
    "    return_attention_mask=True, \n",
    "    pad_to_max_length=True, \n",
    "    max_length=256, \n",
    "    return_tensors='pt'\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_ids_train = encoded_data_train['input_ids']\n",
    "attention_masks_train = encoded_data_train['attention_mask']\n",
    "labels_train = torch.tensor(df[df.data_type=='train'].label.values)\n",
    "\n",
    "input_ids_val = encoded_data_val['input_ids']\n",
    "attention_masks_val = encoded_data_val['attention_mask']\n",
    "labels_val = torch.tensor(df[df.data_type=='val'].label.values)\n",
    "\n",
    "dataset_train = TensorDataset(input_ids_train, attention_masks_train, labels_train)\n",
    "dataset_val = TensorDataset(input_ids_val, attention_masks_val, labels_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "075b99a931764fca8ac62e004898d5b4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=433.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e88b86bf9dd042429dbf9f1cf20dbcac",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=440473133.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
      "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "model = BertForSequenceClassification.from_pretrained(\"bert-base-uncased\",\n",
    "                                                      num_labels=len(label_dict),\n",
    "                                                      output_attentions=False,\n",
    "                                                      output_hidden_states=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n",
    "\n",
    "batch_size = 3\n",
    "\n",
    "dataloader_train = DataLoader(dataset_train, \n",
    "                              sampler=RandomSampler(dataset_train), \n",
    "                              batch_size=batch_size)\n",
    "\n",
    "dataloader_validation = DataLoader(dataset_val, \n",
    "                                   sampler=SequentialSampler(dataset_val), \n",
    "                                   batch_size=batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AdamW, get_linear_schedule_with_warmup\n",
    "\n",
    "optimizer = AdamW(model.parameters(),\n",
    "                  lr=1e-5, \n",
    "                  eps=1e-8)\n",
    "                  \n",
    "epochs = 5\n",
    "\n",
    "scheduler = get_linear_schedule_with_warmup(optimizer, \n",
    "                                            num_warmup_steps=0,\n",
    "                                            num_training_steps=len(dataloader_train)*epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import f1_score\n",
    "\n",
    "def f1_score_func(preds, labels):\n",
    "    preds_flat = np.argmax(preds, axis=1).flatten()\n",
    "    labels_flat = labels.flatten()\n",
    "    return f1_score(labels_flat, preds_flat, average='weighted')\n",
    "\n",
    "def accuracy_per_class(preds, labels):\n",
    "    label_dict_inverse = {v: k for k, v in label_dict.items()}\n",
    "    \n",
    "    preds_flat = np.argmax(preds, axis=1).flatten()\n",
    "    labels_flat = labels.flatten()\n",
    "\n",
    "    for label in np.unique(labels_flat):\n",
    "        y_preds = preds_flat[labels_flat==label]\n",
    "        y_true = labels_flat[labels_flat==label]\n",
    "        print(f'Class: {label_dict_inverse[label]}')\n",
    "        print(f'Accuracy: {len(y_preds[y_preds==label])}/{len(y_true)}\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "07117c5699114e39aadb991b40252be4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9451ee1d9fb94e1b9274e329b765eb8c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Epoch 1', max=11242.0, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-66-eb5eba7882fe>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     54\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mbatch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mprogress_bar\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     55\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 56\u001b[0;31m         \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     57\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     58\u001b[0m         \u001b[0mbatch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mb\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/NLP/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36mzero_grad\u001b[0;34m(self, set_to_none)\u001b[0m\n\u001b[1;32m   1338\u001b[0m                     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1339\u001b[0m                         \u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgrad\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrequires_grad_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1340\u001b[0;31m                     \u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgrad\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1341\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1342\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mshare_memory\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mT\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mT\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "import random\n",
    "import numpy as np\n",
    "\n",
    "seed_val = 17\n",
    "random.seed(seed_val)\n",
    "np.random.seed(seed_val)\n",
    "torch.manual_seed(seed_val)\n",
    "torch.cuda.manual_seed_all(seed_val)\n",
    "\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "def evaluate(dataloader_val):\n",
    "\n",
    "    model.eval()\n",
    "    \n",
    "    loss_val_total = 0\n",
    "    predictions, true_vals = [], []\n",
    "    \n",
    "    for batch in dataloader_val:\n",
    "        \n",
    "        batch = tuple(b.to(device) for b in batch)\n",
    "        \n",
    "        inputs = {'input_ids':      batch[0],\n",
    "                  'attention_mask': batch[1],\n",
    "                  'labels':         batch[2],\n",
    "                 }\n",
    "\n",
    "        with torch.no_grad():        \n",
    "            outputs = model(**inputs)\n",
    "            \n",
    "        loss = outputs[0]\n",
    "        logits = outputs[1]\n",
    "        loss_val_total += loss.item()\n",
    "\n",
    "        logits = logits.detach().cpu().numpy()\n",
    "        label_ids = inputs['labels'].cpu().numpy()\n",
    "        predictions.append(logits)\n",
    "        true_vals.append(label_ids)\n",
    "    \n",
    "    loss_val_avg = loss_val_total/len(dataloader_val) \n",
    "    \n",
    "    predictions = np.concatenate(predictions, axis=0)\n",
    "    true_vals = np.concatenate(true_vals, axis=0)\n",
    "            \n",
    "    return loss_val_avg, predictions, true_vals\n",
    "    \n",
    "for epoch in tqdm(range(1, epochs+1)):\n",
    "    \n",
    "    model.train()\n",
    "    \n",
    "    loss_train_total = 0\n",
    "\n",
    "    progress_bar = tqdm(dataloader_train, desc='Epoch {:1d}'.format(epoch), leave=False, disable=False)\n",
    "    for batch in progress_bar:\n",
    "\n",
    "        model.zero_grad()\n",
    "        \n",
    "        batch = tuple(b.to(device) for b in batch)\n",
    "        \n",
    "        inputs = {'input_ids':      batch[0],\n",
    "                  'attention_mask': batch[1],\n",
    "                  'labels':         batch[2],\n",
    "                 }       \n",
    "\n",
    "        outputs = model(**inputs)\n",
    "        \n",
    "        loss = outputs[0]\n",
    "        loss_train_total += loss.item()\n",
    "        loss.backward()\n",
    "\n",
    "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
    "\n",
    "        optimizer.step()\n",
    "        scheduler.step()\n",
    "        \n",
    "        progress_bar.set_postfix({'training_loss': '{:.3f}'.format(loss.item()/len(batch))})\n",
    "         \n",
    "        \n",
    "    torch.save(model.state_dict(), f'data_volume/finetuned_BERT_epoch_{epoch}.model')\n",
    "        \n",
    "    tqdm.write(f'\\nEpoch {epoch}')\n",
    "    \n",
    "    loss_train_avg = loss_train_total/len(dataloader_train)            \n",
    "    tqdm.write(f'Training loss: {loss_train_avg}')\n",
    "    \n",
    "    val_loss, predictions, true_vals = evaluate(dataloader_validation)\n",
    "    val_f1 = f1_score_func(predictions, true_vals)\n",
    "    tqdm.write(f'Validation loss: {val_loss}')\n",
    "    tqdm.write(f'F1 Score (Weighted): {val_f1}')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}