DMTM-Project-2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data cleaning\n",
    "\n",
    "Aspect based sentiment analysis. Project - 2 of CS 583 - Data Mining and Text Mining taught by professor Bing Liu at UIC."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Importing required packages."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\sanku\\Anaconda3\\lib\\site-packages\\sklearn\\cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n",
      "C:\\Users\\sanku\\Anaconda3\\lib\\site-packages\\sklearn\\grid_search.py:42: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. This module will be removed in 0.20.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "# imports\n",
    "import csv\n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem.snowball import SnowballStemmer\n",
    "from nltk.util import ngrams\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.naive_bayes import BernoulliNB\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn import cross_validation\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.model_selection import cross_val_predict\n",
    "from sklearn import svm\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk.tag.stanford import StanfordPOSTagger\n",
    "from nltk.parse.stanford import StanfordDependencyParser\n",
    "import re\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn import grid_search\n",
    "import nltk"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Knowledge and tools required for pre-processing."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\sanku\\Anaconda3\\lib\\site-packages\\nltk\\tag\\stanford.py:149: DeprecationWarning: \n",
      "The StanfordTokenizer will be deprecated in version 3.2.5.\n",
      "Please use nltk.tag.corenlp.CoreNLPPOSTagger or nltk.tag.corenlp.CoreNLPNERTagger instead.\n",
      "  super(StanfordPOSTagger, self).__init__(*args, **kwargs)\n"
     ]
    }
   ],
   "source": [
    "# data required for pre processing\n",
    "happy_emoji = [':‑)',':)',':-]',':]',':-3',':3',':->',':>','8-)','8)',':-}', ':}',':o)',':c)',':^)','=]','=)',\n",
    "                  ':‑D',':D','8‑D','8D','x‑D','xD','X‑D','XD','=D','=3','B^D',':-))']\n",
    "                \n",
    "sad_emoji = [':‑(',':(',':‑c',':c',':‑<',':<',':‑[',':[',':-||','>:[',':{',':@','>:(']\n",
    "    \n",
    "negative_words = [\"doesn't\", \"isn't\", \"wasn't\", \"shouldn't\", \"wouldn't\", \"couldn't\", \"won't\", \"can't\", \"don't\"]\n",
    "    \n",
    "stop_words = set(stopwords.words('english'))\n",
    "    \n",
    "# pre-processing\n",
    "tokenizer = RegexpTokenizer(r'\\w+')\n",
    "wnl = WordNetLemmatizer()\n",
    "st = StanfordPOSTagger('english-left3words-distsim.tagger')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DATA 1\n",
    "\n",
    "### Reading the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('Data_1_train.csv', newline='') as csvfile1:\n",
    "    reader1 = csv.DictReader(csvfile1)\n",
    "    \"\"\"\n",
    "    for row in reader2:\n",
    "        print(row['example_id'])\n",
    "        print(row[' text'])\n",
    "        print(row[' aspect_term'])\n",
    "        print(row[' term_location'])\n",
    "        print(row[' class'])\n",
    "        print()        \n",
    "    \"\"\"\n",
    "    \n",
    "    \n",
    "    singles1 = []\n",
    "    texts1 = []\n",
    "    tempText1 = ''\n",
    "    classes1 = []\n",
    "    aspectTerm1 = []\n",
    "    \n",
    "    for row in reader1:\n",
    "        # aspect term\n",
    "        aspTL = []\n",
    "        for asp in tokenizer.tokenize(row[' aspect_term']):\n",
    "            aspTL.append(wnl.lemmatize(asp))\n",
    "        \n",
    "        aspectTerm1.append(aspTL)\n",
    "        \n",
    "        # labels\n",
    "        classes1.append(row[' class'])\n",
    "        \n",
    "        # review text\n",
    "        tempText1 = row[' text'].replace('[comma]',',').lower()\n",
    "               \n",
    "        for hem in happy_emoji:\n",
    "            if hem in tempText1:\n",
    "                tempText1 = tempText1.replace(hem, 'happy')\n",
    "        \n",
    "        for sem in sad_emoji:\n",
    "            if sem in tempText1:\n",
    "                tempText1 = tempText1.replace(sem, 'sad')\n",
    "                \n",
    "        for negw in negative_words:\n",
    "            if negw in tempText1:\n",
    "                tempText1 = tempText1.replace(negw, 'not')\n",
    "        \n",
    "        for t in tokenizer.tokenize(tempText1):\n",
    "            if t not in stop_words or t == 'not':\n",
    "                singles1.append(wnl.lemmatize(t))\n",
    "        \n",
    "        texts1.append(singles1)\n",
    "        singles1 = []\n",
    "        \n",
    "    newText1 = []\n",
    "    \n",
    "    for tk in texts1:\n",
    "        newText1.append(' '.join(tk))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Approach\n",
    "\n",
    "### Sentiment analysis without considering aspect\n",
    "\n",
    "Tf-idf vectorize the review text."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfi = TfidfVectorizer(ngram_range=(1, 1))\n",
    "tfidf_data1 = tfi.fit_transform(newText1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<2203x2884 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 22058 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 185,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf_data1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using Bernoulli Naive Bayes with cross validation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "         -1       0.69      0.75      0.72       828\n",
      "          0       0.64      0.32      0.43       436\n",
      "          1       0.73      0.84      0.78       939\n",
      "\n",
      "avg / total       0.70      0.70      0.69      2203\n",
      "\n",
      "The accuracy score is 70.31%\n"
     ]
    }
   ],
   "source": [
    "clf1NB = BernoulliNB()\n",
    "predicted1NB = cross_val_predict(clf1NB, tfidf_data1, classes1, cv=10)\n",
    "\n",
    "print(classification_report(classes1,predicted1NB))\n",
    "print(\"The accuracy score is {:.2%}\".format(metrics.accuracy_score(classes1, predicted1NB)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we will try the SVM but before doing that we use `GridSearchCV` for parameter tuning."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'C': 2, 'kernel': 'linear'}\n"
     ]
    }
   ],
   "source": [
    "parameters = {'kernel':('linear', 'rbf', 'sigmoid', 'poly'), 'C':[0.001, 0.01, 0.1, 1, 2, 3, 4, 5, 10]}\n",
    "clf1SVCGS = grid_search.GridSearchCV(svm.SVC(), parameters, cv=10)\n",
    "clf1SVCGS.fit(tfidf_data1, classes1)\n",
    "print(clf1SVCGS.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now that we have the optimum parameters, we can pass the same data to a linear SVM with 10 fold cross validation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "         -1       0.72      0.78      0.75       828\n",
      "          0       0.57      0.47      0.52       436\n",
      "          1       0.79      0.80      0.79       939\n",
      "\n",
      "avg / total       0.72      0.73      0.72      2203\n",
      "\n",
      "The accuracy score is 72.54%\n"
     ]
    }
   ],
   "source": [
    "clf1SVC = svm.SVC(kernel='linear', C=2, random_state=0)\n",
    "predicted1SVC = cross_val_predict(clf1SVC, tfidf_data1, classes1, cv=10)\n",
    "\n",
    "print(classification_report(classes1, predicted1SVC))\n",
    "print(\"The accuracy score is {:.2%}\".format(metrics.accuracy_score(classes1, predicted1SVC)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### If using this as final classifier."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SVC(C=2, cache_size=200, class_weight=None, coef0=0.0,\n",
       "  decision_function_shape='ovr', degree=3, gamma='auto', kernel='linear',\n",
       "  max_iter=-1, probability=False, random_state=0, shrinking=True,\n",
       "  tol=0.001, verbose=False)"
      ]
     },
     "execution_count": 188,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf1Final = svm.SVC(kernel='linear', C=2, random_state=0)\n",
    "clf1Final.fit(tfidf_data1, classes1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Approach\n",
    "\n",
    "### Read POS tagged data\n",
    "\n",
    "Read from file and find nouns, adjectives, adverbs in the review text."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we do POS tagging of the pre-processed text and save the generated POS tags in a text file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "textPOS1 = []\n",
    "    \n",
    "for nt in newText1:\n",
    "    textPOS1.append(nltk.pos_tag(nt.split()))\n",
    "        \n",
    "# save pos tags to txt file\n",
    "f1pos = open('PosTagsData1Full.txt','w')\n",
    "for row in textPOS1:\n",
    "    f1pos.write(str(row)+\"\\n\")\n",
    "f1pos.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We read the pos tags data from txt file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "combinedList1 = ['NN', 'NNS', 'NNP', 'NNPS', 'JJ', 'JJS', 'JJR', 'RB', 'RBR', 'RBS']\n",
    "\n",
    "posTextList1 = []\n",
    "\n",
    "finalReviewList1 = []\n",
    "strTemp1 = \"\"\n",
    "\n",
    "fpos1 = open('PosTagsData1Full.txt','r')\n",
    "postags1 = fpos1.readline()\n",
    "\n",
    "while postags1:            \n",
    "    posTextList1 = re.findall(r\"[\\w]+\", postags1)\n",
    "    \n",
    "    for i in range(len(posTextList1)):\n",
    "        if posTextList1[i] in combinedList1:\n",
    "            strTemp1 = strTemp1 + str(' ') + posTextList1[i - 1]\n",
    "    finalReviewList1.append(strTemp1)\n",
    "    strTemp1 = \"\"\n",
    "\n",
    "    postags1 = fpos1.readline()\n",
    "fpos1.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we have the new review text in a list named `finalReviewList1`. Now we use this list for count and tfidf transformation (which is equivalent to tfidf verctorization)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "count_vectorizer1 = CountVectorizer(ngram_range=(1, 3))\n",
    "data_tfidf1 = count_vectorizer2.fit_transform(finalReviewList1)\n",
    "tfidf_data1 = TfidfTransformer(use_idf=False).fit_transform(data_tfidf1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After vectorization, we now pass this data to Naive Bayes classifier with 10 fold cross validation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "         -1       0.69      0.80      0.74       828\n",
      "          0       0.65      0.24      0.35       436\n",
      "          1       0.72      0.83      0.77       939\n",
      "\n",
      "avg / total       0.69      0.70      0.68      2203\n",
      "\n",
      "The accuracy score is 70.22%\n"
     ]
    }
   ],
   "source": [
    "clf1NB = BernoulliNB()\n",
    "predicted1NB = cross_val_predict(clf1NB, tfidf_data1, classes1, cv=10)\n",
    "\n",
    "print(classification_report(classes1,predicted1NB))\n",
    "print(\"The accuracy score is {:.2%}\".format(metrics.accuracy_score(classes1, predicted1NB)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we will try the SVM but before doing that we use `GridSearchCV` for parameter tuning."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'C': 2, 'kernel': 'linear'}\n"
     ]
    }
   ],
   "source": [
    "parameters = {'kernel':('linear', 'rbf', 'sigmoid', 'poly'), 'C':[0.001, 0.01, 0.1, 1, 2, 3, 4, 5, 10]}\n",
    "clf1SVCGS = grid_search.GridSearchCV(svm.SVC(), parameters, cv=10)\n",
    "clf1SVCGS.fit(tfidf_data1, classes1)\n",
    "print(clf1SVCGS.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now that we have the optimum parameters, we can pass the same data to a linear SVM with 10 fold cross validation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "         -1       0.72      0.78      0.75       828\n",
      "          0       0.61      0.48      0.54       436\n",
      "          1       0.78      0.80      0.79       939\n",
      "\n",
      "avg / total       0.72      0.73      0.72      2203\n",
      "\n",
      "The accuracy score is 72.86%\n"
     ]
    }
   ],
   "source": [
    "clf1SVC = svm.SVC(kernel='linear', C=2, random_state=0)\n",
    "predicted1SVC = cross_val_predict(clf1SVC, tfidf_data1, classes1, cv=10)\n",
    "\n",
    "print(classification_report(classes1, predicted1SVC))\n",
    "print(\"The accuracy score is {:.2%}\".format(metrics.accuracy_score(classes1, predicted1SVC)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### If using this as final classifier."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SVC(C=2, cache_size=200, class_weight=None, coef0=0.0,\n",
       "  decision_function_shape='ovr', degree=3, gamma='auto', kernel='linear',\n",
       "  max_iter=-1, probability=False, random_state=0, shrinking=True,\n",
       "  tol=0.001, verbose=False)"
      ]
     },
     "execution_count": 188,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf1Final = svm.SVC(kernel='linear', C=2, random_state=0)\n",
    "clf1Final.fit(tfidf_data1, classes1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Approach\n",
    "\n",
    "### Read dependency parsed data\n",
    "\n",
    "Read dependency parsing data and extract nouns and the adjectives that affect them. We use the code below to get the dependency parsing output as a list."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we do the dependecy parsing using stanford corenlp."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "depParsingList1 = []\n",
    "    \n",
    "dep_parser1=StanfordDependencyParser(model_path=\"edu/stanford/nlp/models/lexparser/englishPCFG.ser.gz\")\n",
    "\n",
    "# send it in parts so that jvm does not run out of memory\n",
    "for nt in newText1[0:1000]:\n",
    "    result1 = dep_parser1.raw_parse(nt)    \n",
    "    dep1 = result1.__next__()    \n",
    "    depParsingList1.append(list(dep1.triples()))\n",
    "    \n",
    "fdep1 = open('depParseData1P1.txt','w')\n",
    "for dp in depParsingList1:\n",
    "    fdep1.write(str(dp)+\"\\n\")\n",
    "fdep1.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As we now have our dependency parsed list in the txt file so now we can extract the aspect terms and the adjectives and adverbs which affect them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "fulldeplist1 = []\n",
    "\n",
    "fdep1 = open('depParseData2Final.txt','r')\n",
    "deppars1 = fdep2.readline()\n",
    "while deppars1:\n",
    "    deppars1 = deppars1.replace('[','')\n",
    "    deppars1 = deppars1.replace(']','')\n",
    "\n",
    "    striptxt1 = deppars1.split(')),')\n",
    "    \n",
    "    relationlist1 = []\n",
    "    \n",
    "    for st in striptxt1:\n",
    "        relationlist1.append(re.findall(r\"[\\w]+\", st))\n",
    "        \n",
    "    fulldeplist1.append(relationlist1)\n",
    "\n",
    "    deppars1 = fdep1.readline()\n",
    "fdep1.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we take the dependency parsed list and extract noun-adjective and noun-adverb pairs for each review."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "nounList = ['NN', 'NNS', 'NNP', 'NNPS']\n",
    "adjAdvList = ['JJ', 'JJS', 'JJR', 'RB', 'RBR', 'RBS']\n",
    "newReviewText1 = \"\"\n",
    "newDepList1 = []\n",
    "cnt1 = 0\n",
    "remClasses1 = []\n",
    "\n",
    "for l in fulldeplist1:\n",
    "    for m in l:\n",
    "        if len(m) > 4:\n",
    "            if m[1] in nounList:\n",
    "                if m[4] in adjAdvList:\n",
    "                    newReviewText1 = newReviewText1 + str(' ') + m[3] + str(' ') + m[0]\n",
    "\n",
    "            elif m[4] in nounList:\n",
    "                if m[1] in adjAdvList:\n",
    "                    newReviewText1 = newReviewText1 + str(' ') + m[0] + str(' ') + m[3]\n",
    "\n",
    "    if newReviewText1 != \"\":\n",
    "        remClasses1.append(classes1[cnt2])\n",
    "        newDepList1.append(newReviewText1)\n",
    "    newReviewText1 = \"\"\n",
    "    cnt1 += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then we do Tfidf vectorization (count vectorization and tfidf transformation) so that they can be used in Naive Bayes and SVM classifier."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "count_vectorizer1 = CountVectorizer(ngram_range=(1, 2))\n",
    "data_tfidf1 = count_vectorizer2.fit_transform(newDepList1)\n",
    "tfidf_data1 = TfidfTransformer(use_idf=False).fit_transform(data_tfidf1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now train a Bernoulli Naive Bayes classifier also doing 10 fold cross validation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "         -1       0.36      0.22      0.28       666\n",
      "          0       0.14      0.08      0.10       473\n",
      "          1       0.64      0.80      0.71      1864\n",
      "\n",
      "avg / total       0.50      0.56      0.52      3003\n",
      "\n",
      "The accuracy score is 55.61%\n"
     ]
    }
   ],
   "source": [
    "clf1NBA2 = BernoulliNB()\n",
    "predicted1NBA2 = cross_val_predict(clf1NBA2, tfidf_data1, remClasses1, cv=10)\n",
    "\n",
    "print(classification_report(remClasses1,predicted1NBA2))\n",
    "print(\"The accuracy score is {:.2%}\".format(metrics.accuracy_score(remClasses1, predicted1NBA2)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we use `GridSearchCV` to find the optimal parameters for SVM."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'C': 1, 'kernel': 'linear'}\n"
     ]
    }
   ],
   "source": [
    "parameters = {'kernel':('linear', 'rbf', 'sigmoid', 'poly'), 'C':[0.001, 0.01, 0.1, 1, 2, 3, 4, 5, 10]}\n",
    "clf1SVCA2GS = grid_search.GridSearchCV(svm.SVC(), parameters, cv=10)\n",
    "clf1SVCA2GS.fit(tfidf_data1, remClasses1)\n",
    "print(clf1SVCA2GS.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we train SVM classifier with optimal parameters also doing 10 fold cross validation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "         -1       0.54      0.34      0.42       666\n",
      "          0       0.26      0.08      0.12       473\n",
      "          1       0.70      0.91      0.79      1864\n",
      "\n",
      "avg / total       0.59      0.65      0.60      3003\n",
      "\n",
      "The accuracy score is 65.33%\n"
     ]
    }
   ],
   "source": [
    "clf1SVCA2 = svm.SVC(kernel='linear', C=1, random_state=0)\n",
    "predicted1SVCA2 = cross_val_predict(clf1SVCA2, tfidf_data1, remClasses1, cv=10)\n",
    "\n",
    "print(classification_report(remClasses1,predicted1SVCA2))\n",
    "print(\"The accuracy score is {:.2%}\".format(metrics.accuracy_score(remClasses1, predicted1SVCA2)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### If using this as final classifier."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SVC(C=2, cache_size=200, class_weight=None, coef0=0.0,\n",
       "  decision_function_shape='ovr', degree=3, gamma='auto', kernel='linear',\n",
       "  max_iter=-1, probability=False, random_state=0, shrinking=True,\n",
       "  tol=0.001, verbose=False)"
      ]
     },
     "execution_count": 188,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf1Final = svm.SVC(kernel='linear', C=1, random_state=0)\n",
    "clf1Final.fit(tfidf_data1, classes1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DATA 2\n",
    "\n",
    "### Reading the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('Data_2_train.csv', newline='') as csvfile2:\n",
    "    reader2 = csv.DictReader(csvfile2)\n",
    "    \"\"\"\n",
    "    for row in reader2:\n",
    "        print(row['example_id'])\n",
    "        print(row[' text'])\n",
    "        print(row[' aspect_term'])\n",
    "        print(row[' term_location'])\n",
    "        print(row[' class'])\n",
    "        print()        \n",
    "    \"\"\"\n",
    "    \n",
    "    \n",
    "    singles2 = []\n",
    "    texts2 = []\n",
    "    tempText2 = ''\n",
    "    classes2 = []\n",
    "    aspectTerm2 = []\n",
    "    \n",
    "    for row in reader2:\n",
    "        # aspect term\n",
    "        aspTL = []\n",
    "        for asp in tokenizer.tokenize(row[' aspect_term']):\n",
    "            aspTL.append(wnl.lemmatize(asp))\n",
    "        \n",
    "        aspectTerm2.append(aspTL)\n",
    "        \n",
    "        # labels\n",
    "        classes2.append(row[' class'])\n",
    "        \n",
    "        # review text\n",
    "        tempText2 = row[' text'].replace('[comma]',',').lower()\n",
    "               \n",
    "        for hem in happy_emoji:\n",
    "            if hem in tempText2:\n",
    "                tempText2 = tempText2.replace(hem, 'happy')\n",
    "        \n",
    "        for sem in sad_emoji:\n",
    "            if sem in tempText2:\n",
    "                tempText2 = tempText2.replace(sem, 'sad')\n",
    "                \n",
    "        for negw in negative_words:\n",
    "            if negw in tempText2:\n",
    "                tempText2 = tempText2.replace(negw, 'not')\n",
    "        \n",
    "        for t in tokenizer.tokenize(tempText2):\n",
    "            if t not in stop_words or t == 'not':\n",
    "                singles2.append(wnl.lemmatize(t))\n",
    "        \n",
    "        texts2.append(singles2)\n",
    "        singles2 = []\n",
    "        \n",
    "    newText2 = []\n",
    "    \n",
    "    for tk in texts2:\n",
    "        newText2.append(' '.join(tk))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Approach\n",
    "\n",
    "### Sentiment analysis without considering aspect\n",
    "\n",
    "Tf-idf vectorize the review text."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [],
   "source": [
    "#count_vectorizer2 = CountVectorizer(ngram_range=(1, 1))\n",
    "#data_tfidf2 = count_vectorizer2.fit_transform(newText2)\n",
    "#tfidf_data2 = TfidfTransformer(use_idf=False).fit_transform(data_tfidf2)\n",
    "\n",
    "tfi2 = TfidfVectorizer(ngram_range=(1, 1))\n",
    "tfidf_data2 = tfi2.fit_transform(newText2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using Bernoulli Naive Bayes with cross validation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "         -1       0.49      0.33      0.40       805\n",
      "          0       0.28      0.20      0.23       633\n",
      "          1       0.70      0.84      0.76      2164\n",
      "\n",
      "avg / total       0.58      0.61      0.59      3602\n",
      "\n",
      "The accuracy score is 61.35%\n"
     ]
    }
   ],
   "source": [
    "clf2NB = BernoulliNB()\n",
    "predicted2NB = cross_val_predict(clf2NB, tfidf_data2, classes2, cv=10)\n",
    "\n",
    "print(classification_report(classes2,predicted2NB))\n",
    "print(\"The accuracy score is {:.2%}\".format(metrics.accuracy_score(classes2, predicted2NB)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we will try the SVM but before doing that we use `GridSearchCV` for parameter tuning."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'C': 1, 'kernel': 'linear'}\n"
     ]
    }
   ],
   "source": [
    "parameters = {'kernel':('linear', 'rbf', 'sigmoid', 'poly'), 'C':[0.001, 0.01, 0.1, 1, 2, 3, 4, 5, 10]}\n",
    "clf2SVCGS = grid_search.GridSearchCV(svm.SVC(), parameters, cv=10)\n",
    "clf2SVCGS.fit(tfidf_data2, classes2)\n",
    "print(clf2SVCGS.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now that we have the optimum parameters, we can pass the same data to a linear SVM with 10 fold cross validation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "         -1       0.55      0.43      0.48       805\n",
      "          0       0.46      0.27      0.34       633\n",
      "          1       0.74      0.89      0.81      2164\n",
      "\n",
      "avg / total       0.65      0.68      0.65      3602\n",
      "\n",
      "The accuracy score is 67.66%\n"
     ]
    }
   ],
   "source": [
    "clf2SVC = svm.SVC(kernel='linear', C=1, random_state=0)\n",
    "predicted2SVC = cross_val_predict(clf2SVC, tfidf_data2, classes2, cv=10)\n",
    "\n",
    "print(classification_report(classes2, predicted2SVC))\n",
    "print(\"The accuracy score is {:.2%}\".format(metrics.accuracy_score(classes2, predicted2SVC)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### If using this as final classifier."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SVC(C=1, cache_size=200, class_weight=None, coef0=0.0,\n",
       "  decision_function_shape='ovr', degree=3, gamma='auto', kernel='linear',\n",
       "  max_iter=-1, probability=False, random_state=0, shrinking=True,\n",
       "  tol=0.001, verbose=False)"
      ]
     },
     "execution_count": 208,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf2Final = svm.SVC(kernel='linear', C=1, random_state=0)\n",
    "clf2Final.fit(tfidf_data2, classes2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Approach\n",
    "\n",
    "### Read POS tagged data\n",
    "\n",
    "Read from file and find nouns, adjectives, adverbs in the review text."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we do POS tagging of the pre-processed text and save the generated POS tags in a text file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "# POS tagging\n",
    "textPOS2 = []\n",
    "    \n",
    "for nt in newText2:\n",
    "    textPOS2.append(nltk.pos_tag(nt.split()))\n",
    "        \n",
    "# save pos tags to txt file\n",
    "f2pos = open('PostagsData2Full.txt','w')\n",
    "for row in textPOS2:\n",
    "    f2pos.write(str(row)+\"\\n\")\n",
    "f2pos.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we read the pos tags from the txt file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "combinedList2 = ['NN', 'NNS', 'NNP', 'NNPS', 'JJ', 'JJS', 'JJR', 'RB', 'RBR', 'RBS']\n",
    "\n",
    "posTextList2 = []\n",
    "\n",
    "finalReviewList2 = []\n",
    "strTemp2 = \"\"\n",
    "\n",
    "fpos2 = open('postagsData2Full.txt','r')\n",
    "postags2 = fpos2.readline()\n",
    "\n",
    "while postags2:            \n",
    "    posTextList2 = re.findall(r\"[\\w]+\", postags2)\n",
    "    \n",
    "    for i in range(len(posTextList2)):\n",
    "        if posTextList2[i] in combinedList2:\n",
    "            strTemp2 = strTemp2 + str(' ') + posTextList2[i - 1]\n",
    "    finalReviewList2.append(strTemp2)\n",
    "    strTemp2 = \"\"\n",
    "\n",
    "    postags2 = fpos2.readline()\n",
    "fpos2.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we have the new review text in a list named `finalReviewList2`. Now we use this list for count and tfidf transformation (which is equivalent to tfidf verctorization)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "count_vectorizer2 = CountVectorizer(ngram_range=(1, 2))\n",
    "data_tfidf2 = count_vectorizer2.fit_transform(finalReviewList2)\n",
    "tfidf_data2 = TfidfTransformer(use_idf=False).fit_transform(data_tfidf2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After vectorization, we now pass this data to Naive Bayes classifier with 10 fold cross validation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "         -1       0.44      0.25      0.32       805\n",
      "          0       0.20      0.10      0.13       633\n",
      "          1       0.65      0.85      0.73      2164\n",
      "\n",
      "avg / total       0.52      0.58      0.53      3602\n",
      "\n",
      "The accuracy score is 58.22%\n"
     ]
    }
   ],
   "source": [
    "clf2 = BernoulliNB()\n",
    "predicted2 = cross_val_predict(clf2, tfidf_data2, classes2, cv=10)\n",
    "\n",
    "print(classification_report(classes2,predicted2))\n",
    "print(\"The accuracy score is {:.2%}\".format(metrics.accuracy_score(classes2, predicted2)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we will try the SVM but before doing that we use `GridSearchCV` for parameter tuning."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'C': 1, 'kernel': 'linear'}\n"
     ]
    }
   ],
   "source": [
    "parameters = {'kernel':('linear', 'rbf', 'sigmoid', 'poly'), 'C':[0.001, 0.01, 0.1, 1, 2, 3, 4, 5, 10]}\n",
    "clf2 = grid_search.GridSearchCV(svm.SVC(), parameters, cv=10)\n",
    "clf2.fit(tfidf_data2, classes2)\n",
    "print(clf2.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now that we have the optimum parameters, we can pass the same data to a linear SVM with 10 fold cross validation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "         -1       0.54      0.40      0.46       805\n",
      "          0       0.42      0.19      0.27       633\n",
      "          1       0.71      0.89      0.79      2164\n",
      "\n",
      "avg / total       0.62      0.66      0.62      3602\n",
      "\n",
      "The accuracy score is 65.80%\n"
     ]
    }
   ],
   "source": [
    "clf2 = svm.SVC(kernel='linear', C=1, random_state=0)\n",
    "predicted2 = cross_val_predict(clf2, tfidf_data2, classes2, cv=10)\n",
    "\n",
    "print(classification_report(classes2, predicted2))\n",
    "print(\"The accuracy score is {:.2%}\".format(metrics.accuracy_score(classes2, predicted2)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### If using this as final classifier."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SVC(C=1, cache_size=200, class_weight=None, coef0=0.0,\n",
       "  decision_function_shape='ovr', degree=3, gamma='auto', kernel='linear',\n",
       "  max_iter=-1, probability=False, random_state=0, shrinking=True,\n",
       "  tol=0.001, verbose=False)"
      ]
     },
     "execution_count": 208,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf2Final = svm.SVC(kernel='linear', C=1, random_state=0)\n",
    "clf2Final.fit(tfidf_data2, classes2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Approach\n",
    "\n",
    "### Read dependency parsed data\n",
    "\n",
    "Read dependency parsing data and extract nouns and the adjectives that affect them. We use the code below to get the dependency parsing output as a list."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we do the dependecy parsing using stanford corenlp."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "depParsingList2 = []\n",
    "    \n",
    "dep_parser2=StanfordDependencyParser(model_path=\"edu/stanford/nlp/models/lexparser/englishPCFG.ser.gz\")\n",
    "\n",
    "for nt in newText2[3500:3602]:\n",
    "    result2 = dep_parser2.raw_parse(nt)    \n",
    "    dep2 = result2.__next__()    \n",
    "    depParsingList2.append(list(dep2.triples()))\n",
    "    \n",
    "fdep2 = open('depParseData2P8.txt','w')\n",
    "for dp in depParsingList2:\n",
    "    fdep2.write(str(dp)+\"\\n\")\n",
    "fdep2.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As we now have our dependency parsed list in the txt file so now we can extract the aspect terms and the adjectives and adverbs which affect them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "fulldeplist2 = []\n",
    "\n",
    "fdep2 = open('depParseData2Final.txt','r')\n",
    "deppars2 = fdep2.readline()\n",
    "while deppars2:\n",
    "    #code for parsing dep\n",
    "    deppars2 = deppars2.replace('[','')\n",
    "    deppars2 = deppars2.replace(']','')\n",
    "\n",
    "    striptxt2 = deppars2.split(')),')\n",
    "    \n",
    "    relationlist2 = []\n",
    "    \n",
    "    for st in striptxt2:\n",
    "        relationlist2.append(re.findall(r\"[\\w]+\", st))\n",
    "        \n",
    "    fulldeplist2.append(relationlist2)\n",
    "\n",
    "    deppars2 = fdep2.readline()\n",
    "fdep2.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we take the dependency parsed list and extract noun-adjective and noun-adverb pairs for each review."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "nounList = ['NN', 'NNS', 'NNP', 'NNPS']\n",
    "adjAdvList = ['JJ', 'JJS', 'JJR', 'RB', 'RBR', 'RBS']\n",
    "newReviewText2 = \"\"\n",
    "newDepList2 = []\n",
    "cnt2 = 0\n",
    "remClasses2 = []\n",
    "\n",
    "for l in fulldeplist2:\n",
    "    for m in l:\n",
    "        if len(m) > 4:\n",
    "            if m[1] in nounList:\n",
    "                if m[4] in adjAdvList:\n",
    "                    newReviewText2 = newReviewText2 + str(' ') + m[3] + str(' ') + m[0]\n",
    "\n",
    "            elif m[4] in nounList:\n",
    "                if m[1] in adjAdvList:\n",
    "                    newReviewText2 = newReviewText2 + str(' ') + m[0] + str(' ') + m[3]\n",
    "\n",
    "    if newReviewText2 != \"\":\n",
    "        remClasses2.append(classes2[cnt2])\n",
    "        newDepList2.append(newReviewText2)\n",
    "    newReviewText2 = \"\"\n",
    "    cnt2 += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then we do Tfidf vectorization (count vectorization and tfidf transformation) so that they can be used in Naive Bayes and SVM classifier."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "count_vectorizer2 = CountVectorizer(ngram_range=(1, 2))\n",
    "data_tfidf2 = count_vectorizer2.fit_transform(newDepList2)\n",
    "tfidf_data2 = TfidfTransformer(use_idf=False).fit_transform(data_tfidf2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now train a Bernoulli Naive Bayes classifier also doing 10 fold cross validation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "         -1       0.36      0.22      0.28       666\n",
      "          0       0.14      0.08      0.10       473\n",
      "          1       0.64      0.80      0.71      1864\n",
      "\n",
      "avg / total       0.50      0.56      0.52      3003\n",
      "\n",
      "The accuracy score is 55.61%\n"
     ]
    }
   ],
   "source": [
    "clf2 = BernoulliNB()\n",
    "predicted2 = cross_val_predict(clf2, tfidf_data2, remClasses2, cv=10)\n",
    "\n",
    "print(classification_report(remClasses2,predicted2))\n",
    "print(\"The accuracy score is {:.2%}\".format(metrics.accuracy_score(remClasses2, predicted2)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we use `GridSearchCV` to find the optimal parameters for SVM."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'C': 1, 'kernel': 'linear'}\n"
     ]
    }
   ],
   "source": [
    "parameters = {'kernel':('linear', 'rbf', 'sigmoid', 'poly'), 'C':[0.001, 0.01, 0.1, 1, 2, 3, 4, 5, 10]}\n",
    "clf = grid_search.GridSearchCV(svm.SVC(), parameters, cv=10)\n",
    "clf.fit(tfidf_data2, remClasses2)\n",
    "print(clf.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we train SVM classifier with optimal parameters also doing 10 fold cross validation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "         -1       0.54      0.34      0.42       666\n",
      "          0       0.26      0.08      0.12       473\n",
      "          1       0.70      0.91      0.79      1864\n",
      "\n",
      "avg / total       0.59      0.65      0.60      3003\n",
      "\n",
      "The accuracy score is 65.33%\n"
     ]
    }
   ],
   "source": [
    "clf = svm.SVC(kernel='linear', C=1, random_state=0)\n",
    "predicted = cross_val_predict(clf, tfidf_data2, remClasses2, cv=10)\n",
    "\n",
    "print(classification_report(remClasses2,predicted))\n",
    "print(\"The accuracy score is {:.2%}\".format(metrics.accuracy_score(remClasses2, predicted)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### If using this as final classifier."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SVC(C=1, cache_size=200, class_weight=None, coef0=0.0,\n",
       "  decision_function_shape='ovr', degree=3, gamma='auto', kernel='linear',\n",
       "  max_iter=-1, probability=False, random_state=0, shrinking=True,\n",
       "  tol=0.001, verbose=False)"
      ]
     },
     "execution_count": 208,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf2Final = svm.SVC(kernel='linear', C=1, random_state=0)\n",
    "clf2Final.fit(tfidf_data2, classes2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test Data\n",
    "\n",
    "#### For Test Data 1:\n",
    "\n",
    "Reading data from csv file and pre-processing it for prediction."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('Data-1_test.csv', newline='') as csvfile1:\n",
    "    reader1 = csv.DictReader(csvfile1)\n",
    "    '''\n",
    "    for row in reader1:\n",
    "        print(row['example_id'])\n",
    "        print(row[' text'])\n",
    "        print(row[' aspect_term'])\n",
    "        print(row[' term_location'])\n",
    "        print()        \n",
    "    '''\n",
    "    \n",
    "     \n",
    "    singles1 = []\n",
    "    texts1 = []\n",
    "    tempText1 = ''\n",
    "    aspectTerm1 = []\n",
    "    ids = []\n",
    "    \n",
    "    for row in reader1:\n",
    "        #eample id\n",
    "        ids.append(row['example_id'])\n",
    "        \n",
    "        # aspect term\n",
    "        aspTL = []\n",
    "        for asp in tokenizer.tokenize(row[' aspect_term']):\n",
    "            aspTL.append(wnl.lemmatize(asp))\n",
    "        \n",
    "        aspectTerm1.append(aspTL)\n",
    "        \n",
    "        # review text\n",
    "        tempText1 = row[' text'].replace('[comma]',',').lower()\n",
    "               \n",
    "        for hem in happy_emoji:\n",
    "            if hem in tempText1:\n",
    "                tempText1 = tempText1.replace(hem, 'happy')\n",
    "        \n",
    "        for sem in sad_emoji:\n",
    "            if sem in tempText1:\n",
    "                tempText1 = tempText1.replace(sem, 'sad')\n",
    "                \n",
    "        for negw in negative_words:\n",
    "            if negw in tempText1:\n",
    "                tempText1 = tempText1.replace(negw, 'not')\n",
    "        \n",
    "        for t in tokenizer.tokenize(tempText1):\n",
    "            if t not in stop_words or t == 'not':\n",
    "                singles1.append(wnl.lemmatize(t))\n",
    "        \n",
    "        texts1.append(singles1)\n",
    "        singles1 = []\n",
    "        \n",
    "    newText1 = []\n",
    "    \n",
    "    for tk in texts1:\n",
    "        newText1.append(' '.join(tk))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Converting to Tf-idf vectors."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [],
   "source": [
    "tt = tfi.transform(newText1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Predicting using trained classifier."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [],
   "source": [
    "predicted1 = clf1Final.predict(tt)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Printing the final output txt file for data 1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [],
   "source": [
    "fout1 = open('outData2.txt','w')\n",
    "\n",
    "cnt1 = 0\n",
    "\n",
    "for dp in predicted1:\n",
    "    fout1.write(str(ids[cnt1]) + \";;\" +str(dp)+\"\\n\")\n",
    "    cnt1 += 1\n",
    "fout1.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### For Test Data 2:\n",
    "\n",
    "Reading data from csv file and pre-processing it for prediction."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('Data-2_test.csv', newline='') as csvfile2:\n",
    "    reader2 = csv.DictReader(csvfile2)\n",
    "    \"\"\"\n",
    "    for row in reader2:\n",
    "        print(row['example_id'])\n",
    "        print(row[' text'])\n",
    "        print(row[' aspect_term'])\n",
    "        print(row[' term_location'])\n",
    "        print(row[' class'])\n",
    "        print()        \n",
    "    \"\"\"\n",
    "    \n",
    "    singles2 = []\n",
    "    texts2 = []\n",
    "    tempText2 = ''\n",
    "    aspectTerm2 = []\n",
    "    ids2 = []\n",
    "    \n",
    "    for row in reader2:\n",
    "        ids2.append(row['example_id'])\n",
    "        \n",
    "        # aspect term\n",
    "        aspTL = []\n",
    "        for asp in tokenizer.tokenize(row[' aspect_term']):\n",
    "            aspTL.append(wnl.lemmatize(asp))\n",
    "        \n",
    "        aspectTerm2.append(aspTL)\n",
    "        \n",
    "        # review text\n",
    "        tempText2 = row[' text'].replace('[comma]',',').lower()\n",
    "               \n",
    "        for hem in happy_emoji:\n",
    "            if hem in tempText2:\n",
    "                tempText2 = tempText2.replace(hem, 'happy')\n",
    "        \n",
    "        for sem in sad_emoji:\n",
    "            if sem in tempText2:\n",
    "                tempText2 = tempText2.replace(sem, 'sad')\n",
    "                \n",
    "        for negw in negative_words:\n",
    "            if negw in tempText2:\n",
    "                tempText2 = tempText2.replace(negw, 'not')\n",
    "        \n",
    "        for t in tokenizer.tokenize(tempText2):\n",
    "            if t not in stop_words or t == 'not':\n",
    "                singles2.append(wnl.lemmatize(t))\n",
    "        \n",
    "        texts2.append(singles2)\n",
    "        singles2 = []\n",
    "        \n",
    "    newText2 = []\n",
    "    \n",
    "    for tk in texts2:\n",
    "        newText2.append(' '.join(tk))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Converting to Tf-idf vectors."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [],
   "source": [
    "tt2 = tfi2.transform(newText2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Predicting using trained classifier."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [],
   "source": [
    "predicted2 = clf2Final.predict(tt2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Printing the final output txt file for data 2."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [],
   "source": [
    "fout2 = open('outData2.txt','w')\n",
    "\n",
    "cnt2 = 0\n",
    "\n",
    "for dp in predicted2:\n",
    "    fout2.write(str(ids2[cnt2]) + \";;\" + str(dp) + \"\\n\")\n",
    "    cnt2 += 1\n",
    "fout2.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}