Exercise session 1 (Unsolved).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class DataSet():\n",
    "    def __init__(self, name=\"train\", path=\"dataset\"):\n",
    "        pass\n",
    "\n",
    "    def read(self,bodies,stances):\n",
    "        pass\n",
    "    \n",
    "    def print(self):\n",
    "        print(self.data.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "training = DataSet()\n",
    "test = DataSet(name=\"competition_test\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print(training.data.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Cleaning the text and tokenizing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /Users/skynet/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package wordnet to /Users/skynet/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import re\n",
    "import nltk\n",
    "import numpy as np\n",
    "from sklearn import feature_extraction\n",
    "from tqdm import tqdm\n",
    "\n",
    "nltk.download('punkt')\n",
    "nltk.download('wordnet')\n",
    "\n",
    "_wnl = nltk.WordNetLemmatizer() \n",
    "\n",
    "def normalize_word(w):\n",
    "    pass\n",
    "\n",
    "def get_tokenized_lemmas(s):\n",
    "    pass\n",
    "\n",
    "def clean(s):\n",
    "    # Cleans a string: Lowercasing, trimming, removing non-alphanumeric\n",
    "    pass\n",
    "\n",
    "def remove_stopwords(l):\n",
    "    # Removes stopwords from a list of tokens\n",
    "    pass\n",
    "\n",
    "def get_clean_tokens(s):\n",
    "    s = clean(s)\n",
    "    tokens = get_tokenized_lemmas(s)\n",
    "    clean_tokens = remove_stopwords(tokens)\n",
    "    return clean_tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "training.data['Headline tokens'] = #..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print(training.data.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Reload the dataframe because it takes a long time to transform the body :-) \n",
    "training.data = pd.read_pickle(\"trainingdatatokens.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print(training.data.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# view most common words\n",
    "from collections import Counter\n",
    "\n",
    "min_len = 2 #tunable parameter\n",
    "\n",
    "count_words = Counter()\n",
    "for tokens in training.data['articleBody tokens'].values:\n",
    "    # ...?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "count_words.most_common(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Vectorize: from text to a vector space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vocabulary = #..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "listed_docs = #...\n",
    "listed_docs2 = #..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Vectorize - fit: Fit vocabulary means generate the vector space for the words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "vectorizer = TfidfVectorizer(stop_words=\"english\", ngram_range=(2,2), min_df=5)\n",
    "#\n",
    "#"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Vectorize - transform: Represent a document on the vector space that was fit previously"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "training.data['tfidf headlines'] = #..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print(training.data.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load data just to make it less heavy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "training.data = pd.read_pickle(\"data_tfidf.pkl\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "How does the vector of a document looks like?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#\n",
    "#"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Measuring similarity between two documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "a = vectorizer.transform([listed_docs[0]])\n",
    "b = vectorizer.transform([listed_docs[200]])\n",
    "print(cosine_similarity(a,b))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Find the headline closest to the first body"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# find the headline closest to the first body:\n",
    "#\n",
    "#\n",
    "#\n",
    "#\n",
    "#\n",
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print(\"Similarity: %f\" %sim_old)\n",
    "print(\"Most similar: %s\" %listed_docs2[most_similar_indx])\n",
    "print(\"Match for: %s\" %listed_docs[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Reintroduce the task\n",
    "\n",
    "Ok, so let's do some fake news detection.\n",
    "\n",
    "Given a headline and a body of text, we want to say whether these two are:\n",
    "\n",
    "* unrelated\n",
    "* agree with each other\n",
    "* disagree with each other\n",
    "* discuss each other\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Features ?\n",
    "\n",
    "First, we have to generate features.\n",
    "\n",
    "Let's check:\n",
    "\n",
    "* Word overlap (between headline and body)\n",
    "* Refuting features: words which are refuting\n",
    "* Polarity features: words which contain polarity\n",
    "* n-gram\n",
    "* char-grams\n",
    "* co-occurences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def word_overlap_features(headlines, bodies):\n",
    "    # Computes the percentage of overlap between the headline and the body of text (numerical)\n",
    "    pass\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "word_overlap_features(list(training.data['Headline'].values[0:10]), list(training.data['articleBody'].values[0:10]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def refuting_features(headlines, bodies):\n",
    "    _refuting_words = [\n",
    "        'fake',\n",
    "        'fraud',\n",
    "        'hoax',\n",
    "        'false',\n",
    "        'deny', 'denies',\n",
    "        'not',\n",
    "        'despite',\n",
    "        'nope',\n",
    "        'doubt', 'doubts',\n",
    "        'bogus',\n",
    "        'debunk',\n",
    "        'pranks',\n",
    "        'retract'\n",
    "    ]\n",
    "    # pass\n",
    "    return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "refuting_features(list(training.data['Headline'].values)[0:1], list(training.data['articleBody'].values)[0:1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def polarity_features(headlines, bodies):\n",
    "    _refuting_words = [\n",
    "        'fake',\n",
    "        'fraud',\n",
    "        'hoax',\n",
    "        'false',\n",
    "        'deny', 'denies',\n",
    "        'not',\n",
    "        'despite',\n",
    "        'nope',\n",
    "        'doubt', 'doubts',\n",
    "        'bogus',\n",
    "        'debunk',\n",
    "        'pranks',\n",
    "        'retract'\n",
    "    ]\n",
    "\n",
    "    def calculate_polarity(text):\n",
    "        tokens = get_tokenized_lemmas(text)\n",
    "        return sum([t in _refuting_words for t in tokens]) % 2\n",
    "    \n",
    "    X = []\n",
    "    #pass\n",
    "    return np.array(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "polarity_features(list(training.data['Headline'].values)[0:10], list(training.data['articleBody'].values)[0:10])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Some other features... "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def ngrams(input, n):\n",
    "    input = input.split(' ')\n",
    "    output = []\n",
    "    for i in range(len(input) - n + 1):\n",
    "        output.append(input[i:i + n])\n",
    "    return output\n",
    "\n",
    "def chargrams(input, n):\n",
    "    output = []\n",
    "    for i in range(len(input) - n + 1):\n",
    "        output.append(input[i:i + n])\n",
    "    return output\n",
    "\n",
    "def append_chargrams(features, text_headline, text_body, size):\n",
    "    grams = [' '.join(x) for x in chargrams(\" \".join(remove_stopwords(text_headline.split())), size)]\n",
    "    grams_hits = 0\n",
    "    grams_early_hits = 0\n",
    "    grams_first_hits = 0\n",
    "    for gram in grams:\n",
    "        if gram in text_body:\n",
    "            grams_hits += 1\n",
    "        if gram in text_body[:255]:\n",
    "            grams_early_hits += 1\n",
    "        if gram in text_body[:100]:\n",
    "            grams_first_hits += 1\n",
    "    features.append(grams_hits)\n",
    "    features.append(grams_early_hits)\n",
    "    features.append(grams_first_hits)\n",
    "    return features\n",
    "\n",
    "def append_ngrams(features, text_headline, text_body, size):\n",
    "    grams = [' '.join(x) for x in ngrams(text_headline, size)]\n",
    "    grams_hits = 0\n",
    "    grams_early_hits = 0\n",
    "    for gram in grams:\n",
    "        if gram in text_body:\n",
    "            grams_hits += 1\n",
    "        if gram in text_body[:255]:\n",
    "            grams_early_hits += 1\n",
    "    features.append(grams_hits)\n",
    "    features.append(grams_early_hits)\n",
    "    return features\n",
    "\n",
    "def hand_features(headlines, bodies):\n",
    "    def binary_co_occurence(headline, body):\n",
    "        # Count how many times a token in the title\n",
    "        # appears in the body text.\n",
    "        bin_count = 0\n",
    "        bin_count_early = 0\n",
    "        for headline_token in clean(headline).split(\" \"):\n",
    "            if headline_token in clean(body):\n",
    "                bin_count += 1\n",
    "            if headline_token in clean(body)[:255]:\n",
    "                bin_count_early += 1\n",
    "        return [bin_count, bin_count_early]\n",
    "\n",
    "    def binary_co_occurence_stops(headline, body):\n",
    "        # Count how many times a token in the title\n",
    "        # appears in the body text. Stopwords in the title\n",
    "        # are ignored.\n",
    "        bin_count = 0\n",
    "        bin_count_early = 0\n",
    "        for headline_token in remove_stopwords(clean(headline).split(\" \")):\n",
    "            if headline_token in clean(body):\n",
    "                bin_count += 1\n",
    "                bin_count_early += 1\n",
    "        return [bin_count, bin_count_early]\n",
    "\n",
    "    def count_grams(headline, body):\n",
    "        # Count how many times an n-gram of the title\n",
    "        # appears in the entire body, and intro paragraph\n",
    "        clean_body = clean(body)\n",
    "        clean_headline = clean(headline)\n",
    "        features = []\n",
    "        features = append_chargrams(features, clean_headline, clean_body, 2)\n",
    "        features = append_chargrams(features, clean_headline, clean_body, 8)\n",
    "        features = append_chargrams(features, clean_headline, clean_body, 4)\n",
    "        features = append_chargrams(features, clean_headline, clean_body, 16)\n",
    "        features = append_ngrams(features, clean_headline, clean_body, 2)\n",
    "        features = append_ngrams(features, clean_headline, clean_body, 3)\n",
    "        features = append_ngrams(features, clean_headline, clean_body, 4)\n",
    "        features = append_ngrams(features, clean_headline, clean_body, 5)\n",
    "        features = append_ngrams(features, clean_headline, clean_body, 6)\n",
    "        return features\n",
    "\n",
    "    X = []\n",
    "    for headline, body in zip(headlines, bodies):\n",
    "        X.append(binary_co_occurence(headline, body)\n",
    "                 + binary_co_occurence_stops(headline, body)\n",
    "                 + count_grams(headline, body))\n",
    "\n",
    "    return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "hand_features(list(training.data['Headline'].values)[0:1], list(training.data['articleBody'].values)[0:1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def gen_or_load_feats(feat_fn, headlines, bodies, feature_file):\n",
    "    if not os.path.isfile(feature_file):\n",
    "        feats = feat_fn(headlines, bodies)\n",
    "        np.save(feature_file, feats)\n",
    "\n",
    "    return np.load(feature_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "LABELSINT = {'agree':0, 'disagree':1, 'discuss':2, 'unrelated':3}\n",
    "\n",
    "def generate_features(dataset,name):\n",
    "    h, b, y = [],[],[]\n",
    "\n",
    "    y = [LABELSINT[label] for label in list(dataset['Stance'].values)]\n",
    "    h = list(dataset['Headline'].values)\n",
    "    b = list(dataset['articleBody'].values)\n",
    "\n",
    "    X_overlap  = #\n",
    "    X_refuting = #\n",
    "    X_polarity = #\n",
    "    X_hand = #\n",
    "    \n",
    "    X = np.c_[X_hand, X_polarity, X_refuting, X_overlap]\n",
    "    return X,y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Classification\n",
    "Now, we choose our \"favourite\" classification algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from utils.dependencies import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "\n",
    "LABELS = ['agree', 'disagree', 'discuss', 'unrelated']\n",
    "LABELS_RELATED = ['unrelated','related']\n",
    "RELATED = LABELS[0:3]\n",
    "\n",
    "d = DataSet(\"train\")\n",
    "folds,hold_out = kfold_split(d,n_folds=10)\n",
    "fold_stances, hold_out_stances = get_stances_for_folds(d,folds,hold_out)\n",
    "\n",
    "# Load the competition dataset\n",
    "competition_dataset = DataSet(\"competition_test\")\n",
    "X_competition, y_competition = generate_features(competition_dataset.data, \"competition\")\n",
    "\n",
    "training = DataSet()\n",
    "test = DataSet(name=\"competition_test\")\n",
    "Xs = dict()\n",
    "ys = dict()\n",
    "\n",
    "# Load/Precompute all features now\n",
    "#\n",
    "#\n",
    "#\n",
    "\n",
    "best_score = 0\n",
    "best_fold = None\n",
    "\n",
    "# Classifier for each fold\n",
    "for fold in fold_stances:\n",
    "    ids = list(range(len(folds)))\n",
    "    del ids[fold]\n",
    "\n",
    "    X_train = np.vstack(tuple([Xs[i] for i in ids]))\n",
    "    y_train = np.hstack(tuple([ys[i] for i in ids]))\n",
    "\n",
    "    X_test = Xs[fold]\n",
    "    y_test = ys[fold]\n",
    "\n",
    "    clf = # \n",
    "    clf.fit(X_train, y_train)\n",
    "\n",
    "    predicted = [LABELS[int(a)] for a in clf.predict(X_test)]\n",
    "    actual = [LABELS[int(a)] for a in y_test]\n",
    "\n",
    "    fold_score, _ = score_submission(actual, predicted)\n",
    "    max_fold_score, _ = score_submission(actual, actual)\n",
    "\n",
    "    score = fold_score/float(max_fold_score)\n",
    "\n",
    "    print(\"Score for fold \"+ str(fold) + \" was - \" + str(score))\n",
    "    if score > best_score:\n",
    "        best_score = score\n",
    "        best_fold = clf\n",
    "\n",
    "\n",
    "#Run on Holdout set and report the final score on the holdout set\n",
    "predicted = [LABELS[int(a)] for a in best_fold.predict(X_holdout)]\n",
    "actual = [LABELS[int(a)] for a in y_holdout]\n",
    "\n",
    "print(\"Scores on the dev set\")\n",
    "report_score(actual,predicted)\n",
    "print(\"\")\n",
    "print(\"\")\n",
    "\n",
    "#Run on competition dataset\n",
    "predicted = [LABELS[int(a)] for a in best_fold.predict(X_competition)]\n",
    "actual = [LABELS[int(a)] for a in y_competition]\n",
    "\n",
    "print(\"Scores on the test set\")\n",
    "report_score(actual,predicted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Run on competition dataset\n",
    "predicted = [LABELS[int(a)] for a in best_fold.predict(X_competition)]\n",
    "actual = [LABELS[int(a)] for a in y_competition]\n",
    "\n",
    "print(\"Scores on the test set\")\n",
    "report_score(actual,predicted)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}