notebooks/NaiveBayes.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "<a href=\"https://colab.research.google.com/github/LUMII-AILab/NLP_Course/blob/main/notebooks/NaiveBayes.ipynb\" target=\"_new\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open in Colab\"/></a>"
      ],
      "metadata": {
        "id": "fkWv6p4KNGlJ"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Naïve Bayes text classifier"
      ],
      "metadata": {
        "id": "ESQRdSMPa8El"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Hands-on dataset: *20 Newsgroup* assembled by Ken Lang @ CMU.\n",
        "\n",
        "https://www.kaggle.com/datasets/au1206/20-newsgroup-original\n",
        "\n",
        "We will use a format-converted, single-file version available from the course GitHub repo."
      ],
      "metadata": {
        "id": "nsj6NXaJc4uD"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Setting up the environment"
      ],
      "metadata": {
        "id": "inrvmAZJ77hy"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!wget https://raw.githubusercontent.com/LUMII-AILab/NLP_Course/main/notebooks/resources/news20/20_newsgroup.tsv\n",
        "\n",
        "!wget https://raw.githubusercontent.com/LUMII-AILab/NLP_Course/main/notebooks/resources/news20/20_newsgroup-freq.tsv\n",
        "\n",
        "!wget https://raw.githubusercontent.com/LUMII-AILab/NLP_Course/main/notebooks/resources/news20/stoplist.txt"
      ],
      "metadata": {
        "id": "32Ed8tE4e3i3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install nltk\n",
        "!pip install scikit-learn\n",
        "!pip install seaborn"
      ],
      "metadata": {
        "id": "bvRF7jVfnb20"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import nltk\n",
        "nltk.download('punkt')\n",
        "\n",
        "from sklearn.model_selection import KFold\n",
        "from sklearn.metrics import classification_report\n",
        "from sklearn.metrics import confusion_matrix\n",
        "\n",
        "import numpy\n",
        "\n",
        "import seaborn\n",
        "import matplotlib.pyplot as mplot"
      ],
      "metadata": {
        "id": "7ZyIOCZFF-dI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import re\n",
        "import os\n",
        "import sys\n",
        "import pickle\n",
        "import datetime"
      ],
      "metadata": {
        "id": "JLi2qLKgjpHI"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Text preprocessing"
      ],
      "metadata": {
        "id": "AyQT5jzp8s0-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def initialise(stop_txt, freq_tsv):\n",
        "\tglobal STOPLIST\n",
        "\tSTOPLIST = set()\n",
        "\n",
        "\twith open(stop_txt) as txt:\n",
        "\t\tfor word in txt:\n",
        "\t\t\tSTOPLIST.add(normalize_text(word.strip()))\n",
        "\n",
        "\tprint(\"[I] Word stoplist is read:\", len(STOPLIST))\n",
        "\n",
        "\tglobal WHITELIST\n",
        "\tWHITELIST = set()\n",
        "\n",
        "\twith open(freq_tsv) as tsv:\n",
        "\t\tfor entry in tsv:\n",
        "\t\t\tfreq, word = entry.strip().split(\"\\t\")\n",
        "\n",
        "\t\t\tif int(freq) < 3:\n",
        "        # TODO: experiment with the threshold (e.g., 3 / 5 / 10)\n",
        "\t\t\t\t# Ignore the long tail: most words occure less than N times\n",
        "\t\t\t\tcontinue\n",
        "\n",
        "\t\t\tWHITELIST.add(normalize_text(word))\n",
        "\n",
        "\tprint(\"[I] Word whitelist is read:\", len(WHITELIST))"
      ],
      "metadata": {
        "id": "oy2iikUGj49E"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def normalize_text(text):\n",
        "\ttext = text.lower()\n",
        "\ttext = re.sub(r'\\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Z|a-z]{2,}\\b', '', text) # e-mail addresses\n",
        "\ttext = re.sub(r'https?://[A-Za-z0-9./-]+|www\\.[A-Za-z0-9./-]+', '', text)\t\t\t\t# URLs\n",
        "\ttext = re.sub(r'\\d+', \"100\", text)\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t    # numbers\n",
        "\n",
        "\treturn text.strip()\n",
        "\n",
        "\n",
        "def normalize_vector(vector):\n",
        "\twords = list(vector.keys())\n",
        "\n",
        "\tfor w in words:\n",
        "\t\tif w in STOPLIST or len(w) == 1 or w not in WHITELIST:\n",
        "\t\t\tvector.pop(w)\n",
        "\n",
        "\treturn vector\n",
        "\n",
        "\n",
        "def vectorize_text(text):\n",
        "\treturn normalize_vector({word: True for word in nltk.word_tokenize(normalize_text(text))})"
      ],
      "metadata": {
        "id": "ahWkVvFbkDp0"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def read_data(file):\n",
        "\tdata_set = {}  # topic => samples\n",
        "\n",
        "\twith open(file) as data:\n",
        "\t\tfor entry in data:\n",
        "\t\t\ttopic, text = entry.strip().split(\"\\t\")\n",
        "\n",
        "\t\t\tsub_set = []\n",
        "\t\t\tif topic in data_set:\n",
        "\t\t\t\tsub_set = data_set[topic]\n",
        "\n",
        "\t\t\tsub_set.append((vectorize_text(text), topic))\n",
        "\t\t\tdata_set[topic] = sub_set\n",
        "\n",
        "\treturn data_set\n",
        "\n",
        "\n",
        "def join_data(data_set):\n",
        "\tunion = []\n",
        "\n",
        "\tfor cat in data_set:\n",
        "\t\tunion += data_set[cat]\n",
        "\n",
        "\treturn union"
      ],
      "metadata": {
        "id": "cP1KYQxYmD-C"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Experimentation & evaluation"
      ],
      "metadata": {
        "id": "2rPWMJS_9DhV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def cross_validate(data_set, k):\n",
        "\tglobal LABELS\n",
        "\tLABELS = []\n",
        "\n",
        "\tkfold = KFold(n_splits=k, shuffle=True)\n",
        "\n",
        "\tdata_split = {}\n",
        "\n",
        "\tfor cat in data_set:\n",
        "\t\tLABELS.append(cat)\n",
        "\n",
        "\t\t# K-Fold split for each class to ensure balanced training and test data sets\n",
        "\t\tfolds = []\n",
        "\n",
        "\t\tfor train, test in kfold.split(data_set[cat]):    # k loops\n",
        "\t\t\ttrain_data = numpy.array(data_set[cat])[train]  # vs. data[train]\n",
        "\t\t\ttest_data = numpy.array(data_set[cat])[test]    # vs. data[test]\n",
        "\t\t\tfolds.append({\"train\": train_data, \"test\": test_data})\n",
        "\n",
        "\t\tdata_split[cat] = folds\n",
        "\n",
        "\tvalidations = []\n",
        "\tgold_result = []\n",
        "\tsilver_result = []\n",
        "\n",
        "\tfor i in range(k):\n",
        "\t\t# Join the training and test data into two respective sets\n",
        "\t\ttrain_data = numpy.array([])\n",
        "\t\ttest_data = numpy.array([])\n",
        "\n",
        "\t\tfor cat in data_split:\n",
        "\t\t\tif len(train_data) > 0:\n",
        "\t\t\t\ttrain_data = numpy.append(train_data, data_split[cat][i][\"train\"], axis=0)\n",
        "\t\t\telse:\n",
        "\t\t\t\ttrain_data = data_split[cat][i][\"train\"]\n",
        "\n",
        "\t\t\tif len(test_data) > 0:\n",
        "\t\t\t\ttest_data = numpy.append(test_data, data_split[cat][i][\"test\"], axis=0)\n",
        "\t\t\telse:\n",
        "\t\t\t\ttest_data = data_split[cat][i][\"test\"]\n",
        "\n",
        "\t\t# Naive Bayes classifier: training and evaluation\n",
        "\t\tnb = nltk.NaiveBayesClassifier.train(train_data)\n",
        "\t\tvalidations.append(nltk.classify.accuracy(nb, test_data))\n",
        "\n",
        "\t\tfor t in test_data:\n",
        "\t\t\tgold_result.append(t[1])\n",
        "\t\t\tsilver_result.append(nb.classify(t[0]))\n",
        "\n",
        "\treturn (validations, gold_result, silver_result)"
      ],
      "metadata": {
        "id": "RDuowS50bQuc"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def run_validation(data_path, k):\n",
        "\t\tprint(\"{0}-fold cross-validation:\\n\".format(k))\n",
        "\n",
        "\t\tstart_time = datetime.datetime.now().replace(microsecond=0)\n",
        "\n",
        "\t\t# Run k-fold cross-validation\n",
        "\t\tvalidations, gold, silver = cross_validate(read_data(data_path), k)\n",
        "\n",
        "\t\t# Print the average accuracy: for each cross-validation step, and overall\n",
        "\t\tfor step in validations:\n",
        "\t\t\t\tprint(\"{0:.2f}  \".format(step), end='')\n",
        "\t\tprint(\"{0:.0%}\".format(numpy.mean(validations)))\n",
        "\n",
        "\t\tend_time = datetime.datetime.now().replace(microsecond=0)\n",
        "\t\tprint(\"\\nTotal validation time:\", end_time - start_time, \"\\n\")\n",
        "\n",
        "\t\t# Print an evaluation report\n",
        "\t\tprint(classification_report(gold, silver))\n",
        "\n",
        "\t\t# Print a fancy confusion matrix\n",
        "\t\tmatrix = confusion_matrix(gold, silver)\n",
        "\t\tseaborn.heatmap(matrix, xticklabels=LABELS, yticklabels=LABELS)\n",
        "\t\tmplot.xticks(rotation=90)\n",
        "\t\tmplot.show()\n",
        "\t\t# cf. print(nltk.ConfusionMatrix(gold_total, silver_total))"
      ],
      "metadata": {
        "id": "RmYktSqhl0pB"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Training for production"
      ],
      "metadata": {
        "id": "9hMPAJtq9MWH"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def run_training(data_path, verbose):\n",
        "\t\tprint(\"[I] Training an NB classifier...\")\n",
        "\t\tstart_time = datetime.datetime.now().replace(microsecond=0)\n",
        "\n",
        "\t\t# TRAINING\n",
        "\t\t# The final (production) model is trained by using all available data (train+test)\n",
        "\t\tnb = nltk.NaiveBayesClassifier.train(join_data(read_data(data_path)))\n",
        "\n",
        "\t\tend_time = datetime.datetime.now().replace(microsecond=0)\n",
        "\t\tprint(\"[I] Training time:\", end_time - start_time)\n",
        "\n",
        "\t\tif verbose:\n",
        "\t\t\t\tnb.show_most_informative_features(n=10) # Try with n=100\n",
        "\n",
        "\t\t# Save the model for later use\n",
        "\t\twith open(\"nb_classifier.pickle\", \"wb\") as dmp:\n",
        "\t\t\t\tpickle.dump(nb, dmp)\n",
        "\t\t\t\tprint(\"[I] NB classifier stored in a file\")"
      ],
      "metadata": {
        "id": "mb76hkhclwFC"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## The inference part"
      ],
      "metadata": {
        "id": "0bOz45sF9oyM"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def run_inference():\n",
        "\t\t# Load the pre-trained model\n",
        "\t\twith open(\"nb_classifier.pickle\", \"rb\") as dmp:\n",
        "\t\t\t\tnb = pickle.load(dmp)\n",
        "\t\t\t\tprint(\"[I] NB classifier loaded from a file\")\n",
        "\n",
        "\t\twhile True:\n",
        "\t\t\t\ttext = input(\"\\nEnter a text to classify: \")\n",
        "\t\t\t\tif len(text) == 0: break\n",
        "\n",
        "\t\t\t\t# Extract text features for classification\n",
        "\t\t\t\ttext_feat = vectorize_text(text)\n",
        "\t\t\t\tprint(\"\\nFeatures:\", text_feat.keys(), \"\\n\")\n",
        "\n",
        "\t\t\t\t# INFERENCE\n",
        "\t\t\t\t# Calculate a probability distribution over the classes\n",
        "\t\t\t\tprob_dist = nb.prob_classify(text_feat)\n",
        "\n",
        "\t\t\t\t# Return the probability distribution\n",
        "\t\t\t\tfor label in prob_dist.samples():\n",
        "\t\t\t\t\t\tprint(\"{0}: {1:.3f}\".format(label, prob_dist.prob(label)))\n",
        "\n",
        "\t\t\t\t# Return the most probable class\n",
        "\t\t\t\tprint(\"\\nPrediction:\", prob_dist.max())"
      ],
      "metadata": {
        "id": "Kc3BPltRlQqB"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Execution"
      ],
      "metadata": {
        "id": "Dht3RHRG-isE"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Initialise the stopword and word frequency lists\n",
        "initialise('stoplist.txt', '20_newsgroup-freq.tsv')"
      ],
      "metadata": {
        "id": "4SCQbnBGk5_W"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Run k-fold cross-validation\n",
        "run_validation(\"20_newsgroup.tsv\", k=5)\n",
        "\n",
        "# TODO:\n",
        "# * experiment with preprocessing, feature extraction and 'hyperparameters'\n",
        "# * evaluate and compare results"
      ],
      "metadata": {
        "id": "OLdwTyUHt2aJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Train and save the final model\n",
        "run_training(\"20_newsgroup.tsv\", True) # True=verbose"
      ],
      "metadata": {
        "id": "9JIDVtoRowU3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Run the pre-trained model\n",
        "run_inference()"
      ],
      "metadata": {
        "id": "31sq0_Gsk9Bf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Some test cases (generated with ChatGPT):\n",
        "* `alt.atheism`: *Religion lacks empirical evidence to justify supernatural claims.*\n",
        "* `soc.religion.christian`: *Faith in Jesus brings profound peace and eternal salvation.*\n",
        "* `sci.med`: *Regular exercise contributes to overall well-being and disease prevention.*\n",
        "* `sci.space`: *Satellite technology advances have revolutionized global communication networks.*\n",
        "* `sci.space`: *The Kepler telescope's discovery of exoplanets revolutionizes our search for extraterrestrial life.*"
      ],
      "metadata": {
        "id": "pkeyn-AJzs37"
      }
    }
  ]
}