FastText.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "FastText.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wr3TxbEbwVYM",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "01413e69-7b49-48e0-ef06-9ff08dcd06d6"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import csv"
      ],
      "metadata": {
        "id": "tGD9T2ij3Ka2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data=pd.read_csv('drive/MyDrive/preprocessed_text_1.csv', header=None)\n",
        "data.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "fngxsMe42RSV",
        "outputId": "f7bdd366-8df3-4b35-9ed6-9a9843918e92"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-d0d8f517-af95-49c0-afe4-2d65f698cb86\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>knowing ki vikas kitna samjhata hai priyanka a...</td>\n",
              "      <td>no</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>i am muhajir  aur mere lye sab se pehly pakist...</td>\n",
              "      <td>no</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>doctor  sab sahi me ke phd in hate politics wa...</td>\n",
              "      <td>no</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>poore desh me patel obc me aate hain sirf gujr...</td>\n",
              "      <td>no</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>sarkar banne ke bad hindu hit me ek bhi faisla...</td>\n",
              "      <td>yes</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d0d8f517-af95-49c0-afe4-2d65f698cb86')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-d0d8f517-af95-49c0-afe4-2d65f698cb86 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-d0d8f517-af95-49c0-afe4-2d65f698cb86');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                                                   0    1\n",
              "0  knowing ki vikas kitna samjhata hai priyanka a...   no\n",
              "1  i am muhajir  aur mere lye sab se pehly pakist...   no\n",
              "2  doctor  sab sahi me ke phd in hate politics wa...   no\n",
              "3  poore desh me patel obc me aate hain sirf gujr...   no\n",
              "4  sarkar banne ke bad hindu hit me ek bhi faisla...  yes"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "with open(\"data.txt\", \"w\") as data_file:\n",
        "  [data_file.write(\"__label__{} {}\\n\".format(row[1], row[0])) for row in data.values]\n",
        "data_file.close"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QEjgUF-V3QFc",
        "outputId": "c2b56a47-52a8-4754-f4d5-5879ead5ce1e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<function TextIOWrapper.close>"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!head data.txt"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QDVV28RU57AX",
        "outputId": "914b5354-3ccc-477c-b47f-24d7351a5f34"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "__label__no knowing ki vikas kitna samjhata hai priyanka aur itch guard luv ko usne bola tha ben wali baat me ab sallu ne bhi agree kiya\n",
            "__label__no i am muhajir  aur mere lye sab se pehly pakistan he  agr 10 lakh altaf jese leaders bh is zameen ki behurmati kren un sbko sar e aam phansi deni chahye  proud to be a muhajir and pakistani\n",
            "__label__no doctor  sab sahi me ke phd in hate politics wale bhai padhe likhe ho fir kyu ye sab baate karte ho tum bas bowling  khelo aur maje lo pictwittercomfk1qubqstw\n",
            "__label__no poore desh me patel obc me aate hain sirf gujrat ko chor kar may be ye manuwadiyon bramanwadi kabhi aapko aarackchan nahi denge ye to jis obc ko mila hai usse bhi nafrat karte hain ye khoon aur chamdi ka frak karne waale bharmhanwadi kisi ke sage nahi hain\n",
            "__label__yes sarkar banne ke bad hindu hit me ek bhi faisla jo bjp ke dwara liya gaya hobjp ko  gaygobarmandirmasjid aur nafrat faila kar vot chahiye\n",
            "__label__yes hate it jab test ata ho phr bh acha na ho \n",
            "__label__no ek dil ek jaan akal ke imitihaan kal kp ke rape aur sawing between two legs after rape honge justified due to cinematic liberty aaj toh rajput ki ho gayi shaan urdu ki gulaam\n",
            "__label__yes aise logo se sakht nafrat karta hu jo caste ko naam ke sath jod ke chaude hote h but real me vo piddu hote h\n",
            "__label__yes sir jyada sentimental mat ho jao peene ke baadjiss desh ki buniyad sirf hindu hatred pe hui ho woh kabhi nahi sudrega aap pok de do yaa phir kabootar udao jawab sirf goliyon se milega\n",
            "__label__no i am very sorry to say saaf dil shilpa ke fans hiten ke dil mein kya hai woh bhi samjhte hain hadh hoti hai nafrat ki bhi\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!wc data.txt"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fVELccWS6CoD",
        "outputId": "765401e5-2b3b-482d-ea6b-914246a9ae71"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  4571  94976 537175 data.txt\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!head -n 4000 data.txt > tweet.train\n",
        "!tail -n 579 data.txt > tweet.valid"
      ],
      "metadata": {
        "id": "A0oI5AMK6FK9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!git clone https://github.com/facebookresearch/fastText.git\n",
        "%cd fastText\n",
        "!make\n",
        "!cp fasttext ../\n",
        "%cd .."
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "l5U7SQn56SDZ",
        "outputId": "ee0bb8dc-0f22-4b14-a630-ac8a32a12f81"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'fastText'...\n",
            "remote: Enumerating objects: 3854, done.\u001b[K\n",
            "remote: Total 3854 (delta 0), reused 0 (delta 0), pack-reused 3854\u001b[K\n",
            "Receiving objects: 100% (3854/3854), 8.22 MiB | 14.04 MiB/s, done.\n",
            "Resolving deltas: 100% (2417/2417), done.\n",
            "/content/fastText\n",
            "c++ -pthread -std=c++11 -march=native -O3 -funroll-loops -DNDEBUG -c src/args.cc\n",
            "c++ -pthread -std=c++11 -march=native -O3 -funroll-loops -DNDEBUG -c src/autotune.cc\n",
            "c++ -pthread -std=c++11 -march=native -O3 -funroll-loops -DNDEBUG -c src/matrix.cc\n",
            "c++ -pthread -std=c++11 -march=native -O3 -funroll-loops -DNDEBUG -c src/dictionary.cc\n",
            "c++ -pthread -std=c++11 -march=native -O3 -funroll-loops -DNDEBUG -c src/loss.cc\n",
            "c++ -pthread -std=c++11 -march=native -O3 -funroll-loops -DNDEBUG -c src/productquantizer.cc\n",
            "c++ -pthread -std=c++11 -march=native -O3 -funroll-loops -DNDEBUG -c src/densematrix.cc\n",
            "c++ -pthread -std=c++11 -march=native -O3 -funroll-loops -DNDEBUG -c src/quantmatrix.cc\n",
            "c++ -pthread -std=c++11 -march=native -O3 -funroll-loops -DNDEBUG -c src/vector.cc\n",
            "c++ -pthread -std=c++11 -march=native -O3 -funroll-loops -DNDEBUG -c src/model.cc\n",
            "c++ -pthread -std=c++11 -march=native -O3 -funroll-loops -DNDEBUG -c src/utils.cc\n",
            "c++ -pthread -std=c++11 -march=native -O3 -funroll-loops -DNDEBUG -c src/meter.cc\n",
            "c++ -pthread -std=c++11 -march=native -O3 -funroll-loops -DNDEBUG -c src/fasttext.cc\n",
            "c++ -pthread -std=c++11 -march=native -O3 -funroll-loops -DNDEBUG args.o autotune.o matrix.o dictionary.o loss.o productquantizer.o densematrix.o quantmatrix.o vector.o model.o utils.o meter.o fasttext.o src/main.cc -o fasttext\n",
            "/content\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!./fasttext supervised -input ./tweet.train -output ./hate_speech_model1"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "02SkBHXF6VEL",
        "outputId": "208df8c4-d755-4d58-ba93-a79bb6d559b9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\rRead 0M words\n",
            "Number of words:  14224\n",
            "Number of labels: 4\n",
            "Progress: 100.0% words/sec/thread:  371053 lr:  0.000000 avg.loss:  0.574553 ETA:   0h 0m 0s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!./fasttext test hate_speech_model1.bin ./tweet.valid 3"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AmY3tLM56gV3",
        "outputId": "00956d76-dac4-4e11-bfdb-963ae952e4ef"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "N\t579\n",
            "P@3\t0.333\n",
            "R@3\t0.998\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "words = pd.read_table(\"hate_speech_model1.vec\", sep=\" \", index_col=0, header=None, on_bad_lines='skip')"
      ],
      "metadata": {
        "id": "0xDCV0ZD7OVE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "words.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 112
        },
        "id": "nXbJ-57CTFE5",
        "outputId": "4fe9f187-27b0-40cc-ccdf-5117123d6544"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-edc4e928-1244-4b1a-8e2c-22d0acbb9da9\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>1</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>14224</th>\n",
              "      <td>100</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-edc4e928-1244-4b1a-8e2c-22d0acbb9da9')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-edc4e928-1244-4b1a-8e2c-22d0acbb9da9 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-edc4e928-1244-4b1a-8e2c-22d0acbb9da9');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "         1\n",
              "0         \n",
              "14224  100"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install fasttext"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sVsARVNQ9jZT",
        "outputId": "e49ba3c0-cf0c-40e2-bfed-9d6af0b74f8f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting fasttext\n",
            "  Downloading fasttext-0.9.2.tar.gz (68 kB)\n",
            "\u001b[?25l\r\u001b[K     |████▊                           | 10 kB 38.3 MB/s eta 0:00:01\r\u001b[K     |█████████▌                      | 20 kB 9.2 MB/s eta 0:00:01\r\u001b[K     |██████████████▎                 | 30 kB 7.6 MB/s eta 0:00:01\r\u001b[K     |███████████████████             | 40 kB 7.0 MB/s eta 0:00:01\r\u001b[K     |███████████████████████▉        | 51 kB 4.1 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▋   | 61 kB 4.1 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 68 kB 3.1 MB/s \n",
            "\u001b[?25hCollecting pybind11>=2.2\n",
            "  Using cached pybind11-2.9.1-py2.py3-none-any.whl (211 kB)\n",
            "Requirement already satisfied: setuptools>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from fasttext) (57.4.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from fasttext) (1.19.5)\n",
            "Building wheels for collected packages: fasttext\n",
            "  Building wheel for fasttext (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for fasttext: filename=fasttext-0.9.2-cp37-cp37m-linux_x86_64.whl size=3136532 sha256=a86fad1b8ea7df619eeae9e59b8b90a602fb819e50f413aacbf60d78b4459586\n",
            "  Stored in directory: /root/.cache/pip/wheels/4e/ca/bf/b020d2be95f7641801a6597a29c8f4f19e38f9c02a345bab9b\n",
            "Successfully built fasttext\n",
            "Installing collected packages: pybind11, fasttext\n",
            "Successfully installed fasttext-0.9.2 pybind11-2.9.1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import fasttext  # as installed by `pip install fasttext`\n",
        "from gensim.models import KeyedVectors\n",
        "\n",
        "ft_model = fasttext.load_model('hate_speech_model1.bin')\n",
        "ft_model = fasttext.train_supervised(input=\"tweet.train\", lr=0.5, epoch=25, wordNgrams=2, loss='hs')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZsRUDcZE5uLm",
        "outputId": "9a444186-a48c-4995-d2f8-0f1b4c138377"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Warning : `load_model` does not return WordVectorModel or SupervisedModel any more, but a `FastText` object which is very similar.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\"no\" if ft_model.predict(\"Maine cockroach ka katal kar diya\")[0][0] == '__label__no' else \"hate speech\""
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "ehNyHOiKDmdk",
        "outputId": "77559e50-1076-4a6b-e72f-ef86de69244b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'no'"
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "ft_model.test(\"tweet.valid\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "B2WQ-wmq7KB2",
        "outputId": "22476254-e78c-452b-d722-49da05f1f1c8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(579, 0.6856649395509499, 0.6856649395509499)"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!head"
      ],
      "metadata": {
        "id": "GwFV-BOvDO4k"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "ft2_model = fasttext.load_model('hate_speech_model1.bin')\n",
        "ft2_model = fasttext.train_supervised(input=\"__label__no I hate you\", lr=0.5, epoch=25, wordNgrams=2, loss='hs')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 384
        },
        "id": "e_IXLQl_E_9r",
        "outputId": "b70c6634-ca5d-4833-d1d9-10fa76045ac0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Warning : `load_model` does not return WordVectorModel or SupervisedModel any more, but a `FastText` object which is very similar.\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "ValueError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-13-40ae70fb8ef5>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mft2_model\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfasttext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'hate_speech_model1.bin'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mft2_model\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfasttext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_supervised\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"__label__no I hate you\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.5\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m25\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwordNgrams\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'hs'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/fasttext/FastText.py\u001b[0m in \u001b[0;36mload_model\u001b[0;34m(path)\u001b[0m\n\u001b[1;32m    439\u001b[0m     \u001b[0;34m\"\"\"Load a model given a filepath and return a model object.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    440\u001b[0m     \u001b[0meprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Warning : `load_model` does not return WordVectorModel or SupervisedModel any more, but a `FastText` object which is very similar.\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 441\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_FastText\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_path\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    442\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    443\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/fasttext/FastText.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, model_path, args)\u001b[0m\n\u001b[1;32m     96\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfasttext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfasttext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     97\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mmodel_path\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 98\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloadModel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     99\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_words\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    100\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_labels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mValueError\u001b[0m: hate_speech_model1.bin has wrong file format!"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from gensim.models import FastText\n",
        "from gensim.test.utils import common_texts  # some example sentences\n",
        "print(common_texts[0])\n",
        "['human', 'interface', 'computer']\n",
        "print(len(common_texts))\n",
        "model = FastText(window=3, min_count=1)  # instantiate\n",
        "model.build_vocab(sentences=common_texts)\n",
        "model.train(sentences=common_texts, total_examples=len(common_texts), epochs=10)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 271
        },
        "id": "malXV0zQKBB9",
        "outputId": "37caad2a-ed3e-44f9-f75f-8293b5a48926"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['human', 'interface', 'computer']\n",
            "9\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "TypeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-42-40220b03f981>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'human'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'interface'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'computer'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcommon_texts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfasttext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwindow\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmin_count\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# instantiate\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuild_vocab\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msentences\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcommon_texts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msentences\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcommon_texts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtotal_examples\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcommon_texts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mTypeError\u001b[0m: 'module' object is not callable"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from gensim.models import fasttext\n",
        "from gensim.test.utils import datapath\n",
        "cap_path = datapath(\"hate_speech_model1.bin\")\n",
        "fb_model = fasttext.load_facebook_model(cap_path)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 201
        },
        "id": "fiNO6al4Kd7o",
        "outputId": "582cdcfe-6dc1-4d69-f491-539abeaa75d5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "AttributeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-46-05580f74df44>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mgensim\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtest\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutils\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mdatapath\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mcap_path\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdatapath\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"hate_speech_model1.bin\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mfb_model\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfasttext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_facebook_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcap_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mAttributeError\u001b[0m: module 'gensim.models.fasttext' has no attribute 'load_facebook_model'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "ft_model.predict(\"hate heena too much irritating n bat karne ka tamiz nhi hai unmesab ko insult karna ati hai unkobas aur kuch nhi ati haibohot insecure hai wo ladki\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 184
        },
        "id": "2kN2YArTts87",
        "outputId": "33da63da-d089-41f8-aa36-17a3f352d1b3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-15-8a7cc00234a6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mft_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"hate heena too much irritating n bat karne ka tamiz nhi hai unmesab ko insult karna ati hai unkobas aur kuch nhi ati haibohot insecure hai wo ladki\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m: name 'ft_model' is not defined"
          ]
        }
      ]
    }
  ]
}