Preprocessing_english.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "EG8U4kMttPb2"
   },
   "source": [
    "# Data Loading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "E6DEBPP2tPb6"
   },
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 206
    },
    "id": "NAC2pX3otPb7",
    "outputId": "07331e1e-878d-4d31-ff30-9167e2517ac3"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>hindi</th>\n",
       "      <th>english</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1561836</th>\n",
       "      <td>स्पष्टीकरण.–जहां इस उपधारा के अधीन हानि और लाभ...</td>\n",
       "      <td>स्पष्टीकरण.–जहां इस उपधारा के अधीन हानि और लाभ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1561837</th>\n",
       "      <td>मैंने गौर किया है कि यह न केवल अपने महत्त्वपूर...</td>\n",
       "      <td>है। I note that this is a landmark meeting – n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1561838</th>\n",
       "      <td>उन्होंने मेरे समक्ष जो प्रदर्शन किया उसमें से ...</td>\n",
       "      <td>है। In the presentations that they made before...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1561839</th>\n",
       "      <td>खाद्य और जल सुरक्षा; पर्यावरण की दृष्टि से वहन...</td>\n",
       "      <td>्त है। Issues such as food and water security;...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1561840</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                     hindi  \\\n",
       "1561836  स्पष्टीकरण.–जहां इस उपधारा के अधीन हानि और लाभ...   \n",
       "1561837  मैंने गौर किया है कि यह न केवल अपने महत्त्वपूर...   \n",
       "1561838  उन्होंने मेरे समक्ष जो प्रदर्शन किया उसमें से ...   \n",
       "1561839  खाद्य और जल सुरक्षा; पर्यावरण की दृष्टि से वहन...   \n",
       "1561840                                                NaN   \n",
       "\n",
       "                                                   english  \n",
       "1561836  स्पष्टीकरण.–जहां इस उपधारा के अधीन हानि और लाभ...  \n",
       "1561837  है। I note that this is a landmark meeting – n...  \n",
       "1561838  है। In the presentations that they made before...  \n",
       "1561839  ्त है। Issues such as food and water security;...  \n",
       "1561840                                                NaN  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('./hindi_english_parallel.csv')\n",
    "data.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "eE9CUxmktPb9"
   },
   "source": [
    "### Partition into hindi english"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "aGvxx3fgtPb-",
    "outputId": "9db6cbfe-f015-4a7a-be71-b3718112bb9d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0             Give your application an accessibility workout\n",
      "1                          Accerciser Accessibility Explorer\n",
      "2             The default plugin layout for the bottom panel\n",
      "3                The default plugin layout for the top panel\n",
      "4             A list of plugins that are disabled by default\n",
      "                                 ...                        \n",
      "1561836    स्पष्टीकरण.–जहां इस उपधारा के अधीन हानि और लाभ...\n",
      "1561837    है। I note that this is a landmark meeting – n...\n",
      "1561838    है। In the presentations that they made before...\n",
      "1561839    ्त है। Issues such as food and water security;...\n",
      "1561840                                                  NaN\n",
      "Name: english, Length: 1561841, dtype: object\n",
      "\n",
      "==============================================================\n",
      "\n",
      "0            अपने अनुप्रयोग को पहुंचनीयता व्यायाम का लाभ दें\n",
      "1                            एक्सेर्साइसर पहुंचनीयता अन्वेषक\n",
      "2                      निचले पटल के लिए डिफोल्ट प्लग-इन खाका\n",
      "3                       ऊपरी पटल के लिए डिफोल्ट प्लग-इन खाका\n",
      "4          उन प्लग-इनों की सूची जिन्हें डिफोल्ट रूप से नि...\n",
      "                                 ...                        \n",
      "1561836    स्पष्टीकरण.–जहां इस उपधारा के अधीन हानि और लाभ...\n",
      "1561837    मैंने गौर किया है कि यह न केवल अपने महत्त्वपूर...\n",
      "1561838    उन्होंने मेरे समक्ष जो प्रदर्शन किया उसमें से ...\n",
      "1561839    खाद्य और जल सुरक्षा; पर्यावरण की दृष्टि से वहन...\n",
      "1561840                                                  NaN\n",
      "Name: hindi, Length: 1561841, dtype: object\n"
     ]
    }
   ],
   "source": [
    "english = data['english']\n",
    "hindi = data['hindi']\n",
    "print(english)\n",
    "print()\n",
    "print(\"==============================================================\")\n",
    "print()\n",
    "print(hindi)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "GpuYxLQAtPb-"
   },
   "source": [
    "### Bug in english data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "9jkse_0dtPb_"
   },
   "source": [
    "##### 1. Hindi text in english"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 122
    },
    "id": "3SQXc9qQtPb_",
    "outputId": "bf165c95-94c0-4bbf-8ea1-c926c36faa69"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'स्पष्टीकरण.–जहां इस उपधारा के अधीन हानि और लाभ लेखे में से डेबिट की गर्इ और विनिधान मोक रिजर्व खाते में जमा की गर्इ रकम उस रकम से कम नहीं है जो निर्धारिती द्वारा धारा 139 के अधीन दी गर्इ विवरणी में दावा किए गए विनिधान मोक की बाबत कटौती की रकम के आधार पर इस प्रकार जमा की जानी अपेक्षित है, किन्तु निर्धारण अधिकारी द्वारा धारा 143 के अधीन संगणित किए जाने के लिए यथाप्रस्थापित कुल आय के आधार पर विनिधान मोक की बाबत अधिक कटौती अनुज्ञेय है वहां निर्धारण अधिकारी इस निमित्त लिखित सूचना द्वारा, निर्धारिती को सूचना में विनिर्दिष्ट समय के भीतर या ऐसे अतिरिक्त समय के भीतर जो निर्धारण अधिकारी अनुज्ञात करे, उस पूर्ववर्ष के, जिसमें ऐसी सूचना निर्धारिती पर तामील की जाती है या उसके ठीक पूर्ववर्ती पूर्ववर्ष के लाभ और अभिलाभ में से, यदि उस वर्ष के लेखे पूरे नहीं किए गए हों तो, अतिरिक्त रकम विनिधान मोक रिजर्व खाते में जमा करने का अवसर देगा और यदि निर्धारिती पूर्वोक्त समय के भीतर ऐसे खाते में कोर्इ अतिरिक्त रकम जमा कर देता है तो इस प्रकार जमा की गर्इ रकम उस पूर्ववर्ष के, जिसमें कटौती अनुज्ञेय है, विनिधान मोक रिजर्व खाते में जमा की गर्इ समझी जाएगी और वह रकम निर्धारिती द्वारा उस पूर्ववर्ष की बाबत जिसमें ऐसी अतिरिक्त रकम जमा की जाती है, जमा किए जाने के लिए अपेक्षित रिजर्व की पर्याप्तता के अवधारण के लिए हिसाब में नहीं ली जाएगी:'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "english[1561836]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ec6LVxpftPb_"
   },
   "source": [
    "##### 2. Nan/Floating point numbers in data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "YBIEVyKItPcA"
   },
   "outputs": [],
   "source": [
    "a = list(map(lambda x : type(x[1]) == float and x[0] or None, enumerate(english)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "f9423xcqtPcA"
   },
   "source": [
    "##### Removed NAN and sentences having length less than 100 characters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "vsNPRkuStPcA"
   },
   "outputs": [],
   "source": [
    "SENETNCE_CHAR_LEN = 100\n",
    "a = []\n",
    "for i in english:\n",
    "    if type(i) == float or len(i) <= SENETNCE_CHAR_LEN:\n",
    "        a.append(False)\n",
    "    else:\n",
    "        a.append(True)\n",
    "english = english[a]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "UvbLeEqytPcB",
    "outputId": "1a792b87-ced6-4221-ba99-c98d8c16e643"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "136        Accerciser could not see the applications on y...\n",
       "329        Accerciser could not see the applications on y...\n",
       "585        Accerciser could not see the applications on y...\n",
       "722        Accerciser could not see the applications on y...\n",
       "939        Accerciser could not see the applications on y...\n",
       "                                 ...                        \n",
       "1561834    प्राप्त होगी। By 2020, India is set to become ...\n",
       "1561836    स्पष्टीकरण.–जहां इस उपधारा के अधीन हानि और लाभ...\n",
       "1561837    है। I note that this is a landmark meeting – n...\n",
       "1561838    है। In the presentations that they made before...\n",
       "1561839    ्त है। Issues such as food and water security;...\n",
       "Name: english, Length: 393786, dtype: object"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "english"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "QyxR5dD9tPcB"
   },
   "source": [
    "##### Filter Hindi sentences from english ones"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "x7EoQnnetPcB",
    "outputId": "f0d33c9e-f243-4465-9c23-95895cf9d3ed"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "स्पष्टीकरण.–जहां इस उपधारा के अधीन हानि और लाभ लेखे में से डेबिट की गर्इ और विनिधान मोक रिजर्व खाते में जमा की गर्इ रकम उस रकम से कम नहीं है जो निर्धारिती द्वारा धारा 139 के अधीन दी गर्इ विवरणी में दावा किए गए विनिधान मोक की बाबत कटौती की रकम के आधार पर इस प्रकार जमा की जानी अपेक्षित है, किन्तु निर्धारण अधिकारी द्वारा धारा 143 के अधीन संगणित किए जाने के लिए यथाप्रस्थापित कुल आय के आधार पर विनिधान मोक की बाबत अधिक कटौती अनुज्ञेय है वहां निर्धारण अधिकारी इस निमित्त लिखित सूचना द्वारा, निर्धारिती को सूचना में विनिर्दिष्ट समय के भीतर या ऐसे अतिरिक्त समय के भीतर जो निर्धारण अधिकारी अनुज्ञात करे, उस पूर्ववर्ष के, जिसमें ऐसी सूचना निर्धारिती पर तामील की जाती है या उसके ठीक पूर्ववर्ती पूर्ववर्ष के लाभ और अभिलाभ में से, यदि उस वर्ष के लेखे पूरे नहीं किए गए हों तो, अतिरिक्त रकम विनिधान मोक रिजर्व खाते में जमा करने का अवसर देगा और यदि निर्धारिती पूर्वोक्त समय के भीतर ऐसे खाते में कोर्इ अतिरिक्त रकम जमा कर देता है तो इस प्रकार जमा की गर्इ रकम उस पूर्ववर्ष के, जिसमें कटौती अनुज्ञेय है, विनिधान मोक रिजर्व खाते में जमा की गर्इ समझी जाएगी और वह रकम निर्धारिती द्वारा उस पूर्ववर्ष की बाबत जिसमें ऐसी अतिरिक्त रकम जमा की जाती है, जमा किए जाने के लिए अपेक्षित रिजर्व की पर्याप्तता के अवधारण के लिए हिसाब में नहीं ली जाएगी:\n",
      "Vocab Size =  54\n",
      "2380 = ौ\n",
      "2311 = इ\n",
      "2309 = अ\n",
      "2369 = ु\n",
      "2319 = ए\n",
      "2344 = न\n",
      "2337 = ड\n",
      "2381 = ्\n",
      "2332 = ज\n",
      "2351 = य\n",
      "2370 = ू\n",
      "2354 = ल\n",
      "2334 = ञ\n",
      "2310 = आ\n",
      "2343 = ध\n",
      "2379 = ो\n",
      "2360 = स\n",
      "2326 = ख\n",
      "2330 = च\n",
      "2333 = झ\n",
      "2313 = उ\n",
      "2368 = ी\n",
      "2375 = े\n",
      "2352 = र\n",
      "2376 = ै\n",
      "44 = ,\n",
      "2366 = ा\n",
      "49 = 1\n",
      "2359 = ष\n",
      "58 = :\n",
      "2339 = ण\n",
      "57 = 9\n",
      "2346 = प\n",
      "2340 = त\n",
      "2349 = भ\n",
      "2341 = थ\n",
      "8211 = –\n",
      "46 = .\n",
      "2336 = ठ\n",
      "32 =  \n",
      "2342 = द\n",
      "2357 = व\n",
      "51 = 3\n",
      "2348 = ब\n",
      "2335 = ट\n",
      "2325 = क\n",
      "2361 = ह\n",
      "2306 = ं\n",
      "52 = 4\n",
      "2320 = ऐ\n",
      "2327 = ग\n",
      "2350 = म\n",
      "2367 = ि\n",
      "2324 = औ\n",
      "\n",
      "\n",
      "=================================================================\n",
      "\n",
      "\n",
      "Vocab Size =  45\n",
      "2306 = ौ\n",
      "2309 = इ\n",
      "2310 = अ\n",
      "2311 = ु\n",
      "2313 = ए\n",
      "2319 = न\n",
      "2320 = ड\n",
      "2324 = ्\n",
      "2325 = ज\n",
      "2326 = य\n",
      "2327 = ू\n",
      "2330 = ल\n",
      "2332 = ञ\n",
      "2333 = आ\n",
      "2334 = ध\n",
      "2335 = ो\n",
      "2336 = स\n",
      "2337 = ख\n",
      "2339 = च\n",
      "2340 = झ\n",
      "2341 = उ\n",
      "2342 = ी\n",
      "2343 = े\n",
      "2344 = र\n",
      "2346 = ै\n",
      "2348 = ा\n",
      "2349 = ष\n",
      "2350 = ण\n",
      "2351 = प\n",
      "2352 = त\n",
      "2354 = भ\n",
      "2357 = थ\n",
      "2359 = ठ\n",
      "2360 = द\n",
      "2361 = व\n",
      "2366 = ब\n",
      "2367 = ट\n",
      "2368 = क\n",
      "2369 = ह\n",
      "2370 = ं\n",
      "2375 = ऐ\n",
      "2376 = ग\n",
      "2379 = म\n",
      "2380 = ि\n",
      "2381 = औ\n"
     ]
    }
   ],
   "source": [
    "example_sentense = english[1561836]\n",
    "print(example_sentense)\n",
    "Hindi_voacab = list(set(example_sentense))\n",
    "print(\"Vocab Size = \", len(Hindi_voacab))\n",
    "unicode = list(map(ord, Hindi_voacab))\n",
    "for i, j in zip(Hindi_voacab, unicode):\n",
    "    print(f\"{j} = {i}\")\n",
    "\n",
    "\"\"\"\n",
    "We can see [0-9.,:_] : we need to remove them\n",
    "\"\"\"\n",
    "toRemove = set(['1','2','3','4','5','6','7','8','9','0','_', '-', '–',',','.',':',' '])\n",
    "toRemoveUnicode = set(list(map(ord, toRemove)))\n",
    "\n",
    "Hindi_voacab = list(set(Hindi_voacab) - toRemove)\n",
    "unicode = list(set(unicode) - toRemoveUnicode)\n",
    "\n",
    "print(\"\\n\\n=================================================================\\n\\n\")\n",
    "print(\"Vocab Size = \", len(Hindi_voacab))\n",
    "for i, j in zip(Hindi_voacab, unicode):\n",
    "    print(f\"{j} = {i}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "gWcwu22StPcD",
    "outputId": "c228bb75-195e-458a-c7f3-9cd235047c9a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "============================  SELECTED  ========================\n",
      "1561821    You would soon join different branches of the ...\n",
      "1561822    Y = the amount of tax calculated on the total ...\n",
      "1561823    (y) \"true and first inventor\" does not include...\n",
      "1561825    (za) \"Special Economic Zone\" means each Specia...\n",
      "1561826    (zc) \"Unit\" means a Unit set up by an entrepre...\n",
      "1561827    (zg) \"security receipt\" means a receipt or oth...\n",
      "1561828    Zonal: Eleven annual awards of ₹ 0.50 lakh eac...\n",
      "1561829    Zoroastrianism, which was once the religion of...\n",
      "1561830    (z) \"securitisation\" means acquisition of fina...\n",
      "1561831    ZUBIN MEHTA HAS ENGAGED IN UNTIRING EFFORTS TO...\n",
      "Name: english, dtype: object\n",
      "============================  REJECTED  ========================\n",
      "1524435    India is ranked third in the list of top prosp...\n",
      "1527095    ISRO completed the seven-satellite constellati...\n",
      "1542175    Shram-eva jayate (श्रमेवजयते)is one of the for...\n",
      "1550921    The President of India, Shri Pranab Mukherjee ...\n",
      "1561833    कते हैं। Opening of a Fin-Node Centre in Delhi...\n",
      "1561834    प्राप्त होगी। By 2020, India is set to become ...\n",
      "1561836    स्पष्टीकरण.–जहां इस उपधारा के अधीन हानि और लाभ...\n",
      "1561837    है। I note that this is a landmark meeting – n...\n",
      "1561838    है। In the presentations that they made before...\n",
      "1561839    ्त है। Issues such as food and water security;...\n",
      "Name: english, dtype: object\n"
     ]
    }
   ],
   "source": [
    "def onlyEnglish(sentence : str, Hindi_voacab = Hindi_voacab):\n",
    "    for i in Hindi_voacab:\n",
    "        if i in sentence:\n",
    "            return False\n",
    "    return True\n",
    "\n",
    "filteredEnglishData = [onlyEnglish(str(sen)) for sen in english]\n",
    "print(\"============================  SELECTED  ========================\")\n",
    "print(english[filteredEnglishData][-10:])\n",
    "print(\"============================  REJECTED  ========================\")\n",
    "print(english[map(lambda x : not x, filteredEnglishData)][-10:])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "599DwvGjtPcE"
   },
   "source": [
    "##### Sentence Below got purged"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "7cPWgFs-tPcF",
    "outputId": "d789188d-abc9-4239-a437-0b7b32f904be"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ISRO completed the seven-satellite constellation of Indian Regional Navigational Satellite Systems,NaviC (नाविक).\n"
     ]
    }
   ],
   "source": [
    "print(english[1527095])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "dSgi7MrqtPcF"
   },
   "source": [
    "# Final English Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "JZYuJWpptPcG",
    "outputId": "ddb095ee-e34a-4d10-9cc2-d4341708eb23"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset Length (Sentences) :  393123\n",
      "136        Accerciser could not see the applications on y...\n",
      "329        Accerciser could not see the applications on y...\n",
      "585        Accerciser could not see the applications on y...\n",
      "722        Accerciser could not see the applications on y...\n",
      "939        Accerciser could not see the applications on y...\n",
      "                                 ...                        \n",
      "1561827    (zg) \"security receipt\" means a receipt or oth...\n",
      "1561828    Zonal: Eleven annual awards of ₹ 0.50 lakh eac...\n",
      "1561829    Zoroastrianism, which was once the religion of...\n",
      "1561830    (z) \"securitisation\" means acquisition of fina...\n",
      "1561831    ZUBIN MEHTA HAS ENGAGED IN UNTIRING EFFORTS TO...\n",
      "Name: english, Length: 393123, dtype: object\n"
     ]
    }
   ],
   "source": [
    "english = english[filteredEnglishData]\n",
    "print(\"Dataset Length (Sentences) : \", len(english))\n",
    "print(english)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ZXYqD-DstPcG"
   },
   "source": [
    "# Tokenization - Word Level"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "BmzeRMjCtPcG"
   },
   "source": [
    "##### Space based seperator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "olpG963rtPcH"
   },
   "outputs": [],
   "source": [
    "EnglishTokens = []\n",
    "for index, i in enumerate(english):\n",
    "    EnglishTokens.append(i.split(\" \"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "itn3PqbMtPcH",
    "outputId": "7f508eb9-89e8-4628-a4c1-639e16505178",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "% s is free software; you can redistribute it and / or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version% d of the License, or (at your option) any later version.\n",
      "['%', 's', 'is', 'free', 'software;', 'you', 'can', 'redistribute', 'it', 'and', '/', 'or', 'modify', 'it', 'under', 'the', 'terms', 'of', 'the', 'GNU', 'General', 'Public', 'License', 'as', 'published', 'by', 'the', 'Free', 'Software', 'Foundation;', 'either', 'version%', 'd', 'of', 'the', 'License,', 'or', '(at', 'your', 'option)', 'any', 'later', 'version.']\n",
      "\n",
      "\n",
      "% s is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.\n",
      "['%', 's', 'is', 'distributed', 'in', 'the', 'hope', 'that', 'it', 'will', 'be', 'useful,', 'but', 'WITHOUT', 'ANY', 'WARRANTY;', 'without', 'even', 'the', 'implied', 'warranty', 'of', 'MERCHANTABILITY', 'or', 'FITNESS', 'FOR', 'A', 'PARTICULAR', 'PURPOSE.', 'See', 'the', 'GNU', 'General', 'Public', 'License', 'for', 'more', 'details.']\n",
      "\n",
      "\n",
      "You should have received a copy of the GNU General Public License along with% s; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110 - 1301 USA\n",
      "['You', 'should', 'have', 'received', 'a', 'copy', 'of', 'the', 'GNU', 'General', 'Public', 'License', 'along', 'with%', 's;', 'if', 'not,', 'write', 'to', 'the', 'Free', 'Software', 'Foundation,', 'Inc.,', '51', 'Franklin', 'Street,', 'Fifth', 'Floor,', 'Boston,', 'MA', '02110', '-', '1301', 'USA']\n",
      "\n",
      "\n",
      "You should have received a copy of the GNU General Public License along with this program. If not, see.\n",
      "['You', 'should', 'have', 'received', 'a', 'copy', 'of', 'the', 'GNU', 'General', 'Public', 'License', 'along', 'with', 'this', 'program.', 'If', 'not,', 'see.']\n",
      "\n",
      "\n",
      "A list of strings that come in the form of a quintuple: name, wins, total games played, best time (in seconds) and worst time (also in seconds). Unplayed games do not need to be represented.\n",
      "['A', 'list', 'of', 'strings', 'that', 'come', 'in', 'the', 'form', 'of', 'a', 'quintuple:', 'name,', 'wins,', 'total', 'games', 'played,', 'best', 'time', '(in', 'seconds)', 'and', 'worst', 'time', '(also', 'in', 'seconds).', 'Unplayed', 'games', 'do', 'not', 'need', 'to', 'be', 'represented.']\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "PRINT_FROM = 23\n",
    "PRINT_UPTO = 27\n",
    "for i,j in zip(english[PRINT_FROM:PRINT_UPTO+1], EnglishTokens[PRINT_FROM:PRINT_UPTO+1]):\n",
    "    print(f\"{i}\\n{j}\\n\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "RE2NhDJQtPcH"
   },
   "source": [
    "### Filter\n",
    "\n",
    "1. P.hd. == PHD ------------------------ U.S.A. == USA\n",
    "2.  word. === [\"word\" , \".\"]  --------- . ? !\n",
    "3. Let's === [\"Let\", \"'s\"]   ------------- 's 've 't\n",
    "4. Punctuations\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "id": "zuNf-0lmtPcI"
   },
   "outputs": [],
   "source": [
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "nDVfqTo7tPcI"
   },
   "outputs": [],
   "source": [
    "rules = [\n",
    "    re.compile(r'\\w+\\.\\w+\\.'),   # P.hd.\n",
    "    re.compile(r'\\w+[.?!]'),     # termination of line\n",
    "    re.compile(r\"\\w+'(s|ve|t)\"),  # Let's haven't we've\n",
    "    re.compile(r'.?[,;].?'),      # Remove , ;\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "id": "9YGyfa-ntPcI"
   },
   "outputs": [],
   "source": [
    "def regexEvaluater(word : str, rules : list) -> list:\n",
    "    word = word.lower()\n",
    "    if any(True for _ in iter(rules[0].finditer(word))):\n",
    "        return [word.replace('.', '')]\n",
    "\n",
    "    if any(True for _ in iter(rules[1].finditer(word))):\n",
    "        temp = word[-1]\n",
    "        return [word.replace(temp, ''), temp]\n",
    "\n",
    "    if any(True for _ in iter(rules[2].finditer(word))):\n",
    "        if \"'s\" in word:\n",
    "            return [word.replace(\"'s\", ''), \"'s\"]\n",
    "        if \"'ve\" in word:\n",
    "            return [word.replace(\"'ve\", ''), \"'ve\"]\n",
    "        if \"'t\" in word:\n",
    "            return [word.replace(\"'t\", ''), \"'t\"]\n",
    "\n",
    "    if any(True for _ in iter(rules[3].finditer(word))):\n",
    "        punctuator = ',' if ',' in word else ';'\n",
    "\n",
    "        ans = word.split(punctuator)\n",
    "        for _ in range(ans.count('')):\n",
    "            ans.remove('')\n",
    "        return ans\n",
    "\n",
    "\n",
    "    return [word]\n",
    "\n",
    "def tokenise(words=None, sentence : str=None, rules=rules):\n",
    "    if sentence is not None:\n",
    "        words = sentence.split(\" \")\n",
    "\n",
    "    ans = []\n",
    "    for word in words:\n",
    "        ans.extend(regexEvaluater(word, rules))\n",
    "    return ans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "CcyTE5QHtPcJ",
    "outputId": "1bb587f3-75a5-43d7-b596-ad3f81a13056"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<scientists>\t<are>\t<researching>\t<ways>\t<to>\t<enhance>\t<artificial>\t<intelligence>\t<with>\t<advanced>\t<algorithms>\t<.>\t"
     ]
    }
   ],
   "source": [
    "sentence = \"Scientists are researching ways to enhance artificial intelligence with advanced algorithms.\"\n",
    "out = tokenise(sentence = sentence)\n",
    "\n",
    "for i in out:\n",
    "    print(f'<{i}>', end='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save Tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: tqdm in /home/shay/env/nlp/lib/python3.10/site-packages (4.66.1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████| 393123/393123 [41:49<00:00, 156.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "303858\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "write() argument must be str, not set",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[20], line 9\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;28mlen\u001b[39m(all_tokens))\n\u001b[1;32m      8\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mopen\u001b[39m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m./utils/all_tokens.txt\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mw\u001b[39m\u001b[38;5;124m'\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m f:\n\u001b[0;32m----> 9\u001b[0m     \u001b[43mf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwrite\u001b[49m\u001b[43m(\u001b[49m\u001b[43mall_tokens\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[0;31mTypeError\u001b[0m: write() argument must be str, not set"
     ]
    }
   ],
   "source": [
    "! pip install tqdm\n",
    "from tqdm import tqdm\n",
    "all_tokens = set()\n",
    "for Line in tqdm(EnglishTokens):\n",
    "    all_tokens = all_tokens.union(set(Line))\n",
    "print(len(all_tokens))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "303858\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████| 303858/303858 [00:00<00:00, 1920381.68it/s]\n"
     ]
    }
   ],
   "source": [
    "print(len(all_tokens))\n",
    "all_tokens = list(all_tokens)\n",
    "all_tokens.sort()\n",
    "\n",
    "with open('./utils/all_tokens.txt', 'w') as f:\n",
    "    for tkn in tqdm(all_tokens):\n",
    "        f.write(tkn+'\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Stemming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "id": "QRYyfYYetPcJ"
   },
   "outputs": [],
   "source": [
    "stemming_rules = {\n",
    "    \"s\": \"\",\n",
    "    \"es\": \"\",\n",
    "    \"ies\": \"y\",\n",
    "    \"ied\": \"y\",\n",
    "    \"ing\": \"\",\n",
    "    \"ed\": \"\",\n",
    "    \"er\": \"\",\n",
    "    \"est\": \"\",\n",
    "    \"ion\": \"\",\n",
    "    \"tion\": \"t\",\n",
    "    \"ational\": \"ate\",\n",
    "    \"ed\": \"\",\n",
    "    \"ing\": \"\",\n",
    "    \"er\": \"\",\n",
    "    \"ers\": \"\",\n",
    "    \"ed\": \"\",\n",
    "    \"ing\": \"\",\n",
    "    \"ies\": \"y\",\n",
    "    \"ied\": \"y\",\n",
    "    \"izes\": \"ize\",\n",
    "    \"izing\": \"ize\",\n",
    "    \"ized\": \"ize\",\n",
    "    \"y\": \"\",\n",
    "    \"ier\": \"y\",\n",
    "    \"iest\": \"y\",\n",
    "    \"al\": \"\",\n",
    "    \"ally\": \"\",\n",
    "    \"ed\": \"\",\n",
    "    \"ing\": \"\",\n",
    "    \"er\": \"\",\n",
    "    \"or\": \"\",\n",
    "    \"ar\": \"\",\n",
    "    \"ary\": \"\",\n",
    "    \"ery\": \"\",\n",
    "    \"ful\": \"\",\n",
    "    \"less\": \"\",\n",
    "    \"ness\": \"\",\n",
    "    \"ship\": \"\",\n",
    "    \"sion\": \"\",\n",
    "    \"tion\": \"t\",\n",
    "    \"ive\": \"\",\n",
    "    \"ize\": \"\",\n",
    "    \"ize\": \"\",\n",
    "    \"ize\": \"\",\n",
    "    \"izing\": \"ize\",\n",
    "    \"izing\": \"ize\",\n",
    "    \"izing\": \"ize\",\n",
    "    \"ized\": \"ize\",\n",
    "    \"ized\": \"ize\",\n",
    "    \"ized\": \"ize\",\n",
    "    \"al\": \"\",\n",
    "    \"ally\": \"\",\n",
    "    \"ed\": \"\",\n",
    "    \"ing\": \"\",\n",
    "    \"er\": \"\",\n",
    "    \"or\": \"\",\n",
    "    \"ar\": \"\",\n",
    "    \"ary\": \"\",\n",
    "    \"ery\": \"\",\n",
    "    \"ful\": \"\",\n",
    "    \"less\": \"\",\n",
    "    \"ness\": \"\",\n",
    "    \"ship\": \"\",\n",
    "    \"sion\": \"\",\n",
    "    \"tion\": \"t\",\n",
    "    \"ive\": \"\",\n",
    "    \"ize\": \"\",\n",
    "    \"ize\": \"\",\n",
    "    \"ize\": \"\",\n",
    "    \"izing\": \"ize\",\n",
    "    \"izing\": \"ize\",\n",
    "    \"izing\": \"ize\",\n",
    "    \"ized\": \"ize\",\n",
    "    \"ized\": \"ize\",\n",
    "    \"ized\": \"ize\",\n",
    "    \"ish\": \"\",\n",
    "    \"ism\": \"\",\n",
    "    \"ist\": \"\",\n",
    "    \"al\": \"\",\n",
    "    \"ate\": \"\",\n",
    "    \"en\": \"\",\n",
    "    \"ify\": \"\",\n",
    "    \"ize\": \"\",\n",
    "    \"ise\": \"\",\n",
    "    \"ize\": \"\",\n",
    "    \"fy\": \"\",\n",
    "    \"ize\": \"\",\n",
    "    \"ish\": \"\",\n",
    "    \"en\": \"\",\n",
    "    \"ize\": \"\",\n",
    "    \"ism\": \"\",\n",
    "    \"ist\": \"\",\n",
    "    \"al\": \"\",\n",
    "    \"ate\": \"\",\n",
    "    \"en\": \"\",\n",
    "    \"ify\": \"\",\n",
    "    \"ize\": \"\",\n",
    "    \"ise\": \"\",\n",
    "    \"ize\": \"\",\n",
    "    \"fy\": \"\",\n",
    "    \"ize\": \"\",\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "id": "Po5A8gGFlxIS"
   },
   "outputs": [],
   "source": [
    "def apply_stemming(word):\n",
    "  for suffix, replacement in stemming_rules.items():\n",
    "    if (word.endswith(suffix)):\n",
    "      return word[:-len(suffix)] + replacement\n",
    "  return word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "id": "O2FkEXStmD3_"
   },
   "outputs": [],
   "source": [
    "def return_stemmed_tokens(token_list):\n",
    "  return [apply_stemming(token) for token in token_list]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "E-5lV-nImWS4",
    "outputId": "1201ab23-6584-45cf-cd7a-b710f0e6a4d5"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['scientist',\n",
       " 'are',\n",
       " 'research',\n",
       " 'way',\n",
       " 'to',\n",
       " 'enhance',\n",
       " 'artifici',\n",
       " 'intelligence',\n",
       " 'with',\n",
       " 'advanc',\n",
       " 'algorithm',\n",
       " '.']"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "return_stemmed_tokens(out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "xB5JdpiimbIw",
    "outputId": "8a4c3610-aa21-44c8-b69a-d6703e4e6edf"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['scientists', 'are', 'researching', 'ways', 'to', 'enhance', 'artificial', 'intelligence', 'with', 'advanced', 'algorithms', '.']\n"
     ]
    }
   ],
   "source": [
    "print(out)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save all stems"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "id": "-xaBXQuK9T8E"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████| 303858/303858 [00:01<00:00, 251942.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "280371\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "all_stems = set()\n",
    "for tkn in tqdm(all_tokens):\n",
    "    all_stems.add(return_stemmed_tokens([tkn])[0])\n",
    "print(len(all_stems))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████| 280371/280371 [00:00<00:00, 1831957.82it/s]\n"
     ]
    }
   ],
   "source": [
    "all_stems = list(all_stems)\n",
    "all_stems.sort()\n",
    "\n",
    "with open('./utils/all_stems.txt', 'w') as f:\n",
    "    for stem in tqdm(all_stems):\n",
    "        f.write(stem+'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}