lda.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Enhanced Tweet Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Text Gathering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pymysql\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import nltk\n",
    "from nltk import word_tokenize, FreqDist\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import LancasterStemmer, WordNetLemmatizer\n",
    "# nltk.download\n",
    "# nltk.download('wordnet')\n",
    "# nltk.download('stopwords')\n",
    "from nltk.tokenize import TweetTokenizer\n",
    "\n",
    "# Database configuration\n",
    "DB_HOST = \"localhost\"\n",
    "DB_USER = \"root\"\n",
    "DB_PASSWD = \"\"\n",
    "DB_DB = \"db_twitter_sa\"\n",
    "DB_CHARSET = \"utf8\"\n",
    "\n",
    "# mysql_connection = pymysql.connect(\n",
    "#             host=DB_HOST,\n",
    "#             user=DB_USER,\n",
    "#             passwd=DB_PASSWD,\n",
    "#             db=DB_DB,\n",
    "#             charset=DB_CHARSET,\n",
    "#             cursorclass=pymysql.cursors.DictCursor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# keywords = [\n",
    "#     'Cyberpunk2077', \n",
    "# #     '#Cyberpunk2077', \n",
    "#     'Cyberpunk 2077', \n",
    "#     'GenshinImpact', \n",
    "# #     '#GenshinImpact', \n",
    "#     'Genshin Impact', \n",
    "#     '@redditdota2', \n",
    "#     '#ESLOne', \n",
    "#     '#Animajor', \n",
    "#     'Weplay', \n",
    "#     'Animajor', \n",
    "#     '#fortnite', \n",
    "#     '#Dota2', \n",
    "#     '#DPC', \n",
    "#     'Dota Pro Circuit', \n",
    "#     'DPC', \n",
    "#     'Dota 2', \n",
    "# #     'esport', \n",
    "# #     'e-sport', \n",
    "#     'StardewValley', \n",
    "#     '#Stardew', \n",
    "# #     '#StardewValley', \n",
    "# #     '#LeagueofLegendsWildRift', \n",
    "# #     '#lolwr', \n",
    "#     '#Valorant', \n",
    "# #     'League of Legends'\n",
    "# ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "keywords = [\n",
    "    'Cyberpunk2077',\n",
    "    '#Cyberpunk2077',\n",
    "    'Cyberpunk 2077',\n",
    "    'GenshinImpact',\n",
    "    '#GenshinImpact',\n",
    "    'Genshin Impact',\n",
    "    'Minecraft',\n",
    "    '#fortnite',\n",
    "    'Dota 2',\n",
    "    'StardewValley',\n",
    "    '#Stardew',\n",
    "    '#StardewValley',\n",
    "    '#Valorant',\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>i just wanted to see what he would look like in black...?\\n#GenshinImpact #原神 #venti https://t.co/dvy3gKAL8G</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>@GenshinImpact Hey genshin, I was playing today and I wished 30 rolls, I got not a single 5 or 4 star, I’m sure something went wrong, but I’m wondering if you could help me? I haven’t been able to get any 4-5 stars at all when pulled 50 for the 3 days, been playing this game.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Thank you so much @_Marii_Pily for this commission, it’s so beautiful \\nI LOVE IT SO MUCH ??\\n\\nfor you all Aeventi nation, i love this fandom so much. &amp;lt;3\\n\\n#aeventi #venther #aether #venti #GenshinImpact #原神 https://t.co/VWop9QloHe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Comforting ocean #kaebedo #GenshinImpact https://t.co/JGLZRrVjU3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Is Genshin Impact fun?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>i read Cascading ( In a good way)  by Hubbleablubble on ao3 and now i am a firm Kaebedo enjoyer\\n#GenshinImpact #kaebedo https://t.co/kyhxeSfKN5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Genshin impact (similar to) Kitsune, Inazuma-OC!\\n\\nShe is a deceased human spirit who lives as fox now waiting for the fated person to come by, then she met Venti---\\n\\n#原神 #GenshinImpact  #genshinimpactfanart  #oc https://t.co/GWS5ON6Gjc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>....I wanna be a genshin beta tester. Hey, @GenshinImpact pls?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Check out my broadcast from my PlayStation 4! #PS4live (Genshin Impact)  live at https://t.co/dLkBZsAyhg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Stone cold crazy bitches I know that’s right #GenshinImpact #Kakegurui #RaidenShogun #KirariMomobami https://t.co/aky527x9L5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                                                                                                  tweet\n",
       "0                                                                                                                                                                          i just wanted to see what he would look like in black...?\\n#GenshinImpact #原神 #venti https://t.co/dvy3gKAL8G\n",
       "1  @GenshinImpact Hey genshin, I was playing today and I wished 30 rolls, I got not a single 5 or 4 star, I’m sure something went wrong, but I’m wondering if you could help me? I haven’t been able to get any 4-5 stars at all when pulled 50 for the 3 days, been playing this game.\n",
       "2                                          Thank you so much @_Marii_Pily for this commission, it’s so beautiful \\nI LOVE IT SO MUCH ??\\n\\nfor you all Aeventi nation, i love this fandom so much. &lt;3\\n\\n#aeventi #venther #aether #venti #GenshinImpact #原神 https://t.co/VWop9QloHe\n",
       "3                                                                                                                                                                                                                      Comforting ocean #kaebedo #GenshinImpact https://t.co/JGLZRrVjU3\n",
       "4                                                                                                                                                                                                                                                                Is Genshin Impact fun?\n",
       "5                                                                                                                                      i read Cascading ( In a good way)  by Hubbleablubble on ao3 and now i am a firm Kaebedo enjoyer\\n#GenshinImpact #kaebedo https://t.co/kyhxeSfKN5\n",
       "6                                       Genshin impact (similar to) Kitsune, Inazuma-OC!\\n\\nShe is a deceased human spirit who lives as fox now waiting for the fated person to come by, then she met Venti---\\n\\n#原神 #GenshinImpact  #genshinimpactfanart  #oc https://t.co/GWS5ON6Gjc\n",
       "7                                                                                                                                                                                                                        ....I wanna be a genshin beta tester. Hey, @GenshinImpact pls?\n",
       "8                                                                                                                                                                              Check out my broadcast from my PlayStation 4! #PS4live (Genshin Impact)  live at https://t.co/dLkBZsAyhg\n",
       "9                                                                                                                                                          Stone cold crazy bitches I know that’s right #GenshinImpact #Kakegurui #RaidenShogun #KirariMomobami https://t.co/aky527x9L5"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.set_option('display.max_colwidth', None)\n",
    "\n",
    "# sql = \"SELECT\"\n",
    "# sql += \" CASE WHEN full_text IS NULL THEN text ELSE full_text END AS tweet\"\n",
    "# sql += \" FROM tweets\"\n",
    "# sql += \" WHERE\"\n",
    "\n",
    "# for i in range(len(keywords)):\n",
    "#     sql += \" (full_text LIKE '%\"\n",
    "#     sql += keywords[i]\n",
    "#     sql += \"%' \"\n",
    "#     sql += \" OR text LIKE '%\"\n",
    "#     sql += keywords[i]\n",
    "#     sql += \"%') \"\n",
    "    \n",
    "#     if (i + 1) != len(keywords):\n",
    "#         sql += \"OR \" \n",
    "\n",
    "# sql += \" ORDER BY RAND()\"\n",
    "# sql += \" LIMIT 1000\"\n",
    "\n",
    "# df = pd.read_sql(sql, mysql_connection)\n",
    "\n",
    "# Read CSV\n",
    "df = pd.read_csv(\"tweets/tweets.csv\")\n",
    "\n",
    "df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Store to CSV\n",
    "\n",
    "# from datetime import datetime\n",
    "\n",
    "# today = datetime.now().strftime(\"%Y_%m_%d_%H_%M_%S\")\n",
    "\n",
    "# compression_opts = dict(method='zip',\n",
    "#                         archive_name='tweets_of_game__' + today + '.csv')  \n",
    "# df.to_csv('tweets/tweets_of_game__' + today + '.zip', index=False,\n",
    "#           compression=compression_opts)  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Noise Removal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet</th>\n",
       "      <th>clean_tweet</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>i just wanted to see what he would look like in black...?\\n#GenshinImpact #原神 #venti https://t.co/dvy3gKAL8G</td>\n",
       "      <td>i just wanted to see what he would look like in black...?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>@GenshinImpact Hey genshin, I was playing today and I wished 30 rolls, I got not a single 5 or 4 star, I’m sure something went wrong, but I’m wondering if you could help me? I haven’t been able to get any 4-5 stars at all when pulled 50 for the 3 days, been playing this game.</td>\n",
       "      <td>Hey genshin, I was playing today and I wished 30 rolls, I got not a single 5 or 4 star, Im sure something went wrong, but Im wondering if you could help me? I havent been able to get any 4-5 stars at all when pulled 50 for the 3 days, been playing this game.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Thank you so much @_Marii_Pily for this commission, it’s so beautiful \\nI LOVE IT SO MUCH ??\\n\\nfor you all Aeventi nation, i love this fandom so much. &amp;lt;3\\n\\n#aeventi #venther #aether #venti #GenshinImpact #原神 https://t.co/VWop9QloHe</td>\n",
       "      <td>Thank you so much for this commission, its so beautiful I LOVE IT SO MUCH ?? for you all Aeventi nation, i love this fandom so much. &amp;lt;3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Comforting ocean #kaebedo #GenshinImpact https://t.co/JGLZRrVjU3</td>\n",
       "      <td>Comforting ocean</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Is Genshin Impact fun?</td>\n",
       "      <td>Is Genshin Impact fun?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>i read Cascading ( In a good way)  by Hubbleablubble on ao3 and now i am a firm Kaebedo enjoyer\\n#GenshinImpact #kaebedo https://t.co/kyhxeSfKN5</td>\n",
       "      <td>i read Cascading ( In a good way) by Hubbleablubble on ao3 and now i am a firm Kaebedo enjoyer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Genshin impact (similar to) Kitsune, Inazuma-OC!\\n\\nShe is a deceased human spirit who lives as fox now waiting for the fated person to come by, then she met Venti---\\n\\n#原神 #GenshinImpact  #genshinimpactfanart  #oc https://t.co/GWS5ON6Gjc</td>\n",
       "      <td>Genshin impact (similar to) Kitsune, Inazuma-OC! She is a deceased human spirit who lives as fox now waiting for the fated person to come by, then she met Venti---</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>....I wanna be a genshin beta tester. Hey, @GenshinImpact pls?</td>\n",
       "      <td>....I wanna be a genshin beta tester. Hey, pls?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Check out my broadcast from my PlayStation 4! #PS4live (Genshin Impact)  live at https://t.co/dLkBZsAyhg</td>\n",
       "      <td>Check out my broadcast from my PlayStation 4! (Genshin Impact) live at</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Stone cold crazy bitches I know that’s right #GenshinImpact #Kakegurui #RaidenShogun #KirariMomobami https://t.co/aky527x9L5</td>\n",
       "      <td>Stone cold crazy bitches I know thats right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>-The Fair Lady-\\nLa Signora photoset will be available at the end of this month on my P?tre?n \\nCosplay set from tier \"Silver \"❗️\\n▶️ https://t.co/rp1YPFkXFw\\n\\nPhoto by @Akunohako \\n#GenshinImpact https://t.co/TYipFC0cm7</td>\n",
       "      <td>-The Fair Lady- La Signora photoset will be available at the end of this month on my P?tre?n Cosplay set from tier \"Silver \" Photo by</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Complete the Sakura Cleansing Ritual (3/3) - Genshin Impact INAZUMA https://t.co/bV3FxrprH2 via @YouTube @GenshinImpact #Genshin #Inazuma https://t.co/rbVvZ8Wzrd</td>\n",
       "      <td>Complete the Sakura Cleansing Ritual (3/3) - Genshin Impact INAZUMA via</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>@Minecraft Its over</td>\n",
       "      <td>Its over</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>@GenshinImpact mihoyo WHY DO U HATE ME?</td>\n",
       "      <td>mihoyo WHY DO U HATE ME?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Welcome home baby ?❄️\\n#GenshinImpact\\n#KamisatoAyaka https://t.co/j9hKXBS24n</td>\n",
       "      <td>Welcome home baby ?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>elysium x genshin impact?? ?come home good sir\\n\\n#アークナイツ #Arknights #明日方舟 \\n#GenshinImpact #原神 https://t.co/QuACTIHjkY</td>\n",
       "      <td>elysium x genshin impact?? ?come home good sir</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Today’s requested character is Xiao from Genshin Impact ! https://t.co/xKWG4tC2pI</td>\n",
       "      <td>Todays requested character is Xiao from Genshin Impact !</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>i want you more than any blue sky\\n#kokomiya #kokomi #yoimiya #GenshinImpact #原神 https://t.co/CljFsgH4Vp</td>\n",
       "      <td>i want you more than any blue sky</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Check out my broadcast from my PlayStation 4! #PS4live (Genshin Impact)  live at https://t.co/dSTLq2qhsi</td>\n",
       "      <td>Check out my broadcast from my PlayStation 4! (Genshin Impact) live at</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Genshin Impact sexual orientation hc thread!!!</td>\n",
       "      <td>Genshin Impact sexual orientation hc thread!!!</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                                                                                                   tweet  \\\n",
       "0                                                                                                                                                                           i just wanted to see what he would look like in black...?\\n#GenshinImpact #原神 #venti https://t.co/dvy3gKAL8G   \n",
       "1   @GenshinImpact Hey genshin, I was playing today and I wished 30 rolls, I got not a single 5 or 4 star, I’m sure something went wrong, but I’m wondering if you could help me? I haven’t been able to get any 4-5 stars at all when pulled 50 for the 3 days, been playing this game.   \n",
       "2                                           Thank you so much @_Marii_Pily for this commission, it’s so beautiful \\nI LOVE IT SO MUCH ??\\n\\nfor you all Aeventi nation, i love this fandom so much. &lt;3\\n\\n#aeventi #venther #aether #venti #GenshinImpact #原神 https://t.co/VWop9QloHe   \n",
       "3                                                                                                                                                                                                                       Comforting ocean #kaebedo #GenshinImpact https://t.co/JGLZRrVjU3   \n",
       "4                                                                                                                                                                                                                                                                 Is Genshin Impact fun?   \n",
       "5                                                                                                                                       i read Cascading ( In a good way)  by Hubbleablubble on ao3 and now i am a firm Kaebedo enjoyer\\n#GenshinImpact #kaebedo https://t.co/kyhxeSfKN5   \n",
       "6                                        Genshin impact (similar to) Kitsune, Inazuma-OC!\\n\\nShe is a deceased human spirit who lives as fox now waiting for the fated person to come by, then she met Venti---\\n\\n#原神 #GenshinImpact  #genshinimpactfanart  #oc https://t.co/GWS5ON6Gjc   \n",
       "7                                                                                                                                                                                                                         ....I wanna be a genshin beta tester. Hey, @GenshinImpact pls?   \n",
       "8                                                                                                                                                                               Check out my broadcast from my PlayStation 4! #PS4live (Genshin Impact)  live at https://t.co/dLkBZsAyhg   \n",
       "9                                                                                                                                                           Stone cold crazy bitches I know that’s right #GenshinImpact #Kakegurui #RaidenShogun #KirariMomobami https://t.co/aky527x9L5   \n",
       "10                                                         -The Fair Lady-\\nLa Signora photoset will be available at the end of this month on my P?tre?n \\nCosplay set from tier \"Silver \"❗️\\n▶️ https://t.co/rp1YPFkXFw\\n\\nPhoto by @Akunohako \\n#GenshinImpact https://t.co/TYipFC0cm7   \n",
       "11                                                                                                                     Complete the Sakura Cleansing Ritual (3/3) - Genshin Impact INAZUMA https://t.co/bV3FxrprH2 via @YouTube @GenshinImpact #Genshin #Inazuma https://t.co/rbVvZ8Wzrd   \n",
       "12                                                                                                                                                                                                                                                                   @Minecraft Its over   \n",
       "13                                                                                                                                                                                                                                               @GenshinImpact mihoyo WHY DO U HATE ME?   \n",
       "14                                                                                                                                                                                                         Welcome home baby ?❄️\\n#GenshinImpact\\n#KamisatoAyaka https://t.co/j9hKXBS24n   \n",
       "15                                                                                                                                                               elysium x genshin impact?? ?come home good sir\\n\\n#アークナイツ #Arknights #明日方舟 \\n#GenshinImpact #原神 https://t.co/QuACTIHjkY   \n",
       "16                                                                                                                                                                                                     Today’s requested character is Xiao from Genshin Impact ! https://t.co/xKWG4tC2pI   \n",
       "17                                                                                                                                                                              i want you more than any blue sky\\n#kokomiya #kokomi #yoimiya #GenshinImpact #原神 https://t.co/CljFsgH4Vp   \n",
       "18                                                                                                                                                                              Check out my broadcast from my PlayStation 4! #PS4live (Genshin Impact)  live at https://t.co/dSTLq2qhsi   \n",
       "19                                                                                                                                                                                                                                        Genshin Impact sexual orientation hc thread!!!   \n",
       "\n",
       "                                                                                                                                                                                                                                                           clean_tweet  \n",
       "0                                                                                                                                                                                                            i just wanted to see what he would look like in black...?  \n",
       "1   Hey genshin, I was playing today and I wished 30 rolls, I got not a single 5 or 4 star, Im sure something went wrong, but Im wondering if you could help me? I havent been able to get any 4-5 stars at all when pulled 50 for the 3 days, been playing this game.  \n",
       "2                                                                                                                           Thank you so much for this commission, its so beautiful I LOVE IT SO MUCH ?? for you all Aeventi nation, i love this fandom so much. &lt;3  \n",
       "3                                                                                                                                                                                                                                                     Comforting ocean  \n",
       "4                                                                                                                                                                                                                                               Is Genshin Impact fun?  \n",
       "5                                                                                                                                                                       i read Cascading ( In a good way) by Hubbleablubble on ao3 and now i am a firm Kaebedo enjoyer  \n",
       "6                                                                                                  Genshin impact (similar to) Kitsune, Inazuma-OC! She is a deceased human spirit who lives as fox now waiting for the fated person to come by, then she met Venti---  \n",
       "7                                                                                                                                                                                                                      ....I wanna be a genshin beta tester. Hey, pls?  \n",
       "8                                                                                                                                                                                               Check out my broadcast from my PlayStation 4! (Genshin Impact) live at  \n",
       "9                                                                                                                                                                                                                          Stone cold crazy bitches I know thats right  \n",
       "10                                                                                                                               -The Fair Lady- La Signora photoset will be available at the end of this month on my P?tre?n Cosplay set from tier \"Silver \" Photo by  \n",
       "11                                                                                                                                                                                             Complete the Sakura Cleansing Ritual (3/3) - Genshin Impact INAZUMA via  \n",
       "12                                                                                                                                                                                                                                                            Its over  \n",
       "13                                                                                                                                                                                                                                            mihoyo WHY DO U HATE ME?  \n",
       "14                                                                                                                                                                                                                                                 Welcome home baby ?  \n",
       "15                                                                                                                                                                                                                      elysium x genshin impact?? ?come home good sir  \n",
       "16                                                                                                                                                                                                            Todays requested character is Xiao from Genshin Impact !  \n",
       "17                                                                                                                                                                                                                                   i want you more than any blue sky  \n",
       "18                                                                                                                                                                                              Check out my broadcast from my PlayStation 4! (Genshin Impact) live at  \n",
       "19                                                                                                                                                                                                                      Genshin Impact sexual orientation hc thread!!!  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "import string\n",
    "import preprocessor as p\n",
    "\n",
    "p.set_options(p.OPT.MENTION, p.OPT.URL, p.OPT.EMOJI, p.OPT.HASHTAG)\n",
    "\n",
    "# forming a separate feature for cleaned tweets\n",
    "for i, v in enumerate(df['tweet']):\n",
    "    cleaned_text = p.clean(v)\n",
    "    cleaned_text = re.compile(r'[\\n\\r\\t]').sub(\"\", cleaned_text)\n",
    "    \n",
    "    df.loc[i,'clean_tweet'] = cleaned_text\n",
    "    \n",
    "df.head(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Normalisasi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet</th>\n",
       "      <th>clean_tweet</th>\n",
       "      <th>clean_tweet_normalize</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>i just wanted to see what he would look like in black...?\\n#GenshinImpact #原神 #venti https://t.co/dvy3gKAL8G</td>\n",
       "      <td>i just wanted to see what he would look like in black...?</td>\n",
       "      <td>i just wanted to see what he would look like in black...?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>@GenshinImpact Hey genshin, I was playing today and I wished 30 rolls, I got not a single 5 or 4 star, I’m sure something went wrong, but I’m wondering if you could help me? I haven’t been able to get any 4-5 stars at all when pulled 50 for the 3 days, been playing this game.</td>\n",
       "      <td>Hey genshin, I was playing today and I wished 30 rolls, I got not a single 5 or 4 star, Im sure something went wrong, but Im wondering if you could help me? I havent been able to get any 4-5 stars at all when pulled 50 for the 3 days, been playing this game.</td>\n",
       "      <td>hey genshin, i was playing today and i wished 30 rolls, i got not a single 5 or 4 star, im sure something went wrong, but im wondering if you could help me? i havent been able to get any 4-5 stars at all when pulled 50 for the 3 days, been playing this game.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Thank you so much @_Marii_Pily for this commission, it’s so beautiful \\nI LOVE IT SO MUCH ??\\n\\nfor you all Aeventi nation, i love this fandom so much. &amp;lt;3\\n\\n#aeventi #venther #aether #venti #GenshinImpact #原神 https://t.co/VWop9QloHe</td>\n",
       "      <td>Thank you so much for this commission, its so beautiful I LOVE IT SO MUCH ?? for you all Aeventi nation, i love this fandom so much. &amp;lt;3</td>\n",
       "      <td>thank you so much for this commission, its so beautiful i love it so much ?? for you all aeventi nation, i love this fandom so much. &amp;lt;3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Comforting ocean #kaebedo #GenshinImpact https://t.co/JGLZRrVjU3</td>\n",
       "      <td>Comforting ocean</td>\n",
       "      <td>comforting ocean</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Is Genshin Impact fun?</td>\n",
       "      <td>Is Genshin Impact fun?</td>\n",
       "      <td>is genshin impact fun?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>i read Cascading ( In a good way)  by Hubbleablubble on ao3 and now i am a firm Kaebedo enjoyer\\n#GenshinImpact #kaebedo https://t.co/kyhxeSfKN5</td>\n",
       "      <td>i read Cascading ( In a good way) by Hubbleablubble on ao3 and now i am a firm Kaebedo enjoyer</td>\n",
       "      <td>i read cascading ( in a good way) by hubbleablubble on ao3 and now i am a firm kaebedo enjoyer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Genshin impact (similar to) Kitsune, Inazuma-OC!\\n\\nShe is a deceased human spirit who lives as fox now waiting for the fated person to come by, then she met Venti---\\n\\n#原神 #GenshinImpact  #genshinimpactfanart  #oc https://t.co/GWS5ON6Gjc</td>\n",
       "      <td>Genshin impact (similar to) Kitsune, Inazuma-OC! She is a deceased human spirit who lives as fox now waiting for the fated person to come by, then she met Venti---</td>\n",
       "      <td>genshin impact (similar to) kitsune, inazuma-oc! she is a deceased human spirit who lives as fox now waiting for the fated person to come by, then she met venti---</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>....I wanna be a genshin beta tester. Hey, @GenshinImpact pls?</td>\n",
       "      <td>....I wanna be a genshin beta tester. Hey, pls?</td>\n",
       "      <td>....i wanna be a genshin beta tester. hey, pls?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Check out my broadcast from my PlayStation 4! #PS4live (Genshin Impact)  live at https://t.co/dLkBZsAyhg</td>\n",
       "      <td>Check out my broadcast from my PlayStation 4! (Genshin Impact) live at</td>\n",
       "      <td>check out my broadcast from my playstation 4! (genshin impact) live at</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Stone cold crazy bitches I know that’s right #GenshinImpact #Kakegurui #RaidenShogun #KirariMomobami https://t.co/aky527x9L5</td>\n",
       "      <td>Stone cold crazy bitches I know thats right</td>\n",
       "      <td>stone cold crazy bitches i know thats right</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                                                                                                  tweet  \\\n",
       "0                                                                                                                                                                          i just wanted to see what he would look like in black...?\\n#GenshinImpact #原神 #venti https://t.co/dvy3gKAL8G   \n",
       "1  @GenshinImpact Hey genshin, I was playing today and I wished 30 rolls, I got not a single 5 or 4 star, I’m sure something went wrong, but I’m wondering if you could help me? I haven’t been able to get any 4-5 stars at all when pulled 50 for the 3 days, been playing this game.   \n",
       "2                                          Thank you so much @_Marii_Pily for this commission, it’s so beautiful \\nI LOVE IT SO MUCH ??\\n\\nfor you all Aeventi nation, i love this fandom so much. &lt;3\\n\\n#aeventi #venther #aether #venti #GenshinImpact #原神 https://t.co/VWop9QloHe   \n",
       "3                                                                                                                                                                                                                      Comforting ocean #kaebedo #GenshinImpact https://t.co/JGLZRrVjU3   \n",
       "4                                                                                                                                                                                                                                                                Is Genshin Impact fun?   \n",
       "5                                                                                                                                      i read Cascading ( In a good way)  by Hubbleablubble on ao3 and now i am a firm Kaebedo enjoyer\\n#GenshinImpact #kaebedo https://t.co/kyhxeSfKN5   \n",
       "6                                       Genshin impact (similar to) Kitsune, Inazuma-OC!\\n\\nShe is a deceased human spirit who lives as fox now waiting for the fated person to come by, then she met Venti---\\n\\n#原神 #GenshinImpact  #genshinimpactfanart  #oc https://t.co/GWS5ON6Gjc   \n",
       "7                                                                                                                                                                                                                        ....I wanna be a genshin beta tester. Hey, @GenshinImpact pls?   \n",
       "8                                                                                                                                                                              Check out my broadcast from my PlayStation 4! #PS4live (Genshin Impact)  live at https://t.co/dLkBZsAyhg   \n",
       "9                                                                                                                                                          Stone cold crazy bitches I know that’s right #GenshinImpact #Kakegurui #RaidenShogun #KirariMomobami https://t.co/aky527x9L5   \n",
       "\n",
       "                                                                                                                                                                                                                                                          clean_tweet  \\\n",
       "0                                                                                                                                                                                                           i just wanted to see what he would look like in black...?   \n",
       "1  Hey genshin, I was playing today and I wished 30 rolls, I got not a single 5 or 4 star, Im sure something went wrong, but Im wondering if you could help me? I havent been able to get any 4-5 stars at all when pulled 50 for the 3 days, been playing this game.   \n",
       "2                                                                                                                          Thank you so much for this commission, its so beautiful I LOVE IT SO MUCH ?? for you all Aeventi nation, i love this fandom so much. &lt;3   \n",
       "3                                                                                                                                                                                                                                                    Comforting ocean   \n",
       "4                                                                                                                                                                                                                                              Is Genshin Impact fun?   \n",
       "5                                                                                                                                                                      i read Cascading ( In a good way) by Hubbleablubble on ao3 and now i am a firm Kaebedo enjoyer   \n",
       "6                                                                                                 Genshin impact (similar to) Kitsune, Inazuma-OC! She is a deceased human spirit who lives as fox now waiting for the fated person to come by, then she met Venti---   \n",
       "7                                                                                                                                                                                                                     ....I wanna be a genshin beta tester. Hey, pls?   \n",
       "8                                                                                                                                                                                              Check out my broadcast from my PlayStation 4! (Genshin Impact) live at   \n",
       "9                                                                                                                                                                                                                         Stone cold crazy bitches I know thats right   \n",
       "\n",
       "                                                                                                                                                                                                                                                clean_tweet_normalize  \n",
       "0                                                                                                                                                                                                           i just wanted to see what he would look like in black...?  \n",
       "1  hey genshin, i was playing today and i wished 30 rolls, i got not a single 5 or 4 star, im sure something went wrong, but im wondering if you could help me? i havent been able to get any 4-5 stars at all when pulled 50 for the 3 days, been playing this game.  \n",
       "2                                                                                                                          thank you so much for this commission, its so beautiful i love it so much ?? for you all aeventi nation, i love this fandom so much. &lt;3  \n",
       "3                                                                                                                                                                                                                                                    comforting ocean  \n",
       "4                                                                                                                                                                                                                                              is genshin impact fun?  \n",
       "5                                                                                                                                                                      i read cascading ( in a good way) by hubbleablubble on ao3 and now i am a firm kaebedo enjoyer  \n",
       "6                                                                                                 genshin impact (similar to) kitsune, inazuma-oc! she is a deceased human spirit who lives as fox now waiting for the fated person to come by, then she met venti---  \n",
       "7                                                                                                                                                                                                                     ....i wanna be a genshin beta tester. hey, pls?  \n",
       "8                                                                                                                                                                                              check out my broadcast from my playstation 4! (genshin impact) live at  \n",
       "9                                                                                                                                                                                                                         stone cold crazy bitches i know thats right  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Lowertext\n",
    "df['clean_tweet_normalize'] = [tweet.lower() for tweet in df.clean_tweet.astype(str)]\n",
    "\n",
    "df.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Membersihkan angka dan tanda baca"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet</th>\n",
       "      <th>clean_tweet</th>\n",
       "      <th>clean_tweet_normalize</th>\n",
       "      <th>clean_tweet_3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>i just wanted to see what he would look like in black...?\\n#GenshinImpact #原神 #venti https://t.co/dvy3gKAL8G</td>\n",
       "      <td>i just wanted to see what he would look like in black...?</td>\n",
       "      <td>i just wanted to see what he would look like in black...?</td>\n",
       "      <td>i just wanted to see what he would look like in black</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>@GenshinImpact Hey genshin, I was playing today and I wished 30 rolls, I got not a single 5 or 4 star, I’m sure something went wrong, but I’m wondering if you could help me? I haven’t been able to get any 4-5 stars at all when pulled 50 for the 3 days, been playing this game.</td>\n",
       "      <td>Hey genshin, I was playing today and I wished 30 rolls, I got not a single 5 or 4 star, Im sure something went wrong, but Im wondering if you could help me? I havent been able to get any 4-5 stars at all when pulled 50 for the 3 days, been playing this game.</td>\n",
       "      <td>hey genshin, i was playing today and i wished 30 rolls, i got not a single 5 or 4 star, im sure something went wrong, but im wondering if you could help me? i havent been able to get any 4-5 stars at all when pulled 50 for the 3 days, been playing this game.</td>\n",
       "      <td>hey genshin  i was playing today and i wished  rolls  i got not a single  or  star  im sure something went wrong  but im wondering if you could help me  i havent been able to get any   stars at all when pulled  for the  days  been playing this game</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Thank you so much @_Marii_Pily for this commission, it’s so beautiful \\nI LOVE IT SO MUCH ??\\n\\nfor you all Aeventi nation, i love this fandom so much. &amp;lt;3\\n\\n#aeventi #venther #aether #venti #GenshinImpact #原神 https://t.co/VWop9QloHe</td>\n",
       "      <td>Thank you so much for this commission, its so beautiful I LOVE IT SO MUCH ?? for you all Aeventi nation, i love this fandom so much. &amp;lt;3</td>\n",
       "      <td>thank you so much for this commission, its so beautiful i love it so much ?? for you all aeventi nation, i love this fandom so much. &amp;lt;3</td>\n",
       "      <td>thank you so much for this commission  its so beautiful i love it so much    for you all aeventi nation  i love this fandom so much   lt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Comforting ocean #kaebedo #GenshinImpact https://t.co/JGLZRrVjU3</td>\n",
       "      <td>Comforting ocean</td>\n",
       "      <td>comforting ocean</td>\n",
       "      <td>comforting ocean</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Is Genshin Impact fun?</td>\n",
       "      <td>Is Genshin Impact fun?</td>\n",
       "      <td>is genshin impact fun?</td>\n",
       "      <td>is genshin impact fun</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>i read Cascading ( In a good way)  by Hubbleablubble on ao3 and now i am a firm Kaebedo enjoyer\\n#GenshinImpact #kaebedo https://t.co/kyhxeSfKN5</td>\n",
       "      <td>i read Cascading ( In a good way) by Hubbleablubble on ao3 and now i am a firm Kaebedo enjoyer</td>\n",
       "      <td>i read cascading ( in a good way) by hubbleablubble on ao3 and now i am a firm kaebedo enjoyer</td>\n",
       "      <td>i read cascading   in a good way  by hubbleablubble on  and now i am a firm kaebedo enjoyer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Genshin impact (similar to) Kitsune, Inazuma-OC!\\n\\nShe is a deceased human spirit who lives as fox now waiting for the fated person to come by, then she met Venti---\\n\\n#原神 #GenshinImpact  #genshinimpactfanart  #oc https://t.co/GWS5ON6Gjc</td>\n",
       "      <td>Genshin impact (similar to) Kitsune, Inazuma-OC! She is a deceased human spirit who lives as fox now waiting for the fated person to come by, then she met Venti---</td>\n",
       "      <td>genshin impact (similar to) kitsune, inazuma-oc! she is a deceased human spirit who lives as fox now waiting for the fated person to come by, then she met venti---</td>\n",
       "      <td>genshin impact  similar to  kitsune  inazuma oc  she is a deceased human spirit who lives as fox now waiting for the fated person to come by  then she met venti</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>....I wanna be a genshin beta tester. Hey, @GenshinImpact pls?</td>\n",
       "      <td>....I wanna be a genshin beta tester. Hey, pls?</td>\n",
       "      <td>....i wanna be a genshin beta tester. hey, pls?</td>\n",
       "      <td>i wanna be a genshin beta tester  hey  pls</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Check out my broadcast from my PlayStation 4! #PS4live (Genshin Impact)  live at https://t.co/dLkBZsAyhg</td>\n",
       "      <td>Check out my broadcast from my PlayStation 4! (Genshin Impact) live at</td>\n",
       "      <td>check out my broadcast from my playstation 4! (genshin impact) live at</td>\n",
       "      <td>check out my broadcast from my playstation    genshin impact  live at</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Stone cold crazy bitches I know that’s right #GenshinImpact #Kakegurui #RaidenShogun #KirariMomobami https://t.co/aky527x9L5</td>\n",
       "      <td>Stone cold crazy bitches I know thats right</td>\n",
       "      <td>stone cold crazy bitches i know thats right</td>\n",
       "      <td>stone cold crazy bitches i know thats right</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                                                                                                  tweet  \\\n",
       "0                                                                                                                                                                          i just wanted to see what he would look like in black...?\\n#GenshinImpact #原神 #venti https://t.co/dvy3gKAL8G   \n",
       "1  @GenshinImpact Hey genshin, I was playing today and I wished 30 rolls, I got not a single 5 or 4 star, I’m sure something went wrong, but I’m wondering if you could help me? I haven’t been able to get any 4-5 stars at all when pulled 50 for the 3 days, been playing this game.   \n",
       "2                                          Thank you so much @_Marii_Pily for this commission, it’s so beautiful \\nI LOVE IT SO MUCH ??\\n\\nfor you all Aeventi nation, i love this fandom so much. &lt;3\\n\\n#aeventi #venther #aether #venti #GenshinImpact #原神 https://t.co/VWop9QloHe   \n",
       "3                                                                                                                                                                                                                      Comforting ocean #kaebedo #GenshinImpact https://t.co/JGLZRrVjU3   \n",
       "4                                                                                                                                                                                                                                                                Is Genshin Impact fun?   \n",
       "5                                                                                                                                      i read Cascading ( In a good way)  by Hubbleablubble on ao3 and now i am a firm Kaebedo enjoyer\\n#GenshinImpact #kaebedo https://t.co/kyhxeSfKN5   \n",
       "6                                       Genshin impact (similar to) Kitsune, Inazuma-OC!\\n\\nShe is a deceased human spirit who lives as fox now waiting for the fated person to come by, then she met Venti---\\n\\n#原神 #GenshinImpact  #genshinimpactfanart  #oc https://t.co/GWS5ON6Gjc   \n",
       "7                                                                                                                                                                                                                        ....I wanna be a genshin beta tester. Hey, @GenshinImpact pls?   \n",
       "8                                                                                                                                                                              Check out my broadcast from my PlayStation 4! #PS4live (Genshin Impact)  live at https://t.co/dLkBZsAyhg   \n",
       "9                                                                                                                                                          Stone cold crazy bitches I know that’s right #GenshinImpact #Kakegurui #RaidenShogun #KirariMomobami https://t.co/aky527x9L5   \n",
       "\n",
       "                                                                                                                                                                                                                                                          clean_tweet  \\\n",
       "0                                                                                                                                                                                                           i just wanted to see what he would look like in black...?   \n",
       "1  Hey genshin, I was playing today and I wished 30 rolls, I got not a single 5 or 4 star, Im sure something went wrong, but Im wondering if you could help me? I havent been able to get any 4-5 stars at all when pulled 50 for the 3 days, been playing this game.   \n",
       "2                                                                                                                          Thank you so much for this commission, its so beautiful I LOVE IT SO MUCH ?? for you all Aeventi nation, i love this fandom so much. &lt;3   \n",
       "3                                                                                                                                                                                                                                                    Comforting ocean   \n",
       "4                                                                                                                                                                                                                                              Is Genshin Impact fun?   \n",
       "5                                                                                                                                                                      i read Cascading ( In a good way) by Hubbleablubble on ao3 and now i am a firm Kaebedo enjoyer   \n",
       "6                                                                                                 Genshin impact (similar to) Kitsune, Inazuma-OC! She is a deceased human spirit who lives as fox now waiting for the fated person to come by, then she met Venti---   \n",
       "7                                                                                                                                                                                                                     ....I wanna be a genshin beta tester. Hey, pls?   \n",
       "8                                                                                                                                                                                              Check out my broadcast from my PlayStation 4! (Genshin Impact) live at   \n",
       "9                                                                                                                                                                                                                         Stone cold crazy bitches I know thats right   \n",
       "\n",
       "                                                                                                                                                                                                                                                clean_tweet_normalize  \\\n",
       "0                                                                                                                                                                                                           i just wanted to see what he would look like in black...?   \n",
       "1  hey genshin, i was playing today and i wished 30 rolls, i got not a single 5 or 4 star, im sure something went wrong, but im wondering if you could help me? i havent been able to get any 4-5 stars at all when pulled 50 for the 3 days, been playing this game.   \n",
       "2                                                                                                                          thank you so much for this commission, its so beautiful i love it so much ?? for you all aeventi nation, i love this fandom so much. &lt;3   \n",
       "3                                                                                                                                                                                                                                                    comforting ocean   \n",
       "4                                                                                                                                                                                                                                              is genshin impact fun?   \n",
       "5                                                                                                                                                                      i read cascading ( in a good way) by hubbleablubble on ao3 and now i am a firm kaebedo enjoyer   \n",
       "6                                                                                                 genshin impact (similar to) kitsune, inazuma-oc! she is a deceased human spirit who lives as fox now waiting for the fated person to come by, then she met venti---   \n",
       "7                                                                                                                                                                                                                     ....i wanna be a genshin beta tester. hey, pls?   \n",
       "8                                                                                                                                                                                              check out my broadcast from my playstation 4! (genshin impact) live at   \n",
       "9                                                                                                                                                                                                                         stone cold crazy bitches i know thats right   \n",
       "\n",
       "                                                                                                                                                                                                                                               clean_tweet_3  \n",
       "0                                                                                                                                                                                                  i just wanted to see what he would look like in black      \n",
       "1  hey genshin  i was playing today and i wished  rolls  i got not a single  or  star  im sure something went wrong  but im wondering if you could help me  i havent been able to get any   stars at all when pulled  for the  days  been playing this game   \n",
       "2                                                                                                                  thank you so much for this commission  its so beautiful i love it so much    for you all aeventi nation  i love this fandom so much   lt   \n",
       "3                                                                                                                                                                                                                                           comforting ocean  \n",
       "4                                                                                                                                                                                                                                     is genshin impact fun   \n",
       "5                                                                                                                                                                i read cascading   in a good way  by hubbleablubble on  and now i am a firm kaebedo enjoyer  \n",
       "6                                                                                        genshin impact  similar to  kitsune  inazuma oc  she is a deceased human spirit who lives as fox now waiting for the fated person to come by  then she met venti     \n",
       "7                                                                                                                                                                                                                i wanna be a genshin beta tester  hey  pls   \n",
       "8                                                                                                                                                                                      check out my broadcast from my playstation    genshin impact  live at  \n",
       "9                                                                                                                                                                                                                stone cold crazy bitches i know thats right  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Replace escaped HTML\n",
    "df['clean_tweet_3'] = [re.sub('&amp;', \"\", tweet) for tweet in df.clean_tweet_normalize]\n",
    "\n",
    "# Remove ticks and the next character\n",
    "df['clean_tweet_3'] = [re.sub(\"\\'\\w+\", '', tweet) for tweet in df.clean_tweet_3]\n",
    "\n",
    "# Remove punctuations\n",
    "df['clean_tweet_3'] = [re.sub('[%s]' % re.escape(string.punctuation), ' ', tweet) for tweet in df.clean_tweet_3]\n",
    "\n",
    "# Remove numbers\n",
    "df['clean_tweet_3'] = [re.sub(r'\\w*\\d+\\w*', '', tweet) for tweet in df.clean_tweet_3]\n",
    "\n",
    "df.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>clean_tweet_3</th>\n",
       "      <th>clean_token</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>i just wanted to see what he would look like in black</td>\n",
       "      <td>[i, just, wanted, to, see, what, he, would, look, like, in, black]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>hey genshin  i was playing today and i wished  rolls  i got not a single  or  star  im sure something went wrong  but im wondering if you could help me  i havent been able to get any   stars at all when pulled  for the  days  been playing this game</td>\n",
       "      <td>[hey, genshin, i, was, playing, today, and, i, wished, rolls, i, got, not, a, single, or, star, im, sure, something, went, wrong, but, im, wondering, if, you, could, help, me, i, havent, been, able, to, get, any, stars, at, all, when, pulled, for, the, days, been, playing, this, game]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>thank you so much for this commission  its so beautiful i love it so much    for you all aeventi nation  i love this fandom so much   lt</td>\n",
       "      <td>[thank, you, so, much, for, this, commission, its, so, beautiful, i, love, it, so, much, for, you, all, aeventi, nation, i, love, this, fandom, so, much, lt]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>comforting ocean</td>\n",
       "      <td>[comforting, ocean]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>is genshin impact fun</td>\n",
       "      <td>[is, genshin, impact, fun]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>i read cascading   in a good way  by hubbleablubble on  and now i am a firm kaebedo enjoyer</td>\n",
       "      <td>[i, read, cascading, in, a, good, way, by, hubbleablubble, on, and, now, i, am, a, firm, kaebedo, enjoyer]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>genshin impact  similar to  kitsune  inazuma oc  she is a deceased human spirit who lives as fox now waiting for the fated person to come by  then she met venti</td>\n",
       "      <td>[genshin, impact, similar, to, kitsune, inazuma, oc, she, is, a, deceased, human, spirit, who, lives, as, fox, now, waiting, for, the, fated, person, to, come, by, then, she, met, venti]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>i wanna be a genshin beta tester  hey  pls</td>\n",
       "      <td>[i, wan, na, be, a, genshin, beta, tester, hey, pls]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>check out my broadcast from my playstation    genshin impact  live at</td>\n",
       "      <td>[check, out, my, broadcast, from, my, playstation, genshin, impact, live, at]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>stone cold crazy bitches i know thats right</td>\n",
       "      <td>[stone, cold, crazy, bitches, i, know, thats, right]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                                                               clean_tweet_3  \\\n",
       "0                                                                                                                                                                                                  i just wanted to see what he would look like in black       \n",
       "1  hey genshin  i was playing today and i wished  rolls  i got not a single  or  star  im sure something went wrong  but im wondering if you could help me  i havent been able to get any   stars at all when pulled  for the  days  been playing this game    \n",
       "2                                                                                                                  thank you so much for this commission  its so beautiful i love it so much    for you all aeventi nation  i love this fandom so much   lt    \n",
       "3                                                                                                                                                                                                                                           comforting ocean   \n",
       "4                                                                                                                                                                                                                                     is genshin impact fun    \n",
       "5                                                                                                                                                                i read cascading   in a good way  by hubbleablubble on  and now i am a firm kaebedo enjoyer   \n",
       "6                                                                                        genshin impact  similar to  kitsune  inazuma oc  she is a deceased human spirit who lives as fox now waiting for the fated person to come by  then she met venti      \n",
       "7                                                                                                                                                                                                                i wanna be a genshin beta tester  hey  pls    \n",
       "8                                                                                                                                                                                      check out my broadcast from my playstation    genshin impact  live at   \n",
       "9                                                                                                                                                                                                                stone cold crazy bitches i know thats right   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                     clean_token  \n",
       "0                                                                                                                                                                                                                             [i, just, wanted, to, see, what, he, would, look, like, in, black]  \n",
       "1  [hey, genshin, i, was, playing, today, and, i, wished, rolls, i, got, not, a, single, or, star, im, sure, something, went, wrong, but, im, wondering, if, you, could, help, me, i, havent, been, able, to, get, any, stars, at, all, when, pulled, for, the, days, been, playing, this, game]  \n",
       "2                                                                                                                                  [thank, you, so, much, for, this, commission, its, so, beautiful, i, love, it, so, much, for, you, all, aeventi, nation, i, love, this, fandom, so, much, lt]  \n",
       "3                                                                                                                                                                                                                                                                            [comforting, ocean]  \n",
       "4                                                                                                                                                                                                                                                                     [is, genshin, impact, fun]  \n",
       "5                                                                                                                                                                                     [i, read, cascading, in, a, good, way, by, hubbleablubble, on, and, now, i, am, a, firm, kaebedo, enjoyer]  \n",
       "6                                                                                                     [genshin, impact, similar, to, kitsune, inazuma, oc, she, is, a, deceased, human, spirit, who, lives, as, fox, now, waiting, for, the, fated, person, to, come, by, then, she, met, venti]  \n",
       "7                                                                                                                                                                                                                                           [i, wan, na, be, a, genshin, beta, tester, hey, pls]  \n",
       "8                                                                                                                                                                                                                  [check, out, my, broadcast, from, my, playstation, genshin, impact, live, at]  \n",
       "9                                                                                                                                                                                                                                           [stone, cold, crazy, bitches, i, know, thats, right]  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['clean_token'] = [nltk.word_tokenize(i) for i in df.clean_tweet_3]\n",
    "\n",
    "df[['clean_tweet_3', 'clean_token']].head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5. Stopword removal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['cyberpunk2077',\n",
       " 'cyberpunk2077',\n",
       " 'cyberpunk',\n",
       " '2077',\n",
       " 'genshinimpact',\n",
       " 'genshinimpact',\n",
       " 'genshin',\n",
       " 'impact',\n",
       " 'minecraft',\n",
       " 'fortnite',\n",
       " 'dota',\n",
       " '2',\n",
       " 'stardewvalley',\n",
       " 'stardew',\n",
       " 'stardewvalley',\n",
       " 'valorant']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_keywords = []\n",
    "\n",
    "for keyword in keywords:\n",
    "    cleaned_keywords = [text.lower() for text in keyword.split()]\n",
    "    cleaned_keywords = [re.sub('[%s]' % re.escape(string.punctuation), '', text) for text in cleaned_keywords]\n",
    "    new_keywords = new_keywords + cleaned_keywords\n",
    "    \n",
    "new_keywords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>clean_token</th>\n",
       "      <th>clean_token_no_stopwords</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[i, just, wanted, to, see, what, he, would, look, like, in, black]</td>\n",
       "      <td>[wanted, see, look, like, black]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[hey, genshin, i, was, playing, today, and, i, wished, rolls, i, got, not, a, single, or, star, im, sure, something, went, wrong, but, im, wondering, if, you, could, help, me, i, havent, been, able, to, get, any, stars, at, all, when, pulled, for, the, days, been, playing, this, game]</td>\n",
       "      <td>[hey, playing, today, wished, rolls, got, single, star, sure, something, went, wrong, wondering, could, help, able, get, stars, pulled, days, playing, game]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[thank, you, so, much, for, this, commission, its, so, beautiful, i, love, it, so, much, for, you, all, aeventi, nation, i, love, this, fandom, so, much, lt]</td>\n",
       "      <td>[thank, much, commission, beautiful, love, much, aeventi, nation, love, fandom, much, lt]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[comforting, ocean]</td>\n",
       "      <td>[comforting, ocean]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[is, genshin, impact, fun]</td>\n",
       "      <td>[fun]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>[i, read, cascading, in, a, good, way, by, hubbleablubble, on, and, now, i, am, a, firm, kaebedo, enjoyer]</td>\n",
       "      <td>[read, cascading, good, way, hubbleablubble, firm, kaebedo, enjoyer]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>[genshin, impact, similar, to, kitsune, inazuma, oc, she, is, a, deceased, human, spirit, who, lives, as, fox, now, waiting, for, the, fated, person, to, come, by, then, she, met, venti]</td>\n",
       "      <td>[similar, kitsune, inazuma, oc, deceased, human, spirit, lives, fox, waiting, fated, person, come, met, venti]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>[i, wan, na, be, a, genshin, beta, tester, hey, pls]</td>\n",
       "      <td>[wan, beta, tester, hey]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>[check, out, my, broadcast, from, my, playstation, genshin, impact, live, at]</td>\n",
       "      <td>[check, broadcast, playstation, live]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>[stone, cold, crazy, bitches, i, know, thats, right]</td>\n",
       "      <td>[stone, cold, crazy, bitches, know, thats, right]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                                                                                                     clean_token  \\\n",
       "0                                                                                                                                                                                                                             [i, just, wanted, to, see, what, he, would, look, like, in, black]   \n",
       "1  [hey, genshin, i, was, playing, today, and, i, wished, rolls, i, got, not, a, single, or, star, im, sure, something, went, wrong, but, im, wondering, if, you, could, help, me, i, havent, been, able, to, get, any, stars, at, all, when, pulled, for, the, days, been, playing, this, game]   \n",
       "2                                                                                                                                  [thank, you, so, much, for, this, commission, its, so, beautiful, i, love, it, so, much, for, you, all, aeventi, nation, i, love, this, fandom, so, much, lt]   \n",
       "3                                                                                                                                                                                                                                                                            [comforting, ocean]   \n",
       "4                                                                                                                                                                                                                                                                     [is, genshin, impact, fun]   \n",
       "5                                                                                                                                                                                     [i, read, cascading, in, a, good, way, by, hubbleablubble, on, and, now, i, am, a, firm, kaebedo, enjoyer]   \n",
       "6                                                                                                     [genshin, impact, similar, to, kitsune, inazuma, oc, she, is, a, deceased, human, spirit, who, lives, as, fox, now, waiting, for, the, fated, person, to, come, by, then, she, met, venti]   \n",
       "7                                                                                                                                                                                                                                           [i, wan, na, be, a, genshin, beta, tester, hey, pls]   \n",
       "8                                                                                                                                                                                                                  [check, out, my, broadcast, from, my, playstation, genshin, impact, live, at]   \n",
       "9                                                                                                                                                                                                                                           [stone, cold, crazy, bitches, i, know, thats, right]   \n",
       "\n",
       "                                                                                                                                       clean_token_no_stopwords  \n",
       "0                                                                                                                              [wanted, see, look, like, black]  \n",
       "1  [hey, playing, today, wished, rolls, got, single, star, sure, something, went, wrong, wondering, could, help, able, get, stars, pulled, days, playing, game]  \n",
       "2                                                                     [thank, much, commission, beautiful, love, much, aeventi, nation, love, fandom, much, lt]  \n",
       "3                                                                                                                                           [comforting, ocean]  \n",
       "4                                                                                                                                                         [fun]  \n",
       "5                                                                                          [read, cascading, good, way, hubbleablubble, firm, kaebedo, enjoyer]  \n",
       "6                                                [similar, kitsune, inazuma, oc, deceased, human, spirit, lives, fox, waiting, fated, person, come, met, venti]  \n",
       "7                                                                                                                                      [wan, beta, tester, hey]  \n",
       "8                                                                                                                         [check, broadcast, playstation, live]  \n",
       "9                                                                                                             [stone, cold, crazy, bitches, know, thats, right]  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stopword_list = stopwords.words('english')\n",
    "\n",
    "# Adding non-formal stopwords\n",
    "non_formal_stopwords = ['dont', 'u', 'lol', 'cant', 'na', 'isnt', 'ive', 'would', 'gon', 'didnt', 'havent', 'pls', 'im']\n",
    "stopword_list.extend(non_formal_stopwords)\n",
    "stopword_list.extend(new_keywords)\n",
    "\n",
    "def clean_stopwords(text):\n",
    "    res = []\n",
    "    for word in text:\n",
    "        if word not in stopword_list:\n",
    "            res.append(word)\n",
    "    return res\n",
    "\n",
    "df['clean_token_no_stopwords'] = [clean_stopwords(i) for i in df.clean_token]\n",
    "\n",
    "df[['clean_token', 'clean_token_no_stopwords']].head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tokens information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "77566 words total, with a vocabulary size of 90\n",
      "Max tweet length is 279\n"
     ]
    }
   ],
   "source": [
    "all_words = [word for tokens in df['clean_tweet'] for word in tokens]\n",
    "tweet_lengths = [len(tokens) for tokens in df['clean_tweet']]\n",
    "vocab = sorted(list(set(all_words)))\n",
    "\n",
    "print('{} words total, with a vocabulary size of {}'.format(len(all_words), len(vocab)))\n",
    "print('Max tweet length is {}'.format(max(tweet_lengths)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TF-IDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Hariz\\anaconda3\\lib\\site-packages\\gensim\\similarities\\__init__.py:15: UserWarning: The gensim.similarities.levenshtein submodule is disabled, because the optional Levenshtein package <https://pypi.org/project/python-Levenshtein/> is unavailable. Install Levenhstein (e.g. `pip install python-Levenshtein`) to suppress this warning.\n",
      "  warnings.warn(msg)\n"
     ]
    }
   ],
   "source": [
    "from gensim.corpora import Dictionary\n",
    "from gensim.models.ldamodel import LdaModel\n",
    "from gensim.models.lsimodel import LsiModel\n",
    "from gensim.models import CoherenceModel\n",
    "\n",
    "text_dict = Dictionary(df.clean_token_no_stopwords)\n",
    "\n",
    "text_dict.filter_extremes(no_below = 10, no_above = .90)\n",
    "\n",
    "# Create BoW\n",
    "tweets_bow = [text_dict.doc2bow(token) for token in df['clean_token_no_stopwords']]\n",
    "\n",
    "# for doc in tweets_bow:\n",
    "#     print(doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: 'like',\n",
       " 1: 'see',\n",
       " 2: 'days',\n",
       " 3: 'game',\n",
       " 4: 'get',\n",
       " 5: 'got',\n",
       " 6: 'help',\n",
       " 7: 'hey',\n",
       " 8: 'playing',\n",
       " 9: 'today',\n",
       " 10: 'love',\n",
       " 11: 'lt',\n",
       " 12: 'much',\n",
       " 13: 'thank',\n",
       " 14: 'fun',\n",
       " 15: 'good',\n",
       " 16: 'come',\n",
       " 17: 'inazuma',\n",
       " 18: 'wan',\n",
       " 19: 'check',\n",
       " 20: 'live',\n",
       " 21: 'know',\n",
       " 22: 'right',\n",
       " 23: 'sakura',\n",
       " 24: 'via',\n",
       " 25: 'mihoyo',\n",
       " 26: 'home',\n",
       " 27: 'character',\n",
       " 28: 'xiao',\n",
       " 29: 'want',\n",
       " 30: 'ayaka',\n",
       " 31: 'lumine',\n",
       " 32: 'update',\n",
       " 33: 'need',\n",
       " 34: 'one',\n",
       " 35: 'people',\n",
       " 36: 'made',\n",
       " 37: 'gorou',\n",
       " 38: 'happy',\n",
       " 39: 'games',\n",
       " 40: 'going',\n",
       " 41: 'new',\n",
       " 42: 'also',\n",
       " 43: 'play',\n",
       " 44: 'baal',\n",
       " 45: 'aether',\n",
       " 46: 'think',\n",
       " 47: 'always',\n",
       " 48: 'art',\n",
       " 49: 'players',\n",
       " 50: 'real',\n",
       " 51: 'world',\n",
       " 52: 'twitch',\n",
       " 53: 'better',\n",
       " 54: 'looking',\n",
       " 55: 'next',\n",
       " 56: 'anyone',\n",
       " 57: 'guys',\n",
       " 58: 'please',\n",
       " 59: 'quest',\n",
       " 60: 'full',\n",
       " 61: 'favorite',\n",
       " 62: 'really',\n",
       " 63: 'free',\n",
       " 64: 'characters',\n",
       " 65: 'join',\n",
       " 66: 'looks',\n",
       " 67: 'friends',\n",
       " 68: 'let',\n",
       " 69: 'streaming',\n",
       " 70: 'thoma',\n",
       " 71: 'god',\n",
       " 72: 'already',\n",
       " 73: 'since',\n",
       " 74: 'time',\n",
       " 75: 'outfit',\n",
       " 76: 'childe',\n",
       " 77: 'collab',\n",
       " 78: 'v',\n",
       " 79: 'first',\n",
       " 80: 'use',\n",
       " 81: 'getting',\n",
       " 82: 'story',\n",
       " 83: 'team',\n",
       " 84: 'video',\n",
       " 85: 'day',\n",
       " 86: 'event',\n",
       " 87: 'make',\n",
       " 88: 'go',\n",
       " 89: 'summer',\n",
       " 90: 'ever',\n",
       " 91: 'back',\n",
       " 92: 'stream',\n",
       " 93: 'version',\n",
       " 94: 'rt',\n",
       " 95: 'still',\n",
       " 96: 'birthday',\n",
       " 97: 'finally',\n",
       " 98: 'best',\n",
       " 99: 'even',\n",
       " 100: 'last',\n",
       " 101: 'map',\n",
       " 102: 'many',\n",
       " 103: 'kokomi',\n",
       " 104: 'raiden',\n",
       " 105: 'hope',\n",
       " 106: 'link',\n",
       " 107: 'mona'}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dict(text_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from gensim.models import TfidfModel\n",
    "\n",
    "tfidf = TfidfModel(tweets_bow)\n",
    "corpus_tfidf = tfidf[tweets_bow]\n",
    "\n",
    "# for doc in corpus_tfidf:\n",
    "#     print(doc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_topics = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 756 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<gensim.models.ldamodel.LdaModel at 0x234702723a0>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time \n",
    "LdaModel(corpus_tfidf, num_topics=num_topics, id2word=text_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0::['inazuma', 'check', 'sakura', 'stream', 'live', 'mona', 'day', 'childe', 'thoma', 'raiden']\n",
      "1::['game', 'thank', 'since', 'like', 'art', 'make', 'version', 'inazuma', 'also', 'video']\n",
      "2::['like', 'get', 'summer', 'game', 'still', 'next', 'friends', 'play', 'right', 'guys']\n",
      "3::['time', 'quest', 'ayaka', 'playing', 'even', 'new', 'come', 'days', 'like', 'story']\n",
      "4::['please', 'need', 'much', 'like', 'best', 'help', 'raiden', 'join', 'see', 'first']\n",
      "5::['playing', 'finally', 'one', 'back', 'event', 'today', 'better', 'day', 'time', 'character']\n",
      "6::['new', 'baal', 'love', 'outfit', 'ayaka', 'game', 'link', 'ever', 'kokomi', 'got']\n",
      "7::['world', 'made', 'character', 'story', 'get', 'team', 'kokomi', 'always', 'join', 'use']\n",
      "8::['via', 'baal', 'full', 'love', 'day', 'live', 'aether', 'like', 'favorite', 'lumine']\n",
      "9::['good', 'happy', 'god', 'game', 'come', 'play', 'birthday', 'like', 'home', 'players']\n",
      "\n",
      "inazuma, check, sakura, stream, live, mona, day, childe, thoma, raiden\n",
      "game, thank, since, like, art, make, version, inazuma, also, video\n",
      "like, get, summer, game, still, next, friends, play, right, guys\n",
      "time, quest, ayaka, playing, even, new, come, days, like, story\n",
      "please, need, much, like, best, help, raiden, join, see, first\n",
      "playing, finally, one, back, event, today, better, day, time, character\n",
      "new, baal, love, outfit, ayaka, game, link, ever, kokomi, got\n",
      "world, made, character, story, get, team, kokomi, always, join, use\n",
      "via, baal, full, love, day, live, aether, like, favorite, lumine\n",
      "good, happy, god, game, come, play, birthday, like, home, players\n"
     ]
    }
   ],
   "source": [
    "tweets_lda_tfidf = LdaModel(corpus_tfidf, \n",
    "                            num_topics=num_topics, \n",
    "                            id2word=text_dict,\n",
    "                           random_state=2)\n",
    "\n",
    "x = tweets_lda_tfidf.show_topics(formatted=False)\n",
    "# x\n",
    "\n",
    "topics_words = [(tp[0], [wd[0] for wd in tp[1]]) for tp in x]\n",
    "\n",
    "#Below Code Prints Topics and Words\n",
    "for topic,words in topics_words:\n",
    "    print(str(topic)+ \"::\"+ str(words))\n",
    "print()\n",
    "\n",
    "#Below Code Prints Only Words \n",
    "for topic,words in topics_words:\n",
    "    print(\", \".join(words))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LSA with TF-IDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 101 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<gensim.models.lsimodel.LsiModel at 0x234702b22e0>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time \n",
    "LsiModel(corpus_tfidf, num_topics=num_topics, id2word = text_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0::['like', 'inazuma', 'game', 'get', 'got', 'new', 'first', 'play', 'one', 'love']\n",
      "1::['inazuma', 'like', 'game', 'get', 'new', 'play', 'one', 'day', 'hope', 'got']\n",
      "2::['game', 'like', 'get', 'check', 'first', 'video', 'live', 'got', 'playing', 'new']\n",
      "3::['baal', 'game', 'raiden', 'day', 'mihoyo', 'characters', 'anyone', 'quest', 'please', 'inazuma']\n",
      "4::['check', 'live', 'game', 'playing', 'time', 'ayaka', 'come', 'new', 'twitch', 'baal']\n",
      "5::['love', 'check', 'time', 'live', 'game', 'ayaka', 'like', 'get', 'first', 'new']\n",
      "6::['get', 'got', 'new', 'day', 'first', 'one', 'playing', 'time', 'happy', 'ayaka']\n",
      "7::['love', 'get', 'ayaka', 'playing', 'got', 'game', 'check', 'new', 'one', 'time']\n",
      "8::['day', 'ayaka', 'get', 'new', 'quest', 'story', 'world', 'summer', 'time', 'playing']\n",
      "9::['day', 'new', 'got', 'playing', 'get', 'first', 'summer', 'ayaka', 'love', 'happy']\n",
      "\n",
      "like, inazuma, game, get, got, new, first, play, one, love\n",
      "inazuma, like, game, get, new, play, one, day, hope, got\n",
      "game, like, get, check, first, video, live, got, playing, new\n",
      "baal, game, raiden, day, mihoyo, characters, anyone, quest, please, inazuma\n",
      "check, live, game, playing, time, ayaka, come, new, twitch, baal\n",
      "love, check, time, live, game, ayaka, like, get, first, new\n",
      "get, got, new, day, first, one, playing, time, happy, ayaka\n",
      "love, get, ayaka, playing, got, game, check, new, one, time\n",
      "day, ayaka, get, new, quest, story, world, summer, time, playing\n",
      "day, new, got, playing, get, first, summer, ayaka, love, happy\n"
     ]
    }
   ],
   "source": [
    "tweets_lsa_tfidf = LsiModel(corpus_tfidf, \n",
    "                            num_topics=num_topics, \n",
    "                            id2word = text_dict)\n",
    "\n",
    "x = tweets_lsa_tfidf.show_topics(formatted=False)\n",
    "topics_words = [(tp[0], [wd[0] for wd in tp[1]]) for tp in x]\n",
    "\n",
    "#Below Code Prints Topics and Words\n",
    "for topic,words in topics_words:\n",
    "    print(str(topic)+ \"::\"+ str(words))\n",
    "print()\n",
    "\n",
    "#Below Code Prints Only Words \n",
    "for topic,words in topics_words:\n",
    "    print(\", \".join(words))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Coherence with TF-IDF"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### LDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5208846826706315"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm = CoherenceModel(model=tweets_lda_tfidf, \n",
    "                    corpus=corpus_tfidf, \n",
    "                    texts=df.clean_token,\n",
    "                    coherence='c_v')\n",
    "\n",
    "cm.get_coherence()  # get coherence value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.6449596960147188,\n",
       " 0.5447719545358762,\n",
       " 0.49854020337788957,\n",
       " 0.4966459262127292,\n",
       " 0.4170716838917861,\n",
       " 0.4553206827219702,\n",
       " 0.553681240178838,\n",
       " 0.546527969087466,\n",
       " 0.5575026857695158,\n",
       " 0.4938247849155236]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.get_coherence_per_topic()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### LSA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4824653592143938"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm = CoherenceModel(model=tweets_lsa_tfidf, \n",
    "                    corpus=corpus_tfidf,\n",
    "                    texts=df.clean_token,\n",
    "                    coherence='c_v')\n",
    "\n",
    "cm.get_coherence()  # get coherence value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.28625394797373716,\n",
       " 0.35758720058297483,\n",
       " 0.4717624204284266,\n",
       " 0.4593804614765701,\n",
       " 0.5601275260421781,\n",
       " 0.5738477919218333,\n",
       " 0.4772940679534304,\n",
       " 0.5204308006549255,\n",
       " 0.5433969113258239,\n",
       " 0.5745724637840379]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.get_coherence_per_topic()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: pyLDAvis==2.1.2 in c:\\users\\hariz\\anaconda3\\lib\\site-packages (2.1.2)\n",
      "Requirement already satisfied: future in c:\\users\\hariz\\anaconda3\\lib\\site-packages (from pyLDAvis==2.1.2) (0.18.2)\n",
      "Requirement already satisfied: wheel>=0.23.0 in c:\\users\\hariz\\anaconda3\\lib\\site-packages (from pyLDAvis==2.1.2) (0.35.1)\n",
      "Requirement already satisfied: jinja2>=2.7.2 in c:\\users\\hariz\\anaconda3\\lib\\site-packages (from pyLDAvis==2.1.2) (2.11.2)\n",
      "Requirement already satisfied: numpy>=1.9.2 in c:\\users\\hariz\\anaconda3\\lib\\site-packages (from pyLDAvis==2.1.2) (1.20.3)\n",
      "Requirement already satisfied: joblib>=0.8.4 in c:\\users\\hariz\\anaconda3\\lib\\site-packages (from pyLDAvis==2.1.2) (0.17.0)\n",
      "Requirement already satisfied: funcy in c:\\users\\hariz\\anaconda3\\lib\\site-packages (from pyLDAvis==2.1.2) (1.16)\n",
      "Requirement already satisfied: numexpr in c:\\users\\hariz\\anaconda3\\lib\\site-packages (from pyLDAvis==2.1.2) (2.7.1)\n",
      "Requirement already satisfied: pytest in c:\\users\\hariz\\anaconda3\\lib\\site-packages (from pyLDAvis==2.1.2) (0.0.0)\n",
      "Requirement already satisfied: pandas>=0.17.0 in c:\\users\\hariz\\anaconda3\\lib\\site-packages (from pyLDAvis==2.1.2) (1.2.4)\n",
      "Requirement already satisfied: scipy>=0.18.0 in c:\\users\\hariz\\anaconda3\\lib\\site-packages (from pyLDAvis==2.1.2) (1.5.2)\n",
      "Requirement already satisfied: MarkupSafe>=0.23 in c:\\users\\hariz\\anaconda3\\lib\\site-packages (from jinja2>=2.7.2->pyLDAvis==2.1.2) (1.1.1)\n",
      "Requirement already satisfied: attrs>=17.4.0 in c:\\users\\hariz\\anaconda3\\lib\\site-packages (from pytest->pyLDAvis==2.1.2) (20.3.0)\n",
      "Requirement already satisfied: iniconfig in c:\\users\\hariz\\anaconda3\\lib\\site-packages (from pytest->pyLDAvis==2.1.2) (1.1.1)\n",
      "Requirement already satisfied: packaging in c:\\users\\hariz\\anaconda3\\lib\\site-packages (from pytest->pyLDAvis==2.1.2) (20.4)\n",
      "Requirement already satisfied: pluggy<1.0,>=0.12 in c:\\users\\hariz\\anaconda3\\lib\\site-packages (from pytest->pyLDAvis==2.1.2) (0.13.1)\n",
      "Requirement already satisfied: py>=1.8.2 in c:\\users\\hariz\\anaconda3\\lib\\site-packages (from pytest->pyLDAvis==2.1.2) (1.9.0)\n",
      "Requirement already satisfied: toml in c:\\users\\hariz\\anaconda3\\lib\\site-packages (from pytest->pyLDAvis==2.1.2) (0.10.1)\n",
      "Requirement already satisfied: atomicwrites>=1.0 in c:\\users\\hariz\\anaconda3\\lib\\site-packages (from pytest->pyLDAvis==2.1.2) (1.4.0)\n",
      "Requirement already satisfied: colorama in c:\\users\\hariz\\anaconda3\\lib\\site-packages (from pytest->pyLDAvis==2.1.2) (0.4.4)\n",
      "Requirement already satisfied: pytz>=2017.3 in c:\\users\\hariz\\anaconda3\\lib\\site-packages (from pandas>=0.17.0->pyLDAvis==2.1.2) (2020.1)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in c:\\users\\hariz\\anaconda3\\lib\\site-packages (from pandas>=0.17.0->pyLDAvis==2.1.2) (2.8.1)\n",
      "Requirement already satisfied: six in c:\\users\\hariz\\anaconda3\\lib\\site-packages (from packaging->pytest->pyLDAvis==2.1.2) (1.15.0)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in c:\\users\\hariz\\anaconda3\\lib\\site-packages (from packaging->pytest->pyLDAvis==2.1.2) (2.4.7)\n"
     ]
    }
   ],
   "source": [
    "!pip install pyLDAvis==2.1.2\n",
    "\n",
    "# import pyLDAvis\n",
    "# import pyLDAvis.gensim\n",
    "\n",
    "# pyLDAvis.enable_notebook()\n",
    "\n",
    "# vis = pyLDAvis.gensim.prepare(tweets_lda_tfidf, corpus_tfidf, dictionary=text_dict)\n",
    "# vis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Hariz\\anaconda3\\lib\\site-packages\\sklearn\\decomposition\\_lda.py:28: DeprecationWarning: `np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.\n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  EPS = np.finfo(np.float).eps\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "<link rel=\"stylesheet\" type=\"text/css\" href=\"https://cdn.rawgit.com/bmabey/pyLDAvis/files/ldavis.v1.0.0.css\">\n",
       "\n",
       "\n",
       "<div id=\"ldavis_el730424243454494081166134438\"></div>\n",
       "<script type=\"text/javascript\">\n",
       "\n",
       "var ldavis_el730424243454494081166134438_data = {\"mdsDat\": {\"x\": [0.162010433229195, 0.04581089344463234, -0.06759386969565286, -0.0335704992101443, 0.025326808313912506, 0.03772972659041965, -0.1544533749744783, 0.03989565473041673, -0.039447107277180794, -0.015708665151119853], \"y\": [-0.008102619659023185, -0.01831121068633171, 0.03719239141232467, -0.017970953119663617, 0.09922879087483992, -0.07130803931500924, -0.020046116056521814, 0.05727322755756093, 0.0864427569999841, -0.1443982280081591], \"topics\": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], \"cluster\": [1, 1, 1, 1, 1, 1, 1, 1, 1, 1], \"Freq\": [10.817909502218638, 9.593573230619198, 12.573685045356484, 9.04829924426555, 10.167086251171838, 9.736959641553677, 10.338047267414941, 8.454159337441594, 8.877448892398641, 10.392831587559444]}, \"tinfo\": {\"Term\": [\"inazuma\", \"baal\", \"good\", \"new\", \"quest\", \"time\", \"check\", \"happy\", \"via\", \"please\", \"playing\", \"get\", \"finally\", \"ayaka\", \"thank\", \"since\", \"full\", \"story\", \"god\", \"world\", \"love\", \"sakura\", \"summer\", \"raiden\", \"aether\", \"join\", \"live\", \"still\", \"day\", \"need\", \"inazuma\", \"sakura\", \"check\", \"stream\", \"thoma\", \"mona\", \"games\", \"raiden\", \"childe\", \"let\", \"live\", \"know\", \"mihoyo\", \"team\", \"video\", \"via\", \"see\", \"gorou\", \"anyone\", \"day\", \"map\", \"twitch\", \"going\", \"event\", \"come\", \"use\", \"players\", \"rt\", \"want\", \"today\", \"summer\", \"one\", \"like\", \"fun\", \"since\", \"thank\", \"version\", \"art\", \"collab\", \"make\", \"video\", \"think\", \"people\", \"already\", \"game\", \"also\", \"finally\", \"mona\", \"map\", \"raiden\", \"really\", \"want\", \"one\", \"check\", \"come\", \"fun\", \"ayaka\", \"looking\", \"like\", \"twitch\", \"character\", \"many\", \"hey\", \"made\", \"inazuma\", \"gorou\", \"got\", \"time\", \"love\", \"still\", \"friends\", \"summer\", \"next\", \"get\", \"right\", \"guys\", \"collab\", \"home\", \"play\", \"free\", \"hey\", \"like\", \"favorite\", \"fun\", \"want\", \"version\", \"real\", \"going\", \"days\", \"got\", \"game\", \"v\", \"also\", \"go\", \"hope\", \"see\", \"character\", \"update\", \"link\", \"much\", \"playing\", \"love\", \"quest\", \"even\", \"time\", \"many\", \"ayaka\", \"days\", \"playing\", \"story\", \"looking\", \"go\", \"twitch\", \"come\", \"map\", \"childe\", \"make\", \"mona\", \"really\", \"already\", \"check\", \"world\", \"new\", \"streaming\", \"let\", \"hope\", \"ever\", \"guys\", \"mihoyo\", \"xiao\", \"first\", \"baal\", \"home\", \"like\", \"update\", \"get\", \"got\", \"inazuma\", \"please\", \"help\", \"need\", \"much\", \"best\", \"join\", \"raiden\", \"event\", \"mihoyo\", \"see\", \"lt\", \"think\", \"people\", \"last\", \"characters\", \"wan\", \"sakura\", \"games\", \"first\", \"quest\", \"days\", \"play\", \"like\", \"better\", \"next\", \"know\", \"character\", \"thoma\", \"going\", \"ever\", \"get\", \"love\", \"live\", \"game\", \"world\", \"baal\", \"finally\", \"back\", \"streaming\", \"event\", \"v\", \"better\", \"playing\", \"today\", \"getting\", \"aether\", \"looking\", \"stream\", \"one\", \"update\", \"character\", \"people\", \"also\", \"mihoyo\", \"lt\", \"know\", \"going\", \"xiao\", \"day\", \"twitch\", \"many\", \"games\", \"time\", \"thoma\", \"happy\", \"favorite\", \"first\", \"come\", \"game\", \"childe\", \"like\", \"outfit\", \"ever\", \"new\", \"link\", \"baal\", \"kokomi\", \"better\", \"ayaka\", \"love\", \"always\", \"free\", \"hey\", \"real\", \"really\", \"wan\", \"childe\", \"got\", \"birthday\", \"rt\", \"know\", \"first\", \"anyone\", \"still\", \"friends\", \"twitch\", \"players\", \"day\", \"gorou\", \"game\", \"time\", \"go\", \"get\", \"world\", \"team\", \"made\", \"story\", \"use\", \"anyone\", \"always\", \"v\", \"join\", \"character\", \"kokomi\", \"last\", \"fun\", \"getting\", \"let\", \"gorou\", \"today\", \"wan\", \"characters\", \"best\", \"full\", \"get\", \"real\", \"hope\", \"guys\", \"update\", \"already\", \"via\", \"make\", \"really\", \"check\", \"inazuma\", \"like\", \"full\", \"via\", \"aether\", \"favorite\", \"lumine\", \"getting\", \"rt\", \"characters\", \"live\", \"day\", \"baal\", \"back\", \"love\", \"last\", \"guys\", \"go\", \"hope\", \"help\", \"going\", \"right\", \"got\", \"even\", \"update\", \"use\", \"map\", \"want\", \"days\", \"first\", \"kokomi\", \"streaming\", \"get\", \"like\", \"one\", \"game\", \"happy\", \"god\", \"good\", \"birthday\", \"players\", \"looks\", \"xiao\", \"lumine\", \"come\", \"home\", \"right\", \"play\", \"gorou\", \"story\", \"finally\", \"always\", \"today\", \"streaming\", \"playing\", \"lt\", \"rt\", \"one\", \"fun\", \"game\", \"want\", \"aether\", \"wan\", \"looking\", \"real\", \"live\", \"quest\", \"got\", \"like\", \"new\", \"also\", \"first\"], \"Freq\": [23.0, 17.0, 11.0, 17.0, 9.0, 15.0, 14.0, 8.0, 10.0, 8.0, 14.0, 19.0, 8.0, 14.0, 7.0, 6.0, 8.0, 8.0, 7.0, 10.0, 17.0, 7.0, 11.0, 8.0, 8.0, 8.0, 13.0, 8.0, 14.0, 9.0, 14.612153870090207, 4.3091014096991165, 6.680638918948421, 4.0508152428721225, 2.703957477505223, 3.057725470437093, 2.3126572335553663, 2.5627634112656437, 2.9242475270435646, 1.9748877508535092, 3.646836482201682, 2.180391320997943, 2.4345277079428462, 1.7483073100368627, 1.564812583199855, 2.4014473434960277, 2.3525075438985183, 2.16919323305618, 1.4598327347951132, 3.047920253335492, 1.4507094307920487, 1.11770454158514, 1.3854076389165118, 1.4697318699761952, 1.8667664168917617, 1.1430295623175013, 1.0460125042203827, 0.9541126034244821, 1.5973359053129557, 1.31928114878097, 1.5571851843182802, 1.8970790600572527, 1.6696216618235085, 1.1926007118940067, 4.491034882624761, 4.89652466782045, 3.432033122838161, 3.718524930197192, 2.247741607696459, 3.4332717372200974, 2.295781157453674, 2.0616302082656324, 1.8492499561565752, 1.6056499660445245, 5.644557133360785, 2.3214632287891566, 1.5792496938986293, 1.6624837155156262, 1.2906333658176972, 1.3438675567506435, 1.3330441577313163, 1.8007894504793303, 2.1783008018530503, 2.1308918399278047, 1.7133169160167994, 1.2223816335411612, 2.006475770218249, 0.961096700515435, 3.900732099777488, 0.7601236205975193, 1.3859349471511166, 0.7190906649768604, 0.7277900603036657, 0.9688267597615376, 2.5329504301522823, 1.0844094295297273, 1.5817455357201546, 1.3719334155403724, 1.4046573330155399, 5.0316163204248445, 4.119416997769709, 6.466547395616007, 4.463989731046116, 8.004617355994203, 3.8125884244695025, 3.6006855239562694, 2.1396316479803983, 3.3329048851007386, 3.9498908333066356, 2.3475086427544642, 1.908513043684643, 8.983828616454499, 2.0767848396102435, 2.1853978033867976, 2.833479672075523, 1.7093804058706057, 1.3879687048618359, 1.8070513728086417, 1.8159395121164978, 3.4220006580295315, 5.693808079463088, 1.319743549593901, 2.1049390742861074, 1.9255384999973046, 1.6249488731660644, 1.7633315593174654, 1.9148516644632856, 1.8151947269471607, 1.1586728221202864, 1.3115583051398196, 1.5771474644573853, 1.568581510958928, 5.762501464095404, 3.067350991667897, 6.698655339761186, 1.9191904357053136, 4.373135597884128, 2.3676366492130767, 3.8041990644117667, 2.3219963012705787, 1.651977097761014, 2.3012000220835294, 1.3541769439167692, 2.4829586652060374, 1.4262540905355718, 1.8551946822520715, 1.779752268606104, 1.52626456713407, 1.4778655797595135, 1.1582546724599927, 2.286414161585791, 1.603941625899456, 2.526024614017943, 0.9323167477702804, 0.9997761834261577, 1.2456773161419108, 0.9680026223697528, 1.237987810431827, 1.2093868531806975, 0.9999023499078643, 1.9100413708793804, 2.210177510875082, 1.242909367166227, 2.3619700630016705, 1.2329487761231044, 1.4954606172118488, 1.2728341415209132, 1.231137582245169, 5.537087340521092, 3.7889825533357477, 4.935398933041822, 4.437794416529596, 3.9794059145745524, 3.402362358727495, 3.466897487399292, 2.66119546420804, 2.6532017477389185, 3.009888792681435, 2.3606196504104253, 2.055730071026517, 1.9545568261419008, 2.2218026738435195, 1.8896174189308619, 1.4762516784374504, 1.7256134310938547, 1.4668609195938116, 2.9048210045104588, 1.5840740002041633, 1.2670858525393087, 1.8824495943026263, 4.281565169255043, 1.0449115825842477, 1.0806304582643944, 1.0391419958313843, 1.4462615432309434, 0.8546814939837201, 0.9533997778619293, 0.8546652550971019, 2.1726235544082124, 1.8919904881286793, 1.402672685420201, 2.015964126075886, 1.176119893582517, 1.3404459692749626, 4.039687745551156, 3.2123788414488357, 2.283996024739541, 3.1380650524679456, 2.1077925646250284, 2.72437129135094, 4.28203615095831, 2.8314497591761665, 1.7934188829878814, 2.1962831872851636, 1.8251552290368716, 2.0768798007517137, 3.3554470489041, 2.2447754997025617, 2.2922173441224873, 1.4876374761900093, 2.056706426323952, 1.7110784117461926, 1.5567472982330106, 1.4962787919512366, 1.4302184756425882, 1.3584831514783828, 2.578719943971645, 1.0804072895043202, 1.0498688204497237, 1.1561668315792801, 2.4716342249595145, 1.0751729820738611, 1.171552896085044, 0.9546803909392165, 1.910549213247781, 1.359665858226491, 1.7594293260189717, 1.2124386180302786, 1.6044374958043819, 4.562969459632392, 3.599667807523073, 8.515379433415363, 3.6709694286255767, 8.165968020400198, 2.999711422103279, 2.457291367779489, 4.52025437267376, 5.22340515191662, 2.131897035825367, 2.0184309960926488, 1.636212068817994, 1.4234621300160544, 1.906329876348359, 1.4331098674940066, 1.915293330427036, 2.9298231425236008, 1.2982288470225511, 1.2423731888913692, 1.5131226036748031, 2.5428576892746397, 1.1853870196223768, 1.4076758994002965, 1.1347964436047886, 1.0323023386254222, 1.0453826361306786, 2.2098826520797545, 1.440801091341522, 3.790568592581199, 2.328702614272649, 1.3314065125094205, 1.2820259683503583, 4.510584187833615, 2.7955441864613033, 3.3330622120592515, 2.8975990092221178, 2.2986890693971263, 2.155628378547375, 2.347893327663983, 1.7559600467319851, 2.3337261157786138, 3.227504231407405, 2.4824413315840514, 2.285282333047126, 1.9091246243026356, 1.4244777076357664, 1.4118089142126797, 1.7999213315093552, 1.7634639023904566, 1.1413465356230463, 1.4236953622157205, 1.3510030134864202, 1.245368985281037, 2.89237906538392, 0.8997187715756715, 1.3107961417524538, 1.3238933860980817, 1.4893886851245162, 0.8922788687617864, 1.3181324111355386, 1.2036327127481625, 0.8751884651316947, 1.3211608102800345, 1.6463985517366422, 1.358953379290386, 4.346773646292374, 4.850885196727492, 3.389109128058082, 2.834926476830295, 2.7260033389160387, 2.18143845045048, 2.0567551393676564, 2.5002520978795495, 3.6919884519527324, 3.8200632587717247, 4.470102725159627, 1.9414773398519178, 4.106174738250045, 1.9692631090128652, 1.9728376279107478, 2.0794298301589946, 1.7074858811049591, 1.2697114409834391, 1.3205802235217912, 1.5182235781122446, 2.3069070909553346, 1.0717920728511046, 1.6418063145553328, 1.0314911381378318, 1.0316047447201504, 1.5934744557281129, 1.0316195327132547, 1.7248339367801444, 1.0316738133467667, 0.6686195358650618, 2.065419163562855, 2.999871056009594, 1.202781071181777, 1.2253975668708401, 6.02825818187738, 4.674988942611292, 7.5290091326067365, 3.0511821127262904, 2.6703950657291653, 2.615800603324067, 2.618948999133243, 2.6074972134710146, 3.191918745653365, 2.684451640756354, 2.536275807613847, 3.127865449649228, 2.315143060741874, 2.0181903518877777, 1.8186085759670743, 1.691667509577768, 2.0966550233783856, 1.255879466469042, 2.61135704326657, 1.501760665584059, 1.1425486230939985, 2.3123272649040003, 1.2542391762860172, 3.6421265148908297, 1.525420784730562, 1.096934767262454, 0.807904424748665, 0.8670009526256858, 0.7237796981126273, 1.563278930637303, 1.134247575494265, 1.6135033593717012, 2.827333166701053, 1.6646014485940968, 1.206806306159496, 1.2243753446313181], \"Total\": [23.0, 17.0, 11.0, 17.0, 9.0, 15.0, 14.0, 8.0, 10.0, 8.0, 14.0, 19.0, 8.0, 14.0, 7.0, 6.0, 8.0, 8.0, 7.0, 10.0, 17.0, 7.0, 11.0, 8.0, 8.0, 8.0, 13.0, 8.0, 14.0, 9.0, 23.976732096239914, 7.717500445418899, 14.0985949416667, 8.582991339727192, 7.1332466229942115, 8.748415983061673, 7.256879664726282, 8.449075889635735, 9.93710688672879, 7.181914559053274, 13.540883085891252, 8.259630323196935, 9.269944313970626, 7.232717393358451, 6.571611116461869, 10.564017317622351, 10.599956821370863, 9.778033245173194, 6.964091396399735, 14.748463681226498, 7.301404487658755, 6.274173816115245, 7.986120690073386, 9.04286745101746, 11.636493367672621, 7.235600289168596, 6.9105332832738995, 6.716394835220124, 11.633932770357752, 9.639848862324492, 11.401296924133474, 14.115354155222166, 30.544435347573874, 8.761432124176334, 6.858228639796706, 7.832337744429806, 7.208173029689513, 9.338688093673321, 6.1304517026169725, 9.781078150436231, 6.571611116461869, 8.039832652894688, 7.6484811720316745, 6.966670564216999, 25.900443546257986, 10.986021529220826, 8.097199468834685, 8.748415983061673, 7.301404487658755, 8.449075889635735, 8.561453055503303, 11.633932770357752, 14.115354155222166, 14.0985949416667, 11.636493367672621, 8.761432124176334, 14.731543440295692, 7.2328725177626465, 30.544435347573874, 6.274173816115245, 11.61890412281585, 6.170549582352218, 6.418660528889638, 8.630669306415726, 23.976732096239914, 9.778033245173194, 15.318379533762593, 15.956041260705835, 17.159201045346542, 8.322362775443688, 6.84451337927289, 11.401296924133474, 8.351341181212137, 19.558248506862657, 9.770758483670688, 9.36971874599892, 6.1304517026169725, 9.97933468779571, 12.511905891042712, 7.553092725333408, 6.418660528889638, 30.544435347573874, 7.3445680348035545, 8.761432124176334, 11.633932770357752, 7.208173029689513, 6.128211521771355, 7.986120690073386, 8.034482081841366, 15.318379533762593, 25.900443546257986, 6.126651476909231, 10.986021529220826, 10.311964793237223, 9.163939780420968, 10.599956821370863, 11.61890412281585, 11.203070298591898, 7.420942008439898, 8.70409132972195, 14.169199076656696, 17.159201045346542, 9.98976702699138, 7.2140899480553, 15.956041260705835, 6.170549582352218, 14.731543440295692, 8.034482081841366, 14.169199076656696, 8.77509504854822, 7.2328725177626465, 10.311964793237223, 6.274173816115245, 11.636493367672621, 7.301404487658755, 9.93710688672879, 9.781078150436231, 8.748415983061673, 8.561453055503303, 6.966670564216999, 14.0985949416667, 10.69792135120574, 17.009355140155566, 6.317041825436303, 7.181914559053274, 9.163939780420968, 7.161591934401106, 9.36971874599892, 9.269944313970626, 7.719354755227819, 14.865539629089612, 17.371602878975157, 9.97933468779571, 30.544435347573874, 11.203070298591898, 19.558248506862657, 15.318379533762593, 23.976732096239914, 8.39923824046573, 6.86007736480153, 9.477082134040282, 8.70409132972195, 7.874318773453494, 8.258642067262153, 8.449075889635735, 9.04286745101746, 9.269944313970626, 10.599956821370863, 8.573993697992549, 8.039832652894688, 7.6484811720316745, 8.962996265075121, 8.23919732776256, 6.497632128084591, 7.717500445418899, 7.256879664726282, 14.865539629089612, 9.98976702699138, 8.034482081841366, 12.511905891042712, 30.544435347573874, 7.9799588689219245, 8.351341181212137, 8.259630323196935, 11.61890412281585, 7.1332466229942115, 7.986120690073386, 7.161591934401106, 19.558248506862657, 17.159201045346542, 13.540883085891252, 25.900443546257986, 10.69792135120574, 17.371602878975157, 8.097199468834685, 7.731258562190888, 6.317041825436303, 9.04286745101746, 6.126651476909231, 7.9799588689219245, 14.169199076656696, 9.639848862324492, 6.756720871654773, 8.446605824513238, 7.2328725177626465, 8.582991339727192, 14.115354155222166, 11.203070298591898, 11.61890412281585, 7.6484811720316745, 10.986021529220826, 9.269944313970626, 8.573993697992549, 8.259630323196935, 7.986120690073386, 7.719354755227819, 14.748463681226498, 6.274173816115245, 6.170549582352218, 7.256879664726282, 15.956041260705835, 7.1332466229942115, 8.938552319998415, 7.3445680348035545, 14.865539629089612, 11.636493367672621, 25.900443546257986, 9.93710688672879, 30.544435347573874, 8.472553058187266, 7.161591934401106, 17.009355140155566, 7.420942008439898, 17.371602878975157, 9.329921369764929, 7.9799588689219245, 14.731543440295692, 17.159201045346542, 7.770546345760238, 7.553092725333408, 6.418660528889638, 6.128211521771355, 8.561453055503303, 6.497632128084591, 9.93710688672879, 15.318379533762593, 6.827196940302032, 6.716394835220124, 8.259630323196935, 14.865539629089612, 6.964091396399735, 8.322362775443688, 6.84451337927289, 6.274173816115245, 6.9105332832738995, 14.748463681226498, 9.778033245173194, 25.900443546257986, 15.956041260705835, 10.311964793237223, 19.558248506862657, 10.69792135120574, 7.232717393358451, 8.630669306415726, 8.77509504854822, 7.235600289168596, 6.964091396399735, 7.770546345760238, 6.126651476909231, 8.258642067262153, 11.61890412281585, 9.329921369764929, 8.962996265075121, 8.761432124176334, 6.756720871654773, 7.181914559053274, 9.778033245173194, 9.639848862324492, 6.497632128084591, 8.23919732776256, 7.874318773453494, 8.151579243010994, 19.558248506862657, 6.128211521771355, 9.163939780420968, 9.36971874599892, 11.203070298591898, 6.966670564216999, 10.564017317622351, 9.781078150436231, 8.561453055503303, 14.0985949416667, 23.976732096239914, 30.544435347573874, 8.151579243010994, 10.564017317622351, 8.446605824513238, 7.3445680348035545, 8.198443225063766, 6.756720871654773, 6.716394835220124, 8.23919732776256, 13.540883085891252, 14.748463681226498, 17.371602878975157, 7.731258562190888, 17.159201045346542, 8.962996265075121, 9.36971874599892, 10.311964793237223, 9.163939780420968, 6.86007736480153, 7.986120690073386, 9.770758483670688, 15.318379533762593, 7.2140899480553, 11.203070298591898, 7.235600289168596, 7.301404487658755, 11.633932770357752, 8.034482081841366, 14.865539629089612, 9.329921369764929, 6.317041825436303, 19.558248506862657, 30.544435347573874, 14.115354155222166, 25.900443546257986, 8.938552319998415, 7.047604886599236, 11.634630598524351, 6.827196940302032, 6.9105332832738995, 7.320196610441036, 7.719354755227819, 8.198443225063766, 11.636493367672621, 9.97933468779571, 9.770758483670688, 12.511905891042712, 9.778033245173194, 8.77509504854822, 8.097199468834685, 7.770546345760238, 9.639848862324492, 6.317041825436303, 14.169199076656696, 8.573993697992549, 6.716394835220124, 14.115354155222166, 8.761432124176334, 25.900443546257986, 11.633932770357752, 8.446605824513238, 6.497632128084591, 7.2328725177626465, 6.128211521771355, 13.540883085891252, 9.98976702699138, 15.318379533762593, 30.544435347573874, 17.009355140155566, 10.986021529220826, 14.865539629089612], \"Category\": [\"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\"], \"logprob\": [30.0, 29.0, 28.0, 27.0, 26.0, 25.0, 24.0, 23.0, 22.0, 21.0, 20.0, 19.0, 18.0, 17.0, 16.0, 15.0, 14.0, 13.0, 12.0, 11.0, 10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, -2.052799940109253, -3.273900032043457, -2.835400104522705, -3.335700035095215, -3.7399001121520996, -3.6168999671936035, -3.896199941635132, -3.7934999465942383, -3.661600112915039, -4.054100036621094, -3.4407999515533447, -3.9551000595092773, -3.844899892807007, -4.176000118255615, -4.286799907684326, -3.8585000038146973, -3.8791000843048096, -3.9602999687194824, -4.356299877166748, -3.6201999187469482, -4.362599849700928, -4.623300075531006, -4.408599853515625, -4.3495001792907715, -4.110400199890137, -4.600900173187256, -4.689599990844727, -4.781599998474121, -4.266300201416016, -4.457499980926514, -4.2916998863220215, -4.094299793243408, -4.2220001220703125, -4.558499813079834, -3.1124000549316406, -3.0260000228881836, -3.3812999725341797, -3.3011999130249023, -3.8046000003814697, -3.38100004196167, -3.783400058746338, -3.8910000324249268, -3.9997000694274902, -4.140999794006348, -2.8838000297546387, -3.7723000049591064, -4.157599925994873, -4.106200218200684, -4.359399795532227, -4.318900108337402, -4.327000141143799, -4.026299953460693, -3.8359999656677246, -3.8580000400543213, -4.076099872589111, -4.413700103759766, -3.918100118637085, -4.654200077056885, -3.2532999515533447, -4.888800144195557, -4.288099765777588, -4.944300174713135, -4.932199954986572, -4.646200180053711, -3.6851000785827637, -4.5335001945495605, -4.156000137329102, -4.298299789428711, -4.274700164794922, -3.2692999839782715, -3.4693000316619873, -3.018399953842163, -3.3889999389648438, -2.805000066757202, -3.5467000007629395, -3.6038999557495117, -4.1244001388549805, -3.6812000274658203, -3.5113000869750977, -4.031700134277344, -4.238699913024902, -2.6895999908447266, -4.154200077056885, -4.1031999588012695, -3.8434998989105225, -4.348899841308594, -4.557199954986572, -4.293300151824951, -4.288400173187256, -3.6547999382019043, -3.1456000804901123, -4.607600212097168, -4.140699863433838, -4.229800224304199, -4.399499893188477, -4.317800045013428, -4.235400199890137, -4.28879976272583, -4.73769998550415, -4.613800048828125, -4.4293999671936035, -4.434800148010254, -2.8046000003814697, -3.4351999759674072, -2.654099941253662, -3.904099941253662, -3.0804998874664307, -3.6940999031066895, -3.219899892807007, -3.713599920272827, -4.053999900817871, -3.722599983215332, -4.252799987792969, -3.6465001106262207, -4.200900077819824, -3.937999963760376, -3.9795000553131104, -4.133200168609619, -4.16540002822876, -4.40910005569458, -3.7290000915527344, -4.083499908447266, -3.629300117492676, -4.626100063323975, -4.55620002746582, -4.336299896240234, -4.588500022888184, -4.34250020980835, -4.365900039672852, -4.556099891662598, -3.908900022506714, -3.7629001140594482, -4.338500022888184, -3.696500062942505, -4.34660005569458, -4.153600215911865, -4.314700126647949, -4.3480000495910645, -2.9611001014709473, -3.3405001163482666, -3.0761001110076904, -3.1823999881744385, -3.2913999557495117, -3.4481000900268555, -3.42930006980896, -3.6937999725341797, -3.6967999935150146, -3.57069993019104, -3.8136000633239746, -3.951900005340576, -4.002399921417236, -3.8742001056671143, -4.036200046539307, -4.283100128173828, -4.126999855041504, -4.289400100708008, -3.6061999797821045, -4.212600231170654, -4.435800075531006, -4.039999961853027, -3.2181999683380127, -4.628600120544434, -4.59499979019165, -4.634200096130371, -4.303599834442139, -4.829599857330322, -4.720300197601318, -4.829599857330322, -3.8966000080108643, -4.034900188446045, -4.334199905395508, -3.9714999198913574, -4.510300159454346, -4.3796000480651855, -3.2332000732421875, -3.4623000621795654, -3.8034000396728516, -3.4856998920440674, -3.883699893951416, -3.6270999908447266, -3.1749000549316406, -3.5885000228881836, -4.045199871063232, -3.842600107192993, -4.027699947357178, -3.8984999656677246, -3.418800115585327, -3.82069993019104, -3.799799919128418, -4.232100009918213, -3.9082000255584717, -4.092199802398682, -4.186699867248535, -4.226399898529053, -4.271500110626221, -4.322999954223633, -3.681999921798706, -4.552000045776367, -4.580699920654297, -4.4842000007629395, -3.7244999408721924, -4.5569000244140625, -4.4710001945495605, -4.6757001876831055, -3.9818999767303467, -4.3221001625061035, -4.064300060272217, -4.436699867248535, -4.156599998474121, -3.171299934387207, -3.408400058746338, -2.5473999977111816, -3.3887999057769775, -2.5892999172210693, -3.5906999111175537, -3.7901999950408936, -3.1807000637054443, -3.036099910736084, -3.9321999549865723, -3.9869000911712646, -4.196899890899658, -4.336100101470947, -4.044099807739258, -4.329400062561035, -4.039400100708008, -3.614300012588501, -4.428199768066406, -4.4721999168396, -4.275100231170654, -3.75600004196167, -4.519199848175049, -4.347300052642822, -4.56279993057251, -4.657400131225586, -4.644899845123291, -3.8963000774383545, -4.323999881744385, -3.3566999435424805, -3.843899965286255, -4.4029998779296875, -4.440800189971924, -2.981600046157837, -3.4600000381469727, -3.2841999530792236, -3.4242000579833984, -3.6556999683380127, -3.7200000286102295, -3.634500026702881, -3.925100088119507, -3.6405999660491943, -3.3164000511169434, -3.5787999629974365, -3.661600112915039, -3.841399908065796, -4.134300231933594, -4.143199920654297, -3.9003000259399414, -3.920799970626831, -4.355899810791016, -4.134799957275391, -4.18720006942749, -4.268599987030029, -3.4260001182556152, -4.593699932098389, -4.217400074005127, -4.207499980926514, -4.089700222015381, -4.6020002365112305, -4.211900234222412, -4.302700042724609, -4.621399879455566, -4.20959997177124, -3.989500045776367, -4.181399822235107, -3.067500114440918, -2.9577999114990234, -3.3164000511169434, -3.4948999881744385, -3.53410005569458, -3.7569000720977783, -3.8157999515533447, -3.620500087738037, -3.230799913406372, -3.196700096130371, -3.0394999980926514, -3.873500108718872, -3.1243999004364014, -3.859299898147583, -3.8573999404907227, -3.804800033569336, -4.0019001960754395, -4.298099994659424, -4.258900165557861, -4.1194000244140625, -3.7009999752044678, -4.467599868774414, -4.041100025177002, -4.505899906158447, -4.505799770355225, -4.071000099182129, -4.505799770355225, -3.99180006980896, -4.50570011138916, -4.939499855041504, -3.8115999698638916, -3.4384000301361084, -4.35230016708374, -4.333700180053711, -2.898099899291992, -3.1522998809814453, -2.675800085067749, -3.5789999961853027, -3.7123000621795654, -3.7330000400543213, -3.731800079345703, -3.7360999584198, -3.533900022506714, -3.7070000171661377, -3.7637999057769775, -3.5541999340057373, -3.85509991645813, -3.992300033569336, -4.096499919891357, -4.168799877166748, -3.954200029373169, -4.466700077056885, -3.7346999645233154, -4.287899971008301, -4.561299800872803, -3.856300115585327, -4.4679999351501465, -3.4019999504089355, -4.272299766540527, -4.6020002365112305, -4.907800197601318, -4.837200164794922, -5.0177998542785645, -4.247700214385986, -4.568600177764893, -4.216100215911865, -3.6552000045776367, -4.184899806976318, -4.506499767303467, -4.492099761962891], \"loglift\": [30.0, 29.0, 28.0, 27.0, 26.0, 25.0, 24.0, 23.0, 22.0, 21.0, 20.0, 19.0, 18.0, 17.0, 16.0, 15.0, 14.0, 13.0, 12.0, 11.0, 10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, 1.7287, 1.6412, 1.4771, 1.4731, 1.2539, 1.1728, 1.0804, 1.031, 1.0007, 0.9329, 0.9121, 0.8921, 0.8869, 0.804, 0.789, 0.7426, 0.7186, 0.7182, 0.6615, 0.6473, 0.608, 0.4988, 0.4723, 0.4071, 0.394, 0.3786, 0.3359, 0.2724, 0.2384, 0.2351, 0.2331, 0.217, -0.6826, 0.2297, 1.9207, 1.8743, 1.602, 1.4232, 1.3407, 1.2971, 1.2924, 0.9832, 0.9243, 0.8765, 0.8205, 0.7897, 0.7095, 0.6835, 0.6111, 0.5056, 0.4843, 0.4784, 0.4754, 0.4545, 0.4284, 0.3745, 0.3505, 0.3258, 0.2861, 0.2334, 0.2178, 0.1945, 0.1671, 0.1571, 0.0964, 0.145, 0.0736, -0.1095, -0.1587, 1.5704, 1.5658, 1.5065, 1.4472, 1.1802, 1.1325, 1.1172, 1.0209, 0.9769, 0.9206, 0.905, 0.8607, 0.8498, 0.8104, 0.685, 0.6611, 0.6345, 0.5885, 0.5876, 0.5864, 0.5747, 0.5587, 0.5384, 0.4212, 0.3955, 0.3438, 0.2799, 0.2706, 0.2536, 0.2165, 0.181, -0.1219, -0.3188, 1.8524, 1.5474, 1.5347, 1.2347, 1.1881, 1.1807, 1.0876, 1.0731, 0.9259, 0.9027, 0.8693, 0.8579, 0.7696, 0.7243, 0.6986, 0.6565, 0.6459, 0.6084, 0.5835, 0.505, 0.4955, 0.4893, 0.4308, 0.407, 0.4013, 0.3786, 0.3659, 0.3588, 0.3507, 0.3408, 0.3195, -0.1571, 0.1958, -0.1684, -0.0852, -0.5666, 1.8693, 1.6924, 1.6336, 1.6124, 1.6035, 1.3992, 1.3952, 1.0628, 1.035, 1.0271, 0.9962, 0.9222, 0.9217, 0.8912, 0.8135, 0.8041, 0.7881, 0.6872, 0.6533, 0.4445, 0.439, 0.3919, 0.3212, 0.253, 0.2411, 0.213, 0.2024, 0.1642, 0.1606, 0.1602, 0.0886, 0.0811, 0.0187, -0.2671, 0.0782, -0.2758, 1.6339, 1.451, 1.3119, 1.2709, 1.2622, 1.2545, 1.1326, 1.1041, 1.0028, 0.9822, 0.9523, 0.9103, 0.8926, 0.7217, 0.7061, 0.6919, 0.6537, 0.6396, 0.6231, 0.6208, 0.6094, 0.5919, 0.5854, 0.5701, 0.5581, 0.4924, 0.4643, 0.437, 0.2972, 0.2889, 0.2776, 0.1823, -0.36, 0.2256, -0.6172, 1.6505, 1.5814, 1.5774, 1.5655, 1.5145, 1.1346, 1.0915, 1.0879, 1.08, 0.976, 0.9497, 0.9025, 0.8095, 0.7672, 0.7577, 0.6229, 0.6152, 0.6094, 0.5818, 0.5721, 0.5036, 0.4986, 0.4923, 0.4723, 0.4647, 0.3807, 0.3711, 0.3544, 0.3476, 0.3448, 0.2223, -0.4556, 1.6069, 1.5199, 1.5191, 1.3625, 1.3238, 1.2978, 1.2737, 1.2209, 1.2067, 1.1896, 1.1465, 1.1039, 0.9468, 0.9138, 0.8438, 0.7781, 0.7719, 0.7313, 0.7149, 0.7078, 0.5917, 0.5592, 0.5519, 0.5259, 0.5136, 0.4527, 0.4154, 0.3893, 0.3754, 0.1899, 0.1029, -0.208, -0.642, 1.7929, 1.6434, 1.5085, 1.4697, 1.3205, 1.2911, 1.2382, 1.2291, 1.1221, 1.0708, 1.0642, 1.0398, 0.9916, 0.9062, 0.8636, 0.8204, 0.7414, 0.7347, 0.622, 0.5598, 0.5285, 0.515, 0.5013, 0.4736, 0.4647, 0.4336, 0.369, 0.2677, 0.2196, 0.1759, 0.1736, 0.101, -0.041, -0.6293, 1.8701, 1.8536, 1.8288, 1.4587, 1.3132, 1.235, 1.1831, 1.1185, 0.9705, 0.951, 0.9154, 0.8777, 0.8234, 0.7943, 0.7706, 0.7394, 0.7385, 0.6486, 0.5729, 0.522, 0.4928, 0.455, 0.3202, 0.3024, 0.2324, 0.2228, 0.1793, 0.1427, 0.1279, 0.1051, 0.0885, 0.0134, -0.1158, -0.0601, 0.0554, -0.2326]}, \"token.table\": {\"Topic\": [1, 2, 6, 9, 10, 2, 3, 4, 5, 6, 8, 9, 1, 2, 3, 4, 6, 8, 10, 1, 6, 7, 8, 10, 1, 2, 3, 7, 8, 1, 2, 3, 4, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 10, 4, 5, 7, 9, 4, 6, 8, 9, 2, 5, 8, 10, 2, 5, 6, 7, 8, 5, 6, 7, 9, 10, 2, 3, 4, 5, 6, 8, 2, 5, 8, 9, 10, 1, 2, 4, 5, 8, 1, 2, 4, 5, 6, 7, 1, 2, 3, 1, 2, 4, 6, 10, 1, 2, 5, 6, 7, 9, 10, 3, 4, 5, 6, 7, 9, 1, 3, 4, 9, 1, 2, 5, 6, 8, 2, 4, 5, 7, 2, 3, 6, 9, 2, 6, 10, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 7, 8, 3, 5, 7, 2, 7, 8, 9, 1, 2, 3, 5, 8, 10, 1, 2, 3, 5, 6, 7, 8, 9, 10, 1, 5, 6, 7, 9, 10, 2, 3, 4, 5, 7, 8, 9, 10, 6, 8, 9, 1, 3, 4, 6, 7, 8, 9, 6, 9, 10, 1, 3, 5, 6, 8, 9, 3, 6, 8, 10, 1, 2, 4, 7, 8, 10, 2, 3, 4, 6, 7, 8, 9, 10, 3, 4, 7, 8, 9, 6, 7, 9, 10, 1, 5, 9, 2, 3, 6, 7, 8, 2, 3, 4, 7, 8, 10, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 8, 9, 10, 2, 5, 8, 10, 1, 3, 5, 6, 7, 10, 2, 3, 6, 7, 8, 9, 10, 2, 5, 6, 8, 9, 1, 3, 4, 5, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 2, 3, 4, 7, 8, 1, 3, 4, 5, 8, 9, 10, 1, 2, 3, 4, 6, 7, 10, 1, 2, 3, 6, 7, 8, 10, 2, 3, 5, 6, 7, 9, 10, 1, 2, 3, 5, 6, 7, 10, 1, 4, 6, 7, 9, 10, 2, 3, 4, 6, 7, 8, 1, 2, 4, 5, 6, 8, 9, 1, 2, 3, 4, 6, 8, 1, 2, 4, 7, 8, 9, 1, 4, 5, 6, 1, 2, 3, 4, 6, 10, 2, 3, 4, 5, 6, 8, 1, 2, 3, 4, 5, 6, 2, 3, 4, 5, 6, 7, 8, 10, 1, 3, 4, 5, 7, 1, 2, 3, 6, 7, 8, 9, 10, 1, 4, 5, 7, 10, 2, 5, 6, 7, 8, 9, 1, 2, 3, 5, 6, 8, 10, 1, 3, 5, 6, 7, 10, 1, 2, 3, 4, 6, 10, 3, 4, 5, 8, 3, 4, 5, 10, 1, 2, 5, 6, 2, 3, 7, 8, 10, 2, 3, 4, 6, 7, 8, 10, 3, 6, 7, 9, 10, 1, 3, 7, 9, 10, 1, 5, 9, 1, 2, 3, 4, 5, 6, 7, 10, 2, 5, 8, 10, 3, 7, 10, 4, 5, 8, 10, 1, 6, 7, 8, 9, 4, 5, 6, 9, 10, 1, 3, 5, 6, 8, 1, 6, 8, 10, 2, 4, 5, 6, 9, 2, 3, 5, 7, 8, 10, 1, 3, 4, 5, 6, 7, 2, 3, 4, 5, 6, 7, 8, 1, 3, 4, 6, 8, 10, 1, 2, 4, 6, 7, 9, 1, 3, 4, 5, 6, 8, 9, 10, 1, 2, 5, 6, 8, 9, 3, 6, 8, 2, 3, 6, 10, 1, 2, 4, 8, 9, 1, 2, 6, 9, 10, 5, 7, 8, 10, 1, 2, 3, 6, 8, 9, 10, 1, 2, 4, 5, 8, 10, 1, 4, 6, 7, 8, 10], \"Freq\": [0.118390750175397, 0.118390750175397, 0.236781500350794, 0.355172250526191, 0.118390750175397, 0.28708117910334763, 0.14354058955167381, 0.14354058955167381, 0.14354058955167381, 0.14354058955167381, 0.14354058955167381, 0.14354058955167381, 0.09102476245292085, 0.1820495249058417, 0.1820495249058417, 0.09102476245292085, 0.1820495249058417, 0.09102476245292085, 0.09102476245292085, 0.12869107981649444, 0.12869107981649444, 0.2573821596329889, 0.2573821596329889, 0.2573821596329889, 0.14359375015051865, 0.14359375015051865, 0.14359375015051865, 0.14359375015051865, 0.2871875003010373, 0.10708142192664832, 0.4283256877065933, 0.10708142192664832, 0.10708142192664832, 0.10708142192664832, 0.10708142192664832, 0.10708142192664832, 0.10708142192664832, 0.06788154982217721, 0.13576309964435443, 0.06788154982217721, 0.27152619928870886, 0.06788154982217721, 0.06788154982217721, 0.3394077491108861, 0.06788154982217721, 0.11513042371125114, 0.05756521185562557, 0.46052169484500455, 0.23026084742250227, 0.12934504672892733, 0.38803514018678203, 0.12934504672892733, 0.25869009345785465, 0.1269951126910529, 0.5079804507642116, 0.1269951126910529, 0.1269951126910529, 0.12531392910990002, 0.12531392910990002, 0.3759417873297001, 0.25062785821980005, 0.12531392910990002, 0.14647299744597092, 0.14647299744597092, 0.14647299744597092, 0.14647299744597092, 0.43941899233791276, 0.08606663670081557, 0.17213327340163115, 0.08606663670081557, 0.08606663670081557, 0.17213327340163115, 0.2581999101024467, 0.12137104625840543, 0.24274209251681086, 0.12137104625840543, 0.3641131387752163, 0.12137104625840543, 0.49650337703598696, 0.1418581077245677, 0.1418581077245677, 0.07092905386228385, 0.07092905386228385, 0.3018987351345251, 0.10063291171150836, 0.20126582342301672, 0.10063291171150836, 0.10063291171150836, 0.20126582342301672, 0.16312011716414293, 0.32624023432828586, 0.32624023432828586, 0.17187308382405014, 0.17187308382405014, 0.17187308382405014, 0.08593654191202507, 0.25780962573607524, 0.20341101723149219, 0.0678036724104974, 0.0678036724104974, 0.20341101723149219, 0.1356073448209948, 0.2712146896419896, 0.0678036724104974, 0.24892705959481512, 0.24892705959481512, 0.12446352979740756, 0.12446352979740756, 0.12446352979740756, 0.12446352979740756, 0.1386176229019115, 0.1386176229019115, 0.41585286870573457, 0.1386176229019115, 0.11058439210977099, 0.11058439210977099, 0.33175317632931295, 0.33175317632931295, 0.11058439210977099, 0.1396337586893836, 0.1396337586893836, 0.1396337586893836, 0.5585350347575344, 0.13615504618669477, 0.27231009237338955, 0.13615504618669477, 0.4084651385600843, 0.24699897880715438, 0.49399795761430876, 0.24699897880715438, 0.06726967368498021, 0.06726967368498021, 0.13453934736996043, 0.20180902105494064, 0.13453934736996043, 0.20180902105494064, 0.06726967368498021, 0.13453934736996043, 0.06726967368498021, 0.13239609738219627, 0.13239609738219627, 0.26479219476439253, 0.26479219476439253, 0.13239609738219627, 0.5844096984473907, 0.14610242461184766, 0.14610242461184766, 0.12267561538549485, 0.12267561538549485, 0.12267561538549485, 0.4907024615419794, 0.11413659157851552, 0.11413659157851552, 0.22827318315703105, 0.11413659157851552, 0.22827318315703105, 0.11413659157851552, 0.03860937741139483, 0.23165626446836896, 0.23165626446836896, 0.07721875482278966, 0.07721875482278966, 0.1544375096455793, 0.03860937741139483, 0.03860937741139483, 0.1544375096455793, 0.2756005462956009, 0.13780027314780044, 0.13780027314780044, 0.13780027314780044, 0.13780027314780044, 0.13780027314780044, 0.05112932273302065, 0.4090345818641652, 0.05112932273302065, 0.1022586454660413, 0.05112932273302065, 0.15338796819906195, 0.1022586454660413, 0.05112932273302065, 0.29600157206289696, 0.14800078603144848, 0.29600157206289696, 0.09697472984545277, 0.19394945969090555, 0.19394945969090555, 0.09697472984545277, 0.09697472984545277, 0.09697472984545277, 0.19394945969090555, 0.14189217700065218, 0.14189217700065218, 0.7094608850032609, 0.12521724111219396, 0.2504344822243879, 0.12521724111219396, 0.12521724111219396, 0.12521724111219396, 0.12521724111219396, 0.08595030083093763, 0.08595030083093763, 0.08595030083093763, 0.687602406647501, 0.2045401104549604, 0.1022700552274802, 0.1022700552274802, 0.1022700552274802, 0.2045401104549604, 0.2045401104549604, 0.1305621130219345, 0.19584316953290173, 0.06528105651096724, 0.06528105651096724, 0.19584316953290173, 0.06528105651096724, 0.1305621130219345, 0.1305621130219345, 0.4269071578811359, 0.10672678947028398, 0.10672678947028398, 0.10672678947028398, 0.21345357894056796, 0.11187493949805256, 0.11187493949805256, 0.11187493949805256, 0.6712496369883153, 0.14577095079581967, 0.5830838031832787, 0.14577095079581967, 0.15579574515572484, 0.31159149031144967, 0.15579574515572484, 0.31159149031144967, 0.15579574515572484, 0.1002070810615217, 0.30062124318456507, 0.1002070810615217, 0.1002070810615217, 0.1002070810615217, 0.30062124318456507, 0.21824674189512464, 0.10912337094756232, 0.10912337094756232, 0.10912337094756232, 0.10912337094756232, 0.10912337094756232, 0.21824674189512464, 0.6256065230153836, 0.1251213046030767, 0.0417071015343589, 0.0417071015343589, 0.0417071015343589, 0.0417071015343589, 0.0834142030687178, 0.0417071015343589, 0.0417071015343589, 0.121085281557857, 0.36325584467357097, 0.242170563115714, 0.121085281557857, 0.24214158766683022, 0.12107079383341511, 0.12107079383341511, 0.12107079383341511, 0.24214158766683022, 0.12107079383341511, 0.10718203941575077, 0.10718203941575077, 0.10718203941575077, 0.3215461182472523, 0.21436407883150155, 0.10718203941575077, 0.10718203941575077, 0.11156983339338909, 0.22313966678677818, 0.11156983339338909, 0.22313966678677818, 0.22313966678677818, 0.2784772756003995, 0.13923863780019974, 0.13923863780019974, 0.13923863780019974, 0.13923863780019974, 0.13923863780019974, 0.13923863780019974, 0.0654783752667688, 0.1309567505335376, 0.29465268870045963, 0.0654783752667688, 0.1309567505335376, 0.0654783752667688, 0.0327391876333844, 0.0327391876333844, 0.09821756290015321, 0.09821756290015321, 0.13475378177901023, 0.13475378177901023, 0.13475378177901023, 0.5390151271160409, 0.13475378177901023, 0.29540170863507037, 0.07385042715876759, 0.07385042715876759, 0.07385042715876759, 0.07385042715876759, 0.29540170863507037, 0.14770085431753519, 0.13825765593741327, 0.13825765593741327, 0.13825765593741327, 0.27651531187482653, 0.27651531187482653, 0.13825765593741327, 0.13825765593741327, 0.13660835264638485, 0.13660835264638485, 0.13660835264638485, 0.13660835264638485, 0.13660835264638485, 0.13660835264638485, 0.4098250579391545, 0.058277771637345155, 0.11655554327469031, 0.11655554327469031, 0.058277771637345155, 0.2913888581867258, 0.23311108654938062, 0.058277771637345155, 0.11663176288946102, 0.11663176288946102, 0.11663176288946102, 0.23326352577892204, 0.23326352577892204, 0.11663176288946102, 0.23326352577892204, 0.12197437642098476, 0.12197437642098476, 0.12197437642098476, 0.12197437642098476, 0.3659231292629542, 0.3659231292629542, 0.11586586908811769, 0.11586586908811769, 0.11586586908811769, 0.11586586908811769, 0.11586586908811769, 0.34759760726435307, 0.10223821797757546, 0.30671465393272634, 0.20447643595515091, 0.10223821797757546, 0.10223821797757546, 0.10223821797757546, 0.10223821797757546, 0.16206011906297643, 0.16206011906297643, 0.16206011906297643, 0.32412023812595286, 0.16206011906297643, 0.16206011906297643, 0.136959950882088, 0.136959950882088, 0.136959950882088, 0.136959950882088, 0.136959950882088, 0.136959950882088, 0.21575102635577034, 0.10787551317788517, 0.3236265395336555, 0.21575102635577034, 0.34291922169778827, 0.2286128144651922, 0.1143064072325961, 0.2286128144651922, 0.1143064072325961, 0.1143064072325961, 0.11488850037513851, 0.11488850037513851, 0.11488850037513851, 0.45955400150055403, 0.11488850037513851, 0.11488850037513851, 0.10551770954987795, 0.10551770954987795, 0.10551770954987795, 0.10551770954987795, 0.5275885477493898, 0.10551770954987795, 0.05879117648847292, 0.05879117648847292, 0.17637352946541876, 0.05879117648847292, 0.05879117648847292, 0.5291205883962563, 0.05879117648847292, 0.11758235297694584, 0.11974124614255757, 0.4789649845702303, 0.11974124614255757, 0.11974124614255757, 0.11974124614255757, 0.14168967905492283, 0.14168967905492283, 0.07084483952746141, 0.21253451858238423, 0.07084483952746141, 0.07084483952746141, 0.07084483952746141, 0.14168967905492283, 0.11802817794497869, 0.11802817794497869, 0.11802817794497869, 0.5901408897248934, 0.11802817794497869, 0.261489824582877, 0.261489824582877, 0.1307449122914385, 0.1307449122914385, 0.1307449122914385, 0.1307449122914385, 0.07992387480438941, 0.07992387480438941, 0.31969549921755763, 0.15984774960877882, 0.07992387480438941, 0.07992387480438941, 0.2397716244131682, 0.14470663247080767, 0.14470663247080767, 0.14470663247080767, 0.14470663247080767, 0.14470663247080767, 0.434119897412423, 0.07057561931270118, 0.07057561931270118, 0.14115123862540235, 0.2823024772508047, 0.2823024772508047, 0.21172685793810353, 0.11905841593850908, 0.11905841593850908, 0.7143504956310545, 0.11905841593850908, 0.10010243455108583, 0.6006146073065151, 0.20020486910217167, 0.10010243455108583, 0.35506841685254875, 0.11835613895084958, 0.35506841685254875, 0.11835613895084958, 0.16317974607230115, 0.16317974607230115, 0.16317974607230115, 0.16317974607230115, 0.16317974607230115, 0.11680260272608746, 0.11680260272608746, 0.11680260272608746, 0.11680260272608746, 0.23360520545217492, 0.11680260272608746, 0.11680260272608746, 0.4093847992133847, 0.10234619980334618, 0.10234619980334618, 0.20469239960669236, 0.3070385994100385, 0.14888940042001356, 0.14888940042001356, 0.14888940042001356, 0.2977788008400271, 0.14888940042001356, 0.5183025292048277, 0.25915126460241383, 0.12957563230120692, 0.18868001386267397, 0.09434000693133698, 0.18868001386267397, 0.09434000693133698, 0.28302002079401095, 0.09434000693133698, 0.09434000693133698, 0.09434000693133698, 0.5832409810295519, 0.14581024525738798, 0.14581024525738798, 0.14581024525738798, 0.6007909213899217, 0.12015818427798436, 0.12015818427798436, 0.22791775917354723, 0.11395887958677361, 0.3418766387603209, 0.22791775917354723, 0.4660379862537691, 0.23301899312688454, 0.11650949656344227, 0.11650949656344227, 0.11650949656344227, 0.1583019437948604, 0.1583019437948604, 0.3166038875897208, 0.1583019437948604, 0.1583019437948604, 0.1754186399414385, 0.5262559198243154, 0.08770931997071925, 0.08770931997071925, 0.08770931997071925, 0.27652124246365956, 0.13826062123182978, 0.41478186369548936, 0.13826062123182978, 0.6383790080497863, 0.12767580160995726, 0.12767580160995726, 0.12767580160995726, 0.12767580160995726, 0.2487613967039368, 0.1243806983519684, 0.2487613967039368, 0.1243806983519684, 0.1243806983519684, 0.1243806983519684, 0.42056585991705653, 0.14018861997235216, 0.14018861997235216, 0.14018861997235216, 0.14018861997235216, 0.14018861997235216, 0.06267218689529534, 0.06267218689529534, 0.4387053082670674, 0.06267218689529534, 0.12534437379059069, 0.12534437379059069, 0.06267218689529534, 0.10373606622696223, 0.10373606622696223, 0.10373606622696223, 0.3112081986808867, 0.20747213245392446, 0.20747213245392446, 0.1593835346785413, 0.1593835346785413, 0.1593835346785413, 0.1593835346785413, 0.1593835346785413, 0.1593835346785413, 0.08926124476124093, 0.17852248952248187, 0.08926124476124093, 0.08926124476124093, 0.17852248952248187, 0.08926124476124093, 0.17852248952248187, 0.08926124476124093, 0.13820553375467132, 0.13820553375467132, 0.13820553375467132, 0.13820553375467132, 0.27641106750934263, 0.13820553375467132, 0.16322129694644869, 0.32644259389289737, 0.32644259389289737, 0.41619422669841527, 0.27746281779894355, 0.13873140889947178, 0.13873140889947178, 0.18932191607294155, 0.09466095803647077, 0.09466095803647077, 0.09466095803647077, 0.4733047901823539, 0.30433937196770594, 0.30433937196770594, 0.15216968598385297, 0.15216968598385297, 0.15216968598385297, 0.15390221857555142, 0.15390221857555142, 0.15390221857555142, 0.15390221857555142, 0.17191091262757044, 0.17191091262757044, 0.2578663689413557, 0.08595545631378522, 0.08595545631378522, 0.17191091262757044, 0.17191091262757044, 0.09347610317655702, 0.09347610317655702, 0.18695220635311405, 0.09347610317655702, 0.4673805158827851, 0.09347610317655702, 0.12954450620665733, 0.12954450620665733, 0.12954450620665733, 0.12954450620665733, 0.12954450620665733, 0.388633518619972], \"Term\": [\"aether\", \"aether\", \"aether\", \"aether\", \"aether\", \"already\", \"already\", \"already\", \"already\", \"already\", \"already\", \"already\", \"also\", \"also\", \"also\", \"also\", \"also\", \"also\", \"also\", \"always\", \"always\", \"always\", \"always\", \"always\", \"anyone\", \"anyone\", \"anyone\", \"anyone\", \"anyone\", \"art\", \"art\", \"art\", \"art\", \"art\", \"art\", \"art\", \"art\", \"ayaka\", \"ayaka\", \"ayaka\", \"ayaka\", \"ayaka\", \"ayaka\", \"ayaka\", \"ayaka\", \"baal\", \"baal\", \"baal\", \"baal\", \"back\", \"back\", \"back\", \"back\", \"best\", \"best\", \"best\", \"best\", \"better\", \"better\", \"better\", \"better\", \"better\", \"birthday\", \"birthday\", \"birthday\", \"birthday\", \"birthday\", \"character\", \"character\", \"character\", \"character\", \"character\", \"character\", \"characters\", \"characters\", \"characters\", \"characters\", \"characters\", \"check\", \"check\", \"check\", \"check\", \"check\", \"childe\", \"childe\", \"childe\", \"childe\", \"childe\", \"childe\", \"collab\", \"collab\", \"collab\", \"come\", \"come\", \"come\", \"come\", \"come\", \"day\", \"day\", \"day\", \"day\", \"day\", \"day\", \"day\", \"days\", \"days\", \"days\", \"days\", \"days\", \"days\", \"even\", \"even\", \"even\", \"even\", \"event\", \"event\", \"event\", \"event\", \"event\", \"ever\", \"ever\", \"ever\", \"ever\", \"favorite\", \"favorite\", \"favorite\", \"favorite\", \"finally\", \"finally\", \"finally\", \"first\", \"first\", \"first\", \"first\", \"first\", \"first\", \"first\", \"first\", \"first\", \"free\", \"free\", \"free\", \"free\", \"free\", \"friends\", \"friends\", \"friends\", \"full\", \"full\", \"full\", \"full\", \"fun\", \"fun\", \"fun\", \"fun\", \"fun\", \"fun\", \"game\", \"game\", \"game\", \"game\", \"game\", \"game\", \"game\", \"game\", \"game\", \"games\", \"games\", \"games\", \"games\", \"games\", \"games\", \"get\", \"get\", \"get\", \"get\", \"get\", \"get\", \"get\", \"get\", \"getting\", \"getting\", \"getting\", \"go\", \"go\", \"go\", \"go\", \"go\", \"go\", \"go\", \"god\", \"god\", \"god\", \"going\", \"going\", \"going\", \"going\", \"going\", \"going\", \"good\", \"good\", \"good\", \"good\", \"gorou\", \"gorou\", \"gorou\", \"gorou\", \"gorou\", \"gorou\", \"got\", \"got\", \"got\", \"got\", \"got\", \"got\", \"got\", \"got\", \"guys\", \"guys\", \"guys\", \"guys\", \"guys\", \"happy\", \"happy\", \"happy\", \"happy\", \"help\", \"help\", \"help\", \"hey\", \"hey\", \"hey\", \"hey\", \"hey\", \"home\", \"home\", \"home\", \"home\", \"home\", \"home\", \"hope\", \"hope\", \"hope\", \"hope\", \"hope\", \"hope\", \"hope\", \"inazuma\", \"inazuma\", \"inazuma\", \"inazuma\", \"inazuma\", \"inazuma\", \"inazuma\", \"inazuma\", \"inazuma\", \"join\", \"join\", \"join\", \"join\", \"know\", \"know\", \"know\", \"know\", \"know\", \"know\", \"kokomi\", \"kokomi\", \"kokomi\", \"kokomi\", \"kokomi\", \"kokomi\", \"kokomi\", \"last\", \"last\", \"last\", \"last\", \"last\", \"let\", \"let\", \"let\", \"let\", \"let\", \"let\", \"let\", \"like\", \"like\", \"like\", \"like\", \"like\", \"like\", \"like\", \"like\", \"like\", \"like\", \"link\", \"link\", \"link\", \"link\", \"link\", \"live\", \"live\", \"live\", \"live\", \"live\", \"live\", \"live\", \"looking\", \"looking\", \"looking\", \"looking\", \"looking\", \"looking\", \"looking\", \"looks\", \"looks\", \"looks\", \"looks\", \"looks\", \"looks\", \"looks\", \"love\", \"love\", \"love\", \"love\", \"love\", \"love\", \"love\", \"lt\", \"lt\", \"lt\", \"lt\", \"lt\", \"lt\", \"lt\", \"lumine\", \"lumine\", \"lumine\", \"lumine\", \"lumine\", \"lumine\", \"made\", \"made\", \"made\", \"made\", \"made\", \"made\", \"make\", \"make\", \"make\", \"make\", \"make\", \"make\", \"make\", \"many\", \"many\", \"many\", \"many\", \"many\", \"many\", \"map\", \"map\", \"map\", \"map\", \"map\", \"map\", \"mihoyo\", \"mihoyo\", \"mihoyo\", \"mihoyo\", \"mona\", \"mona\", \"mona\", \"mona\", \"mona\", \"mona\", \"much\", \"much\", \"much\", \"much\", \"much\", \"much\", \"need\", \"need\", \"need\", \"need\", \"need\", \"need\", \"new\", \"new\", \"new\", \"new\", \"new\", \"new\", \"new\", \"new\", \"next\", \"next\", \"next\", \"next\", \"next\", \"one\", \"one\", \"one\", \"one\", \"one\", \"one\", \"one\", \"one\", \"outfit\", \"outfit\", \"outfit\", \"outfit\", \"outfit\", \"people\", \"people\", \"people\", \"people\", \"people\", \"people\", \"play\", \"play\", \"play\", \"play\", \"play\", \"play\", \"play\", \"players\", \"players\", \"players\", \"players\", \"players\", \"players\", \"playing\", \"playing\", \"playing\", \"playing\", \"playing\", \"playing\", \"please\", \"please\", \"please\", \"please\", \"quest\", \"quest\", \"quest\", \"quest\", \"raiden\", \"raiden\", \"raiden\", \"raiden\", \"real\", \"real\", \"real\", \"real\", \"real\", \"really\", \"really\", \"really\", \"really\", \"really\", \"really\", \"really\", \"right\", \"right\", \"right\", \"right\", \"right\", \"rt\", \"rt\", \"rt\", \"rt\", \"rt\", \"sakura\", \"sakura\", \"sakura\", \"see\", \"see\", \"see\", \"see\", \"see\", \"see\", \"see\", \"see\", \"since\", \"since\", \"since\", \"since\", \"still\", \"still\", \"still\", \"story\", \"story\", \"story\", \"story\", \"stream\", \"stream\", \"stream\", \"stream\", \"stream\", \"streaming\", \"streaming\", \"streaming\", \"streaming\", \"streaming\", \"summer\", \"summer\", \"summer\", \"summer\", \"summer\", \"team\", \"team\", \"team\", \"team\", \"thank\", \"thank\", \"thank\", \"thank\", \"thank\", \"think\", \"think\", \"think\", \"think\", \"think\", \"think\", \"thoma\", \"thoma\", \"thoma\", \"thoma\", \"thoma\", \"thoma\", \"time\", \"time\", \"time\", \"time\", \"time\", \"time\", \"time\", \"today\", \"today\", \"today\", \"today\", \"today\", \"today\", \"twitch\", \"twitch\", \"twitch\", \"twitch\", \"twitch\", \"twitch\", \"update\", \"update\", \"update\", \"update\", \"update\", \"update\", \"update\", \"update\", \"use\", \"use\", \"use\", \"use\", \"use\", \"use\", \"v\", \"v\", \"v\", \"version\", \"version\", \"version\", \"version\", \"via\", \"via\", \"via\", \"via\", \"via\", \"video\", \"video\", \"video\", \"video\", \"video\", \"wan\", \"wan\", \"wan\", \"wan\", \"want\", \"want\", \"want\", \"want\", \"want\", \"want\", \"want\", \"world\", \"world\", \"world\", \"world\", \"world\", \"world\", \"xiao\", \"xiao\", \"xiao\", \"xiao\", \"xiao\", \"xiao\"]}, \"R\": 30, \"lambda.step\": 0.01, \"plot.opts\": {\"xlab\": \"PC1\", \"ylab\": \"PC2\"}, \"topic.order\": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]};\n",
       "\n",
       "function LDAvis_load_lib(url, callback){\n",
       "  var s = document.createElement('script');\n",
       "  s.src = url;\n",
       "  s.async = true;\n",
       "  s.onreadystatechange = s.onload = callback;\n",
       "  s.onerror = function(){console.warn(\"failed to load library \" + url);};\n",
       "  document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "}\n",
       "\n",
       "if(typeof(LDAvis) !== \"undefined\"){\n",
       "   // already loaded: just create the visualization\n",
       "   !function(LDAvis){\n",
       "       new LDAvis(\"#\" + \"ldavis_el730424243454494081166134438\", ldavis_el730424243454494081166134438_data);\n",
       "   }(LDAvis);\n",
       "}else if(typeof define === \"function\" && define.amd){\n",
       "   // require.js is available: use it to load d3/LDAvis\n",
       "   require.config({paths: {d3: \"https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min\"}});\n",
       "   require([\"d3\"], function(d3){\n",
       "      window.d3 = d3;\n",
       "      LDAvis_load_lib(\"https://cdn.rawgit.com/bmabey/pyLDAvis/files/ldavis.v1.0.0.js\", function(){\n",
       "        new LDAvis(\"#\" + \"ldavis_el730424243454494081166134438\", ldavis_el730424243454494081166134438_data);\n",
       "      });\n",
       "    });\n",
       "}else{\n",
       "    // require.js not available: dynamically load d3 & LDAvis\n",
       "    LDAvis_load_lib(\"https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min.js\", function(){\n",
       "         LDAvis_load_lib(\"https://cdn.rawgit.com/bmabey/pyLDAvis/files/ldavis.v1.0.0.js\", function(){\n",
       "                 new LDAvis(\"#\" + \"ldavis_el730424243454494081166134438\", ldavis_el730424243454494081166134438_data);\n",
       "            })\n",
       "         });\n",
       "}\n",
       "</script>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pyLDAvis\n",
    "import pyLDAvis.gensim\n",
    "\n",
    "vis = pyLDAvis.gensim.prepare(tweets_lda_tfidf, corpus_tfidf, dictionary=text_dict, sort_topics=False)\n",
    "pyLDAvis.enable_notebook()\n",
    "pyLDAvis.display(vis)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}