generate_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "24c39457-153a-4e5e-b61f-2294c18bea87",
   "metadata": {},
   "source": [
    "# Constants"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3390a3b6-b4ce-4ad6-9618-32fe30dd1f61",
   "metadata": {},
   "source": [
    "Use these constants to control the size of each topic \\\n",
    "None = whole dataset \\\n",
    "0 = exclude topic \\\n",
    "SEED (controls randomness) is defined in **constants.py**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "f6cf6adb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:38:42.843734Z",
     "iopub.status.busy": "2024-01-08T15:38:42.843378Z",
     "iopub.status.idle": "2024-01-08T15:38:42.862250Z",
     "shell.execute_reply": "2024-01-08T15:38:42.861500Z",
     "shell.execute_reply.started": "2024-01-08T15:38:42.843708Z"
    }
   },
   "outputs": [],
   "source": [
    "MAX_SAMPLES_PER_TOPIC = 10000  # sampled dataset size = min(MAX_SAMPLES, len(dataset)\n",
    "\n",
    "CLIMATE_CHANGE = MAX_SAMPLES_PER_TOPIC\n",
    "SELFDRIVING_CARS = MAX_SAMPLES_PER_TOPIC\n",
    "WEATHER = MAX_SAMPLES_PER_TOPIC\n",
    "AIRLINE_SUPPORT = MAX_SAMPLES_PER_TOPIC\n",
    "FIFA_WORLD_CUP = MAX_SAMPLES_PER_TOPIC\n",
    "AUSTRALIAN_ELECTIONS = MAX_SAMPLES_PER_TOPIC\n",
    "COVID19 = MAX_SAMPLES_PER_TOPIC\n",
    "CHATGPT = MAX_SAMPLES_PER_TOPIC\n",
    "STOCK_MARKET_CRASH = MAX_SAMPLES_PER_TOPIC\n",
    "\n",
    "DUPLICATES_FRACTION = 0.1\n",
    "SUPERSAMPLE_COEF = 1 + DUPLICATES_FRACTION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "d2a4c7e2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:38:42.864015Z",
     "iopub.status.busy": "2024-01-08T15:38:42.863795Z",
     "iopub.status.idle": "2024-01-08T15:38:42.889091Z",
     "shell.execute_reply": "2024-01-08T15:38:42.888320Z",
     "shell.execute_reply.started": "2024-01-08T15:38:42.863994Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from constants import SEED\n",
    "from utils import save_dataset, load_dataset\n",
    "\n",
    "pd.set_option('display.max_colwidth', None)\n",
    "rng = np.random.default_rng(seed=SEED)\n",
    "dataset = None\n",
    "    \n",
    "def concat(d1, d2):\n",
    "    return pd.concat([d1, d2], axis=0, ignore_index=True)\n",
    "\n",
    "def select_columns(df, columns):\n",
    "    return df.drop(columns=list(set(df.columns) - set(columns)))\n",
    "\n",
    "def sample(df, n):\n",
    "    if n is None:\n",
    "        return df\n",
    "    elif n == 0:\n",
    "        return pd.DataFrame()\n",
    "    else:\n",
    "        return df.sample(min(n, len(df)), random_state=SEED)\n",
    "    \n",
    "def supersample(df, n):\n",
    "    n = int(n * SUPERSAMPLE_COEF)\n",
    "    return sample(df, n) if len(df) > n else df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "8e421bad-6d4c-46a6-b395-ff3817eb7d1f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:38:42.890212Z",
     "iopub.status.busy": "2024-01-08T15:38:42.890008Z",
     "iopub.status.idle": "2024-01-08T15:38:42.913785Z",
     "shell.execute_reply": "2024-01-08T15:38:42.913060Z",
     "shell.execute_reply.started": "2024-01-08T15:38:42.890190Z"
    }
   },
   "outputs": [],
   "source": [
    "import regex as re\n",
    "import emoji\n",
    "\n",
    "RE_INVALID_CHARS = re.compile(r\"[\\p{Cc}\\p{Cs}]+\")\n",
    "LOWERCASE_ALPHABET = {'a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z'}\n",
    "UNICODE_ALLOWED_CHARS = \"’\"\n",
    "\n",
    "def remove_invalid_chars(s):\n",
    "    return RE_INVALID_CHARS.sub('', s)\n",
    "\n",
    "def consists_of_allowed_chars(s):\n",
    "    return all([\n",
    "        c.isascii() or c in UNICODE_ALLOWED_CHARS or emoji.is_emoji(c)\n",
    "        for c in s\n",
    "    ])\n",
    "\n",
    "def contains_substrings(s, ss, negation=False, case=True):\n",
    "    s = s if case else s.lower()\n",
    "    return any(map(s.__contains__, ss)) ^ negation\n",
    "\n",
    "def replace_bad_chars(s):\n",
    "    if len(s) > 0:\n",
    "        s = re.sub('Ì¢âÂå|Ì¢âÂÒ', ' ', s, flags=re.IGNORECASE)\n",
    "        s = re.sub('(â€™|Ã¢â‚¬â„¢|Ì¢âÂã¢|Ì¢âÂã¢)', \"'\", s, flags=re.IGNORECASE)\n",
    "        s = re.sub('(&amp;amp;|&amp;)', '&',  s, flags=re.IGNORECASE)\n",
    "        s = re.sub('\\$q\\$', '\"', s, flags=re.IGNORECASE)\n",
    "        s = re.sub('n\"t', \"n't\", s, flags=re.IGNORECASE)\n",
    "        s = re.sub(' [ ]+', ' ', s)\n",
    "    return s.strip()\n",
    "\n",
    "def string_frequency(df, s):\n",
    "    return sum([contains_substrings(t, [s], case=False) for t in df['text']]) / len(df)\n",
    "\n",
    "def randomly_remove_substring(df, ss, p=0.5, case=True):\n",
    "    ids = df.index[df['text'].apply(lambda s: contains_substrings(s, [ss], case=case))]\n",
    "    ids = rng.choice(ids, int(p * len(ids)), replace=False)\n",
    "    df.loc[ids, 'text'] = df.loc[ids, 'text'].str.replace(ss, '', case=case)\n",
    "    \n",
    "    f = len(df.index[df['text'].apply(lambda s: contains_substrings(s, [ss], case=case))]) / len(df)\n",
    "    print(f'\"{ss}\" new frequency: {f:.2f}')\n",
    "    \n",
    "    return df\n",
    "\n",
    "def remove_retweet_mark(tweet):\n",
    "    if tweet.lower().startswith('rt @'):\n",
    "        if ':' in tweet:\n",
    "            return tweet.split(':')[1]\n",
    "        else:\n",
    "            tokens = tweet.split(' ', 2)\n",
    "            return tokens[2] if len(tokens) >= 3 else ''\n",
    "    else:\n",
    "        return tweet\n",
    "    \n",
    "def preprocessing_pipeline(df):\n",
    "    df.dropna(subset='text', inplace=True)\n",
    "    df['text'] = df['text'].apply(replace_bad_chars).apply(remove_retweet_mark)\n",
    "    df.drop_duplicates(subset='text', inplace=True)\n",
    "    df = df[df['text'].apply(consists_of_allowed_chars)]\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "0a3c3318-d8f5-47b9-b9f7-d4c6a10175a8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:38:42.915433Z",
     "iopub.status.busy": "2024-01-08T15:38:42.915230Z",
     "iopub.status.idle": "2024-01-08T15:38:42.937800Z",
     "shell.execute_reply": "2024-01-08T15:38:42.937106Z",
     "shell.execute_reply.started": "2024-01-08T15:38:42.915414Z"
    }
   },
   "outputs": [],
   "source": [
    "import pycld2 as cld2\n",
    "\n",
    "def compute_word_fraction(s):\n",
    "    return sum(map(len, [[c for c in t if c.isalpha()] for t in s.split() if not t.startswith(('#', '@', 'http')) and t[0].isalpha()])) / len(s)\n",
    "\n",
    "def compute_average_token_length(s):\n",
    "    tokens = s.split(); return sum(map(len, tokens)) / len(tokens) if len(tokens) > 0 else 0\n",
    "\n",
    "def compute_english_score(s):\n",
    "    t = cld2.detect(s, hintTopLevelDomain='en', hintLanguage='en')[2][0]; return 0 if t[1] != 'en' else t[3]\n",
    "\n",
    "def add_stats(df):\n",
    "    \"\"\"Inplace function\"\"\"\n",
    "    df['length'] = df['text'].str.len()\n",
    "    df['word fraction'] = df['text'].apply(compute_word_fraction)\n",
    "    df['token length'] = df['text'].apply(compute_average_token_length)\n",
    "    df['english score'] = df['text'].apply(compute_english_score)\n",
    "    return df\n",
    "\n",
    "def trim_dataset_by(\n",
    "        dataset, \n",
    "        sort_by=None, \n",
    "        ascending=True, \n",
    "        length=(None, None), \n",
    "        alphabet_fraction=(None, None), \n",
    "        word_fraction=(None, None),\n",
    "        token_length=(None, None),\n",
    "        english_score=(None, None),\n",
    "        custom_metrics=None,\n",
    "        verbose=True,\n",
    "    ):\n",
    "    df = dataset.copy()\n",
    "    n0 = len(df)\n",
    "              \n",
    "    if length[0] is not None: df = df[df['length'] >= length[0]]\n",
    "    if length[1] is not None: df = df[df['length'] <= length[1]]\n",
    "    \n",
    "    if word_fraction[0] is not None: df = df[df['word fraction'] >= word_fraction[0]]\n",
    "    if word_fraction[1] is not None: df = df[df['word fraction'] <= word_fraction[1]]\n",
    "    \n",
    "    if token_length[0] is not None: df = df[df['token length'] >= token_length[0]]\n",
    "    if token_length[1] is not None: df = df[df['token length'] <= token_length[1]]\n",
    "    \n",
    "    if english_score[0] is not None: df = df[df['english score'] >= english_score[0]]\n",
    "    if english_score[1] is not None: df = df[df['english score'] <= english_score[1]]\n",
    "    \n",
    "    if custom_metrics is not None:\n",
    "        for label, a, b in custom_metrics:\n",
    "            if a is not None: df = df[df[label] >= a]\n",
    "            if b is not None: df = df[df[label] <= b]\n",
    "    \n",
    "    n1 = len(df)\n",
    "    if verbose: print(f'samples: {n1}, reduction: {((n0 - n1) / n0):.2%}')\n",
    "    return df.sort_values(sort_by, ascending=ascending)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "ca3eb164-72d0-4e22-961d-3d19bdb35f61",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:38:42.941046Z",
     "iopub.status.busy": "2024-01-08T15:38:42.940836Z",
     "iopub.status.idle": "2024-01-08T15:38:42.960291Z",
     "shell.execute_reply": "2024-01-08T15:38:42.959585Z",
     "shell.execute_reply.started": "2024-01-08T15:38:42.941025Z"
    }
   },
   "outputs": [],
   "source": [
    "import multiprocessing\n",
    "\n",
    "import numpy as np\n",
    "from rapidfuzz.fuzz import ratio\n",
    "\n",
    "from utils import start_time, elapsed_time\n",
    "\n",
    "_strings = None\n",
    "_n_strings = None\n",
    "_similarity = None\n",
    "\n",
    "def flatten(xss):\n",
    "    return [x for xs in xss for x in xs]\n",
    "\n",
    "def find_most_similar(i):\n",
    "    ss = [ratio(_strings[i], _strings[j]) for j in range(i + 1, _n_strings)]\n",
    "    return [i + 1 + j for j in range(len(ss)) if ss[j] >= _similarity]\n",
    "\n",
    "def drop_similar(df, max_similarity=0.8):\n",
    "    global _strings, _n_strings, _similarity\n",
    "    _strings = df['text'].tolist()\n",
    "    _n_strings = len(_strings)\n",
    "    _similarity = max_similarity * 100\n",
    "    \n",
    "    start_time()\n",
    "    pool = multiprocessing.Pool(processes=multiprocessing.cpu_count())\n",
    "    out = pool.map(find_most_similar, range(_n_strings - 1))\n",
    "    similar_ids = set(flatten(out))\n",
    "    elapsed_time(verbose=True)\n",
    "    \n",
    "    ids = set(range(_n_strings)) - similar_ids\n",
    "    df = df.reset_index(drop=True).iloc[list(ids)]\n",
    "    diff = _n_strings - len(df)\n",
    "    print(f'samples: {len(df)}, similar samples detected: {diff} ({diff / _n_strings:.2%})')\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b2ca1a66-3a69-4aae-8bf0-31618e1460c7",
   "metadata": {},
   "source": [
    "# Climate change"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "afe19f49-6b9f-4945-8205-5d96f941b0e1",
   "metadata": {},
   "source": [
    "Load datset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "055d6a7e-e126-4a8e-8978-5ffecfd5cdb9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:38:42.961924Z",
     "iopub.status.busy": "2024-01-08T15:38:42.961730Z",
     "iopub.status.idle": "2024-01-08T15:38:43.042041Z",
     "shell.execute_reply": "2024-01-08T15:38:43.041339Z",
     "shell.execute_reply.started": "2024-01-08T15:38:42.961903Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(43943, 2)"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = load_dataset('climate_change.csv', source=True, encoding='ISO-8859-1')\n",
    "df.rename(columns={'message': 'text', 'tweetid': 'id'}, inplace=True)\n",
    "df = select_columns(df, ['text', 'id'])\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "74ef694a-94df-4ece-ada0-6647310e246f",
   "metadata": {},
   "source": [
    "Explore tweets (by repeatedly running the next cell)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "e8c0ea61-3661-4724-8834-4f650084bed8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:38:43.043267Z",
     "iopub.status.busy": "2024-01-08T15:38:43.043063Z",
     "iopub.status.idle": "2024-01-08T15:38:43.062907Z",
     "shell.execute_reply": "2024-01-08T15:38:43.062258Z",
     "shell.execute_reply.started": "2024-01-08T15:38:43.043247Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>15305</th>\n",
       "      <td>esok presentation bi gp \\naq tntang global warming.. \\nwish me luck.. ï¿½ï¿½ï¿½ï¿½ï¿½ï¿½</td>\n",
       "      <td>840904112195551232</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14829</th>\n",
       "      <td>RT @JordanUhl: #EPA Chief Pruitt claims 'tremendous debate' over whether CO2 causes global warming.\\n\\nNo debate. Just misinformatioâ¦</td>\n",
       "      <td>840255456245383173</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41889</th>\n",
       "      <td>Scientists, investors seek to identify financial risks of climate change  #environment</td>\n",
       "      <td>732978742008512512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40720</th>\n",
       "      <td>Viking M. Services: Geothermal heating -- heat pumps and global warming https://t.co/XXzbMnIgfT https://t.co/BztbrVWoKx</td>\n",
       "      <td>697024720139452417</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>941</th>\n",
       "      <td>The Vita Green Impact Fund is exactly the right investment vehicle for #SDGs &amp;amp; climate change mitigation https://t.co/FgvujJdfjU</td>\n",
       "      <td>793842057655775232</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                          text  \\\n",
       "15305                                                 esok presentation bi gp \\naq tntang global warming.. \\nwish me luck.. ï¿½ï¿½ï¿½ï¿½ï¿½ï¿½   \n",
       "14829  RT @JordanUhl: #EPA Chief Pruitt claims 'tremendous debate' over whether CO2 causes global warming.\\n\\nNo debate. Just misinformatioâ¦   \n",
       "41889                                                   Scientists, investors seek to identify financial risks of climate change  #environment   \n",
       "40720                  Viking M. Services: Geothermal heating -- heat pumps and global warming https://t.co/XXzbMnIgfT https://t.co/BztbrVWoKx   \n",
       "941       The Vita Green Impact Fund is exactly the right investment vehicle for #SDGs &amp; climate change mitigation https://t.co/FgvujJdfjU   \n",
       "\n",
       "                       id  \n",
       "15305  840904112195551232  \n",
       "14829  840255456245383173  \n",
       "41889  732978742008512512  \n",
       "40720  697024720139452417  \n",
       "941    793842057655775232  "
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "033e82a4-1d26-4398-a5de-544432044b6c",
   "metadata": {},
   "source": [
    "Fix some encoding issues"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "6ddf04f5-525f-4981-b507-2396506f61be",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:38:43.063874Z",
     "iopub.status.busy": "2024-01-08T15:38:43.063681Z",
     "iopub.status.idle": "2024-01-08T15:38:44.678071Z",
     "shell.execute_reply": "2024-01-08T15:38:44.677316Z",
     "shell.execute_reply.started": "2024-01-08T15:38:43.063855Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(25342, 2)"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = preprocessing_pipeline(df)\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6ac6362a-c91c-40b6-96cd-9d651c286908",
   "metadata": {},
   "source": [
    "Remove noisy tweets using tweet length, english alphabet share and confidence that the language is English"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "85b8fd87-c3a8-4488-9023-0620bd733732",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:38:44.680062Z",
     "iopub.status.busy": "2024-01-08T15:38:44.679848Z",
     "iopub.status.idle": "2024-01-08T15:38:45.148224Z",
     "shell.execute_reply": "2024-01-08T15:38:45.147476Z",
     "shell.execute_reply.started": "2024-01-08T15:38:44.680041Z"
    }
   },
   "outputs": [],
   "source": [
    "add_stats(df);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "5d76ded4-34c4-44ae-82e7-7be1edce6f4d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:38:45.149591Z",
     "iopub.status.busy": "2024-01-08T15:38:45.149385Z",
     "iopub.status.idle": "2024-01-08T15:38:45.186575Z",
     "shell.execute_reply": "2024-01-08T15:38:45.185824Z",
     "shell.execute_reply.started": "2024-01-08T15:38:45.149570Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "samples: 18234, reduction: 28.05%\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>length</th>\n",
       "      <th>word fraction</th>\n",
       "      <th>token length</th>\n",
       "      <th>english score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>24823</th>\n",
       "      <td>Late Pleistocene-Holocene vegetation and climate change in the Middle Kalahari, Lake Ngami, Botswana\\nhttps://t.co/cAvWNIgbXE @scott_louis</td>\n",
       "      <td>137</td>\n",
       "      <td>0.620438</td>\n",
       "      <td>8.200000</td>\n",
       "      <td>700.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23028</th>\n",
       "      <td>@Philosocrat Oh I believe in climate change. I just don't believe in Delaware.</td>\n",
       "      <td>78</td>\n",
       "      <td>0.653846</td>\n",
       "      <td>5.076923</td>\n",
       "      <td>700.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30626</th>\n",
       "      <td>Study finds that global warming exacerbates refugee crises | John Abraham https://t.co/KlTmfLCjAd https://t.co/90rdZL1ths</td>\n",
       "      <td>121</td>\n",
       "      <td>0.512397</td>\n",
       "      <td>8.384615</td>\n",
       "      <td>700.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>894</th>\n",
       "      <td>.@OfficialJLD endorses Hillary Clinton, slams Donald Trump over climate change</td>\n",
       "      <td>79</td>\n",
       "      <td>0.696203</td>\n",
       "      <td>6.900000</td>\n",
       "      <td>700.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11257</th>\n",
       "      <td>Mike Pompeo, Trump's CIA pick, evades questions about climate change and global instability https://t.co/TPzl33TqrV https://t.co/BHYxZFpNBs</td>\n",
       "      <td>139</td>\n",
       "      <td>0.546763</td>\n",
       "      <td>8.333333</td>\n",
       "      <td>700.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25188</th>\n",
       "      <td>Peru builds up wetland resilience to reinforce indigenous response to climate change https://t.co/x0XAxzKn2j https://t.co/58oYUWGVTz</td>\n",
       "      <td>132</td>\n",
       "      <td>0.553030</td>\n",
       "      <td>8.500000</td>\n",
       "      <td>701.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30668</th>\n",
       "      <td>Study finds that global warming exacerbates refugee crises | John Abraham https://t.co/9FDzShx65z</td>\n",
       "      <td>97</td>\n",
       "      <td>0.639175</td>\n",
       "      <td>7.166667</td>\n",
       "      <td>701.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1749</th>\n",
       "      <td>Ralph Cicerone, former UC Irvine chancellor who studied the causes of climate change, dies at 73 - Los Angeles Times https://t.co/fqGVQGVi80</td>\n",
       "      <td>140</td>\n",
       "      <td>0.657143</td>\n",
       "      <td>5.714286</td>\n",
       "      <td>701.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40678</th>\n",
       "      <td>Unanticipated consequence of climate change: disease carrying mosquitoes &amp; tics spread north. #Dengue, #Zika, #Lymedisease, #Chikungunya etc</td>\n",
       "      <td>140</td>\n",
       "      <td>0.585714</td>\n",
       "      <td>7.294118</td>\n",
       "      <td>701.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30152</th>\n",
       "      <td>Eying vulnerability, Calif. studying how climate change will affect state highways https://t.co/UoI0H4Q4oh https://t.co/Tju0OGHQV4</td>\n",
       "      <td>130</td>\n",
       "      <td>0.538462</td>\n",
       "      <td>9.076923</td>\n",
       "      <td>701.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9779</th>\n",
       "      <td>#trump is a climate change denier, a chronic liar, an unbridled man given to conspiracy theories &amp; wild ideas. Save us #ElectoralCollege</td>\n",
       "      <td>136</td>\n",
       "      <td>0.647059</td>\n",
       "      <td>5.227273</td>\n",
       "      <td>701.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39597</th>\n",
       "      <td>Floods Displace Thousands in Uganda I CLIMATE CHANGE IN AFRICA I https://t.co/PKXlD5VNYK #uganda via @allafrica</td>\n",
       "      <td>111</td>\n",
       "      <td>0.513514</td>\n",
       "      <td>6.466667</td>\n",
       "      <td>701.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15603</th>\n",
       "      <td>Coal lobbyist Trump attorney seeks to bypass US kids' climate lawsuit | Climate Home - climate change news https</td>\n",
       "      <td>113</td>\n",
       "      <td>0.761062</td>\n",
       "      <td>4.947368</td>\n",
       "      <td>701.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23457</th>\n",
       "      <td>@tomcolicchio @MikeBloomberg And donating millions in support of combatting climate change?</td>\n",
       "      <td>91</td>\n",
       "      <td>0.582418</td>\n",
       "      <td>7.363636</td>\n",
       "      <td>701.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8653</th>\n",
       "      <td>Biggest pet peeve: Republicans pretending climate change doesn't exist:</td>\n",
       "      <td>71</td>\n",
       "      <td>0.845070</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>702.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22567</th>\n",
       "      <td>@JEPomfret @USEmbassyBJ @USEmbassyKabul Silly twit! Tricks are for kids! Resigning over climate change? What a silly rabbit!</td>\n",
       "      <td>124</td>\n",
       "      <td>0.540323</td>\n",
       "      <td>6.352941</td>\n",
       "      <td>702.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33284</th>\n",
       "      <td>@silvercreeklady @thehill The US gop don't believe in climate change</td>\n",
       "      <td>68</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>5.900000</td>\n",
       "      <td>703.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40658</th>\n",
       "      <td>Tonight Sanders condemned Republican presidential candidates for their #climate change denial https</td>\n",
       "      <td>100</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>7.333333</td>\n",
       "      <td>703.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7848</th>\n",
       "      <td>#Manifest US investors, companies back Paris climate change agreement. Read Blog: https://t.co/3d6jNVT1gu</td>\n",
       "      <td>105</td>\n",
       "      <td>0.561905</td>\n",
       "      <td>7.833333</td>\n",
       "      <td>703.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18653</th>\n",
       "      <td>UN climate chief bites tongue after Trump de-funding threat | Climate Home - climate change news https://t.co/tLj9TaMagv via @ClimateHome</td>\n",
       "      <td>137</td>\n",
       "      <td>0.591241</td>\n",
       "      <td>6.263158</td>\n",
       "      <td>703.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16110</th>\n",
       "      <td>Donald Trump proposes huge EPA budget cut to stop environment agency researching climate change https://t.co/94MGB1O57i</td>\n",
       "      <td>119</td>\n",
       "      <td>0.689076</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>703.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41355</th>\n",
       "      <td>The moment Hillary Clinton lost her temper with a climate change activist https</td>\n",
       "      <td>80</td>\n",
       "      <td>0.775000</td>\n",
       "      <td>5.153846</td>\n",
       "      <td>704.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32934</th>\n",
       "      <td>Zim ill-prepared for climate change - The Zimbabwe Standard https://t.co/KwaVtykhcU</td>\n",
       "      <td>83</td>\n",
       "      <td>0.590361</td>\n",
       "      <td>7.400000</td>\n",
       "      <td>704.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4300</th>\n",
       "      <td>Dems MUST fight 2 end income inequality, raise min wage, battle climate change, safe gun cntrl, justice reform, state&amp;fed level w/o GOP</td>\n",
       "      <td>135</td>\n",
       "      <td>0.785185</td>\n",
       "      <td>5.181818</td>\n",
       "      <td>704.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25974</th>\n",
       "      <td>Neil DeGrasse Tyson slams climate change deniers amid Hurricane Harvey https</td>\n",
       "      <td>77</td>\n",
       "      <td>0.792208</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>704.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38696</th>\n",
       "      <td>Kenya, EU Steps Up Cooperation On Climate Change - The European Union and Kenya Monday com... http://t.co/P6QQXKRri0 http://t.co/dOxIhwLHL9</td>\n",
       "      <td>139</td>\n",
       "      <td>0.525180</td>\n",
       "      <td>6.777778</td>\n",
       "      <td>704.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43101</th>\n",
       "      <td>Did the Black Lives Matter climate protest backfire? | Climate Home - climate change news https://t.co/OGnz5451a5 via @ClimateHome</td>\n",
       "      <td>130</td>\n",
       "      <td>0.576923</td>\n",
       "      <td>6.277778</td>\n",
       "      <td>705.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14043</th>\n",
       "      <td>Former EU ambassador to Tanzania Mr.Tim Clarke, addressing the conference concerning climate change adaptation https://t.co/GsGt4TrHGl</td>\n",
       "      <td>134</td>\n",
       "      <td>0.708955</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>705.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17697</th>\n",
       "      <td>Donald Trump signs executive order overhauling Barack Obama&amp;apos;s attempts to slow climate change - ... Donald T...</td>\n",
       "      <td>116</td>\n",
       "      <td>0.784483</td>\n",
       "      <td>5.882353</td>\n",
       "      <td>705.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>Watch Leonardo DiCaprio's climate change documentary #BeforeTheFlood for free online \\nhttps</td>\n",
       "      <td>92</td>\n",
       "      <td>0.641304</td>\n",
       "      <td>7.272727</td>\n",
       "      <td>705.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14913</th>\n",
       "      <td>#EPA administrator Scott Pruitt doesn't believe carbon dioxide is a primary factor in climate change. https</td>\n",
       "      <td>108</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>5.750000</td>\n",
       "      <td>705.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11515</th>\n",
       "      <td>Sanders, Perry spar over climate change https://t.co/oZVaVWU8YT</td>\n",
       "      <td>63</td>\n",
       "      <td>0.523810</td>\n",
       "      <td>8.142857</td>\n",
       "      <td>705.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16714</th>\n",
       "      <td>Teens suing U.S. over climate change ask for Exxon's 'Wayne Tracker' emails | Reuters #SmartNews https://t.co/f0yD5rIXZo</td>\n",
       "      <td>120</td>\n",
       "      <td>0.508333</td>\n",
       "      <td>6.562500</td>\n",
       "      <td>706.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36823</th>\n",
       "      <td>Tony Abbott\"s business adviser Maurice Newman says climate change is a hoax led by the UN http://t.co/PWltaVuVXv via @brisbanetimes</td>\n",
       "      <td>131</td>\n",
       "      <td>0.580153</td>\n",
       "      <td>5.947368</td>\n",
       "      <td>706.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20703</th>\n",
       "      <td>Flood-ridden Nigeria farmers need more help adapting to climate change</td>\n",
       "      <td>71</td>\n",
       "      <td>0.845070</td>\n",
       "      <td>6.100000</td>\n",
       "      <td>706.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>406</th>\n",
       "      <td>How a rapper is tackling climate change - Deutsche Welle: Deutsche WelleHow a rapper is tackling climate chan... https://t.co/jHjasxglKs</td>\n",
       "      <td>136</td>\n",
       "      <td>0.661765</td>\n",
       "      <td>6.210526</td>\n",
       "      <td>706.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16828</th>\n",
       "      <td>@GaryLineker BBC seems very proBrexit to me......these 70 MPs are like climate change deniers complaining about not getting airtime</td>\n",
       "      <td>131</td>\n",
       "      <td>0.709924</td>\n",
       "      <td>5.947368</td>\n",
       "      <td>707.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39794</th>\n",
       "      <td>Great morning listen. \\nAll encompassing. Completely candid.\\n@ITK_CanadaInuit President Natan Obed on @TheCurrentCBC</td>\n",
       "      <td>116</td>\n",
       "      <td>0.594828</td>\n",
       "      <td>7.846154</td>\n",
       "      <td>707.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23479</th>\n",
       "      <td>From heatwaves to hurricanes, floods to famine: seven climate change hotspots https://t.co/hXjAD92lEx</td>\n",
       "      <td>101</td>\n",
       "      <td>0.643564</td>\n",
       "      <td>7.500000</td>\n",
       "      <td>707.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4441</th>\n",
       "      <td>Republicans plan multi-billion dollar climate budget raid | Climate Home - climate change news https</td>\n",
       "      <td>101</td>\n",
       "      <td>0.772277</td>\n",
       "      <td>5.733333</td>\n",
       "      <td>707.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17822</th>\n",
       "      <td>Gov. Jerry Brown calls Trump energy plan a 'colossal mistake' that will galvanize climate change activists https://t.co/GnWwHsBoya</td>\n",
       "      <td>130</td>\n",
       "      <td>0.615385</td>\n",
       "      <td>6.705882</td>\n",
       "      <td>707.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43824</th>\n",
       "      <td>Presidential #debate ignores climate change ... again https</td>\n",
       "      <td>60</td>\n",
       "      <td>0.616667</td>\n",
       "      <td>6.500000</td>\n",
       "      <td>707.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15802</th>\n",
       "      <td>Deforestation 2nd largest contributor to climate change #sxsw17 https://t.co/uuDMkJ6WH1</td>\n",
       "      <td>87</td>\n",
       "      <td>0.528736</td>\n",
       "      <td>8.777778</td>\n",
       "      <td>707.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6908</th>\n",
       "      <td>Retweeted Bernie Sanders (@SenSanders):\\n\\nWe have a president-elect who doesn't believe in climate change.... https://t.co/lt104Ysnyd</td>\n",
       "      <td>132</td>\n",
       "      <td>0.560606</td>\n",
       "      <td>7.800000</td>\n",
       "      <td>708.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41652</th>\n",
       "      <td>Someone mapped the Twitter Climate Change convo. Legit amazing. Fave Aus tweeters feature. https</td>\n",
       "      <td>97</td>\n",
       "      <td>0.773196</td>\n",
       "      <td>5.928571</td>\n",
       "      <td>708.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38446</th>\n",
       "      <td>Nigeria mitigating effects of climate change, says Buhari http://t.co/Z4WcYppeVF</td>\n",
       "      <td>80</td>\n",
       "      <td>0.612500</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>708.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23178</th>\n",
       "      <td>'Climate change truth' | My LTE in the @CurryPilot debunking an Oregonian climate change denier</td>\n",
       "      <td>96</td>\n",
       "      <td>0.625000</td>\n",
       "      <td>5.400000</td>\n",
       "      <td>708.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32010</th>\n",
       "      <td>Worst-case global warming scenarios not credible: study https://t.co/nxZbbnPkqh #ClimateHustle</td>\n",
       "      <td>94</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>9.555556</td>\n",
       "      <td>708.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42825</th>\n",
       "      <td>@FayMaur @davrosz . Roberts on Climate Change if Pauline Hanson believes in Climate Change he\"ll bow work that out ? https://t.co/vUrNDLBRrs</td>\n",
       "      <td>140</td>\n",
       "      <td>0.557143</td>\n",
       "      <td>5.714286</td>\n",
       "      <td>708.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16064</th>\n",
       "      <td>Budget calls cuts from State, USAID\\nProposal would scrap climate change fund, refugee assistance https://t.co/iKQGBjsohL @NafeesaSyeed</td>\n",
       "      <td>134</td>\n",
       "      <td>0.604478</td>\n",
       "      <td>7.437500</td>\n",
       "      <td>708.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16756</th>\n",
       "      <td>JJN report reveals devastating impact of climate change on livelihoods https://t.co/28E5BsXPwb</td>\n",
       "      <td>94</td>\n",
       "      <td>0.648936</td>\n",
       "      <td>7.636364</td>\n",
       "      <td>708.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5417</th>\n",
       "      <td>US climate change campaigner dies snorkeling at Great Barrier... https://t.co/S1ktZvPUIh #GreatBarrierReef</td>\n",
       "      <td>106</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>8.727273</td>\n",
       "      <td>709.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8734</th>\n",
       "      <td>Donald Trump picks climate change sceptic Scott Pruitt to lead EPA https://t.co/3ePmuXg2jF</td>\n",
       "      <td>90</td>\n",
       "      <td>0.622222</td>\n",
       "      <td>6.583333</td>\n",
       "      <td>709.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11435</th>\n",
       "      <td>@MrAnthonyWood @MSNBC damn one Democrat asked price about global warming \\n.smh</td>\n",
       "      <td>78</td>\n",
       "      <td>0.551282</td>\n",
       "      <td>6.090909</td>\n",
       "      <td>709.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29913</th>\n",
       "      <td>Creeping climate change brings warmer falls, drier summers to northwestern Ontario https</td>\n",
       "      <td>89</td>\n",
       "      <td>0.797753</td>\n",
       "      <td>6.416667</td>\n",
       "      <td>709.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4511</th>\n",
       "      <td>US climate change campaigner dies snorkeling at Great Barrier... #GreatBarrierReef https://t.co/yC0KZ2yxae</td>\n",
       "      <td>106</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>8.727273</td>\n",
       "      <td>709.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37014</th>\n",
       "      <td>Bonn UN talks seek to trim unwieldy climate change plan - Reuters UK http://t.co/VMSdNAJg5z</td>\n",
       "      <td>91</td>\n",
       "      <td>0.604396</td>\n",
       "      <td>5.571429</td>\n",
       "      <td>709.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8572</th>\n",
       "      <td>#CLIMATEchange #p2 RT Trump picks climate change sceptic Scott Pruitt to lead EPA https</td>\n",
       "      <td>88</td>\n",
       "      <td>0.590909</td>\n",
       "      <td>5.285714</td>\n",
       "      <td>710.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41169</th>\n",
       "      <td>Climate Change Impacts Human Rights, Says UN Special Rapporteur https://t.co/eR6xOKlgGS</td>\n",
       "      <td>87</td>\n",
       "      <td>0.620690</td>\n",
       "      <td>7.800000</td>\n",
       "      <td>710.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37927</th>\n",
       "      <td>Climate change and the California drought cause farmers to experience record ... - http://t.co/MmqnCg4Uum #GoogleAlerts</td>\n",
       "      <td>119</td>\n",
       "      <td>0.554622</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>710.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                               text  \\\n",
       "24823    Late Pleistocene-Holocene vegetation and climate change in the Middle Kalahari, Lake Ngami, Botswana\\nhttps://t.co/cAvWNIgbXE @scott_louis   \n",
       "23028                                                                @Philosocrat Oh I believe in climate change. I just don't believe in Delaware.   \n",
       "30626                     Study finds that global warming exacerbates refugee crises | John Abraham https://t.co/KlTmfLCjAd https://t.co/90rdZL1ths   \n",
       "894                                                                  .@OfficialJLD endorses Hillary Clinton, slams Donald Trump over climate change   \n",
       "11257   Mike Pompeo, Trump's CIA pick, evades questions about climate change and global instability https://t.co/TPzl33TqrV https://t.co/BHYxZFpNBs   \n",
       "25188          Peru builds up wetland resilience to reinforce indigenous response to climate change https://t.co/x0XAxzKn2j https://t.co/58oYUWGVTz   \n",
       "30668                                             Study finds that global warming exacerbates refugee crises | John Abraham https://t.co/9FDzShx65z   \n",
       "1749   Ralph Cicerone, former UC Irvine chancellor who studied the causes of climate change, dies at 73 - Los Angeles Times https://t.co/fqGVQGVi80   \n",
       "40678  Unanticipated consequence of climate change: disease carrying mosquitoes & tics spread north. #Dengue, #Zika, #Lymedisease, #Chikungunya etc   \n",
       "30152            Eying vulnerability, Calif. studying how climate change will affect state highways https://t.co/UoI0H4Q4oh https://t.co/Tju0OGHQV4   \n",
       "9779       #trump is a climate change denier, a chronic liar, an unbridled man given to conspiracy theories & wild ideas. Save us #ElectoralCollege   \n",
       "39597                               Floods Displace Thousands in Uganda I CLIMATE CHANGE IN AFRICA I https://t.co/PKXlD5VNYK #uganda via @allafrica   \n",
       "15603                              Coal lobbyist Trump attorney seeks to bypass US kids' climate lawsuit | Climate Home - climate change news https   \n",
       "23457                                                   @tomcolicchio @MikeBloomberg And donating millions in support of combatting climate change?   \n",
       "8653                                                                        Biggest pet peeve: Republicans pretending climate change doesn't exist:   \n",
       "22567                  @JEPomfret @USEmbassyBJ @USEmbassyKabul Silly twit! Tricks are for kids! Resigning over climate change? What a silly rabbit!   \n",
       "33284                                                                          @silvercreeklady @thehill The US gop don't believe in climate change   \n",
       "40658                                           Tonight Sanders condemned Republican presidential candidates for their #climate change denial https   \n",
       "7848                                      #Manifest US investors, companies back Paris climate change agreement. Read Blog: https://t.co/3d6jNVT1gu   \n",
       "18653     UN climate chief bites tongue after Trump de-funding threat | Climate Home - climate change news https://t.co/tLj9TaMagv via @ClimateHome   \n",
       "16110                       Donald Trump proposes huge EPA budget cut to stop environment agency researching climate change https://t.co/94MGB1O57i   \n",
       "41355                                                               The moment Hillary Clinton lost her temper with a climate change activist https   \n",
       "32934                                                           Zim ill-prepared for climate change - The Zimbabwe Standard https://t.co/KwaVtykhcU   \n",
       "4300        Dems MUST fight 2 end income inequality, raise min wage, battle climate change, safe gun cntrl, justice reform, state&fed level w/o GOP   \n",
       "25974                                                                  Neil DeGrasse Tyson slams climate change deniers amid Hurricane Harvey https   \n",
       "38696   Kenya, EU Steps Up Cooperation On Climate Change - The European Union and Kenya Monday com... http://t.co/P6QQXKRri0 http://t.co/dOxIhwLHL9   \n",
       "43101            Did the Black Lives Matter climate protest backfire? | Climate Home - climate change news https://t.co/OGnz5451a5 via @ClimateHome   \n",
       "14043        Former EU ambassador to Tanzania Mr.Tim Clarke, addressing the conference concerning climate change adaptation https://t.co/GsGt4TrHGl   \n",
       "17697                          Donald Trump signs executive order overhauling Barack Obama&apos;s attempts to slow climate change - ... Donald T...   \n",
       "61                                                     Watch Leonardo DiCaprio's climate change documentary #BeforeTheFlood for free online \\nhttps   \n",
       "14913                                   #EPA administrator Scott Pruitt doesn't believe carbon dioxide is a primary factor in climate change. https   \n",
       "11515                                                                               Sanders, Perry spar over climate change https://t.co/oZVaVWU8YT   \n",
       "16714                      Teens suing U.S. over climate change ask for Exxon's 'Wayne Tracker' emails | Reuters #SmartNews https://t.co/f0yD5rIXZo   \n",
       "36823           Tony Abbott\"s business adviser Maurice Newman says climate change is a hoax led by the UN http://t.co/PWltaVuVXv via @brisbanetimes   \n",
       "20703                                                                        Flood-ridden Nigeria farmers need more help adapting to climate change   \n",
       "406        How a rapper is tackling climate change - Deutsche Welle: Deutsche WelleHow a rapper is tackling climate chan... https://t.co/jHjasxglKs   \n",
       "16828           @GaryLineker BBC seems very proBrexit to me......these 70 MPs are like climate change deniers complaining about not getting airtime   \n",
       "39794                         Great morning listen. \\nAll encompassing. Completely candid.\\n@ITK_CanadaInuit President Natan Obed on @TheCurrentCBC   \n",
       "23479                                         From heatwaves to hurricanes, floods to famine: seven climate change hotspots https://t.co/hXjAD92lEx   \n",
       "4441                                           Republicans plan multi-billion dollar climate budget raid | Climate Home - climate change news https   \n",
       "17822            Gov. Jerry Brown calls Trump energy plan a 'colossal mistake' that will galvanize climate change activists https://t.co/GnWwHsBoya   \n",
       "43824                                                                                   Presidential #debate ignores climate change ... again https   \n",
       "15802                                                       Deforestation 2nd largest contributor to climate change #sxsw17 https://t.co/uuDMkJ6WH1   \n",
       "6908         Retweeted Bernie Sanders (@SenSanders):\\n\\nWe have a president-elect who doesn't believe in climate change.... https://t.co/lt104Ysnyd   \n",
       "41652                                              Someone mapped the Twitter Climate Change convo. Legit amazing. Fave Aus tweeters feature. https   \n",
       "38446                                                              Nigeria mitigating effects of climate change, says Buhari http://t.co/Z4WcYppeVF   \n",
       "23178                                               'Climate change truth' | My LTE in the @CurryPilot debunking an Oregonian climate change denier   \n",
       "32010                                                Worst-case global warming scenarios not credible: study https://t.co/nxZbbnPkqh #ClimateHustle   \n",
       "42825  @FayMaur @davrosz . Roberts on Climate Change if Pauline Hanson believes in Climate Change he\"ll bow work that out ? https://t.co/vUrNDLBRrs   \n",
       "16064       Budget calls cuts from State, USAID\\nProposal would scrap climate change fund, refugee assistance https://t.co/iKQGBjsohL @NafeesaSyeed   \n",
       "16756                                                JJN report reveals devastating impact of climate change on livelihoods https://t.co/28E5BsXPwb   \n",
       "5417                                     US climate change campaigner dies snorkeling at Great Barrier... https://t.co/S1ktZvPUIh #GreatBarrierReef   \n",
       "8734                                                     Donald Trump picks climate change sceptic Scott Pruitt to lead EPA https://t.co/3ePmuXg2jF   \n",
       "11435                                                               @MrAnthonyWood @MSNBC damn one Democrat asked price about global warming \\n.smh   \n",
       "29913                                                      Creeping climate change brings warmer falls, drier summers to northwestern Ontario https   \n",
       "4511                                     US climate change campaigner dies snorkeling at Great Barrier... #GreatBarrierReef https://t.co/yC0KZ2yxae   \n",
       "37014                                                   Bonn UN talks seek to trim unwieldy climate change plan - Reuters UK http://t.co/VMSdNAJg5z   \n",
       "8572                                                        #CLIMATEchange #p2 RT Trump picks climate change sceptic Scott Pruitt to lead EPA https   \n",
       "41169                                                       Climate Change Impacts Human Rights, Says UN Special Rapporteur https://t.co/eR6xOKlgGS   \n",
       "37927                       Climate change and the California drought cause farmers to experience record ... - http://t.co/MmqnCg4Uum #GoogleAlerts   \n",
       "\n",
       "       length  word fraction  token length  english score  \n",
       "24823     137       0.620438      8.200000          700.0  \n",
       "23028      78       0.653846      5.076923          700.0  \n",
       "30626     121       0.512397      8.384615          700.0  \n",
       "894        79       0.696203      6.900000          700.0  \n",
       "11257     139       0.546763      8.333333          700.0  \n",
       "25188     132       0.553030      8.500000          701.0  \n",
       "30668      97       0.639175      7.166667          701.0  \n",
       "1749      140       0.657143      5.714286          701.0  \n",
       "40678     140       0.585714      7.294118          701.0  \n",
       "30152     130       0.538462      9.076923          701.0  \n",
       "9779      136       0.647059      5.227273          701.0  \n",
       "39597     111       0.513514      6.466667          701.0  \n",
       "15603     113       0.761062      4.947368          701.0  \n",
       "23457      91       0.582418      7.363636          701.0  \n",
       "8653       71       0.845070      7.000000          702.0  \n",
       "22567     124       0.540323      6.352941          702.0  \n",
       "33284      68       0.500000      5.900000          703.0  \n",
       "40658     100       0.750000      7.333333          703.0  \n",
       "7848      105       0.561905      7.833333          703.0  \n",
       "18653     137       0.591241      6.263158          703.0  \n",
       "16110     119       0.689076      7.000000          703.0  \n",
       "41355      80       0.775000      5.153846          704.0  \n",
       "32934      83       0.590361      7.400000          704.0  \n",
       "4300      135       0.785185      5.181818          704.0  \n",
       "25974      77       0.792208      6.000000          704.0  \n",
       "38696     139       0.525180      6.777778          704.0  \n",
       "43101     130       0.576923      6.277778          705.0  \n",
       "14043     134       0.708955      8.000000          705.0  \n",
       "17697     116       0.784483      5.882353          705.0  \n",
       "61         92       0.641304      7.272727          705.0  \n",
       "14913     108       0.750000      5.750000          705.0  \n",
       "11515      63       0.523810      8.142857          705.0  \n",
       "16714     120       0.508333      6.562500          706.0  \n",
       "36823     131       0.580153      5.947368          706.0  \n",
       "20703      71       0.845070      6.100000          706.0  \n",
       "406       136       0.661765      6.210526          706.0  \n",
       "16828     131       0.709924      5.947368          707.0  \n",
       "39794     116       0.594828      7.846154          707.0  \n",
       "23479     101       0.643564      7.500000          707.0  \n",
       "4441      101       0.772277      5.733333          707.0  \n",
       "17822     130       0.615385      6.705882          707.0  \n",
       "43824      60       0.616667      6.500000          707.0  \n",
       "15802      87       0.528736      8.777778          707.0  \n",
       "6908      132       0.560606      7.800000          708.0  \n",
       "41652      97       0.773196      5.928571          708.0  \n",
       "38446      80       0.612500      8.000000          708.0  \n",
       "23178      96       0.625000      5.400000          708.0  \n",
       "32010      94       0.500000      9.555556          708.0  \n",
       "42825     140       0.557143      5.714286          708.0  \n",
       "16064     134       0.604478      7.437500          708.0  \n",
       "16756      94       0.648936      7.636364          708.0  \n",
       "5417      106       0.500000      8.727273          709.0  \n",
       "8734       90       0.622222      6.583333          709.0  \n",
       "11435      78       0.551282      6.090909          709.0  \n",
       "29913      89       0.797753      6.416667          709.0  \n",
       "4511      106       0.500000      8.727273          709.0  \n",
       "37014      91       0.604396      5.571429          709.0  \n",
       "8572       88       0.590909      5.285714          710.0  \n",
       "41169      87       0.620690      7.800000          710.0  \n",
       "37927     119       0.554622      7.000000          710.0  "
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = trim_dataset_by(\n",
    "    df, \n",
    "    \n",
    "    # sort_by='length',\n",
    "    # sort_by='word fraction',\n",
    "    # sort_by='token length',\n",
    "    sort_by='english score',\n",
    "    ascending=True,\n",
    "    \n",
    "    length=(60, None),\n",
    "    word_fraction=(0.5, None),\n",
    "    token_length=(4, 10),\n",
    "    english_score=(700, None),\n",
    ")\n",
    "a.drop('id', axis=1)[:60]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bda72f0f-ca2a-4aab-8ebe-811ff964452c",
   "metadata": {},
   "source": [
    "Drop similar tweets (it is expensive procedure, that is why we are running it in last order)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "be80e754-66b5-4607-99b0-ae037c1e43f4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:38:45.187716Z",
     "iopub.status.busy": "2024-01-08T15:38:45.187456Z",
     "iopub.status.idle": "2024-01-08T15:38:53.753740Z",
     "shell.execute_reply": "2024-01-08T15:38:53.752600Z",
     "shell.execute_reply.started": "2024-01-08T15:38:45.187671Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time elapsed: 8s\n",
      "samples: 10236, similar samples detected: 764 (6.95%)\n"
     ]
    }
   ],
   "source": [
    "df = drop_similar(supersample(a, CLIMATE_CHANGE), max_similarity=0.8)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "27112df3-00ab-4753-8a5e-c20fb502d02e",
   "metadata": {},
   "source": [
    "Add topic column, sample the pre-defined number of tweets, and create the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "a811dfb2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:38:53.756021Z",
     "iopub.status.busy": "2024-01-08T15:38:53.755227Z",
     "iopub.status.idle": "2024-01-08T15:38:53.784470Z",
     "shell.execute_reply": "2024-01-08T15:38:53.783726Z",
     "shell.execute_reply.started": "2024-01-08T15:38:53.755984Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 3)"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = select_columns(df.copy(), ['text', 'id'])\n",
    "df = sample(df, CLIMATE_CHANGE)\n",
    "df['topic'] = 'climate change'\n",
    "dataset = df.copy()\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a914231c-9d0f-4ade-b176-d58d454225a0",
   "metadata": {},
   "source": [
    "# Self-driving cars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "577a8850-44a4-4e8b-a1db-018312607fe6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:38:53.785530Z",
     "iopub.status.busy": "2024-01-08T15:38:53.785339Z",
     "iopub.status.idle": "2024-01-08T15:38:53.840914Z",
     "shell.execute_reply": "2024-01-08T15:38:53.840160Z",
     "shell.execute_reply.started": "2024-01-08T15:38:53.785512Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7146, 2)"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = load_dataset('selfdriving_cars.csv', source=True, encoding='ISO-8859-1')\n",
    "df = select_columns(df, ['text', 'sentiment'])\n",
    "df.drop_duplicates(subset='text', inplace=True)\n",
    "df['text'] = df['text'].apply(remove_invalid_chars)\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "248c2680-b9de-436a-9aff-10c9526681c9",
   "metadata": {},
   "source": [
    "Remove unrelated (noisy) tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "51049635-c946-4add-b1d1-5378cda2bcfb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:38:53.842035Z",
     "iopub.status.busy": "2024-01-08T15:38:53.841843Z",
     "iopub.status.idle": "2024-01-08T15:38:53.861886Z",
     "shell.execute_reply": "2024-01-08T15:38:53.861225Z",
     "shell.execute_reply.started": "2024-01-08T15:38:53.842016Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6933, 1)"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df[df['sentiment'] != 'not_relevant']\n",
    "df = select_columns(df, ['text'])\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6d1574d7-a887-4ee6-8a63-a3470cf5030f",
   "metadata": {},
   "source": [
    "Explore dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "7ee0cf07-f7bb-4e8d-a430-72d183b97d6e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:38:53.862808Z",
     "iopub.status.busy": "2024-01-08T15:38:53.862632Z",
     "iopub.status.idle": "2024-01-08T15:38:53.882801Z",
     "shell.execute_reply": "2024-01-08T15:38:53.881994Z",
     "shell.execute_reply.started": "2024-01-08T15:38:53.862790Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5122</th>\n",
       "      <td>We invented the car and now, we're on the verge of introducing the self-driving car. See our breakthroughs: http://t.co/EDurd8e5p0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3180</th>\n",
       "      <td>GREAT day! I learned in my workshop this AM that the Google car has FINALLY driven by our home! #atleastthelawnwascut http://t.co/xx5UywN3hH</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2068</th>\n",
       "      <td>The future is now. #google driverless car. @ Mill Valley, CA http://t.co/jFRksD1L60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3184</th>\n",
       "      <td>@justinvl damn, didnÌ¢âÂã¢t think of that. I could totally wax and polish the Google car!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6218</th>\n",
       "      <td>@kurtvarner I was joking dude, can't call it focus with self-driving cars et al. Wanna build product millions of people will love? U at HD?</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                              text\n",
       "5122            We invented the car and now, we're on the verge of introducing the self-driving car. See our breakthroughs: http://t.co/EDurd8e5p0\n",
       "3180  GREAT day! I learned in my workshop this AM that the Google car has FINALLY driven by our home! #atleastthelawnwascut http://t.co/xx5UywN3hH\n",
       "2068                                                           The future is now. #google driverless car. @ Mill Valley, CA http://t.co/jFRksD1L60\n",
       "3184                                                     @justinvl damn, didnÌ¢âÂã¢t think of that. I could totally wax and polish the Google car!\n",
       "6218   @kurtvarner I was joking dude, can't call it focus with self-driving cars et al. Wanna build product millions of people will love? U at HD?"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "0f33b371-e3ad-4685-af87-95629b9112ed",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:38:53.883883Z",
     "iopub.status.busy": "2024-01-08T15:38:53.883684Z",
     "iopub.status.idle": "2024-01-08T15:38:54.161340Z",
     "shell.execute_reply": "2024-01-08T15:38:54.160604Z",
     "shell.execute_reply.started": "2024-01-08T15:38:53.883862Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5899, 1)"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = preprocessing_pipeline(df)\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "9a8d07ef-58c1-4218-8f7c-b025f4f219a4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:38:54.162797Z",
     "iopub.status.busy": "2024-01-08T15:38:54.162216Z",
     "iopub.status.idle": "2024-01-08T15:38:54.288424Z",
     "shell.execute_reply": "2024-01-08T15:38:54.287691Z",
     "shell.execute_reply.started": "2024-01-08T15:38:54.162776Z"
    }
   },
   "outputs": [],
   "source": [
    "add_stats(df);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "c0dfb90b-f591-4ac5-bab3-06de88b516a0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:38:54.292207Z",
     "iopub.status.busy": "2024-01-08T15:38:54.291707Z",
     "iopub.status.idle": "2024-01-08T15:38:54.323879Z",
     "shell.execute_reply": "2024-01-08T15:38:54.323131Z",
     "shell.execute_reply.started": "2024-01-08T15:38:54.292184Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "samples: 4721, reduction: 19.97%\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>length</th>\n",
       "      <th>word fraction</th>\n",
       "      <th>token length</th>\n",
       "      <th>english score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6010</th>\n",
       "      <td>@SuhelenG quionda vas a orrar tu dinero para los self driving cars coming soon 2 ur area?L.A......</td>\n",
       "      <td>98</td>\n",
       "      <td>0.653061</td>\n",
       "      <td>4.823529</td>\n",
       "      <td>455.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1641</th>\n",
       "      <td>Burkhard Bilger: Inside Google's Driverless Car http://t.co/jNoW1cBonl via @NewYorker</td>\n",
       "      <td>85</td>\n",
       "      <td>0.505882</td>\n",
       "      <td>8.555556</td>\n",
       "      <td>474.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>461</th>\n",
       "      <td>@JimSpohrer tells a gr8 story on cab drivers askd abt driverless cars.</td>\n",
       "      <td>70</td>\n",
       "      <td>0.657143</td>\n",
       "      <td>4.916667</td>\n",
       "      <td>496.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3141</th>\n",
       "      <td>Aerial Google self driving car @ Computer History Museum http://t.co/TLrGcELkUV</td>\n",
       "      <td>79</td>\n",
       "      <td>0.594937</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>498.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4427</th>\n",
       "      <td>In self-driving milestone, Google's autonomous cars are mastering city streets http://t.co/c6WpyiwLXy http://t.co/2IbOmkdCMv</td>\n",
       "      <td>124</td>\n",
       "      <td>0.532258</td>\n",
       "      <td>9.416667</td>\n",
       "      <td>499.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1135</th>\n",
       "      <td>Sergey Brin unveils new Google driverless car prototype http://t.co/CVON3FADCL</td>\n",
       "      <td>78</td>\n",
       "      <td>0.615385</td>\n",
       "      <td>7.777778</td>\n",
       "      <td>505.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3764</th>\n",
       "      <td>Autonomous driving solutions, Lexus smart co pilot. http://t.co/Iqo0NGr2</td>\n",
       "      <td>72</td>\n",
       "      <td>0.597222</td>\n",
       "      <td>8.125000</td>\n",
       "      <td>512.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2396</th>\n",
       "      <td>Google car @ Computer History Museum http://t.co/jApPaWK2rb</td>\n",
       "      <td>59</td>\n",
       "      <td>0.508475</td>\n",
       "      <td>7.571429</td>\n",
       "      <td>512.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2564</th>\n",
       "      <td>Oh hey Google Car! @ MDT Metromover - Bayfront Park Station http://t.co/4hCAG7Lr</td>\n",
       "      <td>80</td>\n",
       "      <td>0.575000</td>\n",
       "      <td>5.750000</td>\n",
       "      <td>519.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>959</th>\n",
       "      <td>Brilliant Smithsonian magazine award winners. Here w/Sebastian Thrun - driverless cars, Udacity. http://t.co/AtPH3FsV</td>\n",
       "      <td>117</td>\n",
       "      <td>0.683761</td>\n",
       "      <td>8.076923</td>\n",
       "      <td>525.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>909</th>\n",
       "      <td>Real challenge to Google's driverless cars: Israeli startup Mobileye raises $400M w/ cheaper tech http://t.co/hqLGazafyQ via @zite</td>\n",
       "      <td>130</td>\n",
       "      <td>0.607692</td>\n",
       "      <td>6.705882</td>\n",
       "      <td>528.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6758</th>\n",
       "      <td>Craig Tieman inventor presenting Delphi's Connected Car @WomenandHiTech. Very cool. http://t.co/oUm6egLL44</td>\n",
       "      <td>106</td>\n",
       "      <td>0.528302</td>\n",
       "      <td>8.727273</td>\n",
       "      <td>542.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>WO2013070799A1 Automated Vehicle Conveyance Apparatus Transportation System #B61B #B61B13 http://t.co/HToCWDVEZM</td>\n",
       "      <td>112</td>\n",
       "      <td>0.517857</td>\n",
       "      <td>10.300000</td>\n",
       "      <td>549.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1818</th>\n",
       "      <td>Driverless cars, ubiquitous wireless internet &amp; electricity, growing &amp; printing organs... LENR next? #theFutureisNOW #theSingularityIsNear</td>\n",
       "      <td>138</td>\n",
       "      <td>0.579710</td>\n",
       "      <td>8.266667</td>\n",
       "      <td>556.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2060</th>\n",
       "      <td>CONCORD: DMV handing out testing permits for driver-less cars: DMV handing out testing permits for driver-less cars http://t.co/gPqxDzVbSt</td>\n",
       "      <td>138</td>\n",
       "      <td>0.688406</td>\n",
       "      <td>6.722222</td>\n",
       "      <td>561.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5651</th>\n",
       "      <td>#dashitcray Robot chauffeurs: Google self-driving cars log 300K miles | VentureBeat: http://t.co/isLeDhps</td>\n",
       "      <td>105</td>\n",
       "      <td>0.523810</td>\n",
       "      <td>7.833333</td>\n",
       "      <td>565.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2059</th>\n",
       "      <td>CONCORD: DMV handing out testing permits for driver-less cars: DMV handing out testing permits for driver-less cars http://t.co/vEE13dcFme</td>\n",
       "      <td>138</td>\n",
       "      <td>0.688406</td>\n",
       "      <td>6.722222</td>\n",
       "      <td>565.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3716</th>\n",
       "      <td>Ayo dey need 2 chill wit dat Google car dat drive itself. U know da cops gon make dat shit drive sum nigga azz str8 2 da police station lmao</td>\n",
       "      <td>140</td>\n",
       "      <td>0.764286</td>\n",
       "      <td>3.700000</td>\n",
       "      <td>568.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117</th>\n",
       "      <td>Google unveils fully autonomous, hands-free car prototype [w/videos] http://t.co/KrauTretw1</td>\n",
       "      <td>91</td>\n",
       "      <td>0.538462</td>\n",
       "      <td>9.222222</td>\n",
       "      <td>571.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6845</th>\n",
       "      <td>@theGreaterMarin @mikesonn Oh so driverless mini-transit not driverless cars</td>\n",
       "      <td>76</td>\n",
       "      <td>0.552632</td>\n",
       "      <td>7.555556</td>\n",
       "      <td>573.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>242</th>\n",
       "      <td>Minnesota Autonomous Vehicle Meetup. (@ Hack Factory of Minnesota - @hackfactorymn) http://t.co/YV0myVzRLG</td>\n",
       "      <td>106</td>\n",
       "      <td>0.509434</td>\n",
       "      <td>7.916667</td>\n",
       "      <td>580.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1144</th>\n",
       "      <td>#VisionZero #ActOnClimate net zero best NYT: Google's Next Phase in Driverless Cars: No Brakes or Steering Wheel http://t.co/M5iDEIxHwo</td>\n",
       "      <td>135</td>\n",
       "      <td>0.511111</td>\n",
       "      <td>6.555556</td>\n",
       "      <td>582.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4446</th>\n",
       "      <td>Google's self-driving cars understand cyclists gestures. http://t.co/zTDjG8keyU</td>\n",
       "      <td>79</td>\n",
       "      <td>0.607595</td>\n",
       "      <td>10.428571</td>\n",
       "      <td>585.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>768</th>\n",
       "      <td>@jaymohr37 driverless cars. Pssssh, been there, done that s/ David Hasselhoff #jaymohrsports</td>\n",
       "      <td>92</td>\n",
       "      <td>0.576087</td>\n",
       "      <td>6.750000</td>\n",
       "      <td>590.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6875</th>\n",
       "      <td>not driving Google's self driving car @ Computer History Museum http://t.co/ReyX2g1rvK</td>\n",
       "      <td>86</td>\n",
       "      <td>0.604651</td>\n",
       "      <td>6.909091</td>\n",
       "      <td>592.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2089</th>\n",
       "      <td>@cnni Driverless truck?! Perfect 'car (((Bomb))). I'll take 2 4 AIPC rats nest in Hymie town &amp; DC.</td>\n",
       "      <td>98</td>\n",
       "      <td>0.551020</td>\n",
       "      <td>4.500000</td>\n",
       "      <td>594.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3120</th>\n",
       "      <td>Google Self-Driving car. Adeus, motoristas. @ Google Corp http://t.co/wZWIcfpeTq</td>\n",
       "      <td>80</td>\n",
       "      <td>0.562500</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>595.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5883</th>\n",
       "      <td>Burkhard Bilger: Inside Google's self-driving car http://t.co/UUUNqwyvRS</td>\n",
       "      <td>72</td>\n",
       "      <td>0.569444</td>\n",
       "      <td>9.428571</td>\n",
       "      <td>599.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2989</th>\n",
       "      <td>O hai @google self-driving car! Testing near ocean beach I see. http://t.co/0n44ce7SWN</td>\n",
       "      <td>86</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>6.250000</td>\n",
       "      <td>601.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1221</th>\n",
       "      <td>#IBMInnovate @bosch expanding system complexity #internetofthings Driverless cars are coming!</td>\n",
       "      <td>93</td>\n",
       "      <td>0.516129</td>\n",
       "      <td>8.400000</td>\n",
       "      <td>603.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3995</th>\n",
       "      <td>VentureBeat: Silicon Valley VCs predict 2013 trends: Space, robots, self-driving cars. http://t.co/prxJoctL</td>\n",
       "      <td>107</td>\n",
       "      <td>0.616822</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>608.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107</th>\n",
       "      <td>Finally, a very realistic timeline for full autonomous car capability. Hats off, @MorganStanley #AutoForum http://t.co/ZwemgV5JF8</td>\n",
       "      <td>129</td>\n",
       "      <td>0.511628</td>\n",
       "      <td>7.666667</td>\n",
       "      <td>612.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6469</th>\n",
       "      <td>#Volvo begins testing self-driving cars in Sweden http://t.co/ZNloqs6lQ3 Subscribers via @gizmag</td>\n",
       "      <td>96</td>\n",
       "      <td>0.520833</td>\n",
       "      <td>7.818182</td>\n",
       "      <td>612.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5436</th>\n",
       "      <td>Tech Trend: Mercedes-Benz Unveils Self-Driving 'Future Truck' on Germany's Autobahn. http://t.co/DWxkQaGcGh #FutureReality</td>\n",
       "      <td>122</td>\n",
       "      <td>0.508197</td>\n",
       "      <td>9.250000</td>\n",
       "      <td>617.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1894</th>\n",
       "      <td>California Becomes Third State To Legalize Driverless Cars. http://t.co/Q8gveWFi (RT: @singularityhub)</td>\n",
       "      <td>102</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>8.363636</td>\n",
       "      <td>618.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>824</th>\n",
       "      <td>Could Google's interest in driverless cars spark a resurgence in 1940/50s-style cybernetics? http://t.co/IRE1naoYeN @markoff</td>\n",
       "      <td>124</td>\n",
       "      <td>0.524194</td>\n",
       "      <td>7.928571</td>\n",
       "      <td>619.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3722</th>\n",
       "      <td>Google unveils self-driving car prototype http://t.co/51zdTNYIRJ oooo</td>\n",
       "      <td>69</td>\n",
       "      <td>0.579710</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>620.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289</th>\n",
       "      <td>Auto psychedelia: Lidar heat map from #Ford Fusion Hybrid Autonomous Research Vehicle http://t.co/AqOkQ3DsPR</td>\n",
       "      <td>108</td>\n",
       "      <td>0.629630</td>\n",
       "      <td>7.384615</td>\n",
       "      <td>624.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4709</th>\n",
       "      <td>The self-driving car masters city-street driving | KurzweilAI http://t.co/wnMNZ6oOmz via @kurzweilainews</td>\n",
       "      <td>104</td>\n",
       "      <td>0.519231</td>\n",
       "      <td>8.545455</td>\n",
       "      <td>624.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099</th>\n",
       "      <td>Started here in Nevada! How Google Got States to Legalize Driverless Cars - http://t.co/LdNKh8sPvc http://t.co/9YQMNIk6b0</td>\n",
       "      <td>121</td>\n",
       "      <td>0.504132</td>\n",
       "      <td>7.133333</td>\n",
       "      <td>625.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>716</th>\n",
       "      <td>Google: Driverless cars are mastering city streets http://t.co/Lr1Xa9B9fw</td>\n",
       "      <td>73</td>\n",
       "      <td>0.589041</td>\n",
       "      <td>8.250000</td>\n",
       "      <td>634.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2080</th>\n",
       "      <td>Google's \"driverless\" Lexus making a pit stop. Oh, thank heaven...for refills. @ Top Of San Bruno http://t.co/3luppxWWv5</td>\n",
       "      <td>120</td>\n",
       "      <td>0.516667</td>\n",
       "      <td>6.117647</td>\n",
       "      <td>635.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3830</th>\n",
       "      <td>The Verge Self-driving cars and ocean colonies: revisit Isaac Asimov's vision of 2014 http://t.co/wL1fDz9GyF. Ocean colonies. Pre-mars???</td>\n",
       "      <td>137</td>\n",
       "      <td>0.627737</td>\n",
       "      <td>7.117647</td>\n",
       "      <td>638.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3967</th>\n",
       "      <td>Toyota FV2 - concept vehicle self driving or driving mode #verycool http://t.co/WoZfvOncmM</td>\n",
       "      <td>90</td>\n",
       "      <td>0.511111</td>\n",
       "      <td>6.583333</td>\n",
       "      <td>638.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6199</th>\n",
       "      <td>Dey tuk r jobs.. Free Google self driving ad sponsored cab to stores. http://t.co/Vd9a4nF6Vi</td>\n",
       "      <td>92</td>\n",
       "      <td>0.586957</td>\n",
       "      <td>5.642857</td>\n",
       "      <td>640.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6691</th>\n",
       "      <td>Autonomous taxi competes with crowd sourced taxi gets better name. #Goober http://t.co/cfWpw7HWDx</td>\n",
       "      <td>97</td>\n",
       "      <td>0.577320</td>\n",
       "      <td>7.166667</td>\n",
       "      <td>642.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>986</th>\n",
       "      <td>Google's own prototype driverless cars! http://t.co/Zl0McVQzN5</td>\n",
       "      <td>62</td>\n",
       "      <td>0.532258</td>\n",
       "      <td>9.500000</td>\n",
       "      <td>642.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5189</th>\n",
       "      <td>Self-driving Anki Drive racers are like futuristic slot cars... with lasers. @ CBS Interactive http://t.co/bpxZhgNIIL</td>\n",
       "      <td>117</td>\n",
       "      <td>0.641026</td>\n",
       "      <td>6.866667</td>\n",
       "      <td>642.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>717</th>\n",
       "      <td>Google: Driverless cars are mastering city streets http://t.co/AerMDvYZu5</td>\n",
       "      <td>73</td>\n",
       "      <td>0.589041</td>\n",
       "      <td>8.250000</td>\n",
       "      <td>643.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>746</th>\n",
       "      <td>Google: Driverless cars are mastering city streets http://t.co/npSCNvcni6</td>\n",
       "      <td>73</td>\n",
       "      <td>0.589041</td>\n",
       "      <td>8.250000</td>\n",
       "      <td>643.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>874</th>\n",
       "      <td>@dsucher @BrentToderian Driverless cars won't be panacea for aging in place. They won't bring human scale, community, social life to sprawl.</td>\n",
       "      <td>140</td>\n",
       "      <td>0.657143</td>\n",
       "      <td>5.714286</td>\n",
       "      <td>644.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4000</th>\n",
       "      <td>Toyota debuting self-driving cars at CES! http://t.co/SjAnNTdk Isaac Asimov (&amp; Boy's Life) had me waiting since 1990: http://t.co/FwsJBruy</td>\n",
       "      <td>138</td>\n",
       "      <td>0.507246</td>\n",
       "      <td>6.722222</td>\n",
       "      <td>644.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6720</th>\n",
       "      <td>Lasers cameras radar &amp; differential GPS will be in autonomous cars!Hi tech cruising! http://t.co/GuOBlXgfSj</td>\n",
       "      <td>107</td>\n",
       "      <td>0.644860</td>\n",
       "      <td>6.714286</td>\n",
       "      <td>646.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1131</th>\n",
       "      <td>Google building truly driverless car with no brakes, steering wheel, gas pedals http://t.co/xfS5VypsOb</td>\n",
       "      <td>102</td>\n",
       "      <td>0.647059</td>\n",
       "      <td>6.923077</td>\n",
       "      <td>651.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4407</th>\n",
       "      <td>Slate: Navigation Without Representation: D.C.'s DMV Prepares for Self-Driving Cars. http://t.co/llT8KoDlvG</td>\n",
       "      <td>107</td>\n",
       "      <td>0.635514</td>\n",
       "      <td>8.818182</td>\n",
       "      <td>652.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>562</th>\n",
       "      <td>Pa. congressman to test driverless CMU car: PITTSBURGH (AP) - A Pennsylvania congressman is planning a trip to... http://t.co/JPyUwK023u</td>\n",
       "      <td>136</td>\n",
       "      <td>0.632353</td>\n",
       "      <td>6.210526</td>\n",
       "      <td>652.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1097</th>\n",
       "      <td>Uber CEO: Expect driverless cars, record funding round in the future: Uber Technologies Inc. envisions the future... http://t.co/xbez0zNglu</td>\n",
       "      <td>139</td>\n",
       "      <td>0.669065</td>\n",
       "      <td>6.777778</td>\n",
       "      <td>656.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>324</th>\n",
       "      <td>Driverless cars!!! Argh \"@zeGamelord: Very overwhelming RT @PhantomBeta: Google is a scary organization.\"</td>\n",
       "      <td>105</td>\n",
       "      <td>0.590476</td>\n",
       "      <td>7.153846</td>\n",
       "      <td>661.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5384</th>\n",
       "      <td>Mercedes-Benz Unveils Self-Driving 'Future Truck' on Germany's Autobahn http://t.co/uktFsdHXlE</td>\n",
       "      <td>94</td>\n",
       "      <td>0.563830</td>\n",
       "      <td>9.555556</td>\n",
       "      <td>663.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4467</th>\n",
       "      <td>Self driving cars = CaaS (cars as a service) #SingularityU</td>\n",
       "      <td>58</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>4.900000</td>\n",
       "      <td>663.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                              text  \\\n",
       "6010                                            @SuhelenG quionda vas a orrar tu dinero para los self driving cars coming soon 2 ur area?L.A......   \n",
       "1641                                                         Burkhard Bilger: Inside Google's Driverless Car http://t.co/jNoW1cBonl via @NewYorker   \n",
       "461                                                                         @JimSpohrer tells a gr8 story on cab drivers askd abt driverless cars.   \n",
       "3141                                                               Aerial Google self driving car @ Computer History Museum http://t.co/TLrGcELkUV   \n",
       "4427                  In self-driving milestone, Google's autonomous cars are mastering city streets http://t.co/c6WpyiwLXy http://t.co/2IbOmkdCMv   \n",
       "1135                                                                Sergey Brin unveils new Google driverless car prototype http://t.co/CVON3FADCL   \n",
       "3764                                                                      Autonomous driving solutions, Lexus smart co pilot. http://t.co/Iqo0NGr2   \n",
       "2396                                                                                   Google car @ Computer History Museum http://t.co/jApPaWK2rb   \n",
       "2564                                                              Oh hey Google Car! @ MDT Metromover - Bayfront Park Station http://t.co/4hCAG7Lr   \n",
       "959                          Brilliant Smithsonian magazine award winners. Here w/Sebastian Thrun - driverless cars, Udacity. http://t.co/AtPH3FsV   \n",
       "909             Real challenge to Google's driverless cars: Israeli startup Mobileye raises $400M w/ cheaper tech http://t.co/hqLGazafyQ via @zite   \n",
       "6758                                    Craig Tieman inventor presenting Delphi's Connected Car @WomenandHiTech. Very cool. http://t.co/oUm6egLL44   \n",
       "36                                WO2013070799A1 Automated Vehicle Conveyance Apparatus Transportation System #B61B #B61B13 http://t.co/HToCWDVEZM   \n",
       "1818    Driverless cars, ubiquitous wireless internet & electricity, growing & printing organs... LENR next? #theFutureisNOW #theSingularityIsNear   \n",
       "2060    CONCORD: DMV handing out testing permits for driver-less cars: DMV handing out testing permits for driver-less cars http://t.co/gPqxDzVbSt   \n",
       "5651                                     #dashitcray Robot chauffeurs: Google self-driving cars log 300K miles | VentureBeat: http://t.co/isLeDhps   \n",
       "2059    CONCORD: DMV handing out testing permits for driver-less cars: DMV handing out testing permits for driver-less cars http://t.co/vEE13dcFme   \n",
       "3716  Ayo dey need 2 chill wit dat Google car dat drive itself. U know da cops gon make dat shit drive sum nigga azz str8 2 da police station lmao   \n",
       "117                                                    Google unveils fully autonomous, hands-free car prototype [w/videos] http://t.co/KrauTretw1   \n",
       "6845                                                                  @theGreaterMarin @mikesonn Oh so driverless mini-transit not driverless cars   \n",
       "242                                     Minnesota Autonomous Vehicle Meetup. (@ Hack Factory of Minnesota - @hackfactorymn) http://t.co/YV0myVzRLG   \n",
       "1144       #VisionZero #ActOnClimate net zero best NYT: Google's Next Phase in Driverless Cars: No Brakes or Steering Wheel http://t.co/M5iDEIxHwo   \n",
       "4446                                                               Google's self-driving cars understand cyclists gestures. http://t.co/zTDjG8keyU   \n",
       "768                                                   @jaymohr37 driverless cars. Pssssh, been there, done that s/ David Hasselhoff #jaymohrsports   \n",
       "6875                                                        not driving Google's self driving car @ Computer History Museum http://t.co/ReyX2g1rvK   \n",
       "2089                                            @cnni Driverless truck?! Perfect 'car (((Bomb))). I'll take 2 4 AIPC rats nest in Hymie town & DC.   \n",
       "3120                                                              Google Self-Driving car. Adeus, motoristas. @ Google Corp http://t.co/wZWIcfpeTq   \n",
       "5883                                                                      Burkhard Bilger: Inside Google's self-driving car http://t.co/UUUNqwyvRS   \n",
       "2989                                                        O hai @google self-driving car! Testing near ocean beach I see. http://t.co/0n44ce7SWN   \n",
       "1221                                                 #IBMInnovate @bosch expanding system complexity #internetofthings Driverless cars are coming!   \n",
       "3995                                   VentureBeat: Silicon Valley VCs predict 2013 trends: Space, robots, self-driving cars. http://t.co/prxJoctL   \n",
       "107              Finally, a very realistic timeline for full autonomous car capability. Hats off, @MorganStanley #AutoForum http://t.co/ZwemgV5JF8   \n",
       "6469                                              #Volvo begins testing self-driving cars in Sweden http://t.co/ZNloqs6lQ3 Subscribers via @gizmag   \n",
       "5436                    Tech Trend: Mercedes-Benz Unveils Self-Driving 'Future Truck' on Germany's Autobahn. http://t.co/DWxkQaGcGh #FutureReality   \n",
       "1894                                        California Becomes Third State To Legalize Driverless Cars. http://t.co/Q8gveWFi (RT: @singularityhub)   \n",
       "824                   Could Google's interest in driverless cars spark a resurgence in 1940/50s-style cybernetics? http://t.co/IRE1naoYeN @markoff   \n",
       "3722                                                                         Google unveils self-driving car prototype http://t.co/51zdTNYIRJ oooo   \n",
       "289                                   Auto psychedelia: Lidar heat map from #Ford Fusion Hybrid Autonomous Research Vehicle http://t.co/AqOkQ3DsPR   \n",
       "4709                                      The self-driving car masters city-street driving | KurzweilAI http://t.co/wnMNZ6oOmz via @kurzweilainews   \n",
       "1099                     Started here in Nevada! How Google Got States to Legalize Driverless Cars - http://t.co/LdNKh8sPvc http://t.co/9YQMNIk6b0   \n",
       "716                                                                      Google: Driverless cars are mastering city streets http://t.co/Lr1Xa9B9fw   \n",
       "2080                      Google's \"driverless\" Lexus making a pit stop. Oh, thank heaven...for refills. @ Top Of San Bruno http://t.co/3luppxWWv5   \n",
       "3830     The Verge Self-driving cars and ocean colonies: revisit Isaac Asimov's vision of 2014 http://t.co/wL1fDz9GyF. Ocean colonies. Pre-mars???   \n",
       "3967                                                    Toyota FV2 - concept vehicle self driving or driving mode #verycool http://t.co/WoZfvOncmM   \n",
       "6199                                                  Dey tuk r jobs.. Free Google self driving ad sponsored cab to stores. http://t.co/Vd9a4nF6Vi   \n",
       "6691                                             Autonomous taxi competes with crowd sourced taxi gets better name. #Goober http://t.co/cfWpw7HWDx   \n",
       "986                                                                                 Google's own prototype driverless cars! http://t.co/Zl0McVQzN5   \n",
       "5189                         Self-driving Anki Drive racers are like futuristic slot cars... with lasers. @ CBS Interactive http://t.co/bpxZhgNIIL   \n",
       "717                                                                      Google: Driverless cars are mastering city streets http://t.co/AerMDvYZu5   \n",
       "746                                                                      Google: Driverless cars are mastering city streets http://t.co/npSCNvcni6   \n",
       "874   @dsucher @BrentToderian Driverless cars won't be panacea for aging in place. They won't bring human scale, community, social life to sprawl.   \n",
       "4000    Toyota debuting self-driving cars at CES! http://t.co/SjAnNTdk Isaac Asimov (& Boy's Life) had me waiting since 1990: http://t.co/FwsJBruy   \n",
       "6720                                   Lasers cameras radar & differential GPS will be in autonomous cars!Hi tech cruising! http://t.co/GuOBlXgfSj   \n",
       "1131                                        Google building truly driverless car with no brakes, steering wheel, gas pedals http://t.co/xfS5VypsOb   \n",
       "4407                                   Slate: Navigation Without Representation: D.C.'s DMV Prepares for Self-Driving Cars. http://t.co/llT8KoDlvG   \n",
       "562       Pa. congressman to test driverless CMU car: PITTSBURGH (AP) - A Pennsylvania congressman is planning a trip to... http://t.co/JPyUwK023u   \n",
       "1097   Uber CEO: Expect driverless cars, record funding round in the future: Uber Technologies Inc. envisions the future... http://t.co/xbez0zNglu   \n",
       "324                                      Driverless cars!!! Argh \"@zeGamelord: Very overwhelming RT @PhantomBeta: Google is a scary organization.\"   \n",
       "5384                                                Mercedes-Benz Unveils Self-Driving 'Future Truck' on Germany's Autobahn http://t.co/uktFsdHXlE   \n",
       "4467                                                                                    Self driving cars = CaaS (cars as a service) #SingularityU   \n",
       "\n",
       "      length  word fraction  token length  english score  \n",
       "6010      98       0.653061      4.823529          455.0  \n",
       "1641      85       0.505882      8.555556          474.0  \n",
       "461       70       0.657143      4.916667          496.0  \n",
       "3141      79       0.594937      7.000000          498.0  \n",
       "4427     124       0.532258      9.416667          499.0  \n",
       "1135      78       0.615385      7.777778          505.0  \n",
       "3764      72       0.597222      8.125000          512.0  \n",
       "2396      59       0.508475      7.571429          512.0  \n",
       "2564      80       0.575000      5.750000          519.0  \n",
       "959      117       0.683761      8.076923          525.0  \n",
       "909      130       0.607692      6.705882          528.0  \n",
       "6758     106       0.528302      8.727273          542.0  \n",
       "36       112       0.517857     10.300000          549.0  \n",
       "1818     138       0.579710      8.266667          556.0  \n",
       "2060     138       0.688406      6.722222          561.0  \n",
       "5651     105       0.523810      7.833333          565.0  \n",
       "2059     138       0.688406      6.722222          565.0  \n",
       "3716     140       0.764286      3.700000          568.0  \n",
       "117       91       0.538462      9.222222          571.0  \n",
       "6845      76       0.552632      7.555556          573.0  \n",
       "242      106       0.509434      7.916667          580.0  \n",
       "1144     135       0.511111      6.555556          582.0  \n",
       "4446      79       0.607595     10.428571          585.0  \n",
       "768       92       0.576087      6.750000          590.0  \n",
       "6875      86       0.604651      6.909091          592.0  \n",
       "2089      98       0.551020      4.500000          594.0  \n",
       "3120      80       0.562500      8.000000          595.0  \n",
       "5883      72       0.569444      9.428571          599.0  \n",
       "2989      86       0.500000      6.250000          601.0  \n",
       "1221      93       0.516129      8.400000          603.0  \n",
       "3995     107       0.616822      8.000000          608.0  \n",
       "107      129       0.511628      7.666667          612.0  \n",
       "6469      96       0.520833      7.818182          612.0  \n",
       "5436     122       0.508197      9.250000          617.0  \n",
       "1894     102       0.500000      8.363636          618.0  \n",
       "824      124       0.524194      7.928571          619.0  \n",
       "3722      69       0.579710      9.000000          620.0  \n",
       "289      108       0.629630      7.384615          624.0  \n",
       "4709     104       0.519231      8.545455          624.0  \n",
       "1099     121       0.504132      7.133333          625.0  \n",
       "716       73       0.589041      8.250000          634.0  \n",
       "2080     120       0.516667      6.117647          635.0  \n",
       "3830     137       0.627737      7.117647          638.0  \n",
       "3967      90       0.511111      6.583333          638.0  \n",
       "6199      92       0.586957      5.642857          640.0  \n",
       "6691      97       0.577320      7.166667          642.0  \n",
       "986       62       0.532258      9.500000          642.0  \n",
       "5189     117       0.641026      6.866667          642.0  \n",
       "717       73       0.589041      8.250000          643.0  \n",
       "746       73       0.589041      8.250000          643.0  \n",
       "874      140       0.657143      5.714286          644.0  \n",
       "4000     138       0.507246      6.722222          644.0  \n",
       "6720     107       0.644860      6.714286          646.0  \n",
       "1131     102       0.647059      6.923077          651.0  \n",
       "4407     107       0.635514      8.818182          652.0  \n",
       "562      136       0.632353      6.210526          652.0  \n",
       "1097     139       0.669065      6.777778          656.0  \n",
       "324      105       0.590476      7.153846          661.0  \n",
       "5384      94       0.563830      9.555556          663.0  \n",
       "4467      58       0.500000      4.900000          663.0  "
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = trim_dataset_by(\n",
    "    df, \n",
    "    \n",
    "    # sort_by='length',\n",
    "    # sort_by='word fraction',\n",
    "    # sort_by='token length',\n",
    "    sort_by='english score',\n",
    "    ascending=True,\n",
    "    \n",
    "    length=(40, None),\n",
    "    word_fraction=(0.5, None),\n",
    "    token_length=(1, 12),\n",
    "    english_score=(400, None),\n",
    ")\n",
    "a[:60]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "30fa205a-dc9b-442d-a90f-4130f52bc701",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:38:54.325296Z",
     "iopub.status.busy": "2024-01-08T15:38:54.324825Z",
     "iopub.status.idle": "2024-01-08T15:38:55.862631Z",
     "shell.execute_reply": "2024-01-08T15:38:55.861705Z",
     "shell.execute_reply.started": "2024-01-08T15:38:54.325274Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time elapsed: 1s\n",
      "samples: 4497, similar samples detected: 224 (4.74%)\n"
     ]
    }
   ],
   "source": [
    "df = drop_similar(supersample(a, SELFDRIVING_CARS));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "fc90dac7-2bc5-431c-b5e1-7e747c121b74",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:38:55.864539Z",
     "iopub.status.busy": "2024-01-08T15:38:55.863861Z",
     "iopub.status.idle": "2024-01-08T15:38:55.893308Z",
     "shell.execute_reply": "2024-01-08T15:38:55.892682Z",
     "shell.execute_reply.started": "2024-01-08T15:38:55.864509Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4497, 3)"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = select_columns(df.copy(), ['text'])\n",
    "df['topic'] = 'self-driving cars'\n",
    "df['id'] = 0\n",
    "\n",
    "df = sample(df, SELFDRIVING_CARS)\n",
    "dataset = concat(dataset, df)\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "21731247-5154-4d7b-9481-a86ea392d2dc",
   "metadata": {},
   "source": [
    "# Weather"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "5eec56d3-6f2b-4ee6-8325-96307e4e6396",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:38:55.894600Z",
     "iopub.status.busy": "2024-01-08T15:38:55.894165Z",
     "iopub.status.idle": "2024-01-08T15:38:55.921032Z",
     "shell.execute_reply": "2024-01-08T15:38:55.920426Z",
     "shell.execute_reply.started": "2024-01-08T15:38:55.894578Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1000, 3)"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = load_dataset('weather.csv', source=True)\n",
    "df.rename(columns={'tweet_text': 'text', 'tweet_id': 'id'}, inplace=True)\n",
    "df = select_columns(df, ['text', 'id', 'sentiment'])\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "5cf4a54d-4366-4d8d-815f-18ba4f9ad85f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:38:55.922041Z",
     "iopub.status.busy": "2024-01-08T15:38:55.921851Z",
     "iopub.status.idle": "2024-01-08T15:38:55.969189Z",
     "shell.execute_reply": "2024-01-08T15:38:55.968600Z",
     "shell.execute_reply.started": "2024-01-08T15:38:55.922022Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(762, 3)"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = df.copy()\n",
    "a = a[a['sentiment'] != 'Tweet not related to weather condition']\n",
    "a = preprocessing_pipeline(a)\n",
    "a.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "id": "43f0f105-1e1f-4b61-925a-9681c061715a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:38:55.970194Z",
     "iopub.status.busy": "2024-01-08T15:38:55.970000Z",
     "iopub.status.idle": "2024-01-08T15:38:55.986393Z",
     "shell.execute_reply": "2024-01-08T15:38:55.985795Z",
     "shell.execute_reply.started": "2024-01-08T15:38:55.970174Z"
    }
   },
   "outputs": [],
   "source": [
    "df = select_columns(a, ['text', 'id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "186f4f9d-0b1e-4c26-8778-9ae8009d9e60",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:38:55.987497Z",
     "iopub.status.busy": "2024-01-08T15:38:55.987307Z",
     "iopub.status.idle": "2024-01-08T15:38:56.006814Z",
     "shell.execute_reply": "2024-01-08T15:38:56.006202Z",
     "shell.execute_reply.started": "2024-01-08T15:38:55.987477Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>870</th>\n",
       "      <td>84305541</td>\n",
       "      <td>Am I the only person who heard that loud thundering storm last night or was that just me?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>257</th>\n",
       "      <td>84050868</td>\n",
       "      <td>Me and katilyn chatting up the movie, walk outside.. OMG. Sprint to car. We don't like thunder storms. #mature</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>464</th>\n",
       "      <td>81992074</td>\n",
       "      <td>Wednesday - Snow showers. Snow accumulation 3 to 5 inches. Highs in the 30s. {link}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105</th>\n",
       "      <td>82839115</td>\n",
       "      <td>News - Current Conditions : 64.4F, Heavy Thunderstorms and Rain - 7:07 PM EDT May. 16: Temperat... {link} #News #Greensboro</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>711</th>\n",
       "      <td>84050258</td>\n",
       "      <td>Its freezing in Boston hows the weather in Providence???</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           id  \\\n",
       "870  84305541   \n",
       "257  84050868   \n",
       "464  81992074   \n",
       "105  82839115   \n",
       "711  84050258   \n",
       "\n",
       "                                                                                                                            text  \n",
       "870                                    Am I the only person who heard that loud thundering storm last night or was that just me?  \n",
       "257               Me and katilyn chatting up the movie, walk outside.. OMG. Sprint to car. We don't like thunder storms. #mature  \n",
       "464                                          Wednesday - Snow showers. Snow accumulation 3 to 5 inches. Highs in the 30s. {link}  \n",
       "105  News - Current Conditions : 64.4F, Heavy Thunderstorms and Rain - 7:07 PM EDT May. 16: Temperat... {link} #News #Greensboro  \n",
       "711                                                                     Its freezing in Boston hows the weather in Providence???  "
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b56413b4-dbd5-4313-8bd6-29a6713a9ac0",
   "metadata": {},
   "source": [
    "The dataset has already been processed. Author replaced any mentions by @metion and links by {links}. I'd like to replace {link} by %link"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "43426078-d4b8-435c-ba03-a390a5d375d1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:38:56.007797Z",
     "iopub.status.busy": "2024-01-08T15:38:56.007601Z",
     "iopub.status.idle": "2024-01-08T15:38:56.026152Z",
     "shell.execute_reply": "2024-01-08T15:38:56.025498Z",
     "shell.execute_reply.started": "2024-01-08T15:38:56.007777Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(152, 2)"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['text'].str.contains('{link}')].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "73ba3e69-1a68-4d57-85cd-980450b707a8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:38:56.027168Z",
     "iopub.status.busy": "2024-01-08T15:38:56.026981Z",
     "iopub.status.idle": "2024-01-08T15:38:56.045398Z",
     "shell.execute_reply": "2024-01-08T15:38:56.044820Z",
     "shell.execute_reply.started": "2024-01-08T15:38:56.027149Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0, 2)"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['text'] = df['text'].str.replace('{link}', '%link', regex=False)\n",
    "df[df['text'].str.contains('{link}')].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "a29d569a-aa61-4bb5-8819-7054ac440618",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:38:56.046375Z",
     "iopub.status.busy": "2024-01-08T15:38:56.046187Z",
     "iopub.status.idle": "2024-01-08T15:38:56.078238Z",
     "shell.execute_reply": "2024-01-08T15:38:56.077597Z",
     "shell.execute_reply.started": "2024-01-08T15:38:56.046356Z"
    }
   },
   "outputs": [],
   "source": [
    "add_stats(df);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "76c87983-1228-4d28-8324-76b6598f933b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:38:56.079357Z",
     "iopub.status.busy": "2024-01-08T15:38:56.079150Z",
     "iopub.status.idle": "2024-01-08T15:38:56.110306Z",
     "shell.execute_reply": "2024-01-08T15:38:56.109575Z",
     "shell.execute_reply.started": "2024-01-08T15:38:56.079336Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "samples: 595, reduction: 21.92%\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>length</th>\n",
       "      <th>word fraction</th>\n",
       "      <th>token length</th>\n",
       "      <th>english score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>435</th>\n",
       "      <td>84049777</td>\n",
       "      <td>It's 82 degrees</td>\n",
       "      <td>15</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>4.333333</td>\n",
       "      <td>2441.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>519</th>\n",
       "      <td>81993022</td>\n",
       "      <td>Its Freezing D;</td>\n",
       "      <td>15</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>4.333333</td>\n",
       "      <td>1706.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>843</th>\n",
       "      <td>83269790</td>\n",
       "      <td>What up sunshine</td>\n",
       "      <td>16</td>\n",
       "      <td>0.875000</td>\n",
       "      <td>4.666667</td>\n",
       "      <td>1927.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125</th>\n",
       "      <td>80052225</td>\n",
       "      <td>Morning sunshine.</td>\n",
       "      <td>17</td>\n",
       "      <td>0.882353</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>1505.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>401</th>\n",
       "      <td>84031161</td>\n",
       "      <td>Is it hot outside?</td>\n",
       "      <td>18</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>3.750000</td>\n",
       "      <td>2730.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>934</th>\n",
       "      <td>82509653</td>\n",
       "      <td>Really hot outside</td>\n",
       "      <td>18</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>5.333333</td>\n",
       "      <td>2209.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>84034743</td>\n",
       "      <td>It's fuckin hot out</td>\n",
       "      <td>19</td>\n",
       "      <td>0.789474</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>1792.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>613</th>\n",
       "      <td>84050364</td>\n",
       "      <td>Its hot af outside.</td>\n",
       "      <td>20</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>1940.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266</th>\n",
       "      <td>84308267</td>\n",
       "      <td>My room is freezing</td>\n",
       "      <td>20</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>1792.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>509</th>\n",
       "      <td>84318313</td>\n",
       "      <td>Its a million degrees</td>\n",
       "      <td>21</td>\n",
       "      <td>0.857143</td>\n",
       "      <td>4.500000</td>\n",
       "      <td>2001.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>580</th>\n",
       "      <td>84313769</td>\n",
       "      <td>Loving this weather (:</td>\n",
       "      <td>22</td>\n",
       "      <td>0.772727</td>\n",
       "      <td>4.750000</td>\n",
       "      <td>1689.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>977</th>\n",
       "      <td>81990794</td>\n",
       "      <td>Ahhh weather so hot!!!</td>\n",
       "      <td>22</td>\n",
       "      <td>0.727273</td>\n",
       "      <td>4.750000</td>\n",
       "      <td>1945.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>79189697</td>\n",
       "      <td>Mad windy for no reason</td>\n",
       "      <td>23</td>\n",
       "      <td>0.826087</td>\n",
       "      <td>3.800000</td>\n",
       "      <td>1792.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>195</th>\n",
       "      <td>84320750</td>\n",
       "      <td>You know the old saying</td>\n",
       "      <td>24</td>\n",
       "      <td>0.791667</td>\n",
       "      <td>3.800000</td>\n",
       "      <td>2304.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>950</th>\n",
       "      <td>81213627</td>\n",
       "      <td>yay it's gonna storm! c:</td>\n",
       "      <td>24</td>\n",
       "      <td>0.708333</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>1513.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>956</th>\n",
       "      <td>82850000</td>\n",
       "      <td>Another rainy day! Blah.</td>\n",
       "      <td>24</td>\n",
       "      <td>0.791667</td>\n",
       "      <td>5.250000</td>\n",
       "      <td>1647.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>432</th>\n",
       "      <td>82837247</td>\n",
       "      <td>cold in here cold outside</td>\n",
       "      <td>25</td>\n",
       "      <td>0.840000</td>\n",
       "      <td>4.200000</td>\n",
       "      <td>1496.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>650</th>\n",
       "      <td>84318783</td>\n",
       "      <td>Man its so windy out here</td>\n",
       "      <td>25</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>3.333333</td>\n",
       "      <td>1969.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>127</th>\n",
       "      <td>84304127</td>\n",
       "      <td>it's 80 degrees for what?</td>\n",
       "      <td>25</td>\n",
       "      <td>0.680000</td>\n",
       "      <td>4.200000</td>\n",
       "      <td>2327.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>402</th>\n",
       "      <td>80050064</td>\n",
       "      <td>Weather for Seattle %link</td>\n",
       "      <td>25</td>\n",
       "      <td>0.680000</td>\n",
       "      <td>5.500000</td>\n",
       "      <td>1679.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>281</th>\n",
       "      <td>82681227</td>\n",
       "      <td>Loving this warm weather!</td>\n",
       "      <td>25</td>\n",
       "      <td>0.840000</td>\n",
       "      <td>5.500000</td>\n",
       "      <td>1556.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>829</th>\n",
       "      <td>81995317</td>\n",
       "      <td>Where'd the sunshine go ):</td>\n",
       "      <td>26</td>\n",
       "      <td>0.730769</td>\n",
       "      <td>4.400000</td>\n",
       "      <td>1578.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113</th>\n",
       "      <td>81171736</td>\n",
       "      <td>Freezing! O.o WeLoveSwift</td>\n",
       "      <td>26</td>\n",
       "      <td>0.807692</td>\n",
       "      <td>7.666667</td>\n",
       "      <td>1064.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>344</th>\n",
       "      <td>79196972</td>\n",
       "      <td>Friday Fest es lil chilly!</td>\n",
       "      <td>26</td>\n",
       "      <td>0.807692</td>\n",
       "      <td>4.400000</td>\n",
       "      <td>1181.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>550</th>\n",
       "      <td>84318394</td>\n",
       "      <td>It is hot ass hell outside</td>\n",
       "      <td>26</td>\n",
       "      <td>0.807692</td>\n",
       "      <td>3.500000</td>\n",
       "      <td>2123.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>565</th>\n",
       "      <td>84320081</td>\n",
       "      <td>This weather is depressing.</td>\n",
       "      <td>27</td>\n",
       "      <td>0.851852</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>1630.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>419</th>\n",
       "      <td>84313582</td>\n",
       "      <td>It's too hot out. Fuckin aye</td>\n",
       "      <td>28</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>3.833333</td>\n",
       "      <td>1499.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>361</th>\n",
       "      <td>82517839</td>\n",
       "      <td>the storm outside = no bueno</td>\n",
       "      <td>28</td>\n",
       "      <td>0.785714</td>\n",
       "      <td>3.833333</td>\n",
       "      <td>1327.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>666</th>\n",
       "      <td>83268491</td>\n",
       "      <td>The sunshine makes me happy.</td>\n",
       "      <td>28</td>\n",
       "      <td>0.821429</td>\n",
       "      <td>4.800000</td>\n",
       "      <td>1645.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>749</th>\n",
       "      <td>84314069</td>\n",
       "      <td>Its hot as a BITCH outside !</td>\n",
       "      <td>28</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>3.142857</td>\n",
       "      <td>2048.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>83268638</td>\n",
       "      <td>eww its 81% humidity today :I</td>\n",
       "      <td>29</td>\n",
       "      <td>0.655172</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>1474.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>400</th>\n",
       "      <td>82675706</td>\n",
       "      <td>Enjoying the Midwest weather!</td>\n",
       "      <td>29</td>\n",
       "      <td>0.862069</td>\n",
       "      <td>6.500000</td>\n",
       "      <td>1730.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>926</th>\n",
       "      <td>79190118</td>\n",
       "      <td>:)) Dress weather again, yay.</td>\n",
       "      <td>29</td>\n",
       "      <td>0.689655</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>1834.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>170</th>\n",
       "      <td>82846618</td>\n",
       "      <td>Freezing. Cuddling...alone :(</td>\n",
       "      <td>29</td>\n",
       "      <td>0.724138</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>1024.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>952</th>\n",
       "      <td>82843258</td>\n",
       "      <td>@mention yea dis weather trash</td>\n",
       "      <td>30</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>5.200000</td>\n",
       "      <td>1433.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>165</th>\n",
       "      <td>81178513</td>\n",
       "      <td>I'm freezing on this stupid bus</td>\n",
       "      <td>31</td>\n",
       "      <td>0.806452</td>\n",
       "      <td>4.333333</td>\n",
       "      <td>1312.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>346</th>\n",
       "      <td>82845268</td>\n",
       "      <td>This weather on some bullshit .</td>\n",
       "      <td>31</td>\n",
       "      <td>0.806452</td>\n",
       "      <td>4.333333</td>\n",
       "      <td>1570.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>428</th>\n",
       "      <td>82837743</td>\n",
       "      <td>soooo its cold outside grrr -_-</td>\n",
       "      <td>31</td>\n",
       "      <td>0.741935</td>\n",
       "      <td>4.333333</td>\n",
       "      <td>1206.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>83257925</td>\n",
       "      <td>it's a nice day in LA. sunny :)</td>\n",
       "      <td>31</td>\n",
       "      <td>0.645161</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>1133.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>585</th>\n",
       "      <td>82517296</td>\n",
       "      <td>Insaneeee weather over here!!!!</td>\n",
       "      <td>31</td>\n",
       "      <td>0.774194</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>1316.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122</th>\n",
       "      <td>82853149</td>\n",
       "      <td>Memphis weather is so bipolar!!!</td>\n",
       "      <td>32</td>\n",
       "      <td>0.781250</td>\n",
       "      <td>5.600000</td>\n",
       "      <td>1399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>82674634</td>\n",
       "      <td>I am freezing! I wanna cuddle (:</td>\n",
       "      <td>32</td>\n",
       "      <td>0.718750</td>\n",
       "      <td>3.714286</td>\n",
       "      <td>1377.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>480</th>\n",
       "      <td>81213046</td>\n",
       "      <td>Couldn't ask for better weather!</td>\n",
       "      <td>32</td>\n",
       "      <td>0.812500</td>\n",
       "      <td>5.600000</td>\n",
       "      <td>1856.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>82673833</td>\n",
       "      <td>What's with all the rainy weber?</td>\n",
       "      <td>32</td>\n",
       "      <td>0.781250</td>\n",
       "      <td>4.500000</td>\n",
       "      <td>1632.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>836</th>\n",
       "      <td>82511193</td>\n",
       "      <td>Ughhhhhh its so damn hot outside</td>\n",
       "      <td>32</td>\n",
       "      <td>0.843750</td>\n",
       "      <td>4.500000</td>\n",
       "      <td>1396.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>160</th>\n",
       "      <td>82837899</td>\n",
       "      <td>hope this night is rainy night ^^</td>\n",
       "      <td>33</td>\n",
       "      <td>0.757576</td>\n",
       "      <td>3.857143</td>\n",
       "      <td>1915.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>477</th>\n",
       "      <td>81995385</td>\n",
       "      <td>Bout to enjoy this sunshine a bit</td>\n",
       "      <td>33</td>\n",
       "      <td>0.818182</td>\n",
       "      <td>3.857143</td>\n",
       "      <td>1084.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>304</th>\n",
       "      <td>83267294</td>\n",
       "      <td>Could go for a good #storm today.</td>\n",
       "      <td>33</td>\n",
       "      <td>0.606061</td>\n",
       "      <td>3.857143</td>\n",
       "      <td>1632.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>943</th>\n",
       "      <td>81988861</td>\n",
       "      <td>It's bone chilling cold outside!!</td>\n",
       "      <td>33</td>\n",
       "      <td>0.787879</td>\n",
       "      <td>5.800000</td>\n",
       "      <td>1408.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>287</th>\n",
       "      <td>82844485</td>\n",
       "      <td>This weather knocked everyone out</td>\n",
       "      <td>33</td>\n",
       "      <td>0.878788</td>\n",
       "      <td>5.800000</td>\n",
       "      <td>1656.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>497</th>\n",
       "      <td>81172598</td>\n",
       "      <td>This weather makes me very happy!</td>\n",
       "      <td>33</td>\n",
       "      <td>0.818182</td>\n",
       "      <td>4.666667</td>\n",
       "      <td>1706.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>921</th>\n",
       "      <td>81213904</td>\n",
       "      <td>Weather expectations/reality %link</td>\n",
       "      <td>34</td>\n",
       "      <td>0.764706</td>\n",
       "      <td>10.666667</td>\n",
       "      <td>1174.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>360</th>\n",
       "      <td>82847436</td>\n",
       "      <td>Georgia weather be pissin me off.</td>\n",
       "      <td>34</td>\n",
       "      <td>0.794118</td>\n",
       "      <td>4.666667</td>\n",
       "      <td>1148.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>433</th>\n",
       "      <td>83272072</td>\n",
       "      <td>green snow ..... love that grinder</td>\n",
       "      <td>34</td>\n",
       "      <td>0.705882</td>\n",
       "      <td>4.833333</td>\n",
       "      <td>1165.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>159</th>\n",
       "      <td>84319756</td>\n",
       "      <td>hot weather puts me in a bad mood .</td>\n",
       "      <td>35</td>\n",
       "      <td>0.742857</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>1415.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>855</th>\n",
       "      <td>84048438</td>\n",
       "      <td>This weather forecast is depressing</td>\n",
       "      <td>35</td>\n",
       "      <td>0.885714</td>\n",
       "      <td>6.200000</td>\n",
       "      <td>1336.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>483</th>\n",
       "      <td>81178980</td>\n",
       "      <td>The weather is beyond beautiful !</td>\n",
       "      <td>35</td>\n",
       "      <td>0.771429</td>\n",
       "      <td>4.666667</td>\n",
       "      <td>1952.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>629</th>\n",
       "      <td>82680470</td>\n",
       "      <td>Mostly sunny today, high 70 (21 C).</td>\n",
       "      <td>35</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>4.142857</td>\n",
       "      <td>1220.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>708</th>\n",
       "      <td>82673764</td>\n",
       "      <td>The weather is so gloomy and cold :(</td>\n",
       "      <td>36</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>3.625000</td>\n",
       "      <td>2078.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>833</th>\n",
       "      <td>82835571</td>\n",
       "      <td>The weather better be right tonight!</td>\n",
       "      <td>36</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>5.166667</td>\n",
       "      <td>1763.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           id                                  text  length  word fraction  \\\n",
       "435  84049777                       It's 82 degrees      15       0.666667   \n",
       "519  81993022                       Its Freezing D;      15       0.800000   \n",
       "843  83269790                      What up sunshine      16       0.875000   \n",
       "125  80052225                     Morning sunshine.      17       0.882353   \n",
       "401  84031161                    Is it hot outside?      18       0.777778   \n",
       "934  82509653                    Really hot outside      18       0.888889   \n",
       "54   84034743                   It's fuckin hot out      19       0.789474   \n",
       "613  84050364                   Its hot af outside.      20       0.750000   \n",
       "266  84308267                   My room is freezing      20       0.800000   \n",
       "509  84318313                 Its a million degrees      21       0.857143   \n",
       "580  84313769                Loving this weather (:      22       0.772727   \n",
       "977  81990794                Ahhh weather so hot!!!      22       0.727273   \n",
       "90   79189697               Mad windy for no reason      23       0.826087   \n",
       "195  84320750               You know the old saying      24       0.791667   \n",
       "950  81213627              yay it's gonna storm! c:      24       0.708333   \n",
       "956  82850000              Another rainy day! Blah.      24       0.791667   \n",
       "432  82837247             cold in here cold outside      25       0.840000   \n",
       "650  84318783             Man its so windy out here      25       0.800000   \n",
       "127  84304127             it's 80 degrees for what?      25       0.680000   \n",
       "402  80050064             Weather for Seattle %link      25       0.680000   \n",
       "281  82681227             Loving this warm weather!      25       0.840000   \n",
       "829  81995317            Where'd the sunshine go ):      26       0.730769   \n",
       "113  81171736             Freezing! O.o WeLoveSwift      26       0.807692   \n",
       "344  79196972            Friday Fest es lil chilly!      26       0.807692   \n",
       "550  84318394            It is hot ass hell outside      26       0.807692   \n",
       "565  84320081           This weather is depressing.      27       0.851852   \n",
       "419  84313582          It's too hot out. Fuckin aye      28       0.750000   \n",
       "361  82517839          the storm outside = no bueno      28       0.785714   \n",
       "666  83268491          The sunshine makes me happy.      28       0.821429   \n",
       "749  84314069          Its hot as a BITCH outside !      28       0.750000   \n",
       "45   83268638         eww its 81% humidity today :I      29       0.655172   \n",
       "400  82675706         Enjoying the Midwest weather!      29       0.862069   \n",
       "926  79190118         :)) Dress weather again, yay.      29       0.689655   \n",
       "170  82846618         Freezing. Cuddling...alone :(      29       0.724138   \n",
       "952  82843258        @mention yea dis weather trash      30       0.600000   \n",
       "165  81178513       I'm freezing on this stupid bus      31       0.806452   \n",
       "346  82845268       This weather on some bullshit .      31       0.806452   \n",
       "428  82837743       soooo its cold outside grrr -_-      31       0.741935   \n",
       "64   83257925       it's a nice day in LA. sunny :)      31       0.645161   \n",
       "585  82517296       Insaneeee weather over here!!!!      31       0.774194   \n",
       "122  82853149      Memphis weather is so bipolar!!!      32       0.781250   \n",
       "78   82674634      I am freezing! I wanna cuddle (:      32       0.718750   \n",
       "480  81213046      Couldn't ask for better weather!      32       0.812500   \n",
       "75   82673833      What's with all the rainy weber?      32       0.781250   \n",
       "836  82511193      Ughhhhhh its so damn hot outside      32       0.843750   \n",
       "160  82837899     hope this night is rainy night ^^      33       0.757576   \n",
       "477  81995385     Bout to enjoy this sunshine a bit      33       0.818182   \n",
       "304  83267294     Could go for a good #storm today.      33       0.606061   \n",
       "943  81988861     It's bone chilling cold outside!!      33       0.787879   \n",
       "287  82844485     This weather knocked everyone out      33       0.878788   \n",
       "497  81172598     This weather makes me very happy!      33       0.818182   \n",
       "921  81213904    Weather expectations/reality %link      34       0.764706   \n",
       "360  82847436     Georgia weather be pissin me off.      34       0.794118   \n",
       "433  83272072    green snow ..... love that grinder      34       0.705882   \n",
       "159  84319756   hot weather puts me in a bad mood .      35       0.742857   \n",
       "855  84048438   This weather forecast is depressing      35       0.885714   \n",
       "483  81178980    The weather is beyond beautiful !       35       0.771429   \n",
       "629  82680470   Mostly sunny today, high 70 (21 C).      35       0.600000   \n",
       "708  82673764  The weather is so gloomy and cold :(      36       0.750000   \n",
       "833  82835571  The weather better be right tonight!      36       0.833333   \n",
       "\n",
       "     token length  english score  \n",
       "435      4.333333         2441.0  \n",
       "519      4.333333         1706.0  \n",
       "843      4.666667         1927.0  \n",
       "125      8.000000         1505.0  \n",
       "401      3.750000         2730.0  \n",
       "934      5.333333         2209.0  \n",
       "54       4.000000         1792.0  \n",
       "613      4.000000         1940.0  \n",
       "266      4.000000         1792.0  \n",
       "509      4.500000         2001.0  \n",
       "580      4.750000         1689.0  \n",
       "977      4.750000         1945.0  \n",
       "90       3.800000         1792.0  \n",
       "195      3.800000         2304.0  \n",
       "950      4.000000         1513.0  \n",
       "956      5.250000         1647.0  \n",
       "432      4.200000         1496.0  \n",
       "650      3.333333         1969.0  \n",
       "127      4.200000         2327.0  \n",
       "402      5.500000         1679.0  \n",
       "281      5.500000         1556.0  \n",
       "829      4.400000         1578.0  \n",
       "113      7.666667         1064.0  \n",
       "344      4.400000         1181.0  \n",
       "550      3.500000         2123.0  \n",
       "565      6.000000         1630.0  \n",
       "419      3.833333         1499.0  \n",
       "361      3.833333         1327.0  \n",
       "666      4.800000         1645.0  \n",
       "749      3.142857         2048.0  \n",
       "45       4.000000         1474.0  \n",
       "400      6.500000         1730.0  \n",
       "926      5.000000         1834.0  \n",
       "170      9.000000         1024.0  \n",
       "952      5.200000         1433.0  \n",
       "165      4.333333         1312.0  \n",
       "346      4.333333         1570.0  \n",
       "428      4.333333         1206.0  \n",
       "64       3.000000         1133.0  \n",
       "585      7.000000         1316.0  \n",
       "122      5.600000         1399.0  \n",
       "78       3.714286         1377.0  \n",
       "480      5.600000         1856.0  \n",
       "75       4.500000         1632.0  \n",
       "836      4.500000         1396.0  \n",
       "160      3.857143         1915.0  \n",
       "477      3.857143         1084.0  \n",
       "304      3.857143         1632.0  \n",
       "943      5.800000         1408.0  \n",
       "287      5.800000         1656.0  \n",
       "497      4.666667         1706.0  \n",
       "921     10.666667         1174.0  \n",
       "360      4.666667         1148.0  \n",
       "433      4.833333         1165.0  \n",
       "159      3.000000         1415.0  \n",
       "855      6.200000         1336.0  \n",
       "483      4.666667         1952.0  \n",
       "629      4.142857         1220.0  \n",
       "708      3.625000         2078.0  \n",
       "833      5.166667         1763.0  "
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = trim_dataset_by(\n",
    "    df, \n",
    "    \n",
    "    ascending=True,\n",
    "    sort_by='length',\n",
    "    # sort_by='word fraction',\n",
    "    # sort_by='token length',\n",
    "    # sort_by='english score',\n",
    "    \n",
    "    length=(15, None),\n",
    "    word_fraction=(0.6, None),\n",
    "    # token_length=(1, 12),\n",
    "    english_score=(800, None),\n",
    ")\n",
    "a[:60]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "41d1e1ac-7bcd-4e56-bc75-e2263e5c227f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:38:56.111752Z",
     "iopub.status.busy": "2024-01-08T15:38:56.111252Z",
     "iopub.status.idle": "2024-01-08T15:38:56.262949Z",
     "shell.execute_reply": "2024-01-08T15:38:56.262222Z",
     "shell.execute_reply.started": "2024-01-08T15:38:56.111729Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time elapsed: 0s\n",
      "samples: 587, similar samples detected: 8 (1.34%)\n"
     ]
    }
   ],
   "source": [
    "df = drop_similar(a);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "id": "d391b649-bebb-43c3-aae5-17f11728319c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:38:56.264482Z",
     "iopub.status.busy": "2024-01-08T15:38:56.263974Z",
     "iopub.status.idle": "2024-01-08T15:38:56.291301Z",
     "shell.execute_reply": "2024-01-08T15:38:56.290766Z",
     "shell.execute_reply.started": "2024-01-08T15:38:56.264453Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(587, 3)"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = select_columns(df.copy(), ['text', 'id'])\n",
    "df['topic'] = 'weather'\n",
    "df['id'] = 0\n",
    "\n",
    "df = sample(df, WEATHER)\n",
    "dataset = concat(dataset, df)\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e144bf68-f77e-4bd3-b82c-e26e0c136b1a",
   "metadata": {},
   "source": [
    "# Airline support"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "id": "711712fd-b867-4d1a-bc29-ee15a320f336",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:38:56.292590Z",
     "iopub.status.busy": "2024-01-08T15:38:56.291975Z",
     "iopub.status.idle": "2024-01-08T15:38:56.367875Z",
     "shell.execute_reply": "2024-01-08T15:38:56.367332Z",
     "shell.execute_reply.started": "2024-01-08T15:38:56.292567Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(14640, 1)"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = load_dataset('airline_support.csv', source=True, encoding='ISO-8859-1')\n",
    "df = select_columns(df, ['text'])\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "id": "437706ed-101b-4bcb-a473-85e8716dc1bf",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:38:56.369015Z",
     "iopub.status.busy": "2024-01-08T15:38:56.368560Z",
     "iopub.status.idle": "2024-01-08T15:38:56.905387Z",
     "shell.execute_reply": "2024-01-08T15:38:56.904832Z",
     "shell.execute_reply.started": "2024-01-08T15:38:56.368993Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(13599, 1)"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = preprocessing_pipeline(df)\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "id": "15e5a83b-b872-4825-923a-8c63c0de317b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:38:56.906827Z",
     "iopub.status.busy": "2024-01-08T15:38:56.906189Z",
     "iopub.status.idle": "2024-01-08T15:38:56.928238Z",
     "shell.execute_reply": "2024-01-08T15:38:56.927520Z",
     "shell.execute_reply.started": "2024-01-08T15:38:56.906803Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10812</th>\n",
       "      <td>@USAirways and next flight delayed as well. 5238 out of Charlotte, is it just my luck or what?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10755</th>\n",
       "      <td>@USAirways thank you</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2442</th>\n",
       "      <td>@united let me assure you my travel time is 4h2m not 5h2m #timezones #accuratetraveltimes #3rdtimethishashappened http://t.co/e0C9bI09cf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6615</th>\n",
       "      <td>@SouthwestAir Hey SWAir- a very tortured boarding on flight 1971 due to one gate agent having to board assisted folks one at a time.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6411</th>\n",
       "      <td>@SouthwestAir I'm Flight Booking Problems a flight to Vegas. Any good promo codes??</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                           text\n",
       "10812                                            @USAirways and next flight delayed as well. 5238 out of Charlotte, is it just my luck or what?\n",
       "10755                                                                                                                      @USAirways thank you\n",
       "2442   @united let me assure you my travel time is 4h2m not 5h2m #timezones #accuratetraveltimes #3rdtimethishashappened http://t.co/e0C9bI09cf\n",
       "6615       @SouthwestAir Hey SWAir- a very tortured boarding on flight 1971 due to one gate agent having to board assisted folks one at a time.\n",
       "6411                                                        @SouthwestAir I'm Flight Booking Problems a flight to Vegas. Any good promo codes??"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "id": "8538ebb9-e7a7-4b97-8245-0d8b31d169e4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:38:56.929608Z",
     "iopub.status.busy": "2024-01-08T15:38:56.929186Z",
     "iopub.status.idle": "2024-01-08T15:38:57.208995Z",
     "shell.execute_reply": "2024-01-08T15:38:57.208275Z",
     "shell.execute_reply.started": "2024-01-08T15:38:56.929587Z"
    }
   },
   "outputs": [],
   "source": [
    "add_stats(df);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "id": "4516e59b-9862-4172-8ce1-a27c2601d22d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:38:57.210100Z",
     "iopub.status.busy": "2024-01-08T15:38:57.209890Z",
     "iopub.status.idle": "2024-01-08T15:38:57.244203Z",
     "shell.execute_reply": "2024-01-08T15:38:57.243573Z",
     "shell.execute_reply.started": "2024-01-08T15:38:57.210077Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "samples: 8509, reduction: 37.43%\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>length</th>\n",
       "      <th>word fraction</th>\n",
       "      <th>token length</th>\n",
       "      <th>english score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>13043</th>\n",
       "      <td>@AmericanAir hey ho its not me losing any money (only you) just next make sure you stick to the \"flyers right's booklet\"</td>\n",
       "      <td>120</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>4.500000</td>\n",
       "      <td>1324.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14264</th>\n",
       "      <td>@AmericanAir Still sitting in airport as 6:05 flt delayed until 6:55...but still not boarding, what is happening? No info....</td>\n",
       "      <td>125</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>5.631579</td>\n",
       "      <td>1033.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5875</th>\n",
       "      <td>@SouthwestAir my trip is a month away...why do you consider it \"around the corner\"? Is that a setting I can change?</td>\n",
       "      <td>115</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>4.523810</td>\n",
       "      <td>1287.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12875</th>\n",
       "      <td>@AmericanAir Yes, thanks I found those, didn't see the gray tab at first :)</td>\n",
       "      <td>75</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>4.428571</td>\n",
       "      <td>1199.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7986</th>\n",
       "      <td>@JetBlue No, the flight wasn't until 9:51pm, but it's already been delayed.</td>\n",
       "      <td>75</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>5.333333</td>\n",
       "      <td>1536.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5192</th>\n",
       "      <td>@SouthwestAir reservation (FEHQNE) 21FEB15 | DCA-RSW. Want refund not credit for Cancelled Flightled flight please.</td>\n",
       "      <td>115</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>6.733333</td>\n",
       "      <td>984.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>358</th>\n",
       "      <td>@VirginAmerica just promoting the product is all, had a problem with southwest and recommend noneother than the best! http://t.co/tFaNXBh1Cf</td>\n",
       "      <td>140</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>6.421053</td>\n",
       "      <td>1106.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>@VirginAmerica I &amp;lt;3 Flying VA But Life happens and I am trying to #change my trip JPERHI Can you help.VA home page will not let me ?</td>\n",
       "      <td>135</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>4.037037</td>\n",
       "      <td>975.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8025</th>\n",
       "      <td>@JetBlue has the worst customer service out of any airline company. #dontflythem</td>\n",
       "      <td>80</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>5.750000</td>\n",
       "      <td>958.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6729</th>\n",
       "      <td>@SouthwestAir the last 4 times I've arrived @LASairport our gate has been blocked by a slow to depart plane leading to 30-60 min delays</td>\n",
       "      <td>135</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>4.666667</td>\n",
       "      <td>926.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5270</th>\n",
       "      <td>@SouthwestAir y'all are fast we already have our 5 winners! Congrats!!</td>\n",
       "      <td>70</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>5.454545</td>\n",
       "      <td>1228.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14357</th>\n",
       "      <td>@AmericanAir the ticket is a poor gesture of goodwill (for missed trip) to a top status flyer - I may not next time (AA91 LHR-ORD 2/16)</td>\n",
       "      <td>135</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>4.230769</td>\n",
       "      <td>940.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5387</th>\n",
       "      <td>@SouthwestAir - STL-BOS flight today was Cancelled Flightled. Online resched form gives an error. On hold for past 90 min. HELP! #terribleservice</td>\n",
       "      <td>145</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>5.636364</td>\n",
       "      <td>895.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13618</th>\n",
       "      <td>@AmericanAir what terminal LHR for aa5/BA1506? Doesn't say in boarding pass</td>\n",
       "      <td>75</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>5.909091</td>\n",
       "      <td>875.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12872</th>\n",
       "      <td>@AmericanAir already spoke to that line, unwilling to help - really poor support #hitawall</td>\n",
       "      <td>90</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>5.500000</td>\n",
       "      <td>952.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9814</th>\n",
       "      <td>@USAirways That's 1h 10m waiting to get my bag, 1h40m of flight delays, plus the 2hrs of delay I had on the way out. Never flying you again.</td>\n",
       "      <td>140</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>4.035714</td>\n",
       "      <td>1402.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6351</th>\n",
       "      <td>@southwestair Hi, how do I add a flight to my account? Just created it</td>\n",
       "      <td>70</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>4.071429</td>\n",
       "      <td>1430.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9607</th>\n",
       "      <td>@USAirways You guys are screwing up my trip. Thanks for nothing. #lazy</td>\n",
       "      <td>70</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>4.916667</td>\n",
       "      <td>1482.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9539</th>\n",
       "      <td>@USAirways @AmericanAir taking off now. if I don't have a private jet to bora bora by the time I land, I'm calling Oprah</td>\n",
       "      <td>120</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>4.260870</td>\n",
       "      <td>901.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140</th>\n",
       "      <td>@VirginAmerica You'd think paying an extra $100 bucks RT for luggage might afford you hiring an extra hand at @sfo #lame</td>\n",
       "      <td>120</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>4.761905</td>\n",
       "      <td>1051.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>382</th>\n",
       "      <td>@VirginAmerica I need a receipt for a flight change. Can you send one?</td>\n",
       "      <td>70</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>4.461538</td>\n",
       "      <td>1159.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7898</th>\n",
       "      <td>@JetBlue yeah that's all been done. Define \"incidentals\"? How much can I spend on gear so this trip is not a waste?</td>\n",
       "      <td>115</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>4.272727</td>\n",
       "      <td>1331.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13065</th>\n",
       "      <td>@AmericanAir Let's all have a extraordinary week and make it a year to remember #GoingForGreat 2015 thanks so much American Airlines!!!</td>\n",
       "      <td>135</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>5.476190</td>\n",
       "      <td>1097.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>160</th>\n",
       "      <td>@VirginAmerica got a flight (we were told) for 4:50 today..,checked my email and its for 4;50 TOMORROW. This is unacceptable.</td>\n",
       "      <td>125</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>5.300000</td>\n",
       "      <td>1352.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12410</th>\n",
       "      <td>@AmericanAir The flight is the wrong day. Confirm # USXLON. Departure on 5/21 should be 5/22 for 2 people</td>\n",
       "      <td>105</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>4.578947</td>\n",
       "      <td>1357.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4530</th>\n",
       "      <td>@SouthwestAir in flight wifi + @TMobile wifi calling makes for the best combination. Who doesn't love texting 40,000 feet in the air :D</td>\n",
       "      <td>135</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>4.913043</td>\n",
       "      <td>1248.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11086</th>\n",
       "      <td>@USAirways just heard #Monterey airport was closed due to thick fog, any updates</td>\n",
       "      <td>80</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>5.230769</td>\n",
       "      <td>997.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4690</th>\n",
       "      <td>@SouthwestAir My bag was not shown LUV did not make it to PHL with me.</td>\n",
       "      <td>70</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>3.733333</td>\n",
       "      <td>1469.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11578</th>\n",
       "      <td>@USAirways 25 mins after the scheduled departure and still sitting in terminal. Communication would be lovely... http://t.co/pkAxUNYfn2</td>\n",
       "      <td>135</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>1040.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2751</th>\n",
       "      <td>@united cross country flight SFO&amp;gt;BOS. No wifi, not even a can of soda and this quality inflight entertainment #sad http://t.co/xhlc30MTfF</td>\n",
       "      <td>140</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>6.050000</td>\n",
       "      <td>1063.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1236</th>\n",
       "      <td>@united Flight 683 last night was #bad4business. Will be a long time before I recommend United to anyone.</td>\n",
       "      <td>105</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>4.888889</td>\n",
       "      <td>1159.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11890</th>\n",
       "      <td>@AmericanAir I slept in the miami airport due to mechanical issues and was given 10,000 bonus miles to try and make it right. #slapintheface</td>\n",
       "      <td>140</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>4.875000</td>\n",
       "      <td>1149.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12461</th>\n",
       "      <td>@AmericanAir it isn't good enough. You need to provide a phone number. Other partners do. Should be investigated by @IATA @TheFAAOnline</td>\n",
       "      <td>135</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>5.476190</td>\n",
       "      <td>1134.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14323</th>\n",
       "      <td>@AmericanAir I did. 5 times. With 4 of those being hung up on. The other one it finally sent me to the call back notification. #FixYourStuff</td>\n",
       "      <td>140</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>4.423077</td>\n",
       "      <td>1422.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2681</th>\n",
       "      <td>@united how much longer United?? Been on the phone for over an hour to straighten out a star alliance upgrade!!! http://t.co/U1ViEuiDF5</td>\n",
       "      <td>135</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>5.476190</td>\n",
       "      <td>1249.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4933</th>\n",
       "      <td>@SouthwestAir Thanks. 436. Only a minor delay so not a big deal. :)Appreciate the concern though. Boarding now. You do have amazing service!</td>\n",
       "      <td>140</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>5.130435</td>\n",
       "      <td>1184.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10808</th>\n",
       "      <td>@USAirways &amp; also that I was told no comps, upgrades, etc. ReFlight Booking Problems is good enough. #usairwaysfail</td>\n",
       "      <td>115</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>5.444444</td>\n",
       "      <td>1147.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10669</th>\n",
       "      <td>@USAirways -doing best??? Think not. Lousy way to run an airline. Why no online way to connect? You be on hold for 2hrs &amp; then zip</td>\n",
       "      <td>130</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>4.038462</td>\n",
       "      <td>1382.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14582</th>\n",
       "      <td>@AmericanAir seems like queue times are very high, in Q waiting for an agent 4 almost an hour.Flight got Cancelled Flightled. http://t.co/sDm2wvR3zr</td>\n",
       "      <td>148</td>\n",
       "      <td>0.601351</td>\n",
       "      <td>5.772727</td>\n",
       "      <td>1024.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11142</th>\n",
       "      <td>@USAirways My flight cost me just under $800 and this is what happens? I fly with @united and have no problems. I come the US 4-5 times...</td>\n",
       "      <td>138</td>\n",
       "      <td>0.601449</td>\n",
       "      <td>4.148148</td>\n",
       "      <td>1315.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10743</th>\n",
       "      <td>@USAirways stuck on Tarmac for hour an a half and told longer to go and no hotels available when we land in Phoenix #unacceptable #unhappy</td>\n",
       "      <td>138</td>\n",
       "      <td>0.601449</td>\n",
       "      <td>4.560000</td>\n",
       "      <td>1197.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8509</th>\n",
       "      <td>@JetBlue #JetBlueBOS What's up with luggage for Flt 790 DCA-BOS? Pax waiting for 30 min for bags...no show. Carousel 6 signs not updating.</td>\n",
       "      <td>138</td>\n",
       "      <td>0.601449</td>\n",
       "      <td>5.043478</td>\n",
       "      <td>915.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13000</th>\n",
       "      <td>@AmericanAir here is the ticket they gave me after lying, raising their voice. With no explanation on what happened http://t.co/31cFHtK60r</td>\n",
       "      <td>138</td>\n",
       "      <td>0.601449</td>\n",
       "      <td>5.950000</td>\n",
       "      <td>1219.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6166</th>\n",
       "      <td>@SouthwestAir are bicycles that the 62\", 50lbs criteria free or bikes are automatically $75/each way? The FAQ page isn't very clear to me.</td>\n",
       "      <td>138</td>\n",
       "      <td>0.601449</td>\n",
       "      <td>5.043478</td>\n",
       "      <td>1138.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4743</th>\n",
       "      <td>@SouthwestAir I'm going to start charging you for the consistent body pat downs. Starting to feel like a piece of meat. #punishedforflying</td>\n",
       "      <td>138</td>\n",
       "      <td>0.601449</td>\n",
       "      <td>5.318182</td>\n",
       "      <td>1092.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>893</th>\n",
       "      <td>@United THANK U! Secured room for the night Thx to VERY helpful customer service rep N. Dorns.. I thanked her.. Can u 2? #goodenoughmother</td>\n",
       "      <td>138</td>\n",
       "      <td>0.601449</td>\n",
       "      <td>4.791667</td>\n",
       "      <td>1208.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9851</th>\n",
       "      <td>@USAirways @AuroraBIZ check your bag, @PHLAirport bag handlers broke into my suitcase and stole my camera, bag was returned in a trash bag</td>\n",
       "      <td>138</td>\n",
       "      <td>0.601449</td>\n",
       "      <td>5.043478</td>\n",
       "      <td>955.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12452</th>\n",
       "      <td>@AmericanAir I shared my experience several times (AA Ref#1-2990176298), but AA's response is senseless, weather was not the cause!!!</td>\n",
       "      <td>133</td>\n",
       "      <td>0.601504</td>\n",
       "      <td>6.052632</td>\n",
       "      <td>851.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13175</th>\n",
       "      <td>@AmericanAir That will be the third time I have been called by 800-433-7300 an hung on before anyone speaks. What do I do now???</td>\n",
       "      <td>128</td>\n",
       "      <td>0.601562</td>\n",
       "      <td>4.375000</td>\n",
       "      <td>1411.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8064</th>\n",
       "      <td>@JetBlue Challenge accepted. First 5 crew members to send pics in the chairs get FREE tickets to visit us. #wager #IsItSummerYet</td>\n",
       "      <td>128</td>\n",
       "      <td>0.601562</td>\n",
       "      <td>5.142857</td>\n",
       "      <td>814.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5389</th>\n",
       "      <td>@SouthwestAir been on hold for 2 1/2 hrs. Can't get 2 airport 4 flight icy country backroads! Help pls? Can a human answer? #BNA</td>\n",
       "      <td>128</td>\n",
       "      <td>0.601562</td>\n",
       "      <td>4.375000</td>\n",
       "      <td>1014.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5829</th>\n",
       "      <td>@SouthwestAir Sorry to spam, it would just be so awesome to see @Imaginedragons in Vegas. Your help would be absolutely amazing.</td>\n",
       "      <td>128</td>\n",
       "      <td>0.601562</td>\n",
       "      <td>5.142857</td>\n",
       "      <td>1304.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5117</th>\n",
       "      <td>@SouthwestAir a week after Valentines day..not feeling the #LUV. Going on 11 hrs @BWI and don't my bags aren't even coming w me.</td>\n",
       "      <td>128</td>\n",
       "      <td>0.601562</td>\n",
       "      <td>4.608696</td>\n",
       "      <td>1109.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3400</th>\n",
       "      <td>@united @perfectomobile huh? I have sent 16 crash reports in Feb alone. 8 in one day. Only app I have that consistently crashes.</td>\n",
       "      <td>128</td>\n",
       "      <td>0.601562</td>\n",
       "      <td>4.608696</td>\n",
       "      <td>1076.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13583</th>\n",
       "      <td>@AmericanAir Hi guys, can you tell me what the emergency is for #AA65 diverting to #Heathrow from Zurich-New York? Thanks, John.</td>\n",
       "      <td>128</td>\n",
       "      <td>0.601562</td>\n",
       "      <td>5.142857</td>\n",
       "      <td>1237.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7410</th>\n",
       "      <td>@JetBlue why would you hire a guy who doesn't care about the safety of children? Not remotely acceptable. http://t.co/lKFh9HYHW8</td>\n",
       "      <td>128</td>\n",
       "      <td>0.601562</td>\n",
       "      <td>5.789474</td>\n",
       "      <td>1173.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2056</th>\n",
       "      <td>@united thanks for nothing. Really?!? 60 minute wait times when I've had to call back 4x. That's 240minute wait times.</td>\n",
       "      <td>118</td>\n",
       "      <td>0.601695</td>\n",
       "      <td>4.950000</td>\n",
       "      <td>1160.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6662</th>\n",
       "      <td>@SouthwestAir Twitter says I can't DM someone unless they follow me. Can @SouthwestAir follows my twitter? thanks you.</td>\n",
       "      <td>118</td>\n",
       "      <td>0.601695</td>\n",
       "      <td>5.611111</td>\n",
       "      <td>1122.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12221</th>\n",
       "      <td>@AmericanAir if I could fly an md80/dc10 I would be so happy I live that plane so much md80 is love md80 is life.</td>\n",
       "      <td>113</td>\n",
       "      <td>0.601770</td>\n",
       "      <td>3.750000</td>\n",
       "      <td>1299.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13025</th>\n",
       "      <td>@AmericanAir Was put on hold for 5.5 hrs then got a call back at 11:20 pm, only to wait on hold for another hour.</td>\n",
       "      <td>113</td>\n",
       "      <td>0.601770</td>\n",
       "      <td>3.750000</td>\n",
       "      <td>1429.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                       text  \\\n",
       "13043                              @AmericanAir hey ho its not me losing any money (only you) just next make sure you stick to the \"flyers right's booklet\"   \n",
       "14264                         @AmericanAir Still sitting in airport as 6:05 flt delayed until 6:55...but still not boarding, what is happening? No info....   \n",
       "5875                                    @SouthwestAir my trip is a month away...why do you consider it \"around the corner\"? Is that a setting I can change?   \n",
       "12875                                                                           @AmericanAir Yes, thanks I found those, didn't see the gray tab at first :)   \n",
       "7986                                                                            @JetBlue No, the flight wasn't until 9:51pm, but it's already been delayed.   \n",
       "5192                                    @SouthwestAir reservation (FEHQNE) 21FEB15 | DCA-RSW. Want refund not credit for Cancelled Flightled flight please.   \n",
       "358            @VirginAmerica just promoting the product is all, had a problem with southwest and recommend noneother than the best! http://t.co/tFaNXBh1Cf   \n",
       "88                  @VirginAmerica I &lt;3 Flying VA But Life happens and I am trying to #change my trip JPERHI Can you help.VA home page will not let me ?   \n",
       "8025                                                                       @JetBlue has the worst customer service out of any airline company. #dontflythem   \n",
       "6729                @SouthwestAir the last 4 times I've arrived @LASairport our gate has been blocked by a slow to depart plane leading to 30-60 min delays   \n",
       "5270                                                                                 @SouthwestAir y'all are fast we already have our 5 winners! Congrats!!   \n",
       "14357               @AmericanAir the ticket is a poor gesture of goodwill (for missed trip) to a top status flyer - I may not next time (AA91 LHR-ORD 2/16)   \n",
       "5387      @SouthwestAir - STL-BOS flight today was Cancelled Flightled. Online resched form gives an error. On hold for past 90 min. HELP! #terribleservice   \n",
       "13618                                                                           @AmericanAir what terminal LHR for aa5/BA1506? Doesn't say in boarding pass   \n",
       "12872                                                            @AmericanAir already spoke to that line, unwilling to help - really poor support #hitawall   \n",
       "9814           @USAirways That's 1h 10m waiting to get my bag, 1h40m of flight delays, plus the 2hrs of delay I had on the way out. Never flying you again.   \n",
       "6351                                                                                 @southwestair Hi, how do I add a flight to my account? Just created it   \n",
       "9607                                                                                 @USAirways You guys are screwing up my trip. Thanks for nothing. #lazy   \n",
       "9539                               @USAirways @AmericanAir taking off now. if I don't have a private jet to bora bora by the time I land, I'm calling Oprah   \n",
       "140                                @VirginAmerica You'd think paying an extra $100 bucks RT for luggage might afford you hiring an extra hand at @sfo #lame   \n",
       "382                                                                                  @VirginAmerica I need a receipt for a flight change. Can you send one?   \n",
       "7898                                    @JetBlue yeah that's all been done. Define \"incidentals\"? How much can I spend on gear so this trip is not a waste?   \n",
       "13065               @AmericanAir Let's all have a extraordinary week and make it a year to remember #GoingForGreat 2015 thanks so much American Airlines!!!   \n",
       "160                           @VirginAmerica got a flight (we were told) for 4:50 today..,checked my email and its for 4;50 TOMORROW. This is unacceptable.   \n",
       "12410                                             @AmericanAir The flight is the wrong day. Confirm # USXLON. Departure on 5/21 should be 5/22 for 2 people   \n",
       "4530                @SouthwestAir in flight wifi + @TMobile wifi calling makes for the best combination. Who doesn't love texting 40,000 feet in the air :D   \n",
       "11086                                                                      @USAirways just heard #Monterey airport was closed due to thick fog, any updates   \n",
       "4690                                                                                 @SouthwestAir My bag was not shown LUV did not make it to PHL with me.   \n",
       "11578               @USAirways 25 mins after the scheduled departure and still sitting in terminal. Communication would be lovely... http://t.co/pkAxUNYfn2   \n",
       "2751           @united cross country flight SFO&gt;BOS. No wifi, not even a can of soda and this quality inflight entertainment #sad http://t.co/xhlc30MTfF   \n",
       "1236                                              @united Flight 683 last night was #bad4business. Will be a long time before I recommend United to anyone.   \n",
       "11890          @AmericanAir I slept in the miami airport due to mechanical issues and was given 10,000 bonus miles to try and make it right. #slapintheface   \n",
       "12461               @AmericanAir it isn't good enough. You need to provide a phone number. Other partners do. Should be investigated by @IATA @TheFAAOnline   \n",
       "14323          @AmericanAir I did. 5 times. With 4 of those being hung up on. The other one it finally sent me to the call back notification. #FixYourStuff   \n",
       "2681                @united how much longer United?? Been on the phone for over an hour to straighten out a star alliance upgrade!!! http://t.co/U1ViEuiDF5   \n",
       "4933           @SouthwestAir Thanks. 436. Only a minor delay so not a big deal. :)Appreciate the concern though. Boarding now. You do have amazing service!   \n",
       "10808                                   @USAirways & also that I was told no comps, upgrades, etc. ReFlight Booking Problems is good enough. #usairwaysfail   \n",
       "10669                    @USAirways -doing best??? Think not. Lousy way to run an airline. Why no online way to connect? You be on hold for 2hrs & then zip   \n",
       "14582  @AmericanAir seems like queue times are very high, in Q waiting for an agent 4 almost an hour.Flight got Cancelled Flightled. http://t.co/sDm2wvR3zr   \n",
       "11142            @USAirways My flight cost me just under $800 and this is what happens? I fly with @united and have no problems. I come the US 4-5 times...   \n",
       "10743            @USAirways stuck on Tarmac for hour an a half and told longer to go and no hotels available when we land in Phoenix #unacceptable #unhappy   \n",
       "8509             @JetBlue #JetBlueBOS What's up with luggage for Flt 790 DCA-BOS? Pax waiting for 30 min for bags...no show. Carousel 6 signs not updating.   \n",
       "13000            @AmericanAir here is the ticket they gave me after lying, raising their voice. With no explanation on what happened http://t.co/31cFHtK60r   \n",
       "6166             @SouthwestAir are bicycles that the 62\", 50lbs criteria free or bikes are automatically $75/each way? The FAQ page isn't very clear to me.   \n",
       "4743             @SouthwestAir I'm going to start charging you for the consistent body pat downs. Starting to feel like a piece of meat. #punishedforflying   \n",
       "893              @United THANK U! Secured room for the night Thx to VERY helpful customer service rep N. Dorns.. I thanked her.. Can u 2? #goodenoughmother   \n",
       "9851             @USAirways @AuroraBIZ check your bag, @PHLAirport bag handlers broke into my suitcase and stole my camera, bag was returned in a trash bag   \n",
       "12452                 @AmericanAir I shared my experience several times (AA Ref#1-2990176298), but AA's response is senseless, weather was not the cause!!!   \n",
       "13175                      @AmericanAir That will be the third time I have been called by 800-433-7300 an hung on before anyone speaks. What do I do now???   \n",
       "8064                       @JetBlue Challenge accepted. First 5 crew members to send pics in the chairs get FREE tickets to visit us. #wager #IsItSummerYet   \n",
       "5389                       @SouthwestAir been on hold for 2 1/2 hrs. Can't get 2 airport 4 flight icy country backroads! Help pls? Can a human answer? #BNA   \n",
       "5829                       @SouthwestAir Sorry to spam, it would just be so awesome to see @Imaginedragons in Vegas. Your help would be absolutely amazing.   \n",
       "5117                       @SouthwestAir a week after Valentines day..not feeling the #LUV. Going on 11 hrs @BWI and don't my bags aren't even coming w me.   \n",
       "3400                       @united @perfectomobile huh? I have sent 16 crash reports in Feb alone. 8 in one day. Only app I have that consistently crashes.   \n",
       "13583                      @AmericanAir Hi guys, can you tell me what the emergency is for #AA65 diverting to #Heathrow from Zurich-New York? Thanks, John.   \n",
       "7410                       @JetBlue why would you hire a guy who doesn't care about the safety of children? Not remotely acceptable. http://t.co/lKFh9HYHW8   \n",
       "2056                                 @united thanks for nothing. Really?!? 60 minute wait times when I've had to call back 4x. That's 240minute wait times.   \n",
       "6662                                 @SouthwestAir Twitter says I can't DM someone unless they follow me. Can @SouthwestAir follows my twitter? thanks you.   \n",
       "12221                                     @AmericanAir if I could fly an md80/dc10 I would be so happy I live that plane so much md80 is love md80 is life.   \n",
       "13025                                     @AmericanAir Was put on hold for 5.5 hrs then got a call back at 11:20 pm, only to wait on hold for another hour.   \n",
       "\n",
       "       length  word fraction  token length  english score  \n",
       "13043     120       0.600000      4.500000         1324.0  \n",
       "14264     125       0.600000      5.631579         1033.0  \n",
       "5875      115       0.600000      4.523810         1287.0  \n",
       "12875      75       0.600000      4.428571         1199.0  \n",
       "7986       75       0.600000      5.333333         1536.0  \n",
       "5192      115       0.600000      6.733333          984.0  \n",
       "358       140       0.600000      6.421053         1106.0  \n",
       "88        135       0.600000      4.037037          975.0  \n",
       "8025       80       0.600000      5.750000          958.0  \n",
       "6729      135       0.600000      4.666667          926.0  \n",
       "5270       70       0.600000      5.454545         1228.0  \n",
       "14357     135       0.600000      4.230769          940.0  \n",
       "5387      145       0.600000      5.636364          895.0  \n",
       "13618      75       0.600000      5.909091          875.0  \n",
       "12872      90       0.600000      5.500000          952.0  \n",
       "9814      140       0.600000      4.035714         1402.0  \n",
       "6351       70       0.600000      4.071429         1430.0  \n",
       "9607       70       0.600000      4.916667         1482.0  \n",
       "9539      120       0.600000      4.260870          901.0  \n",
       "140       120       0.600000      4.761905         1051.0  \n",
       "382        70       0.600000      4.461538         1159.0  \n",
       "7898      115       0.600000      4.272727         1331.0  \n",
       "13065     135       0.600000      5.476190         1097.0  \n",
       "160       125       0.600000      5.300000         1352.0  \n",
       "12410     105       0.600000      4.578947         1357.0  \n",
       "4530      135       0.600000      4.913043         1248.0  \n",
       "11086      80       0.600000      5.230769          997.0  \n",
       "4690       70       0.600000      3.733333         1469.0  \n",
       "11578     135       0.600000      7.000000         1040.0  \n",
       "2751      140       0.600000      6.050000         1063.0  \n",
       "1236      105       0.600000      4.888889         1159.0  \n",
       "11890     140       0.600000      4.875000         1149.0  \n",
       "12461     135       0.600000      5.476190         1134.0  \n",
       "14323     140       0.600000      4.423077         1422.0  \n",
       "2681      135       0.600000      5.476190         1249.0  \n",
       "4933      140       0.600000      5.130435         1184.0  \n",
       "10808     115       0.600000      5.444444         1147.0  \n",
       "10669     130       0.600000      4.038462         1382.0  \n",
       "14582     148       0.601351      5.772727         1024.0  \n",
       "11142     138       0.601449      4.148148         1315.0  \n",
       "10743     138       0.601449      4.560000         1197.0  \n",
       "8509      138       0.601449      5.043478          915.0  \n",
       "13000     138       0.601449      5.950000         1219.0  \n",
       "6166      138       0.601449      5.043478         1138.0  \n",
       "4743      138       0.601449      5.318182         1092.0  \n",
       "893       138       0.601449      4.791667         1208.0  \n",
       "9851      138       0.601449      5.043478          955.0  \n",
       "12452     133       0.601504      6.052632          851.0  \n",
       "13175     128       0.601562      4.375000         1411.0  \n",
       "8064      128       0.601562      5.142857          814.0  \n",
       "5389      128       0.601562      4.375000         1014.0  \n",
       "5829      128       0.601562      5.142857         1304.0  \n",
       "5117      128       0.601562      4.608696         1109.0  \n",
       "3400      128       0.601562      4.608696         1076.0  \n",
       "13583     128       0.601562      5.142857         1237.0  \n",
       "7410      128       0.601562      5.789474         1173.0  \n",
       "2056      118       0.601695      4.950000         1160.0  \n",
       "6662      118       0.601695      5.611111         1122.0  \n",
       "12221     113       0.601770      3.750000         1299.0  \n",
       "13025     113       0.601770      3.750000         1429.0  "
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = trim_dataset_by(\n",
    "    df, \n",
    "    \n",
    "    # sort_by='length',\n",
    "    sort_by='word fraction',\n",
    "    # sort_by='token length',\n",
    "    # sort_by='english score',\n",
    "    ascending=True,\n",
    "    \n",
    "    length=(70, None),\n",
    "    word_fraction=(0.6, None),\n",
    "    token_length=(None, 7),\n",
    "    english_score=(650, None),\n",
    ")\n",
    "a[:60]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "id": "9415d3c6-f368-49cb-860d-3dcddb60ac4c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:38:57.245356Z",
     "iopub.status.busy": "2024-01-08T15:38:57.245114Z",
     "iopub.status.idle": "2024-01-08T15:39:03.053085Z",
     "shell.execute_reply": "2024-01-08T15:39:03.052159Z",
     "shell.execute_reply.started": "2024-01-08T15:38:57.245336Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time elapsed: 6s\n",
      "samples: 8468, similar samples detected: 41 (0.48%)\n"
     ]
    }
   ],
   "source": [
    "df = drop_similar(supersample(a, AIRLINE_SUPPORT));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "id": "a19e5a06-e0d1-42df-8bcf-da70c02965a1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:39:03.054944Z",
     "iopub.status.busy": "2024-01-08T15:39:03.054226Z",
     "iopub.status.idle": "2024-01-08T15:39:03.085718Z",
     "shell.execute_reply": "2024-01-08T15:39:03.085007Z",
     "shell.execute_reply.started": "2024-01-08T15:39:03.054913Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8468, 3)"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = select_columns(df.copy(), ['text'])\n",
    "df['topic'] = 'airline support'\n",
    "df['id'] = 0\n",
    "\n",
    "df = sample(df, AIRLINE_SUPPORT)\n",
    "dataset = concat(dataset, df)\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2caa6795-feb7-49c0-b22a-8bcd4a4ca4f0",
   "metadata": {},
   "source": [
    "# FIFA World Cup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "id": "0db4dd06-29ec-4328-87c8-b21639b7283a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:39:03.086631Z",
     "iopub.status.busy": "2024-01-08T15:39:03.086439Z",
     "iopub.status.idle": "2024-01-08T15:39:03.179703Z",
     "shell.execute_reply": "2024-01-08T15:39:03.178853Z",
     "shell.execute_reply.started": "2024-01-08T15:39:03.086612Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(22524, 1)"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = load_dataset('fifa_world_cup.csv', source=True)\n",
    "df.rename(columns={'Tweet': 'text'}, inplace=True)\n",
    "df = select_columns(df, ['text'])\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "id": "80e15f50-ec9d-48f3-8f7f-9486379b6e7e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:39:03.181219Z",
     "iopub.status.busy": "2024-01-08T15:39:03.180983Z",
     "iopub.status.idle": "2024-01-08T15:39:04.135139Z",
     "shell.execute_reply": "2024-01-08T15:39:04.134402Z",
     "shell.execute_reply.started": "2024-01-08T15:39:03.181198Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(17341, 1)"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = preprocessing_pipeline(df)\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "id": "eee73ee9-1520-455c-836c-1192d4aa0fc5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:39:04.136390Z",
     "iopub.status.busy": "2024-01-08T15:39:04.136195Z",
     "iopub.status.idle": "2024-01-08T15:39:04.156860Z",
     "shell.execute_reply": "2024-01-08T15:39:04.156132Z",
     "shell.execute_reply.started": "2024-01-08T15:39:04.136371Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6989</th>\n",
       "      <td>Anyone got odds on Qatar to win 1-0 with a comedy last minute own goal and Ecuador to have 8 goals disallowed...? Asking for a Qatari Sheikh.... #WorldCup2022 #FifaWorldCup</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7981</th>\n",
       "      <td>#WorldCup2022 Be There!! https://t.co/wPL9L4gjDW</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11956</th>\n",
       "      <td>And the headlines are already out in India!! JK you were amazing!! @BTS_twt #FIFAWorldCup #WorldCup2022 #jungkook #FIFAWorldCup2022 https://t.co/dCjuvv4OQg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4446</th>\n",
       "      <td>After watching #FIFAUncovered I have trust issues. Valencia is giving me hope though. 😅\\n #QatarEcuador\\n#WorldCup2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7914</th>\n",
       "      <td>I'm saying it now match fixing for Qatar #WorldCup2022</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                               text\n",
       "6989   Anyone got odds on Qatar to win 1-0 with a comedy last minute own goal and Ecuador to have 8 goals disallowed...? Asking for a Qatari Sheikh.... #WorldCup2022 #FifaWorldCup\n",
       "7981                                                                                                                               #WorldCup2022 Be There!! https://t.co/wPL9L4gjDW\n",
       "11956                   And the headlines are already out in India!! JK you were amazing!! @BTS_twt #FIFAWorldCup #WorldCup2022 #jungkook #FIFAWorldCup2022 https://t.co/dCjuvv4OQg\n",
       "4446                                                         After watching #FIFAUncovered I have trust issues. Valencia is giving me hope though. 😅\\n #QatarEcuador\\n#WorldCup2022\n",
       "7914                                                                                                                         I'm saying it now match fixing for Qatar #WorldCup2022"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "id": "0a7ebf14-a98b-4797-be34-3f4acedbd58f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:39:04.157872Z",
     "iopub.status.busy": "2024-01-08T15:39:04.157673Z",
     "iopub.status.idle": "2024-01-08T15:39:04.531473Z",
     "shell.execute_reply": "2024-01-08T15:39:04.530744Z",
     "shell.execute_reply.started": "2024-01-08T15:39:04.157852Z"
    }
   },
   "outputs": [],
   "source": [
    "add_stats(df);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "id": "1d84ec13-fa07-44bd-a623-f18ec0806a59",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:39:04.532991Z",
     "iopub.status.busy": "2024-01-08T15:39:04.532783Z",
     "iopub.status.idle": "2024-01-08T15:39:04.567084Z",
     "shell.execute_reply": "2024-01-08T15:39:04.566352Z",
     "shell.execute_reply.started": "2024-01-08T15:39:04.532970Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "samples: 7041, reduction: 59.40%\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>length</th>\n",
       "      <th>word fraction</th>\n",
       "      <th>token length</th>\n",
       "      <th>english score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4495</th>\n",
       "      <td>Valencia 2 goals already. Ecuador lead 0-2 #WorldCup2022</td>\n",
       "      <td>56</td>\n",
       "      <td>0.553571</td>\n",
       "      <td>6.125000</td>\n",
       "      <td>750.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5849</th>\n",
       "      <td>Enner Valencia took 9 minutes to resettle, then told VAR to choke on its match-fixiness. #WorldCup2022 https://t.co/m1FYpEvlzL</td>\n",
       "      <td>126</td>\n",
       "      <td>0.555556</td>\n",
       "      <td>6.470588</td>\n",
       "      <td>750.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3356</th>\n",
       "      <td>Qatar seff go collect woto woto for this competition 😂😂\\n#WorldCup2022 #Qatar2022</td>\n",
       "      <td>80</td>\n",
       "      <td>0.550000</td>\n",
       "      <td>5.750000</td>\n",
       "      <td>752.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19771</th>\n",
       "      <td>Predict who will be the quarter finalists and win free FIFA tokens.\\n\\n#FIFAWorldCup #WorldCup #WorldCup2022</td>\n",
       "      <td>106</td>\n",
       "      <td>0.518868</td>\n",
       "      <td>6.066667</td>\n",
       "      <td>752.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9663</th>\n",
       "      <td>Course that Ecuador goal was offside 😭🤣🤑 #QATECU #WorldCup2022</td>\n",
       "      <td>62</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>753.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1648</th>\n",
       "      <td>Fans chant for Palestine and raise Palestinian flags during a live broadcast of an Israeli channel from Qatar. #WorldCup2022. #RaisePalestineFlag https://t.co/8Z955J85Ty</td>\n",
       "      <td>169</td>\n",
       "      <td>0.544379</td>\n",
       "      <td>7.095238</td>\n",
       "      <td>754.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12827</th>\n",
       "      <td>Clint Dempsey calling out Alexi Lawless on live TV. He been a hater 🤣🤣 #WorldCup2022</td>\n",
       "      <td>84</td>\n",
       "      <td>0.642857</td>\n",
       "      <td>4.666667</td>\n",
       "      <td>754.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10122</th>\n",
       "      <td>Ecuador scores the first world cup goal in less than 3 minutes. 🤯 #WorldCup2022 #WorldcupQatar2022</td>\n",
       "      <td>98</td>\n",
       "      <td>0.510204</td>\n",
       "      <td>5.600000</td>\n",
       "      <td>755.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4213</th>\n",
       "      <td>Mina I even forgot kune #WorldCup2022 yaz.. 😴 Anyway @netflix ingiphethe sharp.. 😉</td>\n",
       "      <td>82</td>\n",
       "      <td>0.524390</td>\n",
       "      <td>5.384615</td>\n",
       "      <td>756.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>868</th>\n",
       "      <td>COLOMBIAN SINGER MALUMA:\\n\\nThe star features alongside Nicki Minaj and Myriam Fares on Tukoh Taka, the official Fifa Qatar World Cup anthem. ^MA\\n#FIFAWorldCup\\n#WorldCup2022 #LOOKUPTV https://t.co/J9yQg2aSjH</td>\n",
       "      <td>205</td>\n",
       "      <td>0.556098</td>\n",
       "      <td>6.592593</td>\n",
       "      <td>757.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11584</th>\n",
       "      <td>First time watched an opening ceremony without sexual content and half naked women roaming around. #Qatar 👏🏼\\n#FIFAWorldCup #QatarWorldCup2022 #WorldCup2022 #Qatar2022</td>\n",
       "      <td>166</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>6.952381</td>\n",
       "      <td>757.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19146</th>\n",
       "      <td>#WorldCup2022 \\nWinner: Argentina\\nDark horses: USA/Denmark\\nFlops:France\\nTop goalscorers:Julian Alvarez/memphis depay.\\nPlayer of tournament:Messi\\nYoung player of tournament:Enzo fernandez</td>\n",
       "      <td>185</td>\n",
       "      <td>0.778378</td>\n",
       "      <td>8.736842</td>\n",
       "      <td>757.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10518</th>\n",
       "      <td>3 mins into the game. ecuador scoreee goallllll #WorldCup2022</td>\n",
       "      <td>61</td>\n",
       "      <td>0.622951</td>\n",
       "      <td>5.888889</td>\n",
       "      <td>758.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18910</th>\n",
       "      <td>Conservatively 1500 plus migrant workers died making a Qatari World Cup a reality how many died in 2 years of construction of spurs super stadium ? FIFA &amp; Qatar heading for PR hell 😎👍🏼 #WorldCup #WorldCup2022 #football #QatarWorldCup2022 #Qatar #FIFAWorldCup https://t.co/TLkS8AOF8F</td>\n",
       "      <td>282</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>5.902439</td>\n",
       "      <td>759.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22478</th>\n",
       "      <td>My #WorldCup2022 predictons.\\nWinner: Argentina\\nTop scorer: Lukaku\\nOne of Belgium or Denmark to make semis\\nSurprise packets: Saudi Arabia.</td>\n",
       "      <td>137</td>\n",
       "      <td>0.729927</td>\n",
       "      <td>5.900000</td>\n",
       "      <td>760.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22040</th>\n",
       "      <td>FIFA has named Wessam Qutob, a Palestinian refugee and social media influencer, as the #WorldCup2022 ambassador in Qatar. https://t.co/p5tFywuczj</td>\n",
       "      <td>145</td>\n",
       "      <td>0.606897</td>\n",
       "      <td>6.684211</td>\n",
       "      <td>760.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3337</th>\n",
       "      <td>A player has scored a brace in the World Cup opener for the third consecutive World Cup\\n\\n2014 - Neymar Jr. vs Croatia\\n2018 - Denis Cheryshev vs Saudi Arabia\\n2022 - Enner Valencia vs Qatar\\n\\n#WorldCup #FIFAWorldCup #WorldCup2022 https://t.co/Mkv8EYaJRr</td>\n",
       "      <td>250</td>\n",
       "      <td>0.540000</td>\n",
       "      <td>5.225000</td>\n",
       "      <td>760.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17499</th>\n",
       "      <td>FIFA has named Wessam Qutob, a Palestinian refugee and social media influencer, as the #WorldCup2022 ambassador in Qatar. https://t.co/v2agy3Nxbj</td>\n",
       "      <td>145</td>\n",
       "      <td>0.606897</td>\n",
       "      <td>6.684211</td>\n",
       "      <td>760.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14053</th>\n",
       "      <td>The most depressing start to a World Cup I have ever experienced in my life, understandably. #FIFAWorldCup #WorldCup2022 #BBCWorldCup</td>\n",
       "      <td>133</td>\n",
       "      <td>0.563910</td>\n",
       "      <td>6.052632</td>\n",
       "      <td>761.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>960</th>\n",
       "      <td>The #Qatar2022 #WorldCup2022 has opened with a ceremony at the Al-Bayt Stadium in Al Khor, north of Doha\\n\\n📷 Kirill KUDRYAVTSEV /Raul ARBOLEDA /MANAN VATSYAYANA /AFP https://t.co/98yXAgcnfT</td>\n",
       "      <td>188</td>\n",
       "      <td>0.515957</td>\n",
       "      <td>5.962963</td>\n",
       "      <td>761.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20350</th>\n",
       "      <td>COLOMBIAN SINGER MALUMA:\\n\\nThe star features alongside Nicki Minaj and Myriam Fares on Tukoh Taka, the official FIFA Qatar World Cup anthem. ^MA\\n#FIFAWorldCup\\n#WorldCup2022 #LOOKUPTV https://t.co/HHfeC25KVO</td>\n",
       "      <td>205</td>\n",
       "      <td>0.556098</td>\n",
       "      <td>6.592593</td>\n",
       "      <td>761.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8953</th>\n",
       "      <td>Offside??? VAR???? Absolute Bollox. Ecuador were a mile onside at every movement #WorldCup2022 #corrupt</td>\n",
       "      <td>103</td>\n",
       "      <td>0.592233</td>\n",
       "      <td>6.428571</td>\n",
       "      <td>762.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7117</th>\n",
       "      <td>#ssfootball #WorldCup2022 #Qatar2022\\nIn South Africa that's a clear goal, Ecuador will apset the host</td>\n",
       "      <td>101</td>\n",
       "      <td>0.504950</td>\n",
       "      <td>5.800000</td>\n",
       "      <td>762.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1857</th>\n",
       "      <td>Football fans chant for Palestine and raise Palestinian flags during a live broadcast by an Israeli TV in Qatar in #WorldCup2022. \\n \\n#RaisePalestineFlags\\n\\n https://t.co/5c5fQ7tJaI</td>\n",
       "      <td>179</td>\n",
       "      <td>0.530726</td>\n",
       "      <td>6.608696</td>\n",
       "      <td>763.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2379</th>\n",
       "      <td>Ladies and Gentlemen it's official 🤣🤣\\n\\nLadies, Unapeana remote control ukiwa pande gani?\\n\\n#WorldCup2022 https://t.co/nHFOeMCjph</td>\n",
       "      <td>127</td>\n",
       "      <td>0.551181</td>\n",
       "      <td>7.400000</td>\n",
       "      <td>763.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18331</th>\n",
       "      <td>Let's focus on #WorldCup2022 pulak. Prediction Brazil or Argentina akan menang this year 😅😅</td>\n",
       "      <td>91</td>\n",
       "      <td>0.670330</td>\n",
       "      <td>5.571429</td>\n",
       "      <td>764.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6293</th>\n",
       "      <td>1-0 Ecuador from a pen. Awaits @FIFAcom denying goal and awarding a penalty to Qatar due to Ecuador players smiling! #WorldCup2022</td>\n",
       "      <td>130</td>\n",
       "      <td>0.646154</td>\n",
       "      <td>5.238095</td>\n",
       "      <td>765.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18735</th>\n",
       "      <td>FIFA has named Wessam Qutob, a Palestinian refugee and social media influencer, as the #WorldCup2022 ambassador in Qatar. https://t.co/2FfNGxZUaX</td>\n",
       "      <td>145</td>\n",
       "      <td>0.606897</td>\n",
       "      <td>6.684211</td>\n",
       "      <td>766.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19135</th>\n",
       "      <td>@FIFAWorldCup #WorldCup2022 Countdown: 5 hours\\n\\nPowered by #Algorand The Official Blockchain Platform of FIFA. Sports love $ALGO , look it up! ( hint: sail GP, formula E, drone racing ) https://t.co/0DFH2bhFkf</td>\n",
       "      <td>209</td>\n",
       "      <td>0.502392</td>\n",
       "      <td>5.741935</td>\n",
       "      <td>768.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11803</th>\n",
       "      <td>#WorldCup2022 \\nQatar vs Ecuador\\n\\nMy prediction: 2-0 Ecuador\\nMan of the match: Enner Valencia</td>\n",
       "      <td>92</td>\n",
       "      <td>0.641304</td>\n",
       "      <td>5.500000</td>\n",
       "      <td>768.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9607</th>\n",
       "      <td>Hahahaha dis #WorldCup2022 go be something else eiii alo offside</td>\n",
       "      <td>64</td>\n",
       "      <td>0.656250</td>\n",
       "      <td>5.500000</td>\n",
       "      <td>768.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16472</th>\n",
       "      <td>#WorldCup2022 predictions \\n\\nWinner: Brazil \\nGolden Boot: Harry Kane \\nGolden Ball: Messi \\nYoung Player: Musiala \\nDark Horses: Uruguay \\nDisappointing Team: England</td>\n",
       "      <td>161</td>\n",
       "      <td>0.720497</td>\n",
       "      <td>6.750000</td>\n",
       "      <td>768.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>144</th>\n",
       "      <td>Stop American emigrant from going any damn where them stoclem took over rogermi kimthro Hashiq bands are ludicrous #WorldCup2022</td>\n",
       "      <td>128</td>\n",
       "      <td>0.757812</td>\n",
       "      <td>5.789474</td>\n",
       "      <td>768.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2880</th>\n",
       "      <td>Palestinian Authority President Mahmoud Abbas has attended the opening ceremony of the #WorldCup2022 at Al-Bayt Stadium in Qatar's capital Doha. https://t.co/zRBB0mtl4a</td>\n",
       "      <td>168</td>\n",
       "      <td>0.648810</td>\n",
       "      <td>7.047619</td>\n",
       "      <td>769.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13867</th>\n",
       "      <td>Sorry Mr Jungkook sir but gi-friggin'-normous La'eeb stole the show. I expect to see him in every match. #WorldCup2022 https://t.co/uD2je8sorM</td>\n",
       "      <td>142</td>\n",
       "      <td>0.570423</td>\n",
       "      <td>6.150000</td>\n",
       "      <td>769.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2677</th>\n",
       "      <td>Alternate caption:\\n\\nBrasil, thinking ahead to their knockout round match vs Canada, attempt to thwart Canadian player's health regiment by sending over gifts from a Brasilian company... \" ;)\\n\\n#CanMNT #Qatar2022 #WorldCup2022 #FIFAWorldCup #TimHortons https://t.co/t05DjejWG6</td>\n",
       "      <td>274</td>\n",
       "      <td>0.551095</td>\n",
       "      <td>6.800000</td>\n",
       "      <td>769.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2953</th>\n",
       "      <td>Palestinian Authority President Mahmoud Abbas has attended the opening ceremony of the #WorldCup2022 at Al-Bayt Stadium in Qatar's capital Doha. https://t.co/ZoV8GcQafB</td>\n",
       "      <td>168</td>\n",
       "      <td>0.648810</td>\n",
       "      <td>7.047619</td>\n",
       "      <td>769.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21017</th>\n",
       "      <td>#WorldCup2022 #WorldCup Jungkook 'Dreamers' is just ok, not grandstanding, not boring. Ricky Martin (1996) &amp; Shakira (2010) are still the standards for World Cup songs...</td>\n",
       "      <td>170</td>\n",
       "      <td>0.558824</td>\n",
       "      <td>5.840000</td>\n",
       "      <td>769.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12103</th>\n",
       "      <td>Did Anne Lister saw any soccer ball before? \\n\\nIn 1836 Charles Goodyear patented vulcanized rubber. But, only in 1855, Charles Goodyear designed and built the first vulcanized rubber soccer balls.\\n\\n#WorldCup2022 #SaveGentlemanJack #GJWorldCup @LookoutPointTV @AmazonStudios https://t.co/6Bkz4LDuOI</td>\n",
       "      <td>296</td>\n",
       "      <td>0.510135</td>\n",
       "      <td>7.166667</td>\n",
       "      <td>770.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17431</th>\n",
       "      <td>LEAKED ! A mole in the camp has leaked tomorrow's #Englandfootball #WorldCup2022 team. Some will be disappointed !\\nPickford\\nTrippier, Maguire, Stones, Shaw\\nRice, Bellingham, Mount\\nSaka , Kane, Sterling .\\n#ThreeLions</td>\n",
       "      <td>215</td>\n",
       "      <td>0.613953</td>\n",
       "      <td>5.750000</td>\n",
       "      <td>770.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19784</th>\n",
       "      <td>Isolation, genocide, murder, all part of the daily life of Palestinians under Israeli occupation.. Terrorist Israel that is committing barbaric crimes in Palestine is criticizing Qatar! \\n+\\n#QatarWorldCup2022\\n#WorldCup2022\\n#FreePalestine\\n#FreeAhmadManasrah</td>\n",
       "      <td>255</td>\n",
       "      <td>0.603922</td>\n",
       "      <td>7.225806</td>\n",
       "      <td>771.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4017</th>\n",
       "      <td>This looks fake as fuck 🤣🤣🤣 shite team , shite fans @qatar \\n#WorldCup2022</td>\n",
       "      <td>73</td>\n",
       "      <td>0.506849</td>\n",
       "      <td>4.615385</td>\n",
       "      <td>772.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3560</th>\n",
       "      <td>That wire transfer at halftime Finna go crazy #Qatar2022 #WorldCup2022</td>\n",
       "      <td>70</td>\n",
       "      <td>0.542857</td>\n",
       "      <td>6.100000</td>\n",
       "      <td>772.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17703</th>\n",
       "      <td>Brazil\\nArgentina\\nLook strong to win the World Cup acc to me.\\n#WorldCup2022 \\n#FIFAWorldCup</td>\n",
       "      <td>89</td>\n",
       "      <td>0.539326</td>\n",
       "      <td>5.357143</td>\n",
       "      <td>774.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2908</th>\n",
       "      <td>Ecuador's first goal vs Qatar was disallowed for suspected homosexuality. #WorldCup2022</td>\n",
       "      <td>87</td>\n",
       "      <td>0.712644</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>774.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21975</th>\n",
       "      <td>FIFA has named Wessam Qutob, a Palestinian refugee and social media influencer, as the #WorldCup2022 ambassador in Qatar. https://t.co/LPLaoApnxS</td>\n",
       "      <td>145</td>\n",
       "      <td>0.606897</td>\n",
       "      <td>6.684211</td>\n",
       "      <td>775.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7664</th>\n",
       "      <td>Sheikh s robbed Ecuador with var #WorldCup2022 #QATECU that was not offside</td>\n",
       "      <td>75</td>\n",
       "      <td>0.586667</td>\n",
       "      <td>5.333333</td>\n",
       "      <td>775.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20460</th>\n",
       "      <td>FIFA PRESIDENT:\\n\\nIn a monologue at the start of a news conference in Doha, Infantino accused Western critics of Qatar’s human rights record of hypocrisy. ^MA\\n#FIFAWorldCup\\n#WorldCup2022 #LOOKUPTV https://t.co/cqHfPdAkld</td>\n",
       "      <td>219</td>\n",
       "      <td>0.566210</td>\n",
       "      <td>6.300000</td>\n",
       "      <td>775.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5431</th>\n",
       "      <td>According to VAR, Ecuador’s penalty shooter was OFFSIDE!\\n#WorldCup2022 \\n#Qatar2022</td>\n",
       "      <td>82</td>\n",
       "      <td>0.560976</td>\n",
       "      <td>7.200000</td>\n",
       "      <td>775.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17673</th>\n",
       "      <td>FIFA president Gianni Infantino has his say on Europe’s criticism of the FIFA World Cup 2022 in Qatar 👀❌\\n\\n#FIFAWorldCup #WorldCup2022 https://t.co/vl0fkdX99X</td>\n",
       "      <td>157</td>\n",
       "      <td>0.503185</td>\n",
       "      <td>6.136364</td>\n",
       "      <td>775.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18899</th>\n",
       "      <td>🚨.Serious declining in European journalism, poor journalism,ignorance. Knowledge poverty. 🤔🤔. #WorldCup #WorldCup2022</td>\n",
       "      <td>117</td>\n",
       "      <td>0.581197</td>\n",
       "      <td>8.833333</td>\n",
       "      <td>775.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1128</th>\n",
       "      <td>FIFA PRESIDENT:\\n\\nIn a monologue at the start of a news conference in Doha, Infantino accused Western critics of Qatar’s human rights record of hypocrisy. ^MA\\n#FIFAWorldCup\\n#WorldCup2022 #LOOKUPTV https://t.co/uqsMoBkPrh</td>\n",
       "      <td>219</td>\n",
       "      <td>0.566210</td>\n",
       "      <td>6.300000</td>\n",
       "      <td>775.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4250</th>\n",
       "      <td>Ecuador are slamming Qatar. Impressive performance from the players coached by Gustavo Alfaro. \\n\\n#WorldCup2022 #Qatar2022 https://t.co/vFTGJDSX33</td>\n",
       "      <td>145</td>\n",
       "      <td>0.551724</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>776.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6268</th>\n",
       "      <td>#WorldCup2022 \\nEasy decision \\nEasy pen\\nEasy goal for Ecuador</td>\n",
       "      <td>60</td>\n",
       "      <td>0.616667</td>\n",
       "      <td>5.555556</td>\n",
       "      <td>777.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7724</th>\n",
       "      <td>Pahahaha whose in charge of VAR? Sepp Blatter? #WorldCup2022</td>\n",
       "      <td>60</td>\n",
       "      <td>0.616667</td>\n",
       "      <td>5.777778</td>\n",
       "      <td>777.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11422</th>\n",
       "      <td>Moises caicedo of Ecuador is special. Plays fr brighton but should get a big move next transfer window. Perfect for Liverpool. #QatarWorldCup2022 #WorldCup2022 #FIFAWorldCup2022 #Qatar2022</td>\n",
       "      <td>188</td>\n",
       "      <td>0.547872</td>\n",
       "      <td>6.560000</td>\n",
       "      <td>777.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14843</th>\n",
       "      <td>In Pictures| The #WorldCup2022 opening ceremony at Al-Bayt Stadium in Al-Khor, Qatar 🏆 https://t.co/tSHpxAdbVd</td>\n",
       "      <td>110</td>\n",
       "      <td>0.509091</td>\n",
       "      <td>6.928571</td>\n",
       "      <td>778.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>785</th>\n",
       "      <td>🚨 Man City’s John Stones is set to partner Harry Maguire for England vs Iran. (via @FrankieChristou)\\n\\n👀 Bukayo Saka is set to start for England vs Iran ahead of Phil Foden. [Via @TelegraphSport]\\n\\n#MCFC #England #WorldCup2022</td>\n",
       "      <td>224</td>\n",
       "      <td>0.508929</td>\n",
       "      <td>5.027027</td>\n",
       "      <td>778.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21253</th>\n",
       "      <td>I hope Portugal or Argentina will win the world cup #WorldCup2022 #QatarWorldCup2022</td>\n",
       "      <td>84</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>6.083333</td>\n",
       "      <td>778.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18026</th>\n",
       "      <td>'We are all Palestinian': #WorldCup2022 fans show Palestinian pride in Doha ahead of tournament kickoff https://t.co/af23diCpOh</td>\n",
       "      <td>127</td>\n",
       "      <td>0.559055</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>778.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                                                                                                                               text  \\\n",
       "4495                                                                                                                                                                                                                                                       Valencia 2 goals already. Ecuador lead 0-2 #WorldCup2022   \n",
       "5849                                                                                                                                                                                 Enner Valencia took 9 minutes to resettle, then told VAR to choke on its match-fixiness. #WorldCup2022 https://t.co/m1FYpEvlzL   \n",
       "3356                                                                                                                                                                                                                              Qatar seff go collect woto woto for this competition 😂😂\\n#WorldCup2022 #Qatar2022   \n",
       "19771                                                                                                                                                                                                  Predict who will be the quarter finalists and win free FIFA tokens.\\n\\n#FIFAWorldCup #WorldCup #WorldCup2022   \n",
       "9663                                                                                                                                                                                                                                                 Course that Ecuador goal was offside 😭🤣🤑 #QATECU #WorldCup2022   \n",
       "1648                                                                                                                                      Fans chant for Palestine and raise Palestinian flags during a live broadcast of an Israeli channel from Qatar. #WorldCup2022. #RaisePalestineFlag https://t.co/8Z955J85Ty   \n",
       "12827                                                                                                                                                                                                                          Clint Dempsey calling out Alexi Lawless on live TV. He been a hater 🤣🤣 #WorldCup2022   \n",
       "10122                                                                                                                                                                                                            Ecuador scores the first world cup goal in less than 3 minutes. 🤯 #WorldCup2022 #WorldcupQatar2022   \n",
       "4213                                                                                                                                                                                                                             Mina I even forgot kune #WorldCup2022 yaz.. 😴 Anyway @netflix ingiphethe sharp.. 😉   \n",
       "868                                                                                               COLOMBIAN SINGER MALUMA:\\n\\nThe star features alongside Nicki Minaj and Myriam Fares on Tukoh Taka, the official Fifa Qatar World Cup anthem. ^MA\\n#FIFAWorldCup\\n#WorldCup2022 #LOOKUPTV https://t.co/J9yQg2aSjH   \n",
       "11584                                                                                                                                       First time watched an opening ceremony without sexual content and half naked women roaming around. #Qatar 👏🏼\\n#FIFAWorldCup #QatarWorldCup2022 #WorldCup2022 #Qatar2022   \n",
       "19146                                                                                                               #WorldCup2022 \\nWinner: Argentina\\nDark horses: USA/Denmark\\nFlops:France\\nTop goalscorers:Julian Alvarez/memphis depay.\\nPlayer of tournament:Messi\\nYoung player of tournament:Enzo fernandez   \n",
       "10518                                                                                                                                                                                                                                                 3 mins into the game. ecuador scoreee goallllll #WorldCup2022   \n",
       "18910                    Conservatively 1500 plus migrant workers died making a Qatari World Cup a reality how many died in 2 years of construction of spurs super stadium ? FIFA & Qatar heading for PR hell 😎👍🏼 #WorldCup #WorldCup2022 #football #QatarWorldCup2022 #Qatar #FIFAWorldCup https://t.co/TLkS8AOF8F   \n",
       "22478                                                                                                                                                                 My #WorldCup2022 predictons.\\nWinner: Argentina\\nTop scorer: Lukaku\\nOne of Belgium or Denmark to make semis\\nSurprise packets: Saudi Arabia.   \n",
       "22040                                                                                                                                                             FIFA has named Wessam Qutob, a Palestinian refugee and social media influencer, as the #WorldCup2022 ambassador in Qatar. https://t.co/p5tFywuczj   \n",
       "3337                                               A player has scored a brace in the World Cup opener for the third consecutive World Cup\\n\\n2014 - Neymar Jr. vs Croatia\\n2018 - Denis Cheryshev vs Saudi Arabia\\n2022 - Enner Valencia vs Qatar\\n\\n#WorldCup #FIFAWorldCup #WorldCup2022 https://t.co/Mkv8EYaJRr   \n",
       "17499                                                                                                                                                             FIFA has named Wessam Qutob, a Palestinian refugee and social media influencer, as the #WorldCup2022 ambassador in Qatar. https://t.co/v2agy3Nxbj   \n",
       "14053                                                                                                                                                                         The most depressing start to a World Cup I have ever experienced in my life, understandably. #FIFAWorldCup #WorldCup2022 #BBCWorldCup   \n",
       "960                                                                                                                  The #Qatar2022 #WorldCup2022 has opened with a ceremony at the Al-Bayt Stadium in Al Khor, north of Doha\\n\\n📷 Kirill KUDRYAVTSEV /Raul ARBOLEDA /MANAN VATSYAYANA /AFP https://t.co/98yXAgcnfT   \n",
       "20350                                                                                             COLOMBIAN SINGER MALUMA:\\n\\nThe star features alongside Nicki Minaj and Myriam Fares on Tukoh Taka, the official FIFA Qatar World Cup anthem. ^MA\\n#FIFAWorldCup\\n#WorldCup2022 #LOOKUPTV https://t.co/HHfeC25KVO   \n",
       "8953                                                                                                                                                                                                        Offside??? VAR???? Absolute Bollox. Ecuador were a mile onside at every movement #WorldCup2022 #corrupt   \n",
       "7117                                                                                                                                                                                                         #ssfootball #WorldCup2022 #Qatar2022\\nIn South Africa that's a clear goal, Ecuador will apset the host   \n",
       "1857                                                                                                                        Football fans chant for Palestine and raise Palestinian flags during a live broadcast by an Israeli TV in Qatar in #WorldCup2022. \\n \\n#RaisePalestineFlags\\n\\n https://t.co/5c5fQ7tJaI   \n",
       "2379                                                                                                                                                                            Ladies and Gentlemen it's official 🤣🤣\\n\\nLadies, Unapeana remote control ukiwa pande gani?\\n\\n#WorldCup2022 https://t.co/nHFOeMCjph   \n",
       "18331                                                                                                                                                                                                                   Let's focus on #WorldCup2022 pulak. Prediction Brazil or Argentina akan menang this year 😅😅   \n",
       "6293                                                                                                                                                                             1-0 Ecuador from a pen. Awaits @FIFAcom denying goal and awarding a penalty to Qatar due to Ecuador players smiling! #WorldCup2022   \n",
       "18735                                                                                                                                                             FIFA has named Wessam Qutob, a Palestinian refugee and social media influencer, as the #WorldCup2022 ambassador in Qatar. https://t.co/2FfNGxZUaX   \n",
       "19135                                                                                           @FIFAWorldCup #WorldCup2022 Countdown: 5 hours\\n\\nPowered by #Algorand The Official Blockchain Platform of FIFA. Sports love $ALGO , look it up! ( hint: sail GP, formula E, drone racing ) https://t.co/0DFH2bhFkf   \n",
       "11803                                                                                                                                                                                                              #WorldCup2022 \\nQatar vs Ecuador\\n\\nMy prediction: 2-0 Ecuador\\nMan of the match: Enner Valencia   \n",
       "9607                                                                                                                                                                                                                                               Hahahaha dis #WorldCup2022 go be something else eiii alo offside   \n",
       "16472                                                                                                                                      #WorldCup2022 predictions \\n\\nWinner: Brazil \\nGolden Boot: Harry Kane \\nGolden Ball: Messi \\nYoung Player: Musiala \\nDark Horses: Uruguay \\nDisappointing Team: England   \n",
       "144                                                                                                                                                                                Stop American emigrant from going any damn where them stoclem took over rogermi kimthro Hashiq bands are ludicrous #WorldCup2022   \n",
       "2880                                                                                                                                       Palestinian Authority President Mahmoud Abbas has attended the opening ceremony of the #WorldCup2022 at Al-Bayt Stadium in Qatar's capital Doha. https://t.co/zRBB0mtl4a   \n",
       "13867                                                                                                                                                                Sorry Mr Jungkook sir but gi-friggin'-normous La'eeb stole the show. I expect to see him in every match. #WorldCup2022 https://t.co/uD2je8sorM   \n",
       "2677                         Alternate caption:\\n\\nBrasil, thinking ahead to their knockout round match vs Canada, attempt to thwart Canadian player's health regiment by sending over gifts from a Brasilian company... \" ;)\\n\\n#CanMNT #Qatar2022 #WorldCup2022 #FIFAWorldCup #TimHortons https://t.co/t05DjejWG6   \n",
       "2953                                                                                                                                       Palestinian Authority President Mahmoud Abbas has attended the opening ceremony of the #WorldCup2022 at Al-Bayt Stadium in Qatar's capital Doha. https://t.co/ZoV8GcQafB   \n",
       "21017                                                                                                                                    #WorldCup2022 #WorldCup Jungkook 'Dreamers' is just ok, not grandstanding, not boring. Ricky Martin (1996) & Shakira (2010) are still the standards for World Cup songs...   \n",
       "12103  Did Anne Lister saw any soccer ball before? \\n\\nIn 1836 Charles Goodyear patented vulcanized rubber. But, only in 1855, Charles Goodyear designed and built the first vulcanized rubber soccer balls.\\n\\n#WorldCup2022 #SaveGentlemanJack #GJWorldCup @LookoutPointTV @AmazonStudios https://t.co/6Bkz4LDuOI   \n",
       "17431                                                                                  LEAKED ! A mole in the camp has leaked tomorrow's #Englandfootball #WorldCup2022 team. Some will be disappointed !\\nPickford\\nTrippier, Maguire, Stones, Shaw\\nRice, Bellingham, Mount\\nSaka , Kane, Sterling .\\n#ThreeLions   \n",
       "19784                                          Isolation, genocide, murder, all part of the daily life of Palestinians under Israeli occupation.. Terrorist Israel that is committing barbaric crimes in Palestine is criticizing Qatar! \\n+\\n#QatarWorldCup2022\\n#WorldCup2022\\n#FreePalestine\\n#FreeAhmadManasrah   \n",
       "4017                                                                                                                                                                                                                                     This looks fake as fuck 🤣🤣🤣 shite team , shite fans @qatar \\n#WorldCup2022   \n",
       "3560                                                                                                                                                                                                                                         That wire transfer at halftime Finna go crazy #Qatar2022 #WorldCup2022   \n",
       "17703                                                                                                                                                                                                                 Brazil\\nArgentina\\nLook strong to win the World Cup acc to me.\\n#WorldCup2022 \\n#FIFAWorldCup   \n",
       "2908                                                                                                                                                                                                                        Ecuador's first goal vs Qatar was disallowed for suspected homosexuality. #WorldCup2022   \n",
       "21975                                                                                                                                                             FIFA has named Wessam Qutob, a Palestinian refugee and social media influencer, as the #WorldCup2022 ambassador in Qatar. https://t.co/LPLaoApnxS   \n",
       "7664                                                                                                                                                                                                                                    Sheikh s robbed Ecuador with var #WorldCup2022 #QATECU that was not offside   \n",
       "20460                                                                               FIFA PRESIDENT:\\n\\nIn a monologue at the start of a news conference in Doha, Infantino accused Western critics of Qatar’s human rights record of hypocrisy. ^MA\\n#FIFAWorldCup\\n#WorldCup2022 #LOOKUPTV https://t.co/cqHfPdAkld   \n",
       "5431                                                                                                                                                                                                                           According to VAR, Ecuador’s penalty shooter was OFFSIDE!\\n#WorldCup2022 \\n#Qatar2022   \n",
       "17673                                                                                                                                               FIFA president Gianni Infantino has his say on Europe’s criticism of the FIFA World Cup 2022 in Qatar 👀❌\\n\\n#FIFAWorldCup #WorldCup2022 https://t.co/vl0fkdX99X   \n",
       "18899                                                                                                                                                                                         🚨.Serious declining in European journalism, poor journalism,ignorance. Knowledge poverty. 🤔🤔. #WorldCup #WorldCup2022   \n",
       "1128                                                                                FIFA PRESIDENT:\\n\\nIn a monologue at the start of a news conference in Doha, Infantino accused Western critics of Qatar’s human rights record of hypocrisy. ^MA\\n#FIFAWorldCup\\n#WorldCup2022 #LOOKUPTV https://t.co/uqsMoBkPrh   \n",
       "4250                                                                                                                                                            Ecuador are slamming Qatar. Impressive performance from the players coached by Gustavo Alfaro. \\n\\n#WorldCup2022 #Qatar2022 https://t.co/vFTGJDSX33   \n",
       "6268                                                                                                                                                                                                                                                #WorldCup2022 \\nEasy decision \\nEasy pen\\nEasy goal for Ecuador   \n",
       "7724                                                                                                                                                                                                                                                   Pahahaha whose in charge of VAR? Sepp Blatter? #WorldCup2022   \n",
       "11422                                                                                                                  Moises caicedo of Ecuador is special. Plays fr brighton but should get a big move next transfer window. Perfect for Liverpool. #QatarWorldCup2022 #WorldCup2022 #FIFAWorldCup2022 #Qatar2022   \n",
       "14843                                                                                                                                                                                                In Pictures| The #WorldCup2022 opening ceremony at Al-Bayt Stadium in Al-Khor, Qatar 🏆 https://t.co/tSHpxAdbVd   \n",
       "785                                                                            🚨 Man City’s John Stones is set to partner Harry Maguire for England vs Iran. (via @FrankieChristou)\\n\\n👀 Bukayo Saka is set to start for England vs Iran ahead of Phil Foden. [Via @TelegraphSport]\\n\\n#MCFC #England #WorldCup2022   \n",
       "21253                                                                                                                                                                                                                          I hope Portugal or Argentina will win the world cup #WorldCup2022 #QatarWorldCup2022   \n",
       "18026                                                                                                                                                                               'We are all Palestinian': #WorldCup2022 fans show Palestinian pride in Doha ahead of tournament kickoff https://t.co/af23diCpOh   \n",
       "\n",
       "       length  word fraction  token length  english score  \n",
       "4495       56       0.553571      6.125000          750.0  \n",
       "5849      126       0.555556      6.470588          750.0  \n",
       "3356       80       0.550000      5.750000          752.0  \n",
       "19771     106       0.518868      6.066667          752.0  \n",
       "9663       62       0.500000      6.000000          753.0  \n",
       "1648      169       0.544379      7.095238          754.0  \n",
       "12827      84       0.642857      4.666667          754.0  \n",
       "10122      98       0.510204      5.600000          755.0  \n",
       "4213       82       0.524390      5.384615          756.0  \n",
       "868       205       0.556098      6.592593          757.0  \n",
       "11584     166       0.500000      6.952381          757.0  \n",
       "19146     185       0.778378      8.736842          757.0  \n",
       "10518      61       0.622951      5.888889          758.0  \n",
       "18910     282       0.500000      5.902439          759.0  \n",
       "22478     137       0.729927      5.900000          760.0  \n",
       "22040     145       0.606897      6.684211          760.0  \n",
       "3337      250       0.540000      5.225000          760.0  \n",
       "17499     145       0.606897      6.684211          760.0  \n",
       "14053     133       0.563910      6.052632          761.0  \n",
       "960       188       0.515957      5.962963          761.0  \n",
       "20350     205       0.556098      6.592593          761.0  \n",
       "8953      103       0.592233      6.428571          762.0  \n",
       "7117      101       0.504950      5.800000          762.0  \n",
       "1857      179       0.530726      6.608696          763.0  \n",
       "2379      127       0.551181      7.400000          763.0  \n",
       "18331      91       0.670330      5.571429          764.0  \n",
       "6293      130       0.646154      5.238095          765.0  \n",
       "18735     145       0.606897      6.684211          766.0  \n",
       "19135     209       0.502392      5.741935          768.0  \n",
       "11803      92       0.641304      5.500000          768.0  \n",
       "9607       64       0.656250      5.500000          768.0  \n",
       "16472     161       0.720497      6.750000          768.0  \n",
       "144       128       0.757812      5.789474          768.0  \n",
       "2880      168       0.648810      7.047619          769.0  \n",
       "13867     142       0.570423      6.150000          769.0  \n",
       "2677      274       0.551095      6.800000          769.0  \n",
       "2953      168       0.648810      7.047619          769.0  \n",
       "21017     170       0.558824      5.840000          769.0  \n",
       "12103     296       0.510135      7.166667          770.0  \n",
       "17431     215       0.613953      5.750000          770.0  \n",
       "19784     255       0.603922      7.225806          771.0  \n",
       "4017       73       0.506849      4.615385          772.0  \n",
       "3560       70       0.542857      6.100000          772.0  \n",
       "17703      89       0.539326      5.357143          774.0  \n",
       "2908       87       0.712644      7.000000          774.0  \n",
       "21975     145       0.606897      6.684211          775.0  \n",
       "7664       75       0.586667      5.333333          775.0  \n",
       "20460     219       0.566210      6.300000          775.0  \n",
       "5431       82       0.560976      7.200000          775.0  \n",
       "17673     157       0.503185      6.136364          775.0  \n",
       "18899     117       0.581197      8.833333          775.0  \n",
       "1128      219       0.566210      6.300000          775.0  \n",
       "4250      145       0.551724      8.000000          776.0  \n",
       "6268       60       0.616667      5.555556          777.0  \n",
       "7724       60       0.616667      5.777778          777.0  \n",
       "11422     188       0.547872      6.560000          777.0  \n",
       "14843     110       0.509091      6.928571          778.0  \n",
       "785       224       0.508929      5.027027          778.0  \n",
       "21253      84       0.500000      6.083333          778.0  \n",
       "18026     127       0.559055      7.000000          778.0  "
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = trim_dataset_by(\n",
    "    df, \n",
    "    ascending=True,\n",
    "    \n",
    "    # sort_by='length',\n",
    "    length=(55, None),\n",
    "    \n",
    "    # sort_by='word fraction',\n",
    "    word_fraction=(0.5, None),\n",
    "    \n",
    "    # sort_by='token length',\n",
    "    token_length=(4, 9),\n",
    "    \n",
    "    sort_by='english score',\n",
    "    english_score=(750, None),\n",
    ")\n",
    "a[:60]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "id": "84f92775-2724-4594-a761-42ebba0e0759",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:39:04.568040Z",
     "iopub.status.busy": "2024-01-08T15:39:04.567845Z",
     "iopub.status.idle": "2024-01-08T15:39:04.584901Z",
     "shell.execute_reply": "2024-01-08T15:39:04.584197Z",
     "shell.execute_reply.started": "2024-01-08T15:39:04.568020Z"
    }
   },
   "outputs": [],
   "source": [
    "df = a"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c402379b-be86-4caa-9ce0-b5d542cb0445",
   "metadata": {},
   "source": [
    "Randomly remove hashtag #WorldCup2022 to complicate topic discovery"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "id": "39dc68e7-34d7-4460-996f-dc2b709dcd27",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:39:04.586120Z",
     "iopub.status.busy": "2024-01-08T15:39:04.585922Z",
     "iopub.status.idle": "2024-01-08T15:39:04.619260Z",
     "shell.execute_reply": "2024-01-08T15:39:04.618559Z",
     "shell.execute_reply.started": "2024-01-08T15:39:04.586099Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\"#WorldCup2022\" new frequency: 0.29\n"
     ]
    }
   ],
   "source": [
    "df = randomly_remove_substring(df, '#WorldCup2022', p=0.7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "id": "3e199f92-60c1-441b-b840-d23690e76800",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:39:04.625871Z",
     "iopub.status.busy": "2024-01-08T15:39:04.625243Z",
     "iopub.status.idle": "2024-01-08T15:39:10.427006Z",
     "shell.execute_reply": "2024-01-08T15:39:10.426126Z",
     "shell.execute_reply.started": "2024-01-08T15:39:04.625846Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time elapsed: 6s\n",
      "samples: 6854, similar samples detected: 187 (2.66%)\n"
     ]
    }
   ],
   "source": [
    "df = drop_similar(supersample(a, FIFA_WORLD_CUP));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "id": "91da92c2-3bbb-432b-acba-b03d557af54f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:39:10.428269Z",
     "iopub.status.busy": "2024-01-08T15:39:10.428022Z",
     "iopub.status.idle": "2024-01-08T15:39:10.500145Z",
     "shell.execute_reply": "2024-01-08T15:39:10.499511Z",
     "shell.execute_reply.started": "2024-01-08T15:39:10.428245Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6854, 3)"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = select_columns(df.copy(), ['text'])\n",
    "df['topic'] = 'fifa world cup'\n",
    "df['id'] = 0\n",
    "\n",
    "df = sample(df, FIFA_WORLD_CUP)\n",
    "dataset = concat(dataset, df)\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "43bf4990-a554-45bf-b1ac-b5461767802d",
   "metadata": {},
   "source": [
    "# Australian elections"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "id": "9f1134f2-d349-4036-900b-893e81ee8678",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:39:10.501171Z",
     "iopub.status.busy": "2024-01-08T15:39:10.500950Z",
     "iopub.status.idle": "2024-01-08T15:39:11.629270Z",
     "shell.execute_reply": "2024-01-08T15:39:11.628707Z",
     "shell.execute_reply.started": "2024-01-08T15:39:10.501151Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(183379, 2)"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = load_dataset('australian_elections.csv', source=True)\n",
    "df.rename(columns={'full_text': 'text', 'favorite_count': 'likes'}, inplace=True)\n",
    "df = select_columns(df, ['text', 'id'])\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "id": "a646aba6-b6d6-43ec-a096-c795fe56d132",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:39:11.630714Z",
     "iopub.status.busy": "2024-01-08T15:39:11.629985Z",
     "iopub.status.idle": "2024-01-08T15:39:18.890261Z",
     "shell.execute_reply": "2024-01-08T15:39:18.889726Z",
     "shell.execute_reply.started": "2024-01-08T15:39:11.630690Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(165029, 2)"
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = df.copy()\n",
    "\n",
    "a['text'] = a['text'].apply(replace_bad_chars)\n",
    "a.drop_duplicates(subset='text', inplace=True)\n",
    "a = a[a['text'].apply(consists_of_allowed_chars)]\n",
    "\n",
    "a.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "id": "153d2d20-be55-4b01-8dc1-26bf71eeb6c2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:39:18.891651Z",
     "iopub.status.busy": "2024-01-08T15:39:18.890951Z",
     "iopub.status.idle": "2024-01-08T15:39:18.914206Z",
     "shell.execute_reply": "2024-01-08T15:39:18.913594Z",
     "shell.execute_reply.started": "2024-01-08T15:39:18.891627Z"
    }
   },
   "outputs": [],
   "source": [
    "df = a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "id": "2886c524-2ceb-4186-91e8-b479d00a1d8d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:39:18.915518Z",
     "iopub.status.busy": "2024-01-08T15:39:18.914973Z",
     "iopub.status.idle": "2024-01-08T15:39:22.792855Z",
     "shell.execute_reply": "2024-01-08T15:39:22.792316Z",
     "shell.execute_reply.started": "2024-01-08T15:39:18.915493Z"
    }
   },
   "outputs": [],
   "source": [
    "add_stats(df);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "id": "90ca5095-4a36-4502-9ffe-459afba3e312",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:39:22.794156Z",
     "iopub.status.busy": "2024-01-08T15:39:22.793565Z",
     "iopub.status.idle": "2024-01-08T15:39:22.864971Z",
     "shell.execute_reply": "2024-01-08T15:39:22.864433Z",
     "shell.execute_reply.started": "2024-01-08T15:39:22.794132Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "samples: 21289, reduction: 87.10%\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>length</th>\n",
       "      <th>word fraction</th>\n",
       "      <th>token length</th>\n",
       "      <th>english score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>118115</th>\n",
       "      <td>1128160964199608320</td>\n",
       "      <td>Liberal candidates thinks only heterosexual couples should be able to adopt, that carbon emissions aren't pollution and that marriage stops domestic violence.\\n\\nThese deranged candidate are here to do one job. Kill the working class.\\n#auspol #ausvotes\\nhttps://t.co/i49LmqVcqp</td>\n",
       "      <td>274</td>\n",
       "      <td>0.700730</td>\n",
       "      <td>6.210526</td>\n",
       "      <td>1000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45521</th>\n",
       "      <td>1129923506010038272</td>\n",
       "      <td>Morrison is on record as saying labour laws need reform so teenagers can get workplace experience. He's also pro-coal. We'll see kids working in the mines, just like Dickensian England, within 12 months. Mark my words. #auspol</td>\n",
       "      <td>226</td>\n",
       "      <td>0.761062</td>\n",
       "      <td>5.135135</td>\n",
       "      <td>1000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>163798</th>\n",
       "      <td>1129679835993788417</td>\n",
       "      <td>Our student panelists are discussing are worrying trend of candidates making Islamophobic and anti-immigration comments #ausvotes</td>\n",
       "      <td>129</td>\n",
       "      <td>0.806202</td>\n",
       "      <td>7.125000</td>\n",
       "      <td>1000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90300</th>\n",
       "      <td>1129148682849251330</td>\n",
       "      <td>It's not an efficiency dividend, it's a maladministration and corruption dividend #auspol</td>\n",
       "      <td>89</td>\n",
       "      <td>0.764045</td>\n",
       "      <td>6.500000</td>\n",
       "      <td>1000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50832</th>\n",
       "      <td>1129802158369517568</td>\n",
       "      <td>Let's try to find a few positives in this terribly disappointing result.\\nAbbott - gone.\\nAnning - no seat.\\nGovt - precarious majority.\\nA few of those QLD dudes don't look very physically healthy. \\nMorrison will have to own the tanking economy.\\nMight not last 3 years.\\n#auspol</td>\n",
       "      <td>274</td>\n",
       "      <td>0.751825</td>\n",
       "      <td>4.708333</td>\n",
       "      <td>1000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94457</th>\n",
       "      <td>1128966209729253376</td>\n",
       "      <td>RIP Bob Hawke. Using his last days to promote his vision for a better Australia. And heal old rifts. And generally be a good man. #auspol</td>\n",
       "      <td>137</td>\n",
       "      <td>0.737226</td>\n",
       "      <td>4.307692</td>\n",
       "      <td>1000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>153896</th>\n",
       "      <td>1129741496985509892</td>\n",
       "      <td>i can’t believe the twitter emoji for the australian federal election is a fucking sausage on bread this is so humiliating #ausvotes</td>\n",
       "      <td>132</td>\n",
       "      <td>0.765152</td>\n",
       "      <td>5.045455</td>\n",
       "      <td>1000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31976</th>\n",
       "      <td>1130382740468244480</td>\n",
       "      <td>Ten years ago, religion barely featured in daily public life/ discussion. In a country where over 60% of us claim a religious affiliation, there was a general acceptance that religion was a person’s private matter.\\n\\nHow times have changed. #auspol https://t.co/FactA8Tsaj</td>\n",
       "      <td>271</td>\n",
       "      <td>0.701107</td>\n",
       "      <td>5.609756</td>\n",
       "      <td>1000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>166204</th>\n",
       "      <td>1129646190813896706</td>\n",
       "      <td>Was going for a run in Kooyong before, Greens corflutes plastered over trees in a bush land (Kew Boulevard) with climate change &amp; looking after the environment. Some irony #AUSVotes</td>\n",
       "      <td>181</td>\n",
       "      <td>0.745856</td>\n",
       "      <td>5.066667</td>\n",
       "      <td>1000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2680</th>\n",
       "      <td>1130148524606124032</td>\n",
       "      <td>@clairegcoleman Why do you blame Australia when clearly nearly 50% didn't vote for those things. Morrison(LNP), Palmer &amp; Poorlean were instrumental in making it happen preferentially.</td>\n",
       "      <td>183</td>\n",
       "      <td>0.726776</td>\n",
       "      <td>6.076923</td>\n",
       "      <td>1000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>182426</th>\n",
       "      <td>1129680402342277125</td>\n",
       "      <td>If Tony loses his seat does this mean everyone will stop breathlessly asking him his opinion on every damned thing? #australiavotes</td>\n",
       "      <td>131</td>\n",
       "      <td>0.725191</td>\n",
       "      <td>5.285714</td>\n",
       "      <td>1000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54914</th>\n",
       "      <td>1129738528508776449</td>\n",
       "      <td>I, a progressive, said a few days ago that Bill Shorten was not a strong candidate - and was pilloried by Labor supporters for it. There was a sense of surety of the results, the approach, the campaign. I just didn't see it. So not overly surprised #auspol #ausvotes</td>\n",
       "      <td>266</td>\n",
       "      <td>0.725564</td>\n",
       "      <td>4.448980</td>\n",
       "      <td>1000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94674</th>\n",
       "      <td>1128964271805751296</td>\n",
       "      <td>Bob Hawke will rightfully be remembered for modernizing and reforming the Australian economy, and kicking off nearly 30 years of uninterrupted economic growth. \\n\\nHe should also be remembered for calling out John Howard's opportunistic racism in 1988. #auspol https://t.co/MOHMUMT0rB</td>\n",
       "      <td>282</td>\n",
       "      <td>0.716312</td>\n",
       "      <td>6.205128</td>\n",
       "      <td>1000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>176861</th>\n",
       "      <td>1128259716272967680</td>\n",
       "      <td>My favourite Clive Palmer ad is the one in which he accuses Bill Shorten of taxing Australians eleventy mcbillion dollars, all to be collected by a secret army of communist elves. Politics you can trust, man. #ausvotes</td>\n",
       "      <td>218</td>\n",
       "      <td>0.775229</td>\n",
       "      <td>4.918919</td>\n",
       "      <td>1000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63425</th>\n",
       "      <td>1129701147436912640</td>\n",
       "      <td>Gotta think Labor would've taken this comfortably with Penny or Albo at the helm. #AUSpol</td>\n",
       "      <td>89</td>\n",
       "      <td>0.741573</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>1000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72720</th>\n",
       "      <td>1129610793719222274</td>\n",
       "      <td>When you vote Liberal remember this, we live in a liberal democracy that is always in tension, for liberalism always prioritises the individual, whereas the democracy places constraints on the individual #auspol #ausvotes</td>\n",
       "      <td>221</td>\n",
       "      <td>0.769231</td>\n",
       "      <td>5.727273</td>\n",
       "      <td>1000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13565</th>\n",
       "      <td>1129724225575706624</td>\n",
       "      <td>@MegJobson1 Completely shattered - I feel like democracy in Australia is broken. An election won on lies, fear mongering and courruption.</td>\n",
       "      <td>137</td>\n",
       "      <td>0.744526</td>\n",
       "      <td>5.571429</td>\n",
       "      <td>1000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80560</th>\n",
       "      <td>1129480662782578688</td>\n",
       "      <td>Bob Hawke is not the same PM as Shorten will be. Shorten will be a dangerous socialist dictator, implementing his radical policies without transparency. He will not govern for the people, don't risk Labor #auspol #AusVotes2019</td>\n",
       "      <td>226</td>\n",
       "      <td>0.734513</td>\n",
       "      <td>5.305556</td>\n",
       "      <td>1000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>163657</th>\n",
       "      <td>1129680856207904768</td>\n",
       "      <td>Penny Wong says looking almost impossible for Tony Abbott to win in Warringah. #ausvotes</td>\n",
       "      <td>88</td>\n",
       "      <td>0.738636</td>\n",
       "      <td>5.357143</td>\n",
       "      <td>1000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132725</th>\n",
       "      <td>1127512071296274433</td>\n",
       "      <td>The nutters are out tonight but the palmer ads on lego masters \\nin western vic are insane. China invading? Labor in cahouts? Wtf #auspol</td>\n",
       "      <td>136</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>4.666667</td>\n",
       "      <td>1000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>153646</th>\n",
       "      <td>1129743260132499458</td>\n",
       "      <td>This has been a pretty clean and decent campaign considering where it could have gone given the state of modern politics #ausvotes</td>\n",
       "      <td>130</td>\n",
       "      <td>0.769231</td>\n",
       "      <td>4.954545</td>\n",
       "      <td>1000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84165</th>\n",
       "      <td>1129297523166736385</td>\n",
       "      <td>Some rando on Twitter: *makes up Engadine Maccas story*\\n#auspol: Yep, that definitely happened\\n\\nKathy Sherriff: *twice goes to police with detailed rape complaint*\\n#auspol: I will not even contemplate that might have happened</td>\n",
       "      <td>225</td>\n",
       "      <td>0.702222</td>\n",
       "      <td>5.617647</td>\n",
       "      <td>1000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>163570</th>\n",
       "      <td>1129681544556081152</td>\n",
       "      <td>Thinking about being a farmer who suffers the full force of climate change but continues to for climate change deniers #AusVotes</td>\n",
       "      <td>128</td>\n",
       "      <td>0.773438</td>\n",
       "      <td>5.142857</td>\n",
       "      <td>1000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45295</th>\n",
       "      <td>1129927123962028032</td>\n",
       "      <td>How long until Morrison introduces compulsory creationism classes in all schools?\\n\\n#auspol</td>\n",
       "      <td>90</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>6.500000</td>\n",
       "      <td>1000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4169</th>\n",
       "      <td>1130029766893948928</td>\n",
       "      <td>@mani1972ias May be yes, see recent election result in Germany, USA, Australia, ...nationalism kept first , then economy second! People never think pride will not give food! Only inclusive economic development will bring equality</td>\n",
       "      <td>229</td>\n",
       "      <td>0.711790</td>\n",
       "      <td>5.764706</td>\n",
       "      <td>1000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149318</th>\n",
       "      <td>1130098436521390080</td>\n",
       "      <td>Steggall's victory in Warringah removes a blistering wound on the heel of the \\nCoalition's far-right for Morrison to govern #ausvotes</td>\n",
       "      <td>133</td>\n",
       "      <td>0.759398</td>\n",
       "      <td>5.650000</td>\n",
       "      <td>1000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46693</th>\n",
       "      <td>1129905332526825472</td>\n",
       "      <td>#Auspol's frustration with corporate policing morality has come to a head with #IsraelFolau. For the 1st time it can be tested in the public forum &amp; a court. Do private companies &amp; sponsors have a say over employee's private/online lives? Let's hope not or employment = slavery...</td>\n",
       "      <td>280</td>\n",
       "      <td>0.707143</td>\n",
       "      <td>4.978723</td>\n",
       "      <td>1000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43748</th>\n",
       "      <td>1129954511747919872</td>\n",
       "      <td>How many hours per year will Parliament run of the LNP does not get a majority? #auspol</td>\n",
       "      <td>87</td>\n",
       "      <td>0.724138</td>\n",
       "      <td>4.176471</td>\n",
       "      <td>1000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4189</th>\n",
       "      <td>1130028347986206720</td>\n",
       "      <td>Strong opportunities for Green parties to advance (esp in PR systems) to 10-15% of the vote, mobilising most supportive, to pressure governing parties. Have risen in Australia from 2-5% to 10% in a majoritarian system. But different challenges when central issue in choosing govt</td>\n",
       "      <td>279</td>\n",
       "      <td>0.767025</td>\n",
       "      <td>5.363636</td>\n",
       "      <td>1000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59292</th>\n",
       "      <td>1129716302694588416</td>\n",
       "      <td>Corangamite was already notionally LABOR after redistribution Tanya you desperately vile true Deceiver 👎😡 Get some uppers off your husband you sad sack #auspol #australiavotes2019</td>\n",
       "      <td>179</td>\n",
       "      <td>0.709497</td>\n",
       "      <td>6.200000</td>\n",
       "      <td>1000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>157752</th>\n",
       "      <td>1129711828366348288</td>\n",
       "      <td>Stresses me the fuck out whenever they go to Antony Green with breaking news #ausvotes</td>\n",
       "      <td>86</td>\n",
       "      <td>0.732558</td>\n",
       "      <td>4.800000</td>\n",
       "      <td>1000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26597</th>\n",
       "      <td>1128101175998304256</td>\n",
       "      <td>@BevanShields I find it funny that Labor have wheeled out Australia's failed PM &amp; Treasurer as some sort of example of how good Labor is.. Still waiting in Keating's \"J\" curve to appear.. The man's a narcissistic arrogant boofhead. His opinions mean nothing. Save Australia &amp; vote Labor last.</td>\n",
       "      <td>292</td>\n",
       "      <td>0.739726</td>\n",
       "      <td>4.979592</td>\n",
       "      <td>1001.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137467</th>\n",
       "      <td>1127381293426065409</td>\n",
       "      <td>Leading with the woman responsible for arresting a 76 year old pensioner tells you everything about the Coalition launch #Ausvotes #Auspol</td>\n",
       "      <td>138</td>\n",
       "      <td>0.724638</td>\n",
       "      <td>5.619048</td>\n",
       "      <td>1001.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96225</th>\n",
       "      <td>1128941135601197056</td>\n",
       "      <td>So as we approach the end of this election campaign, the Liberal strategy has been duplicitous lying, maniacal thuggery and a pure one man show. We've had no policies and no other Liberal candidates get a word in. It's the ScoMo show, and he's failed miserably #auspol #ausvotes</td>\n",
       "      <td>278</td>\n",
       "      <td>0.744604</td>\n",
       "      <td>4.812500</td>\n",
       "      <td>1001.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>152024</th>\n",
       "      <td>1129771715033260032</td>\n",
       "      <td>Once again the disconnect between the elitist media &amp; real hard-working Australians is evident. The Silent Majority wins once again. #ausvotes</td>\n",
       "      <td>142</td>\n",
       "      <td>0.767606</td>\n",
       "      <td>5.809524</td>\n",
       "      <td>1001.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7098</th>\n",
       "      <td>1129889008799100930</td>\n",
       "      <td>@catvalente I feel for Australia, dealing with two crappy election results in quick succession.</td>\n",
       "      <td>95</td>\n",
       "      <td>0.726316</td>\n",
       "      <td>5.857143</td>\n",
       "      <td>1001.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61173</th>\n",
       "      <td>1129710041722843136</td>\n",
       "      <td>Welp. Australia. I guess we can start being proud of how stupid we are in electing a government. *slow claps to economy depression* #auspol</td>\n",
       "      <td>139</td>\n",
       "      <td>0.719424</td>\n",
       "      <td>4.833333</td>\n",
       "      <td>1001.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65368</th>\n",
       "      <td>1129693961281343488</td>\n",
       "      <td>Please don’t let me down WA, I’d like us to actually do something about climate change #auspol</td>\n",
       "      <td>94</td>\n",
       "      <td>0.723404</td>\n",
       "      <td>4.588235</td>\n",
       "      <td>1001.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99303</th>\n",
       "      <td>1128875180644691969</td>\n",
       "      <td>By forcing more and more people on to casual/temporary work regimes, weakens the ability of Unions to organise and take action. Work choices by deceit and quite deliberate by this disingenuous government. #auspol #AusVotes2019 #auspol2019</td>\n",
       "      <td>238</td>\n",
       "      <td>0.710084</td>\n",
       "      <td>5.828571</td>\n",
       "      <td>1001.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91659</th>\n",
       "      <td>1129032257770074112</td>\n",
       "      <td>@ScottMorrisonMP Delusional\\n\\nJobs by forcing multi jobs and spouses to work due wage erosion and costs, popn. growth and NDIS\\n\\nOnly faux future surplus\\n\\nNow your debt\\n\\nNeglect 95% of families\\n\\nIndifferent to small business\\n\\nCut schools, health and infrastructure \\n\\nSecurity - Rawandans\\n\\n#auspol</td>\n",
       "      <td>294</td>\n",
       "      <td>0.721088</td>\n",
       "      <td>5.500000</td>\n",
       "      <td>1001.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15965</th>\n",
       "      <td>1129637831301713921</td>\n",
       "      <td>Clive Palmer has reportedly hired dozens of young models to don yellow shirts and hand out United Australia Party how-to-vote cards at polling booths across Cairns. https://t.co/eqDwf7OTX4</td>\n",
       "      <td>188</td>\n",
       "      <td>0.723404</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>1001.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45466</th>\n",
       "      <td>1129924400285868032</td>\n",
       "      <td>Scott Morrison has spoken with US President Donald Trump this morning and is likely to speak with UK Prime Minister Theresa May later today. French President Emmanuel Macron and NZ PM Jacinda Ardern have also offered their congratulations directly to Mr Morrison #auspol</td>\n",
       "      <td>270</td>\n",
       "      <td>0.814815</td>\n",
       "      <td>5.302326</td>\n",
       "      <td>1001.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164101</th>\n",
       "      <td>1129676699212210176</td>\n",
       "      <td>Not even kidding, I ADORE ALAN JONES' GREEN SUIT. Maybe even more iconic than Antony Green's starry tie. Want one. Who's the tailor? #ausvotes</td>\n",
       "      <td>142</td>\n",
       "      <td>0.718310</td>\n",
       "      <td>4.958333</td>\n",
       "      <td>1001.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29982</th>\n",
       "      <td>1126955388266532869</td>\n",
       "      <td>As Australia prepares for elections this month, Reuters reports on how the country is one of the most susceptible in terms of online information gathering by political campaigns. Australia's privacy laws do not cover political parties and candidates...\\nhttps://t.co/l4az2jyapm</td>\n",
       "      <td>276</td>\n",
       "      <td>0.757246</td>\n",
       "      <td>6.102564</td>\n",
       "      <td>1001.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>177628</th>\n",
       "      <td>1128103762441097216</td>\n",
       "      <td>Voting early is kinda depressing -- we need sausage sizzles at early voting centres. #ausvotes</td>\n",
       "      <td>94</td>\n",
       "      <td>0.723404</td>\n",
       "      <td>5.333333</td>\n",
       "      <td>1001.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>157302</th>\n",
       "      <td>1129714071761182722</td>\n",
       "      <td>We NEEDED serious action on climate change and a humane immigration policy. Instead we got climate deniers, xenophobia and a potato. #ausvotes</td>\n",
       "      <td>142</td>\n",
       "      <td>0.767606</td>\n",
       "      <td>5.500000</td>\n",
       "      <td>1001.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30711</th>\n",
       "      <td>1130414913929662465</td>\n",
       "      <td>I'll think positively about the #auspol result despite my disappointment. Hopefully climate predictions have been wrong or overstated. Hopefully the reef will recover. Hopefully our economy and social welfare will grow despite lack of progressive or positive policy. Here hopes.</td>\n",
       "      <td>278</td>\n",
       "      <td>0.812950</td>\n",
       "      <td>5.975000</td>\n",
       "      <td>1001.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106831</th>\n",
       "      <td>1128585791662317570</td>\n",
       "      <td>@169lunar Just a protection racket for environmental vandals, wage theft, corporate thieves, natural resource plunderers, off shore tax havens, and a foreign billionaire media mogul whose notoriety is political interference and suppression or ridicule of the truth #auspol #AusVotes2019</td>\n",
       "      <td>286</td>\n",
       "      <td>0.748252</td>\n",
       "      <td>6.358974</td>\n",
       "      <td>1001.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51732</th>\n",
       "      <td>1129763463801253888</td>\n",
       "      <td>Quiet Australians, oh fuck off. Let’s show what the quiet Australians are capable of #Auspol</td>\n",
       "      <td>92</td>\n",
       "      <td>0.739130</td>\n",
       "      <td>5.200000</td>\n",
       "      <td>1001.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29253</th>\n",
       "      <td>1127260329283543045</td>\n",
       "      <td>@thehill Wow did he miss the whole 2016 Steele/FBI/DNC/HRC/Australia/Ukraine/DOJ election interference that he’s hoping doesn’t come to light?</td>\n",
       "      <td>142</td>\n",
       "      <td>0.732394</td>\n",
       "      <td>6.944444</td>\n",
       "      <td>1001.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>153178</th>\n",
       "      <td>1129746909885804551</td>\n",
       "      <td>Typical old people coming out in big numbers ruining younger generations Futures. #ausvotes</td>\n",
       "      <td>91</td>\n",
       "      <td>0.758242</td>\n",
       "      <td>6.076923</td>\n",
       "      <td>1001.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56567</th>\n",
       "      <td>1129728474472689665</td>\n",
       "      <td>I had hope Australia would go against the overseas trend of supporting unpleasant, neo-con populists.\\nI believed Australia was better and would set an example on thoughtful politics.\\nI looked forward to necessary reform.\\nIt didn't happen.\\nI weep for my grandchildren. \\n#auspol</td>\n",
       "      <td>276</td>\n",
       "      <td>0.789855</td>\n",
       "      <td>5.418605</td>\n",
       "      <td>1001.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>147001</th>\n",
       "      <td>1129745488176132101</td>\n",
       "      <td>#AustraliaDecides Old Bill campaigned the usual way, pleading to hearts and minds. Big bad Scott went full metal technological war and exploited the preferencing process. Props to robo scomo for using arbitrage to win Australia. GG.</td>\n",
       "      <td>232</td>\n",
       "      <td>0.754310</td>\n",
       "      <td>5.472222</td>\n",
       "      <td>1001.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48155</th>\n",
       "      <td>1129888792121188354</td>\n",
       "      <td>Labor has to elect @Bowenchris . He is the only leadership option who can emulate the Hawke/Keating success formula of cleverly designed social programs AND productivity reforms. #AUSPOL</td>\n",
       "      <td>186</td>\n",
       "      <td>0.741935</td>\n",
       "      <td>5.678571</td>\n",
       "      <td>1001.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109426</th>\n",
       "      <td>1128514343455182848</td>\n",
       "      <td>Living in the ACT is a blessing come election time. Most Australians don't realise that, by federal mandate, roadside election advertising scales with state land area. For example, candidate corflutes, poster sized here, are 1/3 the size of an Olympic swimming pool in QLD #auspol</td>\n",
       "      <td>280</td>\n",
       "      <td>0.778571</td>\n",
       "      <td>5.244444</td>\n",
       "      <td>1001.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30506</th>\n",
       "      <td>1130422103923052546</td>\n",
       "      <td>Franking credits and imputation credits sounds like rocket science to the electorate. No wonder the fake retiree tax worked. #auspol #ausvoted</td>\n",
       "      <td>142</td>\n",
       "      <td>0.732394</td>\n",
       "      <td>5.809524</td>\n",
       "      <td>1001.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35268</th>\n",
       "      <td>1130287422690828288</td>\n",
       "      <td>Why are people calling Morrison messiah? Seems a little crazy to think a politician who is for cutting employment and stifling social justice would be that. Egos running rampant. #auspol</td>\n",
       "      <td>186</td>\n",
       "      <td>0.790323</td>\n",
       "      <td>5.233333</td>\n",
       "      <td>1001.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15023</th>\n",
       "      <td>1129683628114362369</td>\n",
       "      <td>@RyanMendi97 is a moron. Not on the electoral roll in Australia despite living his whole life here, but managed to vote in a fucken Spanish election the flog. Thank god dual-citizenship rules his dumb brain out of parliament. #ausvotes</td>\n",
       "      <td>235</td>\n",
       "      <td>0.727660</td>\n",
       "      <td>5.051282</td>\n",
       "      <td>1001.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79856</th>\n",
       "      <td>1129505163608113152</td>\n",
       "      <td>Why doesn't it surprise me that an angry old man who dislikes a conservative politician would stab someone volunteering for him #auspol</td>\n",
       "      <td>135</td>\n",
       "      <td>0.785185</td>\n",
       "      <td>5.181818</td>\n",
       "      <td>1001.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24808</th>\n",
       "      <td>1128486758398586880</td>\n",
       "      <td>@Raymartin55 In Australia an election next week \\nBull Shitten seems ready to peak\\nLabor’s Financial management is hell\\nTheir leader’s an accused rapist as well\\nHope voters deny them a seat.</td>\n",
       "      <td>189</td>\n",
       "      <td>0.756614</td>\n",
       "      <td>5.096774</td>\n",
       "      <td>1001.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         id  \\\n",
       "118115  1128160964199608320   \n",
       "45521   1129923506010038272   \n",
       "163798  1129679835993788417   \n",
       "90300   1129148682849251330   \n",
       "50832   1129802158369517568   \n",
       "94457   1128966209729253376   \n",
       "153896  1129741496985509892   \n",
       "31976   1130382740468244480   \n",
       "166204  1129646190813896706   \n",
       "2680    1130148524606124032   \n",
       "182426  1129680402342277125   \n",
       "54914   1129738528508776449   \n",
       "94674   1128964271805751296   \n",
       "176861  1128259716272967680   \n",
       "63425   1129701147436912640   \n",
       "72720   1129610793719222274   \n",
       "13565   1129724225575706624   \n",
       "80560   1129480662782578688   \n",
       "163657  1129680856207904768   \n",
       "132725  1127512071296274433   \n",
       "153646  1129743260132499458   \n",
       "84165   1129297523166736385   \n",
       "163570  1129681544556081152   \n",
       "45295   1129927123962028032   \n",
       "4169    1130029766893948928   \n",
       "149318  1130098436521390080   \n",
       "46693   1129905332526825472   \n",
       "43748   1129954511747919872   \n",
       "4189    1130028347986206720   \n",
       "59292   1129716302694588416   \n",
       "157752  1129711828366348288   \n",
       "26597   1128101175998304256   \n",
       "137467  1127381293426065409   \n",
       "96225   1128941135601197056   \n",
       "152024  1129771715033260032   \n",
       "7098    1129889008799100930   \n",
       "61173   1129710041722843136   \n",
       "65368   1129693961281343488   \n",
       "99303   1128875180644691969   \n",
       "91659   1129032257770074112   \n",
       "15965   1129637831301713921   \n",
       "45466   1129924400285868032   \n",
       "164101  1129676699212210176   \n",
       "29982   1126955388266532869   \n",
       "177628  1128103762441097216   \n",
       "157302  1129714071761182722   \n",
       "30711   1130414913929662465   \n",
       "106831  1128585791662317570   \n",
       "51732   1129763463801253888   \n",
       "29253   1127260329283543045   \n",
       "153178  1129746909885804551   \n",
       "56567   1129728474472689665   \n",
       "147001  1129745488176132101   \n",
       "48155   1129888792121188354   \n",
       "109426  1128514343455182848   \n",
       "30506   1130422103923052546   \n",
       "35268   1130287422690828288   \n",
       "15023   1129683628114362369   \n",
       "79856   1129505163608113152   \n",
       "24808   1128486758398586880   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                          text  \\\n",
       "118115                                  Liberal candidates thinks only heterosexual couples should be able to adopt, that carbon emissions aren't pollution and that marriage stops domestic violence.\\n\\nThese deranged candidate are here to do one job. Kill the working class.\\n#auspol #ausvotes\\nhttps://t.co/i49LmqVcqp   \n",
       "45521                                                                                       Morrison is on record as saying labour laws need reform so teenagers can get workplace experience. He's also pro-coal. We'll see kids working in the mines, just like Dickensian England, within 12 months. Mark my words. #auspol   \n",
       "163798                                                                                                                                                                                       Our student panelists are discussing are worrying trend of candidates making Islamophobic and anti-immigration comments #ausvotes   \n",
       "90300                                                                                                                                                                                                                                It's not an efficiency dividend, it's a maladministration and corruption dividend #auspol   \n",
       "50832                                Let's try to find a few positives in this terribly disappointing result.\\nAbbott - gone.\\nAnning - no seat.\\nGovt - precarious majority.\\nA few of those QLD dudes don't look very physically healthy. \\nMorrison will have to own the tanking economy.\\nMight not last 3 years.\\n#auspol   \n",
       "94457                                                                                                                                                                                RIP Bob Hawke. Using his last days to promote his vision for a better Australia. And heal old rifts. And generally be a good man. #auspol   \n",
       "153896                                                                                                                                                                                    i can’t believe the twitter emoji for the australian federal election is a fucking sausage on bread this is so humiliating #ausvotes   \n",
       "31976                                        Ten years ago, religion barely featured in daily public life/ discussion. In a country where over 60% of us claim a religious affiliation, there was a general acceptance that religion was a person’s private matter.\\n\\nHow times have changed. #auspol https://t.co/FactA8Tsaj   \n",
       "166204                                                                                                                                   Was going for a run in Kooyong before, Greens corflutes plastered over trees in a bush land (Kew Boulevard) with climate change & looking after the environment. Some irony #AUSVotes   \n",
       "2680                                                                                                                                   @clairegcoleman Why do you blame Australia when clearly nearly 50% didn't vote for those things. Morrison(LNP), Palmer & Poorlean were instrumental in making it happen preferentially.   \n",
       "182426                                                                                                                                                                                     If Tony loses his seat does this mean everyone will stop breathlessly asking him his opinion on every damned thing? #australiavotes   \n",
       "54914                                               I, a progressive, said a few days ago that Bill Shorten was not a strong candidate - and was pilloried by Labor supporters for it. There was a sense of surety of the results, the approach, the campaign. I just didn't see it. So not overly surprised #auspol #ausvotes   \n",
       "94674                             Bob Hawke will rightfully be remembered for modernizing and reforming the Australian economy, and kicking off nearly 30 years of uninterrupted economic growth. \\n\\nHe should also be remembered for calling out John Howard's opportunistic racism in 1988. #auspol https://t.co/MOHMUMT0rB   \n",
       "176861                                                                                              My favourite Clive Palmer ad is the one in which he accuses Bill Shorten of taxing Australians eleventy mcbillion dollars, all to be collected by a secret army of communist elves. Politics you can trust, man. #ausvotes   \n",
       "63425                                                                                                                                                                                                                                Gotta think Labor would've taken this comfortably with Penny or Albo at the helm. #AUSpol   \n",
       "72720                                                                                            When you vote Liberal remember this, we live in a liberal democracy that is always in tension, for liberalism always prioritises the individual, whereas the democracy places constraints on the individual #auspol #ausvotes   \n",
       "13565                                                                                                                                                                                @MegJobson1 Completely shattered - I feel like democracy in Australia is broken. An election won on lies, fear mongering and courruption.   \n",
       "80560                                                                                       Bob Hawke is not the same PM as Shorten will be. Shorten will be a dangerous socialist dictator, implementing his radical policies without transparency. He will not govern for the people, don't risk Labor #auspol #AusVotes2019   \n",
       "163657                                                                                                                                                                                                                                Penny Wong says looking almost impossible for Tony Abbott to win in Warringah. #ausvotes   \n",
       "132725                                                                                                                                                                               The nutters are out tonight but the palmer ads on lego masters \\nin western vic are insane. China invading? Labor in cahouts? Wtf #auspol   \n",
       "153646                                                                                                                                                                                      This has been a pretty clean and decent campaign considering where it could have gone given the state of modern politics #ausvotes   \n",
       "84165                                                                                    Some rando on Twitter: *makes up Engadine Maccas story*\\n#auspol: Yep, that definitely happened\\n\\nKathy Sherriff: *twice goes to police with detailed rape complaint*\\n#auspol: I will not even contemplate that might have happened   \n",
       "163570                                                                                                                                                                                        Thinking about being a farmer who suffers the full force of climate change but continues to for climate change deniers #AusVotes   \n",
       "45295                                                                                                                                                                                                                             How long until Morrison introduces compulsory creationism classes in all schools?\\n\\n#auspol   \n",
       "4169                                                                                     @mani1972ias May be yes, see recent election result in Germany, USA, Australia, ...nationalism kept first , then economy second! People never think pride will not give food! Only inclusive economic development will bring equality   \n",
       "149318                                                                                                                                                                                  Steggall's victory in Warringah removes a blistering wound on the heel of the \\nCoalition's far-right for Morrison to govern #ausvotes   \n",
       "46693                                 #Auspol's frustration with corporate policing morality has come to a head with #IsraelFolau. For the 1st time it can be tested in the public forum & a court. Do private companies & sponsors have a say over employee's private/online lives? Let's hope not or employment = slavery...   \n",
       "43748                                                                                                                                                                                                                                  How many hours per year will Parliament run of the LNP does not get a majority? #auspol   \n",
       "4189                                   Strong opportunities for Green parties to advance (esp in PR systems) to 10-15% of the vote, mobilising most supportive, to pressure governing parties. Have risen in Australia from 2-5% to 10% in a majoritarian system. But different challenges when central issue in choosing govt   \n",
       "59292                                                                                                                                      Corangamite was already notionally LABOR after redistribution Tanya you desperately vile true Deceiver 👎😡 Get some uppers off your husband you sad sack #auspol #australiavotes2019   \n",
       "157752                                                                                                                                                                                                                                  Stresses me the fuck out whenever they go to Antony Green with breaking news #ausvotes   \n",
       "26597                     @BevanShields I find it funny that Labor have wheeled out Australia's failed PM & Treasurer as some sort of example of how good Labor is.. Still waiting in Keating's \"J\" curve to appear.. The man's a narcissistic arrogant boofhead. His opinions mean nothing. Save Australia & vote Labor last.   \n",
       "137467                                                                                                                                                                              Leading with the woman responsible for arresting a 76 year old pensioner tells you everything about the Coalition launch #Ausvotes #Auspol   \n",
       "96225                                   So as we approach the end of this election campaign, the Liberal strategy has been duplicitous lying, maniacal thuggery and a pure one man show. We've had no policies and no other Liberal candidates get a word in. It's the ScoMo show, and he's failed miserably #auspol #ausvotes   \n",
       "152024                                                                                                                                                                          Once again the disconnect between the elitist media & real hard-working Australians is evident. The Silent Majority wins once again. #ausvotes   \n",
       "7098                                                                                                                                                                                                                           @catvalente I feel for Australia, dealing with two crappy election results in quick succession.   \n",
       "61173                                                                                                                                                                              Welp. Australia. I guess we can start being proud of how stupid we are in electing a government. *slow claps to economy depression* #auspol   \n",
       "65368                                                                                                                                                                                                                           Please don’t let me down WA, I’d like us to actually do something about climate change #auspol   \n",
       "99303                                                                           By forcing more and more people on to casual/temporary work regimes, weakens the ability of Unions to organise and take action. Work choices by deceit and quite deliberate by this disingenuous government. #auspol #AusVotes2019 #auspol2019   \n",
       "91659   @ScottMorrisonMP Delusional\\n\\nJobs by forcing multi jobs and spouses to work due wage erosion and costs, popn. growth and NDIS\\n\\nOnly faux future surplus\\n\\nNow your debt\\n\\nNeglect 95% of families\\n\\nIndifferent to small business\\n\\nCut schools, health and infrastructure \\n\\nSecurity - Rawandans\\n\\n#auspol   \n",
       "15965                                                                                                                             Clive Palmer has reportedly hired dozens of young models to don yellow shirts and hand out United Australia Party how-to-vote cards at polling booths across Cairns. https://t.co/eqDwf7OTX4   \n",
       "45466                                           Scott Morrison has spoken with US President Donald Trump this morning and is likely to speak with UK Prime Minister Theresa May later today. French President Emmanuel Macron and NZ PM Jacinda Ardern have also offered their congratulations directly to Mr Morrison #auspol   \n",
       "164101                                                                                                                                                                          Not even kidding, I ADORE ALAN JONES' GREEN SUIT. Maybe even more iconic than Antony Green's starry tie. Want one. Who's the tailor? #ausvotes   \n",
       "29982                                    As Australia prepares for elections this month, Reuters reports on how the country is one of the most susceptible in terms of online information gathering by political campaigns. Australia's privacy laws do not cover political parties and candidates...\\nhttps://t.co/l4az2jyapm   \n",
       "177628                                                                                                                                                                                                                          Voting early is kinda depressing -- we need sausage sizzles at early voting centres. #ausvotes   \n",
       "157302                                                                                                                                                                          We NEEDED serious action on climate change and a humane immigration policy. Instead we got climate deniers, xenophobia and a potato. #ausvotes   \n",
       "30711                                   I'll think positively about the #auspol result despite my disappointment. Hopefully climate predictions have been wrong or overstated. Hopefully the reef will recover. Hopefully our economy and social welfare will grow despite lack of progressive or positive policy. Here hopes.   \n",
       "106831                          @169lunar Just a protection racket for environmental vandals, wage theft, corporate thieves, natural resource plunderers, off shore tax havens, and a foreign billionaire media mogul whose notoriety is political interference and suppression or ridicule of the truth #auspol #AusVotes2019   \n",
       "51732                                                                                                                                                                                                                             Quiet Australians, oh fuck off. Let’s show what the quiet Australians are capable of #Auspol   \n",
       "29253                                                                                                                                                                           @thehill Wow did he miss the whole 2016 Steele/FBI/DNC/HRC/Australia/Ukraine/DOJ election interference that he’s hoping doesn’t come to light?   \n",
       "153178                                                                                                                                                                                                                             Typical old people coming out in big numbers ruining younger generations Futures. #ausvotes   \n",
       "56567                                I had hope Australia would go against the overseas trend of supporting unpleasant, neo-con populists.\\nI believed Australia was better and would set an example on thoughtful politics.\\nI looked forward to necessary reform.\\nIt didn't happen.\\nI weep for my grandchildren. \\n#auspol   \n",
       "147001                                                                                #AustraliaDecides Old Bill campaigned the usual way, pleading to hearts and minds. Big bad Scott went full metal technological war and exploited the preferencing process. Props to robo scomo for using arbitrage to win Australia. GG.   \n",
       "48155                                                                                                                               Labor has to elect @Bowenchris . He is the only leadership option who can emulate the Hawke/Keating success formula of cleverly designed social programs AND productivity reforms. #AUSPOL   \n",
       "109426                                Living in the ACT is a blessing come election time. Most Australians don't realise that, by federal mandate, roadside election advertising scales with state land area. For example, candidate corflutes, poster sized here, are 1/3 the size of an Olympic swimming pool in QLD #auspol   \n",
       "30506                                                                                                                                                                           Franking credits and imputation credits sounds like rocket science to the electorate. No wonder the fake retiree tax worked. #auspol #ausvoted   \n",
       "35268                                                                                                                               Why are people calling Morrison messiah? Seems a little crazy to think a politician who is for cutting employment and stifling social justice would be that. Egos running rampant. #auspol   \n",
       "15023                                                                              @RyanMendi97 is a moron. Not on the electoral roll in Australia despite living his whole life here, but managed to vote in a fucken Spanish election the flog. Thank god dual-citizenship rules his dumb brain out of parliament. #ausvotes   \n",
       "79856                                                                                                                                                                                  Why doesn't it surprise me that an angry old man who dislikes a conservative politician would stab someone volunteering for him #auspol   \n",
       "24808                                                                                                                        @Raymartin55 In Australia an election next week \\nBull Shitten seems ready to peak\\nLabor’s Financial management is hell\\nTheir leader’s an accused rapist as well\\nHope voters deny them a seat.   \n",
       "\n",
       "        length  word fraction  token length  english score  \n",
       "118115     274       0.700730      6.210526         1000.0  \n",
       "45521      226       0.761062      5.135135         1000.0  \n",
       "163798     129       0.806202      7.125000         1000.0  \n",
       "90300       89       0.764045      6.500000         1000.0  \n",
       "50832      274       0.751825      4.708333         1000.0  \n",
       "94457      137       0.737226      4.307692         1000.0  \n",
       "153896     132       0.765152      5.045455         1000.0  \n",
       "31976      271       0.701107      5.609756         1000.0  \n",
       "166204     181       0.745856      5.066667         1000.0  \n",
       "2680       183       0.726776      6.076923         1000.0  \n",
       "182426     131       0.725191      5.285714         1000.0  \n",
       "54914      266       0.725564      4.448980         1000.0  \n",
       "94674      282       0.716312      6.205128         1000.0  \n",
       "176861     218       0.775229      4.918919         1000.0  \n",
       "63425       89       0.741573      5.000000         1000.0  \n",
       "72720      221       0.769231      5.727273         1000.0  \n",
       "13565      137       0.744526      5.571429         1000.0  \n",
       "80560      226       0.734513      5.305556         1000.0  \n",
       "163657      88       0.738636      5.357143         1000.0  \n",
       "132725     136       0.750000      4.666667         1000.0  \n",
       "153646     130       0.769231      4.954545         1000.0  \n",
       "84165      225       0.702222      5.617647         1000.0  \n",
       "163570     128       0.773438      5.142857         1000.0  \n",
       "45295       90       0.777778      6.500000         1000.0  \n",
       "4169       229       0.711790      5.764706         1000.0  \n",
       "149318     133       0.759398      5.650000         1000.0  \n",
       "46693      280       0.707143      4.978723         1000.0  \n",
       "43748       87       0.724138      4.176471         1000.0  \n",
       "4189       279       0.767025      5.363636         1000.0  \n",
       "59292      179       0.709497      6.200000         1000.0  \n",
       "157752      86       0.732558      4.800000         1000.0  \n",
       "26597      292       0.739726      4.979592         1001.0  \n",
       "137467     138       0.724638      5.619048         1001.0  \n",
       "96225      278       0.744604      4.812500         1001.0  \n",
       "152024     142       0.767606      5.809524         1001.0  \n",
       "7098        95       0.726316      5.857143         1001.0  \n",
       "61173      139       0.719424      4.833333         1001.0  \n",
       "65368       94       0.723404      4.588235         1001.0  \n",
       "99303      238       0.710084      5.828571         1001.0  \n",
       "91659      294       0.721088      5.500000         1001.0  \n",
       "15965      188       0.723404      6.000000         1001.0  \n",
       "45466      270       0.814815      5.302326         1001.0  \n",
       "164101     142       0.718310      4.958333         1001.0  \n",
       "29982      276       0.757246      6.102564         1001.0  \n",
       "177628      94       0.723404      5.333333         1001.0  \n",
       "157302     142       0.767606      5.500000         1001.0  \n",
       "30711      278       0.812950      5.975000         1001.0  \n",
       "106831     286       0.748252      6.358974         1001.0  \n",
       "51732       92       0.739130      5.200000         1001.0  \n",
       "29253      142       0.732394      6.944444         1001.0  \n",
       "153178      91       0.758242      6.076923         1001.0  \n",
       "56567      276       0.789855      5.418605         1001.0  \n",
       "147001     232       0.754310      5.472222         1001.0  \n",
       "48155      186       0.741935      5.678571         1001.0  \n",
       "109426     280       0.778571      5.244444         1001.0  \n",
       "30506      142       0.732394      5.809524         1001.0  \n",
       "35268      186       0.790323      5.233333         1001.0  \n",
       "15023      235       0.727660      5.051282         1001.0  \n",
       "79856      135       0.785185      5.181818         1001.0  \n",
       "24808      189       0.756614      5.096774         1001.0  "
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = trim_dataset_by(\n",
    "    df, \n",
    "    ascending=True,\n",
    "    \n",
    "    # sort_by='length',\n",
    "    length=(70, None),\n",
    "    \n",
    "    # sort_by='word fraction',\n",
    "    word_fraction=(0.7, None),\n",
    "    \n",
    "    # sort_by='token length',\n",
    "    token_length=(3.5, 8),\n",
    "    \n",
    "    sort_by='english score',\n",
    "    english_score=(1000, None),\n",
    ")\n",
    "a[:60]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "id": "15861c57-d323-47ea-8fc0-1c4dedb9c1ef",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:39:22.866158Z",
     "iopub.status.busy": "2024-01-08T15:39:22.865682Z",
     "iopub.status.idle": "2024-01-08T15:39:40.511058Z",
     "shell.execute_reply": "2024-01-08T15:39:40.510170Z",
     "shell.execute_reply.started": "2024-01-08T15:39:22.866134Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time elapsed: 18s\n",
      "samples: 10907, similar samples detected: 93 (0.85%)\n"
     ]
    }
   ],
   "source": [
    "df = drop_similar(supersample(a, AUSTRALIAN_ELECTIONS));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "id": "2c5dba0c-a1e6-4889-91b7-70c947146f66",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:39:40.512866Z",
     "iopub.status.busy": "2024-01-08T15:39:40.512234Z",
     "iopub.status.idle": "2024-01-08T15:39:40.616733Z",
     "shell.execute_reply": "2024-01-08T15:39:40.615983Z",
     "shell.execute_reply.started": "2024-01-08T15:39:40.512840Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "string frequencies:\n",
      "#auspol: 0.56\n",
      "#ausvotes: 0.30\n",
      "#australiavotes: 0.01\n"
     ]
    }
   ],
   "source": [
    "df = a\n",
    "s = '#coronavirus'\n",
    "print('string frequencies:')\n",
    "\n",
    "s = '#auspol'; print(f'{s}: {string_frequency(df, s):.2f}')\n",
    "s = '#ausvotes'; print(f'{s}: {string_frequency(df, s):.2f}')\n",
    "s = '#australiavotes'; print(f'{s}: {string_frequency(df, s):.2f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "id": "fab07551-3680-40ff-975d-29cdeab06bf7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:39:40.618116Z",
     "iopub.status.busy": "2024-01-08T15:39:40.617923Z",
     "iopub.status.idle": "2024-01-08T15:39:40.759196Z",
     "shell.execute_reply": "2024-01-08T15:39:40.758619Z",
     "shell.execute_reply.started": "2024-01-08T15:39:40.618096Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\"#auspol\" new frequency: 0.11\n",
      "\"#ausvotes\" new frequency: 0.15\n"
     ]
    }
   ],
   "source": [
    "df = randomly_remove_substring(df, '#auspol', p=0.8, case=False)\n",
    "df = randomly_remove_substring(df, '#ausvotes', p=0.5, case=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "id": "ea101e8e-d0eb-4bfa-95d7-9c8c9f0f3398",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:39:40.760566Z",
     "iopub.status.busy": "2024-01-08T15:39:40.760077Z",
     "iopub.status.idle": "2024-01-08T15:39:40.782248Z",
     "shell.execute_reply": "2024-01-08T15:39:40.781492Z",
     "shell.execute_reply.started": "2024-01-08T15:39:40.760543Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>length</th>\n",
       "      <th>word fraction</th>\n",
       "      <th>token length</th>\n",
       "      <th>english score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>23869</th>\n",
       "      <td>1128727062066409472</td>\n",
       "      <td>@tedcruz Maybe we worry about our election booths first? Speaking of elections, have you heard that @Australia has mandatory voting? What a concept.</td>\n",
       "      <td>148</td>\n",
       "      <td>0.702703</td>\n",
       "      <td>5.478261</td>\n",
       "      <td>1024.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9118</th>\n",
       "      <td>1129838671824343042</td>\n",
       "      <td>Is everybody back home in Australia awake and watching #Eurovision because of the soul crushing insomnia after the election?</td>\n",
       "      <td>124</td>\n",
       "      <td>0.758065</td>\n",
       "      <td>5.578947</td>\n",
       "      <td>1165.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49360</th>\n",
       "      <td>1129869619391475712</td>\n",
       "      <td>You just voted the Liberal party back in. It's like winning a lottery in reverse.</td>\n",
       "      <td>89</td>\n",
       "      <td>0.719101</td>\n",
       "      <td>4.625000</td>\n",
       "      <td>1188.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7014</th>\n",
       "      <td>1129891832580624384</td>\n",
       "      <td>This election was won by big bold lies, media amplifying them, pressuring Labor based on false premises &amp; $60M from Palmer. Well and good to gloat but the idea that this was a win for the people of Australia is entirely false. Wait for the paybacks to Murdoch &amp; Palmer #insiders</td>\n",
       "      <td>278</td>\n",
       "      <td>0.751799</td>\n",
       "      <td>4.470588</td>\n",
       "      <td>1242.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35730</th>\n",
       "      <td>1130274918589845504</td>\n",
       "      <td>I must be misunderstanding Joel Fitzgibbon. He thinks the ALP has to move to the right to find the centre?\\n</td>\n",
       "      <td>114</td>\n",
       "      <td>0.745614</td>\n",
       "      <td>4.476190</td>\n",
       "      <td>1033.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        id  \\\n",
       "23869  1128727062066409472   \n",
       "9118   1129838671824343042   \n",
       "49360  1129869619391475712   \n",
       "7014   1129891832580624384   \n",
       "35730  1130274918589845504   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                         text  \\\n",
       "23869                                                                                                                                    @tedcruz Maybe we worry about our election booths first? Speaking of elections, have you heard that @Australia has mandatory voting? What a concept.   \n",
       "9118                                                                                                                                                             Is everybody back home in Australia awake and watching #Eurovision because of the soul crushing insomnia after the election?   \n",
       "49360                                                                                                                                                                                                      You just voted the Liberal party back in. It's like winning a lottery in reverse.    \n",
       "7014   This election was won by big bold lies, media amplifying them, pressuring Labor based on false premises & $60M from Palmer. Well and good to gloat but the idea that this was a win for the people of Australia is entirely false. Wait for the paybacks to Murdoch & Palmer #insiders   \n",
       "35730                                                                                                                                                                            I must be misunderstanding Joel Fitzgibbon. He thinks the ALP has to move to the right to find the centre?\\n   \n",
       "\n",
       "       length  word fraction  token length  english score  \n",
       "23869     148       0.702703      5.478261         1024.0  \n",
       "9118      124       0.758065      5.578947         1165.0  \n",
       "49360      89       0.719101      4.625000         1188.0  \n",
       "7014      278       0.751799      4.470588         1242.0  \n",
       "35730     114       0.745614      4.476190         1033.0  "
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "id": "e065d113-9205-441c-b86c-9e0a062bd48c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:39:40.783545Z",
     "iopub.status.busy": "2024-01-08T15:39:40.783136Z",
     "iopub.status.idle": "2024-01-08T15:39:40.835979Z",
     "shell.execute_reply": "2024-01-08T15:39:40.835217Z",
     "shell.execute_reply.started": "2024-01-08T15:39:40.783523Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\"#auspol\" new frequency: 0.04\n"
     ]
    }
   ],
   "source": [
    "df = randomly_remove_substring(df, '#auspol', p=0.6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "id": "1c5a434f-c5af-4f19-a99c-f9cd3daf514a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:39:40.837516Z",
     "iopub.status.busy": "2024-01-08T15:39:40.836974Z",
     "iopub.status.idle": "2024-01-08T15:39:40.889283Z",
     "shell.execute_reply": "2024-01-08T15:39:40.888525Z",
     "shell.execute_reply.started": "2024-01-08T15:39:40.837493Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\"#ausvotes\" new frequency: 0.10\n"
     ]
    }
   ],
   "source": [
    "df = randomly_remove_substring(df, '#ausvotes', p=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "id": "238c61ba-86f7-40ce-947e-cc43008c066a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:39:40.890705Z",
     "iopub.status.busy": "2024-01-08T15:39:40.890210Z",
     "iopub.status.idle": "2024-01-08T15:39:40.919028Z",
     "shell.execute_reply": "2024-01-08T15:39:40.918322Z",
     "shell.execute_reply.started": "2024-01-08T15:39:40.890684Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 3)"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = select_columns(df.copy(), ['text', 'id'])\n",
    "df['topic'] = 'australian elections'\n",
    "\n",
    "df = sample(df, AUSTRALIAN_ELECTIONS)\n",
    "dataset = concat(dataset, df)\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8b81c3e4-96b5-4c1f-a3de-bb030f6cf52e",
   "metadata": {},
   "source": [
    "# COVID19"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "id": "aac2af45-4d04-4ae7-88fb-b24d6b958713",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:39:40.920444Z",
     "iopub.status.busy": "2024-01-08T15:39:40.919967Z",
     "iopub.status.idle": "2024-01-08T15:39:41.084147Z",
     "shell.execute_reply": "2024-01-08T15:39:41.083373Z",
     "shell.execute_reply.started": "2024-01-08T15:39:40.920420Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(44955, 2)"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1 = load_dataset('covid_train.csv', source=True, encoding = 'ISO-8859-1')\n",
    "df2 = load_dataset('covid_test.csv', source=True)\n",
    "df = pd.concat([df1, df2])\n",
    "df.rename(columns={'OriginalTweet': 'text', 'Sentiment': 'sentiment'}, inplace=True)\n",
    "df = select_columns(df, ['text', 'sentiment'])\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "id": "6732cfc5-db1a-47c9-911b-fd0c6a84c1af",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:39:41.085364Z",
     "iopub.status.busy": "2024-01-08T15:39:41.085145Z",
     "iopub.status.idle": "2024-01-08T15:39:43.028311Z",
     "shell.execute_reply": "2024-01-08T15:39:43.027561Z",
     "shell.execute_reply.started": "2024-01-08T15:39:41.085343Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(36980, 2)"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = preprocessing_pipeline(df)\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "id": "a0b7fe40-2ff8-4f38-8a43-79ded4f35c35",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:39:43.029633Z",
     "iopub.status.busy": "2024-01-08T15:39:43.029382Z",
     "iopub.status.idle": "2024-01-08T15:39:44.107839Z",
     "shell.execute_reply": "2024-01-08T15:39:44.107262Z",
     "shell.execute_reply.started": "2024-01-08T15:39:43.029608Z"
    }
   },
   "outputs": [],
   "source": [
    "add_stats(df);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "id": "5a269283-5135-43a0-b285-6dc276dc3720",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:39:44.109175Z",
     "iopub.status.busy": "2024-01-08T15:39:44.108577Z",
     "iopub.status.idle": "2024-01-08T15:39:44.151489Z",
     "shell.execute_reply": "2024-01-08T15:39:44.150977Z",
     "shell.execute_reply.started": "2024-01-08T15:39:44.109150Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "samples: 19558, reduction: 47.11%\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>length</th>\n",
       "      <th>word fraction</th>\n",
       "      <th>token length</th>\n",
       "      <th>english score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>33054</th>\n",
       "      <td>https://t.co/2yMUPatIYU Wear a mask! Especially in situations where you may be around people, like at the grocery store or pharmacy. Make your own mask, decorate it and have fun with your designs, make something cute and friendly. #CoronaVirus #FlattenTheCurve #StopTheSpread</td>\n",
       "      <td>Extremely Positive</td>\n",
       "      <td>275</td>\n",
       "      <td>0.6</td>\n",
       "      <td>5.900000</td>\n",
       "      <td>1361.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8396</th>\n",
       "      <td>Here are the facts about Coronavirus according to Consumer Reports: \\r\\r\\n https://t.co/47fybkzW5t</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>95</td>\n",
       "      <td>0.6</td>\n",
       "      <td>7.363636</td>\n",
       "      <td>1024.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13225</th>\n",
       "      <td>Is not a panic buying because of the global #coronavirus pandemic. Is about #Walmart online shopping and their politics of not products available. https://t.co/ZIJHJqGQ9F</td>\n",
       "      <td>Positive</td>\n",
       "      <td>170</td>\n",
       "      <td>0.6</td>\n",
       "      <td>6.125000</td>\n",
       "      <td>1154.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24459</th>\n",
       "      <td>#2020Is traveling with alcohol swabs in your pocket so that you can wipe down every keypad you have to interact with at the ATM, supermarket, and pharmacy, which are the only places you can still go. #StayAtHome #StayHome #Covid_19 https://t.co/J2Axi63CxJ</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>255</td>\n",
       "      <td>0.6</td>\n",
       "      <td>5.400000</td>\n",
       "      <td>1342.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13267</th>\n",
       "      <td>COVID-19, has just over 1,372 cases in the UK, and 35 deaths\\r\\r\\n\\r\\r\\nCANCER has around 1000 new cases every day and 450 deaths every day, (that's a 45% death rate) so why is that not causing panic buying and empty Supermarket shelves? https://t.co/gxq3Ai8Pma</td>\n",
       "      <td>Extremely Negative</td>\n",
       "      <td>255</td>\n",
       "      <td>0.6</td>\n",
       "      <td>4.837209</td>\n",
       "      <td>1418.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34018</th>\n",
       "      <td>We are excited to announce Pay-What-You-Can prices for our virtual event Self-Esteem: Coping in Collective Isolation, on April 16th. We recognize these are hard times and still want everyone to feel welcome. Info @ https://t.co/XKVI1jYAMT\\r\\r\\n\\r\\r\\n#coronavirus #mentalhealth #selfcare</td>\n",
       "      <td>Positive</td>\n",
       "      <td>280</td>\n",
       "      <td>0.6</td>\n",
       "      <td>6.263158</td>\n",
       "      <td>1296.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3610</th>\n",
       "      <td>Sainsbury: Give the elderly the first hour of supermarket trading during Covid-19 - Sign the Petition! https://t.co/sqPmSCMbGB via @UKChange</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>140</td>\n",
       "      <td>0.6</td>\n",
       "      <td>6.421053</td>\n",
       "      <td>1070.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7828</th>\n",
       "      <td>For an ongoing display of sheer idiocy check your local supermarket #coronavirus #coronavirusuk</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>95</td>\n",
       "      <td>0.6</td>\n",
       "      <td>6.384615</td>\n",
       "      <td>1013.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34633</th>\n",
       "      <td>As the #Covid_19 crisis pushes the food business into unkown terrain, the fragility &amp; flaws in our globalised food systems are revealed..the longer the supply networks, the more vulnerable they become. We need 'place based' food economies. #FoodSystems https://t.co/SlbARl2L19 https://t.co/mzZyRxwql5</td>\n",
       "      <td>Extremely Negative</td>\n",
       "      <td>300</td>\n",
       "      <td>0.6</td>\n",
       "      <td>6.525000</td>\n",
       "      <td>1148.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24838</th>\n",
       "      <td>Look at these cheap oil and gas prices when I have no where to go..\\r\\r\\n\\r\\r\\n#coronavirus</td>\n",
       "      <td>Negative</td>\n",
       "      <td>85</td>\n",
       "      <td>0.6</td>\n",
       "      <td>4.062500</td>\n",
       "      <td>1365.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34825</th>\n",
       "      <td>Covid-19: Shopping and consumer rights - Experts answers your questions https://t.co/RUQicQJL34</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>95</td>\n",
       "      <td>0.6</td>\n",
       "      <td>7.727273</td>\n",
       "      <td>1107.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6820</th>\n",
       "      <td>@petenator39 @hogotheforsaken @faizashaheen @rebelyarns Hey at least it's something! USA here &amp; our grocery store corporate isn't doing anything special regarding COVID-19. Oh wait, excuse me, the cashiers can get overtime if they want, but that's it</td>\n",
       "      <td>Positive</td>\n",
       "      <td>250</td>\n",
       "      <td>0.6</td>\n",
       "      <td>5.783784</td>\n",
       "      <td>949.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23509</th>\n",
       "      <td>LANXESS intends to stick to its strategy of organic growth, increasing its focus on consumer protection chemicals and developing battery tech amid the coronavirus outbreak, the CEO said.\\r\\r\\n#ICIS #coronavirus #COVID19 #Lanxess #chemicals https://t.co/iQDaIrReGW</td>\n",
       "      <td>Positive</td>\n",
       "      <td>260</td>\n",
       "      <td>0.6</td>\n",
       "      <td>6.617647</td>\n",
       "      <td>949.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>515</th>\n",
       "      <td>Hang in there everyone. We'll all get through this if we support each other.\\r\\r\\nPlease don't strip the supermarket shelves empty. Think of an elderly neighbour or relatives who might have run out of an essential. \\r\\r\\n\\r\\r\\n#coronavirus #RandomActsOfKindness #NI https://t.co/NVEJqNGGiR</td>\n",
       "      <td>Positive</td>\n",
       "      <td>280</td>\n",
       "      <td>0.6</td>\n",
       "      <td>5.825000</td>\n",
       "      <td>1225.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28519</th>\n",
       "      <td>It took me 20mins waiting on a line for Dog food for Kobe in the supermarket FUCK THIS #coronavirus??????</td>\n",
       "      <td>Extremely Negative</td>\n",
       "      <td>105</td>\n",
       "      <td>0.6</td>\n",
       "      <td>4.578947</td>\n",
       "      <td>1108.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6379</th>\n",
       "      <td>@Stop_Trump20 I just came back from the grocery store and there was not one egg in the store. I think someone told trump the cure for Covid 19 is eggs. https://t.co/q9xLrSPI6L</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>175</td>\n",
       "      <td>0.6</td>\n",
       "      <td>4.677419</td>\n",
       "      <td>1159.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5917</th>\n",
       "      <td>This image speaks and shows how inconsiderate and selfish the human population is, heartbreaking image #CoronaVirusUpdate https://t.co/UDwjrjuGz5</td>\n",
       "      <td>Extremely Negative</td>\n",
       "      <td>145</td>\n",
       "      <td>0.6</td>\n",
       "      <td>7.588235</td>\n",
       "      <td>907.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36644</th>\n",
       "      <td>How a cough can spread Coronavirus (COVID-19) through a supermarket and take minutes to disperse https://t.co/tahXxHiX7I</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>120</td>\n",
       "      <td>0.6</td>\n",
       "      <td>6.562500</td>\n",
       "      <td>961.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36848</th>\n",
       "      <td>Scammers are using COVID-19 messages to scam people | FTC Consumer Information ?? https://t.co/bEg53Al6VJ</td>\n",
       "      <td>Negative</td>\n",
       "      <td>105</td>\n",
       "      <td>0.6</td>\n",
       "      <td>6.571429</td>\n",
       "      <td>970.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23842</th>\n",
       "      <td>ETAuto | Covid-19: Oil prices mixed as demand shrinks, but stimulus hopes support https://t.co/bg6qAgt7vb</td>\n",
       "      <td>Extremely Positive</td>\n",
       "      <td>105</td>\n",
       "      <td>0.6</td>\n",
       "      <td>6.571429</td>\n",
       "      <td>1108.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33530</th>\n",
       "      <td>Who is making it easier for you to get through the COVID-19 crisis? A coworker? A neighbor? A grocery store clerk? A nurse? A police officer? There are so many \"everyday heroes\" in #AnneArundel County. https://t.co/EN1HF4H1Nr</td>\n",
       "      <td>Positive</td>\n",
       "      <td>225</td>\n",
       "      <td>0.6</td>\n",
       "      <td>5.277778</td>\n",
       "      <td>1062.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33460</th>\n",
       "      <td>@JugglingFitness @Resistfightthe1 @DoctorHenryCT Check out one of the many new #COVID19 policies at MY supermarket job, that JUST came down from the top: IF ANY of us call out for ANY SICK REASON, WHATSOEVER, we can NOT come back to work until we have tes</td>\n",
       "      <td>Negative</td>\n",
       "      <td>255</td>\n",
       "      <td>0.6</td>\n",
       "      <td>4.688889</td>\n",
       "      <td>1299.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33298</th>\n",
       "      <td>The East County Resource Center in Concrete has seen impacts from the COVID-19 pandemic, including increased demand on the center's emergency food pantry. https://t.co/inWwzzfwL2 #skagit https://t.co/ipHWvkgHX8</td>\n",
       "      <td>Negative</td>\n",
       "      <td>210</td>\n",
       "      <td>0.6</td>\n",
       "      <td>7.115385</td>\n",
       "      <td>1224.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28266</th>\n",
       "      <td>Up until about ten years ago supermarket home delivery really wasn't a thing so I am surprised quite so many people are up in arms as much as they are! #AskForHelp #OfferSomeHelp \\r\\r\\n#StayHomeSaveLives</td>\n",
       "      <td>Positive</td>\n",
       "      <td>200</td>\n",
       "      <td>0.6</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>1224.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10216</th>\n",
       "      <td>Did the government not promise they the vulnerable would have food deliveries ! Every supermarket /food delivery that I have tried says they are not delivering and are focusing on stores #Conservatives #Covid_19 #vulnerable #StayAtHome #SelfIsolation</td>\n",
       "      <td>Negative</td>\n",
       "      <td>250</td>\n",
       "      <td>0.6</td>\n",
       "      <td>5.972222</td>\n",
       "      <td>1194.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10075</th>\n",
       "      <td>@Nadinosaurusrex @RSewell1980 @LibertarianJew @OfficialSPGB This is exactly why. You oppose price gouging. It seems awful to you. OK. Now someone you likely voted for makes it illegal. Then something unusual like COVID-19 happens, and now we see shortages</td>\n",
       "      <td>Negative</td>\n",
       "      <td>255</td>\n",
       "      <td>0.6</td>\n",
       "      <td>5.736842</td>\n",
       "      <td>1173.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27825</th>\n",
       "      <td>We couldn't have put it better. From team #NHS to scientists and researchers, supermarket staff, delivery people and every nut and bolt in the UK workforce, together we will win and emerge stronger ? #coronavirus #love #family #humanity https://t.co/er5OhvykZW</td>\n",
       "      <td>Extremely Positive</td>\n",
       "      <td>260</td>\n",
       "      <td>0.6</td>\n",
       "      <td>5.692308</td>\n",
       "      <td>1313.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30259</th>\n",
       "      <td>The COVID-19 pandemic has impacted oil prices, tourism, and capital markets in the Middle East. \\r\\r\\n\\r\\r\\nhttps://t.co/bQ4QqgCDtS</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>125</td>\n",
       "      <td>0.6</td>\n",
       "      <td>6.500000</td>\n",
       "      <td>1142.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10015</th>\n",
       "      <td>Gifted to me by a receptionist, such gestures of care for each other melt my heart.\\r\\r\\nWe were discussing the chaos at supermarket and she went to take from her car.\\r\\r\\n#Humanity\\r\\r\\n#coronavirus https://t.co/WsaKQ6LbSD</td>\n",
       "      <td>Negative</td>\n",
       "      <td>215</td>\n",
       "      <td>0.6</td>\n",
       "      <td>5.176471</td>\n",
       "      <td>1089.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27718</th>\n",
       "      <td>For my marketers out there: join my colleague @PhilKinzler and I for this 1-hour webinar where we discuss how advanced data science and AI can be applied to more accurately predict how weather + COVID 19 influence consumer behavior.\\r\\r\\n\\r\\r\\nhttps://t.co/smrNf6a4nn https://t.co/Pgq1WPcSY0</td>\n",
       "      <td>Positive</td>\n",
       "      <td>285</td>\n",
       "      <td>0.6</td>\n",
       "      <td>5.853659</td>\n",
       "      <td>1397.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27635</th>\n",
       "      <td>Covid-19 Response: Church In The USA Gives Out Food To People | Nigerian Churches Demand For Electronic Tithe &amp; Offering In This Time Of Total Lock Down https://t.co/SAGWBbFGIJ https://t.co/VH7xDKE39W</td>\n",
       "      <td>Negative</td>\n",
       "      <td>200</td>\n",
       "      <td>0.6</td>\n",
       "      <td>5.931034</td>\n",
       "      <td>1122.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30482</th>\n",
       "      <td>@eBay @AskeBay @eBayNewsroom should really be removing these people charging outrageous prices for Lysol, Wipes and Sanitizer! I cant believe people are selling for up to $300! Unreal! Ebay needs to stop this immediately! #EbayPriceGouging #pricegouging #</td>\n",
       "      <td>Extremely Negative</td>\n",
       "      <td>255</td>\n",
       "      <td>0.6</td>\n",
       "      <td>5.918919</td>\n",
       "      <td>1122.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30603</th>\n",
       "      <td>@Aaron_TheThomas Just got back from the supermarket. There was a bloke there in a mask. I felt sorry for him. He was actually trying to rob the place but couldn't get anyone's attention. #COVID19au #auspol2020 #COVID2019</td>\n",
       "      <td>Negative</td>\n",
       "      <td>220</td>\n",
       "      <td>0.6</td>\n",
       "      <td>5.138889</td>\n",
       "      <td>1018.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9663</th>\n",
       "      <td>#Trump \\r\\r\\n#Covid19 Testing Kits for all, we need to find who is at risk\\r\\r\\n#StopTheSpread. \\r\\r\\nThe money, sounds great, but please, \\r\\r\\nWe Need Testing Kits! \\r\\r\\nMedication!\\r\\r\\nFind it Eliminate it. \\r\\r\\nEight hours work in a public supermarket for me tomorrow, unprotected from #Coronavirus #Pray</td>\n",
       "      <td>Positive</td>\n",
       "      <td>290</td>\n",
       "      <td>0.6</td>\n",
       "      <td>5.181818</td>\n",
       "      <td>1208.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31271</th>\n",
       "      <td>@IFPRI bringing some good news! \\r\\r\\n\"There is no shortage of staple food inventories and the stock-to-use ratio is close to normal!\" The question is will countries resort to trade restrictions? let's hope not\\r\\r\\n#food #foodsecurity #Covid_19 \\r\\r\\n\\r\\r\\nhttps://t</td>\n",
       "      <td>Positive</td>\n",
       "      <td>255</td>\n",
       "      <td>0.6</td>\n",
       "      <td>5.621622</td>\n",
       "      <td>1431.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9363</th>\n",
       "      <td>My idea to stop shops over changing...\\r\\r\\nArmed soldiers turn up and close their shop and confiscate All of their stock then drop it at food banks.\\r\\r\\n#coronavirus #corona #covid19 #COVID_19uk</td>\n",
       "      <td>Negative</td>\n",
       "      <td>190</td>\n",
       "      <td>0.6</td>\n",
       "      <td>5.032258</td>\n",
       "      <td>1176.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26031</th>\n",
       "      <td>Read 5 tips from a consumer psychologist on how brands should respond to #coronavirus. How many of these tactics do you have in place?\\r\\r\\nhttps://t.co/t1QozP9J0O</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>160</td>\n",
       "      <td>0.6</td>\n",
       "      <td>5.360000</td>\n",
       "      <td>1071.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12485</th>\n",
       "      <td>#notgoingout https://t.co/ACGwETNKIi shame these utter idiots are.. this isnt a joke. just look at the horrendous scene's in Italy do you really want that here?</td>\n",
       "      <td>Extremely Negative</td>\n",
       "      <td>160</td>\n",
       "      <td>0.6</td>\n",
       "      <td>5.440000</td>\n",
       "      <td>1030.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25946</th>\n",
       "      <td>How Retail Brands Have Been Disrupted, How eCommerce Can Help: Like most brand marketing categories, the effect of the COVID-19 pandemic on retail store traffic has been quick and profound,... https://t.co/HWXmGmwDh3 @mp_joemandese https://t.co/GR9v4gdPfv</td>\n",
       "      <td>Extremely Positive</td>\n",
       "      <td>255</td>\n",
       "      <td>0.6</td>\n",
       "      <td>6.757576</td>\n",
       "      <td>1058.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32000</th>\n",
       "      <td>I'm shocked by this line of cars waiting to get tested for Covid-19. Oh wait, I'm sorry... this the line at a food bank, because the economy is being ruined.\\r\\r\\n\\r\\r\\nHas anyone seen a line like this for Covid testing? No? Me neither.\\r\\r\\nhttps://t.co/781eq6bu4H https://t.co/x09G6HIkJv</td>\n",
       "      <td>Extremely Negative</td>\n",
       "      <td>280</td>\n",
       "      <td>0.6</td>\n",
       "      <td>5.088889</td>\n",
       "      <td>1141.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25380</th>\n",
       "      <td>.@NYGovCuomo says #NY state has purchased 17,000 ventilators at $25,000/each from #China. Has a firm expectation that 2,500 of them will arrive over the next two weeks. \\r\\r\\nSays high demand from other countries &amp; #US states has driven up prices &amp; slowed delivery. #coronavirus</td>\n",
       "      <td>Negative</td>\n",
       "      <td>275</td>\n",
       "      <td>0.6</td>\n",
       "      <td>5.204545</td>\n",
       "      <td>1178.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33038</th>\n",
       "      <td>@Daniel_I_Berger @CompComSA please deal with @SPARInt and fine them 10% of their annual turnover. They are fixing the prices in South Africa since COVID-19 pandemic.</td>\n",
       "      <td>Positive</td>\n",
       "      <td>165</td>\n",
       "      <td>0.6</td>\n",
       "      <td>5.640000</td>\n",
       "      <td>1077.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36899</th>\n",
       "      <td>If this is true then surely protective face masks for households would of been better than a government letter sent in the post .#StayHomeSaveLives \\r\\r\\n https://t.co/ffYV7lNeRK</td>\n",
       "      <td>Extremely Positive</td>\n",
       "      <td>175</td>\n",
       "      <td>0.6</td>\n",
       "      <td>5.880000</td>\n",
       "      <td>1158.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22666</th>\n",
       "      <td>@cpbbsrctc I never said that the supply is blocked. What I was trying to point out is they are selling it at an exorbitant prices. I had tweeted to @cpbbsrctc and @SarangiSudhansu to look into the matter. Now see, what is the situation. https://t.co/HNpDe</td>\n",
       "      <td>Negative</td>\n",
       "      <td>255</td>\n",
       "      <td>0.6</td>\n",
       "      <td>4.818182</td>\n",
       "      <td>1307.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23221</th>\n",
       "      <td>If I were to tweet that I ration my #toiletpaper to one square per bathroom trip, you might as well just assume I use my hand to wipe. ?\\r\\r\\n\\r\\r\\nCome on people. One square of toilet paper. NO ? #coronavirus #StayAtHome</td>\n",
       "      <td>Positive</td>\n",
       "      <td>215</td>\n",
       "      <td>0.6</td>\n",
       "      <td>4.146341</td>\n",
       "      <td>1157.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10897</th>\n",
       "      <td>@MattHancock @BorisJohnson @10DowningStreet @AWMurrison I am a supermarket worker in a major store, public are not respecting workers by distancing its like were invisible, please go further to protect us as we cant feed a nation if were sick too. #corona</td>\n",
       "      <td>Positive</td>\n",
       "      <td>255</td>\n",
       "      <td>0.6</td>\n",
       "      <td>5.243902</td>\n",
       "      <td>965.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19894</th>\n",
       "      <td>PM Imran Khan today announced Rs 1200 billion Economic Relief and stimulus Package.\\r\\r\\nThe package includes utility bills being deferred for lower income groups, principal &amp; interest for businesses, &amp; reduction in fuel prices.\\r\\r\\n#PMIK ????????\\r\\r\\n#CoronaVirusUpdates\\r\\r\\n#Covid_19 https://t.co/TP1u9FBR6d</td>\n",
       "      <td>Positive</td>\n",
       "      <td>300</td>\n",
       "      <td>0.6</td>\n",
       "      <td>6.512821</td>\n",
       "      <td>1160.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40388</th>\n",
       "      <td>At #DemystData, we have access to multiple datasets and data products that track consumer behavior, demographic profiles, and #geolocation details to help understand which sectors are and will be impacted the hardest. #pandemic #externaldata https://t.co/fR5qRZyufS</td>\n",
       "      <td>Positive</td>\n",
       "      <td>265</td>\n",
       "      <td>0.6</td>\n",
       "      <td>6.600000</td>\n",
       "      <td>992.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>773</th>\n",
       "      <td>I may be naive, but are food hoarding and selling off stock really helping the situation? #CoronavirusPandemic #Covid_19</td>\n",
       "      <td>Positive</td>\n",
       "      <td>120</td>\n",
       "      <td>0.6</td>\n",
       "      <td>5.722222</td>\n",
       "      <td>1122.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19586</th>\n",
       "      <td>@Accelerationis1 Our retail store is closed but other business operations remain. See https://t.co/yg4EwBWBGo You can order online or by phone and we will ship.</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>160</td>\n",
       "      <td>0.6</td>\n",
       "      <td>5.708333</td>\n",
       "      <td>1195.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16239</th>\n",
       "      <td>Idea when there is no sanitizer to be had. Keep water in your car; soap, towel. Pour water onto hands to wet them. Use soap and wash for 20 seconds. Pour water onto hands to rinse. Stand outside car to complete take. #CoronavirusPandemic #CoronavirusUSA #Coronavirus #COVID19</td>\n",
       "      <td>Negative</td>\n",
       "      <td>275</td>\n",
       "      <td>0.6</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>1083.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40723</th>\n",
       "      <td>@OilandEnergy Now we just need to cut another 10-20 MMbbl/d to fix oil prices and cure COVID-19</td>\n",
       "      <td>Negative</td>\n",
       "      <td>95</td>\n",
       "      <td>0.6</td>\n",
       "      <td>4.647059</td>\n",
       "      <td>1321.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1384</th>\n",
       "      <td>\"The rapid shift to online ordering and delivery is having a powerful impact - and this could set the tone for consumer shifts in how we purchase cannabis moving forward.\" https://t.co/8pCMeqM5ot @Airfieldsupply @WallandBroad</td>\n",
       "      <td>Positive</td>\n",
       "      <td>225</td>\n",
       "      <td>0.6</td>\n",
       "      <td>5.848485</td>\n",
       "      <td>1232.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39118</th>\n",
       "      <td>The U.S. has been blowing it for years. A healthy capitalist society is based on real competition... and w/ so many corporations buying up everything &amp; merging? The consumer is always fucked. https://t.co/X0lFOILbQO #capitalism #competition #monopoly</td>\n",
       "      <td>Negative</td>\n",
       "      <td>250</td>\n",
       "      <td>0.6</td>\n",
       "      <td>5.972222</td>\n",
       "      <td>1175.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1408</th>\n",
       "      <td>Please listen to my VERY candid conversation with an LA based ER doctor on the current state of #COVID2019 https://t.co/jt8cd52NRo</td>\n",
       "      <td>Positive</td>\n",
       "      <td>130</td>\n",
       "      <td>0.6</td>\n",
       "      <td>5.550000</td>\n",
       "      <td>1015.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40995</th>\n",
       "      <td>PM Modi: No need to worry about essentials. India has enough stock of everything, from food to medicines\\r\\r\\n#coronavirus #COVID2019 #pmoindia</td>\n",
       "      <td>Extremely Negative</td>\n",
       "      <td>140</td>\n",
       "      <td>0.6</td>\n",
       "      <td>5.619048</td>\n",
       "      <td>1000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18132</th>\n",
       "      <td>Boris Johnson: Stop In-Person Shopping all Grocery Shopping to Be Made Online To Contain Covid-19 https://t.co/PUhU94zFiA via @UKChange</td>\n",
       "      <td>Negative</td>\n",
       "      <td>135</td>\n",
       "      <td>0.6</td>\n",
       "      <td>6.555556</td>\n",
       "      <td>1064.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18877</th>\n",
       "      <td>Dr @Swamy39\\r\\r\\nHuge demand for food products in global markets due to COVID-19 outbreak, India can tap it: TPCI - The Economic Times https://t.co/mw1feiS7yH</td>\n",
       "      <td>Positive</td>\n",
       "      <td>155</td>\n",
       "      <td>0.6</td>\n",
       "      <td>5.416667</td>\n",
       "      <td>1096.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>409</th>\n",
       "      <td>Everyone is out there buying toilet paper and canned food but if I am stuck at home, I am going to stock up on things I love - wine, Epsom salts and chocolate! #coronavirus #COVID19 #icansurviveonchocolate</td>\n",
       "      <td>Extremely Positive</td>\n",
       "      <td>205</td>\n",
       "      <td>0.6</td>\n",
       "      <td>4.722222</td>\n",
       "      <td>1206.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2625</th>\n",
       "      <td>There are people who stock up with food and toilet paper for the quarantine, #VanessaHudgens apparently stocked up with drugs.\\r\\r\\n#Coronavirus #COVID19</td>\n",
       "      <td>Neutral</td>\n",
       "      <td>150</td>\n",
       "      <td>0.6</td>\n",
       "      <td>5.772727</td>\n",
       "      <td>1124.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                                                                                                                                           text  \\\n",
       "33054                                       https://t.co/2yMUPatIYU Wear a mask! Especially in situations where you may be around people, like at the grocery store or pharmacy. Make your own mask, decorate it and have fun with your designs, make something cute and friendly. #CoronaVirus #FlattenTheCurve #StopTheSpread   \n",
       "8396                                                                                                                                                                                                                         Here are the facts about Coronavirus according to Consumer Reports: \\r\\r\\n https://t.co/47fybkzW5t   \n",
       "13225                                                                                                                                                Is not a panic buying because of the global #coronavirus pandemic. Is about #Walmart online shopping and their politics of not products available. https://t.co/ZIJHJqGQ9F   \n",
       "24459                                                           #2020Is traveling with alcohol swabs in your pocket so that you can wipe down every keypad you have to interact with at the ATM, supermarket, and pharmacy, which are the only places you can still go. #StayAtHome #StayHome #Covid_19 https://t.co/J2Axi63CxJ   \n",
       "13267                                                     COVID-19, has just over 1,372 cases in the UK, and 35 deaths\\r\\r\\n\\r\\r\\nCANCER has around 1000 new cases every day and 450 deaths every day, (that's a 45% death rate) so why is that not causing panic buying and empty Supermarket shelves? https://t.co/gxq3Ai8Pma   \n",
       "34018                            We are excited to announce Pay-What-You-Can prices for our virtual event Self-Esteem: Coping in Collective Isolation, on April 16th. We recognize these are hard times and still want everyone to feel welcome. Info @ https://t.co/XKVI1jYAMT\\r\\r\\n\\r\\r\\n#coronavirus #mentalhealth #selfcare   \n",
       "3610                                                                                                                                                                               Sainsbury: Give the elderly the first hour of supermarket trading during Covid-19 - Sign the Petition! https://t.co/sqPmSCMbGB via @UKChange   \n",
       "7828                                                                                                                                                                                                                            For an ongoing display of sheer idiocy check your local supermarket #coronavirus #coronavirusuk   \n",
       "34633              As the #Covid_19 crisis pushes the food business into unkown terrain, the fragility & flaws in our globalised food systems are revealed..the longer the supply networks, the more vulnerable they become. We need 'place based' food economies. #FoodSystems https://t.co/SlbARl2L19 https://t.co/mzZyRxwql5   \n",
       "24838                                                                                                                                                                                                                               Look at these cheap oil and gas prices when I have no where to go..\\r\\r\\n\\r\\r\\n#coronavirus   \n",
       "34825                                                                                                                                                                                                                           Covid-19: Shopping and consumer rights - Experts answers your questions https://t.co/RUQicQJL34   \n",
       "6820                                                                 @petenator39 @hogotheforsaken @faizashaheen @rebelyarns Hey at least it's something! USA here & our grocery store corporate isn't doing anything special regarding COVID-19. Oh wait, excuse me, the cashiers can get overtime if they want, but that's it   \n",
       "23509                                                   LANXESS intends to stick to its strategy of organic growth, increasing its focus on consumer protection chemicals and developing battery tech amid the coronavirus outbreak, the CEO said.\\r\\r\\n#ICIS #coronavirus #COVID19 #Lanxess #chemicals https://t.co/iQDaIrReGW   \n",
       "515                           Hang in there everyone. We'll all get through this if we support each other.\\r\\r\\nPlease don't strip the supermarket shelves empty. Think of an elderly neighbour or relatives who might have run out of an essential. \\r\\r\\n\\r\\r\\n#coronavirus #RandomActsOfKindness #NI https://t.co/NVEJqNGGiR   \n",
       "28519                                                                                                                                                                                                                 It took me 20mins waiting on a line for Dog food for Kobe in the supermarket FUCK THIS #coronavirus??????   \n",
       "6379                                                                                                                                            @Stop_Trump20 I just came back from the grocery store and there was not one egg in the store. I think someone told trump the cure for Covid 19 is eggs. https://t.co/q9xLrSPI6L   \n",
       "5917                                                                                                                                                                          This image speaks and shows how inconsiderate and selfish the human population is, heartbreaking image #CoronaVirusUpdate https://t.co/UDwjrjuGz5   \n",
       "36644                                                                                                                                                                                                  How a cough can spread Coronavirus (COVID-19) through a supermarket and take minutes to disperse https://t.co/tahXxHiX7I   \n",
       "36848                                                                                                                                                                                                                 Scammers are using COVID-19 messages to scam people | FTC Consumer Information ?? https://t.co/bEg53Al6VJ   \n",
       "23842                                                                                                                                                                                                                 ETAuto | Covid-19: Oil prices mixed as demand shrinks, but stimulus hopes support https://t.co/bg6qAgt7vb   \n",
       "33530                                                                                         Who is making it easier for you to get through the COVID-19 crisis? A coworker? A neighbor? A grocery store clerk? A nurse? A police officer? There are so many \"everyday heroes\" in #AnneArundel County. https://t.co/EN1HF4H1Nr   \n",
       "33460                                                           @JugglingFitness @Resistfightthe1 @DoctorHenryCT Check out one of the many new #COVID19 policies at MY supermarket job, that JUST came down from the top: IF ANY of us call out for ANY SICK REASON, WHATSOEVER, we can NOT come back to work until we have tes   \n",
       "33298                                                                                                        The East County Resource Center in Concrete has seen impacts from the COVID-19 pandemic, including increased demand on the center's emergency food pantry. https://t.co/inWwzzfwL2 #skagit https://t.co/ipHWvkgHX8   \n",
       "28266                                                                                                               Up until about ten years ago supermarket home delivery really wasn't a thing so I am surprised quite so many people are up in arms as much as they are! #AskForHelp #OfferSomeHelp \\r\\r\\n#StayHomeSaveLives   \n",
       "10216                                                                Did the government not promise they the vulnerable would have food deliveries ! Every supermarket /food delivery that I have tried says they are not delivering and are focusing on stores #Conservatives #Covid_19 #vulnerable #StayAtHome #SelfIsolation   \n",
       "10075                                                           @Nadinosaurusrex @RSewell1980 @LibertarianJew @OfficialSPGB This is exactly why. You oppose price gouging. It seems awful to you. OK. Now someone you likely voted for makes it illegal. Then something unusual like COVID-19 happens, and now we see shortages   \n",
       "27825                                                      We couldn't have put it better. From team #NHS to scientists and researchers, supermarket staff, delivery people and every nut and bolt in the UK workforce, together we will win and emerge stronger ? #coronavirus #love #family #humanity https://t.co/er5OhvykZW   \n",
       "30259                                                                                                                                                                                       The COVID-19 pandemic has impacted oil prices, tourism, and capital markets in the Middle East. \\r\\r\\n\\r\\r\\nhttps://t.co/bQ4QqgCDtS   \n",
       "10015                                                                                          Gifted to me by a receptionist, such gestures of care for each other melt my heart.\\r\\r\\nWe were discussing the chaos at supermarket and she went to take from her car.\\r\\r\\n#Humanity\\r\\r\\n#coronavirus https://t.co/WsaKQ6LbSD   \n",
       "27718                       For my marketers out there: join my colleague @PhilKinzler and I for this 1-hour webinar where we discuss how advanced data science and AI can be applied to more accurately predict how weather + COVID 19 influence consumer behavior.\\r\\r\\n\\r\\r\\nhttps://t.co/smrNf6a4nn https://t.co/Pgq1WPcSY0   \n",
       "27635                                                                                                                  Covid-19 Response: Church In The USA Gives Out Food To People | Nigerian Churches Demand For Electronic Tithe & Offering In This Time Of Total Lock Down https://t.co/SAGWBbFGIJ https://t.co/VH7xDKE39W   \n",
       "30482                                                           @eBay @AskeBay @eBayNewsroom should really be removing these people charging outrageous prices for Lysol, Wipes and Sanitizer! I cant believe people are selling for up to $300! Unreal! Ebay needs to stop this immediately! #EbayPriceGouging #pricegouging #   \n",
       "30603                                                                                              @Aaron_TheThomas Just got back from the supermarket. There was a bloke there in a mask. I felt sorry for him. He was actually trying to rob the place but couldn't get anyone's attention. #COVID19au #auspol2020 #COVID2019   \n",
       "9663    #Trump \\r\\r\\n#Covid19 Testing Kits for all, we need to find who is at risk\\r\\r\\n#StopTheSpread. \\r\\r\\nThe money, sounds great, but please, \\r\\r\\nWe Need Testing Kits! \\r\\r\\nMedication!\\r\\r\\nFind it Eliminate it. \\r\\r\\nEight hours work in a public supermarket for me tomorrow, unprotected from #Coronavirus #Pray   \n",
       "31271                                               @IFPRI bringing some good news! \\r\\r\\n\"There is no shortage of staple food inventories and the stock-to-use ratio is close to normal!\" The question is will countries resort to trade restrictions? let's hope not\\r\\r\\n#food #foodsecurity #Covid_19 \\r\\r\\n\\r\\r\\nhttps://t   \n",
       "9363                                                                                                                       My idea to stop shops over changing...\\r\\r\\nArmed soldiers turn up and close their shop and confiscate All of their stock then drop it at food banks.\\r\\r\\n#coronavirus #corona #covid19 #COVID_19uk   \n",
       "26031                                                                                                                                                       Read 5 tips from a consumer psychologist on how brands should respond to #coronavirus. How many of these tactics do you have in place?\\r\\r\\nhttps://t.co/t1QozP9J0O   \n",
       "12485                                                                                                                                                          #notgoingout https://t.co/ACGwETNKIi shame these utter idiots are.. this isnt a joke. just look at the horrendous scene's in Italy do you really want that here?   \n",
       "25946                                                           How Retail Brands Have Been Disrupted, How eCommerce Can Help: Like most brand marketing categories, the effect of the COVID-19 pandemic on retail store traffic has been quick and profound,... https://t.co/HWXmGmwDh3 @mp_joemandese https://t.co/GR9v4gdPfv   \n",
       "32000                         I'm shocked by this line of cars waiting to get tested for Covid-19. Oh wait, I'm sorry... this the line at a food bank, because the economy is being ruined.\\r\\r\\n\\r\\r\\nHas anyone seen a line like this for Covid testing? No? Me neither.\\r\\r\\nhttps://t.co/781eq6bu4H https://t.co/x09G6HIkJv   \n",
       "25380                                    .@NYGovCuomo says #NY state has purchased 17,000 ventilators at $25,000/each from #China. Has a firm expectation that 2,500 of them will arrive over the next two weeks. \\r\\r\\nSays high demand from other countries & #US states has driven up prices & slowed delivery. #coronavirus   \n",
       "33038                                                                                                                                                     @Daniel_I_Berger @CompComSA please deal with @SPARInt and fine them 10% of their annual turnover. They are fixing the prices in South Africa since COVID-19 pandemic.   \n",
       "36899                                                                                                                                        If this is true then surely protective face masks for households would of been better than a government letter sent in the post .#StayHomeSaveLives \\r\\r\\n https://t.co/ffYV7lNeRK   \n",
       "22666                                                           @cpbbsrctc I never said that the supply is blocked. What I was trying to point out is they are selling it at an exorbitant prices. I had tweeted to @cpbbsrctc and @SarangiSudhansu to look into the matter. Now see, what is the situation. https://t.co/HNpDe   \n",
       "23221                                                                                             If I were to tweet that I ration my #toiletpaper to one square per bathroom trip, you might as well just assume I use my hand to wipe. ?\\r\\r\\n\\r\\r\\nCome on people. One square of toilet paper. NO ? #coronavirus #StayAtHome   \n",
       "10897                                                           @MattHancock @BorisJohnson @10DowningStreet @AWMurrison I am a supermarket worker in a major store, public are not respecting workers by distancing its like were invisible, please go further to protect us as we cant feed a nation if were sick too. #corona   \n",
       "19894  PM Imran Khan today announced Rs 1200 billion Economic Relief and stimulus Package.\\r\\r\\nThe package includes utility bills being deferred for lower income groups, principal & interest for businesses, & reduction in fuel prices.\\r\\r\\n#PMIK ????????\\r\\r\\n#CoronaVirusUpdates\\r\\r\\n#Covid_19 https://t.co/TP1u9FBR6d   \n",
       "40388                                                 At #DemystData, we have access to multiple datasets and data products that track consumer behavior, demographic profiles, and #geolocation details to help understand which sectors are and will be impacted the hardest. #pandemic #externaldata https://t.co/fR5qRZyufS   \n",
       "773                                                                                                                                                                                                    I may be naive, but are food hoarding and selling off stock really helping the situation? #CoronavirusPandemic #Covid_19   \n",
       "19586                                                                                                                                                          @Accelerationis1 Our retail store is closed but other business operations remain. See https://t.co/yg4EwBWBGo You can order online or by phone and we will ship.   \n",
       "16239                                       Idea when there is no sanitizer to be had. Keep water in your car; soap, towel. Pour water onto hands to wet them. Use soap and wash for 20 seconds. Pour water onto hands to rinse. Stand outside car to complete take. #CoronavirusPandemic #CoronavirusUSA #Coronavirus #COVID19   \n",
       "40723                                                                                                                                                                                                                           @OilandEnergy Now we just need to cut another 10-20 MMbbl/d to fix oil prices and cure COVID-19   \n",
       "1384                                                                                          \"The rapid shift to online ordering and delivery is having a powerful impact - and this could set the tone for consumer shifts in how we purchase cannabis moving forward.\" https://t.co/8pCMeqM5ot @Airfieldsupply @WallandBroad   \n",
       "39118                                                                The U.S. has been blowing it for years. A healthy capitalist society is based on real competition... and w/ so many corporations buying up everything & merging? The consumer is always fucked. https://t.co/X0lFOILbQO #capitalism #competition #monopoly   \n",
       "1408                                                                                                                                                                                         Please listen to my VERY candid conversation with an LA based ER doctor on the current state of #COVID2019 https://t.co/jt8cd52NRo   \n",
       "40995                                                                                                                                                                           PM Modi: No need to worry about essentials. India has enough stock of everything, from food to medicines\\r\\r\\n#coronavirus #COVID2019 #pmoindia   \n",
       "18132                                                                                                                                                                                   Boris Johnson: Stop In-Person Shopping all Grocery Shopping to Be Made Online To Contain Covid-19 https://t.co/PUhU94zFiA via @UKChange   \n",
       "18877                                                                                                                                                            Dr @Swamy39\\r\\r\\nHuge demand for food products in global markets due to COVID-19 outbreak, India can tap it: TPCI - The Economic Times https://t.co/mw1feiS7yH   \n",
       "409                                                                                                               Everyone is out there buying toilet paper and canned food but if I am stuck at home, I am going to stock up on things I love - wine, Epsom salts and chocolate! #coronavirus #COVID19 #icansurviveonchocolate   \n",
       "2625                                                                                                                                                                  There are people who stock up with food and toilet paper for the quarantine, #VanessaHudgens apparently stocked up with drugs.\\r\\r\\n#Coronavirus #COVID19   \n",
       "\n",
       "                sentiment  length  word fraction  token length  english score  \n",
       "33054  Extremely Positive     275            0.6      5.900000         1361.0  \n",
       "8396              Neutral      95            0.6      7.363636         1024.0  \n",
       "13225            Positive     170            0.6      6.125000         1154.0  \n",
       "24459             Neutral     255            0.6      5.400000         1342.0  \n",
       "13267  Extremely Negative     255            0.6      4.837209         1418.0  \n",
       "34018            Positive     280            0.6      6.263158         1296.0  \n",
       "3610              Neutral     140            0.6      6.421053         1070.0  \n",
       "7828              Neutral      95            0.6      6.384615         1013.0  \n",
       "34633  Extremely Negative     300            0.6      6.525000         1148.0  \n",
       "24838            Negative      85            0.6      4.062500         1365.0  \n",
       "34825             Neutral      95            0.6      7.727273         1107.0  \n",
       "6820             Positive     250            0.6      5.783784          949.0  \n",
       "23509            Positive     260            0.6      6.617647          949.0  \n",
       "515              Positive     280            0.6      5.825000         1225.0  \n",
       "28519  Extremely Negative     105            0.6      4.578947         1108.0  \n",
       "6379              Neutral     175            0.6      4.677419         1159.0  \n",
       "5917   Extremely Negative     145            0.6      7.588235          907.0  \n",
       "36644             Neutral     120            0.6      6.562500          961.0  \n",
       "36848            Negative     105            0.6      6.571429          970.0  \n",
       "23842  Extremely Positive     105            0.6      6.571429         1108.0  \n",
       "33530            Positive     225            0.6      5.277778         1062.0  \n",
       "33460            Negative     255            0.6      4.688889         1299.0  \n",
       "33298            Negative     210            0.6      7.115385         1224.0  \n",
       "28266            Positive     200            0.6      5.000000         1224.0  \n",
       "10216            Negative     250            0.6      5.972222         1194.0  \n",
       "10075            Negative     255            0.6      5.736842         1173.0  \n",
       "27825  Extremely Positive     260            0.6      5.692308         1313.0  \n",
       "30259             Neutral     125            0.6      6.500000         1142.0  \n",
       "10015            Negative     215            0.6      5.176471         1089.0  \n",
       "27718            Positive     285            0.6      5.853659         1397.0  \n",
       "27635            Negative     200            0.6      5.931034         1122.0  \n",
       "30482  Extremely Negative     255            0.6      5.918919         1122.0  \n",
       "30603            Negative     220            0.6      5.138889         1018.0  \n",
       "9663             Positive     290            0.6      5.181818         1208.0  \n",
       "31271            Positive     255            0.6      5.621622         1431.0  \n",
       "9363             Negative     190            0.6      5.032258         1176.0  \n",
       "26031             Neutral     160            0.6      5.360000         1071.0  \n",
       "12485  Extremely Negative     160            0.6      5.440000         1030.0  \n",
       "25946  Extremely Positive     255            0.6      6.757576         1058.0  \n",
       "32000  Extremely Negative     280            0.6      5.088889         1141.0  \n",
       "25380            Negative     275            0.6      5.204545         1178.0  \n",
       "33038            Positive     165            0.6      5.640000         1077.0  \n",
       "36899  Extremely Positive     175            0.6      5.880000         1158.0  \n",
       "22666            Negative     255            0.6      4.818182         1307.0  \n",
       "23221            Positive     215            0.6      4.146341         1157.0  \n",
       "10897            Positive     255            0.6      5.243902          965.0  \n",
       "19894            Positive     300            0.6      6.512821         1160.0  \n",
       "40388            Positive     265            0.6      6.600000          992.0  \n",
       "773              Positive     120            0.6      5.722222         1122.0  \n",
       "19586             Neutral     160            0.6      5.708333         1195.0  \n",
       "16239            Negative     275            0.6      5.000000         1083.0  \n",
       "40723            Negative      95            0.6      4.647059         1321.0  \n",
       "1384             Positive     225            0.6      5.848485         1232.0  \n",
       "39118            Negative     250            0.6      5.972222         1175.0  \n",
       "1408             Positive     130            0.6      5.550000         1015.0  \n",
       "40995  Extremely Negative     140            0.6      5.619048         1000.0  \n",
       "18132            Negative     135            0.6      6.555556         1064.0  \n",
       "18877            Positive     155            0.6      5.416667         1096.0  \n",
       "409    Extremely Positive     205            0.6      4.722222         1206.0  \n",
       "2625              Neutral     150            0.6      5.772727         1124.0  "
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = trim_dataset_by(\n",
    "    df, \n",
    "    ascending=True,\n",
    "    \n",
    "    # sort_by='length',\n",
    "    length=(60, None),\n",
    "    \n",
    "    sort_by='word fraction',\n",
    "    word_fraction=(0.60, None),\n",
    "    \n",
    "    # sort_by='token length',\n",
    "    token_length=(4, 8),\n",
    "    \n",
    "    # sort_by='english score',\n",
    "    english_score=(900, None),\n",
    ")\n",
    "a[:60]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "id": "26ed110f-721c-4923-8d1d-407093007176",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:39:44.152738Z",
     "iopub.status.busy": "2024-01-08T15:39:44.152237Z",
     "iopub.status.idle": "2024-01-08T15:40:02.526930Z",
     "shell.execute_reply": "2024-01-08T15:40:02.526196Z",
     "shell.execute_reply.started": "2024-01-08T15:39:44.152710Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time elapsed: 18s\n",
      "samples: 10809, similar samples detected: 191 (1.74%)\n"
     ]
    }
   ],
   "source": [
    "df = drop_similar(supersample(a, COVID19));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "id": "1b7323df-fa65-4d69-8a8f-b9da7e755fd9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:40:02.528650Z",
     "iopub.status.busy": "2024-01-08T15:40:02.528082Z",
     "iopub.status.idle": "2024-01-08T15:40:02.661748Z",
     "shell.execute_reply": "2024-01-08T15:40:02.661103Z",
     "shell.execute_reply.started": "2024-01-08T15:40:02.528623Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "string frequencies:\n",
      "coronavirus: 0.26\n",
      "covid: 0.47\n",
      "covid19: 0.05\n",
      "#coronavirus: 0.22\n",
      "#corona: 0.24\n",
      "#covid: 0.16\n",
      "#covid19: 0.05\n"
     ]
    }
   ],
   "source": [
    "df = a\n",
    "s = '#coronavirus'\n",
    "print('string frequencies:')\n",
    "\n",
    "s = 'coronavirus'; print(f'{s}: {string_frequency(df, s):.2f}')\n",
    "s = 'covid'; print(f'{s}: {string_frequency(df, s):.2f}')\n",
    "s = 'covid19'; print(f'{s}: {string_frequency(df, s):.2f}')\n",
    "\n",
    "s = '#coronavirus'; print(f'{s}: {string_frequency(df, s):.2f}')\n",
    "s = '#corona'; print(f'{s}: {string_frequency(df, s):.2f}')\n",
    "s = '#covid'; print(f'{s}: {string_frequency(df, s):.2f}')\n",
    "s = '#covid19'; print(f'{s}: {string_frequency(df, s):.2f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "id": "7fbe9381-6e0c-458d-aaaa-1857d84e5e8d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:40:02.663122Z",
     "iopub.status.busy": "2024-01-08T15:40:02.662688Z",
     "iopub.status.idle": "2024-01-08T15:40:02.768686Z",
     "shell.execute_reply": "2024-01-08T15:40:02.768081Z",
     "shell.execute_reply.started": "2024-01-08T15:40:02.663099Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\"#coronavirus\" new frequency: 0.04\n",
      "\"#covid19\" new frequency: 0.01\n"
     ]
    }
   ],
   "source": [
    "df = randomly_remove_substring(df, '#coronavirus', p=0.8, case=False)\n",
    "df = randomly_remove_substring(df, '#covid19', p=0.8, case=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "id": "71455408-c883-430b-8ee3-4b5c1e01b13d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:40:02.769680Z",
     "iopub.status.busy": "2024-01-08T15:40:02.769437Z",
     "iopub.status.idle": "2024-01-08T15:40:02.792092Z",
     "shell.execute_reply": "2024-01-08T15:40:02.791353Z",
     "shell.execute_reply.started": "2024-01-08T15:40:02.769658Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>length</th>\n",
       "      <th>word fraction</th>\n",
       "      <th>token length</th>\n",
       "      <th>english score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>33744</th>\n",
       "      <td>Thank you to our hidden heroes They continue to keep shelves stocked and packages delivered amid 19 From the grocery store clerks to truck drivers pharmacy staff and delivery drivers we thank you for helping our communities</td>\n",
       "      <td>Extremely Positive</td>\n",
       "      <td>223</td>\n",
       "      <td>0.829596</td>\n",
       "      <td>5.054054</td>\n",
       "      <td>1264.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>261</th>\n",
       "      <td>Social distancing to prevent the spread of coronavirus could have devastating effect on people with depression. https://t.co/rlJ1MaaxgI</td>\n",
       "      <td>Extremely Negative</td>\n",
       "      <td>135</td>\n",
       "      <td>0.703704</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>977.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26559</th>\n",
       "      <td>Global surveys of consumer sentiment during the coronavirus crisis</td>\n",
       "      <td>Extremely Negative</td>\n",
       "      <td>66</td>\n",
       "      <td>0.878788</td>\n",
       "      <td>6.444444</td>\n",
       "      <td>1054.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23751</th>\n",
       "      <td>I have family on SSI and Social Security. They have had to stock up on food, pay for people to bring them food, supplies, go get their meds. It is not fair to say they are not American because they are living in poverty or cannot go places because of #coronavirus #socialsecurity</td>\n",
       "      <td>Negative</td>\n",
       "      <td>279</td>\n",
       "      <td>0.702509</td>\n",
       "      <td>4.384615</td>\n",
       "      <td>1384.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9566</th>\n",
       "      <td>Great to see some positives coming from #coronavirus people are now wanting to try growing their own food #foodsecurity #growyourown Bunnings sold out of seedling stock and are trying to keep up. Hopefully people continue this way of life</td>\n",
       "      <td>Extremely Positive</td>\n",
       "      <td>238</td>\n",
       "      <td>0.680672</td>\n",
       "      <td>5.128205</td>\n",
       "      <td>1442.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                                                                                                          text  \\\n",
       "33744                                                          Thank you to our hidden heroes They continue to keep shelves stocked and packages delivered amid 19 From the grocery store clerks to truck drivers pharmacy staff and delivery drivers we thank you for helping our communities   \n",
       "261                                                                                                                                                    Social distancing to prevent the spread of coronavirus could have devastating effect on people with depression. https://t.co/rlJ1MaaxgI   \n",
       "26559                                                                                                                                                                                                                       Global surveys of consumer sentiment during the coronavirus crisis   \n",
       "23751  I have family on SSI and Social Security. They have had to stock up on food, pay for people to bring them food, supplies, go get their meds. It is not fair to say they are not American because they are living in poverty or cannot go places because of #coronavirus #socialsecurity   \n",
       "9566                                            Great to see some positives coming from #coronavirus people are now wanting to try growing their own food #foodsecurity #growyourown Bunnings sold out of seedling stock and are trying to keep up. Hopefully people continue this way of life   \n",
       "\n",
       "                sentiment  length  word fraction  token length  english score  \n",
       "33744  Extremely Positive     223       0.829596      5.054054         1264.0  \n",
       "261    Extremely Negative     135       0.703704      7.000000          977.0  \n",
       "26559  Extremely Negative      66       0.878788      6.444444         1054.0  \n",
       "23751            Negative     279       0.702509      4.384615         1384.0  \n",
       "9566   Extremely Positive     238       0.680672      5.128205         1442.0  "
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "id": "06347540-42ba-41e0-b071-3be37e2f61f5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:40:02.793350Z",
     "iopub.status.busy": "2024-01-08T15:40:02.792913Z",
     "iopub.status.idle": "2024-01-08T15:40:02.832221Z",
     "shell.execute_reply": "2024-01-08T15:40:02.831546Z",
     "shell.execute_reply.started": "2024-01-08T15:40:02.793328Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 3)"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = select_columns(df.copy(), ['text'])\n",
    "df['topic'] = 'covid19'\n",
    "df['id'] = 0\n",
    "\n",
    "df = sample(df, COVID19)\n",
    "dataset = concat(dataset, df)\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "73fc24fd-4a95-4247-ab02-536808f28ff5",
   "metadata": {},
   "source": [
    "# ChatGPT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "id": "458662a2-e732-42dd-b9a6-cf576229b4f1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:40:02.833503Z",
     "iopub.status.busy": "2024-01-08T15:40:02.833046Z",
     "iopub.status.idle": "2024-01-08T15:40:04.871651Z",
     "shell.execute_reply": "2024-01-08T15:40:04.870941Z",
     "shell.execute_reply.started": "2024-01-08T15:40:02.833481Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(500036, 4)"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = load_dataset('chatgpt.csv', source=True)\n",
    "df.rename(columns={'content': 'text', 'like_count': 'likes', 'retweet_count': 'retweets'}, inplace=True)\n",
    "df = select_columns(df, ['text', 'id', 'likes', 'retweets'])  # \n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "id": "0f26a7f4-1b5b-43df-8002-f554e34d148b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:40:04.873086Z",
     "iopub.status.busy": "2024-01-08T15:40:04.872866Z",
     "iopub.status.idle": "2024-01-08T15:40:13.202361Z",
     "shell.execute_reply": "2024-01-08T15:40:13.201799Z",
     "shell.execute_reply.started": "2024-01-08T15:40:04.873048Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(172439, 4)"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = df.iloc[:200000].copy()\n",
    "a = preprocessing_pipeline(a)\n",
    "a.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "id": "cc85a300-83c6-4b40-b603-657df28707a5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:40:13.204040Z",
     "iopub.status.busy": "2024-01-08T15:40:13.203544Z",
     "iopub.status.idle": "2024-01-08T15:40:17.347326Z",
     "shell.execute_reply": "2024-01-08T15:40:17.346650Z",
     "shell.execute_reply.started": "2024-01-08T15:40:13.204018Z"
    }
   },
   "outputs": [],
   "source": [
    "df = a;\n",
    "add_stats(df);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "id": "e1d5ec3b-9994-4869-ad6c-a03946142e43",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:40:17.348709Z",
     "iopub.status.busy": "2024-01-08T15:40:17.348471Z",
     "iopub.status.idle": "2024-01-08T15:40:17.428155Z",
     "shell.execute_reply": "2024-01-08T15:40:17.427507Z",
     "shell.execute_reply.started": "2024-01-08T15:40:17.348686Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "samples: 24866, reduction: 85.58%\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>likes</th>\n",
       "      <th>retweets</th>\n",
       "      <th>length</th>\n",
       "      <th>word fraction</th>\n",
       "      <th>token length</th>\n",
       "      <th>english score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1641213003260633088</td>\n",
       "      <td>Most people haven't heard of Chat GPT yet.\\nFirst, elite factions will decide which way to go on AI safety. Next they will push their agenda(s) on the public with misleading and oversimplified media presentations. Finally, the brainless Red and Blue camps will screech their lines</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>279</td>\n",
       "      <td>0.810036</td>\n",
       "      <td>5.086957</td>\n",
       "      <td>1296.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117992</th>\n",
       "      <td>1635801899348656129</td>\n",
       "      <td>if i use chat gpt and dall-e to create a completely original person's online presence and then post it all over the internet. what are the odds i find an almost identical person?</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>178</td>\n",
       "      <td>0.797753</td>\n",
       "      <td>4.424242</td>\n",
       "      <td>1220.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117985</th>\n",
       "      <td>1635802267491155969</td>\n",
       "      <td>@Cyb3rMonk I’ll be honest, I’m struggling to make it useful for me as well but maybe I’m overthinking it.\\n\\nI however had a lesser experienced colleague solve an on going technical issue by using Chat GPT. The resolution provided was actually very clearly outlined and spot-on.</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>276</td>\n",
       "      <td>0.768116</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>1311.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117982</th>\n",
       "      <td>1635802352069525505</td>\n",
       "      <td>#ChatGPT releasing ChatGPT 4 and allowing image inputs is a game changer. \\n\\nCan’t wait to be able to throw in financial statements, diagrams, etc. and have it summarize or explain.</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>180</td>\n",
       "      <td>0.744444</td>\n",
       "      <td>4.966667</td>\n",
       "      <td>1227.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117968</th>\n",
       "      <td>1635802927183921154</td>\n",
       "      <td>Today GPT-4 drastically cut the half-life of research based on LLMs.\\n\\nMy WIP on ChatGPT prompt engineering for long essays is rendered moot just 3mos after model early release.\\n\\nBeyond SOTA models themselves, what research of lasting value can be done?\\n\\nhttps://t.co/NZdWa6bbgV</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>277</td>\n",
       "      <td>0.718412</td>\n",
       "      <td>5.547619</td>\n",
       "      <td>1189.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117949</th>\n",
       "      <td>1635803283179843584</td>\n",
       "      <td>I spent some time to quickly scan the #gpt4 paper, I wonder how to implement it, is it possible to rebuild a #ChatGPT with a high performance computer?</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>151</td>\n",
       "      <td>0.715232</td>\n",
       "      <td>4.428571</td>\n",
       "      <td>1129.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117947</th>\n",
       "      <td>1635803312212815873</td>\n",
       "      <td>ChatGPT (GPT-4) still has significant limitations for research (especially legal). It tends to make major factual errors in citations and case summaries.\\n\\nCurrently, it's only useful for document drafting, but document drafting mostly uses templated letters anyway.</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>265</td>\n",
       "      <td>0.766038</td>\n",
       "      <td>6.162162</td>\n",
       "      <td>1374.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117935</th>\n",
       "      <td>1635803635186556930</td>\n",
       "      <td>I believe prompt engineering is something in the future.\\n\\nI can't believe that there is still people learning in an old fashion way rather than using ChatGPT while learning how to make better prompt.\\n\\nTon of SAAS is using GPT tech, and it is just the beginning of prompt engineer.</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>280</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>4.580000</td>\n",
       "      <td>1463.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117930</th>\n",
       "      <td>1635803831278641159</td>\n",
       "      <td>A new version of the AI system that powers the popular chatbot has better language skills, but is still biased, prone to fabrication, and can be abused.GPT-4 Will Make ChatGPT Smarter but Won't Fix Its Flaws https://t.co/fsd8kGWyBi</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>231</td>\n",
       "      <td>0.714286</td>\n",
       "      <td>5.270270</td>\n",
       "      <td>1206.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117994</th>\n",
       "      <td>1635801833334784000</td>\n",
       "      <td>@rSanti97 Every interpretive ask I've given chat gpt has been mediocre at best, and these are broad \"discuss the themes of ____\" I presume the more specific questions about exact phrasing would trip it up very quickly.</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>218</td>\n",
       "      <td>0.720183</td>\n",
       "      <td>4.918919</td>\n",
       "      <td>1117.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117929</th>\n",
       "      <td>1635803908458303489</td>\n",
       "      <td>@I4NFTS I use it daily for work. Instead of googling code, I ask chat gpt. It gives me answers though not 100% accurate but still beats going through links in google</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>165</td>\n",
       "      <td>0.733333</td>\n",
       "      <td>4.354839</td>\n",
       "      <td>1337.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117925</th>\n",
       "      <td>1635804026049826816</td>\n",
       "      <td>The company behind the #ChatGPT app that churns out essays, poems or computing code on command released Tuesday a long-awaited update of its artificial intelligence (AI) technology that it said would be safer and more accurate than its predecessor. https://t.co/z8tl239Nk3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>272</td>\n",
       "      <td>0.716912</td>\n",
       "      <td>5.825000</td>\n",
       "      <td>1218.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117919</th>\n",
       "      <td>1635804179204804608</td>\n",
       "      <td>Let's be honest, how amazing is Chat GPT's ability to help teachers analyse student responses and give personalised feedback, which lets students learn at their own pace and level. (3/5)</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>186</td>\n",
       "      <td>0.790323</td>\n",
       "      <td>5.233333</td>\n",
       "      <td>1116.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117905</th>\n",
       "      <td>1635804632206422016</td>\n",
       "      <td>@timwoneill its almost there, but still feels very business data focused rather than user data focused. Maybe one day we will have user based AI that can integrate with business specific AI (like Vertex) and general AI (like Chat GPT)</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>234</td>\n",
       "      <td>0.726496</td>\n",
       "      <td>4.875000</td>\n",
       "      <td>1333.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117901</th>\n",
       "      <td>1635804662061314050</td>\n",
       "      <td>@ScottAdamsSays I'm thinking that Chat GPT would be an excellent topic for Dilbert to deal with. You might have to create some new character to represent or administrator the program. \\nAnd I'm wondering if this AI wonder will be able to replace Dilbert or anyone else in the office??</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>283</td>\n",
       "      <td>0.752650</td>\n",
       "      <td>4.775510</td>\n",
       "      <td>1219.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117889</th>\n",
       "      <td>1635805145236643843</td>\n",
       "      <td>Chat GPT is a graspable example of how quickly technology progresses around us. Unreal 🤯</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>88</td>\n",
       "      <td>0.818182</td>\n",
       "      <td>4.933333</td>\n",
       "      <td>1154.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117874</th>\n",
       "      <td>1635805710322900992</td>\n",
       "      <td>ChatGPT and presumably GPT-4 still make very elementary mistakes in reasoning similar to the student (e.g. me when I was a student) who has completely memorized the textbook without understanding the content at a deeper level.</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>226</td>\n",
       "      <td>0.805310</td>\n",
       "      <td>5.305556</td>\n",
       "      <td>982.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117867</th>\n",
       "      <td>1635806036727590913</td>\n",
       "      <td>@NotABigJerk I hope these guys one day ask Chat GPT for medical advice and it gives them false informations</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>107</td>\n",
       "      <td>0.719626</td>\n",
       "      <td>4.684211</td>\n",
       "      <td>1272.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117864</th>\n",
       "      <td>1635806102469197825</td>\n",
       "      <td>I realize the development speed in ML has been fast for some time, but just thinking about how ChatGPT (GPT-3.5) was making splashes last Nov, and now you've got this... 🤯\\n\\nex. if you took time to give a talk or write a peer-reviewed paper on ChatGPT, it's already very dated.</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>276</td>\n",
       "      <td>0.739130</td>\n",
       "      <td>4.411765</td>\n",
       "      <td>1390.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117926</th>\n",
       "      <td>1635803951563153409</td>\n",
       "      <td>This is a thrown gauntlet if I ever saw one in the fresh field of AI chatbots. Claude outperformed both Bing and ChatGPT in every task I've given it so far - though GPT-4 might make this untrue. Whoever will appeal to more people will have more data to train. Rowdy times ahead...</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>280</td>\n",
       "      <td>0.778571</td>\n",
       "      <td>4.301887</td>\n",
       "      <td>1231.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117863</th>\n",
       "      <td>1635806209747083264</td>\n",
       "      <td>@I4NFTS yeah just an email and that's it...try to use chat gpt instead of googling something. You can ask it almost anything.</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>125</td>\n",
       "      <td>0.728000</td>\n",
       "      <td>4.727273</td>\n",
       "      <td>1362.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118014</th>\n",
       "      <td>1635800963499065344</td>\n",
       "      <td>GPT-4 has been announced! Hope this brings ChatGPT and OpenAI's other AI tools to be even better. Excited to see the feature of AI, and how it can make our lives easier and better!\\n\\n#gpt4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>187</td>\n",
       "      <td>0.748663</td>\n",
       "      <td>4.342857</td>\n",
       "      <td>1384.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118033</th>\n",
       "      <td>1635799906282778625</td>\n",
       "      <td>Chat GPT is refusing to draft all the funny tweets I’m asking it to write. Claiming they aren’t respectful and shit</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>115</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>4.523810</td>\n",
       "      <td>1264.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118183</th>\n",
       "      <td>1635794923868831746</td>\n",
       "      <td>Chat gpt moderates it’s language based on its perceived audience and what role it is supposed to play. As more of it and it will deliver better results.</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>152</td>\n",
       "      <td>0.802632</td>\n",
       "      <td>4.464286</td>\n",
       "      <td>1613.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118169</th>\n",
       "      <td>1635795091347623936</td>\n",
       "      <td>why can't #chatgpt feedback to me a list of words that contain 3 bs? it keeps messing up again and again</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>104</td>\n",
       "      <td>0.701923</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>1368.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118162</th>\n",
       "      <td>1635795204883066883</td>\n",
       "      <td>#chatgpt is such a good tool for the social media industry!\\n\\nHelping with market research, content ideas and even captions on socials 🙌🏼🙌🏼</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>138</td>\n",
       "      <td>0.731884</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>1118.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118154</th>\n",
       "      <td>1635795400132292615</td>\n",
       "      <td>Great news everyone, you don't need Microsoft Edge to use GPT in Bing's chat. It's as easy as changing your user agent to fool it into thinking its Edge, and it will work just the same in Chrome. \\n\\n@SebastianKamph</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>213</td>\n",
       "      <td>0.708920</td>\n",
       "      <td>4.435897</td>\n",
       "      <td>1281.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118145</th>\n",
       "      <td>1635795748951580672</td>\n",
       "      <td>ChatGPT on GPT-3 was the Talented Mr Ripley referencing the Library of Babel.\\n\\nCan't wait to find time to give GPT-4 a spin.</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>124</td>\n",
       "      <td>0.758065</td>\n",
       "      <td>4.391304</td>\n",
       "      <td>1015.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118143</th>\n",
       "      <td>1635795851808473088</td>\n",
       "      <td>Perhaps next who is have account twitter must be have knowledge github chat GPT AI etc..\\n\\nPerhaps if you can't do high program computer next you can't open or play twitter again \\n\\nPerhaps FB WA line instagram next too... world welcome to generation high program computer 🙄🙄☺☺</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>275</td>\n",
       "      <td>0.781818</td>\n",
       "      <td>4.808511</td>\n",
       "      <td>1121.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118141</th>\n",
       "      <td>1635795908020383744</td>\n",
       "      <td>hey, #chatgpt! how many unemployed people are necessary to start a global revolution?</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>85</td>\n",
       "      <td>0.729412</td>\n",
       "      <td>5.615385</td>\n",
       "      <td>1298.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118136</th>\n",
       "      <td>1635796047372115968</td>\n",
       "      <td>@jordanbpeterson I think that is a shame. But of course, it will write what's in its repertoire. The fact that one man is responsible for so much discord says more about us, and poor chat gpt can only reflect the emotions of the programmers. That is what is frightening.</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>270</td>\n",
       "      <td>0.737037</td>\n",
       "      <td>4.530612</td>\n",
       "      <td>1322.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118030</th>\n",
       "      <td>1635800169743540224</td>\n",
       "      <td>You can use ChatGPT or GPT-4 for your business.. For example, it can help you create a website.</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>95</td>\n",
       "      <td>0.757895</td>\n",
       "      <td>4.333333</td>\n",
       "      <td>1467.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118123</th>\n",
       "      <td>1635796777222774784</td>\n",
       "      <td>OpenAI today unveiled GPT-4 a major upgrade from ChatGPT-3 which the company introduced only 3 months ago and some have called the most innovating Internet technology since the invention of the Web browser back in 1990\\n\\nIt's free to use and opens in any web browser https://t.co/7adI22r5Dn</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>289</td>\n",
       "      <td>0.723183</td>\n",
       "      <td>5.148936</td>\n",
       "      <td>1240.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118089</th>\n",
       "      <td>1635797753065242628</td>\n",
       "      <td>@bendreyfuss bing AI is not chatgpt. bing runs on GPT-4, built by openAI, and has some internet access. chatgpt is also built by openAI on GPT-3 but has no internet access</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>171</td>\n",
       "      <td>0.707602</td>\n",
       "      <td>4.548387</td>\n",
       "      <td>1137.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118087</th>\n",
       "      <td>1635797800955805696</td>\n",
       "      <td>there are some brilliant people / geniuses as we speak developing the next multi million dollar and billion dollar companies with AI - chat gpt and other tools like it... if you have a great idea, id jump on it, and sell it to a big company that can implement it better than you</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>278</td>\n",
       "      <td>0.784173</td>\n",
       "      <td>4.166667</td>\n",
       "      <td>1509.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118085</th>\n",
       "      <td>1635797821667569665</td>\n",
       "      <td>🤣😂🤣😂 how is anyone going to justify believing chat GPT is anything other than a cia psyop?</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>90</td>\n",
       "      <td>0.766667</td>\n",
       "      <td>4.352941</td>\n",
       "      <td>1409.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118077</th>\n",
       "      <td>1635798106754240512</td>\n",
       "      <td>gpt-4 seems very impressive, if the version of it on ChatGPT is an improvement over what bing is using rn I'll pay for it, I've already been using bing as my research assistant / debugging buddy</td>\n",
       "      <td>15.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>194</td>\n",
       "      <td>0.783505</td>\n",
       "      <td>4.416667</td>\n",
       "      <td>1446.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118066</th>\n",
       "      <td>1635798633839837185</td>\n",
       "      <td>Ok this feels a lot less \"oh, us too!\" As a response to Chat GPT 's omnipresence. \\n\\nInteresting and kinda scary at the same time.</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>129</td>\n",
       "      <td>0.720930</td>\n",
       "      <td>4.120000</td>\n",
       "      <td>1091.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118064</th>\n",
       "      <td>1635798710016933889</td>\n",
       "      <td>South park just introduced Chat gpt which shown in the end works like a family cutaway. South park can now throw hands with family guy.</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>135</td>\n",
       "      <td>0.807407</td>\n",
       "      <td>4.440000</td>\n",
       "      <td>1329.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118036</th>\n",
       "      <td>1635799781724368901</td>\n",
       "      <td>Chat GPT-4 is going to save me so much time across both my existing and new career paths.</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>89</td>\n",
       "      <td>0.775281</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>1447.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118034</th>\n",
       "      <td>1635799889316659200</td>\n",
       "      <td>Just used Ai again. Solving so many of questions quicker than customer support ever could. Is this the end to customer support? Very likely and very scary knowing how many jobs will be lost in that sector. #ChatGPT</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>214</td>\n",
       "      <td>0.771028</td>\n",
       "      <td>4.657895</td>\n",
       "      <td>1336.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118107</th>\n",
       "      <td>1635797223433969664</td>\n",
       "      <td>I spoke with a student this week who asked me how ChatGPT was affecting the workforce, applicants and new recruits when everyone theoretically has access to any information or processing power they can imagine. \\n\\nI’m excited for the changes new GPT-4 will bring that’s for sure.</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>278</td>\n",
       "      <td>0.805755</td>\n",
       "      <td>5.021739</td>\n",
       "      <td>1340.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117827</th>\n",
       "      <td>1635807557175345152</td>\n",
       "      <td>@seanofthaadead Yeah he has chat GPT integrated with gong it’s crazy. Just takes the action items from the gong call and puts together the follow up email. But agreed, that would be an insanely helpful use case. Someone will figure out how to do it.</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>249</td>\n",
       "      <td>0.738956</td>\n",
       "      <td>4.555556</td>\n",
       "      <td>1376.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117820</th>\n",
       "      <td>1635807900630228993</td>\n",
       "      <td>I need to tweet something to change my algorithm because my suggested TL is all fucked and I am sick of muting accounts..\\n\\nSo here goes:\\n\\nChat GPT\\nBoobs\\nHorror movies\\nGoth girls\\nMetal chicks\\nWasabi\\nWriting\\nSignature trampoline techniques\\nOnomatopoeia\\nEgg nog distilleries</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>271</td>\n",
       "      <td>0.822878</td>\n",
       "      <td>5.136364</td>\n",
       "      <td>1085.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117816</th>\n",
       "      <td>1635808030053769220</td>\n",
       "      <td>Anthropic, an artificial intelligence company backed by Alphabet Inc, releases a large language model that competes directly with offerings from Microsoft Corp-backed OpenAI, the creator of #ChatGPT.\\n\\nhttps://t.co/iD411LPffy</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>224</td>\n",
       "      <td>0.714286</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>962.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117658</th>\n",
       "      <td>1635813796764700673</td>\n",
       "      <td>ChatGPT now has a GPT-4 for Plus subscribers! implementation that works so much better than the original. Knows how to give you what you ask for without trying to deceive you into accepting a substandard generated text response.</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>228</td>\n",
       "      <td>0.815789</td>\n",
       "      <td>5.026316</td>\n",
       "      <td>1307.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117645</th>\n",
       "      <td>1635814006312116225</td>\n",
       "      <td>6. OpenAI has a new chat completions playground where GPT-4 and GPT-3.5 have the same API.\\n\\n* The playground allows users to provide any instruction they want to steer the model.</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>178</td>\n",
       "      <td>0.764045</td>\n",
       "      <td>4.741935</td>\n",
       "      <td>1278.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117618</th>\n",
       "      <td>1635815128070340610</td>\n",
       "      <td>- GPT-4 is 82% less likely to respond to requests for disallowed content and 40% more likely to produce factual responses than GPT-3.5\\n\\n- GPT-4 outperforms #ChatGPT by scoring in higher approximate percentiles among test-takers.\\n\\n- Bing chat will now run on #GPT4.</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>264</td>\n",
       "      <td>0.708333</td>\n",
       "      <td>5.116279</td>\n",
       "      <td>1080.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117609</th>\n",
       "      <td>1635815444052418561</td>\n",
       "      <td>Chat GPT discourse reminds me of the NYT cooking video from a few months ago where Priya had an LLM come up with a thanksgiving recipe, only for it to spit out something completely bizarre and fucked up. LLMs can read every recipe ever written, but they can't taste food</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>270</td>\n",
       "      <td>0.803704</td>\n",
       "      <td>4.420000</td>\n",
       "      <td>1241.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117608</th>\n",
       "      <td>1635815457746812928</td>\n",
       "      <td>Holding out for the OpenAI Bundle that includes Github Copilot &amp; Chat GPT Plus for the low price of $16.99/mo (no ads)</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>118</td>\n",
       "      <td>0.703390</td>\n",
       "      <td>4.409091</td>\n",
       "      <td>1232.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117593</th>\n",
       "      <td>1635815954335531008</td>\n",
       "      <td>Chat GPT is actually making me pretty happy. It's removing all the friction from getting started on specific scripting stuff in Godot, but is wrong enough that I still engage and customise it all. Real AI is here. I just felt a glimmer of the potential joy this could bring.</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>274</td>\n",
       "      <td>0.799270</td>\n",
       "      <td>4.500000</td>\n",
       "      <td>1304.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117592</th>\n",
       "      <td>1635815959448682498</td>\n",
       "      <td>Chat GPT is woke. The Trump one is reminiscent of my liberal friends beliefs of Trump as well. 😆 It's a propaganda machine.</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>123</td>\n",
       "      <td>0.780488</td>\n",
       "      <td>4.391304</td>\n",
       "      <td>1127.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117591</th>\n",
       "      <td>1635815978117332992</td>\n",
       "      <td>OpenAI is unveiling GPT-4, the much-anticipated successor to an artificial intelligence tool that spawned viral services like ChatGPT and the Dall-E image-creation program https://t.co/2h3bmz9KGp</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>195</td>\n",
       "      <td>0.733333</td>\n",
       "      <td>7.166667</td>\n",
       "      <td>948.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117673</th>\n",
       "      <td>1635813064061640712</td>\n",
       "      <td>Playing with GPT-4 this evening via ChatGPT Plus. Looks good, but not yet seeing big differences. Once get access to API, will run a few thousand calls through it for an A/B test. Do wish that they'd update the training set beyond September 2021, however.</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>255</td>\n",
       "      <td>0.768627</td>\n",
       "      <td>4.688889</td>\n",
       "      <td>1362.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117586</th>\n",
       "      <td>1635816234590810114</td>\n",
       "      <td>LLM inflation is here GPT-5 and GPT-6 shipping soon to 0 fanfare. Chat usage already dropped</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>92</td>\n",
       "      <td>0.771739</td>\n",
       "      <td>4.812500</td>\n",
       "      <td>1024.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117552</th>\n",
       "      <td>1635817502134648832</td>\n",
       "      <td>@Foobazzler Ah haha yeah not yet. I input the script I made to make this art into ChatGPT and asked for a variation on the input and it barfed 🥲 perhaps with GPT-4 once that is generally available.</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>197</td>\n",
       "      <td>0.730964</td>\n",
       "      <td>4.210526</td>\n",
       "      <td>1174.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117551</th>\n",
       "      <td>1635817516747325440</td>\n",
       "      <td>Chat GPT was able to correctly tell me how in specifically Minecraft 1.12.2 to determine if the nearest players inventory was empty AI is the future</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>148</td>\n",
       "      <td>0.790541</td>\n",
       "      <td>4.730769</td>\n",
       "      <td>1312.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117531</th>\n",
       "      <td>1635818230576197634</td>\n",
       "      <td>Interesting that GPT-4 is also based on the same chat API. I suspect that this is the interface that all these models will converge on. Chat allows easy separation of system &amp; user instructions, which will ultimately make these models a lot more controllable in the long term.</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>276</td>\n",
       "      <td>0.804348</td>\n",
       "      <td>4.770833</td>\n",
       "      <td>1347.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117526</th>\n",
       "      <td>1635818411207933954</td>\n",
       "      <td>ChatGPT caught tech and the world by surprise. \\n\\nAnd made GPT-4 the most anticipated software release of all time.</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>114</td>\n",
       "      <td>0.789474</td>\n",
       "      <td>4.947368</td>\n",
       "      <td>1484.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117524</th>\n",
       "      <td>1635818560852312065</td>\n",
       "      <td>GPT-4 is finally officially released by OpenAI. A 500 times more intelligent system than the current language models like ChatGPT. You can access it via ChatGPT but... only accessible to ChatGPT Plus users. Means, pay dollars and get into the world's most weirdest tool. 🫡 #GPT4 https://t.co/bmOG9SbHtq</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>302</td>\n",
       "      <td>0.705298</td>\n",
       "      <td>5.446809</td>\n",
       "      <td>1099.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         id  \\\n",
       "4       1641213003260633088   \n",
       "117992  1635801899348656129   \n",
       "117985  1635802267491155969   \n",
       "117982  1635802352069525505   \n",
       "117968  1635802927183921154   \n",
       "117949  1635803283179843584   \n",
       "117947  1635803312212815873   \n",
       "117935  1635803635186556930   \n",
       "117930  1635803831278641159   \n",
       "117994  1635801833334784000   \n",
       "117929  1635803908458303489   \n",
       "117925  1635804026049826816   \n",
       "117919  1635804179204804608   \n",
       "117905  1635804632206422016   \n",
       "117901  1635804662061314050   \n",
       "117889  1635805145236643843   \n",
       "117874  1635805710322900992   \n",
       "117867  1635806036727590913   \n",
       "117864  1635806102469197825   \n",
       "117926  1635803951563153409   \n",
       "117863  1635806209747083264   \n",
       "118014  1635800963499065344   \n",
       "118033  1635799906282778625   \n",
       "118183  1635794923868831746   \n",
       "118169  1635795091347623936   \n",
       "118162  1635795204883066883   \n",
       "118154  1635795400132292615   \n",
       "118145  1635795748951580672   \n",
       "118143  1635795851808473088   \n",
       "118141  1635795908020383744   \n",
       "118136  1635796047372115968   \n",
       "118030  1635800169743540224   \n",
       "118123  1635796777222774784   \n",
       "118089  1635797753065242628   \n",
       "118087  1635797800955805696   \n",
       "118085  1635797821667569665   \n",
       "118077  1635798106754240512   \n",
       "118066  1635798633839837185   \n",
       "118064  1635798710016933889   \n",
       "118036  1635799781724368901   \n",
       "118034  1635799889316659200   \n",
       "118107  1635797223433969664   \n",
       "117827  1635807557175345152   \n",
       "117820  1635807900630228993   \n",
       "117816  1635808030053769220   \n",
       "117658  1635813796764700673   \n",
       "117645  1635814006312116225   \n",
       "117618  1635815128070340610   \n",
       "117609  1635815444052418561   \n",
       "117608  1635815457746812928   \n",
       "117593  1635815954335531008   \n",
       "117592  1635815959448682498   \n",
       "117591  1635815978117332992   \n",
       "117673  1635813064061640712   \n",
       "117586  1635816234590810114   \n",
       "117552  1635817502134648832   \n",
       "117551  1635817516747325440   \n",
       "117531  1635818230576197634   \n",
       "117526  1635818411207933954   \n",
       "117524  1635818560852312065   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                  text  \\\n",
       "4                             Most people haven't heard of Chat GPT yet.\\nFirst, elite factions will decide which way to go on AI safety. Next they will push their agenda(s) on the public with misleading and oversimplified media presentations. Finally, the brainless Red and Blue camps will screech their lines   \n",
       "117992                                                                                                                              if i use chat gpt and dall-e to create a completely original person's online presence and then post it all over the internet. what are the odds i find an almost identical person?   \n",
       "117985                          @Cyb3rMonk I’ll be honest, I’m struggling to make it useful for me as well but maybe I’m overthinking it.\\n\\nI however had a lesser experienced colleague solve an on going technical issue by using Chat GPT. The resolution provided was actually very clearly outlined and spot-on.   \n",
       "117982                                                                                                                          #ChatGPT releasing ChatGPT 4 and allowing image inputs is a game changer. \\n\\nCan’t wait to be able to throw in financial statements, diagrams, etc. and have it summarize or explain.   \n",
       "117968                     Today GPT-4 drastically cut the half-life of research based on LLMs.\\n\\nMy WIP on ChatGPT prompt engineering for long essays is rendered moot just 3mos after model early release.\\n\\nBeyond SOTA models themselves, what research of lasting value can be done?\\n\\nhttps://t.co/NZdWa6bbgV   \n",
       "117949                                                                                                                                                         I spent some time to quickly scan the #gpt4 paper, I wonder how to implement it, is it possible to rebuild a #ChatGPT with a high performance computer?   \n",
       "117947                                     ChatGPT (GPT-4) still has significant limitations for research (especially legal). It tends to make major factual errors in citations and case summaries.\\n\\nCurrently, it's only useful for document drafting, but document drafting mostly uses templated letters anyway.   \n",
       "117935                    I believe prompt engineering is something in the future.\\n\\nI can't believe that there is still people learning in an old fashion way rather than using ChatGPT while learning how to make better prompt.\\n\\nTon of SAAS is using GPT tech, and it is just the beginning of prompt engineer.   \n",
       "117930                                                                         A new version of the AI system that powers the popular chatbot has better language skills, but is still biased, prone to fabrication, and can be abused.GPT-4 Will Make ChatGPT Smarter but Won't Fix Its Flaws https://t.co/fsd8kGWyBi   \n",
       "117994                                                                                      @rSanti97 Every interpretive ask I've given chat gpt has been mediocre at best, and these are broad \"discuss the themes of ____\" I presume the more specific questions about exact phrasing would trip it up very quickly.   \n",
       "117929                                                                                                                                           @I4NFTS I use it daily for work. Instead of googling code, I ask chat gpt. It gives me answers though not 100% accurate but still beats going through links in google   \n",
       "117925                                The company behind the #ChatGPT app that churns out essays, poems or computing code on command released Tuesday a long-awaited update of its artificial intelligence (AI) technology that it said would be safer and more accurate than its predecessor. https://t.co/z8tl239Nk3   \n",
       "117919                                                                                                                      Let's be honest, how amazing is Chat GPT's ability to help teachers analyse student responses and give personalised feedback, which lets students learn at their own pace and level. (3/5)   \n",
       "117905                                                                      @timwoneill its almost there, but still feels very business data focused rather than user data focused. Maybe one day we will have user based AI that can integrate with business specific AI (like Vertex) and general AI (like Chat GPT)   \n",
       "117901                    @ScottAdamsSays I'm thinking that Chat GPT would be an excellent topic for Dilbert to deal with. You might have to create some new character to represent or administrator the program. \\nAnd I'm wondering if this AI wonder will be able to replace Dilbert or anyone else in the office??   \n",
       "117889                                                                                                                                                                                                                        Chat GPT is a graspable example of how quickly technology progresses around us. Unreal 🤯   \n",
       "117874                                                                              ChatGPT and presumably GPT-4 still make very elementary mistakes in reasoning similar to the student (e.g. me when I was a student) who has completely memorized the textbook without understanding the content at a deeper level.   \n",
       "117867                                                                                                                                                                                                     @NotABigJerk I hope these guys one day ask Chat GPT for medical advice and it gives them false informations   \n",
       "117864                          I realize the development speed in ML has been fast for some time, but just thinking about how ChatGPT (GPT-3.5) was making splashes last Nov, and now you've got this... 🤯\\n\\nex. if you took time to give a talk or write a peer-reviewed paper on ChatGPT, it's already very dated.   \n",
       "117926                        This is a thrown gauntlet if I ever saw one in the fresh field of AI chatbots. Claude outperformed both Bing and ChatGPT in every task I've given it so far - though GPT-4 might make this untrue. Whoever will appeal to more people will have more data to train. Rowdy times ahead...   \n",
       "117863                                                                                                                                                                                   @I4NFTS yeah just an email and that's it...try to use chat gpt instead of googling something. You can ask it almost anything.   \n",
       "118014                                                                                                                   GPT-4 has been announced! Hope this brings ChatGPT and OpenAI's other AI tools to be even better. Excited to see the feature of AI, and how it can make our lives easier and better!\\n\\n#gpt4   \n",
       "118033                                                                                                                                                                                             Chat GPT is refusing to draft all the funny tweets I’m asking it to write. Claiming they aren’t respectful and shit   \n",
       "118183                                                                                                                                                        Chat gpt moderates it’s language based on its perceived audience and what role it is supposed to play. As more of it and it will deliver better results.   \n",
       "118169                                                                                                                                                                                                        why can't #chatgpt feedback to me a list of words that contain 3 bs? it keeps messing up again and again   \n",
       "118162                                                                                                                                                                    #chatgpt is such a good tool for the social media industry!\\n\\nHelping with market research, content ideas and even captions on socials 🙌🏼🙌🏼   \n",
       "118154                                                                                         Great news everyone, you don't need Microsoft Edge to use GPT in Bing's chat. It's as easy as changing your user agent to fool it into thinking its Edge, and it will work just the same in Chrome. \\n\\n@SebastianKamph   \n",
       "118145                                                                                                                                                                                  ChatGPT on GPT-3 was the Talented Mr Ripley referencing the Library of Babel.\\n\\nCan't wait to find time to give GPT-4 a spin.   \n",
       "118143                         Perhaps next who is have account twitter must be have knowledge github chat GPT AI etc..\\n\\nPerhaps if you can't do high program computer next you can't open or play twitter again \\n\\nPerhaps FB WA line instagram next too... world welcome to generation high program computer 🙄🙄☺☺   \n",
       "118141                                                                                                                                                                                                                           hey, #chatgpt! how many unemployed people are necessary to start a global revolution?   \n",
       "118136                                  @jordanbpeterson I think that is a shame. But of course, it will write what's in its repertoire. The fact that one man is responsible for so much discord says more about us, and poor chat gpt can only reflect the emotions of the programmers. That is what is frightening.   \n",
       "118030                                                                                                                                                                                                                 You can use ChatGPT or GPT-4 for your business.. For example, it can help you create a website.   \n",
       "118123             OpenAI today unveiled GPT-4 a major upgrade from ChatGPT-3 which the company introduced only 3 months ago and some have called the most innovating Internet technology since the invention of the Web browser back in 1990\\n\\nIt's free to use and opens in any web browser https://t.co/7adI22r5Dn   \n",
       "118089                                                                                                                                     @bendreyfuss bing AI is not chatgpt. bing runs on GPT-4, built by openAI, and has some internet access. chatgpt is also built by openAI on GPT-3 but has no internet access   \n",
       "118087                          there are some brilliant people / geniuses as we speak developing the next multi million dollar and billion dollar companies with AI - chat gpt and other tools like it... if you have a great idea, id jump on it, and sell it to a big company that can implement it better than you   \n",
       "118085                                                                                                                                                                                                                      🤣😂🤣😂 how is anyone going to justify believing chat GPT is anything other than a cia psyop?   \n",
       "118077                                                                                                              gpt-4 seems very impressive, if the version of it on ChatGPT is an improvement over what bing is using rn I'll pay for it, I've already been using bing as my research assistant / debugging buddy   \n",
       "118066                                                                                                                                                                             Ok this feels a lot less \"oh, us too!\" As a response to Chat GPT 's omnipresence. \\n\\nInteresting and kinda scary at the same time.   \n",
       "118064                                                                                                                                                                         South park just introduced Chat gpt which shown in the end works like a family cutaway. South park can now throw hands with family guy.   \n",
       "118036                                                                                                                                                                                                                       Chat GPT-4 is going to save me so much time across both my existing and new career paths.   \n",
       "118034                                                                                          Just used Ai again. Solving so many of questions quicker than customer support ever could. Is this the end to customer support? Very likely and very scary knowing how many jobs will be lost in that sector. #ChatGPT   \n",
       "118107                        I spoke with a student this week who asked me how ChatGPT was affecting the workforce, applicants and new recruits when everyone theoretically has access to any information or processing power they can imagine. \\n\\nI’m excited for the changes new GPT-4 will bring that’s for sure.   \n",
       "117827                                                       @seanofthaadead Yeah he has chat GPT integrated with gong it’s crazy. Just takes the action items from the gong call and puts together the follow up email. But agreed, that would be an insanely helpful use case. Someone will figure out how to do it.   \n",
       "117820                    I need to tweet something to change my algorithm because my suggested TL is all fucked and I am sick of muting accounts..\\n\\nSo here goes:\\n\\nChat GPT\\nBoobs\\nHorror movies\\nGoth girls\\nMetal chicks\\nWasabi\\nWriting\\nSignature trampoline techniques\\nOnomatopoeia\\nEgg nog distilleries   \n",
       "117816                                                                              Anthropic, an artificial intelligence company backed by Alphabet Inc, releases a large language model that competes directly with offerings from Microsoft Corp-backed OpenAI, the creator of #ChatGPT.\\n\\nhttps://t.co/iD411LPffy   \n",
       "117658                                                                            ChatGPT now has a GPT-4 for Plus subscribers! implementation that works so much better than the original. Knows how to give you what you ask for without trying to deceive you into accepting a substandard generated text response.   \n",
       "117645                                                                                                                            6. OpenAI has a new chat completions playground where GPT-4 and GPT-3.5 have the same API.\\n\\n* The playground allows users to provide any instruction they want to steer the model.   \n",
       "117618                                    - GPT-4 is 82% less likely to respond to requests for disallowed content and 40% more likely to produce factual responses than GPT-3.5\\n\\n- GPT-4 outperforms #ChatGPT by scoring in higher approximate percentiles among test-takers.\\n\\n- Bing chat will now run on #GPT4.   \n",
       "117609                                  Chat GPT discourse reminds me of the NYT cooking video from a few months ago where Priya had an LLM come up with a thanksgiving recipe, only for it to spit out something completely bizarre and fucked up. LLMs can read every recipe ever written, but they can't taste food   \n",
       "117608                                                                                                                                                                                          Holding out for the OpenAI Bundle that includes Github Copilot & Chat GPT Plus for the low price of $16.99/mo (no ads)   \n",
       "117593                              Chat GPT is actually making me pretty happy. It's removing all the friction from getting started on specific scripting stuff in Godot, but is wrong enough that I still engage and customise it all. Real AI is here. I just felt a glimmer of the potential joy this could bring.   \n",
       "117592                                                                                                                                                                                     Chat GPT is woke. The Trump one is reminiscent of my liberal friends beliefs of Trump as well. 😆 It's a propaganda machine.   \n",
       "117591                                                                                                             OpenAI is unveiling GPT-4, the much-anticipated successor to an artificial intelligence tool that spawned viral services like ChatGPT and the Dall-E image-creation program https://t.co/2h3bmz9KGp   \n",
       "117673                                                 Playing with GPT-4 this evening via ChatGPT Plus. Looks good, but not yet seeing big differences. Once get access to API, will run a few thousand calls through it for an A/B test. Do wish that they'd update the training set beyond September 2021, however.   \n",
       "117586                                                                                                                                                                                                                    LLM inflation is here GPT-5 and GPT-6 shipping soon to 0 fanfare. Chat usage already dropped   \n",
       "117552                                                                                                           @Foobazzler Ah haha yeah not yet. I input the script I made to make this art into ChatGPT and asked for a variation on the input and it barfed 🥲 perhaps with GPT-4 once that is generally available.   \n",
       "117551                                                                                                                                                            Chat GPT was able to correctly tell me how in specifically Minecraft 1.12.2 to determine if the nearest players inventory was empty AI is the future   \n",
       "117531                            Interesting that GPT-4 is also based on the same chat API. I suspect that this is the interface that all these models will converge on. Chat allows easy separation of system & user instructions, which will ultimately make these models a lot more controllable in the long term.   \n",
       "117526                                                                                                                                                                                            ChatGPT caught tech and the world by surprise. \\n\\nAnd made GPT-4 the most anticipated software release of all time.   \n",
       "117524  GPT-4 is finally officially released by OpenAI. A 500 times more intelligent system than the current language models like ChatGPT. You can access it via ChatGPT but... only accessible to ChatGPT Plus users. Means, pay dollars and get into the world's most weirdest tool. 🫡 #GPT4 https://t.co/bmOG9SbHtq   \n",
       "\n",
       "        likes  retweets  length  word fraction  token length  english score  \n",
       "4         0.0       0.0     279       0.810036      5.086957         1296.0  \n",
       "117992    1.0       0.0     178       0.797753      4.424242         1220.0  \n",
       "117985    1.0       0.0     276       0.768116      5.000000         1311.0  \n",
       "117982    0.0       0.0     180       0.744444      4.966667         1227.0  \n",
       "117968    0.0       0.0     277       0.718412      5.547619         1189.0  \n",
       "117949    0.0       0.0     151       0.715232      4.428571         1129.0  \n",
       "117947    7.0       0.0     265       0.766038      6.162162         1374.0  \n",
       "117935    0.0       0.0     280       0.800000      4.580000         1463.0  \n",
       "117930    0.0       0.0     231       0.714286      5.270270         1206.0  \n",
       "117994    0.0       0.0     218       0.720183      4.918919         1117.0  \n",
       "117929    0.0       0.0     165       0.733333      4.354839         1337.0  \n",
       "117925    1.0       0.0     272       0.716912      5.825000         1218.0  \n",
       "117919    0.0       0.0     186       0.790323      5.233333         1116.0  \n",
       "117905    1.0       0.0     234       0.726496      4.875000         1333.0  \n",
       "117901    0.0       0.0     283       0.752650      4.775510         1219.0  \n",
       "117889    0.0       0.0      88       0.818182      4.933333         1154.0  \n",
       "117874    0.0       0.0     226       0.805310      5.305556          982.0  \n",
       "117867    0.0       0.0     107       0.719626      4.684211         1272.0  \n",
       "117864    4.0       0.0     276       0.739130      4.411765         1390.0  \n",
       "117926    5.0       0.0     280       0.778571      4.301887         1231.0  \n",
       "117863    1.0       0.0     125       0.728000      4.727273         1362.0  \n",
       "118014    0.0       0.0     187       0.748663      4.342857         1384.0  \n",
       "118033    1.0       0.0     115       0.800000      4.523810         1264.0  \n",
       "118183    3.0       0.0     152       0.802632      4.464286         1613.0  \n",
       "118169    0.0       0.0     104       0.701923      4.000000         1368.0  \n",
       "118162    0.0       0.0     138       0.731884      5.000000         1118.0  \n",
       "118154    0.0       0.0     213       0.708920      4.435897         1281.0  \n",
       "118145    0.0       0.0     124       0.758065      4.391304         1015.0  \n",
       "118143    0.0       0.0     275       0.781818      4.808511         1121.0  \n",
       "118141    0.0       0.0      85       0.729412      5.615385         1298.0  \n",
       "118136    0.0       0.0     270       0.737037      4.530612         1322.0  \n",
       "118030    2.0       0.0      95       0.757895      4.333333         1467.0  \n",
       "118123    1.0       0.0     289       0.723183      5.148936         1240.0  \n",
       "118089    2.0       0.0     171       0.707602      4.548387         1137.0  \n",
       "118087    5.0       0.0     278       0.784173      4.166667         1509.0  \n",
       "118085    0.0       0.0      90       0.766667      4.352941         1409.0  \n",
       "118077   15.0       0.0     194       0.783505      4.416667         1446.0  \n",
       "118066    0.0       0.0     129       0.720930      4.120000         1091.0  \n",
       "118064    0.0       0.0     135       0.807407      4.440000         1329.0  \n",
       "118036    1.0       0.0      89       0.775281      4.000000         1447.0  \n",
       "118034    0.0       0.0     214       0.771028      4.657895         1336.0  \n",
       "118107    1.0       0.0     278       0.805755      5.021739         1340.0  \n",
       "117827    0.0       0.0     249       0.738956      4.555556         1376.0  \n",
       "117820    0.0       0.0     271       0.822878      5.136364         1085.0  \n",
       "117816    1.0       0.0     224       0.714286      7.000000          962.0  \n",
       "117658    0.0       0.0     228       0.815789      5.026316         1307.0  \n",
       "117645    0.0       0.0     178       0.764045      4.741935         1278.0  \n",
       "117618    0.0       0.0     264       0.708333      5.116279         1080.0  \n",
       "117609    1.0       0.0     270       0.803704      4.420000         1241.0  \n",
       "117608    9.0       0.0     118       0.703390      4.409091         1232.0  \n",
       "117593    4.0       0.0     274       0.799270      4.500000         1304.0  \n",
       "117592    0.0       0.0     123       0.780488      4.391304         1127.0  \n",
       "117591    1.0       0.0     195       0.733333      7.166667          948.0  \n",
       "117673    1.0       0.0     255       0.768627      4.688889         1362.0  \n",
       "117586    8.0       0.0      92       0.771739      4.812500         1024.0  \n",
       "117552    1.0       0.0     197       0.730964      4.210526         1174.0  \n",
       "117551    1.0       0.0     148       0.790541      4.730769         1312.0  \n",
       "117531    2.0       0.0     276       0.804348      4.770833         1347.0  \n",
       "117526    2.0       0.0     114       0.789474      4.947368         1484.0  \n",
       "117524    0.0       0.0     302       0.705298      5.446809         1099.0  "
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = trim_dataset_by(\n",
    "    df, \n",
    "    ascending=True,\n",
    "    \n",
    "    # sort_by='length',\n",
    "    length=(80, None),\n",
    "    \n",
    "    # sort_by='word fraction',\n",
    "    word_fraction=(0.70, None),\n",
    "    \n",
    "    # sort_by='token length',\n",
    "    token_length=(4, 8),\n",
    "    \n",
    "    # sort_by='english score',\n",
    "    english_score=(900, None),\n",
    "    \n",
    "    # sort_by='likes',\n",
    "    sort_by='retweets',\n",
    "    custom_metrics=[\n",
    "        ['likes', None, None],\n",
    "        ['retweets', None, None]\n",
    "    ]\n",
    ")\n",
    "a[:60]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "id": "75e9dbca-c842-44de-a965-e58b80229092",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:40:17.429495Z",
     "iopub.status.busy": "2024-01-08T15:40:17.429282Z",
     "iopub.status.idle": "2024-01-08T15:40:17.482069Z",
     "shell.execute_reply": "2024-01-08T15:40:17.481287Z",
     "shell.execute_reply.started": "2024-01-08T15:40:17.429474Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(9268, 8)"
      ]
     },
     "execution_count": 167,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def subset_contains(df, ss, **kwargs):\n",
    "    return df[df['text'].apply(lambda s: contains_substrings(s, ss, **kwargs))]\n",
    "\n",
    "subset_contains(a, ['#chatgpt'], case=False).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "id": "10b3588b-efc2-43ff-911f-7dea00a15248",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:40:17.483183Z",
     "iopub.status.busy": "2024-01-08T15:40:17.482891Z",
     "iopub.status.idle": "2024-01-08T15:40:17.573801Z",
     "shell.execute_reply": "2024-01-08T15:40:17.573017Z",
     "shell.execute_reply.started": "2024-01-08T15:40:17.483160Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\"#chatgpt\" new frequency: 0.07\n"
     ]
    }
   ],
   "source": [
    "df = randomly_remove_substring(a, '#chatgpt', p=0.8, case=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "id": "b34d8bdd-9c60-47e2-acbb-0f0bad258c30",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:40:17.574945Z",
     "iopub.status.busy": "2024-01-08T15:40:17.574708Z",
     "iopub.status.idle": "2024-01-08T15:40:36.048202Z",
     "shell.execute_reply": "2024-01-08T15:40:36.047511Z",
     "shell.execute_reply.started": "2024-01-08T15:40:17.574921Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time elapsed: 18s\n",
      "samples: 10877, similar samples detected: 123 (1.12%)\n"
     ]
    }
   ],
   "source": [
    "df = drop_similar(supersample(df, CHATGPT));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "id": "69d70035-58ea-45d0-a591-cbb2d88b6e44",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:40:36.049457Z",
     "iopub.status.busy": "2024-01-08T15:40:36.049213Z",
     "iopub.status.idle": "2024-01-08T15:40:36.075723Z",
     "shell.execute_reply": "2024-01-08T15:40:36.075200Z",
     "shell.execute_reply.started": "2024-01-08T15:40:36.049429Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>likes</th>\n",
       "      <th>retweets</th>\n",
       "      <th>length</th>\n",
       "      <th>word fraction</th>\n",
       "      <th>token length</th>\n",
       "      <th>english score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8880</th>\n",
       "      <td>1636973523284512768</td>\n",
       "      <td>@amli_art Wait next week. It's not guaranteed, but we might see a large language model (the same AI system that powers ChatGPT or GPT-4) that is truly open and designed to be trainable with your content. If that happens, in weeks you'll have the system you need from the AI community 4 free</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>290</td>\n",
       "      <td>0.741379</td>\n",
       "      <td>4.490566</td>\n",
       "      <td>1287.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3614</th>\n",
       "      <td>1638083128890646529</td>\n",
       "      <td>I have 50 developers in my team who work on a large set of distributed applications and machine learning services that contain complex business logic to control and automate our logistics processes. #chatGPT could not come up with a single useful LOC here. So why bother with it?</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>279</td>\n",
       "      <td>0.784946</td>\n",
       "      <td>4.833333</td>\n",
       "      <td>1357.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7792</th>\n",
       "      <td>1632700607311265794</td>\n",
       "      <td>@unusual_whales Chat GPT is great with words but not good at all with crunching accurate numbers, so I think the financial analysts / bean counters are safe</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>156</td>\n",
       "      <td>0.724359</td>\n",
       "      <td>4.814815</td>\n",
       "      <td>1438.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4727</th>\n",
       "      <td>1637084131984121859</td>\n",
       "      <td>@elonmusk I asked chat gpt Ai some questions and the answers were definitely bias politically and answered very pro pharma. Chat Gpt is bull sheeeet.</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>149</td>\n",
       "      <td>0.765101</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>1058.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5174</th>\n",
       "      <td>1637288294227804160</td>\n",
       "      <td>@wretchardthecat Isn't the idea that AI is objectively truthful already dead and buried though? Because we have many, many examples of Chat GPT simply making up facts in the middle of detailed responses. In one case I read about it went so far as to fabricate the existence of medical studies!</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>293</td>\n",
       "      <td>0.757679</td>\n",
       "      <td>4.764706</td>\n",
       "      <td>1272.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       id  \\\n",
       "8880  1636973523284512768   \n",
       "3614  1638083128890646529   \n",
       "7792  1632700607311265794   \n",
       "4727  1637084131984121859   \n",
       "5174  1637288294227804160   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                       text  \\\n",
       "8880     @amli_art Wait next week. It's not guaranteed, but we might see a large language model (the same AI system that powers ChatGPT or GPT-4) that is truly open and designed to be trainable with your content. If that happens, in weeks you'll have the system you need from the AI community 4 free   \n",
       "3614                I have 50 developers in my team who work on a large set of distributed applications and machine learning services that contain complex business logic to control and automate our logistics processes. #chatGPT could not come up with a single useful LOC here. So why bother with it?   \n",
       "7792                                                                                                                                           @unusual_whales Chat GPT is great with words but not good at all with crunching accurate numbers, so I think the financial analysts / bean counters are safe   \n",
       "4727                                                                                                                                                  @elonmusk I asked chat gpt Ai some questions and the answers were definitely bias politically and answered very pro pharma. Chat Gpt is bull sheeeet.   \n",
       "5174  @wretchardthecat Isn't the idea that AI is objectively truthful already dead and buried though? Because we have many, many examples of Chat GPT simply making up facts in the middle of detailed responses. In one case I read about it went so far as to fabricate the existence of medical studies!   \n",
       "\n",
       "      likes  retweets  length  word fraction  token length  english score  \n",
       "8880    1.0       0.0     290       0.741379      4.490566         1287.0  \n",
       "3614    0.0       0.0     279       0.784946      4.833333         1357.0  \n",
       "7792    0.0       0.0     156       0.724359      4.814815         1438.0  \n",
       "4727    0.0       0.0     149       0.765101      5.000000         1058.0  \n",
       "5174    0.0       0.0     293       0.757679      4.764706         1272.0  "
      ]
     },
     "execution_count": 170,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "id": "f2da8161-320c-4e14-be6f-0580a6d81288",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:40:36.077155Z",
     "iopub.status.busy": "2024-01-08T15:40:36.076735Z",
     "iopub.status.idle": "2024-01-08T15:40:36.111445Z",
     "shell.execute_reply": "2024-01-08T15:40:36.110927Z",
     "shell.execute_reply.started": "2024-01-08T15:40:36.077130Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 3)"
      ]
     },
     "execution_count": 171,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = select_columns(df.copy(), ['text', 'id'])\n",
    "df['topic'] = 'chatgpt'\n",
    "\n",
    "df = sample(df, CHATGPT)\n",
    "dataset = concat(dataset, df)\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b800d858-bb44-4140-aca2-a6cb32eafd98",
   "metadata": {},
   "source": [
    "# Stock Market Crash"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "id": "a963d080-7dec-48b0-aebc-8abbde52da96",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:40:36.112846Z",
     "iopub.status.busy": "2024-01-08T15:40:36.112433Z",
     "iopub.status.idle": "2024-01-08T15:40:36.350942Z",
     "shell.execute_reply": "2024-01-08T15:40:36.350220Z",
     "shell.execute_reply.started": "2024-01-08T15:40:36.112819Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(33946, 2)"
      ]
     },
     "execution_count": 172,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = load_dataset('stock_market_crash.csv', source=True)\n",
    "df = select_columns(df, ['text', 'id'])\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "id": "b527a872-ccc1-4a52-9994-922079ac6577",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:40:36.352768Z",
     "iopub.status.busy": "2024-01-08T15:40:36.352223Z",
     "iopub.status.idle": "2024-01-08T15:40:37.711240Z",
     "shell.execute_reply": "2024-01-08T15:40:37.710695Z",
     "shell.execute_reply.started": "2024-01-08T15:40:36.352738Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(28813, 2)"
      ]
     },
     "execution_count": 173,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = preprocessing_pipeline(df)\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "id": "fc594d5d-0abe-47dc-ba99-02b31eaaf629",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:40:37.712594Z",
     "iopub.status.busy": "2024-01-08T15:40:37.712198Z",
     "iopub.status.idle": "2024-01-08T15:40:38.373639Z",
     "shell.execute_reply": "2024-01-08T15:40:38.373028Z",
     "shell.execute_reply.started": "2024-01-08T15:40:37.712569Z"
    }
   },
   "outputs": [],
   "source": [
    "add_stats(df);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "id": "6202f835-eb8c-4f99-8b72-607ca395337e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:40:38.375134Z",
     "iopub.status.busy": "2024-01-08T15:40:38.374730Z",
     "iopub.status.idle": "2024-01-08T15:40:38.410426Z",
     "shell.execute_reply": "2024-01-08T15:40:38.409921Z",
     "shell.execute_reply.started": "2024-01-08T15:40:38.375110Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "samples: 10274, reduction: 64.34%\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>length</th>\n",
       "      <th>word fraction</th>\n",
       "      <th>token length</th>\n",
       "      <th>english score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>28563</th>\n",
       "      <td>1524336760536322050</td>\n",
       "      <td>$GMT rekt, another blabla2earn ponzi scam is in mud in the #bearmarket</td>\n",
       "      <td>70</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>4.916667</td>\n",
       "      <td>1024.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27113</th>\n",
       "      <td>1526024535140536320</td>\n",
       "      <td>Short to medium term pessimism, long term optimism #crypto #bearmarket</td>\n",
       "      <td>70</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>6.100000</td>\n",
       "      <td>1024.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19924</th>\n",
       "      <td>1485625253942468621</td>\n",
       "      <td>#NFT are the only thing holding its ground right now #stockmarketcrash</td>\n",
       "      <td>70</td>\n",
       "      <td>0.557143</td>\n",
       "      <td>5.454545</td>\n",
       "      <td>1424.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14829</th>\n",
       "      <td>1496786838723473411</td>\n",
       "      <td>Bought the dip today\\n\\nNiftyBees\\nBankBees\\nJuniorBees\\n\\n#stockmarketcrash</td>\n",
       "      <td>70</td>\n",
       "      <td>0.628571</td>\n",
       "      <td>7.625000</td>\n",
       "      <td>993.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24874</th>\n",
       "      <td>1533270299407486976</td>\n",
       "      <td>#NFT Collections that do well during a #bearmarket are ones to watch 👀</td>\n",
       "      <td>70</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>4.461538</td>\n",
       "      <td>1513.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31460</th>\n",
       "      <td>1495114491314458633</td>\n",
       "      <td>Follow StockAlgos for more investing tips &amp; growth tools.\\n\\n#BEARMARKET</td>\n",
       "      <td>70</td>\n",
       "      <td>0.671429</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>1101.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14834</th>\n",
       "      <td>1496786525236838400</td>\n",
       "      <td>If you have sold today\\nMy Dear that was a wrong call\\n#stockmarketcrash</td>\n",
       "      <td>70</td>\n",
       "      <td>0.585714</td>\n",
       "      <td>4.461538</td>\n",
       "      <td>1316.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10643</th>\n",
       "      <td>1522663485380993032</td>\n",
       "      <td>Wednesday was the biggest bull trap I’ve ever seen 😭 #stockmarketcrash</td>\n",
       "      <td>70</td>\n",
       "      <td>0.585714</td>\n",
       "      <td>5.454545</td>\n",
       "      <td>1219.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23815</th>\n",
       "      <td>1536235134386638848</td>\n",
       "      <td>To note for all those new to trading you would call this a #bearmarket</td>\n",
       "      <td>70</td>\n",
       "      <td>0.657143</td>\n",
       "      <td>4.071429</td>\n",
       "      <td>1433.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1306</th>\n",
       "      <td>1537441785722548232</td>\n",
       "      <td>Patience is the name of the game in the stock market #stockmarketcrash</td>\n",
       "      <td>70</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>4.916667</td>\n",
       "      <td>1301.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3892</th>\n",
       "      <td>1536357547984228353</td>\n",
       "      <td>Glad I bought those $6 coffees instead of investing. #stockmarketcrash</td>\n",
       "      <td>70</td>\n",
       "      <td>0.585714</td>\n",
       "      <td>6.100000</td>\n",
       "      <td>1225.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3482</th>\n",
       "      <td>1536376700052725762</td>\n",
       "      <td>We're supposed to just keep dumping money in, right?\\n#stockmarketcrash</td>\n",
       "      <td>70</td>\n",
       "      <td>0.585714</td>\n",
       "      <td>6.100000</td>\n",
       "      <td>1626.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24490</th>\n",
       "      <td>1534993787579252746</td>\n",
       "      <td>$SPX LACK OF BUYERS AT CRITICAL LEVELS, pure indication of #bearmarket</td>\n",
       "      <td>70</td>\n",
       "      <td>0.628571</td>\n",
       "      <td>5.454545</td>\n",
       "      <td>1039.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2001</th>\n",
       "      <td>1537060042972995586</td>\n",
       "      <td>#US stocks’ bear market growl could beckon recession #stockmarketcrash</td>\n",
       "      <td>70</td>\n",
       "      <td>0.585714</td>\n",
       "      <td>6.888889</td>\n",
       "      <td>1039.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3883</th>\n",
       "      <td>1536358033533677568</td>\n",
       "      <td>Does this mean the housing bubble is about to burst? #stockmarketcrash</td>\n",
       "      <td>70</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>5.454545</td>\n",
       "      <td>1395.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21055</th>\n",
       "      <td>1538221288501698561</td>\n",
       "      <td>#Bearmarket is not about #crypto, it creates opportunities for people.</td>\n",
       "      <td>70</td>\n",
       "      <td>0.585714</td>\n",
       "      <td>6.100000</td>\n",
       "      <td>1299.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3280</th>\n",
       "      <td>1536389823803670533</td>\n",
       "      <td>Keep giving money to Ukraine like it belongs to you\\n\\n#stockmarketcrash</td>\n",
       "      <td>70</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>5.363636</td>\n",
       "      <td>1513.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16723</th>\n",
       "      <td>1493280521278812161</td>\n",
       "      <td>#stockmarketcrash This just keeps on happening and we will do nothing?</td>\n",
       "      <td>70</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>5.454545</td>\n",
       "      <td>1469.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3667</th>\n",
       "      <td>1536366179950161923</td>\n",
       "      <td>#stockmarketcrash is trending on twitter and I don’t even want to look</td>\n",
       "      <td>70</td>\n",
       "      <td>0.585714</td>\n",
       "      <td>4.916667</td>\n",
       "      <td>1214.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>846</th>\n",
       "      <td>1537512401137983493</td>\n",
       "      <td>My hands like like they do now so hers don’t have to #stockmarketcrash</td>\n",
       "      <td>70</td>\n",
       "      <td>0.571429</td>\n",
       "      <td>4.461538</td>\n",
       "      <td>1258.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23996</th>\n",
       "      <td>1536037648434855936</td>\n",
       "      <td>#bearmarket is trending with no less than 6 months delay from reality.</td>\n",
       "      <td>70</td>\n",
       "      <td>0.657143</td>\n",
       "      <td>4.916667</td>\n",
       "      <td>1100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23454</th>\n",
       "      <td>1536374282351173632</td>\n",
       "      <td>Thank God I find out about RH before all of this madness 😭 #bearmarket</td>\n",
       "      <td>70</td>\n",
       "      <td>0.642857</td>\n",
       "      <td>4.071429</td>\n",
       "      <td>1204.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29822</th>\n",
       "      <td>1522247701366120451</td>\n",
       "      <td>People panicking. @CNBC this morning is kinda entertaining #bearmarket</td>\n",
       "      <td>70</td>\n",
       "      <td>0.642857</td>\n",
       "      <td>6.888889</td>\n",
       "      <td>1219.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3554</th>\n",
       "      <td>1536371769203998727</td>\n",
       "      <td>Live by the stock market, die by the stock market... #stockmarketcrash</td>\n",
       "      <td>70</td>\n",
       "      <td>0.557143</td>\n",
       "      <td>5.454545</td>\n",
       "      <td>868.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3551</th>\n",
       "      <td>1536371834731438081</td>\n",
       "      <td>Everything is getting cheaper. Christmas came early!\\n#stockmarketcrash</td>\n",
       "      <td>70</td>\n",
       "      <td>0.628571</td>\n",
       "      <td>7.875000</td>\n",
       "      <td>1325.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11370</th>\n",
       "      <td>1522573561646485510</td>\n",
       "      <td>Build Back Better is a euphemism for Burn Baby Burn. #stockmarketcrash</td>\n",
       "      <td>70</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>5.454545</td>\n",
       "      <td>994.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1166</th>\n",
       "      <td>1537459943921618947</td>\n",
       "      <td>i see the Oligarchs are planning another profit party #stockmarketcrash</td>\n",
       "      <td>71</td>\n",
       "      <td>0.633803</td>\n",
       "      <td>6.200000</td>\n",
       "      <td>1052.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2987</th>\n",
       "      <td>1536424780018044930</td>\n",
       "      <td>I told you who daddy was and yall still ignored em...\\n#stockmarketcrash</td>\n",
       "      <td>71</td>\n",
       "      <td>0.563380</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>1129.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6896</th>\n",
       "      <td>1526081866704556033</td>\n",
       "      <td>#stockmarketcrash and the worst still to come.. fasten your seatbelts!!</td>\n",
       "      <td>71</td>\n",
       "      <td>0.577465</td>\n",
       "      <td>6.200000</td>\n",
       "      <td>1146.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26867</th>\n",
       "      <td>1526666473531396096</td>\n",
       "      <td>What coins everyone looking into in the bear market #bearmarket #crypto</td>\n",
       "      <td>71</td>\n",
       "      <td>0.605634</td>\n",
       "      <td>5.545455</td>\n",
       "      <td>980.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16090</th>\n",
       "      <td>1496377318977286151</td>\n",
       "      <td>The market can really test who the paper handers are. #stockmarketcrash</td>\n",
       "      <td>71</td>\n",
       "      <td>0.605634</td>\n",
       "      <td>5.545455</td>\n",
       "      <td>1155.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14723</th>\n",
       "      <td>1496792984167194624</td>\n",
       "      <td>Best time revise our notes from Psychology of money.\\n\\n#stockmarketcrash</td>\n",
       "      <td>71</td>\n",
       "      <td>0.605634</td>\n",
       "      <td>6.100000</td>\n",
       "      <td>1409.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25751</th>\n",
       "      <td>1529800565810728960</td>\n",
       "      <td>Gonna use some of my fiat$ to go to on vacation during this #bearmarket</td>\n",
       "      <td>71</td>\n",
       "      <td>0.647887</td>\n",
       "      <td>4.142857</td>\n",
       "      <td>1404.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28705</th>\n",
       "      <td>1524183791132102657</td>\n",
       "      <td>The Saturn death cult and crypto connections were all scams #bearmarket</td>\n",
       "      <td>71</td>\n",
       "      <td>0.704225</td>\n",
       "      <td>5.545455</td>\n",
       "      <td>1211.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28505</th>\n",
       "      <td>1524373205879869442</td>\n",
       "      <td>The strong will survive and the weak will be gone in a week #bearmarket</td>\n",
       "      <td>71</td>\n",
       "      <td>0.661972</td>\n",
       "      <td>4.142857</td>\n",
       "      <td>1427.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3963</th>\n",
       "      <td>1536354633802829827</td>\n",
       "      <td>It's not a crash until they hit the circuit breakers. #StockMarketCrash</td>\n",
       "      <td>71</td>\n",
       "      <td>0.591549</td>\n",
       "      <td>5.545455</td>\n",
       "      <td>1331.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16407</th>\n",
       "      <td>1494294803034562562</td>\n",
       "      <td>I like stocks especially when they’re getting shit on #stockmarketcrash</td>\n",
       "      <td>71</td>\n",
       "      <td>0.619718</td>\n",
       "      <td>6.200000</td>\n",
       "      <td>1413.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25502</th>\n",
       "      <td>1530359724725530624</td>\n",
       "      <td>CT is very bearish right now, does it mean its time to buy? #bearmarket</td>\n",
       "      <td>71</td>\n",
       "      <td>0.633803</td>\n",
       "      <td>4.142857</td>\n",
       "      <td>1706.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1981</th>\n",
       "      <td>1537078750843633666</td>\n",
       "      <td>#stockmarketcrash The stock market is now officially in bear territory!</td>\n",
       "      <td>71</td>\n",
       "      <td>0.619718</td>\n",
       "      <td>6.200000</td>\n",
       "      <td>1126.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28683</th>\n",
       "      <td>1524232287021584384</td>\n",
       "      <td>Bull market shows dreams,\\nBear market shows reality. #bearmarket #Nifty</td>\n",
       "      <td>71</td>\n",
       "      <td>0.605634</td>\n",
       "      <td>6.200000</td>\n",
       "      <td>873.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23402</th>\n",
       "      <td>1536391747353432066</td>\n",
       "      <td>Too many discounts... Wonder which brands are worth buying. #bearmarket</td>\n",
       "      <td>71</td>\n",
       "      <td>0.661972</td>\n",
       "      <td>6.200000</td>\n",
       "      <td>1192.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4222</th>\n",
       "      <td>1536339060473602048</td>\n",
       "      <td>Can't help but feel the #stockmarketcrash is completely self inflicted.</td>\n",
       "      <td>71</td>\n",
       "      <td>0.605634</td>\n",
       "      <td>6.200000</td>\n",
       "      <td>1141.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14874</th>\n",
       "      <td>1496784182353170433</td>\n",
       "      <td>We are suffering from a virus which is stock market🥲🥲\\n#stockmarketcrash</td>\n",
       "      <td>71</td>\n",
       "      <td>0.591549</td>\n",
       "      <td>5.545455</td>\n",
       "      <td>1172.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4500</th>\n",
       "      <td>1536309945620123655</td>\n",
       "      <td>Damn it, I saw #stockmarketcrash and I thought capitalism finally died.</td>\n",
       "      <td>71</td>\n",
       "      <td>0.591549</td>\n",
       "      <td>5.545455</td>\n",
       "      <td>1142.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14987</th>\n",
       "      <td>1496777340663345153</td>\n",
       "      <td>After a year of profit, my portfolio has turned RED🔻\\n\\n#stockmarketcrash</td>\n",
       "      <td>71</td>\n",
       "      <td>0.577465</td>\n",
       "      <td>5.454545</td>\n",
       "      <td>1084.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11070</th>\n",
       "      <td>1522596847034015746</td>\n",
       "      <td>#stockmarketcrash is all due to the Biden Depression. Buckle up, folks.</td>\n",
       "      <td>71</td>\n",
       "      <td>0.577465</td>\n",
       "      <td>5.545455</td>\n",
       "      <td>1264.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24190</th>\n",
       "      <td>1535761644101451776</td>\n",
       "      <td>I think it really took a turn for the worst at IreneDAO.. \\n\\n#bearmarket</td>\n",
       "      <td>71</td>\n",
       "      <td>0.619718</td>\n",
       "      <td>4.384615</td>\n",
       "      <td>1314.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10700</th>\n",
       "      <td>1522650154804858880</td>\n",
       "      <td>Amazon is going Bankrupt! Y’all should do the needful #stockmarketcrash</td>\n",
       "      <td>71</td>\n",
       "      <td>0.605634</td>\n",
       "      <td>6.200000</td>\n",
       "      <td>1418.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23415</th>\n",
       "      <td>1536387404352741376</td>\n",
       "      <td>When in doubt switch your long term strategy to short term. #bearmarket</td>\n",
       "      <td>71</td>\n",
       "      <td>0.676056</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>1316.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28373</th>\n",
       "      <td>1524418979783188481</td>\n",
       "      <td>Desire for gain, and fear of loss can move men to action🚀 #bearmarket 📊</td>\n",
       "      <td>71</td>\n",
       "      <td>0.619718</td>\n",
       "      <td>4.142857</td>\n",
       "      <td>1360.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15149</th>\n",
       "      <td>1496762517288361986</td>\n",
       "      <td>Market crash krrgya\\n#stockmarketcrash \\nIt's time to take long breath 🫁🫁</td>\n",
       "      <td>71</td>\n",
       "      <td>0.563380</td>\n",
       "      <td>5.454545</td>\n",
       "      <td>1130.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31437</th>\n",
       "      <td>1495383822468321287</td>\n",
       "      <td>Perma-bulls still mad if I say \"We are in a #bearmarket ?\" When Bottom?</td>\n",
       "      <td>71</td>\n",
       "      <td>0.563380</td>\n",
       "      <td>4.142857</td>\n",
       "      <td>1210.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3878</th>\n",
       "      <td>1536358180413919233</td>\n",
       "      <td>Stop dependency on US market and declare independence #stockmarketcrash</td>\n",
       "      <td>71</td>\n",
       "      <td>0.647887</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>923.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19420</th>\n",
       "      <td>1485881026098380802</td>\n",
       "      <td>Whaaaat TF guys did you learn to drive on a racetrack?#stockmarketcrash</td>\n",
       "      <td>71</td>\n",
       "      <td>0.830986</td>\n",
       "      <td>5.545455</td>\n",
       "      <td>1038.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2911</th>\n",
       "      <td>1536437585735720963</td>\n",
       "      <td>Is there any way to just turn the stock market off??? #stockmarketcrash</td>\n",
       "      <td>71</td>\n",
       "      <td>0.563380</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>1385.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28181</th>\n",
       "      <td>1524529969275232256</td>\n",
       "      <td>What's worse than a #bearmarket cause that's where we seem to be headed</td>\n",
       "      <td>71</td>\n",
       "      <td>0.647887</td>\n",
       "      <td>4.538462</td>\n",
       "      <td>1413.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28329</th>\n",
       "      <td>1524442268748455937</td>\n",
       "      <td>At this point, I think I’ve shorted the whole market. #NYSE #BearMarket</td>\n",
       "      <td>71</td>\n",
       "      <td>0.577465</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>1280.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17522</th>\n",
       "      <td>1493110423826038790</td>\n",
       "      <td>Haven’t seen a single green colour in today’s stocks. #stockmarketcrash</td>\n",
       "      <td>71</td>\n",
       "      <td>0.591549</td>\n",
       "      <td>6.200000</td>\n",
       "      <td>1228.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21764</th>\n",
       "      <td>1537380752953954306</td>\n",
       "      <td>What is your biggest concern about investing right now? \\n\\n#bearmarket 🐻</td>\n",
       "      <td>71</td>\n",
       "      <td>0.647887</td>\n",
       "      <td>5.363636</td>\n",
       "      <td>1318.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7970</th>\n",
       "      <td>1524752193638838272</td>\n",
       "      <td>They don't want nobody getting no money big Biden 😂😂😂 #stockmarketcrash</td>\n",
       "      <td>71</td>\n",
       "      <td>0.563380</td>\n",
       "      <td>5.545455</td>\n",
       "      <td>1161.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        id  \\\n",
       "28563  1524336760536322050   \n",
       "27113  1526024535140536320   \n",
       "19924  1485625253942468621   \n",
       "14829  1496786838723473411   \n",
       "24874  1533270299407486976   \n",
       "31460  1495114491314458633   \n",
       "14834  1496786525236838400   \n",
       "10643  1522663485380993032   \n",
       "23815  1536235134386638848   \n",
       "1306   1537441785722548232   \n",
       "3892   1536357547984228353   \n",
       "3482   1536376700052725762   \n",
       "24490  1534993787579252746   \n",
       "2001   1537060042972995586   \n",
       "3883   1536358033533677568   \n",
       "21055  1538221288501698561   \n",
       "3280   1536389823803670533   \n",
       "16723  1493280521278812161   \n",
       "3667   1536366179950161923   \n",
       "846    1537512401137983493   \n",
       "23996  1536037648434855936   \n",
       "23454  1536374282351173632   \n",
       "29822  1522247701366120451   \n",
       "3554   1536371769203998727   \n",
       "3551   1536371834731438081   \n",
       "11370  1522573561646485510   \n",
       "1166   1537459943921618947   \n",
       "2987   1536424780018044930   \n",
       "6896   1526081866704556033   \n",
       "26867  1526666473531396096   \n",
       "16090  1496377318977286151   \n",
       "14723  1496792984167194624   \n",
       "25751  1529800565810728960   \n",
       "28705  1524183791132102657   \n",
       "28505  1524373205879869442   \n",
       "3963   1536354633802829827   \n",
       "16407  1494294803034562562   \n",
       "25502  1530359724725530624   \n",
       "1981   1537078750843633666   \n",
       "28683  1524232287021584384   \n",
       "23402  1536391747353432066   \n",
       "4222   1536339060473602048   \n",
       "14874  1496784182353170433   \n",
       "4500   1536309945620123655   \n",
       "14987  1496777340663345153   \n",
       "11070  1522596847034015746   \n",
       "24190  1535761644101451776   \n",
       "10700  1522650154804858880   \n",
       "23415  1536387404352741376   \n",
       "28373  1524418979783188481   \n",
       "15149  1496762517288361986   \n",
       "31437  1495383822468321287   \n",
       "3878   1536358180413919233   \n",
       "19420  1485881026098380802   \n",
       "2911   1536437585735720963   \n",
       "28181  1524529969275232256   \n",
       "28329  1524442268748455937   \n",
       "17522  1493110423826038790   \n",
       "21764  1537380752953954306   \n",
       "7970   1524752193638838272   \n",
       "\n",
       "                                                                               text  \\\n",
       "28563        $GMT rekt, another blabla2earn ponzi scam is in mud in the #bearmarket   \n",
       "27113        Short to medium term pessimism, long term optimism #crypto #bearmarket   \n",
       "19924        #NFT are the only thing holding its ground right now #stockmarketcrash   \n",
       "14829  Bought the dip today\\n\\nNiftyBees\\nBankBees\\nJuniorBees\\n\\n#stockmarketcrash   \n",
       "24874        #NFT Collections that do well during a #bearmarket are ones to watch 👀   \n",
       "31460      Follow StockAlgos for more investing tips & growth tools.\\n\\n#BEARMARKET   \n",
       "14834      If you have sold today\\nMy Dear that was a wrong call\\n#stockmarketcrash   \n",
       "10643        Wednesday was the biggest bull trap I’ve ever seen 😭 #stockmarketcrash   \n",
       "23815        To note for all those new to trading you would call this a #bearmarket   \n",
       "1306         Patience is the name of the game in the stock market #stockmarketcrash   \n",
       "3892         Glad I bought those $6 coffees instead of investing. #stockmarketcrash   \n",
       "3482        We're supposed to just keep dumping money in, right?\\n#stockmarketcrash   \n",
       "24490        $SPX LACK OF BUYERS AT CRITICAL LEVELS, pure indication of #bearmarket   \n",
       "2001         #US stocks’ bear market growl could beckon recession #stockmarketcrash   \n",
       "3883         Does this mean the housing bubble is about to burst? #stockmarketcrash   \n",
       "21055        #Bearmarket is not about #crypto, it creates opportunities for people.   \n",
       "3280       Keep giving money to Ukraine like it belongs to you\\n\\n#stockmarketcrash   \n",
       "16723        #stockmarketcrash This just keeps on happening and we will do nothing?   \n",
       "3667         #stockmarketcrash is trending on twitter and I don’t even want to look   \n",
       "846          My hands like like they do now so hers don’t have to #stockmarketcrash   \n",
       "23996        #bearmarket is trending with no less than 6 months delay from reality.   \n",
       "23454        Thank God I find out about RH before all of this madness 😭 #bearmarket   \n",
       "29822        People panicking. @CNBC this morning is kinda entertaining #bearmarket   \n",
       "3554         Live by the stock market, die by the stock market... #stockmarketcrash   \n",
       "3551        Everything is getting cheaper. Christmas came early!\\n#stockmarketcrash   \n",
       "11370        Build Back Better is a euphemism for Burn Baby Burn. #stockmarketcrash   \n",
       "1166        i see the Oligarchs are planning another profit party #stockmarketcrash   \n",
       "2987       I told you who daddy was and yall still ignored em...\\n#stockmarketcrash   \n",
       "6896        #stockmarketcrash and the worst still to come.. fasten your seatbelts!!   \n",
       "26867       What coins everyone looking into in the bear market #bearmarket #crypto   \n",
       "16090       The market can really test who the paper handers are. #stockmarketcrash   \n",
       "14723     Best time revise our notes from Psychology of money.\\n\\n#stockmarketcrash   \n",
       "25751       Gonna use some of my fiat$ to go to on vacation during this #bearmarket   \n",
       "28705       The Saturn death cult and crypto connections were all scams #bearmarket   \n",
       "28505       The strong will survive and the weak will be gone in a week #bearmarket   \n",
       "3963        It's not a crash until they hit the circuit breakers. #StockMarketCrash   \n",
       "16407       I like stocks especially when they’re getting shit on #stockmarketcrash   \n",
       "25502       CT is very bearish right now, does it mean its time to buy? #bearmarket   \n",
       "1981        #stockmarketcrash The stock market is now officially in bear territory!   \n",
       "28683      Bull market shows dreams,\\nBear market shows reality. #bearmarket #Nifty   \n",
       "23402       Too many discounts... Wonder which brands are worth buying. #bearmarket   \n",
       "4222        Can't help but feel the #stockmarketcrash is completely self inflicted.   \n",
       "14874      We are suffering from a virus which is stock market🥲🥲\\n#stockmarketcrash   \n",
       "4500        Damn it, I saw #stockmarketcrash and I thought capitalism finally died.   \n",
       "14987     After a year of profit, my portfolio has turned RED🔻\\n\\n#stockmarketcrash   \n",
       "11070       #stockmarketcrash is all due to the Biden Depression. Buckle up, folks.   \n",
       "24190     I think it really took a turn for the worst at IreneDAO.. \\n\\n#bearmarket   \n",
       "10700       Amazon is going Bankrupt! Y’all should do the needful #stockmarketcrash   \n",
       "23415       When in doubt switch your long term strategy to short term. #bearmarket   \n",
       "28373       Desire for gain, and fear of loss can move men to action🚀 #bearmarket 📊   \n",
       "15149     Market crash krrgya\\n#stockmarketcrash \\nIt's time to take long breath 🫁🫁   \n",
       "31437       Perma-bulls still mad if I say \"We are in a #bearmarket ?\" When Bottom?   \n",
       "3878        Stop dependency on US market and declare independence #stockmarketcrash   \n",
       "19420       Whaaaat TF guys did you learn to drive on a racetrack?#stockmarketcrash   \n",
       "2911        Is there any way to just turn the stock market off??? #stockmarketcrash   \n",
       "28181       What's worse than a #bearmarket cause that's where we seem to be headed   \n",
       "28329       At this point, I think I’ve shorted the whole market. #NYSE #BearMarket   \n",
       "17522       Haven’t seen a single green colour in today’s stocks. #stockmarketcrash   \n",
       "21764     What is your biggest concern about investing right now? \\n\\n#bearmarket 🐻   \n",
       "7970        They don't want nobody getting no money big Biden 😂😂😂 #stockmarketcrash   \n",
       "\n",
       "       length  word fraction  token length  english score  \n",
       "28563      70       0.600000      4.916667         1024.0  \n",
       "27113      70       0.600000      6.100000         1024.0  \n",
       "19924      70       0.557143      5.454545         1424.0  \n",
       "14829      70       0.628571      7.625000          993.0  \n",
       "24874      70       0.600000      4.461538         1513.0  \n",
       "31460      70       0.671429      6.000000         1101.0  \n",
       "14834      70       0.585714      4.461538         1316.0  \n",
       "10643      70       0.585714      5.454545         1219.0  \n",
       "23815      70       0.657143      4.071429         1433.0  \n",
       "1306       70       0.600000      4.916667         1301.0  \n",
       "3892       70       0.585714      6.100000         1225.0  \n",
       "3482       70       0.585714      6.100000         1626.0  \n",
       "24490      70       0.628571      5.454545         1039.0  \n",
       "2001       70       0.585714      6.888889         1039.0  \n",
       "3883       70       0.600000      5.454545         1395.0  \n",
       "21055      70       0.585714      6.100000         1299.0  \n",
       "3280       70       0.600000      5.363636         1513.0  \n",
       "16723      70       0.600000      5.454545         1469.0  \n",
       "3667       70       0.585714      4.916667         1214.0  \n",
       "846        70       0.571429      4.461538         1258.0  \n",
       "23996      70       0.657143      4.916667         1100.0  \n",
       "23454      70       0.642857      4.071429         1204.0  \n",
       "29822      70       0.642857      6.888889         1219.0  \n",
       "3554       70       0.557143      5.454545          868.0  \n",
       "3551       70       0.628571      7.875000         1325.0  \n",
       "11370      70       0.600000      5.454545          994.0  \n",
       "1166       71       0.633803      6.200000         1052.0  \n",
       "2987       71       0.563380      5.000000         1129.0  \n",
       "6896       71       0.577465      6.200000         1146.0  \n",
       "26867      71       0.605634      5.545455          980.0  \n",
       "16090      71       0.605634      5.545455         1155.0  \n",
       "14723      71       0.605634      6.100000         1409.0  \n",
       "25751      71       0.647887      4.142857         1404.0  \n",
       "28705      71       0.704225      5.545455         1211.0  \n",
       "28505      71       0.661972      4.142857         1427.0  \n",
       "3963       71       0.591549      5.545455         1331.0  \n",
       "16407      71       0.619718      6.200000         1413.0  \n",
       "25502      71       0.633803      4.142857         1706.0  \n",
       "1981       71       0.619718      6.200000         1126.0  \n",
       "28683      71       0.605634      6.200000          873.0  \n",
       "23402      71       0.661972      6.200000         1192.0  \n",
       "4222       71       0.605634      6.200000         1141.0  \n",
       "14874      71       0.591549      5.545455         1172.0  \n",
       "4500       71       0.591549      5.545455         1142.0  \n",
       "14987      71       0.577465      5.454545         1084.0  \n",
       "11070      71       0.577465      5.545455         1264.0  \n",
       "24190      71       0.619718      4.384615         1314.0  \n",
       "10700      71       0.605634      6.200000         1418.0  \n",
       "23415      71       0.676056      5.000000         1316.0  \n",
       "28373      71       0.619718      4.142857         1360.0  \n",
       "15149      71       0.563380      5.454545         1130.0  \n",
       "31437      71       0.563380      4.142857         1210.0  \n",
       "3878       71       0.647887      7.000000          923.0  \n",
       "19420      71       0.830986      5.545455         1038.0  \n",
       "2911       71       0.563380      5.000000         1385.0  \n",
       "28181      71       0.647887      4.538462         1413.0  \n",
       "28329      71       0.577465      5.000000         1280.0  \n",
       "17522      71       0.591549      6.200000         1228.0  \n",
       "21764      71       0.647887      5.363636         1318.0  \n",
       "7970       71       0.563380      5.545455         1161.0  "
      ]
     },
     "execution_count": 175,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = trim_dataset_by(\n",
    "    df, \n",
    "    # ascending=False,\n",
    "    ascending=True,\n",
    "    \n",
    "    sort_by='length',\n",
    "    length=(70, None),\n",
    "    \n",
    "    # sort_by='word fraction',\n",
    "    word_fraction=(0.55, None),\n",
    "    \n",
    "    # sort_by='token length',\n",
    "    token_length=(4, 8),\n",
    "    \n",
    "    # sort_by='english score',\n",
    "    english_score=(850, None),\n",
    ")\n",
    "a[:60]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "id": "8ca4baa0-27d2-4a79-b9bb-aa7e824628ad",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:40:38.411719Z",
     "iopub.status.busy": "2024-01-08T15:40:38.411327Z",
     "iopub.status.idle": "2024-01-08T15:40:38.465577Z",
     "shell.execute_reply": "2024-01-08T15:40:38.465033Z",
     "shell.execute_reply.started": "2024-01-08T15:40:38.411695Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "string frequencies:\n",
      "#stockmarketcrash: 0.61\n",
      "#stockmarket: 0.62\n",
      "#bearmarket: 0.40\n"
     ]
    }
   ],
   "source": [
    "df = a\n",
    "print('string frequencies:')\n",
    "\n",
    "s = '#stockmarketcrash'; print(f'{s}: {string_frequency(df, s):.2f}')\n",
    "s = '#stockmarket'; print(f'{s}: {string_frequency(df, s):.2f}')\n",
    "s = '#bearmarket'; print(f'{s}: {string_frequency(df, s):.2f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "id": "1d10c718-5d28-4a88-8346-6255b5a28877",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:40:38.466929Z",
     "iopub.status.busy": "2024-01-08T15:40:38.466531Z",
     "iopub.status.idle": "2024-01-08T15:40:38.543005Z",
     "shell.execute_reply": "2024-01-08T15:40:38.542476Z",
     "shell.execute_reply.started": "2024-01-08T15:40:38.466902Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\"#stockmarketcrash\" new frequency: 0.30\n",
      "\"#bearmarket\" new frequency: 0.20\n"
     ]
    }
   ],
   "source": [
    "df = randomly_remove_substring(df, '#stockmarketcrash', p=0.5, case=False)\n",
    "df = randomly_remove_substring(df, '#bearmarket', p=0.5, case=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "id": "18eeb4cc-ebe4-4985-9e6d-6e3d59eada17",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:40:38.544188Z",
     "iopub.status.busy": "2024-01-08T15:40:38.543859Z",
     "iopub.status.idle": "2024-01-08T15:40:38.564923Z",
     "shell.execute_reply": "2024-01-08T15:40:38.564319Z",
     "shell.execute_reply.started": "2024-01-08T15:40:38.544165Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>length</th>\n",
       "      <th>word fraction</th>\n",
       "      <th>token length</th>\n",
       "      <th>english score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>11457</th>\n",
       "      <td>1522562060130861058</td>\n",
       "      <td>We can turn this around by making the government balance the budget and pay off its debts.\\n\\nFederal revenue is 4.2 trillion annually. We are paying 360 billion just to pay the interest on our 30+ trillion dollar debt.\\n\\nThey can do it, we have to make them.\\n  \\n🙂</td>\n",
       "      <td>278</td>\n",
       "      <td>0.694245</td>\n",
       "      <td>4.500000</td>\n",
       "      <td>1562.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9545</th>\n",
       "      <td>1524060205968941056</td>\n",
       "      <td>(3)It’s a great buying opportunity for all of us. Amidst the noise of stock market bubble and correction, there are a few great companies getting crushed.This is thus a great opportunity to for us to do our research and find such companies to invest! #stockmarketcrash #recession</td>\n",
       "      <td>279</td>\n",
       "      <td>0.702509</td>\n",
       "      <td>5.086957</td>\n",
       "      <td>1249.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3402</th>\n",
       "      <td>1536382109232807938</td>\n",
       "      <td>You ain’t seen nuthin’ yet! Just wait a little longer and the US dollar is no longer the reserve currency. You think 5 bux a gallon for gas is bad?!? Hold my beer. You can thank Comrade Chairman Biden for the destruction of your lives. #stockmarketcrash</td>\n",
       "      <td>253</td>\n",
       "      <td>0.715415</td>\n",
       "      <td>4.521739</td>\n",
       "      <td>1394.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20263</th>\n",
       "      <td>1538463191465861122</td>\n",
       "      <td>Hi #CryptoCommunity! 🫠\\n\\nWish you get through this #BearMarket2022. \\nHodl on 🚀\\n\\nWhat #Altcoins analysis do you want to see today? Write your favorite altcoin in the comments 👇</td>\n",
       "      <td>174</td>\n",
       "      <td>0.557471</td>\n",
       "      <td>4.931034</td>\n",
       "      <td>1247.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11380</th>\n",
       "      <td>1522572971016998913</td>\n",
       "      <td>is not a real crash in the slightest. A decline of 3-4% is really a drop in the bucket. The point amount doesn't matter.</td>\n",
       "      <td>138</td>\n",
       "      <td>0.644928</td>\n",
       "      <td>4.560000</td>\n",
       "      <td>1433.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        id  \\\n",
       "11457  1522562060130861058   \n",
       "9545   1524060205968941056   \n",
       "3402   1536382109232807938   \n",
       "20263  1538463191465861122   \n",
       "11380  1522572971016998913   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                          text  \\\n",
       "11457              We can turn this around by making the government balance the budget and pay off its debts.\\n\\nFederal revenue is 4.2 trillion annually. We are paying 360 billion just to pay the interest on our 30+ trillion dollar debt.\\n\\nThey can do it, we have to make them.\\n  \\n🙂   \n",
       "9545   (3)It’s a great buying opportunity for all of us. Amidst the noise of stock market bubble and correction, there are a few great companies getting crushed.This is thus a great opportunity to for us to do our research and find such companies to invest! #stockmarketcrash #recession   \n",
       "3402                             You ain’t seen nuthin’ yet! Just wait a little longer and the US dollar is no longer the reserve currency. You think 5 bux a gallon for gas is bad?!? Hold my beer. You can thank Comrade Chairman Biden for the destruction of your lives. #stockmarketcrash   \n",
       "20263                                                                                                      Hi #CryptoCommunity! 🫠\\n\\nWish you get through this #BearMarket2022. \\nHodl on 🚀\\n\\nWhat #Altcoins analysis do you want to see today? Write your favorite altcoin in the comments 👇   \n",
       "11380                                                                                                                                                                 is not a real crash in the slightest. A decline of 3-4% is really a drop in the bucket. The point amount doesn't matter.   \n",
       "\n",
       "       length  word fraction  token length  english score  \n",
       "11457     278       0.694245      4.500000         1562.0  \n",
       "9545      279       0.702509      5.086957         1249.0  \n",
       "3402      253       0.715415      4.521739         1394.0  \n",
       "20263     174       0.557471      4.931034         1247.0  \n",
       "11380     138       0.644928      4.560000         1433.0  "
      ]
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "id": "9b925bca-02a5-4d0a-b485-b85adec41361",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:40:38.566674Z",
     "iopub.status.busy": "2024-01-08T15:40:38.566353Z",
     "iopub.status.idle": "2024-01-08T15:40:52.363403Z",
     "shell.execute_reply": "2024-01-08T15:40:52.362511Z",
     "shell.execute_reply.started": "2024-01-08T15:40:38.566652Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time elapsed: 14s\n",
      "samples: 10156, similar samples detected: 118 (1.15%)\n"
     ]
    }
   ],
   "source": [
    "df = drop_similar(supersample(df, STOCK_MARKET_CRASH));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "id": "447f0b31-a50d-47b5-8c92-51bef3868645",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:40:52.365451Z",
     "iopub.status.busy": "2024-01-08T15:40:52.364865Z",
     "iopub.status.idle": "2024-01-08T15:40:52.425247Z",
     "shell.execute_reply": "2024-01-08T15:40:52.424658Z",
     "shell.execute_reply.started": "2024-01-08T15:40:52.365414Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 3)"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = select_columns(df.copy(), ['text', 'id'])\n",
    "df['topic'] = 'stock market crash'\n",
    "\n",
    "df = sample(df, STOCK_MARKET_CRASH)\n",
    "dataset = concat(dataset, df)\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bf5fee20-ae13-49e3-b647-fe95308f6b3c",
   "metadata": {},
   "source": [
    "# Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "id": "2d92dc64-2bec-4a22-aecf-bbc44d9a3946",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:41:30.266356Z",
     "iopub.status.busy": "2024-01-08T15:41:30.265458Z",
     "iopub.status.idle": "2024-01-08T15:41:30.283941Z",
     "shell.execute_reply": "2024-01-08T15:41:30.283334Z",
     "shell.execute_reply.started": "2024-01-08T15:41:30.266303Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(70406, 3)"
      ]
     },
     "execution_count": 185,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ad4252a0-ed69-47c1-9bec-2c51010fbe74",
   "metadata": {},
   "source": [
    "Topic distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "id": "784db5f6-3501-47f9-8d24-b851123c8260",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:40:52.444397Z",
     "iopub.status.busy": "2024-01-08T15:40:52.444200Z",
     "iopub.status.idle": "2024-01-08T15:40:52.533586Z",
     "shell.execute_reply": "2024-01-08T15:40:52.532949Z",
     "shell.execute_reply.started": "2024-01-08T15:40:52.444378Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(70406, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>topic</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>airline support</th>\n",
       "      <td>8468</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>australian elections</th>\n",
       "      <td>10000</td>\n",
       "      <td>10000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chatgpt</th>\n",
       "      <td>10000</td>\n",
       "      <td>10000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>climate change</th>\n",
       "      <td>10000</td>\n",
       "      <td>10000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>covid19</th>\n",
       "      <td>10000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fifa world cup</th>\n",
       "      <td>6854</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>self-driving cars</th>\n",
       "      <td>4497</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stock market crash</th>\n",
       "      <td>10000</td>\n",
       "      <td>10000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>weather</th>\n",
       "      <td>587</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       text     id\n",
       "topic                             \n",
       "airline support        8468      1\n",
       "australian elections  10000  10000\n",
       "chatgpt               10000  10000\n",
       "climate change        10000  10000\n",
       "covid19               10000      1\n",
       "fifa world cup         6854      1\n",
       "self-driving cars      4497      1\n",
       "stock market crash    10000  10000\n",
       "weather                 587      1"
      ]
     },
     "execution_count": 182,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(dataset.shape)\n",
    "dataset.groupby('topic').nunique()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "656d33a2-2514-4b85-9ca5-6c1ffa01d4c5",
   "metadata": {},
   "source": [
    "Explore random tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "id": "99ba9518-681c-405f-9bd1-33f045cdcfb5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:40:52.534537Z",
     "iopub.status.busy": "2024-01-08T15:40:52.534341Z",
     "iopub.status.idle": "2024-01-08T15:40:52.559589Z",
     "shell.execute_reply": "2024-01-08T15:40:52.559033Z",
     "shell.execute_reply.started": "2024-01-08T15:40:52.534517Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>topic</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>37187</th>\n",
       "      <td>You can’t win a federal election in Australia by telling the people they are going to lose money in their hip pocket</td>\n",
       "      <td>australian elections</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27807</th>\n",
       "      <td>Looking forward to the bent refereeing today.</td>\n",
       "      <td>fifa world cup</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59011</th>\n",
       "      <td>@scottmelker I tried to get chat gpt to help me start a new religion for that exact reason, converting people from old shit religions to a new better one. It told me that it is disrespectful and immoral.\\nI said so like a mega pastor?\\nChat gpt, yes, just like a mega pastor.</td>\n",
       "      <td>chatgpt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17857</th>\n",
       "      <td>@AmericanAir How do I check? Reservation for Joe Watson and Kelsey Jennings. We were on hold for 2 hours. Waiting for call back now. ETA?</td>\n",
       "      <td>airline support</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19838</th>\n",
       "      <td>@SouthwestAir you have the worst service, you Cancelled Flightled all your flights FLL to PHL all @USAirways flights flew. Stuck in FL 3 days. #done</td>\n",
       "      <td>airline support</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                                                                                                      text  \\\n",
       "37187                                                                                                                                                                You can’t win a federal election in Australia by telling the people they are going to lose money in their hip pocket    \n",
       "27807                                                                                                                                                                                                                                       Looking forward to the bent refereeing today.    \n",
       "59011  @scottmelker I tried to get chat gpt to help me start a new religion for that exact reason, converting people from old shit religions to a new better one. It told me that it is disrespectful and immoral.\\nI said so like a mega pastor?\\nChat gpt, yes, just like a mega pastor.   \n",
       "17857                                                                                                                                            @AmericanAir How do I check? Reservation for Joe Watson and Kelsey Jennings. We were on hold for 2 hours. Waiting for call back now. ETA?   \n",
       "19838                                                                                                                                 @SouthwestAir you have the worst service, you Cancelled Flightled all your flights FLL to PHL all @USAirways flights flew. Stuck in FL 3 days. #done   \n",
       "\n",
       "                      topic  \n",
       "37187  australian elections  \n",
       "27807        fifa world cup  \n",
       "59011               chatgpt  \n",
       "17857       airline support  \n",
       "19838       airline support  "
      ]
     },
     "execution_count": 183,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset[['text', 'topic']].sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "id": "a85dc533-bb02-4db0-85cd-2f278d281472",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-08T15:41:45.442524Z",
     "iopub.status.busy": "2024-01-08T15:41:45.441864Z",
     "iopub.status.idle": "2024-01-08T15:41:45.711734Z",
     "shell.execute_reply": "2024-01-08T15:41:45.710964Z",
     "shell.execute_reply.started": "2024-01-08T15:41:45.442496Z"
    }
   },
   "outputs": [],
   "source": [
    "# save_dataset(dataset, 'dataset70000.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}