web_data_text_classification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1211f51b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import re\n",
    "import pickle\n",
    "from string import punctuation\n",
    "from nltk.corpus import stopwords\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.model_selection import train_test_split\n",
    "import torch\n",
    "import torchvision\n",
    "import spacy\n",
    "import torch.nn as nn\n",
    "from collections import Counter\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "import torch.nn.functional as F\n",
    "import string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3cce7c6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_json('raw_data.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ee6b3323",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>brand_id</th>\n",
       "      <th>raw_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>unknown</td>\n",
       "      <td>CPL  Loading Done  OK Trading Perpetual Spot T...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>roblox</td>\n",
       "      <td>BloxFarm - Earn Your Robux Today!  Login to Bl...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  brand_id                                           raw_text\n",
       "0  unknown  CPL  Loading Done  OK Trading Perpetual Spot T...\n",
       "1   roblox  BloxFarm - Earn Your Robux Today!  Login to Bl..."
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1cafb10e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(76069, 2)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "cd7f6e8a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 76069 entries, 0 to 76068\n",
      "Data columns (total 2 columns):\n",
      " #   Column    Non-Null Count  Dtype \n",
      "---  ------    --------------  ----- \n",
      " 0   brand_id  76069 non-null  object\n",
      " 1   raw_text  76069 non-null  object\n",
      "dtypes: object(2)\n",
      "memory usage: 1.2+ MB\n"
     ]
    }
   ],
   "source": [
    "data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "cd2d0e71",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "brand_id    0\n",
       "raw_text    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "1f81ecb8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>brand_id</th>\n",
       "      <th>brand_id_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>779</th>\n",
       "      <td>bain_capital</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>780</th>\n",
       "      <td>amfirst</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>781</th>\n",
       "      <td>del_one</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>782</th>\n",
       "      <td>trumark_financial_credit_union</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>783</th>\n",
       "      <td>symantec_vip</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1026</th>\n",
       "      <td>orange_county_s_credit_union</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1027</th>\n",
       "      <td>credit_union_west</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1028</th>\n",
       "      <td>spark_nz</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1029</th>\n",
       "      <td>absa</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1030</th>\n",
       "      <td>post_nord</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>252 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                            brand_id  brand_id_count\n",
       "779                     bain_capital               1\n",
       "780                          amfirst               1\n",
       "781                          del_one               1\n",
       "782   trumark_financial_credit_union               1\n",
       "783                     symantec_vip               1\n",
       "...                              ...             ...\n",
       "1026    orange_county_s_credit_union               1\n",
       "1027               credit_union_west               1\n",
       "1028                        spark_nz               1\n",
       "1029                            absa               1\n",
       "1030                       post_nord               1\n",
       "\n",
       "[252 rows x 2 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 1031 unique classes\n",
    "\n",
    "brand_id_count = pd.DataFrame(data['brand_id'].value_counts()).reset_index().rename(columns={'brand_id':'brand_id_count', 'index':'brand_id'})\n",
    "brand_id_count[brand_id_count['brand_id_count'] == 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "c565caa3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# join the count on the brand id\n",
    "data = pd.merge(data, brand_id_count, on='brand_id', how='inner')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "7f561461",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>brand_id</th>\n",
       "      <th>raw_text</th>\n",
       "      <th>brand_id_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>unknown</td>\n",
       "      <td>CPL  Loading Done  OK Trading Perpetual Spot T...</td>\n",
       "      <td>17882</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>unknown</td>\n",
       "      <td>Home Page - aflac.sparkfly.com  iframe srcwwwg...</td>\n",
       "      <td>17882</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  brand_id                                           raw_text  brand_id_count\n",
       "0  unknown  CPL  Loading Done  OK Trading Perpetual Spot T...           17882\n",
       "1  unknown  Home Page - aflac.sparkfly.com  iframe srcwwwg...           17882"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "52b91158",
   "metadata": {},
   "source": [
    "unknown means something that is not associated with any major brand, a random piece of text hosted on a website"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "28824697",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'CPL  Loading Done  OK Trading Perpetual Spot Trading Markets Assets Liquidity Mining Help Center Bulletin APP Download  Download Login Register Language English   Spanish English  Register Login Choose language Convert currency CPL The Most Secure Cryptocurrency Trading Platform 6448013135USD 24hour trading volume 30 Integrated liquidity providers 2000000 Users Sign up now CPL option trading system maintenance announcement More Currency Pairs Latest Price 24H Change 24H Vol 24H Turnover Markets BTC  USDT 230934 230934 01347 01347 10845K 250B ETH  USDT 158218 158218 01987 01987 13155K 20776M DOGE  USDT 009206 009206 42343 42343 113B 10535M LTC  USDT 9603 9603 20512 20512 69923K 6745M ADA  USDT 03821 03821 20759 20759 Loading 1551M TRX  USDT 006237 006237 03056 03056 Loading 1481M FIL  USDT 5138 5138 17403 17403 Loading 1205M LINK  USDT 6842 6842 15823 15823 Loading 991M EOS  USDT 1033 1033 28222 28222 Loading 842M USDC  USDT 09999 09999 001 001 Loading 561M DASH  USDT 5978 5978 07471 07471 Loading 484M XMR  USDT 1776 1776 00563 00563 Loading 419M BCH  USDT 1319 1319 10503 10503 Loading 244M ZEC  USDT 448 448 0885 0885 Loading 159M YFI  USDT 7193 7193 04016 04016 Loading 96331K DAI  USDT 10189 10189 01278 01278 Loading 76050K ETH  BTC 006852 006852 00394 00394 Loading 167K More Our Advantages CPL is a highperformance open source blockchain platform designed to support and operate a secure compliant and predictable digital infrastructure Safe and stable The topnotch technical team comprehensive security protection independent research and development of highspeed transaction engine stable and reliable operation under massive concurrent transactions Professional compliance Professional operation team years of experience in blockchain and financial industry holding a compliant digital asset trading license and a 100 deposit guarantee User first Facing the global market multilanguage support 724 hours uninterrupted operation strong community support and professional customer service High performance With 300000 transactions per second the order response time is less than 1 millisecond CPL Cryptocurrencies News More 20221214 Outlook for digital assets in 2022 Capital influx and supervision are all the focus Investors and industry insiders say regulation will play an important role in the direction of digital assets as many digital exchanges continue to operate in gray areas in many parts of Asia As digital assets grow in popularity among investors new types of blockchainbased securities and exchanges have emerged prompting regulators to act But rolling out the framework will take time leaving companies and investors in a regulatory bind A lot of regulators take an eitheror approach Either I dont want to do this I dont want to watch this or I think I can come up with convincing rules I think regulators should want to regulate and that takes a group that wants to come up with and Companies that follow these rules will cooperate Henry Chong founder of Hong Kongbased digital exchange Fusang told AsianInvestor Hong Kong and Singapore arguably the leading jurisdictions for digital assets in the region have been rolling out consultation papers and regulatory frameworks to regulate digital transactions In May Hong Kongs Financial Services and Treasury Bureau FSTB said it would require virtual asset exchanges to obtain a license from the Securities and Futures Commission SFC Currently virtual exchanges operate under an optin regime and are licensed under the same regulations as traditional trading and securities firms In Singapore which bills itself as an innovation hub fintech companies are setting up shop in the country where fintech funding has tripled to 3 billion in 2021 Crypto exchange Cryptocom moved its headquarters from Hong Kong to Singapore this year where other global cryptocurrency firms such as Coinbase Binance and Gemini have set up operations That said the Monetary Authority of Singapore MAS is still screening 400 license applications from digital token players Only a handful have been approved and about 70 are currently operating on temporary exempt licenses It remains to be seen how long these fintech companies are willing to wait for regulatory approval Binance Singapore announced on Monday December 13 that it will withdraw from the country breaking rumors that it will set up a global headquarters in Singapore In response to Binance Singapores departure a MAS spokesperson said Applicants may withdraw their applications as they deem fit in which case those operating under the exempt conditions will be required to cease providing regulated payment services Binance Asia Services has provided MAS with a plan for an orderly cessation of its regulated payment services MAS approach to regulation under the Payment Services Act is designed to foster innovation while ensuring appropriate controls are in place to address key risks such as money laundering and terrorist financing the spokesman said investor interest More than a decade after Bitcoin was founded its clear that the digital asset is here to stay While conservative asset owners continue to shy away from cryptocurrencies they have begun investing in the underlying technology and other blockchainenabled securities Cryptocurrency is a digital asset said Ankit Khandelwal chief investment officer at multifamily office Maitri Asset Management We are more interested in the underlying technology in these cryptocurrencies mainly blockchain technology which can have huge potential uses in various industries This technology makes a lot of sense and could have many use cases as well as the world of new developments in virtual spaces and metaverses he said Despite expressed interest Maitris family office clients have reservations about the digital asset space with most still in the learning and evaluation phase Very few clients are really investing in this space because its an evolving field in itself Theres a lot thats unknown here he said But one of the things weve heard from most investor feedback is that this technology shows a lot of promise Some of these uses include REITs funds or even just contracts legal contracts smart contracts which can be used in There are various use cases in different industries and in supply chain logistics Regulatory plus or minus Regardless of which direction regulation is headed experts agree that regulation is on the rise  and that will only give investors more confidence in digital assets and the underlying technology There are several digital asset classes that will increasingly come under the watchful eye of regulators including tethered stablecoins including private native cryptocurrencies said Benjamin Quinlan chairman of the Hong Kong Fintech Association All of this means that the regulatory arbitrage window is now closing fast or even disappearing people are going to need to play more of the regulated game That said its a plus and a minus for the industry But overall In general the tilt towards legality means that in order to achieve real institutional adoption the product set around the digital asset space needs to be inherently safer he said One benefit of increased regulation he added is that fintech companies are starting to realize that if they want to operate as a licensed financial institution theyre starting to understand that you cant just keep making products like crazy to create securitylike tools On the other hand a lot of brokers understand that business is being lost from them that their client base wants to trade digital assetsand now brokerages are even saying to us do we need to launch our own exchange to remain relevant is a very commoditized space so it will be interesting to see which exchanges thrive he added View Details Crypto Prices Today Bitcoin Holds Near 17000 Litecoin Up Up To 5 20221226 New Delhi Cryptocurrency markets were mixed on Monday Bitcoin Litecoin and Yearn Finance gained on the trade while Binance and Dogecoin lost money on the trade Bitcoin gained 031 and held near 17000 while Ethereum was comfortably trading above 1200 Litecoin meanwhile outperformed its larger peers gaining 5 Cardano was up 02 while XRP Polygon and Tron were also higher However Dogecoin is down around 15 The global cryptocurrency market cap stands at just over 812 billion up 012 over the past 24 hours The total transaction volume of DeFi is currently 161 billion accounting for 711 of the entire cryptocurrency market The trading volume of all stablecoins is about 2068 billion accounting for 9130 of the entire cryptocurrency market Bitcoin the worlds largest cryptocurrency according to CoinMarketCap Hovering around 325 billion Chinas 4002 percent dominance was up 007 percent on the day Cryptocurrency shopping carts a quick glance Source coinmarketcapcom data as of 1041 hours IST 26 December 2022 Bitcoin 1688888 034 Ethereum 122133 010 Tether 1 001 USD Coin 1 USD 001 Binance Coin 24420 035 Ripple 035 016 Dogecoin 007607  144 Cardano 026 027 Polygon 08012 064 Polkadot 449 105 TRON 005483 028 Litecoin 6892 483 Shiba Inu 00000008338 008 Solana 1137 063 Note Price change in last 24 hours what is india doing Policies related to cryptoassets and their ecosystems are set by the Ministry of Finance and any relevant legislation will only come into force through international cooperation after a proper risk assessment Parliament was informed last week By definition cryptoassets are borderless and require international cooperation to prevent regulatory arbitrage Therefore any legislation on this topic will only be effective if there is significant international cooperation in assessing the risks and benefits and evolution of common taxonomies and standards  Minister of State for Finance Pankaj Chaudhary told the Lok Sabha in a written reply View Details Under the chaos of the digital asset trading platform FTX will the cryptocurrency be in trouble 20221216 If Bitcoin Bitcoin is a boxer it must be the kind of tough fighter who refuses to give up forever But in the past few weeks it has taken a beating with the collapse of digital asset industry giant FTX and the arrest of its founder Sam BankmanFried in the Caribbean Bitcoin is used to blows of all kinds but after its most battered year to date the antiestablishment warrior is now being pushed to the brink If Bitcoin were a boxer it would be lying in the ring by now staring at gold stars It fell but has it lost The Rise of Bitcoin Bitcoin has a doganddog story like the American movie Rocky In 2009 it was still equivalent to an Internet forum in the no matter what zone Bitcoins prospects were insignificant at that time and its value was only a few pennies But over the years its crowd of promoters has grown helping Bitcoin to develop against the mainstream Bitcoins value rose to tens of thousands of dollars and it began to gain recognition and popularity in some places It began to be accepted by some exclusive websites or trendy coffee shops Gradually against everyones expectations Bitcoin began to be sought after  like a champion boxer Thousands of imitators like Ethereum Dogecoin and Litecoin started appearing before Bitcoin peaked in 2021 with a surge in fame and value People started pouring money into it and other cryptocurrencies with a virtual coin worth nearly 70000 57200 Mainstream groups have also begun to turn to invest in Bitcoin and cryptocurrency projects watershed Then in November of that year Bitcoin started to drop and has been going lower ever since Ongoing price drops and scandals have brought it to multiyear lows in terms of value credibility and appeal Its been a bad time for the cryptocurrency space and we may have worse days to come after the recent FTX scandal Its a watershed moment for cryptocurrencies said the Caribbean Blockchain Alliance Chairman Stefen Deleveaux said FTXs crash last month was the biggest shakeout in the crypto space in years It is the second largest trading platform in the world  the entry point for millions of people into the cryptocurrency space It was once regarded as one of the most trusted platforms but collapsed into bankruptcy just days after it was revealed that it was financially unstable FTX founder Sam BankmanFried is currently in custody accused by the US of building a house of cards based on fraud while telling investors its one of the safest structures in crypto  BankmanFried told the BBC that he wishes he hadnt killed cryptocurrencies because of it FTX certainly took a hit but 2022 itself has dealt blows to cryptocurrencies repeatedly Weve never seen anything like this in the cryptocurrency space before said Prof Carol Alexander from the University of Sussex Business School She predicted last year that cryptocurrencies would crash in 2022 but admitted she was taken aback by how often these events occurred wave after wave The first major blow came in May when two popular digital currencies collapsed costing Bitcoin and the entire crypto ecosystem 400 billion in value Do Kwon who created the Terra currency is now wanted by South Korean authorities allegedly hiding in Serbia A series of other scandals large and small have shaken trust in the cryptocurrency ecosystem such as Kim Kardashian being fined 126 million for promoting an ultimately failed cryptocurrency In addition there is also the collapse of the nonfungible token Non Fungible Token market that was once booming In the past NFTs were sold for millions of dollars but now they are difficult to sell Confidence has also been hit by a spate of hacks against cryptocurrency companies the largest of which stole 600 million from operator Ronin Network As Bitcoin and all other virtual currencies shrink in value big virtual currency firms like Celsius Three Arrows Capital and BlockFi go bust costing investors big and small police Also investigating what happened The value of a single bitcoin often seen as a barometer of the entire virtual currency ecosystem is now hovering at less than 18000  a 70 drop from its November 2021 peak Can cryptocurrencies make a comeback The fall of trading platforms such as FTX has left a huge hole in the industry which has raised other concerns More than 14 billion was withdrawn from the Binance exchange on Tuesday This appears to be due to negative media coverage of cryptocurrencies Changpeng Zhao the companys founder urged calm on Twitter saying it was business as usual However Andy Renshaw senior vice president of product management at Feedzai said that cryptocurrencies will need to rely on a strong and diverse crowdfunding platform to survive this ongoing battle Without a trusted place to trade safely cryptocurrencies are unlikely to return to the status of champion boxers let alone grab gold beltsat least some basic skills still need to be corrected in training camp Digital currency is of course only a part of the encryption ecosystem but most peoples estimates are not optimistic about its value rising in the short term As a speculative asset cryptocurrencies appear to have a bleak future according to Prof Omid Malekan of Columbia Business School Prices have fallen and many of the largest providers of investmentrelated services have gone bust But he said that cryptocurrency is still first and foremost a technology to which he said Its in good shape better than ever in some ways The adoption of bitcoin and socalled stablecoins in developing countries without reliable financial infrastructure shows how the technology is improving peoples lives Professor Malekan said DeLorevo also believes that the cryptocurrencys ups and downs overshadow its overall progress He noted that cryptocurrencies are being used to help charities access funds in places where normal currency cannot be obtained due to war and economic depression He also said that the recent scandal is an opportunity to drive the scammers away in the crypto space Crypto is basically a very short summary of the future of the digital global economy said Professor Alexander She pointed out that the metaverse environment built with encryption technology has been regarded by some people as the form of future work and life View Details What are cryptocurrency prices today December 24 2022 20221224 Cryptocurrency Prices Today December 24 2022 Solana has lost 495 in value over the past 24 hours while Terra Classic and Decentraland have seen notable increases in value Latest Release Celebrities Who Lost Millions on NFTs The market fell for a third straight day after trading in the green flag for two straight days Despite this the value of various cryptocurrencies has increased At the time of writing Solana is down 495 making it the biggest loss in the past 24 hours Dogecoin price surged yesterday but is down 173 as of today The price of Uniswap fell by 153 The value of Toncoin fell by 136 In the past day the price of Terra Classic increased by 551 The value of Decentraland has increased by 509 Cronos gained 327 The total value of all cryptocurrencies currently on the market is estimated at 81048 billion down 023 over the past 24 hours The total cryptocurrency market fell by 1628 and is currently worth 2493 billion The top 4 cryptocurrencies today are as follows Solana SOL down 495 Over the past day the value of the cryptocurrency Solana has dropped by a massive 495 The current market cap is 4023 billion The total transaction volume decreased by 1673 The price of one SOL token is currently 1134 Dogecoin DOGE has lost 173 in value Dogecoins market capitalization fell 173 to 10389 billion in the past 24 hours Currently one DOGE coin can be purchased for 007701 Transaction volume also fell by 5450 Terra Classic LUNC gained 551 After rising 551 Terra Classics current market cap is 99458 billion Compared with the previous trading day the trading volume decreased by 753 percentage points The current price of one LUNC cryptocurrency coin is 00001452 The value of Decentraland MANA has increased by 509 In the past twentyfour hours the value of the cryptocurrency Decentraland has increased by 509 The current market value is 613706 million At this point one MANA token can be purchased for 033302 The recent transaction volume has increased by 32137 finally Cryptocurrency Prices Today December 24 2022 The cryptocurrency market has yet to recover from an adverse trend following the FTX fiasco and the Feds decision to raise interest rates No evidence of dust buildup Still the fact that some coins are doing well makes the end of the year very exciting View Details About US About CPL CPLGlobal Pro Station an innovative digital asset trading platform serving professional trading users worldwide is dedicated to discovering highquality innovative digital asset investment opportunities It currently provides trading and investment services for over forty digital asset products CPL Group is a leading global provider of financial services for blockchain assets It has provided highquality services to millions of users from over 130 countries worldwide It has independent offices in USASingapore Korea Hong Kong and other countries and regions Trading business and operation centers CPL Group and its subbrands are global leaders in technology platform product branches security and risk control system operation and customer service system In additionCPL other subbrands with global operations include CPL Japans digital asset trading platform and CPL Wallet digital asset management services Trade anytime anywhere Perfectly compatible with multiple terminals meeting the transaction needs of various scenarios at any time View live quotes at any time Buy and sell BTC ETH XRP and other digital assets with ease Get alerts on price changes in the Cryptocurrencies you follow View bitcoin spot loan term loan and option prices Compare Cryptocurrency prices on various exchanges App Store Android Download CPL English   Spanish English About Bulletin Service Agreement Support Help Center Online Customer Service Suggestions and Feedback Service Download Coin Application Privacy Statement CopyRight 20182023 CPL All Rights Reserved About Bulletin Service Agreement Support Help Center Online Customer Service Suggestions and Feedback Service Download Coin Application Privacy Statement English   Spanish English CopyRight 20182023 CPL All Rights Reserved Contact Us '"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# seeing what an unknown class looks like\n",
    "\n",
    "data[data['brand_id'] == 'unknown']['raw_text'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "dc6a3f90",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Roblox Robux Generator 2022  Account Information Enter your username Platform Android iOS PC Android Android iOS PC Continue Roblox Robux Generator You can safely use Roblox Generator to add Unlimited Robux to your Roblox account This is the official Roblox Robux Generator updated for 2022 Generator is easy to use Just Enter your username and select how much Robux you want to add in your account Robux Amount 1000 MAX AMOUNT 1000 5000 25000 50000 100000 20 Generate Processing Human Verification  Complete ONE offer After completion your robux will be send to your account Verify Now Account Username Selected Platform Robux Amount 0 1 Recent Activity Suadero on PC 50000 Robux Ermes Posted a minute ago Thank you it really works  Adelia Conti Posted a minute ago Once again I got all resources D I cant thank you enough 3 Abelardo Posted few seconds ago wow this website is giving some real stuff  D Add a new comment Post To prevent spam commenting is only allowed for users who already used our generator Live Chat Generator Chatroom   Enter your Chat Username Save Welcome to the chatroom posting links or spamming will result in a kick 1259 LearningToPIay Do Robux expire Send Copyright  2022 All rights reserved a href target_blankimg srcsstatic1histatscom0gif4275781101 altcounter border0a '"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# seeing what an non-unknown class looks like\n",
    "\n",
    "data[data['brand_id'] == 'roblox']['raw_text'][17885]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "45ae6490",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average length of text with class unknown:  4540.652\n",
      "Average length of text with class non-unknown:  3842.516\n"
     ]
    }
   ],
   "source": [
    "# comparing avg length of unknown vs avg length of non-unknowns\n",
    "\n",
    "avg_len_text_unknown = data[data['brand_id'] == 'unknown']['raw_text'].apply(lambda x: len(x)).mean()\n",
    "avg_len_text_not_unknown = data[data['brand_id'] != 'unknown']['raw_text'].apply(lambda x: len(x)).mean()\n",
    "\n",
    "print('Average length of text with class unknown: ', round(avg_len_text_unknown,3))\n",
    "print('Average length of text with class non-unknown: ', round(avg_len_text_not_unknown,3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "c58ce8c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "data['length_raw_text'] = data['raw_text'].apply(lambda x: len(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f220bb0d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    76069.000000\n",
       "mean      7763.082898\n",
       "std       7160.162808\n",
       "min          1.000000\n",
       "0%           1.000000\n",
       "1%           4.000000\n",
       "10%         58.000000\n",
       "25%        395.000000\n",
       "50%       7659.000000\n",
       "75%      13000.000000\n",
       "90%      17882.000000\n",
       "99%      17882.000000\n",
       "100%     17882.000000\n",
       "max      17882.000000\n",
       "Name: brand_id_count, dtype: float64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# the frewucny dashboard can be updated regularly to see what website is having the most traffic or for a brand use case visualize the daily traffic on web\n",
    "\n",
    "data['brand_id_count'].describe(percentiles=[0, 0.01, 0.1, 0.25, 0.5, 0.75, 0.9, 0.99, 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "0bc6439c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def map_brand_freqeuncy(x):\n",
    "    if x <= 58:\n",
    "        return 'Low'\n",
    "    elif x > 58 and x <= 7659:\n",
    "        return 'Medium'\n",
    "    elif x > 7659 and x < 17882:\n",
    "        return 'High'\n",
    "    else:\n",
    "        return 'unknown_very_high'\n",
    "      \n",
    "data['brand_freq_cat'] = data['brand_id_count'].apply(map_brand_freqeuncy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "91d93dee",
   "metadata": {},
   "outputs": [],
   "source": [
    "data['length_raw_text_log_scaled'] = np.log10(data['length_raw_text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "22a09533",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAG4CAYAAAAQZHNbAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAAA0eElEQVR4nO3deZxcVZn/8c/TnSYLBALpsCQBMhqDAiYsAeUHBBjtQKMguOGMS4siroTBYQRnGAhMBpjRGaXRQRGBxg0RBgQlQMuQBRAwQBJkMURoJIQlHbYQtizP749zKlSV1dVV6bpVt6q+79erX12n7q17nrpVdZ865566x9wdERGRtGmpdQAiIiKFKEGJiEgqKUGJiEgqKUGJiEgqKUGJiEgqKUGJiEgqKUHJZjEzN7PJNaj3UDNbkcB2HzSzQ4ssn2dmJ5S4rURiLNVg9ZvZgWb2qJm9YmbHVDE0kbIoQSXAzPrM7LV4AHjGzC43s60SrvNyM5uTtm0NVbUSobvv4e7zYp2zzeynSddZQ+cA33P3rdz9uloHUy/i5/r9tY6jmShBJecod98K2AvYG/hmbcORtDKz1ipXuSvw4ACxmJnpuJAiZjas1jHUjLvrr8J/QB/w/qzyfwK/zSqfDvwZWAM8BBybtewJYN94+1OAA7vH8gnAdQXqOxFYB7wJvALcEO8fD1wDrAIeB2bF+7cDVhCSKMBWwHLgMwNtq0CdDkyOt4cD3wb+AjwL/AAYGZcdGuv6R+A54Gng+KztjAVuAF4G/gDMAW6PyxbEetbGWI4bbHt5MR4GPJBV/h1wT1b5duCY7NcMOCI+93WxziVx+Tzg34A74ut2C9A+QL2ZGP8Z6I/b/mTW8suBi4Ab43N7P/AB4P64H54EZmetPynuh664j/uBf8laPjJu8wXC++mfgBUDxPZnYCPwWnx+w+Nz+/f43F4DJgPvBHqB54E/AR/Pe82uj7HeE/fL7XmxDstafx5wQlb5c8DDMd6bgV3z3ldfAh6Ny78PWNbyL8THZj47+8Tne03e87wQ+O4A+2Bn4H8Jn4vVhNYkwNuB/4v39QM/A8bEZT/J22/fiPe/F7gTeBFYAhyaVc/fEN7Dawjvve8DP81afjThi8KLcR+9K+8YchqwFHij3OfYKH81D6AR/8hKUMBE4AHggqzlHyMkjxbCQXctsFNcdgXwj/H2xfGA8uWsZacMUOflwJyscgtwL3AmsAXwNuAx4PC4fCbwDLA98CPg6oG2NUB92Qnqu4QD1nbAaELCOS8uOxRYT+hWagOOBF4Fto3Lr4x/o4DdCQfn2wvVU8r28mIcEQ8o7cCw+HxXxhhHxmVjC7xms7MPJPG+efG1mBIfOw84f4B9k4nxvwkJ4JD4Gu+WtX9fAg6Mr9OI+Jh3x/JUQqI/Jq4/Ke6HH8W6pxEOWu+Ky88HFsb9vzPwRwZIUPnPNeu5/QXYI+6nbeLrcHws70M4YO+R9ZpdBWwJ7Ak8RYkJCjiG8GXoXXHbZwB35r3evwHGALsQksgRWZ+bp4D9ACMk0l2BneL+HRPXG0b48rJvgefeSkgk34nxjwAOissmAx3xNRtHSC7fLbLfJhCS2ZHxdeuI5XFx+e8JX9y2AA4iJPSfxmVTYswdhPfxN+J+2SKrrsXx9RxZznNspL+aB9CIf/HN9Qrhm5MDt2beWAOsvxj4ULz9eeD6ePthQqvpylh+AthngG1cTm6Ceg/wl7x1vglcllW+kJA8VxIP1IW2NUB9Hj/QFj84b89adgDweLx9KCERZB+wniN882wltFR2y1q2qQWVXU9WecDtDRDnQuDDsb5bCAfWIwitq6V5r9lgCeqMrPJXgJsGqPNQQoLaMuu+q4B/zdq/Vwyyf78LfCfenhT3w8Ss5fcAn4i3HyMexGP5RMpPUOdklY8DFuY95ofAWVmv2Tuzlp1L6QlqLvD5rGUthC8Yu2a93gfl7bfT4+2bgZMHeE5zgS/E2x8EHhpgvQMISW9YoeV56x4D3F9kv50G/CTvMTcTWrq7xPfAqKxlP+WtBPWvwFV5++EpYgss1vW5zXmOjfSnvubkHOPuowkHq3cSvsUDYGafMbPFZvaimb1I+BaaWT4fONjMdiQcDH4JHGhmkwjfbBeXWP+uwPhMHbGefwZ2yFrn4lj3Ze6+enOeJOGb5ijg3qx6bor3Z6x29/VZ5VcJ3YrjCN8En8xaln17IANtr5D5hNdgRrw9j9CiOSSWy/FMiXUCvODua7PKTxBazRk5z9PM3mNmt5nZKjN7idDN1U6ugeofn7e9J4rENZDsx+8KvCfvvfNJYEcKv2bl1LcrcEHWdp8nfMmZkLXOQM9zZ0IrtpAeQpc48f9PBlhvZ+CJvPcPAGa2vZldaWZPmdnLhISS/xrkP5eP5e2ngwitnfHA8+7+atb62ftsPFn7zd03xuUTBli/nOfYMJSgEubu8wnfmL8NYGa7ErpqvkZotYwhdMlYXH854UM5C1jg7msIH9gTCd9SNw5UVV75SUIrZkzW32h3PzLG0Ur4VnwF8OW8kXL52yqmn9Ci2SOrnm08DBAZzCrCt8yJWfftXEbdpchPUPMZPEGV8/wHsq2ZbZlV3oXQUh2ojp8Tukl3dvdtCOfxrMS6niZ3v+1SZqz58TwJzM9772zl7l/mrddsoPoySXlU1n075m37i3nbHunud5YQ45OE80SFXAdMNbM9Ca2LnxXZxi4DDDw4j7Afprr71oQkkP0aFPqM/STvuWzp7ucTXpPtzCx7P2Tvs5WEBAeEwSlx+VNF6iv1OTYMJajq+C7QYWZ7Efq9nfBBx8yOJ7Riss0nJLDMAXReXrmQZwnnmTLuAV42s9PMbKSZtZrZnma2X1z+z/H/5wjJ84qs0WT52xpQTJg/Ar5jZtvH5zTBzA4v4bEbCCerZ5vZKDN7J2GgRrHnVa47gd2A/QkDJB4kthAI5xgKeRaYVIHRbGeb2RZmdjDhgPKrIuuOJnzjft3M9gf+vox6rgK+aWbbmtlE4KTNDxkI54CmmNmnzawt/u1nZu8q8JrtTujSAsDdVxEOsp+K77nPkZtUfhBj3QPAzLYxs4+VGNclwKlmtm8cbTg5fuHD3V8HriYk+nvc/S8DbOMeQvI438y2NLMRZnZgXDaa0DX/oplNIAxMyJb/XvwpcJSZHR6f64j4G7SJ7v4EsCjupy3M7ADgqKzHXgV8wMzeZ2ZthEE/bxDerwWV8RwbhhJUFcQP7RWEcxAPAf9FOIH6LOHE+B15D5lP+LAsGKBcyI+B3WNXw3XxQHIUYZj744SWziXANma2L/B14DNxvf8gJM3TC22rhKd4GuEE712xa+R3hKRQiq8Rui6fIXRZ/ILwQc2YDfTEWD5e4jY3id1s9wEPuvub8e7fE7p5nhvgYZlEstrM7iu3zugZwii0lYRvul9y90eKrP8V4BwzW0MY2HJVGXWdTeguepxwnm1IXT+x1T4T+AQh/mcI75HhcZWvEbrdniH0DlyWt4kvEA7uqwkDLzYddN392ritK+N75Y9AZ4lx/Yow2vDnhPO71xEGhmT0ED5PAz7/rM/FZMLAkBWEc24Q9uM+hAEsvyUk4mznAWfE9+Kp7v4k8CHCl71VhBbVP/HWcfWThHNeqwnnVn9JfG+7+58ILbQLCZ/Nowijat+kuEGfYyOxeMJNJBXM7D+AHd29a9CVJRXM7LOEQRAH1TiOXYBHCO+fl2sZSyFm9kvgEXc/awjbSPVzrDS1oKSmzOydZjY1dtnsTxjFeG2t45L6Ertjv04Y8ZqKA3fsFn27mbWY2RGE1tZ1Q9he6p5j0pr3F8qSFqMJ3XrjCcPF/wv4dU0jkroSB6M8S+jmPKLG4WTbkdBNOJbQlfhld79/czaU4ueYKHXxiYhIKqmLT0REUilVXXxHHHGE33TTTbUOQ0REqqvgb/5S1YLq7++vdQgiIpISqUpQIiIiGUpQIiKSSkpQIiKSSkpQIiKSSkpQIiKSSkpQIiKSSkpQIiKSSkpQIiKSSkpQIiKSSkpQIiKSSkpQIiKSSkpQIiKSSkpQIiJSsv7+fk466SRWr16deF1KUCIiUrKenh6WLl1KT09P4nUpQYmISEn6+/uZO3cu7s7cuXMTb0UpQYmISEl6enpwdwA2btyYeCtKCUpERErS29vLunXrAFi3bh233HJLovUpQYmISEk6Ojpoa2sDoK2tjZkzZyZanxKUiIiUpKurCzMDoKWlha6urkTrU4ISEZGStLe309nZiZnR2dnJ2LFjE61vWKJbFxGRhtLV1UVfX1/irScAy4zISIPp06f7okWLah2GiIhUlxW6U118IiKSSkpQIiKSSkpQkmrVvO6XiKSLEpSkWjWv+yUi6aIEJalV7et+iUi6KEFJalX7ul8iki5KUJJa1b7ul4ikixKUpFZHR8emy6qYWeLX/RKRdFGCktQ66qijNnXxuTtHH310jSMSkWpSgpLUuuGGG3JaUNdff32NIxKRalKCktTq7e3NaUHpHJRIc1GCktSq9twzIpIuSlCSWtWee0ZE0iXxBGVmY8zsajN7xMweNrMDkq5TGkO1554RkXSpxnxQFwA3uftHzWwLYFQV6pQGUc25Z0QkXRKdD8rMtgaWAG/zEirSfFAiIk2pJvNBvQ1YBVxmZveb2SVmtmVOVGYnmtkiM1u0atWqhMMREZF6kXSCGgbsA1zk7nsDa4HTs1dw94vdfbq7Tx83blzC4YiISL1IOkGtAFa4+92xfDUhYYmIiBSVaIJy92eAJ81st3jX+4CHkqxTREQaQzVG8Z0E/CyO4HsMOL4KdYqISJ1L/HdQ7r44nmOa6u7HuPsLSdcJmipcRKTeNeyVJDRVuIhIfWvIBKWpwkVE6l9DJihNFS4iUv8aMkFpqnARkfrXkAmqo6ODYcPCAMVhw4ZpmgYRkTrUkAmqq6uLjRs3AqGLTxcaFRGpPw2ZoEREpP41ZILq6emhpSU8tZaWFg2SEBGpQw2ZoHp7e1m/fj0A69ev1yAJEZE61JAJqqOjg7a2NgDa2to0SEJEpA41ZILq6urCLMx/1dLSokESIiJ1qCETVHt7O52dnZgZnZ2djB07ttYhiYhImapxNfOa6Orqoq+vT60nEZE6ZZlLAqXB9OnTfdGiRbUOQ0REqssK3dmQXXwiIlL/lKBERCSVlKBERCSVlKBERCSVlKBERCSVlKBERCSVlKBERCSVlKBERCSVlKBERCSVlKBERCSVlKBERCSVlKBERCSVlKBERCSVlKAk1ZYtW0ZnZyfLly+vdSgiAvT393PSSSexevXqxOtSgpJUO/vss1m7di1nnXVWrUMREaCnp4elS5fS09OTeF1KUJJay5Yt48knnwTgySefVCtKpMb6+/uZO3cu7s7cuXMTb0UpQUlqnX322TlltaJEaqunp4fMJLcbN25MvBWlBCWplWk9DVQWkerq7e1l3bp1AKxbt45bbrkl0fqUoEREpCQdHR20tbUB0NbWxsyZMxOtTwlKUuvQQw/NKR922GG1CUREAOjq6tp028xyyklQgpLUmjVrVtGyiFRXe3s7EyZMAGD8+PGMHTs20fqUoCS12tvbN7WiDjvssMQ/DCJSXH9/PytXrgRg5cqVGsW3uar5YzJJzqxZs5g2bZpaTyIp0NPTw8aNGwGN4huSav6YTJLT3t7OhRdeqNaTSAr09vayfv16ANavX69RfJuj2j8mExFpBgcffHBOecaMGYnW15AJqto/JpPkqKtWpHk1ZIKq9o/JJDnqqhVJj4ULF+aUFyxYkGh9DZmgOjo6aG1tBaC1tTXxH5NJMvr7+7nxxhtxd37729+qFSVSYx0dHQwbNgyAYcOG6Ye6m6Orq4sNGzYAsGHDhsR/TCbJ6OnpyWkJqxUlUltdXV20tIS00draqh/qbo7nn38+p/zCCy/UKBIZiptvvjmnfNNNN9UoEhGBMKq2s7MTM6Ozs1M/1N0cugp2Y8h0JQxUFpHq6+rqYurUqVXpmWrIBKWrYDeGV155pWhZRBpbQyYoaQyTJk0qWhaR6tOMuiLAGWeckVM+88wzaxSJiIBm1BXZZLvttsspb7vttjWKRERAM+pWxPjx44uWpT7kv/k1zFyktjSjbgWcc845OeU5c+bUKBIZivw3f/6wcxGpLv1QtwLUNdQY8l83vY4itdXV1ZUz3YZ+qLsZ1DXUGDITow1UFpHG1pAJSlcgEBGpPE1YWAGZC8UOVJb6oMEuIulS7S//DZmg1q5dW7Qs9WGnnXbKKU+YMKFGkYgIVP/yYw2ZoKQx3HfffTnlRYsW1SgSEYHqX36sIROUmRUtS33I/CBwoLKIVFe1Lz/WkAlKB7bGoHNQIunymc98Jqd8/PHHJ1pfQyYoaQy77rprTlkXixWprSuuuCKnfNlllyVanybYkdS65557csp33313jSIRaV7d3d0sX74cgL6+vpxlfX19zJo1C4DJkydvul0piScoM+sD1gAbgPXuPj3pOqUxbNiwoWhZRKpr+PDhvPHGGznlJFWrBXWYu/dXqS4REamQ7FbRsmXLOOGEEzaVL7roIiZPnpxY3ToHJSIiJZkyZcqmVtOkSZMSTU5QnQTlwC1mdq+ZnViF+kREJCG77rorLS0tVZlAtBoJ6kB33wfoBL5qZjOyF5rZiWa2yMwWrVq1qgrhiEgt9Pf3c9JJJyU+C6ska9SoUUydOjXx1hNUIUG5+8r4/zngWmD/vOUXu/t0d58+bty4pMMRkRrp7u5myZIldHd31zoUqROJJigz29LMRmduAzOBPyZZp4ikT39/P/Pnzwdg3rx5akVJSZJuQe0A3G5mS4B7gN+6u+a+EGky3d3dm67o4u5qRUlJEh1m7u6PAdOSrKOQ7bbbjueffz6nLCK1k2k9ZcybN682gUhdachh5tnJqVBZRKpL18eUzdGQCUpE0mXixIlFyyKFNGSCGjlyZNGyiFTX7Nmzc8rnnHNObQKRutKQCer1118vWhYRkfRryASl/u7G0NLSUrQs9WPOnDk5ZbWgpBT6xEtq6YtG4yg0TYPIYBoyQY0YMaJoWeqDElTj0Hlh2RwNmaDWrVtXtCwi1fXaa68VLYsUMugPdc3sQsIVyQty98pOoVgBmuhORKT+ldKCWgTcC4wA9gEejX97EWbJFRERqbhBW1Du3gNgZp8lzIy7LpZ/ANySaHQi0hC23357nnvuuU3lHXbYoYbRSL0o5xzUeGB0VnmreJ+ISFHnnntuTvm8886rUSRST8q5WOz5wP1mdlssHwLMrnhEItJwpkyZsqkVtcMOO1RlsjupfyW3oNz9MuA9hEkHrwUOyHT/iYgM5txzz2XLLbdU60lKVnKCMjMD3g9Mc/dfA1uY2f6DPExEBAitqLlz56r1JCUr5xzU/wAHAH8Xy2uA71c8IhFpSP39/Zx00kmaTVdKVk6Ceo+7fxV4HcDdXwC2SCQqEWk43d3dLFmyRLPpSsnKSVDrzKyV+KNdMxsHbEwkKhFpKP39/Ztm0b3tttvUipKSlJOgugmDI7Y3s38HbgfOLf4QERH+qtWkVpSUopxRfD8DvgGcBzwNHOPuv0oqMBFpHJnWU8Ztt91WeEWRLKVci2+7rOJzwC+yl7n780kEJiIiza2UH+reSzjvZFn3ZcoOvC2BuEREpMmVci2+v6lGICLSuEaOHJkzxYbmg5JSlDUflJlta2b7m9mMzF9SgYlI49hrr71yynvvvXdtApG6UvK1+MzsBOBkYCKwGHgv8HvgbxOJTEQaxuLFi3PK999/f20CkbpSTgvqZGA/4Al3PwzYG1iVSFQi0lDGjh1btCxSSDkJ6nV3fx3AzIa7+yPAbsmEJSKNZOXKlUXLIoWUM93GCjMbA1wH9JrZC4DeZSIyqI0bNxYtixRScoJy92PjzdlxTqhtgJsSiUpEGoqZ4e45ZZHBlDPdxnvNbDSAu88HbiOchxIRKSo7ORUqixRSzjmoi4BXsspr430iIiIVV06CMs/62uPuGynvHJaINKnW1taiZZFCyklQj5nZLDNri38nA48lFZiINI4NGzYULYsUUk6C+hLw/4CngBXAe4ATkwhKRESknFF8zwGfSDAWERGRTcoZxfefZrZ17N671cz6zexTSQYnIiLNq5wuvpnu/jLwQUIX3xTgnxKJSkQayrBhw4qWRQopJ0G1xf9HAr/QRIUiUqrzzz8/p/ytb32rRpFIPSnna8wNZvYI8BrwFTMbB7yeTFgi0kj233//nPK+++5bo0iknpTcgnL304EDgOnuvg54FfhQZrmZdVQ+PBFpBMuWLcspL1++vEaRSD0pqyPY3V/Iur2WcDWJjP8AeisUV1m6u7sHfcPPmjWLyZMnM2vWrCpFJSIZZ555Zk75jDPO4Morr6xRNFIvyppRdxC6+qOIFKTpNmRzVHIoTc2u/pjfKpox469nou/u7q5WODIEg7WGM6+1WsMija+SLajUOOWUU3LKp556ao0iERGRzVXJFlRfBbc1JMceeyzf+c53NpWPPvroGkYj5chuFaklLNLcyrmSxIcL/L3PzLYHcPcPJxdm+SZMmACo9VTPLrnkkpzypZdeWqNIRKQWymlBfZ4wzPy2WD4UuAuYYmbnuPtPKhzbkIwbN45x48ap9VTHpkyZklOePHlyjSIRkVoo5xzURuBd7v4Rd/8IsDvwBuGq5qclEZzIlClTaGlpUetJpAmVk6AmufuzWeXngCnxkkfrKhuWSDBq1CimTp2q1pNIEyqni2+hmf0G+FUsfxRYYGZbAi9WOjAREWlu5SSorwIfBg4i/Ci3B7gmTgN/WAKxiYhIEytnwkI3s9uBNwk/yr0nJicREZGKK2eY+ceBewhdex8H7jazjyYVmIiINLdyuvj+BdgvTv1OnG7jd8DVSQQmIiLNrZxRfC2Z5BStLvPxIiIiJSunBXWTmd0M/CKWjwNurHxIIiIi5Q2S+Ccz+whwIGEU38Xufm1ikYmISFMrd8LCa4BrEopFRERkk0ETlJmtofBcT0YYfb51xaMSEZGmN2iCcvfRpWzIzLbNnhJeRERkKCo5Cu/WCm5LRESaXCUTlFVwWyIi0uQqmaAGvOyRmbWa2f3xYrMiIiKDqtYPbU8GHq5SXSIi0gAS7+Izs4nAB4BLCi0XEREppJyLxf7VlO55971vgId+F/gGYUbeQts90cwWmdmiVatWlRqOiIg0uHJaUHtkF8ysFdg3U44z65K3zgeB59z93oE26u4Xu/t0d58+bty4MsIREZFGNmiCMrNvxh/rTjWzl+PfGsKU79cP8vADgaPNrA+4EvhbM/vpUIMWEZHGN2iCcvfz4o91v+XuW8e/0e4+1t1PH+Sx33T3ie4+CfgE8H/u/qnKhC4iIo2snC6+5dmFOHT8rArHIyIiApSXoN5nZjea2U5m9m7gLqCkyyABuPs8d/9g2RGKiEhTKme6jb83s+OAB4BXgb9z9zsSi0xERJpaOcPM30H4we01QB/waTMblVBcIiLS5Mrp4rsBONPdvwgcAjwK/CGRqEREpOmVM2Hh/u7+MoRJoID/MrPBhpmLiIhslnJaUCPN7MdmdhOAme0OzEgmLBERaXblJKjLgZuBnWJ5GfAPFY5HREQEKC9Btbv7VcRr6rn7emBDIlGJiEjTKydBrTWzscR5n8zsvcBLiUQlIiJNr5xBEl8nXHvv7WZ2BzAO+FgiUYmISNMrJ0E9SBhevhth7qc/Ub0JD0VEpMmUk2B+7+7r3f1Bd/+ju68Dfp9UYCIi0twGbUGZ2Y7ABMIw8715a+bcrQFdSUJERBJRShff4cBngYnAf/FWgnoZ+OdkwhIRkWY3aIJy9x6gx8w+4u7XDLSemXXFdUVERIas5HNQxZJTdPIQYxEREdmkkqPwbPBVRERESlPOMPPBeAW3JSIiVdbd3c3y5cuLrvPoo48CMGvWrEG3N3ny5JLWG0glE5RaUCIidWz58uU8sngxOxZZJ9Pt9uLixUW39UwF4qlkgtLsuiIidW5H4PMVaG/8uAKdaiUnKDP7M3AXsBBY4O4PZS93968NORoREZGonEESuwM/BMYC3zazx8zs2mTCEhGRZldOF98GYF38vxF4FnguiaCKKeUkHpR+Im+oJ/Fk86XthKyIpEs5Cepl4AHgv4EfufvqZEIqbvny5dz/wENsHLVd0fXszdD/ee+fBz5V1/Lq8xWNTcqzfPlylv3xPnbZauBpxbZYFxr5r/f9oei2/vJKa0VjE5HaKydB/R1wEPAV4AQzu5NwLurWRCIrYuOo7Xh99w8OeTsjHvpNBaKRodhlqw2cMf2VIW9nzqKtKhCNiKRJyQnK3X8N/NrM3gl0EqZ7/wYwMpnQRESkmZU8SMLMrokj+S4AtgQ+A2ybVGAiItLcyuniOx+4z90HPmEgIkJpA2AyA1o0uEUGUk4X3x/MbE8z2x0YkXX/FYlEJiIiTa2cH+qeBRxK+D3UjYTzULcDSlAikiO/RTRjxoy/Wqe7u7ta4UidKueHuh8F3gc84+7HA9OA4YlEJSIiTa+cBPWau28E1pvZ1oQf6b4tmbBEpJEsWLCgaFmkkHIGSSwyszHAj4B7gVeAe5IISkREpKQEZWYGnOfuLwI/MLObgK3dfWmSwYlI49hrr70AnXuS0pWUoNzdzew6YN9Y7kswJhGpI7o+piSlnC6+u8xsP3cvflE0EWkqy5cv5/4H74cxg6y4Mfy7/6n7B17nxQoFJQ2hnAR1GPBFM3sCWEuYQdfdfWoikYlI/RgDGw/dOOTNtMwrZ9yWNLpyElRnsYVmtq27vzDEeESkzqxYsQJeqlByeRFW+Iqhb0caQjlXknhikFVuBfYZWjgiIiJBJdvTQ5/EXkTqzsSJE0tb8ZX4V6ntScMrp4tvMF7BbYlInZg8eXJJ62VG8b1jwjsGXmlC6duTxlfJBFUVK1asoOXVlyoy2WDLq6tZsWJ9BaISaV6lDgnPrKffQUmpKpmg1MUnZVmxYgVr17RWZDbcJ9a0suUKnVwXaSRlJSgzawV2yH6cu/8l3nxfBeMa0MSJE3n2jWEVm/J94sQdKxCViAxm8eLFABxyyCHMnz+/tsFIXShnuo2TgLOAZ9n0kzscmArg7s9XPDppaBMnTuT19U9zxvQSzpwPYs6irRihk+t1wV2nq9NqxYoVrAF+XIEhBU8DrwyxV6OcUXwnA7u5+x7u/u74px/pisig8ueDOuSQQ2oUidSTcrr4ngReSioQEWkcg12fz9015XsKTZw4kRf7+/l8BYYU/BhnzBB7NQZNUGb29XjzMWCemf0WeCOz3N3/e0gRiIiIFFBKC2p0/P+X+LdF/AP99klECtCU71IJgyYodz8bwMw+5u6/yl5mZh9LKjAREWlu5QyS+GaJ94mIiAxZKeegOoEjgQlmlt0m3xrQZRhERCQRpZyDWgksAo4G7s26fw1wShJBiYiIlHIOagmwxMx+7u7rqhCTiIhIWb+Dus/M8kftvURoXc1x99WVC0tERJpdOQlqLrAB+Hksf4JwgdiXgMuBoyoamYiINLVyEtSB7n5gVvkBM7vD3Q80s09VOjBpDn95pfjVzJ99NQw03WHUxgHXyWxnSkUjE5FaKydBbWVm73H3uwHMbH8gc2Sp6mi+llefH3Q+KHv9ZQB8xNZFtwO6mnmtlDIx3ZtxkrsRk4pMcgdMKXF7IlI/yklQJwCXmtlWhK69l4ETzGxL4Lwkgiuk9Nk71wDwjrcXS0A76qBWQ6Vcf02T3Ik0r5ITlLv/AXi3mW0DmLu/mLX4qkoHNhDN3iki0hzKmQ9qOPARYBIwzCxc7dbdz0kkMhERaWrldPH9mjBi716yrmYuIiKShHIS1ER3P6KcjZvZCGABMDzWdbW7n1XONkREpDmVc7HYO83s3WVu/w3gb919GrAXcISZvbfMbYiISBMqpwV1EPBZM3uckHgM8GLTvru7A6/EYlv80xxSIiIyqHISVOfmVGBmrYTzVpOB72d+R5W1/ETgRIBddtllc6oQEZEGVHIXn7s/AexM6LJ7Ani1lMe7+wZ33wuYCOxvZnvmLb/Y3ae7+/Rx48aVFbyIiDSukhOUmZ0FnMZbkxS2AT8t9fHxd1PzgLIGWoiISHMqZ5DEsYQ5odYCuPtKYHSxB5jZODMbE2+PBN4PPLJZkYqISFMp5xzUm+7umSk34iWOBrMT0BPPQ7UAV7l78YvoiYiIUF6CusrMfgiMMbMvAJ8DflTsAe6+FNh7CPGJiEiTKudafN82sw7CRWJ3A850997EIhMRkaZWTguKmJCUlEREJHGDJigzW0PhH9dmfqg78IRLIiIim2nQBOXuRUfqiYiIJKGcYeYiIiJVowQlIiKppAQlIiKppAQlIiKppAQlIiKppAQlIiKpVNYPdUVEpLE9A/y4yLyyq+P/sSVsZ8wQY1GCEhERACZPnjzoOqsefRSAMe94R9H1xpS4vWKUoEREBIBZs2aVvE53d3fS4egclIiIpJMSlIiIpJISlIiIpJISlIiIpJISlIiIpJISlIiIpJISlIiIpJISlIiIpJISlIgk7pRTTskpn3rqqTWKROqJEpSIJG7hwoU55dtvv71GkUg9UYISkcQtWrQop3zXXXfVKBKpJ0pQIiKSSkpQIiKSSkpQIpK4sWNzZw8aN25cjSKReqIEJSKJe+2113LKa9eurVEkUk+UoEQkcR0dHTnlmTNn1igSqSdKUCKSuK6uLrbYYgsAhg8fTldXV40jknqgBCUiiWtvb6ezsxMzo7Oz86/OSYkUoinfRaQqurq66OvrU+tJSqYEJSJV0d7ezoUXXljrMKSOqItPUm3x4sUsXryYGTNm1DoUEakyJSgREUklJShJrfxWk1pRIs2lYc9BLVmyBHfn2GOP5dprr611OFKi7u5uli9fPuDyWbNmATB58uRNt0WkMTVsC8rdAVi9enWNIxERkc3RkC2oD33oQzlltaLqR3arqFCXXnd3dzXDEZEaaogEld8t9MILL+QsX716NbNmzVK3kIhIHWnYLj4REalvDdGCym8VqWtIRKT+qQUlIlWxbNkyOjs7i47SFMmmBCUiVTFnzhzWrl3LOeecU+tQpE4oQYlI4pYtW0ZfXx8AfX19akVJSZSgRCRxc+bMySmrFSWlUIISkcRlWk8DlUUKUYISkcRNmjSpaFmkECUoEUnc1772tZzyySefXKNIpJ4oQYlI4np7e3PKN998c40ikXqiBCUiibv11ltzyr/73e9qFInUEyUoEUlcZnaBgcoihShBiUjiDj744JyyJp+UUihBiUjihg8fXrQsUogSlIgkbuHChTnlBQsW1CgSqSdKUCKSuI6ODlpbWwFobW1l5syZNY5I6oESlIgkrqura9PACHenq6urxhFJPVCCEhGRVFKCEpHE9fT00NISDjctLS309PTUOCKpB0pQIpK43t5e1q9fD8D69eu55ZZbahyR1AMlKBFJXEdHB2YGgJlpkISURAlKRBJ31FFH5QySOProo2sckdSDRBOUme1sZreZ2cNm9qCZ6RLGIk3ohhtuyGlBXX/99TWOSOpB0i2o9cA/uvu7gPcCXzWz3ROuU0RSpre3N6cFpXNQUopEE5S7P+3u98Xba4CHgQlJ1iki6dPR0ZFT1jkoKUXVzkGZ2SRgb+DuvPtPNLNFZrZo1apV1QpHRKoo/2KxhxxySI0ikXpSlQRlZlsB1wD/4O4vZy9z94vdfbq7Tx83blw1whGRKvve976XU77gggtqFInUk8QTlJm1EZLTz9z9f5OuT0TSp6+vr2hZpJCkR/EZ8GPgYXf/7yTrEpH0mjRpUtGySCFJt6AOBD4N/K2ZLY5/RyZcp4ikzBlnnJFTPvPMM2sUidSTYUlu3N1vByzJOkQk/bbbbruc8rbbblujSKSe6EoSIpK4np6enPmgdLFYKYUSlIgkrre3lw0bNgCwYcMG/VBXSqIEJSKJ6+jooK2tDYC2tjb9UFdKogQlIonr6urKuRafZtSVUihBiUji2tvbGT9+PADjx49n7NixNY5I6oESlIgkrr+/n6eeegqAlStXsnr16hpHJPVACUpEEpc9as/dNYpPSqIEJSKJ6+3tZd26dQCsW7dOo/ikJEpQIpI4jeKTzaEEJSKJyx7F19LSolF8UhIlKBFJXHt7O52dnZgZnZ2dGsUnJUn0Wny10tLSwsaNG3PKIlJbXV1d9PX1qfUkJWvII7emlxZJn/b2di688EK1nqRkDZmgvvjFL266bWY5ZakfU6dOzSlPmzatRpGISC00ZIJqb2/n8MMPB+Dwww/XN7Y6NXv27KJlEWlsDZmgILSipk2bptZTHWtvb9/Uipo2bZq+aIg0mYZNUOrvbgyzZ89m2rRpaj2JNKGGHMUnjSPzRUNEmk/DtqBERKS+KUGJiEgqKUGJiEgqKUGJiEgqKUGJiEgqKUGJiEgqKUGJiEgqKUGJiEgqKUGJiEgqKUGJiEgqKUGJiEgqKUGJiEgqKUGJiEgqKUGJiEgqaboNEREZUHd3N8uXL99UfvTRRwGYNWtWznqTJ0/+q/uGSglKRERKNnLkyKrVZe5etcoGM336dF+0aFGtwxARkeqyQnfqHJSIiKSSEpSIiKSSEpSIiKSSEpSIiKSSEpSIiKSSEpSIiKSSEpSIiKSSEpSIiKSSEpSIiKSSEpSIiKSSEpSIiKSSEpSIiKSSEpSIiKSSEpSIiKSSEpSIiKRSquaDMrNVwBO1jmMI2oH+WgfRxLT/a0f7vrbqff/3u/sR+XemKkHVOzNb5O7Tax1Hs9L+rx3t+9pq1P2vLj4REUklJSgREUklJajKurjWATQ57f/a0b6vrYbc/zoHJSIiqaQWlIiIpJISlIiIpJISlNSUmbmZ/SSrPMzMVpnZb8rczjwzmx5v32hmYyocalMys1fyyp81s+/F218ys88M8vhN68vmyX8NmkndJii98XMPynn3Tzez7kEeO8nM/phcdCVbC+xpZiNjuQN4aigbdPcj3f3FoQYmxbn7D9z9ilrHIY2rbhNUIzKzYZXYjrsvcvdZldhWlcwFPhBv/x3wi8wCM9vSzC41sz+Y2f1m9qF4/0gzu9LMlprZL4GRWY/pM7P2/CRsZqea2ex4e56ZfcfMFpjZw2a2n5n9r5k9amZzqvCc656ZzTazU+Pt/eJr8Xsz+1bel5/xZnZT3Lf/WaNwG4qZ7WVmd8V9fq2ZbWtm25vZvXH5tNg7sUss/9nMRtU26vKlJkENdDCJB5L/MLN7zGyZmR1c4LEfiB+MdjO73My6zexOM3vMzD4a17HMB8fMHjCz4+L9/2NmR8fb15rZpfH2581sTozrYTP7kZk9aGa3ZH3bz4/jXWZ2T95zWhpv72tm883sXjO72cx2ivfPM7NzzWw+8C9m9riZtcVlW8eDbVuRXfex/H1jZodmusjMbJyZ9ZrZfWb2QzN7wsza42NbS3leVXAl8AkzGwFMBe7OWvYvwP+5+37AYcC3zGxL4MvAq+4+Ffh3YN/NqPdNd58B/AD4NfBVYE/gs2Y2drOfTWMZaWaLM3/AOQOsdxnwJXc/ANiQt2wv4Djg3cBxZrZzUsE2kSuA0+L7/wHgLHd/DhhhZlsDBwOLgIPNbFfgOXd/tXbhbp7UJKhBDHP3/YF/AM7KXmBmxwKnA0e6e+ZaVDsBBwEfBM6P932Y8EGZBryfcKDbCVhAeDEBJgC7x9sHAQvj7XcA33f3PYAXgY8UCtLdHwa2MLO3xbuOA66KCeZC4KPuvi9wKeGgmjHG3Q9x97OBebzVmvgEcI27r9ucfROdRTjA7wNcC+yStayk55U0d18KTCK0nm7MWzwTOD0eHOcBIwjPYQbw06zHL92Mqq+P/x8AHnT3p939DeAxQAfR4DV33yvzB5yZv4KF832j3f3OeNfP81a51d1fcvfXgYeAXZMMuNGZ2TaEY8b8eFcP4fMAcCdwYCyfG/8fzFvHsrpSkS6lKvjf+P9ewoEs4zBgOjDT3V/Ouv86d98IPGRmO8T7DgJ+4e4bgGdji2U/wgv3D2a2O+HDs21MXAcAs4CxwOPuvniAGPJdBXyckBiPi3+7Eb6Z95oZQCvwdNZjfpl1+xLgG8B1wPHAF4rUBQPvm4yDgGMB3P0mM3sha1k5zytp1wPfBg4l7PMMAz7i7n/KXjnux8F+xLee3C9hI/KWvxH/b8y6nSnXy2cjDWyQ5dn7dgPat0laSEhIuxJ6BU4jfE7KGnSUFmlqQRU7mGTe4Plv7seA0cCUvG1lfyAs738Od38K2BY4gtCaWkhIMK+4+5oC2xvsA/ZL4ONmNiVs3h+NdT+Y9U303e4+M+sxa7PiuQOYZGaHAK3uPthAhoH2TUaxg0eaDhyXAue4+wN5998MnGQxI5nZ3vH+BcAn4317EroG8z0LbG9mY81sOKFFLRXm7i8Aa8zsvfGuT9Qynkbn7i8BL2Sd7vg0kGlNLQA+BTwav6Q/DxwJ3FH1QCsgTQlqcw4mTxC67q4wsz0GWXcBof+71czGEZq+mfNFvyd0kWUS1KlsZpPY3f9MONj/K2+1jP4EjDOzAwDMrG2QeK8gDBS4bHNiyHM7IeFiZjMJyTh13H2Fu19QYNG/AW3A0niO8t/i/RcBW8VzfN/grdcye5vrCOdM7iZ8g3wkidgFgM8DF5vZ7wlfil6qcTyNZJSZrcj6+zrQRThNsZRw6uIcAHfvi49ZEP/fDrwYv0TUndQ0td19nZllDiaPU+LBxN3/ZGafBH5lZkcVWfVaQrfdEkKT9xvu/kxctpDQTbjczJ4AtmNofba/BL4F/E2M8c04WKM79h8PA74LPDjA438GzCFrNNsQnA38Ig4KmU/oWlwDbFWBbQ+Zu/9VHO4+j3C+CXd/DfhigXVeY4Bv6u4+Ket2N/BXQ+7d/dBC9eUva3b5r4+7Xw5cHm/Pzlr0YDxhj5mdTjhBn7N+LKsVWyZ3H6gh8d5Cd7r7Llm3zyWci6pLuhZfCsVk9iF3/3QFtjUc2ODu62ML7qJ4slukYuIXoG8Svnw9AXzW3VfVNiqpd0pQKWNmFwKdhFGJyyqwvXcQBm60AG8CX3H3Pwx1uyIiSVOC2kxm9n3CcM5sF7h7Jc4b1awuEZG0UIISEZFUStMoPhERkU2UoEREJJWUoEREJJWUoKQpWZWmG4kXL/5okeUHx4v1Lq7hxXqz45lkZn9f6zhEQAlKZEBm1lqFaj4JfDteAuu1KtddyCRACUpSQQlKmtkwM+uxMKfO1WY2Kk5vcqaZ3U6YyuQLFuaiWmJm11icU8eKT+vyPTN7yMx+C2w/UOVmdgLhMlRnmtnP4jQpt5nZz4EH4mW5vhXrX2pmXyxUh4UZhIu10vaLcS6xMDXL6NhSWmhhGpb7zOz/xdXPJ0zRsNjMTqnIXhbZTKm51JFIDewGfN7d77AwD9hX4v2vu/tBAGY21t1/FG/PIVxz7sK4XmZal3cSrsZ+NeHK8bsR5j7agXCF/EsLVe7ul5jZQcBv3P1qMzsU2B/Y090fN7MTgZfcfb94RZA7zOwWYO9S6zCzLQiX3jrO3f9gYa6g14DngA53fz3+mPsXhJkBTgdO1SWJJA2UoKSZPRmvHg9hbqnMLMTZ05/sGRPTGML1C2/OWlZoWpcZvDWty0oz+78yY7rH3R+Pt2cCU7NaR9sQ5vAqp47dgKczVw/JTEtjYdLH75nZXoSLG+fPCCBSc0pQ0szyf6WeKa/Nuu9y4Bh3X2JmnyXMV5VRaFqXQtstR3bdBpzk7tlJETM7sow6bIB1TyHMIDCN0NX/evmhiiRL56Ckme2SmQKFMJvv7QXWGQ08bWFW5E+WsM0FhOnrWy1MfHnYEOK7GfhyrBszmxJbPuXU8Qgw3sz2i9sYbWbDCK2xp2ML8NOESTQhXOl+9BBiFqkYJShpZg8DXXFOne0Ic0zl+1fCFDC9lDYFzLXAo4Rp5C/irYnkNsclhPNL98Uh8T8k9HqUXIe7v0mY1flCM1tCeB4jgP8hPPe7CN17mZbbUmB9HFChQRJSU7oWn0idM7PLiQMtah2LSCWpBSUiIqmkFpRIFZjZtcQZlrOclj8AIu11iFSTEpSIiKSSuvhERCSVlKBERCSVlKBERCSVlKBERCSV/j8X386b4sn0/wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.catplot(y='length_raw_text_log_scaled', x='brand_freq_cat', hue='brand_freq_cat',\n",
    "            data=data, height=6, kind=\"box\")\n",
    "plt.title('Raw text length with brand frequency category')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "4382fa3f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    7.606900e+04\n",
       "mean     4.006631e+03\n",
       "std      1.295899e+04\n",
       "min      2.800000e+01\n",
       "25%      1.333000e+03\n",
       "50%      2.280000e+03\n",
       "75%      3.718000e+03\n",
       "max      2.089116e+06\n",
       "Name: length_raw_text, dtype: float64"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['length_raw_text'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "dc3e9775",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmoAAAE/CAYAAAD2ee+mAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAAAn5klEQVR4nO3de7hV9X3n8fdXIEIUvCJRjhFsULloMFDUZkAajJIxqZpqaiaNGG2IKblNLl5Sm8FMnTGOzUNsG6c2RtDYKvUyoolGQ2XUDAkeI8YAGqkSPYECEi8kKhH8zh97gZvDuWzg7LPXOef9ep797LV+e63f+p6zFT78fusSmYkkSZLKZ49GFyBJkqS2GdQkSZJKyqAmSZJUUgY1SZKkkjKoSZIklZRBTZIkqaQMapI6FBH/OyL+uov6emdE/DYi+hXriyLiL7qi76K/eyJiRlf1txPH/ZuIeCEi/qO7j91GLXMj4m8adOxVEXFSI44t9VYGNakPK/5ifS0iNkbESxHx/yLigojY9mdDZl6Qmf+9xr46/Es6M5/LzL0zc0sX1D47Ir7Xqv8PZOa83e17J+s4FPgSMCYz39HG51MjoqWLjtVlfe2uRgZCqS8xqEn6UGYOBg4DrgAuAq7r6oNERP+u7rMkDgM2ZOa6RhciqfcxqEkCIDNfzswFwJ8BMyJiHGw/chIRB0bE3cXo228i4qGI2CMibgTeCdxVTG1eGBEjIiIj4vyIeA74t6q26tD2BxGxJCJejog7I2L/4lg7jB5tHbWLiOnAV4E/K473ePH5tqnUoq5LI+JXEbEuIm6IiH2Kz7bWMSMiniumLf+qvd9NROxT7L++6O/Sov+TgPuBQ4o65rbaby/gnqrPfxsRhxT7XhwR/x4RGyJiftXPfU1E3FrVxzciYmF7fXX2vUbEByNiadWI6TGtfp9fjoifF7//WyJiYNXnF0bEmohYHRF/UfzO3hURM4GPARcWddxVdcjx7fUnaecZ1CRtJzOXAC3A5DY+/lLx2VBgGJWwlJn5ceA5KqNze2fmlVX7nAiMBk5p55DnAOcBhwCbgatrqPFe4H8AtxTHe3cbm51bvP4YOBzYG/j7Vtv8J+BIYBrwtYgY3c4h/w7Yp+jnxKLmT2Tmj4APAKuLOs5tVefvWn2+d2auBj4HnF70dQjwIvAPxW5fAo6JiHMjYjJwPjCjg77aFRHvAb4LfAo4APhHYEFE7Fm12UeA6cBI4Jjid0YRhr8InAS8q6h16891LXATcGVRx4c660/SrjGoSWrLamD/NtrfAA4GDsvMNzLzoez8gcGzM/N3mflaO5/fmJm/KILIXwMfieJig930MeCbmflMZv4WuAQ4u9Vo3mWZ+VpmPg48DuwQ+Ipa/gy4JDM3ZuYq4G+Bj+9GbZ8C/iozWzJzEzAbODMi+mfmq8CfA98Evgd8NjN39by0TwL/mJk/zcwtxfl7m4Djq7a5OjNXZ+ZvgLuA8UX7R4DrM3NZUdNlNR6zvf4k7QKDmqS2DAd+00b7/wJWAvdFxDMRcXENfT2/E5//ChgAHFhTlR07pOivuu/+VEYCt6q+SvNVKqNurR0IvK2NvobvRm2HAXcU05EvASuALVtrK0Y1nwECmL+bx/nS1uMUxzqUyu9mq/Z+B4ew/XfT2ffYWX+SdoFBTdJ2IuIPqYSQh1t/VowofSkzDwc+BHwxIqZt/bidLjsbcTu0avmdVEbtXgB+B7y9qq5+VKZca+13NZWgUt33ZmBtJ/u19kJRU+u+fl3j/m3V+Tzwgczct+o1MDN/DRARs4A9i5/hwk766sjzwOWtjvP2zPyXGvZdAzRVrR/a6vOdrUXSLjCoSQIgIoZExAeBm4HvZeYTbWzzweJk8gBeoTIKtPVWG2upnMO1s/48IsZExNuBrwO3Frfv+CUwMCJOjYgBwKVUwstWa4ERUXUrkVb+BfivETEyIvbmrXPaNu9McUUt84HLI2JwRBxG5dyt73W853Z1HrD1QobC/y76OwwgIoZGxGnF8hHA31CZ/vw4lRP2x3fQV0f+CbggIo6Lir2K3+fgGvadD3wiIkYX383X2vi5duX7lrQTDGqS7oqIjVRGX/6KyrlRn2hn21HAj4DfAouBb2fmouKz/wlcWkyxfXknjn8jMJfKlNlAKifak5kvA38JfIfK6NXvqFzIsNW/Fu8bIuJnbfT73aLvB4FngdeBz+5EXdU+Wxz/GSojjf9c9N+pzHySSmh8pvjdHAJ8C1hAZQp5I/AT4Lji/LnvAd/IzMcz82kqF2zcGBF7ttNXR8dupnKe2t9TuWBhJTWe3J+Z91C5sOOBYr/FxUebivfrgDFFHf+nlj4l7bzo/DxgSVJfV1wR+wtgz50dlZS06xxRkyS1KSLOiIi3RcR+wDeAuwxpUvcyqEmS2vMpYD3w71TORfx0Y8uR+h6nPiVJkkrKETVJkqSSMqhJkiSVVP/ON+mZDjzwwBwxYkSjy5AkSerUo48++kJmDm3d3muD2ogRI2hubm50GZIkSZ2KiF+11e7UpyRJUkkZ1CRJkkrKoCZJklRSvfYcNUmSBG+88QYtLS28/vrrjS5FwMCBA2lqamLAgAE1bW9QkySpF2tpaWHw4MGMGDGCiGh0OX1aZrJhwwZaWloYOXJkTfs49SlJUi/2+uuvc8ABBxjSSiAiOOCAA3ZqdNOgJklSL2dIK4+d/S4MapIkqVTmzp3LZz7zmUaXUQqeoyZJUh8ye3a5+9P2HFGTJEl1tWrVKsaNG7dt/aqrrmL27NlMnTqViy66iEmTJnHEEUfw0EMP7bDv97//fU444QReeOEFzj33XD73uc/xR3/0Rxx++OHceuutQOUk/a985SuMGzeOo48+mltuuQWAv/zLv2TBggUAnHHGGZx33nkAXHfddVx66aWsWrWK0aNH88lPfpKxY8dy8skn89prr9X717FTDGqSJKlhNm/ezJIlS5gzZw6XXXbZdp/dcccdXHHFFfzgBz/gwAMPBGDNmjU8/PDD3H333Vx88cUA3H777SxdupTHH3+cH/3oR3zlK19hzZo1TJkyZVv4+/Wvf83y5csBePjhh5k8eTIATz/9NLNmzWLZsmXsu+++3Hbbbd31o9fEqU9JXauteRDnRiS148Mf/jAAEyZMYNWqVdvaH3jgAZqbm7nvvvsYMmTItvbTTz+dPfbYgzFjxrB27VqgErw++tGP0q9fP4YNG8aJJ57II488wuTJk5kzZw7Lly9nzJgxvPjii6xZs4bFixdz9dVXs2HDBkaOHMn48ePbrKEMHFGTJEl11b9/f958881t69W3p9hzzz0B6NevH5s3b97Wfvjhh7Nx40Z++ctfbtfX1u2hMuVZ/d7a8OHDefHFF7n33nuZMmUKkydPZv78+ey9994MHjx4h/5a11AGBjVJklRXw4YNY926dWzYsIFNmzZx9913d7rPYYcdxu23384555zDsmXLOtx2ypQp3HLLLWzZsoX169fz4IMPMmnSJABOOOEE5syZsy2oXXXVVdumPXsCpz4lSVJdDRgwgK997Wscd9xxjBw5kqOOOqqm/Y488khuuukmzjrrLO666652tzvjjDNYvHgx7373u4kIrrzySt7xjncAMHnyZO677z7e9a53cdhhh/Gb3/ymRwW1aG+4sKebOHFiNjc3N7oMqe/xHDWpVFasWMHo0aMbXYaqtPWdRMSjmTmx9bZOfUqSJJWUQU2SJKmkDGqSJEklZVCTJEkqqboFtYg4MiKWVr1eiYgvRMT+EXF/RDxdvO9Xtc8lEbEyIp6KiFOq2idExBPFZ1fHzj56XpIkqQeqW1DLzKcyc3xmjgcmAK8CdwAXAwszcxSwsFgnIsYAZwNjgenAtyOiX9HdNcBMYFTxml6vuiVJksqiu6Y+pwH/npm/Ak4D5hXt84DTi+XTgJszc1NmPgusBCZFxMHAkMxcnJV7idxQtY8kSepl5s6dy+rVq7usv7333rvL+upu3XXD27OBfymWh2XmGoDMXBMRBxXtw4GfVO3TUrS9USy3bpckSTurq+9rWIf7JM6dO5dx48ZxyCGHdHnfPU3dR9Qi4m3AnwD/2tmmbbRlB+1tHWtmRDRHRPP69et3rlBJktTlVq1axejRo/nkJz/J2LFjOfnkk3nttdcAWLp0KccffzzHHHMMZ5xxBi+++CK33norzc3NfOxjH2P8+PHbtt1q6tSpbL2h/QsvvMCIESOASrj78Ic/zPTp0xk1ahQXXnjhDrW88MILnHDCCXz/+99n0aJFTJ06lTPPPJOjjjqKj33sY9ueGbpw4UKOPfZYjj76aM477zw2bdrEkiVLtj1A/s4772TQoEH8/ve/5/XXX+fwww/fVttFF13EpEmTOOKII3jooYd2+/fXHVOfHwB+lplri/W1xXQmxfu6or0FOLRqvyZgddHe1Eb7DjLz2sycmJkThw4d2oU/giRJ2lVPP/00s2bNYtmyZey7777cdtttAJxzzjl84xvf4Oc//zlHH300l112GWeeeSYTJ07kpptuYunSpQwaNKjm4yxdupRbbrmFJ554gltuuYXnn39+22dr167l1FNP5etf/zqnnnoqAI899hhz5sxh+fLlPPPMM/z4xz/m9ddf59xzz93Wz+bNm7nmmmt4z3vew2OPPQbAQw89xLhx43jkkUf46U9/ynHHHbftOJs3b2bJkiXMmTOHyy67bLd/d90R1D7KW9OeAAuAGcXyDODOqvazI2LPiBhJ5aKBJcU06caIOL642vOcqn0kSVLJjRw5kvHjxwMwYcIEVq1axcsvv8xLL73EiSeeCMCMGTN48MEHd+s406ZNY5999mHgwIGMGTOGX/3qVwC88cYbTJs2jSuvvJL3v//927afNGkSTU1N7LHHHowfP55Vq1bx1FNPMXLkSI444ojt6urfvz/vete7WLFiBUuWLOGLX/wiDz74IA899NB2zw7dOuq29efcXXU9Ry0i3g68H/hUVfMVwPyIOB94DjgLIDOXRcR8YDmwGZiVmVuKfT4NzAUGAfcUL0mN5jM8JdVgzz333Lbcr1+/HaYzd0b//v158803AXj99dc7PM7mzZu37TNhwgR++MMfbguG7W3f0TPQJ0+ezD333MOAAQM46aSTOPfcc9myZQtXXXXVDn1WH3931HVELTNfzcwDMvPlqrYNmTktM0cV77+p+uzyzPyDzDwyM++pam/OzHHFZ5/J3vokeUmS+oh99tmH/fbbb9t5XDfeeOO2EDV48GA2btzY5n4jRozg0UcfBeDWW2+t6VgRwXe/+12efPJJrrjiig63Peqoo1i1ahUrV67coa4pU6YwZ84cTjjhBIYOHcqGDRt48sknGTt2bE117IruuupTkiRpO/PmzeOCCy7g1Vdf5fDDD+f6668H4Nxzz+WCCy5g0KBBLF68eLvz1L785S/zkY98hBtvvJH3ve99NR+rX79+3HzzzXzoQx9iyJAhjBkzps3tBg4cyPXXX89ZZ53F5s2b+cM//EMuuOACAI477jjWrl3LlClTADjmmGM46KCDqOd9+KO3Dk5NnDgxt14VIqlOap36dIpUapgVK1YwevToRpehKm19JxHxaGZObL2tz/qUJEkqKYOaJElSSRnUJEmSSsqgJklSL9dbz0fviXb2uzCoSZLUiw0cOJANGzYY1kogM9mwYQMDBw6seR9vzyFJUi/W1NRES0sLPgO7HAYOHEhTU1PnGxYMapIk9WIDBgxg5MiRjS5Du8ipT0mSpJIyqEmSJJWUQU2SJKmkDGqSJEklZVCTJEkqKYOaJElSSRnUJEmSSsqgJkmSVFIGNUmSpJIyqEmSJJWUQU2SJKmkDGqSJEklZVCTJEkqKYOaJElSSRnUJEmSSsqgJkmSVFIGNUmSpJKqa1CLiH0j4taIeDIiVkTECRGxf0TcHxFPF+/7VW1/SUSsjIinIuKUqvYJEfFE8dnVERH1rFuSJKkM6j2i9i3g3sw8Cng3sAK4GFiYmaOAhcU6ETEGOBsYC0wHvh0R/Yp+rgFmAqOK1/Q61y1JktRwdQtqETEEmAJcB5CZv8/Ml4DTgHnFZvOA04vl04CbM3NTZj4LrAQmRcTBwJDMXJyZCdxQtY8kSVKvVc8RtcOB9cD1EfFYRHwnIvYChmXmGoDi/aBi++HA81X7txRtw4vl1u2SJEm9Wj2DWn/gPcA1mXks8DuKac52tHXeWXbQvmMHETMjojkimtevX7+z9UqSJJVKPYNaC9CSmT8t1m+lEtzWFtOZFO/rqrY/tGr/JmB10d7URvsOMvPazJyYmROHDh3aZT+IJElSI9QtqGXmfwDPR8SRRdM0YDmwAJhRtM0A7iyWFwBnR8SeETGSykUDS4rp0Y0RcXxxtec5VftIkiT1Wv3r3P9ngZsi4m3AM8AnqITD+RFxPvAccBZAZi6LiPlUwtxmYFZmbin6+TQwFxgE3FO8JEmSerW6BrXMXApMbOOjae1sfzlweRvtzcC4Li1OkiSp5HwygSRJUkkZ1CRJkkrKoCZJklRSBjVJkqSSMqhJkiSVlEFNkiSppAxqkiRJJWVQkyRJKimDmiRJUkkZ1CRJkkrKoCZJklRSBjVJkqSSMqhJkiSVlEFNkiSppAxqkiRJJWVQkyRJKimDmiRJUkkZ1CRJkkrKoCZJklRSBjVJkqSSMqhJkiSVlEFNkiSppAxqkiRJJWVQkyRJKimDmiRJUkkZ1CRJkkqqrkEtIlZFxBMRsTQimou2/SPi/oh4unjfr2r7SyJiZUQ8FRGnVLVPKPpZGRFXR0TUs25JkqQy6I4RtT/OzPGZObFYvxhYmJmjgIXFOhExBjgbGAtMB74dEf2Kfa4BZgKjitf0bqhbkiSpofo34JinAVOL5XnAIuCiov3mzNwEPBsRK4FJEbEKGJKZiwEi4gbgdOCebq1a0q6bPbu2NknSduo9opbAfRHxaETMLNqGZeYagOL9oKJ9OPB81b4tRdvwYrl1uyRJUq9W7xG192bm6og4CLg/Ip7sYNu2zjvLDtp37KASBmcCvPOd79zZWiVJkkqlriNqmbm6eF8H3AFMAtZGxMEAxfu6YvMW4NCq3ZuA1UV7UxvtbR3v2sycmJkThw4d2pU/iiRJUrerW1CLiL0iYvDWZeBk4BfAAmBGsdkM4M5ieQFwdkTsGREjqVw0sKSYHt0YEccXV3ueU7WPJElSr1XPqc9hwB3FnTT6A/+cmfdGxCPA/Ig4H3gOOAsgM5dFxHxgObAZmJWZW4q+Pg3MBQZRuYjACwmkXeFJ/ZLUo9QtqGXmM8C722jfAExrZ5/LgcvbaG8GxnV1jZIkSWXmkwkkSZJKqtOgFhE31tImSZKkrlXLiNrY6pXiaQET6lOOJEmStmo3qBXP3dwIHBMRrxSvjVRup7Gg2yqUJEnqo9oNapn5PzNzMPC/MnNI8RqcmQdk5sXdWKMkSVKfVMvU58rqlYjoFxH/rU71SJIkqVBLUJsWET+IiIMj4mjgJ8DgOtclSZLU53V6H7XM/C8R8WfAE8CrwEcz88d1r0ySJKmPq+X2HKOAzwO3AauAj0fE2+tclyRJUp9Xy9TnXcDXMvNTwInA08Ajda1KkiRJNT1CalJmvgKQmQn8bUR4ew5JkqQ6q2VEbVBEXBcR9wJExBhgSn3LkiRJUi1BbS7wQ+DgYv2XwBfqVI8kSZIKtQS1AzNzPvAmQGZuBrbUtSpJkiTVFNR+FxEHAAkQEccDL9e1KkmSJNV0McEXqTzb8w8i4sfAUOCsulYlSZKkmoLaMiq35TgSCOApahuJkyRJ0m6oJXAtzszNmbksM3+RmW8Ai+tdmCRJUl/X7ohaRLwDGE7l9hzHUhlNAxgC+GQCSZKkOuto6vMU4FygCfhb3gpqrwBfrW9ZkiRJajeoZeY8YF5E/Glm3taNNUmSJIkazlEzpEmSJDWGV29KkiSVVLtBLSLOKt5Hdl85kiRJ2qqjEbVLinenPiVJkhqgo6s+N0TEA8DIiFjQ+sPM/JP6lSVJkqSOgtqpwHuAG6ncnmOXREQ/oBn4dWZ+MCL2B24BRgCrgI9k5ovFtpcA51N56PvnMvOHRfsEYC4wCPgB8PnMzF2tSZIkqSdod+ozM3+fmT8B/igz/y/wM+DRzPy/xXqtPg+sqFq/GFiYmaOAhcU6ETEGOBsYC0wHvl2EPIBrgJnAqOI1fSeOL0mS1CPVctXnsIh4DPgFsDwiHo2IcbV0HhFNVEbmvlPVfBowr1ieB5xe1X5zZm7KzGeBlcCkiDgYGJKZi4tRtBuq9pEkSeq1aglq1wJfzMzDMvOdwJeKtlrMAS4E3qxqG5aZawCK94OK9uHA81XbtRRtw4vl1u2SJEm9Wi1Bba/MfGDrSmYuAvbqbKeI+CCwLjMfrbGWaKMtO2hv65gzI6I5IprXr19f42ElSZLKqZag9kxE/HVEjChelwLP1rDfe4E/iYhVwM3A+yLie8DaYjqT4n1dsX0LcGjV/k3A6qK9qY32HWTmtZk5MTMnDh06tIYSJUmSyquWoHYeMBS4vXgdCHyis50y85LMbMrMEVQuEvi3zPxzYAEwo9hsBnBnsbwAODsi9ixusjsKWFJMj26MiOMjIoBzqvaRJEnqtTq6PQcAxa0zPteFx7wCmB8R5wPPAWcVx1kWEfOB5cBmYFZmbin2+TRv3Z7jnuIlSZLUq3Ua1LpCcV7bomJ5AzCtne0uBy5vo70ZqOlKU0mSpN7Ch7JLkiSVlEFNkiSppDqd+oyIocAnqTzyadv2mXle/cqSJElSLeeo3Qk8BPyIyjM4JUmS1A1qCWpvz8yL6l6JJEmStlPLOWp3R8R/rnslkiRJ2k67I2oRsZG3HuH01YjYBLxRrGdmDumeEiVJkvqmdoNaZg7uzkIkSZK0vU6nPiNiYS1tkiRJ6lodTX0OBPYCDoyI/ahMeQIMAQ7phtokSZL6tI6u+vwU8AUqoexnVe2vAP9Qx5okSZJEx+eofQv4VkR8NjP/rhtrkiRJErXdR+3XEfHhVm0vA09k5ro61CRJkiRqC2rnAycADxTrU4GfAEdExNcz88Y61SZJktSn1RLU3gRGZ+ZagIgYBlwDHAc8CBjUJEmS6qCWJxOM2BrSCuuAIzLzN1RugCtJkqQ6qGVE7aGIuBv412L9T4EHI2Iv4KV6FSZJktTX1RLUZlEJZ++lci+1G4DbMjOBP65jbZIkSX1ap0GtCGS3Fi9JkiR1k1oeIfXhiHg6Il6OiFciYmNEvNIdxUmSJPVltUx9Xgl8KDNX1LsYSZIkvaWWqz7XGtIkSZK6Xy0jas0RcQvwf4BNWxsz8/Z6FSVJkqTagtoQ4FXg5Kq2BAxqkiRJdVTLVZ+f6I5CJEmStL1arvo8IiIWRsQvivVjIuLS+pcmSZLUt9VyMcE/AZdQPC4qM38OnF3PoiRJklRbUHt7Zi5p1ba5s50iYmBELImIxyNiWURcVrTvHxH3F/dmuz8i9qva55KIWBkRT0XEKVXtEyLiieKzqyMiav0BJUmSeqpagtoLEfEHVC4gICLOBNbUsN8m4H2Z+W5gPDA9Io4HLgYWZuYoYGGxTkSMoTJSNxaYDnw7IvoVfV0DzARGFa/pNf10kiRJPVgtQW0W8I/AURHxa+ALwAWd7ZQVvy1WBxSvBE4D5hXt84DTi+XTgJszc1NmPgusBCZFxMHAkMxcXDzO6oaqfSRJknqtToNaZj6TmScBQ4GjMvM/AWfU0nlE9IuIpcA64P7M/CkwLDPXFH2vAQ4qNh8OPF+1e0vRNrxYbt3e1vFmRkRzRDSvX7++lhIlSZJKq5YRNQAy83eZubFY/WKN+2zJzPFAE5XRsXEdbN7WeWfZQXtbx7s2Mydm5sShQ4fWUqIkSVJp1RzUWtmpk/kz8yVgEZVzy9YW05kU7+uKzVqAQ6t2awJWF+1NbbRLkiT1arsa1Noc0aoWEUMjYt9ieRBwEvAksACYUWw2A7izWF4AnB0Re0bESCoXDSwppkc3RsTxxdWe51TtI0mS1Gu1+2SCiNhI24EsgEE19H0wMK+4cnMPYH5m3h0Ri4H5EXE+8BxwFkBmLouI+cByKrf/mJWZW4q+Pg3MLY57T/GSJEnq1doNapk5eHc6Lm6Me2wb7RuAae3sczlweRvtzUBH57dJkiT1Ors69SlJkqQ6M6hJkiSVlEFNkiSppAxqkiRJJWVQkyRJKimDmiRJUkkZ1CRJkkrKoCZJklRSBjVJkqSSavfJBJJUV7Nn19YmSX2YI2qSJEklZVCTJEkqKYOaJElSSRnUJEmSSsqgJkmSVFIGNUmSpJIyqEmSJJWUQU2SJKmkDGqSJEklZVCTJEkqKYOaJElSSRnUJEmSSsqgJkmSVFIGNUmSpJIyqEmSJJWUQU2SJKmk6hbUIuLQiHggIlZExLKI+HzRvn9E3B8RTxfv+1Xtc0lErIyIpyLilKr2CRHxRPHZ1RER9apbkiSpLOo5orYZ+FJmjgaOB2ZFxBjgYmBhZo4CFhbrFJ+dDYwFpgPfjoh+RV/XADOBUcVreh3rliRJKoW6BbXMXJOZPyuWNwIrgOHAacC8YrN5wOnF8mnAzZm5KTOfBVYCkyLiYGBIZi7OzARuqNpHkiSp1+qWc9QiYgRwLPBTYFhmroFKmAMOKjYbDjxftVtL0Ta8WG7dLkmS1KvVPahFxN7AbcAXMvOVjjZtoy07aG/rWDMjojkimtevX7/zxUqSJJVIXYNaRAygEtJuyszbi+a1xXQmxfu6or0FOLRq9yZgddHe1Eb7DjLz2sycmJkThw4d2nU/iCRJUgP0r1fHxZWZ1wErMvObVR8tAGYAVxTvd1a1/3NEfBM4hMpFA0syc0tEbIyI46lMnZ4D/F296pbUudmzK+9TF73VNnVqAwqRpF6ubkENeC/wceCJiFhatH2VSkCbHxHnA88BZwFk5rKImA8sp3LF6KzM3FLs92lgLjAIuKd4SZIk9Wp1C2qZ+TBtn18GMK2dfS4HLm+jvRkY13XVSZIklZ9PJpAkSSopg5okSVJJGdQkSZJKyqAmSZJUUgY1SZKkkjKoSZIklZRBTZIkqaQMapIkSSVlUJMkSSopg5okSVJJGdQkSZJKyqAmSZJUUgY1SZKkkurf6AIk9RCzZ29bnLqoYVVIUp/iiJokSVJJOaImqW+pGhmsqV2SGsgRNUmSpJIyqEmSJJWUU5+Seoe2pi6dzpTUwzmiJkmSVFKOqEnqvRxRk9TDGdQk7ahRAcfpS0najlOfkiRJJeWImqSex1E2SX2EI2qSJEkl5YiapHJz9ExSH1a3EbWI+G5ErIuIX1S17R8R90fE08X7flWfXRIRKyPiqYg4pap9QkQ8UXx2dUREvWqWJEkqk3qOqM0F/h64oartYmBhZl4RERcX6xdFxBjgbGAscAjwo4g4IjO3ANcAM4GfAD8ApgP31LFuSX2RV5xKKqG6BbXMfDAiRrRqPg2YWizPAxYBFxXtN2fmJuDZiFgJTIqIVcCQzFwMEBE3AKdjUJO6jmFEkkqruy8mGJaZawCK94OK9uHA81XbtRRtw4vl1u2SJEm9Xlmu+mzrvLPsoL3tTiJmRkRzRDSvX7++y4qTJElqhO6+6nNtRBycmWsi4mBgXdHeAhxatV0TsLpob2qjvU2ZeS1wLcDEiRPbDXSSVBPPW5PUYN09orYAmFEszwDurGo/OyL2jIiRwChgSTE9ujEiji+u9jynah9JkqRerW4jahHxL1QuHDgwIlqA/wZcAcyPiPOB54CzADJzWUTMB5YDm4FZxRWfAJ+mcgXpICoXEXghgfouR3gkqU+p51WfH23no2ntbH85cHkb7c3AuC4sTZIkqUcoy8UEkiRJasVHSEm9kdOhPUet09lOe0t9kkFNknaGgUlSNzKoSerUokW7v83UqV1QSE9ioJPUBQxqUk/nX/6S1GsZ1KSyMoDVNJLX50bqJPUpBjVJ3cLQJUk7z9tzSJIklZQjaiofT8JWT+N/n5LqxKAmlYF/0QO1TY/2aH7PknaSQU3qbv5lLUmqkUFNjdVTQ0svn57t9SNbktRDeDGBJElSSTmipp6hJ4xgla0eSVKPZ1CTepDWU5Jddd+xnjzVWa/fiSSVgUFNPdfujmA1eJSurXBkyJAkVTOoqfuUbWqwbPV0k548erYremQg3p3/NnvCaQKSamZQk0qsr4Wq7uJ0qaSewqCm+vBf8GqQPhVu/f9M6vUMaupzevtoSp8KKnXS2/8bkdRzGNQk9Xmdhdtawq9hTlI9GNSkHszRs/JwFE5SPRjUtPtmz+6ZV9Z1s1pClcFLdeGVoFKPZVDTzvEP95oZuiRJu8ugJrWhs5DlaKF2RU+eHm39bzT/zSZ1D4OaKvxTV5Kk0ukxQS0ipgPfAvoB38nMKxpcUs/QwADWU0YPdmWK0mlNdab0/42092eD/2iTSqVHBLWI6Af8A/B+oAV4JCIWZObyxlamndEVwa2nhD9pV3RFuNvt/yfauzio1fqiVg1Tp9JpyHP6VNp5PSKoAZOAlZn5DEBE3AycBhjUqjX4geI9tQ+pUerx329Dr8Bu9WfQDv+w2to+dfvt2tq33Tapj4nMbHQNnYqIM4HpmfkXxfrHgeMy8zPt7TNx4sRsbm7urhK7Xzt/gBlcJPVUbQXKmkbRaw15RVv1R1MXzd6xz9mzHf1Tt4uIRzNz4g7tPSSonQWc0iqoTcrMz7babiYws1g9EniqWwvtvQ4EXmh0EWqX30+5+f2Um99PufWl7+ewzBzaurGnTH22AIdWrTcBq1tvlJnXAtd2V1F9RUQ0t5XyVQ5+P+Xm91Nufj/l5vcDezS6gBo9AoyKiJER8TbgbGBBg2uSJEmqqx4xopaZmyPiM8APqdye47uZuazBZUmSJNVVjwhqAJn5A+AHja6jj3I6udz8fsrN76fc/H7Krc9/Pz3iYgJJkqS+qKecoyZJktTnGNTUpog4NCIeiIgVEbEsIj7f6Jr0logYGBFLIuLx4vu5rNE1aUcR0S8iHouIuxtdi3YUEasi4omIWBoRvfjGmz1TROwbEbdGxJPF30UnNLqmRugx56ip220GvpSZP4uIwcCjEXG/j+0qjU3A+zLztxExAHg4Iu7JzJ80ujBt5/PACmBIowtRu/44M/vKfbp6mm8B92bmmcUdH97e6IIawRE1tSkz12Tmz4rljVT+shne2Kq0VVb8tlgdULw84bREIqIJOBX4TqNrkXqaiBgCTAGuA8jM32fmSw0tqkEMaupURIwAjgV+2uBSVKWYVlsKrAPuz0y/n3KZA1wIvNngOtS+BO6LiEeLJ9uoPA4H1gPXF6cPfCci9mp0UY1gUFOHImJv4DbgC5n5SqPr0Vsyc0tmjqfypI5JETGuwSWpEBEfBNZl5qONrkUdem9mvgf4ADArIqY0uiBt0x94D3BNZh4L/A64uLElNYZBTe0qzn26DbgpM29vdD1qWzEdsAiY3thKVOW9wJ9ExCrgZuB9EfG9xpak1jJzdfG+DrgDmNTYilSlBWipmim4lUpw63MMampTRASVcwNWZOY3G12PthcRQyNi32J5EHAS8GRDi9I2mXlJZjZl5ggqj7z7t8z88waXpSoRsVdxoRTFlNrJwC8aW5W2ysz/AJ6PiCOLpmlAn7yYzas+1Z73Ah8HnijOgwL4avGECDXewcC8iOhH5R9c8zPTW0BItRsG3FH5Nyn9gX/OzHsbW5Ja+SxwU3HF5zPAJxpcT0P4ZAJJkqSScupTkiSppAxqkiRJJWVQkyRJKimDmiRJUkkZ1CRJkkrKoCZJklRSBjVJkqSSMqhJkiSV1P8H1INBD+7PMJAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 720x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# distribution of length of text\n",
    "data['length_raw_text'] = data['raw_text'].apply(lambda x: len(x))\n",
    "\n",
    "# histogram\n",
    "\n",
    "kwargs = dict(alpha=0.5, bins=100)\n",
    "plt.figure(figsize=(10, 5))\n",
    "plt.hist(data[data['brand_id'] == 'unknown']\n",
    "         ['length_raw_text_log_scaled'], **kwargs, color='blue', density=False, label='unknown')\n",
    "plt.hist(data[data['brand_id'] != 'unknown']\n",
    "         ['length_raw_text_log_scaled'], **kwargs,  color='red', density=False, label='not unknown')\n",
    "plt.legend()\n",
    "plt.ylabel('Length of text')\n",
    "plt.title('Distribution of text length')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "0774b468",
   "metadata": {},
   "outputs": [],
   "source": [
    "# presence of numeric data\n",
    "\n",
    "data['numeric_count'] = data['raw_text'].apply(lambda x: sum(c.isdigit() for c in x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "be8d2db0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgUAAAEWCAYAAAD2NuSlAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAAApVUlEQVR4nO3dd7wdVbn/8c+XJKSQkNCFUI4UUQREioC04BVEiiAW4NICKOIFhfsTAb2ggCAB5FoQpShNpCoghitFIBSpAUJClRakxCBCQkIJEJ7fH2ttMmx2zTn77LNPvu/X67zO1DXPmpk988ya2bMVEZiZmZkt1O4AzMzMrG9wUmBmZmaAkwIzMzPLnBSYmZkZ4KTAzMzMMicFZmZmBjgp6DWSHpI0psXLCEmr5u7TJR3VwDyzJa3cyrj6Gkkr5noP6GY5EyR9rafiamK5XXlbD8z9f5G0d2/H0S4LWn37mlrHGUnflDQ9f76WkLSJpMdz/05tC7qfaeVxW35PQfdJuha4KyJ+UDZ8R+AMYPmIeKcX4ghgtYh4otXLspQUABdExG96ebldwNPAoN7Yr3qTpLHA1yJi0xaVPzWX/9e+VFYnqXackTQIeBXYKCIeyMNuAK6KiJ+3Ic5zgeci4sgWlN1vj7VuKegZ5wJ7SlLZ8D2B3/e3A3cnK11dm3W6PrgvLwMMAR4qDFuprL9hfbB+bdcr6yQi/NfNP2AoMBPYvDBsMeBN4BO5fyrw2dz9KWAiKaueDvxvHj6GlNkWyy6f7w5gBjAN+CWwcGHaAFbN3ecCx+XuPwOzC3/vAmOrzHMacDUwC7gLWKVQ/tbAY7muvwJuJl0pVVonRwOXAufnsh4C1q8Ua4V4xwDPAYcBL+a67gRsC/wdeBn4fmHehYAjgCeBf+flLp7HdeVl7Qf8A7ilMGxgnmZx4BzgBeAV4MrCNhwP/CsPH09q9Sktd0Kd+l8GXJDrPwX4CPC9XKdnga0L048Efpvr+jxwHDAgjxsA/AR4CXgKOLAs/vfiAFYBbszr4SXg98Cosv3pUGBy3o6XAENq7NtfBx7JdXgYWDcP/1he7oy8bb9Qbb0AY4Hbyrb9AcDjeb2eBiiX+SYwl7SfzqgSU7G+Y4Hb8vp5hdSC8vkq8/2OtO+/kcs/LA/fCLg91+UBYEwe/um8DlfI/Z/I03y0UlmkE+IFed3PAO4BlqkSS2l/La3XLza43qcCh+ftNwcYCHwhb4MZed18rFDO4aT9aRbps/sftY5BVWL9Lmm/fAHYlwrHGdK+/VoeN5u0Dz5Zto4GU3s/Hwv8Dfgp6TN+XJ7nJ6TP7nTgdGBo2XHiO8w7TuyTx+0PvA28lZf95yp1+zhwfV7edPJxhRrHWtIxJHJ9ZwO75OHbA5PyPLcDaxeWsy5wf94Ol5E+d8eVbe8nchxXAcuVfV4OJH1enq5w3K61jpYkHbdm5LJvBRaqeT6rNdJ/jf8BZwG/KfR/A5hU6J/KvJP7HcCeuXs4qbntvZ28rNzifOuRDmADSSe2R4BDynaeDyQFZeVtQ/pwr1BlnpfzB2Ig6YRycWHnehXYOY87OH/oap0U3ySdyAcAJwB3Voq1PN68Ht4BfgAMyh+YfwEXAiNIH+Q3gZXz9IcAdwLL5w/IGcBFeVxXXtb5wCKkBK40rHRSvZr0IV0sL2+LPHwJ4EvAsLzcy8gJQx4/oYH6fy6vr/NJJ6z/KdTp6cL0V+a4FwGWBu4GvpHHHQA8CqxASmBuonpSsCqwVV4PS5EOYD8r25/uBpbLZT0CHFClDl8hHbg3IJ20VyVd+Q0iHcC+DywMfIZ0sFu90nqhclIwHhgFrJi37TaVpq0SV7G+Y0n74ddJ+9k3Sfu3qsw7lfx5yv2jSSfxbUnJ5Va5f6k8/njSCW4o6UR8UI2yvkFKwIflWNYDFq2xbpfLy9yFdIJZttZ6LyxzUt4XhjLvZLxV3i6H5W2zMLA6KflcrvBZWKXWMajK8WI6sCZp37yQ6hcfXRT2yyrr6Eqq7+djSZ/7b5E+M0OBn5FOkouTPoN/Bk4oO04cm+u+LfA6sFitY2AhlhGkE/53SAndCGDDZo+1uX9dUmKyYd72e+e6D87b4hnSMXMQ6Rj6VmG9fYaUfK6bpz8VuKVsWdfndTC0fPl11tEJpCRhUP7bjCqfjfeWV2uk/xr/AzYlXXmVNtrfgP+u9OEgHaiPAZYsK2MMNZKCCss8BLii0o5a6QNBOoC8CGxWY55iYrMt8Gju3gu4ozBOpANOrZPiXwv9awBvVFpuebx5PbzBvCuIEXn6DQvT3wvslLsfIV8B5f5lSSeK0gc6yAlEHl8aNjBP+y75QFJnG68DvFLon1Cn/tcX+ncgXVWU12kUqdl1TmnfyeN3A27K3TdSOHGTWmwqJgUV4tgJuL9sf9qj0H8ScHqVea8FDq4wfDPgnxSuOICLgKMrxUPlpGDTQv+lwBGVpq0S13vl5+mfKIwblsv/UJV5p/L+k9ThwO8q1Hvv3D0o72tTgGsoHFArlLUvZVeIjf6RTvQ71lrvhWXuW+g/Cri00L8QKaEYQ0omXgQ+S3r+pFhOxWNQheWdDYwr9H+E+UwKqL+fjwX+URgnUsJTbK3cmHlXy2NIx4ni8l5k3kXWe7FVqdtuFD4bddbDIVQ51ub+XwM/KpvnMWALYPO8TYr7zm2F9fZb4KTCuOGk41dXYVmfKSs78vatt46OBf5UjLXen58p6CERcRvpimfH/FToBqSsupL9SB+uRyXdI2n7RpYh6SOSxkv6p6RXgR+TruAbmXckaec4KiJurTHpPwvdr5N2UEhXNc+WRkTa456rs9jysoY0cU/s3xExN3e/kf9PL4x/oxDbSsAVkmZImkFKEuaSDkIlz1LZCsDLEfFK+QhJwySdIemZvL5vAUY18a2F8nhfqlCn4cy7+p5WqMMZpCspKFv3pKuOiiQtLeliSc/nmC/gg/tItW1cbgVSE3C55YBnI+LdsphGV4urgkZjaKqsiHg9dzZa3krAV0rrPa/7TUnJIhHxNunksiZwSt7vq/kd6YR+saQXJJ2UH777AEl7SZpUWOaazNtO1dZ7SXFfWI7C/pC3ybPA6EgPwR1CSlBfzPvFcnnSRo9BDe97Dai3n1O2rKVISd69hemvycNL/h3vf2armX2p6nqej2PtSsB3yvajFUjrbzng+bJ9p9Y2nE1qrRpdZfqieuvoZFLL0XWSnpJ0RI06AH7QsKedT7qi3hO4LiKmV5ooIh6PiN1IH4YTgT9IWoSU8Q0rTZdPPsUPwK9JzcirRcSipObb8ocbP0DSQqQE5aaIOGN+KkZqZlu+UKaK/fPhdQp1BT7UjbKeJd1HHlX4GxIRzxemqXYwfxZYXNKoCuO+Q2qC3TCv783z8LrrvEnPkq6glizEv2hEfDyPn0Y6wJSsWKOsE0h1XTvHvEc34n2W9IxCuReAFfJ+VYyptL7ftx/T3LatddLtCeXlP0tqKSjuO4tExDgASaOBH5KeOTlF0uBqZUXE2xFxTESsQXoeYXvS8eB9JK1Eut14ELBERIwCHmTedqq23ist9wXSCalUtkj7yvM5pgsjfZNjpTzfiXl4tWNQuWb2vXrq7efldXuJlDx/vDD9yIho9KRfb1+qtZ6bPdY+Cxxfth8Ni4iLSOtwdNmD6MV1Wr4NFyHdumzk+FVzHUXErIj4TkSsTGqt/H+S/qNGPZwU9LDzSU11XwfOqzaRpD0kLZWz+hl58FzSQ3RDJG2XrzCOJN1jKhlBuq8/W9JHSfdPG3E86R7ewU3UpdzVwFqSdspX+wfSvRP5JOA/JQ2QtA2pmW1+nQ4cnw+2SFoqfx20roiYBvwF+JWkxSQNklQ6+Y8gfeBmSFqcdHLocTmG60gnnUUlLSRpFUmldXIp8G1Jy0tajPSQWjUjyA/p5RPad7sR2m+AQyWtp2TVvI7vIp34D8vrawzpgHNxnm8SsHNuaVmVdFXaqOnA8pIW7kbc9covfr/7AmAHSZ/L++IQSWPyuhapleC3pDpMA35UrSxJW0paKyfzr5KagOfyQYuQDvL/yvPtQ2opKKm23iu5FNhO0n/kY8Z3SCfe2yWtLukzOZF5k7Qvz83LrHYMqlT+WElrSBpGNz4DDezn5dO/S0qefipp6Rz3aEmfa3CR5du63HjgQ5IOkTRY0ghJG+Zx9Y615WWfBRwgacO8zRbJx/ERpOc35gIHSRqYj02fKsx7IbCPpHXytvox6SvuU+tVsN46krR93n+U6zOXytv5PU4KelDeiLeTPvRX1Zh0G+AhSbOBnwO7RsSbETET+C/SQeF50oG32ER/KPCfpIe6ziI9HNeI3UgPzbyi9NKL2ZJ2b7hiQES8RHoA6iRS09YapKeX5zRTTsHBpBPJDGB30gNI8+vnpPV9naRZpIcON6w9y/vsSTqAP0q6J3lIHv4z0sNOL+Uyr+lGjPXsRXog6WHSU/R/IDdhk7b1taQn4+8DLq9RzjGkB5ZmkhK5WtPWFBGXkRLKC0n73JWkb3W8RXri/fOkdfMrYK+IeDTP+lPSg1TTScnx75tY7I2kJ+n/Keml+Y29hhOAI3NT66ER8SywI+lK8F+kK77vko6N3ybdgjoqN/3uQzp4b1apLFKS/AfSwfcR0rdzLigPICIeBk4hnSymA2uRnkEqja+43itVJiIeI7UGnUraFjsAO+RtNBgYl4f/k9Qq8P08a8VjUIXy/0L6HNxIaoa+sVIcTai1n1dyeF7unUrN+H8ltd414rfAGnn7XFk+MiJmkR7Q3IG0fh4Htsyj6x1rjwbOy2V/NSImki4Gf5nr9QTpGQnyttiZlFjOIG2v8eRjZ0TcQHo25I+kxHMVYNcG6wi119FquX82aX/7VURMqFWYX15k8yU3HT8H7B4RN7U7HjOzTiHpLtIDvue0O5ZybimwhuUm1lG5iat0j+3ONodlZtanSdpC0ofy7YO9gbVpbcvjfPMbo6wZG5OaNEvNfztFxBu1ZzEzW+CtTno+YzjpGw9fzs9Y9Dm+fWBmZmaAbx+YmZlZ5tsHC7gll1wyurq62h2GmVlHuffee1+KiKXqT9lZnBQs4Lq6upg4cWK7wzAz6yiSuvN2xz7Ltw/MzMwMcFJgZmZmmZMCMzMzA5wUmJmZWeakwMzMzAAnBWZmZpY5KTAzMzPASYGZmZllTgrMzMwMcFJgZmZmmZMCMzMzA5wUmJmZWeYfRFrATXl+Jl1HXN3uMKzNpo7brt0hmFkf4JYCMzMzA5wUmJmZWeakwMzMzAAnBWZmZpY5KTAzMzPASYGZmZllTgrMzMwMcFJgZmZmmZMCMzMzA5wUmJmZWdankgJJYyX9st1xtJOkCZLWrzB8fUm/qDNvl6QHWxedmZn1Z/7tgxaTNDAi3uluORExEZjYAyGZmZlV1NKWgvIrV0mHSjo6Xw2fKOluSX+XtFmFebeTdIekJSWdK+kXkm6X9JSkL+dpJOlkSQ9KmiJplzz8V5K+kLuvkHR27t5P0nE5rkcknSXpIUnXSRpapQ4fk3R3WZ0m5+71JN0s6V5J10paNg+fIOnHkm4G/kfS05IG5XGLSppa6q/iK+XrRtIYSeNz91KSrpd0n6QzJD0jack874BG6mVmZlaunbcPBkbEp4BDgB8WR0j6InAEsG1EvJQHLwtsCmwPjMvDdgbWAT4BfBY4OZ+YbwFKicZoYI3cvSlwa+5eDTgtIj4OzAC+VCnIiHgEWFjSynnQLsCl+aR+KvDliFgPOBs4vjDrqIjYIiKOASYApZ+h2xX4Y0S8PT/rJvshcGNErAtcAaxYGFe3XpL2lzRR0sS5r8+sEYaZmS1I2pkUXJ7/3wt0FYZvCRwObBcRrxSGXxkR70bEw8AyedimwEURMTcipgM3AxuQTvybSVoDeBiYnpOFjYHb87xPR8SkKjGUuxT4au7eBbgEWB1YE7he0iTgSGD5wjyXFLp/A+yTu/cBzqmxLKi+bko2BS4GiIhrgOJ6qluviDgzItaPiPUHDBtZJxQzM1tQtPqZgnd4f+IxpNA9J/+fWxbHU8DKwEd4/z30OYVulf1/n4h4XtJiwDakVoPFSSf12RExS9ISZeXNBWo1s18CXCbp8lR8PC5pLeChiNi4yjyvFeL5W77tsAUwICLqPQxYbd2UVKx32byl+X37wMzMGtLqloLpwNKSlpA0mNT0X88zpNsC50v6eJ1pbwF2kTRA0lLA5kDp/v8dpOb3W0gtB4cy79ZBUyLiSdIJ9ijmtQA8BiwlaWMASYPqxHs+cBH1WwkacRu55ULS1sBiPVCmmZkt4FqaFOT75scCdwHjgUcbnO8xYHfS1fkqNSa9ApgMPADcCBwWEf/M424l3Zt/AriP1FowX0lBdgmwB+lWAhHxFvBl4ERJDwCTgE/XmP/3pJP3Rd2IoeQYYGtJ9wGfB6YBs3qgXDMzW4ApItodwwIhf2Nix4jYswfKGgzMjYh3ckvFryNinfkpa/Cyq8Wye/+suyFZh5s6brv6E5nZeyTdGxEfeKdMp/N7CnqBpFNJV/Tb9lCRK5K+AbEQ8Bbw9R4q18zMFmBOCgoknQZsUjb45xHRrecAIuJbPbmsiHgc+GR3YjIzMyvnpKAgIg7sj8syMzNrRJ/67QMzMzNrHycFZmZmBjgpMDMzs8xJgZmZmQF+0HCBt9bokUz0d9TNzAy3FJiZmVnmpMDMzMwAJwVmZmaWOSkwMzMzwEmBmZmZZU4KzMzMDHBSYGZmZpmTAjMzMwOcFJiZmVnmpMDMzMwAJwVmZmaWOSkwMzMzwEmBmZmZZU4KzMzMDHBSYGZmZpmTAjMzMwOcFJiZmVnmpMDMzMwAJwVmZmaWOSkwMzMzwEmBmZmZZU4KzMzMDHBSYGZmZpmTAjMzMwOcFJiZmVnmpMDMzMwAGNjuAKy9pjw/k64jrm53GNZLpo7brt0hmFkf5pYCMzMzA5wUmJmZWeakwMzMzAAnBWZmZpY5KTAzMzOgiaRA0sGNDDMzM7PO1ExLwd4Vho3toTjMzMyszeq+p0DSbsB/Ah+WdFVh1Ajg360KzMzMzHpXIy8vuh2YBiwJnFIYPguY3IqgzMzMrPfVTQoi4hngGWDj1odjZmZm7dLMg4YbSbpH0mxJb0maK+nVVgbXSSSFpN8V+gdK+pek8U2WM0HS+rn7/ySN6uFQzczMKmrmQcNfArsBjwNDga8Bp7YiqA71GrCmpKG5fyvg+e4UGBHbRsSM7gZmZmbWiKbeUxARTwADImJuRJwDbNmasDrWX4DSL87sBlxUGiFpEUln59aW+yXtmIcPlXSxpMmSLiElXKV5pkpaUlKXpAcLww+VdHTuniDpp5JukfSIpA0kXS7pcUnH9UKdzcysn2gmKXhd0sLAJEknSfpvYJEWxdWpLgZ2lTQEWBu4qzDuf4AbI2IDUjJ1sqRFgG8Cr0fE2sDxwHrzsdy3ImJz4HTgT8CBwJrAWElLlE8saX9JEyVNnPv6zPlYnJmZ9UfNJAV75ukPIjWVrwB8qRVBdaqImAx0kVoJ/q9s9NbAEZImAROAIcCKwObABYX55+cbHaWvik4BHoqIaRExB3iKtJ3K4zwzItaPiPUHDBs5H4szM7P+qJGvJJa8RLoifRM4RtIAYHBrwupoVwE/AcYAxat0AV+KiMeKE0sCiDplvsP7E7ghZePn5P/vFrpL/c1sYzMzW4A101JwAzCs0D8U+GvPhtMvnA0cGxFTyoZfC3xLOQuQ9Mk8/BZg9zxsTdJth3LTgaUlLSFpMLB9SyI3M7MFWjNJwZCImF3qyd3Daky/QIqI5yLi5xVG/QgYBEzODw3+KA//NTBc0mTgMODuCmW+DRxLekZhPPBoK2I3M7MFWzNNy69JWjci7gOQtB7wRmvC6jwRMbzCsAmk5weIiDeAb1SY5g1g1ypldhW6fwH8osI0Yyotr3ycmZlZPc0kBYcAl0l6IfcvC+zS4xGZmZlZWzScFETEPZI+CqxOemju0dysDYCkrSLi+hbEaGZmZr2g2ZcXvR0RD0bElGJCkJ3Yg3GZmZlZL2sqKahDPViWmZmZ9bKeTArqfdfezMzM+rCeTArMzMysg/VkUjC1B8syMzOzXtbwtw8k7VxrfETUHG9901qjRzJx3Hb1JzQzs36vmfcU7Ad8Grgx929JelHOTNLzBJf3aGRmZmbWq5pJCgJYIyKmAUhaFjgtIvZpSWRmZmbWq5p5pqCrlBBk04GP9HA8ZmZm1ibNtBRMkHQtcBGp1WBX4KaWRGVmZma9rpnXHB8k6YvA5nnQmRFxRWvCMjMzs97WTEsBwH3ArIj4q6RhkkZExKxWBGZmZma9q+FnCiR9HfgDcEYeNBq4sgUxmZmZWRs086DhgcAmwKsAEfE4sHQrgjIzM7Pe10xSMCci3ir1SBqIf+/AzMys32gmKbhZ0veBoZK2Ai4D/tyasMzMzKy3NZMUHA78C5gCfAP4P+DIVgRlZmZmva+hbx9IWgiYHBFrAme1NiQzMzNrh4ZaCiLiXeABSSu2OB4zMzNrk2beU7As8JCku4HXSgMj4gs9HpWZmZn1urpJgaTBETEHOKYX4jEzM7M2aaSl4A5gXeBrEbFni+MxMzOzNmkkKVhY0t7ApyXtXD4yIi7v+bDMzMystzWSFBwA7A6MAnYoGxeAkwIzM7N+oG5SEBG3AbdJmhgRv602naStIuL6Ho3OzMzMek3DLy+qlRBkJ3YzFjMzM2ujZt5oWI96sCwzMzPrZT2ZFPjHkczMzDpYTyYFZmZm1sF6MimY2oNlmZmZWS9r5I2GH3g3QVHpPQURUXM6MzMz69saeU9B6d0ESwOfBm7M/VsCE/B7CszMzPqFRt5TsA+ApPHAGhExLfcvC5zW2vDMzMystzTzK4ldpYQgmw58pIfjsV425fmZdB1xdbvDqGvquO3aHYKZWb/XTFIwQdK1wEWkrx/uCtzUkqjMzMys1zWcFETEQfmhw83yoDMj4orWhGVmZma9rZmWgtI3DfxgoZmZWT/U8HsKJO0s6XFJMyW9KmmWpFdbGZyZmZn1nmZaCk4CdoiIR1oVjJmZmbVPM280nO6EwMzMrP9qpqVgoqRLgCuBOaWBpTcampmZWWdrJilYFHgd2LowLPCDh2ZmZv1CM19J3KeVgZiZmVl7NZwUSBoC7Ad8HBhSGh4R+7YgLgMkzY6I4e2Ow8zMFgzNPGj4O+BDwOeAm4HlgVmtCMrMzMx6XzNJwaoRcRTwWkScB2wHrNWasKwaSetIulPSZElXSFpM0tKS7s3jPyEpJK2Y+5+UNKy9UZuZWSdoJil4O/+fIWlNYCTQ1eMRWT3nA4dHxNrAFOCHEfEiMETSoqTXUE8ENpO0EvBiRLxeLEDS/pImSpo49/WZvR2/mZn1Uc18++BMSYsBRwJXAcOBo1oSlVUkaSQwKiJuzoPOAy7L3bcDmwCbAz8GtgEE3FpeTkScCZwJMHjZ1aLFYZuZWYdoKCmQtBDwakS8AtwCrNzSqGx+3EpqJVgJ+BNwOOkro+PbGZSZmXWOhm4fRMS7wEEtjsXqiIiZwCuSSr9UuSfpoU9IydoewON5e70MbAv8rdcDNTOzjtTM7YPrJR0KXAK8VhoYES/3eFRWMkzSc4X+/wX2Bk7PDw8+BewDEBFTJUFKDgBuA5bPrTtmZmZ1NZMU7Etqjv6vsuG+ldAiEVGtJWejKtOvWOj+MenZAjMzs4Y0kxSsQUoINiUlB7cCp7ciKDMzM+t9zSQF5wGvAr/I/bvlYV/t6aDMzMys9zWTFKweEZ8o9N8k6YGeDsjMzMzao5mXF90v6b172ZI2xE+2m5mZ9Rt1WwokTSE9QzAI2EvSP3L/SsDDrQ3PzMzMeksjtw+2b3kUZmZm1nZ1k4KIeKY3AjEzM7P2auaZAjMzM+vHmvn2gfVDa40eycRx27U7DDMz6wPcUmBmZmaAkwIzMzPLnBSYmZkZ4KTAzMzMMicFZmZmBjgpMDMzs8xJgZmZmQFOCszMzCxzUmBmZmaAkwIzMzPLnBSYmZkZ4KTAzMzMMicFZmZmBjgpMDMzs8xJgZmZmQFOCszMzCxzUmBmZmaAkwIzMzPLnBSYmZkZ4KTAzMzMMicFZmZmBjgpMDMzs8xJgZmZmQFOCszMzCxzUmBmZmaAkwIzMzPLBrY7AGuvKc/PpOuIq9sdhplZr5o6brt2h9AnuaXAzMzMACcFZmZmljkpMDMzM8BJgZmZmWVOCszMzAxwUmBmZmaZkwIzMzMDnBSYmZlZ5qTAzMzMACcFfYqk2WX9YyX9MncfIGmvOvO/N72ZmVmz/JrjDhERp7c7BjMz69/cUtAhJB0t6dDcvYGkyZLukHSypAcLky4n6RpJj0s6qU3hmplZB3JLQd8yVNKkQv/iwFUVpjsH2D8ibpc0rmzcOsAngTnAY5JOjYhnWxGsmZn1L24p6FveiIh1Sn/AD8onkDQKGBERt+dBF5ZNckNEzIyIN4GHgZUqlLG/pImSJs59fWbP1sDMzDqWk4LOozrj5xS651KhNSgizoyI9SNi/QHDRvZocGZm1rmcFHSYiHgFmCVpozxo13bGY2Zm/YeTgs60H3CmpDtILQe+B2BmZt3mBw37kIgYXtZ/LnBu7j66MOqhiFgbQNIRwMTy6XP/9i0M18zM+hknBZ1pO0nfI22/Z4Cx7Q3HzMz6AycFHSgiLgEuaXccZmbWv/iZAjMzMwOcFJiZmVnmpMDMzMwAJwVmZmaWOSkwMzMzwEmBmZmZZf5K4gJurdEjmThuu3aHYWZmfYBbCszMzAxwUmBmZmaZkwIzMzMDnBSYmZlZ5qTAzMzMACcFZmZmljkpMDMzM8BJgZmZmWVOCszMzAxwUmBmZmaZkwIzMzMDnBSYmZlZ5qTAzMzMAFBEtDsGayNJs4DH2h1HCywJvNTuIFrA9eosrldnaaZeK0XEUq0Mph3808n2WESs3+4gepqkia5X53C9Oovr1X/59oGZmZkBTgrMzMwsc1JgZ7Y7gBZxvTqL69VZXK9+yg8ampmZGeCWAjMzM8ucFJiZmRngpGCBJmkbSY9JekLSEe2OpzskTZU0RdIkSRPzsMUlXS/p8fx/sXbHWY+ksyW9KOnBwrCq9ZD0vbz9HpP0ufZEXV+Veh0t6fm8zSZJ2rYwrs/XS9IKkm6S9IikhyQdnId39PaqUa9O315DJN0t6YFcr2Py8I7eXj0uIvy3AP4BA4AngZWBhYEHgDXaHVc36jMVWLJs2EnAEbn7CODEdsfZQD02B9YFHqxXD2CNvN0GAx/O23NAu+vQRL2OBg6tMG1H1AtYFlg3d48A/p5j7+jtVaNenb69BAzP3YOAu4CNOn179fSfWwoWXJ8CnoiIpyLiLeBiYMc2x9TTdgTOy93nATu1L5TGRMQtwMtlg6vVY0fg4oiYExFPA0+QtmufU6Ve1XREvSJiWkTcl7tnAY8Ao+nw7VWjXtV0Sr0iImbn3kH5L+jw7dXTnBQsuEYDzxb6n6P2B7+vC+A6SfdK2j8PWyYipkE60AFLty267qlWj/6wDQ+SNDnfXig123ZcvSR1AZ8kXX32m+1VVi/o8O0laYCkScCLwPUR0a+2V09wUrDgUoVhnfz91E0iYl3g88CBkjZvd0C9oNO34a+BVYB1gGnAKXl4R9VL0nDgj8AhEfFqrUkrDOukenX89oqIuRGxDrA88ClJa9aYvGPq1ZOcFCy4ngNWKPQvD7zQpli6LSJeyP9fBK4gNfNNl7QsQP7/Yvsi7JZq9ejobRgR0/NB+l3gLOY1zXZMvSQNIp04fx8Rl+fBHb+9KtWrP2yvkoiYAUwAtqEfbK+e5KRgwXUPsJqkD0taGNgVuKrNMc0XSYtIGlHqBrYGHiTVZ+882d7An9oTYbdVq8dVwK6SBkv6MLAacHcb4psvpQNx9kXSNoMOqZckAb8FHomI/y2M6ujtVa1e/WB7LSVpVO4eCnwWeJQO3149zb+SuICKiHckHQRcS/omwtkR8VCbw5pfywBXpGMZA4ELI+IaSfcAl0raD/gH8JU2xtgQSRcBY4AlJT0H/BAYR4V6RMRDki4FHgbeAQ6MiLltCbyOKvUaI2kdUpPsVOAb0FH12gTYE5iS71MDfJ/O317V6rVbh2+vZYHzJA0gXRBfGhHjJd1BZ2+vHuXXHJuZmRng2wdmZmaWOSkwMzMzwEmBmZmZZU4KzMzMDHBSYGZmZpmTAjNAUkg6pdB/qKSj2xhSvybpo/mX9u6XtEq74ymStL6kX8znvBMkrd/gtGMl/XJ+ljM/JI2S9F+9tTzrTE4KzJI5wM6Slmx3ID1BSV/+fO8E/CkiPhkRT7Y7mBJJAyNiYkR8u92x1JO/b9+MUUBTSUEH7EfWw7yxzZJ3gDOB/y4fIelcSV8u9M/O/8dIulnSpZL+LmmcpN3zb7ZPqXQFnH+7/cr8ozJ3Slo7Dx8u6Zw832RJX8rDt5F0n9JvwN+Qhx0t6dBCmQ9K6sp/j0j6FXAfsIKk70q6J5dZ+v340nRnKf2u/HX5DW9IWlXSX/Py7ivVoUo5i0i6Ok/7oKRdKtR3nVzPyZKukLSYpG2BQ4CvSbqpwjyzJR2fy71T0jI9sR2U3mj3x1yPeyRtUlifZ0q6Djg/lze+znb5taSJef0dU2mHKqvTBpJuz3W6W/kNnMBykq6R9LikkwrTVyxf0lRJP5B0G/AVSV/PdXkg121Ynm6ZvL4fyH+fJr1UaRWlFpqTa2zXSvvRuXkbT5H0gc+I9SPt/u1m//mvL/wBs4FFSW9qGwkcChydx50LfLk4bf4/BphBelPaYOB54Jg87mDgZxWWcyrww9z9GWBS7j6xOD2wGLAU6VfaPpyHLZ7/H03hd+1Jr5vtyn/vAhvl4VuTEh2RLgDGA5vn6d4B1snTXQrskbvvAr6Yu4cAw2qU8yXgrEIcIyvUdzKwRe4+tlTH8jqUzRPADrn7JODIntgOwIXAprl7RdJrfEux3AsMLZQ3vtp2KdsWA0jv0F87908A1i+rz8LAU8AGuX9R0ps3x+bhI/O6fgZYoU75U4HDCmUvUeg+DvhW7r6E9ENGpTJG5u3+YGH6WvtHcT9aj/SLgqX5RrX78+q/1v35NcdmWUS8Kul84NvAGw3Odk/kn12V9CRwXR4+BdiywvSbkk6mRMSNkpaQNJL0HvZdC7G8ImkH4JZIv+VORLzcQDzPRMSduXvr/Hd/7h9Oen/7P4CnI2JSHn4v0JWvXkdHxBV5eW/melUr51bgJ5JOJJ1Eby0Gkus1KiJuzoPOAy5roA5vkU5Qpdi2amCeRrbDZ4E1pPd+/G7RwhX7VRFRaZt/YLvkzq8q/UT3QFIysgYpAapkdWBaRNyTy3g1xwlwQ0TMzP0PAyuREsFa5V9SKHtNSceRbg0MJ722HFLCuVde3lxgpub91HFJrf2juB89Baws6VTgauatW+uHnBSYvd/PSE2m5xSGvUO+1aZ0JF+4MG5OofvdQv+7VP58Vfs5VvHBn2WtNOx98WRDCt2vlc1/QkSc8b5Cpa6yuOcCQ6vEVrWcXNZ6wLbACZKui4hjq5TRjLcjolTvucxbj93dDgsBG5ef/PPJubje3jeasm2g9OM4h5Ku/F+RdC7v3wZ1y6gS91xgYAPlF2M9F9gpIh6QNJbUytGoWvvHe8vIMXwC+BxwIPBVYN8mlmMdxM8UmBXkq/FLgf0Kg6eSmlABdgQGdWMRtwC7Q7oXDryUrxyvAw4qTZSv6u4AtsgnCSQtXohn3TxsXeDDVZZ1LbCvpOF52tGSlq4WWI7jOUk75ekH53vUFcuRtBzwekRcAPykFFOhvJnAK5I2y4P2BG5m/k2le9uhfB2vMx/zLEZq/n+NdPW9DPD5OmU8Snp2YINcxghJtS7Imil/BDBN6aeOdy8MvwH4Zl7eAEmLArPy9CUN7R9KD98uFBF/BI6ibDtb/+KWArMPOoXCiYD02/F/knQ36WBb7aqyEUcD50iaDLzOvJ9sPQ44TdKDpCvGYyLi8tyEfLnSE+AvkprS/wjspfQLdvcAf6+0oIi4TtLHgDvy1fBsYI9cfjV7AmdIOhZ4G/hKjXJWBU6W9G6e9psVytsbOD0nF08B+9RZP7V0dzt8m7SOJ5OOfbcAB9SZp9p2uR94iFSnv9UqICLeUnoI81SlBzrfIN2WqDb9A02UfxTpOZBnSLdKSif9g4EzlX75by7wzYi4Q9Lfcl3+EhHfbXD/GE3aZ0sXkd+rVV/rbP6VRDMzMwN8+8DMzMwyJwVmZmYGOCkwMzOzzEmBmZmZAU4KzMzMLHNSYGZmZoCTAjMzM8v+P43KD9Rh6p3NAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "data.groupby('brand_freq_cat').median()['numeric_count'].plot.barh()\n",
    "plt.xlabel('Num occurences of numerical characters')\n",
    "plt.title('Visualizing numerical median count in texts across different categories')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "1812c049",
   "metadata": {},
   "outputs": [],
   "source": [
    "def count_puncts(x):\n",
    "    count = sum([1 for i in x if i in punctuation]) \n",
    "    return count\n",
    "\n",
    "data['punctuation_count'] = data['raw_text'].apply(lambda x: count_puncts(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "afbec71f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgsAAAEWCAYAAADo/9QWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAAAq6UlEQVR4nO3dabgcVbn28f9NAoRASIAAhjBEQURmGRQQMJ4jiAREEQWESAAnjnrAVxSOigSOQ3AWRBkUEVAMKiCCCiiGeQoQEhCQKTlMhimEDBggPO+HtRoqTXd1d/budO+979917WvXXE+tWl311KrqakUEZmZmZvUs1+kAzMzMrLs5WTAzM7NSThbMzMyslJMFMzMzK+VkwczMzEo5WTAzM7NSThZ6maS7JY1t8zpC0ka5+zRJxzUxz3xJb2pnXP2JpC9L+lmn42jV0tSN/mKgbW+3kTRT0nty9xKfH0kflPRIPg69TdJbJN0haZ6k/+5c1P1LO88/8nsWmifpcuDmiPha1fB9gNOBdSPi5WUQRwBvjogH2r2ubiRpDPAwsHxvlHf+cJ0XEev2dFmd1l/rRrv3kaQpefk9ThB7c1l9iaSZwMcj4q81xj0I/L+I+EPu/znwfER8ftlGCZImAhtFxMFtWPZM6pRBX+eWhdacDYyXpKrh44FfLYtEwcysjJJuO7ZvANxd0t80SYN7JaJ+ZJmUSUT4r8k/YCVgLrBrYdhqwL+BrXL/TOA9ufvtwFTgeWA28P08fCzwaNWyq+e7EXgOeAL4MbBCYdogZcaQEpiv5+4/AvMLf68AE+rMcypwGTAPuBnYsLD83YH78rb+BLialC3XKpOJwO+AyXlZt1fKonq9NeIdCzwKfAF4Mm/roVXl/T1gVo7lujzs//JyK9u5Y47jvMK8Y/I0g3P/ocA9OcaHgE/l4SsDL+SyqixvnRrLez/p4PYcMAV4a9W+OxqYnuOcDAypU14TgOuBH+RlPQTslIc/ksvhkML0KwLfzds8GzgNWKkw/ou53B4HDquxnytlvRpwKfAUMCd3r1tYzhTgf3Ns84ArgJEln4V9gGmkuv0gsEcevg5wCfAs8ADwiVr7vtbnoF451ttHNWIqbu9YSupW1XzfABaTPsfzgR/n4ZsAV+ZtuQ/4SB6+YR62TWGbn87rfN2yAOX9/WTerunA5nViqVlPmyj3KXnd1+ey2ohUr27N67wV2KmqHj6U1/MwcFAevhHp8z43b9PkkjownvTZfAb4CksewyYC55Hq73xSvVyQY76qqow2pqSeF/blMcC/gHNJF7rH5uU9A1wArF712T8kL+9p4Ct53B7Ai8BLed131tm29YALSZ+XZwp1YsMc/zN5ub8CRuRx55Lq6At52V/Kw3cAbiB93u8ExhbW80bgmrwf/ko6Lrdy3DmGVJ8WAYOr9kFZGQ3J++eZvOxbgbVLz39lI/1XsxKdCfys0P8pYFrVDqzsrBuB8bl7FWCHYuWvWm5xvm1zBRucK/49wFGFaWueEKqWtwfpBLJenXmeJSUlg3OF/00eN5J0INo3jzuS9MEqSxZeAvYDlicd7Cu3CJZYb3W8uRxeBk7M8+4JLARWy+NPzR+Q0cAg0sFvRaoSgUIcZcnCONIHXcC78nq2Kdkfry6PdDBbAOyW4/wS6US4QmHf3UI6aaye99en65TXhLzNh+Zt+jrpgHZq3rbdSQeOVfL0PySdfFcHhpESwm8V9vFsYHPSCfXXNfZzpazXAD4EDM3L+S1wcSGuKaSDysakhGwKMKnONryddDLZjXRAGg1sksddTUowhwBbkw62/1mrrlaXe1k51tpHNeIqbu9YSupWjXmnUKjjuTwfyftpMLAN6eSwWR7/iRzfUOBy4Lsly3ovcBswglT/3gqMqhNHWT0tK/cppHq0WY53bVJSOD73H5j718jb9jzwljzvqMJ2nU868S+X9+HOdeLclHRC3JVUb7+fy3uJZKHWMatOGf2Q+vW8si9PyutaCTgKuAlYNw87HTi/6rN/Zp52K9LJ9K21YquxbYNIJ/Uf5LJ6tRxIydRueZ1rkk70P6x1HM/9o0kn5D1zme6W+9csnCO+C6wA7Jz3SyvHnWmkxGal6vU3KKNP5TIemrd3W2DV0s9X2Uj/1axIO5M+sJWdcz3w+VqVJVekE6i6QqNBslBjnUcBF9X64FEjWciV7Elgl5J5ignPnsC9uftjwI2FcSIdNMuShZsK/cuRruJ2qV5vdby5HF5gyZP+k6REabk8bqsa6xxDi8lCjWVcDBxZsj9eXR5wHHBB1TY+Rr5CyPvu4ML4bwOn1VnvBOD+Qv8WOc61C8OeIZ1oRTpYFFt9dgQezt1nUTih5/3eMJHM47YG5hT6pwBfLfT/F/CXOvOeDvygxvD1SFeMwwrDvgWcXSue6nIvK8da+6jG+puqW3XmncKSJ6/9gWtrbPfxhf5LgBmkK7sVS5b1H8A/yfW6bBsa1NOa5V5Y54mF/vHALVXT3Jjr38qkq8kPUWilytOcA5xBodWpzvq+Rr7AyP0rk67YW04WaFzPx+ZlDymMv4echOb+UaSLlsoFVrBky9ktwAG1YquxbTuSktyax46qaT8A3FFVh4vJwjHAuVXzXE5q9ViflAQNLYw7j9aOO4dVLfvV9Tcoo8NIrR1bNlsXu+2+VteLiOtIFWmf/O2C7UlXdLUcTjqA3yvpVkl7NbMOSRtLulTSvyQ9D3yTdMXfzLzDgT8Ax0XEtSWT/qvQvZDU8gHpqu6RyohItezRBqstTv9Knn6dZuIFnokln/WoxDKSlNE/2ORySkl6n6SbJD0r6TlSgtRUmZK2ZValJ2/jI6Srhop65VnL7EL3C3mZ1cNWIV25DAVuk/RcjvsveXglrkcK882iDklDJZ0uaVauU9cAIyQNWoptWI/a+2Ud4NmImFcV0+ga09bTSjk2Uq9uNWMD4B2Vcs9lfxDwhsI0Z5JadU6JiEX1FhQRV5FuR5wKzJZ0hqRVa03boJ7WK/eKYl1Yos5ms4DREbGAlAx9GnhC0mWSNsnTfIl08r4lP1l/WJ11VR8nFpCS3KXRqJ4DPBUR/y70bwBcVJj+HlKiunZhmqWtS+sBs6LGM2iS1pL0G0mP5c/ReZQfRzYAPlxVj3Ymnbgrn5eFhenr7sM6x53i9LXWXa+MziUlLb+R9Likb0tavmRZThaW0jmkK/DxwBVVB/pXRcT9EXEgsBapCe13klYmZdFDK9PlA3bxg/FT4F7SU+2rAl8mfYBL5Yeafg38PSJOX5oNI7UKvPrEeX6Ys9ET6OtVxbAu6RYIpA/p0MK0xYNtmadJ9zQ3rDEuagxbokyL65G0IvB7UnPf2hExAvgTr5VpreUVPU764FWWJ9I2P9Zgvp56mpQ4bBYRI/Lf8IioHPSeoFD2pCuVer4AvAV4R65Tu+bhDetVDY9Qe788DqwuaVhVTJVyqruPmtBoH/VU9fIfAa4ulPuIiFglIo4AkLQKqen858BESauXxRoRJ0fEtqTbBBuTnjVZQhP1tF6511rvEnU2e3VfRMTlEbEb6aR1LynxISL+FRGfiIh1SE3VP6l8FbfKEnVP0lDSLY6l0aieV28bpLJ4X9X+GRIRzXwmG9WlR4D16zw0+K08/5b5c3QwS36GasV5blWcK0fEJFIZrp7LrqL4eW7muFO2LXXLKCJeiogTImJT0u3dvUjntLqcLCydc4D3kO5b/rLeRJIOlrRmzgify4MXk5okh0gal7O5r5LuKVUMI927mp8z/iOajOsbpObAI1vYlmqXAVtI+kD+sHyGxgf1bSXtm6c/inR/8KY8bhrwUUmDJO1Bug/bUC6zs4DvS1onz79jPqA+RXqQqPjeiGnArpLWz60r/1MYtwKpfJ8CXpb0PtKzARWzgTXyfLVcAIyT9J95f30hb+MNzWzL0splcCbwA0lrAUgaLem9hbgmSNo0H3COL1ncMNIB+bl8YiubtpGfA4fm8lgux7RJRDxCKpNvSRoiaUtS69qv8nzTgD0lrS7pDaS60qxG+6inZrNkfboU2FjSeEnL57/tJb01j/8RcFtEfJz0mTmt3rLyfO/IdWcBKQleXCOGRvW0ZrnX2Z4/5fg/KmmwpP1JzxlcKmltSe/PFy6LSM8eLM6xflhS5eJgDulkVCvW3wF7SdpZ0gqkZ0OW6nzSRD2v5TTgG5I2yNOvqfQV9mbMBsao/jdGbiGdyCdJWjnX5XfmccNI5fWcpNG8PumrrkfnAXtLem8+hg2RNFbSuhExi/QA/ERJK0jaEdi7MG9Pjzt1y0jSuyVtkS9Unyfdnqi1n1/lZGEpRMRM0g5bmXTfsp49gLslzScdXA6IiH9HxFzSPeGfkbLEBSzZ1H808FHSg25nkp4Kb8aBpPuic5RefjJf0kFNbxgQEU8DHybdL36GdICZSqqk9fyB1Kw5h9Tasm9EvJTHHUn6ADxHasa9uIVwjibdE76V9EDmSaR7vgvJT37nJrYdIuJKUjlNJz1Mdmlhm+YB/0368M0hle0lhfH3kh7seigvb4lbKBFxH+kK4hTSVdDewN4R8WIL27K0jiE91HRTbvb8K6mFgIj4M+nq9qo8zVUly/kh6WGvp0mJ3F+WNqCIuIX04N8PSM/vXM1rV0AHku4ZPw5cRLrHf2Uedy7pwbGZpG9bNFuvG+6jXvAjYD9JcySdnOvM7sABpG35F/kBu3zA3YPUjA/w/4BtCp+1JZYFrEr6HM/htW8PfLfGNjaqp2XlXr2sZ0hXi1/I6/sSsFf+fC+Xhz9O+ly9i3Q8gnRb9eZ8zLqE9LzEwzWWfzfpQuLXpBPrHBrfrixTt57X8aMc3xWS5pHq9DuaXNdv8/9nJN1ePTIiFpM+4xuRHhp9lHR8g/QM2jak8r+M9I2Jom8BX8119OicQO9Dah1+inS1/0VeO/ceRHpG4hnSw86TycfaXjjulJXRG0gJ3/Ok2xNXkxKbuvxSJiuVs+9HSV+t+nuN8RNp0wtOzMwGEkmTSQ+b96Tlry3csmCvk5vMRuQm/8rzEjc1mM3MzFqQb1FtmG8r7UFqhbi4w2HV5DdhWS07kpoXVwD+AXwgIl7obEhmZv3OG0i3MtYgteAeERF3dDak2nwbwszMzEr5NoSZmZmV8m2IAW7kyJExZsyYTodhZtan3HbbbU9HxJqNp+wfnCwMcGPGjGHq1KmdDsPMrE+RVPeNqf2Rb0OYmZlZKScLZmZmVsrJgpmZmZVysmBmZmalnCyYmZlZKScLZmZmVsrJgpmZmZVysmBmZmalnCyYmZlZKScLZmZmVsrJgpmZmZVysmBmZmal/ENSA9yMx+Yy5tjLOh2GmdkyNXPSuE6H0Ke4ZcHMzMxKOVkwMzOzUk4WzMzMrJSTBTMzMyvlZMHMzMxKOVkwMzOzUk4WzMzMrJSTBTMzMyvlZMHMzMxKOVkwMzOzUl2VLEiaIOnHnY6jkyRNkbRdjeHbSTq5wbxjJN3VvujMzGwg8m9DtJmkwRHxck+XExFTgam9EJKZmVlL2tqyUH2lK+loSRPz1fNJkm6R9E9Ju9SYd5ykGyWNlHS2pJMl3SDpIUn75Wkk6TuS7pI0Q9L+efhPJL0/d18k6azcfbikr+e47pF0pqS7JV0haaU62/BWSbdUbdP03L2tpKsl3Sbpckmj8vApkr4p6WrgK5IelrR8HreqpJmV/jo+XF02ksZKujR3rynpSkm3Szpd0ixJI/O8g5rZLjMzs2Z18jbE4Ih4O3AUcHxxhKQPAscCe0bE03nwKGBnYC9gUh62L7A1sBXwHuA7+YR9DVBJQEYDm+bunYFrc/ebgVMjYjPgOeBDtYKMiHuAFSS9KQ/aH7ggn+xPAfaLiG2Bs4BvFGYdERHviogTgClA5SfODgB+HxEvLU3ZZMcDV0XENsBFwPqFcQ23S9InJU2VNHXxwrklYZiZmXU2Wbgw/78NGFMY/m7gGGBcRMwpDL84Il6JiH8Aa+dhOwPnR8TiiJgNXA1sT0oIdpG0KfAPYHZOInYEbsjzPhwR0+rEUO0C4CO5e39gMvAWYHPgSknTgK8C6xbmmVzo/hlwaO4+FPhFybqgftlU7Az8BiAi/gIUy6nhdkXEGRGxXURsN2jo8AahmJnZQNfuZxZeZsmEZEihe1H+v7gqjoeANwEbs+Q9+kWFblX9X0JEPCZpNWAPUivD6qST/fyImCdpjarlLQbKmusnA7+VdGFafNwvaQvg7ojYsc48CwrxXJ9vX7wLGBQRjR5CrFc2FTW3u2reyvy+DWFmZj3S7paF2cBaktaQtCLpFkIjs0i3F86RtFmDaa8B9pc0SNKawK5A5fmCG0nN+NeQWhqO5rVbEC2JiAdJJ97jeK3F4D5gTUk7AkhavkG85wDn07hVoRnXkVs6JO0OrNYLyzQzM6uprclCvi9/InAzcClwb5Pz3QccRLqa37Bk0ouA6cCdwFXAlyLiX3nctaR7/w8At5NaF5YqWcgmAweTbkkQES8C+wEnSboTmAbsVDL/r0gn9fN7EEPFCcDukm4H3gc8AczrheWamZm9jiKi0zEMCPkbHPtExPheWNaKwOKIeDm3bPw0IrZemmWtOOrNMeqQH/Y0JDOzPmXmpHGNJyoh6baIeN07cforv2dhGZB0CqkFYM9eWuT6pG9kLAe8CHyil5ZrZmb2Ok4WCiSdCryzavCPIqJHzxlExOd6c10RcT/wtp7EZGZm1iwnCwUR8Zn+uC4zM7Oe6KrfhjAzM7Pu42TBzMzMSjlZMDMzs1JOFszMzKyUH3Ac4LYYPZypPfy+sZmZ9W9uWTAzM7NSThbMzMyslJMFMzMzK+VkwczMzEo5WTAzM7NSThbMzMyslJMFMzMzK+VkwczMzEo5WTAzM7NSThbMzMyslJMFMzMzK+VkwczMzEo5WTAzM7NSThbMzMyslJMFMzMzK+VkwczMzEo5WTAzM7NSThbMzMyslJMFMzMzK+VkwczMzEo5WTAzM7NSThbMzMyslJMFMzMzK+VkwczMzEo5WTAzM7NSThbMzMys1OBOB2CdNeOxuYw59rJOh2ED1MxJ4zodgpk1wS0LZmZmVsrJgpmZmZVysmBmZmalnCyYmZlZKScLZmZmVqrpZEHSkc0MMzMzs/6llZaFQ2oMm9BLcZiZmVmXavieBUkHAh8F3ijpksKoYcAz7QrMzMzMukMzL2W6AXgCGAl8rzB8HjC9HUGZmZlZ92iYLETELGAWsGP7wzEzM7Nu08oDjjtIulXSfEkvSlos6fl2BteXSApJ5xb6B0t6StKlLS5niqTtcvefJI3o5VDNzMxa0soDjj8GDgTuB1YCPg6c0o6g+qgFwOaSVsr9uwGP9WSBEbFnRDzX08DMzMx6oqX3LETEA8CgiFgcEb8A3t2esPqsPwOVX8Y5EDi/MkLSypLOyq0zd0jaJw9fSdJvJE2XNJmUiFXmmSlppKQxku4qDD9a0sTcPUXSDyRdI+keSdtLulDS/ZK+vgy22czM+rlWkoWFklYApkn6tqTPAyu3Ka6+6jfAAZKGAFsCNxfGfQW4KiK2JyVZ35G0MnAEsDAitgS+AWy7FOt9MSJ2BU4D/gB8BtgcmCBpjeqJJX1S0lRJUxcvnLsUqzMzs4GklWRhfJ7+s6Qm9/WAD7UjqL4qIqYDY0itCn+qGr07cKykacAUYAiwPrArcF5h/qX5hknlK60zgLsj4omIWAQ8RNpP1XGeERHbRcR2g4YOX4rVmZnZQNLMVycrniZdwf4bOEHSIGDF9oTVp10CfBcYCxSv6gV8KCLuK04sCSAaLPNllkzshlSNX5T/v1LorvS3so/NzMxep5WWhb8BQwv9KwF/7d1w+oWzgBMjYkbV8MuBzylnB5LelodfAxyUh21Oun1RbTawlqQ1JK0I7NWWyM3MzGpoJVkYEhHzKz25e2jJ9ANSRDwaET+qMep/geWB6flhxf/Nw38KrCJpOvAl4JYay3wJOJH0DMSlwL3tiN3MzKyWVpqoF0jaJiJuB5C0LfBCe8LqeyJilRrDppCeTyAiXgA+VWOaF4AD6ixzTKH7ZODkGtOMrbW+6nFmZmZLq5Vk4Sjgt5Iez/2jgP17PSIzMzPrKk0nCxFxq6RNgLeQHta7NzePAyBpt4i4sg0xmpmZWQe1+lKmlyLiroiYUUwUspN6MS4zMzPrEi0lCw2oF5dlZmZmXaI3k4VG7wowMzOzPqg3kwUzMzPrh3ozWZjZi8syMzOzLtH0tyEk7Vs2PiJKx1t32mL0cKZOGtd4QjMzG7Baec/C4cBOwFW5/92kFwDNJT2vcGGvRmZmZmZdoZVkIYBNI+IJAEmjgFMj4tC2RGZmZmZdoZVnFsZUEoVsNrBxL8djZmZmXaaVloUpki4Hzie1MhwA/L0tUZmZmVnXaOV1z5+V9EFg1zzojIi4qD1hmZmZWbdopWUB4HZgXkT8VdJQScMiYl47AjMzM7Pu0PQzC5I+AfwOOD0PGg1c3IaYzMzMrIu08oDjZ4B3As8DRMT9wFrtCMrMzMy6RyvJwqKIeLHSI2kw/j0IMzOzfq+VZOFqSV8GVpK0G/Bb4I/tCcvMzMy6RSvJwjHAU8AM4FPAn4CvtiMoMzMz6x5NfRtC0nLA9IjYHDizvSGZmZlZN2mqZSEiXgHulLR+m+MxMzOzLtPKexZGAXdLugVYUBkYEe/v9ajMzMysazRMFiStGBGLgBOWQTxmZmbWZZppWbgR2Ab4eESMb3M8ZmZm1mWaSRZWkHQIsJOkfatHRsSFvR+WmZmZdYtmkoVPAwcBI4C9q8YF4GTBzMysH2uYLETEdcB1kqZGxM/rTSdpt4i4slejMzMzs45r+qVMZYlCdlIPYzEzM7Mu1MobHBtRLy7LzMzMukRvJgv+USkzM7N+qDeTBTMzM+uHejNZmNmLyzIzM7Mu0cwbHF/3boWiynsWIqJ0OjMzM+ubmnnPQuXdCmsBOwFX5f53A1PwexbMzMz6tWbes3AogKRLgU0j4oncPwo4tb3hmZmZWae18quTYyqJQjYb2LiX47FlbMZjcxlz7GWdDsMGqJmTxnU6BDNrQivJwhRJlwPnk74meQDw97ZEZWZmZl2j6WQhIj6bH3bcJQ86IyIuak9YZmZm1i1aaVmofPPBDzSamZkNIE2/Z0HSvpLulzRX0vOS5kl6vp3BmZmZWee10rLwbWDviLinXcGYmZlZ92nlDY6znSiYmZkNPK20LEyVNBm4GFhUGVh5g6OZmZn1T60kC6sCC4HdC8MCP/BoZmbWr7Xy1clD2xmImZmZdaemkwVJQ4DDgc2AIZXhEXFYG+IyQNL8iFil03GYmdnA1soDjucCbwDeC1wNrAvMa0dQZmZm1j1aSRY2iojjgAUR8UtgHLBFe8KyeiRtLekmSdMlXSRpNUlrSbotj99KUkhaP/c/KGloZ6M2M7O+rJVk4aX8/zlJmwPDgTG9HpE1cg5wTERsCcwAjo+IJ4EhklYlvY57KrCLpA2AJyNiYXEBkj4paaqkqYsXzl3W8ZuZWR/TyrchzpC0GvBV4BJgFeC4tkRlNUkaDoyIiKvzoF8Cv83dNwDvBHYFvgnsAQi4tno5EXEGcAbAiqPeHG0O28zM+rimkgVJywHPR8Qc4BrgTW2NypbGtaRWhQ2APwDHkL7aemkngzIzs76vqdsQEfEK8Nk2x2INRMRcYI6kyi9/jic9bAopiTsYuD/vr2eBPYHrl3mgZmbWr7RyG+JKSUcDk4EFlYER8WyvR2UVQyU9Wuj/PnAIcFp+aPEh4FCAiJgpCVLSAHAdsG5uDTIzM1tqrSQLh5Gatf+rarhvSbRJRNRr+dmhzvTrF7q/SXp2wczMrEdaSRY2JSUKO5OShmuB09oRlJmZmXWPVpKFXwLPAyfn/gPzsI/0dlBmZmbWPVpJFt4SEVsV+v8u6c7eDsjMzMy6SysvZbpD0qv3yiW9Az9pb2Zm1u81bFmQNIP0jMLywMck/V/u3wD4R3vDMzMzs05r5jbEXm2PwszMzLpWw2QhImYti0DMzMysO7XyzIKZmZkNQK18G8L6oS1GD2fqpHGdDsPMzLqYWxbMzMyslJMFMzMzK+VkwczMzEo5WTAzM7NSThbMzMyslJMFMzMzK+VkwczMzEo5WTAzM7NSThbMzMyslJMFMzMzK+VkwczMzEo5WTAzM7NSThbMzMyslJMFMzMzK+VkwczMzEo5WTAzM7NSThbMzMyslJMFMzMzK+VkwczMzEo5WTAzM7NSThbMzMyslJMFMzMzK+VkwczMzEo5WTAzM7NSThbMzMyslJMFMzMzKzW40wFYZ814bC5jjr2s02HYADVz0rhOh2BmTXDLgpmZmZVysmBmZmalnCyYmZlZKScLZmZmVsrJgpmZmZVysmBmZmalnCyYmZlZKScLZmZmVsrJgpmZmZVystBFJM2v6p8g6ce5+9OSPtZg/lenNzMz6y1+3XMfERGndToGMzMbmNyy0EdImijp6Ny9vaTpkm6U9B1JdxUmXUfSXyTdL+nbHQrXzMz6EbcsdJeVJE0r9K8OXFJjul8An4yIGyRNqhq3NfA2YBFwn6RTIuKRdgRrZmYDg1sWussLEbF15Q/4WvUEkkYAwyLihjzo11WT/C0i5kbEv4F/ABvUWMYnJU2VNHXxwrm9uwVmZtbvOFnoe9Rg/KJC92JqtB5FxBkRsV1EbDdo6PBeDc7MzPofJwt9TETMAeZJ2iEPOqCT8ZiZWf/nZKFvOhw4Q9KNpJYG30swM7O28QOOXSQiVqnqPxs4O3dPLIy6OyK2BJB0LDC1evrcv1cbwzUzswHCyULfNE7S/5D23yxgQmfDMTOz/szJQh8UEZOByZ2Ow8zMBgY/s2BmZmalnCyYmZlZKScLZmZmVsrJgpmZmZVysmBmZmalnCyYmZlZKX91coDbYvRwpk4a1+kwzMysi7llwczMzEo5WTAzM7NSThbMzMyslJMFMzMzK+VkwczMzEo5WTAzM7NSThbMzMyslJMFMzMzK+VkwczMzEo5WTAzM7NSThbMzMyslJMFMzMzK+VkwczMzEopIjodg3WQpHnAfZ2Oo8RI4OlOB1HC8fWM4+uZbo6vm2ODnse3QUSs2VvBdDv/RLXdFxHbdTqIeiRNdXxLz/H1jONbet0cG3R/fN3GtyHMzMyslJMFMzMzK+Vkwc7odAANOL6ecXw94/iWXjfHBt0fX1fxA45mZmZWyi0LZmZmVsrJgpmZmZVystCPSdpD0n2SHpB0bI3xknRyHj9d0jbNzruM4jsoxzVd0g2StiqMmylphqRpkqZ2KL6xkubmGKZJ+lqz8y6D2L5YiOsuSYslrZ7HLYuyO0vSk5LuqjO+03WvUXydrnuN4utk3WsUW6fr3nqS/i7pHkl3SzqyxjQdrX99UkT4rx/+AYOAB4E3ASsAdwKbVk2zJ/BnQMAOwM3NzruM4tsJWC13v68SX+6fCYzscPmNBS5dmnnbHVvV9HsDVy2rssvr2BXYBrirzviO1b0m4+tY3Wsyvo7UvWZi64K6NwrYJncPA/7ZTce+vvrnloX+6+3AAxHxUES8CPwG2Kdqmn2AcyK5CRghaVST87Y9voi4ISLm5N6bgHV7OYYexdemedux/AOB83tx/Q1FxDXAsyWTdLLuNYyvw3WvmfKrp+3l12Jsnah7T0TE7bl7HnAPMLpqso7Wv77IyUL/NRp4pND/KK//wNSbppl5l0V8RYeTrgQqArhC0m2SPtnLsbUS346S7pT0Z0mbtThvu2ND0lBgD+D3hcHtLrtmdLLutWpZ171mdaLuNa0b6p6kMcDbgJurRvWl+tcV/Lrn/ks1hlV/T7beNM3M21NNr0PSu0kH7J0Lg98ZEY9LWgu4UtK9+YpnWcZ3O+n98PMl7QlcDLy5yXnbHVvF3sD1EVG8Emx32TWjk3WvaR2qe83oVN1rRUfrnqRVSInKURHxfPXoGrN0Xf3rJm5Z6L8eBdYr9K8LPN7kNM3MuyziQ9KWwM+AfSLimcrwiHg8/38SuIjUfLhM44uI5yNifu7+E7C8pJHNzNvu2AoOoKoZeBmUXTM6Wfea0sG611AH614rOlb3JC1PShR+FREX1pik6+tf1+n0QxP+a88fqdXoIeCNvPagzmZV04xjyYd8bml23mUU3/rAA8BOVcNXBoYVum8A9uhAfG/gtRebvR34v1yWbS2/ZpcPDCfdW155WZZdYV1jqP+AXsfqXpPxdazuNRlfR+peM7F1uu7lcjgH+GHJNB2vf33tz7ch+qmIeFnSZ4HLSU/4nhURd0v6dB5/GvAn0lPBDwALgUPL5u1AfF8D1gB+Igng5Ui/Erc2cFEeNhj4dUT8pQPx7QccIell4AXggEhHnLaWX5OxAXwQuCIiFhRmb3vZAUg6n/TE/khJjwLHA8sX4utY3Wsyvo7VvSbj60jdazI26GDdA94JjAdmSJqWh32ZlAB2Rf3ri/y6ZzMzMyvlZxbMzMyslJMFMzMzK+VkwczMzEo5WTAzM7NSThbMzMyslJMFG3AkhaTvFfqPljSxgyH1a5I2yb8yeIekDZfxusdK2qkH84+Q9F+F/nUk/a53oqu5vgmSftyu5ddY3xLbZ1aPkwUbiBYB++Y33vV5+ed2u/mz/AHgDxHxtoh4cBmveyzpFySX1gjg1ZNpRDweEfv1MKa2kTSoxVlGUNi+JtfR7fXN2sA73Aail4EzgM9Xj5B0tqT9Cv3z8/+xkq6WdIGkf0qaJOkgSbdImlHrilnS6pIuljRd0k359cFIWkXSL/J80yV9KA/fQ9Lt+ceB/paHTZR0dGGZd0kak//ukfQT0u8ErCfpi5Juzcs8IU9fme5MSXdLukLSSnncRpL+mtd3e2Ub6ixnZUmX5WnvkrR/je3dOm/ndEkXSVpN6XcLjgI+LunvNeaZL+l7ef1/k7RmHj5F0na5e6Skmbl7gqQLJf1F0v2Svl1Y1hLlp/QjQp8GPp9bNnYp2b+r5Hluz/ul8kuDk4AN8/zfyeV5V55nSGE/3qH0OxKlMVZt+/aSbsjx3iJpWB61Tp3t+6mkqXk/nlAYPlPS1yRdB3xY0ify/rtT0u+VftAJSWvn/XJn/tupevtK9n+t+nZ2rgszJL3us2T9TKdfIek//y3rP2A+sCowk/Ra2qOBiXnc2cB+xWnz/7HAc8AoYEXgMeCEPO5IarxaFjgFOD53/wcwLXefVJweWA1Yk/Rrd2/Mw1bP/ycCRxemvYv0qt0xwCvADnn47qQESKSLgEuBXfN0LwNb5+kuAA7O3TcDH8zdQ4ChJcv5EHBmIY7hNbZ3OvCu3H1iZRurt6FqngAOyt1fA36cu6cA2+XukcDM3D2B9Dre4TnmWaR3+TdbfvX272Bg1cL6HshlMIbCa42L/cAXgF/k7k1Ir1weUi/Gqu1eIU+zfe5fNcdQd97CNg3K5bNl7p8JfKmw7DUK3V8HPpe7J5N+VKmyjOE1tq+sHhXr27bAlYX5RnT6c+2/9v65ZcEGpEi/QncO8N8tzHZrRDwREYuAB4Er8vAZpINptZ2Bc/P6rgLWkDQceA9waiGWOaT3018TEQ/nYc++fnGvMysibsrdu+e/O0hXfpuQfoUQ4OGImJa7bwPG5KvY0RFxUV7fvyNiYclyZgDvkXSSpF0iYm4xkLxdIyLi6jzol6STTCOvkE5iAOex5K871vO3iJgbEf8G/gFswNKVX5GAb0qaDvyV9LPEazeYp7h/7yWd2DcuibHoLcATEXFrnv/5iHi5wbwfkXQ7ad9sBmxaWN7kQvfmkq6VNAM4KE8LKWH9aV7f4up9mJXVo2J9ewh4k6RTJO0BVP+qo/Uz/m0IG8h+SDog/qIw7GXy7TlJIl0BViwqdL9S6H+F2p+lsp/BrfVz4bXevf5qPNmQQnfxvfsCvhURpy+x0NQUX4x7MbBSndjqLicva1vS+/S/JemKiDixzjJ6olIGxe0eUjVN9fYMpn75Vau3fw8itU5sGxEv5dse1eutVq8M68VYPW+9eF83r6Q3klrAto+IOZLOpn5dOBv4QETcKWkCqVWsWWX16NV15Bi2At4LfAb4CHBYC+uxPsYtCzZg5avPC4DDC4NnkppYAfYh/0DOUrqGdBJC0ljg6dyicQXw2cpEklYDbgTelU8KSFq9EM82edg2pF/Dq+Vy4DBJq+RpR0taq15gOY5HJX0gT79ivrddczmS1gEWRsR5wHcrMRWWNxeYI2mXPGg8cDWNLUf6USSAjwLXFba7sh+aeaCwXvnNA4YVpisut7h/hwNP5kTh3bx2NV89f1Fx/25M+qGi+5qIFeBe0rMJ2+f5h0kqu3hblXSynitpbeB9JdMOA55Q+pnmgwrD/wYckdc3SNKqvH77mqpHSg8HLxcRvweOo6o+WP/jlgUb6L5H4cQNnAn8QdItpIPrgppzNWci8IvctL0QOCQP/zpwan5QbjHp2YcLJX0SuFDpSfMngd2A3wMfU/r1vFuBf9ZaUURcIemtwI3pgpn5wMF5+fWMB06XdCLwEvDhkuVsBHxH0it52iNqLO8Q4LScdDxE/iW/BhYAm0m6DZgLVB6c/C5wgaTxwFWNFhIRT9Upvz8Cv1N6YPFz1N+/vwL+KGkqMI10MicinpF0fd5Xf6Zw+wj4Sd7eGaQWiwkRsSiXW6N4X1R6SPQUpQdOXyDdnqo3/Z2S7gDuJpXt9SWLP470PMos0u2jSjJwJHCGpMNJ9eKIiLixuH0R8cUm69FoUt2uXHD+T8ONtj7NvzppZh0jaX5ErNLpOMysnG9DmJmZWSm3LJiZmVkptyyYmZlZKScLZmZmVsrJgpmZmZVysmBmZmalnCyYmZlZqf8PNUPi38dI/fEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# can remove punctuation as it is not making much difference in data\n",
    "\n",
    "data.groupby('brand_freq_cat').median()['punctuation_count'].plot.barh()\n",
    "plt.xlabel('Num occurences of punctuation characters')\n",
    "plt.title('Visualizing punctuation median count in texts across different categories')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "63a993a8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "brand_id                                                                unknown\n",
       "raw_text                      RBX TIPS 2022  LEGIT WAYS TO GET ROBLOX ROBUX ...\n",
       "brand_id_count                                                            17882\n",
       "length_raw_text                                                              89\n",
       "brand_freq_cat                                                unknown_very_high\n",
       "length_raw_text_log_scaled                                              1.94939\n",
       "numeric_count                                                                 8\n",
       "punctuation_count                                                             0\n",
       "Name: 17727, dtype: object"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# if we can find the brand id in every text\n",
    "# the text has roblox but is not mapped to roblox, ir else we could have used keyword extraction models\n",
    "\n",
    "data.loc[17727]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "b0b89fdc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "brand_id                                                       neuberger_berman\n",
       "raw_text                      路博迈  KLUvQBY5B8Fq8Hxg88AFeSDAcXufT1uYRoepmp0Uo...\n",
       "brand_id_count                                                                8\n",
       "length_raw_text                                                         2089116\n",
       "brand_freq_cat                                                              Low\n",
       "length_raw_text_log_scaled                                             6.319963\n",
       "numeric_count                                                            314832\n",
       "punctuation_count                                                             0\n",
       "Name: 68605, dtype: object"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# some of the data is in different language\n",
    "\n",
    "data.loc[68605]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "186e1374",
   "metadata": {},
   "source": [
    "# Text cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "d16f1aca",
   "metadata": {},
   "outputs": [],
   "source": [
    "sub_data = data.sample(1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "1307bbde",
   "metadata": {},
   "outputs": [],
   "source": [
    "#tokenization\n",
    "\n",
    "tok = spacy.load('en_core_web_sm')\n",
    "def tokenize(text):\n",
    "    text = re.sub(r\"[^\\x00-\\x7F]+\", \" \", text)\n",
    "    regex = re.compile('[' + re.escape(string.punctuation) + '0-9\\\\r\\\\t\\\\n]') # remove punctuation and numbers\n",
    "    nopunct = regex.sub(\" \", text.lower())\n",
    "    return [token.text for token in tok.tokenizer(nopunct)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "590c8cae",
   "metadata": {},
   "outputs": [],
   "source": [
    "counts = Counter()\n",
    "for index, row in sub_data.iterrows():\n",
    "    counts.update(tokenize(row['raw_text']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "409ca09a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "34739"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "aac560fc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num_words before: 34739\n",
      "num_words after: 16991\n"
     ]
    }
   ],
   "source": [
    "#deleting infrequent words\n",
    "\n",
    "print(\"num_words before:\",len(counts.keys()))\n",
    "for word in list(counts):\n",
    "    if counts[word] < 2:\n",
    "        del counts[word]\n",
    "print(\"num_words after:\",len(counts.keys()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "2a107f1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "#creating vocabulary\n",
    "\n",
    "vocab2index = {\"\":0, \"UNK\":1}\n",
    "words = [\"\", \"UNK\"]\n",
    "for word in counts:\n",
    "    vocab2index[word] = len(words)\n",
    "    words.append(word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "693d5d35",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "16993"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(vocab2index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "8641c089",
   "metadata": {},
   "outputs": [],
   "source": [
    "# saving vocab2index for future\n",
    "\n",
    "with open(\"vocab2index.pkl\", \"wb\") as myFile:\n",
    "    pickle.dump(vocab2index, myFile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "8dd796d7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3682.994"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub_data['length_raw_text'].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "4a2ee9d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 3863 is the median length\n",
    "\n",
    "def encode_sentence(text, vocab2index, N=4000):\n",
    "    tokenized = tokenize(text)\n",
    "    encoded = np.zeros(N, dtype=int)\n",
    "    enc1 = np.array([vocab2index.get(word, vocab2index[\"UNK\"]) for word in tokenized])\n",
    "    length = min(N, len(enc1))\n",
    "    encoded[:length] = enc1[:length]\n",
    "    return encoded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "ab9ec4b3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>brand_id</th>\n",
       "      <th>raw_text</th>\n",
       "      <th>brand_id_count</th>\n",
       "      <th>length_raw_text</th>\n",
       "      <th>brand_freq_cat</th>\n",
       "      <th>length_raw_text_log_scaled</th>\n",
       "      <th>numeric_count</th>\n",
       "      <th>punctuation_count</th>\n",
       "      <th>encoded_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>57081</th>\n",
       "      <td>runescape</td>\n",
       "      <td>RS Money,RS Gold,Cheap RS gold ,wow gold,b&amp;S g...</td>\n",
       "      <td>1131</td>\n",
       "      <td>3092</td>\n",
       "      <td>Medium</td>\n",
       "      <td>3.490239</td>\n",
       "      <td>254</td>\n",
       "      <td>53</td>\n",
       "      <td>[2, 3, 2, 4, 5, 2, 4, 6, 7, 4, 8, 9, 4, 10, 11...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59371</th>\n",
       "      <td>facebook</td>\n",
       "      <td>Log in to Facebook | Facebook  Sign Up Log Int...</td>\n",
       "      <td>858</td>\n",
       "      <td>376</td>\n",
       "      <td>Medium</td>\n",
       "      <td>2.575188</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>[160, 48, 40, 161, 28, 161, 6, 47, 162, 160, 9...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72766</th>\n",
       "      <td>prosperity_bank</td>\n",
       "      <td>Personal| Home  IMPORTANT INFORMATION ABOUT CO...</td>\n",
       "      <td>11</td>\n",
       "      <td>4345</td>\n",
       "      <td>Low</td>\n",
       "      <td>3.637990</td>\n",
       "      <td>38</td>\n",
       "      <td>1</td>\n",
       "      <td>[192, 6, 36, 6, 193, 194, 124, 195, 28, 196, 1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43870</th>\n",
       "      <td>hm_fashion</td>\n",
       "      <td>🎁Free gift card for everyone!🎁  HM Free gift f...</td>\n",
       "      <td>1136</td>\n",
       "      <td>2008</td>\n",
       "      <td>Medium</td>\n",
       "      <td>3.302764</td>\n",
       "      <td>131</td>\n",
       "      <td>1</td>\n",
       "      <td>[6, 210, 376, 241, 13, 377, 22, 378, 210, 376,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62884</th>\n",
       "      <td>opensea</td>\n",
       "      <td>OpenSea, the largest NFT marketplace  strongWe...</td>\n",
       "      <td>442</td>\n",
       "      <td>470</td>\n",
       "      <td>Medium</td>\n",
       "      <td>2.672098</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>[496, 6, 91, 497, 498, 180, 6, 499, 500, 462, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              brand_id                                           raw_text  \\\n",
       "57081        runescape  RS Money,RS Gold,Cheap RS gold ,wow gold,b&S g...   \n",
       "59371         facebook  Log in to Facebook | Facebook  Sign Up Log Int...   \n",
       "72766  prosperity_bank  Personal| Home  IMPORTANT INFORMATION ABOUT CO...   \n",
       "43870       hm_fashion  🎁Free gift card for everyone!🎁  HM Free gift f...   \n",
       "62884          opensea  OpenSea, the largest NFT marketplace  strongWe...   \n",
       "\n",
       "       brand_id_count  length_raw_text brand_freq_cat  \\\n",
       "57081            1131             3092         Medium   \n",
       "59371             858              376         Medium   \n",
       "72766              11             4345            Low   \n",
       "43870            1136             2008         Medium   \n",
       "62884             442              470         Medium   \n",
       "\n",
       "       length_raw_text_log_scaled  numeric_count  punctuation_count  \\\n",
       "57081                    3.490239            254                 53   \n",
       "59371                    2.575188              4                  1   \n",
       "72766                    3.637990             38                  1   \n",
       "43870                    3.302764            131                  1   \n",
       "62884                    2.672098             17                  1   \n",
       "\n",
       "                                            encoded_text  \n",
       "57081  [2, 3, 2, 4, 5, 2, 4, 6, 7, 4, 8, 9, 4, 10, 11...  \n",
       "59371  [160, 48, 40, 161, 28, 161, 6, 47, 162, 160, 9...  \n",
       "72766  [192, 6, 36, 6, 193, 194, 124, 195, 28, 196, 1...  \n",
       "43870  [6, 210, 376, 241, 13, 377, 22, 378, 210, 376,...  \n",
       "62884  [496, 6, 91, 497, 498, 180, 6, 499, 500, 462, ...  "
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub_data['encoded_text'] = sub_data['raw_text'].apply(\n",
    "    lambda x: np.array(encode_sentence(x, vocab2index)))\n",
    "sub_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "bbf9d796",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# label encoding the target\n",
    "\n",
    "le = LabelEncoder()\n",
    "sub_data['brand_id_encoded'] = le.fit_transform(sub_data['brand_id'])\n",
    "brand_name_mapping = dict(zip(le.classes_, le.transform(le.classes_)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "27af8978",
   "metadata": {},
   "outputs": [],
   "source": [
    "# saving brand_name_mapping for future\n",
    "\n",
    "with open(\"brand_name_mapping.pkl\", \"wb\") as myFile:\n",
    "    pickle.dump(brand_name_mapping, myFile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "c7aceb7a",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = sub_data['encoded_text'].values\n",
    "y = sub_data['brand_id_encoded'].values\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    X, y, test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "3a3b7587",
   "metadata": {},
   "outputs": [],
   "source": [
    "class WebDataset(Dataset):\n",
    "    def __init__(self, X, Y):\n",
    "        self.X = X\n",
    "        self.y = Y\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.y)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        return self.X[idx], self.y[idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "0f4982b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_ds = WebDataset(X_train, y_train)\n",
    "test_ds = WebDataset(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "6f3eacbb",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dl = DataLoader(train_ds, batch_size=16, shuffle=True)\n",
    "test_dl = DataLoader(test_ds, batch_size=16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "6af03b40",
   "metadata": {},
   "outputs": [],
   "source": [
    "class LSTMModel(torch.nn.Module):\n",
    "    def __init__(self, vocab_size, embedding_dim, hidden_dim):\n",
    "        super().__init__()\n",
    "        self.embeddings = nn.Embedding(\n",
    "            vocab_size, embedding_dim, padding_idx=0)\n",
    "        self.lstm = nn.LSTM(embedding_dim, hidden_dim, batch_first=True)\n",
    "        self.linear = nn.Linear(hidden_dim, len(brand_name_mapping))\n",
    "        self.dropout = nn.Dropout(0.2)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.embeddings(x)\n",
    "        x = self.dropout(x)\n",
    "        lstm_out, (ht, ct) = self.lstm(x)\n",
    "        return self.linear(ht[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "1e676475",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(model, epochs=10, lr=0.001, wd=1e-5):\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=lr,\n",
    "                                 weight_decay=wd)\n",
    "    for i in range(epochs):\n",
    "        model.train()\n",
    "        sum_loss = 0.0\n",
    "        correct = 0\n",
    "        total = 0\n",
    "        for x, y in train_dl:\n",
    "            x = x.long()\n",
    "            y = y.long()\n",
    "            y_pred = model(x)\n",
    "            optimizer.zero_grad()\n",
    "            loss = F.cross_entropy(y_pred, y)\n",
    "            pred = torch.max(y_pred, 1)[1]\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            sum_loss += loss.item()*y.shape[0]\n",
    "            correct += (pred == y).float().sum()\n",
    "            total += y.shape[0]\n",
    "            train_loss = sum_loss/total\n",
    "            train_acc = correct/total\n",
    "        val_loss, val_acc = validation_metrics(model, test_dl)\n",
    "        print(\"train loss %.3f, train acc %.3f, val loss %.3f, and val acc %.3f\" % (\n",
    "            train_loss, train_acc, val_loss, val_acc))\n",
    "\n",
    "\n",
    "def validation_metrics(model, test_dl):\n",
    "    model.eval()\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    sum_loss = 0.0\n",
    "    for x, y in test_dl:\n",
    "        x = x.long()\n",
    "        y = y.long()\n",
    "        y_pred = model(x)\n",
    "        loss = F.cross_entropy(y_pred, y)\n",
    "        pred = torch.max(y_pred, 1)[1]\n",
    "        correct += (pred == y).float().sum()\n",
    "        total += y.shape[0]\n",
    "        sum_loss += loss.item()*y.shape[0]\n",
    "    return sum_loss/total, correct/total"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "78afab40",
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab_size = len(words)\n",
    "model_fixed = LSTMModel(vocab_size, 50, 50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "9ef59f56",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train loss 3.744, train acc 0.216, val loss 3.829, and val acc 0.165\n",
      "train loss 3.362, train acc 0.234, val loss 4.004, and val acc 0.220\n",
      "train loss 3.347, train acc 0.226, val loss 4.048, and val acc 0.165\n",
      "train loss 3.335, train acc 0.241, val loss 4.049, and val acc 0.165\n",
      "train loss 3.332, train acc 0.241, val loss 4.064, and val acc 0.165\n",
      "train loss 3.325, train acc 0.229, val loss 4.060, and val acc 0.165\n",
      "train loss 3.322, train acc 0.243, val loss 4.105, and val acc 0.165\n",
      "train loss 3.324, train acc 0.225, val loss 4.143, and val acc 0.165\n",
      "train loss 3.321, train acc 0.226, val loss 4.141, and val acc 0.165\n",
      "train loss 3.313, train acc 0.237, val loss 4.104, and val acc 0.165\n"
     ]
    }
   ],
   "source": [
    "train_model(model_fixed, epochs=10, lr=0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "07422281",
   "metadata": {},
   "outputs": [],
   "source": [
    "# saving the model for future predictions\n",
    "\n",
    "with open(\"finalized_model.pkl\", \"wb\") as myFile:\n",
    "    pickle.dump(model_fixed, myFile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "b7f1f3d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "filepath = 'model_fixed_state_dict'\n",
    "torch.save(model_fixed.state_dict(), filepath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "293e8757",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_model = LSTMModel(vocab_size, 50, 50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "ac1d9ae5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LSTMModel(\n",
       "  (embeddings): Embedding(16993, 50, padding_idx=0)\n",
       "  (lstm): LSTM(50, 50, batch_first=True)\n",
       "  (linear): Linear(in_features=50, out_features=178, bias=True)\n",
       "  (dropout): Dropout(p=0.2, inplace=False)\n",
       ")"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Later to restore:\n",
    "new_model.load_state_dict(torch.load(filepath))\n",
    "new_model.eval()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c53ad72",
   "metadata": {},
   "source": [
    "# Making predictions\n",
    "\n",
    "Because we used a subset of the data, we will test on one class which was present in the train data and one not present in the train data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "2d54dfab",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Just a moment...  wwwapkmirrorcom Checking if the site connection is secure div idchallengeerrortitle div classh2 span classiconwrapper div classheadingicon warningicondiv span span idchallengeerrortext Enable JavaScript and cookies to continue span div div wwwapkmirrorcom needs to review the security of your connection before proceeding Why am I seeing this page Requests from malicious bots can pose as legitimate traffic Occasionally you may see this page while the site ensures that the connection is secure Connection is secure Proceeding error code 1020 Ray ID 78b461416da5c37c Performance  security by Cloudflare '"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.loc[3456]['raw_text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "ed3f5780",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'unknown'"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.loc[3456]['brand_id']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "de86690f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'unknown': 2.862473964691162,\n",
       " 'wayfair': 2.7404298782348633,\n",
       " 'roblox': 2.0151207447052}"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def predict(text):\n",
    "#     with open('finalized_model.pkl', 'rb') as f:\n",
    "#         model_ = pickle.load(f)\n",
    "    model = torch.load('finalized_model.pt')\n",
    "    model.eval()\n",
    "    \n",
    "    with open('vocab2index.txt', 'rb') as f:\n",
    "        vocab2index = pickle.load(f)\n",
    "    \n",
    "    with open('brand_name_mapping.txt', 'rb') as f:\n",
    "        brand_name_mapping = pickle.load(f)\n",
    "    \n",
    "    def encode_sentence(text, vocab2index, N=3863):\n",
    "        tokenized = tokenize(text)\n",
    "        encoded = np.zeros(N, dtype=int)\n",
    "        enc1 = np.array([vocab2index.get(word, vocab2index[\"UNK\"]) for word in tokenized])\n",
    "        length = min(N, len(enc1))\n",
    "        encoded[:length] = enc1[:length]\n",
    "        return encoded\n",
    "    \n",
    "    # preprocess_data\n",
    "    encoded_input = encode_sentence(text, vocab2index, N=3863)\n",
    "    \n",
    "\n",
    "    \n",
    "    # Make a prediction\n",
    "    output = model(torch.tensor(encoded_input).unsqueeze(0))\n",
    "\n",
    "    top_values, top_indices = torch.topk(output, k=3)\n",
    "    top_values = top_values.tolist()[0]\n",
    "    top_indices = top_indices.tolist()[0]\n",
    "    result_dict = dict(zip(top_values, top_indices))\n",
    "    output_dict = {}\n",
    "    for k, v in result_dict.items():\n",
    "        brand = next(key for key, value in brand_name_mapping.items() if value == v)\n",
    "        output_dict[brand] = k\n",
    "\n",
    "    # Interpret the prediction\n",
    "    return output_dict\n",
    "  \n",
    "predict(text=data.loc[3456]['raw_text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "0005eefe",
   "metadata": {},
   "outputs": [],
   "source": [
    "# def load_glove_vectors(glove_file=\"/Users/tansha/Documents/bolster/glove.6B/glove.6B.50d.txt\"):\n",
    "#     \"\"\"Load the glove word vectors\"\"\"\n",
    "#     word_vectors = {}\n",
    "#     with open(glove_file) as f:\n",
    "#         for line in f:\n",
    "#             split = line.split()\n",
    "#             word_vectors[split[0]] = np.array([float(x) for x in split[1:]])\n",
    "#     return word_vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "30244f01",
   "metadata": {},
   "outputs": [],
   "source": [
    "# def get_emb_matrix(pretrained, word_counts, emb_size = 50):\n",
    "#     \"\"\" Creates embedding matrix from word vectors\"\"\"\n",
    "#     vocab_size = len(word_counts) + 2\n",
    "#     vocab_to_idx = {}\n",
    "#     vocab = [\"\", \"UNK\"]\n",
    "#     W = np.zeros((vocab_size, emb_size), dtype=\"float32\")\n",
    "#     W[0] = np.zeros(emb_size, dtype='float32') # adding a vector for padding\n",
    "#     W[1] = np.random.uniform(-0.25, 0.25, emb_size) # adding a vector for unknown words \n",
    "#     vocab_to_idx[\"UNK\"] = 1\n",
    "#     i = 2\n",
    "#     for word in word_counts:\n",
    "#         if word in word_vecs:\n",
    "#             W[i] = word_vecs[word]\n",
    "#         else:\n",
    "#             W[i] = np.random.uniform(-0.25,0.25, emb_size)\n",
    "#         vocab_to_idx[word] = i\n",
    "#         vocab.append(word)\n",
    "#         i += 1   \n",
    "#     return W, np.array(vocab), vocab_to_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "564973b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# word_vecs = load_glove_vectors()\n",
    "# pretrained_weights, vocab, vocab2index = get_emb_matrix(word_vecs, counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "4d1e30f7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>brand_id</th>\n",
       "      <th>raw_text</th>\n",
       "      <th>brand_id_count</th>\n",
       "      <th>length_raw_text</th>\n",
       "      <th>brand_freq_cat</th>\n",
       "      <th>length_raw_text_log_scaled</th>\n",
       "      <th>numeric_count</th>\n",
       "      <th>punctuation_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>72887</th>\n",
       "      <td>digital_federal_credit_union</td>\n",
       "      <td>Digital Federal Credit Union | Personal &amp; Busi...</td>\n",
       "      <td>24</td>\n",
       "      <td>14780</td>\n",
       "      <td>Low</td>\n",
       "      <td>4.169674</td>\n",
       "      <td>90</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           brand_id  \\\n",
       "72887  digital_federal_credit_union   \n",
       "\n",
       "                                                raw_text  brand_id_count  \\\n",
       "72887  Digital Federal Credit Union | Personal & Busi...              24   \n",
       "\n",
       "       length_raw_text brand_freq_cat  length_raw_text_log_scaled  \\\n",
       "72887            14780            Low                    4.169674   \n",
       "\n",
       "       numeric_count  punctuation_count  \n",
       "72887             90                  2  "
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.sample(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e070a792",
   "metadata": {},
   "source": [
    "# Deploying model into production\n",
    "\n",
    "* Using MLFlow server to keep track of experiements with metadata and metrics.\n",
    "* Connecting MLFlow with AirFlow to schedule model deployment\n",
    "* Scheduling the ML pipeline for retraining"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "725ddb77",
   "metadata": {},
   "source": [
    "# When might the model require retraining\n",
    "\n",
    "* Around any big event(IPO, festivals and sales, elections, tax filing days)\n",
    "* Upon special request from a client\n",
    "* Schedule an AirFlow Dag using scheduler to re-train the model at a desired frequency(daily, weekly, etc) with new data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f1994d6b",
   "metadata": {},
   "source": [
    "# Maintaining the ML Pipeline\n",
    "\n",
    "* Keep a check on data quality and distribution\n",
    "* Keeping check on data drift\n",
    "* Using load balancers and distributed training pipelines(fault tolerance)\n",
    "* Scheduling alerts and messages for failed nodes and load transfer\n",
    "* Keeping a check on model latency, throughput and accuracy\n",
    "* Fine tuning the architecture in production with changes in data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca2a3e5f",
   "metadata": {},
   "source": [
    "# Limitations of the process described above\n",
    "\n",
    "* Due to time and computational constraints, did not try complex architecutres\n",
    "* Using complex weighted metrics for improved model evaluation on precision and recall\n",
    "* Combining other text features(length, count nums, etc)\n",
    "* Using parallelized training and preprocessing for faster computation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4325a0af",
   "metadata": {},
   "source": [
    "# Methodology\n",
    "\n",
    "* EDA: \n",
    "    * Checking for label distribution\n",
    "    * Checking for length of raw text for unknown vs not-unknown classes\n",
    "    * Checking for log_scaled length distribution of raw text across brands categorized with frequency\n",
    "    * Visible difference between the median count of numbers across different categories\n",
    "    \n",
    "* preprocessing:\n",
    "    * Removing stopwords\n",
    "    * removing punctuation and non ASCII characters\n",
    "    * Removing less frequenct words and creating a word to index mapping dict(non-frequent words are mapped to token-UNK)\n",
    "    \n",
    "* baseline: LSTM\n",
    "    * Does not perform well with sequential data as it takes one input at a time\n",
    "    * Takes longer to train\n",
    "    \n",
    "* xlm-roberta-base\n",
    "    * XLM-RoBERTa is a multilingual version of RoBERTa. It is pre-trained on 2.5TB of filtered CommonCrawl data containing 100 languages.RoBERTa is a transformers model pretrained on a large corpus in a self-supervised fashion with the Masked language modeling (MLM) objective. Taking a sentence, the model randomly masks 15% of the words in the input then run the entire masked sentence through the model and has to predict the masked words. \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7c5e9c02",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}