Tags exploration and selection .ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tags exploration and selection \n",
    "\n",
    "In this notebook, I'll get closer look at the tags provided. Also, we will select the tags that we will further use for recommendation. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tags overview"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "import numpy as np "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_tags = pd.read_csv(\"data/tags.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tag_id</th>\n",
       "      <th>tag_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>--1-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>--10-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>--12-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>--122-</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   tag_id tag_name\n",
       "0       0        -\n",
       "1       1     --1-\n",
       "2       2    --10-\n",
       "3       3    --12-\n",
       "4       4   --122-"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_tags.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_tags[\"tag_len\"] = df_tags[\"tag_name\"].apply(len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def num_words(tag): \n",
    "    tag = tag.replace(\"-\", \" \") #remove \"-\"\n",
    "    num_words =len(tag.split()) \n",
    "    return num_words\n",
    "df_tags[\"num_words\"] = df_tags[\"tag_name\"].apply(num_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tag_id</th>\n",
       "      <th>tag_name</th>\n",
       "      <th>tag_len</th>\n",
       "      <th>num_words</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>-</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>--1-</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>--10-</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>--12-</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>--122-</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   tag_id tag_name  tag_len  num_words\n",
       "0       0        -        1          0\n",
       "1       1     --1-        4          1\n",
       "2       2    --10-        5          1\n",
       "3       3    --12-        5          1\n",
       "4       4   --122-        6          1"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_tags.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tag_id</th>\n",
       "      <th>tag_len</th>\n",
       "      <th>num_words</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>34252.000000</td>\n",
       "      <td>34252.000000</td>\n",
       "      <td>34252.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>17125.500000</td>\n",
       "      <td>13.149743</td>\n",
       "      <td>2.146473</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>9887.845047</td>\n",
       "      <td>6.111680</td>\n",
       "      <td>1.071366</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>8562.750000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>17125.500000</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>25688.250000</td>\n",
       "      <td>17.000000</td>\n",
       "      <td>3.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>34251.000000</td>\n",
       "      <td>35.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             tag_id       tag_len     num_words\n",
       "count  34252.000000  34252.000000  34252.000000\n",
       "mean   17125.500000     13.149743      2.146473\n",
       "std     9887.845047      6.111680      1.071366\n",
       "min        0.000000      1.000000      0.000000\n",
       "25%     8562.750000      9.000000      1.000000\n",
       "50%    17125.500000     13.000000      2.000000\n",
       "75%    25688.250000     17.000000      3.000000\n",
       "max    34251.000000     35.000000     10.000000"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_tags.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Distribution of tags over books "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_bt = pd.read_csv(\"data/book_tags.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>goodreads_book_id</th>\n",
       "      <th>tag_id</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>30574</td>\n",
       "      <td>167697</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>11305</td>\n",
       "      <td>37174</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>11557</td>\n",
       "      <td>34173</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>8717</td>\n",
       "      <td>12986</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>33114</td>\n",
       "      <td>12716</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   goodreads_book_id  tag_id   count\n",
       "0                  1   30574  167697\n",
       "1                  1   11305   37174\n",
       "2                  1   11557   34173\n",
       "3                  1    8717   12986\n",
       "4                  1   33114   12716"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_bt.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = pd.read_csv(\"data/tags.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>goodreads_book_id</th>\n",
       "      <th>tag_id</th>\n",
       "      <th>count</th>\n",
       "      <th>tag_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>30574</td>\n",
       "      <td>167697</td>\n",
       "      <td>to-read</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>30574</td>\n",
       "      <td>24549</td>\n",
       "      <td>to-read</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>30574</td>\n",
       "      <td>496107</td>\n",
       "      <td>to-read</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5</td>\n",
       "      <td>30574</td>\n",
       "      <td>11909</td>\n",
       "      <td>to-read</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6</td>\n",
       "      <td>30574</td>\n",
       "      <td>298</td>\n",
       "      <td>to-read</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   goodreads_book_id  tag_id   count tag_name\n",
       "0                  1   30574  167697  to-read\n",
       "1                  2   30574   24549  to-read\n",
       "2                  3   30574  496107  to-read\n",
       "3                  5   30574   11909  to-read\n",
       "4                  6   30574     298  to-read"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#merge with tags name\n",
    "df_bt = df_bt.merge(t, on = 'tag_id', copy=False)\n",
    "df_bt.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "b = pd.read_csv(\"data/books.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "#marge with books titles\n",
    "df_bt = df_bt.merge(b[[ 'goodreads_book_id', 'title']], on = 'goodreads_book_id', copy=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>goodreads_book_id</th>\n",
       "      <th>tag_id</th>\n",
       "      <th>count</th>\n",
       "      <th>tag_name</th>\n",
       "      <th>title_x</th>\n",
       "      <th>title_y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>30574</td>\n",
       "      <td>167697</td>\n",
       "      <td>to-read</td>\n",
       "      <td>Harry Potter and the Half-Blood Prince (Harry ...</td>\n",
       "      <td>Harry Potter and the Half-Blood Prince (Harry ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>11305</td>\n",
       "      <td>37174</td>\n",
       "      <td>fantasy</td>\n",
       "      <td>Harry Potter and the Half-Blood Prince (Harry ...</td>\n",
       "      <td>Harry Potter and the Half-Blood Prince (Harry ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>11557</td>\n",
       "      <td>34173</td>\n",
       "      <td>favorites</td>\n",
       "      <td>Harry Potter and the Half-Blood Prince (Harry ...</td>\n",
       "      <td>Harry Potter and the Half-Blood Prince (Harry ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>8717</td>\n",
       "      <td>12986</td>\n",
       "      <td>currently-reading</td>\n",
       "      <td>Harry Potter and the Half-Blood Prince (Harry ...</td>\n",
       "      <td>Harry Potter and the Half-Blood Prince (Harry ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>33114</td>\n",
       "      <td>12716</td>\n",
       "      <td>young-adult</td>\n",
       "      <td>Harry Potter and the Half-Blood Prince (Harry ...</td>\n",
       "      <td>Harry Potter and the Half-Blood Prince (Harry ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   goodreads_book_id  tag_id   count           tag_name  \\\n",
       "0                  1   30574  167697            to-read   \n",
       "1                  1   11305   37174            fantasy   \n",
       "2                  1   11557   34173          favorites   \n",
       "3                  1    8717   12986  currently-reading   \n",
       "4                  1   33114   12716        young-adult   \n",
       "\n",
       "                                             title_x  \\\n",
       "0  Harry Potter and the Half-Blood Prince (Harry ...   \n",
       "1  Harry Potter and the Half-Blood Prince (Harry ...   \n",
       "2  Harry Potter and the Half-Blood Prince (Harry ...   \n",
       "3  Harry Potter and the Half-Blood Prince (Harry ...   \n",
       "4  Harry Potter and the Half-Blood Prince (Harry ...   \n",
       "\n",
       "                                             title_y  \n",
       "0  Harry Potter and the Half-Blood Prince (Harry ...  \n",
       "1  Harry Potter and the Half-Blood Prince (Harry ...  \n",
       "2  Harry Potter and the Half-Blood Prince (Harry ...  \n",
       "3  Harry Potter and the Half-Blood Prince (Harry ...  \n",
       "4  Harry Potter and the Half-Blood Prince (Harry ...  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_bt.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_bt[['goodreads_book_id','tag_id','count']].groupby(['tag_id']).count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://stackoverflow.com/questions/14529838/apply-multiple-functions-to-multiple-groupby-columns\n",
    "df_group_tags = df_bt.groupby('tag_id').agg(\n",
    "    {'goodreads_book_id':'count', \n",
    "     'count':'sum', })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>goodreads_book_id</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tag_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        goodreads_book_id  count\n",
       "tag_id                          \n",
       "0                       7     24\n",
       "1                       2      6\n",
       "2                       1      3\n",
       "3                       1      3\n",
       "4                       1      3"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_group_tags.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>goodreads_book_id</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>34252.000000</td>\n",
       "      <td>3.425200e+04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>29.192806</td>\n",
       "      <td>6.097491e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>277.254715</td>\n",
       "      <td>7.627092e+05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>-1.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>5.000000</td>\n",
       "      <td>5.200000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>9983.000000</td>\n",
       "      <td>1.407188e+08</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       goodreads_book_id         count\n",
       "count       34252.000000  3.425200e+04\n",
       "mean           29.192806  6.097491e+03\n",
       "std           277.254715  7.627092e+05\n",
       "min             1.000000 -1.000000e+00\n",
       "25%             1.000000  3.000000e+00\n",
       "50%             1.000000  1.000000e+01\n",
       "75%             5.000000  5.200000e+01\n",
       "max          9983.000000  1.407188e+08"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_group_tags.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tag_name\n",
       "to-read              9983\n",
       "favorites            9881\n",
       "owned                9858\n",
       "books-i-own          9799\n",
       "currently-reading    9776\n",
       "library              9415\n",
       "owned-books          9221\n",
       "fiction              9097\n",
       "to-buy               8692\n",
       "kindle               8316\n",
       "Name: tag_name, dtype: int64"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Most popular tags\n",
    "tag_counts = df_bt.groupby('tag_name').tag_name.count().sort_values(ascending = False)\n",
    "tag_counts.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tags selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>goodreads_book_id</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tag_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>2</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>3</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34239</th>\n",
       "      <td>5</td>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34241</th>\n",
       "      <td>5</td>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34242</th>\n",
       "      <td>16</td>\n",
       "      <td>64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34247</th>\n",
       "      <td>5</td>\n",
       "      <td>353</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34248</th>\n",
       "      <td>10</td>\n",
       "      <td>7456</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>17095 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        goodreads_book_id  count\n",
       "tag_id                          \n",
       "0                       7     24\n",
       "1                       2      6\n",
       "15                      2      6\n",
       "21                      2     19\n",
       "22                      3     27\n",
       "...                   ...    ...\n",
       "34239                   5     31\n",
       "34241                   5     31\n",
       "34242                  16     64\n",
       "34247                   5    353\n",
       "34248                  10   7456\n",
       "\n",
       "[17095 rows x 2 columns]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Select tags that are only given to at least two books and at least 5 times \n",
    "df_sel = df_group_tags[df_group_tags[\"goodreads_book_id\"]>=2]\n",
    "df_sel "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>goodreads_book_id</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tag_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>2</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>3</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34239</th>\n",
       "      <td>5</td>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34241</th>\n",
       "      <td>5</td>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34242</th>\n",
       "      <td>16</td>\n",
       "      <td>64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34247</th>\n",
       "      <td>5</td>\n",
       "      <td>353</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34248</th>\n",
       "      <td>10</td>\n",
       "      <td>7456</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16012 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        goodreads_book_id  count\n",
       "tag_id                          \n",
       "0                       7     24\n",
       "1                       2      6\n",
       "15                      2      6\n",
       "21                      2     19\n",
       "22                      3     27\n",
       "...                   ...    ...\n",
       "34239                   5     31\n",
       "34241                   5     31\n",
       "34242                  16     64\n",
       "34247                   5    353\n",
       "34248                  10   7456\n",
       "\n",
       "[16012 rows x 2 columns]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_sel= df_sel[df_sel[\"count\"]>=5]\n",
    "df_sel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tag_id</th>\n",
       "      <th>goodreads_book_id</th>\n",
       "      <th>count</th>\n",
       "      <th>tag_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>24</td>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>--1-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>15</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>--6-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>21</td>\n",
       "      <td>2</td>\n",
       "      <td>19</td>\n",
       "      <td>-calif--</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>22</td>\n",
       "      <td>3</td>\n",
       "      <td>27</td>\n",
       "      <td>-d-c--</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   tag_id  goodreads_book_id  count  tag_name\n",
       "0       0                  7     24         -\n",
       "1       1                  2      6      --1-\n",
       "2      15                  2      6      --6-\n",
       "3      21                  2     19  -calif--\n",
       "4      22                  3     27    -d-c--"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_sel_tags = df_sel.merge(t, on = 'tag_id', copy=False, how='inner')\n",
    "df_sel_tags.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sel_tags.rename(columns={\"goodreads_book_id\": \"book_count\"}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sel_tags.sort_values(\"tag_id\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tag_id</th>\n",
       "      <th>book_count</th>\n",
       "      <th>count</th>\n",
       "      <th>tag_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>24</td>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>--1-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>15</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>--6-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>21</td>\n",
       "      <td>2</td>\n",
       "      <td>19</td>\n",
       "      <td>-calif--</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>22</td>\n",
       "      <td>3</td>\n",
       "      <td>27</td>\n",
       "      <td>-d-c--</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   tag_id  book_count  count  tag_name\n",
       "0       0           7     24         -\n",
       "1       1           2      6      --1-\n",
       "2      15           2      6      --6-\n",
       "3      21           2     19  -calif--\n",
       "4      22           3     27    -d-c--"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_sel_tags.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sel_tags.to_csv('data/tags_selected.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO delete the following cells "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'seaborn'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-38-dbb9b51cc2e8>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[1;32mimport\u001b[0m \u001b[0mseaborn\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0msns\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mmatplotlib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpyplot\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mplt\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mscipy\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mstats\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0msns\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcolor_codes\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0msns\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdistplot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtag_counts_array\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'seaborn'"
     ]
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy import stats\n",
    "sns.set(color_codes=True)\n",
    "sns.distplot(tag_counts_array)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEJCAYAAABlmAtYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAG7NJREFUeJzt3XuUnHWd5/F3dxKhJQkwoTkEERgW84VxhHgBnMNFHFg9WS8Mq8AhESajhGEBhzkruu5MMghH19Fxw8U1OnIxzEaBXRBXhbCOyEC8gKKCI5HvsgpxgHjICWoIGkjo7B/Pr4eyt5NfdVd3+vZ+ndMnVd/6PVW/bz2d+tTzPFVPd23fvh1Jkname6wnIEka/wwLSVKVYSFJqjIsJElVhoUkqcqwkCRVGRYa9yJie0TsM6C2OCK+Wi5fFhFnV+7jbyLilNGc52iJiPkR8dOI+H5EHDzgtgnblyaW6WM9AalTmfk3bQz7Y2DtaM9llLwduCszzxnktonclyYQw0ITXkSsBH6cmZ+IiEuBU4HngY3AYuDfA68D/i4iXgC+AXwKmA9sB1YDf5WZ2yLi3wEfA14AHgBOBo4DTgTeA+wB/Bp4K/Bp4BXAHOAZYGFmZkT8E/B94PXAvsBngf2AN5TlT8/Mfx6kj2XAmcA24P8AFwInAecD0yKiJzMXtYy/YEBfD5W+ZgFzy/zPyMwtO+lrC/APQP+W222ZuaztJ19ThruhNFHcFREP9P8Alw0cEBEvB/4SOCozXwd8DTgmMz8F3A+8PzNvBa6iCZJX0bzYHglcHBFzgP8OvCsz5wN3AS9reYhXAidm5huBBcCvMvOPMnMe8D2aF/d+B2fmscC7gI8D/1TmdAfw3kHm/mflPo/KzCOAHwMrM/PzwGeAm1qDAmCQvpYA12fm64FDgd8H3lLpawnws8x8DXA88IqI2HPwVaCpzLDQRPHGzJzf/wMMtuvpCeBB4AcR8Qnggcz80iDjFgD/LTO3Z+ZzNC/GC4ATgLWZ+SBAZl4PbGpZ7keZuancdjOwMiLeGxFX0mx5zGwZ+8Xy70/Lv3e0XP+9Hczpc5n5bLl+JXBSRLxksCdjB/4TsCEiPkCz1bN/mdPO+roDeEdE3A78OfDBzPz1EB5TU4RhoUkjM/todvUsptlyuDwiPj7I0G6a3U+t12fQ7P7pGjC2r+Xy5v4LEfEfgGuB3wBfAG4YsOxzA+a2tTL9aYPMafog89mZG4BzgXXA5cAPyvI77Cszv0ezBfJZ4GDguxHx2iE8pqYIw0KTRkQcSbP75ieZ+VGaF8yjys3baAIB4H8DF0ZEV0TsRvMC+4/At4B5EXFEub93AHvxuy/i/d5Ms5voWiCBt9G84A/XHcC7I2KPcv0vgHvKls/OtPb1ZuCyzLypXD+mzGmHfUXE3wLLyhbYRTTHPf6wgz40SXmAW5NGZj4YEf8DuD8iNgO/pXnRBfgy8NGyW+cvgE8C/wy8hOaF+iOZ+XxEnAn8Q0T00RwP2Eaz9TDQJ4DPRsR7aN61f4fmGMhwXQu8nOadfTfwf4FFO1/k/+vrr4BbI+JZmoPwdwOHZubTO+nrCuD6iPgxzdbQg8CNHfShSarLU5RLjYiYDSwFPpSZv4mI1wC3Aftn5oT9jzJZ+9KuZVhILSLiwzQfvd1afv5jZq4Z21l1brL2pV3HsJAkVXmAW5JUZVhIkqom8qehdqP5WOR6mlMYSJLqptGcDuZ7DPg+0M60FRYRcRnwTprPm1+bmcsj4nM055bp/8bppZl5a0ScDCwHemhOUbC03Md84BpgNnAPcF45F8+BwCqac+gksCgzN1N3FOABOkkanuOBb7Y7uBoWEfEGmjNbHkHz5Z+1EXEbzTl1TsjM9S1je4DraL5F+y/AbRGxIDNX0wTCOZl5b0RcS3NOmk8DK4AVmXljOZHaMprTFtSsB/jlL5+lr294B+nnzJnJxo3t5NLkMNX6BXueKuy5fd3dXey99x5QXkPbVQ2LzLw7It5YtgJeVpb5LXAgcF2p3QpcChwNPJKZjwJExCrgtIhYC/Rk5r3lblcCl0bENTTnrfmTlvrdtBcWLwD09W0fdlj0Lz+VTLV+wZ6nCnsesiHtvm/rAHdmbi2nfl4L3EmzhfEN4N00p2E+nub0zfvzu2m1HjhgJ/V9gE2ZuW1AXZI0jrR9gDszL4mIjwFfAU7KzFP7b4uITwJnAzfzu+fR6aI5YdnAE7ftqA6/e+K2qjlzZtYH7URv76yOlp9oplq/YM9ThT2PrnaOWRwG7J6ZD5RTBXwROCMiNmbmLWVYF823Qh+nOcrebz/gyZ3UnwL2jIhpmflCGfPkUBrYuHHzsDfFentnsWHDM8NadiKaav2CPU8V9ty+7u6uYb3Jbmc31CHA1RGxWzlZ2Sk0xxWuiIi9I2IGzVk7bwXuAyIiDo2IacBCYHVmrgO2RMSx5T7PKvWtNJ9oOqPUz6b5q2WSpHGkGhaZeTvNScd+SPOnIr+dmZcBH6U59fFamj8yc0NmbqH5WwK3lPrDNLumoDmD5uUR8TDNH2S5qtTPB84tB8GPpznhmSRpHJnI54Y6GHjU3VDtm2r9gj1PFfbcvpbdUL8PPNb2ckN+JEnSlDORT/fRsee3vjDsTxNseW4bz2z67QjPSJLGpykdFi+ZMY23ve9/DWvZr/zXU5haG72SpjJ3Q0mSqgwLSVKVYSFJqjIsJElVhoUkqcqwkCRVGRaSpCrDQpJUZVhIkqoMC0lSlWEhSaoyLCRJVYaFJKnKsJAkVRkWkqQqw0KSVGVYSJKqDAtJUpVhIUmqautvcEfEZcA7ge3AtZm5PCJOBpYDPcBNmbm0jJ0PXAPMBu4BzsvMbRFxILAK2BdIYFFmbo6IvYDPA4cAG4DTM/MXI9mkJKkz1S2LiHgD8MfAEcDrgPdGxJHAdcApwOHAURGxoCyyCrgwM+cBXcCSUl8BrMjMw4D7gWWl/mFgTWYeDlwNXDkSjUmSRk41LDLzbuCNmbmNZqtgOrAX8EhmPlrqq4DTIuIgoCcz7y2Lryz1GcAJwM2t9XL5LTRbFgA3AAvKeEnSONHWbqjM3BoRlwIXA/8T2B9Y3zJkPXDATur7AJtKsLTWaV2m7K7aBPQCT7YztzlzZrYzbFT09s4as8cerok4507Z89Rgz6OrrbAAyMxLIuJjwFeAeTTHL/p1AX00Wyrt1Cn1/jGtulpuq9q4cTN9fQPvuj2dPtEbNjzT0fK7Wm/vrAk3507Z89Rgz+3r7u4a1pvsdo5ZHFYOWpOZvwG+CJwIzG0Zth/NlsDjO6g/BewZEdNKfS4vbjk8UcYREdOBWcDGIXciSRo17Xx09hDg6ojYLSJeQnNQ+++BiIhDSwAsBFZn5jpgS0QcW5Y9q9S3AmuAM0r9bGB1uXx7uU65fU0ZL0kaJ9o5wH07cBvwQ+D7wLcz80ZgMXALsBZ4mBcPXi8CLo+Ih4GZwFWlfj5wbkSsBY4Hlpb6MuD1EfFQGXNB521JkkZSuwe4PwR8aEDtTuDIQcY+CBw9SH0dze6rgfWngbe3Mw9J0tjwG9ySpCrDQpJUZVhIkqoMC0lSlWEhSaoyLCRJVYaFJKnKsJAkVRkWkqQqw0KSVGVYSJKqDAtJUpVhIUmqMiwkSVWGhSSpyrCQJFUZFpKkKsNCklRlWEiSqgwLSVKVYSFJqprezqCIuAQ4vVy9LTM/EBGfA44Dni31SzPz1og4GVgO9AA3ZebSch/zgWuA2cA9wHmZuS0iDgRWAfsCCSzKzM0j054kaSRUtyzKi/+bgFcD84HXRsSpwOuAEzJzfvm5NSJ6gOuAU4DDgaMiYkG5q1XAhZk5D+gClpT6CmBFZh4G3A8sG7n2JEkjoZ3dUOuB92Xm85m5FfgJcGD5uS4ifhQRl0ZEN3A08EhmPpqZ22gC4rSIOAjoycx7y32uLPUZwAnAza31EepNkjRCqruhMvOh/ssR8Qqa3VHHAycC5wO/Br4KvAfYTBMu/dYDBwD776C+D7CpBEtrXZI0jrR1zAIgIl4J3Aa8PzMTOLXltk8CZ9NsIWxvWawL6KPZgmmnTqm3bc6cmUMZPqJ6e2eN2WMP10Scc6fseWqw59HV7gHuY4FbgL/MzBsj4lXAvMy8pQzpArYCjwNzWxbdD3hyJ/WngD0jYlpmvlDGPDmUBjZu3Exf38C8aU+nT/SGDc90tPyu1ts7a8LNuVP2PDXYc/u6u7uG9Sa7nQPcLwe+BCzMzBtLuQu4IiL2LscdzgVuBe5rFolDI2IasBBYnZnrgC0ldADOKvWtwBrgjFI/G1g95C4kSaOqnS2Li4HdgeUR0V/7DPBR4FvADOCWzLwBICIW02yF7A7czosHrxcBV0fEbOAHwFWlfj5wfUQsBX4OnNlZS5KkkdbOAe6LgIt2cPOKQcbfCRw5SP1Bmk9LDayvozlYLkkap/wGtySpyrCQJFUZFpKkKsNCklRlWEiSqgwLSVKVYSFJqjIsJElVhoUkqcqwkCRVGRaSpCrDQpJUZVhIkqoMC0lSlWEhSaoyLCRJVYaFJKnKsJAkVRkWkqQqw0KSVGVYSJKqDAtJUtX0dgZFxCXA6eXqbZn5gYg4GVgO9AA3ZebSMnY+cA0wG7gHOC8zt0XEgcAqYF8ggUWZuTki9gI+DxwCbABOz8xfjFiHkqSOVbcsSii8CXg1MB94bUScCVwHnAIcDhwVEQvKIquACzNzHtAFLCn1FcCKzDwMuB9YVuofBtZk5uHA1cCVI9GYJGnktLMbaj3wvsx8PjO3Aj8B5gGPZOajmbmNJiBOi4iDgJ7MvLcsu7LUZwAnADe31svlt9BsWQDcACwo4yVJ40R1N1RmPtR/OSJeQbM76pM0IdJvPXAAsP8O6vsAm0qwtNZpXabsrtoE9AJPttPAnDkz2xk2Knp7Z43ZYw/XRJxzp+x5arDn0dXWMQuAiHglcBvwfmAbzdZFvy6gj2ZLZXsbdUq9f0yrrpbbqjZu3Exf38C7bk+nT/SGDc90tPyu1ts7a8LNuVP2PDXYc/u6u7uG9Sa7rU9DRcSxwJ3ABzPzeuBxYG7LkP1otgR2VH8K2DMippX6XF7ccniijCMipgOzgI1D7kSSNGraOcD9cuBLwMLMvLGU72tuikNLACwEVmfmOmBLCReAs0p9K7AGOKPUzwZWl8u3l+uU29eU8ZKkcaKd3VAXA7sDyyOiv/YZYDFwS7ntdl48eL0IuDoiZgM/AK4q9fOB6yNiKfBz4MxSXwasjIiHgF+V5SVJ40g7B7gvAi7awc1HDjL+QeDoQerrgBMHqT8NvL02D0nS2PEb3JKkKsNCklRlWEiSqgwLSVKVYSFJqjIsJElVhoUkqcqwkCRVGRaSpCrDQpJUZVhIkqoMC0lSlWEhSaoyLCRJVYaFJKnKsJAkVRkWkqQqw0KSVGVYSJKqDAtJUpVhIUmqmt7uwIiYDXwbeGtmPhYRnwOOA54tQy7NzFsj4mRgOdAD3JSZS8vy84FrgNnAPcB5mbktIg4EVgH7AgksyszNI9OeJGkktLVlERHHAN8E5rWUXweckJnzy8+tEdEDXAecAhwOHBURC8r4VcCFmTkP6AKWlPoKYEVmHgbcDyzrtClJ0shqdzfUEuAC4EmAiHgpcCBwXUT8KCIujYhu4Gjgkcx8NDO30QTEaRFxENCTmfeW+1tZ6jOAE4CbW+udtyVJGklt7YbKzHMAIqK/tB/wDeB84NfAV4H3AJuB9S2LrgcOAPbfQX0fYFMJlta6JGkcafuYRavM/Blwav/1iPgkcDbNFsL2lqFdQB/NFkw7dUq9bXPmzBzK8BHV2ztrzB57uCbinDtlz1ODPY+uYYVFRLwKmJeZt5RSF7AVeByY2zJ0P5pdVzuqPwXsGRHTMvOFMubJocxl48bN9PUNzJv2dPpEb9jwTEfL72q9vbMm3Jw7Zc9Tgz23r7u7a1hvsof70dku4IqI2LscdzgXuBW4D4iIODQipgELgdWZuQ7YEhHHluXPKvWtwBrgjFI/G1g9zDlJkkbJsMIiM38EfBT4FrAWeCAzb8jMLcBi4JZSf5gXD14vAi6PiIeBmcBVpX4+cG5ErAWOB5YOrxVJ0mgZ0m6ozDy45fIKmo+9DhxzJ3DkIPUHaT4tNbC+DjhxKPOQJO1afoNbklRlWEiSqgwLSVKVYSFJqjIsJElVhoUkqcqwkCRVGRaSpCrDQpJUZVhIkqoMC0lSlWEhSaoyLCRJVYaFJKnKsJAkVRkWkqQqw0KSVGVYSJKqDAtJUpVhIUmqMiwkSVWGhSSpano7gyJiNvBt4K2Z+VhEnAwsB3qAmzJzaRk3H7gGmA3cA5yXmdsi4kBgFbAvkMCizNwcEXsBnwcOATYAp2fmL0a0Q0lSx6pbFhFxDPBNYF653gNcB5wCHA4cFRELyvBVwIWZOQ/oApaU+gpgRWYeBtwPLCv1DwNrMvNw4GrgypFoSpI0strZDbUEuAB4slw/GngkMx/NzG00AXFaRBwE9GTmvWXcylKfAZwA3NxaL5ffQrNlAXADsKCMlySNI9XdUJl5DkBE9Jf2B9a3DFkPHLCT+j7AphIsrfXfua+yu2oT0MuLwVQ1Z87MdoeOuN7eWWP22MM1EefcKXueGux5dLV1zGKAbmB7y/UuoG8IdUq9f0yrrpbb2rJx42b6+gbefXs6faI3bHimo+V3td7eWRNuzp2y56nBntvX3d01rDfZw/k01OPA3Jbr+9FsCeyo/hSwZ0RMK/W5vLjl8EQZR0RMB2YBG4cxJ0nSKBpOWNwHREQcWgJgIbA6M9cBWyLi2DLurFLfCqwBzij1s4HV5fLt5Trl9jVlvCRpHBlyWGTmFmAxcAuwFniYFw9eLwIuj4iHgZnAVaV+PnBuRKwFjgeWlvoy4PUR8VAZc8Hw2pAkjaa2j1lk5sEtl+8EjhxkzIM0n5YaWF8HnDhI/Wng7e3OQZI0NvwGtySpyrCQJFUZFpKkKsNCklRlWEiSqgwLSVKVYSFJqjIsJElVhoUkqcqwkCRVGRaSpCrDQpJUZVhIkqoMC0lSlWEhSaoyLCRJVYaFJKnKsJAkVRkWkqQqw0KSVGVYSJKqpneycETcBewLbC2lPwf+DbAUmAFckZmfKmNPBpYDPcBNmbm01OcD1wCzgXuA8zJzWyfzkiSNrGFvWUREFzAPODIz52fmfOBx4CPAccB84NyI+IOI6AGuA04BDgeOiogF5a5WARdm5jygC1gy7G4kSaOiky2LKP9+LSLmAFcDzwDfyMynASLiZuCdwN3AI5n5aKmvAk6LiLVAT2beW+5rJXAp8OkO5iVJGmGdHLPYG7gTOBU4CTgPOBBY3zJmPXAAsP8Q65KkcWTYWxaZ+R3gO/3XI+JammMSH24Z1gX00YTS9iHU2zZnzswhzXsk9fbOGrPHHq6JOOdO2fPUYM+ja9hhERHHAbtl5p2l1AU8BsxtGbYf8CTNsYyh1Nu2ceNm+vq21wcOotMnesOGZzpaflfr7Z014ebcKXueGuy5fd3dXcN6k93Jbqi9gL+LiN0jYhbwp8C7gJMiojciXgq8A7gDuA+IiDg0IqYBC4HVmbkO2BIRx5b7PAtY3cGcJEmjYNhhkZlfBW4Dfgh8H7guM78F/DVwF/AA8IXM/G5mbgEWA7cAa4GHgZvLXS0CLo+Ih4GZwFXDnZMkaXR09D2LzFwGLBtQ+wLwhUHG3gkcOUj9QeDoTuYhSRpdfoNbklRlWEiSqgwLSVKVYSFJqjIsJElVhoUkqcqwkCRVGRaSpCrDQpJUZVhIkqoMC0lSlWEhSaoyLCRJVYaFJKnKsJAkVRkWkqQqw0KSVGVYSJKqDAtJUpVhIUmqMiwkSVWGhSSpavpYTwAgIhYCS4EZwBWZ+akxnpIkqcWYb1lExMuAjwDHAfOBcyPiD8Z2VpKkVuNhy+Jk4BuZ+TRARNwMvBO4rLLcNIDu7q6OHnzfvXuGtdzzW1+gt3fWsJZ97rltbN68ZVjLdqrT52sisuepwZ6HvMy0oSw3HsJif2B9y/X1wNFtLDcXYO+99+jowa9d+qaOlh+O3Xabzm67zdzljwswZ87YPO5YsuepwZ6HbC7w03YHj4ew6Aa2t1zvAvraWO57wPE04fLCKMxLkiajaTRB8b2hLDQewuJxmhf9fvsBT7ax3HPAN0dlRpI0ubW9RdFvPITF14EPRUQv8CzwDuDcsZ2SJKnVmH8aKjOfAP4auAt4APhCZn53bGclSWrVtX379vooSdKUNuZbFpKk8c+wkCRVGRaSpCrDQpJUNR4+OrvLTbYTF0bEJcDp5eptmfmBiDgZWA70ADdl5tIydj5wDTAbuAc4LzO3RcSBwCpgXyCBRZm5eRe3MiQR8Qlgn8xcPNS+ImIv4PPAIcAG4PTM/MWYNNKmiHgbcAmwB/C1zLxosq/niHgX8J/L1dWZefFkXNcRMRv4NvDWzHxspNbrSPY+5bYsJtuJC8sv1ZuAV9P089qIOBO4DjgFOBw4KiIWlEVWARdm5jyab8svKfUVwIrMPAy4H1i267oYuog4CfjTltJQ+/owsCYzDweuBq7cJRMfpog4BPgM8CfAEcBryjqdtOs5Il4KXAW8ATgSOL78vk+qdR0Rx9B8wXheud7DyK3XEet9yoUFLScuzMxngf4TF05U64H3ZebzmbkV+AnNL90jmfloZm6j+QU7LSIOAnoy896y7MpSnwGcQPNc/Gt9F/YwJBHxezSB/1/K9eH09Raad1wANwALyvjx6lSad5iPl/V8BvAbJvF6pjktRTfNltSM8rOVybeulwAX8OKZK45m5NbriPU+FcNisBMXHjBGc+lYZj7U/8sTEa+g2R3Vx+A97qj3fYBN5ReztT5e/T3NFzl/Wa4Pp69/XabcvgnoHd1pd+RQYFpEfDkiHgDOZ8d9T4r1nJnP0LxDfpjmtECPAc8zydZ1Zp6TmWtaSiO5Xkes96kYFsM9ceG4FhGvBP4ReD/wMwbvcUe9D6zDOH1OIuIc4F8y886W8nD6Gnhu5/H+ezCdZqv4PcAfAcfQ7IeelOsZICKOAN4NHETzovcCzS7Xyb6u211/u7T3qRgWj1NOb160e+LCcSsijgXuBD6Ymdez4x53VH8K2DMi+s9vP5fx+5ycAbypvLu+DHg7cA5D7+uJMo6ImA7MAjaO+uyH7xfA1zNzQ2b+FriVJjwm63oGeDNwZ2Y+lZnP0exeOZHJv65H8v/viPU+FcPi68BJEdFbDqC9A7hjjOc0bBHxcuBLwMLMvLGU72tuikPLL9BCmk+SrAO2lHABOKvUtwJraF6IAc4GVu+yJoYgM/9tZv5hZs4H/gb4cmb+GUPv6/ZynXL7mjJ+vPoq8OaI2Kus0wU0+6gn5XouHgROjog9IqILeBtwN5N/XY/k/98R633KfXQ2M5+IiP4TF74EuGaCn7jwYmB3YHlE9Nc+AywGbim33c6LB78WAVeXj+r9gObTJtDsA78+IpYCPwfO3BWTH0FD7WsZsDIiHgJ+VZYftzLzvoj4OM2nZmbQ7HL8NM3+/Em5njPzaxHxauD7NAe2vwv8Lc1W1WRe11siYjEjs15HrHdPJChJqpqKu6EkSUNkWEiSqgwLSVKVYSFJqjIsJElVhoUkqcqwkCRVGRaSpKr/B2IwOLFoFTpeAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "_ = plt.hist(tag_counts_array, bins=20)  # arguments are passed to np.histogram\n",
    "plt.title(\"Histogram of tags\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://www.youtube.com/watch?v=2I2E1ZbF8pg - about aggregation in python \n",
    "https://www.shanelynn.ie/summarising-aggregation-and-grouping-data-in-python-pandas/ \n",
    "https://www.geeksforgeeks.org/python-pandas-dataframe-groupby/ "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([34066,    92,    38,    15,    14,     5,     5,     5,     4,\n",
       "            8], dtype=int64),\n",
       " array([1.0000e+00, 9.9920e+02, 1.9974e+03, 2.9956e+03, 3.9938e+03,\n",
       "        4.9920e+03, 5.9902e+03, 6.9884e+03, 7.9866e+03, 8.9848e+03,\n",
       "        9.9830e+03]))"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.histogram(tag_counts, bins=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}