EDA.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exploratory Data Analysis\n",
    "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1vF9IsVn5jHnbWOO7RfDRPjEaqok1Z1hK?usp=sharing)\n",
    "\n",
    "[Open In Kaggle](https://www.kaggle.com/kurianbenoy/eda-of-msc-dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import plotly.express as px\n",
    "import librosa\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "speech = pd.read_csv('speech.tsv', sep='\\t')\n",
    "sentences = pd.read_csv('sentences.tsv', sep='\\t')\n",
    "users = pd.read_csv('users.tsv', sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Peeking at the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "                     id              sentence                          user  \\\n0  007b0xQfxvsmaYVRN7wS  mmujzi8AdOyGyLt95lNH  sutUe8ScwBdhopQvzr2VzUO28q03   \n1  022Bgg06giNtYdeKfvOs  eRBPHg3VacjjQeLRN1vB  2dEDqqeI3PZx0m2D9EBHUn7noTM2   \n2  048AWoisCWs2UWf0eEtA  rRKWjXZu1vlzFi0uqngC  mYTyO3jrvgZv1XU0pZwRbSwwuhC3   \n3  06o5F5pl9YRbHpSGQTKT  tQoJUX0JnMgsAnVOKfZw  2dEDqqeI3PZx0m2D9EBHUn7noTM2   \n4  07LFdQOyNS7TFj26pJUC  KZzOTLufc3E5QUuiGeJs  sutUe8ScwBdhopQvzr2VzUO28q03   \n\n                                          fileName  \\\n0  audio/01f276e0-b451-439f-b686-6856fc11e2b4.webm   \n1  audio/cd7ec9d9-adb4-45d8-a299-a631e3252991.webm   \n2  audio/8264035d-f9e6-4933-9f9e-97eb1ec83234.webm   \n3  audio/a21ab006-57bd-485e-bc3c-e5aa70e02575.webm   \n4  audio/f42dc879-fae8-4386-b6f4-187d5f29e828.webm   \n\n                                              sample           time     vote  \n0  https://firebasestorage.googleapis.com/v0/b/ma...  1584562139945  default  \n1  https://firebasestorage.googleapis.com/v0/b/ma...  1573228579118       11  \n2  https://firebasestorage.googleapis.com/v0/b/ma...  1580404212435  default  \n3  https://firebasestorage.googleapis.com/v0/b/ma...  1573229287922       11  \n4  https://firebasestorage.googleapis.com/v0/b/ma...  1584562993254  default  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>id</th>\n      <th>sentence</th>\n      <th>user</th>\n      <th>fileName</th>\n      <th>sample</th>\n      <th>time</th>\n      <th>vote</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>007b0xQfxvsmaYVRN7wS</td>\n      <td>mmujzi8AdOyGyLt95lNH</td>\n      <td>sutUe8ScwBdhopQvzr2VzUO28q03</td>\n      <td>audio/01f276e0-b451-439f-b686-6856fc11e2b4.webm</td>\n      <td>https://firebasestorage.googleapis.com/v0/b/ma...</td>\n      <td>1584562139945</td>\n      <td>default</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>022Bgg06giNtYdeKfvOs</td>\n      <td>eRBPHg3VacjjQeLRN1vB</td>\n      <td>2dEDqqeI3PZx0m2D9EBHUn7noTM2</td>\n      <td>audio/cd7ec9d9-adb4-45d8-a299-a631e3252991.webm</td>\n      <td>https://firebasestorage.googleapis.com/v0/b/ma...</td>\n      <td>1573228579118</td>\n      <td>11</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>048AWoisCWs2UWf0eEtA</td>\n      <td>rRKWjXZu1vlzFi0uqngC</td>\n      <td>mYTyO3jrvgZv1XU0pZwRbSwwuhC3</td>\n      <td>audio/8264035d-f9e6-4933-9f9e-97eb1ec83234.webm</td>\n      <td>https://firebasestorage.googleapis.com/v0/b/ma...</td>\n      <td>1580404212435</td>\n      <td>default</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>06o5F5pl9YRbHpSGQTKT</td>\n      <td>tQoJUX0JnMgsAnVOKfZw</td>\n      <td>2dEDqqeI3PZx0m2D9EBHUn7noTM2</td>\n      <td>audio/a21ab006-57bd-485e-bc3c-e5aa70e02575.webm</td>\n      <td>https://firebasestorage.googleapis.com/v0/b/ma...</td>\n      <td>1573229287922</td>\n      <td>11</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>07LFdQOyNS7TFj26pJUC</td>\n      <td>KZzOTLufc3E5QUuiGeJs</td>\n      <td>sutUe8ScwBdhopQvzr2VzUO28q03</td>\n      <td>audio/f42dc879-fae8-4386-b6f4-187d5f29e828.webm</td>\n      <td>https://firebasestorage.googleapis.com/v0/b/ma...</td>\n      <td>1584562993254</td>\n      <td>default</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 4
    }
   ],
   "source": [
    "speech.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "                     id                                      sentence  \\\n0  08vCWSixKguCxbN8Noel  സൂചികൊണ്ടെടുക്കേണ്ടത് തൂമ്പ കൊണ്ടെടുക്കരുത്.   \n1  09qAOEXP8rX17zgrzNb0                              കയ്യിൽ കാശുണ്ടോ?   \n2  0AVDnHgMnGBJ9aexjJAk                       ഞാൻ ഒരു പാട്ട് പാടട്ടെ.   \n3  0LztDTFtWsAw6JCpvx1S                                 സമയമെത്രയായി?   \n4  0RUQfytlKJx3E0rM2hwM                        ഓണം വരാനൊരു മൂലം വേണം.   \n\n       category  \n0       proverb  \n1  conversation  \n2  conversation  \n3  conversation  \n4       proverb  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>id</th>\n      <th>sentence</th>\n      <th>category</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>08vCWSixKguCxbN8Noel</td>\n      <td>സൂചികൊണ്ടെടുക്കേണ്ടത് തൂമ്പ കൊണ്ടെടുക്കരുത്.</td>\n      <td>proverb</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>09qAOEXP8rX17zgrzNb0</td>\n      <td>കയ്യിൽ കാശുണ്ടോ?</td>\n      <td>conversation</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0AVDnHgMnGBJ9aexjJAk</td>\n      <td>ഞാൻ ഒരു പാട്ട് പാടട്ടെ.</td>\n      <td>conversation</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0LztDTFtWsAw6JCpvx1S</td>\n      <td>സമയമെത്രയായി?</td>\n      <td>conversation</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0RUQfytlKJx3E0rM2hwM</td>\n      <td>ഓണം വരാനൊരു മൂലം വേണം.</td>\n      <td>proverb</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 5
    }
   ],
   "source": [
    "sentences.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "                             id              name\n0  09eJLYsUg8XByGwaPX3P6oykfui2      Manu Kurakar\n1  0aokYrDll4TMRR3WvXVvSMtbQBA3  VIDENSHUS VIMALA\n2  10gWfkC1DFYukBgs1OI0EJEz41m2    Raju Hariharan\n3  11nKjHmpoZPhPDxX60Dt2Gxb0DD2        NuraaruN88\n4  19n9jw6zdmVj7kfN6FnU9PZTxk43     Anirudh Sankr",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>id</th>\n      <th>name</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>09eJLYsUg8XByGwaPX3P6oykfui2</td>\n      <td>Manu Kurakar</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0aokYrDll4TMRR3WvXVvSMtbQBA3</td>\n      <td>VIDENSHUS VIMALA</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>10gWfkC1DFYukBgs1OI0EJEz41m2</td>\n      <td>Raju Hariharan</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>11nKjHmpoZPhPDxX60Dt2Gxb0DD2</td>\n      <td>NuraaruN88</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>19n9jw6zdmVj7kfN6FnU9PZTxk43</td>\n      <td>Anirudh Sankr</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 6
    }
   ],
   "source": [
    "users.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "2074 sounds has been recorded by 195 users\n"
    }
   ],
   "source": [
    "print(f\"{speech.shape[0]} sounds has been recorded by {users.shape[0]} users\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Checking for null values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "id          0\nsentence    0\nuser        0\nfileName    0\nsample      0\ntime        0\nvote        0\ndtype: int64"
     },
     "metadata": {},
     "execution_count": 8
    }
   ],
   "source": [
    "speech.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "id          0\nsentence    0\ncategory    0\ndtype: int64"
     },
     "metadata": {},
     "execution_count": 9
    }
   ],
   "source": [
    "sentences.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Fortunately there is no null values in our dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sentences based on various categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "plotlyServerURL": "https://plot.ly"
       },
       "data": [
        {
         "domain": {
          "x": [
           0,
           1
          ],
          "y": [
           0,
           1
          ]
         },
         "hoverinfo": "label+percent",
         "hovertemplate": "Categories=%{label}<br>value=%{value}<extra></extra>",
         "labels": [
          "conversation",
          "proverb",
          "story",
          "default"
         ],
         "legendgroup": "",
         "marker": {
          "colors": [
           "gold",
           "mediumturquoise",
           "darkorange",
           "lightgreen"
          ],
          "line": {
           "color": "#000000",
           "width": 2
          }
         },
         "name": "",
         "showlegend": true,
         "textfont": {
          "size": 20
         },
         "textinfo": "value",
         "type": "pie",
         "values": [
          203,
          173,
          118,
          15
         ]
        }
       ],
       "layout": {
        "legend": {
         "tracegroupgap": 0
        },
        "margin": {
         "t": 60
        },
        "template": {
         "data": {
          "bar": [
           {
            "error_x": {
             "color": "#2a3f5f"
            },
            "error_y": {
             "color": "#2a3f5f"
            },
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             }
            },
            "type": "bar"
           }
          ],
          "barpolar": [
           {
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             }
            },
            "type": "barpolar"
           }
          ],
          "carpet": [
           {
            "aaxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "baxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "type": "carpet"
           }
          ],
          "choropleth": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "choropleth"
           }
          ],
          "contour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "contour"
           }
          ],
          "contourcarpet": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "contourcarpet"
           }
          ],
          "heatmap": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmap"
           }
          ],
          "heatmapgl": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmapgl"
           }
          ],
          "histogram": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "histogram"
           }
          ],
          "histogram2d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2d"
           }
          ],
          "histogram2dcontour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2dcontour"
           }
          ],
          "mesh3d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "mesh3d"
           }
          ],
          "parcoords": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "parcoords"
           }
          ],
          "pie": [
           {
            "automargin": true,
            "type": "pie"
           }
          ],
          "scatter": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter"
           }
          ],
          "scatter3d": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter3d"
           }
          ],
          "scattercarpet": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattercarpet"
           }
          ],
          "scattergeo": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergeo"
           }
          ],
          "scattergl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergl"
           }
          ],
          "scattermapbox": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattermapbox"
           }
          ],
          "scatterpolar": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolar"
           }
          ],
          "scatterpolargl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolargl"
           }
          ],
          "scatterternary": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterternary"
           }
          ],
          "surface": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "surface"
           }
          ],
          "table": [
           {
            "cells": {
             "fill": {
              "color": "#EBF0F8"
             },
             "line": {
              "color": "white"
             }
            },
            "header": {
             "fill": {
              "color": "#C8D4E3"
             },
             "line": {
              "color": "white"
             }
            },
            "type": "table"
           }
          ]
         },
         "layout": {
          "annotationdefaults": {
           "arrowcolor": "#2a3f5f",
           "arrowhead": 0,
           "arrowwidth": 1
          },
          "coloraxis": {
           "colorbar": {
            "outlinewidth": 0,
            "ticks": ""
           }
          },
          "colorscale": {
           "diverging": [
            [
             0,
             "#8e0152"
            ],
            [
             0.1,
             "#c51b7d"
            ],
            [
             0.2,
             "#de77ae"
            ],
            [
             0.3,
             "#f1b6da"
            ],
            [
             0.4,
             "#fde0ef"
            ],
            [
             0.5,
             "#f7f7f7"
            ],
            [
             0.6,
             "#e6f5d0"
            ],
            [
             0.7,
             "#b8e186"
            ],
            [
             0.8,
             "#7fbc41"
            ],
            [
             0.9,
             "#4d9221"
            ],
            [
             1,
             "#276419"
            ]
           ],
           "sequential": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ],
           "sequentialminus": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ]
          },
          "colorway": [
           "#636efa",
           "#EF553B",
           "#00cc96",
           "#ab63fa",
           "#FFA15A",
           "#19d3f3",
           "#FF6692",
           "#B6E880",
           "#FF97FF",
           "#FECB52"
          ],
          "font": {
           "color": "#2a3f5f"
          },
          "geo": {
           "bgcolor": "white",
           "lakecolor": "white",
           "landcolor": "#E5ECF6",
           "showlakes": true,
           "showland": true,
           "subunitcolor": "white"
          },
          "hoverlabel": {
           "align": "left"
          },
          "hovermode": "closest",
          "mapbox": {
           "style": "light"
          },
          "paper_bgcolor": "white",
          "plot_bgcolor": "#E5ECF6",
          "polar": {
           "angularaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "radialaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "scene": {
           "xaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "yaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "zaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           }
          },
          "shapedefaults": {
           "line": {
            "color": "#2a3f5f"
           }
          },
          "ternary": {
           "aaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "baxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "caxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "title": {
           "x": 0.05
          },
          "xaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          },
          "yaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          }
         }
        },
        "title": {
         "text": "No of sentences belonging to each category in MSC dataset"
        }
       }
      }
     },
     "metadata": {}
    }
   ],
   "source": [
    "df = sentences.category.value_counts()\n",
    "colors = [\"gold\", \"mediumturquoise\", \"darkorange\", \"lightgreen\"]\n",
    "fig = px.pie(df, df.index, df.values, labels={\"index\": \"Categories\"})\n",
    "fig.update_traces(\n",
    "    hoverinfo=\"label+percent\",\n",
    "    textinfo=\"value\",\n",
    "    textfont_size=20,\n",
    "    marker=dict(colors=colors, line=dict(color=\"#000000\", width=2)),\n",
    ")\n",
    "fig.update_layout(title=\"No of sentences belonging to each category in MSC dataset\")\n",
    "\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Top 15 most contributed users to MSC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "Muneer Jinnen          532\nKavya Manohar          253\nSanthosh Thottingal    158\nkavitha manohar        126\nabdul azeez vengara    120\nnotsoperfectkarthi     118\ndefault                 70\nJaya K                  56\nManohar MUNNUMAKAL      42\nAshish Thampi           42\nMajo Davis              35\nHrishikesh K.B          34\nvipin vijayan           31\nJyothis Jagan           25\nDr Jada                 25\nName: name, dtype: int64"
     },
     "metadata": {},
     "execution_count": 45
    }
   ],
   "source": [
    "users_speech = pd.merge(speech, users, left_on='user', right_on='id')\n",
    "users_speech.name.value_counts()[:15]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Total time duration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "output_type": "error",
     "ename": "NameError",
     "evalue": "name 'librosa' is not defined",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-12-1fe487fdf646>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mfilenames\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlistdir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'audio/'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mf\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mfilenames\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m     \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlibrosa\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'audio/'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m     \u001b[0msum\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msum\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mlibrosa\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_duration\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msr\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'librosa' is not defined"
     ]
    }
   ],
   "source": [
    "sum = 0\n",
    "filenames = os.listdir('audio/')\n",
    "for f in filenames:\n",
    "    y, sr = librosa.load(os.path.join('audio/', f))\n",
    "    sum = sum + librosa.get_duration(y=y, sr=sr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Categories of Speech sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "plotlyServerURL": "https://plot.ly"
       },
       "data": [
        {
         "domain": {
          "x": [
           0,
           1
          ],
          "y": [
           0,
           1
          ]
         },
         "hoverinfo": "label+percent",
         "hovertemplate": "Categories=%{label}<br>value=%{value}<extra></extra>",
         "labels": [
          "proverb",
          "conversation",
          "story",
          "default"
         ],
         "legendgroup": "",
         "marker": {
          "colors": [
           "gold",
           "mediumturquoise",
           "darkorange",
           "lightgreen"
          ],
          "line": {
           "color": "#000000",
           "width": 2
          }
         },
         "name": "",
         "showlegend": true,
         "textfont": {
          "size": 20
         },
         "textinfo": "value",
         "type": "pie",
         "values": [
          1081,
          560,
          347,
          86
         ]
        }
       ],
       "layout": {
        "legend": {
         "tracegroupgap": 0
        },
        "margin": {
         "t": 60
        },
        "template": {
         "data": {
          "bar": [
           {
            "error_x": {
             "color": "#2a3f5f"
            },
            "error_y": {
             "color": "#2a3f5f"
            },
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             }
            },
            "type": "bar"
           }
          ],
          "barpolar": [
           {
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             }
            },
            "type": "barpolar"
           }
          ],
          "carpet": [
           {
            "aaxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "baxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "type": "carpet"
           }
          ],
          "choropleth": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "choropleth"
           }
          ],
          "contour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "contour"
           }
          ],
          "contourcarpet": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "contourcarpet"
           }
          ],
          "heatmap": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmap"
           }
          ],
          "heatmapgl": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmapgl"
           }
          ],
          "histogram": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "histogram"
           }
          ],
          "histogram2d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2d"
           }
          ],
          "histogram2dcontour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2dcontour"
           }
          ],
          "mesh3d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "mesh3d"
           }
          ],
          "parcoords": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "parcoords"
           }
          ],
          "pie": [
           {
            "automargin": true,
            "type": "pie"
           }
          ],
          "scatter": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter"
           }
          ],
          "scatter3d": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter3d"
           }
          ],
          "scattercarpet": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattercarpet"
           }
          ],
          "scattergeo": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergeo"
           }
          ],
          "scattergl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergl"
           }
          ],
          "scattermapbox": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattermapbox"
           }
          ],
          "scatterpolar": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolar"
           }
          ],
          "scatterpolargl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolargl"
           }
          ],
          "scatterternary": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterternary"
           }
          ],
          "surface": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "surface"
           }
          ],
          "table": [
           {
            "cells": {
             "fill": {
              "color": "#EBF0F8"
             },
             "line": {
              "color": "white"
             }
            },
            "header": {
             "fill": {
              "color": "#C8D4E3"
             },
             "line": {
              "color": "white"
             }
            },
            "type": "table"
           }
          ]
         },
         "layout": {
          "annotationdefaults": {
           "arrowcolor": "#2a3f5f",
           "arrowhead": 0,
           "arrowwidth": 1
          },
          "coloraxis": {
           "colorbar": {
            "outlinewidth": 0,
            "ticks": ""
           }
          },
          "colorscale": {
           "diverging": [
            [
             0,
             "#8e0152"
            ],
            [
             0.1,
             "#c51b7d"
            ],
            [
             0.2,
             "#de77ae"
            ],
            [
             0.3,
             "#f1b6da"
            ],
            [
             0.4,
             "#fde0ef"
            ],
            [
             0.5,
             "#f7f7f7"
            ],
            [
             0.6,
             "#e6f5d0"
            ],
            [
             0.7,
             "#b8e186"
            ],
            [
             0.8,
             "#7fbc41"
            ],
            [
             0.9,
             "#4d9221"
            ],
            [
             1,
             "#276419"
            ]
           ],
           "sequential": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ],
           "sequentialminus": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ]
          },
          "colorway": [
           "#636efa",
           "#EF553B",
           "#00cc96",
           "#ab63fa",
           "#FFA15A",
           "#19d3f3",
           "#FF6692",
           "#B6E880",
           "#FF97FF",
           "#FECB52"
          ],
          "font": {
           "color": "#2a3f5f"
          },
          "geo": {
           "bgcolor": "white",
           "lakecolor": "white",
           "landcolor": "#E5ECF6",
           "showlakes": true,
           "showland": true,
           "subunitcolor": "white"
          },
          "hoverlabel": {
           "align": "left"
          },
          "hovermode": "closest",
          "mapbox": {
           "style": "light"
          },
          "paper_bgcolor": "white",
          "plot_bgcolor": "#E5ECF6",
          "polar": {
           "angularaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "radialaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "scene": {
           "xaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "yaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "zaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           }
          },
          "shapedefaults": {
           "line": {
            "color": "#2a3f5f"
           }
          },
          "ternary": {
           "aaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "baxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "caxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "title": {
           "x": 0.05
          },
          "xaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          },
          "yaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          }
         }
        },
        "title": {
         "text": "Categories of user utterances in MSC dataset"
        }
       }
      }
     },
     "metadata": {}
    }
   ],
   "source": [
    "speech_sentences = pd.merge(speech, sentences, left_on='sentence', right_on='id')\n",
    "df = speech_sentences.category.value_counts()\n",
    "colors = [\"gold\", \"mediumturquoise\", \"darkorange\", \"lightgreen\"]\n",
    "fig = px.pie(df, df.index, df.values, labels={\"index\": \"Categories\"})\n",
    "fig.update_traces(\n",
    "    hoverinfo=\"label+percent\",\n",
    "    textinfo=\"value\",\n",
    "    textfont_size=20,\n",
    "    marker=dict(colors=colors, line=dict(color=\"#000000\", width=2)),\n",
    ")\n",
    "fig.update_layout(title=\"Categories of user utterances in MSC dataset\")\n",
    "\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Good quality speech \n",
    "\n",
    "- criteria: min 3 upvotes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sound = speech[speech.vote != 'default']\n",
    "sound['vote'] = sound['vote'].astype(str).astype(float)\n",
    "good_sound = sound[sound['vote']>=3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "No of good sound sampeles is 283\n"
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "   index                    id              sentence  \\\n0      1  022Bgg06giNtYdeKfvOs  eRBPHg3VacjjQeLRN1vB   \n1      3  06o5F5pl9YRbHpSGQTKT  tQoJUX0JnMgsAnVOKfZw   \n2      7  0ErcCyR2OdDcFi31xmSz  0AVDnHgMnGBJ9aexjJAk   \n3      8  0Gbc6JQLVd6ah2P5Nz0D  2hkQEfpIiHauS7txxAH7   \n4      9  0I2iIh7OSMdWhvb85tMp  5FiRVyH58OhlpIak6NCX   \n\n                           user  \\\n0  2dEDqqeI3PZx0m2D9EBHUn7noTM2   \n1  2dEDqqeI3PZx0m2D9EBHUn7noTM2   \n2  59Tt9EuW2DOtyckjvkMVmuPqRW13   \n3  6FzGR8AJFJSEml6AIs55fEgJmqX2   \n4  oeaNxrE0uxNSfoDpdrCYGBGC7uC3   \n\n                                          fileName  \\\n0  audio/cd7ec9d9-adb4-45d8-a299-a631e3252991.webm   \n1  audio/a21ab006-57bd-485e-bc3c-e5aa70e02575.webm   \n2  audio/eb8b87cf-82a3-4a8e-91b8-775ae8250e7a.webm   \n3  audio/3f6ec7ce-e480-49ba-b53f-29990a3918e7.webm   \n4  audio/63a28877-988f-4f87-a6dd-f0d2b5c24944.webm   \n\n                                              sample           time  vote  \n0  https://firebasestorage.googleapis.com/v0/b/ma...  1573228579118  11.0  \n1  https://firebasestorage.googleapis.com/v0/b/ma...  1573229287922  11.0  \n2  https://firebasestorage.googleapis.com/v0/b/ma...  1586442218313   4.0  \n3  https://firebasestorage.googleapis.com/v0/b/ma...  1572600034618  30.0  \n4  https://firebasestorage.googleapis.com/v0/b/ma...  1572577161778  42.0  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>index</th>\n      <th>id</th>\n      <th>sentence</th>\n      <th>user</th>\n      <th>fileName</th>\n      <th>sample</th>\n      <th>time</th>\n      <th>vote</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>022Bgg06giNtYdeKfvOs</td>\n      <td>eRBPHg3VacjjQeLRN1vB</td>\n      <td>2dEDqqeI3PZx0m2D9EBHUn7noTM2</td>\n      <td>audio/cd7ec9d9-adb4-45d8-a299-a631e3252991.webm</td>\n      <td>https://firebasestorage.googleapis.com/v0/b/ma...</td>\n      <td>1573228579118</td>\n      <td>11.0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>3</td>\n      <td>06o5F5pl9YRbHpSGQTKT</td>\n      <td>tQoJUX0JnMgsAnVOKfZw</td>\n      <td>2dEDqqeI3PZx0m2D9EBHUn7noTM2</td>\n      <td>audio/a21ab006-57bd-485e-bc3c-e5aa70e02575.webm</td>\n      <td>https://firebasestorage.googleapis.com/v0/b/ma...</td>\n      <td>1573229287922</td>\n      <td>11.0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>7</td>\n      <td>0ErcCyR2OdDcFi31xmSz</td>\n      <td>0AVDnHgMnGBJ9aexjJAk</td>\n      <td>59Tt9EuW2DOtyckjvkMVmuPqRW13</td>\n      <td>audio/eb8b87cf-82a3-4a8e-91b8-775ae8250e7a.webm</td>\n      <td>https://firebasestorage.googleapis.com/v0/b/ma...</td>\n      <td>1586442218313</td>\n      <td>4.0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>8</td>\n      <td>0Gbc6JQLVd6ah2P5Nz0D</td>\n      <td>2hkQEfpIiHauS7txxAH7</td>\n      <td>6FzGR8AJFJSEml6AIs55fEgJmqX2</td>\n      <td>audio/3f6ec7ce-e480-49ba-b53f-29990a3918e7.webm</td>\n      <td>https://firebasestorage.googleapis.com/v0/b/ma...</td>\n      <td>1572600034618</td>\n      <td>30.0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>9</td>\n      <td>0I2iIh7OSMdWhvb85tMp</td>\n      <td>5FiRVyH58OhlpIak6NCX</td>\n      <td>oeaNxrE0uxNSfoDpdrCYGBGC7uC3</td>\n      <td>audio/63a28877-988f-4f87-a6dd-f0d2b5c24944.webm</td>\n      <td>https://firebasestorage.googleapis.com/v0/b/ma...</td>\n      <td>1572577161778</td>\n      <td>42.0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 67
    }
   ],
   "source": [
    "good_sound = good_sound.reset_index()\n",
    "print(\"No of good sound sampeles is\", good_sound.shape[0])\n",
    "good_sound.head()"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python_defaultSpec_1594009888764",
   "display_name": "Python 3.7.6 64-bit ('base': conda)"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}