CNN+LSTM.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NRz4JQqM7tiH",
        "outputId": "81bf0b71-540a-4a51-8a3d-9d8bc2c57931"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "e0u37_Us8BoG"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "from glob import glob\n",
        "import pickle\n",
        "import itertools\n",
        "import numpy as np\n",
        "from scipy.stats import zscore\n",
        "from sklearn.utils import shuffle\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "### Graph imports ###\n",
        "import matplotlib.pyplot as plt\n",
        "from PIL import Image\n",
        "import pandas as pd\n",
        "\n",
        "### Audio import ###\n",
        "import librosa\n",
        "import IPython\n",
        "from IPython.display import Audio\n",
        "\n",
        "### Plot imports ###\n",
        "from IPython.display import Image\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "### Time Distributed ConvNet imports ###\n",
        "import tensorflow as tf\n",
        "from tensorflow.keras.models import Sequential, Model\n",
        "from tensorflow.keras.layers import Input, Dense, Dropout, Activation, TimeDistributed, concatenate\n",
        "from tensorflow.keras.layers import Conv2D, MaxPooling2D, AveragePooling2D, BatchNormalization, LeakyReLU, Flatten\n",
        "from tensorflow.keras.layers import LSTM\n",
        "from tensorflow.keras.optimizers import Adam, SGD\n",
        "from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\n",
        "from tensorflow.keras import backend as K\n",
        "from keras.utils import np_utils\n",
        "from keras.utils.vis_utils import plot_model\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "\n",
        "### Warning ###\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "GYNFSfto8O6C"
      },
      "outputs": [],
      "source": [
        "Ravdess = \"/content/drive/MyDrive/actors/\"\n",
        "Crema = \"/content/drive/MyDrive/AudioWAV/\"\n",
        "Tess = \"/content/drive/MyDrive/TESS Toronto emotional speech set data/\"\n",
        "Savee = \"/content/drive/MyDrive/ALL/\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0ky7L-fO8Jq2",
        "outputId": "e56713fa-9ad2-4226-90d5-3e5d9e7f5ee1"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['angry', 'disgust', 'fear', 'happy', 'neutral', 'sad'],\n",
              "      dtype=object)"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ],
      "source": [
        "ravdess_directory_list = os.listdir(Ravdess)\n",
        "\n",
        "file_emotion = []\n",
        "file_path = []\n",
        "for dir in ravdess_directory_list:\n",
        "    # as their are 20 different actors in our previous directory we need to extract files for each actor.\n",
        "    actor = os.listdir(Ravdess + dir)\n",
        "    for file in actor:\n",
        "        part = file.split('.')[0]\n",
        "        part = part.split('-')\n",
        "        # third part in each file represents the emotion associated to that file.\n",
        "        try:\n",
        "          file_emotion.append(int(part[2]))\n",
        "          file_path.append(Ravdess + dir + '/' + file)\n",
        "        except IndexError:\n",
        "          pass\n",
        "        \n",
        "# dataframe for emotion of files\n",
        "emotion_df = pd.DataFrame(file_emotion, columns=['Emotions'])\n",
        "\n",
        "# dataframe for path of files.\n",
        "path_df = pd.DataFrame(file_path, columns=['Path'])\n",
        "Ravdess_df = pd.concat([emotion_df, path_df], axis=1)\n",
        "\n",
        "# changing integers to actual emotions.\n",
        "Ravdess_df.Emotions.replace({1:'neutral', 2:'calm', 3:'happy', 4:'sad', 5:'angry', 6:'fear', 7:'disgust', 8:'surprise'}, inplace=True)\n",
        "Ravdess_df = Ravdess_df[(Ravdess_df[\"Emotions\"] != 'surprise') &(Ravdess_df[\"Emotions\"] != 'calm')] \n",
        "Ravdess_df.head()\n",
        "np.unique(Ravdess_df.Emotions)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "u_J7SN6c8S7O",
        "outputId": "bbb77d4f-3964-4b8d-d00f-69f3660312c8"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['angry', 'disgust', 'fear', 'happy', 'neutral', 'sad'],\n",
              "      dtype=object)"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ],
      "source": [
        "crema_directory_list = os.listdir(Crema)\n",
        "\n",
        "file_emotion = []\n",
        "file_path = []\n",
        "\n",
        "for file in crema_directory_list:\n",
        "    # storing file paths\n",
        "    file_path.append(Crema + file)\n",
        "    # storing file emotions\n",
        "    part=file.split('_')\n",
        "    if part[2] == 'SAD':\n",
        "        file_emotion.append('sad')\n",
        "    elif part[2] == 'ANG':\n",
        "        file_emotion.append('angry')\n",
        "    elif part[2] == 'DIS':\n",
        "        file_emotion.append('disgust')\n",
        "    elif part[2] == 'FEA':\n",
        "        file_emotion.append('fear')\n",
        "    elif part[2] == 'HAP':\n",
        "        file_emotion.append('happy')\n",
        "    elif part[2] == 'NEU':\n",
        "        file_emotion.append('neutral')\n",
        "    else:\n",
        "        file_emotion.append('Unknown')\n",
        "        \n",
        "# dataframe for emotion of files\n",
        "emotion_df = pd.DataFrame(file_emotion, columns=['Emotions'])\n",
        "\n",
        "# dataframe for path of files.\n",
        "path_df = pd.DataFrame(file_path, columns=['Path'])\n",
        "Crema_df = pd.concat([emotion_df, path_df], axis=1)\n",
        "np.unique(Crema_df.Emotions)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "y5r4wCrH8Xtv",
        "outputId": "cd9e33c4-6393-41ab-dd38-9087f8ca3bab"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['angry', 'disgust', 'fear', 'happy', 'neutral', 'sad', 'sad (1)'],\n",
              "      dtype=object)"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ],
      "source": [
        "tess_directory_list = os.listdir(Tess)\n",
        "\n",
        "file_emotion = []\n",
        "file_path = []\n",
        "\n",
        "for dir in tess_directory_list:\n",
        "    directories = os.listdir(Tess + dir)\n",
        "    for file in directories:\n",
        "        part = file.split('.')[0]\n",
        "        try:\n",
        "          part = part.split('_')[2]\n",
        "          if part=='ps':\n",
        "              # file_emotion.append('surprise')\n",
        "              continue\n",
        "          else:\n",
        "              file_emotion.append(part)\n",
        "          file_path.append(Tess + dir + '/' + file)\n",
        "        except IndexError:\n",
        "          pass\n",
        "        \n",
        "# dataframe for emotion of files\n",
        "emotion_df = pd.DataFrame(file_emotion, columns=['Emotions'])\n",
        "\n",
        "# dataframe for path of files.\n",
        "path_df = pd.DataFrame(file_path, columns=['Path'])\n",
        "Tess_df = pd.concat([emotion_df, path_df], axis=1)\n",
        "np.unique(Tess_df.Emotions)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "P2-SmZth8dLh",
        "outputId": "36ebcc34-105d-41bb-a0e2-dd2c04fc1aa8"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['angry', 'disgust', 'fear', 'happy', 'neutral', 'sad'],\n",
              "      dtype=object)"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ],
      "source": [
        "savee_directory_list = os.listdir(Savee)\n",
        "\n",
        "file_emotion = []\n",
        "file_path = []\n",
        "\n",
        "for file in savee_directory_list:\n",
        "    file_path.append(Savee + file)\n",
        "    part = file.split('_')[1]\n",
        "    ele = part[:-6]\n",
        "    if ele=='a':\n",
        "        file_emotion.append('angry')\n",
        "    elif ele=='d':\n",
        "        file_emotion.append('disgust')\n",
        "    elif ele=='f':\n",
        "        file_emotion.append('fear')\n",
        "    elif ele=='h':\n",
        "        file_emotion.append('happy')\n",
        "    elif ele=='n':\n",
        "        file_emotion.append('neutral')\n",
        "    elif ele=='sa':\n",
        "        file_emotion.append('sad')\n",
        "    else:\n",
        "        file_emotion.append('surprise')        \n",
        "# dataframe for emotion of files\n",
        "emotion_df = pd.DataFrame(file_emotion, columns=['Emotions'])\n",
        "\n",
        "# dataframe for path of files.\n",
        "path_df = pd.DataFrame(file_path, columns=['Path'])\n",
        "\n",
        "Savee_df = pd.concat([emotion_df, path_df], axis=1)\n",
        "Savee_df = Savee_df[Savee_df['Emotions'] != 'surprise']\n",
        "np.unique(Savee_df['Emotions'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 676
        },
        "id": "kgiWaLJG810h",
        "outputId": "fbb22b4d-a024-48e2-da34-c949c29a49d8"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-7a0b14a5-a9d6-4362-a52e-5b7e53a7250b\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Emotions</th>\n",
              "      <th>Path</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>happy</td>\n",
              "      <td>/content/drive/MyDrive/actors/Actor_03/03-01-0...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>sad</td>\n",
              "      <td>/content/drive/MyDrive/actors/Actor_17/03-01-0...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>neutral</td>\n",
              "      <td>/content/drive/MyDrive/actors/Actor_15/03-01-0...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>angry</td>\n",
              "      <td>/content/drive/MyDrive/actors/Actor_19/03-01-0...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>fear</td>\n",
              "      <td>/content/drive/MyDrive/actors/Actor_14/03-01-0...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>fear</td>\n",
              "      <td>/content/drive/MyDrive/actors/Actor_21/03-01-0...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>sad</td>\n",
              "      <td>/content/drive/MyDrive/actors/Actor_01/03-01-0...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>happy</td>\n",
              "      <td>/content/drive/MyDrive/actors/Actor_03/03-01-0...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>angry</td>\n",
              "      <td>/content/drive/MyDrive/actors/Actor_20/03-01-0...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>disgust</td>\n",
              "      <td>/content/drive/MyDrive/actors/Actor_15/03-01-0...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>neutral</td>\n",
              "      <td>/content/drive/MyDrive/actors/Actor_04/03-01-0...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>angry</td>\n",
              "      <td>/content/drive/MyDrive/actors/Actor_11/03-01-0...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>disgust</td>\n",
              "      <td>/content/drive/MyDrive/actors/Actor_18/03-01-0...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>angry</td>\n",
              "      <td>/content/drive/MyDrive/actors/Actor_23/03-01-0...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>sad</td>\n",
              "      <td>/content/drive/MyDrive/actors/Actor_02/03-01-0...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>angry</td>\n",
              "      <td>/content/drive/MyDrive/actors/Actor_15/03-01-0...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>neutral</td>\n",
              "      <td>/content/drive/MyDrive/actors/Actor_12/03-01-0...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>angry</td>\n",
              "      <td>/content/drive/MyDrive/actors/Actor_20/03-01-0...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>sad</td>\n",
              "      <td>/content/drive/MyDrive/actors/Actor_13/03-01-0...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>happy</td>\n",
              "      <td>/content/drive/MyDrive/actors/Actor_12/03-01-0...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-7a0b14a5-a9d6-4362-a52e-5b7e53a7250b')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-7a0b14a5-a9d6-4362-a52e-5b7e53a7250b button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-7a0b14a5-a9d6-4362-a52e-5b7e53a7250b');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "   Emotions                                               Path\n",
              "0     happy  /content/drive/MyDrive/actors/Actor_03/03-01-0...\n",
              "1       sad  /content/drive/MyDrive/actors/Actor_17/03-01-0...\n",
              "2   neutral  /content/drive/MyDrive/actors/Actor_15/03-01-0...\n",
              "3     angry  /content/drive/MyDrive/actors/Actor_19/03-01-0...\n",
              "4      fear  /content/drive/MyDrive/actors/Actor_14/03-01-0...\n",
              "5      fear  /content/drive/MyDrive/actors/Actor_21/03-01-0...\n",
              "6       sad  /content/drive/MyDrive/actors/Actor_01/03-01-0...\n",
              "7     happy  /content/drive/MyDrive/actors/Actor_03/03-01-0...\n",
              "8     angry  /content/drive/MyDrive/actors/Actor_20/03-01-0...\n",
              "9   disgust  /content/drive/MyDrive/actors/Actor_15/03-01-0...\n",
              "10  neutral  /content/drive/MyDrive/actors/Actor_04/03-01-0...\n",
              "11    angry  /content/drive/MyDrive/actors/Actor_11/03-01-0...\n",
              "12  disgust  /content/drive/MyDrive/actors/Actor_18/03-01-0...\n",
              "13    angry  /content/drive/MyDrive/actors/Actor_23/03-01-0...\n",
              "14      sad  /content/drive/MyDrive/actors/Actor_02/03-01-0...\n",
              "15    angry  /content/drive/MyDrive/actors/Actor_15/03-01-0...\n",
              "16  neutral  /content/drive/MyDrive/actors/Actor_12/03-01-0...\n",
              "17    angry  /content/drive/MyDrive/actors/Actor_20/03-01-0...\n",
              "18      sad  /content/drive/MyDrive/actors/Actor_13/03-01-0...\n",
              "19    happy  /content/drive/MyDrive/actors/Actor_12/03-01-0..."
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ],
      "source": [
        "# data_path = pd.concat([Ravdess_df, Crema_df, Tess_df, Savee_df], axis = 0)\n",
        "data_path = Ravdess_df\n",
        "data_path = shuffle(data_path).reset_index(drop=True)\n",
        "data_path.to_csv(\"data_path.csv\",index=False)\n",
        "data_path = data_path\n",
        "RAV_df = data_path.Path\n",
        "data_path.head(20)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "NKxi6W3lefpl"
      },
      "outputs": [],
      "source": [
        "labels = data_path.Emotions.replace({'neutral':1, 'happy':2, 'sad':3,'sad (1)':3, 'angry':4, 'fear':5, 'disgust':6})\n",
        "labels = labels.ravel()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "unique, counts = np.unique(labels, return_counts=True)\n",
        "\n",
        "result = np.column_stack((unique, counts)) \n",
        "print (result)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "E_o51-RRWTOe",
        "outputId": "66f633d7-4e52-4938-a753-933ba3cd3471"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[  1  96]\n",
            " [  2 192]\n",
            " [  3 192]\n",
            " [  4 192]\n",
            " [  5 192]\n",
            " [  6 192]]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "hhlWV6WcjRym"
      },
      "outputs": [],
      "source": [
        "signal = []\n",
        "\n",
        "# Sample rate (16.0 kHz)\n",
        "sample_rate = 16000     \n",
        "\n",
        "# Max pad lenght (3.0 sec)\n",
        "max_pad_len = 49100\n",
        "\n",
        "for index,path in enumerate(RAV_df[:1]):\n",
        "    X, sample_rate = librosa.load(path\n",
        "                                  ,duration=3\n",
        "                                  ,offset=0.5\n",
        "                                 )\n",
        "    sample_rate = np.array(sample_rate)\n",
        "    \n",
        "    y = zscore(X)\n",
        "        \n",
        "    # Padding or truncated signal \n",
        "    if len(y) < max_pad_len:    \n",
        "        y_padded = np.zeros(max_pad_len)\n",
        "        y_padded[:len(y)] = y\n",
        "        y = y_padded\n",
        "    elif len(y) > max_pad_len:\n",
        "        y = np.asarray(y[:max_pad_len])\n",
        "\n",
        "    # Add to signal list\n",
        "    signal.append(y)\n",
        "    "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "h5r-PSD5jYyv"
      },
      "outputs": [],
      "source": [
        "def noisy_signal(signal, snr_low=15, snr_high=30, nb_augmented=2):\n",
        "    \n",
        "    # Signal length\n",
        "    signal_len = len(signal)\n",
        "\n",
        "    # Generate White noise\n",
        "    noise = np.random.normal(size=(nb_augmented, signal_len))\n",
        "    \n",
        "    # Compute signal and noise power\n",
        "    s_power = np.sum((signal / (2.0 ** 15)) ** 2) / signal_len\n",
        "    n_power = np.sum((noise / (2.0 ** 15)) ** 2, axis=1) / signal_len\n",
        "    \n",
        "    # Random SNR: Uniform [15, 30]\n",
        "    snr = np.random.randint(snr_low, snr_high)\n",
        "    \n",
        "    # Compute K coeff for each noise\n",
        "    K = np.sqrt((s_power / n_power) * 10 ** (- snr / 10))\n",
        "    K = np.ones((signal_len, nb_augmented)) * K\n",
        "    \n",
        "    # Generate noisy signal\n",
        "    return signal + K.T * noise"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mXkrBs3_nMr4",
        "outputId": "02a067eb-312e-4e77-d13d-f2e96d99385b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Data Augmentation: START\n",
            "Data Augmentation: END!\n"
          ]
        }
      ],
      "source": [
        "print(\"Data Augmentation: START\")\n",
        "augmented_signal = list(map(noisy_signal, signal))\n",
        "print(\"Data Augmentation: END!\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "HWO9ncO_nQE9"
      },
      "outputs": [],
      "source": [
        "def mel_spectrogram(y, sr=16000, n_fft=512, win_length=256, hop_length=128, window='hamming', n_mels=128, fmax=4000):\n",
        "    \n",
        "    np.nan_to_num(y, copy = False, nan = 0)\n",
        "    # Compute spectogram\n",
        "    mel_spect = np.abs(librosa.stft(y, n_fft=n_fft, window=window, win_length=win_length, hop_length=hop_length)) ** 2\n",
        "    \n",
        "    # Compute mel spectrogram\n",
        "    mel_spect = librosa.feature.melspectrogram(S=mel_spect, sr=sr, n_mels=n_mels, fmax=fmax)\n",
        "    \n",
        "    # Compute log-mel spectrogram\n",
        "    mel_spect = librosa.power_to_db(mel_spect, ref=np.max)\n",
        "    \n",
        "    return mel_spect"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "Dp9tSH0mnVBl"
      },
      "outputs": [],
      "source": [
        "mel_spect = np.asarray(list(map(mel_spectrogram, signal)))\n",
        "augmented_mel_spect = [np.asarray(list(map(mel_spectrogram, augmented_signal[i]))) for i in range(len(augmented_signal))]\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "id": "TDkwxILwnYHI",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 398
        },
        "outputId": "46cebb18-5acb-46f9-efa2-ff9c97fbf351"
      },
      "outputs": [
        {
          "output_type": "error",
          "ename": "ValueError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-17-c8602d48f627>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mMEL_SPECT_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mMEL_SPECT_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mAUG_MEL_SPECT_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mAUG_MEL_SPECT_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabel_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabel_test\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_test_split\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmel_spect\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maugmented_mel_spect\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.2\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mrandom_state\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;31m# Build augmented labels and train\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0maug_label_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mitertools\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchain\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_iterable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlabel\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0;36m2\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mlabel\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mlabel_train\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mAUG_MEL_SPECT_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mitertools\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchain\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_iterable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mAUG_MEL_SPECT_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/sklearn/model_selection/_split.py\u001b[0m in \u001b[0;36mtrain_test_split\u001b[0;34m(test_size, train_size, random_state, shuffle, stratify, *arrays)\u001b[0m\n\u001b[1;32m   2417\u001b[0m         \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"At least one array required as input\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2418\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2419\u001b[0;31m     \u001b[0marrays\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mindexable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0marrays\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2420\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2421\u001b[0m     \u001b[0mn_samples\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_num_samples\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marrays\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/sklearn/utils/validation.py\u001b[0m in \u001b[0;36mindexable\u001b[0;34m(*iterables)\u001b[0m\n\u001b[1;32m    368\u001b[0m     \"\"\"\n\u001b[1;32m    369\u001b[0m     \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0m_make_indexable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mX\u001b[0m \u001b[0;32min\u001b[0m \u001b[0miterables\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 370\u001b[0;31m     \u001b[0mcheck_consistent_length\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    371\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    372\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/sklearn/utils/validation.py\u001b[0m in \u001b[0;36mcheck_consistent_length\u001b[0;34m(*arrays)\u001b[0m\n\u001b[1;32m    331\u001b[0m         raise ValueError(\n\u001b[1;32m    332\u001b[0m             \u001b[0;34m\"Found input variables with inconsistent numbers of samples: %r\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 333\u001b[0;31m             \u001b[0;34m%\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ml\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0ml\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mlengths\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    334\u001b[0m         )\n\u001b[1;32m    335\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mValueError\u001b[0m: Found input variables with inconsistent numbers of samples: [1, 1, 1056]"
          ]
        }
      ],
      "source": [
        "MEL_SPECT_train, MEL_SPECT_test, AUG_MEL_SPECT_train, AUG_MEL_SPECT_test, label_train, label_test = train_test_split(mel_spect, augmented_mel_spect,labels, test_size=0.2,random_state=1)\n",
        "\n",
        "# Build augmented labels and train\n",
        "aug_label_train = np.asarray(list(itertools.chain.from_iterable([[label] * 2 for label in label_train])))\n",
        "AUG_MEL_SPECT_train = np.asarray(list(itertools.chain.from_iterable(AUG_MEL_SPECT_train)))\n",
        "\n",
        "# Concatenate original and augmented\n",
        "X_train = np.concatenate((MEL_SPECT_train, AUG_MEL_SPECT_train))\n",
        "y_train = np.concatenate((label_train, aug_label_train))\n",
        "\n",
        "# Build test set\n",
        "X_test = MEL_SPECT_test\n",
        "y_test = label_test"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tGzqMeoMdmIt"
      },
      "outputs": [],
      "source": [
        "MEL_SPECT_train, MEL_SPECT_test, label_train, label_test = train_test_split(mel_spect, labels, test_size=0.2,random_state=1)\n",
        "\n",
        "# Concatenate original and augmented\n",
        "X_train = MEL_SPECT_train\n",
        "y_train = label_train\n",
        "\n",
        "# Build test set\n",
        "X_test = MEL_SPECT_test\n",
        "y_test = label_test"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "id": "iAwv6FnQncvN",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 217
        },
        "outputId": "d33aa538-c69d-4b74-e2a4-8030c14e00c1"
      },
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-16-d305aaec8d85>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0;31m# Frame for TimeDistributed model\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m \u001b[0mX_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mframe\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhop_ts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwin_ts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     14\u001b[0m \u001b[0mX_test\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mframe\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhop_ts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwin_ts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'X_train' is not defined"
          ]
        }
      ],
      "source": [
        "win_ts = 128\n",
        "hop_ts = 64\n",
        "\n",
        "# Split spectrogram into frames\n",
        "def frame(x, win_step=128, win_size=64):\n",
        "    nb_frames = 1 + int((x.shape[2] - win_size) / win_step)\n",
        "    frames = np.zeros((x.shape[0], nb_frames, x.shape[1], win_size)).astype(np.float32)\n",
        "    for t in range(nb_frames):\n",
        "        frames[:,t,:,:] = np.copy(x[:,:,(t * win_step):(t * win_step + win_size)]).astype(np.float32)\n",
        "    return frames\n",
        "\n",
        "# Frame for TimeDistributed model\n",
        "X_train = frame(X_train, hop_ts, win_ts)\n",
        "X_test = frame(X_test, hop_ts, win_ts)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Lte0385LIOI0"
      },
      "source": [
        "# Новый раздел"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sRiYiOq4JHGX"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OG-mzmTmoDgp"
      },
      "outputs": [],
      "source": [
        "lb = LabelEncoder()\n",
        "y_train = np_utils.to_categorical(lb.fit_transform(np.ravel(y_train)))\n",
        "y_test = np_utils.to_categorical(lb.transform(np.ravel(y_test)))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GBdmsSmzoHNn"
      },
      "outputs": [],
      "source": [
        "X_train = X_train.reshape(X_train.shape[0], X_train.shape[1] , X_train.shape[2], X_train.shape[3], 1)\n",
        "X_test = X_test.reshape(X_test.shape[0], X_test.shape[1] , X_test.shape[2], X_test.shape[3], 1)\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "np.save('/content/drive/MyDrive/X_train', X_train)\n",
        "np.save('/content/drive/MyDrive/X_test', X_test)\n",
        "np.save('/content/drive/MyDrive/y_test', y_test)\n",
        "np.save('/content/drive/MyDrive/y_train', y_train)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 217
        },
        "id": "lYQe0wqx2GRi",
        "outputId": "7481f873-db5d-42eb-816b-bc9003aeed1e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-6-71773ecb1e5e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'/content/drive/MyDrive/X_train'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'/content/drive/MyDrive/X_test'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'/content/drive/MyDrive/y_test'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'/content/drive/MyDrive/y_train'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'X_train' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "X_train = np.load('/content/drive/MyDrive/X_train.npy')\n",
        "X_test = np.load('/content/drive/MyDrive/X_test.npy')\n",
        "y_test = np.load('/content/drive/MyDrive/y_test.npy')\n",
        "y_train = np.load('/content/drive/MyDrive/y_train.npy')"
      ],
      "metadata": {
        "id": "7EllGjMx5DIn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "X_train.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LvXCj6UPMlm5",
        "outputId": "c5c2ba3a-a739-4c33-a822-e4f2b4945d68"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(2532, 5, 128, 128, 1)"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uZ3Zp53YoJWm"
      },
      "outputs": [],
      "source": [
        "K.clear_session()\n",
        "\n",
        "# Define two sets of inputs: MFCC and FBANK\n",
        "input_y = Input(shape=X_train.shape[1:], name='InputMELSPECT')\n",
        "\n",
        "## First LFLB (local feature learning block)\n",
        "y = TimeDistributed(Conv2D(64, kernel_size=(3, 3), strides=(1, 1), padding='same'), name='Conv1MELSPECT')(input_y)\n",
        "y = TimeDistributed(BatchNormalization(), name='BatchNorm1MELSPECT')(y)\n",
        "y = TimeDistributed(Activation('elu'), name='Activ1MELSPECT')(y)\n",
        "y = TimeDistributed(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same'), name='MaxPool1MELSPECT')(y)\n",
        "y = TimeDistributed(Dropout(0.2), name='Drop1MELSPECT')(y)     \n",
        "\n",
        "## Second LFLB (local feature learning block)\n",
        "y = TimeDistributed(Conv2D(64, kernel_size=(3, 3), strides=(1, 1), padding='same'), name='Conv2MELSPECT')(y)\n",
        "y = TimeDistributed(BatchNormalization(), name='BatchNorm2MELSPECT')(y)\n",
        "y = TimeDistributed(Activation('elu'), name='Activ2MELSPECT')(y)\n",
        "y = TimeDistributed(MaxPooling2D(pool_size=(4, 4), strides=(4, 4), padding='same'), name='MaxPool2MELSPECT')(y)\n",
        "y = TimeDistributed(Dropout(0.2), name='Drop2MELSPECT')(y)\n",
        "\n",
        "## Second LFLB (local feature learning block)\n",
        "y = TimeDistributed(Conv2D(128, kernel_size=(3, 3), strides=(1, 1), padding='same'), name='Conv3MELSPECT')(y)\n",
        "y = TimeDistributed(BatchNormalization(), name='BatchNorm3MELSPECT')(y)\n",
        "y = TimeDistributed(Activation('elu'), name='Activ3MELSPECT')(y)\n",
        "y = TimeDistributed(MaxPooling2D(pool_size=(4, 4), strides=(4, 4), padding='same'), name='MaxPool3MELSPECT')(y)\n",
        "y = TimeDistributed(Dropout(0.2), name='Drop3MELSPECT')(y)\n",
        "\n",
        "## Second LFLB (local feature learning block)\n",
        "y = TimeDistributed(Conv2D(128, kernel_size=(3, 3), strides=(1, 1), padding='same'), name='Conv4MELSPECT')(y)\n",
        "y = TimeDistributed(BatchNormalization(), name='BatchNorm4MELSPECT')(y)\n",
        "y = TimeDistributed(Activation('elu'), name='Activ4MELSPECT')(y)\n",
        "y = TimeDistributed(MaxPooling2D(pool_size=(4, 4), strides=(4, 4), padding='same'), name='MaxPool4MELSPECT')(y)\n",
        "y = TimeDistributed(Dropout(0.2), name='Drop4MELSPECT')(y)  \n",
        "\n",
        "## Flat\n",
        "y = TimeDistributed(Flatten(), name='FlatMELSPECT')(y)                      \n",
        "                               \n",
        "# Apply 2 LSTM layer and one FC\n",
        "y = LSTM(256, return_sequences=False, dropout=0.2, name='LSTM1')(y)\n",
        "y = Dense(y_train.shape[1], activation='softmax', name='FC')(y)\n",
        "\n",
        "# Build final model\n",
        "model = Model(inputs=input_y, outputs=y)\n",
        "\n",
        "# Early stopping\n",
        "early_stopping = EarlyStopping(monitor='val_accuracy', patience=30, verbose=1, mode='max')\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_FHPTKVAoPUF",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6f91e2e4-99a5-4c04-a2c8-656213fcc1cf"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/5\n",
            "40/40 [==============================] - 35s 790ms/step - loss: 0.3115 - accuracy: 0.8819 - val_loss: 1.2328 - val_accuracy: 0.6509\n",
            "Epoch 2/5\n",
            "40/40 [==============================] - 30s 753ms/step - loss: 0.3188 - accuracy: 0.8847 - val_loss: 1.2147 - val_accuracy: 0.6368\n",
            "Epoch 3/5\n",
            "40/40 [==============================] - 30s 753ms/step - loss: 0.2982 - accuracy: 0.8965 - val_loss: 1.0097 - val_accuracy: 0.7170\n",
            "Epoch 4/5\n",
            "40/40 [==============================] - 30s 753ms/step - loss: 0.2765 - accuracy: 0.9084 - val_loss: 1.0839 - val_accuracy: 0.6792\n",
            "Epoch 5/5\n",
            "40/40 [==============================] - 30s 751ms/step - loss: 0.2744 - accuracy: 0.9044 - val_loss: 0.8822 - val_accuracy: 0.7358\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:absl:Function `_wrapped_model` contains input name(s) InputMELSPECT with unsupported characters which will be renamed to inputmelspect in the SavedModel.\n",
            "WARNING:absl:Found untraced functions such as lstm_cell_8_layer_call_fn, lstm_cell_8_layer_call_and_return_conditional_losses, lstm_cell_8_layer_call_fn, lstm_cell_8_layer_call_and_return_conditional_losses, lstm_cell_8_layer_call_and_return_conditional_losses while saving (showing 5 of 5). These functions will not be directly callable after loading.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:Assets written to: /content/drive/MyDrive/my_model_1/assets\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:Assets written to: /content/drive/MyDrive/my_model_1/assets\n",
            "WARNING:absl:<keras.layers.recurrent.LSTMCell object at 0x7fee2804f5d0> has the same name 'LSTMCell' as a built-in Keras object. Consider renaming <class 'keras.layers.recurrent.LSTMCell'> to avoid naming conflicts when loading with `tf.keras.models.load_model`. If renaming is not possible, pass the object in the `custom_objects` parameter of the load function.\n"
          ]
        }
      ],
      "source": [
        "model.compile(optimizer=SGD(lr=0.01, decay=1e-6, momentum=0.8), loss='categorical_crossentropy', metrics=['accuracy'])\n",
        "\n",
        "# Fit model\n",
        "history = model.fit(X_train, y_train, batch_size=64, epochs=5, validation_data=(X_test, y_test), callbacks=[early_stopping])\n",
        "model.save('/content/drive/MyDrive/my_model_1')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Early stopping\n",
        "early_stopping = EarlyStopping(monitor='val_accuracy', patience=30, verbose=1, mode='max')"
      ],
      "metadata": {
        "id": "gyS1jY2Z6o8M"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from tensorflow import keras\n",
        "\n",
        "for i in range(19): \n",
        "  model = keras.models.load_model('/content/drive/MyDrive/my_model_1')\n",
        "  history = model.fit(X_train, y_train, batch_size=64, epochs=5, validation_data=(X_test, y_test), callbacks=[early_stopping])\n",
        "  model.save('/content/drive/MyDrive/my_model_1')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pkAKPdgaunyQ",
        "outputId": "822f89fc-d511-45ed-82a0-17c3bbc06719"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/5\n",
            "40/40 [==============================] - 34s 779ms/step - loss: 0.1974 - accuracy: 0.9261 - val_loss: 1.0210 - val_accuracy: 0.7406\n",
            "Epoch 2/5\n",
            "40/40 [==============================] - 30s 757ms/step - loss: 0.1816 - accuracy: 0.9360 - val_loss: 1.2672 - val_accuracy: 0.6840\n",
            "Epoch 3/5\n",
            "40/40 [==============================] - 30s 756ms/step - loss: 0.1759 - accuracy: 0.9419 - val_loss: 1.0463 - val_accuracy: 0.6698\n",
            "Epoch 4/5\n",
            "40/40 [==============================] - 30s 756ms/step - loss: 0.1723 - accuracy: 0.9380 - val_loss: 2.4911 - val_accuracy: 0.4575\n",
            "Epoch 5/5\n",
            "40/40 [==============================] - 30s 755ms/step - loss: 0.1712 - accuracy: 0.9412 - val_loss: 1.0168 - val_accuracy: 0.7170\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:absl:Function `_wrapped_model` contains input name(s) InputMELSPECT with unsupported characters which will be renamed to inputmelspect in the SavedModel.\n",
            "WARNING:absl:Found untraced functions such as lstm_cell_14_layer_call_fn, lstm_cell_14_layer_call_and_return_conditional_losses, lstm_cell_14_layer_call_fn, lstm_cell_14_layer_call_and_return_conditional_losses, lstm_cell_14_layer_call_and_return_conditional_losses while saving (showing 5 of 5). These functions will not be directly callable after loading.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:Assets written to: /content/drive/MyDrive/my_model_2/assets\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:Assets written to: /content/drive/MyDrive/my_model_2/assets\n",
            "WARNING:absl:<keras.layers.recurrent.LSTMCell object at 0x7fee29153b10> has the same name 'LSTMCell' as a built-in Keras object. Consider renaming <class 'keras.layers.recurrent.LSTMCell'> to avoid naming conflicts when loading with `tf.keras.models.load_model`. If renaming is not possible, pass the object in the `custom_objects` parameter of the load function.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "g_b9tBhloz87",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "73fda292-7893-42b1-c4b3-363d68d6cdb5"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.8821555376052856, 0.7358490824699402]"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ],
      "source": [
        "+model = keras.models.load_model('/content/drive/MyDrive/my_model_1')\n",
        "score = model.evaluate(X_test, y_test, verbose=0)\n",
        "score\n"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "name": "CNN+LSTM.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}