Ensembling.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "92fbd82c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import random\n",
    "import functools\n",
    "\n",
    "import datasets as DSS\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from PIL import Image\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torchtext\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torchvision import models, transforms\n",
    "from torchvision.transforms.functional import InterpolationMode\n",
    "from torchinfo import summary\n",
    "\n",
    "import transformers\n",
    "\n",
    "from tqdm.notebook import tqdm\n",
    "import matplotlib.pyplot as plt\n",
    "import plotly.graph_objects as go\n",
    "from plotly.subplots import make_subplots\n",
    "import plotly.express as px"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c2e6a848",
   "metadata": {},
   "outputs": [],
   "source": [
    "seed = 2021\n",
    "\n",
    "torch.random.manual_seed(seed)\n",
    "torch.cuda.manual_seed(seed)\n",
    "np.random.seed(seed)\n",
    "random.seed(seed)\n",
    "torch.backends.cudnn.deterministic = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "cbf1d34f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('data_multimodal.csv')\n",
    "\n",
    "df = df.loc[(df['Text'].notnull())].reset_index(drop=True).copy()\n",
    "\n",
    "df['Label'] = df['Sentiment'].apply(lambda x: 1 if x == 'positive' else 0)\n",
    "\n",
    "df_text = df.loc[\n",
    "    (df['Type'] == 'test'), \n",
    "    ['Filename', 'Type', 'Text', 'Sentiment', 'Label']\n",
    "].drop_duplicates().reset_index(drop=True).copy()\n",
    "\n",
    "\n",
    "path_spectograms = os.path.join(os.getcwd(), 'spectograms')\n",
    "\n",
    "df_spect = df.loc[:, ['Sentiment', 'Score', 'Type', 'Spectogram']].drop_duplicates().reset_index(drop=True).copy()\n",
    "df_spect['PathSpectogram'] = df_spect['Spectogram'].apply(lambda x: os.path.join(path_spectograms, x))\n",
    "\n",
    "\n",
    "path_faces = os.path.join(os.getcwd(), 'extracted_faces')\n",
    "\n",
    "df_faces = df.loc[\n",
    "    (df['Face'].notnull()), \n",
    "    ['Filename', 'Face', 'Sentiment', 'Type', 'Score']\n",
    "].reset_index(drop=True).copy()\n",
    "\n",
    "df_faces['Face'] = df_faces['Face'].apply(lambda x: os.path.join(path_faces, x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "6b26ddc1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Filename</th>\n",
       "      <th>FrameName</th>\n",
       "      <th>Fps</th>\n",
       "      <th>Count</th>\n",
       "      <th>Size</th>\n",
       "      <th>Face</th>\n",
       "      <th>Spectogram</th>\n",
       "      <th>Start</th>\n",
       "      <th>End</th>\n",
       "      <th>Score</th>\n",
       "      <th>Sentiment</th>\n",
       "      <th>Type</th>\n",
       "      <th>Text</th>\n",
       "      <th>Nwords</th>\n",
       "      <th>Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>03bSnISJMiM_1</td>\n",
       "      <td>03bSnISJMiM_1_frame_1.jpg</td>\n",
       "      <td>30.0</td>\n",
       "      <td>1</td>\n",
       "      <td>224x224</td>\n",
       "      <td>03bSnISJMiM_1_face_1.jpg</td>\n",
       "      <td>03bSnISJMiM_1_spect.jpg</td>\n",
       "      <td>51.904533</td>\n",
       "      <td>55.945351</td>\n",
       "      <td>2.4</td>\n",
       "      <td>positive</td>\n",
       "      <td>train</td>\n",
       "      <td>anyhow it was really good</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>03bSnISJMiM_1</td>\n",
       "      <td>03bSnISJMiM_1_frame_6.jpg</td>\n",
       "      <td>30.0</td>\n",
       "      <td>6</td>\n",
       "      <td>224x224</td>\n",
       "      <td>03bSnISJMiM_1_face_6.jpg</td>\n",
       "      <td>03bSnISJMiM_1_spect.jpg</td>\n",
       "      <td>51.904533</td>\n",
       "      <td>55.945351</td>\n",
       "      <td>2.4</td>\n",
       "      <td>positive</td>\n",
       "      <td>train</td>\n",
       "      <td>anyhow it was really good</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>03bSnISJMiM_1</td>\n",
       "      <td>03bSnISJMiM_1_frame_11.jpg</td>\n",
       "      <td>30.0</td>\n",
       "      <td>11</td>\n",
       "      <td>224x224</td>\n",
       "      <td>03bSnISJMiM_1_face_11.jpg</td>\n",
       "      <td>03bSnISJMiM_1_spect.jpg</td>\n",
       "      <td>51.904533</td>\n",
       "      <td>55.945351</td>\n",
       "      <td>2.4</td>\n",
       "      <td>positive</td>\n",
       "      <td>train</td>\n",
       "      <td>anyhow it was really good</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>03bSnISJMiM_1</td>\n",
       "      <td>03bSnISJMiM_1_frame_16.jpg</td>\n",
       "      <td>30.0</td>\n",
       "      <td>16</td>\n",
       "      <td>224x224</td>\n",
       "      <td>03bSnISJMiM_1_face_16.jpg</td>\n",
       "      <td>03bSnISJMiM_1_spect.jpg</td>\n",
       "      <td>51.904533</td>\n",
       "      <td>55.945351</td>\n",
       "      <td>2.4</td>\n",
       "      <td>positive</td>\n",
       "      <td>train</td>\n",
       "      <td>anyhow it was really good</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>03bSnISJMiM_1</td>\n",
       "      <td>03bSnISJMiM_1_frame_21.jpg</td>\n",
       "      <td>30.0</td>\n",
       "      <td>21</td>\n",
       "      <td>224x224</td>\n",
       "      <td>03bSnISJMiM_1_face_21.jpg</td>\n",
       "      <td>03bSnISJMiM_1_spect.jpg</td>\n",
       "      <td>51.904533</td>\n",
       "      <td>55.945351</td>\n",
       "      <td>2.4</td>\n",
       "      <td>positive</td>\n",
       "      <td>train</td>\n",
       "      <td>anyhow it was really good</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        Filename                   FrameName   Fps  Count     Size  \\\n",
       "0  03bSnISJMiM_1   03bSnISJMiM_1_frame_1.jpg  30.0      1  224x224   \n",
       "1  03bSnISJMiM_1   03bSnISJMiM_1_frame_6.jpg  30.0      6  224x224   \n",
       "2  03bSnISJMiM_1  03bSnISJMiM_1_frame_11.jpg  30.0     11  224x224   \n",
       "3  03bSnISJMiM_1  03bSnISJMiM_1_frame_16.jpg  30.0     16  224x224   \n",
       "4  03bSnISJMiM_1  03bSnISJMiM_1_frame_21.jpg  30.0     21  224x224   \n",
       "\n",
       "                        Face               Spectogram      Start        End  \\\n",
       "0   03bSnISJMiM_1_face_1.jpg  03bSnISJMiM_1_spect.jpg  51.904533  55.945351   \n",
       "1   03bSnISJMiM_1_face_6.jpg  03bSnISJMiM_1_spect.jpg  51.904533  55.945351   \n",
       "2  03bSnISJMiM_1_face_11.jpg  03bSnISJMiM_1_spect.jpg  51.904533  55.945351   \n",
       "3  03bSnISJMiM_1_face_16.jpg  03bSnISJMiM_1_spect.jpg  51.904533  55.945351   \n",
       "4  03bSnISJMiM_1_face_21.jpg  03bSnISJMiM_1_spect.jpg  51.904533  55.945351   \n",
       "\n",
       "   Score Sentiment   Type                       Text  Nwords  Label  \n",
       "0    2.4  positive  train  anyhow it was really good       5      1  \n",
       "1    2.4  positive  train  anyhow it was really good       5      1  \n",
       "2    2.4  positive  train  anyhow it was really good       5      1  \n",
       "3    2.4  positive  train  anyhow it was really good       5      1  \n",
       "4    2.4  positive  train  anyhow it was really good       5      1  "
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d4cce18c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4bd643ee",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "0d4dd6f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def collate_fn(batch, pad_index):\n",
    "    \n",
    "    batch_ids = [i['ids'] for i in batch]\n",
    "    batch_ids = nn.utils.rnn.pad_sequence(batch_ids, padding_value=pad_index, batch_first=True)\n",
    "    batch_label = [i['label'] for i in batch]\n",
    "    batch_label = torch.stack(batch_label)\n",
    "    batch = {\n",
    "        'ids': batch_ids,\n",
    "        'label': batch_label\n",
    "    }\n",
    "    return batch\n",
    "\n",
    "\n",
    "def evaluate(dataloader, model, criterion, device):\n",
    "    \n",
    "    model.eval()\n",
    "    epoch_losses = []\n",
    "    epoch_accs = []\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        \n",
    "        for batch in tqdm(dataloader):\n",
    "            \n",
    "            ids = batch['ids'].to(device)\n",
    "            label = batch['label'].to(device)\n",
    "            prediction = model(ids)\n",
    "            loss = criterion(prediction, label)\n",
    "            accuracy = get_accuracy(prediction ,label)\n",
    "            epoch_losses.append(loss.item())\n",
    "            epoch_accs.append(accuracy.item())\n",
    "        \n",
    "    return np.mean(epoch_losses), np.mean(epoch_accs)\n",
    "\n",
    "\n",
    "def get_accuracy(prediction, label):\n",
    "    \n",
    "    batch_size, _ = prediction.shape\n",
    "    predicted_classes = torch.sigmoid(prediction).argmax(dim=-1)\n",
    "    correct_predictions = predicted_classes.eq(label).sum()\n",
    "    accuracy = correct_predictions / batch_size\n",
    "    \n",
    "    return accuracy\n",
    "\n",
    "'''\n",
    "Transformer\n",
    "'''\n",
    "class TextDatasetTransformer(torch.utils.data.Dataset):\n",
    "    \n",
    "    def __init__(self, df, tokenizer):\n",
    "        \n",
    "        self.ids = list(map(\n",
    "            lambda x: tokenizer(x, truncation=True)['input_ids'], \n",
    "            df['Text'].tolist()\n",
    "        ))\n",
    "        self.labels = df['Label'].tolist()\n",
    "        \n",
    "    def __len__(self):\n",
    "        \n",
    "        return len(self.ids)\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        \n",
    "        ids = torch.tensor(self.ids[idx])\n",
    "        \n",
    "        label = torch.tensor(self.labels[idx])\n",
    "        \n",
    "        sample = {'ids': ids, 'label': label}\n",
    "        \n",
    "        return sample\n",
    "    \n",
    "\n",
    "class BERTLSTM(nn.Module):\n",
    "    def __init__(self, bert, hidden_dim, output_dim, n_layers, bidirectional, dropout):\n",
    "        \n",
    "        super().__init__()\n",
    "        \n",
    "        self.bert = bert\n",
    "        embedding_dim = bert.config.hidden_size\n",
    "        self.rnn = nn.LSTM(\n",
    "            embedding_dim, \n",
    "            hidden_dim, \n",
    "            num_layers=n_layers,\n",
    "            bidirectional=bidirectional, \n",
    "            batch_first=True, \n",
    "            dropout = 0 if n_layers < 2 else dropout\n",
    "        )\n",
    "        self.out = nn.Linear(hidden_dim * 2 if bidirectional else hidden_dim, output_dim)\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        \n",
    "        for param in self.bert.parameters():\n",
    "            param.requires_grad = False\n",
    "        \n",
    "    def forward(self, text):\n",
    "        \n",
    "        with torch.no_grad():\n",
    "            embedded = self.bert(text)[0]\n",
    "    \n",
    "        _, (hidden, cell) = self.rnn(embedded)\n",
    "        \n",
    "        if self.rnn.bidirectional:\n",
    "            hidden = self.dropout(torch.cat((hidden[-2, :, :], hidden[-1, :, :]), dim=1))\n",
    "        else:\n",
    "            hidden = self.dropout(hidden[-1, :, :])\n",
    "        \n",
    "        output = self.out(hidden)\n",
    "        \n",
    "        return output\n",
    "    \n",
    "\n",
    "def get_transformer_model(path_saved_models, batch_size, df_text):\n",
    "    \n",
    "    model_name = 'BERTLSTM.pt'\n",
    "    path_transformer = os.path.join(path_saved_models, model_name)\n",
    "\n",
    "    transformer_name = 'bert-base-uncased'\n",
    "\n",
    "    transformer_tokenizer = transformers.AutoTokenizer.from_pretrained(transformer_name)\n",
    "\n",
    "    pad_index = transformer_tokenizer.pad_token_id\n",
    "\n",
    "    transformer_dataset = TextDatasetTransformer(df_text, transformer_tokenizer)\n",
    "\n",
    "    collate = functools.partial(collate_fn, pad_index=pad_index)\n",
    "\n",
    "    transformer_dataloader = DataLoader(\n",
    "        transformer_dataset,\n",
    "        batch_size=batch_size, \n",
    "        collate_fn=collate, \n",
    "        shuffle=False\n",
    "    )\n",
    "    transformer = transformers.AutoModel.from_pretrained(transformer_name)\n",
    "\n",
    "    hidden_dim = 256\n",
    "    n_layers = 2\n",
    "    bidirectional = True\n",
    "    dropout = 0.5\n",
    "\n",
    "    bert_lstm = BERTLSTM(transformer, hidden_dim, output_dim, n_layers, bidirectional, dropout)\n",
    "    bert_lstm.load_state_dict(torch.load(path_transformer))\n",
    "    \n",
    "    return bert_lstm, transformer_dataloader\n",
    "\n",
    "'''\n",
    "FastText\n",
    "'''\n",
    "def generate_bigrams(x):\n",
    "    \n",
    "    n_grams = set(zip(*[x[i:] for i in range(2)]))\n",
    "    \n",
    "    for n_gram in n_grams:\n",
    "        x.append(' '.join(n_gram))\n",
    "    \n",
    "    return x\n",
    "\n",
    "\n",
    "def tokenize_data(example, tokenizer, generate_ngrams):\n",
    "    \n",
    "    tokens = generate_ngrams(tokenizer(example['text']))\n",
    "    length = len(tokens)\n",
    "    \n",
    "    return {'tokens': tokens, 'length': length}\n",
    "\n",
    "\n",
    "# Bag of Tricks: https://arxiv.org/pdf/1607.01759.pdf\n",
    "    \n",
    "class FastText(nn.Module):\n",
    "    \n",
    "    def __init__(self, vocab_size, embedding_dim, output_dimm, pad_index):\n",
    "        \n",
    "        super().__init__()\n",
    "        self.embedding = nn.Embedding(vocab_size, embedding_dim, padding_idx=pad_index)\n",
    "        self.fc = nn.Linear(embedding_dim, output_dim)\n",
    "        \n",
    "    def forward(self, text, *args):\n",
    "        \n",
    "        embedded = self.embedding(text)\n",
    "        pooled = torch.mean(embedded, axis=1).squeeze(1)\n",
    "        \n",
    "        return self.fc(pooled)\n",
    "    \n",
    "    \n",
    "class FastTextDataset(torch.utils.data.Dataset):\n",
    "    \n",
    "    def __init__(self, df, tokenizer, max_length, vocab, preprocess=None):\n",
    "        # preprocess is a function that can be called after tokenization! \n",
    "         # (mainly used to generate bi-grams)\n",
    "        \n",
    "        self.texts = list(map(\n",
    "            lambda x: preprocess(tokenizer(x)) if preprocess else tokenizer(x), \n",
    "            df['Text'].tolist()\n",
    "        ))\n",
    "        self.numericalized = [list(map(lambda x: vocab[x], i)) for i in self.texts]\n",
    "        self.labels = df['Label'].tolist()\n",
    "        self.max_length = max_length\n",
    "        \n",
    "    def __len__(self):\n",
    "        \n",
    "        return len(self.texts)\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        \n",
    "        n_ids = len(self.numericalized[idx])\n",
    "        \n",
    "        if n_ids > self.max_length:\n",
    "            start_index = random.choice(range(n_ids - self.max_length))\n",
    "            ids = torch.tensor(self.numericalized[idx][start_index: start_index + self.max_length])\n",
    "        else:\n",
    "            ids = torch.tensor(self.numericalized[idx])\n",
    "        \n",
    "        label = torch.tensor(self.labels[idx])\n",
    "        \n",
    "        sample = {'ids': ids, 'label': label}\n",
    "        \n",
    "        return sample\n",
    "\n",
    "    \n",
    "def get_fast_text_model(path_saved_models, batch_size, df_text):\n",
    "    \n",
    "    model_name = 'FastText_IMDB.pt'\n",
    "    vocab_name = 'FastTextVocab.pth'\n",
    "\n",
    "    path_fast_text = os.path.join(path_saved_models, model_name)\n",
    "    path_vocab = os.path.join(path_saved_models, vocab_name)\n",
    "\n",
    "    vocab = torch.load(path_vocab)\n",
    "\n",
    "    unk_index = vocab['<unk>']\n",
    "    pad_index = vocab['<pad>']\n",
    "\n",
    "    vocab.set_default_index(unk_index)\n",
    "    \n",
    "    collate = functools.partial(collate_fn, pad_index=pad_index)\n",
    "\n",
    "    max_length = 30\n",
    "    tokenizer = torchtext.data.utils.get_tokenizer('basic_english')\n",
    "\n",
    "    fast_text_dataset = FastTextDataset(\n",
    "        df_text, \n",
    "        tokenizer,\n",
    "        max_length,\n",
    "        vocab,\n",
    "        generate_bigrams\n",
    "    )\n",
    "    fast_text_dataloader = DataLoader(\n",
    "        fast_text_dataset,\n",
    "        batch_size=batch_size, \n",
    "        collate_fn=collate, \n",
    "        shuffle=False\n",
    "    )\n",
    "    pad_index = vocab['<pad>']\n",
    "    vocab_size = len(vocab)\n",
    "    embedding_dim = 300\n",
    "\n",
    "    fast_text_model = FastText(vocab_size, embedding_dim, output_dim, pad_index)\n",
    "    fast_text_model.load_state_dict(torch.load(path_fast_text))\n",
    "\n",
    "    return fast_text_model, fast_text_dataloader\n",
    "\n",
    "\n",
    "'''\n",
    "Spectogram CNN\n",
    "'''\n",
    "class SpectogramDataset(Dataset):\n",
    "    \n",
    "    def __init__(self, df, transforms):\n",
    "        \n",
    "        self.path_spectograms = df['PathSpectogram'].tolist()\n",
    "        self.labels = list(map(lambda x: 1 if x == 'positive' else 0, df['Sentiment'].tolist()))\n",
    "        self.transforms = transforms\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.path_spectograms)\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        \n",
    "        image = Image.open(self.path_spectograms[idx])\n",
    "        label = self.labels[idx]\n",
    "        image = self.transforms(image)\n",
    "            \n",
    "        return image, label\n",
    "    \n",
    "\n",
    "def get_cnn_spectogram(path_saved_models, batch_size, df_spect):\n",
    "    \n",
    "    model_name = 'CNNSpectogram.pt'\n",
    "    path_cnn_spect = os.path.join(path_saved_models, model_name)\n",
    "    \n",
    "    num_classes = df_spect['Sentiment'].nunique()\n",
    "    \n",
    "    tensor_transforms = transforms.Compose([\n",
    "    transforms.ToTensor()\n",
    "    ])\n",
    "\n",
    "    channel_means = []\n",
    "    channel_stds = []\n",
    "\n",
    "    for pf in tqdm(df_spect.loc[(df_spect['Type'] == 'train'), 'PathSpectogram'].tolist()):\n",
    "\n",
    "        img = tensor_transforms(Image.open(pf))\n",
    "\n",
    "        channel_means.append(img.mean(axis=(1, 2)).numpy())\n",
    "        channel_stds.append(img.std(axis=(1, 2)).numpy())\n",
    "\n",
    "    channel_means = np.stack(channel_means).mean(axis=0).tolist()\n",
    "    channel_stds = np.stack(channel_stds).mean(axis=0).tolist()\n",
    "    \n",
    "    size = (224, 224)\n",
    "    \n",
    "    test_transforms = transforms.Compose([\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize(mean=channel_means, std=channel_stds),\n",
    "        transforms.Resize(size, interpolation=InterpolationMode.NEAREST)\n",
    "    ])\n",
    "    \n",
    "    spect_dataset = SpectogramDataset(\n",
    "        df_spect.loc[(df_spect['Type'] == 'test')].reset_index(drop=True).copy(),\n",
    "        test_transforms\n",
    "    )\n",
    "    spect_dataloader = DataLoader(\n",
    "        spect_dataset,\n",
    "        batch_size=batch_size,\n",
    "        shuffle=False\n",
    "    )\n",
    "    cnn_spect = models.resnet18(pretrained=False)\n",
    "\n",
    "    num_ftrs = cnn_spect.fc.in_features\n",
    "    cnn_spect.fc = nn.Sequential(\n",
    "        nn.Dropout(0.5),\n",
    "        nn.Linear(num_ftrs, num_classes),\n",
    "    )\n",
    "    cnn_spect.load_state_dict(torch.load(path_cnn_spect))\n",
    "    \n",
    "    return cnn_spect, spect_dataloader\n",
    "    \n",
    "\n",
    "'''\n",
    "CNN Faces - extracted faces\n",
    "'''\n",
    "class FaceDataset(Dataset):\n",
    "    \n",
    "    def __init__(self, df, transforms):\n",
    "        \n",
    "        self.path_faces = df['Face'].tolist()\n",
    "        self.labels = list(map(lambda x: 1 if x == 'positive' else 0, df['Sentiment'].tolist()))\n",
    "        self.transforms = transforms\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.path_faces)\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        \n",
    "        image = Image.open(self.path_faces[idx])\n",
    "        label = self.labels[idx]\n",
    "        image = self.transforms(image)\n",
    "            \n",
    "        return image, label\n",
    "    \n",
    "\n",
    "def get_cnn_faces(path_saved_models, batch_size, df_faces):\n",
    "    \n",
    "    model_name = 'CNNFaces.pt'\n",
    "    path_cnn_faces = os.path.join(path_saved_models, model_name)\n",
    "    \n",
    "    num_classes = df_faces['Sentiment'].nunique()\n",
    "    \n",
    "    tensor_transforms = transforms.Compose([\n",
    "    transforms.ToTensor()\n",
    "    ])\n",
    "    channel_means = []\n",
    "    channel_stds = []\n",
    "\n",
    "    for pf in tqdm(df_faces.loc[(df_faces['Type'] == 'train'), 'Face'].tolist()):\n",
    "\n",
    "        img = tensor_transforms(Image.open(pf))\n",
    "\n",
    "        channel_means.append(img.mean(axis=(1, 2)).numpy())\n",
    "        channel_stds.append(img.std(axis=(1, 2)).numpy())\n",
    "\n",
    "    channel_means = np.stack(channel_means).mean(axis=0).tolist()\n",
    "    channel_stds = np.stack(channel_stds).mean(axis=0).tolist()\n",
    "    \n",
    "    size = (200, 70)\n",
    "    \n",
    "    test_transforms = transforms.Compose([\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize(mean=channel_means, std=channel_stds),\n",
    "        transforms.Resize(size, interpolation=InterpolationMode.NEAREST)\n",
    "    ])\n",
    "    faces_dataset = FaceDataset(\n",
    "        df_faces.loc[(df_faces['Type'] == 'test')].reset_index(drop=True).copy(),\n",
    "        test_transforms\n",
    "    )\n",
    "    faces_dataloader = DataLoader(\n",
    "        faces_dataset,\n",
    "        batch_size=batch_size,\n",
    "        shuffle=False\n",
    "    )\n",
    "    \n",
    "    cnn_faces = models.resnet18(pretrained=True)\n",
    "\n",
    "    num_ftrs = cnn_faces.fc.in_features\n",
    "    cnn_faces.fc = nn.Sequential(\n",
    "        nn.Dropout(0.5),\n",
    "        nn.Linear(num_ftrs, num_classes)\n",
    "    )\n",
    "    cnn_faces.load_state_dict(torch.load(path_cnn_faces))\n",
    "\n",
    "    return cnn_faces, faces_dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "cb244c94",
   "metadata": {},
   "outputs": [],
   "source": [
    "path_saved_models = os.path.join(os.getcwd(), 'saved_models')\n",
    "\n",
    "output_dim = df['Label'].nunique()\n",
    "\n",
    "batch_size = 32\n",
    "\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "device = torch.device('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f3dd5d05",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertModel: ['cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.bias']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "source": [
    "bert_lstm, transformer_dataloader = get_transformer_model(path_saved_models, batch_size, df_text)\n",
    "\n",
    "fast_text_model, fast_text_dataloader = get_fast_text_model(path_saved_models, batch_size, df_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d1993a28",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c0333c50d36c454896248339c39f27d7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/22 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Transformer test_loss: 0.620, test_acc: 0.681\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "57e28a9d4535444091cb1194f6c2b82a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/22 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FastText test_loss: 3.544, test_acc: 0.690\n"
     ]
    }
   ],
   "source": [
    "bert_lstm.to(device)\n",
    "test_loss, test_acc = evaluate(transformer_dataloader, bert_lstm, criterion, device)\n",
    "print(f'Transformer test_loss: {test_loss:.3f}, test_acc: {test_acc:.3f}')\n",
    "\n",
    "fast_text_model.to(device)\n",
    "test_loss, test_acc = evaluate(fast_text_dataloader, fast_text_model, criterion, device)\n",
    "print(f'FastText test_loss: {test_loss:.3f}, test_acc: {test_acc:.3f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b2cbda92",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "c055862f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_predictions_text(dataloader, model, device):\n",
    "    \n",
    "    predictions = []\n",
    "    model.to(device)\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        \n",
    "        for batch in tqdm(dataloader):\n",
    "            \n",
    "            ids = batch['ids'].to(device)\n",
    "            preds = torch.sigmoid(model(ids)).cpu()\n",
    "            predictions.append(preds)\n",
    "            \n",
    "    return torch.cat(predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "20294b89",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "caf4d187e4ff4c5d9c221ef798c3d947",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/22 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f44663c1181744fa9ddb9a3dbcb15f3d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/22 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "trf_preds = get_predictions_text(transformer_dataloader, bert_lstm, device)\n",
    "ft_preds = get_predictions_text(fast_text_dataloader, fast_text_model, device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "3e84a2f6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Averaged predictions accuracy: 0.7124\n"
     ]
    }
   ],
   "source": [
    "# Average predictions\n",
    "labels = torch.tensor(transformer_dataloader.dataset.labels)\n",
    "\n",
    "avg_preds = torch.stack([trf_preds, ft_preds]).mean(dim=0).argmax(dim=1)\n",
    "\n",
    "print(f'Averaged predictions accuracy: {avg_preds.eq(labels).sum() / len(labels):.4f}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "03db4b3e",
   "metadata": {},
   "source": [
    "## The averaged predictions yields 1.6% more accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "4a3e5f8d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ac90c02c28374a369d5975e0e854c5bf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1284 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cnn_spect, spect_dataloader = get_cnn_spectogram(path_saved_models, batch_size, df_spect)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "92b80f27",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_predictions_image(dataloader, model, device):\n",
    "    \n",
    "    predictions = []\n",
    "    model.to(device)\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        \n",
    "        for images, labels in tqdm(dataloader):\n",
    "            \n",
    "            images = images.to(device)\n",
    "            preds = torch.sigmoid(model(images)).cpu()\n",
    "            predictions.append(preds)\n",
    "            \n",
    "    return torch.cat(predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "cf9b0b3f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "72dd5292a9084d75a2a9c448a934d4f2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/22 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CNN Spectogram test_acc: 0.493\n"
     ]
    }
   ],
   "source": [
    "assert labels.eq(torch.tensor(spect_dataloader.dataset.labels)).sum().item() == len(labels)\n",
    "\n",
    "cnn_spect_preds = get_predictions_image(spect_dataloader, cnn_spect, device)\n",
    "\n",
    "cnn_spect_accuracy = cnn_spect_preds.argmax(dim=1).eq(labels).sum() / len(labels)\n",
    "\n",
    "print(f'CNN Spectogram test_acc: {cnn_spect_accuracy:.3f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "7e0bc1da",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Averaged predictions accuracy: 0.7139\n"
     ]
    }
   ],
   "source": [
    "avg_preds = torch.stack([trf_preds, ft_preds, cnn_spect_preds]).mean(dim=0).argmax(dim=1)\n",
    "\n",
    "print(f'Averaged predictions accuracy: {avg_preds.eq(labels).sum() / len(labels):.4f}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "98b4492e",
   "metadata": {},
   "source": [
    "## Although CNN Spectogram performance is worse, by averaging its predictions with\n",
    "## the text models the overall accuracy slightly increases. Lets try majority voting."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "2b48700d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([3, 685, 2])"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.stack([trf_preds, ft_preds, cnn_spect_preds]).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "1b5e3055",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Majority voting accuracy: 0.6803\n"
     ]
    }
   ],
   "source": [
    "majority_preds, _ = torch.stack([trf_preds, ft_preds, cnn_spect_preds]).argmax(dim=2).mode(dim=0)\n",
    "\n",
    "print(f'Majority voting accuracy: {majority_preds.eq(labels).sum() / len(labels):.4f}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "777ee29b",
   "metadata": {},
   "source": [
    "## In this case the accuracy decreases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "1268ce97",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2d591f037dcd475f8bea11ee4cd6c22f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/31146 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cnn_faces, faces_dataloader = get_cnn_faces(path_saved_models, batch_size, df_faces)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "20df7df9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1780f875dcd94249a54eb53a74b0eac0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/643 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cnn_faces_preds = get_predictions_image(faces_dataloader, cnn_faces, device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "bfdefa50",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CNN Faces test_acc: 0.505\n"
     ]
    }
   ],
   "source": [
    "df_res = df_faces.loc[(df_faces['Type'] == 'test'), ['Filename', 'Face']].reset_index(drop=True).copy()\n",
    "df_res['preds'] = cnn_faces_preds.argmax(dim=1).numpy().tolist()\n",
    "\n",
    "cnn_faces_preds_mean = torch.round(torch.tensor(df_res.groupby('Filename')['preds'].mean().tolist()))\n",
    "\n",
    "print(f'CNN Faces test_acc: {cnn_faces_preds_mean.int().eq(labels).sum() / len(labels):.3f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "d37d5202",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[1, 0, 1,  ..., 1, 0, 0]],\n",
       "\n",
       "        [[0, 0, 0,  ..., 0, 1, 0]],\n",
       "\n",
       "        [[1, 1, 1,  ..., 1, 1, 1]]])"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.stack([\n",
    "    trf_preds.argmax(dim=1).unsqueeze(0), \n",
    "    ft_preds.argmax(dim=1).unsqueeze(0), \n",
    "    cnn_spect_preds.argmax(dim=1).unsqueeze(0)\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "2ae77312",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Averaged predictions accuracy: 0.6467\n"
     ]
    }
   ],
   "source": [
    "all_averaged = torch.stack([\n",
    "    trf_preds.argmax(dim=1), \n",
    "    ft_preds.argmax(dim=1), \n",
    "    cnn_spect_preds.argmax(dim=1),\n",
    "    cnn_faces_preds_mean\n",
    "]).mean(axis=0)\n",
    "\n",
    "print(f'Averaged predictions accuracy: {all_averaged.int().eq(labels).sum() / len(labels):.4f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "9295679d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Majority voting accuracy: 0.6788\n"
     ]
    }
   ],
   "source": [
    "all_mode, _= torch.stack([\n",
    "    trf_preds.argmax(dim=1), \n",
    "    ft_preds.argmax(dim=1), \n",
    "    cnn_spect_preds.argmax(dim=1),\n",
    "    cnn_faces_preds_mean\n",
    "]).mode(axis=0)\n",
    "\n",
    "print(f'Majority voting accuracy: {all_mode.int().eq(labels).sum() / len(labels):.4f}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "02b97de9",
   "metadata": {},
   "source": [
    "## The vision based methods could not increase the overall performance. NLP based models dominated the performance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1f1776a5",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}