FNN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_LAUNCH_BLOCKING'] = \"1\""
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import the libraries\n",
    "import math\n",
    "import copy\n",
    "import itertools\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torch.optim.lr_scheduler import StepLR, ReduceLROnPlateau"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading a Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import the dataset\n",
    "train_calc = pd.read_csv('calc_case_description_train_set.csv')\n",
    "test_calc = pd.read_csv('calc_case_description_test_set.csv')\n",
    "train_mass = pd.read_csv('mass_case_description_train_set.csv')\n",
    "test_mass = pd.read_csv('mass_case_description_test_set.csv')\n",
    "\n",
    "calc = train_calc.values.tolist() + test_calc.values.tolist()\n",
    "calc = pd.DataFrame(calc, columns = train_calc.columns)\n",
    "mass = train_mass.values.tolist() + test_mass.values.tolist()\n",
    "mass = pd.DataFrame(mass, columns = train_mass.columns)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Processing Settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# embedding sizes\n",
    "calc_type_embedding_size = 0\n",
    "calc_dist_embedding_size = 0\n",
    "mass_shape_embedding_size = 0\n",
    "mass_margins_embedding_size = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to create dictionaries for categorical features\n",
    "def create_embedding_dict(series):\n",
    "    unique_values = series.unique()\n",
    "    embedding_dict = {value: i for i, value in enumerate(unique_values)}\n",
    "    return embedding_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process(data):\n",
    "        # make a copy of the data to avoid SettingWithCopyWarning\n",
    "        data = data.copy()\n",
    "        \n",
    "        # set the limitations on the numerical columns\n",
    "        try:\n",
    "                data['breast density'] = data['breast density'].clip(1, 4)\n",
    "        except KeyError:\n",
    "                data['breast_density'] = data['breast_density'].clip(1, 4)\n",
    "        data['abnormality id'] = data['abnormality id'].clip(0)\n",
    "        data['assessment'] = data['assessment'].clip(0, 5)\n",
    "        data['subtlety'] = data['subtlety'].clip(1, 5)\n",
    "        \n",
    "        # change the name of index\n",
    "        data.index = data['patient_id'] + '_' + data['image view'] + '_' \\\n",
    "        + data['left or right breast'] + '_' + data['abnormality id'].astype(str)\n",
    "\n",
    "        # Remove useless columns\n",
    "        data = data[data.columns.drop(list(data.filter(regex='file path')) \n",
    "                + ['image view', 'patient_id', 'left or right breast', 'abnormality type'])]\n",
    "\n",
    "        # Fill NaN values with appropriate placeholders\n",
    "        try:\n",
    "                data['calc type'] = data['calc type'].fillna('None')\n",
    "                data['calc distribution'] = data['calc distribution'].fillna('None')\n",
    "        except KeyError:\n",
    "                data['mass shape'] = data['mass shape'].fillna('None')\n",
    "                data['mass margins'] = data['mass margins'].fillna('None')\n",
    "        # '''\n",
    "        # pathology :\n",
    "        # BENIGN_WITHOUT_CALLBACK = 0\n",
    "        # BENIGN = 1\n",
    "        # MALIGNANT = 2\n",
    "        # '''\n",
    "        data['pathology'] = data['pathology'].map({'BENIGN_WITHOUT_CALLBACK': 0, 'BENIGN': 1, 'MALIGNANT': 2})\n",
    "        \n",
    "        # Create embedding dictionaries for categorical features\n",
    "        # and define embedding sizes\n",
    "        \n",
    "        global calc_type_embedding_size\n",
    "        global calc_dist_embedding_size\n",
    "        global mass_shape_embedding_size\n",
    "        global mass_margins_embedding_size\n",
    "        \n",
    "        try:\n",
    "                calc_type_embedding_dict = create_embedding_dict(data['calc type'])\n",
    "                calc_dist_embedding_dict = create_embedding_dict(data['calc distribution'])\n",
    "                calc_type_embedding_size = len(calc_type_embedding_dict)\n",
    "                calc_dist_embedding_size = len(calc_dist_embedding_dict)\n",
    "        except KeyError:\n",
    "                mass_shape_embedding_dict = create_embedding_dict(data['mass shape'])\n",
    "                mass_margins_embedding_dict = create_embedding_dict(data['mass margins'])\n",
    "                mass_shape_embedding_size = len(mass_shape_embedding_dict)\n",
    "                mass_margins_embedding_size = len(mass_margins_embedding_dict)\n",
    "        \n",
    "        # Replace categorical values with their embedding indices        \n",
    "        try:\n",
    "                data['calc type'] = data['calc type'].map(calc_type_embedding_dict)\n",
    "                data['calc distribution'] = data['calc distribution'].map(calc_dist_embedding_dict)\n",
    "        except KeyError:\n",
    "                data['mass shape'] = data['mass shape'].map(mass_shape_embedding_dict)\n",
    "                data['mass margins'] = data['mass margins'].map(mass_margins_embedding_dict)\n",
    "        \n",
    "        # rename columns\n",
    "        data.rename(columns={'abnormality id': 'number of abnormalities', \n",
    "                             'assessment' : 'overall BI-RADS assessment'}, inplace=True)\n",
    "        try:\n",
    "                data.rename(columns={'breast_density' : 'breast density'}, inplace=True)\n",
    "                # split\n",
    "        except KeyError:\n",
    "                return data\n",
    "        \n",
    "        return data"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dataprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reload the csv files and dataset\n",
    "train_calc = pd.read_csv('calc_case_description_train_set.csv')\n",
    "test_calc = pd.read_csv('calc_case_description_test_set.csv')\n",
    "train_mass = pd.read_csv('mass_case_description_train_set.csv')\n",
    "test_mass = pd.read_csv('mass_case_description_test_set.csv')\n",
    "\n",
    "calc = train_calc.values.tolist() + test_calc.values.tolist()\n",
    "calc = pd.DataFrame(calc, columns = train_calc.columns)\n",
    "mass = train_mass.values.tolist() + test_mass.values.tolist()\n",
    "mass = pd.DataFrame(mass, columns = train_mass.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Execute the data process function\n",
    "calc = process(calc)\n",
    "mass = process(mass)\n",
    "train_calc, test_calc = calc[:train_calc.shape[0]], calc[train_calc.shape[0]:]\n",
    "train_mass, test_mass = mass[:train_mass.shape[0]], mass[train_mass.shape[0]:]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Creating a Custom Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# label maps\n",
    "labels_map_assessment = {\n",
    "    0: \"0\",\n",
    "    1: \"1\",\n",
    "    2: \"2\",\n",
    "    3: \"3\",\n",
    "    4: \"4\",\n",
    "    5: \"5\"\n",
    "}\n",
    "\n",
    "labels_map_pathology = {\n",
    "    0: \"BENIGN_WITHOUT_CALLBACK\",\n",
    "    1: \"BENIGN\",\n",
    "    2: \"MALIGNANT\"\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CreateDataset(Dataset):\n",
    "    def __init__(self, X, y):\n",
    "        self.X = torch.tensor(X, dtype=torch.float32)\n",
    "        self.y = torch.tensor(y, dtype=torch.float32)\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.X)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        return self.X[idx], self.y[idx]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preparing data for training with DataLoaders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_dataloaders(train, test, type, batch_size):\n",
    "    if type == \"a\":\n",
    "        # calc, assessement prediction\n",
    "        X_train = train.drop(['overall BI-RADS assessment', 'pathology'], axis=1).values\n",
    "        y_train = train['overall BI-RADS assessment'].values\n",
    "        X_test = test.drop(['overall BI-RADS assessment', 'pathology'], axis=1).values\n",
    "        y_test = test['overall BI-RADS assessment'].values\n",
    "    elif type == \"p\":\n",
    "        # calc, pathology prediction\n",
    "        X_train = train.drop('pathology', axis=1).values\n",
    "        y_train = train['pathology'].values\n",
    "        X_test = test.drop('pathology', axis=1).values\n",
    "        y_test = test['pathology'].values\n",
    "        \n",
    "    # Assuming you have already preprocessed the data and split it into training and testing sets    \n",
    "    train_dataset = CreateDataset(X_train, y_train)\n",
    "    test_dataset = CreateDataset(X_test, y_test)\n",
    "\n",
    "    train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n",
    "    test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n",
    "    \n",
    "    return train_dataset, test_dataset, train_loader, test_loader"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Using cuda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using cuda device\n"
     ]
    }
   ],
   "source": [
    "device = (\n",
    "    \"cuda\"\n",
    "    if torch.cuda.is_available()\n",
    "    else \"mps\"\n",
    "    if torch.backends.mps.is_available()\n",
    "    else \"cpu\"\n",
    ")\n",
    "print(f\"Using {device} device\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deine the FNN Class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FNN(nn.Module):\n",
    "    def __init__(self, input_size, hidden_size, num_classes, hidden_layers=1, activation=\"relu\", weight_init=\"lecun\"):\n",
    "        super().__init__()\n",
    "        self.linear_non_linear_stack = nn.Sequential().to(device)\n",
    "        \n",
    "        modules = []\n",
    "        modules.append(nn.Linear(input_size, hidden_size))\n",
    "        \n",
    "        # weight initialization\n",
    "        if weight_init == \"lecun\":\n",
    "            pass\n",
    "        elif weight_init == \"zero\":\n",
    "            nn.init.zeros_(modules[0].weight)\n",
    "        elif weight_init == \"normal\":\n",
    "            nn.init.normal_(modules[0].weight, mean=0, std=0.01)\n",
    "        elif weight_init == \"xavier\" or weight_init == \"glorot\":\n",
    "            nn.init.xavier_normal_(modules[0].weight)\n",
    "        elif weight_init == \"kaiming\" or weight_init == \"he\":\n",
    "            nn.init.kaiming_normal_(modules[0].weight)\n",
    "        \n",
    "        # activation function\n",
    "        if activation == \"relu\":\n",
    "            modules.append(nn.ReLU())\n",
    "        elif activation == \"sigmoid\":\n",
    "            modules.append(nn.Sigmoid())\n",
    "        elif activation == \"tanh\":\n",
    "            modules.append(nn.Tanh())\n",
    "        elif activation == \"leaky_relu\":\n",
    "            modules.append(nn.LeakyReLU())\n",
    "        \n",
    "        for i in range(hidden_layers):\n",
    "            ln = nn.Linear(hidden_size, hidden_size)\n",
    "            \n",
    "            if weight_init == \"lecun\":\n",
    "                pass\n",
    "            elif weight_init == \"zero\":\n",
    "                nn.init.zeros_(ln.weight)\n",
    "            elif weight_init == \"normal\":\n",
    "                nn.init.normal_(ln.weight, mean=0, std=0.01)\n",
    "            elif weight_init == \"xavier\" or weight_init == \"glorot\":\n",
    "                nn.init.xavier_normal_(ln.weight)\n",
    "            elif weight_init == \"kaiming\" or weight_init == \"he\":\n",
    "                nn.init.kaiming_normal_(ln.weight)\n",
    "            \n",
    "            modules.append(ln)\n",
    "            if activation == \"relu\":\n",
    "                modules.append(nn.ReLU())\n",
    "            elif activation == \"sigmoid\":\n",
    "                modules.append(nn.Sigmoid())\n",
    "            elif activation == \"tanh\":\n",
    "                modules.append(nn.Tanh())\n",
    "            elif activation == \"leaky_relu\":\n",
    "                modules.append(nn.LeakyReLU())\n",
    "                \n",
    "        modules.append(nn.Linear(hidden_size, num_classes))\n",
    "        \n",
    "        self.linear_non_linear_stack = nn.Sequential(*modules).to(device)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = x.to(device)\n",
    "        logits = self.linear_non_linear_stack(x)\n",
    "        return logits"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train and test the model"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Function for Train and Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train and test the model \n",
    "def train_loop(dataloader, model, loss_fn, optimizer, p=\"False\", scheduler=None):\n",
    "    size = len(dataloader.dataset)\n",
    "    for batch, (X, y) in enumerate(dataloader):\n",
    "        X, y = X.to(device), y.to(device).long()  # Move input and target tensors to the same device as the model\n",
    "        # Compute prediction and loss\n",
    "        pred = model(X).to(device)\n",
    "        # print(pred.shape)\n",
    "        # print(y.shape)\n",
    "        loss = loss_fn(pred, y)\n",
    "\n",
    "        # Backpropagation\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        if batch % 10 == 0 or batch == len(dataloader) - 1:\n",
    "            loss, current = loss.item(), (batch + 1) * len(X)\n",
    "            if batch == len(dataloader) - 1:\n",
    "                current = size\n",
    "            if p == True:\n",
    "                print(f\"loss: {loss:>7f}  [{current:>5d}/{size:>5d}]\")\n",
    "\n",
    "\n",
    "def test_loop(dataloader, model, loss_fn, p=\"False\"):\n",
    "    size = len(dataloader.dataset)\n",
    "    num_batches = len(dataloader)\n",
    "    test_loss, correct = 0, 0\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for X, y in dataloader:\n",
    "            X, y = X.to(device), y.to(device).long()  # Move input and target tensors to the same device as the model\n",
    "            pred = model(X)\n",
    "            test_loss += loss_fn(pred, y).item()\n",
    "            correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n",
    "\n",
    "    test_loss /= num_batches\n",
    "    correct /= size\n",
    "    if p == True:\n",
    "        print(f\"Test Error: \\n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \\n\")\n",
    "    return round(100*correct, 1), test_loss"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Class for saving each models' information effectively"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model:\n",
    "    def __init__(self):\n",
    "        pass"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initial Settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 64\n",
    "hidden_size = 512\n",
    "learning_rate = 1e-3\n",
    "epochs = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "def l1_regularizer(model, l1_lambda):\n",
    "    l1_loss = 0\n",
    "    for param in model.parameters():\n",
    "        l1_loss += torch.sum(torch.abs(param))\n",
    "    return l1_lambda * l1_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_L1(dataloader, model, loss_fn, optimizer, l1_lambda, p=\"False\", scheduler=None):\n",
    "    size = len(dataloader.dataset)\n",
    "    for batch, (X, y) in enumerate(dataloader):\n",
    "        X, y = X.to(device), y.to(device).long()  # Move input and target tensors to the same device as the model\n",
    "        # Compute prediction and loss\n",
    "        pred = model(X).to(device)\n",
    "        \n",
    "        loss = loss_fn(pred, y)\n",
    "\n",
    "        # Compute L1 regularization penalty\n",
    "        l1_penalty = l1_regularizer(model, l1_lambda)\n",
    "\n",
    "        # Combine original loss and L1 penalty\n",
    "        loss = loss + l1_penalty\n",
    "        \n",
    "        # Backpropagation\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        if batch % 10 == 0 or batch == len(dataloader) - 1:\n",
    "            loss, current = loss.item(), (batch + 1) * len(X)\n",
    "            if batch == len(dataloader) - 1:\n",
    "                current = size\n",
    "            if p == True:\n",
    "                print(f\"loss: {loss:>7f}  [{current:>5d}/{size:>5d}]\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## features -> assessment (calc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default settings\n",
    "fac = Model()\n",
    "\n",
    "fac.epochs = epochs\n",
    "fac.batch_size = batch_size\n",
    "fac.train_dataset, fac.test_dataset, fac.train_dataloader, fac.test_dataloader = get_dataloaders(train_calc, test_calc, \"a\", batch_size)\n",
    "\n",
    "fac.input_size = fac.train_dataloader.dataset.X.shape[1]\n",
    "fac.hidden_size = hidden_size\n",
    "fac.num_classes = 6\n",
    "\n",
    "fac.lr = 1e-3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 281,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "73.0"
      ]
     },
     "execution_count": 281,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# default model\n",
    "fac.model = FNN(fac.input_size, fac.hidden_size, fac.num_classes).to(device)\n",
    "fac.loss_fn = nn.CrossEntropyLoss()\n",
    "fac.optimizer = torch.optim.SGD(fac.model.parameters(), lr=fac.lr)\n",
    "\n",
    "for t in range(fac.epochs):\n",
    "    train_loop(fac.train_dataloader, fac.model, fac.loss_fn, fac.optimizer)\n",
    "    accuracy = test_loop(fac.test_dataloader, fac.model, fac.loss_fn)[0]\n",
    "fac.accuracy = accuracy\n",
    "fac.accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 283,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Mini-batch gradient descent\n",
    "for i in range(20):\n",
    "    model = FNN(fac.input_size, fac.hidden_size, fac.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.SGD(model.parameters(), lr=fac.lr)\n",
    "\n",
    "    for t in range(fac.epochs):\n",
    "        train_loop(fac.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fac.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fac.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fac.accuracy = accuracy\n",
    "        fac.model = model\n",
    "        fac.loss_fn = loss_fn\n",
    "        fac.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 282,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "New best model found with accuracy:  73.6\n",
      "New best model found with accuracy:  74.5\n"
     ]
    }
   ],
   "source": [
    "# SGD momentum\n",
    "for i in range(20):\n",
    "    model = FNN(fac.input_size, fac.hidden_size, fac.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.SGD(model.parameters(), lr=fac.lr, momentum=0.9)\n",
    "\n",
    "    for t in range(fac.epochs):\n",
    "        train_loop(fac.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fac.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fac.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fac.accuracy = accuracy\n",
    "        fac.model = model\n",
    "        fac.loss_fn = loss_fn\n",
    "        fac.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 285,
   "metadata": {},
   "outputs": [],
   "source": [
    "# SGD Nestrov\n",
    "for i in range(20):\n",
    "    model = FNN(fac.input_size, fac.hidden_size, fac.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.SGD(model.parameters(), lr=fac.lr, momentum=0.9, nesterov=True)\n",
    "\n",
    "    for t in range(fac.epochs):\n",
    "        train_loop(fac.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fac.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fac.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fac.accuracy = accuracy\n",
    "        fac.model = model\n",
    "        fac.loss_fn = loss_fn\n",
    "        fac.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 292,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "New best model found with accuracy:  76.7\n"
     ]
    }
   ],
   "source": [
    "# Adam\n",
    "for i in range(20):\n",
    "    model = FNN(fac.input_size, fac.hidden_size, fac.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.Adam(model.parameters())\n",
    "\n",
    "    for t in range(fac.epochs):\n",
    "        train_loop(fac.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fac.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fac.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fac.accuracy = accuracy\n",
    "        fac.model = model\n",
    "        fac.loss_fn = loss_fn\n",
    "        fac.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 297,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adagrad\n",
    "for i in range(20):\n",
    "    model = FNN(fac.input_size, fac.hidden_size, fac.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.Adagrad(model.parameters())\n",
    "\n",
    "    for t in range(fac.epochs):\n",
    "        train_loop(fac.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fac.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fac.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fac.accuracy = accuracy\n",
    "        fac.model = model\n",
    "        fac.loss_fn = loss_fn\n",
    "        fac.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 298,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adadelta\n",
    "for i in range(20):\n",
    "    model = FNN(fac.input_size, fac.hidden_size, fac.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.Adadelta(model.parameters())\n",
    "\n",
    "    for t in range(fac.epochs):\n",
    "        train_loop(fac.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fac.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fac.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fac.accuracy = accuracy\n",
    "        fac.model = model\n",
    "        fac.loss_fn = loss_fn\n",
    "        fac.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 299,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adamax\n",
    "for i in range(20):\n",
    "    model = FNN(fac.input_size, fac.hidden_size, fac.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.Adamax(model.parameters())\n",
    "\n",
    "    for t in range(fac.epochs):\n",
    "        train_loop(fac.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fac.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fac.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fac.accuracy = accuracy\n",
    "        fac.model = model\n",
    "        fac.loss_fn = loss_fn\n",
    "        fac.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 300,
   "metadata": {},
   "outputs": [],
   "source": [
    "# RMSprop\n",
    "for i in range(20):\n",
    "    model = FNN(fac.input_size, fac.hidden_size, fac.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters())\n",
    "\n",
    "    for t in range(fac.epochs):\n",
    "        train_loop(fac.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fac.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fac.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fac.accuracy = accuracy\n",
    "        fac.model = model\n",
    "        fac.loss_fn = loss_fn\n",
    "        fac.optimizer = optimizer"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Adam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 310,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Number of Neurons: int((input_size + num_classes)/2)\n",
    "n = int((fac.input_size + fac.num_classes)/2)\n",
    "for i in range(40):\n",
    "    model = FNN(fac.input_size, n, fac.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.Adam(model.parameters())\n",
    "\n",
    "    for t in range(fac.epochs):\n",
    "        train_loop(fac.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fac.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fac.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fac.accuracy = accuracy\n",
    "        fac.hidden_size = n\n",
    "        fac.model = model\n",
    "        fac.loss_fn = loss_fn\n",
    "        fac.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 311,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Number of Neurons: int(2/3*input_size + num_classes)\n",
    "n = int(2/3*fac.input_size + fac.num_classes)\n",
    "for i in range(40):\n",
    "    model = FNN(fac.input_size, n, fac.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.Adam(model.parameters())\n",
    "\n",
    "    for t in range(fac.epochs):\n",
    "        train_loop(fac.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fac.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fac.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fac.accuracy = accuracy\n",
    "        fac.hidden_size = n\n",
    "        fac.model = model\n",
    "        fac.loss_fn = loss_fn\n",
    "        fac.optimizer = optimizer"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "512"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 314,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Number of Layers: 2\n",
    "for i in range(40):\n",
    "    model = FNN(fac.input_size, fac.hidden_size, fac.num_classes, hidden_layers=2).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.Adam(model.parameters())\n",
    "\n",
    "    for t in range(fac.epochs):\n",
    "        train_loop(fac.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fac.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fac.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fac.accuracy = accuracy\n",
    "        fac.hidden_layers = 2\n",
    "        fac.model = model\n",
    "        fac.loss_fn = loss_fn\n",
    "        fac.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 315,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Number of Layers: 3\n",
    "for i in range(40):\n",
    "    model = FNN(fac.input_size, fac.hidden_size, fac.num_classes, hidden_layers=3).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.Adam(model.parameters())\n",
    "\n",
    "    for t in range(fac.epochs):\n",
    "        train_loop(fac.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fac.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fac.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fac.accuracy = accuracy\n",
    "        fac.hidden_layers = 3\n",
    "        fac.model = model\n",
    "        fac.loss_fn = loss_fn\n",
    "        fac.optimizer = optimizer"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 316,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Activation Function: leaky_relu (default is ReLU)\n",
    "for i in range(40):\n",
    "    model = FNN(fac.input_size, fac.hidden_size, fac.num_classes, activation=\"leaky_relu\").to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.Adam(model.parameters())\n",
    "\n",
    "    for t in range(fac.epochs):\n",
    "        train_loop(fac.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fac.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fac.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fac.accuracy = accuracy\n",
    "        fac.model = model\n",
    "        fac.loss_fn = loss_fn\n",
    "        fac.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 317,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Activation Function: tanh (default is ReLU)\n",
    "for i in range(40):\n",
    "    model = FNN(fac.input_size, fac.hidden_size, fac.num_classes, activation=\"tanh\").to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.Adam(model.parameters())\n",
    "\n",
    "    for t in range(fac.epochs):\n",
    "        train_loop(fac.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fac.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fac.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fac.accuracy = accuracy\n",
    "        fac.model = model\n",
    "        fac.loss_fn = loss_fn\n",
    "        fac.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 318,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Activation Function: sigmoid (default is ReLU)\n",
    "for i in range(40):\n",
    "    model = FNN(fac.input_size, fac.hidden_size, fac.num_classes, activation=\"sigmoid\").to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.Adam(model.parameters())\n",
    "\n",
    "    for t in range(fac.epochs):\n",
    "        train_loop(fac.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fac.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fac.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fac.accuracy = accuracy\n",
    "        fac.model = model\n",
    "        fac.loss_fn = loss_fn\n",
    "        fac.optimizer = optimizer"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "ReLu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 319,
   "metadata": {},
   "outputs": [],
   "source": [
    "# weight_init : zero (default is lecun)\n",
    "for i in range(40):\n",
    "    model = FNN(fac.input_size, fac.hidden_size, fac.num_classes, weight_init=\"zero\").to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.Adam(model.parameters())\n",
    "\n",
    "    for t in range(fac.epochs):\n",
    "        train_loop(fac.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fac.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fac.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fac.accuracy = accuracy\n",
    "        fac.model = model\n",
    "        fac.loss_fn = loss_fn\n",
    "        fac.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 320,
   "metadata": {},
   "outputs": [],
   "source": [
    "# weight_init : normal (default is lecun)\n",
    "for i in range(40):\n",
    "    model = FNN(fac.input_size, fac.hidden_size, fac.num_classes, weight_init=\"normal\").to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.Adam(model.parameters())\n",
    "\n",
    "    for t in range(fac.epochs):\n",
    "        train_loop(fac.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fac.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fac.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fac.accuracy = accuracy\n",
    "        fac.model = model\n",
    "        fac.loss_fn = loss_fn\n",
    "        fac.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 321,
   "metadata": {},
   "outputs": [],
   "source": [
    "# weight_init : xavier (default is lecun)\n",
    "for i in range(40):\n",
    "    model = FNN(fac.input_size, fac.hidden_size, fac.num_classes, weight_init=\"xavier\").to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.Adam(model.parameters())\n",
    "\n",
    "    for t in range(fac.epochs):\n",
    "        train_loop(fac.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fac.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fac.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fac.accuracy = accuracy\n",
    "        fac.model = model\n",
    "        fac.loss_fn = loss_fn\n",
    "        fac.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 322,
   "metadata": {},
   "outputs": [],
   "source": [
    "# weight_init : kaiming (default is lecun)\n",
    "for i in range(40):\n",
    "    model = FNN(fac.input_size, fac.hidden_size, fac.num_classes, weight_init=\"kaiming\").to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.Adam(model.parameters())\n",
    "\n",
    "    for t in range(fac.epochs):\n",
    "        train_loop(fac.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fac.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fac.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fac.accuracy = accuracy\n",
    "        fac.model = model\n",
    "        fac.loss_fn = loss_fn\n",
    "        fac.optimizer = optimizer"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lecun Normal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 324,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "New best model found with accuracy:  77.3\n"
     ]
    }
   ],
   "source": [
    "# Batch size: 32 (default is 64)\n",
    "train_dataset, test_dataset, train_dataloader, test_dataloader = get_dataloaders(train_calc, test_calc, \"a\", 32)\n",
    " \n",
    "for i in range(40):\n",
    "    model = FNN(fac.input_size, fac.hidden_size, fac.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.Adam(model.parameters())\n",
    "\n",
    "    for t in range(fac.epochs):\n",
    "        train_loop(fac.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fac.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fac.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fac.accuracy = accuracy\n",
    "        fac.batch_size = 32\n",
    "        fac.model = model\n",
    "        fac.loss_fn = loss_fn\n",
    "        fac.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 325,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Batch size: 128 (default is 64)\n",
    "train_dataset, test_dataset, train_dataloader, test_dataloader = get_dataloaders(train_calc, test_calc, \"a\", 128)\n",
    " \n",
    "for i in range(40):\n",
    "    model = FNN(fac.input_size, fac.hidden_size, fac.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.Adam(model.parameters())\n",
    "\n",
    "    for t in range(fac.epochs):\n",
    "        train_loop(fac.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fac.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fac.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fac.accuracy = accuracy\n",
    "        fac.batch_size = 16\n",
    "        fac.model = model\n",
    "        fac.loss_fn = loss_fn\n",
    "        fac.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 333,
   "metadata": {},
   "outputs": [],
   "source": [
    "fac.train_dataset, fac.test_dataset, fac.train_dataloader, fac.test_dataloader = get_dataloaders(train_calc, test_calc, \"a\", 32)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 338,
   "metadata": {},
   "outputs": [],
   "source": [
    "# number of epochs (default): 10 -> increase until 3-\n",
    "e = fac.epochs\n",
    "for i in range(3):\n",
    "    for j in range(10):\n",
    "        model = FNN(fac.input_size, fac.hidden_size, fac.num_classes).to(device)\n",
    "        loss_fn = nn.CrossEntropyLoss()\n",
    "        optimizer = torch.optim.Adam(model.parameters())\n",
    "\n",
    "        for t in range(e):\n",
    "            train_loop(fac.train_dataloader, model, loss_fn, optimizer)\n",
    "            accuracy = test_loop(fac.test_dataloader, model, loss_fn)[0]\n",
    "        \n",
    "        if accuracy > fac.accuracy:\n",
    "            print(\"New best epochs: \", e)\n",
    "            print(\"New best model found with accuracy: \", accuracy)\n",
    "            fac.accuracy = accuracy\n",
    "            fac.epochs = e\n",
    "            fac.model = model\n",
    "            fac.loss_fn = loss_fn\n",
    "            fac.optimizer = optimizer\n",
    "    e += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 343,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\sonog\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\torch\\optim\\lr_scheduler.py:138: UserWarning: Detected call of `lr_scheduler.step()` before `optimizer.step()`. In PyTorch 1.1.0 and later, you should call them in the opposite order: `optimizer.step()` before `lr_scheduler.step()`.  Failure to do this will result in PyTorch skipping the first value of the learning rate schedule. See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\n",
      "  warnings.warn(\"Detected call of `lr_scheduler.step()` before `optimizer.step()`. \"\n"
     ]
    }
   ],
   "source": [
    "# learning rate scheduler: StepLR\n",
    "# learning rate: 0.001 (default) same as Adam's default\n",
    "\n",
    "for i in range(40):\n",
    "    model = FNN(fac.input_size, fac.hidden_size, fac.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=fac.lr)\n",
    "    scheduler = StepLR(optimizer, step_size=1, gamma=0.1)\n",
    "    \n",
    "    for t in range(fac.epochs):\n",
    "        scheduler.step()\n",
    "        train_loop(fac.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fac.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fac.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fac.accuracy = accuracy\n",
    "        fac.model = model\n",
    "        fac.loss_fn = loss_fn\n",
    "        fac.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 344,
   "metadata": {},
   "outputs": [],
   "source": [
    "# learning rate scheduler: ReduceLROnPlateau\n",
    "# learning rate: 0.001 (default) same as Adam's default\n",
    "\n",
    "for i in range(40):\n",
    "    model = FNN(fac.input_size, fac.hidden_size, fac.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=fac.lr)\n",
    "    scheduler = ReduceLROnPlateau(optimizer, mode='max', factor=0.1, patience=0)\n",
    "    \n",
    "    for t in range(fac.epochs):\n",
    "        train_loop(fac.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fac.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fac.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fac.accuracy = accuracy\n",
    "        fac.model = model\n",
    "        fac.loss_fn = loss_fn\n",
    "        fac.optimizer = optimizer\n",
    "    scheduler.step(accuracy)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "No scheduler required"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 348,
   "metadata": {},
   "outputs": [],
   "source": [
    "# L2 Regularization\n",
    "# Define the hyperparameter search space\n",
    "epochs_candidates = [10, 20, 30]\n",
    "l2_lambda_candidates = [0.001, 0.01, 0.1]\n",
    "\n",
    "for num_epochs, l2_lambda in itertools.product(epochs_candidates, l2_lambda_candidates):\n",
    "    model = FNN(fac.input_size, fac.hidden_size, fac.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = optim.Adam(model.parameters(), lr=learning_rate, weight_decay=l2_lambda)\n",
    "\n",
    "    for t in range(num_epochs):\n",
    "        train_loop(fac.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fac.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fac.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        print(\"New best hyperparameters: \", num_epochs, l2_lambda)\n",
    "        \n",
    "        fac.accuracy = accuracy\n",
    "        \n",
    "        fac.model = model\n",
    "        fac.loss_fn = loss_fn\n",
    "        fac.optimizer = optimizer\n",
    "        \n",
    "        fac.epochs = num_epochs\n",
    "        fac.l2_lambda = l2_lambda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 487,
   "metadata": {},
   "outputs": [],
   "source": [
    "# L1 Regularization\n",
    "epochs_candidates = [10, 20, 30]\n",
    "l1_lambda_candidates = [0.001, 0.01, 0.1]\n",
    "\n",
    "for num_epochs, l1_lambda in itertools.product(epochs_candidates, l1_lambda_candidates):\n",
    "    model = FNN(fac.input_size, fac.hidden_size, fac.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = optim.Adam(model.parameters(), lr=learning_rate, weight_decay=l1_lambda)\n",
    "\n",
    "    for t in range(num_epochs):\n",
    "        train_L1(fac.train_dataloader, model, loss_fn, optimizer, l1_lambda)\n",
    "        accuracy = test_loop(fac.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fac.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        print(\"New best hyperparameters: \", num_epochs, l1_lambda)\n",
    "        \n",
    "        fac.accuracy = accuracy\n",
    "        \n",
    "        fac.model = model\n",
    "        fac.loss_fn = loss_fn\n",
    "        fac.optimizer = optimizer\n",
    "        \n",
    "        fac.epochs = num_epochs\n",
    "        fac.l1_lambda = l1_lambda"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "No regularization"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Try with lower batch size then because the change in batch size improved the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 429,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Batch size: 16 (default is 64)\n",
    "train_dataset, test_dataset, train_dataloader, test_dataloader = get_dataloaders(train_calc, test_calc, \"a\", 16)\n",
    " \n",
    "for i in range(60):\n",
    "    model = FNN(fac.input_size, fac.hidden_size, fac.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.Adam(model.parameters())\n",
    "\n",
    "    for t in range(fac.epochs):\n",
    "        train_loop(fac.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fac.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fac.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fac.accuracy = accuracy\n",
    "        fac.train_dataset, fac.test_dataset, fac.train_dataloader, fac.test_dataloader = train_dataset, test_dataset, train_dataloader, test_dataloader\n",
    "        fac.batch_size = 16\n",
    "        fac.model = model\n",
    "        fac.loss_fn = loss_fn\n",
    "        fac.optimizer = optimizer"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## features -> assessment (mass)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 354,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default settings\n",
    "fam = Model()\n",
    "\n",
    "fam.epochs = epochs\n",
    "fam.batch_size = batch_size\n",
    "fam.train_dataset, fam.test_dataset, fam.train_dataloader, fam.test_dataloader = get_dataloaders(train_mass, test_mass, \"a\", batch_size)\n",
    "\n",
    "fam.input_size = fam.train_dataloader.dataset.X.shape[1]\n",
    "fam.hidden_size = hidden_size\n",
    "fam.num_classes = 6\n",
    "\n",
    "fam.lr = 1e-3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 381,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "52.6"
      ]
     },
     "execution_count": 381,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# default model\n",
    "fam.model = FNN(fam.input_size, fam.hidden_size, fam.num_classes).to(device)\n",
    "fam.loss_fn = nn.CrossEntropyLoss()\n",
    "fam.optimizer = torch.optim.SGD(fam.model.parameters(), lr=fam.lr)\n",
    "\n",
    "for t in range(fac.epochs):\n",
    "    train_loop(fam.train_dataloader, fam.model, fam.loss_fn, fam.optimizer)\n",
    "    accuracy = test_loop(fam.test_dataloader, fam.model, fam.loss_fn)[0]\n",
    "fam.accuracy = accuracy\n",
    "fam.accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 383,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "New best model found with accuracy:  53.7\n",
      "New best model found with accuracy:  60.1\n"
     ]
    }
   ],
   "source": [
    "# Mini-batch gradient descent\n",
    "for i in range(30):\n",
    "    model = FNN(fam.input_size, fam.hidden_size, fam.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.SGD(model.parameters(), lr=fam.lr, momentum=0.9)\n",
    "\n",
    "    for t in range(fam.epochs):\n",
    "        train_loop(fam.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fac.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fam.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fam.accuracy = accuracy\n",
    "        fam.model = model\n",
    "        fam.loss_fn = loss_fn\n",
    "        fam.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 384,
   "metadata": {},
   "outputs": [],
   "source": [
    "# SGD momentum\n",
    "for i in range(40):\n",
    "    model = FNN(fam.input_size, fam.hidden_size, fam.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.SGD(model.parameters(), lr=fam.lr, momentum=0.9)\n",
    "\n",
    "    for t in range(fam.epochs):\n",
    "        train_loop(fam.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fam.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fam.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fam.accuracy = accuracy\n",
    "        fam.model = model\n",
    "        fam.loss_fn = loss_fn\n",
    "        fam.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 386,
   "metadata": {},
   "outputs": [],
   "source": [
    "# SGD Nestrov\n",
    "for i in range(20):\n",
    "    model = FNN(fam.input_size, fam.hidden_size, fam.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.SGD(model.parameters(), lr=fam.lr, momentum=0.9, nesterov=True)\n",
    "\n",
    "    for t in range(fam.epochs):\n",
    "        train_loop(fam.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fam.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fam.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fam.accuracy = accuracy\n",
    "        fam.model = model\n",
    "        fam.loss_fn = loss_fn\n",
    "        fam.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 387,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "New best model found with accuracy:  61.1\n",
      "New best model found with accuracy:  61.9\n",
      "New best model found with accuracy:  62.7\n",
      "New best model found with accuracy:  63.0\n"
     ]
    }
   ],
   "source": [
    "# Adam\n",
    "for i in range(40):\n",
    "    model = FNN(fam.input_size, fam.hidden_size, fam.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.Adam(model.parameters())\n",
    "\n",
    "    for t in range(fam.epochs):\n",
    "        train_loop(fam.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fam.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fam.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fam.accuracy = accuracy\n",
    "        fam.model = model\n",
    "        fam.loss_fn = loss_fn\n",
    "        fam.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 388,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adagrad\n",
    "for i in range(20):\n",
    "    model = FNN(fam.input_size, fam.hidden_size, fam.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.Adagrad(model.parameters())\n",
    "\n",
    "    for t in range(fam.epochs):\n",
    "        train_loop(fam.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fam.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fam.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fam.accuracy = accuracy\n",
    "        fam.model = model\n",
    "        fam.loss_fn = loss_fn\n",
    "        fam.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 389,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adadelta\n",
    "for i in range(20):\n",
    "    model = FNN(fam.input_size, fam.hidden_size, fam.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.Adadelta(model.parameters())\n",
    "\n",
    "    for t in range(fam.epochs):\n",
    "        train_loop(fam.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fam.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fam.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fam.accuracy = accuracy\n",
    "        fam.model = model\n",
    "        fam.loss_fn = loss_fn\n",
    "        fam.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 390,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adamax\n",
    "for i in range(20):\n",
    "    model = FNN(fam.input_size, fam.hidden_size, fam.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.Adamax(model.parameters())\n",
    "\n",
    "    for t in range(fam.epochs):\n",
    "        train_loop(fam.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fam.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fam.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fam.accuracy = accuracy\n",
    "        fam.model = model\n",
    "        fam.loss_fn = loss_fn\n",
    "        fam.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 391,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "New best model found with accuracy:  63.5\n"
     ]
    }
   ],
   "source": [
    "# RMSprop\n",
    "for i in range(20):\n",
    "    model = FNN(fam.input_size, fam.hidden_size, fam.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters())\n",
    "\n",
    "    for t in range(fam.epochs):\n",
    "        train_loop(fam.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fam.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fam.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fam.accuracy = accuracy\n",
    "        fam.model = model\n",
    "        fam.loss_fn = loss_fn\n",
    "        fam.optimizer = optimizer"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "RMSprop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 393,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\sonog\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\torch\\optim\\lr_scheduler.py:138: UserWarning: Detected call of `lr_scheduler.step()` before `optimizer.step()`. In PyTorch 1.1.0 and later, you should call them in the opposite order: `optimizer.step()` before `lr_scheduler.step()`.  Failure to do this will result in PyTorch skipping the first value of the learning rate schedule. See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\n",
      "  warnings.warn(\"Detected call of `lr_scheduler.step()` before `optimizer.step()`. \"\n"
     ]
    }
   ],
   "source": [
    "# learning rate scheduler: StepLR\n",
    "\n",
    "for i in range(40):\n",
    "    model = FNN(fam.input_size, fam.hidden_size, fam.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters(), lr=0.1)\n",
    "    scheduler = StepLR(optimizer, step_size=1, gamma=0.1)\n",
    "    \n",
    "    for t in range(fam.epochs):\n",
    "        scheduler.step()\n",
    "        train_loop(fam.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fam.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fam.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fam.accuracy = accuracy\n",
    "        fam.model = model\n",
    "        fam.loss_fn = loss_fn\n",
    "        fam.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 394,
   "metadata": {},
   "outputs": [],
   "source": [
    "# learning rate scheduler: ReduceLROnPlateau\n",
    "\n",
    "for i in range(40):\n",
    "    model = FNN(fam.input_size, fam.hidden_size, fam.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters(), lr=0.1)\n",
    "    scheduler = ReduceLROnPlateau(optimizer, mode='max', factor=0.1, patience=0)\n",
    "    \n",
    "    for t in range(fam.epochs):\n",
    "        train_loop(fam.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fam.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fam.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fam.accuracy = accuracy\n",
    "        fam.model = model\n",
    "        fam.loss_fn = loss_fn\n",
    "        fam.optimizer = optimizer\n",
    "    scheduler.step(accuracy)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "No scheduler required"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 396,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Number of Neurons: int((input_size + num_classes)/2)\n",
    "n = int((fam.input_size + fam.num_classes)/2)\n",
    "for i in range(40):\n",
    "    model = FNN(fam.input_size, n, fam.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters())\n",
    "\n",
    "    for t in range(fam.epochs):\n",
    "        train_loop(fam.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fam.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fam.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fam.accuracy = accuracy\n",
    "        fam.hidden_size = n\n",
    "        fam.model = model\n",
    "        fam.loss_fn = loss_fn\n",
    "        fam.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 398,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Number of Neurons: int(2/3*input_size + num_classes)\n",
    "n = int(2/3*fam.input_size + fam.num_classes)\n",
    "for i in range(40):\n",
    "    model = FNN(fam.input_size, n, fam.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters())\n",
    "\n",
    "    for t in range(fam.epochs):\n",
    "        train_loop(fam.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fam.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fam.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fam.accuracy = accuracy\n",
    "        fam.hidden_size = n\n",
    "        fam.model = model\n",
    "        fam.loss_fn = loss_fn\n",
    "        fam.optimizer = optimizer"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get the default number of neurons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 401,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Batch size: 32 (default is 64)\n",
    "train_dataset, test_dataset, train_dataloader, test_dataloader = get_dataloaders(train_mass, test_mass, \"a\", 32)\n",
    " \n",
    "for i in range(40):\n",
    "    model = FNN(fam.input_size, fam.hidden_size, fam.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.Adam(model.parameters())\n",
    "\n",
    "    for t in range(fam.epochs):\n",
    "        train_loop(fam.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fam.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fam.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fam.accuracy = accuracy\n",
    "        fam.train_dataset, fam.test_dataset, fam.train_dataloader, fam.test_dataloader = train_dataset, test_dataset, train_dataloader, test_dataloader\n",
    "        fam.batch_size = 32\n",
    "        fam.model = model\n",
    "        fam.loss_fn = loss_fn\n",
    "        fam.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 402,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Batch size: 16 (default is 64)\n",
    "train_dataset, test_dataset, train_dataloader, test_dataloader = get_dataloaders(train_mass, test_mass, \"a\", 16)\n",
    " \n",
    "for i in range(40):\n",
    "    model = FNN(fam.input_size, fam.hidden_size, fam.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.Adam(model.parameters())\n",
    "\n",
    "    for t in range(fam.epochs):\n",
    "        train_loop(fam.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fam.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fam.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fam.accuracy = accuracy\n",
    "        fam.train_dataset, fam.test_dataset, fam.train_dataloader, fam.test_dataloader = train_dataset, test_dataset, train_dataloader, test_dataloader\n",
    "        fam.batch_size = 16\n",
    "        fam.model = model\n",
    "        fam.loss_fn = loss_fn\n",
    "        fam.optimizer = optimizer"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "No need to change the batch size. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 407,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Hidden Layers: 2\n",
    "for i in range(40):\n",
    "    model = FNN(fam.input_size, fam.hidden_size, fam.num_classes, hidden_layers=2).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters())\n",
    "\n",
    "    for t in range(fam.epochs):\n",
    "        train_loop(fam.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fam.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fam.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fam.accuracy = accuracy\n",
    "        fam.hidden_layers = 2\n",
    "        fam.model = model\n",
    "        fam.loss_fn = loss_fn\n",
    "        fam.optimizer = optimizer"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Hidden Layers = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 408,
   "metadata": {},
   "outputs": [],
   "source": [
    "# number of epochs (default): 10 -> increase until 3-\n",
    "e = fam.epochs\n",
    "for i in range(3):\n",
    "    for j in range(10):\n",
    "        model = FNN(fam.input_size, fam.hidden_size, fam.num_classes).to(device)\n",
    "        loss_fn = nn.CrossEntropyLoss()\n",
    "        optimizer = torch.optim.RMSprop(model.parameters())\n",
    "\n",
    "        for t in range(e):\n",
    "            train_loop(fam.train_dataloader, model, loss_fn, optimizer)\n",
    "            accuracy = test_loop(fam.test_dataloader, model, loss_fn)[0]\n",
    "        \n",
    "        if accuracy > fam.accuracy:\n",
    "            print(\"New best epochs: \", e)\n",
    "            print(\"New best model found with accuracy: \", accuracy)\n",
    "            fam.accuracy = accuracy\n",
    "            fam.epochs = e\n",
    "            fam.model = model\n",
    "            fam.loss_fn = loss_fn\n",
    "            fam.optimizer = optimizer\n",
    "    e += 1"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 417,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Activation Function: leaky_relu (default is ReLU)\n",
    "for i in range(40):\n",
    "    model = FNN(fam.input_size, fam.hidden_size, fam.num_classes, activation=\"leaky_relu\").to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters())\n",
    "\n",
    "    for t in range(fam.epochs):\n",
    "        train_loop(fam.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fam.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fam.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fam.accuracy = accuracy\n",
    "        fam.model = model\n",
    "        fam.loss_fn = loss_fn\n",
    "        fam.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 418,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Activation Function: tanh (default is ReLU)\n",
    "for i in range(40):\n",
    "    model = FNN(fam.input_size, fam.hidden_size, fam.num_classes, activation=\"tanh\").to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters())\n",
    "\n",
    "    for t in range(fam.epochs):\n",
    "        train_loop(fam.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fam.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fam.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fam.accuracy = accuracy\n",
    "        fam.model = model\n",
    "        fam.loss_fn = loss_fn\n",
    "        fam.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 420,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Activation Function: sigmoid (default is ReLU)\n",
    "for i in range(40):\n",
    "    model = FNN(fam.input_size, fam.hidden_size, fam.num_classes, activation=\"sigmoid\").to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters())\n",
    "\n",
    "    for t in range(fam.epochs):\n",
    "        train_loop(fam.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fam.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fam.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fam.accuracy = accuracy\n",
    "        fam.model = model\n",
    "        fam.loss_fn = loss_fn\n",
    "        fam.optimizer = optimizer"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "ReLu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 421,
   "metadata": {},
   "outputs": [],
   "source": [
    "# weight_init : zero (default is lecun)\n",
    "for i in range(40):\n",
    "    model = FNN(fam.input_size, fam.hidden_size, fam.num_classes, weight_init=\"zero\").to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters())\n",
    "\n",
    "    for t in range(fam.epochs):\n",
    "        train_loop(fam.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fam.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fam.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fam.accuracy = accuracy\n",
    "        fam.model = model\n",
    "        fam.loss_fn = loss_fn\n",
    "        fam.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 422,
   "metadata": {},
   "outputs": [],
   "source": [
    "# weight_init : normal (default is lecun)\n",
    "for i in range(40):\n",
    "    model = FNN(fam.input_size, fam.hidden_size, fam.num_classes, weight_init=\"normal\").to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters())\n",
    "\n",
    "    for t in range(fam.epochs):\n",
    "        train_loop(fam.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fam.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fam.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fam.accuracy = accuracy\n",
    "        fam.model = model\n",
    "        fam.loss_fn = loss_fn\n",
    "        fam.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 423,
   "metadata": {},
   "outputs": [],
   "source": [
    "# weight_init : xavier (default is lecun)\n",
    "for i in range(40):\n",
    "    model = FNN(fam.input_size, fam.hidden_size, fam.num_classes, weight_init=\"xavier\").to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters())\n",
    "\n",
    "    for t in range(fam.epochs):\n",
    "        train_loop(fam.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fam.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fam.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fam.accuracy = accuracy\n",
    "        fam.model = model\n",
    "        fam.loss_fn = loss_fn\n",
    "        fam.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 424,
   "metadata": {},
   "outputs": [],
   "source": [
    "# weight_init : kaiming (default is lecun)\n",
    "for i in range(40):\n",
    "    model = FNN(fam.input_size, fam.hidden_size, fam.num_classes, weight_init=\"kaiming\").to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters())\n",
    "\n",
    "    for t in range(fam.epochs):\n",
    "        train_loop(fam.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fam.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fam.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fam.accuracy = accuracy\n",
    "        fam.model = model\n",
    "        fam.loss_fn = loss_fn\n",
    "        fam.optimizer = optimizer"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lecun"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 425,
   "metadata": {},
   "outputs": [],
   "source": [
    "# L2 Regularization\n",
    "# Define the hyperparameter search space\n",
    "epochs_candidates = [10, 20, 30]\n",
    "l2_lambda_candidates = [0.001, 0.01, 0.1]\n",
    "\n",
    "for num_epochs, l2_lambda in itertools.product(epochs_candidates, l2_lambda_candidates):\n",
    "    model = FNN(fam.input_size, fam.hidden_size, fam.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = optim.RMSprop(model.parameters(), lr=learning_rate, weight_decay=l2_lambda)\n",
    "\n",
    "    for t in range(num_epochs):\n",
    "        train_loop(fam.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fam.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fam.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        print(\"New best hyperparameters: \", num_epochs, l2_lambda)\n",
    "        \n",
    "        fam.accuracy = accuracy\n",
    "        \n",
    "        fam.model = model\n",
    "        fam.loss_fn = loss_fn\n",
    "        fam.optimizer = optimizer\n",
    "        \n",
    "        fam.epochs = num_epochs\n",
    "        fam.l2_lambda = l2_lambda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 486,
   "metadata": {},
   "outputs": [],
   "source": [
    "# L1 Regularization\n",
    "epochs_candidates = [10, 20, 30]\n",
    "l1_lambda_candidates = [0.001, 0.01, 0.1]\n",
    "\n",
    "for num_epochs, l1_lambda in itertools.product(epochs_candidates, l1_lambda_candidates):\n",
    "    model = FNN(fam.input_size, fam.hidden_size, fam.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = optim.Adam(model.parameters(), lr=learning_rate, weight_decay=l1_lambda)\n",
    "\n",
    "    for t in range(num_epochs):\n",
    "        train_L1(fam.train_dataloader, model, loss_fn, optimizer, l1_lambda)\n",
    "        accuracy = test_loop(fam.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fam.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        print(\"New best hyperparameters: \", num_epochs, l1_lambda)\n",
    "        \n",
    "        fam.accuracy = accuracy\n",
    "        \n",
    "        fam.model = model\n",
    "        fam.loss_fn = loss_fn\n",
    "        fam.optimizer = optimizer\n",
    "        \n",
    "        fam.epochs = num_epochs\n",
    "        fam.l1_lambda = l1_lambda"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "No regularization so this is the best:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 428,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "New best model found with accuracy:  64.0\n"
     ]
    }
   ],
   "source": [
    "# RMSprop\n",
    "for i in range(100):\n",
    "    model = FNN(fam.input_size, fam.hidden_size, fam.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters())\n",
    "\n",
    "    for t in range(fam.epochs):\n",
    "        train_loop(fam.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fam.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fam.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fam.accuracy = accuracy\n",
    "        fam.model = model\n",
    "        fam.loss_fn = loss_fn\n",
    "        fam.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 430,
   "metadata": {},
   "outputs": [],
   "source": [
    "# RMSprop\n",
    "for i in range(100):\n",
    "    model = FNN(fam.input_size, fam.hidden_size, fam.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters())\n",
    "\n",
    "    for t in range(fam.epochs):\n",
    "        train_loop(fam.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fam.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fam.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fam.accuracy = accuracy\n",
    "        fam.model = model\n",
    "        fam.loss_fn = loss_fn\n",
    "        fam.optimizer = optimizer"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## features including assessment -> pathology (calc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default settings\n",
    "fpc = Model()\n",
    "\n",
    "fpc.epochs = epochs\n",
    "fpc.batch_size = batch_size\n",
    "fpc.train_dataset, fpc.test_dataset, fpc.train_dataloader, fpc.test_dataloader = get_dataloaders(train_calc, test_calc, \"p\", batch_size)\n",
    "\n",
    "fpc.input_size = fpc.train_dataloader.dataset.X.shape[1]\n",
    "fpc.hidden_size = hidden_size\n",
    "fpc.num_classes = 3\n",
    "\n",
    "fpc.lr = 1e-3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "63.8"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# default model\n",
    "fpc.model = FNN(fpc.input_size, fpc.hidden_size, fpc.num_classes).to(device)\n",
    "fpc.loss_fn = nn.CrossEntropyLoss()\n",
    "fpc.optimizer = torch.optim.SGD(fpc.model.parameters(), lr=fpc.lr)\n",
    "\n",
    "for t in range(fpc.epochs):\n",
    "    train_loop(fpc.train_dataloader, fpc.model, fpc.loss_fn, fpc.optimizer)\n",
    "    accuracy = test_loop(fpc.test_dataloader, fpc.model, fpc.loss_fn)[0]\n",
    "fpc.accuracy = accuracy\n",
    "fpc.accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "New best model found with accuracy:  65.0\n",
      "New best model found with accuracy:  65.3\n",
      "New best model found with accuracy:  66.0\n",
      "New best model found with accuracy:  66.9\n"
     ]
    }
   ],
   "source": [
    "# Mini-batch gradient descent\n",
    "for i in range(40):\n",
    "    model = FNN(fpc.input_size, fpc.hidden_size, fpc.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.SGD(model.parameters(), lr=fpc.lr, momentum=0.9)\n",
    "\n",
    "    for t in range(fpc.epochs):\n",
    "        train_loop(fpc.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpc.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpc.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpc.accuracy = accuracy\n",
    "        fpc.model = model\n",
    "        fpc.loss_fn = loss_fn\n",
    "        fpc.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "New best model found with accuracy:  67.2\n",
      "New best model found with accuracy:  67.5\n",
      "New best model found with accuracy:  68.1\n"
     ]
    }
   ],
   "source": [
    "# SGD momentum\n",
    "for i in range(40):\n",
    "    model = FNN(fpc.input_size, fpc.hidden_size, fpc.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.SGD(model.parameters(), lr=fpc.lr, momentum=0.9)\n",
    "\n",
    "    for t in range(fpc.epochs):\n",
    "        train_loop(fpc.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpc.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpc.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpc.accuracy = accuracy\n",
    "        fpc.model = model\n",
    "        fpc.loss_fn = loss_fn\n",
    "        fpc.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "# SGD Nestrov\n",
    "for i in range(40):\n",
    "    model = FNN(fpc.input_size, fpc.hidden_size, fpc.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.SGD(model.parameters(), lr=fpc.lr, momentum=0.9, nesterov=True)\n",
    "\n",
    "    for t in range(fpc.epochs):\n",
    "        train_loop(fpc.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpc.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpc.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpc.accuracy = accuracy\n",
    "        fpc.model = model\n",
    "        fpc.loss_fn = loss_fn\n",
    "        fpc.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "New best model found with accuracy:  68.4\n",
      "New best model found with accuracy:  68.7\n",
      "New best model found with accuracy:  69.0\n"
     ]
    }
   ],
   "source": [
    "# Adam\n",
    "for i in range(40):\n",
    "    model = FNN(fpc.input_size, fpc.hidden_size, fpc.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.Adam(model.parameters())\n",
    "\n",
    "    for t in range(fpc.epochs):\n",
    "        train_loop(fpc.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpc.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpc.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpc.accuracy = accuracy\n",
    "        fpc.model = model\n",
    "        fpc.loss_fn = loss_fn\n",
    "        fpc.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "New best model found with accuracy:  69.3\n"
     ]
    }
   ],
   "source": [
    "# Adagrad\n",
    "for i in range(40):\n",
    "    model = FNN(fpc.input_size, fpc.hidden_size, fpc.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.Adagrad(model.parameters())\n",
    "\n",
    "    for t in range(fpc.epochs):\n",
    "        train_loop(fpc.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpc.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpc.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpc.accuracy = accuracy\n",
    "        fpc.model = model\n",
    "        fpc.loss_fn = loss_fn\n",
    "        fpc.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adadelta\n",
    "for i in range(40):\n",
    "    model = FNN(fpc.input_size, fpc.hidden_size, fpc.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.Adadelta(model.parameters())\n",
    "\n",
    "    for t in range(fpc.epochs):\n",
    "        train_loop(fpc.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpc.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpc.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpc.accuracy = accuracy\n",
    "        fpc.model = model\n",
    "        fpc.loss_fn = loss_fn\n",
    "        fpc.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adamax\n",
    "for i in range(40):\n",
    "    model = FNN(fpc.input_size, fpc.hidden_size, fpc.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.Adamax(model.parameters())\n",
    "\n",
    "    for t in range(fpc.epochs):\n",
    "        train_loop(fpc.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpc.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpc.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpc.accuracy = accuracy\n",
    "        fpc.model = model\n",
    "        fpc.loss_fn = loss_fn\n",
    "        fpc.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "New best model found with accuracy:  69.6\n",
      "New best model found with accuracy:  71.2\n"
     ]
    }
   ],
   "source": [
    "# RMSprop\n",
    "for i in range(40):\n",
    "    model = FNN(fpc.input_size, fpc.hidden_size, fpc.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters())\n",
    "\n",
    "    for t in range(fpc.epochs):\n",
    "        train_loop(fpc.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpc.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpc.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpc.accuracy = accuracy\n",
    "        fpc.model = model\n",
    "        fpc.loss_fn = loss_fn\n",
    "        fpc.optimizer = optimizer"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "RMSprop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "New best model found with accuracy:  72.7\n"
     ]
    }
   ],
   "source": [
    "# learning rate scheduler: StepLR\n",
    "for i in range(40):\n",
    "    model = FNN(fpc.input_size, fpc.hidden_size, fpc.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters(), lr=0.1)\n",
    "    scheduler = StepLR(optimizer, step_size=1, gamma=0.1)\n",
    "\n",
    "    for t in range(fpc.epochs):\n",
    "        scheduler.step()\n",
    "        train_loop(fpc.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpc.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpc.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpc.accuracy = accuracy\n",
    "        fpc.model = model\n",
    "        fpc.loss_fn = loss_fn\n",
    "        fpc.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "# learning rate scheduler: ReduceLROnPlateau\n",
    "for i in range(40):\n",
    "    model = FNN(fpc.input_size, fpc.hidden_size, fpc.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters(), lr=0.1)\n",
    "    scheduler = ReduceLROnPlateau(optimizer, mode='max', factor=0.1, patience=0)\n",
    "\n",
    "    for t in range(fpc.epochs):\n",
    "        train_loop(fpc.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpc.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpc.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpc.accuracy = accuracy\n",
    "        fpc.model = model\n",
    "        fpc.loss_fn = loss_fn\n",
    "        fpc.optimizer = optimizer\n",
    "    scheduler.step(accuracy)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "learning rate scheduler: StepLR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Number of Neurons: int((input_size + num_classes)/2)\n",
    "n = int((fpc.input_size + fpc.num_classes)/2)\n",
    "for i in range(40):\n",
    "    model = FNN(fpc.input_size, n, fpc.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters(), lr=0.1)\n",
    "    scheduler = StepLR(optimizer, step_size=1, gamma=0.1)\n",
    "\n",
    "    for t in range(fpc.epochs):\n",
    "        scheduler.step()\n",
    "        train_loop(fpc.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpc.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpc.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpc.accuracy = accuracy\n",
    "        fpc.model = model\n",
    "        fpc.loss_fn = loss_fn\n",
    "        fpc.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Number of Neurons: int(2/3*input_size + num_classes)\n",
    "n = int(2/3*fpc.input_size + fpc.num_classes)\n",
    "for i in range(40):\n",
    "    model = FNN(fpc.input_size, n, fpc.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters(), lr=0.1)\n",
    "    scheduler = StepLR(optimizer, step_size=1, gamma=0.1)\n",
    "\n",
    "    for t in range(fpc.epochs):\n",
    "        scheduler.step()\n",
    "        train_loop(fpc.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpc.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpc.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpc.accuracy = accuracy\n",
    "        fpc.model = model\n",
    "        fpc.loss_fn = loss_fn\n",
    "        fpc.optimizer = optimizer"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Number of Layers: 2\n",
    "for i in range(40):\n",
    "    model = FNN(fpc.input_size, fpc.hidden_size, fpc.num_classes, hidden_layers=2).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters(), lr=0.1)\n",
    "    scheduler = ReduceLROnPlateau(optimizer, mode='max', factor=0.1, patience=0)\n",
    "\n",
    "    for t in range(fpc.epochs):\n",
    "        train_loop(fpc.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpc.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpc.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpc.accuracy = accuracy\n",
    "        fpc.model = model\n",
    "        fpc.loss_fn = loss_fn\n",
    "        fpc.optimizer = optimizer\n",
    "    scheduler.step(accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "fpc.hidden_layers = 1"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\sonog\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\torch\\optim\\lr_scheduler.py:138: UserWarning: Detected call of `lr_scheduler.step()` before `optimizer.step()`. In PyTorch 1.1.0 and later, you should call them in the opposite order: `optimizer.step()` before `lr_scheduler.step()`.  Failure to do this will result in PyTorch skipping the first value of the learning rate schedule. See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\n",
      "  warnings.warn(\"Detected call of `lr_scheduler.step()` before `optimizer.step()`. \"\n"
     ]
    }
   ],
   "source": [
    "# Batch size: 32 (default is 64)\n",
    "train_dataset, test_dataset, train_dataloader, test_dataloader = get_dataloaders(train_calc, test_calc, \"p\", 32)\n",
    "\n",
    "for i in range(40):\n",
    "    model = FNN(fpc.input_size, fpc.hidden_size, fpc.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters(), lr=0.1)\n",
    "    scheduler = StepLR(optimizer, step_size=1, gamma=0.1)\n",
    "\n",
    "    for t in range(fpc.epochs):\n",
    "        scheduler.step()\n",
    "        train_loop(fpc.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpc.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpc.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpc.accuracy = accuracy\n",
    "        fpc.train_dataset, fpc.test_dataset, fpc.train_dataloader, fpc.test_dataloader = fpc.train_dataset, test_dataset, train_dataloader, test_dataloader\n",
    "        fpc.model = model\n",
    "        fpc.loss_fn = loss_fn\n",
    "        fpc.optimizer = optimizer\n",
    "        fpc.batch_size = 32"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\sonog\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\torch\\optim\\lr_scheduler.py:138: UserWarning: Detected call of `lr_scheduler.step()` before `optimizer.step()`. In PyTorch 1.1.0 and later, you should call them in the opposite order: `optimizer.step()` before `lr_scheduler.step()`.  Failure to do this will result in PyTorch skipping the first value of the learning rate schedule. See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\n",
      "  warnings.warn(\"Detected call of `lr_scheduler.step()` before `optimizer.step()`. \"\n"
     ]
    }
   ],
   "source": [
    "# number of epochs (default): 10 -> increase until 3-\n",
    "e = fpc.epochs\n",
    "for i in range(3):\n",
    "    for j in range(10):\n",
    "        model = FNN(fpc.input_size, fpc.hidden_size, fpc.num_classes).to(device)\n",
    "        optimizer = torch.optim.RMSprop(model.parameters(), lr=0.1)\n",
    "        scheduler = StepLR(optimizer, step_size=1, gamma=0.1)\n",
    "\n",
    "        for t in range(e):\n",
    "            scheduler.step()\n",
    "            train_loop(fpc.train_dataloader, model, loss_fn, optimizer)\n",
    "            accuracy = test_loop(fpc.test_dataloader, model, loss_fn)[0]\n",
    "        \n",
    "        if accuracy > fpc.accuracy:\n",
    "            print(\"New best epochs: \", e)\n",
    "            print(\"New best model found with accuracy: \", accuracy)\n",
    "            fpc.accuracy = accuracy\n",
    "            fac.epochs = e\n",
    "            fpc.model = model\n",
    "            fpc.loss_fn = loss_fn\n",
    "            fpc.optimizer = optimizer\n",
    "    e += 1"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Activation Function: leaky_relu (default is ReLU)\n",
    "for i in range(40):\n",
    "    model = FNN(fpc.input_size, fpc.hidden_size, fpc.num_classes, activation=\"leaky_relu\").to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters(), lr=0.1)\n",
    "    scheduler = StepLR(optimizer, step_size=1, gamma=0.1)\n",
    "\n",
    "    for t in range(fpc.epochs):\n",
    "        scheduler.step()\n",
    "        train_loop(fpc.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpc.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpc.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpc.accuracy = accuracy\n",
    "        fpc.model = model\n",
    "        fpc.loss_fn = loss_fn\n",
    "        fpc.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Activation Function: tanh (default is ReLU)\n",
    "for i in range(40):\n",
    "    model = FNN(fpc.input_size, fpc.hidden_size, fpc.num_classes, activation=\"tanh\").to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters(), lr=0.1)\n",
    "    scheduler = StepLR(optimizer, step_size=1, gamma=0.1)\n",
    "\n",
    "    for t in range(fpc.epochs):\n",
    "        scheduler.step()\n",
    "        train_loop(fpc.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpc.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpc.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpc.accuracy = accuracy\n",
    "        fpc.model = model\n",
    "        fpc.loss_fn = loss_fn\n",
    "        fpc.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Activation Function: sigmoid (default is ReLU)\n",
    "for i in range(40):\n",
    "    model = FNN(fpc.input_size, fpc.hidden_size, fpc.num_classes, activation=\"sigmoid\").to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters(), lr=0.1)\n",
    "    scheduler = StepLR(optimizer, step_size=1, gamma=0.1)\n",
    "\n",
    "    for t in range(fpc.epochs):\n",
    "        scheduler.step()\n",
    "        train_loop(fpc.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpc.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpc.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpc.accuracy = accuracy\n",
    "        fpc.model = model\n",
    "        fpc.loss_fn = loss_fn\n",
    "        fpc.optimizer = optimizer"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "ReLu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "# weight_init : zero (default is lecun)\n",
    "for i in range(40):\n",
    "    model = FNN(fpc.input_size, fpc.hidden_size, fpc.num_classes, weight_init=\"zero\").to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters(), lr=0.1)\n",
    "    scheduler = StepLR(optimizer, step_size=1, gamma=0.1)\n",
    "\n",
    "    for t in range(fpc.epochs):\n",
    "        scheduler.step()\n",
    "        train_loop(fpc.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpc.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpc.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpc.accuracy = accuracy\n",
    "        fpc.model = model\n",
    "        fpc.loss_fn = loss_fn\n",
    "        fpc.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "New best model found with accuracy:  73.3\n",
      "New best model found with accuracy:  75.5\n"
     ]
    }
   ],
   "source": [
    "# weight_init : normal (default is lecun) # no lr scheduler and 2 hidden layers by mistake\n",
    "for i in range(40):\n",
    "    model = FNN(fpc.input_size, fpc.hidden_size, fpc.num_classes, weight_init=\"normal\", hidden_layers=2).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters())\n",
    "\n",
    "    for t in range(fpc.epochs):\n",
    "        train_loop(fpc.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpc.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpc.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpc.accuracy = accuracy\n",
    "        fpc.model = model\n",
    "        fpc.loss_fn = loss_fn\n",
    "        fpc.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "# weight_init : normal (default is lecun)\n",
    "for i in range(40):\n",
    "    model = FNN(fpc.input_size, fpc.hidden_size, fpc.num_classes, weight_init=\"normal\").to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters(), lr=0.1)\n",
    "    scheduler = StepLR(optimizer, step_size=1, gamma=0.1)\n",
    "\n",
    "    for t in range(fpc.epochs):\n",
    "        scheduler.step()\n",
    "        train_loop(fpc.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpc.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpc.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpc.accuracy = accuracy\n",
    "        fpc.model = model\n",
    "        fpc.loss_fn = loss_fn\n",
    "        fpc.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "# weight_init : xavier (default is lecun)\n",
    "for i in range(40):\n",
    "    model = FNN(fpc.input_size, fpc.hidden_size, fpc.num_classes, weight_init=\"xavier\").to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters(), lr=0.1)\n",
    "    scheduler = StepLR(optimizer, step_size=1, gamma=0.1)\n",
    "\n",
    "    for t in range(fpc.epochs):\n",
    "        scheduler.step()\n",
    "        train_loop(fpc.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpc.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpc.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpc.accuracy = accuracy\n",
    "        fpc.model = model\n",
    "        fpc.loss_fn = loss_fn\n",
    "        fpc.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "# weight_init : kaiming (default is lecun)\n",
    "for i in range(40):\n",
    "    model = FNN(fpc.input_size, fpc.hidden_size, fpc.num_classes, weight_init=\"kaiming\").to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters(), lr=0.1)\n",
    "    scheduler = StepLR(optimizer, step_size=1, gamma=0.1)\n",
    "\n",
    "    for t in range(fpc.epochs):\n",
    "        scheduler.step()\n",
    "        train_loop(fpc.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpc.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpc.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpc.accuracy = accuracy\n",
    "        fpc.model = model\n",
    "        fpc.loss_fn = loss_fn\n",
    "        fpc.optimizer = optimizer"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Normal without lr scheduler and hidden_layers=2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "# L2 Regularization\n",
    "# Define the hyperparameter search space\n",
    "epochs_candidates = [10, 20, 30]\n",
    "l2_lambda_candidates = [0.001, 0.01, 0.1]\n",
    "\n",
    "for num_epochs, l2_lambda in itertools.product(epochs_candidates, l2_lambda_candidates):\n",
    "    model = FNN(fpc.input_size, fpc.hidden_size, fpc.num_classes, hidden_layers=2, weight_init=\"normal\").to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = optim.RMSprop(model.parameters(), lr=learning_rate, weight_decay=l2_lambda)\n",
    "\n",
    "    for t in range(num_epochs):\n",
    "        train_loop(fpc.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpc.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpc.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        print(\"New best hyperparameters: \", num_epochs, l2_lambda)\n",
    "        \n",
    "        fpc.accuracy = accuracy\n",
    "        fpc.model = model\n",
    "        fpc.loss_fn = loss_fn\n",
    "        fpc.optimizer = optimizer\n",
    "        \n",
    "        fac.epochs = num_epochs\n",
    "        fac.l2_lambda = l2_lambda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "# L1 Regularization\n",
    "epochs_candidates = [10, 20, 30]\n",
    "l1_lambda_candidates = [0.001, 0.01, 0.1]\n",
    "\n",
    "for num_epochs, l1_lambda in itertools.product(epochs_candidates, l1_lambda_candidates):\n",
    "    model = FNN(fpc.input_size, fpc.hidden_size, fpc.num_classes, hidden_layers=2, weight_init=\"normal\").to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = optim.RMSprop(model.parameters(), lr=learning_rate, weight_decay=l1_lambda)\n",
    "\n",
    "    for t in range(num_epochs):\n",
    "        train_L1(fpc.train_dataloader, model, loss_fn, optimizer, l1_lambda)\n",
    "        accuracy = test_loop(fpc.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpc.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        print(\"New best hyperparameters: \", num_epochs, l2_lambda)\n",
    "        \n",
    "        fpc.accuracy = accuracy\n",
    "        \n",
    "        fpc.model = model\n",
    "        fpc.loss_fn = loss_fn\n",
    "        fpc.optimizer = optimizer\n",
    "        \n",
    "        fpc.epochs = num_epochs\n",
    "        fpc.l1_lambda = l1_lambda"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "No regularization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(100):\n",
    "    model = FNN(fpc.input_size, fpc.hidden_size, fpc.num_classes, weight_init=\"normal\", hidden_layers=2).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters())\n",
    "\n",
    "    for t in range(fpc.epochs):\n",
    "        train_loop(fpc.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpc.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpc.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpc.accuracy = accuracy\n",
    "        fpc.model = model\n",
    "        fpc.loss_fn = loss_fn\n",
    "        fpc.optimizer = optimizer"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## features including assessment -> pathology (mass)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default settings\n",
    "fpm = Model()\n",
    "\n",
    "fpm.epochs = epochs\n",
    "fpm.batch_size = batch_size\n",
    "fpm.train_dataset, fpm.test_dataset, fpm.train_dataloader, fpm.test_dataloader = get_dataloaders(train_mass, test_mass, \"p\", batch_size)\n",
    "\n",
    "fpm.input_size = fpm.train_dataloader.dataset.X.shape[1]\n",
    "fpm.hidden_size = hidden_size\n",
    "fpm.num_classes = 3\n",
    "\n",
    "fpm.lr = 1e-3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "63.5"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# default model\n",
    "fpm.model = FNN(fpm.input_size, fpm.hidden_size, fpm.num_classes).to(device)\n",
    "fpm.loss_fn = nn.CrossEntropyLoss()\n",
    "fpm.optimizer = torch.optim.SGD(fpm.model.parameters(), lr=fpm.lr)\n",
    "\n",
    "for t in range(fpm.epochs):\n",
    "    train_loop(fpm.train_dataloader, fpm.model, fpm.loss_fn, fpm.optimizer)\n",
    "    accuracy = test_loop(fpm.test_dataloader, fpm.model, fpm.loss_fn)[0]\n",
    "fpm.accuracy = accuracy\n",
    "fpm.accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "New best model found with accuracy:  64.0\n"
     ]
    }
   ],
   "source": [
    "# Mini-batch gradient descent\n",
    "for i in range(40):\n",
    "    model = FNN(fpm.input_size, fpm.hidden_size, fpm.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.SGD(model.parameters(), lr=fpm.lr)\n",
    "\n",
    "    for t in range(fpm.epochs):\n",
    "        train_loop(fpm.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpm.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpm.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpm.accuracy = accuracy\n",
    "        fpm.model = model\n",
    "        fpm.loss_fn = loss_fn\n",
    "        fpm.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "New best model found with accuracy:  69.6\n",
      "New best model found with accuracy:  70.6\n"
     ]
    }
   ],
   "source": [
    "# SGD momentum\n",
    "for i in range(40):\n",
    "    model = FNN(fpm.input_size, fpm.hidden_size, fpm.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.SGD(model.parameters(), lr=fpm.lr, momentum=0.9)\n",
    "\n",
    "    for t in range(fpm.epochs):\n",
    "        train_loop(fpm.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpm.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpm.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpm.accuracy = accuracy\n",
    "        fpm.model = model\n",
    "        fpm.loss_fn = loss_fn\n",
    "        fpm.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "# SGD Nestrov\n",
    "for i in range(40):\n",
    "    model = FNN(fpm.input_size, fpm.hidden_size, fpm.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.SGD(model.parameters(), lr=fpm.lr, momentum=0.9, nesterov=True)\n",
    "\n",
    "    for t in range(fpm.epochs):\n",
    "        train_loop(fpm.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpm.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpm.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpm.accuracy = accuracy\n",
    "        fpm.model = model\n",
    "        fpm.loss_fn = loss_fn\n",
    "        fpm.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adam\n",
    "for i in range(40):\n",
    "    model = FNN(fpm.input_size, fpm.hidden_size, fpm.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.Adam(model.parameters())\n",
    "\n",
    "    for t in range(fpm.epochs):\n",
    "        train_loop(fpm.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpm.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpm.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpm.accuracy = accuracy\n",
    "        fpm.model = model\n",
    "        fpm.loss_fn = loss_fn\n",
    "        fpm.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adagrad\n",
    "for i in range(40):\n",
    "    model = FNN(fpm.input_size, fpm.hidden_size, fpm.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.Adagrad(model.parameters())\n",
    "\n",
    "    for t in range(fpm.epochs):\n",
    "        train_loop(fpm.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpm.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpm.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpm.accuracy = accuracy\n",
    "        fpm.model = model\n",
    "        fpm.loss_fn = loss_fn\n",
    "        fpm.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adadelta\n",
    "for i in range(40):\n",
    "    model = FNN(fpm.input_size, fpm.hidden_size, fpm.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.Adadelta(model.parameters())\n",
    "\n",
    "    for t in range(fpm.epochs):\n",
    "        train_loop(fpm.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpm.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpm.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpm.accuracy = accuracy\n",
    "        fpm.model = model\n",
    "        fpm.loss_fn = loss_fn\n",
    "        fpm.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adamax\n",
    "for i in range(40):\n",
    "    model = FNN(fpm.input_size, fpm.hidden_size, fpm.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.Adamax(model.parameters())\n",
    "\n",
    "    for t in range(fpm.epochs):\n",
    "        train_loop(fpm.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpm.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpm.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpm.accuracy = accuracy\n",
    "        fpm.model = model\n",
    "        fpm.loss_fn = loss_fn\n",
    "        fpm.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "New best model found with accuracy:  70.9\n",
      "New best model found with accuracy:  72.5\n"
     ]
    }
   ],
   "source": [
    "# RMSprop\n",
    "for i in range(40):\n",
    "    model = FNN(fpm.input_size, fpm.hidden_size, fpm.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters())\n",
    "\n",
    "    for t in range(fpm.epochs):\n",
    "        train_loop(fpm.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpm.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpm.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpm.accuracy = accuracy\n",
    "        fpm.model = model\n",
    "        fpm.loss_fn = loss_fn\n",
    "        fpm.optimizer = optimizer"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "RMSprop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Number of Neurons: int(2/3*input_size + num_classes)\n",
    "n = int(2/3*fpm.input_size + fpm.num_classes)\n",
    "\n",
    "for i in range(40):\n",
    "    model = FNN(fpm.input_size, n, fpm.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters())\n",
    "\n",
    "    for t in range(fpm.epochs):\n",
    "        train_loop(fpm.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpm.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpm.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpm.accuracy = accuracy\n",
    "        fpm.model = model\n",
    "        fpm.loss_fn = loss_fn\n",
    "        fpm.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Number of Neurons: int(2/3*input_size + num_classes)\n",
    "n = int(2/3*fpm.input_size + fpm.num_classes)\n",
    "\n",
    "for i in range(40):\n",
    "    model = FNN(fpm.input_size, n, fpm.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters())\n",
    "\n",
    "    for t in range(fpm.epochs):\n",
    "        train_loop(fpm.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpm.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpm.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpm.accuracy = accuracy\n",
    "        fpm.model = model\n",
    "        fpm.loss_fn = loss_fn\n",
    "        fpm.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "New best model found with accuracy:  73.3\n"
     ]
    }
   ],
   "source": [
    "# Number of Neurons: default\n",
    "\n",
    "for i in range(40):\n",
    "    model = FNN(fpm.input_size, fpm.hidden_size, fpm.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters())\n",
    "\n",
    "    for t in range(fpm.epochs):\n",
    "        train_loop(fpm.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpm.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpm.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpm.accuracy = accuracy\n",
    "        fpm.model = model\n",
    "        fpm.loss_fn = loss_fn\n",
    "        fpm.optimizer = optimizer"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "512"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Number of Layers: 2\n",
    "for i in range(40):\n",
    "    model = FNN(fpm.input_size, fpm.hidden_size, fpm.num_classes, hidden_layers=2).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters())\n",
    "\n",
    "    for t in range(fpm.epochs):\n",
    "        train_loop(fpm.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpm.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpm.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpm.accuracy = accuracy\n",
    "        fpm.model = model\n",
    "        fpm.loss_fn = loss_fn\n",
    "        fpm.optimizer = optimizer"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Batch size: 32 (default is 64)\n",
    "train_dataset, test_dataset, train_dataloader, test_dataloader = get_dataloaders(train_mass, test_mass, \"p\", 32)\n",
    "\n",
    "for i in range(40):\n",
    "    model = FNN(fpm.input_size, fpm.hidden_size, fpm.num_classes).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters())\n",
    "\n",
    "    for t in range(fpm.epochs):\n",
    "        train_loop(fpm.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpm.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpm.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpm.accuracy = accuracy\n",
    "        fpm.train_dataset, fpm.test_dataset, fpm.train_dataloader, fpm.test_dataloader = train_dataset, test_dataset, train_dataloader, test_dataloader\n",
    "        fpm.model = model\n",
    "        fpm.loss_fn = loss_fn\n",
    "        fpm.optimizer = optimizer"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# number of epochs (default): 10 -> increase until 3-\n",
    "e = fpm.epochs\n",
    "for i in range(3):\n",
    "    for j in range(10):\n",
    "        model = FNN(fpm.input_size, fpm.hidden_size, fpm.num_classes).to(device)\n",
    "        loss_fn = nn.CrossEntropyLoss()\n",
    "        optimizer = torch.optim.RMSprop(model.parameters())\n",
    "\n",
    "        for t in range(e):\n",
    "            train_loop(fpm.train_dataloader, model, loss_fn, optimizer)\n",
    "            accuracy = test_loop(fpm.test_dataloader, model, loss_fn)[0]\n",
    "        \n",
    "        if accuracy > fpm.accuracy:\n",
    "            print(\"New best epochs: \", e)\n",
    "            print(\"New best model found with accuracy: \", accuracy)\n",
    "            fpm.accuracy = accuracy\n",
    "            fpm.epochs = e\n",
    "            fpm.model = model\n",
    "            fpm.loss_fn = loss_fn\n",
    "            fpm.optimizer = optimizer\n",
    "    e += 1"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Activation Function: leaky_relu (default is ReLU)\n",
    "for i in range(40):\n",
    "    model = FNN(fpm.input_size, fpm.hidden_size, fpm.num_classes, activation=\"leaky_relu\").to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters())\n",
    "\n",
    "    for t in range(fpm.epochs):\n",
    "        train_loop(fpm.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpm.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpm.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpm.accuracy = accuracy\n",
    "        fpm.model = model\n",
    "        fpm.loss_fn = loss_fn\n",
    "        fpm.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Activation Function: tanh (default is ReLU)\n",
    "for i in range(40):\n",
    "    model = FNN(fpm.input_size, fpm.hidden_size, fpm.num_classes, activation=\"tanh\").to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters())\n",
    "\n",
    "    for t in range(fpm.epochs):\n",
    "        train_loop(fpm.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpm.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpm.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpm.accuracy = accuracy\n",
    "        fpm.model = model\n",
    "        fpm.loss_fn = loss_fn\n",
    "        fpm.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Activation Function: sigmoid (default is ReLU)\n",
    "for i in range(40):\n",
    "    model = FNN(fpm.input_size, fpm.hidden_size, fpm.num_classes, activation=\"sigmoid\").to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters())\n",
    "\n",
    "    for t in range(fpm.epochs):\n",
    "        train_loop(fpm.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpm.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpm.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpm.accuracy = accuracy\n",
    "        fpm.model = model\n",
    "        fpm.loss_fn = loss_fn\n",
    "        fpm.optimizer = optimizer"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "ReLu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "# weight_init : zero (default is lecun)\n",
    "for i in range(40):\n",
    "    model = FNN(fpm.input_size, fpm.hidden_size, fpm.num_classes, weight_init=\"zero\").to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters())\n",
    "\n",
    "    for t in range(fpm.epochs):\n",
    "        train_loop(fpm.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpm.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpm.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpm.accuracy = accuracy\n",
    "        fpm.model = model\n",
    "        fpm.loss_fn = loss_fn\n",
    "        fpm.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "# weight_init : normal (default is lecun)\n",
    "for i in range(40):\n",
    "    model = FNN(fpm.input_size, fpm.hidden_size, fpm.num_classes, weight_init=\"normal\").to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters())\n",
    "\n",
    "    for t in range(fpm.epochs):\n",
    "        train_loop(fpm.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpm.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpm.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpm.accuracy = accuracy\n",
    "        fpm.model = model\n",
    "        fpm.loss_fn = loss_fn\n",
    "        fpm.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "New best model found with accuracy:  73.8\n"
     ]
    }
   ],
   "source": [
    "# weight_init : xavier (default is lecun)\n",
    "for i in range(40):\n",
    "    model = FNN(fpm.input_size, fpm.hidden_size, fpm.num_classes, weight_init=\"xavier\").to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters())\n",
    "\n",
    "    for t in range(fpm.epochs):\n",
    "        train_loop(fpm.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpm.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpm.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpm.accuracy = accuracy\n",
    "        fpm.model = model\n",
    "        fpm.loss_fn = loss_fn\n",
    "        fpm.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "# weight_init : kaiming (default is lecun)\n",
    "for i in range(40):\n",
    "    model = FNN(fpm.input_size, fpm.hidden_size, fpm.num_classes, weight_init=\"kaiming\").to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters())\n",
    "\n",
    "    for t in range(fpm.epochs):\n",
    "        train_loop(fpm.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpm.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpm.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpm.accuracy = accuracy\n",
    "        fpm.model = model\n",
    "        fpm.loss_fn = loss_fn\n",
    "        fpm.optimizer = optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "# weight_init : normal (default is lecun) # same condition as best of fpc\n",
    "for i in range(40):\n",
    "    model = FNN(fpm.input_size, fpm.hidden_size, fpm.num_classes, weight_init=\"normal\", hidden_layers=2).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.RMSprop(model.parameters())\n",
    "\n",
    "    for t in range(fpm.epochs):\n",
    "        train_loop(fpm.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpm.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpm.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        fpm.accuracy = accuracy\n",
    "        fpm.model = model\n",
    "        fpm.loss_fn = loss_fn\n",
    "        fpm.optimizer = optimizer"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lecun"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "# L2 Regularization\n",
    "# Define the hyperparameter search space\n",
    "epochs_candidates = [10, 20, 30]\n",
    "l2_lambda_candidates = [0.001, 0.01, 0.1]\n",
    "\n",
    "for num_epochs, l2_lambda in itertools.product(epochs_candidates, l2_lambda_candidates):\n",
    "    model = FNN(fpm.input_size, fpm.hidden_size, fpm.num_classes, weight_init=\"normal\", hidden_layers=2).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = optim.RMSprop(model.parameters(), lr=learning_rate, weight_decay=l2_lambda)\n",
    "\n",
    "    for t in range(num_epochs):\n",
    "        train_loop(fpm.train_dataloader, model, loss_fn, optimizer)\n",
    "        accuracy = test_loop(fpm.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpm.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        print(\"New best hyperparameters: \", num_epochs, l2_lambda)\n",
    "        \n",
    "        fpm.accuracy = accuracy\n",
    "        fpm.model = model\n",
    "        fpm.loss_fn = loss_fn\n",
    "        fpm.optimizer = optimizer\n",
    "        \n",
    "        fpm.epochs = num_epochs\n",
    "        fpm.l2_lambda = l2_lambda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "# L1 Regularization\n",
    "epochs_candidates = [10, 20, 30]\n",
    "l1_lambda_candidates = [0.001, 0.01, 0.1]\n",
    "\n",
    "for num_epochs, l1_lambda in itertools.product(epochs_candidates, l1_lambda_candidates):\n",
    "    model = FNN(fpm.input_size, fpm.hidden_size, fpm.num_classes, weight_init=\"normal\", hidden_layers=2).to(device)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = optim.RMSprop(model.parameters(), lr=learning_rate, weight_decay=l1_lambda)\n",
    "\n",
    "    for t in range(num_epochs):\n",
    "        train_L1(fpm.train_dataloader, model, loss_fn, optimizer, l1_lambda)\n",
    "        accuracy = test_loop(fpm.test_dataloader, model, loss_fn)[0]\n",
    "    \n",
    "    if accuracy > fpm.accuracy:\n",
    "        print(\"New best model found with accuracy: \", accuracy)\n",
    "        print(\"New best hyperparameters: \", num_epochs, l1_lambda)\n",
    "        \n",
    "        fpm.accuracy = accuracy\n",
    "        \n",
    "        fpm.model = model\n",
    "        fpm.loss_fn = loss_fn\n",
    "        fpm.optimizer = optimizer\n",
    "        \n",
    "        fpm.epochs = num_epochs\n",
    "        fpm.l1_lambda = l1_lambda"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "No regularization"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Accuracies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 489,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "77.3"
      ]
     },
     "execution_count": 489,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# fac.accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 490,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "64.0"
      ]
     },
     "execution_count": 490,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# fam.accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "75.5"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# fpc.accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "73.8"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fpm.accuracy"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Save the models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 434,
   "metadata": {},
   "outputs": [],
   "source": [
    "# torch.save(fac.model.state_dict(), \"FNN_fac.pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 435,
   "metadata": {},
   "outputs": [],
   "source": [
    "# torch.save(fam.model.state_dict(), \"FNN_fam.pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "# torch.save(fpc.model.state_dict(), \"FNN_fpc.pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(fpm.model.state_dict(), \"FNN_fpm.pt\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}