3_5_random_factors.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import math\n",
    "import statistics\n",
    "import pandas as pd\n",
    "import random\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.patches as mpatches\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load output test probabilitites and true labels for each model\n",
    "\n",
    "For every model there are 2 files generated in numpy binary format:\n",
    "\n",
    "  **probabilities.npy** - numpy array of size **TEST POINT CLOUDS CLOUD x CLASSES COUNT x MODELS COUNT** with output classification probability for each of 10 individually trained models\n",
    "\n",
    "  **true_labels.npy** - numpy array of size **TEST POINT CLOUDS CLOUD** with true labels data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Base models\n",
    "BASE_MODELS = ['pointnet', 'pointnet++', 'so-net', 'kcnet', 'deepsets', 'dgcnn', 'pointcnn']\n",
    "BASE_MODELS_ALIASES = ['PointNet', 'PointNet++', 'SO-Net', 'KCNet', 'DeepSets', 'DGCNN', 'PointCNN']\n",
    "SONET_BAGGING_MODELS = ['so-net_sz_01', 'so-net_sz_02', 'so-net_sz_03', 'so-net_sz_04', 'so-net_sz_05', 'so-net_sz_06',\n",
    "                        'so-net_sz_07', 'so-net_sz_08', 'so-net_sz_09', 'so-net_bagging']\n",
    "SONET_CONST_MODELS = ['so-net_TTT', 'so-net_FFT', 'so-net_FTF', 'so-net_TFF', 'so-net_TTF', 'so-net_TFT', 'so-net_FTT']\n",
    "SONET_CLASSIFIER_ENSEMBLE = ['so-net_' + str(i) for i in range(1, 11)]\n",
    "ALL_MODELS = BASE_MODELS + SONET_BAGGING_MODELS + SONET_CONST_MODELS + SONET_CLASSIFIER_ENSEMBLE\n",
    "\n",
    "CLASS_NAMES = ['airplane', 'bathtub', 'bed', 'bench', 'bookshelf', 'bottle', 'bowl',\n",
    "               'car', 'chair', 'cone', 'cup', 'curtain', 'desk', 'door', 'dresser',\n",
    "               'flower_pot', 'glass_box', 'guitar', 'keyboard', 'lamp', 'laptop',\n",
    "               'mantel', 'monitor', 'night_stand', 'person', 'piano', 'plant', 'radio',\n",
    "               'range_hood', 'sink', 'sofa', 'stairs', 'stool', 'table', 'tent', 'toilet',\n",
    "               'tv_stand', 'vase', 'wardrobe', 'xbox']\n",
    "CLASS_TEST_INSTANCES = [100, 50, 100, 20, 100, 100, 20, 100, 100, 20, 20, 20, 86, 20,\n",
    "                        86, 20, 100, 100, 20, 20, 20, 100, 100, 86, 20, 100, 100, 20,\n",
    "                        100, 20, 100, 20, 20, 100, 20, 100, 100, 100, 20, 20]\n",
    "CLASSES_COUNT = 40\n",
    "# MODELS_COUNT = 10\n",
    "# ENSEMBLE_MODELS_NUMBER = 10\n",
    "\n",
    "probabilities = {}\n",
    "true_labels = {}\n",
    "for model in ALL_MODELS:\n",
    "    for dir_name in ['models_reordered', 'models_raw', 'models_bagging', 'models_const', 'models_classifier_ensemble']:\n",
    "        if os.path.exists(os.path.join(dir_name, model)):\n",
    "            break\n",
    "    prob_filepath = os.path.join(os.path.join(dir_name, model), 'probabilities.npy')\n",
    "    label_filepath = os.path.join(os.path.join(dir_name, model), 'true_labels.npy')\n",
    "    probabilities[model] = np.load(prob_filepath)\n",
    "    if dir_name != 'models_reordered':\n",
    "        probabilities[model] = np.transpose(probabilities[model], axes=(2, 0, 1))\n",
    "\n",
    "    # probabilities[model] =  softmax(probabilities[model])\n",
    "    idx = np.argmax(probabilities[model], axis=-1)\n",
    "    hardvote = np.squeeze(np.eye(40)[idx.reshape(-1)]).reshape(probabilities[model].shape)\n",
    "    e_x = np.exp(probabilities[model])\n",
    "    softvote = e_x /np.expand_dims(np.sum(e_x, axis=-1), axis=-1)\n",
    "    probabilities[model] = {'activations': probabilities[model]/np.std(probabilities[model]),\n",
    "                           'hardvote': hardvote, 'softvote': softvote}\n",
    "    true_labels[model] = np.load(label_filepath)\n",
    "    \n",
    "    for class_idx in range(CLASSES_COUNT):\n",
    "        error_info = (\"Test cloud instances for model \" + model + \" and class \"\n",
    "                      + str(class_idx) + \" differs! Should be \" + str(CLASS_TEST_INSTANCES[class_idx])\n",
    "                      + \" but got \" + str(np.sum(true_labels[model] == class_idx)))\n",
    "        assert np.sum(true_labels[model] == class_idx) == CLASS_TEST_INSTANCES[class_idx] or \\\n",
    "          (model=='deepsets' and class_idx==12 and np.sum(true_labels[model] == class_idx) == 85) or \\\n",
    "          (model=='deepsets' and class_idx==39 and np.sum(true_labels[model] == class_idx) == 19) or \\\n",
    "          (model=='deepsets_ep100' and class_idx==12 and np.sum(true_labels[model] == class_idx) == 85) or \\\n",
    "          (model=='deepsets_ep100' and class_idx==39 and np.sum(true_labels[model] == class_idx) == 19), \\\n",
    "          error_info\n",
    "                \n",
    "#     assert (probabilities[model].shape[0] == ENSEMBLE_MODELS_NUMBER)\n",
    "    assert (probabilities[model]['activations'].shape[1] == 2468 or \\\n",
    "            (model=='deepsets' and probabilities[model]['activations'].shape[1] == 2466) or \\\n",
    "           (model=='deepsets_ep100' and probabilities[model]['activations'].shape[1] == 2466))\n",
    "    assert (probabilities[model]['activations'].shape[2] == CLASSES_COUNT)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Caclulate statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "####################################################################################\n",
    "# Get random subsets\n",
    "####################################################################################\n",
    "\n",
    "def get_k_combinations_of_n_elems(n, k, rep):\n",
    "    \"\"\"\n",
    "    Get rep different k-combinations of set of n numbers. If requested reps is greater\n",
    "    than possible combinations number just return all possible combinations. In our case:\n",
    "    n - maximum number of ensembled models\n",
    "    k - actual number of ensembled models\n",
    "    rep - how many different k subsets of n we want to generate.    \n",
    "    \"\"\"\n",
    "    c_n_k = math.factorial(n) / math.factorial(k) / math.factorial(n-k)\n",
    "    combinations_number = min(c_n_k, rep)\n",
    "    \n",
    "    subsets = []   \n",
    "    while len(subsets) != combinations_number:\n",
    "        candidate = np.sort(np.random.choice(n, k, replace=False))\n",
    "        if not np.array([(el == candidate).all() for el in subsets]).any():\n",
    "            subsets.append(candidate)\n",
    "    return np.array(subsets)\n",
    "        \n",
    "####################################################################################\n",
    "# Calc accuracy statistics\n",
    "####################################################################################\n",
    "\n",
    "def get_stats_for_num_ens_param(params_list):\n",
    "    stats = {}\n",
    "    stats['mean'] = statistics.mean(params_list)\n",
    "    stats['stddev'] = statistics.stdev(params_list) if len(params_list) > 1 else 0.0\n",
    "    stats['median'] = statistics.median(params_list)\n",
    "    stats['min'] = min(params_list)\n",
    "    stats['max'] = max(params_list)\n",
    "    return stats\n",
    "\n",
    "def get_accuracy_stats(models_list, voting_method='activations'):\n",
    "    accuracy_stats = {}\n",
    "    for model in models_list:\n",
    "        subsets = {}\n",
    "        \n",
    "        num_ensemble_models = probabilities[model][voting_method].shape[0]\n",
    "        for num_ens in range(1, num_ensemble_models+1):\n",
    "            subset = get_k_combinations_of_n_elems(len(probabilities[model][voting_method]),\n",
    "                                                   num_ens, 100)\n",
    "            subsets[num_ens] = subset\n",
    "            \n",
    "        accuracy_stats[model] = {}\n",
    "\n",
    "        for num_ens in subsets:\n",
    "            num_ens_accs = []\n",
    "            for subset in subsets[num_ens]:\n",
    "                preds = np.argmax(np.mean(probabilities[model][voting_method][subset], axis=0), axis=-1)\n",
    "                true = true_labels[model]\n",
    "\n",
    "                acc = float(np.sum(preds == true))/len(true)\n",
    "                class_accs = []\n",
    "                for class_idx in range(CLASSES_COUNT):\n",
    "                    class_acc = float(np.sum((preds == true) * (true == class_idx)))/float(np.sum(true == class_idx))\n",
    "                    class_accs.append(class_acc)\n",
    "                mean_class_acc = statistics.mean(class_accs)\n",
    "                num_ens_accs.append({'acc': acc, 'mean_class_acc': mean_class_acc, 'class_accs': class_accs})\n",
    "\n",
    "            num_ens_summary = {}\n",
    "            num_ens_summary['acc'] = get_stats_for_num_ens_param([x['acc'] for x in num_ens_accs])\n",
    "            num_ens_summary['mean_class_acc'] = get_stats_for_num_ens_param([x['mean_class_acc'] for x in num_ens_accs])\n",
    "            num_ens_summary['class_accs'] = []\n",
    "            for class_idx in range(CLASSES_COUNT):\n",
    "                num_ens_summary['class_accs'].append(get_stats_for_num_ens_param([x['class_accs'][class_idx] \n",
    "                                                                                  for x in num_ens_accs]))\n",
    "            accuracy_stats[model][num_ens] = num_ens_summary\n",
    "    return accuracy_stats\n",
    "\n",
    "font = {'family' : 'normal',\n",
    "        'weight' : 'normal',\n",
    "        'size'   : 16}\n",
    "\n",
    "plt.rc('font', **font)\n",
    "\n",
    "def plot_acc_in_num_models(ax, acc_stats, acc_type='acc', styles={}, h_factor=0.1, times=None, without_errorbar=False):\n",
    "    for model in acc_stats:\n",
    "        x = acc_stats[model].keys()\n",
    "        y = [acc_stats[model][i][acc_type]['mean'] for i in x]\n",
    "        if 'stddev' in acc_stats[model][1][acc_type] and not without_errorbar:\n",
    "            e = [acc_stats[model][i][acc_type]['stddev'] for i in x]\n",
    "        if times is not None:\n",
    "            x = [v * times[model] for v in x]\n",
    "        if 'stddev' in acc_stats[model][1][acc_type] and not without_errorbar:\n",
    "            ax.errorbar(x, y, e, label=styles[model]['label'], capsize=6, \n",
    "                               color=styles[model]['color'],\n",
    "                              linestyle=styles[model]['linestyle'])\n",
    "        else:\n",
    "            ax.plot(x, y, label=styles[model]['label'], \n",
    "                               color=styles[model]['color'],\n",
    "                              linestyle=styles[model]['linestyle'])\n",
    "       \n",
    "    if times is None:\n",
    "        ax.set_xlabel('Number of ensembled models')\n",
    "    else:\n",
    "        ax.set_xlabel('Inference time for batch of 4 point clouds [ms]')\n",
    "        \n",
    "    if acc_type == 'acc':\n",
    "        ax.set_ylabel('Instance classification accuracy')\n",
    "    elif acc_type == 'mean_class_acc':\n",
    "        ax.set_ylabel('Mean class accuracy')\n",
    "    box = ax.get_position()\n",
    "    ax.set_position([box.x0, box.y0 + box.height * h_factor,\n",
    "                     box.width, box.height * (1.0 - h_factor)])\n",
    "\n",
    "    # Show tick labels\n",
    "    for tk in ax.get_yticklabels():\n",
    "        tk.set_visible(True)\n",
    "        \n",
    "    return ax.get_legend_handles_labels()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SO-Net random factors analisys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "acc_stats_base_models = get_accuracy_stats(BASE_MODELS, 'activations')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "acc_stats_sonet_const = get_accuracy_stats(SONET_CONST_MODELS, 'activations')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Plain acc mean</th>\n",
       "      <th>Ensemble acc mean</th>\n",
       "      <th>Plain class acc mean</th>\n",
       "      <th>Ensemble class acc mean</th>\n",
       "      <th>Increase acc</th>\n",
       "      <th>Increase class acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>so-net_TTT</th>\n",
       "      <td>92.34%</td>\n",
       "      <td>93.19%</td>\n",
       "      <td>89.58%</td>\n",
       "      <td>90.96%</td>\n",
       "      <td>0.85%</td>\n",
       "      <td>1.38%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>so-net_FFT</th>\n",
       "      <td>92.20%</td>\n",
       "      <td>92.99%</td>\n",
       "      <td>89.62%</td>\n",
       "      <td>90.25%</td>\n",
       "      <td>0.79%</td>\n",
       "      <td>0.63%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>so-net_FTF</th>\n",
       "      <td>92.47%</td>\n",
       "      <td>93.35%</td>\n",
       "      <td>89.78%</td>\n",
       "      <td>90.51%</td>\n",
       "      <td>0.88%</td>\n",
       "      <td>0.73%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>so-net_TFF</th>\n",
       "      <td>92.49%</td>\n",
       "      <td>93.15%</td>\n",
       "      <td>89.84%</td>\n",
       "      <td>90.65%</td>\n",
       "      <td>0.66%</td>\n",
       "      <td>0.81%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>so-net_TTF</th>\n",
       "      <td>92.48%</td>\n",
       "      <td>93.11%</td>\n",
       "      <td>89.89%</td>\n",
       "      <td>90.36%</td>\n",
       "      <td>0.63%</td>\n",
       "      <td>0.48%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>so-net_TFT</th>\n",
       "      <td>92.50%</td>\n",
       "      <td>93.15%</td>\n",
       "      <td>90.10%</td>\n",
       "      <td>90.58%</td>\n",
       "      <td>0.65%</td>\n",
       "      <td>0.48%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>so-net_FTT</th>\n",
       "      <td>92.51%</td>\n",
       "      <td>93.07%</td>\n",
       "      <td>89.88%</td>\n",
       "      <td>90.40%</td>\n",
       "      <td>0.56%</td>\n",
       "      <td>0.52%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>so-net</th>\n",
       "      <td>92.65%</td>\n",
       "      <td>93.57%</td>\n",
       "      <td>89.98%</td>\n",
       "      <td>90.89%</td>\n",
       "      <td>0.92%</td>\n",
       "      <td>0.91%</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Plain acc mean  Ensemble acc mean  Plain class acc mean  \\\n",
       "so-net_TTT          92.34%             93.19%                89.58%   \n",
       "so-net_FFT          92.20%             92.99%                89.62%   \n",
       "so-net_FTF          92.47%             93.35%                89.78%   \n",
       "so-net_TFF          92.49%             93.15%                89.84%   \n",
       "so-net_TTF          92.48%             93.11%                89.89%   \n",
       "so-net_TFT          92.50%             93.15%                90.10%   \n",
       "so-net_FTT          92.51%             93.07%                89.88%   \n",
       "so-net              92.65%             93.57%                89.98%   \n",
       "\n",
       "            Ensemble class acc mean  Increase acc  Increase class acc  \n",
       "so-net_TTT                   90.96%         0.85%               1.38%  \n",
       "so-net_FFT                   90.25%         0.79%               0.63%  \n",
       "so-net_FTF                   90.51%         0.88%               0.73%  \n",
       "so-net_TFF                   90.65%         0.66%               0.81%  \n",
       "so-net_TTF                   90.36%         0.63%               0.48%  \n",
       "so-net_TFT                   90.58%         0.65%               0.48%  \n",
       "so-net_FTT                   90.40%         0.56%               0.52%  \n",
       "so-net                       90.89%         0.92%               0.91%  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = []\n",
    "for model in SONET_CONST_MODELS:\n",
    "    data.append([100 * acc_stats_sonet_const[model][1]['acc']['mean'],\n",
    "                100 * acc_stats_sonet_const[model][5]['acc']['mean'],\n",
    "                100 * acc_stats_sonet_const[model][1]['mean_class_acc']['mean'],\n",
    "                100 * acc_stats_sonet_const[model][5]['mean_class_acc']['mean']])\n",
    "model = 'so-net'\n",
    "data.append([100 * acc_stats_base_models[model][1]['acc']['mean'],\n",
    "                100 * acc_stats_base_models[model][5]['acc']['mean'],\n",
    "                100 * acc_stats_base_models[model][1]['mean_class_acc']['mean'],\n",
    "                100 * acc_stats_base_models[model][5]['mean_class_acc']['mean']])\n",
    "pd.options.display.float_format = '{:,.2f}%'.format\n",
    "df = pd.DataFrame(data, columns=['Plain acc mean', 'Ensemble acc mean', \n",
    "                                 'Plain class acc mean', 'Ensemble class acc mean'], \n",
    "                  index=SONET_CONST_MODELS + ['so-net'])\n",
    "df['Increase acc'] = df['Ensemble acc mean'] - df['Plain acc mean']\n",
    "df['Increase class acc'] = df['Ensemble class acc mean'] - df['Plain class acc mean']\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Plain acc mean</th>\n",
       "      <th>Ensemble acc mean</th>\n",
       "      <th>Plain class acc mean</th>\n",
       "      <th>Ensemble class acc mean</th>\n",
       "      <th>Increase acc</th>\n",
       "      <th>Increase class acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>so-net_FTT</th>\n",
       "      <td>92.51%</td>\n",
       "      <td>93.07%</td>\n",
       "      <td>89.88%</td>\n",
       "      <td>90.40%</td>\n",
       "      <td>0.56%</td>\n",
       "      <td>0.52%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>so-net_TTF</th>\n",
       "      <td>92.48%</td>\n",
       "      <td>93.11%</td>\n",
       "      <td>89.89%</td>\n",
       "      <td>90.36%</td>\n",
       "      <td>0.63%</td>\n",
       "      <td>0.48%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>so-net_TFT</th>\n",
       "      <td>92.50%</td>\n",
       "      <td>93.15%</td>\n",
       "      <td>90.10%</td>\n",
       "      <td>90.58%</td>\n",
       "      <td>0.65%</td>\n",
       "      <td>0.48%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>so-net_TFF</th>\n",
       "      <td>92.49%</td>\n",
       "      <td>93.15%</td>\n",
       "      <td>89.84%</td>\n",
       "      <td>90.65%</td>\n",
       "      <td>0.66%</td>\n",
       "      <td>0.81%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>so-net_FFT</th>\n",
       "      <td>92.20%</td>\n",
       "      <td>92.99%</td>\n",
       "      <td>89.62%</td>\n",
       "      <td>90.25%</td>\n",
       "      <td>0.79%</td>\n",
       "      <td>0.63%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>so-net_TTT</th>\n",
       "      <td>92.34%</td>\n",
       "      <td>93.19%</td>\n",
       "      <td>89.58%</td>\n",
       "      <td>90.96%</td>\n",
       "      <td>0.85%</td>\n",
       "      <td>1.38%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>so-net_FTF</th>\n",
       "      <td>92.47%</td>\n",
       "      <td>93.35%</td>\n",
       "      <td>89.78%</td>\n",
       "      <td>90.51%</td>\n",
       "      <td>0.88%</td>\n",
       "      <td>0.73%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>so-net_FFF</th>\n",
       "      <td>92.65%</td>\n",
       "      <td>93.57%</td>\n",
       "      <td>89.98%</td>\n",
       "      <td>90.89%</td>\n",
       "      <td>0.92%</td>\n",
       "      <td>0.91%</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Plain acc mean  Ensemble acc mean  Plain class acc mean  \\\n",
       "so-net_FTT          92.51%             93.07%                89.88%   \n",
       "so-net_TTF          92.48%             93.11%                89.89%   \n",
       "so-net_TFT          92.50%             93.15%                90.10%   \n",
       "so-net_TFF          92.49%             93.15%                89.84%   \n",
       "so-net_FFT          92.20%             92.99%                89.62%   \n",
       "so-net_TTT          92.34%             93.19%                89.58%   \n",
       "so-net_FTF          92.47%             93.35%                89.78%   \n",
       "so-net_FFF          92.65%             93.57%                89.98%   \n",
       "\n",
       "            Ensemble class acc mean  Increase acc  Increase class acc  \n",
       "so-net_FTT                   90.40%         0.56%               0.52%  \n",
       "so-net_TTF                   90.36%         0.63%               0.48%  \n",
       "so-net_TFT                   90.58%         0.65%               0.48%  \n",
       "so-net_TFF                   90.65%         0.66%               0.81%  \n",
       "so-net_FFT                   90.25%         0.79%               0.63%  \n",
       "so-net_TTT                   90.96%         0.85%               1.38%  \n",
       "so-net_FTF                   90.51%         0.88%               0.73%  \n",
       "so-net_FFF                   90.89%         0.92%               0.91%  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = []\n",
    "TOTAL_NUM_MODELS = 5\n",
    "for model in SONET_CONST_MODELS:\n",
    "    data.append([100 * acc_stats_sonet_const[model][1]['acc']['mean'],\n",
    "                 100 * acc_stats_sonet_const[model][TOTAL_NUM_MODELS]['acc']['mean'],\n",
    "                 100 * acc_stats_sonet_const[model][1]['mean_class_acc']['mean'],\n",
    "                 100 * acc_stats_sonet_const[model][TOTAL_NUM_MODELS]['mean_class_acc']['mean']])\n",
    "model = 'so-net'\n",
    "data.append([100 * acc_stats_base_models[model][1]['acc']['mean'],\n",
    "                 100 * acc_stats_base_models[model][TOTAL_NUM_MODELS]['acc']['mean'],\n",
    "                 100 * acc_stats_base_models[model][1]['mean_class_acc']['mean'],\n",
    "                 100 * acc_stats_base_models[model][TOTAL_NUM_MODELS]['mean_class_acc']['mean']])\n",
    "pd.options.display.float_format = '{:,.2f}%'.format\n",
    "df = pd.DataFrame(data, columns=['Plain acc mean', 'Ensemble acc mean', \n",
    "                                 'Plain class acc mean', 'Ensemble class acc mean'], \n",
    "                  index=SONET_CONST_MODELS + ['so-net_FFF'])\n",
    "df['Increase acc'] = df['Ensemble acc mean'] - df['Plain acc mean']\n",
    "df['Increase class acc'] = df['Ensemble class acc mean'] - df['Plain class acc mean']\n",
    "df.sort_values(by=['Increase acc'])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}