Titanic Modeling- CabinXYZ.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "# filter warnings\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "from sklearn.exceptions import DataConversionWarning\n",
    "warnings.filterwarnings(action='ignore')\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "gender_submission = pd.read_csv('titanic/gender_submission.csv')\n",
    "test_df = pd.read_csv('titanic/test.csv')\n",
    "train_df = pd.read_csv('titanic/train.csv')\n",
    "train_df_copy = train_df.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Z    687\n",
      "Y    184\n",
      "X     20\n",
      "Name: Cabin, dtype: int64\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x220783f2748>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAELCAYAAAA2mZrgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAE79JREFUeJzt3X+QXWd93/H3R3IVD6pJId5EGf3AmiJwVMfB8SKSOhNwsYnczlilECLFneAZgibTCNrSoBHTjEpFaaaChEkbQVASpy4dEI4zaTeZbZSCnbS4ONXaVkwkR3SRjbUSG9aYH4amyLK+/WOvTi5XV7tX9h7dlfR+zdzReZ7z3LNfzdXos+e55zwnVYUkSQBLhl2AJGnxMBQkSQ1DQZLUMBQkSQ1DQZLUMBQkSY1WQyHJxiRHkkwm2dFn/5ok9yd5JMmjSf5+m/VIkuaWtu5TSLIU+DxwKzAFHAC2VNXhrjF7gUeq6iNJ1gPjVXVNKwVJkubV5pnCBmCyqo5W1UlgH7CpZ0wBL+5sfzdwosV6JEnzuKLFY68EjnW1p4DX9Ix5L/BHSd4BLAduabEeSdI82gyF9OnrnavaAvzHqvrlJD8KfCzJdVV1+jsOlGwFtgIsX778xmuvvbaVgiXpUvXQQw89VVUj841rMxSmgNVd7VWcPT30NmAjQFV9NsmVwNXAl7sHVdVeYC/A6OhoTUxMtFWzJF2SknxxkHFtfqdwAFiXZG2SZcBmYKxnzJPA6wGS/ABwJTDTYk2SpDm0FgpVdQrYBuwHHgPuqapDSXYlub0z7F8Ab0/yZ8AngDvLZVslaWjanD6iqsaB8Z6+nV3bh4Gb2qxBkjQ472iWJDUMBUlSw1CQJDUMBUlSw1CQJDVavfpIulxs376d6elpVqxYwe7du4ddjvS8GQrSApienub48ePDLkN6wZw+kiQ1DAVJUsNQkCQ1DAVJUsNQkCQ1DAVJUsNQkCQ1DAVJUsNQkCQ1DAVJUsNQkCQ1Wg2FJBuTHEkymWRHn/0fSnKw8/p8kq+1WY8kaW6tLYiXZCmwB7gVmAIOJBnrPJcZgKr6513j3wHc0FY9kqT5tXmmsAGYrKqjVXUS2AdsmmP8FuATLdYjSZpHm6GwEjjW1Z7q9J0lycuAtcB9LdYjSZpHm6GQPn11jrGbgXur6rm+B0q2JplIMjEzM7NgBUqSvlOboTAFrO5qrwJOnGPsZuaYOqqqvVU1WlWjIyMjC1iiJKlbm6FwAFiXZG2SZcz+xz/WOyjJK4GXAJ9tsRZJ0gBau/qoqk4l2QbsB5YCd1XVoSS7gImqOhMQW4B9VXWuqSVdpp7c9YPDLmFgp55+KXAFp57+4kVV95qdnxt2CVpkWn1Gc1WNA+M9fTt72u9tswZJ0uC8o1mS1DAUJEkNQ0GS1DAUJEkNQ0GS1DAUJEkNQ0GS1DAUJEmNVm9ek6SLwfbt25menmbFihXs3r172OUMlaEg6bI3PT3N8ePHh13GouD0kSSpYShIkhqGgiSpYShIkhqGgiSpYShIkhqGgiSpYShIkhqGgiSp0WooJNmY5EiSySQ7zjHmLUkOJzmU5ONt1iNJmltry1wkWQrsAW4FpoADScaq6nDXmHXAe4CbquqrSb63rXqkNl195WngVOdP6eLV5tpHG4DJqjoKkGQfsAk43DXm7cCeqvoqQFV9ucV6pNb8wvVfG3YJ0oJoc/poJXCsqz3V6ev2CuAVSR5I8mCSjf0OlGRrkokkEzMzMy2VK0lqMxTSp6962lcA64DXAVuA30zyt856U9XeqhqtqtGRkZEFL1SSNKvNUJgCVne1VwEn+oz5r1X1bFU9DhxhNiQkSUPQZigcANYlWZtkGbAZGOsZ81+AmwGSXM3sdNLRFmuSJM2htVCoqlPANmA/8BhwT1UdSrIrye2dYfuBryQ5DNwPvLuqvtJWTZKkubX65LWqGgfGe/p2dm0X8K7OS5I0ZN7RLElqGAqSpIahIElqGAqSpIahIElqGAqSpIahIElqGAqSpIahIElqGAqSpIahIElqGAqSpIahIElqGAqSpIahIElqGAqSpIahIElqGAqSpEaroZBkY5IjSSaT7Oiz/84kM0kOdl4/22Y9kqS5tfaM5iRLgT3ArcAUcCDJWFUd7hn6yara1lYdkqTBtXmmsAGYrKqjVXUS2AdsavHnSZJeoDZDYSVwrKs91enr9aYkjya5N8nqFuuRJM2jzVBIn77qaf8+cE1VXQ98Cri774GSrUkmkkzMzMwscJmSpDPaDIUpoPs3/1XAie4BVfWVqvp2p/kbwI39DlRVe6tqtKpGR0ZGWilWktRuKBwA1iVZm2QZsBkY6x6Q5Pu7mrcDj7VYjyRpHq1dfVRVp5JsA/YDS4G7qupQkl3ARFWNAe9McjtwCngauLOteiRJ82stFACqahwY7+nb2bX9HuA9bdYgSRqcdzRLkhqGgiSpYShIkhpzfqeQ5BnOvregUVUvXvCKJElDM2coVNVVAJ0rhqaBjzF7U9odwFWtVydJuqAGnT76iar6cFU9U1XfqKqPAG9qszBJ0oU3aCg8l+SOJEuTLElyB/Bcm4VJki68QUPhp4G3AH/Zef1kp0+SdAkZ6Oa1qnoCl72WNKCb/sNNwy7hvCz72jKWsIRjXzt2UdX+wDseWPBjDnSmkOQVST6d5M877euT/OKCVyNJGqpBp49+g9nlKJ4FqKpHmV3gTpJ0CRk0FF5UVf+7p+/UQhcjSRquQUPhqSR/m86NbEneDHyptaokSUMx6CqpPw/sBa5Nchx4nNkb2CRJl5BBQ+GLVXVLkuXAkqp6ps2iJEnDMej00eNJ9gI/AnyzxXokSUM0aCi8EvgUs9NIjyf5tSQ/1l5ZkqRhGCgUquqvquqeqvpHwA3Ai4E/abUySdIFN/DzFJK8NsmHgYeBK5ld9mK+92xMciTJZJIdc4x7c5JKMjpoPZKkhTfQF81JHgcOAvcA766qbw3wnqXAHuBWYAo4kGSsqg73jLsKeCfwp+dZuyRpgQ169dEPVdU3zvPYG4DJqjoKkGQfs+snHe4Z9z5gN/AL53l8SdICm+/Ja9urajfw/iRnPYGtqt45x9tXAse62lPAa3qOfwOwuqr+IImhIElDNt+ZwmOdPyeex7HTp68JliRLgA8Bd857oGQrsBVgzZo1z6MUSdIg5nsc5+93Nh+tqkfO89hTwOqu9irgRFf7KuA64I+TAKwAxpLcXlXfEUJVtZfZO6oZHR095zOjJUkvzKBXH/1Kkr9I8r4kf2fA9xwA1iVZm2QZs6uqjp3ZWVVfr6qrq+qaqroGeBA4KxAkSRfOoPcp3Ay8DpgB9ib53HzPU6iqU8A2YD+z01D3VNWhJLuS3P7CypYktWHQq4+oqmng3ye5H9gO7AT+zTzvGQfGe/p2nmPs6watRZLUjkGfvPYDSd7befLarwH/i9nvCCRJl5BBzxR+G/gE8IaqOjHfYEnSxWneUOjcmfyFqvrVC1CPJGmI5p0+qqrngO/pXEEkSbqEDfyQHeCBJGNAs+5RVf1KK1VJkoZi0FA40XktYfamM0nSJWigUKiqf912IZKk4Rt06ez76Vq36Iyq+nsLXpEkaWgGnT7qXsH0SuBNwKmFL0eSNEyDTh891NP1QBIfxylJl5hBp49e2tVcAowyu6qpJOkSMuj00UP89XcKp4AngLe1UdDlbPv27UxPT7NixQp279497HIkXYbme/Laq4FjVbW2034rs98nPMHZj9XUCzQ9Pc3x48eHXYaky9h8dzR/FDgJkOTHgV8C7ga+TuehN5KkS8d800dLq+rpzvZPAXur6neB301ysN3SJEkX2nxnCkuTnAmO1wP3de0b+FkMkqSLw3z/sX8C+JMkTwF/BfxPgCQvZ3YKSZJ0CZkzFKrq/Uk+DXw/8EdVdeYKpCXAO9ouTpJ0YQ2ydPaDVfV7VdW9Ournq+rh+d6bZGOSI0kmk+zos//nOs97PpjkM0nWn/9fQZK0UAZ6HOfz0Xk4zx7gNmA9sKXPf/ofr6ofrKpXAbsBl+KWpCFqLRSADcBkVR2tqpPAPmBT94Cq+kZXczl9Ft2TJF04bV5BtBI41tWeAl7TOyjJzwPvApYBrroqSUPUZiikT1+/5bf3AHuS/DTwi8BbzzpQshXYCrBmzZqBC7jx3f9p4LGLwVVPPcNS4Mmnnrmoan/oAz8z7BIkLZA2p4+mgNVd7VXMPr3tXPYB/7DfjqraW1WjVTU6MjKygCVKkrq1GQoHgHVJ1iZZBmwGxroHJFnX1fwHwP9psR5J0jxamz6qqlNJtgH7gaXAXVV1KMkuYKKqxoBtSW4BngW+Sp+pI0nShdPqUhVVNQ6M9/Tt7Nr+p23+fEnS+Wlz+kiSdJExFCRJDUNBktQwFCRJDUNBktQwFCRJDZ+etoicXrb8O/6UpAvNUFhEvrXuDcMuQdJlzukjSVLDUJAkNZw+knTZqxcVpzlNvcjnfBkKki57z9707LBLWDScPpIkNQwFSVLDUJAkNQwFSVLDUJAkNQwFSVLDUJAkNVoNhSQbkxxJMplkR5/970pyOMmjST6d5GVt1iNJmltroZBkKbAHuA1YD2xJsr5n2CPAaFVdD9wL7G6rHknS/No8U9gATFbV0ao6CewDNnUPqKr7q+r/dpoPAqtarEeSNI82Q2ElcKyrPdXpO5e3Af+t344kW5NMJJmYmZlZwBIlSd3aDIX06eu72lSSfwyMAh/ot7+q9lbVaFWNjoyMLGCJkqRubS6INwWs7mqvAk70DkpyC/AvgddW1bdbrEeSNI82zxQOAOuSrE2yDNgMjHUPSHID8FHg9qr6cou1SJIG0FooVNUpYBuwH3gMuKeqDiXZleT2zrAPAH8T+J0kB5OMneNwkqQLoNXnKVTVODDe07eza/uWNn++JOn8eEezJKlhKEiSGoaCJKlhKEiSGoaCJKlhKEiSGoaCJKlhKEiSGoaCJKlhKEiSGoaCJKlhKEiSGoaCJKlhKEiSGoaCJKlhKEiSGoaCJKlhKEiSGq2GQpKNSY4kmUyyo8/+H0/ycJJTSd7cZi2SpPm1FgpJlgJ7gNuA9cCWJOt7hj0J3Al8vK06JEmDu6LFY28AJqvqKECSfcAm4PCZAVX1RGff6RbrkCQNqM3po5XAsa72VKdPkrRItRkK6dNXz+tAydYkE0kmZmZmXmBZkqRzaTMUpoDVXe1VwInnc6Cq2ltVo1U1OjIysiDFSZLO1mYoHADWJVmbZBmwGRhr8edJkl6g1kKhqk4B24D9wGPAPVV1KMmuJLcDJHl1kingJ4GPJjnUVj2SpPm1efURVTUOjPf07ezaPsDstJIkaRHwjmZJUsNQkCQ1DAVJUsNQkCQ1DAVJUsNQkCQ1DAVJUsNQkCQ1DAVJUsNQkCQ1DAVJUsNQkCQ1DAVJUsNQkCQ1DAVJUsNQkCQ1DAVJUsNQkCQ1Wg2FJBuTHEkymWRHn/3fleSTnf1/muSaNuuRJM2ttVBIshTYA9wGrAe2JFnfM+xtwFer6uXAh4B/11Y9kqT5tXmmsAGYrKqjVXUS2Ads6hmzCbi7s30v8PokabEmSdIc2gyFlcCxrvZUp6/vmKo6BXwd+J4Wa5IkzeGKFo/d7zf+eh5jSLIV2NppfjPJkRdY22J2NfDUsIs4H/ngW4ddwmJx0X12/CtPzLtcdJ9f3nlen9/LBhnUZihMAau72quAE+cYM5XkCuC7gad7D1RVe4G9LdW5qCSZqKrRYdeh8+dnd3Hz85vV5vTRAWBdkrVJlgGbgbGeMWPAmV8z3wzcV1VnnSlIki6M1s4UqupUkm3AfmApcFdVHUqyC5ioqjHgt4CPJZlk9gxhc1v1SJLmF38xX1ySbO1Ml+ki42d3cfPzm2UoSJIaLnMhSWoYCotAkjcmOdjzOp3ktmHXprll1me6P6skb0nyh8OsS4NJsjrJ40le2mm/pNMe6PLNS5HTR4tQ576MO4Cbq+r0sOvR3JJcB/wOcAOzF1UcBDZW1ReGWpgGkmQ78PKq2prko8ATVfVLw65rWAyFRSbJK4D7gL9bVU8Oux4NJslu4FvAcuCZqnrfkEvSgJL8DeAh4C7g7cANnaV5LkuGwiLS+cf5WeCDVbVv2PVocEmWAw8DJ4HRqvr2kEvSeUjyE8AfAm+oqv8+7HqGqc07mnX+3gccMhAuPlX1rSSfBL5pIFyUbgO+BFwHGAoaviSvA94E/PCQS9Hzd7rz0kUkyauAW4EfAT6TZF9VfWnIZQ2NVx8tAkleAvw28DNV9cyw65EuF52l+j8C/LPOd3gfAD443KqGy1BYHH4O+F7gIz2Xpf7UsAuTLnFvB57s+h7hw8C1SV47xJqGyi+aJUkNzxQkSQ1DQZLUMBQkSQ1DQZLUMBQkSQ1DQTqHJCuS7EvyhSSHk4x31qbqN/aaJH9+jn2/mWR9u9VKC8M7mqU+Ojc1/R5wd1Vt7vS9Cvg+4PPnc6yq+tmFr1Bqh2cKUn83A89W1a+f6aiqg8AjST6d5OEkn0uyqes9VyS5O8mjSe5N8iKAJH+cZLSz/c0k70/yZ0keTPJ9F/RvJc3DUJD6u47Z5ZR7/T/gjVX1w8wGxy93zioAXgnsrarrgW8A/6TP+5cDD1bVDwH/g9k7aqVFw1CQzk+Af5vkUeBTwEpmp5QAjlXVA53t/wz8WJ/3nwT+oLP9EHBNe6VK589QkPo7BNzYp/8OYAS4sapeBfwlcGVnX++aMf3WkHm2/nptmefwez0tMoaC1N99wHclaaZ3krwaeBnw5ap6NsnNnfYZa5L8aGd7C/CZC1attEAMBamPzm/zbwRu7VySegh4LzAOjCaZYPas4S+63vYY8NbO1NJLmV2SWbqouEqqJKnhmYIkqWEoSJIahoIkqWEoSJIahoIkqWEoSJIahoIkqWEoSJIa/x+OxVcr6sgXhQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "a = (train_df.Cabin\n",
    "     .apply(lambda x: x[0] if pd.notna(x) else 'Z')\n",
    "     .apply(lambda x: 'Y' if x in ['C','E','D','B','F'] else x)\n",
    "     .apply(lambda x: 'X' if x in ['G','A','T'] else x))\n",
    "\n",
    "print(a.value_counts())\n",
    "sns.barplot(x=a, y=train_df.Survived)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Cleaning and Encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import patsy as pts\n",
    "\n",
    "def data_cleaning(df):\n",
    "    # PassengerId\n",
    "    df.drop(['PassengerId'],axis=1,inplace=True)\n",
    "\n",
    "    # Age\n",
    "    for ind in df[df.Age.isnull()].index:\n",
    "        # filters\n",
    "        filter1 = df['Sex'] == df.loc[ind, 'Sex']\n",
    "        filter2 = df['Pclass'] == df.loc[ind, 'Pclass']\n",
    "        filter3 = df['SibSp'] == df.loc[ind, 'SibSp']\n",
    "        filter4 = df['Parch'] == df.loc[ind, 'Parch']\n",
    "        fill_value = df[filter1][filter2][filter3][filter4]['Age'].median()\n",
    "\n",
    "        # if filter result is nan, we fill with the global median\n",
    "        if pd.isna(fill_value):\n",
    "            fill_value = df['Age'].median()\n",
    "\n",
    "        # fill in values\n",
    "        df.loc[ind, 'Age'] = fill_value\n",
    "\n",
    "    # Cabin\n",
    "    df['Cabin'] = (df.Cabin\n",
    "     .apply(lambda x: x[0] if pd.notna(x) else 'Z')\n",
    "     .apply(lambda x: 'Y' if x in ['C','E','D','B','F'] else x)\n",
    "     .apply(lambda x: 'X' if x in ['G','A','T'] else x))\n",
    "\n",
    "    # Embarked\n",
    "    df.Embarked.fillna(df['Embarked'].mode()[0],inplace=True)\n",
    "\n",
    "    # title\n",
    "    df['Title'] = df.Name.apply(lambda x: x.split(', ')[1].split('. ')[0])\n",
    "    df[\"Title\"] = df[\"Title\"].replace(['Lady', 'the Countess','Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')\n",
    "    df[\"Title\"] = df[\"Title\"].map({\"Master\":\"Master\", \"Miss\":\"Female\", \"Ms\" : \"Female\" , \"Mme\":\"Female\", \"Mlle\":\"Female\", \"Mrs\":\"Female\", \"Mr\":\"Male\", \"Rare\":\"Rare\"})\n",
    "\n",
    "    # family size and alone status\n",
    "    df['FamilySize'] = df['Parch'] + df['SibSp'] + 1\n",
    "    df['IsAlone'] = df['FamilySize'].apply(lambda x: 1 if x == 1 else 0)\n",
    "    \n",
    "    \n",
    "    # data encoding\n",
    "    y, X = pts.dmatrices('Survived ~ Pclass + C(Sex) + Age + SibSp + Parch + Fare + ' +\n",
    "                        'C(Embarked) + C(Title) + C(Cabin) + FamilySize + IsAlone', data=df,\n",
    "                        return_type='dataframe')\n",
    "    X.columns = [i.replace('[','').replace(']','') for i in X.columns]\n",
    "    pd.concat([X,y]).info()\n",
    "\n",
    "    return X, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 1782 entries, 0 to 890\n",
      "Data columns (total 17 columns):\n",
      "Age                 891 non-null float64\n",
      "C(Cabin)T.Y         891 non-null float64\n",
      "C(Cabin)T.Z         891 non-null float64\n",
      "C(Embarked)T.Q      891 non-null float64\n",
      "C(Embarked)T.S      891 non-null float64\n",
      "C(Sex)T.male        891 non-null float64\n",
      "C(Title)T.Male      891 non-null float64\n",
      "C(Title)T.Master    891 non-null float64\n",
      "C(Title)T.Rare      891 non-null float64\n",
      "FamilySize          891 non-null float64\n",
      "Fare                891 non-null float64\n",
      "Intercept           891 non-null float64\n",
      "IsAlone             891 non-null float64\n",
      "Parch               891 non-null float64\n",
      "Pclass              891 non-null float64\n",
      "SibSp               891 non-null float64\n",
      "Survived            891 non-null float64\n",
      "dtypes: float64(17)\n",
      "memory usage: 250.6 KB\n"
     ]
    }
   ],
   "source": [
    "# clean it in one line\n",
    "X, y = data_cleaning(train_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Modeling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "from sklearn import ensemble, linear_model, svm, naive_bayes, discriminant_analysis, neighbors, tree\n",
    "from sklearn.preprocessing import normalize\n",
    "from sklearn.model_selection import GridSearchCV, cross_val_score, StratifiedKFold, learning_curve\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Naive Bayes Parameters tunning \n",
    "# NBC = naive_bayes.BernoulliNB()\n",
    "# nb_param_grid = {\n",
    "#                     'alpha': range(1,100,20),\n",
    "#                     'fit_prior': [True,False]}\n",
    "\n",
    "\n",
    "# RFC Parameters tunning \n",
    "RFC = ensemble.RandomForestClassifier()\n",
    "rf_param_grid = {\"max_depth\": [None],\n",
    "              \"max_features\": [1, 3, 10],\n",
    "              \"min_samples_split\": [3, 10],\n",
    "              \"min_samples_leaf\": [1, 3, 10],\n",
    "              \"bootstrap\": [False],\n",
    "              \"n_estimators\" :[100,300],\n",
    "              \"criterion\": [\"gini\"]}\n",
    "\n",
    "\n",
    "# XGB Parameters tunning, reference\n",
    "# https://www.analyticsvidhya.com/blog/2016/03/complete-guide-parameter-tuning-xgboost-with-codes-python/\n",
    "xgb = XGBClassifier()\n",
    "xgb_param_grid = {\n",
    "                'max_depth':[4,6],\n",
    "                'min_child_weight':[4,6],\n",
    "                'gamma':[i/10.0 for i in range(0,5,2)],\n",
    "                'subsample':[i/10.0 for i in range(6,10,2)],\n",
    "                'colsample_bytree':[i/10.0 for i in range(6,10,2)],\n",
    "                'reg_alpha':[1e-2, 0.1, 1],\n",
    "}\n",
    "\n",
    "# DA Parameters tunning \n",
    "DAC = discriminant_analysis.LinearDiscriminantAnalysis()\n",
    "dac_param_grid = {\n",
    "                'solver':['svd','lsqr'],\n",
    "                'n_components':[i for i in range(0,5,1)],\n",
    "}\n",
    "\n",
    "# LRC tunning \n",
    "LRC = linear_model.LogisticRegression()\n",
    "lrc_param_grid = {\n",
    "    'C': [0.1,1,10],\n",
    "    'solver':['newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga'],\n",
    "}\n",
    "\n",
    "# GB tunning \n",
    "GBC = ensemble.gradient_boosting.GradientBoostingClassifier()\n",
    "gbc_param_grid = {\n",
    "              'learning_rate': [0.1, 0.05],\n",
    "              'max_depth': [5,6],\n",
    "              'min_samples_leaf': [31],\n",
    "              'n_estimators':[30,70],\n",
    "              'max_features':range(4,8,4),\n",
    "              'subsample':[0.6,0.75]\n",
    "              }\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# put all models into one box\n",
    "dic = {\n",
    "# 'NBC':[NBC,nb_param_grid],\n",
    "'RFC':[RFC,rf_param_grid],\n",
    "'XGB':[xgb,xgb_param_grid],\n",
    "'DAC':[DAC,dac_param_grid],\n",
    "'LRC':[LRC,lrc_param_grid],\n",
    "'GBC':[GBC,gbc_param_grid]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "kfold = StratifiedKFold(n_splits=10)\n",
    "\n",
    "def para_tuning(dic, X, y, n_jobs=4):\n",
    "    '''dictionary format should be:\n",
    "    { <Name of Model> : [<model>, <parameter grid>]}\n",
    "    '''\n",
    "    model_ls = []\n",
    "    \n",
    "    for model in dic:\n",
    "        # grid search cross validation for hyper-parameter tunings\n",
    "        gs = GridSearchCV(dic[model][0],param_grid = dic[model][1], cv=kfold, scoring=\"accuracy\", n_jobs = n_jobs, verbose = 1)\n",
    "        gs.fit(X,y)\n",
    "\n",
    "        # this is the best classifier\n",
    "        model_ls.append([model,gs.best_estimator_])\n",
    "    \n",
    "    return model_ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 10 folds for each of 36 candidates, totalling 360 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    3.7s\n",
      "[Parallel(n_jobs=4)]: Done 192 tasks      | elapsed:   12.9s\n",
      "[Parallel(n_jobs=4)]: Done 360 out of 360 | elapsed:   26.9s finished\n",
      "[Parallel(n_jobs=4)]: Using backend LokyBackend with 4 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 10 folds for each of 144 candidates, totalling 1440 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    1.9s\n",
      "[Parallel(n_jobs=4)]: Done 192 tasks      | elapsed:    7.4s\n",
      "[Parallel(n_jobs=4)]: Done 442 tasks      | elapsed:   16.8s\n",
      "[Parallel(n_jobs=4)]: Done 792 tasks      | elapsed:   29.9s\n",
      "[Parallel(n_jobs=4)]: Done 1242 tasks      | elapsed:   48.1s\n",
      "[Parallel(n_jobs=4)]: Done 1440 out of 1440 | elapsed:   56.1s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 10 folds for each of 10 candidates, totalling 100 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done 100 out of 100 | elapsed:    0.2s finished\n",
      "[Parallel(n_jobs=4)]: Using backend LokyBackend with 4 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 10 folds for each of 15 candidates, totalling 150 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Done 150 out of 150 | elapsed:    1.5s finished\n",
      "[Parallel(n_jobs=4)]: Using backend LokyBackend with 4 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 10 folds for each of 16 candidates, totalling 160 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Done 160 out of 160 | elapsed:    2.1s finished\n"
     ]
    }
   ],
   "source": [
    "# tune features in one line.. just kidding... we need all the above lines\n",
    "model_ls = para_tuning(dic,X,y,4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import VotingClassifier\n",
    "# this is the voting classifier\n",
    "voting_clf = VotingClassifier(estimators=model_ls, voting='soft')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cross validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 10 fold cross validation\n",
    "def cv_test(clf,X,y):\n",
    "    cv_scores = cross_val_score(clf, X, y = y, scoring = \"accuracy\", cv = kfold, n_jobs=4)\n",
    "    score = cv_scores.mean()\n",
    "    standard_deviation = cv_scores.std()\n",
    "    print(\"{:.4%}\".format(score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RFC\n",
      "82.4965%\n",
      "XGB\n",
      "84.2880%\n",
      "DAC\n",
      "82.8298%\n",
      "LRC\n",
      "82.9448%\n",
      "GBC\n",
      "83.0596%\n",
      "Voting Classifier\n",
      "83.7300%\n"
     ]
    }
   ],
   "source": [
    "# test models\n",
    "for model in model_ls:\n",
    "    print(model[0])\n",
    "    cv_test(model[1],X,y)\n",
    "\n",
    "# this voting classifier does not like to be put into a list...\n",
    "# it will break down the whole kernel if you do it...\n",
    "print('Voting Classifier')\n",
    "cv_test(voting_clf,X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RFC\n",
      "82.6746%\n",
      "XGB\n",
      "83.2814%\n",
      "DAC\n",
      "82.8102%\n",
      "LRC\n",
      "82.6407%\n",
      "GBC\n",
      "82.6814%\n",
      "voting\n",
      "83.3085%\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# 20 times train test split and take the mean.\n",
    "# validation set :)\n",
    "\n",
    "dic={}\n",
    "\n",
    "# --------------- loop --------------- #\n",
    "for model in model_ls:\n",
    "    for i in range(20):\n",
    "        X_train, X_test, y_train, y_test = \\\n",
    "            train_test_split(X, y, test_size=0.33)\n",
    "        \n",
    "        for model in model_ls:\n",
    "            model[1].fit(X_train,y_train)\n",
    "            sc = model[1].score(X_test,y_test)\n",
    "        \n",
    "            if model[0] not in dic.keys():\n",
    "                dic[model[0]] = [sc]\n",
    "            elif model[0] in dic.keys():\n",
    "                dic[model[0]].append(sc)\n",
    "            else:\n",
    "                print('what?')\n",
    "        \n",
    "        voting_clf.fit(X_train,y_train)\n",
    "        sc = voting_clf.score(X_test,y_test)\n",
    "        if 'voting' not in dic.keys():\n",
    "            dic['voting'] = [sc]\n",
    "        elif 'voting' in dic.keys():\n",
    "            dic['voting'].append(sc)\n",
    "        else:\n",
    "            print('what?')\n",
    "# --------------- loop --------------- #\n",
    "            \n",
    "import numpy as np\n",
    "for i in dic.keys():\n",
    "    print(i)\n",
    "    print(\"{:.4%}\".format(np.mean(dic[i])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": true,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}