Titanic Modeling-Cabin + delete size and alone.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "# filter warnings\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "from sklearn.exceptions import DataConversionWarning\n",
    "warnings.filterwarnings(action='ignore')\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "gender_submission = pd.read_csv('titanic/gender_submission.csv')\n",
    "test_df = pd.read_csv('titanic/test.csv')\n",
    "train_df = pd.read_csv('titanic/train.csv')\n",
    "train_df_copy = train_df.copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Cleaning and Encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import patsy as pts\n",
    "\n",
    "def data_cleaning(df):\n",
    "    # PassengerId\n",
    "    df.drop(['PassengerId'],axis=1,inplace=True)\n",
    "\n",
    "    # Age\n",
    "    for ind in df[df.Age.isnull()].index:\n",
    "        # filters\n",
    "        filter1 = df['Sex'] == df.loc[ind, 'Sex']\n",
    "        filter2 = df['Pclass'] == df.loc[ind, 'Pclass']\n",
    "        filter3 = df['SibSp'] == df.loc[ind, 'SibSp']\n",
    "        filter4 = df['Parch'] == df.loc[ind, 'Parch']\n",
    "        fill_value = df[filter1][filter2][filter3][filter4]['Age'].median()\n",
    "\n",
    "        # if filter result is nan, we fill with the global median\n",
    "        if pd.isna(fill_value):\n",
    "            fill_value = df['Age'].median()\n",
    "\n",
    "        # fill in values\n",
    "        df.loc[ind, 'Age'] = fill_value\n",
    "\n",
    "    # Cabin\n",
    "    df['Cabin'] = (df.Cabin\n",
    "     .apply(lambda x: x[0] if pd.notna(x) else 'Z')\n",
    "     .apply(lambda x: 'Y' if x in ['C','E','D','B','F'] else x)\n",
    "     .apply(lambda x: 'X' if x in ['G','A','T'] else x))\n",
    "    \n",
    "    # Embarked\n",
    "    df.Embarked.fillna(df['Embarked'].mode()[0],inplace=True)\n",
    "\n",
    "    # title\n",
    "    df['Title'] = df.Name.apply(lambda x: x.split(', ')[1].split('. ')[0])\n",
    "    df[\"Title\"] = df[\"Title\"].replace(['Lady', 'the Countess','Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')\n",
    "    df[\"Title\"] = df[\"Title\"].map({\"Master\":\"Master\", \"Miss\":\"Female\", \"Ms\" : \"Female\" , \"Mme\":\"Female\", \"Mlle\":\"Female\", \"Mrs\":\"Female\", \"Mr\":\"Male\", \"Rare\":\"Rare\"})\n",
    "\n",
    "    # family size and alone status\n",
    "    df['FamilySize'] = df['Parch'] + df['SibSp'] + 1\n",
    "    df['IsAlone'] = df['FamilySize'].apply(lambda x: 1 if x == 1 else 0)\n",
    "    \n",
    "    \n",
    "    # data encoding\n",
    "    y, X = pts.dmatrices('Survived ~ Pclass + C(Sex) + Age + SibSp + Parch + Fare + ' +\n",
    "                        'C(Embarked) + C(Title)', data=df,\n",
    "                        return_type='dataframe')\n",
    "    X.columns = [i.replace('[','').replace(']','') for i in X.columns]\n",
    "    pd.concat([X,y]).info()\n",
    "\n",
    "    return X, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 1782 entries, 0 to 890\n",
      "Data columns (total 13 columns):\n",
      "Age                 891 non-null float64\n",
      "C(Embarked)T.Q      891 non-null float64\n",
      "C(Embarked)T.S      891 non-null float64\n",
      "C(Sex)T.male        891 non-null float64\n",
      "C(Title)T.Male      891 non-null float64\n",
      "C(Title)T.Master    891 non-null float64\n",
      "C(Title)T.Rare      891 non-null float64\n",
      "Fare                891 non-null float64\n",
      "Intercept           891 non-null float64\n",
      "Parch               891 non-null float64\n",
      "Pclass              891 non-null float64\n",
      "SibSp               891 non-null float64\n",
      "Survived            891 non-null float64\n",
      "dtypes: float64(13)\n",
      "memory usage: 194.9 KB\n"
     ]
    }
   ],
   "source": [
    "# clean it in one line\n",
    "X, y = data_cleaning(train_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Modeling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "from sklearn import ensemble, linear_model, svm, naive_bayes, discriminant_analysis, neighbors, tree\n",
    "from sklearn.preprocessing import normalize\n",
    "from sklearn.model_selection import GridSearchCV, cross_val_score, StratifiedKFold, learning_curve\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Naive Bayes Parameters tunning \n",
    "# NBC = naive_bayes.BernoulliNB()\n",
    "# nb_param_grid = {\n",
    "#                     'alpha': range(1,100,20),\n",
    "#                     'fit_prior': [True,False]}\n",
    "\n",
    "\n",
    "# RFC Parameters tunning \n",
    "RFC = ensemble.RandomForestClassifier()\n",
    "rf_param_grid = {\"max_depth\": [None],\n",
    "              \"max_features\": [1, 3, 10],\n",
    "              \"min_samples_split\": [3, 10],\n",
    "              \"min_samples_leaf\": [1, 3, 10],\n",
    "              \"bootstrap\": [False],\n",
    "              \"n_estimators\" :[100,300],\n",
    "              \"criterion\": [\"gini\"]}\n",
    "\n",
    "\n",
    "# XGB Parameters tunning, reference\n",
    "# https://www.analyticsvidhya.com/blog/2016/03/complete-guide-parameter-tuning-xgboost-with-codes-python/\n",
    "xgb = XGBClassifier()\n",
    "xgb_param_grid = {\n",
    "                'max_depth':[4,6],\n",
    "                'min_child_weight':[4,6],\n",
    "                'gamma':[i/10.0 for i in range(0,5,2)],\n",
    "                'subsample':[i/10.0 for i in range(6,10,2)],\n",
    "                'colsample_bytree':[i/10.0 for i in range(6,10,2)],\n",
    "                'reg_alpha':[1e-2, 0.1, 1],\n",
    "}\n",
    "\n",
    "# DA Parameters tunning \n",
    "DAC = discriminant_analysis.LinearDiscriminantAnalysis()\n",
    "dac_param_grid = {\n",
    "                'solver':['svd','lsqr'],\n",
    "                'n_components':[i for i in range(0,5,1)],\n",
    "}\n",
    "\n",
    "# LRC tunning \n",
    "LRC = linear_model.LogisticRegression()\n",
    "lrc_param_grid = {\n",
    "    'C': [0.1,1,10],\n",
    "    'solver':['newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga'],\n",
    "}\n",
    "\n",
    "# GB tunning \n",
    "GBC = ensemble.gradient_boosting.GradientBoostingClassifier()\n",
    "gbc_param_grid = {\n",
    "              'learning_rate': [0.1, 0.05],\n",
    "              'max_depth': [5,6],\n",
    "              'min_samples_leaf': [31],\n",
    "              'n_estimators':[30,70],\n",
    "              'max_features':range(4,8,4),\n",
    "              'subsample':[0.6,0.75]\n",
    "              }\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# put all models into one box\n",
    "dic = {\n",
    "# 'NBC':[NBC,nb_param_grid],\n",
    "'RFC':[RFC,rf_param_grid],\n",
    "'XGB':[xgb,xgb_param_grid],\n",
    "'DAC':[DAC,dac_param_grid],\n",
    "'LRC':[LRC,lrc_param_grid],\n",
    "'GBC':[GBC,gbc_param_grid]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "kfold = StratifiedKFold(n_splits=10)\n",
    "\n",
    "def para_tuning(dic, X, y, n_jobs=4):\n",
    "    '''dictionary format should be:\n",
    "    { <Name of Model> : [<model>, <parameter grid>]}\n",
    "    '''\n",
    "    model_ls = []\n",
    "    \n",
    "    for model in dic:\n",
    "        # grid search cross validation for hyper-parameter tunings\n",
    "        gs = GridSearchCV(dic[model][0],param_grid = dic[model][1], cv=kfold, scoring=\"accuracy\", n_jobs = n_jobs, verbose = 1)\n",
    "        gs.fit(X,y)\n",
    "\n",
    "        # this is the best classifier\n",
    "        model_ls.append([model,gs.best_estimator_])\n",
    "    \n",
    "    return model_ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 10 folds for each of 36 candidates, totalling 360 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    3.6s\n",
      "[Parallel(n_jobs=4)]: Done 192 tasks      | elapsed:   12.3s\n",
      "[Parallel(n_jobs=4)]: Done 360 out of 360 | elapsed:   25.7s finished\n",
      "[Parallel(n_jobs=4)]: Using backend LokyBackend with 4 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 10 folds for each of 144 candidates, totalling 1440 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    1.7s\n",
      "[Parallel(n_jobs=4)]: Done 192 tasks      | elapsed:    6.4s\n",
      "[Parallel(n_jobs=4)]: Done 442 tasks      | elapsed:   14.5s\n",
      "[Parallel(n_jobs=4)]: Done 792 tasks      | elapsed:   26.0s\n",
      "[Parallel(n_jobs=4)]: Done 1242 tasks      | elapsed:   42.4s\n",
      "[Parallel(n_jobs=4)]: Done 1440 out of 1440 | elapsed:   49.6s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 10 folds for each of 10 candidates, totalling 100 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done 100 out of 100 | elapsed:    0.1s finished\n",
      "[Parallel(n_jobs=4)]: Using backend LokyBackend with 4 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 10 folds for each of 15 candidates, totalling 150 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Done 150 out of 150 | elapsed:    1.4s finished\n",
      "[Parallel(n_jobs=4)]: Using backend LokyBackend with 4 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 10 folds for each of 16 candidates, totalling 160 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Done 160 out of 160 | elapsed:    2.1s finished\n"
     ]
    }
   ],
   "source": [
    "# tune features in one line.. just kidding... we need all the above lines\n",
    "model_ls = para_tuning(dic,X,y,4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import VotingClassifier\n",
    "# this is the voting classifier\n",
    "voting_clf = VotingClassifier(estimators=model_ls, voting='soft')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cross validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 10 fold cross validation\n",
    "def cv_test(clf,X,y):\n",
    "    cv_scores = cross_val_score(clf, X, y = y, scoring = \"accuracy\", cv = kfold, n_jobs=4)\n",
    "    score = cv_scores.mean()\n",
    "    standard_deviation = cv_scores.std()\n",
    "    print(\"{:.4%}\".format(score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RFC\n",
      "83.1669%\n",
      "XGB\n",
      "84.1832%\n",
      "DAC\n",
      "82.9447%\n",
      "LRC\n",
      "83.0546%\n",
      "GBC\n",
      "82.7149%\n",
      "Voting Classifier\n",
      "83.9547%\n"
     ]
    }
   ],
   "source": [
    "# test models\n",
    "for model in model_ls:\n",
    "    print(model[0])\n",
    "    cv_test(model[1],X,y)\n",
    "\n",
    "# this voting classifier does not like to be put into a list...\n",
    "# it will break down the whole kernel if you do it...\n",
    "print('Voting Classifier')\n",
    "cv_test(voting_clf,X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RFC\n",
      "82.8542%\n",
      "XGB\n",
      "82.5322%\n",
      "DAC\n",
      "82.5390%\n",
      "LRC\n",
      "82.7763%\n",
      "GBC\n",
      "81.7153%\n",
      "voting\n",
      "83.2136%\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# 20 times train test split and take the mean.\n",
    "# validation set :)\n",
    "\n",
    "dic={}\n",
    "\n",
    "# --------------- loop --------------- #\n",
    "for model in model_ls:\n",
    "    for i in range(20):\n",
    "        X_train, X_test, y_train, y_test = \\\n",
    "            train_test_split(X, y, test_size=0.33)\n",
    "        \n",
    "        for model in model_ls:\n",
    "            model[1].fit(X_train,y_train)\n",
    "            sc = model[1].score(X_test,y_test)\n",
    "        \n",
    "            if model[0] not in dic.keys():\n",
    "                dic[model[0]] = [sc]\n",
    "            elif model[0] in dic.keys():\n",
    "                dic[model[0]].append(sc)\n",
    "            else:\n",
    "                print('what?')\n",
    "        \n",
    "        voting_clf.fit(X_train,y_train)\n",
    "        sc = voting_clf.score(X_test,y_test)\n",
    "        if 'voting' not in dic.keys():\n",
    "            dic['voting'] = [sc]\n",
    "        elif 'voting' in dic.keys():\n",
    "            dic['voting'].append(sc)\n",
    "        else:\n",
    "            print('what?')\n",
    "# --------------- loop --------------- #\n",
    "            \n",
    "import numpy as np\n",
    "for i in dic.keys():\n",
    "    print(i)\n",
    "    print(\"{:.4%}\".format(np.mean(dic[i])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": true,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}