All Features Training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import gc; gc.enable()\n",
    "import lightgbm as lgb\n",
    "import warnings \n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "from GridSearcher import data_loader, model_loader, fit_params, get_oof_predictions\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "DATA_TYPE = 'sparse'\n",
    "DATA_NORMALIZE = True\n",
    "IS_REGRESSION = True\n",
    "SEED = 519"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Arguments: sparse True True\n",
      "target loaded\n"
     ]
    }
   ],
   "source": [
    "dl = data_loader(data_type=DATA_TYPE, is_regression=IS_REGRESSION, is_train=True, is_pure=True)\n",
    "train_X, train_y = dl.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Arguments: sparse True False\n",
      "target loaded\n"
     ]
    }
   ],
   "source": [
    "dl_test = data_loader(data_type=DATA_TYPE, is_regression=IS_REGRESSION, is_train=False, is_pure=True)\n",
    "test_X = dl_test.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((1503424, 38406), (1503424,), (508438, 38406))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_X.shape, train_y.shape, test_X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.    ,  0.    ,  0.    , ...,  0.7376,  0.    ,  0.7376])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ridge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ml = model_loader(model_type='rg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fit_param = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'alpha': 1.0} train loss: 0.218961, valid loss:0.226523, loss_diff:0.007563\n",
      "{'alpha': 1.0} train loss: 0.219196, valid loss:0.225495, loss_diff:0.006298\n",
      "{'alpha': 1.0} train loss: 0.218992, valid loss:0.226428, loss_diff:0.007435\n",
      "{'alpha': 1.0} train loss: 0.219083, valid loss:0.226062, loss_diff:0.006978\n",
      "{'alpha': 1.0} train loss: 0.219072, valid loss:0.225943, loss_diff:0.006871\n",
      "=================>{'alpha': 1.0} loss:0.226090\n",
      "{'alpha': 2.0} train loss: 0.219400, valid loss:0.226693, loss_diff:0.007294\n",
      "{'alpha': 2.0} train loss: 0.219648, valid loss:0.225647, loss_diff:0.005999\n",
      "{'alpha': 2.0} train loss: 0.219435, valid loss:0.226599, loss_diff:0.007165\n",
      "{'alpha': 2.0} train loss: 0.219528, valid loss:0.226240, loss_diff:0.006712\n",
      "{'alpha': 2.0} train loss: 0.219519, valid loss:0.226102, loss_diff:0.006583\n",
      "=================>{'alpha': 2.0} loss:0.226256\n",
      "{'alpha': 4.0} train loss: 0.219909, valid loss:0.226788, loss_diff:0.006879\n",
      "{'alpha': 4.0} train loss: 0.220164, valid loss:0.225728, loss_diff:0.005564\n",
      "{'alpha': 4.0} train loss: 0.219945, valid loss:0.226694, loss_diff:0.006750\n",
      "{'alpha': 4.0} train loss: 0.220041, valid loss:0.226334, loss_diff:0.006293\n",
      "{'alpha': 4.0} train loss: 0.220036, valid loss:0.226181, loss_diff:0.006145\n",
      "=================>{'alpha': 4.0} loss:0.226345\n",
      "{'alpha': 8.0} train loss: 0.220483, valid loss:0.226757, loss_diff:0.006274\n",
      "{'alpha': 8.0} train loss: 0.220740, valid loss:0.225687, loss_diff:0.004947\n",
      "{'alpha': 8.0} train loss: 0.220517, valid loss:0.226667, loss_diff:0.006149\n",
      "{'alpha': 8.0} train loss: 0.220614, valid loss:0.226285, loss_diff:0.005671\n",
      "{'alpha': 8.0} train loss: 0.220620, valid loss:0.226134, loss_diff:0.005514\n",
      "=================>{'alpha': 8.0} loss:0.226306\n",
      "{'alpha': 16.0} train loss: 0.221172, valid loss:0.226652, loss_diff:0.005480\n",
      "{'alpha': 16.0} train loss: 0.221432, valid loss:0.225580, loss_diff:0.004148\n",
      "{'alpha': 16.0} train loss: 0.221207, valid loss:0.226572, loss_diff:0.005365\n",
      "{'alpha': 16.0} train loss: 0.221303, valid loss:0.226153, loss_diff:0.004849\n",
      "{'alpha': 16.0} train loss: 0.221325, valid loss:0.226020, loss_diff:0.004696\n",
      "=================>{'alpha': 16.0} loss:0.226195\n",
      "Best params: {'alpha': 1.0} \tbest loss: 0.22609012994\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>param</th>\n",
       "      <th>val_loss_mean</th>\n",
       "      <th>val_loss_std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>{'alpha': 1.0}</td>\n",
       "      <td>0.226090</td>\n",
       "      <td>0.000368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>{'alpha': 2.0}</td>\n",
       "      <td>0.226256</td>\n",
       "      <td>0.000375</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>{'alpha': 4.0}</td>\n",
       "      <td>0.226345</td>\n",
       "      <td>0.000381</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>{'alpha': 8.0}</td>\n",
       "      <td>0.226306</td>\n",
       "      <td>0.000386</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>{'alpha': 16.0}</td>\n",
       "      <td>0.226195</td>\n",
       "      <td>0.000390</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             param  val_loss_mean  val_loss_std\n",
       "0   {'alpha': 1.0}       0.226090      0.000368\n",
       "1   {'alpha': 2.0}       0.226256      0.000375\n",
       "2   {'alpha': 4.0}       0.226345      0.000381\n",
       "3   {'alpha': 8.0}       0.226306      0.000386\n",
       "4  {'alpha': 16.0}       0.226195      0.000390"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "default_params = {\n",
    "    'alpha':1.0, \n",
    "    'fit_intercept':True, \n",
    "    'normalize':False, \n",
    "    'copy_X':True, \n",
    "    'max_iter':None, \n",
    "    'tol':0.001, \n",
    "    'solver':'auto', \n",
    "    'random_state': SEED\n",
    "}\n",
    "\n",
    "try_params = {\n",
    "    'alpha': [1., 2., 4., 8., 16.]\n",
    "}\n",
    "\n",
    "fit_params(train_X, train_y, ml, default_params, try_params, fit_params=fit_param, seed=SEED, use_eval_set=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'alpha': 0.01} train loss: 0.218195, valid loss:0.226164, loss_diff:0.007969\n",
      "{'alpha': 0.01} train loss: 0.218413, valid loss:0.225180, loss_diff:0.006768\n",
      "{'alpha': 0.01} train loss: 0.218227, valid loss:0.226079, loss_diff:0.007852\n",
      "{'alpha': 0.01} train loss: 0.218282, valid loss:0.225684, loss_diff:0.007402\n",
      "{'alpha': 0.01} train loss: 0.218264, valid loss:0.225537, loss_diff:0.007273\n",
      "=================>{'alpha': 0.01} loss:0.225729\n",
      "{'alpha': 0.05} train loss: 0.218237, valid loss:0.226182, loss_diff:0.007945\n",
      "{'alpha': 0.05} train loss: 0.218456, valid loss:0.225195, loss_diff:0.006739\n",
      "{'alpha': 0.05} train loss: 0.218269, valid loss:0.226097, loss_diff:0.007828\n",
      "{'alpha': 0.05} train loss: 0.218327, valid loss:0.225702, loss_diff:0.007376\n",
      "{'alpha': 0.05} train loss: 0.218313, valid loss:0.225562, loss_diff:0.007249\n",
      "=================>{'alpha': 0.05} loss:0.225748\n",
      "{'alpha': 0.1} train loss: 0.218289, valid loss:0.226206, loss_diff:0.007917\n",
      "{'alpha': 0.1} train loss: 0.218508, valid loss:0.225213, loss_diff:0.006705\n",
      "{'alpha': 0.1} train loss: 0.218321, valid loss:0.226119, loss_diff:0.007799\n",
      "{'alpha': 0.1} train loss: 0.218381, valid loss:0.225726, loss_diff:0.007344\n",
      "{'alpha': 0.1} train loss: 0.218366, valid loss:0.225588, loss_diff:0.007222\n",
      "=================>{'alpha': 0.1} loss:0.225770\n",
      "{'alpha': 0.5} train loss: 0.218640, valid loss:0.226375, loss_diff:0.007735\n",
      "{'alpha': 0.5} train loss: 0.218868, valid loss:0.225367, loss_diff:0.006498\n",
      "{'alpha': 0.5} train loss: 0.218672, valid loss:0.226282, loss_diff:0.007610\n",
      "{'alpha': 0.5} train loss: 0.218751, valid loss:0.225904, loss_diff:0.007153\n",
      "{'alpha': 0.5} train loss: 0.218736, valid loss:0.225785, loss_diff:0.007049\n",
      "=================>{'alpha': 0.5} loss:0.225942\n",
      "Best params: {'alpha': 0.01} \tbest loss: 0.225729049321\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>param</th>\n",
       "      <th>val_loss_mean</th>\n",
       "      <th>val_loss_std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>{'alpha': 0.01}</td>\n",
       "      <td>0.225729</td>\n",
       "      <td>0.000361</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>{'alpha': 0.05}</td>\n",
       "      <td>0.225748</td>\n",
       "      <td>0.000361</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>{'alpha': 0.1}</td>\n",
       "      <td>0.225770</td>\n",
       "      <td>0.000362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>{'alpha': 0.5}</td>\n",
       "      <td>0.225942</td>\n",
       "      <td>0.000363</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             param  val_loss_mean  val_loss_std\n",
       "0  {'alpha': 0.01}       0.225729      0.000361\n",
       "1  {'alpha': 0.05}       0.225748      0.000361\n",
       "2   {'alpha': 0.1}       0.225770      0.000362\n",
       "3   {'alpha': 0.5}       0.225942      0.000363"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "default_params = {\n",
    "    'alpha':1.0, \n",
    "    'fit_intercept':True, \n",
    "    'normalize':False, \n",
    "    'copy_X':True, \n",
    "    'max_iter':None, \n",
    "    'tol':0.001, \n",
    "    'solver':'auto', \n",
    "    'random_state': SEED\n",
    "}\n",
    "\n",
    "try_params = {\n",
    "    'alpha': [0.01, 0.05, 0.1, 0.5,]\n",
    "}\n",
    "\n",
    "fit_params(train_X, train_y, ml, default_params, try_params, fit_params=fit_param, seed=SEED, use_eval_set=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'alpha': 32} train loss: 0.222047, valid loss:0.226603, loss_diff:0.004556\n",
      "{'alpha': 32} train loss: 0.222307, valid loss:0.225534, loss_diff:0.003228\n",
      "{'alpha': 32} train loss: 0.222079, valid loss:0.226539, loss_diff:0.004460\n",
      "{'alpha': 32} train loss: 0.222180, valid loss:0.226074, loss_diff:0.003895\n",
      "{'alpha': 32} train loss: 0.222193, valid loss:0.225945, loss_diff:0.003752\n",
      "=================>{'alpha': 32} loss:0.226139\n",
      "{'alpha': 64} train loss: 0.223124, valid loss:0.226740, loss_diff:0.003616\n",
      "{'alpha': 64} train loss: 0.223384, valid loss:0.225672, loss_diff:0.002288\n",
      "{'alpha': 64} train loss: 0.223151, valid loss:0.226692, loss_diff:0.003541\n",
      "{'alpha': 64} train loss: 0.223262, valid loss:0.226186, loss_diff:0.002924\n",
      "{'alpha': 64} train loss: 0.223283, valid loss:0.226079, loss_diff:0.002796\n",
      "=================>{'alpha': 64} loss:0.226274\n",
      "{'alpha': 128} train loss: 0.224362, valid loss:0.227131, loss_diff:0.002769\n",
      "{'alpha': 128} train loss: 0.224618, valid loss:0.226051, loss_diff:0.001433\n",
      "{'alpha': 128} train loss: 0.224382, valid loss:0.227093, loss_diff:0.002710\n",
      "{'alpha': 128} train loss: 0.224505, valid loss:0.226558, loss_diff:0.002053\n",
      "{'alpha': 128} train loss: 0.224525, valid loss:0.226460, loss_diff:0.001935\n",
      "=================>{'alpha': 128} loss:0.226659\n",
      "Best params: {'alpha': 32} \tbest loss: 0.226139028697\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>param</th>\n",
       "      <th>val_loss_mean</th>\n",
       "      <th>val_loss_std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>{'alpha': 32}</td>\n",
       "      <td>0.226139</td>\n",
       "      <td>0.000396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>{'alpha': 64}</td>\n",
       "      <td>0.226274</td>\n",
       "      <td>0.000400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>{'alpha': 128}</td>\n",
       "      <td>0.226659</td>\n",
       "      <td>0.000407</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            param  val_loss_mean  val_loss_std\n",
       "0   {'alpha': 32}       0.226139      0.000396\n",
       "1   {'alpha': 64}       0.226274      0.000400\n",
       "2  {'alpha': 128}       0.226659      0.000407"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "default_params = {\n",
    "    'alpha':1.0, \n",
    "    'fit_intercept':True, \n",
    "    'normalize':False, \n",
    "    'copy_X':True, \n",
    "    'max_iter':None, \n",
    "    'tol':0.001, \n",
    "    'solver':'auto', \n",
    "    'random_state': SEED\n",
    "}\n",
    "\n",
    "try_params = {\n",
    "    'alpha': [32, 64, 128]\n",
    "}\n",
    "\n",
    "fit_params(train_X, train_y, ml, default_params, try_params, fit_params=fit_param, seed=SEED, use_eval_set=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 1 completed.\n",
      "Fold 2 completed.\n",
      "Fold 3 completed.\n",
      "Fold 4 completed.\n",
      "Fold 5 completed.\n",
      "Fold 1 completed.\n",
      "Fold 2 completed.\n",
      "Fold 3 completed.\n",
      "Fold 4 completed.\n",
      "Fold 5 completed.\n",
      "Fold 1 completed.\n",
      "Fold 2 completed.\n",
      "Fold 3 completed.\n",
      "Fold 4 completed.\n",
      "Fold 5 completed.\n",
      "Fold 1 completed.\n",
      "Fold 2 completed.\n",
      "Fold 3 completed.\n",
      "Fold 4 completed.\n",
      "Fold 5 completed.\n"
     ]
    }
   ],
   "source": [
    "for alpha in [0.001, 1., 16., 32.]:\n",
    "    name = 'alpha_'+str(alpha).replace(\".\",\"\")\n",
    "    default_params = {\n",
    "        'alpha':alpha, \n",
    "        'fit_intercept':True, \n",
    "        'normalize':False, \n",
    "        'copy_X':True, \n",
    "        'max_iter':None, \n",
    "        'tol':0.001, \n",
    "        'solver':'auto', \n",
    "        'random_state': SEED\n",
    "    }\n",
    "    ret, ret_test, _ = get_oof_predictions(train_X, train_y, test_X, ml, default_params, seed=SEED, fit_params=None, use_eval_set=False, predict_proba=False)\n",
    "    pd.DataFrame(data=ret, columns=[name+'_pred']).to_csv(name+'_oof_val_pred.csv', index=False)\n",
    "    pd.DataFrame(data=ret_test, columns=[name+'_pred']).to_csv(name+'_oof_test_pred.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## KNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'model_loader' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-2-d376953594af>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mml\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel_loader\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel_type\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'knn'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodel_selection\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mKFold\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdecomposition\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mTruncatedSVD\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mscipy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msparse\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mvstack\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'model_loader' is not defined"
     ]
    }
   ],
   "source": [
    "ml = model_loader(model_type='knn')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# try best neighbors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# get n=5, 10, 100, 500 features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''\n",
    "non-cat: 57: 10/11 ==> first=38349 (38409-38349)=57 => 38358\\38359\n",
    "cat: 9 \n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from scipy.sparse import vstack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "reduced_X = TruncatedSVD(n_components=300).fit_transform(vstack([train_X, test_X]).tocsr())\n",
    "\n",
    "import pickle\n",
    "with open('reduced_sparse_features.pickle', 'wb') as handle:\n",
    "    pickle.dump(reduced_X, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2011862, 300)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pickle\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from scipy.sparse import vstack\n",
    "with open('reduced_sparse_features.pickle', 'rb') as handle:\n",
    "    reduced_X = pickle.load(handle)\n",
    "\n",
    "reduced_X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class knn_feature_wrapper():\n",
    "    def __init__(self, params, seed=519):\n",
    "        self.params = params\n",
    "        self.seed = seed\n",
    "        \n",
    "    def transform(self, X, y, test_X, reduced_X,\n",
    "                  n_list=[5, 10, 100, 500],\n",
    "                  feature_inds=[38358, 38359],\n",
    "                  feature_cols=['price', 'item_seq_number']):\n",
    "        def get_model():\n",
    "            return model_loader(model_type='knn').load(self.params)\n",
    "        \n",
    "        def get_feature_names(n):\n",
    "            return ['{}_{}_closest_mean'.format(feat, n) for feat in feature_cols]\n",
    "        \n",
    "        train_res = pd.DataFrame()\n",
    "        test_res = pd.DataFrame()\n",
    "        for n in n_list:\n",
    "            train_res['{}_closest_avg_dist'.format(n)] = np.zeros((X.shape[0],))\n",
    "            test_res['{}_closest_avg_dist'.format(n)] = np.zeros((test_X.shape[0],))\n",
    "            train_res['{}_closest_avg_target'.format(n)]  = np.zeros((X.shape[0],))\n",
    "            test_res['{}_closest_avg_target'.format(n)]  = np.zeros((test_X.shape[0],))\n",
    "            for feat in feature_cols:\n",
    "                train_res['{}_{}_closest_mean'.format(feat, n)] = np.zeros((X.shape[0],))\n",
    "                test_res['{}_{}_closest_mean'.format(feat, n)] = np.zeros((test_X.shape[0],))\n",
    "        \n",
    "        for train_ix, val_ix in KFold(5, shuffle=True, random_state=self.seed).split(X):\n",
    "            md = get_model()\n",
    "            \n",
    "            tr_X, tr_y = X[train_ix,:], y[train_ix]\n",
    "            \n",
    "            md.fit(reduced_X[train_ix, :], tr_y)\n",
    "            all_val_dist, all_val_ind = md.kneighbors(reduced_X[val_ix,:], max(n_list), return_distance=True)\n",
    "            print('Getting validation n-neighbors done.')\n",
    "            all_test_dist, all_test_ind = md.kneighbors(reduced_X[X.shape[0]:,:], max(n_list), return_distance=True)\n",
    "            print('Getting test n-neighbors done.')\n",
    "            \n",
    "            for n in n_list:\n",
    "                val_dist, val_ind = all_val_dist[:, :n], all_val_ind[:, :n]\n",
    "                test_dist, test_ind = all_test_dist[:, :n], all_test_ind[:, :n]\n",
    "                \n",
    "                train_res.loc[val_ix, '{}_closest_avg_dist'.format(n)] = val_dist.mean(axis=1)\n",
    "                train_res.loc[val_ix, get_feature_names(n)] = \\\n",
    "                    np.array([tr_X[ind,:][:,feature_inds].mean(axis=0) for ind in tqdm(val_ind)])\n",
    "                train_res.loc[val_ix, '{}_closest_avg_target'.format(n)] = \\\n",
    "                    np.array([tr_y[ind].mean() for ind in tqdm(val_ind)])\n",
    "                \n",
    "                test_res.loc[:, '{}_closest_avg_dist'.format(n)] += test_dist.mean(axis=1)\n",
    "                test_res.loc[:, get_feature_names(n)] += \\\n",
    "                    np.array([tr_X[ind,:][:,feature_inds].mean(axis=0) for ind in tqdm(test_ind)])\n",
    "                test_res.loc[:, '{}_closest_avg_target'.format(n)] += \\\n",
    "                    np.array([tr_y[ind].mean() for ind in tqdm(test_ind)])\n",
    "                \n",
    "                print(n, ' finished')\n",
    "                \n",
    "            del md, all_val_dist, all_val_ind\n",
    "            \n",
    "        test_res.loc[:, :] /= 5\n",
    "        return train_res, test_res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.  1.  2.]\n",
      " [ 0.  1.  2.]\n",
      " [ 0.  1.  2.]\n",
      " [ 0.  1.  2.]\n",
      " [ 0.  1.  2.]]\n",
      "[ 0.  1.  0.  0.  0.]\n",
      "[[ 1.  1.  1.]\n",
      " [ 0.  1.  1.]\n",
      " [ 0.  0.  1.]\n",
      " [ 0.  0.  0.]]\n",
      "Getting n neighbors done.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 1000.07it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<?, ?it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████| 4/4 [00:00<?, ?it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████| 4/4 [00:00<00:00, 4000.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2  finished\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<?, ?it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<?, ?it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████| 4/4 [00:00<?, ?it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████| 4/4 [00:00<?, ?it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3  finished\n",
      "Getting n neighbors done.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<?, ?it/s]\n",
      "100%|███████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 999.83it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████| 4/4 [00:00<?, ?it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████| 4/4 [00:00<?, ?it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2  finished\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 1000.07it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 1000.07it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████| 4/4 [00:00<00:00, 4000.29it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████| 4/4 [00:00<00:00, 4000.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3  finished\n",
      "Getting n neighbors done.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 999.60it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<?, ?it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████| 4/4 [00:00<?, ?it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████| 4/4 [00:00<?, ?it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2  finished\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 999.83it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<?, ?it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████| 4/4 [00:00<00:00, 4000.29it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████| 4/4 [00:00<00:00, 4000.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3  finished\n",
      "Getting n neighbors done.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 333.36it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<?, ?it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████| 4/4 [00:00<00:00, 4000.29it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████| 4/4 [00:00<?, ?it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2  finished\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 1000.07it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<?, ?it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████| 4/4 [00:00<00:00, 1333.32it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████| 4/4 [00:00<?, ?it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3  finished\n",
      "Getting n neighbors done.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<?, ?it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 1000.07it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████| 4/4 [00:00<?, ?it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████| 4/4 [00:00<?, ?it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2  finished\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<?, ?it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<?, ?it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████| 4/4 [00:00<?, ?it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████| 4/4 [00:00<?, ?it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3  finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(   2_closest_avg_dist  2_closest_avg_target  price_2_closest_mean  \\\n",
       " 0                 0.0                   0.5                   1.0   \n",
       " 1                 0.0                   0.0                   1.0   \n",
       " 2                 0.0                   0.5                   1.0   \n",
       " 3                 0.0                   0.5                   1.0   \n",
       " 4                 0.0                   0.5                   1.0   \n",
       " \n",
       "    item_seq_number_2_closest_mean  3_closest_avg_dist  3_closest_avg_target  \\\n",
       " 0                             2.0                 0.0              0.333333   \n",
       " 1                             2.0                 0.0              0.000000   \n",
       " 2                             2.0                 0.0              0.333333   \n",
       " 3                             2.0                 0.0              0.333333   \n",
       " 4                             2.0                 0.0              0.333333   \n",
       " \n",
       "    price_3_closest_mean  item_seq_number_3_closest_mean  \n",
       " 0                   1.0                             2.0  \n",
       " 1                   1.0                             2.0  \n",
       " 2                   1.0                             2.0  \n",
       " 3                   1.0                             2.0  \n",
       " 4                   1.0                             2.0  ,\n",
       "    2_closest_avg_dist  2_closest_avg_target  price_2_closest_mean  \\\n",
       " 0            1.414214                   0.1                   1.0   \n",
       " 1            1.000000                   0.1                   1.0   \n",
       " 2            1.414214                   0.1                   1.0   \n",
       " 3            2.236068                   0.1                   1.0   \n",
       " \n",
       "    item_seq_number_2_closest_mean  3_closest_avg_dist  3_closest_avg_target  \\\n",
       " 0                             2.0            1.414214              0.066667   \n",
       " 1                             2.0            1.000000              0.066667   \n",
       " 2                             2.0            1.414214              0.066667   \n",
       " 3                             2.0            2.236068              0.066667   \n",
       " \n",
       "    price_3_closest_mean  item_seq_number_3_closest_mean  \n",
       " 0                   1.0                             2.0  \n",
       " 1                   1.0                             2.0  \n",
       " 2                   1.0                             2.0  \n",
       " 3                   1.0                             2.0  )"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# testing code\n",
    "params = {\n",
    "    'p': 2,\n",
    "    'n_jobs': 1,\n",
    "    \n",
    "}\n",
    "kfw = knn_feature_wrapper(params)\n",
    "temp_X, temp_y, temp_test_X = np.zeros((5, 3)), np.zeros((5,)), np.zeros((4, 3))\n",
    "temp_X[:, 0] = 0\n",
    "temp_X[:, 1] = 1\n",
    "temp_X[:, 2] = 2\n",
    "temp_y[1] = 1\n",
    "temp_test_X[:1, 0] = 1\n",
    "temp_test_X[:2, 1] = 1\n",
    "temp_test_X[:3, 2] = 1\n",
    "\n",
    "print(temp_X)\n",
    "print(temp_y)\n",
    "print(temp_test_X)\n",
    "\n",
    "kfw.transform(temp_X, temp_y, temp_test_X, \n",
    "              n_list=[2, 3],\n",
    "              feature_inds=[1, 2],\n",
    "              feature_cols=['price', 'item_seq_number'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "    'p': 2,\n",
    "    'n_jobs': 3,   \n",
    "}\n",
    "kfw = knn_feature_wrapper(params)\n",
    "train_res, test_res = kfw.transform(train_X, train_y, test_X, reduced_X, n_list=[2, 5, 10, 30])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_res.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_res.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_res.to_csv('train_knn_features.csv', index=False)\n",
    "test_res.to_csv('test_knn_features.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MLP (Need tensorflow + keras)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'keras'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-1-c00344ac9ffd>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mpandas\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mtensorflow\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[0mkeras\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpreprocessing\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mtext\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msequence\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      4\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mnumpy\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mkeras\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlayers\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mInput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mSpatialDropout1D\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mDropout\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mGlobalAveragePooling1D\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mGRU\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mBidirectional\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mLSTM\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mDense\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mEmbedding\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mconcatenate\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mEmbedding\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mFlatten\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mActivation\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mBatchNormalization\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mregularizers\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'keras'"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "from keras.preprocessing import text, sequence\n",
    "import numpy as np\n",
    "from keras.layers import Input, SpatialDropout1D,Dropout, GlobalAveragePooling1D, GRU, Bidirectional, LSTM, Dense, Embedding, concatenate, Embedding, Flatten, Activation, BatchNormalization, regularizers\n",
    "from keras.initializers import Orthogonal\n",
    "from keras.models import Model, Sequential\n",
    "from keras.optimizers import Adam\n",
    "from keras.callbacks import ModelCheckpoint, LambdaCallback, Callback, LearningRateScheduler\n",
    "import keras.backend as K\n",
    "import numpy as np\n",
    "from sklearn import metrics\n",
    "from sklearn.model_selection import train_test_split\n",
    "import os\n",
    "import pickle\n",
    "import gc; gc.enable()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[name: \"/cpu:0\"\n",
      "device_type: \"CPU\"\n",
      "memory_limit: 268435456\n",
      "locality {\n",
      "}\n",
      "incarnation: 1596168217501253868\n",
      "]\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'K' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-3-98d92945b18c>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclient\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mdevice_lib\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdevice_lib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlist_local_devices\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 4\u001b[1;33m \u001b[0mK\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtensorflow_backend\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_get_available_gpus\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'K' is not defined"
     ]
    }
   ],
   "source": [
    "sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))\n",
    "from tensorflow.python.client import device_lib\n",
    "print(device_lib.list_local_devices())\n",
    "K.tensorflow_backend._get_available_gpus()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def root_mean_squared_error(y_true, y_pred):\n",
    "    return K.sqrt(K.mean(K.square(K.clip(y_pred, 0., 1.) - y_true), axis=-1))\n",
    "\n",
    "def build_model(X):\n",
    "    # define the architecture of the network\n",
    "    main_in = Input(shape=[X.shape[1]])\n",
    "    main = Dropout(0.1)(main_in)\n",
    "    main = Dense(512, activation='relu')(main)\n",
    "    main = Dense(128, activation='relu')(main)\n",
    "    out = Dense(1, activation = \"linear\")(main)\n",
    "    model = Model(main_in, out)\n",
    "    \n",
    "    '''\n",
    "    model = Sequential()\n",
    "    model.add(Input(shape=(32,)))\n",
    "    model.add(Dropout(0.1))\n",
    "    model.add(Dense(512, input_dim=X.shape[1], kernel_initializer=\"glorot_normal\", activation=\"relu\"))\n",
    "    model.add(Dense(256, kernel_initializer=\"glorot_normal\", activation=\"relu\"))\n",
    "    model.add(Dense(1, activation = \"linear\"))\n",
    "    '''\n",
    "    model.compile(optimizer = Adam(lr=0.0005), loss = 'mean_squared_error',\n",
    "                  metrics =[root_mean_squared_error])\n",
    "    model.summary()\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def clip_rmse(true, prediction):\n",
    "    return np.sqrt(metrics.mean_squared_error(true, np.clip(prediction, 0., 1.)))\n",
    "    \n",
    "class NBatchEvalLogger(Callback):\n",
    "    def __init__(self, display, val_X, val_y, save_path=None, save_start=1000):\n",
    "        self.step = 0\n",
    "        self.display = display\n",
    "        self.val_X = val_X\n",
    "        self.val_y = val_y\n",
    "        self.best_loss = None\n",
    "        self.save_path = save_path\n",
    "        self.save_start = save_start\n",
    "        self.record_count = 0\n",
    "        \n",
    "    def on_batch_end(self, batch, logs={}):\n",
    "        self.step += 1\n",
    "        if self.step % self.display == 0 and self.step >= self.save_start:\n",
    "            #loss, metric = self.model.evaluate(self.val_X, self.val_y, batch_size=128, verbose=1)\n",
    "            prediction = self.model.predict(self.val_X, batch_size=128, verbose=0)\n",
    "            loss = clip_rmse(self.val_y, prediction)\n",
    "            \n",
    "            if self.best_loss is None:\n",
    "                self.best_loss = loss\n",
    "            else:\n",
    "                if loss < self.best_loss:\n",
    "                    self.best_loss = loss\n",
    "                    if self.save_path is not None:\n",
    "                        self.model.save(self.save_path, overwrite=True)\n",
    "                        self.record_count += 1\n",
    "                    \n",
    "            print('\\nstep: {} val loss={:.5f}, best loss={:.5f}'.format(self.step, loss, self.best_loss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 38406)             0         \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 38406)             0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 512)               19664384  \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 128)               65664     \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 1)                 129       \n",
      "=================================================================\n",
      "Total params: 19,730,177\n",
      "Trainable params: 19,730,177\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Train on 1202739 samples, validate on 300685 samples\n",
      "Epoch 1/4\n",
      "\n",
      "Epoch 00001: LearningRateScheduler reducing learning rate to 0.0005.\n",
      " 256000/1202739 [=====>........................] - ETA: 10:22 - loss: 0.0537 - root_mean_squared_error: 0.1553\n",
      "step: 2000 val loss=0.22836, best loss=0.22836\n",
      " 512000/1202739 [===========>..................] - ETA: 9:49 - loss: 0.0529 - root_mean_squared_error: 0.1537\n",
      "step: 4000 val loss=0.22720, best loss=0.22720\n",
      " 768000/1202739 [==================>...........] - ETA: 6:39 - loss: 0.0524 - root_mean_squared_error: 0.1526\n",
      "step: 6000 val loss=0.22630, best loss=0.22630\n",
      "1024000/1202739 [========================>.....] - ETA: 2:50 - loss: 0.0521 - root_mean_squared_error: 0.1519\n",
      "step: 8000 val loss=0.22581, best loss=0.22581\n",
      "1202739/1202739 [==============================] - 1297s 1ms/step - loss: 0.0520 - root_mean_squared_error: 0.1516 - val_loss: 0.0506 - val_root_mean_squared_error: 0.1545\n",
      "Epoch 2/4\n",
      "\n",
      "Epoch 00002: LearningRateScheduler reducing learning rate to 0.0005.\n",
      "  77184/1202739 [>.............................] - ETA: 12:19 - loss: 0.0482 - root_mean_squared_error: 0.1434\n",
      "step: 10000 val loss=0.22519, best loss=0.22519\n",
      " 333184/1202739 [=======>......................] - ETA: 13:55 - loss: 0.0487 - root_mean_squared_error: 0.1445\n",
      "step: 12000 val loss=0.22482, best loss=0.22482\n",
      " 589184/1202739 [=============>................] - ETA: 10:13 - loss: 0.0487 - root_mean_squared_error: 0.1442\n",
      "step: 14000 val loss=0.22467, best loss=0.22467\n",
      " 845184/1202739 [====================>.........] - ETA: 6:03 - loss: 0.0489 - root_mean_squared_error: 0.1448\n",
      "step: 16000 val loss=0.22489, best loss=0.22467\n",
      "1101184/1202739 [==========================>...] - ETA: 1:43 - loss: 0.0490 - root_mean_squared_error: 0.1451\n",
      "step: 18000 val loss=0.22392, best loss=0.22392\n",
      "1202739/1202739 [==============================] - 1396s 1ms/step - loss: 0.0491 - root_mean_squared_error: 0.1452 - val_loss: 0.0503 - val_root_mean_squared_error: 0.1463\n",
      "Epoch 3/4\n",
      "\n",
      "Epoch 00003: LearningRateScheduler reducing learning rate to 0.0001.\n",
      " 154368/1202739 [==>...........................] - ETA: 11:27 - loss: 0.0457 - root_mean_squared_error: 0.1385\n",
      "step: 20000 val loss=0.22401, best loss=0.22392\n",
      " 410368/1202739 [=========>....................] - ETA: 11:53 - loss: 0.0457 - root_mean_squared_error: 0.1382\n",
      "step: 22000 val loss=0.22407, best loss=0.22392\n",
      " 666368/1202739 [===============>..............] - ETA: 8:33 - loss: 0.0456 - root_mean_squared_error: 0.1379\n",
      "step: 24000 val loss=0.22397, best loss=0.22392\n",
      " 922368/1202739 [======================>.......] - ETA: 4:35 - loss: 0.0456 - root_mean_squared_error: 0.1378\n",
      "step: 26000 val loss=0.22401, best loss=0.22392\n",
      "1178368/1202739 [============================>.] - ETA: 24s - loss: 0.0456 - root_mean_squared_error: 0.1377\n",
      "step: 28000 val loss=0.22401, best loss=0.22392\n",
      "1202739/1202739 [==============================] - 1394s 1ms/step - loss: 0.0456 - root_mean_squared_error: 0.1377 - val_loss: 0.0503 - val_root_mean_squared_error: 0.1414\n",
      "Epoch 4/4\n",
      "\n",
      "Epoch 00004: LearningRateScheduler reducing learning rate to 0.0001.\n",
      " 231552/1202739 [====>.........................] - ETA: 10:39 - loss: 0.0442 - root_mean_squared_error: 0.1348\n",
      "step: 30000 val loss=0.22470, best loss=0.22392\n",
      " 487552/1202739 [===========>..................] - ETA: 10:17 - loss: 0.0441 - root_mean_squared_error: 0.1345\n",
      "step: 32000 val loss=0.22487, best loss=0.22392\n",
      " 743552/1202739 [=================>............] - ETA: 7:06 - loss: 0.0442 - root_mean_squared_error: 0.1346\n",
      "step: 34000 val loss=0.22503, best loss=0.22392\n",
      " 999552/1202739 [=======================>......] - ETA: 3:14 - loss: 0.0441 - root_mean_squared_error: 0.1345\n",
      "step: 36000 val loss=0.22523, best loss=0.22392\n",
      "1202739/1202739 [==============================] - 1294s 1ms/step - loss: 0.0441 - root_mean_squared_error: 0.1344 - val_loss: 0.0506 - val_root_mean_squared_error: 0.1440\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         (None, 38406)             0         \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 38406)             0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 512)               19664384  \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 128)               65664     \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 1)                 129       \n",
      "=================================================================\n",
      "Total params: 19,730,177\n",
      "Trainable params: 19,730,177\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Train on 1202739 samples, validate on 300685 samples\n",
      "Epoch 1/4\n",
      "\n",
      "Epoch 00001: LearningRateScheduler reducing learning rate to 0.0005.\n",
      " 256000/1202739 [=====>........................] - ETA: 10:22 - loss: 0.0541 - root_mean_squared_error: 0.1561\n",
      "step: 2000 val loss=0.22750, best loss=0.22750\n",
      " 512000/1202739 [===========>..................] - ETA: 9:49 - loss: 0.0532 - root_mean_squared_error: 0.1543\n",
      "step: 4000 val loss=0.22536, best loss=0.22536\n",
      " 768000/1202739 [==================>...........] - ETA: 6:40 - loss: 0.0527 - root_mean_squared_error: 0.1531\n",
      "step: 6000 val loss=0.22512, best loss=0.22512\n",
      "1024000/1202739 [========================>.....] - ETA: 2:50 - loss: 0.0523 - root_mean_squared_error: 0.1524\n",
      "step: 8000 val loss=0.22385, best loss=0.22385\n",
      "1202739/1202739 [==============================] - 1297s 1ms/step - loss: 0.0521 - root_mean_squared_error: 0.1519 - val_loss: 0.0501 - val_root_mean_squared_error: 0.1513\n",
      "Epoch 2/4\n",
      "\n",
      "Epoch 00002: LearningRateScheduler reducing learning rate to 0.0005.\n",
      "  77184/1202739 [>.............................] - ETA: 12:16 - loss: 0.0487 - root_mean_squared_error: 0.1442\n",
      "step: 10000 val loss=0.22389, best loss=0.22385\n",
      " 333184/1202739 [=======>......................] - ETA: 13:50 - loss: 0.0490 - root_mean_squared_error: 0.1451\n",
      "step: 12000 val loss=0.22397, best loss=0.22385\n",
      " 589184/1202739 [=============>................] - ETA: 10:09 - loss: 0.0490 - root_mean_squared_error: 0.1451\n",
      "step: 14000 val loss=0.22376, best loss=0.22376\n",
      " 845184/1202739 [====================>.........] - ETA: 6:00 - loss: 0.0491 - root_mean_squared_error: 0.1453\n",
      "step: 16000 val loss=0.22384, best loss=0.22376\n",
      "1101184/1202739 [==========================>...] - ETA: 1:43 - loss: 0.0492 - root_mean_squared_error: 0.1454\n",
      "step: 18000 val loss=0.22297, best loss=0.22297\n",
      "1202739/1202739 [==============================] - 1388s 1ms/step - loss: 0.0491 - root_mean_squared_error: 0.1454 - val_loss: 0.0506 - val_root_mean_squared_error: 0.1535\n",
      "Epoch 3/4\n",
      "\n",
      "Epoch 00003: LearningRateScheduler reducing learning rate to 0.0001.\n",
      " 154368/1202739 [==>...........................] - ETA: 11:21 - loss: 0.0462 - root_mean_squared_error: 0.1396\n",
      "step: 20000 val loss=0.22295, best loss=0.22295\n",
      " 410368/1202739 [=========>....................] - ETA: 11:50 - loss: 0.0458 - root_mean_squared_error: 0.1382\n",
      "step: 22000 val loss=0.22294, best loss=0.22294\n",
      " 666368/1202739 [===============>..............] - ETA: 8:31 - loss: 0.0458 - root_mean_squared_error: 0.1382\n",
      "step: 24000 val loss=0.22305, best loss=0.22294\n",
      " 922368/1202739 [======================>.......] - ETA: 4:34 - loss: 0.0457 - root_mean_squared_error: 0.1379\n",
      "step: 26000 val loss=0.22314, best loss=0.22294\n",
      "1178368/1202739 [============================>.] - ETA: 24s - loss: 0.0456 - root_mean_squared_error: 0.1378\n",
      "step: 28000 val loss=0.22324, best loss=0.22294\n",
      "1202739/1202739 [==============================] - 1387s 1ms/step - loss: 0.0456 - root_mean_squared_error: 0.1378 - val_loss: 0.0498 - val_root_mean_squared_error: 0.1410\n",
      "Epoch 4/4\n",
      "\n",
      "Epoch 00004: LearningRateScheduler reducing learning rate to 0.0001.\n",
      " 231552/1202739 [====>.........................] - ETA: 10:35 - loss: 0.0443 - root_mean_squared_error: 0.1349\n",
      "step: 30000 val loss=0.22377, best loss=0.22294\n",
      " 487552/1202739 [===========>..................] - ETA: 10:15 - loss: 0.0441 - root_mean_squared_error: 0.1345\n",
      "step: 32000 val loss=0.22390, best loss=0.22294\n",
      " 743552/1202739 [=================>............] - ETA: 7:04 - loss: 0.0440 - root_mean_squared_error: 0.1341\n",
      "step: 34000 val loss=0.22443, best loss=0.22294\n",
      " 999552/1202739 [=======================>......] - ETA: 3:14 - loss: 0.0441 - root_mean_squared_error: 0.1344\n",
      "step: 36000 val loss=0.22415, best loss=0.22294\n",
      "1202739/1202739 [==============================] - 1291s 1ms/step - loss: 0.0442 - root_mean_squared_error: 0.1345 - val_loss: 0.0503 - val_root_mean_squared_error: 0.1432\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_3 (InputLayer)         (None, 38406)             0         \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 38406)             0         \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 512)               19664384  \n",
      "_________________________________________________________________\n",
      "dense_8 (Dense)              (None, 128)               65664     \n",
      "_________________________________________________________________\n",
      "dense_9 (Dense)              (None, 1)                 129       \n",
      "=================================================================\n",
      "Total params: 19,730,177\n",
      "Trainable params: 19,730,177\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Train on 1202739 samples, validate on 300685 samples\n",
      "Epoch 1/4\n",
      "\n",
      "Epoch 00001: LearningRateScheduler reducing learning rate to 0.0005.\n",
      " 256000/1202739 [=====>........................] - ETA: 10:22 - loss: 0.0538 - root_mean_squared_error: 0.1555\n",
      "step: 2000 val loss=0.22842, best loss=0.22842\n",
      " 512000/1202739 [===========>..................] - ETA: 9:49 - loss: 0.0530 - root_mean_squared_error: 0.1537\n",
      "step: 4000 val loss=0.22653, best loss=0.22653\n",
      " 768000/1202739 [==================>...........] - ETA: 6:39 - loss: 0.0525 - root_mean_squared_error: 0.1527\n",
      "step: 6000 val loss=0.22579, best loss=0.22579\n",
      "1024000/1202739 [========================>.....] - ETA: 2:50 - loss: 0.0522 - root_mean_squared_error: 0.1520\n",
      "step: 8000 val loss=0.22515, best loss=0.22515\n",
      "1202739/1202739 [==============================] - 1297s 1ms/step - loss: 0.0520 - root_mean_squared_error: 0.1516 - val_loss: 0.0505 - val_root_mean_squared_error: 0.1467\n",
      "Epoch 2/4\n",
      "\n",
      "Epoch 00002: LearningRateScheduler reducing learning rate to 0.0005.\n",
      "  77184/1202739 [>.............................] - ETA: 12:18 - loss: 0.0491 - root_mean_squared_error: 0.1454\n",
      "step: 10000 val loss=0.22510, best loss=0.22510\n",
      " 333184/1202739 [=======>......................] - ETA: 13:56 - loss: 0.0488 - root_mean_squared_error: 0.1447\n",
      "step: 12000 val loss=0.22466, best loss=0.22466\n",
      " 589184/1202739 [=============>................] - ETA: 10:14 - loss: 0.0489 - root_mean_squared_error: 0.1449\n",
      "step: 14000 val loss=0.22449, best loss=0.22449\n",
      " 845184/1202739 [====================>.........] - ETA: 6:03 - loss: 0.0490 - root_mean_squared_error: 0.1452\n",
      "step: 16000 val loss=0.22425, best loss=0.22425\n",
      "1101184/1202739 [==========================>...] - ETA: 1:44 - loss: 0.0491 - root_mean_squared_error: 0.1454\n",
      "step: 18000 val loss=0.22464, best loss=0.22425\n",
      "1202739/1202739 [==============================] - 1399s 1ms/step - loss: 0.0491 - root_mean_squared_error: 0.1454 - val_loss: 0.0503 - val_root_mean_squared_error: 0.1452\n",
      "Epoch 3/4\n",
      "\n",
      "Epoch 00003: LearningRateScheduler reducing learning rate to 0.0001.\n",
      " 154368/1202739 [==>...........................] - ETA: 11:28 - loss: 0.0457 - root_mean_squared_error: 0.1383\n",
      "step: 20000 val loss=0.22387, best loss=0.22387\n",
      " 410368/1202739 [=========>....................] - ETA: 11:57 - loss: 0.0459 - root_mean_squared_error: 0.1384\n",
      "step: 22000 val loss=0.22386, best loss=0.22386\n",
      " 666368/1202739 [===============>..............] - ETA: 8:36 - loss: 0.0458 - root_mean_squared_error: 0.1382\n",
      "step: 24000 val loss=0.22404, best loss=0.22386\n",
      " 922368/1202739 [======================>.......] - ETA: 4:36 - loss: 0.0458 - root_mean_squared_error: 0.1382\n",
      "step: 26000 val loss=0.22395, best loss=0.22386\n",
      "1178368/1202739 [============================>.] - ETA: 24s - loss: 0.0457 - root_mean_squared_error: 0.1380\n",
      "step: 28000 val loss=0.22424, best loss=0.22386\n",
      "1202739/1202739 [==============================] - 1398s 1ms/step - loss: 0.0457 - root_mean_squared_error: 0.1379 - val_loss: 0.0503 - val_root_mean_squared_error: 0.1459\n",
      "Epoch 4/4\n",
      "\n",
      "Epoch 00004: LearningRateScheduler reducing learning rate to 0.0001.\n",
      " 231552/1202739 [====>.........................] - ETA: 10:33 - loss: 0.0440 - root_mean_squared_error: 0.1341\n",
      "step: 30000 val loss=0.22529, best loss=0.22386\n",
      " 487552/1202739 [===========>..................] - ETA: 10:12 - loss: 0.0441 - root_mean_squared_error: 0.1344\n",
      "step: 32000 val loss=0.22499, best loss=0.22386\n",
      " 743552/1202739 [=================>............] - ETA: 7:02 - loss: 0.0444 - root_mean_squared_error: 0.1349\n",
      "step: 34000 val loss=0.22502, best loss=0.22386\n",
      " 999552/1202739 [=======================>......] - ETA: 3:13 - loss: 0.0443 - root_mean_squared_error: 0.1348\n",
      "step: 36000 val loss=0.22492, best loss=0.22386\n",
      "1202739/1202739 [==============================] - 1288s 1ms/step - loss: 0.0443 - root_mean_squared_error: 0.1348 - val_loss: 0.0507 - val_root_mean_squared_error: 0.1423\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_4 (InputLayer)         (None, 38406)             0         \n",
      "_________________________________________________________________\n",
      "dropout_4 (Dropout)          (None, 38406)             0         \n",
      "_________________________________________________________________\n",
      "dense_10 (Dense)             (None, 512)               19664384  \n",
      "_________________________________________________________________\n",
      "dense_11 (Dense)             (None, 128)               65664     \n",
      "_________________________________________________________________\n",
      "dense_12 (Dense)             (None, 1)                 129       \n",
      "=================================================================\n",
      "Total params: 19,730,177\n",
      "Trainable params: 19,730,177\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Train on 1202739 samples, validate on 300685 samples\n",
      "Epoch 1/4\n",
      "\n",
      "Epoch 00001: LearningRateScheduler reducing learning rate to 0.0005.\n",
      " 256000/1202739 [=====>........................] - ETA: 10:22 - loss: 0.0541 - root_mean_squared_error: 0.1561\n",
      "step: 2000 val loss=0.22778, best loss=0.22778\n",
      " 512000/1202739 [===========>..................] - ETA: 9:48 - loss: 0.0532 - root_mean_squared_error: 0.1542\n",
      "step: 4000 val loss=0.22582, best loss=0.22582\n",
      " 768000/1202739 [==================>...........] - ETA: 6:39 - loss: 0.0526 - root_mean_squared_error: 0.1529\n",
      "step: 6000 val loss=0.22524, best loss=0.22524\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1024000/1202739 [========================>.....] - ETA: 2:50 - loss: 0.0523 - root_mean_squared_error: 0.1521\n",
      "step: 8000 val loss=0.22457, best loss=0.22457\n",
      "1202739/1202739 [==============================] - 1295s 1ms/step - loss: 0.0521 - root_mean_squared_error: 0.1517 - val_loss: 0.0507 - val_root_mean_squared_error: 0.1477\n",
      "Epoch 2/4\n",
      "\n",
      "Epoch 00002: LearningRateScheduler reducing learning rate to 0.0005.\n",
      "  77184/1202739 [>.............................] - ETA: 12:17 - loss: 0.0486 - root_mean_squared_error: 0.1445\n",
      "step: 10000 val loss=0.22571, best loss=0.22457\n",
      " 333184/1202739 [=======>......................] - ETA: 13:50 - loss: 0.0489 - root_mean_squared_error: 0.1449\n",
      "step: 12000 val loss=0.22427, best loss=0.22427\n",
      " 589184/1202739 [=============>................] - ETA: 10:11 - loss: 0.0489 - root_mean_squared_error: 0.1449\n",
      "step: 14000 val loss=0.22392, best loss=0.22392\n",
      " 845184/1202739 [====================>.........] - ETA: 6:02 - loss: 0.0490 - root_mean_squared_error: 0.1451\n",
      "step: 16000 val loss=0.22528, best loss=0.22392\n",
      "1101184/1202739 [==========================>...] - ETA: 1:43 - loss: 0.0491 - root_mean_squared_error: 0.1452\n",
      "step: 18000 val loss=0.22370, best loss=0.22370\n",
      "1202739/1202739 [==============================] - 1393s 1ms/step - loss: 0.0491 - root_mean_squared_error: 0.1453 - val_loss: 0.0500 - val_root_mean_squared_error: 0.1460\n",
      "Epoch 3/4\n",
      "\n",
      "Epoch 00003: LearningRateScheduler reducing learning rate to 0.0001.\n",
      " 154368/1202739 [==>...........................] - ETA: 11:25 - loss: 0.0462 - root_mean_squared_error: 0.1394\n",
      "step: 20000 val loss=0.22340, best loss=0.22340\n",
      " 410368/1202739 [=========>....................] - ETA: 11:54 - loss: 0.0460 - root_mean_squared_error: 0.1387\n",
      "step: 22000 val loss=0.22359, best loss=0.22340\n",
      " 666368/1202739 [===============>..............] - ETA: 8:32 - loss: 0.0458 - root_mean_squared_error: 0.1382\n",
      "step: 24000 val loss=0.22359, best loss=0.22340\n",
      " 922368/1202739 [======================>.......] - ETA: 4:34 - loss: 0.0458 - root_mean_squared_error: 0.1381\n",
      "step: 26000 val loss=0.22365, best loss=0.22340\n",
      "1178368/1202739 [============================>.] - ETA: 24s - loss: 0.0457 - root_mean_squared_error: 0.1380\n",
      "step: 28000 val loss=0.22372, best loss=0.22340\n",
      "1202739/1202739 [==============================] - 1387s 1ms/step - loss: 0.0458 - root_mean_squared_error: 0.1380 - val_loss: 0.0501 - val_root_mean_squared_error: 0.1420\n",
      "Epoch 4/4\n",
      "\n",
      "Epoch 00004: LearningRateScheduler reducing learning rate to 0.0001.\n",
      " 231552/1202739 [====>.........................] - ETA: 10:31 - loss: 0.0441 - root_mean_squared_error: 0.1345\n",
      "step: 30000 val loss=0.22431, best loss=0.22340\n",
      " 487552/1202739 [===========>..................] - ETA: 10:12 - loss: 0.0442 - root_mean_squared_error: 0.1347\n",
      "step: 32000 val loss=0.22442, best loss=0.22340\n",
      " 743552/1202739 [=================>............] - ETA: 7:02 - loss: 0.0443 - root_mean_squared_error: 0.1348\n",
      "step: 34000 val loss=0.22451, best loss=0.22340\n",
      " 999552/1202739 [=======================>......] - ETA: 3:13 - loss: 0.0444 - root_mean_squared_error: 0.1349\n",
      "step: 36000 val loss=0.22483, best loss=0.22340\n",
      "1202739/1202739 [==============================] - 1284s 1ms/step - loss: 0.0443 - root_mean_squared_error: 0.1347 - val_loss: 0.0506 - val_root_mean_squared_error: 0.1448\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_5 (InputLayer)         (None, 38406)             0         \n",
      "_________________________________________________________________\n",
      "dropout_5 (Dropout)          (None, 38406)             0         \n",
      "_________________________________________________________________\n",
      "dense_13 (Dense)             (None, 512)               19664384  \n",
      "_________________________________________________________________\n",
      "dense_14 (Dense)             (None, 128)               65664     \n",
      "_________________________________________________________________\n",
      "dense_15 (Dense)             (None, 1)                 129       \n",
      "=================================================================\n",
      "Total params: 19,730,177\n",
      "Trainable params: 19,730,177\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Train on 1202740 samples, validate on 300684 samples\n",
      "Epoch 1/4\n",
      "\n",
      "Epoch 00001: LearningRateScheduler reducing learning rate to 0.0005.\n",
      " 256000/1202740 [=====>........................] - ETA: 10:18 - loss: 0.0541 - root_mean_squared_error: 0.1561\n",
      "step: 2000 val loss=0.22922, best loss=0.22922\n",
      " 512000/1202740 [===========>..................] - ETA: 9:45 - loss: 0.0530 - root_mean_squared_error: 0.1536\n",
      "step: 4000 val loss=0.22646, best loss=0.22646\n",
      " 768000/1202740 [==================>...........] - ETA: 6:37 - loss: 0.0525 - root_mean_squared_error: 0.1527\n",
      "step: 6000 val loss=0.22571, best loss=0.22571\n",
      "1024000/1202740 [========================>.....] - ETA: 2:49 - loss: 0.0523 - root_mean_squared_error: 0.1522\n",
      "step: 8000 val loss=0.22454, best loss=0.22454\n",
      "1202740/1202740 [==============================] - 1289s 1ms/step - loss: 0.0521 - root_mean_squared_error: 0.1518 - val_loss: 0.0506 - val_root_mean_squared_error: 0.1532\n",
      "Epoch 2/4\n",
      "\n",
      "Epoch 00002: LearningRateScheduler reducing learning rate to 0.0005.\n",
      "  77184/1202740 [>.............................] - ETA: 12:15 - loss: 0.0494 - root_mean_squared_error: 0.1457\n",
      "step: 10000 val loss=0.22539, best loss=0.22454\n",
      " 333184/1202740 [=======>......................] - ETA: 13:48 - loss: 0.0491 - root_mean_squared_error: 0.1453\n",
      "step: 12000 val loss=0.22420, best loss=0.22420\n",
      " 589184/1202740 [=============>................] - ETA: 10:09 - loss: 0.0491 - root_mean_squared_error: 0.1452\n",
      "step: 14000 val loss=0.22458, best loss=0.22420\n",
      " 845184/1202740 [====================>.........] - ETA: 6:00 - loss: 0.0491 - root_mean_squared_error: 0.1452\n",
      "step: 16000 val loss=0.22394, best loss=0.22394\n",
      "1101184/1202740 [==========================>...] - ETA: 1:43 - loss: 0.0491 - root_mean_squared_error: 0.1452\n",
      "step: 18000 val loss=0.22422, best loss=0.22394\n",
      "1202740/1202740 [==============================] - 1388s 1ms/step - loss: 0.0491 - root_mean_squared_error: 0.1452 - val_loss: 0.0500 - val_root_mean_squared_error: 0.1457\n",
      "Epoch 3/4\n",
      "\n",
      "Epoch 00003: LearningRateScheduler reducing learning rate to 0.0001.\n",
      " 154368/1202740 [==>...........................] - ETA: 11:23 - loss: 0.0462 - root_mean_squared_error: 0.1395\n",
      "step: 20000 val loss=0.22327, best loss=0.22327\n",
      " 410368/1202740 [=========>....................] - ETA: 11:51 - loss: 0.0459 - root_mean_squared_error: 0.1384\n",
      "step: 22000 val loss=0.22356, best loss=0.22327\n",
      " 666368/1202740 [===============>..............] - ETA: 8:31 - loss: 0.0457 - root_mean_squared_error: 0.1380\n",
      "step: 24000 val loss=0.22364, best loss=0.22327\n",
      " 922368/1202740 [======================>.......] - ETA: 4:34 - loss: 0.0456 - root_mean_squared_error: 0.1376\n",
      "step: 26000 val loss=0.22397, best loss=0.22327\n",
      "1178368/1202740 [============================>.] - ETA: 24s - loss: 0.0456 - root_mean_squared_error: 0.1376\n",
      "step: 28000 val loss=0.22394, best loss=0.22327\n",
      "1202740/1202740 [==============================] - 1386s 1ms/step - loss: 0.0455 - root_mean_squared_error: 0.1375 - val_loss: 0.0502 - val_root_mean_squared_error: 0.1438\n",
      "Epoch 4/4\n",
      "\n",
      "Epoch 00004: LearningRateScheduler reducing learning rate to 0.0001.\n",
      " 231552/1202740 [====>.........................] - ETA: 10:32 - loss: 0.0437 - root_mean_squared_error: 0.1335\n",
      "step: 30000 val loss=0.22451, best loss=0.22327\n",
      " 487552/1202740 [===========>..................] - ETA: 10:12 - loss: 0.0437 - root_mean_squared_error: 0.1336\n",
      "step: 32000 val loss=0.22474, best loss=0.22327\n",
      " 743552/1202740 [=================>............] - ETA: 7:02 - loss: 0.0438 - root_mean_squared_error: 0.1339\n",
      "step: 34000 val loss=0.22471, best loss=0.22327\n",
      " 999552/1202740 [=======================>......] - ETA: 3:13 - loss: 0.0440 - root_mean_squared_error: 0.1341\n",
      "step: 36000 val loss=0.22480, best loss=0.22327\n",
      "1202740/1202740 [==============================] - 1285s 1ms/step - loss: 0.0440 - root_mean_squared_error: 0.1340 - val_loss: 0.0506 - val_root_mean_squared_error: 0.1426\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "kf = KFold(5, shuffle=True, random_state=SEED)\n",
    "\n",
    "ret = np.zeros((train_X.shape[0],))\n",
    "ret_test = np.zeros((test_X.shape[0],))\n",
    "ret_models = []\n",
    "\n",
    "EPOCHS = 4\n",
    "fold=1\n",
    "for train_ix, val_ix in kf.split(train_X):\n",
    "    \n",
    "    model = build_model(train_X)\n",
    "    file_path = \"mlp_model_{}.hdf5\".format(fold)\n",
    "    #check_point = ModelCheckpoint(file_path, monitor = \"val_loss\", mode = \"min\", save_best_only = True, verbose = 1)\n",
    "    check_point = NBatchEvalLogger(2000, train_X[val_ix,:], train_y[val_ix], save_path=file_path, save_start=2000)\n",
    "    lr_schd = LearningRateScheduler(lambda epoch: 0.0005*(0.2**(epoch//2)), verbose=1)\n",
    "    \n",
    "    history = model.fit(train_X[train_ix,:], train_y[train_ix], batch_size = 128, epochs = EPOCHS, \n",
    "                        validation_data = (train_X[val_ix,:], train_y[val_ix]),\n",
    "                        verbose = 1, callbacks = [lr_schd, check_point])\n",
    "\n",
    "    model.load_weights(file_path)\n",
    "    ret[val_ix] = model.predict(train_X[val_ix,:])\n",
    "    ret_test += model.predict(test_X).reshape((test_X.shape[0],))\n",
    "    ret_models.append(model)\n",
    "    \n",
    "    del model; gc.collect()\n",
    "    fold += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ret_test /= 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.324957  ,  0.1405607 ,  0.21151634, ...,  0.03753489,\n",
       "        0.44956121,  0.0730801 ])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ret_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pd.DataFrame(data=ret, columns=['mlp_pred']).to_csv('mlp_oof_val_pred.csv', index=False)\n",
    "pd.DataFrame(data=ret_test, columns=['mlp_pred']).to_csv('mlp_oof_test_pred.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_df = pd.read_csv(\"test.csv\", usecols=['item_id'])\n",
    "test_pred = pd.read_csv(\"mlp_oof_test_pred.csv\")\n",
    "test_pred = test_pred.loc[:, 'mlp_pred'].values\n",
    "pd.DataFrame(np.clip(test_pred,0,1), \n",
    "             index=test_df.item_id,\n",
    "             columns=['deal_probability']).to_csv('mlp_submission.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ml = model_loader(model_type='lr')\n",
    "fit_param=None\n",
    "from sklearn.metrics import log_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'C': 1.0, 'penalty': 'l2'} train loss: 0.267944, valid loss:0.290776, loss_diff:0.022831\n",
      "{'C': 1.0, 'penalty': 'l2'} train loss: 0.268541, valid loss:0.287768, loss_diff:0.019226\n",
      "{'C': 1.0, 'penalty': 'l2'} train loss: 0.267758, valid loss:0.291122, loss_diff:0.023364\n",
      "{'C': 1.0, 'penalty': 'l2'} train loss: 0.268069, valid loss:0.289864, loss_diff:0.021795\n",
      "{'C': 1.0, 'penalty': 'l2'} train loss: 0.268056, valid loss:0.289615, loss_diff:0.021559\n",
      "=================>{'C': 1.0, 'penalty': 'l2'} loss:0.289829\n",
      "{'C': 2.0, 'penalty': 'l2'} train loss: 0.265380, valid loss:0.291909, loss_diff:0.026529\n",
      "{'C': 2.0, 'penalty': 'l2'} train loss: 0.266135, valid loss:0.288892, loss_diff:0.022757\n",
      "{'C': 2.0, 'penalty': 'l2'} train loss: 0.265244, valid loss:0.292323, loss_diff:0.027080\n",
      "{'C': 2.0, 'penalty': 'l2'} train loss: 0.265557, valid loss:0.290956, loss_diff:0.025399\n",
      "{'C': 2.0, 'penalty': 'l2'} train loss: 0.265533, valid loss:0.290749, loss_diff:0.025216\n",
      "=================>{'C': 2.0, 'penalty': 'l2'} loss:0.290966\n",
      "{'C': 4.0, 'penalty': 'l2'} train loss: 0.263113, valid loss:0.292816, loss_diff:0.029703\n",
      "{'C': 4.0, 'penalty': 'l2'} train loss: 0.263677, valid loss:0.289527, loss_diff:0.025850\n",
      "{'C': 4.0, 'penalty': 'l2'} train loss: 0.262867, valid loss:0.293266, loss_diff:0.030399\n",
      "{'C': 4.0, 'penalty': 'l2'} train loss: 0.263210, valid loss:0.291726, loss_diff:0.028517\n",
      "{'C': 4.0, 'penalty': 'l2'} train loss: 0.263149, valid loss:0.291559, loss_diff:0.028409\n",
      "=================>{'C': 4.0, 'penalty': 'l2'} loss:0.291779\n",
      "{'C': 8.0, 'penalty': 'l2'} train loss: 0.260850, valid loss:0.293304, loss_diff:0.032453\n",
      "{'C': 8.0, 'penalty': 'l2'} train loss: 0.261613, valid loss:0.290136, loss_diff:0.028523\n",
      "{'C': 8.0, 'penalty': 'l2'} train loss: 0.260667, valid loss:0.294024, loss_diff:0.033357\n",
      "{'C': 8.0, 'penalty': 'l2'} train loss: 0.261060, valid loss:0.292279, loss_diff:0.031219\n",
      "{'C': 8.0, 'penalty': 'l2'} train loss: 0.261260, valid loss:0.292383, loss_diff:0.031123\n",
      "=================>{'C': 8.0, 'penalty': 'l2'} loss:0.292425\n",
      "Best params: {'C': 1.0, 'penalty': 'l2'} \tbest loss: 0.289829100387\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>param</th>\n",
       "      <th>val_loss_mean</th>\n",
       "      <th>val_loss_std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>{'C': 1.0, 'penalty': 'l2'}</td>\n",
       "      <td>0.289829</td>\n",
       "      <td>0.001172</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>{'C': 2.0, 'penalty': 'l2'}</td>\n",
       "      <td>0.290966</td>\n",
       "      <td>0.001190</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>{'C': 4.0, 'penalty': 'l2'}</td>\n",
       "      <td>0.291779</td>\n",
       "      <td>0.001297</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>{'C': 8.0, 'penalty': 'l2'}</td>\n",
       "      <td>0.292425</td>\n",
       "      <td>0.001311</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         param  val_loss_mean  val_loss_std\n",
       "0  {'C': 1.0, 'penalty': 'l2'}       0.289829      0.001172\n",
       "1  {'C': 2.0, 'penalty': 'l2'}       0.290966      0.001190\n",
       "2  {'C': 4.0, 'penalty': 'l2'}       0.291779      0.001297\n",
       "3  {'C': 8.0, 'penalty': 'l2'}       0.292425      0.001311"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "default_params = {\n",
    "    'penalty': 'l2', \n",
    "    'fit_intercept':True,\n",
    "    'C': 1.0\n",
    "}\n",
    "\n",
    "try_params = [\n",
    "    {\n",
    "        'penalty': ['l2'],\n",
    "        'C': [1.0, 2., 4., 8.]\n",
    "    }\n",
    "]\n",
    "\n",
    "fit_params(train_X, train_y, ml, default_params, try_params, fit_params=fit_param, \n",
    "           seed=SEED, use_eval_set=False, loss_func=log_loss, predict_proba=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'C': 0.01, 'penalty': 'l2'} train loss: 0.290417, valid loss:0.293783, loss_diff:0.003366\n",
      "{'C': 0.01, 'penalty': 'l2'} train loss: 0.291078, valid loss:0.290997, loss_diff:-0.000082\n",
      "{'C': 0.01, 'penalty': 'l2'} train loss: 0.290301, valid loss:0.294176, loss_diff:0.003874\n",
      "{'C': 0.01, 'penalty': 'l2'} train loss: 0.290592, valid loss:0.293102, loss_diff:0.002510\n",
      "{'C': 0.01, 'penalty': 'l2'} train loss: 0.290668, valid loss:0.292711, loss_diff:0.002043\n",
      "=================>{'C': 0.01, 'penalty': 'l2'} loss:0.292954\n",
      "{'C': 0.1, 'penalty': 'l2'} train loss: 0.278400, valid loss:0.288481, loss_diff:0.010081\n",
      "{'C': 0.1, 'penalty': 'l2'} train loss: 0.279020, valid loss:0.285823, loss_diff:0.006803\n",
      "{'C': 0.1, 'penalty': 'l2'} train loss: 0.278274, valid loss:0.288989, loss_diff:0.010715\n",
      "{'C': 0.1, 'penalty': 'l2'} train loss: 0.278563, valid loss:0.287816, loss_diff:0.009253\n",
      "{'C': 0.1, 'penalty': 'l2'} train loss: 0.278584, valid loss:0.287514, loss_diff:0.008929\n",
      "=================>{'C': 0.1, 'penalty': 'l2'} loss:0.287724\n",
      "{'C': 0.5, 'penalty': 'l2'} train loss: 0.270687, valid loss:0.289516, loss_diff:0.018829\n",
      "{'C': 0.5, 'penalty': 'l2'} train loss: 0.271335, valid loss:0.286658, loss_diff:0.015323\n",
      "{'C': 0.5, 'penalty': 'l2'} train loss: 0.270574, valid loss:0.289947, loss_diff:0.019373\n",
      "{'C': 0.5, 'penalty': 'l2'} train loss: 0.270859, valid loss:0.288711, loss_diff:0.017852\n",
      "{'C': 0.5, 'penalty': 'l2'} train loss: 0.270859, valid loss:0.288450, loss_diff:0.017591\n",
      "=================>{'C': 0.5, 'penalty': 'l2'} loss:0.288656\n",
      "Best params: {'C': 0.1, 'penalty': 'l2'} \tbest loss: 0.287724477278\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>param</th>\n",
       "      <th>val_loss_mean</th>\n",
       "      <th>val_loss_std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>{'C': 0.01, 'penalty': 'l2'}</td>\n",
       "      <td>0.292954</td>\n",
       "      <td>0.001104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>{'C': 0.1, 'penalty': 'l2'}</td>\n",
       "      <td>0.287724</td>\n",
       "      <td>0.001081</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>{'C': 0.5, 'penalty': 'l2'}</td>\n",
       "      <td>0.288656</td>\n",
       "      <td>0.001135</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          param  val_loss_mean  val_loss_std\n",
       "0  {'C': 0.01, 'penalty': 'l2'}       0.292954      0.001104\n",
       "1   {'C': 0.1, 'penalty': 'l2'}       0.287724      0.001081\n",
       "2   {'C': 0.5, 'penalty': 'l2'}       0.288656      0.001135"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "default_params = {\n",
    "    'penalty': 'l2', \n",
    "    'fit_intercept':True,\n",
    "    'C': 1.0\n",
    "}\n",
    "\n",
    "try_params = [\n",
    "    {\n",
    "        'penalty': ['l2'],\n",
    "        'C': [0.01, 0.1, 0.5]\n",
    "    }\n",
    "]\n",
    "\n",
    "fit_params(train_X, train_y, ml, default_params, try_params, fit_params=fit_param, \n",
    "           seed=SEED, use_eval_set=False, loss_func=log_loss, predict_proba=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "default_params = {\n",
    "    'penalty': 'l2', \n",
    "    'fit_intercept':True,\n",
    "    'C': 1.0\n",
    "}\n",
    "\n",
    "try_params = [\n",
    "    {\n",
    "        'penalty': ['l1'],\n",
    "        'C': [1.0, 2., 4., 8.]\n",
    "    }\n",
    "]\n",
    "\n",
    "fit_params(train_X, train_y, ml, default_params, try_params, fit_params=fit_param, \n",
    "           seed=SEED, use_eval_set=False, loss_func=log_loss, predict_proba=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LGBM (Use Dense features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ml = model_loader(model_type='lgb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open('cat_feature_names', 'rb') as handle:\n",
    "    cat_cols = pickle.load(handle)\n",
    "\n",
    "with open('feature_names', 'rb') as handle:\n",
    "    features = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['image_top_1',\n",
       " 'param_1',\n",
       " 'param_2',\n",
       " 'param_3',\n",
       " 'city',\n",
       " 'region',\n",
       " 'category_name',\n",
       " 'parent_category_name',\n",
       " 'user_type']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 50 rounds.\n",
      "[100]\ttrain's rmse: 0.218926\tvalid's rmse: 0.222944\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-10-c9d4a9d925b7>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     29\u001b[0m }\n\u001b[0;32m     30\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 31\u001b[1;33m \u001b[0mfit_params\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrain_X\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrain_y\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mml\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdefault_params\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtry_params\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfit_params\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mfit_param\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mseed\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mSEED\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0muse_eval_set\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mE:\\Kun-Hao Data\\Kaggle Experiments\\Avito\\GridSearcher.py\u001b[0m in \u001b[0;36mfit_params\u001b[1;34m(X, y, model_loader, default_params, try_params, use_eval_set, fit_params, seed, loss_func, predict_proba)\u001b[0m\n\u001b[0;32m    292\u001b[0m                           \u001b[0meval_set\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtr_X\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtr_y\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mval_X\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mval_y\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    293\u001b[0m                           \u001b[0meval_names\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'train'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'valid'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 294\u001b[1;33m                           **fit_params)\n\u001b[0m\u001b[0;32m    295\u001b[0m                 \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    296\u001b[0m                     \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtr_X\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtr_y\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mE:\\Anaconda3\\lib\\site-packages\\lightgbm\\sklearn.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y, sample_weight, init_score, eval_set, eval_names, eval_sample_weight, eval_init_score, eval_metric, early_stopping_rounds, verbose, feature_name, categorical_feature, callbacks)\u001b[0m\n\u001b[0;32m    610\u001b[0m                                        \u001b[0mverbose\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mverbose\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeature_name\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mfeature_name\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    611\u001b[0m                                        \u001b[0mcategorical_feature\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcategorical_feature\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 612\u001b[1;33m                                        callbacks=callbacks)\n\u001b[0m\u001b[0;32m    613\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    614\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mE:\\Anaconda3\\lib\\site-packages\\lightgbm\\sklearn.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y, sample_weight, init_score, group, eval_set, eval_names, eval_sample_weight, eval_init_score, eval_group, eval_metric, early_stopping_rounds, verbose, feature_name, categorical_feature, callbacks)\u001b[0m\n\u001b[0;32m    457\u001b[0m                               \u001b[0mverbose_eval\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mverbose\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeature_name\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mfeature_name\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    458\u001b[0m                               \u001b[0mcategorical_feature\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcategorical_feature\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 459\u001b[1;33m                               callbacks=callbacks)\n\u001b[0m\u001b[0;32m    460\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    461\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mevals_result\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mE:\\Anaconda3\\lib\\site-packages\\lightgbm\\engine.py\u001b[0m in \u001b[0;36mtrain\u001b[1;34m(params, train_set, num_boost_round, valid_sets, valid_names, fobj, feval, init_model, feature_name, categorical_feature, early_stopping_rounds, evals_result, verbose_eval, learning_rates, keep_training_booster, callbacks)\u001b[0m\n\u001b[0;32m    197\u001b[0m                                     evaluation_result_list=None))\n\u001b[0;32m    198\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 199\u001b[1;33m         \u001b[0mbooster\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfobj\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mfobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    200\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    201\u001b[0m         \u001b[0mevaluation_result_list\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mE:\\Anaconda3\\lib\\site-packages\\lightgbm\\basic.py\u001b[0m in \u001b[0;36mupdate\u001b[1;34m(self, train_set, fobj)\u001b[0m\n\u001b[0;32m   1437\u001b[0m             _safe_call(_LIB.LGBM_BoosterUpdateOneIter(\n\u001b[0;32m   1438\u001b[0m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1439\u001b[1;33m                 ctypes.byref(is_finished)))\n\u001b[0m\u001b[0;32m   1440\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__is_predicted_cur_iter\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;32mFalse\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0m_\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange_\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__num_dataset\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1441\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mis_finished\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalue\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "default_params = {\n",
    "    'boosting_type':'gbdt', \n",
    "    'num_leaves':31, \n",
    "    'max_depth':5, \n",
    "    'learning_rate':0.3, \n",
    "    'n_estimators':2000, \n",
    "    'min_split_gain':0.0, \n",
    "    'min_child_weight':0.001, \n",
    "    'min_child_samples':20, \n",
    "    'subsample':1., \n",
    "    'subsample_freq':1, \n",
    "    'colsample_bytree':.6, \n",
    "    'reg_alpha':0.0, \n",
    "    'reg_lambda':0.0, \n",
    "    'random_state':SEED, \n",
    "    'n_jobs': 3\n",
    "}\n",
    "\n",
    "fit_param = {\n",
    "    'feature_name': list(features), \n",
    "    'categorical_feature': cat_cols,\n",
    "    'early_stopping_rounds': 50,\n",
    "    'verbose': 100,\n",
    "    'eval_metric': 'rmse'\n",
    "}\n",
    "\n",
    "try_params = {\n",
    "    'random_state': [719]\n",
    "}\n",
    "\n",
    "fit_params(train_X, train_y, ml, default_params, try_params, fit_params=fit_param, seed=SEED, use_eval_set=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\nfit_param.update({\\n    'early_stopping_rounds': 30,\\n    'verbose': 100,\\n    'eval_metric': 'rmse'\\n})\\n\""
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fit_param = {\n",
    "    'feature_name': list(features), \n",
    "    'categorical_feature': cat_cols,\n",
    "}\n",
    "'''\n",
    "fit_param.update({\n",
    "    'early_stopping_rounds': 30,\n",
    "    'verbose': 100,\n",
    "    'eval_metric': 'rmse'\n",
    "})\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'max_depth': 5} train loss: 0.218505, valid loss:0.223704, loss_diff:0.005199\n",
      "{'max_depth': 5} train loss: 0.219042, valid loss:0.222470, loss_diff:0.003428\n",
      "{'max_depth': 5} train loss: 0.218784, valid loss:0.223454, loss_diff:0.004670\n",
      "{'max_depth': 5} train loss: 0.219033, valid loss:0.222853, loss_diff:0.003820\n",
      "{'max_depth': 5} train loss: 0.219184, valid loss:0.223251, loss_diff:0.004068\n",
      "=================>{'max_depth': 5} loss:0.223146\n",
      "{'max_depth': 6} train loss: 0.217569, valid loss:0.223429, loss_diff:0.005860\n",
      "{'max_depth': 6} train loss: 0.217589, valid loss:0.221967, loss_diff:0.004378\n",
      "{'max_depth': 6} train loss: 0.217316, valid loss:0.223030, loss_diff:0.005715\n",
      "{'max_depth': 6} train loss: 0.217332, valid loss:0.222378, loss_diff:0.005047\n",
      "{'max_depth': 6} train loss: 0.217610, valid loss:0.222590, loss_diff:0.004980\n",
      "=================>{'max_depth': 6} loss:0.222679\n",
      "{'max_depth': 8} train loss: 0.215871, valid loss:0.222551, loss_diff:0.006680\n",
      "{'max_depth': 8} train loss: 0.216085, valid loss:0.221391, loss_diff:0.005306\n",
      "{'max_depth': 8} train loss: 0.215925, valid loss:0.222422, loss_diff:0.006497\n",
      "{'max_depth': 8} train loss: 0.216365, valid loss:0.221527, loss_diff:0.005163\n",
      "{'max_depth': 8} train loss: 0.216189, valid loss:0.222131, loss_diff:0.005942\n",
      "=================>{'max_depth': 8} loss:0.222005\n",
      "Best params: {'max_depth': 8} \tbest loss: 0.222004558559\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>param</th>\n",
       "      <th>val_loss_mean</th>\n",
       "      <th>val_loss_std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>{'max_depth': 5}</td>\n",
       "      <td>0.223146</td>\n",
       "      <td>0.000438</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>{'max_depth': 6}</td>\n",
       "      <td>0.222679</td>\n",
       "      <td>0.000508</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>{'max_depth': 8}</td>\n",
       "      <td>0.222005</td>\n",
       "      <td>0.000468</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              param  val_loss_mean  val_loss_std\n",
       "0  {'max_depth': 5}       0.223146      0.000438\n",
       "1  {'max_depth': 6}       0.222679      0.000508\n",
       "2  {'max_depth': 8}       0.222005      0.000468"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "default_params = {\n",
    "    'boosting_type':'gbdt', \n",
    "    'num_leaves':31, \n",
    "    'max_depth':5, \n",
    "    'learning_rate':0.3, \n",
    "    'n_estimators':100, \n",
    "    'min_split_gain':0.0, \n",
    "    'min_child_weight':0.001, \n",
    "    'min_child_samples':20, \n",
    "    'subsample':1., \n",
    "    'subsample_freq':1, \n",
    "    'colsample_bytree':.6, \n",
    "    'reg_alpha':0.0, \n",
    "    'reg_lambda':0.0, \n",
    "    'random_state':SEED, \n",
    "    'n_jobs': 3\n",
    "}\n",
    "\n",
    "\n",
    "try_params = {\n",
    "    'max_depth': [5, 6, 8]\n",
    "}\n",
    "\n",
    "fit_params(train_X, train_y, ml, default_params, try_params, fit_params=fit_param, seed=SEED, use_eval_set=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'min_split_gain': 0.0} train loss: 0.215871, valid loss:0.222551, loss_diff:0.006680\n",
      "{'min_split_gain': 0.0} train loss: 0.216085, valid loss:0.221391, loss_diff:0.005306\n",
      "{'min_split_gain': 0.0} train loss: 0.215925, valid loss:0.222422, loss_diff:0.006497\n",
      "{'min_split_gain': 0.0} train loss: 0.216365, valid loss:0.221527, loss_diff:0.005163\n",
      "{'min_split_gain': 0.0} train loss: 0.216189, valid loss:0.222131, loss_diff:0.005942\n",
      "=================>{'min_split_gain': 0.0} loss:0.222005\n",
      "{'min_split_gain': 0.1} train loss: 0.215889, valid loss:0.222735, loss_diff:0.006846\n",
      "{'min_split_gain': 0.1} train loss: 0.216071, valid loss:0.221491, loss_diff:0.005419\n",
      "{'min_split_gain': 0.1} train loss: 0.216219, valid loss:0.222521, loss_diff:0.006302\n",
      "{'min_split_gain': 0.1} train loss: 0.216482, valid loss:0.221848, loss_diff:0.005367\n",
      "{'min_split_gain': 0.1} train loss: 0.216082, valid loss:0.222172, loss_diff:0.006090\n",
      "=================>{'min_split_gain': 0.1} loss:0.222154\n",
      "{'min_split_gain': 0.2} train loss: 0.215611, valid loss:0.222658, loss_diff:0.007047\n",
      "{'min_split_gain': 0.2} train loss: 0.216259, valid loss:0.221452, loss_diff:0.005194\n",
      "{'min_split_gain': 0.2} train loss: 0.216251, valid loss:0.222512, loss_diff:0.006261\n",
      "{'min_split_gain': 0.2} train loss: 0.216553, valid loss:0.221854, loss_diff:0.005302\n",
      "{'min_split_gain': 0.2} train loss: 0.216158, valid loss:0.222111, loss_diff:0.005953\n",
      "=================>{'min_split_gain': 0.2} loss:0.222118\n",
      "{'min_split_gain': 0.3} train loss: 0.215782, valid loss:0.222724, loss_diff:0.006942\n",
      "{'min_split_gain': 0.3} train loss: 0.216362, valid loss:0.221556, loss_diff:0.005194\n",
      "{'min_split_gain': 0.3} train loss: 0.216068, valid loss:0.222538, loss_diff:0.006470\n",
      "{'min_split_gain': 0.3} train loss: 0.216413, valid loss:0.221707, loss_diff:0.005294\n",
      "{'min_split_gain': 0.3} train loss: 0.216364, valid loss:0.222114, loss_diff:0.005750\n",
      "=================>{'min_split_gain': 0.3} loss:0.222128\n",
      "{'min_split_gain': 0.4} train loss: 0.216199, valid loss:0.222857, loss_diff:0.006657\n",
      "{'min_split_gain': 0.4} train loss: 0.215958, valid loss:0.221344, loss_diff:0.005386\n",
      "{'min_split_gain': 0.4} train loss: 0.216123, valid loss:0.222406, loss_diff:0.006282\n",
      "{'min_split_gain': 0.4} train loss: 0.216610, valid loss:0.221837, loss_diff:0.005228\n",
      "{'min_split_gain': 0.4} train loss: 0.216277, valid loss:0.222175, loss_diff:0.005897\n",
      "=================>{'min_split_gain': 0.4} loss:0.222124\n",
      "Best params: {'min_split_gain': 0.0} \tbest loss: 0.222004558559\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>param</th>\n",
       "      <th>val_loss_mean</th>\n",
       "      <th>val_loss_std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>{'min_split_gain': 0.0}</td>\n",
       "      <td>0.222005</td>\n",
       "      <td>0.000468</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>{'min_split_gain': 0.1}</td>\n",
       "      <td>0.222154</td>\n",
       "      <td>0.000449</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>{'min_split_gain': 0.2}</td>\n",
       "      <td>0.222118</td>\n",
       "      <td>0.000438</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>{'min_split_gain': 0.3}</td>\n",
       "      <td>0.222128</td>\n",
       "      <td>0.000453</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>{'min_split_gain': 0.4}</td>\n",
       "      <td>0.222124</td>\n",
       "      <td>0.000512</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     param  val_loss_mean  val_loss_std\n",
       "0  {'min_split_gain': 0.0}       0.222005      0.000468\n",
       "1  {'min_split_gain': 0.1}       0.222154      0.000449\n",
       "2  {'min_split_gain': 0.2}       0.222118      0.000438\n",
       "3  {'min_split_gain': 0.3}       0.222128      0.000453\n",
       "4  {'min_split_gain': 0.4}       0.222124      0.000512"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "default_params = {\n",
    "    'boosting_type':'gbdt', \n",
    "    'num_leaves':31, \n",
    "    'max_depth':8, \n",
    "    'learning_rate':0.3, \n",
    "    'n_estimators':100, \n",
    "    'min_split_gain':0.0, \n",
    "    'min_child_weight':0.001, \n",
    "    'min_child_samples':20, \n",
    "    'subsample':1., \n",
    "    'subsample_freq':1, \n",
    "    'colsample_bytree':.6, \n",
    "    'reg_alpha':0.0, \n",
    "    'reg_lambda':0.0, \n",
    "    'random_state':SEED, \n",
    "    'n_jobs': 3\n",
    "}\n",
    "\n",
    "try_params = {\n",
    "    'min_split_gain': [i/10.0 for i in range(0,5)]\n",
    "}\n",
    "\n",
    "fit_params(train_X, train_y, ml, default_params, try_params, fit_params=fit_param, seed=SEED, use_eval_set=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'colsample_bytree': 0.6} train loss: 0.215871, valid loss:0.222551, loss_diff:0.006680\n",
      "{'colsample_bytree': 0.6} train loss: 0.216085, valid loss:0.221391, loss_diff:0.005306\n",
      "{'colsample_bytree': 0.6} train loss: 0.215925, valid loss:0.222422, loss_diff:0.006497\n",
      "{'colsample_bytree': 0.6} train loss: 0.216365, valid loss:0.221527, loss_diff:0.005163\n",
      "{'colsample_bytree': 0.6} train loss: 0.216189, valid loss:0.222131, loss_diff:0.005942\n",
      "=================>{'colsample_bytree': 0.6} loss:0.222005\n",
      "{'colsample_bytree': 0.7} train loss: 0.215871, valid loss:0.222641, loss_diff:0.006770\n",
      "{'colsample_bytree': 0.7} train loss: 0.216165, valid loss:0.221512, loss_diff:0.005347\n",
      "{'colsample_bytree': 0.7} train loss: 0.215783, valid loss:0.222465, loss_diff:0.006682\n",
      "{'colsample_bytree': 0.7} train loss: 0.216138, valid loss:0.221625, loss_diff:0.005487\n",
      "{'colsample_bytree': 0.7} train loss: 0.216353, valid loss:0.222102, loss_diff:0.005748\n",
      "=================>{'colsample_bytree': 0.7} loss:0.222069\n",
      "{'colsample_bytree': 0.8} train loss: 0.215447, valid loss:0.222617, loss_diff:0.007171\n",
      "{'colsample_bytree': 0.8} train loss: 0.216165, valid loss:0.221426, loss_diff:0.005261\n",
      "{'colsample_bytree': 0.8} train loss: 0.215445, valid loss:0.222445, loss_diff:0.007000\n",
      "{'colsample_bytree': 0.8} train loss: 0.215503, valid loss:0.221694, loss_diff:0.006191\n",
      "{'colsample_bytree': 0.8} train loss: 0.215468, valid loss:0.222147, loss_diff:0.006679\n",
      "=================>{'colsample_bytree': 0.8} loss:0.222066\n",
      "{'colsample_bytree': 0.9} train loss: 0.215318, valid loss:0.222722, loss_diff:0.007404\n",
      "{'colsample_bytree': 0.9} train loss: 0.215407, valid loss:0.221294, loss_diff:0.005887\n",
      "{'colsample_bytree': 0.9} train loss: 0.215664, valid loss:0.222581, loss_diff:0.006917\n",
      "{'colsample_bytree': 0.9} train loss: 0.215636, valid loss:0.221726, loss_diff:0.006090\n",
      "{'colsample_bytree': 0.9} train loss: 0.215372, valid loss:0.222047, loss_diff:0.006675\n",
      "=================>{'colsample_bytree': 0.9} loss:0.222074\n",
      "{'colsample_bytree': 1.0} train loss: 0.215355, valid loss:0.222799, loss_diff:0.007444\n",
      "{'colsample_bytree': 1.0} train loss: 0.215285, valid loss:0.221326, loss_diff:0.006041\n",
      "{'colsample_bytree': 1.0} train loss: 0.215828, valid loss:0.222825, loss_diff:0.006997\n",
      "{'colsample_bytree': 1.0} train loss: 0.215559, valid loss:0.221826, loss_diff:0.006267\n",
      "{'colsample_bytree': 1.0} train loss: 0.215375, valid loss:0.222113, loss_diff:0.006738\n",
      "=================>{'colsample_bytree': 1.0} loss:0.222178\n",
      "Best params: {'colsample_bytree': 0.6} \tbest loss: 0.222004558559\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>param</th>\n",
       "      <th>val_loss_mean</th>\n",
       "      <th>val_loss_std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>{'colsample_bytree': 0.6}</td>\n",
       "      <td>0.222005</td>\n",
       "      <td>0.000468</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>{'colsample_bytree': 0.7}</td>\n",
       "      <td>0.222069</td>\n",
       "      <td>0.000446</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>{'colsample_bytree': 0.8}</td>\n",
       "      <td>0.222066</td>\n",
       "      <td>0.000448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>{'colsample_bytree': 0.9}</td>\n",
       "      <td>0.222074</td>\n",
       "      <td>0.000531</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>{'colsample_bytree': 1.0}</td>\n",
       "      <td>0.222178</td>\n",
       "      <td>0.000576</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       param  val_loss_mean  val_loss_std\n",
       "0  {'colsample_bytree': 0.6}       0.222005      0.000468\n",
       "1  {'colsample_bytree': 0.7}       0.222069      0.000446\n",
       "2  {'colsample_bytree': 0.8}       0.222066      0.000448\n",
       "3  {'colsample_bytree': 0.9}       0.222074      0.000531\n",
       "4  {'colsample_bytree': 1.0}       0.222178      0.000576"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "default_params = {\n",
    "    'boosting_type':'gbdt', \n",
    "    'num_leaves':31, \n",
    "    'max_depth':8, \n",
    "    'learning_rate':0.3, \n",
    "    'n_estimators':100, \n",
    "    'min_split_gain':0.0, \n",
    "    'min_child_weight':0.001, \n",
    "    'min_child_samples':20, \n",
    "    'subsample':1., \n",
    "    'subsample_freq':1, \n",
    "    'colsample_bytree':.6, \n",
    "    'reg_alpha':0.0, \n",
    "    'reg_lambda':0.0, \n",
    "    'random_state':SEED, \n",
    "    'n_jobs': 3\n",
    "}\n",
    "\n",
    "try_params = {\n",
    "    'colsample_bytree':[i/10.0 for i in range(6,11)]\n",
    "}\n",
    "\n",
    "fit_params(train_X, train_y, ml, default_params, try_params, fit_params=fit_param, seed=SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'colsample_bytree': 0.55} train loss: 0.216380, valid loss:0.222791, loss_diff:0.006411\n",
      "{'colsample_bytree': 0.55} train loss: 0.216794, valid loss:0.221431, loss_diff:0.004637\n",
      "{'colsample_bytree': 0.55} train loss: 0.216700, valid loss:0.222528, loss_diff:0.005827\n",
      "{'colsample_bytree': 0.55} train loss: 0.216361, valid loss:0.221781, loss_diff:0.005420\n",
      "{'colsample_bytree': 0.55} train loss: 0.216375, valid loss:0.222048, loss_diff:0.005672\n",
      "=================>{'colsample_bytree': 0.55} loss:0.222116\n",
      "{'colsample_bytree': 0.6} train loss: 0.215871, valid loss:0.222551, loss_diff:0.006680\n",
      "{'colsample_bytree': 0.6} train loss: 0.216085, valid loss:0.221391, loss_diff:0.005306\n",
      "{'colsample_bytree': 0.6} train loss: 0.215925, valid loss:0.222422, loss_diff:0.006497\n",
      "{'colsample_bytree': 0.6} train loss: 0.216365, valid loss:0.221527, loss_diff:0.005163\n",
      "{'colsample_bytree': 0.6} train loss: 0.216189, valid loss:0.222131, loss_diff:0.005942\n",
      "=================>{'colsample_bytree': 0.6} loss:0.222005\n",
      "{'colsample_bytree': 0.65} train loss: 0.215828, valid loss:0.222644, loss_diff:0.006817\n",
      "{'colsample_bytree': 0.65} train loss: 0.216443, valid loss:0.221420, loss_diff:0.004977\n",
      "{'colsample_bytree': 0.65} train loss: 0.216107, valid loss:0.222433, loss_diff:0.006326\n",
      "{'colsample_bytree': 0.65} train loss: 0.216353, valid loss:0.221622, loss_diff:0.005268\n",
      "{'colsample_bytree': 0.65} train loss: 0.216007, valid loss:0.222125, loss_diff:0.006117\n",
      "=================>{'colsample_bytree': 0.65} loss:0.222049\n",
      "Best params: {'colsample_bytree': 0.6} \tbest loss: 0.222004558559\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>param</th>\n",
       "      <th>val_loss_mean</th>\n",
       "      <th>val_loss_std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>{'colsample_bytree': 0.55}</td>\n",
       "      <td>0.222116</td>\n",
       "      <td>0.000492</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>{'colsample_bytree': 0.6}</td>\n",
       "      <td>0.222005</td>\n",
       "      <td>0.000468</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>{'colsample_bytree': 0.65}</td>\n",
       "      <td>0.222049</td>\n",
       "      <td>0.000466</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        param  val_loss_mean  val_loss_std\n",
       "0  {'colsample_bytree': 0.55}       0.222116      0.000492\n",
       "1   {'colsample_bytree': 0.6}       0.222005      0.000468\n",
       "2  {'colsample_bytree': 0.65}       0.222049      0.000466"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "default_params = {\n",
    "    'boosting_type':'gbdt', \n",
    "    'num_leaves':31, \n",
    "    'max_depth':8, \n",
    "    'learning_rate':0.3, \n",
    "    'n_estimators':100, \n",
    "    'min_split_gain':0.0, \n",
    "    'min_child_weight':0.001, \n",
    "    'min_child_samples':20, \n",
    "    'subsample':1., \n",
    "    'subsample_freq':1, \n",
    "    'colsample_bytree':.6, \n",
    "    'reg_alpha':0.0, \n",
    "    'reg_lambda':0.0, \n",
    "    'random_state':SEED, \n",
    "    'n_jobs': 3\n",
    "}\n",
    "\n",
    "\n",
    "try_params = {\n",
    "    'colsample_bytree':[.55, .6, .65]\n",
    "}\n",
    "\n",
    "fit_params(train_X, train_y, ml, default_params, try_params, fit_params=fit_param, seed=SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'subsample': 0.59999999999999998} train loss: 0.218250, valid loss:0.223976, loss_diff:0.005726\n",
      "{'subsample': 0.59999999999999998} train loss: 0.218481, valid loss:0.222729, loss_diff:0.004248\n",
      "{'subsample': 0.59999999999999998} train loss: 0.218495, valid loss:0.223856, loss_diff:0.005361\n",
      "{'subsample': 0.59999999999999998} train loss: 0.218637, valid loss:0.223302, loss_diff:0.004666\n",
      "{'subsample': 0.59999999999999998} train loss: 0.218761, valid loss:0.223491, loss_diff:0.004730\n",
      "=================>{'subsample': 0.59999999999999998} loss:0.223471\n",
      "{'subsample': 0.65000000000000002} train loss: 0.217739, valid loss:0.223566, loss_diff:0.005827\n",
      "{'subsample': 0.65000000000000002} train loss: 0.217995, valid loss:0.222441, loss_diff:0.004445\n",
      "{'subsample': 0.65000000000000002} train loss: 0.218073, valid loss:0.223443, loss_diff:0.005371\n",
      "{'subsample': 0.65000000000000002} train loss: 0.218392, valid loss:0.222925, loss_diff:0.004533\n",
      "{'subsample': 0.65000000000000002} train loss: 0.217732, valid loss:0.223072, loss_diff:0.005340\n",
      "=================>{'subsample': 0.65000000000000002} loss:0.223089\n",
      "{'subsample': 0.70000000000000007} train loss: 0.217250, valid loss:0.223442, loss_diff:0.006193\n",
      "{'subsample': 0.70000000000000007} train loss: 0.217713, valid loss:0.222291, loss_diff:0.004577\n",
      "{'subsample': 0.70000000000000007} train loss: 0.217757, valid loss:0.223420, loss_diff:0.005663\n",
      "{'subsample': 0.70000000000000007} train loss: 0.217654, valid loss:0.222620, loss_diff:0.004966\n",
      "{'subsample': 0.70000000000000007} train loss: 0.217789, valid loss:0.222963, loss_diff:0.005174\n",
      "=================>{'subsample': 0.70000000000000007} loss:0.222947\n",
      "{'subsample': 0.75000000000000011} train loss: 0.217025, valid loss:0.223338, loss_diff:0.006313\n",
      "{'subsample': 0.75000000000000011} train loss: 0.217533, valid loss:0.222300, loss_diff:0.004768\n",
      "{'subsample': 0.75000000000000011} train loss: 0.217306, valid loss:0.223258, loss_diff:0.005952\n",
      "{'subsample': 0.75000000000000011} train loss: 0.217665, valid loss:0.222551, loss_diff:0.004886\n",
      "{'subsample': 0.75000000000000011} train loss: 0.217182, valid loss:0.222646, loss_diff:0.005463\n",
      "=================>{'subsample': 0.75000000000000011} loss:0.222819\n",
      "{'subsample': 0.80000000000000016} train loss: 0.216945, valid loss:0.223157, loss_diff:0.006212\n",
      "{'subsample': 0.80000000000000016} train loss: 0.217240, valid loss:0.221878, loss_diff:0.004639\n",
      "{'subsample': 0.80000000000000016} train loss: 0.216928, valid loss:0.222793, loss_diff:0.005865\n",
      "{'subsample': 0.80000000000000016} train loss: 0.217134, valid loss:0.222293, loss_diff:0.005159\n",
      "{'subsample': 0.80000000000000016} train loss: 0.217080, valid loss:0.222611, loss_diff:0.005531\n",
      "=================>{'subsample': 0.80000000000000016} loss:0.222547\n",
      "{'subsample': 0.8500000000000002} train loss: 0.216458, valid loss:0.222971, loss_diff:0.006512\n",
      "{'subsample': 0.8500000000000002} train loss: 0.216817, valid loss:0.221764, loss_diff:0.004947\n",
      "{'subsample': 0.8500000000000002} train loss: 0.216727, valid loss:0.222854, loss_diff:0.006127\n",
      "{'subsample': 0.8500000000000002} train loss: 0.217027, valid loss:0.222181, loss_diff:0.005154\n",
      "{'subsample': 0.8500000000000002} train loss: 0.216846, valid loss:0.222372, loss_diff:0.005526\n",
      "=================>{'subsample': 0.8500000000000002} loss:0.222428\n",
      "{'subsample': 0.90000000000000024} train loss: 0.216192, valid loss:0.222810, loss_diff:0.006618\n",
      "{'subsample': 0.90000000000000024} train loss: 0.216774, valid loss:0.221707, loss_diff:0.004932\n",
      "{'subsample': 0.90000000000000024} train loss: 0.216482, valid loss:0.222684, loss_diff:0.006202\n",
      "{'subsample': 0.90000000000000024} train loss: 0.216777, valid loss:0.222068, loss_diff:0.005291\n",
      "{'subsample': 0.90000000000000024} train loss: 0.216596, valid loss:0.222526, loss_diff:0.005930\n",
      "=================>{'subsample': 0.90000000000000024} loss:0.222359\n",
      "{'subsample': 0.95000000000000029} train loss: 0.216135, valid loss:0.222752, loss_diff:0.006617\n",
      "{'subsample': 0.95000000000000029} train loss: 0.216745, valid loss:0.221564, loss_diff:0.004819\n",
      "{'subsample': 0.95000000000000029} train loss: 0.216305, valid loss:0.222484, loss_diff:0.006179\n",
      "{'subsample': 0.95000000000000029} train loss: 0.216636, valid loss:0.221985, loss_diff:0.005348\n",
      "{'subsample': 0.95000000000000029} train loss: 0.216686, valid loss:0.222148, loss_diff:0.005462\n",
      "=================>{'subsample': 0.95000000000000029} loss:0.222186\n",
      "{'subsample': 1.0000000000000004} train loss: 0.215871, valid loss:0.222551, loss_diff:0.006680\n",
      "{'subsample': 1.0000000000000004} train loss: 0.216085, valid loss:0.221391, loss_diff:0.005306\n",
      "{'subsample': 1.0000000000000004} train loss: 0.215925, valid loss:0.222422, loss_diff:0.006497\n",
      "{'subsample': 1.0000000000000004} train loss: 0.216365, valid loss:0.221527, loss_diff:0.005163\n",
      "{'subsample': 1.0000000000000004} train loss: 0.216189, valid loss:0.222131, loss_diff:0.005942\n",
      "=================>{'subsample': 1.0000000000000004} loss:0.222005\n",
      "Best params: {'subsample': 1.0000000000000004} \tbest loss: 0.222004558559\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>param</th>\n",
       "      <th>val_loss_mean</th>\n",
       "      <th>val_loss_std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>{'subsample': 0.59999999999999998}</td>\n",
       "      <td>0.223471</td>\n",
       "      <td>0.000443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>{'subsample': 0.65000000000000002}</td>\n",
       "      <td>0.223089</td>\n",
       "      <td>0.000400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>{'subsample': 0.70000000000000007}</td>\n",
       "      <td>0.222947</td>\n",
       "      <td>0.000449</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>{'subsample': 0.75000000000000011}</td>\n",
       "      <td>0.222819</td>\n",
       "      <td>0.000408</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>{'subsample': 0.80000000000000016}</td>\n",
       "      <td>0.222547</td>\n",
       "      <td>0.000435</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>{'subsample': 0.8500000000000002}</td>\n",
       "      <td>0.222428</td>\n",
       "      <td>0.000443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>{'subsample': 0.90000000000000024}</td>\n",
       "      <td>0.222359</td>\n",
       "      <td>0.000412</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>{'subsample': 0.95000000000000029}</td>\n",
       "      <td>0.222186</td>\n",
       "      <td>0.000409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>{'subsample': 1.0000000000000004}</td>\n",
       "      <td>0.222005</td>\n",
       "      <td>0.000468</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                param  val_loss_mean  val_loss_std\n",
       "0  {'subsample': 0.59999999999999998}       0.223471      0.000443\n",
       "1  {'subsample': 0.65000000000000002}       0.223089      0.000400\n",
       "2  {'subsample': 0.70000000000000007}       0.222947      0.000449\n",
       "3  {'subsample': 0.75000000000000011}       0.222819      0.000408\n",
       "4  {'subsample': 0.80000000000000016}       0.222547      0.000435\n",
       "5   {'subsample': 0.8500000000000002}       0.222428      0.000443\n",
       "6  {'subsample': 0.90000000000000024}       0.222359      0.000412\n",
       "7  {'subsample': 0.95000000000000029}       0.222186      0.000409\n",
       "8   {'subsample': 1.0000000000000004}       0.222005      0.000468"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "default_params = {\n",
    "    'boosting_type':'gbdt', \n",
    "    'num_leaves':31, \n",
    "    'max_depth':8, \n",
    "    'learning_rate':0.3, \n",
    "    'n_estimators':100, \n",
    "    'min_split_gain':0.0, \n",
    "    'min_child_weight':0.001, \n",
    "    'min_child_samples':20, \n",
    "    'subsample':1., \n",
    "    'subsample_freq':1, \n",
    "    'colsample_bytree':.6, \n",
    "    'reg_alpha':0.0, \n",
    "    'reg_lambda':0.0, \n",
    "    'random_state':SEED, \n",
    "    'n_jobs': 3\n",
    "}\n",
    "\n",
    "try_params = {\n",
    "    'subsample': np.arange(.6, 1.05, .05)\n",
    "}\n",
    "\n",
    "fit_params(train_X, train_y, ml, default_params, try_params, fit_params=fit_param, seed=SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'reg_alpha': 0.0} train loss: 0.215871, valid loss:0.222551, loss_diff:0.006680\n",
      "{'reg_alpha': 0.0} train loss: 0.216085, valid loss:0.221391, loss_diff:0.005306\n",
      "{'reg_alpha': 0.0} train loss: 0.215925, valid loss:0.222422, loss_diff:0.006497\n",
      "{'reg_alpha': 0.0} train loss: 0.216365, valid loss:0.221527, loss_diff:0.005163\n",
      "{'reg_alpha': 0.0} train loss: 0.216189, valid loss:0.222131, loss_diff:0.005942\n",
      "=================>{'reg_alpha': 0.0} loss:0.222005\n",
      "{'reg_alpha': 0.1} train loss: 0.215805, valid loss:0.222623, loss_diff:0.006818\n",
      "{'reg_alpha': 0.1} train loss: 0.216183, valid loss:0.221403, loss_diff:0.005220\n",
      "{'reg_alpha': 0.1} train loss: 0.216018, valid loss:0.222417, loss_diff:0.006398\n",
      "{'reg_alpha': 0.1} train loss: 0.216569, valid loss:0.221808, loss_diff:0.005239\n",
      "{'reg_alpha': 0.1} train loss: 0.216045, valid loss:0.221995, loss_diff:0.005950\n",
      "=================>{'reg_alpha': 0.1} loss:0.222049\n",
      "{'reg_alpha': 0.2} train loss: 0.215623, valid loss:0.222629, loss_diff:0.007007\n",
      "{'reg_alpha': 0.2} train loss: 0.216231, valid loss:0.221514, loss_diff:0.005283\n",
      "{'reg_alpha': 0.2} train loss: 0.216248, valid loss:0.222519, loss_diff:0.006271\n",
      "{'reg_alpha': 0.2} train loss: 0.216226, valid loss:0.221914, loss_diff:0.005689\n",
      "{'reg_alpha': 0.2} train loss: 0.215946, valid loss:0.221961, loss_diff:0.006015\n",
      "=================>{'reg_alpha': 0.2} loss:0.222108\n",
      "{'reg_alpha': 0.3} train loss: 0.216116, valid loss:0.222654, loss_diff:0.006537\n",
      "{'reg_alpha': 0.3} train loss: 0.216375, valid loss:0.221627, loss_diff:0.005252\n",
      "{'reg_alpha': 0.3} train loss: 0.215923, valid loss:0.222453, loss_diff:0.006530\n",
      "{'reg_alpha': 0.3} train loss: 0.216055, valid loss:0.221728, loss_diff:0.005672\n",
      "{'reg_alpha': 0.3} train loss: 0.216045, valid loss:0.222007, loss_diff:0.005962\n",
      "=================>{'reg_alpha': 0.3} loss:0.222093\n",
      "{'reg_alpha': 0.4} train loss: 0.216254, valid loss:0.222644, loss_diff:0.006389\n",
      "{'reg_alpha': 0.4} train loss: 0.216329, valid loss:0.221515, loss_diff:0.005187\n",
      "{'reg_alpha': 0.4} train loss: 0.216359, valid loss:0.222530, loss_diff:0.006171\n",
      "{'reg_alpha': 0.4} train loss: 0.215997, valid loss:0.221742, loss_diff:0.005745\n",
      "{'reg_alpha': 0.4} train loss: 0.216066, valid loss:0.222176, loss_diff:0.006110\n",
      "=================>{'reg_alpha': 0.4} loss:0.222121\n",
      "{'reg_alpha': 0.5} train loss: 0.216061, valid loss:0.222932, loss_diff:0.006872\n",
      "{'reg_alpha': 0.5} train loss: 0.216021, valid loss:0.221338, loss_diff:0.005316\n",
      "{'reg_alpha': 0.5} train loss: 0.216155, valid loss:0.222473, loss_diff:0.006318\n",
      "{'reg_alpha': 0.5} train loss: 0.216085, valid loss:0.221539, loss_diff:0.005454\n",
      "{'reg_alpha': 0.5} train loss: 0.216263, valid loss:0.221950, loss_diff:0.005688\n",
      "=================>{'reg_alpha': 0.5} loss:0.222047\n",
      "{'reg_alpha': 0.6} train loss: 0.216004, valid loss:0.222870, loss_diff:0.006866\n",
      "{'reg_alpha': 0.6} train loss: 0.216283, valid loss:0.221474, loss_diff:0.005191\n",
      "{'reg_alpha': 0.6} train loss: 0.215954, valid loss:0.222418, loss_diff:0.006464\n",
      "{'reg_alpha': 0.6} train loss: 0.216166, valid loss:0.221749, loss_diff:0.005583\n",
      "{'reg_alpha': 0.6} train loss: 0.216397, valid loss:0.222073, loss_diff:0.005676\n",
      "=================>{'reg_alpha': 0.6} loss:0.222117\n",
      "{'reg_alpha': 0.7} train loss: 0.216121, valid loss:0.222795, loss_diff:0.006674\n",
      "{'reg_alpha': 0.7} train loss: 0.216179, valid loss:0.221245, loss_diff:0.005067\n",
      "{'reg_alpha': 0.7} train loss: 0.216360, valid loss:0.222593, loss_diff:0.006234\n",
      "{'reg_alpha': 0.7} train loss: 0.216409, valid loss:0.221728, loss_diff:0.005319\n",
      "{'reg_alpha': 0.7} train loss: 0.216133, valid loss:0.222131, loss_diff:0.005998\n",
      "=================>{'reg_alpha': 0.7} loss:0.222099\n",
      "{'reg_alpha': 0.8} train loss: 0.215792, valid loss:0.222710, loss_diff:0.006918\n",
      "{'reg_alpha': 0.8} train loss: 0.216329, valid loss:0.221326, loss_diff:0.004996\n",
      "{'reg_alpha': 0.8} train loss: 0.216058, valid loss:0.222363, loss_diff:0.006305\n",
      "{'reg_alpha': 0.8} train loss: 0.216562, valid loss:0.221658, loss_diff:0.005095\n",
      "{'reg_alpha': 0.8} train loss: 0.216131, valid loss:0.222029, loss_diff:0.005899\n",
      "=================>{'reg_alpha': 0.8} loss:0.222017\n",
      "{'reg_alpha': 0.9} train loss: 0.215900, valid loss:0.222803, loss_diff:0.006903\n",
      "{'reg_alpha': 0.9} train loss: 0.216192, valid loss:0.221300, loss_diff:0.005107\n",
      "{'reg_alpha': 0.9} train loss: 0.216138, valid loss:0.222585, loss_diff:0.006446\n",
      "{'reg_alpha': 0.9} train loss: 0.216246, valid loss:0.221809, loss_diff:0.005563\n",
      "{'reg_alpha': 0.9} train loss: 0.216323, valid loss:0.222030, loss_diff:0.005707\n",
      "=================>{'reg_alpha': 0.9} loss:0.222105\n",
      "{'reg_alpha': 1.0} train loss: 0.216124, valid loss:0.222811, loss_diff:0.006687\n",
      "{'reg_alpha': 1.0} train loss: 0.216455, valid loss:0.221485, loss_diff:0.005030\n",
      "{'reg_alpha': 1.0} train loss: 0.216166, valid loss:0.222518, loss_diff:0.006352\n",
      "{'reg_alpha': 1.0} train loss: 0.216482, valid loss:0.221871, loss_diff:0.005389\n",
      "{'reg_alpha': 1.0} train loss: 0.216236, valid loss:0.222013, loss_diff:0.005777\n",
      "=================>{'reg_alpha': 1.0} loss:0.222140\n",
      "Best params: {'reg_alpha': 0.0} \tbest loss: 0.222004558559\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>param</th>\n",
       "      <th>val_loss_mean</th>\n",
       "      <th>val_loss_std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>{'reg_alpha': 0.0}</td>\n",
       "      <td>0.222005</td>\n",
       "      <td>0.000468</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>{'reg_alpha': 0.1}</td>\n",
       "      <td>0.222049</td>\n",
       "      <td>0.000434</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>{'reg_alpha': 0.2}</td>\n",
       "      <td>0.222108</td>\n",
       "      <td>0.000413</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>{'reg_alpha': 0.3}</td>\n",
       "      <td>0.222093</td>\n",
       "      <td>0.000400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>{'reg_alpha': 0.4}</td>\n",
       "      <td>0.222121</td>\n",
       "      <td>0.000437</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>{'reg_alpha': 0.5}</td>\n",
       "      <td>0.222047</td>\n",
       "      <td>0.000589</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>{'reg_alpha': 0.6}</td>\n",
       "      <td>0.222117</td>\n",
       "      <td>0.000492</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>{'reg_alpha': 0.7}</td>\n",
       "      <td>0.222099</td>\n",
       "      <td>0.000565</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>{'reg_alpha': 0.8}</td>\n",
       "      <td>0.222017</td>\n",
       "      <td>0.000491</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>{'reg_alpha': 0.9}</td>\n",
       "      <td>0.222105</td>\n",
       "      <td>0.000540</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>{'reg_alpha': 1.0}</td>\n",
       "      <td>0.222140</td>\n",
       "      <td>0.000471</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 param  val_loss_mean  val_loss_std\n",
       "0   {'reg_alpha': 0.0}       0.222005      0.000468\n",
       "1   {'reg_alpha': 0.1}       0.222049      0.000434\n",
       "2   {'reg_alpha': 0.2}       0.222108      0.000413\n",
       "3   {'reg_alpha': 0.3}       0.222093      0.000400\n",
       "4   {'reg_alpha': 0.4}       0.222121      0.000437\n",
       "5   {'reg_alpha': 0.5}       0.222047      0.000589\n",
       "6   {'reg_alpha': 0.6}       0.222117      0.000492\n",
       "7   {'reg_alpha': 0.7}       0.222099      0.000565\n",
       "8   {'reg_alpha': 0.8}       0.222017      0.000491\n",
       "9   {'reg_alpha': 0.9}       0.222105      0.000540\n",
       "10  {'reg_alpha': 1.0}       0.222140      0.000471"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "default_params = {\n",
    "    'boosting_type':'gbdt', \n",
    "    'num_leaves':31, \n",
    "    'max_depth':8, \n",
    "    'learning_rate':0.3, \n",
    "    'n_estimators':100, \n",
    "    'min_split_gain':0.0, \n",
    "    'min_child_weight':0.001, \n",
    "    'min_child_samples':20, \n",
    "    'subsample':1., \n",
    "    'subsample_freq':1, \n",
    "    'colsample_bytree':.6, \n",
    "    'reg_alpha':0.0, \n",
    "    'reg_lambda':0.0, \n",
    "    'random_state':SEED, \n",
    "    'n_jobs': 3\n",
    "}\n",
    "\n",
    "try_params = {\n",
    "    'reg_alpha':[i/10.0 for i in range(0,11,1)]\n",
    "}\n",
    "\n",
    "fit_params(train_X, train_y, ml, default_params, try_params, fit_params=fit_param, seed=SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'reg_lambda': 0.0} train loss: 0.215871, valid loss:0.222551, loss_diff:0.006680\n",
      "{'reg_lambda': 0.0} train loss: 0.216085, valid loss:0.221391, loss_diff:0.005306\n",
      "{'reg_lambda': 0.0} train loss: 0.215925, valid loss:0.222422, loss_diff:0.006497\n",
      "{'reg_lambda': 0.0} train loss: 0.216365, valid loss:0.221527, loss_diff:0.005163\n",
      "{'reg_lambda': 0.0} train loss: 0.216189, valid loss:0.222131, loss_diff:0.005942\n",
      "=================>{'reg_lambda': 0.0} loss:0.222005\n",
      "{'reg_lambda': 0.1} train loss: 0.216060, valid loss:0.222695, loss_diff:0.006634\n",
      "{'reg_lambda': 0.1} train loss: 0.216194, valid loss:0.221390, loss_diff:0.005195\n",
      "{'reg_lambda': 0.1} train loss: 0.216187, valid loss:0.222522, loss_diff:0.006335\n",
      "{'reg_lambda': 0.1} train loss: 0.216521, valid loss:0.221791, loss_diff:0.005271\n",
      "{'reg_lambda': 0.1} train loss: 0.216255, valid loss:0.222064, loss_diff:0.005809\n",
      "=================>{'reg_lambda': 0.1} loss:0.222092\n",
      "{'reg_lambda': 0.2} train loss: 0.215882, valid loss:0.222628, loss_diff:0.006745\n",
      "{'reg_lambda': 0.2} train loss: 0.216289, valid loss:0.221592, loss_diff:0.005303\n",
      "{'reg_lambda': 0.2} train loss: 0.216097, valid loss:0.222551, loss_diff:0.006454\n",
      "{'reg_lambda': 0.2} train loss: 0.216146, valid loss:0.221645, loss_diff:0.005498\n",
      "{'reg_lambda': 0.2} train loss: 0.216098, valid loss:0.222045, loss_diff:0.005947\n",
      "=================>{'reg_lambda': 0.2} loss:0.222092\n",
      "{'reg_lambda': 0.3} train loss: 0.216088, valid loss:0.222621, loss_diff:0.006533\n",
      "{'reg_lambda': 0.3} train loss: 0.216193, valid loss:0.221447, loss_diff:0.005254\n",
      "{'reg_lambda': 0.3} train loss: 0.216031, valid loss:0.222426, loss_diff:0.006395\n",
      "{'reg_lambda': 0.3} train loss: 0.216222, valid loss:0.221496, loss_diff:0.005274\n",
      "{'reg_lambda': 0.3} train loss: 0.216318, valid loss:0.222093, loss_diff:0.005775\n",
      "=================>{'reg_lambda': 0.3} loss:0.222017\n",
      "{'reg_lambda': 0.4} train loss: 0.215967, valid loss:0.222516, loss_diff:0.006550\n",
      "{'reg_lambda': 0.4} train loss: 0.216272, valid loss:0.221623, loss_diff:0.005350\n",
      "{'reg_lambda': 0.4} train loss: 0.216038, valid loss:0.222452, loss_diff:0.006414\n",
      "{'reg_lambda': 0.4} train loss: 0.216411, valid loss:0.221856, loss_diff:0.005444\n",
      "{'reg_lambda': 0.4} train loss: 0.216112, valid loss:0.222145, loss_diff:0.006033\n",
      "=================>{'reg_lambda': 0.4} loss:0.222118\n",
      "{'reg_lambda': 0.5} train loss: 0.216088, valid loss:0.222714, loss_diff:0.006626\n",
      "{'reg_lambda': 0.5} train loss: 0.216139, valid loss:0.221481, loss_diff:0.005341\n",
      "{'reg_lambda': 0.5} train loss: 0.216197, valid loss:0.222484, loss_diff:0.006287\n",
      "{'reg_lambda': 0.5} train loss: 0.216397, valid loss:0.221803, loss_diff:0.005406\n",
      "{'reg_lambda': 0.5} train loss: 0.216178, valid loss:0.222142, loss_diff:0.005964\n",
      "=================>{'reg_lambda': 0.5} loss:0.222125\n",
      "{'reg_lambda': 0.6} train loss: 0.216023, valid loss:0.222604, loss_diff:0.006581\n",
      "{'reg_lambda': 0.6} train loss: 0.216146, valid loss:0.221326, loss_diff:0.005180\n",
      "{'reg_lambda': 0.6} train loss: 0.216135, valid loss:0.222449, loss_diff:0.006315\n",
      "{'reg_lambda': 0.6} train loss: 0.216118, valid loss:0.221688, loss_diff:0.005570\n",
      "{'reg_lambda': 0.6} train loss: 0.215965, valid loss:0.221999, loss_diff:0.006033\n",
      "=================>{'reg_lambda': 0.6} loss:0.222013\n",
      "{'reg_lambda': 0.7} train loss: 0.216177, valid loss:0.222664, loss_diff:0.006488\n",
      "{'reg_lambda': 0.7} train loss: 0.216085, valid loss:0.221371, loss_diff:0.005287\n",
      "{'reg_lambda': 0.7} train loss: 0.216125, valid loss:0.222552, loss_diff:0.006427\n",
      "{'reg_lambda': 0.7} train loss: 0.216441, valid loss:0.221840, loss_diff:0.005399\n",
      "{'reg_lambda': 0.7} train loss: 0.216342, valid loss:0.222117, loss_diff:0.005775\n",
      "=================>{'reg_lambda': 0.7} loss:0.222109\n",
      "{'reg_lambda': 0.8} train loss: 0.216012, valid loss:0.222524, loss_diff:0.006512\n",
      "{'reg_lambda': 0.8} train loss: 0.216363, valid loss:0.221433, loss_diff:0.005070\n",
      "{'reg_lambda': 0.8} train loss: 0.216098, valid loss:0.222460, loss_diff:0.006362\n",
      "{'reg_lambda': 0.8} train loss: 0.216236, valid loss:0.221776, loss_diff:0.005541\n",
      "{'reg_lambda': 0.8} train loss: 0.216074, valid loss:0.222088, loss_diff:0.006013\n",
      "=================>{'reg_lambda': 0.8} loss:0.222056\n",
      "{'reg_lambda': 0.9} train loss: 0.215940, valid loss:0.222621, loss_diff:0.006681\n",
      "{'reg_lambda': 0.9} train loss: 0.216217, valid loss:0.221475, loss_diff:0.005258\n",
      "{'reg_lambda': 0.9} train loss: 0.216095, valid loss:0.222420, loss_diff:0.006326\n",
      "{'reg_lambda': 0.9} train loss: 0.216406, valid loss:0.221803, loss_diff:0.005397\n",
      "{'reg_lambda': 0.9} train loss: 0.216105, valid loss:0.222083, loss_diff:0.005978\n",
      "=================>{'reg_lambda': 0.9} loss:0.222080\n",
      "{'reg_lambda': 1.0} train loss: 0.215882, valid loss:0.222562, loss_diff:0.006680\n",
      "{'reg_lambda': 1.0} train loss: 0.216187, valid loss:0.221348, loss_diff:0.005161\n",
      "{'reg_lambda': 1.0} train loss: 0.216215, valid loss:0.222341, loss_diff:0.006126\n",
      "{'reg_lambda': 1.0} train loss: 0.216360, valid loss:0.221760, loss_diff:0.005400\n",
      "{'reg_lambda': 1.0} train loss: 0.216212, valid loss:0.222164, loss_diff:0.005952\n",
      "=================>{'reg_lambda': 1.0} loss:0.222035\n",
      "Best params: {'reg_lambda': 0.0} \tbest loss: 0.222004558559\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>param</th>\n",
       "      <th>val_loss_mean</th>\n",
       "      <th>val_loss_std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>{'reg_lambda': 0.0}</td>\n",
       "      <td>0.222005</td>\n",
       "      <td>0.000468</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>{'reg_lambda': 0.1}</td>\n",
       "      <td>0.222092</td>\n",
       "      <td>0.000476</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>{'reg_lambda': 0.2}</td>\n",
       "      <td>0.222092</td>\n",
       "      <td>0.000436</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>{'reg_lambda': 0.3}</td>\n",
       "      <td>0.222017</td>\n",
       "      <td>0.000476</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>{'reg_lambda': 0.4}</td>\n",
       "      <td>0.222118</td>\n",
       "      <td>0.000342</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>{'reg_lambda': 0.5}</td>\n",
       "      <td>0.222125</td>\n",
       "      <td>0.000446</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>{'reg_lambda': 0.6}</td>\n",
       "      <td>0.222013</td>\n",
       "      <td>0.000473</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>{'reg_lambda': 0.7}</td>\n",
       "      <td>0.222109</td>\n",
       "      <td>0.000473</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>{'reg_lambda': 0.8}</td>\n",
       "      <td>0.222056</td>\n",
       "      <td>0.000412</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>{'reg_lambda': 0.9}</td>\n",
       "      <td>0.222080</td>\n",
       "      <td>0.000413</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>{'reg_lambda': 1.0}</td>\n",
       "      <td>0.222035</td>\n",
       "      <td>0.000433</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  param  val_loss_mean  val_loss_std\n",
       "0   {'reg_lambda': 0.0}       0.222005      0.000468\n",
       "1   {'reg_lambda': 0.1}       0.222092      0.000476\n",
       "2   {'reg_lambda': 0.2}       0.222092      0.000436\n",
       "3   {'reg_lambda': 0.3}       0.222017      0.000476\n",
       "4   {'reg_lambda': 0.4}       0.222118      0.000342\n",
       "5   {'reg_lambda': 0.5}       0.222125      0.000446\n",
       "6   {'reg_lambda': 0.6}       0.222013      0.000473\n",
       "7   {'reg_lambda': 0.7}       0.222109      0.000473\n",
       "8   {'reg_lambda': 0.8}       0.222056      0.000412\n",
       "9   {'reg_lambda': 0.9}       0.222080      0.000413\n",
       "10  {'reg_lambda': 1.0}       0.222035      0.000433"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "default_params = {\n",
    "    'boosting_type':'gbdt', \n",
    "    'num_leaves':31, \n",
    "    'max_depth':8, \n",
    "    'learning_rate':0.3, \n",
    "    'n_estimators':100, \n",
    "    'min_split_gain':0.0, \n",
    "    'min_child_weight':0.001, \n",
    "    'min_child_samples':20, \n",
    "    'subsample':1., \n",
    "    'subsample_freq':1, \n",
    "    'colsample_bytree':.6, \n",
    "    'reg_alpha':0.0, \n",
    "    'reg_lambda':0.0, \n",
    "    'random_state':SEED, \n",
    "    'n_jobs': 3\n",
    "}\n",
    "\n",
    "try_params = {\n",
    "    'reg_lambda':[i/10.0 for i in range(0,11,1)]\n",
    "}\n",
    "\n",
    "fit_params(train_X, train_y, ml, default_params, try_params, fit_params=fit_param, seed=SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'reg_alpha': 0.0, 'reg_lambda': 0.0} train loss: 0.215871, valid loss:0.222551, loss_diff:0.006680\n",
      "{'reg_alpha': 0.0, 'reg_lambda': 0.0} train loss: 0.216085, valid loss:0.221391, loss_diff:0.005306\n",
      "{'reg_alpha': 0.0, 'reg_lambda': 0.0} train loss: 0.215925, valid loss:0.222422, loss_diff:0.006497\n",
      "{'reg_alpha': 0.0, 'reg_lambda': 0.0} train loss: 0.216365, valid loss:0.221527, loss_diff:0.005163\n",
      "{'reg_alpha': 0.0, 'reg_lambda': 0.0} train loss: 0.216189, valid loss:0.222131, loss_diff:0.005942\n",
      "=================>{'reg_alpha': 0.0, 'reg_lambda': 0.0} loss:0.222005\n",
      "{'reg_alpha': 0.0, 'reg_lambda': 0.05} train loss: 0.215974, valid loss:0.222709, loss_diff:0.006735\n",
      "{'reg_alpha': 0.0, 'reg_lambda': 0.05} train loss: 0.216038, valid loss:0.221485, loss_diff:0.005448\n",
      "{'reg_alpha': 0.0, 'reg_lambda': 0.05} train loss: 0.216119, valid loss:0.222458, loss_diff:0.006338\n",
      "{'reg_alpha': 0.0, 'reg_lambda': 0.05} train loss: 0.216095, valid loss:0.221461, loss_diff:0.005367\n",
      "{'reg_alpha': 0.0, 'reg_lambda': 0.05} train loss: 0.216204, valid loss:0.222122, loss_diff:0.005917\n",
      "=================>{'reg_alpha': 0.0, 'reg_lambda': 0.05} loss:0.222047\n",
      "{'reg_alpha': 0.05, 'reg_lambda': 0.0} train loss: 0.215857, valid loss:0.222716, loss_diff:0.006859\n",
      "{'reg_alpha': 0.05, 'reg_lambda': 0.0} train loss: 0.216180, valid loss:0.221516, loss_diff:0.005337\n",
      "{'reg_alpha': 0.05, 'reg_lambda': 0.0} train loss: 0.215960, valid loss:0.222518, loss_diff:0.006558\n",
      "{'reg_alpha': 0.05, 'reg_lambda': 0.0} train loss: 0.216200, valid loss:0.221773, loss_diff:0.005572\n",
      "{'reg_alpha': 0.05, 'reg_lambda': 0.0} train loss: 0.216330, valid loss:0.222260, loss_diff:0.005930\n",
      "=================>{'reg_alpha': 0.05, 'reg_lambda': 0.0} loss:0.222157\n",
      "{'reg_alpha': 0.05, 'reg_lambda': 0.05} train loss: 0.216053, valid loss:0.222623, loss_diff:0.006570\n",
      "{'reg_alpha': 0.05, 'reg_lambda': 0.05} train loss: 0.216026, valid loss:0.221494, loss_diff:0.005469\n",
      "{'reg_alpha': 0.05, 'reg_lambda': 0.05} train loss: 0.216279, valid loss:0.222476, loss_diff:0.006197\n",
      "{'reg_alpha': 0.05, 'reg_lambda': 0.05} train loss: 0.216084, valid loss:0.221785, loss_diff:0.005701\n",
      "{'reg_alpha': 0.05, 'reg_lambda': 0.05} train loss: 0.216082, valid loss:0.222072, loss_diff:0.005990\n",
      "=================>{'reg_alpha': 0.05, 'reg_lambda': 0.05} loss:0.222090\n",
      "Best params: {'reg_alpha': 0.0, 'reg_lambda': 0.0} \tbest loss: 0.222004558559\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>param</th>\n",
       "      <th>val_loss_mean</th>\n",
       "      <th>val_loss_std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>{'reg_alpha': 0.0, 'reg_lambda': 0.0}</td>\n",
       "      <td>0.222005</td>\n",
       "      <td>0.000468</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>{'reg_alpha': 0.0, 'reg_lambda': 0.05}</td>\n",
       "      <td>0.222047</td>\n",
       "      <td>0.000504</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>{'reg_alpha': 0.05, 'reg_lambda': 0.0}</td>\n",
       "      <td>0.222157</td>\n",
       "      <td>0.000450</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>{'reg_alpha': 0.05, 'reg_lambda': 0.05}</td>\n",
       "      <td>0.222090</td>\n",
       "      <td>0.000420</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     param  val_loss_mean  val_loss_std\n",
       "0    {'reg_alpha': 0.0, 'reg_lambda': 0.0}       0.222005      0.000468\n",
       "1   {'reg_alpha': 0.0, 'reg_lambda': 0.05}       0.222047      0.000504\n",
       "2   {'reg_alpha': 0.05, 'reg_lambda': 0.0}       0.222157      0.000450\n",
       "3  {'reg_alpha': 0.05, 'reg_lambda': 0.05}       0.222090      0.000420"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "default_params = {\n",
    "    'boosting_type':'gbdt', \n",
    "    'num_leaves':31, \n",
    "    'max_depth':8, \n",
    "    'learning_rate':0.3, \n",
    "    'n_estimators':100, \n",
    "    'min_split_gain':0.0, \n",
    "    'min_child_weight':0.001, \n",
    "    'min_child_samples':20, \n",
    "    'subsample':1., \n",
    "    'subsample_freq':1, \n",
    "    'colsample_bytree':.6, \n",
    "    'reg_alpha':0.0, \n",
    "    'reg_lambda':0.0, \n",
    "    'random_state':SEED, \n",
    "    'n_jobs': 3\n",
    "}\n",
    "\n",
    "try_params = {\n",
    "    'reg_alpha': [0., .05],\n",
    "    'reg_lambda':[0., .05]\n",
    "}\n",
    "\n",
    "fit_params(train_X, train_y, ml, default_params, try_params, fit_params=fit_param, seed=SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'max_depth': 8} train loss: 0.215871, valid loss:0.222551, loss_diff:0.006680\n",
      "{'max_depth': 8} train loss: 0.216085, valid loss:0.221391, loss_diff:0.005306\n",
      "{'max_depth': 8} train loss: 0.215925, valid loss:0.222422, loss_diff:0.006497\n",
      "{'max_depth': 8} train loss: 0.216365, valid loss:0.221527, loss_diff:0.005163\n",
      "{'max_depth': 8} train loss: 0.216189, valid loss:0.222131, loss_diff:0.005942\n",
      "=================>{'max_depth': 8} loss:0.222005\n",
      "{'max_depth': 10} train loss: 0.215346, valid loss:0.222288, loss_diff:0.006942\n",
      "{'max_depth': 10} train loss: 0.215494, valid loss:0.221248, loss_diff:0.005754\n",
      "{'max_depth': 10} train loss: 0.215434, valid loss:0.222286, loss_diff:0.006853\n",
      "{'max_depth': 10} train loss: 0.215442, valid loss:0.221433, loss_diff:0.005990\n",
      "{'max_depth': 10} train loss: 0.215330, valid loss:0.221716, loss_diff:0.006387\n",
      "=================>{'max_depth': 10} loss:0.221794\n",
      "{'max_depth': 15} train loss: 0.214630, valid loss:0.222029, loss_diff:0.007399\n",
      "{'max_depth': 15} train loss: 0.214962, valid loss:0.220893, loss_diff:0.005931\n",
      "{'max_depth': 15} train loss: 0.214613, valid loss:0.221712, loss_diff:0.007099\n",
      "{'max_depth': 15} train loss: 0.214872, valid loss:0.221185, loss_diff:0.006314\n",
      "{'max_depth': 15} train loss: 0.214699, valid loss:0.221432, loss_diff:0.006733\n",
      "=================>{'max_depth': 15} loss:0.221450\n",
      "{'max_depth': 18} train loss: 0.214395, valid loss:0.221990, loss_diff:0.007596\n",
      "{'max_depth': 18} train loss: 0.214600, valid loss:0.220683, loss_diff:0.006084\n",
      "{'max_depth': 18} train loss: 0.214584, valid loss:0.221680, loss_diff:0.007097\n",
      "{'max_depth': 18} train loss: 0.214750, valid loss:0.221067, loss_diff:0.006316\n",
      "{'max_depth': 18} train loss: 0.214597, valid loss:0.221360, loss_diff:0.006763\n",
      "=================>{'max_depth': 18} loss:0.221356\n",
      "Best params: {'max_depth': 18} \tbest loss: 0.221356072184\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>param</th>\n",
       "      <th>val_loss_mean</th>\n",
       "      <th>val_loss_std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>{'max_depth': 8}</td>\n",
       "      <td>0.222005</td>\n",
       "      <td>0.000468</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>{'max_depth': 10}</td>\n",
       "      <td>0.221794</td>\n",
       "      <td>0.000429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>{'max_depth': 15}</td>\n",
       "      <td>0.221450</td>\n",
       "      <td>0.000396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>{'max_depth': 18}</td>\n",
       "      <td>0.221356</td>\n",
       "      <td>0.000457</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               param  val_loss_mean  val_loss_std\n",
       "0   {'max_depth': 8}       0.222005      0.000468\n",
       "1  {'max_depth': 10}       0.221794      0.000429\n",
       "2  {'max_depth': 15}       0.221450      0.000396\n",
       "3  {'max_depth': 18}       0.221356      0.000457"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "default_params = {\n",
    "    'boosting_type':'gbdt', \n",
    "    'num_leaves':31, \n",
    "    'max_depth':8, \n",
    "    'learning_rate':0.3, \n",
    "    'n_estimators':100, \n",
    "    'min_split_gain':0.0, \n",
    "    'min_child_weight':0.001, \n",
    "    'min_child_samples':20, \n",
    "    'subsample':1., \n",
    "    'subsample_freq':1, \n",
    "    'colsample_bytree':.6, \n",
    "    'reg_alpha':0.0, \n",
    "    'reg_lambda':0.0, \n",
    "    'random_state':SEED, \n",
    "    'n_jobs': 3\n",
    "}\n",
    "\n",
    "try_params = {\n",
    "    'max_depth': [8, 10, 15, 18]\n",
    "}\n",
    "\n",
    "fit_params(train_X, train_y, ml, default_params, try_params, fit_params=fit_param, seed=SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'num_leaves': 32} train loss: 0.214193, valid loss:0.222013, loss_diff:0.007820\n",
      "{'num_leaves': 32} train loss: 0.214513, valid loss:0.220503, loss_diff:0.005990\n",
      "{'num_leaves': 32} train loss: 0.214240, valid loss:0.221530, loss_diff:0.007290\n",
      "{'num_leaves': 32} train loss: 0.214466, valid loss:0.220842, loss_diff:0.006375\n",
      "{'num_leaves': 32} train loss: 0.214366, valid loss:0.221435, loss_diff:0.007069\n",
      "=================>{'num_leaves': 32} loss:0.221265\n",
      "{'num_leaves': 48} train loss: 0.210871, valid loss:0.221536, loss_diff:0.010665\n",
      "{'num_leaves': 48} train loss: 0.211536, valid loss:0.220142, loss_diff:0.008606\n",
      "{'num_leaves': 48} train loss: 0.211463, valid loss:0.221319, loss_diff:0.009856\n",
      "{'num_leaves': 48} train loss: 0.211400, valid loss:0.220481, loss_diff:0.009081\n",
      "{'num_leaves': 48} train loss: 0.211229, valid loss:0.220944, loss_diff:0.009715\n",
      "=================>{'num_leaves': 48} loss:0.220885\n",
      "{'num_leaves': 64} train loss: 0.208893, valid loss:0.221343, loss_diff:0.012450\n",
      "{'num_leaves': 64} train loss: 0.209512, valid loss:0.220292, loss_diff:0.010780\n",
      "{'num_leaves': 64} train loss: 0.208816, valid loss:0.221032, loss_diff:0.012216\n",
      "{'num_leaves': 64} train loss: 0.209175, valid loss:0.220316, loss_diff:0.011140\n",
      "{'num_leaves': 64} train loss: 0.208819, valid loss:0.220920, loss_diff:0.012101\n",
      "=================>{'num_leaves': 64} loss:0.220781\n",
      "{'num_leaves': 128} train loss: 0.203078, valid loss:0.221232, loss_diff:0.018154\n",
      "{'num_leaves': 128} train loss: 0.203587, valid loss:0.220168, loss_diff:0.016581\n",
      "{'num_leaves': 128} train loss: 0.203286, valid loss:0.221502, loss_diff:0.018216\n",
      "{'num_leaves': 128} train loss: 0.203469, valid loss:0.220308, loss_diff:0.016839\n",
      "{'num_leaves': 128} train loss: 0.203012, valid loss:0.220906, loss_diff:0.017894\n",
      "=================>{'num_leaves': 128} loss:0.220823\n",
      "{'num_leaves': 256} train loss: 0.197002, valid loss:0.222003, loss_diff:0.025001\n",
      "{'num_leaves': 256} train loss: 0.197107, valid loss:0.220786, loss_diff:0.023680\n",
      "{'num_leaves': 256} train loss: 0.196655, valid loss:0.221747, loss_diff:0.025093\n",
      "{'num_leaves': 256} train loss: 0.196043, valid loss:0.220964, loss_diff:0.024922\n",
      "{'num_leaves': 256} train loss: 0.196332, valid loss:0.221456, loss_diff:0.025124\n",
      "=================>{'num_leaves': 256} loss:0.221391\n",
      "Best params: {'num_leaves': 64} \tbest loss: 0.220780641001\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>param</th>\n",
       "      <th>val_loss_mean</th>\n",
       "      <th>val_loss_std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>{'num_leaves': 32}</td>\n",
       "      <td>0.221265</td>\n",
       "      <td>0.000533</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>{'num_leaves': 48}</td>\n",
       "      <td>0.220885</td>\n",
       "      <td>0.000516</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>{'num_leaves': 64}</td>\n",
       "      <td>0.220781</td>\n",
       "      <td>0.000413</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>{'num_leaves': 128}</td>\n",
       "      <td>0.220823</td>\n",
       "      <td>0.000516</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>{'num_leaves': 256}</td>\n",
       "      <td>0.221391</td>\n",
       "      <td>0.000459</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 param  val_loss_mean  val_loss_std\n",
       "0   {'num_leaves': 32}       0.221265      0.000533\n",
       "1   {'num_leaves': 48}       0.220885      0.000516\n",
       "2   {'num_leaves': 64}       0.220781      0.000413\n",
       "3  {'num_leaves': 128}       0.220823      0.000516\n",
       "4  {'num_leaves': 256}       0.221391      0.000459"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "default_params = {\n",
    "    'boosting_type':'gbdt', \n",
    "    'num_leaves':31, \n",
    "    'max_depth':18, \n",
    "    'learning_rate':0.3, \n",
    "    'n_estimators':100, \n",
    "    'min_split_gain':0.0, \n",
    "    'min_child_weight':0.001, \n",
    "    'min_child_samples':20, \n",
    "    'subsample':1., \n",
    "    'subsample_freq':1, \n",
    "    'colsample_bytree':.6, \n",
    "    'reg_alpha':0.0, \n",
    "    'reg_lambda':0.0, \n",
    "    'random_state':SEED, \n",
    "    'n_jobs': 3\n",
    "}\n",
    "\n",
    "try_params = {\n",
    "    'num_leaves': [32, 48, 64, 128, 256]\n",
    "}\n",
    "\n",
    "fit_params(train_X, train_y, ml, default_params, try_params, fit_params=fit_param, seed=SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 50 rounds.\n",
      "[100]\ttrain's rmse: 0.226186\tvalid's rmse: 0.228302\n",
      "[200]\ttrain's rmse: 0.221284\tvalid's rmse: 0.225013\n",
      "[300]\ttrain's rmse: 0.218433\tvalid's rmse: 0.223555\n",
      "[400]\ttrain's rmse: 0.216636\tvalid's rmse: 0.222766\n",
      "[500]\ttrain's rmse: 0.215245\tvalid's rmse: 0.222235\n",
      "[600]\ttrain's rmse: 0.214198\tvalid's rmse: 0.221888\n",
      "[700]\ttrain's rmse: 0.213315\tvalid's rmse: 0.221636\n",
      "[800]\ttrain's rmse: 0.212522\tvalid's rmse: 0.221388\n",
      "[900]\ttrain's rmse: 0.211793\tvalid's rmse: 0.221178\n",
      "[1000]\ttrain's rmse: 0.211119\tvalid's rmse: 0.221018\n",
      "[1100]\ttrain's rmse: 0.210466\tvalid's rmse: 0.220863\n",
      "[1200]\ttrain's rmse: 0.209909\tvalid's rmse: 0.220733\n",
      "[1300]\ttrain's rmse: 0.209335\tvalid's rmse: 0.220622\n",
      "[1400]\ttrain's rmse: 0.208819\tvalid's rmse: 0.220519\n",
      "[1500]\ttrain's rmse: 0.208334\tvalid's rmse: 0.220424\n",
      "[1600]\ttrain's rmse: 0.207839\tvalid's rmse: 0.220329\n",
      "[1700]\ttrain's rmse: 0.207367\tvalid's rmse: 0.220252\n",
      "[1800]\ttrain's rmse: 0.206918\tvalid's rmse: 0.220175\n",
      "[1900]\ttrain's rmse: 0.206495\tvalid's rmse: 0.220112\n",
      "[2000]\ttrain's rmse: 0.206065\tvalid's rmse: 0.220046\n",
      "[2100]\ttrain's rmse: 0.205669\tvalid's rmse: 0.219985\n",
      "[2200]\ttrain's rmse: 0.205256\tvalid's rmse: 0.219931\n",
      "[2300]\ttrain's rmse: 0.204865\tvalid's rmse: 0.219879\n",
      "[2400]\ttrain's rmse: 0.204471\tvalid's rmse: 0.219834\n",
      "[2500]\ttrain's rmse: 0.20407\tvalid's rmse: 0.219786\n",
      "[2600]\ttrain's rmse: 0.20371\tvalid's rmse: 0.219744\n",
      "[2700]\ttrain's rmse: 0.203348\tvalid's rmse: 0.2197\n",
      "[2800]\ttrain's rmse: 0.202992\tvalid's rmse: 0.219661\n",
      "[2900]\ttrain's rmse: 0.202656\tvalid's rmse: 0.219634\n",
      "[3000]\ttrain's rmse: 0.202303\tvalid's rmse: 0.2196\n",
      "[3100]\ttrain's rmse: 0.201956\tvalid's rmse: 0.219564\n",
      "[3200]\ttrain's rmse: 0.201617\tvalid's rmse: 0.219538\n",
      "[3300]\ttrain's rmse: 0.201277\tvalid's rmse: 0.219502\n",
      "[3400]\ttrain's rmse: 0.20096\tvalid's rmse: 0.219478\n",
      "[3500]\ttrain's rmse: 0.200651\tvalid's rmse: 0.219452\n",
      "[3600]\ttrain's rmse: 0.200336\tvalid's rmse: 0.219424\n",
      "[3700]\ttrain's rmse: 0.200022\tvalid's rmse: 0.219404\n",
      "[3800]\ttrain's rmse: 0.199727\tvalid's rmse: 0.219382\n",
      "[3900]\ttrain's rmse: 0.199421\tvalid's rmse: 0.219359\n",
      "[4000]\ttrain's rmse: 0.199124\tvalid's rmse: 0.219337\n",
      "[4100]\ttrain's rmse: 0.198847\tvalid's rmse: 0.219321\n",
      "[4200]\ttrain's rmse: 0.198533\tvalid's rmse: 0.219299\n",
      "[4300]\ttrain's rmse: 0.198248\tvalid's rmse: 0.219281\n",
      "[4400]\ttrain's rmse: 0.197949\tvalid's rmse: 0.219266\n",
      "[4500]\ttrain's rmse: 0.197629\tvalid's rmse: 0.219242\n",
      "[4600]\ttrain's rmse: 0.197341\tvalid's rmse: 0.219224\n",
      "[4700]\ttrain's rmse: 0.197069\tvalid's rmse: 0.219206\n",
      "[4800]\ttrain's rmse: 0.19679\tvalid's rmse: 0.219198\n",
      "[4900]\ttrain's rmse: 0.196498\tvalid's rmse: 0.219179\n",
      "[5000]\ttrain's rmse: 0.196231\tvalid's rmse: 0.219165\n",
      "[5100]\ttrain's rmse: 0.195956\tvalid's rmse: 0.219149\n",
      "[5200]\ttrain's rmse: 0.195689\tvalid's rmse: 0.219138\n",
      "[5300]\ttrain's rmse: 0.195433\tvalid's rmse: 0.219125\n",
      "[5400]\ttrain's rmse: 0.195151\tvalid's rmse: 0.219112\n",
      "[5500]\ttrain's rmse: 0.19489\tvalid's rmse: 0.219102\n",
      "[5600]\ttrain's rmse: 0.194625\tvalid's rmse: 0.219094\n",
      "[5700]\ttrain's rmse: 0.194364\tvalid's rmse: 0.219087\n",
      "[5800]\ttrain's rmse: 0.194101\tvalid's rmse: 0.219078\n",
      "[5900]\ttrain's rmse: 0.193841\tvalid's rmse: 0.219068\n",
      "[6000]\ttrain's rmse: 0.19357\tvalid's rmse: 0.219053\n",
      "[6100]\ttrain's rmse: 0.193298\tvalid's rmse: 0.219044\n",
      "Early stopping, best iteration is:\n",
      "[6143]\ttrain's rmse: 0.193197\tvalid's rmse: 0.219036\n",
      "Fold 1 completed.\n",
      "Training until validation scores don't improve for 50 rounds.\n",
      "[100]\ttrain's rmse: 0.225952\tvalid's rmse: 0.229408\n",
      "[200]\ttrain's rmse: 0.221017\tvalid's rmse: 0.22601\n",
      "[300]\ttrain's rmse: 0.218156\tvalid's rmse: 0.22452\n",
      "[400]\ttrain's rmse: 0.216298\tvalid's rmse: 0.223752\n",
      "[500]\ttrain's rmse: 0.214935\tvalid's rmse: 0.223256\n",
      "[600]\ttrain's rmse: 0.213927\tvalid's rmse: 0.222926\n",
      "[700]\ttrain's rmse: 0.213057\tvalid's rmse: 0.222648\n",
      "[800]\ttrain's rmse: 0.21228\tvalid's rmse: 0.222421\n",
      "[900]\ttrain's rmse: 0.211609\tvalid's rmse: 0.222241\n",
      "[1000]\ttrain's rmse: 0.210943\tvalid's rmse: 0.222087\n",
      "[1100]\ttrain's rmse: 0.210323\tvalid's rmse: 0.221944\n",
      "[1200]\ttrain's rmse: 0.209706\tvalid's rmse: 0.221812\n",
      "[1300]\ttrain's rmse: 0.209143\tvalid's rmse: 0.221697\n",
      "[1400]\ttrain's rmse: 0.208643\tvalid's rmse: 0.221605\n",
      "[1500]\ttrain's rmse: 0.20813\tvalid's rmse: 0.221506\n",
      "[1600]\ttrain's rmse: 0.207631\tvalid's rmse: 0.221426\n",
      "[1700]\ttrain's rmse: 0.207161\tvalid's rmse: 0.221352\n",
      "[1800]\ttrain's rmse: 0.206714\tvalid's rmse: 0.221285\n",
      "[1900]\ttrain's rmse: 0.206242\tvalid's rmse: 0.221211\n",
      "[2000]\ttrain's rmse: 0.205819\tvalid's rmse: 0.221153\n",
      "[2100]\ttrain's rmse: 0.205382\tvalid's rmse: 0.221094\n",
      "[2200]\ttrain's rmse: 0.204978\tvalid's rmse: 0.221042\n",
      "[2300]\ttrain's rmse: 0.204568\tvalid's rmse: 0.220988\n",
      "[2400]\ttrain's rmse: 0.204184\tvalid's rmse: 0.22094\n",
      "[2500]\ttrain's rmse: 0.203763\tvalid's rmse: 0.220881\n",
      "[2600]\ttrain's rmse: 0.20338\tvalid's rmse: 0.220835\n",
      "[2700]\ttrain's rmse: 0.203036\tvalid's rmse: 0.220808\n",
      "[2800]\ttrain's rmse: 0.202659\tvalid's rmse: 0.220758\n",
      "[2900]\ttrain's rmse: 0.202316\tvalid's rmse: 0.220732\n",
      "[3000]\ttrain's rmse: 0.201977\tvalid's rmse: 0.220703\n",
      "[3100]\ttrain's rmse: 0.201622\tvalid's rmse: 0.220677\n",
      "[3200]\ttrain's rmse: 0.201293\tvalid's rmse: 0.220656\n",
      "[3300]\ttrain's rmse: 0.200967\tvalid's rmse: 0.220631\n",
      "[3400]\ttrain's rmse: 0.200646\tvalid's rmse: 0.220609\n",
      "[3500]\ttrain's rmse: 0.200336\tvalid's rmse: 0.220588\n",
      "[3600]\ttrain's rmse: 0.200032\tvalid's rmse: 0.220565\n",
      "[3700]\ttrain's rmse: 0.199724\tvalid's rmse: 0.220542\n",
      "[3800]\ttrain's rmse: 0.199412\tvalid's rmse: 0.220524\n",
      "[3900]\ttrain's rmse: 0.199116\tvalid's rmse: 0.220506\n",
      "[4000]\ttrain's rmse: 0.198828\tvalid's rmse: 0.220488\n",
      "[4100]\ttrain's rmse: 0.198527\tvalid's rmse: 0.220468\n",
      "[4200]\ttrain's rmse: 0.19824\tvalid's rmse: 0.220447\n",
      "[4300]\ttrain's rmse: 0.197941\tvalid's rmse: 0.220427\n",
      "[4400]\ttrain's rmse: 0.197647\tvalid's rmse: 0.220406\n",
      "[4500]\ttrain's rmse: 0.197361\tvalid's rmse: 0.220388\n",
      "[4600]\ttrain's rmse: 0.197071\tvalid's rmse: 0.22037\n",
      "[4700]\ttrain's rmse: 0.196767\tvalid's rmse: 0.220351\n",
      "[4800]\ttrain's rmse: 0.196498\tvalid's rmse: 0.220339\n",
      "[4900]\ttrain's rmse: 0.196215\tvalid's rmse: 0.220323\n",
      "[5000]\ttrain's rmse: 0.195935\tvalid's rmse: 0.220308\n",
      "[5100]\ttrain's rmse: 0.195677\tvalid's rmse: 0.220298\n",
      "[5200]\ttrain's rmse: 0.195416\tvalid's rmse: 0.220293\n",
      "[5300]\ttrain's rmse: 0.195146\tvalid's rmse: 0.220281\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-20-51738ce93c1b>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     40\u001b[0m lgb_oof_val_pred, lgb_oof_test_pred, models = get_oof_predictions(train_X, train_y, test_X, ml, \n\u001b[0;32m     41\u001b[0m                                                                   \u001b[0mdefault_params\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mseed\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mSEED\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfit_params\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mfit_param\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 42\u001b[1;33m                                                                   use_eval_set=True)\n\u001b[0m",
      "\u001b[1;32mE:\\Kun-Hao Data\\Kaggle Experiments\\Avito\\GridSearcher.py\u001b[0m in \u001b[0;36mget_oof_predictions\u001b[1;34m(X, y, test_X, model_loader, param, seed, fit_params, use_eval_set, predict_proba)\u001b[0m\n\u001b[0;32m    236\u001b[0m                           \u001b[0meval_set\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtr_X\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtr_y\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mval_X\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mval_y\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    237\u001b[0m                           \u001b[0meval_names\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'train'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'valid'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 238\u001b[1;33m                           **fit_params)\n\u001b[0m\u001b[0;32m    239\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    240\u001b[0m                 \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtr_X\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtr_y\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mE:\\Anaconda3\\lib\\site-packages\\lightgbm\\sklearn.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y, sample_weight, init_score, eval_set, eval_names, eval_sample_weight, eval_init_score, eval_metric, early_stopping_rounds, verbose, feature_name, categorical_feature, callbacks)\u001b[0m\n\u001b[0;32m    610\u001b[0m                                        \u001b[0mverbose\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mverbose\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeature_name\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mfeature_name\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    611\u001b[0m                                        \u001b[0mcategorical_feature\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcategorical_feature\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 612\u001b[1;33m                                        callbacks=callbacks)\n\u001b[0m\u001b[0;32m    613\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    614\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mE:\\Anaconda3\\lib\\site-packages\\lightgbm\\sklearn.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y, sample_weight, init_score, group, eval_set, eval_names, eval_sample_weight, eval_init_score, eval_group, eval_metric, early_stopping_rounds, verbose, feature_name, categorical_feature, callbacks)\u001b[0m\n\u001b[0;32m    457\u001b[0m                               \u001b[0mverbose_eval\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mverbose\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeature_name\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mfeature_name\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    458\u001b[0m                               \u001b[0mcategorical_feature\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcategorical_feature\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 459\u001b[1;33m                               callbacks=callbacks)\n\u001b[0m\u001b[0;32m    460\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    461\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mevals_result\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mE:\\Anaconda3\\lib\\site-packages\\lightgbm\\engine.py\u001b[0m in \u001b[0;36mtrain\u001b[1;34m(params, train_set, num_boost_round, valid_sets, valid_names, fobj, feval, init_model, feature_name, categorical_feature, early_stopping_rounds, evals_result, verbose_eval, learning_rates, keep_training_booster, callbacks)\u001b[0m\n\u001b[0;32m    197\u001b[0m                                     evaluation_result_list=None))\n\u001b[0;32m    198\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 199\u001b[1;33m         \u001b[0mbooster\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfobj\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mfobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    200\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    201\u001b[0m         \u001b[0mevaluation_result_list\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mE:\\Anaconda3\\lib\\site-packages\\lightgbm\\basic.py\u001b[0m in \u001b[0;36mupdate\u001b[1;34m(self, train_set, fobj)\u001b[0m\n\u001b[0;32m   1437\u001b[0m             _safe_call(_LIB.LGBM_BoosterUpdateOneIter(\n\u001b[0;32m   1438\u001b[0m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1439\u001b[1;33m                 ctypes.byref(is_finished)))\n\u001b[0m\u001b[0;32m   1440\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__is_predicted_cur_iter\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;32mFalse\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0m_\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange_\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__num_dataset\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1441\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mis_finished\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalue\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "default_params = {\n",
    "    'boosting_type':'gbdt', \n",
    "    'num_leaves':64, \n",
    "    'max_depth':18, \n",
    "    'learning_rate':0.02, \n",
    "    'n_estimators':20000, \n",
    "    'min_split_gain':0.0, \n",
    "    'min_child_weight':0.001, \n",
    "    'min_child_samples':20, \n",
    "    'subsample':1., \n",
    "    'subsample_freq':1, \n",
    "    'colsample_bytree':.6, \n",
    "    'reg_alpha':0.0, \n",
    "    'reg_lambda':0.0, \n",
    "    'random_state':SEED, \n",
    "    'n_jobs': 3\n",
    "}\n",
    "\n",
    "\n",
    "'''\n",
    "default_params = {\n",
    "    'boosting_type':'gbdt', \n",
    "    'objective' : 'regression',\n",
    "    'metric' : 'rmse',\n",
    "    'num_leaves' : 48,\n",
    "    'max_depth': 15,\n",
    "    'learning_rate' : 0.05,\n",
    "    'colsample_bytree' : 0.6,\n",
    "    'verbosity' : -1,\n",
    "    'n_estimators': 20000,\n",
    "    'random_state':SEED, \n",
    "}\n",
    "'''\n",
    "fit_param.update({\n",
    "    'early_stopping_rounds': 50,\n",
    "    'verbose': 100,\n",
    "    'eval_metric': 'rmse'\n",
    "})\n",
    "\n",
    "lgb_oof_val_pred, lgb_oof_test_pred, models = get_oof_predictions(train_X, train_y, test_X, ml, \n",
    "                                                                  default_params, seed=SEED, fit_params=fit_param, \n",
    "                                                                  use_eval_set=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "for model in models:\n",
    "    fig, ax = plt.subplots(figsize=(10, 14))\n",
    "    lgb.plot_importance(model, max_num_features=100, ax=ax)\n",
    "    plt.title(\"Light GBM Feature Importance\")\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pd.DataFrame(data=lgb_oof_val_pred, columns=['all_mean_enc_user_ft_lgb_pred']).to_csv('all_mean_enc_user_ft_oof_val_pred.csv', index=False)\n",
    "pd.DataFrame(data=lgb_oof_test_pred, columns=['all_mean_enc_user_ft_lgb_pred']).to_csv('all_mean_enc_user_ft_oof_test_pred.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_df = pd.read_csv(\"test.csv\", usecols=['item_id'])\n",
    "test_pred = pd.read_csv(\"all_mean_enc_user_ft_oof_test_pred.csv\")\n",
    "test_pred = test_pred.loc[:, 'all_mean_enc_user_ft_lgb_pred'].values\n",
    "pd.DataFrame(np.clip(test_pred,0,1), \n",
    "             index=test_df.item_id,\n",
    "             columns=['deal_probability']).to_csv('all_mean_enc_user_ft_lgb_submission.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ridge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ml = model_loader(model_type='rg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "default_params = {\n",
    "    'alpha': 1.0, \n",
    "    'fit_intercept': True, \n",
    "    'normalize': False, \n",
    "    'copy_X': True, \n",
    "    'max_iter': None, \n",
    "    'tol': 0.001, \n",
    "    'solver':'auto', \n",
    "    'random_state': SEED\n",
    "}\n",
    "\n",
    "try_params = \\\n",
    "[\n",
    "    {\n",
    "        'alpha': [0.01, 0.1, 0.5, 1., 2.],\n",
    "    }\n",
    "]\n",
    "\n",
    "fit_params(train_X, train_y, ml, default_params, try_params, fit_params=None, seed=SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "default_params = {\n",
    "    'alpha': 1.0, \n",
    "    'fit_intercept': True, \n",
    "    'normalize': False, \n",
    "    'copy_X': True, \n",
    "    'max_iter': None, \n",
    "    'tol': 0.001, \n",
    "    'solver':'auto', \n",
    "    'random_state': SEED\n",
    "}\n",
    "\n",
    "fit_params = None\n",
    "\n",
    "rg_oof_val_pred, rg_oof_test_pred, models = get_oof_predictions(train_X, train_y, test_X, ml, \n",
    "                                                                  default_params, seed=SEED, fit_params=fit_params, \n",
    "                                                                  use_eval_set=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pd.DataFrame(data=rg_oof_val_pred, columns=['text_rg_pred']).to_csv('text_rg_oof_val_pred.csv', index=False)\n",
    "pd.DataFrame(data=rg_oof_test_pred, columns=['text_rg_pred']).to_csv('text_rg_oof_test_pred.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}