+cat interact feat script.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "f7971b7f-d99e-4b55-854d-7d3823404dbb",
    "_uuid": "ccc855613dc9c2986afc718508e97d21213b7213",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import gc\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import string\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "import scipy\n",
    "import lightgbm as lgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "a58e8629-e64f-4073-818b-b88dbe41583d",
    "_uuid": "d20f7162e1adb56bc3b304f61b7891e7cc3d0521",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gp = pd.read_csv('aggregated_features.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "_cell_guid": "bc719fcf-f418-4db5-a4fa-ebfa27716521",
    "_uuid": "02aad0b977053435e089bd189ea45e4a88731e87"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>region</th>\n",
       "      <th>city</th>\n",
       "      <th>parent_category_name</th>\n",
       "      <th>category_name</th>\n",
       "      <th>param_1</th>\n",
       "      <th>param_2</th>\n",
       "      <th>param_3</th>\n",
       "      <th>title</th>\n",
       "      <th>...</th>\n",
       "      <th>activation_date</th>\n",
       "      <th>user_type</th>\n",
       "      <th>image</th>\n",
       "      <th>image_top_1</th>\n",
       "      <th>deal_probability</th>\n",
       "      <th>avg_days_up_user</th>\n",
       "      <th>avg_times_up_user</th>\n",
       "      <th>med_days_up_user</th>\n",
       "      <th>med_times_up_user</th>\n",
       "      <th>n_user_items</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>b912c3c6a6ad</td>\n",
       "      <td>e00f8ff2eaf9</td>\n",
       "      <td>Свердловская область</td>\n",
       "      <td>Екатеринбург</td>\n",
       "      <td>Личные вещи</td>\n",
       "      <td>Товары для детей и игрушки</td>\n",
       "      <td>Постельные принадлежности</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Кокоби(кокон для сна)</td>\n",
       "      <td>...</td>\n",
       "      <td>2017-03-28</td>\n",
       "      <td>Private</td>\n",
       "      <td>d10c7e016e03247a3bf2d13348fe959fe6f436c1caf64c...</td>\n",
       "      <td>1008.0</td>\n",
       "      <td>0.12789</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>8.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2dac0150717d</td>\n",
       "      <td>39aeb48f0017</td>\n",
       "      <td>Самарская область</td>\n",
       "      <td>Самара</td>\n",
       "      <td>Для дома и дачи</td>\n",
       "      <td>Мебель и интерьер</td>\n",
       "      <td>Другое</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Стойка для Одежды</td>\n",
       "      <td>...</td>\n",
       "      <td>2017-03-26</td>\n",
       "      <td>Private</td>\n",
       "      <td>79c9392cc51a9c81c6eb91eceb8e552171db39d7142700...</td>\n",
       "      <td>692.0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ba83aefab5dc</td>\n",
       "      <td>91e2f88dd6e3</td>\n",
       "      <td>Ростовская область</td>\n",
       "      <td>Ростов-на-Дону</td>\n",
       "      <td>Бытовая электроника</td>\n",
       "      <td>Аудио и видео</td>\n",
       "      <td>Видео, DVD и Blu-ray плееры</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Philips bluray</td>\n",
       "      <td>...</td>\n",
       "      <td>2017-03-20</td>\n",
       "      <td>Private</td>\n",
       "      <td>b7f250ee3f39e1fedd77c141f273703f4a9be59db4b48a...</td>\n",
       "      <td>3032.0</td>\n",
       "      <td>0.43177</td>\n",
       "      <td>4.428571</td>\n",
       "      <td>1.142857</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>02996f1dd2ea</td>\n",
       "      <td>bf5cccea572d</td>\n",
       "      <td>Татарстан</td>\n",
       "      <td>Набережные Челны</td>\n",
       "      <td>Личные вещи</td>\n",
       "      <td>Товары для детей и игрушки</td>\n",
       "      <td>Автомобильные кресла</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Автокресло</td>\n",
       "      <td>...</td>\n",
       "      <td>2017-03-25</td>\n",
       "      <td>Company</td>\n",
       "      <td>e6ef97e0725637ea84e3d203e82dadb43ed3cc0a1c8413...</td>\n",
       "      <td>796.0</td>\n",
       "      <td>0.80323</td>\n",
       "      <td>16.714286</td>\n",
       "      <td>2.642857</td>\n",
       "      <td>18.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>32.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7c90be56d2ab</td>\n",
       "      <td>ef50846afc0b</td>\n",
       "      <td>Волгоградская область</td>\n",
       "      <td>Волгоград</td>\n",
       "      <td>Транспорт</td>\n",
       "      <td>Автомобили</td>\n",
       "      <td>С пробегом</td>\n",
       "      <td>ВАЗ (LADA)</td>\n",
       "      <td>2110</td>\n",
       "      <td>ВАЗ 2110, 2003</td>\n",
       "      <td>...</td>\n",
       "      <td>2017-03-16</td>\n",
       "      <td>Private</td>\n",
       "      <td>54a687a3a0fc1d68aed99bdaaf551c5c70b761b16fd0a2...</td>\n",
       "      <td>2264.0</td>\n",
       "      <td>0.20797</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        item_id       user_id                 region              city  \\\n",
       "0  b912c3c6a6ad  e00f8ff2eaf9   Свердловская область      Екатеринбург   \n",
       "1  2dac0150717d  39aeb48f0017      Самарская область            Самара   \n",
       "2  ba83aefab5dc  91e2f88dd6e3     Ростовская область    Ростов-на-Дону   \n",
       "3  02996f1dd2ea  bf5cccea572d              Татарстан  Набережные Челны   \n",
       "4  7c90be56d2ab  ef50846afc0b  Волгоградская область         Волгоград   \n",
       "\n",
       "  parent_category_name               category_name  \\\n",
       "0          Личные вещи  Товары для детей и игрушки   \n",
       "1      Для дома и дачи           Мебель и интерьер   \n",
       "2  Бытовая электроника               Аудио и видео   \n",
       "3          Личные вещи  Товары для детей и игрушки   \n",
       "4            Транспорт                  Автомобили   \n",
       "\n",
       "                       param_1     param_2 param_3                  title  \\\n",
       "0    Постельные принадлежности         NaN     NaN  Кокоби(кокон для сна)   \n",
       "1                       Другое         NaN     NaN      Стойка для Одежды   \n",
       "2  Видео, DVD и Blu-ray плееры         NaN     NaN         Philips bluray   \n",
       "3         Автомобильные кресла         NaN     NaN             Автокресло   \n",
       "4                   С пробегом  ВАЗ (LADA)    2110         ВАЗ 2110, 2003   \n",
       "\n",
       "       ...      activation_date  user_type  \\\n",
       "0      ...           2017-03-28    Private   \n",
       "1      ...           2017-03-26    Private   \n",
       "2      ...           2017-03-20    Private   \n",
       "3      ...           2017-03-25    Company   \n",
       "4      ...           2017-03-16    Private   \n",
       "\n",
       "                                               image image_top_1  \\\n",
       "0  d10c7e016e03247a3bf2d13348fe959fe6f436c1caf64c...      1008.0   \n",
       "1  79c9392cc51a9c81c6eb91eceb8e552171db39d7142700...       692.0   \n",
       "2  b7f250ee3f39e1fedd77c141f273703f4a9be59db4b48a...      3032.0   \n",
       "3  e6ef97e0725637ea84e3d203e82dadb43ed3cc0a1c8413...       796.0   \n",
       "4  54a687a3a0fc1d68aed99bdaaf551c5c70b761b16fd0a2...      2264.0   \n",
       "\n",
       "  deal_probability avg_days_up_user  avg_times_up_user  med_days_up_user  \\\n",
       "0          0.12789         8.000000           2.000000               8.0   \n",
       "1          0.00000              NaN                NaN               NaN   \n",
       "2          0.43177         4.428571           1.142857               3.0   \n",
       "3          0.80323        16.714286           2.642857              18.0   \n",
       "4          0.20797              NaN                NaN               NaN   \n",
       "\n",
       "   med_times_up_user  n_user_items  \n",
       "0                2.0           2.0  \n",
       "1                NaN           NaN  \n",
       "2                1.0           9.0  \n",
       "3                3.0          32.0  \n",
       "4                NaN           NaN  \n",
       "\n",
       "[5 rows x 23 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv('train.csv', parse_dates=['activation_date'])\n",
    "test = pd.read_csv('test.csv', parse_dates=['activation_date'])\n",
    "\n",
    "train = train.merge(gp, on='user_id', how='left')\n",
    "test = test.merge(gp, on='user_id', how='left')\n",
    "\n",
    "agg_cols = list(gp.columns)[1:]\n",
    "\n",
    "del gp; gc.collect()\n",
    "\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "_cell_guid": "29edf671-7daf-4037-a286-6ad09992cb1b",
    "_uuid": "6fe0d03951f67ff3f28c7c7240089f3631c83c73",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = train.sort_values(['activation_date']).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "_cell_guid": "1985e85e-d8d7-4a58-b0c0-7317abd125fe",
    "_uuid": "9fb7f90bc8d3ed7a2e0d5513050bc634d945db67",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_index, valid_index = train_test_split(np.arange(len(train)), test_size=0.1, random_state=519)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "_cell_guid": "10353cf1-5b93-4fd1-82ea-f252d054e9d0",
    "_uuid": "88c2d490a28a8a82a0258f1343dd5fed455a2f95"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    207048\n",
       "6    205318\n",
       "1    201131\n",
       "2    196631\n",
       "3    191293\n",
       "4    176527\n",
       "5    175133\n",
       "Name: activation_date, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.loc[train_index, 'activation_date'].dt.weekday.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "_cell_guid": "c89bd334-407a-4b13-b78b-82752d38ae3a",
    "_uuid": "89044abdca31af70a5373f007ee10920c5477613"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    23007\n",
       "6    22614\n",
       "1    22289\n",
       "2    21884\n",
       "3    21422\n",
       "4    19599\n",
       "5    19528\n",
       "Name: activation_date, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.loc[valid_index, 'activation_date'].dt.weekday.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "_cell_guid": "64125deb-a81f-459e-b915-975768ac8f38",
    "_uuid": "285fa47da4734b9ecfe7a7a2b8bbc93e9f8c3d3b"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2    81888\n",
       "1    81114\n",
       "0    80191\n",
       "3    77177\n",
       "4    70366\n",
       "6    58909\n",
       "5    58793\n",
       "Name: activation_date, dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.loc[:, 'activation_date'].dt.weekday.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "e624308f-c957-48e8-87f6-f9e78fc5b69f",
    "_uuid": "abbecd367f012736903c150e6fb989de3e135cca"
   },
   "source": [
    "One more thing about the approach that I haven't mentioned yet is that we will have quite some NaN values because not every ID in `train` and `test` occurs in `train_active` and `test_active`. Let's check how big that problem is."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "_cell_guid": "d6a65f65-04a0-4d0d-81ec-4ed0551308bc",
    "_uuid": "65a2f2fe6ca76cc9f742affe57d4bd491f29c3bc"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "22.407185198586692"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train[agg_cols].isnull().any(axis=1).sum() / len(train) * 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "_cell_guid": "5e26f756-3f9e-4eb7-bcbe-5eaedc7d9f35",
    "_uuid": "e088f55ab84dbd80c5a466e551d9eeb44ca9e623"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "24.354198545348694"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test[agg_cols].isnull().any(axis=1).sum() / len(test) * 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(item_id                      0\n",
       " user_id                      0\n",
       " region                       0\n",
       " city                         0\n",
       " parent_category_name         0\n",
       " category_name                0\n",
       " param_1                  61576\n",
       " param_2                 654542\n",
       " param_3                 862565\n",
       " title                        0\n",
       " description             116276\n",
       " price                    85362\n",
       " item_seq_number              0\n",
       " activation_date              0\n",
       " user_type                    0\n",
       " image                   112588\n",
       " image_top_1             112588\n",
       " deal_probability             0\n",
       " avg_days_up_user        336875\n",
       " avg_times_up_user       336875\n",
       " med_days_up_user        336875\n",
       " med_times_up_user       336875\n",
       " n_user_items            336875\n",
       " dtype: int64, item_id                      0\n",
       " user_id                      0\n",
       " region                       0\n",
       " city                         0\n",
       " parent_category_name         0\n",
       " category_name                0\n",
       " param_1                  22910\n",
       " param_2                 233229\n",
       " param_3                 306331\n",
       " title                        0\n",
       " description                  0\n",
       " price                    30585\n",
       " item_seq_number              0\n",
       " activation_date              0\n",
       " user_type                    0\n",
       " image                    42609\n",
       " image_top_1              42609\n",
       " avg_days_up_user        123826\n",
       " avg_times_up_user       123826\n",
       " med_days_up_user        123826\n",
       " med_times_up_user       123826\n",
       " n_user_items            123826\n",
       " dtype: int64)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.isnull().sum(), test.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "06a9a6c2-a602-48c5-bcb5-d7ed40cc463d",
    "_uuid": "813b1e3a93f96e3bc7d9f2375e452f8b109112fa"
   },
   "source": [
    "We have missing features for 22.41% of train and 24.35% of test data. That's not perfect but certainly acceptable. Onto some more basic feature engineering with ideas from [a great kernel](https://www.kaggle.com/tunguz/bow-meta-text-and-dense-features-lb-0-2241?scriptVersionId=3603709)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "_cell_guid": "36649982-801e-4e2b-a954-a0c8aac37702",
    "_uuid": "ea823fe2ee45c3492ca70d75ce3d9a7041d57024",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "_cell_guid": "41e1111d-674e-4c86-b232-c0086832ee6d",
    "_uuid": "9c60409e1dc6c618f229e15c2b368660385c2ec1",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "count = lambda l1,l2: sum([1 for x in l1 if x in l2])\n",
    "\n",
    "\n",
    "for df in [train, test]:\n",
    "    df['description'].fillna('unknowndescription', inplace=True)\n",
    "    df['title'].fillna('unknowntitle', inplace=True)\n",
    "\n",
    "    df['weekday'] = pd.to_datetime(df['activation_date']).dt.day\n",
    "    \n",
    "    for col in ['description', 'title']:\n",
    "        df['num_words_' + col] = df[col].apply(lambda comment: len(comment.split()))\n",
    "        df['num_unique_words_' + col] = df[col].apply(lambda comment: len(set(w for w in comment.split())))\n",
    "\n",
    "    df['words_vs_unique_title'] = df['num_unique_words_title'] / df['num_words_title'] * 100\n",
    "    df['words_vs_unique_description'] = df['num_unique_words_description'] / df['num_words_description'] * 100\n",
    "    \n",
    "    df['city'] = df['region'] + '_' + df['city']\n",
    "    df['num_desc_punct'] = df['description'].apply(lambda x: count(x, set(string.punctuation)))\n",
    "    \n",
    "    for col in agg_cols:\n",
    "        df[col].fillna(-1, inplace=True)\n",
    "    \n",
    "    for col in ['price', 'image_top_1']:\n",
    "         df[col].fillna(-1, inplace=True)\n",
    "            \n",
    "    for col in ['param_1', 'param_2', 'param_3']:\n",
    "         df[col].fillna('khwinkaggle', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\viola\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('stopwords')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "_cell_guid": "8d61ada6-8f3a-4b58-baad-199589122985",
    "_uuid": "cc558a03cbddc3954b01783636785cdee6cf954a"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((1503424, 16447), (1503424, 15000))"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count_vectorizer_title = CountVectorizer(stop_words=stopwords.words('russian'), lowercase=True, min_df=25)\n",
    "\n",
    "title_counts = count_vectorizer_title.fit_transform(train['title'].append(test['title']))\n",
    "\n",
    "train_title_counts = title_counts[:len(train)]\n",
    "test_title_counts = title_counts[len(train):]\n",
    "\n",
    "\n",
    "count_vectorizer_desc = TfidfVectorizer(stop_words=stopwords.words('russian'), \n",
    "                                        lowercase=True, ngram_range=(1, 2),\n",
    "                                        max_features=15000)\n",
    "\n",
    "desc_counts = count_vectorizer_desc.fit_transform(train['description'].append(test['description']))\n",
    "\n",
    "train_desc_counts = desc_counts[:len(train)]\n",
    "test_desc_counts = desc_counts[len(train):]\n",
    "\n",
    "train_title_counts.shape, train_desc_counts.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "_cell_guid": "56338e56-c266-4342-a8f3-10621db49dee",
    "_uuid": "cc6f15f981f05ad9b1e89ab7acb54ea2350278c6",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "target = 'deal_probability'\n",
    "predictors = [\n",
    "    'num_desc_punct', \n",
    "    'words_vs_unique_description', 'num_unique_words_description', 'num_unique_words_title', 'num_words_description', 'num_words_title',\n",
    "    'avg_times_up_user', 'avg_days_up_user', 'n_user_items', \n",
    "    'price', 'item_seq_number'\n",
    "]\n",
    "categorical = [\n",
    "    'image_top_1', 'param_1', 'param_2', 'param_3', \n",
    "    'city', 'region', 'category_name', 'parent_category_name', 'user_type'\n",
    "]\n",
    "\n",
    "predictors = predictors + categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "_cell_guid": "7695c1a1-06cf-40e6-8792-77b62f48262a",
    "_uuid": "0bcde9bb9b9b26347c7db7da47123c6fec40860e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Transforming image_top_1...\n",
      "Transforming param_1...\n",
      "Transforming param_2...\n",
      "Transforming param_3...\n",
      "Transforming city...\n",
      "Transforming region...\n",
      "Transforming category_name...\n",
      "Transforming parent_category_name...\n",
      "Transforming user_type...\n"
     ]
    }
   ],
   "source": [
    "for feature in categorical:\n",
    "    print(f'Transforming {feature}...')\n",
    "    encoder = LabelEncoder()\n",
    "    encoder.fit(train[feature].append(test[feature]).astype(str))\n",
    "    \n",
    "    train[feature] = encoder.transform(train[feature].astype(str))\n",
    "    test[feature] = encoder.transform(test[feature].astype(str))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cat Interaction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pairs = [('parent_category_name', 'category_name'), \n",
    "         ('parent_category_name', 'param_1'), ('parent_category_name', 'param_2'), ('parent_category_name', 'param_3'), \n",
    "         ('category_name', 'param_1'), ('category_name', 'param_2'), ('category_name', 'param_3'),\n",
    "         ('parent_category_name', 'region'), ('category_name', 'region'),\n",
    "         ('user_type', 'region'), \n",
    "         #('user_type', 'city'), ## too much\n",
    "         ('user_type', 'parent_category_name'), ('user_type', 'category_name'),\n",
    "         #('parent_category_name', 'image_top_1'), ('category_name', 'image_top_1'),\n",
    "        ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "labelize_cols = ['parent_category_name_category_name', \n",
    "                 'parent_category_name_param_1', \n",
    "                 'parent_category_name_param_2', \n",
    "                 'parent_category_name_param_3', \n",
    "                 'category_name_param_1',\n",
    "                 'category_name_param_2', \n",
    "                 'category_name_param_3', \n",
    "                 'parent_category_name_region', 'category_name_region',\n",
    "                 'user_type_region', 'user_type_parent_category_name', 'user_type_category_name']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing pair: ('parent_category_name', 'category_name')\n",
      "Processing pair: ('parent_category_name', 'param_1')\n",
      "Processing pair: ('parent_category_name', 'param_2')\n",
      "Processing pair: ('parent_category_name', 'param_3')\n",
      "Processing pair: ('category_name', 'param_1')\n",
      "Processing pair: ('category_name', 'param_2')\n",
      "Processing pair: ('category_name', 'param_3')\n",
      "Processing pair: ('parent_category_name', 'region')\n",
      "Processing pair: ('category_name', 'region')\n",
      "Processing pair: ('user_type', 'region')\n",
      "Processing pair: ('user_type', 'parent_category_name')\n",
      "Processing pair: ('user_type', 'category_name')\n"
     ]
    }
   ],
   "source": [
    "for pair in pairs:\n",
    "    col1 = pair[0]\n",
    "    col2 = pair[1]\n",
    "    print('Processing pair:', pair)\n",
    "    new_feature = col1+'_'+col2\n",
    "    train.loc[:, new_feature] = train[col1].astype(str)+'_'+train[col2].astype(str)\n",
    "    test.loc[:, new_feature] = test[col1].astype(str)+'_'+test[col2].astype(str)\n",
    "    \n",
    "    encoder = LabelEncoder()\n",
    "    encoder.fit(train[new_feature].append(test[new_feature]).astype(str))\n",
    "    \n",
    "    train[new_feature] = encoder.transform(train[new_feature].astype(str))\n",
    "    test[new_feature] = encoder.transform(test[new_feature].astype(str)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "categorical.extend(labelize_cols)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "target = 'deal_probability'\n",
    "predictors = [\n",
    "    'num_desc_punct', \n",
    "    'words_vs_unique_description', 'num_unique_words_description', 'num_unique_words_title', 'num_words_description', 'num_words_title',\n",
    "    'avg_times_up_user', 'avg_days_up_user', 'n_user_items', \n",
    "    'price', 'item_seq_number'\n",
    "]\n",
    "\n",
    "categorical = [\n",
    "    'image_top_1', 'param_1', 'param_2', 'param_3', \n",
    "    'city', 'region', 'category_name', 'parent_category_name', 'user_type',\n",
    "    'parent_category_name_category_name', \n",
    "    'parent_category_name_param_1', \n",
    "    'parent_category_name_param_2', \n",
    "    'parent_category_name_param_3', \n",
    "    'category_name_param_1',\n",
    "    'category_name_param_2', \n",
    "    'category_name_param_3', \n",
    "    'parent_category_name_region', 'category_name_region',\n",
    "    'user_type_region', 'user_type_parent_category_name', 'user_type_category_name'\n",
    "]\n",
    "\n",
    "predictors = predictors + categorical"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "edf2ac02-764e-42d3-bd00-8c1f79878d52",
    "_uuid": "312ce17bcca12df526c85cb9c85e7ac67c9246c7"
   },
   "source": [
    "After some hyperparameter definitions and creating train / valid / test matrices, we can finally train the model. Let's see if the aggregated features helped.\n",
    "\n",
    "*Note: For further feature engineering, I would recommend restricting the max_depth further (5 worked well for me) and increasing the learning rate (to ~ 0.1) so you don't have to wait forever for the training to finish.*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "_cell_guid": "7e94bcc4-ab56-4a38-bdcf-218791e95c2c",
    "_uuid": "ac57e30cf4452005a2cb20db424d7bcf1f11d1df"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of features: 31484\n"
     ]
    }
   ],
   "source": [
    "rounds = 24000\n",
    "early_stop_rounds = 200\n",
    "params = {\n",
    "    'objective' : 'regression',\n",
    "    'metric' : 'rmse',\n",
    "    'num_leaves' : 48,\n",
    "    'max_depth': 15,\n",
    "    'learning_rate' : 0.02,\n",
    "    'feature_fraction' : 0.6,\n",
    "    'verbosity' : -1\n",
    "}\n",
    "\n",
    "feature_names = np.hstack([\n",
    "    count_vectorizer_desc.get_feature_names(),\n",
    "    count_vectorizer_title.get_feature_names(),\n",
    "    predictors\n",
    "])\n",
    "print('Number of features:', len(feature_names))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "kf = KFold(5, shuffle=True, random_state=411)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_test = scipy.sparse.hstack([\n",
    "    test_desc_counts,\n",
    "    test_title_counts,\n",
    "    test.loc[:, predictors]\n",
    "], format='csr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_train = scipy.sparse.hstack([\n",
    "        train_desc_counts,\n",
    "        train_title_counts,\n",
    "        train.loc[:, predictors]\n",
    "    ], format='csr')\n",
    "y_train = train.loc[:, target].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from GridSearcher import data_loader, model_loader, fit_params, get_oof_predictions\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ml = model_loader('lgb')\n",
    "SEED=719"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Anaconda3\\lib\\site-packages\\lightgbm\\basic.py:1007: UserWarning: categorical_feature in Dataset is overrided. New categorical_feature is ['category_name', 'category_name_param_1', 'category_name_param_2', 'category_name_param_3', 'category_name_region', 'city', 'image_top_1', 'param_1', 'param_2', 'param_3', 'parent_category_name', 'parent_category_name_category_name', 'parent_category_name_param_1', 'parent_category_name_param_2', 'parent_category_name_param_3', 'parent_category_name_region', 'region', 'user_type', 'user_type_category_name', 'user_type_parent_category_name', 'user_type_region']\n",
      "  warnings.warn('categorical_feature in Dataset is overrided. New categorical_feature is {}'.format(sorted(list(categorical_feature))))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 50 rounds.\n",
      "[100]\ttrain's rmse: 0.222178\tvalid's rmse: 0.225392\n",
      "[200]\ttrain's rmse: 0.219733\tvalid's rmse: 0.224142\n",
      "[300]\ttrain's rmse: 0.218028\tvalid's rmse: 0.223395\n",
      "[400]\ttrain's rmse: 0.216692\tvalid's rmse: 0.222917\n",
      "[500]\ttrain's rmse: 0.215484\tvalid's rmse: 0.222488\n",
      "[600]\ttrain's rmse: 0.214536\tvalid's rmse: 0.222198\n",
      "[700]\ttrain's rmse: 0.21368\tvalid's rmse: 0.221961\n",
      "[800]\ttrain's rmse: 0.212794\tvalid's rmse: 0.221718\n",
      "[900]\ttrain's rmse: 0.212131\tvalid's rmse: 0.221577\n",
      "[1000]\ttrain's rmse: 0.211567\tvalid's rmse: 0.221464\n",
      "[1100]\ttrain's rmse: 0.210834\tvalid's rmse: 0.221298\n",
      "[1200]\ttrain's rmse: 0.210197\tvalid's rmse: 0.221179\n",
      "[1300]\ttrain's rmse: 0.209664\tvalid's rmse: 0.221062\n",
      "[1400]\ttrain's rmse: 0.209088\tvalid's rmse: 0.220955\n",
      "[1500]\ttrain's rmse: 0.208599\tvalid's rmse: 0.220886\n",
      "[1600]\ttrain's rmse: 0.208138\tvalid's rmse: 0.220816\n",
      "[1700]\ttrain's rmse: 0.207498\tvalid's rmse: 0.220722\n",
      "[1800]\ttrain's rmse: 0.20707\tvalid's rmse: 0.220665\n",
      "[1900]\ttrain's rmse: 0.206724\tvalid's rmse: 0.220631\n",
      "[2000]\ttrain's rmse: 0.206315\tvalid's rmse: 0.22059\n",
      "{'random_state': 719} train loss: 0.206257, valid loss:0.220519, loss_diff:0.014262\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Anaconda3\\lib\\site-packages\\lightgbm\\basic.py:1007: UserWarning: categorical_feature in Dataset is overrided. New categorical_feature is ['category_name', 'category_name_param_1', 'category_name_param_2', 'category_name_param_3', 'category_name_region', 'city', 'image_top_1', 'param_1', 'param_2', 'param_3', 'parent_category_name', 'parent_category_name_category_name', 'parent_category_name_param_1', 'parent_category_name_param_2', 'parent_category_name_param_3', 'parent_category_name_region', 'region', 'user_type', 'user_type_category_name', 'user_type_parent_category_name', 'user_type_region']\n",
      "  warnings.warn('categorical_feature in Dataset is overrided. New categorical_feature is {}'.format(sorted(list(categorical_feature))))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 50 rounds.\n",
      "[100]\ttrain's rmse: 0.222232\tvalid's rmse: 0.224877\n",
      "[200]\ttrain's rmse: 0.219528\tvalid's rmse: 0.22341\n",
      "[300]\ttrain's rmse: 0.217869\tvalid's rmse: 0.222723\n",
      "[400]\ttrain's rmse: 0.21647\tvalid's rmse: 0.222196\n",
      "[500]\ttrain's rmse: 0.215417\tvalid's rmse: 0.221873\n",
      "[600]\ttrain's rmse: 0.21449\tvalid's rmse: 0.221613\n",
      "[700]\ttrain's rmse: 0.213545\tvalid's rmse: 0.221351\n",
      "[800]\ttrain's rmse: 0.212774\tvalid's rmse: 0.221197\n",
      "[900]\ttrain's rmse: 0.21217\tvalid's rmse: 0.22105\n",
      "[1000]\ttrain's rmse: 0.211541\tvalid's rmse: 0.22093\n",
      "[1100]\ttrain's rmse: 0.210841\tvalid's rmse: 0.220778\n",
      "[1200]\ttrain's rmse: 0.210183\tvalid's rmse: 0.220641\n",
      "[1300]\ttrain's rmse: 0.209634\tvalid's rmse: 0.220534\n",
      "[1400]\ttrain's rmse: 0.20916\tvalid's rmse: 0.220461\n",
      "[1500]\ttrain's rmse: 0.208549\tvalid's rmse: 0.220353\n",
      "[1600]\ttrain's rmse: 0.208124\tvalid's rmse: 0.220303\n",
      "[1700]\ttrain's rmse: 0.207752\tvalid's rmse: 0.220259\n",
      "[1800]\ttrain's rmse: 0.20713\tvalid's rmse: 0.220151\n",
      "[1900]\ttrain's rmse: 0.206666\tvalid's rmse: 0.220061\n",
      "[2000]\ttrain's rmse: 0.206275\tvalid's rmse: 0.220035\n",
      "{'random_state': 719} train loss: 0.206218, valid loss:0.219955, loss_diff:0.013737\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Anaconda3\\lib\\site-packages\\lightgbm\\basic.py:1007: UserWarning: categorical_feature in Dataset is overrided. New categorical_feature is ['category_name', 'category_name_param_1', 'category_name_param_2', 'category_name_param_3', 'category_name_region', 'city', 'image_top_1', 'param_1', 'param_2', 'param_3', 'parent_category_name', 'parent_category_name_category_name', 'parent_category_name_param_1', 'parent_category_name_param_2', 'parent_category_name_param_3', 'parent_category_name_region', 'region', 'user_type', 'user_type_category_name', 'user_type_parent_category_name', 'user_type_region']\n",
      "  warnings.warn('categorical_feature in Dataset is overrided. New categorical_feature is {}'.format(sorted(list(categorical_feature))))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 50 rounds.\n",
      "[100]\ttrain's rmse: 0.222269\tvalid's rmse: 0.22478\n",
      "[200]\ttrain's rmse: 0.219571\tvalid's rmse: 0.223431\n",
      "[300]\ttrain's rmse: 0.217895\tvalid's rmse: 0.222719\n",
      "[400]\ttrain's rmse: 0.216458\tvalid's rmse: 0.222174\n",
      "[500]\ttrain's rmse: 0.215417\tvalid's rmse: 0.221843\n",
      "[600]\ttrain's rmse: 0.214505\tvalid's rmse: 0.221613\n",
      "[700]\ttrain's rmse: 0.213613\tvalid's rmse: 0.221396\n",
      "[800]\ttrain's rmse: 0.212907\tvalid's rmse: 0.221223\n",
      "[900]\ttrain's rmse: 0.212104\tvalid's rmse: 0.221033\n",
      "[1000]\ttrain's rmse: 0.211368\tvalid's rmse: 0.220847\n",
      "[1100]\ttrain's rmse: 0.210714\tvalid's rmse: 0.220734\n",
      "[1200]\ttrain's rmse: 0.210063\tvalid's rmse: 0.220573\n",
      "[1300]\ttrain's rmse: 0.209482\tvalid's rmse: 0.220467\n",
      "[1400]\ttrain's rmse: 0.209001\tvalid's rmse: 0.220385\n",
      "[1500]\ttrain's rmse: 0.208512\tvalid's rmse: 0.22033\n",
      "[1600]\ttrain's rmse: 0.207937\tvalid's rmse: 0.220236\n",
      "[1700]\ttrain's rmse: 0.207506\tvalid's rmse: 0.220159\n",
      "[1800]\ttrain's rmse: 0.206984\tvalid's rmse: 0.220048\n",
      "[1900]\ttrain's rmse: 0.206566\tvalid's rmse: 0.220001\n",
      "[2000]\ttrain's rmse: 0.206175\tvalid's rmse: 0.219965\n",
      "{'random_state': 719} train loss: 0.206117, valid loss:0.219889, loss_diff:0.013772\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Anaconda3\\lib\\site-packages\\lightgbm\\basic.py:1007: UserWarning: categorical_feature in Dataset is overrided. New categorical_feature is ['category_name', 'category_name_param_1', 'category_name_param_2', 'category_name_param_3', 'category_name_region', 'city', 'image_top_1', 'param_1', 'param_2', 'param_3', 'parent_category_name', 'parent_category_name_category_name', 'parent_category_name_param_1', 'parent_category_name_param_2', 'parent_category_name_param_3', 'parent_category_name_region', 'region', 'user_type', 'user_type_category_name', 'user_type_parent_category_name', 'user_type_region']\n",
      "  warnings.warn('categorical_feature in Dataset is overrided. New categorical_feature is {}'.format(sorted(list(categorical_feature))))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 50 rounds.\n",
      "[100]\ttrain's rmse: 0.222132\tvalid's rmse: 0.225229\n",
      "[200]\ttrain's rmse: 0.219486\tvalid's rmse: 0.223918\n",
      "[300]\ttrain's rmse: 0.217795\tvalid's rmse: 0.223207\n",
      "[400]\ttrain's rmse: 0.21644\tvalid's rmse: 0.222704\n",
      "[500]\ttrain's rmse: 0.21526\tvalid's rmse: 0.222308\n",
      "[600]\ttrain's rmse: 0.214264\tvalid's rmse: 0.222003\n",
      "[700]\ttrain's rmse: 0.213476\tvalid's rmse: 0.221779\n",
      "[800]\ttrain's rmse: 0.212684\tvalid's rmse: 0.221573\n",
      "[900]\ttrain's rmse: 0.211945\tvalid's rmse: 0.221394\n",
      "[1000]\ttrain's rmse: 0.211268\tvalid's rmse: 0.221229\n",
      "[1100]\ttrain's rmse: 0.21059\tvalid's rmse: 0.22108\n",
      "[1200]\ttrain's rmse: 0.210039\tvalid's rmse: 0.220977\n",
      "[1300]\ttrain's rmse: 0.209504\tvalid's rmse: 0.220865\n",
      "[1400]\ttrain's rmse: 0.208833\tvalid's rmse: 0.220761\n",
      "[1500]\ttrain's rmse: 0.208296\tvalid's rmse: 0.220668\n",
      "[1600]\ttrain's rmse: 0.207811\tvalid's rmse: 0.220588\n",
      "[1700]\ttrain's rmse: 0.207354\tvalid's rmse: 0.220515\n",
      "[1800]\ttrain's rmse: 0.206945\tvalid's rmse: 0.220454\n",
      "[1900]\ttrain's rmse: 0.206451\tvalid's rmse: 0.220395\n",
      "[2000]\ttrain's rmse: 0.206077\tvalid's rmse: 0.22036\n",
      "{'random_state': 719} train loss: 0.206018, valid loss:0.220281, loss_diff:0.014263\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Anaconda3\\lib\\site-packages\\lightgbm\\basic.py:1007: UserWarning: categorical_feature in Dataset is overrided. New categorical_feature is ['category_name', 'category_name_param_1', 'category_name_param_2', 'category_name_param_3', 'category_name_region', 'city', 'image_top_1', 'param_1', 'param_2', 'param_3', 'parent_category_name', 'parent_category_name_category_name', 'parent_category_name_param_1', 'parent_category_name_param_2', 'parent_category_name_param_3', 'parent_category_name_region', 'region', 'user_type', 'user_type_category_name', 'user_type_parent_category_name', 'user_type_region']\n",
      "  warnings.warn('categorical_feature in Dataset is overrided. New categorical_feature is {}'.format(sorted(list(categorical_feature))))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 50 rounds.\n",
      "[100]\ttrain's rmse: 0.222259\tvalid's rmse: 0.223884\n",
      "[200]\ttrain's rmse: 0.219827\tvalid's rmse: 0.22267\n",
      "[300]\ttrain's rmse: 0.218091\tvalid's rmse: 0.22192\n",
      "[400]\ttrain's rmse: 0.216836\tvalid's rmse: 0.221464\n",
      "[500]\ttrain's rmse: 0.21564\tvalid's rmse: 0.221046\n",
      "[600]\ttrain's rmse: 0.21466\tvalid's rmse: 0.220774\n",
      "[700]\ttrain's rmse: 0.213829\tvalid's rmse: 0.220562\n",
      "[800]\ttrain's rmse: 0.213097\tvalid's rmse: 0.220377\n",
      "[900]\ttrain's rmse: 0.2124\tvalid's rmse: 0.220224\n",
      "[1000]\ttrain's rmse: 0.211716\tvalid's rmse: 0.220069\n",
      "[1100]\ttrain's rmse: 0.211088\tvalid's rmse: 0.219942\n",
      "[1200]\ttrain's rmse: 0.210436\tvalid's rmse: 0.21984\n",
      "[1300]\ttrain's rmse: 0.209933\tvalid's rmse: 0.21976\n",
      "[1400]\ttrain's rmse: 0.209373\tvalid's rmse: 0.219644\n",
      "[1500]\ttrain's rmse: 0.208733\tvalid's rmse: 0.219536\n",
      "[1600]\ttrain's rmse: 0.208259\tvalid's rmse: 0.219479\n",
      "[1700]\ttrain's rmse: 0.207829\tvalid's rmse: 0.219407\n",
      "[1800]\ttrain's rmse: 0.207367\tvalid's rmse: 0.219355\n",
      "[1900]\ttrain's rmse: 0.206893\tvalid's rmse: 0.219302\n",
      "[2000]\ttrain's rmse: 0.206466\tvalid's rmse: 0.219271\n",
      "{'random_state': 719} train loss: 0.206410, valid loss:0.219198, loss_diff:0.012788\n",
      "=================>{'random_state': 719} loss:0.219968\n",
      "Best params: {'random_state': 719} \tbest loss: 0.219968495791\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>param</th>\n",
       "      <th>val_loss_mean</th>\n",
       "      <th>val_loss_std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>{'random_state': 719}</td>\n",
       "      <td>0.219968</td>\n",
       "      <td>0.000448</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   param  val_loss_mean  val_loss_std\n",
       "0  {'random_state': 719}       0.219968      0.000448"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "default_params = {\n",
    "    'boosting_type':'gbdt', \n",
    "    'num_leaves':31, \n",
    "    'max_depth':5, \n",
    "    'learning_rate':0.1, \n",
    "    'n_estimators':2000, \n",
    "    'min_split_gain':0.0, \n",
    "    'min_child_weight':0.001, \n",
    "    'min_child_samples':20, \n",
    "    'subsample':1., \n",
    "    'subsample_freq':1, \n",
    "    'colsample_bytree':.6, \n",
    "    'reg_alpha':0.0, \n",
    "    'reg_lambda':0.0, \n",
    "    'random_state':SEED, \n",
    "    'n_jobs': 3\n",
    "}\n",
    "fit_param = {\n",
    "    'feature_name': list(feature_names), \n",
    "    'categorical_feature': categorical,\n",
    "}\n",
    "fit_param.update({\n",
    "    'early_stopping_rounds': 50,\n",
    "    'verbose': 100,\n",
    "    'eval_metric': 'rmse'\n",
    "})\n",
    "\n",
    "try_params = {\n",
    "    'random_state': [719]\n",
    "}\n",
    "\n",
    "fit_params(x_train, y_train, ml, default_params, try_params, fit_params=fit_param, seed=SEED, use_eval_set=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Gen Submissions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "9e910115-312f-4437-8ba3-9fb6f7426af8",
    "_uuid": "1a50b3e34d76019fdbfd45c3566eec09556f24cb",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Anaconda3\\lib\\site-packages\\lightgbm\\basic.py:1005: UserWarning: Using categorical_feature in Dataset.\n",
      "  warnings.warn('Using categorical_feature in Dataset.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 200 rounds.\n",
      "[100]\ttrain's rmse: 0.227302\tvalid's rmse: 0.228307\n",
      "[200]\ttrain's rmse: 0.222449\tvalid's rmse: 0.224899\n",
      "[300]\ttrain's rmse: 0.219561\tvalid's rmse: 0.223295\n",
      "[400]\ttrain's rmse: 0.217639\tvalid's rmse: 0.222424\n",
      "[500]\ttrain's rmse: 0.216249\tvalid's rmse: 0.221881\n",
      "[600]\ttrain's rmse: 0.215163\tvalid's rmse: 0.221479\n",
      "[700]\ttrain's rmse: 0.214225\tvalid's rmse: 0.221167\n",
      "[800]\ttrain's rmse: 0.213412\tvalid's rmse: 0.220925\n",
      "[900]\ttrain's rmse: 0.212658\tvalid's rmse: 0.220709\n",
      "[1000]\ttrain's rmse: 0.211986\tvalid's rmse: 0.22052\n",
      "[1100]\ttrain's rmse: 0.21138\tvalid's rmse: 0.220356\n",
      "[1200]\ttrain's rmse: 0.210782\tvalid's rmse: 0.220204\n",
      "[1300]\ttrain's rmse: 0.210244\tvalid's rmse: 0.220079\n",
      "[1400]\ttrain's rmse: 0.209696\tvalid's rmse: 0.219966\n",
      "[1500]\ttrain's rmse: 0.20921\tvalid's rmse: 0.219853\n",
      "[1600]\ttrain's rmse: 0.208748\tvalid's rmse: 0.219768\n",
      "[1700]\ttrain's rmse: 0.2083\tvalid's rmse: 0.219678\n",
      "[1800]\ttrain's rmse: 0.207868\tvalid's rmse: 0.2196\n",
      "[1900]\ttrain's rmse: 0.20746\tvalid's rmse: 0.219524\n",
      "[2000]\ttrain's rmse: 0.207084\tvalid's rmse: 0.219458\n",
      "[2100]\ttrain's rmse: 0.206709\tvalid's rmse: 0.219397\n",
      "[2200]\ttrain's rmse: 0.206335\tvalid's rmse: 0.219336\n",
      "[2300]\ttrain's rmse: 0.20599\tvalid's rmse: 0.219279\n",
      "[2400]\ttrain's rmse: 0.205634\tvalid's rmse: 0.219228\n",
      "[2500]\ttrain's rmse: 0.205277\tvalid's rmse: 0.219173\n",
      "[2600]\ttrain's rmse: 0.20494\tvalid's rmse: 0.219123\n",
      "[2700]\ttrain's rmse: 0.204632\tvalid's rmse: 0.219081\n",
      "[2800]\ttrain's rmse: 0.204304\tvalid's rmse: 0.21904\n",
      "[2900]\ttrain's rmse: 0.20396\tvalid's rmse: 0.218984\n",
      "[3000]\ttrain's rmse: 0.203656\tvalid's rmse: 0.218951\n",
      "[3100]\ttrain's rmse: 0.203353\tvalid's rmse: 0.218917\n",
      "[3200]\ttrain's rmse: 0.203069\tvalid's rmse: 0.218881\n",
      "[3300]\ttrain's rmse: 0.202785\tvalid's rmse: 0.218856\n",
      "[3400]\ttrain's rmse: 0.2025\tvalid's rmse: 0.218825\n",
      "[3500]\ttrain's rmse: 0.202221\tvalid's rmse: 0.218791\n",
      "[3600]\ttrain's rmse: 0.201944\tvalid's rmse: 0.218767\n",
      "[3700]\ttrain's rmse: 0.201675\tvalid's rmse: 0.218735\n",
      "[3800]\ttrain's rmse: 0.201416\tvalid's rmse: 0.21871\n",
      "[3900]\ttrain's rmse: 0.20115\tvalid's rmse: 0.218687\n",
      "[4000]\ttrain's rmse: 0.200889\tvalid's rmse: 0.218663\n",
      "[4100]\ttrain's rmse: 0.200626\tvalid's rmse: 0.218637\n",
      "[4200]\ttrain's rmse: 0.200365\tvalid's rmse: 0.218613\n",
      "[4300]\ttrain's rmse: 0.200121\tvalid's rmse: 0.21859\n",
      "[4400]\ttrain's rmse: 0.199866\tvalid's rmse: 0.218567\n",
      "[4500]\ttrain's rmse: 0.199621\tvalid's rmse: 0.218552\n",
      "[4600]\ttrain's rmse: 0.199392\tvalid's rmse: 0.218537\n",
      "[4700]\ttrain's rmse: 0.199152\tvalid's rmse: 0.218519\n",
      "[4800]\ttrain's rmse: 0.198921\tvalid's rmse: 0.218502\n",
      "[4900]\ttrain's rmse: 0.198674\tvalid's rmse: 0.218478\n",
      "[5000]\ttrain's rmse: 0.198428\tvalid's rmse: 0.218461\n",
      "[5100]\ttrain's rmse: 0.198196\tvalid's rmse: 0.218441\n",
      "[5200]\ttrain's rmse: 0.197951\tvalid's rmse: 0.218421\n",
      "[5300]\ttrain's rmse: 0.19772\tvalid's rmse: 0.218404\n",
      "[5400]\ttrain's rmse: 0.197504\tvalid's rmse: 0.21839\n",
      "[5500]\ttrain's rmse: 0.197277\tvalid's rmse: 0.218382\n",
      "[5600]\ttrain's rmse: 0.197023\tvalid's rmse: 0.218364\n",
      "[5700]\ttrain's rmse: 0.196783\tvalid's rmse: 0.218352\n",
      "[5800]\ttrain's rmse: 0.196562\tvalid's rmse: 0.218338\n",
      "[5900]\ttrain's rmse: 0.196345\tvalid's rmse: 0.218327\n",
      "[6000]\ttrain's rmse: 0.196118\tvalid's rmse: 0.218315\n",
      "[6100]\ttrain's rmse: 0.195915\tvalid's rmse: 0.2183\n",
      "[6200]\ttrain's rmse: 0.195695\tvalid's rmse: 0.218288\n",
      "[6300]\ttrain's rmse: 0.195489\tvalid's rmse: 0.218276\n",
      "[6400]\ttrain's rmse: 0.195288\tvalid's rmse: 0.218266\n",
      "[6500]\ttrain's rmse: 0.195077\tvalid's rmse: 0.218253\n",
      "[6600]\ttrain's rmse: 0.194863\tvalid's rmse: 0.218241\n",
      "[6700]\ttrain's rmse: 0.194654\tvalid's rmse: 0.218235\n",
      "[6800]\ttrain's rmse: 0.194451\tvalid's rmse: 0.21823\n",
      "[6900]\ttrain's rmse: 0.19423\tvalid's rmse: 0.218225\n",
      "[7000]\ttrain's rmse: 0.194015\tvalid's rmse: 0.218212\n",
      "[7100]\ttrain's rmse: 0.19381\tvalid's rmse: 0.2182\n",
      "[7200]\ttrain's rmse: 0.19361\tvalid's rmse: 0.218194\n",
      "[7300]\ttrain's rmse: 0.193411\tvalid's rmse: 0.218185\n",
      "[7400]\ttrain's rmse: 0.19321\tvalid's rmse: 0.218177\n",
      "[7500]\ttrain's rmse: 0.19301\tvalid's rmse: 0.218168\n",
      "[7600]\ttrain's rmse: 0.192817\tvalid's rmse: 0.21816\n",
      "[7700]\ttrain's rmse: 0.192614\tvalid's rmse: 0.218155\n",
      "[7800]\ttrain's rmse: 0.192426\tvalid's rmse: 0.218151\n",
      "[7900]\ttrain's rmse: 0.19224\tvalid's rmse: 0.218143\n",
      "[8000]\ttrain's rmse: 0.192042\tvalid's rmse: 0.218137\n",
      "[8100]\ttrain's rmse: 0.191847\tvalid's rmse: 0.21813\n",
      "[8200]\ttrain's rmse: 0.191654\tvalid's rmse: 0.218127\n",
      "[8300]\ttrain's rmse: 0.191443\tvalid's rmse: 0.218123\n",
      "[8400]\ttrain's rmse: 0.191259\tvalid's rmse: 0.218116\n",
      "[8500]\ttrain's rmse: 0.191067\tvalid's rmse: 0.218109\n",
      "[8600]\ttrain's rmse: 0.190891\tvalid's rmse: 0.218105\n",
      "[8700]\ttrain's rmse: 0.190704\tvalid's rmse: 0.218101\n",
      "[8800]\ttrain's rmse: 0.190496\tvalid's rmse: 0.218099\n",
      "[8900]\ttrain's rmse: 0.190307\tvalid's rmse: 0.218097\n",
      "[9000]\ttrain's rmse: 0.19012\tvalid's rmse: 0.218094\n",
      "[9100]\ttrain's rmse: 0.189934\tvalid's rmse: 0.218089\n",
      "[9200]\ttrain's rmse: 0.189758\tvalid's rmse: 0.218087\n",
      "[9300]\ttrain's rmse: 0.18958\tvalid's rmse: 0.218085\n"
     ]
    }
   ],
   "source": [
    "ret = np.zeros((train.shape[0],))\n",
    "ret_test = np.zeros((x_test.shape[0],))\n",
    "ret_models = []\n",
    "    \n",
    "for train_ix, val_ix in kf.split(train):\n",
    "    dtrain = lgb.Dataset(x_train[train_ix,:], label=y_train[train_ix],\n",
    "                         feature_name=list(feature_names), \n",
    "                         categorical_feature=categorical)\n",
    "    dvalid = lgb.Dataset(x_train[val_ix,:], label=y_train[val_ix],\n",
    "                         feature_name=list(feature_names), \n",
    "                         categorical_feature=categorical)\n",
    "    \n",
    "    model = lgb.train(params, dtrain, \n",
    "                      valid_sets=[dtrain, dvalid], \n",
    "                      valid_names=['train', 'valid'],\n",
    "                      num_boost_round=rounds, \n",
    "                      early_stopping_rounds=early_stop_rounds, \n",
    "                      verbose_eval=100)\n",
    "    \n",
    "    ret[val_ix] = model.predict(x_train[val_ix,:])\n",
    "    ret_test += model.predict(x_test)\n",
    "    ret_models.append(model)\n",
    "    \n",
    "    del dtrain, dvalid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ret_test = ret_test / 5."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "cfc958c3-f045-49d4-8017-be7a60d7e18b",
    "_uuid": "40209bda4fe8e803d8e56e0c98ed4ec0375eac94",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for model in ret_models:\n",
    "    fig, ax = plt.subplots(figsize=(10, 14))\n",
    "    lgb.plot_importance(model, max_num_features=50, ax=ax)\n",
    "    plt.title(\"Light GBM Feature Importance\")\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "c1870612-7d3a-4871-a1de-27b379c9e970",
    "_uuid": "723d20c1ee33b7ccbeb3e0a428048300f0a86944"
   },
   "source": [
    "That looks good. But the model is kind of a black box. It is a good idea to plot the feature importances for our model now."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "959449c6-6712-4447-9223-9ac8dd094e22",
    "_uuid": "1b9e07f74d5599578910c4d7486f60dc1f6e5688"
   },
   "source": [
    "`avg_days_up`, `avg_times_up_user` and `n_user_items` are our most important engineered features! Looks like we were successful. Now we just have to predict the test matrix and submit!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "prefix = 'cat_interact_'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "f41e4221-1654-4ce8-86ed-ae39b6099591",
    "_uuid": "5e934b64b9340e9666bc339c605a4b0438e3fef6",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pd.DataFrame(data=ret, columns=[prefix+'lgb_pred']).to_csv(prefix+'lgb_oof_val_pred.csv', index=False)\n",
    "pd.DataFrame(data=ret_test, columns=[prefix+'lgb_pred']).to_csv(prefix+'lgb_oof_test_pred.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "9d49a8cc-4b52-47f4-868a-f7b3006cd9f3",
    "_uuid": "3c93c6232801704e05d2a94c38b7df77d7ed14f4",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "subm = pd.read_csv('sample_submission.csv')\n",
    "subm['deal_probability'] = np.clip(ret_test, 0, 1)\n",
    "subm.to_csv(prefix+'submission.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "for md in ret_models:\n",
    "    with open('models/'+prefix +'lgb.model', 'wb') as handle:\n",
    "        pickle.dump(md, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ret_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "e41f2802-c21c-471b-a7b3-f190aec1a7f2",
    "_uuid": "478c61d627e87de36aa762a69b1c10ff6c718788"
   },
   "source": [
    "I'll end this kernel with some ideas to improve it:\n",
    "- Use K-Fold cross validation.\n",
    "- Try other methods than mean for reducing the aggregated features to one per user (e. g. modus or median).\n",
    "- Try other gradient boosting libraries like CatBoost or XGBoost.\n",
    "- Add a temporal dimension to engineered features (e. g. # of items a user put up for sale *per day*).\n",
    "- Add more advanced text features like pretrained word embeddings.\n",
    "- Add image features. At the moment we completely ignore images! (as discussed [here](https://www.kaggle.com/c/avito-demand-prediction/discussion/56678), two promising approaches could be [NIMA: Neural Image Assessment](https://arxiv.org/abs/1709.05424) and [Multimedia Features for Click Prediction](https://storage.googleapis.com/kaggle-forum-message-attachments/328059/9411/dimitri-clickadvert.pdf)).\n",
    "- Normalize text before creating the Tf-Idf matrix (e. g. using [stemming](http://www.nltk.org/howto/stem.html)).\n",
    "- ~~Learn russian and do in-depth text analysis.~~\n",
    "\n",
    "Thanks for reading and have fun in this competition!"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}