Text Features (Counter Vectorizer (char level based)).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stage 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from IPython.display import display\n",
    "from copy import deepcopy as cp\n",
    "import gc; gc.enable()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_usecols = ['region', 'city', 'parent_category_name', 'category_name', 'title', 'description', 'deal_probability']\n",
    "test_usecols = cp(train_usecols)\n",
    "test_usecols.remove('deal_probability')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_df = pd.read_csv(\"data/train.csv\", usecols=train_usecols)\n",
    "test_df = pd.read_csv(\"data/test.csv\", usecols=test_usecols)\n",
    "print(\"Train file rows and columns are : \", train_df.shape)\n",
    "print(\"Test file rows and columns are : \", test_df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_df['deal_probability'] = np.zeros((test_df.shape[0],))\n",
    "all_df = pd.concat([train_df, test_df], axis=0).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "display(train_df.head(5))\n",
    "display(test_df.head(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def print_contains_info(df, col1, col2):\n",
    "    str1s = df[col1].values\n",
    "    str2s = df[col2].values\n",
    "    \n",
    "    contains_count = 0\n",
    "    for i in range(len(str1s)):\n",
    "        str1 = str(str1s[i])\n",
    "        str2 = str(str2s[i])\n",
    "        str1 = str1.split(\" \")\n",
    "        str2 = str2.split(\" \")\n",
    "        \n",
    "        for s in str1:\n",
    "            if s in str2:\n",
    "                contains_count += 1\n",
    "                break\n",
    "                \n",
    "    print('{} in {} contains counts:\\n'.format(col1, col2), contains_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print_contains_info(all_df, 'region', 'title')\n",
    "print_contains_info(all_df, 'city', 'title')\n",
    "print_contains_info(all_df, 'parent_category_name', 'title')\n",
    "print_contains_info(all_df, 'category_name', 'title')\n",
    "print_contains_info(all_df, 'region', 'description')\n",
    "print_contains_info(all_df, 'city', 'description')\n",
    "print_contains_info(all_df, 'parent_category_name', 'description')\n",
    "print_contains_info(all_df, 'category_name', 'description')\n",
    "print_contains_info(all_df, 'title', 'description')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_contains_feature(df, col1, col2):\n",
    "    print('processing ', col1, col2)\n",
    "    res_df = pd.DataFrame()\n",
    "    str1s = df[col1].values\n",
    "    str2s = df[col2].values\n",
    "    \n",
    "    is_contains = []\n",
    "    contains_counts = []\n",
    "    for i in range(len(str1s)):\n",
    "        str1 = str(str1s[i])\n",
    "        str2 = str(str2s[i])\n",
    "        str1 = str1.split(\" \")\n",
    "        str2 = str2.split(\" \")\n",
    "        \n",
    "        contains_count = 0\n",
    "        for s in str1:\n",
    "            if s in str2:\n",
    "                contains_count += 1\n",
    "                \n",
    "        is_contains.append(1 if contains_count > 0 else 0)\n",
    "        contains_counts.append(contains_count)\n",
    "        \n",
    "    res_df['{}_in_{}'.format(col1,col2)] = is_contains\n",
    "    res_df['{}_in_{}_counts'.format(col1,col2)] = contains_counts\n",
    "    del is_contains, contains_counts; gc.collect()\n",
    "    return res_df "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_df = pd.concat([all_df, get_contains_feature(all_df, 'region', 'title')],  axis=1)\n",
    "all_df = pd.concat([all_df, get_contains_feature(all_df, 'city', 'title')],  axis=1)\n",
    "all_df = pd.concat([all_df, get_contains_feature(all_df, 'parent_category_name', 'title')],  axis=1)\n",
    "all_df = pd.concat([all_df, get_contains_feature(all_df, 'category_name', 'title')],  axis=1)\n",
    "all_df = pd.concat([all_df, get_contains_feature(all_df, 'region', 'description')],  axis=1)\n",
    "all_df = pd.concat([all_df, get_contains_feature(all_df, 'city', 'description')],  axis=1)\n",
    "all_df = pd.concat([all_df, get_contains_feature(all_df, 'parent_category_name', 'description')],  axis=1)\n",
    "all_df = pd.concat([all_df, get_contains_feature(all_df, 'category_name', 'description')],  axis=1)\n",
    "all_df = pd.concat([all_df, get_contains_feature(all_df, 'title', 'description')],  axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "norm_cols = [c for c in all_df.columns if 'counts' in c]\n",
    "for col in norm_cols:\n",
    "    all_df[col] = (all_df[col] - all_df[col].mean())/all_df[col].std() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "text_feature = all_df['region'] + ' ' + all_df['city'] + ' ' + \\\n",
    "                all_df['parent_category_name'] + ' ' + all_df['category_name'] + ' ' + \\\n",
    "                all_df['title'] + ' ' + all_df['description']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_df = all_df.drop(test_usecols, axis=1)\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_df.to_csv('text_other_features.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "stopwords = ['а', 'е', 'и', 'ж', 'м', 'о', 'на', 'не', 'ни', 'об', 'но', 'он', 'мне', 'мои', 'мож', 'она', 'они', 'оно', 'мной', 'много', 'многочисленное', 'многочисленная', 'многочисленные', 'многочисленный', 'мною', 'мой', 'мог', 'могут', 'можно', 'может', 'можхо', 'мор', 'моя', 'моё', 'мочь', 'над', 'нее', 'оба', 'нам', 'нем', 'нами', 'ними', 'мимо', 'немного', 'одной', 'одного', 'менее', 'однажды', 'однако', 'меня', 'нему', 'меньше', 'ней', 'наверху', 'него', 'ниже', 'мало', 'надо', 'один', 'одиннадцать', 'одиннадцатый', 'назад', 'наиболее', 'недавно', 'миллионов', 'недалеко', 'между', 'низко', 'меля', 'нельзя', 'нибудь', 'непрерывно', 'наконец', 'никогда', 'никуда', 'нас', 'наш', 'нет', 'нею', 'неё', 'них', 'мира', 'наша', 'наше', 'наши', 'ничего', 'начала', 'нередко', 'несколько', 'обычно', 'опять', 'около', 'мы', 'ну', 'нх', 'от', 'отовсюду', 'особенно', 'нужно', 'очень', 'отсюда', 'в', 'во', 'вон', 'вниз', 'внизу', 'вокруг', 'вот', 'восемнадцать', 'восемнадцатый', 'восемь', 'восьмой', 'вверх', 'вам', 'вами', 'важное', 'важная', 'важные', 'важный', 'вдали', 'везде', 'ведь', 'вас', 'ваш', 'ваша', 'ваше', 'ваши', 'впрочем', 'весь', 'вдруг', 'вы', 'все', 'второй', 'всем', 'всеми', 'времени', 'время', 'всему', 'всего', 'всегда', 'всех', 'всею', 'всю', 'вся', 'всё', 'всюду', 'г', 'год', 'говорил', 'говорит', 'года', 'году', 'где', 'да', 'ее', 'за', 'из', 'ли', 'же', 'им', 'до', 'по', 'ими', 'под', 'иногда', 'довольно', 'именно', 'долго', 'позже', 'более', 'должно', 'пожалуйста', 'значит', 'иметь', 'больше', 'пока', 'ему', 'имя', 'пор', 'пора', 'потом', 'потому', 'после', 'почему', 'почти', 'посреди', 'ей', 'два', 'две', 'двенадцать', 'двенадцатый', 'двадцать', 'двадцатый', 'двух', 'его', 'дел', 'или', 'без', 'день', 'занят', 'занята', 'занято', 'заняты', 'действительно', 'давно', 'девятнадцать', 'девятнадцатый', 'девять', 'девятый', 'даже', 'алло', 'жизнь', 'далеко', 'близко', 'здесь', 'дальше', 'для', 'лет', 'зато', 'даром', 'первый', 'перед', 'затем', 'зачем', 'лишь', 'десять', 'десятый', 'ею', 'её', 'их', 'бы', 'еще', 'при', 'был', 'про', 'процентов', 'против', 'просто', 'бывает', 'бывь', 'если', 'люди', 'была', 'были', 'было', 'будем', 'будет', 'будете', 'будешь', 'прекрасно', 'буду', 'будь', 'будто', 'будут', 'ещё', 'пятнадцать', 'пятнадцатый', 'друго', 'другое', 'другой', 'другие', 'другая', 'других', 'есть', 'пять', 'быть', 'лучше', 'пятый', 'к', 'ком', 'конечно', 'кому', 'кого', 'когда', 'которой', 'которого', 'которая', 'которые', 'который', 'которых', 'кем', 'каждое', 'каждая', 'каждые', 'каждый', 'кажется', 'как', 'какой', 'какая', 'кто', 'кроме', 'куда', 'кругом', 'с', 'т', 'у', 'я', 'та', 'те', 'уж', 'со', 'то', 'том', 'снова', 'тому', 'совсем', 'того', 'тогда', 'тоже', 'собой', 'тобой', 'собою', 'тобою', 'сначала', 'только', 'уметь', 'тот', 'тою', 'хорошо', 'хотеть', 'хочешь', 'хоть', 'хотя', 'свое', 'свои', 'твой', 'своей', 'своего', 'своих', 'свою', 'твоя', 'твоё', 'раз', 'уже', 'сам', 'там', 'тем', 'чем', 'сама', 'сами', 'теми', 'само', 'рано', 'самом', 'самому', 'самой', 'самого', 'семнадцать', 'семнадцатый', 'самим', 'самими', 'самих', 'саму', 'семь', 'чему', 'раньше', 'сейчас', 'чего', 'сегодня', 'себе', 'тебе', 'сеаой', 'человек', 'разве', 'теперь', 'себя', 'тебя', 'седьмой', 'спасибо', 'слишком', 'так', 'такое', 'такой', 'такие', 'также', 'такая', 'сих', 'тех', 'чаще', 'четвертый', 'через', 'часто', 'шестой', 'шестнадцать', 'шестнадцатый', 'шесть', 'четыре', 'четырнадцать', 'четырнадцатый', 'сколько', 'сказал', 'сказала', 'сказать', 'ту', 'ты', 'три', 'эта', 'эти', 'что', 'это', 'чтоб', 'этом', 'этому', 'этой', 'этого', 'чтобы', 'этот', 'стал', 'туда', 'этим', 'этими', 'рядом', 'тринадцать', 'тринадцатый', 'этих', 'третий', 'тут', 'эту', 'суть', 'чуть', 'тысяч']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def clean_text(txt):\n",
    "    words = str(txt).split(str=\" ./\")\n",
    "    words = [wrd for wrd in words if wrd not in stopwords]\n",
    "    words = [wrd for wrd in words if len(wrd) > 1]\n",
    "    txt = \" \".join(words)\n",
    "    return txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "text_feature = text_feature.apply(lambda x: clean_text(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('text_feature.pickle', 'wb') as handle:\n",
    "    pickle.dump(text_feature, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stage 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "import numpy as np\n",
    "import gc; gc.enable()\n",
    "with open('text_feature.pickle', 'rb') as handle:\n",
    "    text_feature = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer, HashingVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2011862, 262144)\n"
     ]
    }
   ],
   "source": [
    "hash_char_wb_features = HashingVectorizer(ngram_range=(2, 3), \n",
    "                                          analyzer='char_wb',\n",
    "                                          n_features=2**18,\n",
    "                                          norm='l2', \n",
    "                                          dtype=np.float32).fit_transform(text_feature.values)\n",
    "print(hash_char_wb_features.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('hash_char_wb_features.pickle', 'wb') as handle:\n",
    "    pickle.dump(hash_char_wb_features, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "hash_char_wb_features = (hash_char_wb_features > 0).astype(int)\n",
    "with open('hash_char_wb_bin_features.pickle', 'wb') as handle:\n",
    "    pickle.dump(hash_char_wb_features, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "218"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del hash_char_wb_features; gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stage 3: Test with fm_ftrl model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import gc; gc.enable()\n",
    "from sklearn.decomposition import TruncatedSVD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('hash_char_wb_features.pickle', 'rb') as handle:\n",
    "    hash_char_wb_features = pickle.load(handle)\n",
    "    \n",
    "hash_char_wb_features = hash_char_wb_features.astype(np.float64)\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1503424\n"
     ]
    }
   ],
   "source": [
    "train_df = pd.read_csv(\"data/train.csv\", usecols=['deal_probability'])\n",
    "train_len = train_df.shape[0]\n",
    "train_y = train_df['deal_probability'].values\n",
    "del train_df\n",
    "print(train_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "hash_char_wb_train_features = hash_char_wb_features[:train_len, :] \n",
    "hash_char_wb_test_features = hash_char_wb_features[train_len:, :] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "306"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del hash_char_wb_features; gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1503424, 262144)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hash_char_wb_train_features.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(508438, 262144)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hash_char_wb_test_features.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import Ridge\n",
    "from sklearn.model_selection import KFold, GridSearchCV, ParameterGrid, train_test_split\n",
    "from sklearn.metrics import mean_squared_error, make_scorer\n",
    "from copy import deepcopy as cp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def clip_rmse(ground_truth, predictions):\n",
    "    predictions = np.clip(predictions, 0., 1.)\n",
    "    return mean_squared_error(ground_truth, predictions)**.5\n",
    "\n",
    "clip_rmse_scorer = make_scorer(clip_rmse, greater_is_better=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "kfold = KFold(n_splits=3, random_state=719)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from wordbatch.models import FM_FTRL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def simple_train_test_eval(default_params, X, y, params):\n",
    "    tr_X, val_X, tr_y, val_y = train_test_split(X, y, test_size=0.3, random_state=719)\n",
    "    \n",
    "    min_score = None\n",
    "    best_param = None\n",
    "    \n",
    "    for param in list(ParameterGrid(params)):\n",
    "        use_params = cp(default_params)\n",
    "        use_params.update(param)\n",
    "        print('Fitting params:\\n', use_params)\n",
    "        md = FM_FTRL(**use_params)\n",
    "        md.fit(tr_X, tr_y)\n",
    "        score = clip_rmse(val_y, md.predict(val_X))\n",
    "        print(param, score)\n",
    "        \n",
    "        if min_score is None or score < min_score:\n",
    "            best_param = param\n",
    "            min_score = score\n",
    "            \n",
    "    print('Best param:', best_param, '\\nscore:', min_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting params:\n",
      " {'alpha': 0.01, 'beta': 0.005, 'L1': 0.0001, 'L2': 0.1, 'D': 262144, 'D_fm': 300, 'iters': 40, 'seed': 719, 'threads': 4, 'verbose': 0}\n",
      "{'L1': 0.0001, 'L2': 0.1} 0.233972783022\n",
      "Best param: {'L1': 0.0001, 'L2': 0.1} \n",
      "score: 0.233972783022\n"
     ]
    }
   ],
   "source": [
    "fmftrl_default_params = {\n",
    "    'alpha': .01,\n",
    "    'beta': .005,\n",
    "    'L1': 0.0001,\n",
    "    'L2': 0.1,\n",
    "    'D': hash_char_wb_train_features.shape[1],\n",
    "    'D_fm': 300,\n",
    "    'iters': 40,\n",
    "    'seed': 719,\n",
    "    'threads': 4,\n",
    "    'verbose': 0\n",
    "}\n",
    "\n",
    "try_params = {\n",
    "    'L1': [0.0001],\n",
    "    'L2': [0.1]\n",
    "}\n",
    "simple_train_test_eval(fmftrl_default_params, hash_char_wb_train_features, train_y, try_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}