Preprocessing_dateFeatures.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Path to the pickle files\n",
    "PKL_DIR = 'Data/pickle'\n",
    "CSV_DIR = 'Data'\n",
    "\n",
    "# File names\n",
    "train_interpolated = '/train_withInterpolations.pkl'\n",
    "test_interpolated = '/test_withleak.pkl'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Read all\n",
    "train_data = pd.read_pickle(PKL_DIR+train_interpolated)\n",
    "test_data = pd.read_pickle(PKL_DIR+test_interpolated)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Perform further preprocessing to get variations on date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Find minimum and maximum activity dates\n",
    "min_date_train = min(train_data['date'])\n",
    "max_date_train = max(train_data['date'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Find minimum and maximum people dates\n",
    "min_pdate_train = min(train_data['people_date'])\n",
    "max_pdate_train = max(train_data['people_date'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Get reference dates from beginning of record time\n",
    "def getReference(df):\n",
    "    df['adays_till_activity'] = (df['date']-min_date_train)\n",
    "    df['pdays_till_activity'] = (df['people_date'] - min_pdate_train)\n",
    "    df['worked_for_day']=np.busday_count(df['people_date'].values.astype('datetime64[D]'), df['date'].values.astype('datetime64[D]'))\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Generate outcome for prev day and curr day per group_1\n",
    "def getPreviousAndNextDatesOutcomes(df):\n",
    "    df.sort_values(['people_group_1','date'],inplace=True)\n",
    "    # Get average outcome interpolated per group and date\n",
    "    df1 = df[['people_group_1','date','outcome_filled']].groupby(['people_group_1','date']).mean()\n",
    "    df1.columns = ['avg_outcome']\n",
    "    df1.reset_index(inplace=True)\n",
    "    df1['next_outcome']=df1[['people_group_1','date','avg_outcome']].groupby('people_group_1')['avg_outcome'].shift(-1)\n",
    "    df1['prev_outcome']=df1[['people_group_1','date','avg_outcome']].groupby('people_group_1')['avg_outcome'].shift(1)\n",
    "    df1['diff_date']=df1[['people_group_1','date','avg_outcome']].groupby('people_group_1')['date'].apply(lambda x: x-x.shift(1))\n",
    "    df1.drop('avg_outcome',axis=1,inplace=True)\n",
    "    df = pd.merge(df,df1,on=['people_group_1','date'],how='left')\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Get people per group\n",
    "def getPeopleperGroup(df):\n",
    "    # ppg = people per group\n",
    "    ppg = df.groupby('people_group_1')['people_id'].apply(lambda x: len(x.unique()))\n",
    "    ppg = ppg.reset_index()\n",
    "    ppg.columns=['people_group_1','people_per_group']\n",
    "    df = pd.merge(df,ppg,on='people_group_1') \n",
    "    ppgd = df.groupby(['people_group_1','date'])['people_id'].apply(lambda x: len(x.unique()))\n",
    "    ppgd = ppgd.reset_index()\n",
    "    ppgd.columns=['people_group_1','date','people_per_group_date']\n",
    "    df = pd.merge(df,ppgd,on=['people_group_1','date']) \n",
    "    apgd = df.groupby(['people_group_1','date'])['activity_id'].apply(lambda x: len(x.unique()))\n",
    "    apgd = apgd.reset_index()\n",
    "    apgd.columns=['people_group_1','date','activities_per_group_date']\n",
    "    df = pd.merge(df,apgd,on=['people_group_1','date']) \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# using all duplicates?\n",
    "if False:\n",
    "    is_dup = train_data.drop('activity_id',axis=1).duplicated()\n",
    "    df = train_data[~is_dup]\n",
    "else:\n",
    "    df = train_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1398166, 72)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(498687, 72)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df = pd.concat([df,test_data])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df = getReference(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df = getPreviousAndNextDatesOutcomes(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['people_id'] = df['people_id'].astype('int32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = getPeopleperGroup(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['pdays_till_activity',\n",
       " 'adays_till_activity',\n",
       " 'people_per_group',\n",
       " 'next_outcome',\n",
       " 'people_per_group_date',\n",
       " 'prev_outcome',\n",
       " 'diff_date',\n",
       " 'activities_per_group_date',\n",
       " 'worked_for_day']"
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(set(df.columns).difference(set(train_data.columns)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "columns_added = ['activity_id','adays_till_activity','pdays_till_activity','worked_for_day','next_outcome','prev_outcome','diff_date', 'people_per_group','people_per_group_date','activities_per_group_date']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(588273, 81)"
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['outcome']==0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(333083, 81)"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['outcome'].isnull()].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df['adays_till_activity'] = df['adays_till_activity'].astype('<m8[D]').values.astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['pdays_till_activity'] = df['adays_till_activity'].astype('<m8[D]').values.astype(int)\n",
    "df['diff_date'] = df['adays_till_activity'].astype('<m8[D]').values.astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# df[columns_added][10:30]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### Spilt the df\n",
    "train_added = df[df['outcome']>=0]\n",
    "test_added = df[df['outcome'].isnull()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## Pickle results ##\n",
    "######### Saving state of code here ##############\n",
    "train_added.to_pickle(SAVE_AS_DIR+'/train_withDateFeatures_without17304.pkl')\n",
    "test_added.to_pickle(SAVE_AS_DIR+'/test_withDateFeatures_without17304.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}