DimensionalityReduction_with17304_removal.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import time\n",
    "from sklearn.preprocessing import OneHotEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Path to people.csv from ReadHatKaggle data set\n",
    "FEATURE_FILE ='Data/act_train_features.csv'\n",
    "# Path to act_train.csv from RedHatKaggle data set\n",
    "OUTPUT ='Data/act_train_output.csv'\n",
    "# Path to the test file\n",
    "TEST_FILE = 'Data/act_test_features.csv'\n",
    "\n",
    "# Path to the files with reduced dimesions\n",
    "SAVE_AS_DIR = 'Data'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Non feature\n",
    "NON_FEATURE=['activity_id','people_id','date','people_date']\n",
    "\n",
    "# Categorical data that is only label encoded\n",
    "CATEGORICAL_DATA = ['people_char_1', 'people_char_2',\n",
    "                    'people_char_3', 'people_char_4', 'people_char_5',\n",
    "                    'people_char_6', 'people_char_7', 'people_char_8',\n",
    "                    'people_char_9', 'activity_category',\n",
    "                    'char_1', 'char_2', 'char_3', 'char_4', 'char_5', 'char_6',\n",
    "                    'char_7', 'char_8', 'char_9','people_group_1', 'char_10']\n",
    "\n",
    "# Already in a one-hot encoded form\n",
    "CATEGORICAL_BINARY = ['people_char_10', 'people_char_11', 'people_char_12',\n",
    "                      'people_char_13', 'people_char_14', 'people_char_15',\n",
    "                      'people_char_16', 'people_char_17', 'people_char_18',\n",
    "                      'people_char_19', 'people_char_20', 'people_char_21',\n",
    "                      'people_char_22', 'people_char_23', 'people_char_24',\n",
    "                      'people_char_25', 'people_char_26', 'people_char_27',\n",
    "                      'people_char_28', 'people_char_29', 'people_char_30',\n",
    "                      'people_char_31', 'people_char_32', 'people_char_33',\n",
    "                      'people_char_34', 'people_char_35', 'people_char_36',\n",
    "                      'people_char_37' ]\n",
    "\n",
    "# Continuous categories\n",
    "CONT = ['people_days', 'days',\n",
    "      'people_month',  'month', \n",
    "      'people_quarter', 'quarter',\n",
    "      'people_week', 'week',\n",
    "      'people_dayOfMonth', 'dayOfMonth',\n",
    "      'people_year', 'year', \n",
    "      'people_char_38', ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Remove redundant values\n",
    "def remove_redundant(df_train,df_test,column,replacement):\n",
    "    Intersection_test_train=list(\\\n",
    "                                 set(df_test[column].astype('int64').unique())\\\n",
    "                                 .intersection\\\n",
    "                                 (set(df_train[column].astype('int64').unique())))\n",
    "    df_train[column]=df_train[column].apply(lambda x: replacement if x not in Intersection_test_train else x)\n",
    "    df_test[column]=df_test[column].apply(lambda x: replacement if x not in Intersection_test_train else x)\n",
    "    return df_train,df_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Function to change labels of categories to one-hot encoding using scikit's OneHot Encoding\n",
    "# pd.get_dummies(df) does the same, provides sweet header's as well but it it not fast enough, kill's memory\n",
    "def category_to_one_hot(dataset,non_feature,continuous_feature):\n",
    "    ds = dataset.drop(non_feature,axis=1)\n",
    "    boolean_column = []\n",
    "    counter=0\n",
    "    for column in ds.columns:\n",
    "        if column not in continuous_feature:\n",
    "            boolean_column.append(counter)\n",
    "        counter += 1\n",
    "    # boolean_colum is not the column name but index\n",
    "    print (\"Done filtering columns...\")\n",
    "    grd_enc = OneHotEncoder(categorical_features=boolean_column)\n",
    "    encoded_arr=grd_enc.fit_transform(ds).toarray()\n",
    "    return encoded_arr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Read the train data set\n",
    "train_data_df=pd.read_csv(FEATURE_FILE,parse_dates=[\"date\"])\n",
    "train_data_df.sort_values(by=['activity_id'],ascending=True, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Read the train data output\n",
    "train_output = pd.read_csv(OUTPUT)\n",
    "train_output.sort_values(by='activity_id',ascending=True, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Read the test data set\n",
    "test_data_df=pd.read_csv(TEST_FILE,parse_dates=[\"date\"])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_data_df.sort_values(by=['activity_id'],ascending=True, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_data_df.set_index('people_group_1',inplace=True)\n",
    "train_data_df.drop(17304,axis=0,inplace=True)\n",
    "train_data_df.reset_index(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_data_df.set_index('people_group_1',inplace=True)\n",
    "the_group=test_data_df.loc[17304]\n",
    "test_data_df.drop(17304,axis=0,inplace=True)\n",
    "test_data_df.reset_index(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "803.232099056\n"
     ]
    }
   ],
   "source": [
    "# Function to help reduce exploding dimensions 1336.28923082 secs\n",
    "start=time.time()\n",
    "for column in CATEGORICAL_DATA:\n",
    "    train_data_df,test_data_df=remove_redundant(train_data_df,test_data_df,column,99999)\n",
    "end=time.time()\n",
    "print(end-start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done filtering columns...\n",
      "60.7442810535\n"
     ]
    }
   ],
   "source": [
    "# Function to one hot encode all values\n",
    "start=time.time()\n",
    "train_arr=category_to_one_hot(train_data_df,NON_FEATURE,CONT)\n",
    "end=time.time()\n",
    "print(end-start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_file_path(directory, filename):\n",
    "    \"\"\" Combines file path directory/filename\n",
    "    \"\"\"\n",
    "    return os.path.join(directory, filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def write_out(df, output):\n",
    "    df.set_index(['activity_id']).to_csv(output + \"_features_reduced_17304_non_cont.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "file_save=get_file_path(SAVE_AS_DIR,'train')\n",
    "write_out(train_data_df,file_save)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "file_save=get_file_path(SAVE_AS_DIR,'test')\n",
    "write_out(test_data_df,file_save)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/aakash.gupta/anaconda/lib/python2.7/site-packages/ipykernel/__main__.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  from ipykernel import kernelapp as app\n"
     ]
    }
   ],
   "source": [
    "file_save=get_file_path(SAVE_AS_DIR,'manipulated_results.csv')\n",
    "the_group['outcome']=0\n",
    "the_group.set_index(['activity_id']).to_csv(file_save)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}