correlation_test.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# This is correlation test for a preprocessed data, created via cluster1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import csv\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from scipy.stats import ttest_ind"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('tmp.txt', sep = '|', low_memory = False)\n",
    "columns = list(df.columns.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>actcount</th>\n",
       "      <th>med_ml_num</th>\n",
       "      <th>buynum</th>\n",
       "      <th>grind</th>\n",
       "      <th>quests_speed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.048387</td>\n",
       "      <td>0.045605</td>\n",
       "      <td>0.058676</td>\n",
       "      <td>0.011676</td>\n",
       "      <td>0.031583</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>actcount</th>\n",
       "      <td>0.048387</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.075726</td>\n",
       "      <td>0.087715</td>\n",
       "      <td>0.080849</td>\n",
       "      <td>-0.078455</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>med_ml_num</th>\n",
       "      <td>0.045605</td>\n",
       "      <td>0.075726</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.187018</td>\n",
       "      <td>0.068988</td>\n",
       "      <td>0.364767</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>buynum</th>\n",
       "      <td>0.058676</td>\n",
       "      <td>0.087715</td>\n",
       "      <td>0.187018</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.062461</td>\n",
       "      <td>0.120310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>grind</th>\n",
       "      <td>0.011676</td>\n",
       "      <td>0.080849</td>\n",
       "      <td>0.068988</td>\n",
       "      <td>0.062461</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.090823</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>quests_speed</th>\n",
       "      <td>0.031583</td>\n",
       "      <td>-0.078455</td>\n",
       "      <td>0.364767</td>\n",
       "      <td>0.120310</td>\n",
       "      <td>0.090823</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    id  actcount  med_ml_num    buynum     grind  quests_speed\n",
       "id            1.000000  0.048387    0.045605  0.058676  0.011676      0.031583\n",
       "actcount      0.048387  1.000000    0.075726  0.087715  0.080849     -0.078455\n",
       "med_ml_num    0.045605  0.075726    1.000000  0.187018  0.068988      0.364767\n",
       "buynum        0.058676  0.087715    0.187018  1.000000  0.062461      0.120310\n",
       "grind         0.011676  0.080849    0.068988  0.062461  1.000000      0.090823\n",
       "quests_speed  0.031583 -0.078455    0.364767  0.120310  0.090823      1.000000"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.corr(method = 'pearson')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>actcount</th>\n",
       "      <th>med_ml_num</th>\n",
       "      <th>buynum</th>\n",
       "      <th>grind</th>\n",
       "      <th>quests_speed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.002496</td>\n",
       "      <td>-0.007902</td>\n",
       "      <td>0.017055</td>\n",
       "      <td>-0.007359</td>\n",
       "      <td>-0.001928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>actcount</th>\n",
       "      <td>-0.002496</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.620762</td>\n",
       "      <td>0.405479</td>\n",
       "      <td>0.556500</td>\n",
       "      <td>0.683768</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>med_ml_num</th>\n",
       "      <td>-0.007902</td>\n",
       "      <td>0.620762</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.435885</td>\n",
       "      <td>0.552965</td>\n",
       "      <td>0.485756</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>buynum</th>\n",
       "      <td>0.017055</td>\n",
       "      <td>0.405479</td>\n",
       "      <td>0.435885</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.344590</td>\n",
       "      <td>0.313451</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>grind</th>\n",
       "      <td>-0.007359</td>\n",
       "      <td>0.556500</td>\n",
       "      <td>0.552965</td>\n",
       "      <td>0.344590</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.446805</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>quests_speed</th>\n",
       "      <td>-0.001928</td>\n",
       "      <td>0.683768</td>\n",
       "      <td>0.485756</td>\n",
       "      <td>0.313451</td>\n",
       "      <td>0.446805</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    id  actcount  med_ml_num    buynum     grind  quests_speed\n",
       "id            1.000000 -0.002496   -0.007902  0.017055 -0.007359     -0.001928\n",
       "actcount     -0.002496  1.000000    0.620762  0.405479  0.556500      0.683768\n",
       "med_ml_num   -0.007902  0.620762    1.000000  0.435885  0.552965      0.485756\n",
       "buynum        0.017055  0.405479    0.435885  1.000000  0.344590      0.313451\n",
       "grind        -0.007359  0.556500    0.552965  0.344590  1.000000      0.446805\n",
       "quests_speed -0.001928  0.683768    0.485756  0.313451  0.446805      1.000000"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.corr(method = 'spearman')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>actcount</th>\n",
       "      <th>med_ml_num</th>\n",
       "      <th>buynum</th>\n",
       "      <th>grind</th>\n",
       "      <th>quests_speed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.001778</td>\n",
       "      <td>-0.006083</td>\n",
       "      <td>0.013295</td>\n",
       "      <td>-0.005488</td>\n",
       "      <td>-0.001392</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>actcount</th>\n",
       "      <td>-0.001778</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.472066</td>\n",
       "      <td>0.339821</td>\n",
       "      <td>0.459542</td>\n",
       "      <td>0.457740</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>med_ml_num</th>\n",
       "      <td>-0.006083</td>\n",
       "      <td>0.472066</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.404132</td>\n",
       "      <td>0.481119</td>\n",
       "      <td>0.438242</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>buynum</th>\n",
       "      <td>0.013295</td>\n",
       "      <td>0.339821</td>\n",
       "      <td>0.404132</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.307357</td>\n",
       "      <td>0.273219</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>grind</th>\n",
       "      <td>-0.005488</td>\n",
       "      <td>0.459542</td>\n",
       "      <td>0.481119</td>\n",
       "      <td>0.307357</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.365401</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>quests_speed</th>\n",
       "      <td>-0.001392</td>\n",
       "      <td>0.457740</td>\n",
       "      <td>0.438242</td>\n",
       "      <td>0.273219</td>\n",
       "      <td>0.365401</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    id  actcount  med_ml_num    buynum     grind  quests_speed\n",
       "id            1.000000 -0.001778   -0.006083  0.013295 -0.005488     -0.001392\n",
       "actcount     -0.001778  1.000000    0.472066  0.339821  0.459542      0.457740\n",
       "med_ml_num   -0.006083  0.472066    1.000000  0.404132  0.481119      0.438242\n",
       "buynum        0.013295  0.339821    0.404132  1.000000  0.307357      0.273219\n",
       "grind        -0.005488  0.459542    0.481119  0.307357  1.000000      0.365401\n",
       "quests_speed -0.001392  0.457740    0.438242  0.273219  0.365401      1.000000"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.corr(method = 'kendall')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This part shows all pairs of parametres, which has significant correlation.\n",
    "It uses Student-test."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "significance = 0.95\n",
    "for x in columns:\n",
    "    for y in columns:\n",
    "        if x==y:\n",
    "            continue\n",
    "        tt = ttest_ind(df[x],df[y])\n",
    "        if tt[1]>=1-significance:\n",
    "            print x,y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}