Week3_Internship.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Week 3 Hamoye_Internship\n",
    "## Supervised machinel learning(Classification)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import important libraries\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import imblearn\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "#importing the dataset from the website below\n",
    "#df= pd.read_csv( 'https://query.data.world/s/wh6j7rxy2hvrn4ml75ci62apk5hgae' ) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Homet\\Anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py:3058: DtypeWarning: Columns (6) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "df=pd.read_csv('ecology.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>country</th>\n",
       "      <th>year</th>\n",
       "      <th>country_code</th>\n",
       "      <th>record</th>\n",
       "      <th>crop_land</th>\n",
       "      <th>grazing_land</th>\n",
       "      <th>forest_land</th>\n",
       "      <th>fishing_ground</th>\n",
       "      <th>built_up_land</th>\n",
       "      <th>carbon</th>\n",
       "      <th>total</th>\n",
       "      <th>QScore</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>Armenia</td>\n",
       "      <td>1992</td>\n",
       "      <td>1</td>\n",
       "      <td>AreaPerCap</td>\n",
       "      <td>0.140292</td>\n",
       "      <td>0.199546</td>\n",
       "      <td>0.097188051</td>\n",
       "      <td>0.036888</td>\n",
       "      <td>0.029320</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.032351e-01</td>\n",
       "      <td>3A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>Armenia</td>\n",
       "      <td>1992</td>\n",
       "      <td>1</td>\n",
       "      <td>AreaTotHA</td>\n",
       "      <td>483000.000000</td>\n",
       "      <td>687000.000000</td>\n",
       "      <td>334600</td>\n",
       "      <td>127000.000000</td>\n",
       "      <td>100943.000800</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.732543e+06</td>\n",
       "      <td>3A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>Armenia</td>\n",
       "      <td>1992</td>\n",
       "      <td>1</td>\n",
       "      <td>BiocapPerCap</td>\n",
       "      <td>0.159804</td>\n",
       "      <td>0.135261</td>\n",
       "      <td>0.084003213</td>\n",
       "      <td>0.013742</td>\n",
       "      <td>0.033398</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.262086e-01</td>\n",
       "      <td>3A</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   country  year  country_code        record      crop_land   grazing_land  \\\n",
       "0  Armenia  1992             1    AreaPerCap       0.140292       0.199546   \n",
       "1  Armenia  1992             1     AreaTotHA  483000.000000  687000.000000   \n",
       "2  Armenia  1992             1  BiocapPerCap       0.159804       0.135261   \n",
       "\n",
       "   forest_land  fishing_ground  built_up_land  carbon         total QScore  \n",
       "0  0.097188051        0.036888       0.029320     0.0  5.032351e-01     3A  \n",
       "1       334600   127000.000000  100943.000800     0.0  1.732543e+06     3A  \n",
       "2  0.084003213        0.013742       0.033398     0.0  4.262086e-01     3A  "
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "EFConsPerCap    9024\n",
       "BiocapTotGHA    9024\n",
       "AreaPerCap      9023\n",
       "EFConsTotGHA    9023\n",
       "EFProdTotGHA    9023\n",
       "BiocapPerCap    9023\n",
       "EFProdPerCap    9023\n",
       "AreaTotHA       9023\n",
       "Name: record, dtype: int64"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.record.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3A    51481\n",
       "2A    10576\n",
       "2B    10096\n",
       "1A       16\n",
       "1B       16\n",
       "Name: QScore, dtype: int64"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#the target label is the QScore we are trying to predict.\n",
    "#to check if we are dealing with a balanced dataset we run:\n",
    "df.QScore.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "country               0\n",
       "year                  0\n",
       "country_code          0\n",
       "record                0\n",
       "crop_land         20472\n",
       "grazing_land      20472\n",
       "forest_land       20472\n",
       "fishing_ground    20473\n",
       "built_up_land     20473\n",
       "carbon            20473\n",
       "total                 9\n",
       "QScore                1\n",
       "dtype: int64"
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [],
   "source": [
    "#to drop rows with missing data we simply run a generic dropna\n",
    "df.dropna(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "country           0\n",
       "year              0\n",
       "country_code      0\n",
       "record            0\n",
       "crop_land         0\n",
       "grazing_land      0\n",
       "forest_land       0\n",
       "fishing_ground    0\n",
       "built_up_land     0\n",
       "carbon            0\n",
       "total             0\n",
       "QScore            0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3A    51473\n",
       "2A      224\n",
       "1A       16\n",
       "Name: QScore, dtype: int64"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.QScore.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [],
   "source": [
    "#converitng the dataset from a multiclass classification to a binary classification\n",
    "#this is to simplify the classification and also fix the imbalanced data.\n",
    "df[ 'QScore' ] = df[ 'QScore' ].replace([ '1A' ], '2A' ) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3A    51473\n",
       "2A      240\n",
       "Name: QScore, dtype: int64"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.QScore.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "#to further balance the dataset\n",
    "df_2A = df[df.QScore== '2A' ]\n",
    "df_3A = df[df.QScore== '3A' ].sample( 350 )\n",
    "data_df = df_2A.append(df_3A) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3A    350\n",
       "2A    240\n",
       "Name: QScore, dtype: int64"
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sklearn.utils\n",
    "data_df = sklearn.utils.shuffle(data_df)\n",
    "data_df = data_df.reset_index(drop= True )\n",
    "data_df.shape\n",
    "data_df.QScore.value_counts() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_df = data_df.drop(columns=[ 'country_code' , 'country' , 'year' ]) \n",
    "X = data_df.drop(columns= 'QScore' )\n",
    "y = data_df[ 'QScore' ] \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "#record is a a categorical column that needs to be encoded for the model to  read it.\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "encoder=LabelEncoder()\n",
    "X.record = encoder.fit_transform(X.record)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "#spliting the data into train test split\n",
    "from sklearn.model_selection import train_test_split\n",
    "x_train,x_test,y_train,y_test=train_test_split(X,y,random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [],
   "source": [
    "import imblearn\n",
    "#the library imblearn is imported to make final corrections of bias caused by oversampling of one class\n",
    "from imblearn.over_sampling import SMOTE\n",
    "smote = SMOTE(random_state= 1 )\n",
    "x_train_balanced, y_balanced = smote.fit_sample(x_train, y_train)\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "scaler = MinMaxScaler()\n",
    "normalised_train_df = scaler.fit_transform(x_train_balanced.drop(columns=[ 'record' ]))\n",
    "normalised_train_df = pd.DataFrame(normalised_train_df,\n",
    "columns=x_train_balanced.drop(columns=[ 'record' ]).columns)\n",
    "normalised_train_df[ 'record' ] = x_train_balanced[ 'record' ]\n",
    "x_test = x_test.reset_index(drop= True )\n",
    "normalised_test_df = scaler.transform(x_test.drop(columns=[ 'record' ]))\n",
    "normalised_test_df = pd.DataFrame(normalised_test_df,\n",
    "columns=x_test.drop(columns=[ 'record' ]).columns)\n",
    "normalised_test_df[ 'record' ] = x_test[ 'record' ] \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Model development"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(random_state=0)"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#logistic Regression\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "log_reg = LogisticRegression(random_state=0)\n",
    "log_reg.fit(normalised_train_df, y_balanced) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5717054263565892\n",
      "0.5067567567567568\n"
     ]
    }
   ],
   "source": [
    "print(log_reg.score(normalised_train_df,y_balanced))\n",
    "print(log_reg.score(normalised_test_df,y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Model Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import LeaveOneOut\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.56732623 0.52442822 0.46609542 0.46581801 0.55161692]\n",
      "0.5150569588395956\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "scores = cross_val_score(log_reg, normalised_train_df, y_balanced, cv= 5 , scoring= 'f1_macro' )\n",
    "print(scores)\n",
    "print(np.mean(scores))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "kf = KFold(n_splits= 5 )\n",
    "kf.split(normalised_train_df)\n",
    "f1_scores = []\n",
    "#run for every split\n",
    "for train_index, test_index in kf.split(normalised_train_df):\n",
    "    x_train, x_test = normalised_train_df.iloc[train_index],normalised_train_df.iloc[test_index]\n",
    "    y_train, y_test = y_balanced[train_index],y_balanced[test_index]\n",
    "    model = LogisticRegression().fit(x_train, y_train)\n",
    " #save result to list\n",
    "    f1_scores.append(f1_score(y_true=y_test, y_pred=model.predict(x_test),pos_label= '2A' )* 100 ) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import StratifiedKFold\n",
    "skf = StratifiedKFold(n_splits= 5 , shuffle= True , random_state= 1 )\n",
    "f1_scores = []\n",
    "#run for every split\n",
    "for train_index, test_index in skf.split(normalised_train_df, y_balanced):\n",
    "    x_train, x_test = np.array(normalised_train_df)[train_index],np.array(normalised_train_df)[test_index]\n",
    "    y_train, y_test = y_balanced[train_index], y_balanced[test_index]\n",
    "    model = LogisticRegression().fit(x_train, y_train)\n",
    "     #save result to list\n",
    "    f1_scores.append(f1_score(y_true=y_test, y_pred=model.predict(x_test), pos_label= '2A' )) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.4528301886792453,\n",
       " 0.5892857142857143,\n",
       " 0.6194690265486725,\n",
       " 0.4464285714285714,\n",
       " 0.6079999999999999]"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f1_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "#leave one out\n",
    "loo = LeaveOneOut()\n",
    "scores = cross_val_score(LogisticRegression(), normalised_train_df, y_balanced, cv=loo,\n",
    " scoring= 'f1_macro' )\n",
    "average_score = scores.mean() * 100 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50.38910505836576"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "average_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Due to the cross_validation splits y_test and new_predictions are no longer equal. Please rerun the cell at line 153  and 154 and skip to line 163"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[47,  9],\n",
       "       [64, 28]], dtype=int64)"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Confusion Matrix\n",
    "from sklearn.metrics import recall_score, accuracy_score, precision_score, f1_score,confusion_matrix\n",
    "new_predictions = log_reg.predict(normalised_test_df)\n",
    "cnf_mat = confusion_matrix(y_true=y_test, y_pred=new_predictions, labels=[ '2A' , '3A' ])\n",
    "cnf_mat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 51\n"
     ]
    }
   ],
   "source": [
    "#Accuracy\n",
    "accuracy = accuracy_score(y_true=y_test, y_pred=new_predictions)\n",
    "print( 'Accuracy: {}' .format(round(accuracy* 100 ), 2 )) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall: 84\n"
     ]
    }
   ],
   "source": [
    "#Recall\n",
    "recall = recall_score(y_true=y_test, y_pred=new_predictions, pos_label= '2A' )\n",
    "print( 'Recall: {}' .format(round(recall* 100 ), 2 )) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision: 42\n"
     ]
    }
   ],
   "source": [
    "#precision\n",
    "precision = precision_score(y_true=y_test, y_pred=new_predictions, pos_label= '2A' )\n",
    "print( 'Precision: {}' .format(round(precision* 100 ), 2 ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F1: 56\n"
     ]
    }
   ],
   "source": [
    "f1 = f1_score(y_true=y_test, y_pred=new_predictions, pos_label= '2A' )\n",
    "print( 'F1: {}' .format(round(f1* 100 ), 2 )) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "          2A       0.42      0.84      0.56        56\n",
      "          3A       0.76      0.30      0.43        92\n",
      "\n",
      "    accuracy                           0.51       148\n",
      "   macro avg       0.59      0.57      0.50       148\n",
      "weighted avg       0.63      0.51      0.48       148\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#to get a summary of the whole report \n",
    "print(classification_report(y_test, new_predictions,\n",
    " target_names=[\"2A\", \"3A\"]))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Evaluating a model accuracy could get tricky as it all depends on the \"business metiric\" or what the end goal is.\n",
    "sometimes a business goal could deal  with having more false positives than a high false negatives i.e they are targeting  a highre recall value. Setting the model parameters to include a high recall or precision is called setting an <b> Operating Point</b>.\n",
    " most classifiers provide a decision_function or a predict_proba method to\n",
    "assess degrees of certainty about predictions. Making predictions can be seen as\n",
    "thresholding the output of decision_function or predict_proba at a certain fixed\n",
    "point—in binary classification we use 0 for the decision function and 0.5 for\n",
    "predict_proba.Increasing or decreasing the values theat we set as threshold determines if we want a high recall or precision curve.\n",
    "There is a trade of between the precison and recall value as it is impossible to have very high values of both and so picking a value that still balances those metrics become tricky. This is where the precision recall curve comes in handy.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [],
   "source": [
    "#lets convert our target label to binary values\n",
    "y_test_binary=pd.get_dummies(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>2A</th>\n",
       "      <th>3A</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>225</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>85</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>418</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>132</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>357</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>181</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>71</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>422</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>148 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     2A  3A\n",
       "225   0   1\n",
       "14    1   0\n",
       "85    0   1\n",
       "418   0   1\n",
       "132   0   1\n",
       "..   ..  ..\n",
       "357   0   1\n",
       "0     0   1\n",
       "181   0   1\n",
       "71    0   1\n",
       "422   0   1\n",
       "\n",
       "[148 rows x 2 columns]"
      ]
     },
     "execution_count": 171,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test_binary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test_binary.drop('2A',inplace=True,axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>3A</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>225</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>85</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>418</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>132</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>357</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>181</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>71</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>422</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>148 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     3A\n",
       "225   1\n",
       "14    0\n",
       "85    1\n",
       "418   1\n",
       "132   1\n",
       "..   ..\n",
       "357   1\n",
       "0     1\n",
       "181   1\n",
       "71    1\n",
       "422   1\n",
       "\n",
       "[148 rows x 1 columns]"
      ]
     },
     "execution_count": 173,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test_binary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import precision_recall_curve\n",
    "precision, recall, thresholds = precision_recall_curve(\n",
    " y_test_binary, log_reg.predict_proba(normalised_test_df)[:,1])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x151bfe235c0>"
      ]
     },
     "execution_count": 175,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXxU1fn48c+TnZA9gQQIIYGwB9kCWAUVty+iX6mKouhXVNTWr7YurdXa2lrp4lKX2urXuuMCVqwW3KpoxV0gbMpOWBPWhJCVbJM8vz9myC+EBCYkdyZhnvfrlRdz55658xwyuc+cc889R1QVY4wxgSvI3wEYY4zxL0sExhgT4CwRGGNMgLNEYIwxAc4SgTHGBLgQfwfQWklJSZqenu7vMIwxplNZtmxZoap2a25fp0sE6enp5OTk+DsMY4zpVERke0v7rGvIGGMCnCUCY4wJcJYIjDEmwHW6awTGmI6jtraW/Px8qqqq/B2K8YiIiCA1NZXQ0FCvX2OJwBhz3PLz84mOjiY9PR0R8Xc4AU9V2b9/P/n5+WRkZHj9Ose6hkTkBRHZJyKrW9gvIvKEiOSKyHciMsqpWIwxzqiqqiIxMdGSQAchIiQmJra6hebkNYKXgElH2X8e0N/zcyPwfw7GYoxxSGuSgMvl4vHHH+fxxx/H5XI5GFXgOp6k7FjXkKp+LiLpRykyBXhZ3fNgfysicSLSQ1V3OxWT6RhUlQc+WM+0Mb3p2y3K3+EYH5ozZw633347AAkJCVx99dV+jsiAf0cN9QLyGm3ne547gojcKCI5IpJTUFDgk+CMc97IyePvn2/h6837/R2K8SGXy8WsWbMatmfNmtXmVkFxcTFPPfVUw/aiRYu44IIL2nTM5rz00kvccsstrXpNeno6hYWFRzx/33338ec///mI56urq5k2bRqZmZmMGzeObdu2tXjcYcOGMWLECLKzs1sVU0v8mQiaa780u0qOqj6jqtmqmt2tW7N3SJtOYk9JFb9/dx2JXcNI7BqGq67e3yEZH5kzZw65ublkZmaSmZlJbm4uc+bMadMxmyYCb9XV1bXpfZ3w/PPPEx8fT25uLrfffjt33XVXi2U//fRTVq5c2W6zLPgzEeQDvRttpwK7/BSL8ZEVOw5QVu1if0UNN722nCXbivwdkvGBxq2Be++9l3vvvRdoe6vg7rvvZvPmzYwYMYI777wTgPLycqZOncqgQYO48sorObQKY3p6Ovfffz/jx49n3rx5bN68mUmTJjF69GgmTJjA+vXrAZg3bx5ZWVkMHz6c0047reG9du3axaRJk+jfvz+/+MUvGp6fO3cuw4YNIysrq8WT9x/+8AcGDhzI2WefzYYNG5otM3/+fGbMmAHA1KlT+eSTT/DZCpKq6tgPkA6sbmHf+cAHuFsGJwNLvDnm6NGj1XRuy7cX6YBfva8/fPJLddXV+zsc0wZr1671qtzs2bMV0MzMTK2trdXa2lrNzMxUQGfPnn3c779161YdOnRow/ann36qMTExmpeXp3V1dXryySfrF198oaqqffr00QcffLCh7JlnnqkbN25UVdVvv/1WJ06cqKqqWVlZmp+fr6qqBw4cUFXVF198UTMyMrS4uFgrKys1LS1Nd+zYoTt37tTevXvrvn37tLa2VidOnKhvv/12w/sVFBRoTk6OZmVlaUVFhZaUlGi/fv304YcfPqIuQ4cO1by8vIbtvn37akFBwRHl0tPTdeTIkTpq1Cj9+9//3uz/S3O/FyBHWzivOjl8dC7wDTBQRPJFZKaI/FhEfuwp8j6wBcgFngX+16lYTMehqjy1aDMKPDx1OMFBNuzwRNe0NRASEkJISEi7tQqaGjt2LKmpqQQFBTFixIjD+tqnTZsGuFsNX3/9NZdeeikjRozgRz/6Ebt3u8epnHrqqVxzzTU8++yzh3UhnXXWWcTGxhIREcGQIUPYvn07S5cu5YwzzqBbt26EhIRw5ZVX8vnnnx8WzxdffMFFF11EZGQkMTExXHjhhc3Grc18+29uBNBXX33F8uXL+eCDD3jyySePeL/j4eSooSuOsV+Bm516f9MxzV+5i4Vr93LP5EFkdrcRQ4Gg8bWB6dOnNzw/ffp0Zs2a1XCtoL1GEIWHhzc8Dg4OPizJdO3aFYD6+nri4uJYuXLlEa9/+umnWbx4Me+99x4jRoxoKNPccZs7eTfHmyGdqamp5OXlkZqaisvloqSkhISEhCPK9ezZE4Du3btz0UUXsWTJksO6sI6HzTVkfGZfaRW/XbCGUWlxzBzf19/hGB9orjVwSHu0CqKjoykrK2v162JiYsjIyGDevHmA+9v4qlWrANi8eTPjxo3j/vvvJykpiby8vBaPM27cOD777DMKCwupq6tj7ty5nH766YeVOe2003j77beprKykrKyMd955p9ljXXjhhcyePRuAN998kzPPPPOIBFJRUdFQ34qKCj766COysrJaXf+mLBEYn1BV7nl7NVW1dTx8qXUJBYqWWgOHTJ8+vU0jiBITEzn11FPJyspquFjsrddee43nn3+e4cOHM3ToUObPnw/AnXfe2XDx97TTTmP48OEtHqNHjx786U9/YuLEiQwfPpxRo0YxZcqUw8qMGjWKadOmMWLECC655BImTJjQ7LFmzpzJ/v37yczM5NFHH+WBBx4A3BepJ0+eDMDevXsZP348w4cPZ+zYsZx//vlMmnS0+3a9I942bTqK7OxstYVpOp+vcwuZ/txiQoKEuMijT4Y1LiORJ6+0GUc6g3Xr1jF48OBm97lcLgYPHkxubq5Xx8rMzGTdunWHtRrM8Wnu9yIiy1S12RsP7H/c+ES/7lFce2o6Na6W7xvYtK+cJVuLiOliH8sTwaHWgLfa+1qB8Z79xRmfSI6J4Lf/PbTF/ZU1dZz/xBf0iuvCPZOb/4ZpOperr77aTuqdhF0jMB3Cwx9uYEthBQ9NPYnoCO/nUTf+19m6l090x/P7sERg/G7Z9iJe/HorCV3DWLByF6t3lvg7JOOliIgI9u/fb8mgg1DPegQRERGtep11DRm/O1BRS8/YLhSWV/OPnDyG944jq1esv8MyXkhNTSU/Px+bDLLjOLRCWWtYIjB+d/aQZAamRHPeX75geGoc08b0PvaLTIcQGhraqpWwTMdkXUPG7+rqlTveWIkAj1xm9xgY42vWIjB+9/Rnm1m67QCPXjac3gmR/g7HmIBjLQLjV6t3lvDYwo2cf1IPLhrZ7LpExhiHWSIwflNZU8etr68gKSqcP/wwyxZAN8ZPrGvI+M3v31vL5oIKXp05jrjIMH+HY0zAshaB8YsPvt/Na4t3cONpfRnfP8nf4RgT0CwRGJ/LKzrIL/75HcNTY/n5uQP9HY4xAc8SgfGp2rp6bn19BSj89YpRhIXYR9AYf7NrBManHv94I8t3FPPXK0aSlmhDRY3pCOzrmPGZLzcV8tSizUzL7s1/D+/p73CMMR6WCIxPFJRVc/sbK+nXLYrfXjjE3+EYYxqxriHjuLp65bZ/rKC0spZXZo4lMsw+dsZ0JPYXaRz31Ke5fJW7n/7do5i7eEfD86nxkdxwmi1ib4y/WSIwjttdWkVcZCgF5dXMX7ULV51SXu0iJSaCmeMzCLJJ5ozxK0sExnF/vGgYf7xoGOBeOOOmV5ezcN1enrhipCUBYzoAu1hsfOq5L7by7zV7+OV5gxibkeDvcIwxWCIwPrR4y34e+Pd6zstKYeZ4W8zEmI7CEoHxiX2lVdwydwV9EiJ5aOpJNtOoMR2IXSMwjqutq+eWOSsor3Lx6sxxREeE+jskY0wjlgiM4x74YD1LthXx+LQRDEyJ9nc4xpgmrGvIOOrNZfk8/+VWrjklnR/aCmTGdEiWCIxjVuYVc8/b33NKv0R+df5gf4djjGmBo4lARCaJyAYRyRWRu5vZnyYin4rIChH5TkQmOxmP8Z19pVX86JUckmPCeXL6KEKD7TuHMR2VY3+dIhIMPAmcBwwBrhCRprON/Rp4Q1VHApcDTzkVj/GdalcdP3p1GWVVLp69Opv4rrYMpTEdmZNf08YCuaq6RVVrgNeBKU3KKBDjeRwL7HIwHuMDqsqv317Nih3FPHLpcAalxBz7RcYYv3IyEfQC8hpt53uea+w+4CoRyQfeB37S3IFE5EYRyRGRnIKCAidiNe3kpa+3MW9ZPj89qz/nDevh73CMMV5wMhE0d8eQNtm+AnhJVVOBycArInJETKr6jKpmq2p2t27dHAjVtIdFG/Yx6921nDMkmdvO6u/vcIwxXnIyEeQDvRttp3Jk189M4A0AVf0GiACSHIzJOGRrYQU/mbOCeoWUmAg+3bDP3yEZY7zkZCJYCvQXkQwRCcN9MXhBkzI7gLMARGQw7kRgfT+dUO6+cupUCQ4SXvl2O3OX5B37RcaYDsGxRKCqLuAW4ENgHe7RQWtE5H4RudBT7GfADSKyCpgLXKOqTbuPTCdwzpBkPv35GXSLCkcEhqfG8l1+sb/DMsZ4wdEpJlT1fdwXgRs/95tGj9cCpzoZg/GdLzYVsqe0CoBHFm5k0cYC/nnTKX6OyhhzLDbXkGk3l4zqxeg+8cx4YQk7ig5ywUk9WLK1qGF/anwXesZ18WOExpjmWCIw7UZEyNlWxI6igwD87p21h+3v3z2KhXec7o/QjDFHYYnAtKsLR/Skd0IkdfXuSz119cp976xha2EFd5wzwM/RGWOaY4nAtKvwkGBO7pvYsD3r3bVsKajg3guG2A1mxnRQNhOYccwLX25tmILalqY0puOyFoFxxLrdpcx6z32NYF5OHv9clt+wLzw0iFdmjmNwD5uHyJiOwBKBcUTP2C7cdHo/qmrrG57LO3CQhWv3EtOlCykxEX6MzhjTmCUC44jYyFB+MWlQw3ZdvXLuY58B7hlKr3lpKTed3pdJWXbdwBh/s0RgfEKAsRmJ9IqP5MtNBewqqfJ3SMYYD7tYbHwiKEj408XDGNE7jnqFhK5h/Hv1Hl78aqu/QzMm4FkiMD51sNpFn8RIalz1/GvlLv6z3mYpNcbfLBEYn/r1BUN4cvooADK7R/HE5SP9HJExxhKB8amthRVc8+ISYruE8srMsbaesTEdgCUC4zN7S6v4n+cXU1hew6XZqazeWYqrrv7YLzTGOMoSgfGZD77fTf6BSgAe/3gTN7ycw7rdZX6OyhhjicD4zNU/SOc/Pzud809y3ztww4QMsnrZ3cXG+JslAuMzQUHCB6v38N53u5k+Lo17Jg9GRPwdljEBzxKB8ZmXvtrKwx9u4L+GJvPL8wYd+wXGGJ+wRGB8IndfOfd5Fqr5cM1eht33EY99vMnPURljwBKB8ZHeCV2Y9cMsbpjgno66a1gwZw3q7ueojDFgicD4SHhIMOdlpfDphgIiw4J56bqxDO8d5++wjDFYIjA+sr+8miufXUz+gYO8cM0YxqQn+DskY4yHJQLjuPp6ZcaLS9iwt4yYiFAeW7iR+5ssbG+M8R+bhtr4xIDu0XQNC2Ht7lIWby0ipkuov0MyxnhYIjCOCwoSHp02gve+280tc5cj4r5Y/Mu3vmvVcc4ZksyZg5IditKYwGWJwPjMjqKDdIsKB+Drzfu9fl1BeTWq7gvOlgiMaX+WCIzP3HRGP246o1+rXvPM55v54/vrmTQ0hXsmD3YoMmMCmyUC02E9tSiXh/69gfOH9eDxy0cQGmxjG4xxgiUC0yE98ckmHl24kSkjevLIpcMJsSRgjGMsEZgORVV57ONNPPHJJi4e1YuHpw4nOMgmpjPGSY5+zRKRSSKyQURyReTuFspcJiJrRWSNiMxxMh7Tsakqf/5oA098sonLslMtCRjjI461CEQkGHgSOAfIB5aKyAJVXduoTH/gl8CpqnpARGzymQBVX6/c/+5aXvp6G9PHpfH7KVkEWRIwxiec7BoaC+Sq6hYAEXkdmAI0vqX0BuBJVT0AoKr7HIzHdFA1rnrufHMV81fu4vrxGdwzebAlAWN8yMmuoV5AXqPtfM9zjQ0ABojIVyLyrYhMau5AInKjiOSISE5BQYFD4Rp/OFjj4oaXc5i/chd3TRrEr863JGCMrznZImjur1mbef/+wBlAKvCFiGSpavFhL1J9BngGIDs7u+kxTCdVfLCGa19ayqq8Yh68ZBjTxqT5OyRjApKTiSAf6N1oOxXY1UyZb1W1FtgqIhtwJ4alDsZlOoDdJZVc/fwSthcd5KkrRzMpK8XfIRkTsJzsGloK9BeRDBEJAy4HFjQp8y9gIoCIJOHuKtriYEymA9hcUM7U//uG3SVVzL52rCUBY/zMsRaBqrpE5BbgQyAYeEFV14jI/UCOqi7w7DtXRNYCdcCdqur9JDSm01mZV8x1Ly0lSOD1G08mq1esv0MyJuCJaufqcs/OztacnBx/h2GOw8K1e/nJ3OV0iw7n5evGkZHU1d8hGRMwRGSZqmY3t8/u2zc+8eayfH70Sg5VtfWM6B3Py99s49MNNlrYmI7AppgwPvFdfjFdw0NA4Z1V7jEDFdUuJg60ewiN8bejdg2JyB1He7GqPtruER2DdQ11XvX1yl3//I55y/JJ6BrG/57Rj/CQIxulI9Pi7dqBMe3saF1Dx2oRRDsQjwlQBeXV/GvlTgCKKmr4/Xvrmi137pBknrm62c+rMcYBR00Eqvo7XwViTnzJMRGs+M25VNXWHbHv6UWbee7LrcRFhnL+ST34clMhg3tEk+hZ0cwY45yjJgIReeJo+1X1p+0bjjnRRYWHEBV++Mdub2kVz325FYDig7Xc+vpKAC4e2YtHp43weYzGBJpjdQ0t80kUJqAlx0Tw4W2nUVpVy5zFO3h7xU6SY8KZNqY3O4srjygfLEJKbIQfIjXmxHSsrqHZvgrEBLaBKdGs3lnC2yvc1xD2llYz7ZlvWyz/2LThXDQy1VfhGXNC82r4qIh0A+4ChgANX8VU9UyH4jIBaEByNE9cMZKqmiOvIdSr8sJXW9m4t5xRaXGcMcCGnRrTXry9j+A14B/A+cCPgRmAzQdt2lVYSBAXDu95xPMHa1z87I1VbNxbztTRqfzhoizCQ4L9EKExJyZvE0Giqj4vIreq6mfAZyLymZOBGQOws7iSG2bnsG5PKb+aPJjrJ2QgYusVGNOevE0EtZ5/d4vI+bink7YOWuOopduKuOnV5VTX1vHCjDFMHGTdQcY4wdtE8HsRiQV+BvwViAFudywqE9BUlZe/2c6sd9fSOyGSuTeMo3+y3dtojFO8SgSq+q7nYQme9QOMccr9767lxa+2AdAzLoJHF25stpwIzBzfl9F94n0YnTEnHm9HDc0Gbj20hKSIxAOPqOp1TgZnAlNVbR0DkqMAKCirpqCs+rD9RRW1FJa7n5s8rAdgicCYtvC2a+ikxusIq+oBERnpUEwmwP3p4pNa3Lds+wFmznavZDo8NZZ1u0tZt7u0VcdPiYngf36Q3pYQjTmheJsIgkQkXlUPAIhIQitea0y7WbqtiPIqFyFBwppdpazZ5X0ScNW7Z9pNigrj8rFphAbbchzGgPcn80eAr0XkTUCBy4A/OBaVMS348en9+PHp/Vr9uue+2MIf31+HiDB9XB/mr9zV7rGNSY+nT6KtumY6H28vFr8sIjnAmYAAF6vqWkcjM6ad1NbV8+jCjdQroMoTn2xy5H2mZffmwaktd2sZ01G1pnsnAahQ1RdFpJuIZKjqVqcCM6a9hAYH8c3dZ1FaVXvswq1QV6889OF63v9+D+Mzk7hn8uB2Pb4xvuLtqKHfAtnAQOBFIBR4FTjVudCMaT+xkaHERoa22/HKqtzTZf9n/T5m/KAPv75giF1zMJ2Wty2Ci4CRwHIAVd0lInaHjwlIO/YfZObspWwprOD3P8ziqpP7+DskY9rE20RQo6oqIgogInZFzASkzzYWcOvrK1CFV64byymZSf4OyZg28zYRvCEifwfiROQG4DrgOefCMqZjqa9X/vqfXB7/ZCMDk6N5+qrRpCfZ9yFzYvB21NCfReQcoBT3dYLfqOpCRyMzpoMoPljDbf9YyaINBVw8shd/uGgYXcJsGmxz4vB61JDnxL8QQESCReRKVX3NsciM6QC+zy/hx68uY19ZFb//YRZXjkuzabDNCedYi9fHADcDvYAFuBPBzcCdwErcC9YYc0JavGU///PCEmpc9SRFhfHP5fn8c3l+w/7xmUn87NyBfozQmPZxrBbBK8AB4BvgetwJIAyYoqorHY7NGL+KDAvhlH6J1HmmpgD3JHjr95QBkG2znpoTxLESQV9VHQYgIs8BhUCaqpY5HpkxfjYsNZaXrh3bsL1owz7ueGMVXUKDmfXDLKaOtrWZzInhWImg4VZMVa0Tka2WBEygqa2r588fbeDvn21hUEo0f5s+iszuUf4Oy5h2c6xEMFxEDk3vKEAXz7YAqqoxjkZnjJ/lFR3kp6+vYMWOYq4cl8a9FwwhItRGDJkTy1ETgaq26RMvIpOAvwDBwHOq+kAL5aYC84AxqprTlvc0pj2oKvNy8vndO2sIEuFv00dywUk9/R2WMY5wbE0BEQkGngTOAfKBpSKyoOmspZ6pKn4KLHYqFmNao6Csml++9R0fr9vHD/om8vClJ5EaH+nvsIxxjJOLy4wFclV1C4CIvA5MAZpOXz0LeAj4uYOxGOOVf6/ezT1vr6a82sW9Fwzh2lPSCQqy+wbMic3JRNALyGu0nQ+Ma1zAs9xlb1V9V0RaTAQiciNwI0BaWpoDoZpAV1JZy+8WrOGtFTsZ1iuWRy8bTv9km1fRBAYnE0FzX6MaBmSLSBDwGHDNsQ6kqs8AzwBkZ2frMYob0yr/Xr2H3y5YTWF5Dbee1Z9bzsy0KaVNQHEyEeQDvRttpwKN1weMBrKARZ5b9lOABSJyoV0wNr6wt7SK38xfzYdr9jK4RwzPXp3NSalx/g7LGJ9zMhEsBfqLSAawE7gcmH5op6qWAA1z+IrIIuDnlgSM0+rrlTlLdvDgB+upqavnrkmDuH5ChrUCTMByLBGoqktEbgE+xD189AVVXSMi9wM5qrrAqfc2piW5+8r55VvfsXTbAU7pl8gfLxpm00mbgCeqnavLPTs7W3NyrNFgWm9lXjGXPf0NNXX1AKTERNB0ItGZ4zO4fkJfP0RnjLNEZJmqZje3z8muIWM6lOSYcC4bk0qNq77huaraej5YvZvaOiUmIoS+3ax1YAKPtQhMwNpXVsXU//uGHUUHARiYHE1iVFirjiEC/3tGJqfakpWmg7MWgTHNUIWMpK4kx4Q3PFdbV3+UVxwqo6zKL0YVwkOCqHbVORmmMY6zRGACVnJMBLOvG3vsgo0s3VbEvf9ajSqcOag7v7twKL0TbPoJ07lZIjDGC/vLq3ngg/XMW5ZPr7guPPM/ozlnSLItW2lOCJYIjDmG+St38pv5ayiprCUpKozLx/RmR9FBnv9ya6uPNaRHDKe04nqCy+Xib3/7GwC33HILISH2J2van32qjDmGv3+2hZJK9xpNheU1PLJw43Efa1RaHG+1IhHMmTOH22+/HYCEhASuvvrq435vY1pio4aMOYbaunoqa1t/QVgV3l6ezyMLN1Je7eKqcX342bkDiIv0bmSSy+Vi8ODB5ObmApCZmcm6deusVWCOi40aMqYNQoODWj39xNpdpfxm/mpyth9gZFocs6ZkkdUrtlXHmDNnDrm5uWRmZgKQm5vLnDlzrFVg2p21CIxpRyWVtTy2cCMvf7ON4CDhhgl9uWR0KkGtvKjscrk455TRbN+6mT//7RkSu4Zx7bXXWKvAHDdrERjjI797Zw1vLd8JQH2d8tSizTy1aHOrj1O++hP2b91MSHwPntiezI2n9SUzM9NaBcYRlgiMaUe3nTWA0/p3a9Mx6lwubnz5fwGIPeVyJgxIZsapfel9773MmDGDWbNmMX36dGsVmHZjnyRj2lFaYiRpiW27wewPjz9N0e4dhMT3oOuQM/gyt5DxD36K1scREt+D3NxcUn74C6Kyzmp4zZ3/NZCbJ2a2NXwToCwRGNOBuFwuXvjbIwBcev1tZJ8z6LD9OdzG3Ifvom7Zm8jQM1AJJiOpa5tbISawWSIwpgOZM2cOWza7Rwq9/Mc7juj+qZhwKx/PeYqCnduJWP85v77tx1w/IYPwkGA/RWxOBDZqyJgOoul9A8eS3rcfmzast2sFxitHGzVka/MZ00Ecum/AW9u2bGbOnDkORmQChbUIjOmgiipqeOKTTbz67XbCQoK4eWKmdQOZ42b3ERjTiRyscfHiV9t4etFmKmpcXD42jdvO7k/36Ah/h2ZOUJYIjOkgqmrrmLN4B08tyqWwvIazB3fn7vMGkdk92t+hmROcJQJjOoC3V+Tz0L83sLukiiCBi0f1YnSfeBZvLWLx1qIWXzc+M4k+ibbOsmkbSwTG+Fl9vTLr3XUUVdS4txXeWr6zYaqKo7nu1Ax+899DnA7RnOAsERjjZ0FBwpd3TaS8ynXUcmt2l/LoRxv5fmcJSVFh3Dwxk+nj0nwUpTmRWSIwpgOIDAshMqz5P8dVecX89T+b+HjdPmK7hPKLSQO55pT0Fssb01r2STKmg8rZVsQT/8nl840FxEWGcsc5A5hxSjqxXUL9HZo5wVgiMKYDqa9XPl63l2e/2MLSbQdI7BrG3ecN4qqT+xAVbn+uxhn2yTKmA6iqreOt5Tt57ostbCmsIDW+C7/97yFcPiaNLmF2A5lxliUCY/xod0klr327g7lLdrC/ooZhvWL56xUjOS8rhZBWLo9pzPGyRGCMj6kqi7cWMfvrbXy0di+qypmDkpk5PoOT+yYgrVzW0pi2skRgjI+UHKzl7RX5zF2Sx4a9ZcRFhnL9hAyuGteH3gltW8zGmLZwNBGIyCTgL0Aw8JyqPtBk/x3A9YALKACuU9XtTsZkjC8d+vb/+pIdvL96DzWueob1iuWhS07iwhE9iQi1/n/jf44lAhEJBp4EzgHygaUiskBV1zYqtgLIVtWDInIT8BAwzamYjPGVXcWV/GvlTubl5LO1sILoiBAuH9ObaWN6M7RnrL/DM+YwTrYIxgK5qroFQEReB6YADYlAVT9tVP5b4CoH4zHGUWVVtXyweg9vL9/Jt1v3owpj0uO5ZWImk4f1sNE/psNyMhH0AmVSxrkAAA7gSURBVPIabecD445SfibwQXM7RORG4EaAtDS7pd50HNWuOr7cVMj8lbv4aO0eqmrrSU+M5LazBnDRyF5tXsjeGF9wMhE0N/Sh2VVwROQqIBs4vbn9qvoM8Ay4F6ZprwCNOR41rnq+yi3k3e9289HaPZRVuYiLDGXq6FQuHpXKyN5xNvLHdCpOJoJ8oHej7VRgV9NCInI28CvgdFWtdjAeY9pk9c4SZn+9jQ/X7KHUM0Fc17Bgpozoyan9kggNEbbvr2D7/ooWjxEcFMSZg7rbXcKmQ3Hy07gU6C8iGcBO4HJgeuMCIjIS+DswSVX3ORiLMW12/7trWdJkbYCKmjrmr9zF/JVHfMdp0V8uH8GUEb3aOzxjjptjiUBVXSJyC/Ah7uGjL6jqGhG5H8hR1QXAw0AUMM/TlN6hqhc6FZMxbTH72rHsLa06ZrnyaheLNuzjve/3sG53KQBJUWFMykphclYPftAv0elQjWkVW7zemHaQf+Agn6zbx8fr9rJ4SxE1dfUkRYVzXlYKk4f1YGxGAsFBdt3A+I8tXm9MO6uvV1blFzec/NfvKQOgb1JXZpzSh7MHJ5Odbid/0zlYIjDGS3tLq/h8YwFfbCrky9xCiipqCA4SRveJ51eTB3PW4O707Rbl7zCNaTVLBMa0oKq2jsVbi/jCc/LfsNf9rT8pKpwzBnTjtAHdOGNgN+Iiw/wcqTFtY4nAGI+q2jpW5hWzeEsR327Zz7IdB6hx1RMWHMSYjHguHjWICf27MSglmiDr8jEnEEsEJmBV1daxfMeBhhP/irxialz1iMDglBiuGteHCQOSODkj0aaHMCc0SwQmYBSWV7N8+wGW7yhm+fYDrMwrpqauniCBIT1juPrkPozrm8jY9ARiI21dYBM4LBGYE5Krrp71e8pYscNz4t9xgO37DwIQEiQM7RnDjFP6cHLfRLLTE2xBeBPQLBGYTk9VyT9Qyfc7S/h+ZwkrdxSzKr+YgzV1gPvi7ug+cUwfm8aoPvEM6xVr6wAY04glAtOpND3pf59fwupdJRQfrAXc3/YH9Yjm0tGpjOoTz6i0eFLju9gkcMYchSUC02G56urZWljBuj1lrN1VyuqdR570B6ZEM2loClm9YhnWK5aBKdH2bd+YVrJEYPxOVSkoq2b9njLW7yll/e4y1u8pI3dfOTV19YCd9I1xkiUC41MllbVsLihn0173yX797jI27C2jqKKmoUxyTDgDU2KY0D+JgSnRDEqJoV/3roSH2EnfGCdYIjDtrr5e2VVSSe6+cjYXVLC5oJzNnseF5f9/yYkuocEMSInmnMHJDOrhPuEPSokmvqvdqWuML1kiMG2ypaCcNbtK3Sf7ggo27ytnS2E5VbX1h5WLDg+hb/coThuQRL9uUWR2jyItIZKgJhdx95VVs6/s/yeL9KRIawkY4zBLBOa47Squ5OxHP6Pei5nMy6pdrMorZlVecave49LRqTx86fDjjNAY4w1LBOa49YiN4NWZ4yiurG2xTL0qe0qq2L7/INv2V7C1sIJdxZXNJo+w4CDSEiNJT+xKRlIkfRK7cs6QZAdrYIwBSwSmDUSEsRkJ7CmtYueBSvIPVLKzuNL9uPggOw9Usqu4qmHkD7hP9n27RR12ss9I6kqfxEh6xHax+fuN8QNLBOaoql117Cp2n+h3Fh90n+wPVJLvOeHvKa2irsnX+6SocFLjuzC0Vyz/NTSF3gmRdrI3pgOzRBCgVJXSShd7y6rYU1LF3tJDP9Xs8TzeU1J12IVbgCCBlJgIesV3YWxGAr3iutArvgu94rqQGt+FnnFdbGy/MZ2MJYITUFVtHfsandAP/ewprT5su+nIHoC4yFCSoyNIjo1gYHI0veK7kBof2XCiT4mNIDQ4yA+1MsY4xRJBJ1FVW8f+ihr2l1ezv7yGwvLqw7YLyqvZV1rN3rKqhikYGgsPCSIlNoLkmAhOSo0jJSac5JiIhp+UmAi6x4Tbt3ljApAlAj+pr1eKK2vZX15Ngedkvt9zci9s9PjQib6s2tXscSJCg0iKCiexaxhpiZGMzUggudFJPiU2guToCGK6hNjEa8aYZlkiaAd19UpZVS3FB2sprqyl+GANJZWe7YO1HDhYQ1FFDfsrDn2br6GoorrZIZRBAgldw0mKCiMxKoyTUuNIjAprONknRoWTGBVGN8+/kWH2KzTGtI2dRRpx1dW7T+Cek3hJZU3Dyby4spaSgzUcaPT4ULnSqlr0KDdVRYeHkBAVRmLXMHonRDIyLd59om90Yj90oo+LDLNRNcYYnwqYRFBaVcs/l+Wzv7yGYs8JvqTS/W29+GAtJQdrW+x+OZaYiBDiIsOIiwwltkuo+3GXUOIiQ4mJCCUkuPkTu+JePrGwvJoNlLWhdsaYE11ocBAXjuhJTET7r6YXMIkgZ1sRv3tnrSPHLq1yUVrlYkeRI4c3xhgAQoOFaWPS2v24AZMIzhyUzPf3nXvEzU/GGNPRFZZXc/ajn1Nb58z5K2ASAUC0A00qY4xxWuNpWpxgdwYZY0yAs0RgjDEBzhKBMcYEOEcTgYhMEpENIpIrInc3sz9cRP7h2b9YRNKdjMcYY8yRHEsEIhIMPAmcBwwBrhCRIU2KzQQOqGom8BjwoFPxGGOMaZ6TLYKxQK6qblHVGuB1YEqTMlOA2Z7HbwJniU2IY4wxPuVkIugF5DXazvc812wZVXUBJUBi0wOJyI0ikiMiOQUFBQ6Fa4wxHVN4cDCTh6WQlhDpyPGdvI+guW/2Te+G8KYMqvoM8AxAdna23RFmjAkosZGhPHXlaMeO72SLIB/o3Wg7FdjVUhkRCQFiAZuowRhjfMjJRLAU6C8iGSISBlwOLGhSZgEww/N4KvAf1aPN42mMMaa9OdY1pKouEbkF+BAIBl5Q1TUicj+Qo6oLgOeBV0QkF3dL4HKn4jHGGNM8R+caUtX3gfebPPebRo+rgEudjMEYY8zR2Z3FxhgT4CwRGGNMgLNEYIwxAc4SgTHGBDjpbKM1RaQA2O7vOI5DElDo7yAccqLW7UStF1jdOqu21K2PqnZrbkenSwSdlYjkqGq2v+NwwolatxO1XmB166ycqpt1DRljTICzRGCMMQHOEoHvPOPvABx0otbtRK0XWN06K0fqZtcIjDEmwFmLwBhjApwlAmOMCXCWCNqRiEwSkQ0ikisidzez/xoRKRCRlZ6f6/0R5/E4Vt08ZS4TkbUiskZE5vg6xuPlxe/tsUa/s40iUuyPOI+HF3VLE5FPRWSFiHwnIpP9Eefx8KJufUTkE0+9FolIqj/ibC0ReUFE9onI6hb2i4g84an3dyIyqs1vqqr20w4/uKfa3gz0BcKAVcCQJmWuAf7m71gdqlt/YAUQ79nu7u+426tuTcr/BPeU6n6PvZ1+b88AN3keDwG2+TvudqzbPGCG5/GZwCv+jtvLup0GjAJWt7B/MvAB7hUeTwYWt/U9rUXQfsYCuaq6RVVrgNeBKX6Oqb14U7cbgCdV9QCAqu7zcYzHq7W/tyuAuT6JrO28qZsCMZ7HsRy5imBH5U3dhgCfeB5/2sz+DklVP+foKzVOAV5Wt2+BOBHp0Zb3tETQfnoBeY228z3PNXWJpzn3poj0bmZ/R+RN3QYAA0TkKxH5VkQm+Sy6tvH294aI9AEygP/4IK724E3d7gOuEpF83GuH/MQ3obWZN3VbBVzieXwREC0iiT6IzWlef2a9ZYmg/UgzzzUdm/sOkK6qJwEfA7Mdj6p9eFO3ENzdQ2fg/tb8nIjEORxXe/CmbodcDrypqnUOxtOevKnbFcBLqpqKu8vhFRHpDOcFb+r2c+B0EVkBnA7sBFxOB+YDrfnMeqUz/MI7i3yg8Tf8VJo0s1V1v6pWezafBUb7KLa2OmbdPGXmq2qtqm4FNuBODB2dN3U75HI6T7cQeFe3mcAbAKr6DRCBe2Kzjs6bv7ddqnqxqo4EfuV5rsR3ITqmNZ9Zr1giaD9Lgf4ikiEiYbhPGgsaF2jSj3chsM6H8bXFMesG/AuYCCAiSbi7irb4NMrj403dEJGBQDzwjY/jawtv6rYDOAtARAbjTgQFPo3y+Hjz95bUqHXzS+AFH8folAXA1Z7RQycDJaq6uy0HdHTN4kCiqi4RuQX4EPeIhhdUdY2I3A/kqOoC4KciciHu5mkR7lFEHZ6XdfsQOFdE1gJ1wJ2qut9/UXvHy7qBuwvldfUM2+gMvKzbz4BnReR23N0L13SGOnpZtzOAP4mIAp8DN/st4FYQkbm4Y0/yXLv5LRAKoKpP476WMxnIBQ4C17b5PTvB79wYY4yDrGvIGGMCnCUCY4wJcJYIjDEmwFkiMMaYAGeJwBhjApwlAhOQRKTOM5voahGZJyKR7XDMbBF54ij7e4rIm219H2Pamw0fNQFJRMpVNcrz+DVgmao+2mi/4P77qPdXjMb4irUIjIEvgEwRSReRdSLyFLAc6C0i54rINyKy3NNyOJQ8xojI1yKySkSWiEi0iJwhIu969p/eaA2DFZ796YfmmBeRCBF5UUS+9+w/dFf2NSLyloj8W0Q2ichDfvo/MQHEEoEJaCISApwHfO95aiDuKX5HAhXAr4GzVXUUkAPc4ZnS4B/Arao6HDgbqGxy6J8DN6vqCGBCM/tvBlDVYbjvWp4tIhGefSOAacAwYFonmqXWdFKWCEyg6iIiK3Gf3HcAz3ue3+6Z4x3ci34MAb7ylJ0B9MGdLHar6lIAVS1V1aazWn4FPCoiPwXimtk/HnjF8/r1wHbc8zMBfKKqJapaBaz1vKcxjrG5hkygqvR8W2/gvixAReOngIWqekWTcidxjGl/VfUBEXkP95ww34rI2UBVk2O3pLrR4zrs79Q4zFoExrTsW+BUEckEEJFIERkArAd6isgYz/PRni6mBiLST1W/V9UHcbc6BjU59ufAlZ6yA4A03FN3G+NzlgiMaYGqFuCeIXauiHyHOzEM8iyNOA34q4isAhbinr65sds8Q1NX4b4+8EGT/U8BwSLyPe7rDdc0WqvCGJ+y4aPGGBPgrEVgjDEBzhKBMcYEOEsExhgT4CwRGGNMgLNEYIwxAc4SgTHGBDhLBMYYE+D+H0wwbUBs1IScAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(precision, recall)\n",
    "close_default= np.argmin(np.abs(thresholds - 0.5))\n",
    "plt.plot(precision[close_default], recall[close_default], '^', c='k',\n",
    " markersize=10, label=\"threshold 0.5\", fillstyle=\"none\", mew=2)\n",
    "plt.xlabel(\"Precision\")\n",
    "plt.ylabel(\"Recall\")\n",
    "plt.legend(loc=\"best\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Each point along the PRcurve represents a possible threshold of the predict_proba esitmator. from the curve we can see that a precison of 0.7 will correspond to a recall value of about 0.2. The black traingle marks the point that corresponds to a thresh‐\n",
    "old of 0.5, the default threshold for predcit_proba. This point is the trade-off that\n",
    "is chosen when calling the predict method. Values located at the upper right corner are usuall values that correspond to high precision and recall. The curve starts at the upper left corner with a corresponding recall value of 1.0 which is the highes and this implies a low threshold value at that point and thus classifies everything as the positive class. Increasing the threshold i.e choosing a higher pobability value before a data point can be classified as positive we see that the curve moves to the right implying a higher prediction of  True Positive in other words precision level becomes high while the recall value drops.Raising the threshold more and more, we get to a situation where most of the points classified as being positive are true positives, leading\n",
    "to a very high precision but lower recall. The more the model keeps recall high as\n",
    "precision goes up, the better."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Average Precision SCORE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "One way to summarize the precision recall curve i s by computing the are under the curve which is called the average precision.The average precision score function can be used to calculae this."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6776877292194542"
      ]
     },
     "execution_count": 176,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import average_precision_score\n",
    "ap = average_precision_score(y_test_binary, log_reg.predict_proba(normalised_test_df)[:, 1])\n",
    "ap\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ROC  CURVE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The Reciever Operational Character curve is another metric for evaluating the performnce of a classificatin model. It is a plot of the <strong>RECALL</strong> versus the <strong> False Postive RATE</strong> the FPR is given by the formula <strong>FP/FP+TN</strong>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve\n",
    "fpr, tpr, thresholds = roc_curve(y_test_binary, log_reg.predict_proba(normalised_test_df)[:, 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x151bfe6fcf8>"
      ]
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAe50lEQVR4nO3df3BU9b3/8efbRBssYBVwBg2YaFB+BAkQQQelopZSq/B1pBWxg/Ry1bktdUpbqx2LesHWVq3164hVvFK5HSMtzHjBFqutFxVtFUJFv/wQXVEhYEukgoBQE3h//9hNZtlssptkz/46r8dMZnbP+eTs+ySQ93l/Pud8PubuiIhIeB2T6wBERCS3lAhEREJOiUBEJOSUCEREQk6JQEQk5EpzHUBn9e3b1ysqKnIdhohIQVm3bt1H7t4v2b6CSwQVFRXU19fnOgwRkYJiZh+0t09dQyIiIadEICISckoEIiIhV3BjBMk0NTXR0NDAoUOHch2KtKOsrIzy8nKOPfbYXIciIgmKIhE0NDTQq1cvKioqMLNchyMJ3J3du3fT0NBAZWVlrsMRkQSBdQ2Z2SIz22VmG9rZb2b2gJlFzOxNMxvV1c86dOgQffr0URLIU2ZGnz59VLGJ5KkgxwgeByZ1sP8rwKDY1/XAr7rzYUoC+U2/H5H8FVjXkLu/ZGYVHTSZAvy3R+fBftXMvmBm/d39w6BiEhHJd3WvbWP5+h1J9w09pTe3Xz4s45+Zy7uGTgW2x71viG1rw8yuN7N6M6tvbGzMSnCdVVJSQk1NDdXV1Vx++eXs2bOndd/GjRu56KKLOPPMMxk0aBDz588nfh2IZ555htraWoYMGcLgwYP5wQ9+kPQz0m0nIoVr+fodbPrwk6x+Zi4Hi5P1FSRdJcfdFwILAWpra/NyJZ0ePXqwfv16AK699loWLFjArbfeysGDB5k8eTK/+tWvmDhxIp9++ilXXnklDz30EN/+9rfZsGEDs2fP5g9/+AODBw+mubmZhQsXtjl+uu3ac/jwYUpKSjJ2viLSfcmu/jd9+AlD+/fmtzecl7U4clkRNAAD4t6XAztzFEtGnXfeeezYEf3l1tXVMW7cOCZOnAjA8ccfz4MPPsjPfvYzAO6++25uvfVWBg8eDEBpaSnf+ta32hyzo3YzZ85k2bJlrW179uwJwAsvvMCECROYPn06w4cP5+abb+ahhx5qbXfHHXfwi1/8AoB77rmHc845h7PPPpvbb789oz8PEUku2dX/0P69mVKTtHMkMLmsCFYAs81sCTAW2JuJ8YH/fHojm3ZmtqzqTL/c4cOHef7555k1axYQ7RYaPXr0UW3OOOMM9u/fzyeffMKGDRv4/ve/n/K46bZLtGbNGjZs2EBlZSWvv/463/3ud1sTyO9+9zv++Mc/8txzz/HOO++wZs0a3J3Jkyfz0ksvMX78+E5/noh0Trav/pMJLBGY2ZPAhUBfM2sAbgeOBXD3h4GVwKVABPgU+GZQsWTDwYMHqamp4f3332f06NF86UtfAqL30Ld3x0w27qQZM2ZM6737I0eOZNeuXezcuZPGxkZOPPFEBg4cyAMPPMBzzz3HyJEjAdi/fz/vvPOOEoFISAR519DVKfY78O1Mf24QI+rpaBkj2Lt3L5dddhkLFizgxhtvZNiwYbz00ktHtd26dSs9e/akV69eDBs2jHXr1jFixIgOj99Ru9LSUo4cOQJEE89nn33Wuu/zn//8UW2nTp3KsmXL+Pvf/860adNav+dHP/oRN9xwQ5fOXUQKm+YayrATTjiBBx54gHvvvZempiauueYaXn75Zf785z8D0crhxhtv5Ic//CEAN910Ez/96U95++23AThy5Aj33Xdfm+N21K6iooJ169YBsHz5cpqamtqNb9q0aSxZsoRly5YxdepUAL785S+zaNEi9u/fD8COHTvYtWtXJn4cIlIAlAgCMHLkSEaMGMGSJUvo0aMHy5cv58477+Sss85i+PDhnHPOOcyePRuAs88+m/vvv5+rr76aIUOGUF1dzYcfth0q6ajdddddx4svvsiYMWN47bXX2lQB8YYNG8a+ffs49dRT6d+/PwATJ05k+vTpnHfeeQwfPpypU6eyb9++AH4yIpKPLP5+9kJQW1vriQvTbN68mSFDhuQoIkmXfk9SqDp6yKs7snmrqJmtc/faZPtUEYiIpBDUQ165uFU0maKYfVREJGj5cJtnUFQRiIiEnCoCEZE4HU37UKxCWRE0Nzdz//33c//999Pc3JzrcEQkj+TLtA/ZFMqKoK6ujjlz5gBw0kknMWPGjBxHJCL5pJjHA5IJXUXQ3NzM/PnzW9/Pnz+/21XBnj17jprM7YUXXuCyyy7r1jGTefzxx1ufP0hXRUUFH330UZvtd9xxB/fee2+b7f/617+46qqrqKqqYuzYsbz//vvtHnf48OHU1NRQW5v0jjQRKRChSwR1dXVEIhGqqqqoqqoiEolQV1fXrWMmJoJ0HT58uFufG4THHnuME088kUgkwpw5c7j55pvbbbtq1SrWr19P4nMdIlJYQpUI4quBuXPnMnfuXKD7VcEtt9zCu+++S01NDTfddBMQnbht6tSpDB48mGuuuaZ1IZqKigrmzZvH+eefz9KlS3n33XeZNGkSo0eP5oILLuCtt94CYOnSpVRXVzNixIijJn/buXMnkyZNYtCgQa3TVAA8+eSTDB8+nOrq6nb/eP/kJz/hrLPO4pJLLmHLli1J2yxfvpxrr70WiM5L9Pzzz1NoDx2KSCe5e0F9jR492hNt2rSpzbZkFi9e7IBXVVV5U1OTNzU1eVVVlQO+ePHitI6RzHvvvefDhg1rfb9q1Srv3bu3b9++3Q8fPuznnnuur1692t3dTzvtNP/5z3/e2vaiiy7yt99+293dX331VZ8wYYK7u1dXV3tDQ4O7u3/88cfu7v7rX//aKysrfc+ePX7w4EEfOHCgb9u2zXfs2OEDBgzwXbt2eVNTk0+YMMGfeuqp1s9rbGz0+vp6r66u9gMHDvjevXv9jDPO8HvuuafNuQwbNsy3b9/e+v7000/3xsbGNu0qKip85MiRPmrUKH/kkUfS+jml+3sSyaWvP/wX//rDf8l1GBkH1Hs7f1dDUxEkVgOlpaWUlpZmrCpINGbMGMrLyznmmGNap6ducdVVVwHRquEvf/kLX/va16ipqeGGG25onT9o3LhxzJw5k0cfffSoLqSLL76YE044gbKyMoYOHcoHH3zA2rVrufDCC+nXrx+lpaVcc801bWY8Xb16NVdccQXHH388vXv3ZvLkyUnj9iRX/8mmy37llVf429/+xjPPPMOCBQvafJ6IFI7QJIL4sYHp06e3bp8+fXrGxgrife5zn2t9XVJSclSSaZkU7siRI3zhC19g/fr1rV+bN28G4OGHH+bOO+9k+/bt1NTUsHv37naPm+yPdzLprH9QXl7O9u3RpaSbm5vZu3cvJ510Upt2p5xyCgAnn3wyV1xxBWvWrEkrBhHJP6FIBMmqgRaZqAp69erVpdk6e/fuTWVlJUuXLgWiV+NvvPEGAO+++y5jx45l3rx59O3bt/WPczJjx47lxRdf5KOPPuLw4cM8+eSTfPGLXzyqzfjx43nqqac4ePAg+/bt4+mnn056rMmTJ7N48WIAli1bxkUXXdQmgRw4cKD1fA8cOMBzzz1HdXV1p89fRPJDKBJBe9VAi+5WBX369GHcuHFUV1e3Dhan64knnuCxxx5jxIgRDBs2jOXLlwPR9QdaBn/Hjx/f4cI1/fv356677mLChAmMGDGCUaNGMWXKlKPajBo1iquuuoqamhquvPJKLrjggqTHmjVrFrt376aqqor77ruvdW3lnTt3cumllwLwj3/8g/PPP58RI0YwZswYvvrVrzJp0qROnbeI5I+in4a6ubmZIUOGEIlE0jp+VVUVmzdvPqpqkMzQNNRSCK565K8ARfdAWainoW6pBtKV6bECEZF8V/SXvTNmzNAUEiIiHSiaiqDQurjCRr8fkfxVFBVBWVkZu3fvpk+fPmndIinZ5e7s3r2bsrKyXIciIdWZpSaLfcrpZIoiEZSXl9PQ0EBjY2OuQ5F2lJWVUV5enuswJKRappZO5w98sU85nUxRJIJjjz2WysrKXIchInksbFNLd0bRjBGIiEjXKBGIiIScEoGISMgpEYiIhJwSgYhIyCkRiIiEnBKBiEjIBZoIzGySmW0xs4iZ3ZJk/0AzW2Vmr5vZm2Z2aZDxiIhIW4ElAjMrARYAXwGGAleb2dCEZj8GfufuI4FpwENBxSMiIskFWRGMASLuvtXdPwOWAFMS2jjQ8sz3CcDOAOMREZEkgkwEpwLx6ys2xLbFuwP4hpk1ACuB7yQ7kJldb2b1Zlav+YRERDIryESQbBrQxLmIrwYed/dy4FLgN2bWJiZ3X+jute5e269fvwBCFREJryAnnWsABsS9L6dt188sYBKAu//VzMqAvsCuAOMSkSLV3nTTYZxaujOCrAjWAoPMrNLMjiM6GLwioc024GIAMxsClAHq+xGRLmmZbjpRGKeW7ozAKgJ3bzaz2cCzQAmwyN03mtk8oN7dVwDfBx41szlEu41mupayEpE0JLv6b7ny13TTnRPoegTuvpLoIHD8ttviXm8CxgUZg4gUp2SLzejKv2uKYmEaEQknXf1nhhKBiOQNrS2cG5prSETyRnuDvcmoGyhzVBGISF5Rd0/2qSIQEQk5VQQiEij1++c/VQQiEij1++c/VQQikjF6yKswqSIQkYxJdvWvq/z8p4pARDot1eRuuvovLKoIRKTTNLlbcVFFICJdoiv/4qGKQEQk5FQRiEiHOroTSIqDKgIR6ZDuBCp+qghEJCWNBxQ3VQQiIiGnRCAiEnLqGhIRIPVDYlK8VBGICKCHxMJMFYGItNKgcDipIhARCTlVBCIhpIfEJJ4qApEQ0kNiEk8VgUge68wyj52h6aIlnioCkTzWmWUeO0NX/xJPFYFIntOVuwQtrURgZrXABcApwEFgA/Bnd/9ngLGJiEgWdNg1ZGYzzexvwI+AHsAWYBdwPvAnM1tsZgODD1NERIKSqiL4PDDO3Q8m22lmNcAgYFumAxMRkezosCJw9wXtJYHY/vXu/nx7+81skpltMbOImd3STpuvm9kmM9toZnXphy4iIpnQYUVgZg90tN/db+zge0uABcCXgAZgrZmtcPdNcW0GEe12GufuH5vZyZ0JXkREui9V19C6bhx7DBBx960AZrYEmAJsimtzHbDA3T8GcPdd3fg8ERHpgg4Tgbsv7saxTwW2x71vAMYmtDkTwMxeAUqAO9z9j4kHMrPrgesBBg7U2LSISCal6hp6GvD29rv75I6+Pdm3JPn8QcCFQDmw2syq3X1PwucsBBYC1NbWthuPiIh0XqquoXu7cewGYEDc+3JgZ5I2r7p7E/CemW0hmhjWduNzRUSkE1J1Db3YjWOvBQaZWSWwA5gGTE9o8z/A1cDjZtaXaFfR1m58poiIdFK6TxYPAu4ChgJlLdvd/fT2vsfdm81sNvAs0f7/Re6+0czmAfXuviK2b6KZbQIOAze5++4un42IiHRaunMN/Rq4HfglMAH4JsnHAI7i7iuBlQnbbot77cD3Yl8iIpID6SaCHu7+vJmZu38A3GFmq4kmBxHJAC0WI7mS7jTUh8zsGOAdM5ttZlcAevhLJIO0WIzkSroVwXeB44EbgflEu4euDSookWLRmYVltFiM5EpaicDdW27n3E90fEBE0tBylZ9O946u/iVX0r1r6E/A11oe9DKzE4El7v7lIIMTKRTtXfnrKl8KQbpjBH3jn/aNzQ2kMQKRmPaWlNRVvhSCdMcIjpjZQHffBmBmp9HB1BMiYaQrfylU6SaCW4GXzazlSePxxCaBEylm6Q726jZPKWTpDhb/0cxGAecSfZBsjrt/FGhkInkg3cFedQFJIUt3sNiAScDp7j7PzAaa2Rh3XxNseCK5py4fKXbpDhY/BJxHdII4gH1EVx8TEZECl+4YwVh3H2Vmr0P0riEzOy7AuEREJEvSrQiaYmsQO4CZ9QOOBBaViIhkTboVwQPAU8DJZvYTYCrw48CiEglQV6Z9EClm6d419ISZrQMuJnrX0P9x982BRiYSEE37IHK0lIkgNuvom+5eDbwVfEgi7evM1Xx7NO2DyNFSjhG4+xHgDTMbmIV4RDrU3lQOnaGrfJGjpTtG0B/YaGZrgAMtG919ciBRSVHIxNV7Il3Ni2ReuongPwONQopSZ/ri06WreZHM6zARxJamdHd/MVWbzIcmxUBX7yL5L9UYwSoz+07i+ICZHWdmF5nZYrRSmYhIQUvVNTQJ+DfgSTOrBPYAZUAJ8BzwS3dfH2yIIiISpA4TgbsfIjrP0ENmdizQFzgYv0iNiIgUtnQHi3H3JuDDAGMREZEcSHeuIRERKVJKBCIiIZd211C82Eyk09z9iQzHIwWovQfHNGGbSGHosCIws95m9iMze9DMJlrUd4CtwNezE6Lku/amfdDDXyKFIVVF8BvgY+CvwL8DNwHHAVN026jE04NjIoUrVSI43d2HA5jZfwEfAQPdfV/gkYmISFakGixuannh7oeB95QERESKS6pEMMLMPjGzfWa2Dzg77n3KuYDNbJKZbTGziJnd0kG7qWbmZlbb2RMQEZHuSfVkcUlXDxy7s2gB8CWgAVhrZivcfVNCu17AjcBrXf0sERHpulR3DZWZ2Xdjdw1db2adud10DBBx963u/hmwBJiSpN184G7gUCeOLSIiGZKqa2gxUAv8P+BS4BedOPapwPa49w2xba3MbCQwwN1/39GBYkmo3szqGxsbOxGCiIikkuoKf2jcXUOPAWs6cWxLsq113YLYWsi/BGamOpC7LwQWAtTW1mrtAxGRDOrMXUPNnTx2AzAg7n05sDPufS+gGnjBzN4HzgVWaMBYRCS7UlUENXF3BxnQI/beAHf3juYPWAsMiq1jsAOYBkxv2enue4lOax09uNkLwA/cvb7TZyEiIl2WKhG84e4ju3Jgd282s9nAs0QXslnk7hvNbB5Q7+4runJcERHJrFSJoFv98e6+EliZsO22dtpe2J3PkuxINsGcJpcTKWypEsHJZva99na6+30ZjkfyXMsEc/F/+DW5nEhhS5UISoCeJL8DSIpEe9NIJ9OSBDTBnEjxSJUIPnT3eVmJRHIm2VV+e3T1L1J8UiUCVQIhoat8kfBK9RzBxVmJQkREcibVpHP/zFYg0rHm5mYefPBBAGbPnk1pacfFXFf6/UUknLq0ZrFkX11dHXPmzAHgpJNOYsaMGR22V7+/iKRLiaAANDc3M3/+/Nb38+fPZ/r06SmrAvX7i0g6Uo0RSB6oq6sjEolQVVVFVVUVkUiEurq6XIclIkVCiSDPxVcDc+fOZe7cuUC0Kmhu7uw8gCIibalrKM/FVwN2xvmseGMHPfuVE4lEGHfdPCrO/UrS79MAsIikSxVBHkusBn6/4R+8tetThn71mwBs+sOvOXI4eVWgAWARSZcqgjxV99o2/u/D/0UkEqFnv3Ke3l/JW7uiV/lP/Pg2hrz8JJFIhMt7vpfyDiIRkY6oIshTT637gPUrHgNg6Fe/yTElpa1X+aWlpRorEJGMUUWQp7at/ROf/XMnVVVVvPLobW1uFZ0+fTrz589vvYNIVYGIdJW5F9YSwLW1tV5fX9yLmDU3N3PiKZXsb2xIq31VVRWbN29O+VyBiISXma1z96RLAatrKA/V1dWlnQQAPVcgIt2iS8g8NGPGDP5wcBCAngwWkcCpIhARCTklAhGRkFPXUI61N120ngwWkWxRRZBjLdNFJ9KTwSKSLaoI8oCmixaRXFJFICISckoEIiIhp0QgIhJySgQiIiGnRCAiEnJKBCIiIadEICIScoEmAjObZGZbzCxiZrck2f89M9tkZm+a2fNmdlqQ8YiISFuBJQIzKwEWAF8BhgJXm9nQhGavA7XufjawDLg7qHhERCS5ICuCMUDE3be6+2fAEmBKfAN3X+Xun8bevgqUBxiPiIgkEWQiOBXYHve+IbatPbOAZ5LtMLPrzazezOobGxszGKKIiASZCCzJtqTrYprZN4Ba4J5k+919obvXunttv379MhiiiIgEOelcAzAg7n05sDOxkZldAtwKfNHd/xVgPDmXbMppTTctIrkWZEWwFhhkZpVmdhwwDVgR38DMRgKPAJPdfVeAseSFZFNOa7ppEcm1wCoCd282s9nAs0AJsMjdN5rZPKDe3VcQ7QrqCSw1M4Bt7j45qJjygaacFpF8E+h6BO6+EliZsO22uNeXBPn5IiKSmp4sFhEJOSUCEZGQUyIQEQk5JQIRkZBTIhARCTklAhGRkFMiEBEJOSUCEZGQUyIQEQk5JQIRkZBTIhARCTklAhGRkFMiEBEJOSUCEZGQC3Qa6rBKthIZaDUyEclPqggCkGwlMtBqZCKSn1QRBEQrkYlIoVBFICISckoEIiIhp0QgIhJySgQiIiGnRCAiEnJKBCIiIadEICIScqF5jqC9p32DoCeIRaSQhKYiaO9p3yDoCWIRKSShqQhAT/uKiCQTmopARESSUyIQEQk5JQIRkZBTIhARCblAE4GZTTKzLWYWMbNbkuz/nJn9Nrb/NTOrCDIeERFpK7BEYGYlwALgK8BQ4GozG5rQbBbwsbtXAb8Efh5UPCIiklyQFcEYIOLuW939M2AJMCWhzRRgcez1MuBiM7MAYxIRkQRBPkdwKrA97n0DMLa9Nu7ebGZ7gT7AR/GNzOx64HqAgQMHdimYoafoSV8RkWSCTATJruy9C21w94XAQoDa2to2+9Nx++XDuvJtIiJFL8iuoQZgQNz7cmBne23MrBQ4AfhngDGJiEiCIBPBWmCQmVWa2XHANGBFQpsVwLWx11OB/3X3Ll3xi4hI1wTWNRTr858NPAuUAIvcfaOZzQPq3X0F8BjwGzOLEK0EpgUVj4iIJBfopHPuvhJYmbDttrjXh4CvBRmDiIh0TE8Wi4iEnBKBiEjIKRGIiIScEoGISMhZod2taWaNwAdd/Pa+JDy1HAI653DQOYdDd875NHfvl2xHwSWC7jCzenevzXUc2aRzDgedczgEdc7qGhIRCTklAhGRkAtbIliY6wByQOccDjrncAjknEM1RiAiIm2FrSIQEZEESgQiIiFXlInAzCaZ2RYzi5jZLUn2f87Mfhvb/5qZVWQ/ysxK45y/Z2abzOxNM3vezE7LRZyZlOqc49pNNTM3s4K/1TCdczazr8d+1xvNrC7bMWZaGv+2B5rZKjN7Pfbv+9JcxJkpZrbIzHaZ2YZ29puZPRD7ebxpZqO6/aHuXlRfRKe8fhc4HTgOeAMYmtDmW8DDsdfTgN/mOu4snPME4PjY6/8IwznH2vUCXgJeBWpzHXcWfs+DgNeBE2PvT8513Fk454XAf8ReDwXez3Xc3Tzn8cAoYEM7+y8FniG6wuO5wGvd/cxirAjGABF33+runwFLgCkJbaYAi2OvlwEXm1myZTMLRcpzdvdV7v5p7O2rRFeMK2Tp/J4B5gN3A4eyGVxA0jnn64AF7v4xgLvvynKMmZbOOTvQsij5CbRdCbGguPtLdLxS4xTgvz3qVeALZta/O59ZjIngVGB73PuG2Lakbdy9GdgL9MlKdMFI55zjzSJ6RVHIUp6zmY0EBrj777MZWIDS+T2fCZxpZq+Y2atmNilr0QUjnXO+A/iGmTUQXf/kO9kJLWc6+/89pUAXpsmRZFf2iffIptOmkKR9Pmb2DaAW+GKgEQWvw3M2s2OAXwIzsxVQFqTzey4l2j10IdGqb7WZVbv7noBjC0o653w18Li7/8LMziO66mG1ux8JPrycyPjfr2KsCBqAAXHvy2lbKra2MbNSouVkR6VYvkvnnDGzS4Bbgcnu/q8sxRaUVOfcC6gGXjCz94n2pa4o8AHjdP9tL3f3Jnd/D9hCNDEUqnTOeRbwOwB3/ytQRnRytmKV1v/3zijGRLAWGGRmlWZ2HNHB4BUJbVYA18ZeTwX+12OjMAUq5TnHukkeIZoECr3fGFKcs7vvdfe+7l7h7hVEx0Umu3t9bsLNiHT+bf8P0RsDMLO+RLuKtmY1ysxK55y3ARcDmNkQoomgMatRZtcKYEbs7qFzgb3u/mF3Dlh0XUPu3mxms4Fnid5xsMjdN5rZPKDe3VcAjxEtHyNEK4FpuYu4+9I853uAnsDS2Lj4NnefnLOguynNcy4qaZ7zs8BEM9sEHAZucvfduYu6e9I85+8Dj5rZHKJdJDML+cLOzJ4k2rXXNzbucTtwLIC7P0x0HORSIAJ8Cnyz259ZwD8vERHJgGLsGhIRkU5QIhARCTklAhGRkFMiEBEJOSUCEZGQUyIQSZOZHTaz9XFfFWZ2oZntjc18udnMbo+1jd/+lpndm+v4RdpTdM8RiATooLvXxG+ITWG+2t0vM7PPA+vNrGVuo5btPYDXzewpd38luyGLpKaKQCRD3P0AsA44I2H7QWA93ZwYTCQoSgQi6esR1y30VOJOM+tDdE6jjQnbTyQ6389L2QlTpHPUNSSSvjZdQzEXmNnrwBHgZ7EpEC6MbX8TOCu2/e9ZjFUkbUoEIt232t0va2+7mZ0JvBwbI1if7eBEUlHXkEjA3P1t4C7g5lzHIpKMEoFIdjwMjDezylwHIpJIs4+KiIScKgIRkZBTIhARCTklAhGRkFMiEBEJOSUCEZGQUyIQEQk5JQIRkZD7/y8eNohYiGDUAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(fpr, tpr, label=\"ROC Curve\")\n",
    "plt.xlabel(\"FPR\")\n",
    "plt.ylabel(\"TPR (recall)\")\n",
    "default = np.argmin(np.abs(thresholds - 0.5))\n",
    "plt.plot(fpr[default], tpr[default], '^', markersize=10,\n",
    "label=\"threshold 0.5 \", fillstyle=\"none\", c='k', mew=2)\n",
    "plt.legend(loc=\"best\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A quick glance at the curve we can notice something different compared to the precision recall curve. In the roc curve optimum values will be found at the upper left corner where we have a higher recall value and a low false positive rate unlike the preicsion recall curve where the optimum values would be located at the upper right corner. From the above curve the default threshold value gives us a recall value slightly greater than 0.3 and FPR of about 0.19. Depening on the business goal the threshold can be changed and this would lead to either a higher recall and subsequently higher FPR or a lower recall value and therefore a lower FPR."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Area Under the Curve (AUC)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The AUC value gives a summary of the ruc curve. The AUC\n",
    "it can be interpreted as evaluating the ranking of positive samples. It’s equivalent to the\n",
    "probability that a randomly picked point of the positive class will have a higher score\n",
    "according to the classifier than a randomly picked point from the negative class. So, a\n",
    "perfect AUC of 1 means that all positive points have a higher score than all negative\n",
    "points. This can also be simply put as the certainty at which a data instance is classified as positve is  with a certainty of 1 for all positive class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC for Logistic Regressio: 0.547\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "log_auc = roc_auc_score(y_test_binary, log_reg.predict_proba(normalised_test_df)[:, 1])\n",
    "print(\"AUC for Logistic Regressio: {:.3f}\".format(log_auc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Notebook by Adediwura Boluro_Ajayi\n",
    "linkedin:<a href=\"https://linkedin.com/in/adediwura\"> connect on linkedin</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}