machine_learning_model_rough_draft.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "81f7fe01",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import our dependencies\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler,OneHotEncoder\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.svm import SVC\n",
    "import pandas as pd\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7a6c1a38",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Sex</th>\n",
       "      <th>ChestPainType</th>\n",
       "      <th>RestingBP</th>\n",
       "      <th>Cholesterol</th>\n",
       "      <th>FastingBS</th>\n",
       "      <th>RestingECG</th>\n",
       "      <th>MaxHR</th>\n",
       "      <th>ExerciseAngina</th>\n",
       "      <th>Oldpeak</th>\n",
       "      <th>ST_Slope</th>\n",
       "      <th>HeartDisease</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>40</td>\n",
       "      <td>M</td>\n",
       "      <td>ATA</td>\n",
       "      <td>140</td>\n",
       "      <td>289</td>\n",
       "      <td>0</td>\n",
       "      <td>Normal</td>\n",
       "      <td>172</td>\n",
       "      <td>N</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Up</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>49</td>\n",
       "      <td>F</td>\n",
       "      <td>NAP</td>\n",
       "      <td>160</td>\n",
       "      <td>180</td>\n",
       "      <td>0</td>\n",
       "      <td>Normal</td>\n",
       "      <td>156</td>\n",
       "      <td>N</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Flat</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>37</td>\n",
       "      <td>M</td>\n",
       "      <td>ATA</td>\n",
       "      <td>130</td>\n",
       "      <td>283</td>\n",
       "      <td>0</td>\n",
       "      <td>ST</td>\n",
       "      <td>98</td>\n",
       "      <td>N</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Up</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>48</td>\n",
       "      <td>F</td>\n",
       "      <td>ASY</td>\n",
       "      <td>138</td>\n",
       "      <td>214</td>\n",
       "      <td>0</td>\n",
       "      <td>Normal</td>\n",
       "      <td>108</td>\n",
       "      <td>Y</td>\n",
       "      <td>1.5</td>\n",
       "      <td>Flat</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>54</td>\n",
       "      <td>M</td>\n",
       "      <td>NAP</td>\n",
       "      <td>150</td>\n",
       "      <td>195</td>\n",
       "      <td>0</td>\n",
       "      <td>Normal</td>\n",
       "      <td>122</td>\n",
       "      <td>N</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Up</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Age Sex ChestPainType  RestingBP  Cholesterol  FastingBS RestingECG  MaxHR  \\\n",
       "0   40   M           ATA        140          289          0     Normal    172   \n",
       "1   49   F           NAP        160          180          0     Normal    156   \n",
       "2   37   M           ATA        130          283          0         ST     98   \n",
       "3   48   F           ASY        138          214          0     Normal    108   \n",
       "4   54   M           NAP        150          195          0     Normal    122   \n",
       "\n",
       "  ExerciseAngina  Oldpeak ST_Slope  HeartDisease  \n",
       "0              N      0.0       Up             0  \n",
       "1              N      1.0     Flat             1  \n",
       "2              N      0.0       Up             0  \n",
       "3              Y      1.5     Flat             1  \n",
       "4              N      0.0       Up             0  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load in the dataset\n",
    "import pandas as pd\n",
    "heart_df = pd.read_csv(\"Resources/heart.csv\")\n",
    "heart_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "15c7196e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "918"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Find out how many rows\n",
    "heart_df[heart_df.columns[0]].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a56b0a0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "17aa69a0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sex               2\n",
       "ChestPainType     4\n",
       "RestingECG        3\n",
       "ExerciseAngina    2\n",
       "ST_Slope          3\n",
       "dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Generate our categorical variable list\n",
    "heart_cat = heart_df.dtypes[heart_df.dtypes == \"object\"].index.tolist()\n",
    "\n",
    "# Check the number of unique values in each column\n",
    "heart_df[heart_cat].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "66446ed6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\yutac\\.conda\\envs\\mlenv\\lib\\site-packages\\sklearn\\utils\\deprecation.py:87: FutureWarning: Function get_feature_names is deprecated; get_feature_names is deprecated in 1.0 and will be removed in 1.2. Please use get_feature_names_out instead.\n",
      "  warnings.warn(msg, category=FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Sex_F</th>\n",
       "      <th>Sex_M</th>\n",
       "      <th>ChestPainType_ASY</th>\n",
       "      <th>ChestPainType_ATA</th>\n",
       "      <th>ChestPainType_NAP</th>\n",
       "      <th>ChestPainType_TA</th>\n",
       "      <th>RestingECG_LVH</th>\n",
       "      <th>RestingECG_Normal</th>\n",
       "      <th>RestingECG_ST</th>\n",
       "      <th>ExerciseAngina_N</th>\n",
       "      <th>ExerciseAngina_Y</th>\n",
       "      <th>ST_Slope_Down</th>\n",
       "      <th>ST_Slope_Flat</th>\n",
       "      <th>ST_Slope_Up</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Sex_F  Sex_M  ChestPainType_ASY  ChestPainType_ATA  ChestPainType_NAP  \\\n",
       "0    0.0    1.0                0.0                1.0                0.0   \n",
       "1    1.0    0.0                0.0                0.0                1.0   \n",
       "2    0.0    1.0                0.0                1.0                0.0   \n",
       "3    1.0    0.0                1.0                0.0                0.0   \n",
       "4    0.0    1.0                0.0                0.0                1.0   \n",
       "\n",
       "   ChestPainType_TA  RestingECG_LVH  RestingECG_Normal  RestingECG_ST  \\\n",
       "0               0.0             0.0                1.0            0.0   \n",
       "1               0.0             0.0                1.0            0.0   \n",
       "2               0.0             0.0                0.0            1.0   \n",
       "3               0.0             0.0                1.0            0.0   \n",
       "4               0.0             0.0                1.0            0.0   \n",
       "\n",
       "   ExerciseAngina_N  ExerciseAngina_Y  ST_Slope_Down  ST_Slope_Flat  \\\n",
       "0               1.0               0.0            0.0            0.0   \n",
       "1               1.0               0.0            0.0            1.0   \n",
       "2               1.0               0.0            0.0            0.0   \n",
       "3               0.0               1.0            0.0            1.0   \n",
       "4               1.0               0.0            0.0            0.0   \n",
       "\n",
       "   ST_Slope_Up  \n",
       "0          1.0  \n",
       "1          0.0  \n",
       "2          1.0  \n",
       "3          0.0  \n",
       "4          1.0  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a OneHotEncoder instance\n",
    "enc = OneHotEncoder(sparse=False)\n",
    "\n",
    "# Fit and transform the OneHotEncoder using the categorical variable list\n",
    "encode_df = pd.DataFrame(enc.fit_transform(heart_df[heart_cat]))\n",
    "\n",
    "# Add the encoded variable names to the dataframe\n",
    "encode_df.columns = enc.get_feature_names(heart_cat)\n",
    "encode_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "2420febe",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\yutac\\.conda\\envs\\mlenv\\lib\\site-packages\\ipykernel_launcher.py:3: FutureWarning: In a future version of pandas all arguments of DataFrame.drop except for the argument 'labels' will be keyword-only\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>RestingBP</th>\n",
       "      <th>Cholesterol</th>\n",
       "      <th>FastingBS</th>\n",
       "      <th>MaxHR</th>\n",
       "      <th>Oldpeak</th>\n",
       "      <th>HeartDisease</th>\n",
       "      <th>Sex_F</th>\n",
       "      <th>Sex_M</th>\n",
       "      <th>ChestPainType_ASY</th>\n",
       "      <th>...</th>\n",
       "      <th>ChestPainType_NAP</th>\n",
       "      <th>ChestPainType_TA</th>\n",
       "      <th>RestingECG_LVH</th>\n",
       "      <th>RestingECG_Normal</th>\n",
       "      <th>RestingECG_ST</th>\n",
       "      <th>ExerciseAngina_N</th>\n",
       "      <th>ExerciseAngina_Y</th>\n",
       "      <th>ST_Slope_Down</th>\n",
       "      <th>ST_Slope_Flat</th>\n",
       "      <th>ST_Slope_Up</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>40</td>\n",
       "      <td>140</td>\n",
       "      <td>289</td>\n",
       "      <td>0</td>\n",
       "      <td>172</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>49</td>\n",
       "      <td>160</td>\n",
       "      <td>180</td>\n",
       "      <td>0</td>\n",
       "      <td>156</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>37</td>\n",
       "      <td>130</td>\n",
       "      <td>283</td>\n",
       "      <td>0</td>\n",
       "      <td>98</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>48</td>\n",
       "      <td>138</td>\n",
       "      <td>214</td>\n",
       "      <td>0</td>\n",
       "      <td>108</td>\n",
       "      <td>1.5</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>54</td>\n",
       "      <td>150</td>\n",
       "      <td>195</td>\n",
       "      <td>0</td>\n",
       "      <td>122</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Age  RestingBP  Cholesterol  FastingBS  MaxHR  Oldpeak  HeartDisease  \\\n",
       "0   40        140          289          0    172      0.0             0   \n",
       "1   49        160          180          0    156      1.0             1   \n",
       "2   37        130          283          0     98      0.0             0   \n",
       "3   48        138          214          0    108      1.5             1   \n",
       "4   54        150          195          0    122      0.0             0   \n",
       "\n",
       "   Sex_F  Sex_M  ChestPainType_ASY  ...  ChestPainType_NAP  ChestPainType_TA  \\\n",
       "0    0.0    1.0                0.0  ...                0.0               0.0   \n",
       "1    1.0    0.0                0.0  ...                1.0               0.0   \n",
       "2    0.0    1.0                0.0  ...                0.0               0.0   \n",
       "3    1.0    0.0                1.0  ...                0.0               0.0   \n",
       "4    0.0    1.0                0.0  ...                1.0               0.0   \n",
       "\n",
       "   RestingECG_LVH  RestingECG_Normal  RestingECG_ST  ExerciseAngina_N  \\\n",
       "0             0.0                1.0            0.0               1.0   \n",
       "1             0.0                1.0            0.0               1.0   \n",
       "2             0.0                0.0            1.0               1.0   \n",
       "3             0.0                1.0            0.0               0.0   \n",
       "4             0.0                1.0            0.0               1.0   \n",
       "\n",
       "   ExerciseAngina_Y  ST_Slope_Down  ST_Slope_Flat  ST_Slope_Up  \n",
       "0               0.0            0.0            0.0          1.0  \n",
       "1               0.0            0.0            1.0          0.0  \n",
       "2               0.0            0.0            0.0          1.0  \n",
       "3               1.0            0.0            1.0          0.0  \n",
       "4               0.0            0.0            0.0          1.0  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Merge one-hot encoded features and drop the originals\n",
    "heart_df = heart_df.merge(encode_df,left_index=True, right_index=True)\n",
    "heart_df = heart_df.drop(heart_cat,1)\n",
    "heart_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "7ca7b867",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Age\n",
      "RestingBP\n",
      "Cholesterol\n",
      "FastingBS\n",
      "MaxHR\n",
      "Oldpeak\n",
      "HeartDisease\n",
      "Sex_F\n",
      "Sex_M\n",
      "ChestPainType_ASY\n",
      "ChestPainType_ATA\n",
      "ChestPainType_NAP\n",
      "ChestPainType_TA\n",
      "RestingECG_LVH\n",
      "RestingECG_Normal\n",
      "RestingECG_ST\n",
      "ExerciseAngina_N\n",
      "ExerciseAngina_Y\n",
      "ST_Slope_Down\n",
      "ST_Slope_Flat\n",
      "ST_Slope_Up\n"
     ]
    }
   ],
   "source": [
    "for col in heart_df.columns:\n",
    "    print(col)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "26a82773",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove loan status target from features data\n",
    "y = heart_df.HeartDisease.values\n",
    "X = heart_df.drop(columns=[\"HeartDisease\"]).values\n",
    "\n",
    "# Split training/test datasets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, train_size = 0.6, random_state=42, stratify=y)\n",
    "\n",
    "# Create a StandardScaler instance\n",
    "scaler = StandardScaler()\n",
    "\n",
    "# Fit the StandardScaler\n",
    "X_scaler = scaler.fit(X_train)\n",
    "\n",
    "# Scale the data\n",
    "X_train_scaled = X_scaler.transform(X_train)\n",
    "X_test_scaled = X_scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "5c9cf184",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the SVM model\n",
    "svm = SVC(kernel='linear')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "26ff8859",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SVC(kernel='linear')"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train the model\n",
    "svm.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f20407f4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " SVM model accuracy: 0.857\n"
     ]
    }
   ],
   "source": [
    "# Evaluate the model\n",
    "y_pred = svm.predict(X_test_scaled)\n",
    "print(f\" SVM model accuracy: {accuracy_score(y_test,y_pred):.3f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fc9560e1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "2e3f0b72",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the model - deep neural net\n",
    "number_input_features = len(X_train_scaled[0])\n",
    "hidden_nodes_layer1 = 10\n",
    "hidden_nodes_layer2 = 5\n",
    "\n",
    "nn = tf.keras.models.Sequential()\n",
    "\n",
    "# First hidden layer\n",
    "nn.add(\n",
    "    tf.keras.layers.Dense(units=hidden_nodes_layer1, input_dim=number_input_features, activation=\"relu\")\n",
    ")\n",
    "\n",
    "# Second hidden layer\n",
    "nn.add(tf.keras.layers.Dense(units=hidden_nodes_layer2, activation=\"relu\"))\n",
    "\n",
    "\n",
    "# Output layer\n",
    "nn.add(tf.keras.layers.Dense(units=1, activation=\"sigmoid\"))\n",
    "\n",
    "# Compile the Sequential model together and customize metrics\n",
    "nn.compile(loss=\"binary_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "511f460d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n",
      "22/22 [==============================] - 1s 2ms/step - loss: 0.7954 - accuracy: 0.4898\n",
      "Epoch 2/50\n",
      "22/22 [==============================] - 0s 2ms/step - loss: 0.6849 - accuracy: 0.5843\n",
      "Epoch 3/50\n",
      "22/22 [==============================] - 0s 2ms/step - loss: 0.6027 - accuracy: 0.7166\n",
      "Epoch 4/50\n",
      "22/22 [==============================] - 0s 2ms/step - loss: 0.5363 - accuracy: 0.7834\n",
      "Epoch 5/50\n",
      "22/22 [==============================] - 0s 1ms/step - loss: 0.4846 - accuracy: 0.8169\n",
      "Epoch 6/50\n",
      "22/22 [==============================] - 0s 1ms/step - loss: 0.4460 - accuracy: 0.8358\n",
      "Epoch 7/50\n",
      "22/22 [==============================] - 0s 1ms/step - loss: 0.4178 - accuracy: 0.8416\n",
      "Epoch 8/50\n",
      "22/22 [==============================] - 0s 1ms/step - loss: 0.3978 - accuracy: 0.8445\n",
      "Epoch 9/50\n",
      "22/22 [==============================] - 0s 1ms/step - loss: 0.3837 - accuracy: 0.8445\n",
      "Epoch 10/50\n",
      "22/22 [==============================] - 0s 1ms/step - loss: 0.3729 - accuracy: 0.8445\n",
      "Epoch 11/50\n",
      "22/22 [==============================] - 0s 1ms/step - loss: 0.3646 - accuracy: 0.8459\n",
      "Epoch 12/50\n",
      "22/22 [==============================] - 0s 1ms/step - loss: 0.3579 - accuracy: 0.8474\n",
      "Epoch 13/50\n",
      "22/22 [==============================] - 0s 1ms/step - loss: 0.3517 - accuracy: 0.8517\n",
      "Epoch 14/50\n",
      "22/22 [==============================] - 0s 1ms/step - loss: 0.3464 - accuracy: 0.8547\n",
      "Epoch 15/50\n",
      "22/22 [==============================] - 0s 1ms/step - loss: 0.3417 - accuracy: 0.8561\n",
      "Epoch 16/50\n",
      "22/22 [==============================] - 0s 1ms/step - loss: 0.3377 - accuracy: 0.8561\n",
      "Epoch 17/50\n",
      "22/22 [==============================] - ETA: 0s - loss: 0.1130 - accuracy: 0.96 - 0s 1ms/step - loss: 0.3332 - accuracy: 0.8634\n",
      "Epoch 18/50\n",
      "22/22 [==============================] - 0s 1ms/step - loss: 0.3294 - accuracy: 0.8648\n",
      "Epoch 19/50\n",
      "22/22 [==============================] - 0s 1ms/step - loss: 0.3262 - accuracy: 0.8663\n",
      "Epoch 20/50\n",
      "22/22 [==============================] - 0s 1ms/step - loss: 0.3230 - accuracy: 0.8648\n",
      "Epoch 21/50\n",
      "22/22 [==============================] - 0s 1ms/step - loss: 0.3197 - accuracy: 0.8619\n",
      "Epoch 22/50\n",
      "22/22 [==============================] - 0s 1ms/step - loss: 0.3172 - accuracy: 0.8634\n",
      "Epoch 23/50\n",
      "22/22 [==============================] - 0s 1ms/step - loss: 0.3144 - accuracy: 0.8692\n",
      "Epoch 24/50\n",
      "22/22 [==============================] - 0s 999us/step - loss: 0.3118 - accuracy: 0.8706\n",
      "Epoch 25/50\n",
      "22/22 [==============================] - 0s 853us/step - loss: 0.3092 - accuracy: 0.8692\n",
      "Epoch 26/50\n",
      "22/22 [==============================] - 0s 1ms/step - loss: 0.3067 - accuracy: 0.8750\n",
      "Epoch 27/50\n",
      "22/22 [==============================] - 0s 1ms/step - loss: 0.3042 - accuracy: 0.8765\n",
      "Epoch 28/50\n",
      "22/22 [==============================] - 0s 1ms/step - loss: 0.3020 - accuracy: 0.8808\n",
      "Epoch 29/50\n",
      "22/22 [==============================] - 0s 1ms/step - loss: 0.3003 - accuracy: 0.8794\n",
      "Epoch 30/50\n",
      "22/22 [==============================] - 0s 1ms/step - loss: 0.2975 - accuracy: 0.8794\n",
      "Epoch 31/50\n",
      "22/22 [==============================] - 0s 1ms/step - loss: 0.2956 - accuracy: 0.8794\n",
      "Epoch 32/50\n",
      "22/22 [==============================] - 0s 1ms/step - loss: 0.2929 - accuracy: 0.8808\n",
      "Epoch 33/50\n",
      "22/22 [==============================] - 0s 1ms/step - loss: 0.2911 - accuracy: 0.8823\n",
      "Epoch 34/50\n",
      "22/22 [==============================] - 0s 1ms/step - loss: 0.2893 - accuracy: 0.8837\n",
      "Epoch 35/50\n",
      "22/22 [==============================] - 0s 1ms/step - loss: 0.2878 - accuracy: 0.8852\n",
      "Epoch 36/50\n",
      "22/22 [==============================] - 0s 1ms/step - loss: 0.2859 - accuracy: 0.8852\n",
      "Epoch 37/50\n",
      "22/22 [==============================] - 0s 2ms/step - loss: 0.2840 - accuracy: 0.8852\n",
      "Epoch 38/50\n",
      "22/22 [==============================] - 0s 1ms/step - loss: 0.2827 - accuracy: 0.8852\n",
      "Epoch 39/50\n",
      "22/22 [==============================] - 0s 1ms/step - loss: 0.2811 - accuracy: 0.8881\n",
      "Epoch 40/50\n",
      "22/22 [==============================] - 0s 948us/step - loss: 0.2793 - accuracy: 0.8852\n",
      "Epoch 41/50\n",
      "22/22 [==============================] - 0s 1ms/step - loss: 0.2782 - accuracy: 0.8866\n",
      "Epoch 42/50\n",
      "22/22 [==============================] - 0s 1ms/step - loss: 0.2761 - accuracy: 0.8866\n",
      "Epoch 43/50\n",
      "22/22 [==============================] - 0s 1ms/step - loss: 0.2750 - accuracy: 0.8881\n",
      "Epoch 44/50\n",
      "22/22 [==============================] - 0s 1ms/step - loss: 0.2738 - accuracy: 0.8852\n",
      "Epoch 45/50\n",
      "22/22 [==============================] - 0s 1ms/step - loss: 0.2722 - accuracy: 0.8895\n",
      "Epoch 46/50\n",
      "22/22 [==============================] - 0s 949us/step - loss: 0.2704 - accuracy: 0.8895\n",
      "Epoch 47/50\n",
      "22/22 [==============================] - ETA: 0s - loss: 0.2163 - accuracy: 0.90 - 0s 1ms/step - loss: 0.2693 - accuracy: 0.8895\n",
      "Epoch 48/50\n",
      "22/22 [==============================] - 0s 997us/step - loss: 0.2680 - accuracy: 0.8895\n",
      "Epoch 49/50\n",
      "22/22 [==============================] - ETA: 0s - loss: 0.1962 - accuracy: 0.90 - 0s 1ms/step - loss: 0.2671 - accuracy: 0.8895\n",
      "Epoch 50/50\n",
      "22/22 [==============================] - 0s 1ms/step - loss: 0.2657 - accuracy: 0.8910\n",
      "8/8 - 0s - loss: 0.3364 - accuracy: 0.8652 - 141ms/epoch - 18ms/step\n",
      "Loss: 0.3363837003707886, Accuracy: 0.865217387676239\n"
     ]
    }
   ],
   "source": [
    "# Train the model \n",
    "fit_model = nn.fit(X_train_scaled, y_train, epochs=50) \n",
    "# Evaluate the model using the test data \n",
    "model_loss, model_accuracy = nn.evaluate(X_test_scaled,y_test,verbose=2)\n",
    "print(f\"Loss: {model_loss}, Accuracy: {model_accuracy}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "240be635",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "2686a347",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8826086956521739\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\yutac\\.conda\\envs\\mlenv\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,\n"
     ]
    }
   ],
   "source": [
    "# Trying Logistic regression\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "classifier = LogisticRegression(solver='lbfgs',\n",
    "   max_iter=200,\n",
    "   random_state=1)\n",
    "\n",
    "classifier.fit(X_train, y_train)\n",
    "y_pred = classifier.predict(X_test)\n",
    "from sklearn.metrics import accuracy_score\n",
    "print(accuracy_score(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "55c4b4ea",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "2fb690df",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 86,  17],\n",
       "       [ 10, 117]], dtype=int64)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix, accuracy_score, classification_report\n",
    "# Display the confusion matrix\n",
    "confusion_matrix(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "58c1a617",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 40., 140., 289., ...,   0.,   0.,   1.],\n",
       "       [ 49., 160., 180., ...,   0.,   1.,   0.],\n",
       "       [ 37., 130., 283., ...,   0.,   0.,   1.],\n",
       "       ...,\n",
       "       [ 57., 130., 131., ...,   0.,   1.,   0.],\n",
       "       [ 57., 130., 236., ...,   0.,   1.,   0.],\n",
       "       [ 38., 138., 175., ...,   0.,   0.,   1.]])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# List the features sorted in descending order by feature importance\n",
    "X"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "mlenv",
   "language": "python",
   "name": "mlenv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}