homework1_student_ver.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cn8mvvwTUrG-"
      },
      "source": [
        "# Homework 1"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zIRDtwbvUrHC"
      },
      "source": [
        "## Part 1: Mortality Prediction in the ICU"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "The dataset you will use for this homework assignment can be found at the following link: https://drive.google.com/drive/folders/1n-fmx1jS_IWEsUj3EzlZ1qZ0hxqtSnZF?usp=sharing\n",
        "\n",
        "Please make sure to copy the files to your own google drive folder and substitute the path to that file below to load the dataset."
      ],
      "metadata": {
        "id": "Emn_ImD6xjT1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "\n",
        "# Will ask for authentication, click allow\n",
        "drive.mount(\"/content/drive\")\n",
        "\n",
        "# Enter the correct path here\n",
        "#------YOUR CODE HERE--------\n",
        "path_to_data_folder = \"/content/drive/My Drive/hst_953_2022_data\"\n",
        "#------YOUR CODE ENDS--------\n"
      ],
      "metadata": {
        "id": "NRYHtJlb3Qku"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lL7nZI1KUrHD"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "# Load the GOSSIS dataset into a pandas dataframe\n",
        "df = pd.read_csv(os.path.join(path_to_data_folder, \"gossis.csv\"))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1C2xa_zJUrHE"
      },
      "source": [
        "(a)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rcgBewwqUrHE"
      },
      "outputs": [],
      "source": [
        "#------YOUR CODE HERE--------\n",
        "# Enter the features corresponding to each category as a list below\n",
        "# category (a)\n",
        "# relevant features, all the features other than the ones listed below\n",
        "\n",
        "# category (b)\n",
        "irrelevant_features = [\"feature1\", \"feature2\", \"feature3\"]\n",
        "\n",
        "# category (c)\n",
        "potentially_leaky_features = [\"feature1\", \"feature2\", \"feature3\"]\n",
        "#------YOUR CODE ENDS--------\n",
        "\n",
        "# Features to drop: categories (b) and (c)\n",
        "features_to_drop = irrelevant_features + potentially_leaky_features\n",
        "\n",
        "# Ground truth labels\n",
        "label = \"hospital_death\""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4jDFZof1UrHF"
      },
      "source": [
        "(b)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xFNDeiRlUrHG"
      },
      "outputs": [],
      "source": [
        "# Define the dataset and the target variable\n",
        "X = df.drop(columns=features_to_drop+[label])\n",
        "y = df[[label]]\n",
        "# Populate the following list with the names of the binary features\n",
        "# that you didn't drop from the dataset\n",
        "#------YOUR CODE HERE--------\n",
        "remaining_binary_features = [\"feature1\", \"feature2\", \"feature3\"]\n",
        "#------YOUR CODE ENDS--------\n",
        "\n",
        "# Convert binary features into strings\n",
        "# (to apply a specific preprocessing pipeline to those features later)\n",
        "for c in remaining_binary_features:\n",
        "    X[c] = X[c].fillna(-1)\n",
        "    X[c] = X[c].astype(int)\n",
        "    X[c] = X[c].astype(str)\n",
        "    X[c] = X[c].replace(\"-1\", np.nan)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-0hj1gw3UrHG"
      },
      "outputs": [],
      "source": [
        "# Preprocessing pipeline\n",
        "from sklearn.preprocessing import StandardScaler, OneHotEncoder\n",
        "from sklearn.compose import ColumnTransformer, make_column_selector as selector\n",
        "from sklearn.impute import SimpleImputer\n",
        "from sklearn.pipeline import Pipeline\n",
        "\n",
        "#------YOUR CODE HERE--------\n",
        "# Each of the below transformers should be either a preprocessing step\n",
        "# (i.e. a scaler/encoder/imputer) or a pipeline (chain) of preprocessing\n",
        "# steps\n",
        "\n",
        "# For numerical features\n",
        "numeric_transformer = \n",
        "\n",
        "# For categorical features\n",
        "categorical_transformer = \n",
        "#------YOUR CODE ENDS--------\n",
        "\n",
        "preprocessing = ColumnTransformer(\n",
        "    transformers=[\n",
        "        (\n",
        "            \"num\",\n",
        "            numeric_transformer,\n",
        "            selector(dtype_exclude=[\"category\", object, \"string\"]),\n",
        "        ),\n",
        "        (\n",
        "            \"cat\",\n",
        "            categorical_transformer,\n",
        "            selector(dtype_include=[\"category\", object, \"string\"]),\n",
        "        ),\n",
        "    ]\n",
        ")\n",
        "\n",
        "preprocessing_pipeline = Pipeline(\n",
        "    steps=[(\"preprocessing\", preprocessing)]\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iRDVIDe-UrHH"
      },
      "outputs": [],
      "source": [
        "# Split the dataset into an 80%/20% train-test split\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "X_train_raw, X_test_raw, y_train, y_test = train_test_split(X, y, test_size=0.2, shuffle=True, random_state=42)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fhKd2ZQCUrHI"
      },
      "outputs": [],
      "source": [
        "# Train the preprocessing pipeline on the train set and apply it on the test set\n",
        "X_train = preprocessing_pipeline.fit_transform(X_train_raw)  # fit on train set\n",
        "# extract new feature names\n",
        "feature_names = list(preprocessing_pipeline[0].transformers_[0][2]) + list(preprocessing_pipeline[0].transformers_[1][1].get_feature_names_out())\n",
        "X_train = pd.DataFrame(X_train, columns=feature_names)  # to dataframe type\n",
        "# apply preprocessing pipeline to test set\n",
        "X_test = pd.DataFrame(preprocessing_pipeline.transform(X_test_raw), columns=feature_names)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iKYQWhItUrHI"
      },
      "source": [
        "(c)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4xCizBmxUrHI"
      },
      "outputs": [],
      "source": [
        "from sklearn.linear_model import LogisticRegression\n",
        "\n",
        "# Create and fit a logistic regression model\n",
        "#------YOUR CODE HERE--------\n",
        "\n",
        "#------YOUR CODE ENDS--------"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eOi-bTwtUrHJ"
      },
      "outputs": [],
      "source": [
        "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, roc_auc_score\n",
        "from collections import defaultdict\n",
        "\n",
        "#------YOUR CODE HERE--------\n",
        "# Fill out the following function to compute and return the different\n",
        "# performance metrics of the input model. This is to avoid copying\n",
        "# the same code to compute the metrics for each model\n",
        "def evaluate_model(model, X_train, y_train, X_test, y_test, sample_weight=None):\n",
        "    \n",
        "    pass\n",
        "#------YOUR CODE ENDS--------"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZUVgmepsUrHK"
      },
      "outputs": [],
      "source": [
        "# Use evaluate_model to compute the different performance metrics for your linear regression model\n",
        "#------YOUR CODE HERE--------\n",
        "\n",
        "#------YOUR CODE ENDS--------"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uHaRBC2bUrHK"
      },
      "source": [
        "(d)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ihm69dntUrHK"
      },
      "outputs": [],
      "source": [
        "from sklearn.ensemble import RandomForestClassifier\n",
        "\n",
        "# Create and fit a random forest\n",
        "# Use random_state=42 for consistency\n",
        "#------YOUR CODE HERE--------\n",
        "\n",
        "#------YOUR CODE ENDS--------"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "U81UxmaIUrHL"
      },
      "outputs": [],
      "source": [
        "# Use evaluate_model to compute the different performance metrics for your random forest\n",
        "#------YOUR CODE HERE--------\n",
        "\n",
        "#------YOUR CODE ENDS--------"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PseP_LeWUrHK"
      },
      "outputs": [],
      "source": [
        "import xgboost\n",
        "\n",
        "# Create and fit an xgboost model\n",
        "# Use random_state=42 for consistency\n",
        "#------YOUR CODE HERE--------\n",
        "\n",
        "#------YOUR CODE ENDS--------\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vmHhhHkfUrHL"
      },
      "outputs": [],
      "source": [
        "# Use evaluate_model to compute the different performance metrics for your xgboost model\n",
        "#------YOUR CODE HERE--------\n",
        "\n",
        "#------YOUR CODE ENDS--------"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tGkZCRUaUrHL"
      },
      "source": [
        "(e)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IdzBtJRLUrHL"
      },
      "outputs": [],
      "source": [
        "# Plot the performance metrics on a histogram\n",
        "# You may use any python library of your choice\n",
        "#------YOUR CODE HERE--------\n",
        "\n",
        "#------YOUR CODE ENDS--------"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kEZEQAyLUrHM"
      },
      "source": [
        "(f) (answer this in your report)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GDqiBk18UrHN"
      },
      "source": [
        "(g)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "07c15Mh9UrHN"
      },
      "outputs": [],
      "source": [
        "!pip install shap"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UuonrnW3UrHN"
      },
      "outputs": [],
      "source": [
        "import shap\n",
        "\n",
        "# Compute and plot the shapley values of the xgboost model on a beeswarm plot\n",
        "#------YOUR CODE HERE--------\n",
        "\n",
        "#------YOUR CODE ENDS--------"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GxDmxeg2UrHO"
      },
      "source": [
        "(h)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BkWrf7X6UrHO"
      },
      "outputs": [],
      "source": [
        "# Split the test set into cohorts in three different ways\n",
        "# Remember to split both the input features and the ground truth labels!\n",
        "#------YOUR CODE HERE--------\n",
        "\n",
        "#------YOUR CODE ENDS--------"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7QeLY7aKUrHO"
      },
      "source": [
        "(i) "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eBbJNr4uUrHO"
      },
      "outputs": [],
      "source": [
        "# Compute the performance metrics below\n",
        "# You may use the evaluate_model function from before\n",
        "#------YOUR CODE HERE--------\n",
        "\n",
        "#------YOUR CODE ENDS--------"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Plot the performance metrics on a histogram for each of the cohort splits\n",
        "#------YOUR CODE HERE--------\n",
        "\n",
        "#------YOUR CODE ENDS--------"
      ],
      "metadata": {
        "id": "OX1xg6__I_nu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "(j) Answer this question in your report"
      ],
      "metadata": {
        "id": "tHaM5K7WJ_Q_"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2jdo0lN6UrHP"
      },
      "source": [
        "(k)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "q5FFcdrRUrHP"
      },
      "outputs": [],
      "source": [
        "# Visualize the shapley values for the prediction of the first datapoint of each half\n",
        "# of each split using a waterfall plot\n",
        "#------YOUR CODE HERE--------\n",
        "\n",
        "#------YOUR CODE ENDS--------"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GCah6FMHUrHR"
      },
      "source": [
        "(l) Answer this question in your report"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "q2jJUqJrUrHR"
      },
      "source": [
        "## Problem 2. Delving into Disparities "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_XdiNU-IUrHR"
      },
      "source": [
        "(a) "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "31aWB50RUrHR"
      },
      "outputs": [],
      "source": [
        "# Split the training set into cohorts the same way you split the test set before\n",
        "#------YOUR CODE HERE--------\n",
        "\n",
        "#------YOUR CODE ENDS--------"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UH4ykh8wUrHR"
      },
      "outputs": [],
      "source": [
        "# Plot the distributions, each on a different pie chart\n",
        "#------YOUR CODE HERE--------\n",
        "\n",
        "#------YOUR CODE ENDS--------"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BY2DsSRXUrHS"
      },
      "source": [
        "(b) "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "y5aPVOTgUrHU"
      },
      "outputs": [],
      "source": [
        "def drop_rows_criteria(df: pd.DataFrame,\n",
        "                       outcome: pd.DataFrame,\n",
        "                       pct_drop: float) -> pd.DataFrame:\n",
        "    \"\"\"\n",
        "    Return two new versions of a dataset and the corresponding outcome\n",
        "    without some rows based on a certain criteria\n",
        "    (to change in the code of this function)\n",
        "\n",
        "    Args:\n",
        "        df (pd.DataFrame): initial dataset, pandas DataFrame format\n",
        "        outcome (pd.DataFrame): outcome dataset, pandas DataFrame format\n",
        "        pct_drop (float): percentage of chance to drop a value\n",
        "            if a certain criteria is met\n",
        "\n",
        "    Returns:\n",
        "        pd.DataFrame: modified dataset\n",
        "        pd.DataFrame: modified outcome\n",
        "    \n",
        "    Example:\n",
        "        X_train_raw2, y_train2 = drop_rows_criteria(X_train_raw, y_train)\n",
        "    \"\"\"\n",
        "    df2 = df.copy(deep=True).reset_index(drop=True)  # work on a copy of the dataframe\n",
        "    outcome2 = outcome.copy(deep=True).reset_index(drop=True)\n",
        "    indices_to_drop = []\n",
        "\n",
        "    # Local rng for better reproducibility\n",
        "    rng = np.random.default_rng(seed=42)\n",
        "    for i in range(len(df2)):\n",
        "        if df2.iat[i, df2.columns.get_loc(\"gender\")] != \"M\":\n",
        "\n",
        "            if rng.random() < pct_drop:  # pct_drop chance to drop this row\n",
        "                indices_to_drop.append(i)\n",
        "    df2.drop(indices_to_drop, axis=0, inplace=True)\n",
        "    outcome2.drop(indices_to_drop, axis=0, inplace=True)\n",
        "    return df2, outcome2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8Dolnxz9UrHU"
      },
      "outputs": [],
      "source": [
        "from tqdm import tqdm\n",
        "\n",
        "report = {}\n",
        "percentages = [0.2, 0.4, 0.6, 0.8]\n",
        "for pct in tqdm(percentages):\n",
        "\n",
        "    # Use drop_rows_criteria to drop pct% of the training data\n",
        "    #------YOUR CODE HERE--------\n",
        "    X_train_raw2, y_train2 = \n",
        "    #------YOUR CODE ENDS--------\n",
        "    \n",
        "    # Create a fresh preprocessing pipeline\n",
        "    preprocessing_pipeline2 = Pipeline(\n",
        "        steps=[(\"preprocessing\", preprocessing)]\n",
        "    )\n",
        "\n",
        "\n",
        "    # Fit the preprocessing pipeline to the new training set and store the\n",
        "    # normalized version of that set in X_train2\n",
        "    # (Question to think about: why should we do this?)\n",
        "    #------YOUR CODE HERE--------\n",
        "    X_train2 = \n",
        "    #------YOUR CODE ENDS--------\n",
        "\n",
        "    # Extract new feature names\n",
        "    features_names = list(preprocessing_pipeline2[0].transformers_[0][2]) + list(preprocessing_pipeline2[0].transformers_[1][1].get_feature_names_out())\n",
        "    X_train2 = pd.DataFrame(X_train2, columns=features_names)  # to dataframe type\n",
        "    X_test2 = pd.DataFrame(preprocessing_pipeline2.transform(X_test_raw), columns=features_names)\n",
        "\n",
        "    # Split the new test data (X_test2) into male and female cohorts\n",
        "    #------YOUR CODE HERE--------\n",
        "    \n",
        "\n",
        "    #------YOUR CODE ENDS--------\n",
        "\n",
        "    # Create, fit, and evaluate your model\n",
        "    # You may use evaluate_model from before\n",
        "    #------YOUR CODE HERE--------\n",
        "    \n",
        "    \n",
        "    #------YOUR CODE ENDS--------"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3.7.10 ('base')",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.10"
    },
    "orig_nbformat": 4,
    "vscode": {
      "interpreter": {
        "hash": "9a8029a51b57fe2d11fbd3b82c1cfa41a63a260330aa68039b758425bc8dfc97"
      }
    },
    "colab": {
      "provenance": [],
      "collapsed_sections": []
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}