when_to_show/effect_of_latent_state.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from requests import session\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import json\n",
    "import copy\n",
    "import numpy as np, scipy.stats as st\n",
    "import json\n",
    "from tqdm import tqdm\n",
    "import pickle\n",
    "import logging\n",
    "import  argparse\n",
    "import math\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import f1_score\n",
    "from itertools import chain\n",
    "import xgboost as xg\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import mean_squared_error, r2_score, mean_absolute_error\n",
    "logging.basicConfig(level=logging.INFO)\n",
    "from sklearn import preprocessing\n",
    "import matplotlib\n",
    "from re import S\n",
    "from scipy.stats.stats import pearsonr\n",
    "from sklearn.calibration import calibration_curve\n",
    "import matplotlib\n",
    "from sklearn.metrics import roc_auc_score\n",
    "import random\n",
    "import sys\n",
    "import os\n",
    "# sys import ..\n",
    "sys.path.append(\"../\")\n",
    "from data_split import *\n",
    "from metrics_get import *\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = 'featureframe_user_study.pkl'\n",
    "data = pickle.load(open(path, 'rb'))\n",
    "splitbyusers = True\n",
    "output_path = '.'\n",
    "test_percentage = 0.001\n",
    "val_percentage = 0.001\n",
    "REMOVE_S_AND_R = True # remove shown and replay\n",
    "feature_dict = {'Measurements: compCharLen, confidence, documentLength, numLines, numTokens, promptCharLen, promptEndPos, quantile': 0,\n",
    "'edit percentage': 1, 'time_in_state': 2, 'session_features':3, 'suggestion_label':4, 'prompt_label':5,\n",
    "'suggestion_embedding':6, 'prompt_embedding':7, 'suggestion_text_features':8, 'prompt_text_features':9, 'user_id':10, 'statename':11, 'labeled_state': 12}\n",
    "# split into train and test\n",
    "SEQUENCE_MODE = False # keep session as a sequence or split it into events\n",
    "SPLIT_BY_USER = bool(splitbyusers) # otherwise split by session uniformly\n",
    "ADD_PREVIOUS_STATES = True\n",
    "PREDICT_ACTION = True # Otherwise predict time in state\n",
    "NORMALIZE_DATA = False # normalize data\n",
    "\n",
    "previous_states_to_keep = 5\n",
    "if not PREDICT_ACTION and SPLIT_BY_USER:\n",
    "    raise ValueError('Cannot predict time and split by user')\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_TRIALS = 21 # number of programmers\n",
    "so_far_count = 0\n",
    "aucs_with = []\n",
    "aucs_without = []\n",
    "accs_with = []\n",
    "accs_without = []\n",
    "test_set_seenns = []\n",
    "while so_far_count < MAX_TRIALS:\n",
    "    random_number = np.random.randint(0, 100000)\n",
    "\n",
    "    # without latent state\n",
    "    features_to_keep = np.array([0,3,4,5,8,9, 10])\n",
    "    label_index = np.array([11])\n",
    "    df_observations_features, df_observations_labels = get_features_labels_user_study(path, features_to_keep, label_index, REMOVE_S_AND_R)\n",
    "\n",
    "    X_train, X_test, X_val, y_train, y_test, y_val = process_data_user_study(df_observations_features, df_observations_labels,\n",
    "    REMOVE_S_AND_R, SEQUENCE_MODE, SPLIT_BY_USER, ADD_PREVIOUS_STATES, PREDICT_ACTION, NORMALIZE_DATA,\n",
    "    test_percentage, val_percentage, previous_states_to_keep, random_number)\n",
    "    # check if X_test array is already been seen\n",
    "    for seen in test_set_seenns:\n",
    "        if np.array_equal(X_test, seen):\n",
    "            continue\n",
    "        \n",
    "    test_set_seenns.append(X_test)\n",
    "    so_far_count += 1\n",
    "    model = XGBClassifier()\n",
    "    model.fit(X_train, y_train)\n",
    "    y_pred = model.predict(X_test)\n",
    "    accuracy = accuracy_score(y_test, y_pred)\n",
    "    y_pred_proba = model.predict_proba(X_test)\n",
    "    y_pred_proba = y_pred_proba[:,1]\n",
    "    auc = roc_auc_score(y_test, y_pred_proba)\n",
    "    aucs_without.append(auc)\n",
    "    accs_without.append(accuracy)\n",
    "    # with latent state\n",
    "    features_to_keep = np.array([0,3,4,5,8,9, 10,12])\n",
    "    label_index = np.array([11])\n",
    "    df_observations_features, df_observations_labels = get_features_labels_user_study(path, features_to_keep, label_index, REMOVE_S_AND_R)\n",
    "\n",
    "    X_train, X_test, X_val, y_train, y_test, y_val = process_data_user_study(df_observations_features, df_observations_labels,\n",
    "    REMOVE_S_AND_R, SEQUENCE_MODE, SPLIT_BY_USER, ADD_PREVIOUS_STATES, PREDICT_ACTION, NORMALIZE_DATA,\n",
    "    test_percentage, val_percentage, previous_states_to_keep, random_number)\n",
    "    model = XGBClassifier()\n",
    "    model.fit(X_train, y_train)\n",
    "    y_pred = model.predict(X_test)\n",
    "    accuracy = accuracy_score(y_test, y_pred)\n",
    "    y_pred_proba = model.predict_proba(X_test)\n",
    "    y_pred_proba = y_pred_proba[:,1]\n",
    "    auc = roc_auc_score(y_test, y_pred_proba)\n",
    "    aucs_with.append(auc)\n",
    "    accs_with.append(accuracy)\n",
    "    print(f'aucs_with: {aucs_with}')\n",
    "    print(f'aucs_without: {aucs_without}')\n",
    "    print(f'accs_with: {accs_with}')\n",
    "    print(f'accs_without: {accs_without}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print mean and standard error\n",
    "print(f'WITH latent state model: accuracy = {np.mean(accs_with)} +- {np.std(accs_with)/np.sqrt(len(accs_with))}')\n",
    "print(f'WITHOUT latent state model: accuracy = {np.mean(accs_without)} +- {np.std(accs_without)/np.sqrt(len(accs_without))}')\n",
    "print(f'WITH latent state model auc = {np.mean(aucs_with)} +- {np.std(aucs_with)/np.sqrt(len(aucs_with))}')\n",
    "print(f'WITHOUT latent state model auc = {np.mean(aucs_without)} +- {np.std(aucs_without)/np.sqrt(len(aucs_without))}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# perform paired t-test on accuracy and aucs for with and without\n",
    "from scipy.stats import ttest_rel\n",
    "\n",
    "test_auc = ttest_rel(aucs_with, aucs_without)\n",
    "test_acc = ttest_rel(accs_with, accs_without)\n",
    "print(\"T-test auc\")\n",
    "print(test_auc)\n",
    "print(\"T-test acc\")\n",
    "print(test_acc)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "hussein",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}