code/V2/04_5_model_test.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Final Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "X_train_transformed = pd.read_csv('../../data/X_train_transformed.csv')\n",
    "X_test_transformed = pd.read_csv('../../data/X_test_transformed.csv')\n",
    "#X_val_transformed = pd.read_csv('../../data/X_val_transformed.csv')\n",
    "#X_train_pca = pd.read_csv('../../data/X_train_pca.csv')\n",
    "#X_test_pca = pd.read_csv('../../data/X_test_pca.csv')\n",
    "#X_val_pca = pd.read_csv('../../data/X_val_pca.csv')\n",
    "\n",
    "\n",
    "y_train = pd.read_csv('../../data/y_train.csv')\n",
    "y_test = pd.read_csv('../../data/y_test.csv')\n",
    "#y_val = pd.read_csv('../../data/y_val.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [],
   "source": [
    "# copy the data\n",
    "y_train_binary = y_train.copy()\n",
    "\n",
    "# map values to 'intervene' and 'no_intervene'\n",
    "y_train_binary.replace({'Pass': 'no_intervene', 'Distinction': 'no_intervene',\n",
    "                        'Withdrawn': 'intervene', 'Fail': 'intervene'}, inplace=True)\n",
    "\n",
    "y_test_binary = y_test.copy()\n",
    "y_test_binary.replace({'Pass': 'no_intervene', 'Distinction': 'no_intervene',\n",
    "                        'Withdrawn': 'intervene', 'Fail': 'intervene'}, inplace=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import zipfile\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [],
   "source": [
    "zipfile_ou = '../../data/anonymisedData.zip'\n",
    "#prediction_point = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def load_data(zip_file_path):\n",
    "    '''Loads the data from the Open University Learning Analytics dataset zip file.'''\n",
    "    \n",
    "    with zipfile.ZipFile(zip_file_path, 'r') as zip_file:\n",
    "        registrations = pd.read_csv(zip_file.open('studentRegistration.csv'))\n",
    "        courses = pd.read_csv(zip_file.open('courses.csv'))\n",
    "        students = pd.read_csv(zip_file.open('studentInfo.csv'))\n",
    "        student_vle = pd.read_csv(zip_file.open('studentVle.csv'))\n",
    "        vle = pd.read_csv(zip_file.open('vle.csv'))\n",
    "        student_assessments = pd.read_csv(zip_file.open('studentAssessment.csv'))\n",
    "        assessments = pd.read_csv(zip_file.open('assessments.csv'))\n",
    "    \n",
    "    return registrations, courses, students, student_vle, vle, student_assessments, assessments\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [],
   "source": [
    "registrations, courses, students, student_vle, vle, student_assessments, assessments = load_data(zipfile_ou)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [],
   "source": [
    "prediction_point = 150"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [],
   "source": [
    "def student_data(students, courses, registrations, prediction_point=None):\n",
    "    '''Returns a dataframe of student data based on `students`: \n",
    "\n",
    "    prediction_point is an integer representing the number of days from the start of the course for which you want to predict the outcome.\n",
    "\n",
    "    default is max(student_regist['module_presentation_length']) - that is, the last da of the lengthiest course\n",
    "\n",
    "    - merge students and courses\n",
    "    - merge registrations\n",
    "    - drop missing value rows (date_registration, imd_band)\n",
    "    - drop students who unregistered before registering\n",
    "    - fills missing date_unregistration with module_presentation_length - that is, assumes they completed the course\n",
    "    - removes students who withdrew or failed before prediction point\n",
    "    '''\n",
    "\n",
    "    # merge students and courses\n",
    "    student_regist = pd.merge(students, courses, on=['code_module', 'code_presentation'], validate='many_to_one')\n",
    "\n",
    "    # merge registrations\n",
    "    student_regist = pd.merge(student_regist, registrations, on=['code_module', 'code_presentation', 'id_student'], how = 'left', validate='1:1')\n",
    "\n",
    "    # drop missing value rows (date_registration, imd_band)\n",
    "    student_regist.dropna(subset=['date_registration', 'imd_band'], inplace=True)\n",
    "\n",
    "    # set default status to 'keep'\n",
    "    student_regist['status'] = 'keep'\n",
    "\n",
    "    # prediction point must be less than course length, integer, and greater than 0\n",
    "    if prediction_point is not None:\n",
    "        assert isinstance(prediction_point, int) and prediction_point > 0 and prediction_point <= (max(student_regist['module_presentation_length']) + 7), \"Error: Invalid prediction point.\\n\\nPlease provide an integer value greater than 0 and less than or equal to the maximum course length.\\n\\nThis is the number of days from the start of the course for which you want to predict the outcome.\"\n",
    "    \n",
    "        # withdrawn or failed before prediction point - remove\n",
    "        withdrawn_fail_condition = (student_regist['final_result'].isin(['Withdrawn', 'Fail'])) & (student_regist['date_unregistration'] <= prediction_point)\n",
    "        student_regist.loc[withdrawn_fail_condition, 'status'] = 'remove_outcome_known'\n",
    "    \n",
    "        # unregister after prediction point - keep\n",
    "        unregister_after_condition = student_regist['date_unregistration'] > prediction_point\n",
    "        student_regist.loc[unregister_after_condition, 'status'] = 'keep'\n",
    "    \n",
    "        # if no unregistration date - keep\n",
    "        no_unregistration_condition = student_regist['date_unregistration'].isna()\n",
    "        student_regist.loc[no_unregistration_condition, 'status'] = 'keep'\n",
    "    \n",
    "        # query case\n",
    "        student_regist.loc[~(withdrawn_fail_condition | unregister_after_condition | no_unregistration_condition), 'status'] = 'query'\n",
    "\n",
    "\n",
    "    # rows which need investigation\n",
    "    query_rows = student_regist[student_regist['status'] == 'query'] | student_regist[student_regist['status'].isna()]\n",
    "\n",
    "    # print rows which need investigation\n",
    "    if not query_rows.empty:\n",
    "        print(\"The following rows need investigation.  They are excluded from the following analysis: \\n\")\n",
    "        print(query_rows)\n",
    "        student_regist = student_regist[~student_regist.isin(query_rows)].dropna()\n",
    "\n",
    "    # replace missing date_unreg with module_presentation_length\n",
    "    student_regist['date_unregistration'] = student_regist['date_unregistration'].fillna(student_regist['module_presentation_length'])\n",
    "\n",
    "    # drop students who unregistered before starting\n",
    "    student_regist = student_regist[student_regist['date_unregistration'] >= student_regist['date_registration']]\n",
    "\n",
    "    # remove rows from final student df which are not needed\n",
    "    model_final = student_regist[student_regist['status'] != 'remove_outcome_known']\n",
    "   \n",
    "\n",
    "    return model_final\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_final = student_data(students, courses, registrations, prediction_point)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 23122 entries, 0 to 32592\n",
      "Data columns (total 16 columns):\n",
      " #   Column                      Non-Null Count  Dtype  \n",
      "---  ------                      --------------  -----  \n",
      " 0   code_module                 23122 non-null  object \n",
      " 1   code_presentation           23122 non-null  object \n",
      " 2   id_student                  23122 non-null  int64  \n",
      " 3   gender                      23122 non-null  object \n",
      " 4   region                      23122 non-null  object \n",
      " 5   highest_education           23122 non-null  object \n",
      " 6   imd_band                    23122 non-null  object \n",
      " 7   age_band                    23122 non-null  object \n",
      " 8   num_of_prev_attempts        23122 non-null  int64  \n",
      " 9   studied_credits             23122 non-null  int64  \n",
      " 10  disability                  23122 non-null  object \n",
      " 11  final_result                23122 non-null  object \n",
      " 12  module_presentation_length  23122 non-null  int64  \n",
      " 13  date_registration           23122 non-null  float64\n",
      " 14  date_unregistration         23122 non-null  float64\n",
      " 15  status                      23122 non-null  object \n",
      "dtypes: float64(2), int64(4), object(10)\n",
      "memory usage: 3.0+ MB\n"
     ]
    }
   ],
   "source": [
    "model_final.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_assessments(student_assessments, assessments, courses, model_final, prediction_point=None):\n",
    "    '''Returns updated model_final dataframe with student assessment data added:\n",
    "\n",
    "    prediction_point is an integer representing the number of days from the start of the course for which you want to predict the outcome.\n",
    "\n",
    "    default is max(student_regist['module_presentation_length']) - that is, the last da of the lengthiest course\n",
    "\n",
    "    - merge 'assessments' and 'courses' and 'student_assessments'\n",
    "    - populate missing 'date' values with course final week - as suggested in literature\n",
    "    - remove students from 'model_final' who have no score for an assessment\n",
    "    - remove students with odd assessment dates (before registration, after unregistration, or very far into future)\n",
    "    - remove students who withdrew or failed before prediction point\n",
    "    - calculate new features - average score, submission date distance, proportion of assessments submitted\n",
    "    '''\n",
    "    # merge 'assessments' and 'courses' on 'code_module' and 'code_presentation'\n",
    "    course_assess = pd.merge(assessments,courses, on=['code_module', 'code_presentation'], how='left')\n",
    "\n",
    "    # fill in the missing 'date' values with course final week (as per literature)\n",
    "    value_to_fill = course_assess['module_presentation_length'] - 3\n",
    "    course_assess['date'] = course_assess['date'].fillna(value_to_fill)\n",
    "\n",
    "    # merge student_assessments with course_assess\n",
    "    stu_assess = pd.merge(student_assessments, course_assess, on=['id_assessment'], how='left')\n",
    "\n",
    "    # drop no score for an assessment\n",
    "    missing_score_rows = stu_assess[stu_assess['score'].isna()]\n",
    "\n",
    "    if not missing_score_rows.empty:\n",
    "        print(\"\\n\\nThe following students have missing 'scores'. Their are excluded from the analysis: \\n\")\n",
    "        print(missing_score_rows)\n",
    "\n",
    "        # drop rows with missing score\n",
    "        stu_assess.dropna(subset=['score'], inplace=True)\n",
    "\n",
    "    # remove students date_submitted values 21 days after course end date\n",
    "    max_module_length = stu_assess['module_presentation_length'].max() +21\n",
    "    \n",
    "    stu_assess = stu_assess[stu_assess['date_submitted'] <= max_module_length]\n",
    "\n",
    "\n",
    "    \n",
    "    # if prediction_point is None\n",
    "    if prediction_point is None:\n",
    "        # no data reduction\n",
    "        model_student_assess = stu_assess\n",
    "        model_course_assess = course_assess\n",
    "    else:\n",
    "        # prediction point must be less than course length, integer, and greater than 0\n",
    "        assert isinstance(prediction_point, int) and prediction_point > 0 and prediction_point <= (max(courses['module_presentation_length'])+7), \"Error: Invalid prediction point.\\n\\nPlease provide an integer value greater than 0 and less than or equal to the maximum course length.\\n\\nThis is the number of days from the start of the course for which you want to predict the outcome.\"\n",
    "\n",
    "        # reduce data by prediction point\n",
    "        model_student_assess = stu_assess[stu_assess['date'] <= prediction_point]\n",
    "        model_course_assess = course_assess[course_assess['date'] <= prediction_point]\n",
    "\n",
    "    # expected assessment details\n",
    "    expected_ass = model_course_assess.groupby(['code_module', 'code_presentation'])['id_assessment'].count().reset_index()\n",
    "    expected_ass = expected_ass.rename(columns={'id_assessment': 'exp_sub_count'})\n",
    "    date_sum = model_course_assess.groupby(['code_module', 'code_presentation'])['date'].sum().reset_index()\n",
    "    expected_ass = expected_ass.merge(date_sum, on=['code_module', 'code_presentation'], how='left')\n",
    "    expected_ass = expected_ass.rename(columns={'date': 'exp_sub_date_sum'})\n",
    "\n",
    "    # summarise students' assessments by module_presentation\n",
    "    student_assessment_summary = model_student_assess.groupby(['id_student', 'code_module', 'code_presentation']).agg(\n",
    "        count_id_assessment=('id_assessment', 'count'),\n",
    "        sum_score=('score', 'sum'),\n",
    "        sum_date=('date', 'sum')\n",
    "    ).reset_index()\n",
    "\n",
    "    # merge student_assessment_summary and expected_ass on code_module and code_presentation\n",
    "    merged_assess_summary = student_assessment_summary.merge(expected_ass, on=['code_module', 'code_presentation'])\n",
    "\n",
    "    # calculate the new features\n",
    "    merged_assess_summary['prop_submissions'] = merged_assess_summary['count_id_assessment'] / merged_assess_summary['exp_sub_count']\n",
    "    merged_assess_summary['avg_score'] = merged_assess_summary['sum_score'] / merged_assess_summary['exp_sub_count']\n",
    "    merged_assess_summary['submission_distance'] = merged_assess_summary['exp_sub_date_sum'] - merged_assess_summary['sum_date']\n",
    "\n",
    "    # merge with 'model_final' - inner (loses no engagement students)\n",
    "    #model_final = model_final.merge(merged_assess_summary, on=['id_student', 'code_module', 'code_presentation'], how='inner')\n",
    "\n",
    "    # merge with 'model_final' - left (keeps students with no engagement - NaN need updating to 0 in tidy query)\n",
    "    model_final = model_final.merge(merged_assess_summary, on=['id_student', 'code_module', 'code_presentation'], how='left')\n",
    "\n",
    "  \n",
    "\n",
    "    return model_final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "The following students have missing 'scores'. Their are excluded from the analysis: \n",
      "\n",
      "        id_assessment  id_student  date_submitted  is_banked  score  \\\n",
      "215              1752      721259              22          0    NaN   \n",
      "937              1754      260355             127          0    NaN   \n",
      "2364             1760     2606802             180          0    NaN   \n",
      "3358            14984      186780              77          0    NaN   \n",
      "3914            14984      531205              26          0    NaN   \n",
      "...               ...         ...             ...        ...    ...   \n",
      "148929          34903      582670             241          0    NaN   \n",
      "159251          37415      610738              87          0    NaN   \n",
      "166390          37427      631786             221          0    NaN   \n",
      "169725          37435      648110              62          0    NaN   \n",
      "170103          37435      480914              49          0    NaN   \n",
      "\n",
      "       code_module code_presentation assessment_type   date  weight  \\\n",
      "215            AAA             2013J             TMA   19.0    10.0   \n",
      "937            AAA             2013J             TMA  117.0    20.0   \n",
      "2364           AAA             2014J             TMA  117.0    20.0   \n",
      "3358           BBB             2013B             TMA   19.0     5.0   \n",
      "3914           BBB             2013B             TMA   19.0     5.0   \n",
      "...            ...               ...             ...    ...     ...   \n",
      "148929         FFF             2014J             TMA  199.0    25.0   \n",
      "159251         GGG             2013J             TMA   61.0     0.0   \n",
      "166390         GGG             2014B             TMA  166.0     0.0   \n",
      "169725         GGG             2014J             TMA   61.0     0.0   \n",
      "170103         GGG             2014J             TMA   61.0     0.0   \n",
      "\n",
      "        module_presentation_length  \n",
      "215                            268  \n",
      "937                            268  \n",
      "2364                           269  \n",
      "3358                           240  \n",
      "3914                           240  \n",
      "...                            ...  \n",
      "148929                         269  \n",
      "159251                         261  \n",
      "166390                         241  \n",
      "169725                         269  \n",
      "170103                         269  \n",
      "\n",
      "[173 rows x 11 columns]\n"
     ]
    }
   ],
   "source": [
    "model_final = add_assessments(student_assessments, assessments, courses, model_final, prediction_point)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 23122 entries, 0 to 23121\n",
      "Data columns (total 24 columns):\n",
      " #   Column                      Non-Null Count  Dtype  \n",
      "---  ------                      --------------  -----  \n",
      " 0   code_module                 23122 non-null  object \n",
      " 1   code_presentation           23122 non-null  object \n",
      " 2   id_student                  23122 non-null  int64  \n",
      " 3   gender                      23122 non-null  object \n",
      " 4   region                      23122 non-null  object \n",
      " 5   highest_education           23122 non-null  object \n",
      " 6   imd_band                    23122 non-null  object \n",
      " 7   age_band                    23122 non-null  object \n",
      " 8   num_of_prev_attempts        23122 non-null  int64  \n",
      " 9   studied_credits             23122 non-null  int64  \n",
      " 10  disability                  23122 non-null  object \n",
      " 11  final_result                23122 non-null  object \n",
      " 12  module_presentation_length  23122 non-null  int64  \n",
      " 13  date_registration           23122 non-null  float64\n",
      " 14  date_unregistration         23122 non-null  float64\n",
      " 15  status                      23122 non-null  object \n",
      " 16  count_id_assessment         21660 non-null  float64\n",
      " 17  sum_score                   21660 non-null  float64\n",
      " 18  sum_date                    21660 non-null  float64\n",
      " 19  exp_sub_count               21660 non-null  float64\n",
      " 20  exp_sub_date_sum            21660 non-null  float64\n",
      " 21  prop_submissions            21660 non-null  float64\n",
      " 22  avg_score                   21660 non-null  float64\n",
      " 23  submission_distance         21660 non-null  float64\n",
      "dtypes: float64(10), int64(4), object(10)\n",
      "memory usage: 4.4+ MB\n"
     ]
    }
   ],
   "source": [
    "model_final.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_vle_data(model_final, student_vle, vle, courses, prediction_point=None):\n",
    "    '''Returns updated model_final dataframe with vle data:\n",
    "\n",
    "    prediction_point is an integer representing the number of days from the start of the course for which you want to predict the outcome.\n",
    "\n",
    "    default is max(student_regist['module_presentation_length']) - that is, the last da of the lengthiest course\n",
    "\n",
    "    - merge 'vle' and 'courses' and 'student_vle'\n",
    "    - remove columns (week_from, week_to)\n",
    "    - filter the rows where 'date' <= 'prediction_point'\n",
    "    - create new features: vle_activity_count, student vle type count, student total clicks, student days active (engaged)\n",
    "    \n",
    "    '''\n",
    "    # merge 'vle' and 'courses' on 'code_module' and 'code_presentation'\n",
    "    course_vle = vle.merge(courses, on=['code_module', 'code_presentation'], how='left').drop(['week_from', 'week_to'], axis=1)\n",
    "\n",
    "    # merge vle with student_vle\n",
    "    all_stu_vle = pd.merge(student_vle, course_vle, on=['id_site', 'code_module', 'code_presentation'], how='left')\n",
    "\n",
    "        \n",
    "    # if prediction_point is None\n",
    "    if prediction_point is None:\n",
    "        # no filtering\n",
    "        all_stu_vle = all_stu_vle\n",
    "    else:\n",
    "        # prediction point must be less than or equal to the maximum date\n",
    "        assert isinstance(prediction_point, int) and prediction_point <= max(all_stu_vle['date']), \"Error: Invalid prediction point.\\n\\nPlease provide an integer value less than or equal to the maximum date.\\n\\nThis is the cutoff date for filtering the rows.\"\n",
    "\n",
    "        # filter the rows where 'date' <= 'prediction_point'\n",
    "        all_stu_vle = all_stu_vle[all_stu_vle['date'] <= prediction_point]\n",
    "\n",
    "\n",
    "    # filter the rows where 'date' is greater than 'module_presentation_length'\n",
    "    vle_after_done = all_stu_vle[all_stu_vle['date'] > all_stu_vle['module_presentation_length']]\n",
    "\n",
    "    if not vle_after_done.empty:\n",
    "        print(\"The following rows need investigation. They are excluded from the following analysis: \\n\")\n",
    "        print(vle_after_done)\n",
    "\n",
    "        # match rows based on 'code_module', 'code_presentation', and 'id_student'\n",
    "        matching_rows = model_final[model_final[['code_module', 'code_presentation', 'id_student']].isin(vle_after_done).all(axis=1)]\n",
    "\n",
    "        # remove the matching rows from 'model_final'\n",
    "        model_final = model_final[~model_final.index.isin(matching_rows.index)]\n",
    "\n",
    "    # aggregations for each column\n",
    "    aggregations = {\n",
    "        'id_site': 'count',\n",
    "        'activity_type': 'nunique',\n",
    "        'sum_click': 'sum',\n",
    "        'date': lambda x: x.nunique()\n",
    "    }\n",
    "\n",
    "    # group and apply the aggregations\n",
    "    grouped_stu_vle = all_stu_vle.groupby(['code_module', 'code_presentation', 'id_student']).agg(aggregations).reset_index()\n",
    "\n",
    "    # rename the columns\n",
    "    grouped_stu_vle.rename(columns={\n",
    "        'id_site': 'stu_activity_count',\n",
    "        'activity_type': 'stu_activity_type_count',\n",
    "        'sum_click': 'stu_total_clicks',\n",
    "        'date': 'stu_days_active'\n",
    "    }, inplace=True)\n",
    "\n",
    "    # number of vle types per module_presentation\n",
    "    group_vle = vle.groupby(['code_module', 'code_presentation'])['activity_type'].nunique().reset_index()\n",
    "    group_vle.rename(columns={'activity_type': 'mod_pres_vle_type_count'}, inplace=True)\n",
    "\n",
    "    # merge 'grouped_stu_vle' and 'group_vle' on 'code_module' and 'code_presentation'\n",
    "    merged_vle_summary = grouped_stu_vle.merge(group_vle, on=['code_module', 'code_presentation'], how='left')\n",
    "\n",
    "    # merge with 'model_final' - inner join - loses students without vle engagement\n",
    "    #model_final = model_final.merge(merged_vle_summary, on=['id_student', 'code_module', 'code_presentation'], how='inner')\n",
    "\n",
    "    # merge with 'model_final' - left join - keeps students without vle engagement, NaN nees updating to 0 in tidy function\n",
    "    model_final = model_final.merge(merged_vle_summary, on=['id_student', 'code_module', 'code_presentation'], how='left')\n",
    "\n",
    "    return model_final\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_final = add_vle_data(model_final, student_vle, vle, courses, prediction_point)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 23122 entries, 0 to 23121\n",
      "Data columns (total 29 columns):\n",
      " #   Column                      Non-Null Count  Dtype  \n",
      "---  ------                      --------------  -----  \n",
      " 0   code_module                 23122 non-null  object \n",
      " 1   code_presentation           23122 non-null  object \n",
      " 2   id_student                  23122 non-null  int64  \n",
      " 3   gender                      23122 non-null  object \n",
      " 4   region                      23122 non-null  object \n",
      " 5   highest_education           23122 non-null  object \n",
      " 6   imd_band                    23122 non-null  object \n",
      " 7   age_band                    23122 non-null  object \n",
      " 8   num_of_prev_attempts        23122 non-null  int64  \n",
      " 9   studied_credits             23122 non-null  int64  \n",
      " 10  disability                  23122 non-null  object \n",
      " 11  final_result                23122 non-null  object \n",
      " 12  module_presentation_length  23122 non-null  int64  \n",
      " 13  date_registration           23122 non-null  float64\n",
      " 14  date_unregistration         23122 non-null  float64\n",
      " 15  status                      23122 non-null  object \n",
      " 16  count_id_assessment         21660 non-null  float64\n",
      " 17  sum_score                   21660 non-null  float64\n",
      " 18  sum_date                    21660 non-null  float64\n",
      " 19  exp_sub_count               21660 non-null  float64\n",
      " 20  exp_sub_date_sum            21660 non-null  float64\n",
      " 21  prop_submissions            21660 non-null  float64\n",
      " 22  avg_score                   21660 non-null  float64\n",
      " 23  submission_distance         21660 non-null  float64\n",
      " 24  stu_activity_count          22708 non-null  float64\n",
      " 25  stu_activity_type_count     22708 non-null  float64\n",
      " 26  stu_total_clicks            22708 non-null  float64\n",
      " 27  stu_days_active             22708 non-null  float64\n",
      " 28  mod_pres_vle_type_count     22708 non-null  float64\n",
      "dtypes: float64(15), int64(4), object(10)\n",
      "memory usage: 5.3+ MB\n"
     ]
    }
   ],
   "source": [
    "model_final.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tidy_up_model_final_binary(model_final):\n",
    "    '''Returns updated model_final dataframe:\n",
    "    - deleted unnecessary columns\n",
    "    - reordered columns\n",
    "    - added subject, year, month columns\n",
    "    \n",
    "    '''\n",
    "    \n",
    "    # module subject mapping\n",
    "    code_module_mapping = {\n",
    "        'AAA': 'SocSci',\n",
    "        'BBB': 'SocSci',\n",
    "        'GGG': 'SocSci',\n",
    "        'CCC': 'Stem',\n",
    "        'DDD': 'Stem',\n",
    "        'EEE': 'Stem',\n",
    "        'FFF': 'Stem'\n",
    "    }\n",
    "    model_final['subject'] = model_final['code_module'].map(code_module_mapping)\n",
    "\n",
    "    # move 'final_result' to the last column\n",
    "    final_result_column = model_final.pop('final_result')\n",
    "    model_final['final_result'] = final_result_column\n",
    "\n",
    "    # drop columns\n",
    "    model_final.drop(columns=['code_module', 'code_presentation', 'id_student','gender', 'region', 'highest_education', 'imd_band', 'age_band', 'disability', 'date_registration', 'date_unregistration', 'count_id_assessment', 'sum_score', 'sum_date', 'exp_sub_count', 'exp_sub_date_sum', 'status', 'mod_pres_vle_type_count', 'module_presentation_length'], inplace=True)\n",
    "    \n",
    "    # replace NaN values with 0 - these are students who did not engage (vle, assessment)\n",
    "    model_final.fillna(0, inplace=True)\n",
    "\n",
    "    \n",
    "\n",
    "    return model_final\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>num_of_prev_attempts</th>\n",
       "      <th>studied_credits</th>\n",
       "      <th>prop_submissions</th>\n",
       "      <th>avg_score</th>\n",
       "      <th>submission_distance</th>\n",
       "      <th>stu_activity_count</th>\n",
       "      <th>stu_activity_type_count</th>\n",
       "      <th>stu_total_clicks</th>\n",
       "      <th>stu_days_active</th>\n",
       "      <th>subject</th>\n",
       "      <th>final_result</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>240</td>\n",
       "      <td>1.0</td>\n",
       "      <td>81.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>133.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>710.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>SocSci</td>\n",
       "      <td>Pass</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>60</td>\n",
       "      <td>1.0</td>\n",
       "      <td>69.333333</td>\n",
       "      <td>0.0</td>\n",
       "      <td>313.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1010.0</td>\n",
       "      <td>56.0</td>\n",
       "      <td>SocSci</td>\n",
       "      <td>Pass</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>60</td>\n",
       "      <td>1.0</td>\n",
       "      <td>72.333333</td>\n",
       "      <td>0.0</td>\n",
       "      <td>445.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1499.0</td>\n",
       "      <td>83.0</td>\n",
       "      <td>SocSci</td>\n",
       "      <td>Pass</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>60</td>\n",
       "      <td>1.0</td>\n",
       "      <td>54.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>242.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>825.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>SocSci</td>\n",
       "      <td>Pass</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>60</td>\n",
       "      <td>1.0</td>\n",
       "      <td>74.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>404.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1391.0</td>\n",
       "      <td>81.0</td>\n",
       "      <td>SocSci</td>\n",
       "      <td>Pass</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23117</th>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>1.0</td>\n",
       "      <td>82.500000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>83.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>182.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>SocSci</td>\n",
       "      <td>Distinction</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23118</th>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>41.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>SocSci</td>\n",
       "      <td>Fail</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23119</th>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>1.0</td>\n",
       "      <td>80.500000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>93.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>304.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>SocSci</td>\n",
       "      <td>Distinction</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23120</th>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>1.0</td>\n",
       "      <td>70.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>61.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>163.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>SocSci</td>\n",
       "      <td>Pass</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23121</th>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>1.0</td>\n",
       "      <td>80.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>152.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>488.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>SocSci</td>\n",
       "      <td>Distinction</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>23122 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       num_of_prev_attempts  studied_credits  prop_submissions  avg_score  \\\n",
       "0                         0              240               1.0  81.000000   \n",
       "1                         0               60               1.0  69.333333   \n",
       "2                         0               60               1.0  72.333333   \n",
       "3                         0               60               1.0  54.000000   \n",
       "4                         0               60               1.0  74.000000   \n",
       "...                     ...              ...               ...        ...   \n",
       "23117                     0               30               1.0  82.500000   \n",
       "23118                     0               30               0.0   0.000000   \n",
       "23119                     0               30               1.0  80.500000   \n",
       "23120                     0               30               1.0  70.000000   \n",
       "23121                     0               30               1.0  80.000000   \n",
       "\n",
       "       submission_distance  stu_activity_count  stu_activity_type_count  \\\n",
       "0                      0.0               133.0                      6.0   \n",
       "1                      0.0               313.0                      7.0   \n",
       "2                      0.0               445.0                      8.0   \n",
       "3                      0.0               242.0                      7.0   \n",
       "4                      0.0               404.0                      7.0   \n",
       "...                    ...                 ...                      ...   \n",
       "23117                  0.0                83.0                      6.0   \n",
       "23118                  0.0                19.0                      4.0   \n",
       "23119                  0.0                93.0                      7.0   \n",
       "23120                  0.0                61.0                      6.0   \n",
       "23121                  0.0               152.0                      6.0   \n",
       "\n",
       "       stu_total_clicks  stu_days_active subject final_result  \n",
       "0                 710.0             29.0  SocSci         Pass  \n",
       "1                1010.0             56.0  SocSci         Pass  \n",
       "2                1499.0             83.0  SocSci         Pass  \n",
       "3                 825.0             50.0  SocSci         Pass  \n",
       "4                1391.0             81.0  SocSci         Pass  \n",
       "...                 ...              ...     ...          ...  \n",
       "23117             182.0             17.0  SocSci  Distinction  \n",
       "23118              41.0              5.0  SocSci         Fail  \n",
       "23119             304.0             18.0  SocSci  Distinction  \n",
       "23120             163.0             13.0  SocSci         Pass  \n",
       "23121             488.0             29.0  SocSci  Distinction  \n",
       "\n",
       "[23122 rows x 11 columns]"
      ]
     },
     "execution_count": 220,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tidy_up_model_final_binary(model_final)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 23122 entries, 0 to 23121\n",
      "Data columns (total 11 columns):\n",
      " #   Column                   Non-Null Count  Dtype  \n",
      "---  ------                   --------------  -----  \n",
      " 0   num_of_prev_attempts     23122 non-null  int64  \n",
      " 1   studied_credits          23122 non-null  int64  \n",
      " 2   prop_submissions         23122 non-null  float64\n",
      " 3   avg_score                23122 non-null  float64\n",
      " 4   submission_distance      23122 non-null  float64\n",
      " 5   stu_activity_count       23122 non-null  float64\n",
      " 6   stu_activity_type_count  23122 non-null  float64\n",
      " 7   stu_total_clicks         23122 non-null  float64\n",
      " 8   stu_days_active          23122 non-null  float64\n",
      " 9   subject                  23122 non-null  object \n",
      " 10  final_result             23122 non-null  object \n",
      "dtypes: float64(7), int64(2), object(2)\n",
      "memory usage: 2.1+ MB\n"
     ]
    }
   ],
   "source": [
    "model_final.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# drop target from X, save target to y\n",
    "X = model_final.drop('final_result', axis=1)  \n",
    "y = model_final['final_result']  \n",
    "\n",
    "# split data into train-test and validation sets with stratification\n",
    "X_train_test, X_val, y_train_test, y_val = train_test_split(\n",
    "    X, y, test_size=0.2, stratify=y, random_state=567)\n",
    "\n",
    "# split train-test set into train and test sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    X_train_test, y_train_test, test_size=0.25, stratify=y_train_test, random_state=567)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original Proportions:\n",
      "Pass           0.511591\n",
      "Fail           0.298158\n",
      "Distinction    0.122178\n",
      "Withdrawn      0.068074\n",
      "Name: final_result, dtype: float64\n",
      "\n",
      "Train Set Proportions:\n",
      "Pass           0.511606\n",
      "Fail           0.298155\n",
      "Distinction    0.122189\n",
      "Withdrawn      0.068051\n",
      "Name: final_result, dtype: float64\n",
      "\n",
      "Validation Set Proportions:\n",
      "Pass           0.511568\n",
      "Fail           0.298162\n",
      "Distinction    0.122162\n",
      "Withdrawn      0.068108\n",
      "Name: final_result, dtype: float64\n",
      "\n",
      "Test Set Proportions:\n",
      "Pass           0.511568\n",
      "Fail           0.298162\n",
      "Distinction    0.122162\n",
      "Withdrawn      0.068108\n",
      "Name: final_result, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "# proportions of target variable in original data\n",
    "original_proportions = y.value_counts(normalize=True)\n",
    "\n",
    "# proportions of target variable in train, validation, and test sets\n",
    "train_proportions = y_train.value_counts(normalize=True)\n",
    "val_proportions = y_val.value_counts(normalize=True)\n",
    "test_proportions = y_test.value_counts(normalize=True)\n",
    "\n",
    "# results\n",
    "print(\"Original Proportions:\")\n",
    "print(original_proportions)\n",
    "\n",
    "print(\"\\nTrain Set Proportions:\")\n",
    "print(train_proportions)\n",
    "\n",
    "print(\"\\nValidation Set Proportions:\")\n",
    "print(val_proportions)\n",
    "\n",
    "print(\"\\nTest Set Proportions:\")\n",
    "print(test_proportions)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Missing values in X_train: 0\n",
      "Missing values in X_test: 0\n",
      "Missing values in y_train: 0\n",
      "Missing values in y_test: 0\n",
      "Missing values in X_val: 0\n",
      "Missing values in y_val: 0\n"
     ]
    }
   ],
   "source": [
    "# missing values in X_train, X_test, y_train, y_test\n",
    "missing_values_X_train = X_train.isnull().sum()\n",
    "missing_values_X_test = X_test.isnull().sum()\n",
    "missing_values_y_train = y_train.isnull().sum()\n",
    "missing_values_y_test = y_test.isnull().sum()\n",
    "missing_values_X_val = X_val.isnull().sum()\n",
    "missing_values_y_val = y_val.isnull().sum()\n",
    "\n",
    "\n",
    "# rows with missing values\n",
    "rows_with_missing_X_train = X_train[X_train.isnull().any(axis=1)]\n",
    "rows_with_missing_X_test = X_test[X_test.isnull().any(axis=1)]\n",
    "rows_with_missing_y_train = y_train[y_train.isnull()]\n",
    "rows_with_missing_y_test = y_test[y_test.isnull()]\n",
    "rows_with_missing_X_val = X_val[X_val.isnull().any(axis=1)]\n",
    "rows_with_missing_y_val = y_val[y_val.isnull()]\n",
    "\n",
    "\n",
    "\n",
    "# results\n",
    "print(\"Missing values in X_train:\", len(rows_with_missing_X_train))\n",
    "print(\"Missing values in X_test:\", len(rows_with_missing_X_test))\n",
    "print(\"Missing values in y_train:\", len(rows_with_missing_y_train))\n",
    "print(\"Missing values in y_test:\", len(rows_with_missing_y_test))\n",
    "print(\"Missing values in X_val:\", len(rows_with_missing_X_val))\n",
    "print(\"Missing values in y_val:\", len(rows_with_missing_y_val))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Numeric Columns:\n",
      "['num_of_prev_attempts', 'studied_credits', 'prop_submissions', 'avg_score', 'submission_distance', 'stu_activity_count', 'stu_activity_type_count', 'stu_total_clicks', 'stu_days_active']\n",
      "\n",
      "\n",
      "Non-Numeric Columns:\n",
      "['subject']\n"
     ]
    }
   ],
   "source": [
    "numeric_columns = X_train.select_dtypes(include=['int64', 'float64']).columns.tolist()\n",
    "non_numeric_columns = X_train.select_dtypes(exclude=['int64', 'float64']).columns.tolist()\n",
    "\n",
    "print(\"Numeric Columns:\")\n",
    "print(numeric_columns)\n",
    "print(\"\\n\")\n",
    "print(\"Non-Numeric Columns:\")\n",
    "print(non_numeric_columns)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler, OrdinalEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of X_train_nominal_encoded: (13872, 2)\n",
      "Shape of X_test_nominal_encoded: (4625, 2)\n",
      "Shape of X_val_nominal_encoded: (4625, 2)\n"
     ]
    }
   ],
   "source": [
    "nominal_cols = ['subject']\n",
    "\n",
    "# One-Hot Encoding\n",
    "X_train_nominal_encoded = pd.get_dummies(X_train[nominal_cols])\n",
    "X_test_nominal_encoded = pd.get_dummies(X_test[nominal_cols])\n",
    "X_val_nominal_encoded = pd.get_dummies(X_val[nominal_cols])\n",
    "\n",
    "# reset indices\n",
    "X_train_nominal_encoded.reset_index(drop=True, inplace=True)\n",
    "X_test_nominal_encoded.reset_index(drop=True, inplace=True)\n",
    "X_val_nominal_encoded.reset_index(drop=True, inplace=True)\n",
    "\n",
    "print(\"Shape of X_train_nominal_encoded:\", X_train_nominal_encoded.shape)\n",
    "print(\"Shape of X_test_nominal_encoded:\", X_test_nominal_encoded.shape)\n",
    "print(\"Shape of X_val_nominal_encoded:\", X_val_nominal_encoded.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of X_train_transformed: (13872, 11)\n",
      "Shape of X_test_transformed: (4625, 11)\n",
      "Shape of X_val_transformed: (4625, 11)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# standard Scaling\n",
    "X_train_numeric = X_train[numeric_columns]\n",
    "X_test_numeric = X_test[numeric_columns]\n",
    "X_val_numeric = X_val[numeric_columns]\n",
    "\n",
    "scaler = StandardScaler()\n",
    "X_train_scaled = scaler.fit_transform(X_train_numeric)\n",
    "X_test_scaled = scaler.transform(X_test_numeric)\n",
    "X_val_scaled = scaler.transform(X_val_numeric)\n",
    "\n",
    "# reset indices \n",
    "X_train_scaled_reset = pd.DataFrame(X_train_scaled, columns=numeric_columns).reset_index(drop=True)\n",
    "X_test_scaled_reset = pd.DataFrame(X_test_scaled, columns=numeric_columns).reset_index(drop=True)\n",
    "X_val_scaled_reset = pd.DataFrame(X_val_scaled, columns=numeric_columns).reset_index(drop=True)\n",
    "\n",
    "# concatenate merged nominal dataframes with scaled dataframes\n",
    "X_train_transformed = pd.concat([X_train_nominal_encoded, X_train_scaled_reset], axis=1)\n",
    "X_test_transformed = pd.concat([X_test_nominal_encoded, X_test_scaled_reset], axis=1)\n",
    "X_val_transformed = pd.concat([X_val_nominal_encoded, X_val_scaled_reset], axis=1)\n",
    "\n",
    "# merging all dataframes\n",
    "print(\"Shape of X_train_transformed:\", X_train_transformed.shape)\n",
    "print(\"Shape of X_test_transformed:\", X_test_transformed.shape)\n",
    "print(\"Shape of X_val_transformed:\", X_val_transformed.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 13872 entries, 0 to 13871\n",
      "Data columns (total 11 columns):\n",
      " #   Column                   Non-Null Count  Dtype  \n",
      "---  ------                   --------------  -----  \n",
      " 0   subject_SocSci           13872 non-null  uint8  \n",
      " 1   subject_Stem             13872 non-null  uint8  \n",
      " 2   num_of_prev_attempts     13872 non-null  float64\n",
      " 3   studied_credits          13872 non-null  float64\n",
      " 4   prop_submissions         13872 non-null  float64\n",
      " 5   avg_score                13872 non-null  float64\n",
      " 6   submission_distance      13872 non-null  float64\n",
      " 7   stu_activity_count       13872 non-null  float64\n",
      " 8   stu_activity_type_count  13872 non-null  float64\n",
      " 9   stu_total_clicks         13872 non-null  float64\n",
      " 10  stu_days_active          13872 non-null  float64\n",
      "dtypes: float64(9), uint8(2)\n",
      "memory usage: 1002.6 KB\n"
     ]
    }
   ],
   "source": [
    "X_train_transformed.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {},
   "outputs": [],
   "source": [
    "# copy the data\n",
    "y_train_binary = y_train.copy()\n",
    "\n",
    "# map values to 'intervene' and 'no_intervene'\n",
    "y_train_binary.replace({'Pass': 'no_intervene', 'Distinction': 'no_intervene',\n",
    "                        'Withdrawn': 'intervene', 'Fail': 'intervene'}, inplace=True)\n",
    "\n",
    "y_test_binary = y_test.copy()\n",
    "y_test_binary.replace({'Pass': 'no_intervene', 'Distinction': 'no_intervene',\n",
    "                        'Withdrawn': 'intervene', 'Fail': 'intervene'}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test_binary.to_csv('y_test_binary_150.csv', index=False)\n",
    "X_test_transformed.to_csv('X_test_transformed_150.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.series.Series'>\n"
     ]
    }
   ],
   "source": [
    "print(type(y_test_binary))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n"
     ]
    }
   ],
   "source": [
    "print(type(X_test_transformed))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}