shap_pan.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/kchen/.local/lib/python3.9/site-packages/tensorflow/python/compat/v2_compat.py:101: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "non-resource variables are not supported in the long term\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.metrics import confusion_matrix, roc_auc_score, roc_curve, accuracy_score, auc, precision_recall_curve\n",
    "from sklearn.model_selection import train_test_split\n",
    "import tensorflow as tf\n",
    "import shap\n",
    "tf.compat.v1.disable_v2_behavior()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.options.display.max_rows = 20\n",
    "pd.options.display.max_columns = 200\n",
    "data = pd.read_csv('clean_pan.csv', index_col='CASEID')\n",
    "data = data.drop(['Unnamed: 0'], axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_names = ['PAN_JAUNDICE', 'PAN_BILIARYSTENT', 'PAN_CHEMO', 'PAN_RADIO', 'PAN_INTRA_ANTIBIOTICS',  'PAN_DUCTSIZE', 'PAN_GLANDTEXT', 'PAN_RECONSTRUCTION',  'PAN_DRAINS', 'PAN_DRAINS_TYPE', 'PAN_DRAINSYS_TYPE', 'PAN_DRAINSYS_SUCTN', 'PAN_RESECTION']\n",
    "\n",
    "cont_names = ['PAN_AMYLASE_POD1']\n",
    "y_names=['PAN_FISTULA']\n",
    "\n",
    "features = cat_names + cont_names + y_names\n",
    "data = data[features]\n",
    "\n",
    "columns = {'PAN_JAUNDICE':'Jaundice', 'PAN_BILIARYSTENT':'Biliary stent', 'PAN_CHEMO':'Chemotherapy', 'PAN_RADIO':'Radiation therapy', 'PAN_INTRA_ANTIBIOTICS':'Antibiotic',  'PAN_DUCTSIZE':'Duct size', 'PAN_GLANDTEXT':'Gland texture', 'PAN_RECONSTRUCTION':'Reconstruction',  'PAN_DRAINS':'Drains', 'PAN_DRAINS_TYPE':'Drain location', 'PAN_DRAINSYS_TYPE':'Drain type', 'PAN_DRAINSYS_SUCTN':'Drain suction', 'PAN_RESECTION':'Vascular resection','PAN_AMYLASE_POD1':'Drain amylase (POD1)'}\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "col = {value : key for key, value in columns.items()}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.rename(columns=columns, errors='raise', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = data.drop(['PAN_FISTULA'], axis=1)\n",
    "y = data['PAN_FISTULA']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "X_train, X_valid, y_train, y_valid = train_test_split(X, y, train_size=0.8, test_size=0.2, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/kchen/.local/lib/python3.9/site-packages/keras/layers/normalization/batch_normalization.py:520: _colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "Train on 18586 samples, validate on 4647 samples\n",
      "Epoch 1/200\n",
      "16384/18586 [=========================>....] - ETA: 0s - loss: 0.7778 - Sensitivity: 0.3349 - tn: 9761.0000 - auc: 0.5455 - prc: 0.1853"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "`Model.state_updates` will be removed in a future version. This property should not be used in TensorFlow 2.0, as `updates` are applied automatically.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18586/18586 [==============================] - 1s 66us/sample - loss: 0.7621 - Sensitivity: 0.3177 - tn: 11343.0000 - auc: 0.5477 - prc: 0.1866 - val_loss: 0.6056 - val_Sensitivity: 0.1591 - val_tn: 3725.0000 - val_auc: 0.6807 - val_prc: 0.2964\n",
      "Epoch 2/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.6284 - Sensitivity: 0.1436 - tn: 14056.0000 - auc: 0.5798 - prc: 0.2109 - val_loss: 0.5473 - val_Sensitivity: 0.1110 - val_tn: 3803.0000 - val_auc: 0.6889 - val_prc: 0.3042\n",
      "Epoch 3/200\n",
      "18586/18586 [==============================] - 0s 9us/sample - loss: 0.6170 - Sensitivity: 0.1452 - tn: 14225.0000 - auc: 0.5918 - prc: 0.2241 - val_loss: 0.5399 - val_Sensitivity: 0.1003 - val_tn: 3812.0000 - val_auc: 0.6931 - val_prc: 0.3077\n",
      "Epoch 4/200\n",
      "18586/18586 [==============================] - 0s 9us/sample - loss: 0.6017 - Sensitivity: 0.1507 - tn: 14202.0000 - auc: 0.6015 - prc: 0.2305 - val_loss: 0.5344 - val_Sensitivity: 0.0909 - val_tn: 3822.0000 - val_auc: 0.6843 - val_prc: 0.2989\n",
      "Epoch 5/200\n",
      "18586/18586 [==============================] - 0s 9us/sample - loss: 0.5922 - Sensitivity: 0.1371 - tn: 14230.0000 - auc: 0.5960 - prc: 0.2216 - val_loss: 0.5247 - val_Sensitivity: 0.0816 - val_tn: 3834.0000 - val_auc: 0.6830 - val_prc: 0.2970\n",
      "Epoch 6/200\n",
      "18586/18586 [==============================] - 0s 9us/sample - loss: 0.5806 - Sensitivity: 0.1371 - tn: 14232.0000 - auc: 0.5941 - prc: 0.2238 - val_loss: 0.5195 - val_Sensitivity: 0.0695 - val_tn: 3843.0000 - val_auc: 0.6801 - val_prc: 0.2929\n",
      "Epoch 7/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.5723 - Sensitivity: 0.1404 - tn: 14238.0000 - auc: 0.5993 - prc: 0.2254 - val_loss: 0.5112 - val_Sensitivity: 0.0709 - val_tn: 3841.0000 - val_auc: 0.6807 - val_prc: 0.2940\n",
      "Epoch 8/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.5611 - Sensitivity: 0.1218 - tn: 14328.0000 - auc: 0.6061 - prc: 0.2283 - val_loss: 0.5006 - val_Sensitivity: 0.0695 - val_tn: 3847.0000 - val_auc: 0.6815 - val_prc: 0.2928\n",
      "Epoch 9/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.5571 - Sensitivity: 0.1290 - tn: 14400.0000 - auc: 0.6034 - prc: 0.2263 - val_loss: 0.4990 - val_Sensitivity: 0.0695 - val_tn: 3849.0000 - val_auc: 0.6805 - val_prc: 0.2911\n",
      "Epoch 10/200\n",
      "18586/18586 [==============================] - 0s 9us/sample - loss: 0.5412 - Sensitivity: 0.1228 - tn: 14512.0000 - auc: 0.6124 - prc: 0.2352 - val_loss: 0.5011 - val_Sensitivity: 0.0762 - val_tn: 3834.0000 - val_auc: 0.6841 - val_prc: 0.2952\n",
      "Epoch 11/200\n",
      "18586/18586 [==============================] - 0s 9us/sample - loss: 0.5420 - Sensitivity: 0.1257 - tn: 14500.0000 - auc: 0.6100 - prc: 0.2311 - val_loss: 0.4968 - val_Sensitivity: 0.0775 - val_tn: 3836.0000 - val_auc: 0.6838 - val_prc: 0.2935\n",
      "Epoch 12/200\n",
      "18586/18586 [==============================] - 0s 9us/sample - loss: 0.5262 - Sensitivity: 0.1095 - tn: 14585.0000 - auc: 0.6155 - prc: 0.2341 - val_loss: 0.5016 - val_Sensitivity: 0.0909 - val_tn: 3824.0000 - val_auc: 0.6864 - val_prc: 0.2973\n",
      "Epoch 13/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.5219 - Sensitivity: 0.0968 - tn: 14643.0000 - auc: 0.6121 - prc: 0.2270 - val_loss: 0.4963 - val_Sensitivity: 0.0922 - val_tn: 3819.0000 - val_auc: 0.6885 - val_prc: 0.2985\n",
      "Epoch 14/200\n",
      "18586/18586 [==============================] - 0s 11us/sample - loss: 0.5140 - Sensitivity: 0.1040 - tn: 14635.0000 - auc: 0.6196 - prc: 0.2383 - val_loss: 0.4957 - val_Sensitivity: 0.0936 - val_tn: 3822.0000 - val_auc: 0.6881 - val_prc: 0.2980\n",
      "Epoch 15/200\n",
      "18586/18586 [==============================] - 0s 11us/sample - loss: 0.5127 - Sensitivity: 0.0897 - tn: 14743.0000 - auc: 0.6113 - prc: 0.2283 - val_loss: 0.4910 - val_Sensitivity: 0.0936 - val_tn: 3819.0000 - val_auc: 0.6899 - val_prc: 0.2993\n",
      "Epoch 16/200\n",
      "18586/18586 [==============================] - 0s 11us/sample - loss: 0.5003 - Sensitivity: 0.0936 - tn: 14797.0000 - auc: 0.6169 - prc: 0.2352 - val_loss: 0.4918 - val_Sensitivity: 0.1043 - val_tn: 3801.0000 - val_auc: 0.6894 - val_prc: 0.2999\n",
      "Epoch 17/200\n",
      "18586/18586 [==============================] - 0s 11us/sample - loss: 0.4936 - Sensitivity: 0.0910 - tn: 14829.0000 - auc: 0.6243 - prc: 0.2393 - val_loss: 0.4878 - val_Sensitivity: 0.1136 - val_tn: 3787.0000 - val_auc: 0.6911 - val_prc: 0.3009\n",
      "Epoch 18/200\n",
      "18586/18586 [==============================] - 0s 11us/sample - loss: 0.4913 - Sensitivity: 0.0858 - tn: 14806.0000 - auc: 0.6246 - prc: 0.2348 - val_loss: 0.4851 - val_Sensitivity: 0.1163 - val_tn: 3785.0000 - val_auc: 0.6906 - val_prc: 0.3002\n",
      "Epoch 19/200\n",
      "18586/18586 [==============================] - 0s 11us/sample - loss: 0.4816 - Sensitivity: 0.0767 - tn: 14976.0000 - auc: 0.6355 - prc: 0.2447 - val_loss: 0.4763 - val_Sensitivity: 0.1176 - val_tn: 3795.0000 - val_auc: 0.6950 - val_prc: 0.3052\n",
      "Epoch 20/200\n",
      "18586/18586 [==============================] - 0s 11us/sample - loss: 0.4825 - Sensitivity: 0.0754 - tn: 15042.0000 - auc: 0.6288 - prc: 0.2435 - val_loss: 0.4811 - val_Sensitivity: 0.1230 - val_tn: 3780.0000 - val_auc: 0.6904 - val_prc: 0.2999\n",
      "Epoch 21/200\n",
      "18586/18586 [==============================] - 0s 11us/sample - loss: 0.4745 - Sensitivity: 0.0663 - tn: 15006.0000 - auc: 0.6343 - prc: 0.2416 - val_loss: 0.4736 - val_Sensitivity: 0.1190 - val_tn: 3790.0000 - val_auc: 0.6948 - val_prc: 0.3034\n",
      "Epoch 22/200\n",
      "18586/18586 [==============================] - 0s 12us/sample - loss: 0.4682 - Sensitivity: 0.0660 - tn: 15086.0000 - auc: 0.6347 - prc: 0.2468 - val_loss: 0.4789 - val_Sensitivity: 0.1297 - val_tn: 3766.0000 - val_auc: 0.6936 - val_prc: 0.3041\n",
      "Epoch 23/200\n",
      "18586/18586 [==============================] - 0s 12us/sample - loss: 0.4663 - Sensitivity: 0.0705 - tn: 15107.0000 - auc: 0.6372 - prc: 0.2540 - val_loss: 0.4734 - val_Sensitivity: 0.1350 - val_tn: 3764.0000 - val_auc: 0.6934 - val_prc: 0.3025\n",
      "Epoch 24/200\n",
      "18586/18586 [==============================] - 0s 12us/sample - loss: 0.4562 - Sensitivity: 0.0715 - tn: 15201.0000 - auc: 0.6485 - prc: 0.2695 - val_loss: 0.4697 - val_Sensitivity: 0.1230 - val_tn: 3787.0000 - val_auc: 0.6950 - val_prc: 0.3046\n",
      "Epoch 25/200\n",
      "18586/18586 [==============================] - 0s 11us/sample - loss: 0.4618 - Sensitivity: 0.0517 - tn: 15189.0000 - auc: 0.6347 - prc: 0.2471 - val_loss: 0.4752 - val_Sensitivity: 0.1270 - val_tn: 3785.0000 - val_auc: 0.6916 - val_prc: 0.3035\n",
      "Epoch 26/200\n",
      "18586/18586 [==============================] - 0s 12us/sample - loss: 0.4587 - Sensitivity: 0.0565 - tn: 15207.0000 - auc: 0.6384 - prc: 0.2547 - val_loss: 0.4658 - val_Sensitivity: 0.1163 - val_tn: 3797.0000 - val_auc: 0.6944 - val_prc: 0.3063\n",
      "Epoch 27/200\n",
      "18586/18586 [==============================] - 0s 11us/sample - loss: 0.4542 - Sensitivity: 0.0565 - tn: 15230.0000 - auc: 0.6450 - prc: 0.2621 - val_loss: 0.4694 - val_Sensitivity: 0.1190 - val_tn: 3791.0000 - val_auc: 0.6947 - val_prc: 0.3078\n",
      "Epoch 28/200\n",
      "18586/18586 [==============================] - 0s 12us/sample - loss: 0.4479 - Sensitivity: 0.0539 - tn: 15291.0000 - auc: 0.6492 - prc: 0.2661 - val_loss: 0.4720 - val_Sensitivity: 0.1230 - val_tn: 3790.0000 - val_auc: 0.6918 - val_prc: 0.3044\n",
      "Epoch 29/200\n",
      "18586/18586 [==============================] - 0s 11us/sample - loss: 0.4495 - Sensitivity: 0.0432 - tn: 15304.0000 - auc: 0.6447 - prc: 0.2571 - val_loss: 0.4672 - val_Sensitivity: 0.1176 - val_tn: 3796.0000 - val_auc: 0.6925 - val_prc: 0.3051\n",
      "Epoch 30/200\n",
      "18586/18586 [==============================] - 0s 11us/sample - loss: 0.4491 - Sensitivity: 0.0409 - tn: 15327.0000 - auc: 0.6464 - prc: 0.2614 - val_loss: 0.4684 - val_Sensitivity: 0.1150 - val_tn: 3799.0000 - val_auc: 0.6939 - val_prc: 0.3052\n",
      "Epoch 31/200\n",
      "18586/18586 [==============================] - 0s 11us/sample - loss: 0.4417 - Sensitivity: 0.0458 - tn: 15330.0000 - auc: 0.6536 - prc: 0.2720 - val_loss: 0.4675 - val_Sensitivity: 0.1110 - val_tn: 3800.0000 - val_auc: 0.6936 - val_prc: 0.3047\n",
      "Epoch 32/200\n",
      "18586/18586 [==============================] - 0s 11us/sample - loss: 0.4404 - Sensitivity: 0.0315 - tn: 15366.0000 - auc: 0.6528 - prc: 0.2617 - val_loss: 0.4705 - val_Sensitivity: 0.1176 - val_tn: 3796.0000 - val_auc: 0.6929 - val_prc: 0.3040\n",
      "Epoch 33/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4441 - Sensitivity: 0.0416 - tn: 15347.0000 - auc: 0.6428 - prc: 0.2641 - val_loss: 0.4684 - val_Sensitivity: 0.1150 - val_tn: 3803.0000 - val_auc: 0.6944 - val_prc: 0.3060\n",
      "Epoch 34/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4355 - Sensitivity: 0.0419 - tn: 15372.0000 - auc: 0.6589 - prc: 0.2780 - val_loss: 0.4650 - val_Sensitivity: 0.1123 - val_tn: 3806.0000 - val_auc: 0.6954 - val_prc: 0.3071\n",
      "Epoch 35/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4339 - Sensitivity: 0.0406 - tn: 15391.0000 - auc: 0.6626 - prc: 0.2806 - val_loss: 0.4661 - val_Sensitivity: 0.1123 - val_tn: 3806.0000 - val_auc: 0.6937 - val_prc: 0.3057\n",
      "Epoch 36/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4362 - Sensitivity: 0.0351 - tn: 15402.0000 - auc: 0.6536 - prc: 0.2704 - val_loss: 0.4677 - val_Sensitivity: 0.1150 - val_tn: 3802.0000 - val_auc: 0.6927 - val_prc: 0.3044\n",
      "Epoch 37/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4340 - Sensitivity: 0.0374 - tn: 15393.0000 - auc: 0.6640 - prc: 0.2824 - val_loss: 0.4647 - val_Sensitivity: 0.1096 - val_tn: 3805.0000 - val_auc: 0.6938 - val_prc: 0.3058\n",
      "Epoch 38/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4324 - Sensitivity: 0.0341 - tn: 15415.0000 - auc: 0.6650 - prc: 0.2794 - val_loss: 0.4651 - val_Sensitivity: 0.1110 - val_tn: 3807.0000 - val_auc: 0.6947 - val_prc: 0.3060\n",
      "Epoch 39/200\n",
      "18586/18586 [==============================] - 0s 11us/sample - loss: 0.4311 - Sensitivity: 0.0338 - tn: 15420.0000 - auc: 0.6692 - prc: 0.2877 - val_loss: 0.4655 - val_Sensitivity: 0.1096 - val_tn: 3810.0000 - val_auc: 0.6960 - val_prc: 0.3073\n",
      "Epoch 40/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4297 - Sensitivity: 0.0309 - tn: 15430.0000 - auc: 0.6643 - prc: 0.2823 - val_loss: 0.4679 - val_Sensitivity: 0.1056 - val_tn: 3812.0000 - val_auc: 0.6936 - val_prc: 0.3058\n",
      "Epoch 41/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4285 - Sensitivity: 0.0283 - tn: 15441.0000 - auc: 0.6686 - prc: 0.2859 - val_loss: 0.4676 - val_Sensitivity: 0.1029 - val_tn: 3817.0000 - val_auc: 0.6930 - val_prc: 0.3054\n",
      "Epoch 42/200\n",
      "18586/18586 [==============================] - 0s 11us/sample - loss: 0.4272 - Sensitivity: 0.0305 - tn: 15449.0000 - auc: 0.6673 - prc: 0.2885 - val_loss: 0.4693 - val_Sensitivity: 0.1070 - val_tn: 3815.0000 - val_auc: 0.6921 - val_prc: 0.3041\n",
      "Epoch 43/200\n",
      "18586/18586 [==============================] - 0s 11us/sample - loss: 0.4277 - Sensitivity: 0.0318 - tn: 15438.0000 - auc: 0.6666 - prc: 0.2872 - val_loss: 0.4686 - val_Sensitivity: 0.1056 - val_tn: 3813.0000 - val_auc: 0.6910 - val_prc: 0.3035\n",
      "Epoch 44/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4267 - Sensitivity: 0.0253 - tn: 15448.0000 - auc: 0.6688 - prc: 0.2852 - val_loss: 0.4658 - val_Sensitivity: 0.1029 - val_tn: 3817.0000 - val_auc: 0.6920 - val_prc: 0.3062\n",
      "Epoch 45/200\n",
      "18586/18586 [==============================] - 0s 11us/sample - loss: 0.4249 - Sensitivity: 0.0322 - tn: 15436.0000 - auc: 0.6723 - prc: 0.2941 - val_loss: 0.4648 - val_Sensitivity: 0.1003 - val_tn: 3817.0000 - val_auc: 0.6908 - val_prc: 0.3047\n",
      "Epoch 46/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4249 - Sensitivity: 0.0296 - tn: 15440.0000 - auc: 0.6759 - prc: 0.2978 - val_loss: 0.4639 - val_Sensitivity: 0.1003 - val_tn: 3816.0000 - val_auc: 0.6913 - val_prc: 0.3047\n",
      "Epoch 47/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4242 - Sensitivity: 0.0266 - tn: 15444.0000 - auc: 0.6757 - prc: 0.2950 - val_loss: 0.4658 - val_Sensitivity: 0.1029 - val_tn: 3817.0000 - val_auc: 0.6909 - val_prc: 0.3042\n",
      "Epoch 48/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4225 - Sensitivity: 0.0299 - tn: 15436.0000 - auc: 0.6776 - prc: 0.2925 - val_loss: 0.4645 - val_Sensitivity: 0.1016 - val_tn: 3818.0000 - val_auc: 0.6920 - val_prc: 0.3055\n",
      "Epoch 49/200\n",
      "18586/18586 [==============================] - 0s 11us/sample - loss: 0.4238 - Sensitivity: 0.0227 - tn: 15453.0000 - auc: 0.6762 - prc: 0.2947 - val_loss: 0.4618 - val_Sensitivity: 0.1003 - val_tn: 3819.0000 - val_auc: 0.6919 - val_prc: 0.3057\n",
      "Epoch 50/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4238 - Sensitivity: 0.0286 - tn: 15439.0000 - auc: 0.6729 - prc: 0.2928 - val_loss: 0.4653 - val_Sensitivity: 0.1029 - val_tn: 3818.0000 - val_auc: 0.6904 - val_prc: 0.3051\n",
      "Epoch 51/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4215 - Sensitivity: 0.0270 - tn: 15443.0000 - auc: 0.6800 - prc: 0.2985 - val_loss: 0.4609 - val_Sensitivity: 0.1003 - val_tn: 3819.0000 - val_auc: 0.6924 - val_prc: 0.3062\n",
      "Epoch 52/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4219 - Sensitivity: 0.0296 - tn: 15434.0000 - auc: 0.6790 - prc: 0.2967 - val_loss: 0.4610 - val_Sensitivity: 0.1016 - val_tn: 3817.0000 - val_auc: 0.6922 - val_prc: 0.3075\n",
      "Epoch 53/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4214 - Sensitivity: 0.0296 - tn: 15435.0000 - auc: 0.6790 - prc: 0.2999 - val_loss: 0.4608 - val_Sensitivity: 0.1029 - val_tn: 3817.0000 - val_auc: 0.6918 - val_prc: 0.3061\n",
      "Epoch 54/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4215 - Sensitivity: 0.0335 - tn: 15444.0000 - auc: 0.6772 - prc: 0.3029 - val_loss: 0.4627 - val_Sensitivity: 0.1029 - val_tn: 3818.0000 - val_auc: 0.6914 - val_prc: 0.3072\n",
      "Epoch 55/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4205 - Sensitivity: 0.0273 - tn: 15446.0000 - auc: 0.6819 - prc: 0.3048 - val_loss: 0.4578 - val_Sensitivity: 0.0976 - val_tn: 3820.0000 - val_auc: 0.6925 - val_prc: 0.3065\n",
      "Epoch 56/200\n",
      "18586/18586 [==============================] - 0s 9us/sample - loss: 0.4208 - Sensitivity: 0.0263 - tn: 15445.0000 - auc: 0.6811 - prc: 0.3033 - val_loss: 0.4617 - val_Sensitivity: 0.1029 - val_tn: 3818.0000 - val_auc: 0.6909 - val_prc: 0.3068\n",
      "Epoch 57/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4191 - Sensitivity: 0.0309 - tn: 15442.0000 - auc: 0.6869 - prc: 0.3042 - val_loss: 0.4576 - val_Sensitivity: 0.0976 - val_tn: 3822.0000 - val_auc: 0.6931 - val_prc: 0.3076\n",
      "Epoch 58/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4195 - Sensitivity: 0.0234 - tn: 15449.0000 - auc: 0.6852 - prc: 0.3008 - val_loss: 0.4586 - val_Sensitivity: 0.1003 - val_tn: 3822.0000 - val_auc: 0.6933 - val_prc: 0.3093\n",
      "Epoch 59/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4209 - Sensitivity: 0.0344 - tn: 15422.0000 - auc: 0.6803 - prc: 0.3025 - val_loss: 0.4589 - val_Sensitivity: 0.1029 - val_tn: 3819.0000 - val_auc: 0.6924 - val_prc: 0.3077\n",
      "Epoch 60/200\n",
      "18586/18586 [==============================] - 0s 11us/sample - loss: 0.4191 - Sensitivity: 0.0273 - tn: 15450.0000 - auc: 0.6864 - prc: 0.3095 - val_loss: 0.4549 - val_Sensitivity: 0.0949 - val_tn: 3823.0000 - val_auc: 0.6939 - val_prc: 0.3088\n",
      "Epoch 61/200\n",
      "18586/18586 [==============================] - 0s 11us/sample - loss: 0.4201 - Sensitivity: 0.0260 - tn: 15445.0000 - auc: 0.6830 - prc: 0.3021 - val_loss: 0.4589 - val_Sensitivity: 0.0976 - val_tn: 3823.0000 - val_auc: 0.6925 - val_prc: 0.3085\n",
      "Epoch 62/200\n",
      "18586/18586 [==============================] - 0s 11us/sample - loss: 0.4195 - Sensitivity: 0.0309 - tn: 15440.0000 - auc: 0.6848 - prc: 0.3020 - val_loss: 0.4556 - val_Sensitivity: 0.0963 - val_tn: 3823.0000 - val_auc: 0.6929 - val_prc: 0.3080\n",
      "Epoch 63/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4198 - Sensitivity: 0.0172 - tn: 15457.0000 - auc: 0.6845 - prc: 0.3036 - val_loss: 0.4543 - val_Sensitivity: 0.0963 - val_tn: 3824.0000 - val_auc: 0.6936 - val_prc: 0.3093\n",
      "Epoch 64/200\n",
      "18586/18586 [==============================] - 0s 11us/sample - loss: 0.4158 - Sensitivity: 0.0309 - tn: 15444.0000 - auc: 0.6940 - prc: 0.3109 - val_loss: 0.4546 - val_Sensitivity: 0.0989 - val_tn: 3822.0000 - val_auc: 0.6937 - val_prc: 0.3091\n",
      "Epoch 65/200\n",
      "18586/18586 [==============================] - 0s 11us/sample - loss: 0.4186 - Sensitivity: 0.0240 - tn: 15447.0000 - auc: 0.6846 - prc: 0.3016 - val_loss: 0.4516 - val_Sensitivity: 0.0976 - val_tn: 3823.0000 - val_auc: 0.6944 - val_prc: 0.3095\n",
      "Epoch 66/200\n",
      "18586/18586 [==============================] - 0s 11us/sample - loss: 0.4170 - Sensitivity: 0.0309 - tn: 15451.0000 - auc: 0.6900 - prc: 0.3074 - val_loss: 0.4528 - val_Sensitivity: 0.1003 - val_tn: 3820.0000 - val_auc: 0.6936 - val_prc: 0.3104\n",
      "Epoch 67/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4156 - Sensitivity: 0.0302 - tn: 15442.0000 - auc: 0.6931 - prc: 0.3132 - val_loss: 0.4535 - val_Sensitivity: 0.1016 - val_tn: 3817.0000 - val_auc: 0.6934 - val_prc: 0.3091\n",
      "Epoch 68/200\n",
      "18586/18586 [==============================] - 0s 11us/sample - loss: 0.4193 - Sensitivity: 0.0335 - tn: 15429.0000 - auc: 0.6851 - prc: 0.3077 - val_loss: 0.4552 - val_Sensitivity: 0.1016 - val_tn: 3817.0000 - val_auc: 0.6920 - val_prc: 0.3078\n",
      "Epoch 69/200\n",
      "18586/18586 [==============================] - 0s 11us/sample - loss: 0.4169 - Sensitivity: 0.0286 - tn: 15442.0000 - auc: 0.6903 - prc: 0.3166 - val_loss: 0.4522 - val_Sensitivity: 0.1016 - val_tn: 3818.0000 - val_auc: 0.6936 - val_prc: 0.3087\n",
      "Epoch 70/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4174 - Sensitivity: 0.0302 - tn: 15441.0000 - auc: 0.6904 - prc: 0.3137 - val_loss: 0.4521 - val_Sensitivity: 0.0976 - val_tn: 3819.0000 - val_auc: 0.6924 - val_prc: 0.3086\n",
      "Epoch 71/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4188 - Sensitivity: 0.0325 - tn: 15433.0000 - auc: 0.6863 - prc: 0.3062 - val_loss: 0.4515 - val_Sensitivity: 0.0989 - val_tn: 3818.0000 - val_auc: 0.6931 - val_prc: 0.3096\n",
      "Epoch 72/200\n",
      "18586/18586 [==============================] - 0s 11us/sample - loss: 0.4152 - Sensitivity: 0.0270 - tn: 15446.0000 - auc: 0.6958 - prc: 0.3153 - val_loss: 0.4498 - val_Sensitivity: 0.0976 - val_tn: 3822.0000 - val_auc: 0.6932 - val_prc: 0.3095\n",
      "Epoch 73/200\n",
      "18586/18586 [==============================] - 0s 11us/sample - loss: 0.4166 - Sensitivity: 0.0305 - tn: 15445.0000 - auc: 0.6903 - prc: 0.3116 - val_loss: 0.4516 - val_Sensitivity: 0.0989 - val_tn: 3820.0000 - val_auc: 0.6931 - val_prc: 0.3093\n",
      "Epoch 74/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4162 - Sensitivity: 0.0296 - tn: 15443.0000 - auc: 0.6941 - prc: 0.3134 - val_loss: 0.4489 - val_Sensitivity: 0.0976 - val_tn: 3820.0000 - val_auc: 0.6952 - val_prc: 0.3111\n",
      "Epoch 75/200\n",
      "18586/18586 [==============================] - 0s 11us/sample - loss: 0.4162 - Sensitivity: 0.0315 - tn: 15443.0000 - auc: 0.6906 - prc: 0.3173 - val_loss: 0.4526 - val_Sensitivity: 0.1056 - val_tn: 3814.0000 - val_auc: 0.6932 - val_prc: 0.3092\n",
      "Epoch 76/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4159 - Sensitivity: 0.0341 - tn: 15436.0000 - auc: 0.6936 - prc: 0.3150 - val_loss: 0.4505 - val_Sensitivity: 0.1043 - val_tn: 3814.0000 - val_auc: 0.6931 - val_prc: 0.3105\n",
      "Epoch 77/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4160 - Sensitivity: 0.0374 - tn: 15431.0000 - auc: 0.6914 - prc: 0.3146 - val_loss: 0.4504 - val_Sensitivity: 0.1043 - val_tn: 3816.0000 - val_auc: 0.6930 - val_prc: 0.3097\n",
      "Epoch 78/200\n",
      "18586/18586 [==============================] - 0s 12us/sample - loss: 0.4173 - Sensitivity: 0.0357 - tn: 15434.0000 - auc: 0.6897 - prc: 0.3152 - val_loss: 0.4515 - val_Sensitivity: 0.1043 - val_tn: 3815.0000 - val_auc: 0.6912 - val_prc: 0.3088\n",
      "Epoch 79/200\n",
      "18586/18586 [==============================] - 0s 11us/sample - loss: 0.4176 - Sensitivity: 0.0370 - tn: 15429.0000 - auc: 0.6888 - prc: 0.3131 - val_loss: 0.4512 - val_Sensitivity: 0.1043 - val_tn: 3814.0000 - val_auc: 0.6920 - val_prc: 0.3096\n",
      "Epoch 80/200\n",
      "18586/18586 [==============================] - 0s 9us/sample - loss: 0.4172 - Sensitivity: 0.0305 - tn: 15425.0000 - auc: 0.6893 - prc: 0.3089 - val_loss: 0.4471 - val_Sensitivity: 0.0976 - val_tn: 3819.0000 - val_auc: 0.6936 - val_prc: 0.3101\n",
      "Epoch 81/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4164 - Sensitivity: 0.0309 - tn: 15435.0000 - auc: 0.6924 - prc: 0.3131 - val_loss: 0.4487 - val_Sensitivity: 0.1029 - val_tn: 3817.0000 - val_auc: 0.6950 - val_prc: 0.3112\n",
      "Epoch 82/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4171 - Sensitivity: 0.0328 - tn: 15426.0000 - auc: 0.6918 - prc: 0.3127 - val_loss: 0.4464 - val_Sensitivity: 0.1029 - val_tn: 3818.0000 - val_auc: 0.6948 - val_prc: 0.3110\n",
      "Epoch 83/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4164 - Sensitivity: 0.0338 - tn: 15422.0000 - auc: 0.6924 - prc: 0.3108 - val_loss: 0.4470 - val_Sensitivity: 0.0989 - val_tn: 3819.0000 - val_auc: 0.6942 - val_prc: 0.3109\n",
      "Epoch 84/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4160 - Sensitivity: 0.0247 - tn: 15451.0000 - auc: 0.6932 - prc: 0.3121 - val_loss: 0.4460 - val_Sensitivity: 0.0936 - val_tn: 3823.0000 - val_auc: 0.6936 - val_prc: 0.3107\n",
      "Epoch 85/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4164 - Sensitivity: 0.0221 - tn: 15455.0000 - auc: 0.6919 - prc: 0.3134 - val_loss: 0.4463 - val_Sensitivity: 0.0976 - val_tn: 3819.0000 - val_auc: 0.6940 - val_prc: 0.3096\n",
      "Epoch 86/200\n",
      "18586/18586 [==============================] - 0s 11us/sample - loss: 0.4149 - Sensitivity: 0.0322 - tn: 15443.0000 - auc: 0.6956 - prc: 0.3152 - val_loss: 0.4435 - val_Sensitivity: 0.1016 - val_tn: 3816.0000 - val_auc: 0.6932 - val_prc: 0.3117\n",
      "Epoch 87/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4158 - Sensitivity: 0.0409 - tn: 15428.0000 - auc: 0.6934 - prc: 0.3174 - val_loss: 0.4464 - val_Sensitivity: 0.1003 - val_tn: 3818.0000 - val_auc: 0.6914 - val_prc: 0.3093\n",
      "Epoch 88/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4157 - Sensitivity: 0.0276 - tn: 15442.0000 - auc: 0.6943 - prc: 0.3136 - val_loss: 0.4457 - val_Sensitivity: 0.0976 - val_tn: 3818.0000 - val_auc: 0.6929 - val_prc: 0.3090\n",
      "Epoch 89/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4154 - Sensitivity: 0.0325 - tn: 15429.0000 - auc: 0.6940 - prc: 0.3142 - val_loss: 0.4433 - val_Sensitivity: 0.0989 - val_tn: 3816.0000 - val_auc: 0.6920 - val_prc: 0.3110\n",
      "Epoch 90/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4142 - Sensitivity: 0.0299 - tn: 15439.0000 - auc: 0.6984 - prc: 0.3186 - val_loss: 0.4429 - val_Sensitivity: 0.0976 - val_tn: 3818.0000 - val_auc: 0.6922 - val_prc: 0.3085\n",
      "Epoch 91/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4148 - Sensitivity: 0.0309 - tn: 15442.0000 - auc: 0.6957 - prc: 0.3124 - val_loss: 0.4430 - val_Sensitivity: 0.1003 - val_tn: 3816.0000 - val_auc: 0.6932 - val_prc: 0.3095\n",
      "Epoch 92/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4134 - Sensitivity: 0.0322 - tn: 15427.0000 - auc: 0.7009 - prc: 0.3215 - val_loss: 0.4436 - val_Sensitivity: 0.0976 - val_tn: 3818.0000 - val_auc: 0.6922 - val_prc: 0.3095\n",
      "Epoch 93/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4150 - Sensitivity: 0.0296 - tn: 15448.0000 - auc: 0.6953 - prc: 0.3151 - val_loss: 0.4427 - val_Sensitivity: 0.0989 - val_tn: 3816.0000 - val_auc: 0.6937 - val_prc: 0.3093\n",
      "Epoch 94/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4146 - Sensitivity: 0.0299 - tn: 15433.0000 - auc: 0.6966 - prc: 0.3163 - val_loss: 0.4423 - val_Sensitivity: 0.1003 - val_tn: 3818.0000 - val_auc: 0.6942 - val_prc: 0.3104\n",
      "Epoch 95/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4153 - Sensitivity: 0.0403 - tn: 15440.0000 - auc: 0.6935 - prc: 0.3201 - val_loss: 0.4429 - val_Sensitivity: 0.1043 - val_tn: 3812.0000 - val_auc: 0.6940 - val_prc: 0.3102\n",
      "Epoch 96/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4147 - Sensitivity: 0.0328 - tn: 15426.0000 - auc: 0.6974 - prc: 0.3178 - val_loss: 0.4430 - val_Sensitivity: 0.1029 - val_tn: 3812.0000 - val_auc: 0.6944 - val_prc: 0.3106\n",
      "Epoch 97/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4155 - Sensitivity: 0.0338 - tn: 15425.0000 - auc: 0.6953 - prc: 0.3174 - val_loss: 0.4419 - val_Sensitivity: 0.0976 - val_tn: 3817.0000 - val_auc: 0.6934 - val_prc: 0.3098\n",
      "Epoch 98/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4151 - Sensitivity: 0.0247 - tn: 15457.0000 - auc: 0.6948 - prc: 0.3184 - val_loss: 0.4452 - val_Sensitivity: 0.0963 - val_tn: 3818.0000 - val_auc: 0.6923 - val_prc: 0.3094\n",
      "Epoch 99/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4134 - Sensitivity: 0.0348 - tn: 15438.0000 - auc: 0.7000 - prc: 0.3213 - val_loss: 0.4433 - val_Sensitivity: 0.0976 - val_tn: 3817.0000 - val_auc: 0.6933 - val_prc: 0.3101\n",
      "Epoch 100/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4152 - Sensitivity: 0.0257 - tn: 15444.0000 - auc: 0.6941 - prc: 0.3122 - val_loss: 0.4436 - val_Sensitivity: 0.1016 - val_tn: 3817.0000 - val_auc: 0.6935 - val_prc: 0.3101\n",
      "Epoch 101/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4154 - Sensitivity: 0.0416 - tn: 15417.0000 - auc: 0.6947 - prc: 0.3156 - val_loss: 0.4429 - val_Sensitivity: 0.1003 - val_tn: 3818.0000 - val_auc: 0.6942 - val_prc: 0.3108\n",
      "Epoch 102/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4137 - Sensitivity: 0.0276 - tn: 15445.0000 - auc: 0.6981 - prc: 0.3192 - val_loss: 0.4430 - val_Sensitivity: 0.0976 - val_tn: 3819.0000 - val_auc: 0.6941 - val_prc: 0.3113\n",
      "Epoch 103/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4132 - Sensitivity: 0.0322 - tn: 15429.0000 - auc: 0.7008 - prc: 0.3185 - val_loss: 0.4430 - val_Sensitivity: 0.1003 - val_tn: 3817.0000 - val_auc: 0.6948 - val_prc: 0.3113\n",
      "Epoch 104/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4147 - Sensitivity: 0.0387 - tn: 15415.0000 - auc: 0.6969 - prc: 0.3174 - val_loss: 0.4416 - val_Sensitivity: 0.1003 - val_tn: 3816.0000 - val_auc: 0.6945 - val_prc: 0.3109\n",
      "Epoch 105/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4138 - Sensitivity: 0.0302 - tn: 15429.0000 - auc: 0.6992 - prc: 0.3141 - val_loss: 0.4424 - val_Sensitivity: 0.0976 - val_tn: 3819.0000 - val_auc: 0.6940 - val_prc: 0.3107\n",
      "Epoch 106/200\n",
      "18586/18586 [==============================] - 0s 11us/sample - loss: 0.4146 - Sensitivity: 0.0335 - tn: 15430.0000 - auc: 0.6953 - prc: 0.3186 - val_loss: 0.4430 - val_Sensitivity: 0.1016 - val_tn: 3818.0000 - val_auc: 0.6924 - val_prc: 0.3100\n",
      "Epoch 107/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4144 - Sensitivity: 0.0305 - tn: 15437.0000 - auc: 0.6975 - prc: 0.3149 - val_loss: 0.4423 - val_Sensitivity: 0.1016 - val_tn: 3818.0000 - val_auc: 0.6942 - val_prc: 0.3105\n",
      "Epoch 108/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4151 - Sensitivity: 0.0208 - tn: 15460.0000 - auc: 0.6966 - prc: 0.3138 - val_loss: 0.4412 - val_Sensitivity: 0.1016 - val_tn: 3818.0000 - val_auc: 0.6938 - val_prc: 0.3103\n",
      "Epoch 109/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4135 - Sensitivity: 0.0344 - tn: 15419.0000 - auc: 0.7019 - prc: 0.3195 - val_loss: 0.4421 - val_Sensitivity: 0.1043 - val_tn: 3810.0000 - val_auc: 0.6940 - val_prc: 0.3102\n",
      "Epoch 110/200\n",
      "18586/18586 [==============================] - 0s 11us/sample - loss: 0.4129 - Sensitivity: 0.0315 - tn: 15438.0000 - auc: 0.7020 - prc: 0.3191 - val_loss: 0.4398 - val_Sensitivity: 0.0989 - val_tn: 3818.0000 - val_auc: 0.6954 - val_prc: 0.3114\n",
      "Epoch 111/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4129 - Sensitivity: 0.0286 - tn: 15450.0000 - auc: 0.7016 - prc: 0.3191 - val_loss: 0.4401 - val_Sensitivity: 0.0989 - val_tn: 3818.0000 - val_auc: 0.6965 - val_prc: 0.3121\n",
      "Epoch 112/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4143 - Sensitivity: 0.0338 - tn: 15435.0000 - auc: 0.6974 - prc: 0.3181 - val_loss: 0.4403 - val_Sensitivity: 0.0989 - val_tn: 3819.0000 - val_auc: 0.6959 - val_prc: 0.3110\n",
      "Epoch 113/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4151 - Sensitivity: 0.0240 - tn: 15447.0000 - auc: 0.6961 - prc: 0.3128 - val_loss: 0.4416 - val_Sensitivity: 0.0976 - val_tn: 3821.0000 - val_auc: 0.6945 - val_prc: 0.3106\n",
      "Epoch 114/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4147 - Sensitivity: 0.0341 - tn: 15438.0000 - auc: 0.6963 - prc: 0.3162 - val_loss: 0.4435 - val_Sensitivity: 0.1003 - val_tn: 3816.0000 - val_auc: 0.6939 - val_prc: 0.3097\n",
      "Epoch 115/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4138 - Sensitivity: 0.0367 - tn: 15424.0000 - auc: 0.6991 - prc: 0.3193 - val_loss: 0.4403 - val_Sensitivity: 0.0989 - val_tn: 3817.0000 - val_auc: 0.6953 - val_prc: 0.3112\n",
      "Epoch 116/200\n",
      "18586/18586 [==============================] - 0s 11us/sample - loss: 0.4144 - Sensitivity: 0.0244 - tn: 15446.0000 - auc: 0.6987 - prc: 0.3186 - val_loss: 0.4411 - val_Sensitivity: 0.0949 - val_tn: 3824.0000 - val_auc: 0.6942 - val_prc: 0.3102\n",
      "Epoch 117/200\n",
      "18586/18586 [==============================] - 0s 11us/sample - loss: 0.4126 - Sensitivity: 0.0286 - tn: 15441.0000 - auc: 0.7004 - prc: 0.3235 - val_loss: 0.4397 - val_Sensitivity: 0.1016 - val_tn: 3820.0000 - val_auc: 0.6949 - val_prc: 0.3105\n",
      "Epoch 118/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4120 - Sensitivity: 0.0338 - tn: 15426.0000 - auc: 0.7032 - prc: 0.3227 - val_loss: 0.4391 - val_Sensitivity: 0.1016 - val_tn: 3819.0000 - val_auc: 0.6952 - val_prc: 0.3107\n",
      "Epoch 119/200\n",
      "18586/18586 [==============================] - 0s 11us/sample - loss: 0.4125 - Sensitivity: 0.0344 - tn: 15419.0000 - auc: 0.7022 - prc: 0.3234 - val_loss: 0.4359 - val_Sensitivity: 0.1016 - val_tn: 3819.0000 - val_auc: 0.6950 - val_prc: 0.3104\n",
      "Epoch 120/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4127 - Sensitivity: 0.0299 - tn: 15439.0000 - auc: 0.7019 - prc: 0.3200 - val_loss: 0.4388 - val_Sensitivity: 0.1043 - val_tn: 3815.0000 - val_auc: 0.6940 - val_prc: 0.3103\n",
      "Epoch 121/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4149 - Sensitivity: 0.0396 - tn: 15407.0000 - auc: 0.6984 - prc: 0.3159 - val_loss: 0.4385 - val_Sensitivity: 0.1029 - val_tn: 3817.0000 - val_auc: 0.6939 - val_prc: 0.3098\n",
      "Epoch 122/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4147 - Sensitivity: 0.0309 - tn: 15448.0000 - auc: 0.6976 - prc: 0.3185 - val_loss: 0.4371 - val_Sensitivity: 0.1043 - val_tn: 3815.0000 - val_auc: 0.6953 - val_prc: 0.3115\n",
      "Epoch 123/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4152 - Sensitivity: 0.0406 - tn: 15420.0000 - auc: 0.6964 - prc: 0.3200 - val_loss: 0.4393 - val_Sensitivity: 0.1043 - val_tn: 3812.0000 - val_auc: 0.6952 - val_prc: 0.3115\n",
      "Epoch 124/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4134 - Sensitivity: 0.0406 - tn: 15411.0000 - auc: 0.7006 - prc: 0.3199 - val_loss: 0.4377 - val_Sensitivity: 0.1043 - val_tn: 3814.0000 - val_auc: 0.6944 - val_prc: 0.3107\n",
      "Epoch 125/200\n",
      "18586/18586 [==============================] - 0s 11us/sample - loss: 0.4127 - Sensitivity: 0.0296 - tn: 15433.0000 - auc: 0.7017 - prc: 0.3172 - val_loss: 0.4374 - val_Sensitivity: 0.0963 - val_tn: 3819.0000 - val_auc: 0.6955 - val_prc: 0.3115\n",
      "Epoch 126/200\n",
      "18586/18586 [==============================] - 0s 11us/sample - loss: 0.4130 - Sensitivity: 0.0266 - tn: 15442.0000 - auc: 0.7013 - prc: 0.3188 - val_loss: 0.4380 - val_Sensitivity: 0.0936 - val_tn: 3821.0000 - val_auc: 0.6937 - val_prc: 0.3098\n",
      "Epoch 127/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4140 - Sensitivity: 0.0279 - tn: 15439.0000 - auc: 0.6986 - prc: 0.3143 - val_loss: 0.4372 - val_Sensitivity: 0.0963 - val_tn: 3819.0000 - val_auc: 0.6947 - val_prc: 0.3104\n",
      "Epoch 128/200\n",
      "18586/18586 [==============================] - 0s 10us/sample - loss: 0.4137 - Sensitivity: 0.0335 - tn: 15430.0000 - auc: 0.6994 - prc: 0.3177 - val_loss: 0.4383 - val_Sensitivity: 0.1003 - val_tn: 3816.0000 - val_auc: 0.6944 - val_prc: 0.3105\n",
      "Epoch 129/200\n",
      "18586/18586 [==============================] - 0s 18us/sample - loss: 0.4129 - Sensitivity: 0.0367 - tn: 15415.0000 - auc: 0.7029 - prc: 0.3219 - val_loss: 0.4372 - val_Sensitivity: 0.0976 - val_tn: 3821.0000 - val_auc: 0.6938 - val_prc: 0.3091\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fcb78e75f70>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_shape = [X.shape[1]]\n",
    "model4 = tf.keras.models.Sequential()\n",
    "model4.add(tf.keras.layers.Flatten(input_shape=input_shape))\n",
    "model4.add(tf.keras.layers.BatchNormalization())\n",
    "for _ in range(2):\n",
    "    model4.add(tf.keras.layers.Dense(1000))\n",
    "    model4.add(tf.keras.layers.BatchNormalization())\n",
    "    model4.add(tf.keras.layers.Dropout(0.8))\n",
    "    model4.add(tf.keras.layers.Activation(\"relu\"))\n",
    "model4.add(tf.keras.layers.Dense(1, activation=\"sigmoid\"))\n",
    "\n",
    "opt = tf.keras.optimizers.Adam(learning_rate=3e-4)\n",
    "\n",
    "metrics = [tf.keras.metrics.Recall(name='Sensitivity'), tf.keras.metrics.TrueNegatives(name='tn'), tf.keras.metrics.AUC(name='auc'), tf.keras.metrics.AUC(name='prc', curve='PR')]\n",
    "\n",
    "model4.compile(\n",
    "    optimizer=opt,\n",
    "    loss=tf.keras.losses.BinaryCrossentropy(from_logits=False),\n",
    "    metrics=metrics,)\n",
    "\n",
    "early_stopping = tf.keras.callbacks.EarlyStopping(\n",
    "    patience=10,\n",
    "    min_delta=0.00001,\n",
    "    restore_best_weights=True,)\n",
    "model4.fit(X_train, y_train,\n",
    "    validation_data=(X_valid, y_valid),\n",
    "    batch_size=1024,\n",
    "    epochs=200,\n",
    "    callbacks=[early_stopping])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(18586, 14)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4647, 14)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_valid.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras is no longer supported, please use tf.keras instead.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/kchen/.local/lib/python3.9/site-packages/shap/explainers/tf_utils.py:28: The name tf.keras.backend.get_session is deprecated. Please use tf.compat.v1.keras.backend.get_session instead.\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmQAAAGyCAYAAABKntgTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABUKklEQVR4nO3debxd0/3/8dcSCdIMUkKREAT9UkP5KDHUTFXTUjRmKaHaqqmmX4QGMY81VYxBQlpUiFnaphU19KOKRs1uEIkMMidEkv37Y63Dzsm5U+6wb3Lfz8fjPu45e6299tr7DPdzP2vtvUOWZYiIiIhIcVYougMiIiIirZ0CMhEREZGCKSATERERKZgCMhEREZGCKSATERERKZgCMhEREZGCrVh0B6T1GjlyZNa7d++iuyEiItKcQqWFypCJiIiIFEwBmYiIiEjBFJCJiIiIFEwBmYiIiEjBFJCJiIiIFEwBmYiIiEjBFJCJiIiIFEwBmYiIiEjBFJCJiIiIFEwBmYiIiEjBFJCJiIiIFEwBmYiIiEjBFJCJiIiIFEwBmYiIiEjBFJCJiIiIFEwBmYiIiEjBQpZlRfdBWqlw5QK9+UREpEXKTl+xqZoOlRYqQyYiIiJSMAVkIiIiIgVTQCYiIiJSsCYbIF2emNkQYIG792ui9rsBHwHru3tVU2xDREREWq5WH5CZ2TZAf2BnoD0wBXgZuNHd/1pk30rMrC8wwN17NmKbA4Gd3H3PxmpTRERElk6rHrI0s72A54D3AAM6ApsD9wIHFNi1ZYKZtTGzVv0eEhERaQytPUP2B2Cou5+ZWzYLeDD9VGRmFwOHAGsAnwLXu/u1qawH8AFwFPD/gO7A88DR7j4h1fkWcAuwS1r/8hq21Qu4GWhnZrPT4h+5+2gz+w5wFbA1MA8YBpzn7l+a2YXAT4Dt3H2emX0beBE4CPgmMSu4Qq7NLYDvU5aJyw/X5vatH/BbYENgPTP7Mu3D3sDKwN+A37j7p9Xtl4iIiHyt1WY3zGxjYkBx31Ks/gawEzGjdhxwiZntU1anDzHAWQf4BnBBrmwYsBBYN9XpW92G3P154ATgfXfvkH5Gm9kawN+BP6dt9AL2IgaBAAOBqcCNZtYeeAC41t2fcfc/AhcDo3Ntvl+P/T8M2D3t/2RgBJAB3wHWIwa199ajPRERkVatNWfIuqbf40sLzOzHwN3Ei7at5O4rV1rR3Yfmnv7VzB4D9gCeyi0/392npHbvJWaVMLN1iMFMT3efAcwws/OBp+vZ/6OAV919cGk/zOwS4DLgAndfaGaHAq8Qh2UnAufXcxvVOd/dJwKYmQHbAHu6+xdp2ZnAFDPr5u4fN9I2RUREllutOSCbkn53A94EcPdHgFXNbCfg2epWNLOTiJmxbsTgbRWWzAhNyD2eQ8wmlbYHMC5X/sFS9H99YEczm55bFoA2pSfuPtHM/gScBOzu7ouWYjuVVJX1YyXg0xibfeVzYgZQAZmIiEgtWnNA9jbwPnEu2Ki6rmRmOxKzUHsAL6ZM1ANUcyuECkoZufWIJxMA9KhlnUqB1DhglLvvV0NfdwF+DtxBHLrc1t3n1NDmLOLwat7awIc19GccMeD8ZiMGfCIiIq1Kq51D5u4Z8GvgSDO7zMy6m1lI8622q2HVTsT5X5OBzMz2A/atx3Y/BkYDl5tZJzNbEzivltUmAmuYWafcsruJI4bHmNnKZraCmW1gZj8gFqxJnB93MjGb9zHxJIZ8m+uaWbvcsv+k7fwotXcAcY5bjbsEvApcZ2arpW13NbNDallPREREklYbkAG4+5PEyfkbA/8GZgNjgR2J87wqeYoYDL1EHPY8CHionps+jDjM9xFxaPTuWur/DXgG+MDMppvZLmkO127A/sQhxGmpHxukS1EMA5529ztT5uoIYE8zOza1eX/a/sTU5vru/h4xgLsF+Az4ATWcbQqQ2v4JMUP4spnNAl4Adq3jsRAREWn1QpZlRfdBWqlw5QK9+UREpEXKTm+yWV0Vpzi16gyZiIiISEuggExERESkYArIRERERArWmi97IQV7ZJMn6N27d9HdEBERKZwyZCIiIiIFU0AmIiIiUjAFZCIiIiIFU0AmIiIiUjAFZCIiIiIFU0AmIiIiUjDdOkkKo1sniSz/mvD2MyLLKt06SURERKQlUkAmIiIiUjAFZCIiIiIFU0AmIiIiUrBCAzIz629mI4vsQ32ZWWZmOxW07ZXN7B0z26SI7deFmQ03s2OL7oeIiMiypNbTX8xsNNALmA8sAqYCzwHXuvvLDdm4u1/ckPVboZOB5939LQAz2xX4GzAnlc8EngJ+6+6fpTpdgPOBA4DVgSnAQ8Dv3H1aqtMD+ACYS3yNFwDvAY8C17j7jFIHzOw24vthE2CIu/cr6+NA4O9mdq+7z2vEfRcREVlu1TVDdqG7d3T3zsBuwDjgBTM7oLoVzKxtY3RQIjNrA5wI3FpWtNDdO7h7B2AnYrB0bVqnA/As8F3gB0CH9Pu7wLOpPG8Td+8IrAGcBOwBuJmtlqvzGnAa8Eilfrr7m8C7wKFLt6ciIiKtT70vEOPu44ABZrYWcL2ZjXD3zMyqgDuIAdu2QD8z+x9wHbAZ0AZ4ATjR3d8DMLOBwE7uvmd6XgXcQgwEtgOqgOPd/Z+V+mJmW9bS/pC0/Evgp8RM0unA/4iBzbcBBw5390/M7JfACe6+ZW4bGwJvARumfc9vvxtwG7AN0I4YrJxSyhya2XeB64HNgYXAm8B+7j7NzFYEzgT6EgOgscDJ7u7VHHoDugAVjwWAu79vZo8C+6RFpwBrAzuXsmHAWDP7MTEDdgowqEI7XwL/NLOfpD6fBpyTyq5L+1ZTwPUMsD/x/SAiIiK1aMgcsuHAOsShq5LjiH+8OwIPAxlxCGsdoAcwGxhaS7vHELMznYl/2O+qoW5d2j8IeBD4JnAhMRC7gDiEt2Zq4/xUdxiwoZltm1v/WGBUeTCWrADcBKwHfAv4N/DnXHbwRuDptO01icdmfio7H/gJMWO1GjF4eTINMVayNfC2uy+sphwz6wn0Bv6VFv0QeCwXjAGQnj8G7FtdW6neZ8TXYI+a6lXweuqviIiI1EFDLqH8cfqdH8661d1fSY/nETNGJV+Y2fnA62bW3t3nVtPuYHcfC1/NVzrFzDrn5zGVuHtd2v+ruz+W2rsb+ANwj7t/nJY9QAwkcfeZZjacGIT9Kw0THk0MEJfg7h8CH5aem9mAVHcj4A1i8LUu0N3dq4gZPMwspHr7ufv7afXbzewUYD8qB61diHPEyrUxs+nEwHI6cQ7Z2amsK/CPSn0HPgG2r6Ys72Pge3WolzeTGISKiIhIHTQkIOuWfk/NLavKV0jDfVcQhx87EoMGiIFCpYwTwITc49Jk9Y7AEgFZHdv/qj13n2tm5duYm9YtGQyMMrPTiJmhFalmvpSZrQ5cDewKrEqcEF/aPsDPgXOBMWb2JTHQOp8YXHUARppZ/vZBbfn6uJabBnSqsHyhu69azTqTidnDStZO5bXpxuKvcV10Aj6r5zoiIiKtVkOGLPsA44nzq0oWldW5GZgFbOHunYAd0/KK93FaCo3evrv/izi/6mBipmxImlNVySXAWsB2afvd89t39w/c/Rh37wb8GOgHHEU803EOsKe7r5r7+Ya7X1rNtl4BNk5Zu7p6Evihma2aX5ie/xB4oqaV0/DpXsBf67FNgO8Q+ysiIiJ1UO8MmZl1JwYWfYE+7l7TDaI7Ae8A01M26YKl6WQB7d8C/JY46f+MWrY/F5iWzli8LF9oZkcDz7j7J8ThxAXEjFZmZr8HrjSzfu7+Tlp/R+D1VL/cv1IbvYAxddyPa4lnOz6cTlh4C9iYOGw7Cfh9pZXSCQcGXEoMeK/OlbUjBvJtgMzMVgYWufv8XBN7AXfWsY8iIiKtXl0zZOea2Swzm0mck9QT2MHdH6xlvVOBnYlzip4lXteqMTVV+8OA9YHn3P2dGuqdRzxDcipxvtw/iWdTluwOvGxmc4DngXuBe1LZ74gnPjycjus7wAlU85qkyfw3EIPhOnH3maQgj3hywRziJP2xwI6pPO8tM5tFzODdRDym27j7lFydp4nzA48gBuXz0jIA0kVrN0r7KiIiInUQsqymBFfrlCbdvw+c4+4tJrAws1WIgd+PSheHbWnM7D7gL+5+W211w5UL9OYTWc5lpzdkqrLIcqnitCp9Uio7nHhdsQeK7kheuvL9RkX3oyburgvCioiI1JMCsjJmNpk41+uYsnlRIiIiIk1CQ5ZSmJEjR2a9e/cuuhsiIiLNqeKQZUMueyEiIiIijUABmYiIiEjBFJCJiIiIFEwBmYiIiEjBFJCJiIiIFEwBmYiIiEjBFJCJiIiIFEzXIZPC6NZJ0lLpdj8i0oR0HTIRERGRlkgBmYiIiEjBFJCJiIiIFEwBWRMws/5mNrIB6/c1s3cbs0/13P7hZvZqUdsXERFpbVrtzFUzGw30AuYDi4CpwHPAte7+ckPadveLG9zBZmJmQ4AF7t6vtMzdhwHDCuuUiIhIK9PaM2QXuntHd+8M7AaMA14wswOqW8HM2jZb70RERKRVaLUZsnLuPg4YYGZrAdeb2Qh3z8ysCriDGLBtC/Qzs/8B1wGbAW2AF4AT3f09ADMbCOzk7num51XALcAewHZAFXC8u/+zLn0zs/bAJcBPgVWAMcBJ7v5hKm8LnAEcDawNTALOcvcHzGwP4GJgY2AB8Je07iQzOxM4PLVxSNpcZ+BIYIC796zj9kcDLwM9gL3T9k9z94frsn8iIiKtXWvPkFUyHFgH2CS37DjgNKAj8DCQAQNTvR7AbGBoLe0eA5xEDHieAe6qR5+uAbZPP+sBU4CRZtYmlQ8CjgAOBjoBuwBvp7IvgBOBrsDmxIDt9wDufjlxaPIud++QfhYuxfYhBoNXpf27AbgrBXIiIiJSC2XIlvRx+r1abtmt7v5KejwPeC1X9oWZnQ+8bmbt3X1uNe0OdvexAGZ2G3CKmXV29xk1dcbMViAGO73dfXxadgrwGfA9M3sB+DXQx91L/fq4tB/uPibX3EQzu5yY8auT2rYPPJ+q/rGU8TOzW4CrgY0AnRwgIiJSCwVkS+qWfk/NLavKVzCzDYEriMOPHYkZM4hZqHHVtDsh93hO+t0RqDEgS22uBHxQWuDus81sEtAdeA/4Bl9nxBZjZtsQhyy3BNoTrxDcoZZt1mf7pYBsQq58jplB3D8RERGphYYsl9QHGA+8lVu2qKzOzcAsYAt37wTsmJZXvB1CA00mDjv2KC0wsw7AGsBHqXwuMRtVyXDg38DGqa+HlpWX71t9ty8iIiINpAxZYmbdgX5AX+LwX033WewEvANMN7PVgQuaql/uvsjM7gYuNLM3gOnEuVpvAi+lEw9uAi43sw+BscS5bd9MQ5idiFm4WWa2LnB22SYmAtub2QruvkRwVtv2G3+PRUREWp/WniE718xmmdlM4B9AT2AHd3+wlvVOBXYGZgLPAo82bTc5FXDgX8CHwFrAj3MT8M8B/gSMIGbuRhP3BeB4YqA5C/gzcH9Z27cRhzynmtn0son6dd2+iIiINEDIspoSQSJNJ1y5QG8+aZGy0zV4ICJNpuL0ptaeIRMREREpnAIyERERkYIpIBMREREpmCZKSGEe2eQJevfuXXQ3RERECqcMmYiIiEjBFJCJiIiIFEwBmYiIiEjBFJCJiIiIFEwBmYiIiEjBFJCJiIiIFEwBmYiIiEjBdC9LKYzuZbns0T0eRUQaTPeyFBEREWmJFJCJiIiIFEwBmYiIiEjBFJAtY8ysv5mNLLofIiIi0ng0qb8JmNlooBcwH1gETAWeA65195cL7Bpm1hcY4O49i+wHaFL/skiT+kVEGkyT+pvZhe7e0d07A7sB44AXzOyA6lYws7bN1jsRERFpMZQhawIpQzbK3QeVLb8d2Afo7u6ZmVUBdxADtm2BfsD/gOuAzYA2wAvAie7+XmpjILCTu++ZnlcBtwB7ANsBVcDx7v7PCv3qBfwNaAfMTYt/BFwGDHf3a3J1LwB2dPc90jZ3Bl4DjgLmATe4+6W5+t8BrgK2TuXDgPPc/cvqjpMyZMseZchERBpMGbIWYDiwDrBJbtlxwGlAR+BhIAMGpno9gNnA0FraPQY4CegMPAPcVamSuz8PnAC87+4d0s9oYDBwbKmema0A/By4Nbf694FPgbWAnwCnmdlhqf4awN+BP6d+9wL2Av5fLf0WERERQP/uNq+P0+/VcstudfdX0uN5xCxUyRdmdj7wupm1d/e5VDbY3ccCmNltwClm1tndZ9SxX8OBa8xse3d/gZjFWwV4KFdnAnCZu2fAy2Z2C9AXuJeYNXvV3QenuuPN7BJi5u2COvZBRESk1VJA1ry6pd9Tc8uq8hXMbEPgCuLwY0dixgygK3EeWiUTco/npN8dgToFZO4+18yGEodMX0i/73b3L3LVxqVgLN/vn6bH6wM7mtn0XHkgDrmKiIhILRSQNa8+wHjgrdyyRWV1bgY+AbZw96lpbtbrVDPmvBTKt1cyGHjOzC4CegNblZWvZ2YhF5T14OuM3zjinLn9GqmPIiIirYoCsmZgZt2JWae+QJ+yTFO5TsA7wHQzW53GH/KbCKxhZp3cfWZpobu/ZmZjgQeBl9z9jbL11gLOMLNrgO/w9dw3gLuB35rZMcQhzPnEgG1jd3+ykfsvIiKy3NGk/qZzrpnNMrOZwD+AnsAO7v5gLeudSjyjcSbwLPBoI/frb8SJ/x+Y2XQz2yVXNhj4LotP5i95lhiUTUx9+j0x+MLdJxLPFN2fOJQ5jTj/bING7ruIiMhySZe9kK+Y2a7ACGDt/AkE5ZfaaCy67MWyR5e9EBFpMF32QqpnZisDpxPP+qzubE4RERFpAgrIBDP7KfAZsCpwUbG9ERERaX00ZCmF0ZDlskdDliIiDVZxyFLfrlKYRzZ5gt69exfdDRERkcJpyFJERESkYArIRERERAqmgExERESkYArIRERERAqmgExERESkYArIRERERAqm65BJYXQdssan64SJiLR4unWSiIiISEukgExERESkYArIRERERAqmgExERESkYArIADMbbWYDiu5Hbcysr5m9W3Q/REREpHG1mlOyzMyAAcCOwErAROBx4LIi+1UdMxsI7OTuexbdFxEREWlarSJDZmZ7AWOAt4Ct3L0TsAswNf1ulcwsmFmrCcpFRERaqlZxHTIzewd41t2PqaZ8NPAy0APYG5gEnObuD+fq7A+cC2wITAAGufuwVNaXmH27Efgt0BkYDFwC3ALsBXwC9HP3MWmdFYH+QF+gC/Bv4GR3/6+Z9QHuIQbMn6cubAF8P23nOuBM4BvAn4BfufvC1O66wNXATkAGjAR+6+6zUnkGnAIcCWwG7JbauRjYGFgA/AU4yd0n5Y7Pf4CewK7AOOB0d3/CzLqkfdvB3V/JHa9/AM+4+4WVjjnoOmRNQdchExFp8VrndcjMbGNiIHFvLVWPBq4iBlM3AHeZWfvUxl7A7cRA5pup7g1m9v3c+usBqwIbEIOh3wBPAFcQA64/A3fm6p8BHAX8EPgW8CzwjJl1cvc/EgOk0e7eIf28n9vOmsTAcFvgYOCQ1M+Vgb8CbwDrA5sC3YDfl+3rsUAfoAPwCvAFcCLQFdgcWLuadX6f9vFi4CEz6+Hu04D7gX6liumY9wLuQERERGq13AdkxCADYHwt9f7o7v9090XErFZnYKNUdjLwe3d/1t0XuftLwFBiQFUyDzjf3ee7+6vAq8C/3P2FlL0aCvQ0s86p/s+By9z9TXf/ArgAWAjsV0s/5wHnufsX7v4uMZtlqexHQHD389x9XgqWzgUON7M2uTaudPf33H1hameMu//L3Re4+0TgcmCPsu2OcPdnUp1hgAOHpbJbgMNSQAgxeHvS3Ws75iIiIkLrmNQ/Of1eB/hfDfUmlB64+5x4DgAd06L1gd3M7LRc/TbErFbJpBTMlczNt5mel9qcAXQHPshtc5GZVaXlNZlUGp5M5pT1c10zm162TkbMwpUCpKp8oZltQ8x6bQm0J6ZTO5S1UVXhebfU9zFm9glwkJkNJ2YQj69lP0RERCRZ7gMyd387XSriUGDUUjYzDhji7lc0Xs/4iDhnDQAzWyE9/ygtWrTkKrUaB7zt7pvVUq+87eHAA8DB7j7TzH5EnHuW16PC88dzzwcTM2OziZm+x+rebRERkdZtuQ/Ikl8BI83sU+AGd//EzNYEjiGXparBtcAQM3sB+CcxO7Y5cXjQl7JPQ4Az0+T3KuAs4utRCmQmErNd7dx9fh3bfBS4yMz6A9cTg6O1ge+5+0M1rNeJmLWblU4KOLtCnf3NbA9gNPAz4jDpkbnye4gnMfwOuLMsiyciIiI1aA1zyHD3Z4gT7TcFXjezWcTLYKxBDDBqW/9p4DjiBP0pxKHIa1hyWK8+rgDuA54GPgV2B/Z295mp/H5itmyimU03s/Xr0M+5qZ1NgTeJQdZfgK1qWfV44qT8WcSTD+6vUOd24LTU5nnAge6eH3KdRsyybZnqioiISB21isteSMOky16McvdBtdQbSLz8xd51aVeXvWh8uuyFiEiLV/GyF/r2lkaRhoCPQ5P5RURE6q1VDFlK0zKzq4H3gZHursn8IiIi9aQhSymMhiwbn4YsRURaPA1ZSsvyyCZP0Lt376K7ISIiUjgNWYqIiIgUTAGZiIiISMEUkImIiIgUTAGZiIiISMEUkImIiIgUTAGZiIiISMF0HTIpjK5DVj1dT0xEZLlV8TpkypCJiIiIFEwBmYiIiEjBFJCJiIiIFEwBmYiIiEjBFJAt48zscDN7Nfd8tJkNqGv9xtimiIiINIzOsiyAmZ0DDAL6uvtd9VhvCLDA3fvVUGc0MMrdBzW0n3Xd5tLSWZbV01mWIiLLLZ1l2RKY2QrAccBnwPEFd0dERERaAGXImpmZ7Qs8AuwPPAps7u7/TWUZ8Gvg58C3gbHELNqbZnYmcFFq5ov0uzNwJDDA3XumNkYD/wF6ArsC44DT3f2JVN63rH574BLgp8AqwBjgJHf/sB7bbAucARwNrA1MAs5y9wdqOhbKkFVPGTIRkeWWMmQtxPHAE+7+GPAa8Iuy8r7AgcDqwEfA9QDufjkwDLjL3Tukn4XVbONY4PfAqsDFwENm1qOautcA26ef9YApwEgza1OPbQ4CjgAOBjoBuwBv13QQRERE5Gv6N7wZmdnawI+IgQvA7cD5Znamu89Ly65w9w9T/SHA0KXY1Ah3fyY9HmZmvwQOIwZn+f6sQMxq9Xb38WnZKcTh1O8Bz9dhnwIxq9fH3V9Liz9OPyIiIlIHypA1r2OJwc6j6flQ4jBhn1ydCbnHc4COS7GdqgrPu1Wo1xVYCfigtMDdZxOHHLvXcVtdgW+gjJiIiMhSU0DWTFI26ljiMOLHZjYReANow5LDltVZVMd6PSo8r5SxmkycG/ZVfTPrAKxBHC6tyzYnA3OBjerYNxERESmjIcvm8wNi1ul7wPjc8i2BJ81s8zq0MRHY3sxWcPeaAqX9zWwPYDTwM8CIE/EX4+6LzOxu4EIzewOYDlwFvAm8VJdtuntmZjcBl5vZh8QTEdYBvpkbwhQREZEaKCBrPr8gzu16uWz5RDN7nrplyW4D9gCmprlbq1VT73bgNOBhYqbrQHf/oJq6pwKXAv8iDl/+E/hxbvJ+XbZ5DjALGAF8ixjEnUk8aUFERERqocteSGF02Yvq6bIXIiLLLV32QkRERKQlUkAmIiIiUjCNi0hhHtnkCXr37l10N0RERAqnDJmIiIhIwRSQiYiIiBRMAZmIiIhIwRSQiYiIiBRMAZmIiIhIwRSQiYiIiBRMAZmIiIhIwXTrJCmMbp30Nd0qSUSk1dCtk0RERERaIgVkIiIiIgVTQCYiIiJSMAVksgQzG2tmfYruh4iISGux3M8kNrPRQC/gS2Ah8AFwkbvfX2S/amJmPYj97O7uHzf3dtx9s6bapoiIiCyptWTILnT3DsBqwBDgXjPrWWyXGsbMgpkt9wG1iIhIa9Cq/qC7+wIzuxW4BtgKeNfM9gfOBTYEJgCD3H1YaR0z2wUYBGwGLAIedfe+ubLLgW+nda9x98GpbFdgFHA4cDGwOvAUcKy7zzKzkNr9OdARmApc5e7XA6+mzb9lZhlwmbtfmB6fAhyZ+rObmZ0ALHD3frk+VwED3H1oLftQ3XYqrb9U+1m3V0ZERKR1ay0ZMgDMrB3wy/T0bTPbC7idGOR8EzgauMHMvp/qb0EMLm4H1gK6EzNsmNn6wJPAH4iZt77AJWZ2cG6TbYC9gS2BjYHvAielsr3S9rZz947A94AxqWzL9HsTd+/g7hfm2jwW6AN0AF6pwz5Xuw+1bKe0fkP3U0RERGrRWjJk55jZ6cRM1JdAP3d/zcweBX7v7s+mei+Z2VDgKOAfwAnASHcfkmtrdPp9KPDvXNkLZjYY6Afk56ed7e6zgdlmNgKwtHw+sDKwmZlNdvdJwKQ67MuV7v5eerzQzGqsXMs+1EVD91NERERq0VoyZBe5+6rE4bTHgd3S8vWBs8xseumHmAFaO5X3AN6ups3uxAnxee+l5SUL3X1y7vkcYlCIu48G+gMDgElm9rTVIboCqupQJ68H1e9DXTRoP0VERKR2rSVDBoC7TzOzfsB7ZvYTYBwwxN2vqGaVKmCjaso+An5YtmyDtLyu/bkFuMXM2gMDgT8D6xLneVWnvGwWMdAEIE30XyNXXkX1+1DTdkoavJ8iIiJSs1YVkAG4+2dmdjVxAvqpwBAzewH4J3Eu1OZAcHcHBgMvmtmRwJ+IGcXtUnbrPuBcMzsKuBfYGvgFX89Rq5GZfQ9YCXgJ+IIYWC1MxZOJwdJGQG2XvXgZuDzN9foEuABomyuvaR/qsp0G7aeIiIjUrrUMWZb7PXGC+9rAccAVwBTSGYTECfO4+6vE7NAvgU+BD4lnOOLuH6SyE4lnSN4DnOvuf6pjHzqkfkxJ6+9NnKyPu88jnvl5XxpKPaeGdoYBjwD/Jg4lfgiMLxXWsg+1bqcR9lNERERqEbIsK7oP0kqFKxfozZdkp7e6ZLWISGsVKi1srRkyERERkRZDAZmIiIhIwRSQiYiIiBRME1ekMI9s8gS9e/cuuhsiIiKFU4ZMREREpGAKyEREREQKpoBMREREpGAKyEREREQKpoBMREREpGAKyEREREQKpoBMREREpGC6l6UUZnm6l6XuRSkiInWke1mKiIiItEQKyEREREQKpoBMREREpGAKyKQiM9vZzKYX3Q8REZHWQDORC2Bmo4FewHxgETAVeA641t1fbqRt9AUGuHvPpVnf3Z8FVm2MvoiIiEjNlCErzoXu3tHdOwO7AeOAF8zsgIL7JSIiIs1MGbIWwN3HAQPMbC3gejMb4e6ZmVURs1xDAcysB/AB0N3dPzazABwH/AZYD5gBXAa8DNwMtDOz2WkzP3L30fntmtlKwPXA/sDKwKdAf3e/38x2BUa5+4pmtgbwflm32wOnufu1ZtYeuAA4EOgMvASc6O7vNsoBEhERWc4pQ9ayDAfWATapY/0TgIHAL4nDi98FXnT351PZ++7eIf2MrrD+0cC2wP+5eydgd2BseSV3n5Rrp0NqewrwaKpyK/BtYHvgW8CLwKNm1raO+yEiItKqKUPWsnycfq9Wx/q/AS5y9zHp+ZT0U1fzgQ7Apmb2vLt/VNsKZrY7cAPwA3d/18xWBw4D1nP3T1Od84FTgO2AMdW1JSIiIpECspalW/o9tY71ewBvN2B7Q4E1gWuAjczsL8CZ1Q01mtnmwANAX3d/IS1eP/1+zczy1dsC3RvQNxERkVZDAVnL0gcYD7yVns8CvpErX7usfhWwEfBMhbYW1bYxd19AnHN2mZmtSsx83QF8v7yumXUDHgfOc/cRuaJx6fdG7j65tm2KiIjIkhSQtQBm1h3oB/QF+rh76R6PLwOHmtkwYBXg3LJVbwT6m9krxHlb3wTWd/d/AROBNcysk7vPrGa7uxNPBHgNmAfMARZWqNeJGIzd5+435MvcfZKZ3QvcZGanuPv4FNztBjzj7rPL2xMREZHFaVJ/cc41s1lmNhP4B9AT2MHdH8zVGUAMkCYAo4mT/vNuAi4BbgdmAv8mTtIH+Bsxc/aBmU03s10q9GFN4B5gWtrGesDxFeptDWwO/MrMZud+fpXKjyNm9Uab2SzgdeBgYLm5ebiIiEhTClmmv5lSjHDlguXmzZedrmSziIjUSai0UBkyERERkYIpIBMREREpmAIyERERkYJp4osU5pFNnqB3795Fd0NERKRwypCJiIiIFEwBmYiIiEjBFJCJiIiIFEwBmYiIiEjBFJCJiIiIFEwBmYiIiEjBdOskKcyyeusk3SZJREQaQLdOEhEREWmJFJCJiIiIFEwBmYiIiEjBFJCJiIiIFEwBWTMxs/5mNrLoftSVmc02s15F90NERKQ10FmWOWY2GugFzAcWAVOB54Br3f3lArvWZMxsV2CUuzf7qYM6y1JERFohnWVZRxe6e0d37wzsBowDXjCzA6pbwczaNlvvREREZLmjDFlOypCNcvdBZctvB/YBurt7ZmZVwB3EgG1boB/wP+A6YDOgDfACcKK7v5faGAjs5O57pudVwC3AHsB2QBVwvLv/s5q+fRe4HtgcWAi8Cezn7tMq9dvMMmBndx+Tnv8U6A/0BD4HbgduBN4DVgbmpFV/7e53VVj/QOA8oEfq60B3fyiV9QUGpP0/E/gG8CfgV+6+sLrjrQyZiIi0QsqQNcBwYB1gk9yy44DTgI7Aw0AGDEz1egCzgaG1tHsMcBLQGXgGuKuGujcCTwPfBNZM255fl86b2b6p7YHA6sDGwBPu/gmwL7DQ3TuknyX6YGY7AMOAs4HViIHdfWa2Xa7aeqlfGxKD1IOBQ+rSPxERkdZO/+rXzcfp92q5Zbe6+yvp8TzgtVzZF2Z2PvC6mbV397nVtDvY3ccCmNltwClm1tndZ1SoOx9Yl5ilqyJm4OrqN8DN7v5oej4TGFOP9fsCD7r7E+n5Y2b2EDGgfDEtmweclzJi75rZXwAjBnIiIiJSAwVkddMt/Z6aW1aVr2BmGwJXEIcfOxIzZgBdifPQKpmQe1waMuwIVArIfg6cC4wxsy+J2bfz3X1BHfrfA3ioDvWq0x0oP6nhPWDr3PNJZcOTc4j7IiIiIrVQQFY3fYDxwFu5ZYvK6twMfAJs4e5Tzew7wOtUM1ZcX+7+ATEjhZltThy+/IA4l20Wcd4WqXztstWrgI2qabp8Pyr5iBjU5W2QlouIiEgDKSCrgZl1J07Y7wv0cfeaJqF3At4BppvZ6sAFjdyXo4Fn0ryv6cAC4uR+iNmrPmZ2NXHC/kVlq98IDDezvxHnqrUnBo5jgIlAGzNbPwV9ldwFjDKze4BRwN7AT4FdG2n3REREWjVN6l/SuWY2y8xmAv8gnpW4g7s/WMt6pwI7E+dnPQs8WnP1etsdeNnM5gDPA/cC96Sya4hneb4H/Ad4LL+iuz8GHAtcDHxGzPTtk8reBv4AvGRm083syPINu/tzwNHAlcA04HLgCHevzzw2ERERqYYueyGF0WUvRESkFdJlL0RERERaIgVkIiIiIgXT2IsU5pFNnqB3795Fd0NERKRwypCJiIiIFEwBmYiIiEjBFJCJiIiIFEwBmYiIiEjBFJCJiIiIFEwBmYiIiEjBFJCJiIiIFEy3TpLCLIu3TtJtk0REpIF06yQRERGRlkgBmYiIiEjBFJCJiIiIFEwBmYiIiEjBFJA1IjMba2Z90uMeZpaZWbf0/HAze7XYHoqIiEhLpFPG6sjMRgO9gC/ToonADe5+bamOu29W3fruPgwY1oRdrJe0P6PcfVAjttkD+ADo7u4fN1a7IiIiyztlyOrnQnfv4O4dgCOAi8xsr+bYsJm1bY7tiIiISPNThmwpufsLZvYGsDnwDICZVQED3H1oeX0z65vKeqbnhwD/D1gfmAM8Apzm7nNybd0B7AZsCxxnZncAO7j7K7l2/wE84+4XVtjmnsAVwIbAfOA/7r6nmd0A7Az0MrOzgfHuvkla5zjgZKA78D5wlrs/ncoGpvVeBPqlzfzB3X+XHpeGZN8yswy4rFK/REREZHHKkC0FMwtmtiPwbeD5pWxmBnAYsCoxyNkZGFBW5zjgNKAjMAK4n68DIcxsY+Iw6h3VbONu4DqgM7AOMAjA3U8EnuXrjF8+GDsLOBzoApwD/NnMeuba/D7wIbA28GOgfzoWAFum35ukdhWMiYiI1IEyZPVzjpmdDrQDVgEGAy8tTUPu/kTu6btmdhNwVFm1W3PZsHlmdgsw0sx+6+6fA8cCT7r7+Go2M5+YHVvT3ScCo2vp1snABe5eynQ9bmZ/Aw4hBXPA2+5+c3r8gpn9BzDguVraFhERkWooIKufi0qT4NPZk/cSs1NH17ehNPfsPGKWbSWgDTCprFpV/om7jzGzT4CDzGx42u7xNWzmJ0B/4HUzmwzckj8JoYL1gRvN7LrcshWB/AT9CWXrzCFm8ERERGQpKSBbSu7+sZn9CbiEegZkZtaOOAR5JnCHu88zsxOB08uqLqqw+mBiZmw2sBB4rIY+vgr0MbMA7AQ8bWavuftfq2l7HPA7d7+/PvtTS39FRESkFgrIlpKZfQs4mK8nstdHO2JWbFoKxjYFTqzjuvcQg8DfAXe6+8Jq+tcOOBR4zN2nmNk0YsBUqj8R6Fm22jXAQDN7h7hfKwPbAFPc/c069G1y2sZGLJ5VExERkRpoUn/9nGtms81sNjFg+ZQ4Mb9e3H028Evg8tTWjcThz7qsOw14gDiB/vZaqvcB3kzbeISY/fp7KrsGMDObbmZjU9u3ApcDdwLTiJP3zwXqdMkNd5+X6t+X2j2nLuuJiIi0diHLsqL7IPWULj+xg7vvXXRfGiJcuWCZe/NlpyupLCIiDRIqLdRfl2WMma1JvBxGTZP5RUREZBmiIctliJldTbxY60h3r3Yyv4iIiCxbNGQphRk5cmTWu3fvorshIiLSnCoOWSpDJiIiIlIwBWQiIiIiBVNAJiIiIlIwBWQiIiIiBVNAJiIiIlIwBWQiIiIiBVNAJiIiIlIwXYdMCtMSb52kWyOJiEgT03XIRERERFoiBWQiIiIiBVNAJiIiIlIwBWQCgJktMLNd0+PDzezVgrskIiLSamgGcwthZqOBUe4+qOi+uPswYFjR/RAREWktlCETERERKZgyZC2Qmd0J7AmsCnwEDHL3e1PZrsRM2oq5+gOBndx9z/Q8A34N/Bz4NjAW6Ovub6byjsANQG9gFnBe2fb7AgPcvWd63hY4AzgaWBuYBJzl7g+k8uOAk4HuwPup7OlGPCQiIiLLNWXIWqYxwFbEgOwCYIiZbVrPNvoCBwKrE4O663Nl1wIbAZsCWwA/AdrU0NYg4AjgYKATsAvwNnwVjJ0FHA50Ac4B/mxmPevZXxERkVZLGbIWyN1vzz0dbmanA7sCb9SjmSvc/UMAMxsCDE2PVyAGT/u5+8S07CzggEqNmFkgZtv6uPtrafHH6QdiZuwCdy+dBPC4mf0NOIQYyImIiEgtFJC1MClgGgj0Ab4FZMA3gK71bGpC7vEcoGN63BVYCajKlX9QQztd0/bfrqZ8feBGM7sut2xFvg7YREREpBYKyFqeQ4F+wN7AG+6+yMycr2+1MAtoY2YrufsXadna9Wh/CjAf6AG8l5b1qKH+ZGAucYjznQrl44Dfufv99eiDiIiI5Cgga3k6AQuIgdAKaYL9lsCjqfxtYDbQz8z+AOwAHAT8uy6Nu/tCM7sXON/M/gvMAy6toX5mZjcBl5vZh8QTBNYBvpmGMK8BBprZO8CrwMrANsCU0kkEIiIiUjNN6m9ZMuAu4EXgXWA8ceL9s6UK7j6LePbkb4EZxDlcd9VzOycThynfBF4HRgILa6h/DvAnYAQxQzca6Jn6cytwOXAnMA34EDgXaFvPPomIiLRaIcuyovsggJn9Gxjs7oOL7ktzCVcuaHFvvux0JY1FRKRJhUoLlSFrAcxsK+A7wAsFd0VEREQKoHRAwcxsOLAT0D936QgRERFpRRSQFczdDym6DyIiIlIsBWRSmEc2eYLevXsX3Q0REZHCaQ6ZiIiISMEUkImIiIgUTAGZiIiISMEUkImIiIgUTAGZiIiISMEUkImIiIgUTAGZiIiISMF0L0spTEu7l6XuYykiIs1A97IUERERaYkUkImIiIgUTAGZiIiISMEUkNXCzKrM7Iii+9GczGxnM5tedD9ERERaixYxi9nMHgZmuPtRFcr+Box19xObv2fLPzMbCOzk7nuWlrn7s8CqRfVJRESktWkpGbLBwEFmtmp+oZltBOySypc5Zta2MeuJiIjI8qlFZMiAJ4HJwJHA9bnlxwMvuvvrZnYxcAiwBvApcL27XwtgZiul9fYHVk7l/d39/lS+CzAI2AxYBDzq7n3NbFdglLt/dRwqZYxyZe2BocAOQHvgXeAsd38mlfcFBhADyJOBGWmb5e1UAXcAuwHbAv3M7AHgTKBv2sexwMnu7mmdPYErgA2B+cB/Sn1M/boAOBDoDLwEnOju76bytsAZwNHA2sAk4CygDdAfWMHMZqfubQGsmz8uZrZiqtcX6AL8O/Xtv6l8SGrrc+BgYA5wgbsvk4G0iIhIc2sRGTJ3XwTcBhxXWmZm7YgBROmP+hvATkDHVO8SM9snlR1NDGz+z907AbsTAxrMbAvgKeB2YC2gOzBkKbu6AvBnYCNgNeA+4EEz65qr04MY9GyU+lSd44DT0v48DJwP/AT4QWr7DuBJM+uS6t8NXEcMuNYhBpgltwLfBrYHvgW8CDyay7wNAo4gBkudiFnHt939j8DFwGh375B+3q/Q1zOAo4AfpvafBZ4xs065OgcBI4FvAr8BbjCz9WrYfxEREUlaSoYMYsB0nplt5+4vAgcAbYE/Arj70Fzdv5rZY8AexGBrPtAB2NTMnnf3j3J1TwBGuvuQ3LLRS9NBd59NzJCVXGFmZxEDr8fTsi+Bs939i1qau9XdXwEws8+Bk4D9cgHR7WZ2CrBf2uZ8YnZsTXefWNoHM1sdOAxYz90/TcvOB04BtjOz54BfA33c/bXU9sfpp65+Dlzm7m+m9i8A+qW+3Zfq/NXdH0mP/5xOCtgKGFeP7YiIiLRKLSYgc/dPzOxR0jBl+j3U3ecBmNlJxKxSN+JVblcB7k2rDwXWBK4BNjKzvwBnpiG7HsArjdFHM1uFOGz4Q2B14vBnRyCfIZtQh2AMoCr3eHViQDnSzPJXr29L3F+I2bP+wOtmNhm4JQ3Zrp/KXzMzytbtnvr2DeDtOvSpOt2BD0pP3H1RGnbtnqszoWydOcRjIyIiIrVoMQFZcgtwv5ndQJxfdTKAme0IXEbMiL3o7gvTnKsA4O4LUvll6cSAG4hDft8nBj4bVbO9WUAbM1spF0StXUP/Tktt7gFUuXtmZlNY/DYIi+q4r/l6U4gBzJ7u/q9Kld39VaCPmQXi0O3TZvYa8N9UZSN3n1y+Xqo/l3gM3qmlH9X5iBjYltpcIT3/qJr6IiIiUg8tLSB7ihicPAg8X5o0Tpz3tJA48T8zs/2AfYHSpP3diRPoXwPmEYObhWndwcCLZnYk8CfiPLDt3H00MWs0mzip/g/EyfoHESetV9IJ+AKYCrRLw5WrNnSnU2D3e+BKM+vn7u+YWQdgR+D1dEwOBR5z9ylmNo0YSC1090lmdi9wk5md4u7jU1C6G/CMu882s5uAy83sQ+LcunWAb6YhzInAumbWzt3nV9PFIcCZZvYPYoB7FvG981hD911ERERayKT+kjS5/1biMNwtuaKniJPaXyIGJwcBD+XK1wTuAaYRh87WIw55ljJLPwR+STz78kPi2Zy4+yzi/KjfEgO6k4G7auji1cB04BPgPWLmqWqpdnZJvyNO7n/YzGYSs1kn8PVr1Ad4M50N+QjwO3f/eyo7DngLGG1ms4hB3MFAafjzHGIwOoKYFRwN9Exl9xMzXRPNbLqZlYZA864gzhV7mngMdwf2dveZDd9tERERCVmW1V5LpAmEKxe0qDdfdnpLSxiLiMhyKFRa2KIyZCIiIiKtkQIyERERkYIpIBMREREpmCbNSGEe2eQJevfuXXQ3RERECqcMmYiIiEjBFJCJiIiIFEwBmYiIiEjBFJCJiIiIFEwBmYiIiEjBFJCJiIiIFEy3TpLCNPWtk3QrJBERaYF06yQRERGRlkgBmYiIiEjBFJCJiIiIFEwBmYiIiEjBFJBJRWbW38xGFt0PERGR1kBnWS7jzGw00AuYDywCpgLPAde6+8sFdq1WOstSRERaIZ1luRy70N07untnYDdgHPCCmR1QqbKZtW3W3omIiEiNlEJYzrj7OGCAma0FXG9mI4APgDuIwdq2QD8z+x9wHbAZ0AZ4ATjR3d8DMLOBwE7uvmd6XgXcAuwBbAdUAce7+z9T+Z7AFcCGxGzdf0rrioiISM2UIVt+DQfWATZJz48DTgM6Ag8DGTAw1ekBzAaG1tLmMcBJQGfgGeCuXNndxACvc2pzUMN3QUREpHVQhmz59XH6vVr6fau7v5IezwNey9X9wszOB143s/buPreaNge7+1gAM7sNOMXMOrv7DGJWbENgTXefCIxuxH0RERFZrikgW351S7+npt9V+UIz25A4xLgdMWtWmmDflTgHrZIJucdz0u+OwAzgJ0B/YlA3GbjF3a9d+u6LiIi0HhqyXH71AcYDb6Xni8rKbwZmAVu4eydgx7S84tkftXH3V929D7AG8AvgEjPbfWnaEhERaW2UIVvOmFl3oB/QF+jj7pmZVaraCXgHmG5mqwMXNGCb7YBDgcfcfYqZTSMGgAuXtk0REZHWRBmy5cO5ZjbLzGYC/wB6Aju4+4M1rHMqsDMwE3gWeLSBfegDvGlms4FHgN+5+98b2KaIiEiroAvDSmF0YVgREWmFdGFYERERkZZIAZmIiIhIwTSmI4V5ZJMn6N27d9HdEBERKZwyZCIiIiIFU0AmIiIiUjAFZCIiIiIFU0AmIiIiUjAFZCIiIiIFU0AmIiIiUjAFZCIiIiIF062TpDBNeesk3TZJRERaKN06SURERKQlUkAmIiIiUjAFZCIiIiIFazUBmZkNMLPRuedjzaxPA9ucbWa9Gty5+m1ztJkNaM5tioiISNNqkTOfU+DUC/gSWAh8AFzk7vc31jbcfbN69GdXYJS7L3a83L1DY/Wnmu1WAQPcfWhTbkdERESK1ZIzZBemgGc1YAhwr5n1LLZLyx8za1t0H0RERFq7Fpkhy3P3BWZ2K3ANsBXwLoCZ3QnsCawKfAQMcvd7S+uZ2X7AFcC6wOjSernyKlL2yczaA0OBHYD2qe5Z7v6Mma0NPAG0MbPZafVfu/tdZpYBO7v7mNTmgcB5QA+gChjo7g+lsr7AAOA64EzgG8CfgF+5+8Ly/Tazkanvt5nZzcA/3X3vVNzFzB4E9gYmAae5+8O5dfcHzgU2BCakYzOsrB+DgZOBGcBmZnYxcAiwBvApcL27X5vW6UHMUh6X+r5mOqbHufskM/slcIK7b5nrw4bAW8CG7j6ufP9ERETkay05QwaAmbUDfpmevp0rGkMM0FYFLgCGmNmmaZ0NgT8DF6fy64jBRHVWSPU3Imbk7gMeNLOu7v4JsC+w0N07pJ+7KvRzB2AYcHZqoz9wn5ltl6u2HjGY2RDYFjiYGAQtwd17Ax8C/dI2984VHw1cBXQGbgDuSkElZrYXcDtwCvDNVPcGM/t+bv0ewNppf7dNy94AdgI6pmN1iZntU9ato4DvA92BRcQglrTfG5rZtrm6xxKHeRWMiYiI1KIlZ8jOMbPTiQHCl8TA5LVSobvfnqs7PNXdlRhYHAK8lJt79bSZjQDWqbQhd5/N18EFwBVmdhYxWHm8jv3tCzzo7k+k54+Z2UPAMcCLadk84LyUEXvXzP4CGDGgqY8/uvs/AczsFuBqYnD1KjHr9Xt3fzbVfcnMhhKDqX+kZV8CZ7v7F6UGy+ap/dXMHgP2AJ7KLT/f3Sem7Z4BvGNma7v7J2Y2nBiE/cvM2hADwZPquV8iIiKtUksOyC5y90Fm1oWY8dkt/cbMVgAGAn2AbwEZcQiwa1q3G3HIMO8DqgnIzGwV4vDmD4HVidmfjrn26qI78HLZsveArXPPJ5UNT85J26mvCaUH7j7HzMi1sz6wm5mdlqvfBng293xCPhgDMLOTiJmxbsSrCK8C3Mviqio87gZ8QhwCHZW2uwfxvfVIPfdLRESkVWrxQ5buPg3oB+xnZj9Jiw9Nyw4Eurj7qsTsUOl2BOOJw3J55c/zTiMOxe0BdE7tTcu1t6gOXf2owjY2SMuXVl22W24cce7aqrmfju7+w+raNbMdgcuAXwCrp/0fyZK3d+hR4fHHAO7+L2IAejAxUzbE3b9civ6LiIjUycCBAzniiCOK7kajaPEBGYC7f0Yclrs4Zcc6AQuAycAKZnYMsGVuleHAdmZ2qJmtaGZ7AvvXsIlOwBfAVKCdmZ1HnHtWMpE4qX/9Gtq4CzjQzPYxszZmti/wU+DOeuxquYnEocj6uBY41cx2Tv1oZ2bbWEqjVaMT8fIik4EsnRCxb4V655rZmmbWiRjAjUpz7EpuAX5LzDTeVs9+i4iILOHee+/FzOjQoQNrrbUW++67L2PGjCmkL1VVVey22260b9+eb3/724waNarR2m7JQ5blfg+cSpwLdRewO/FsyLnAPeSG5Nz9XTM7iBg03Ar8nRggbFVN21cThxY/AaYTg5qqXHtvm9kfiPOx2gK/cfd78g24+3NmdjRwJXHy/jjgCHd/oQH7PAi4Pg0nvuDulYKkxbj702Z2HHEIdhNiNmws8ezP6jwF3A28RBz+fRh4qEK9ocTjvCZxPtqRZeXD0nafc/d3auuriIi0POHKBU3afnZ63UOPq6++mksvvZSbb76ZffbZh3bt2vHkk0/y8MMPs9NOOzVhLys79NBD6dWrF48//jiPP/44Bx10EO+88w5du9ZnhlNlIcuyRuiiLM9yl73o7u4f11AvAO8D5+QvQVKdcOWCJnvz1ecDLyIiX2spAdmMGTNYZ511uPPOOzn44IMr1hk4cCDvvvsuQ4fG89IOPvhgnn32WebNm8eWW27JH/7wBzbbLF4H/vHHH+f000/no48+olOnTpx66qmcfvrpTJkyhb59+zJmzBhWWGEFNttsM/7+97+zwgqLDyK+/fbbbL755kyZMoWOHeO07Z133pnDDz+cE044oT6HoHw6ELCMDFnKMuNwoB3wQNEdERGRZdvzzz/P559/zgEHHFDndfbdd1/eeecdJk2axNZbb83hhx/+Vdmxxx7L4MGDmTVrFv/973/ZfffdAbjqqqvo1q0bkydP5tNPP+Xiiy8mhCVjprFjx7LBBht8FYwBbLnllowdO7YBe/k1pRGkUZjZZOK8vmPcfX7R/RERkWXb1KlTWX311VlxxbqHKsccc8xXjwcOHEiXLl2YMWMGnTt3pm3btrzxxhtsueWWdOnShS5dugDQtm1bJkyYwLhx4+jZsyc777xzxbZnz55N586dF1vWuXNnxo8fvxR7tyRlyKRW7l7l7qGm4Up37+rua+WuwyYiIrLUVlttNaZMmcKCBXUbQl24cCFnn302G264IZ06daJHjx4ATJkyBYAHH3yQxx9/nPXWW49ddtmF559/HoAzzjiDnj17svfee7PBBhtw6aWXVmy/Q4cOzJw5c7FlM2fOXCxj1hDKkElhHtnkCXr37l10N0REpAXq1asXK620EiNGjOCggw6qtf69997Lww8/zKhRo+jRowczZsygS5culObKb7vttjz88MN8+eWX3HDDDfzsZz/jo48+omPHjlx11VVcddVVXw1lbrvttuyxxx6Ltb/ZZpvx/vvvM2vWrK+CsFdffZXDDjusUfZXGTIRERFpcTp37swFF1zAr3/9a0aMGMHcuXP58ssveeKJJzjzzDOXqD9r1ixWWmklVlttNebOnUv//v2/Kps/fz7Dhg1jxowZtG3blk6dOn01af/RRx/l3XffJcsyOnfuTJs2bZaY0A+w8cYbs9VWW3H++efz+eef89BDD/Haa69x4IEHNsr+KiATERGRFum3v/0tV199NYMGDaJr1650796dG264gf3333+JukcddRTrrbce66yzDptuuinbb7/9YuX33HMPPXr0oFOnTtx8880MGxbvWvjOO++w55570qFDB3r16sWvfvUrdtttt4r9GT58OO5Oly5dOPvss3nggQca5ZIXoMteSIFGjhyZachSRERaGV32QkRERKQlUkAmIiIiUjAFZCIiIiIFU0AmIiIiUjAFZCIiIiIFU0AmIiIiUjAFZCIiIiIFU0AmIiIiUjAFZCIiIiIFU0AmIiIiUjAFZCIiIiIFU0AmIiIiUjAFZCIiIiIFC1mWFd0HaaVWWmml/86fP//zovvR0qy44oqrL1iwYErR/WhJdEwq03FZko5JZTouSyrwmEzJsuwHS/SngI6IALD55pt/7u5WdD9aGjNzHZfF6ZhUpuOyJB2TynRcltTSjomGLEVEREQKpoBMREREpGAKyKRItxTdgRZKx2VJOiaV6bgsScekMh2XJbWoY6JJ/SIiIiIFU4ZMREREpGA6y1IanZltDNwFrAZMBY5y93fK6rQBrgN+AGTApe5+W21ly6pGOCZ7AxcDmwPXu/vpzdj9JtMIx+Vc4BBgIfAl0N/dn2q+PWh8jXBMfg6cCiwC2gC3uvt1zbcHTaOhxyVXZxPgFeCmZf1z1AjvlYHAr4BPUvXn3P3XzdP7ptMY7xUz+xlwLhBS+Z7u/mlT9lsZMmkKNwM3uvvGwI3A4Ap1Dgd6AhsBvYCBZtajDmXLqoYek/eBfsAVTd/VZtXQ4/ISsK27bwEcA/zRzFZp8l43rYYekweBLd19K2AH4LdmtkVTd7oZNPS4lP4IDwZGNHVnm0mDjwlwt7tvlX6W+WAsadBxMTMDBgJ7uft3gJ2AGU3daQVk0qjMbA1ga+C+tOg+YGsz61pWtQ/xP/dF7j6Z+AV5cB3KljmNcUzc/V13/w+woFk63Qwa6bg85e5zU73XiP/NrtbUfW8qjXRMZrp7aXJwe6At8T/8ZVYjfa8AnA08CrzdtD1ueo14TJYrjXRcTgWudPeJAO4+w92b/CLmCsiksXUHxrv7QoD0+5O0PG9dYFzu+Ye5OjWVLYsa45gsjxr7uBwFvOfuHzdBX5tLoxwTM/uxmY1Nda5w99ebtNdNr8HHxcy2BPYBrmny3jaPxvr8HGJmr5nZ02bWqyk73Ewa47hsCmxgZv8ws3+b2QAzC03cbwVkIrLsM7NdgAuBQ4vuS0vg7o+4+2bAxsCRad5Uq2VmbYmXODih9IdagDi0t34a8r8CeNjMltkMcyNqA2wB7AXsAuwLHNnUG1VAJo3tI2CdNFejNGdj7bQ870NgvdzzdXN1aipbFjXGMVkeNcpxSf/VDwX2d/e3mrTHTa9R3yvu/iFxnt2PmqS3zaehx2UtYEPgcTOrAk4BjjOzFnUdqnpq8HvF3Se6+5fp8TNp+XeauN9NrbH+Bj3g7l+4+yzgYeB7TdprFJBJI3P3ScB/+DpTcSjwShqjz7uf+IW4Qhrb3x94oA5ly5xGOibLncY4Lma2LfBH4CB3/3dz9LspNdIx+b9SJTNbHdgNWKaHLBt6XNz9Q3df3d17uHsP4Fri/KHjm6P/TaGR3ivrlCqZ2VZAD2CZ/qemkb5v7wX2NrOQsqt7AK82dd912QtpCicAd5nZecA04twezOxx4Dx3d+AeYDugdCryBe7+QXpcU9myqkHHxMx2AoYDnYBgZocAxy7rl3ig4e+Vm4BVgMHxxCgAjlzG50w19Jgcny6T8iXxJIcb3P3p5tyBJtLQ47I8augxudjMtiFeNmY+8bMzsTl3oIk09LgMBwx4g3j5mKeA25u607pSv4iIiEjBNGQpIiIiUjAFZCIiIiIFU0AmIiIiUjAFZCIiIiIFU0AmIiIiUjAFZCL1EELYJ4TwbO75riGEqgK71GxCCENCCLc1Yns9QghZ7nnXEMK4EMLqdVj3hBDCPY3Vl2VBCGHnEML0ovvRGoUQjqjP57yxPytSs6b6bCzF635pCOHCpd2eAjKROgohBOJ98H5XS71fhhD+G0KYGUKYFkLwEEKfXHlVCOGICustsTxEb6e2OpSV7RpCyEIIs9PPJyGEO0MI32zYnhYjy7LJxAsy1nZ8vwFcAAxshm61GFmWPZtl2apF96M6IYSBIYRRRfejNWiqYx1CGB1CGNDY7Ta18s9Gge/Fy4BfhxDWqbVmBQrIROpub6Ad8LfqKoQQDiUGFMcCnYm37DiVeHHCpbEbsAHx4oSV7tO4MMuyDlmWdQB2AnoRr0K+rLoD+HkIoVMNdY4AXs+y7L1m6tNiQghtQgj67hSRxWRZNg14AvjF0qyvLxVpkVK2aEAI4W8p+/N6CGGLEMKhIYR3QwgzQgi3hRBWzK2zbgjhgRDCxBDChBDCLSGEjrnyi0MI76f23gshnJIr65GyTUeGEN4IIcwKITwdQlgr1639gVFZzVdT3gH4R5ZlL2bRvPTf29JeKf0XwJPEq0rX+CHPsux94FHgu+VlIYQV0zHZv2z5kBDCnenxHiGEF1NWb3IIYXgIYY3qtpeO106557uGEBaUbbN/yvBNDyE8F0Kwyq19tQ/vAFOAPWuotj/wTFlfTg4hvJletw9DCJeEENqksitCCCPK6u+a6n4jPf9OCOGptN+l9dumstJ749gQwhvAXGCNEMIhIYRXU/ZyQghhcKm9tN63Qggj03v17bR+FkLokatzXMqmzgghvBJC2Lu6na5wfIeEEO4JIdyRju/49PnYKoTwr7R/fwshrJ1bpyqEcF4IYUz6HHgIYdtceY3vgRBC2/SavpXafy+EcFCIGeD+wK7h64ztBtXsxy5pGzPSa/aLXNmuIYQFIYQ+qe0ZIYQ/5T/HFdpbmu+KLUIIf037+X5av02u/Hvp2MwOIYwh/lOU32b7EMKVIYQPQgifhRCeDCH0rK6PFfq8Wgjh7hC/qyaGEO4Kucx2KMuW596D3ao71iGEvml/z0rvx0khhKsqvI+75drtG0J4Nz2+AdgZODe1WfEWSiFmn/4SQrgsvUemhhBOCyGsl47prBDCyyGE/8ut06DPSvj6vX5r+Pq9vsT7Jj2u8fiU7ctiQ8uN9Lo/Q/yOqr8sy/Sjnxb3A1QRb2nxf0Bb4s2j3wNuAb5BvBHsJODwVH9l4F3iUNYqQBfgceCOXJtHEDNWAdgdmAfsk8p6ABkxoFmdeIui54Bbc+u/CJxU1s9dgarc84OBz4FBxPufrVrNvh1R23KgK/AF8FNikJUB25Rte0HueU/ifejuqOaYXg6MyD3vAMwGdk7PdwK2Jd5S7VvAP4D7cvWHALflnmfATjX056J0zDYA2hCzhlOALvljXqGfI4FBNbw3PgV+XLbsQGD99Np+N9X5RSrblHhbmK65+ncBt6fHawBTiQFvO2AdwIHzyt4bf0nHpV3an32BzYj/2PYk3mblktw2/gI8mN5LawCjUzs9UvlxxPfslqmNH6bXo2c1+11+fIcQ38P7pfVPSOs/AnQD2gN/ZfH3cBXwCbBN2o+zgclApzq+By5L+7lFOtbdgC1S2UDiPyw1fa7XT33um7axPfAZcHBuHzPibWo6AGsSvwfOacTvis7p/XEusFJa733gjFz51HRs2qXjMZHFP+fDiN8Va6Y65wNvAm0rfVYq9PlJ4vu8S/p5DHishu+CHum4dKvuWKdj+iVwI/E7cEPgbaB/pTZy67ybez4aGFDLazgwbacfX38OFgKjyl6DZ3LrNPSzMoT4vvlxauOnqQ/rVfPZqO74vFu27KvXqTFe91RnG+KIRruajmPFY1vfFfSjn+b4SV9IZ+Se/zB9QPN/VP8EXJMeHwS8V9bGNsSApk0123gAuDw9Ln1ZbZsr/zXwSu7520DfsjZ2zX9g07IfAX8mfukvJA5xfqds3+YA08t+FrH4l/CZxD8kpS/5fwODy7adpXWnAR8AN1MhCEz1/48YmKyRnh8DvF3Da/AjYFLu+VdfXul5tQEZ8Y/1LOD7ZW2+XtpHqg/IhgE31dCv+cCutbx/rgT+lHv+InBqetwxHf8d0/PTgb+WrX8g6cs79974fi3bPBF4KT3ultbZIFe+B4v/kfkvcFRZGyOp5g8ilQOy/B/x9qn9g3PLfsXi7+Eq4MLc8wB8CBxW23sg1Z0N7FdN3YHUHpD1B54rW3YJ8FTZezr/Ob8CeKiGNquo33fFYcBHpFsHpmW/AN5Kjw9PxyRffhHpc078hy0D1s2VrwDMIH0eqCEgI/5TmAEb5ZZtkpatldunpQnIvgDa55b1I33Gy9vIrbM0AdnYsmWTKrwG0xrxszKE3Hs9LZsM/KSaz0Z1x6emgKzBr3tatlGqt0ZNx7HSj24uLi3ZhNzjucT5UpPLlpWGMtYH1g1LnmmTEf/THx9COImYlehG/OOyCnESeXXbnJNrH2LQU9PcprjBLHuU+F8UIYRvE2+A/WgIYf0sfWKJ2Zuh+fVC7myeEEJIfR2aZdmXafHtwKUhhNOzLJuVli3M6jjRO8uy/4UQ/k3MFF4N/By4M7fNbYCLiRmb9sRj1KFCU3Wxelp3ZMidSUn877lb5VW+0okYXFZnidchxLl7pxGzcSsS/3t9IVflTuCXxJMyfgZ8nGXZc6lsfWDHsvdOIP73n1dVts29gPOAbxMzLW2If5ggZtkgfsGXjCtrb33gxhDCdbllKwIfU3dfvV+zLJsb3zZLfG7Kh/uqcutkIYQPSa9JLe+BrsSM09v16F+57iz52r4H/CT3vPxzXv45rKQ+3xXdgXG5z2KpD93T424VyvN9Xj/9fi0d75K2uTZqUqqTb/O9XNkElt6kLMvm5p5XUfvnbWmU93EuNbzvGuGzUmmbdXlf1Edjve6d+Pof5XrRHDJZXowj/ie4atnPylmWjQ8h7EgcbvkFsHoKYkYS/+DU1SvE4a86y7LsTWIQsB5xaKKudiem9o8pzTMhpsc7EP/DX1p3An3TvIftgbtzZcOJWbiNsyzrROWTCPJmE/9Al6ydezyF+IW5Z9nr8Y0syy6tpd3vEI91dRZ7HUII3YlDJIOIGYbOxGGb/Gs7HNg4hLA18T/lO3Nl44j/Tef72TmLJ0rkLcptsx0wIrW7bjpeZ+W2OT79Xje3fv5xabvHlG23Q5Zlv6xh3xtDj9KDFPivy9dBYE3vgcnEP7QbVdPuomqW532U336yQVreXD4C1guL/1XN92F8hfIeucelYGGjsteufZZl99Vx++VtblBWNovqP1tQ/bFeI4TQvqzfpde29E/c0rS71Brps1Jflfaj/JjC4vvfWK/7d4gZxPn17bQCMllePAq0C3HCcccQrRNCOCCVdyIOH04GshDCfsR5DfUxgphKr1YI4ZgQwsEhXUsrTaA9AXgjy7LP6rGtXxDn73wb2Cr9fIcYSBxfz37nDScGetcR53iMz5V1IqbfZ4UQ1iXOpajJy8DRIYR2afLtaaWC9F/m74ErQwgbAYQQOoR4HbfyPwJfSYFiV+J8lOqMYPFJ/x2I32WTgS9DCNsDR+ZXyLJsOvAQMWjbnjiHrORuwNJrt3IIYYU0CfgHNfShHfE//WlZls0LIWxKHIYpbe9j4vDPpen92BUov5zANcDAECfhhxDCKiGEnVJWtSkdE0LYOsTJ3mcQM2GPpbJq3wPpNb0JuDzEkyBCiJPMt0hVJhKz1O1q2PZ9wDYhhKNCPOnje8T3+u2Nuoc1e4z42vVP791NiAFCqQ+PEt9TZ4R4EsPWxPmPAGRZNomYWb8ppMsbhBBWDSEcEMouTVNJlmWfAE8DV6X1ugBXAU9kWVbKAr0MHJo+M12J893yqjvWKwCXpffSBsTh+LvSdqeS/gkI8UzhzYlZ+PJ263xyQh01xmelviodn/8QA9Yfpc/4AcD3c+WN9brvRfyOqjcFZLJcSGn63YmZkzeJf1T+QgxkAJ4i/uF9iZi9OYj4B7o+ngIWhBB2raHONOLQ2P9CCHOIc5emE+fi1EmIZ7XtD1yZZdnE/A8xy/fdUMvZitXJsmwGcb/3JV5iIu944pyTWcQ5cPfX0tyJxC/vz4hzdIaUlf8OeBh4OIQwkzjx+gRq/t45BhiS+lmde4At0x8csiz7X25b04lBRKVMxZ3E/X4q94ePdFx3Ix7zKuJr+BBlZ1jlZVk2m/g6Xx5CmE3MyJUPfx9GDHY+Jp4gUjqeX6Q2biWeaHFn2uaHxD+8bWvY98ZwCzEgnwb0Ic4JKx3v2t4D5xBf6xGpzmi+/gN+PzHDMzHEM+HWL1uXLMs+IM4vOpE4gfoe4Nwsy/7USPtWq7SvexOD+k/5+rvh6lQ+nXiiRB/iMboO+ENZM8cRT6AZHUKYRZwbeTBxqKoujiAev7eI31fTgaNy5QOI/0BOIB7j4WXrV3esxxHfbx8Qv3ueJL7HSo4mfhfNSPtbHghfQ/znZHoIYWwd96VGjfFZWQpLHJ8sXibnZOL7/zPgB8QTCUr9nE4DX/cQwqrE9/fNS9PpsPhwqYjUJGVN+mdZ9v30fFdiANGjwG4tk1JW7YMsy0J63pV4dqOVzf+ptO4JxEn5R9ZUryUJIexDDBpXyQr64g1xnuKA8vmLsuwLIfQlvraNneFqdi3hs7I0QgiXEOcvLlWGT5P6Reohy7Inif91SiNLQdh6dax7M0v5X2hzCSFsRZzL8jpxQvAg4I/L0h8YkeawvHxWsiz7fw1ZX0OWIg1TxbJ9ZfwiTSeeqLC86kIc9psNjAFeIw6ZiMji9FlBQ5YiIiIihVOGTERERKRgCshERERECqaATERERKRgCshERERECqaATERERKRgCshERERECvb/AUdzzgBVhpC4AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 576x511.2 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import shap\n",
    "tf.config.list_physical_devices('GPU')\n",
    "explainer = shap.DeepExplainer(model4, X_train.sample(n=4000))\n",
    "shap_values = explainer.shap_values(X_valid.sample(n=4000).values)\n",
    "shap.summary_plot(shap_values, X_valid.sample(n=4000))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>col_name</th>\n",
       "      <th>feature_importance_vals</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Gland texture</td>\n",
       "      <td>0.061181</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Drain amylase (POD1)</td>\n",
       "      <td>0.035899</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Drain location</td>\n",
       "      <td>0.023997</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Drain type</td>\n",
       "      <td>0.015578</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Chemotherapy</td>\n",
       "      <td>0.012203</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Antibiotic</td>\n",
       "      <td>0.010177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Reconstruction</td>\n",
       "      <td>0.009652</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Duct size</td>\n",
       "      <td>0.007254</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Drain suction</td>\n",
       "      <td>0.006629</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Biliary stent</td>\n",
       "      <td>0.006484</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                col_name  feature_importance_vals\n",
       "6          Gland texture                 0.061181\n",
       "13  Drain amylase (POD1)                 0.035899\n",
       "9         Drain location                 0.023997\n",
       "10            Drain type                 0.015578\n",
       "2           Chemotherapy                 0.012203\n",
       "4             Antibiotic                 0.010177\n",
       "7         Reconstruction                 0.009652\n",
       "5              Duct size                 0.007254\n",
       "11         Drain suction                 0.006629\n",
       "1          Biliary stent                 0.006484"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "shap_values[0].shape\n",
    "vals = np.abs(shap_values[0]).mean(0)\n",
    "feature_names = X_train.columns\n",
    "feature_importance = pd.DataFrame(list(zip(feature_names, vals)),\n",
    "                                  columns=['col_name','feature_importance_vals'])\n",
    "feature_importance.sort_values(by=['feature_importance_vals'],\n",
    "                               ascending=False, inplace=True)\n",
    "feature_importance.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "pan_feature_importance = feature_importance[feature_importance['col_name'].str.contains('PAN')]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "cat_names = ['PAN_JAUNDICE', 'PAN_BILIARYSTENT', 'PAN_CHEMO', 'PAN_RADIO', 'PAN_INTRA_ANTIBIOTICS',  'PAN_DUCTSIZE', 'PAN_GLANDTEXT', 'PAN_RECONSTRUCTION',  'PAN_DRAINS', 'PAN_DRAINS_TYPE', 'PAN_DRAINSYS_TYPE', 'PAN_DRAINSYS_SUCTN', 'PAN_RESECTION', \n",
    "            ]\n",
    "cont_names = ['PAN_AMYLASE_POD1']\n",
    "\n",
    "features = cat_names + cont_names\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>col_name</th>\n",
       "      <th>feature_importance_vals</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [col_name, feature_importance_vals]\n",
       "Index: []"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pan_feature_importance.head(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "names = {value : key for key, value in columns.items()}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "mapping = {'PAN_JAUNDICE':'Jaundice', 'PAN_BILIARYSTENT':'Biliary stent', 'PAN_CHEMO':'Chemotherapy', 'PAN_RADIO':'Radiation therapy', 'PAN_INTRA_ANTIBIOTICS':'Antibiotic',  'PAN_DUCTSIZE':'Duct size', 'PAN_GLANDTEXT':'Gland texture', 'PAN_RECONSTRUCTION':'Reconstruction',  'PAN_DRAINS':'Drains', 'PAN_DRAINS_TYPE':'Drain location', 'PAN_DRAINSYS_TYPE':'Drain type', 'PAN_DRAINSYS_SUCTN':'Drain suction', 'PAN_RESECTION':'Vascular resection','PAN_AMYLASE_POD1':'Drain amylase (POD1)'}\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "f= lambda x: mapping.get(x, x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "mapping = {'PAN_JAUNDICE':'Jaundice', 'PAN_BILIARYSTENT':'Biliary stent', 'PAN_CHEMO':'Chemotherapy', 'PAN_RADIO':'Radiation therapy', 'PAN_INTRA_ANTIBIOTICS':'Antibiotic',  'PAN_DUCTSIZE':'Duct size', 'PAN_GLANDTEXT':'Gland texture', 'PAN_RECONSTRUCTION':'Reconstruction',  'PAN_DRAINS':'Drains', 'PAN_DRAINS_TYPE':'Drain location', 'PAN_DRAINSYS_TYPE':'Drain type', 'PAN_DRAINSYS_SUCTN':'Drain suction', 'PAN_RESECTION':'Vascular resection','PAN_AMYLASE_POD1':'Drain amylase (POD1)'}\n",
    "\n",
    "hep_feature_importance = feature_importance[feature_importance['col_name'].str.contains('HEP')]\n",
    "hep_feature_importance.head(20)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_importance.to_excel('shap_pan.xlsx')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "lr = LogisticRegression()\n",
    "lr.fit(X_train, y_train)\n",
    "\n",
    "coef = lr.coef_\n",
    "coef.shape\n",
    "ors = np.exp(coef)\n",
    "ors = ors.squeeze()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>col_name</th>\n",
       "      <th>odds ratio</th>\n",
       "      <th>diff</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Gland texture</td>\n",
       "      <td>1.374893</td>\n",
       "      <td>0.374893</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Chemotherapy</td>\n",
       "      <td>0.765040</td>\n",
       "      <td>0.234960</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Drain location</td>\n",
       "      <td>0.782095</td>\n",
       "      <td>0.217905</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Vascular resection</td>\n",
       "      <td>0.835935</td>\n",
       "      <td>0.164065</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Drain type</td>\n",
       "      <td>0.862717</td>\n",
       "      <td>0.137283</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Radiation therapy</td>\n",
       "      <td>0.876673</td>\n",
       "      <td>0.123327</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Antibiotic</td>\n",
       "      <td>0.884218</td>\n",
       "      <td>0.115782</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Duct size</td>\n",
       "      <td>0.913628</td>\n",
       "      <td>0.086372</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Jaundice</td>\n",
       "      <td>0.920526</td>\n",
       "      <td>0.079474</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Drains</td>\n",
       "      <td>0.954089</td>\n",
       "      <td>0.045911</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              col_name  odds ratio      diff\n",
       "6        Gland texture    1.374893  0.374893\n",
       "2         Chemotherapy    0.765040  0.234960\n",
       "9       Drain location    0.782095  0.217905\n",
       "12  Vascular resection    0.835935  0.164065\n",
       "10          Drain type    0.862717  0.137283\n",
       "3    Radiation therapy    0.876673  0.123327\n",
       "4           Antibiotic    0.884218  0.115782\n",
       "5            Duct size    0.913628  0.086372\n",
       "0             Jaundice    0.920526  0.079474\n",
       "8               Drains    0.954089  0.045911"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "or_df = pd.DataFrame(list(zip(feature_names, ors)),columns=['col_name','odds ratio'])\n",
    "or_df['diff'] = abs(1 - or_df['odds ratio'])\n",
    "pan_ors = or_df.sort_values(by='diff', ascending=False)\n",
    "pan_ors.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "pan_ors.to_excel('ors_pan.xlsx')"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1"
  },
  "kernelspec": {
   "display_name": "Python 3.9.5 64-bit",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}