stat_significance_test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "14c80db0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "from scipy import stats\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm\n",
    "from itertools import chain, combinations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3025520d",
   "metadata": {},
   "outputs": [],
   "source": [
    "#!/usr/bin/env python\n",
    "\n",
    "# Copyright 2016-2020 Biomedical Imaging Group Rotterdam, Departments of\n",
    "# Medical Informatics and Radiology, Erasmus MC, Rotterdam, The Netherlands\n",
    "#\n",
    "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
    "# you may not use this file except in compliance with the License.\n",
    "# You may obtain a copy of the License at\n",
    "#\n",
    "#     http://www.apache.org/licenses/LICENSE-2.0\n",
    "#\n",
    "# Unless required by applicable law or agreed to in writing, software\n",
    "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
    "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
    "# See the License for the specific language governing permissions and\n",
    "# limitations under the License.\n",
    "\n",
    "\"\"\"Adopted from https://github.com/yandexdataschool/roc_comparison.\"\"\"\n",
    "\n",
    "import numpy as np\n",
    "import scipy.stats\n",
    "\n",
    "\n",
    "# AUC comparison adapted from\n",
    "# https://github.com/Netflix/vmaf/\n",
    "\n",
    "def compute_midrank(x):\n",
    "    \"\"\"Computes midranks.\n",
    "\n",
    "    Args:\n",
    "       x - a 1D numpy array\n",
    "    Returns:\n",
    "       array of midranks\n",
    "\n",
    "    \"\"\"\n",
    "    J = np.argsort(x)\n",
    "    Z = x[J]\n",
    "    N = len(x)\n",
    "    T = np.zeros(N, dtype=float)\n",
    "    i = 0\n",
    "    while i < N:\n",
    "        j = i\n",
    "        while j < N and Z[j] == Z[i]:\n",
    "            j += 1\n",
    "        T[i:j] = 0.5*(i + j - 1)\n",
    "        i = j\n",
    "    T2 = np.empty(N, dtype=float)\n",
    "    # Note(kazeevn) +1 is due to Python using 0-based indexing\n",
    "    # instead of 1-based in the AUC formula in the paper\n",
    "    T2[J] = T + 1\n",
    "    return T2\n",
    "\n",
    "\n",
    "def compute_midrank_weight(x, sample_weight):\n",
    "    \"\"\"Computes midranks.\n",
    "\n",
    "    Args:\n",
    "       x - a 1D numpy array\n",
    "    Returns:\n",
    "       array of midranks\n",
    "\n",
    "    \"\"\"\n",
    "    J = np.argsort(x)\n",
    "    Z = x[J]\n",
    "    cumulative_weight = np.cumsum(sample_weight[J])\n",
    "    N = len(x)\n",
    "    T = np.zeros(N, dtype=float)\n",
    "    i = 0\n",
    "    while i < N:\n",
    "        j = i\n",
    "        while j < N and Z[j] == Z[i]:\n",
    "            j += 1\n",
    "        T[i:j] = cumulative_weight[i:j].mean()\n",
    "        i = j\n",
    "    T2 = np.empty(N, dtype=float)\n",
    "    T2[J] = T\n",
    "    return T2\n",
    "\n",
    "\n",
    "def fastDeLong(predictions_sorted_transposed, label_1_count):\n",
    "    \"\"\"Fast DeLong test computation.\n",
    "\n",
    "    The fast version of DeLong's method for computing the covariance of\n",
    "    unadjusted AUC.\n",
    "    Args:\n",
    "       predictions_sorted_transposed: a 2D numpy.array[n_classifiers, n_examples]\n",
    "          sorted such as the examples with label \"1\" are first\n",
    "    Returns:\n",
    "       (AUC value, DeLong covariance)\n",
    "    Reference:\n",
    "     @article{sun2014fast,\n",
    "       title={Fast Implementation of DeLong's Algorithm for\n",
    "              Comparing the Areas Under Correlated Receiver Oerating Characteristic Curves},\n",
    "       author={Xu Sun and Weichao Xu},\n",
    "       journal={IEEE Signal Processing Letters},\n",
    "       volume={21},\n",
    "       number={11},\n",
    "       pages={1389--1393},\n",
    "       year={2014},\n",
    "       publisher={IEEE}\n",
    "     }\n",
    "\n",
    "    \"\"\"\n",
    "    # Short variables are named as they are in the paper\n",
    "    m = label_1_count\n",
    "    n = predictions_sorted_transposed.shape[1] - m\n",
    "    positive_examples = predictions_sorted_transposed[:, :m]\n",
    "    negative_examples = predictions_sorted_transposed[:, m:]\n",
    "    k = predictions_sorted_transposed.shape[0]\n",
    "\n",
    "    tx = np.empty([k, m], dtype=float)\n",
    "    ty = np.empty([k, n], dtype=float)\n",
    "    tz = np.empty([k, m + n], dtype=float)\n",
    "    for r in tqdm(range(k)):\n",
    "        tx[r, :] = compute_midrank(positive_examples[r, :])\n",
    "        ty[r, :] = compute_midrank(negative_examples[r, :])\n",
    "        tz[r, :] = compute_midrank(predictions_sorted_transposed[r, :])\n",
    "    aucs = tz[:, :m].sum(axis=1) / m / n - float(m + 1.0) / 2.0 / n\n",
    "    v01 = (tz[:, :m] - tx[:, :]) / n\n",
    "    v10 = 1.0 - (tz[:, m:] - ty[:, :]) / m\n",
    "    sx = np.cov(v01)\n",
    "    sy = np.cov(v10)\n",
    "    delongcov = sx / m + sy / n\n",
    "    return aucs, delongcov\n",
    "\n",
    "\n",
    "def calc_pvalue(aucs, sigma):\n",
    "    \"\"\"Computes log(10) of p-values.\n",
    "\n",
    "    Args:\n",
    "       aucs: 1D array of AUCs\n",
    "       sigma: AUC DeLong covariances\n",
    "    Returns:\n",
    "       log10(pvalue)\n",
    "\n",
    "    \"\"\"\n",
    "    l = np.array([[1, -1]])\n",
    "    z = np.abs(np.diff(aucs)) / (np.sqrt(np.dot(np.dot(l, sigma), l.T)))\n",
    "    log10pval = np.log10(2) + scipy.stats.norm.logsf(z, loc=0, scale=1) / np.log(10)\n",
    "    return np.power(10, log10pval)\n",
    "\n",
    "\n",
    "def compute_ground_truth_statistics(ground_truth, sample_weight=None):\n",
    "    assert np.array_equal(np.unique(ground_truth), [0, 1])\n",
    "    order = (-ground_truth).argsort()\n",
    "    label_1_count = int(ground_truth.sum())\n",
    "    if sample_weight is None:\n",
    "        ordered_sample_weight = None\n",
    "    else:\n",
    "        ordered_sample_weight = sample_weight[order]\n",
    "\n",
    "    return order, label_1_count, ordered_sample_weight\n",
    "\n",
    "\n",
    "def delong_roc_variance(ground_truth, predictions):\n",
    "    \"\"\"Computes ROC AUC variance for a single set of predictions.\n",
    "\n",
    "    Args:\n",
    "       ground_truth: np.array of 0 and 1\n",
    "       predictions: np.array of floats of the probability of being class 1\n",
    "\n",
    "    \"\"\"\n",
    "    sample_weight = None\n",
    "    order, label_1_count, ordered_sample_weight = compute_ground_truth_statistics(\n",
    "        ground_truth, sample_weight)\n",
    "    predictions_sorted_transposed = predictions[np.newaxis, order]\n",
    "    aucs, delongcov = fastDeLong(predictions_sorted_transposed, label_1_count, ordered_sample_weight)\n",
    "    assert len(aucs) == 1, \"There is a bug in the code, please forward this to the developers\"\n",
    "    return aucs[0], delongcov\n",
    "\n",
    "\n",
    "def delong_roc_test(ground_truth, predictions_one, predictions_two):\n",
    "    \"\"\"Computes log(p-value) for hypothesis that two ROC AUCs are different.\n",
    "\n",
    "    Args:\n",
    "       ground_truth: np.array of 0 and 1\n",
    "       predictions_one: predictions of the first model,\n",
    "          np.array of floats of the probability of being class 1\n",
    "       predictions_two: predictions of the second model,\n",
    "          np.array of floats of the probability of being class 1\n",
    "\n",
    "    \"\"\"\n",
    "    order, label_1_count, _ = compute_ground_truth_statistics(ground_truth)\n",
    "    predictions_sorted_transposed = np.vstack((predictions_one, predictions_two))[:, order]\n",
    "    aucs, delongcov = fastDeLong(predictions_sorted_transposed, label_1_count)\n",
    "    return calc_pvalue(aucs, delongcov)\n",
    "\n",
    "\n",
    "def calc_delong(y_true, y_p, alpha=0.95):\n",
    "    \n",
    "#     alpha = .95\n",
    "\n",
    "    y_true = y_true.reshape(-1)\n",
    "    y_p = y_p.reshape(-1)\n",
    "\n",
    "    auc, auc_cov = delong_roc_variance(\n",
    "        y_true,\n",
    "        y_p)\n",
    "\n",
    "    auc_std = np.sqrt(auc_cov)\n",
    "    lower_upper_q = np.abs(np.array([0, 1]) - (1 - alpha) / 2)\n",
    "\n",
    "    ci = stats.norm.ppf(\n",
    "        lower_upper_q,\n",
    "        loc=auc,\n",
    "        scale=auc_std)\n",
    "\n",
    "    ci[ci > 1] = 1\n",
    "\n",
    "    print('AUC:', auc)\n",
    "    print('AUC COV:', auc_cov)\n",
    "    print('95% AUC CI:', ci)\n",
    "    \n",
    "    return auc, auc_cov, ci"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "eb1a76d1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "check_invalid_ecg.ipynb\t\t   proposed\r\n",
      "data_check_with_ori.ipynb\t   proposed_cmc\r\n",
      "data_processing\t\t\t   proposed_limited_labeled\r\n",
      "deepbeat_baseline\t\t   __pycache__\r\n",
      "deepmi\t\t\t\t   result_plots\r\n",
      "deepmi_limited_labels\t\t   results_plot.ipynb\r\n",
      "ecg_only_baseline\t\t   run_stat_sig.py\r\n",
      "ecg_only_baseline_limited_labeled  sample_ecg.npy\r\n",
      "model_card.md\t\t\t   sample_ppg.npy\r\n",
      "output2(w_correction).txt\t   stat_significance_test.ipynb\r\n",
      "output.txt\t\t\t   stat_significance_test.py\r\n",
      "ppg_only_baseline\t\t   tmp.ipynb\r\n",
      "ppg_only_baseline_limited_labeled\r\n"
     ]
    }
   ],
   "source": [
    "! ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e185a82c",
   "metadata": {},
   "outputs": [],
   "source": [
    "testing_data_label_pairs = [\n",
    "    ['data_simband_ecg_2400.npy', 'label_simband.npy', '/labs/hulab/Robust_learning_TESTDATA/simband_patient_idx_dict.pkl'],\n",
    "    ['data_simband_ppg_2400.npy', 'label_simband.npy', '/labs/hulab/Robust_learning_TESTDATA/simband_patient_idx_dict.pkl'],\n",
    "    ['data_ucla_ecg_2400.npy', 'label_ucla_ecg.npy', '/labs/hulab/Robust_learning_TESTDATA/UCLA_patient_idx_dict.pkl'],\n",
    "    ['data_ucla_ppg_2400.npy', 'label_ucla_ppg.npy', '/labs/hulab/Robust_learning_TESTDATA/UCLA_patient_idx_dict.pkl'],\n",
    "    ['data_staford_2400.npy', 'label_staford.npy', '/labs/hulab/Robust_learning_TESTDATA/stanford_patient_idx_dict.pkl'],\n",
    "#     ['data_staford_goodquality_2400.npy', 'label_staford_goodquality.npy', '/labs/hulab/Robust_learning_TESTDATA/stanford_patient_idx_dict.pkl'],\n",
    "#     ['data_staford_badquality_2400.npy', 'label_staford_badquality.npy', '/labs/hulab/Robust_learning_TESTDATA/stanford_patient_idx_dict.pkl']\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "529ce475",
   "metadata": {},
   "source": [
    "# PPG Datasets (full)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "de91d46a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('simsiam', 'deepmi'), ('simsiam', 'deepbeat_baseline'), ('simsiam', 'ppg_only_baseline'), ('deepmi', 'deepbeat_baseline'), ('deepmi', 'ppg_only_baseline'), ('deepbeat_baseline', 'ppg_only_baseline')]\n"
     ]
    }
   ],
   "source": [
    "folders = ['simsiam', 'deepmi', 'deepbeat_baseline', 'ppg_only_baseline']\n",
    "model_names = ['simsiam', 'deepmi', 'deepbeat', 'ppgonly']\n",
    "dataset_lengths = ['2400', '2400', '800', '2400']\n",
    "combs = list(combinations(folders, 2))\n",
    "print(combs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "c0dd4820",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------\n",
      "---------------------------------------------------------------\n",
      "data_simband_ppg_2400\n",
      "(856988,) (856988,) (856988,) (1000,) (1000,) (1000,) (1000,)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [00:02<00:00,  1.07s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('simsiam', 'deepmi'): p_auroc=0.0 , p_auprc=9.127480280470798e-194\n",
      "(856988,) (856988,) (856988,) (1000,) (1000,) (1000,) (1000,)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [00:02<00:00,  1.06s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('simsiam', 'deepbeat_baseline'): p_auroc=0.0 , p_auprc=0.0\n",
      "(856988,) (856988,) (856988,) (1000,) (1000,) (1000,) (1000,)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [00:02<00:00,  1.08s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('simsiam', 'ppg_only_baseline'): p_auroc=0.0 , p_auprc=7.441164401512137e-294\n",
      "(856988,) (856988,) (856988,) (1000,) (1000,) (1000,) (1000,)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [00:02<00:00,  1.05s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('deepmi', 'deepbeat_baseline'): p_auroc=8.056389084274472e-145 , p_auprc=7.376168594182096e-229\n",
      "(856988,) (856988,) (856988,) (1000,) (1000,) (1000,) (1000,)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [00:02<00:00,  1.08s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('deepmi', 'ppg_only_baseline'): p_auroc=4.21087162825943e-225 , p_auprc=6.318410264803836e-60\n",
      "(856988,) (856988,) (856988,) (1000,) (1000,) (1000,) (1000,)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [00:02<00:00,  1.08s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('deepbeat_baseline', 'ppg_only_baseline'): p_auroc=5.693480572498069e-38 , p_auprc=1.861076800142062e-189\n",
      "---------------------------------------------------------------\n",
      "---------------------------------------------------------------\n",
      "data_ucla_ppg_2400\n",
      "(250683755,) (250683755,) (250683755,) (1000,) (1000,) (1000,) (1000,)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [18:13<00:00, 546.79s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('simsiam', 'deepmi'): p_auroc=6.640197698403894e-43 , p_auprc=2.1512358932614873e-37\n",
      "(250683755,) (250683755,) (250683755,) (1000,) (1000,) (1000,) (1000,)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [17:54<00:00, 537.01s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('simsiam', 'deepbeat_baseline'): p_auroc=0.0 , p_auprc=1.5087805678964306e-307\n",
      "(250683755,) (250683755,) (250683755,) (1000,) (1000,) (1000,) (1000,)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [18:25<00:00, 552.72s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('simsiam', 'ppg_only_baseline'): p_auroc=2.774549938223466e-40 , p_auprc=6.438839407439185e-23\n",
      "(250683755,) (250683755,) (250683755,) (1000,) (1000,) (1000,) (1000,)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [17:49<00:00, 534.83s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('deepmi', 'deepbeat_baseline'): p_auroc=0.0 , p_auprc=3.391721131538456e-300\n",
      "(250683755,) (250683755,) (250683755,) (1000,) (1000,) (1000,) (1000,)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [18:09<00:00, 544.66s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('deepmi', 'ppg_only_baseline'): p_auroc=1.919082043914492e-22 , p_auprc=8.041028403127641e-114\n",
      "(250683755,) (250683755,) (250683755,) (1000,) (1000,) (1000,) (1000,)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [17:48<00:00, 534.15s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('deepbeat_baseline', 'ppg_only_baseline'): p_auroc=0.0 , p_auprc=9.330772348591663e-290\n",
      "---------------------------------------------------------------\n",
      "data_staford_2400\n",
      "(17556623,) (17556623,) (17556623,) (1000,) (1000,) (1000,) (1000,)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [00:54<00:00, 27.34s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('simsiam', 'deepmi'): p_auroc=0.0 , p_auprc=0.0\n",
      "(17556623,) (17556623,) (17556623,) (1000,) (1000,) (1000,) (1000,)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [00:54<00:00, 27.05s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('simsiam', 'deepbeat_baseline'): p_auroc=4.890782205998843e-125 , p_auprc=0.04602517483857248\n",
      "(17556623,) (17556623,) (17556623,) (1000,) (1000,) (1000,) (1000,)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [00:56<00:00, 28.23s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('simsiam', 'ppg_only_baseline'): p_auroc=0.0 , p_auprc=0.0\n",
      "(17556623,) (17556623,) (17556623,) (1000,) (1000,) (1000,) (1000,)\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_29704/3383375832.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     18\u001b[0m             \u001b[0mauprc_2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_2}_{model_names[folders.index(model_pair[1])]}_auprcs.npy'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtargets\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpred_probs_1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpred_probs_2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mauroc_1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mauroc_2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mauprc_1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mauprc_2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m             \u001b[0mp_auroc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdelong_roc_test\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtargets\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpred_probs_1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpred_probs_2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     21\u001b[0m             \u001b[0mp_auroc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstats\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mttest_rel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mauroc_1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mauroc_2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpvalue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m             \u001b[0mp_auprc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstats\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mttest_rel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mauprc_1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mauprc_2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpvalue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/tmp/ipykernel_29704/1427712099.py\u001b[0m in \u001b[0;36mdelong_roc_test\u001b[0;34m(ground_truth, predictions_one, predictions_two)\u001b[0m\n\u001b[1;32m    182\u001b[0m     \"\"\"\n\u001b[1;32m    183\u001b[0m     \u001b[0morder\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabel_1_count\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompute_ground_truth_statistics\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mground_truth\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 184\u001b[0;31m     \u001b[0mpredictions_sorted_transposed\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvstack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpredictions_one\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpredictions_two\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0morder\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    185\u001b[0m     \u001b[0maucs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdelongcov\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfastDeLong\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpredictions_sorted_transposed\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabel_1_count\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    186\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mcalc_pvalue\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maucs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdelongcov\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<__array_function__ internals>\u001b[0m in \u001b[0;36mvstack\u001b[0;34m(*args, **kwargs)\u001b[0m\n",
      "\u001b[0;32m/labs/hulab/stark_conda/envs/base_pytorch/lib/python3.7/site-packages/numpy/core/shape_base.py\u001b[0m in \u001b[0;36mvstack\u001b[0;34m(tup)\u001b[0m\n\u001b[1;32m    280\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marrs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    281\u001b[0m         \u001b[0marrs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0marrs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 282\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_nx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcatenate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marrs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    283\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    284\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<__array_function__ internals>\u001b[0m in \u001b[0;36mconcatenate\u001b[0;34m(*args, **kwargs)\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "for pair in testing_data_label_pairs:\n",
    "        \n",
    "    dataset = pair[0].split('.')[0]\n",
    "    print('---------------------------------------------------------------')\n",
    "    if 'ppg' in dataset or 'staford' in dataset:\n",
    "        print(dataset)\n",
    "        for model_pair in combs:\n",
    "\n",
    "            dataset_1 = dataset.replace('2400', dataset_lengths[folders.index(model_pair[0])])\n",
    "            dataset_2 = dataset.replace('2400', dataset_lengths[folders.index(model_pair[1])])\n",
    "            \n",
    "            targets = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_1}_{model_names[folders.index(model_pair[0])]}_targets.npy')\n",
    "            pred_probs_1 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_1}_{model_names[folders.index(model_pair[0])]}_pred_probs.npy')\n",
    "            pred_probs_2 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_2}_{model_names[folders.index(model_pair[1])]}_pred_probs.npy')\n",
    "            auroc_1 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_1}_{model_names[folders.index(model_pair[0])]}_aurocs.npy')       \n",
    "            auroc_2 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_2}_{model_names[folders.index(model_pair[1])]}_aurocs.npy')\n",
    "            auprc_1 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_1}_{model_names[folders.index(model_pair[0])]}_auprcs.npy')            \n",
    "            auprc_2 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_2}_{model_names[folders.index(model_pair[1])]}_auprcs.npy')\n",
    "            print(targets.shape, pred_probs_1.shape, pred_probs_2.shape, auroc_1.shape, auroc_2.shape, auprc_1.shape, auprc_2.shape)\n",
    "            p_auroc = delong_roc_test(targets, pred_probs_1, pred_probs_2)[0][0]\n",
    "            p_auprc = stats.ttest_rel(auprc_1, auprc_2).pvalue\n",
    "            print(f'{model_pair}: p_auroc={p_auroc} , p_auprc={p_auprc}')    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "e00cbd73",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------\n",
      "---------------------------------------------------------------\n",
      "data_simband_ppg_2400\n",
      "('simsiam', 'deepmi'): p_auroc=7.004012126930545e-164 , p_auprc=4.882530023580109e-143\n",
      "('simsiam', 'deepbeat_baseline'): p_auroc=4.654951174852043e-165 , p_auprc=9.36400998312439e-165\n",
      "('simsiam', 'ppg_only_baseline'): p_auroc=3.510543381233979e-165 , p_auprc=5.492971800093215e-160\n",
      "('deepmi', 'deepbeat_baseline'): p_auroc=1.0721500110969205e-115 , p_auprc=5.643844824027882e-153\n",
      "('deepmi', 'ppg_only_baseline'): p_auroc=8.447931670801826e-146 , p_auprc=2.4812949873078955e-56\n",
      "('deepbeat_baseline', 'ppg_only_baseline'): p_auroc=1.701765361433702e-34 , p_auprc=1.9795227674018953e-141\n",
      "---------------------------------------------------------------\n",
      "---------------------------------------------------------------\n",
      "data_ucla_ppg_2400\n",
      "('simsiam', 'deepmi'): p_auroc=1.6220970106420827e-38 , p_auprc=2.7555357760491723e-36\n",
      "('simsiam', 'deepbeat_baseline'): p_auroc=1.1232415216595756e-163 , p_auprc=1.1801712143917756e-160\n",
      "('simsiam', 'ppg_only_baseline'): p_auroc=5.9238619323202625e-34 , p_auprc=1.5409218462171982e-19\n",
      "('deepmi', 'deepbeat_baseline'): p_auroc=3.4832743955404717e-163 , p_auprc=3.599857424547753e-160\n",
      "('deepmi', 'ppg_only_baseline'): p_auroc=1.3366838250223455e-21 , p_auprc=2.04394131157475e-102\n",
      "('deepbeat_baseline', 'ppg_only_baseline'): p_auroc=8.456391385620767e-165 , p_auprc=2.173306222437759e-159\n",
      "---------------------------------------------------------------\n",
      "data_staford_2400\n",
      "('simsiam', 'deepmi'): p_auroc=3.325859118934513e-165 , p_auprc=3.325859118934513e-165\n",
      "('simsiam', 'deepbeat_baseline'): p_auroc=1.0609189569037093e-102 , p_auprc=0.0044436695841821435\n",
      "('simsiam', 'ppg_only_baseline'): p_auroc=3.325859118934513e-165 , p_auprc=3.325859118934513e-165\n",
      "('deepmi', 'deepbeat_baseline'): p_auroc=3.325859118934513e-165 , p_auprc=3.325859118934513e-165\n",
      "('deepmi', 'ppg_only_baseline'): p_auroc=2.9625185318405445e-33 , p_auprc=3.325859118934513e-165\n",
      "('deepbeat_baseline', 'ppg_only_baseline'): p_auroc=3.325859118934513e-165 , p_auprc=3.325859118934513e-165\n"
     ]
    }
   ],
   "source": [
    "for pair in testing_data_label_pairs:\n",
    "        \n",
    "    dataset = pair[0].split('.')[0]\n",
    "    print('---------------------------------------------------------------')\n",
    "    if 'ppg' in dataset or 'staford' in dataset:\n",
    "        print(dataset)\n",
    "        for model_pair in combs:\n",
    "\n",
    "            dataset_1 = dataset.replace('2400', dataset_lengths[folders.index(model_pair[0])])\n",
    "            dataset_2 = dataset.replace('2400', dataset_lengths[folders.index(model_pair[1])])\n",
    "            \n",
    "            targets = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_1}_{model_names[folders.index(model_pair[0])]}_targets.npy')\n",
    "            pred_probs_1 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_1}_{model_names[folders.index(model_pair[0])]}_pred_probs.npy')\n",
    "            pred_probs_2 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_2}_{model_names[folders.index(model_pair[1])]}_pred_probs.npy')\n",
    "            auroc_1 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_1}_{model_names[folders.index(model_pair[0])]}_aurocs.npy')       \n",
    "            auroc_2 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_2}_{model_names[folders.index(model_pair[1])]}_aurocs.npy')\n",
    "            auprc_1 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_1}_{model_names[folders.index(model_pair[0])]}_auprcs.npy')            \n",
    "            auprc_2 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_2}_{model_names[folders.index(model_pair[1])]}_auprcs.npy')\n",
    "#             print(targets.shape, pred_probs_1.shape, pred_probs_2.shape, auroc_1.shape, auroc_2.shape, auprc_1.shape, auprc_2.shape)\n",
    "            p_auroc_wc = stats.wilcoxon(auroc_1, auroc_2).pvalue\n",
    "            p_auprc_wc = stats.wilcoxon(auprc_1, auprc_2).pvalue\n",
    "        \n",
    "            print(f'{model_pair}: p_auroc={p_auroc_wc} , p_auprc={p_auprc_wc}')        \n",
    "            \n",
    "#             x, y = auroc_1, auroc_2\n",
    "#             d = x - y\n",
    "#             d = np.compress(np.not_equal(d, 0), d)\n",
    "#             count = len(d)\n",
    "#             r = stats.rankdata(abs(d))\n",
    "#             r_plus = np.sum((d > 0) * r)\n",
    "#             r_minus = np.sum((d < 0) * r)\n",
    "            \n",
    "#             T = min(r_plus, r_minus)\n",
    "            \n",
    "#             mn = count * (count + 1.) * 0.25\n",
    "#             se = count * (count + 1.) * (2. * count + 1.)\n",
    "            \n",
    "#             replist, repnum = stats.find_repeats(r)\n",
    "#             if repnum.size != 0:\n",
    "#                 # Correction for repeated elements.\n",
    "#                 se -= 0.5 * (repnum * (repnum * repnum - 1)).sum()\n",
    "\n",
    "#             se = np.sqrt(se / 24)\n",
    "\n",
    "#             # apply continuity correction if applicable\n",
    "#             d = 0\n",
    "            \n",
    "#             z = (T - mn - d) / se\n",
    "#             prob = 2. * stats.distributions.norm.sf(abs(z))\n",
    "            \n",
    "#             z = z\n",
    "            \n",
    "            \n",
    "#             print(prob)\n",
    "            \n",
    "            \n",
    "#             break\n",
    "            \n",
    "            "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5cc0e900",
   "metadata": {},
   "source": [
    "# ECG datasets (full)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "6d2d0fb0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('simsiam', 'deepmi'), ('simsiam', 'ecg_only_baseline'), ('deepmi', 'ecg_only_baseline')]\n"
     ]
    }
   ],
   "source": [
    "folders = ['simsiam', 'deepmi', 'ecg_only_baseline']\n",
    "model_names = ['simsiam', 'deepmi', 'ecgonly']\n",
    "dataset_lengths = ['2400', '2400', '2400']\n",
    "combs = list(combinations(folders, 2))\n",
    "print(combs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "425e6efa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------\n",
      "data_simband_ecg_2400\n",
      "('simsiam', 'deepmi')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [00:01<00:00,  1.49it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('simsiam', 'deepmi'): p_auroc=0.0 , p_auprc=0.0\n",
      "('simsiam', 'ecg_only_baseline')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [00:01<00:00,  1.49it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('simsiam', 'ecg_only_baseline'): p_auroc=0.0 , p_auprc=0.0\n",
      "('deepmi', 'ecg_only_baseline')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [00:01<00:00,  1.47it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('deepmi', 'ecg_only_baseline'): p_auroc=3.179342579000822e-16 , p_auprc=0.0\n",
      "---------------------------------------------------------------\n",
      "---------------------------------------------------------------\n",
      "data_ucla_ecg_2400\n",
      "('simsiam', 'deepmi')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [09:20<00:00, 280.04s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('simsiam', 'deepmi'): p_auroc=0.0 , p_auprc=4.0864597254483e-98\n",
      "('simsiam', 'ecg_only_baseline')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [09:17<00:00, 278.52s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('simsiam', 'ecg_only_baseline'): p_auroc=0.0 , p_auprc=1.9704262934622083e-216\n",
      "('deepmi', 'ecg_only_baseline')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [09:34<00:00, 287.17s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('deepmi', 'ecg_only_baseline'): p_auroc=0.0 , p_auprc=1.0927738265027907e-152\n",
      "---------------------------------------------------------------\n",
      "---------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "for pair in testing_data_label_pairs:\n",
    "        \n",
    "    dataset = pair[0].split('.')[0]\n",
    "    print('---------------------------------------------------------------')\n",
    "    if 'ecg' in dataset:\n",
    "        print(dataset)\n",
    "        for model_pair in combs:\n",
    "            print(model_pair)\n",
    "            \n",
    "            dataset_1 = dataset.replace('2400', dataset_lengths[folders.index(model_pair[0])])\n",
    "            dataset_2 = dataset.replace('2400', dataset_lengths[folders.index(model_pair[1])])\n",
    "            \n",
    "            targets = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_1}_{model_names[folders.index(model_pair[0])]}_targets.npy')\n",
    "            pred_probs_1 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_1}_{model_names[folders.index(model_pair[0])]}_pred_probs.npy')\n",
    "            pred_probs_2 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_2}_{model_names[folders.index(model_pair[1])]}_pred_probs.npy')\n",
    "            auroc_1 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_1}_{model_names[folders.index(model_pair[0])]}_aurocs.npy')       \n",
    "            auroc_2 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_2}_{model_names[folders.index(model_pair[1])]}_aurocs.npy')\n",
    "            auprc_1 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_1}_{model_names[folders.index(model_pair[0])]}_auprcs.npy')            \n",
    "            auprc_2 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_2}_{model_names[folders.index(model_pair[1])]}_auprcs.npy')\n",
    "#             print(targets.shape, pred_probs_1.shape, pred_probs_2.shape, auroc_1.shape, auroc_2.shape, auprc_1.shape, auprc_2.shape)\n",
    "            p_auroc = delong_roc_test(targets, pred_probs_1, pred_probs_2)[0][0]\n",
    "#             p_auroc = stats.ttest_rel(auroc_1, auroc_2).pvalue\n",
    "            p_auprc = stats.ttest_rel(auprc_1, auprc_2).pvalue\n",
    "            \n",
    "            print(f'{model_pair}: p_auroc={p_auroc} , p_auprc={p_auprc}')\n",
    "            \n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "6668864b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------\n",
      "data_simband_ecg_2400\n",
      "('simsiam', 'deepmi')\n",
      "('simsiam', 'deepmi'): p_auroc=3.5548037607056008e-127 , p_auprc=1.9748820424982348e-164\n",
      "('simsiam', 'ecg_only_baseline')\n",
      "('simsiam', 'ecg_only_baseline'): p_auroc=2.019183825066748e-73 , p_auprc=3.325859118934513e-165\n",
      "('deepmi', 'ecg_only_baseline')\n",
      "('deepmi', 'ecg_only_baseline'): p_auroc=4.419200071432701e-08 , p_auprc=1.5172038211816667e-164\n",
      "---------------------------------------------------------------\n",
      "---------------------------------------------------------------\n",
      "data_ucla_ecg_2400\n",
      "('simsiam', 'deepmi')\n",
      "('simsiam', 'deepmi'): p_auroc=9.980326940189327e-134 , p_auprc=7.264207136436138e-88\n",
      "('simsiam', 'ecg_only_baseline')\n",
      "('simsiam', 'ecg_only_baseline'): p_auroc=3.643623307702546e-162 , p_auprc=4.7522710116359023e-144\n",
      "('deepmi', 'ecg_only_baseline')\n",
      "('deepmi', 'ecg_only_baseline'): p_auroc=4.88392024197109e-165 , p_auprc=1.4544668682991088e-117\n",
      "---------------------------------------------------------------\n",
      "---------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "for pair in testing_data_label_pairs:\n",
    "        \n",
    "    dataset = pair[0].split('.')[0]\n",
    "    print('---------------------------------------------------------------')\n",
    "    if 'ecg' in dataset:\n",
    "        print(dataset)\n",
    "        for model_pair in combs:\n",
    "            print(model_pair)\n",
    "            \n",
    "            dataset_1 = dataset.replace('2400', dataset_lengths[folders.index(model_pair[0])])\n",
    "            dataset_2 = dataset.replace('2400', dataset_lengths[folders.index(model_pair[1])])\n",
    "            \n",
    "            targets = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_1}_{model_names[folders.index(model_pair[0])]}_targets.npy')\n",
    "            pred_probs_1 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_1}_{model_names[folders.index(model_pair[0])]}_pred_probs.npy')\n",
    "            pred_probs_2 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_2}_{model_names[folders.index(model_pair[1])]}_pred_probs.npy')\n",
    "            auroc_1 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_1}_{model_names[folders.index(model_pair[0])]}_aurocs.npy')       \n",
    "            auroc_2 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_2}_{model_names[folders.index(model_pair[1])]}_aurocs.npy')\n",
    "            auprc_1 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_1}_{model_names[folders.index(model_pair[0])]}_auprcs.npy')            \n",
    "            auprc_2 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_2}_{model_names[folders.index(model_pair[1])]}_auprcs.npy')\n",
    "            \n",
    "            p_auroc_wc = stats.wilcoxon(auroc_1, auroc_2).pvalue\n",
    "            p_auprc_wc = stats.wilcoxon(auprc_1, auprc_2).pvalue\n",
    "        \n",
    "            print(f'{model_pair}: p_auroc={p_auroc_wc} , p_auprc={p_auprc_wc}') \n",
    "        \n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "df888aa5",
   "metadata": {},
   "source": [
    "# PPG (limited)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "0f859ce8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('simsiam_limited_labeled', 'deepmi_limited_labels'), ('simsiam_limited_labeled', 'ppg_only_baseline_limited_labeled'), ('deepmi_limited_labels', 'ppg_only_baseline_limited_labeled')]\n"
     ]
    }
   ],
   "source": [
    "folders = ['simsiam_limited_labeled', 'deepmi_limited_labels', 'ppg_only_baseline_limited_labeled']\n",
    "model_names = ['simsiamlimitedlabels', 'deepmilimitedlabels', 'ppgonlylimitedlabel']\n",
    "dataset_lengths = ['2400', '2400', '2400']\n",
    "combs = list(combinations(folders, 2))\n",
    "print(combs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "5e600e0f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------\n",
      "---------------------------------------------------------------\n",
      "data_simband_ppg_2400\n",
      "(856988,) (856988,) (856988,) (1000,) (1000,) (1000,) (1000,)\n",
      "(856988,) (856988,) (856988,) (1000,) (1000,) (1000,) (1000,)\n",
      "(856988,) (856988,) (856988,) (1000,) (1000,) (1000,) (1000,)\n",
      "---------------------------------------------------------------\n",
      "---------------------------------------------------------------\n",
      "data_ucla_ppg_2400\n",
      "(250683755,) (250683755,) (250683755,) (1000,) (1000,) (1000,) (1000,)\n",
      "(250683755,) (250683755,) (250683755,) (1000,) (1000,) (1000,) (1000,)\n",
      "(250683755,) (250683755,) (250683755,) (1000,) (1000,) (1000,) (1000,)\n",
      "---------------------------------------------------------------\n",
      "data_staford_2400\n",
      "(17556623,) (17556623,) (17556623,) (1000,) (1000,) (1000,) (1000,)\n",
      "(17556623,) (17556623,) (17556623,) (1000,) (1000,) (1000,) (1000,)\n",
      "(17556623,) (17556623,) (17556623,) (1000,) (1000,) (1000,) (1000,)\n"
     ]
    }
   ],
   "source": [
    "for pair in testing_data_label_pairs:\n",
    "\n",
    "    dataset = pair[0].split('.')[0]\n",
    "    print('---------------------------------------------------------------')\n",
    "    if 'ppg' in dataset or 'staford' in dataset:\n",
    "        print(dataset)\n",
    "        for model_pair in combs:\n",
    "\n",
    "            dataset_1 = dataset.replace('2400', dataset_lengths[folders.index(model_pair[0])])\n",
    "            dataset_2 = dataset.replace('2400', dataset_lengths[folders.index(model_pair[1])])\n",
    "            \n",
    "            targets = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_1}_{model_names[folders.index(model_pair[0])]}_targets.npy')\n",
    "            pred_probs_1 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_1}_{model_names[folders.index(model_pair[0])]}_pred_probs.npy')\n",
    "            pred_probs_2 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_2}_{model_names[folders.index(model_pair[1])]}_pred_probs.npy')\n",
    "            auroc_1 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_1}_{model_names[folders.index(model_pair[0])]}_aurocs.npy')       \n",
    "            auroc_2 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_2}_{model_names[folders.index(model_pair[1])]}_aurocs.npy')\n",
    "            auprc_1 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_1}_{model_names[folders.index(model_pair[0])]}_auprcs.npy')            \n",
    "            auprc_2 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_2}_{model_names[folders.index(model_pair[1])]}_auprcs.npy')\n",
    "            print(targets.shape, pred_probs_1.shape, pred_probs_2.shape, auroc_1.shape, auroc_2.shape, auprc_1.shape, auprc_2.shape)\n",
    "#             p_auroc = delong_roc_test(targets, pred_probs_1, pred_probs_2)[0][0]\n",
    "# #             p_auroc = stats.ttest_rel(auroc_1, auroc_2).pvalue\n",
    "#             p_auprc = stats.ttest_rel(auprc_1, auprc_2).pvalue\n",
    "            \n",
    "#             print(f'{model_pair}: p_auroc={p_auroc} , p_auprc={p_auprc}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "23e63337",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------\n",
      "---------------------------------------------------------------\n",
      "data_simband_ppg_2400\n",
      "('simsiam_limited_labeled', 'deepmi_limited_labels'): p_auroc=3.325859118934513e-165 , p_auprc=3.325859118934513e-165\n",
      "('simsiam_limited_labeled', 'ppg_only_baseline_limited_labeled'): p_auroc=3.325859118934513e-165 , p_auprc=3.4169553748281145e-165\n",
      "('deepmi_limited_labels', 'ppg_only_baseline_limited_labeled'): p_auroc=2.2605188352998174e-150 , p_auprc=3.325859118934513e-165\n",
      "---------------------------------------------------------------\n",
      "---------------------------------------------------------------\n",
      "data_ucla_ppg_2400\n",
      "('simsiam_limited_labeled', 'deepmi_limited_labels'): p_auroc=3.325859118934513e-165 , p_auprc=3.325859118934513e-165\n",
      "('simsiam_limited_labeled', 'ppg_only_baseline_limited_labeled'): p_auroc=3.325859118934513e-165 , p_auprc=3.325859118934513e-165\n",
      "('deepmi_limited_labels', 'ppg_only_baseline_limited_labeled'): p_auroc=7.636639239723726e-165 , p_auprc=4.785499635158569e-128\n",
      "---------------------------------------------------------------\n",
      "data_staford_2400\n",
      "('simsiam_limited_labeled', 'deepmi_limited_labels'): p_auroc=3.325859118934513e-165 , p_auprc=3.325859118934513e-165\n",
      "('simsiam_limited_labeled', 'ppg_only_baseline_limited_labeled'): p_auroc=3.325859118934513e-165 , p_auprc=3.325859118934513e-165\n",
      "('deepmi_limited_labels', 'ppg_only_baseline_limited_labeled'): p_auroc=3.325859118934513e-165 , p_auprc=4.775614275783113e-14\n"
     ]
    }
   ],
   "source": [
    "for pair in testing_data_label_pairs:\n",
    "\n",
    "    dataset = pair[0].split('.')[0]\n",
    "    print('---------------------------------------------------------------')\n",
    "    if 'ppg' in dataset or 'staford' in dataset:\n",
    "        print(dataset)\n",
    "        for model_pair in combs:\n",
    "\n",
    "            dataset_1 = dataset.replace('2400', dataset_lengths[folders.index(model_pair[0])])\n",
    "            dataset_2 = dataset.replace('2400', dataset_lengths[folders.index(model_pair[1])])\n",
    "            \n",
    "            targets = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_1}_{model_names[folders.index(model_pair[0])]}_targets.npy')\n",
    "            pred_probs_1 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_1}_{model_names[folders.index(model_pair[0])]}_pred_probs.npy')\n",
    "            pred_probs_2 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_2}_{model_names[folders.index(model_pair[1])]}_pred_probs.npy')\n",
    "            auroc_1 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_1}_{model_names[folders.index(model_pair[0])]}_aurocs.npy')       \n",
    "            auroc_2 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_2}_{model_names[folders.index(model_pair[1])]}_aurocs.npy')\n",
    "            auprc_1 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_1}_{model_names[folders.index(model_pair[0])]}_auprcs.npy')            \n",
    "            auprc_2 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_2}_{model_names[folders.index(model_pair[1])]}_auprcs.npy')\n",
    "            p_auroc_wc = stats.wilcoxon(auroc_1, auroc_2).pvalue\n",
    "            p_auprc_wc = stats.wilcoxon(auprc_1, auprc_2).pvalue\n",
    "        \n",
    "            print(f'{model_pair}: p_auroc={p_auroc_wc} , p_auprc={p_auprc_wc}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "72df07d1",
   "metadata": {},
   "source": [
    "# ECG Limited"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "e22b75e9",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('deepmi_limited_labels', 'ecg_only_baseline_limited_labeled')]\n"
     ]
    }
   ],
   "source": [
    "folders = ['deepmi_limited_labels', 'ecg_only_baseline_limited_labeled']\n",
    "model_names = ['deepmilimitedlabels', 'ecgonlylimitedlabel']\n",
    "dataset_lengths = ['2400', '2400']\n",
    "combs = list(combinations(folders, 2))\n",
    "print(combs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "014400ed",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------\n",
      "data_simband_ecg_2400\n",
      "(856988,) (856988,) (856988,) (1000,) (1000,) (1000,) (1000,)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [00:01<00:00,  1.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.00060045 0.0008392 ]\n",
      " [0.0008392  0.05907573]]\n",
      "False False\n",
      "('deepmi_limited_labels', 'ecg_only_baseline_limited_labeled'): p_auroc=5.908506151078022e-97 , p_auprc=9.951774711370074e-219\n",
      "---------------------------------------------------------------\n",
      "---------------------------------------------------------------\n",
      "data_ucla_ecg_2400\n",
      "(250683755,) (250683755,) (250683755,) (1000,) (1000,) (1000,) (1000,)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [09:37<00:00, 288.70s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.00032977 0.00083913]\n",
      " [0.00083913 0.00375018]]\n",
      "False False\n",
      "('deepmi_limited_labels', 'ecg_only_baseline_limited_labeled'): p_auroc=1.1920205966593885e-86 , p_auprc=0.0\n",
      "---------------------------------------------------------------\n",
      "---------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "for pair in testing_data_label_pairs:\n",
    "\n",
    "    dataset = pair[0].split('.')[0]\n",
    "    print('---------------------------------------------------------------')\n",
    "    if 'ecg' in dataset:\n",
    "        print(dataset)\n",
    "        for model_pair in combs:\n",
    "\n",
    "            dataset_1 = dataset.replace('2400', dataset_lengths[folders.index(model_pair[0])])\n",
    "            dataset_2 = dataset.replace('2400', dataset_lengths[folders.index(model_pair[1])])\n",
    "            \n",
    "            targets = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_1}_{model_names[folders.index(model_pair[0])]}_targets.npy')\n",
    "            pred_probs_1 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_1}_{model_names[folders.index(model_pair[0])]}_pred_probs.npy')\n",
    "            pred_probs_2 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_2}_{model_names[folders.index(model_pair[1])]}_pred_probs.npy')\n",
    "            auroc_1 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_1}_{model_names[folders.index(model_pair[0])]}_aurocs.npy')       \n",
    "            auroc_2 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_2}_{model_names[folders.index(model_pair[1])]}_aurocs.npy')\n",
    "            auprc_1 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_1}_{model_names[folders.index(model_pair[0])]}_auprcs.npy')            \n",
    "            auprc_2 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_2}_{model_names[folders.index(model_pair[1])]}_auprcs.npy')\n",
    "            print(targets.shape, pred_probs_1.shape, pred_probs_2.shape, auroc_1.shape, auroc_2.shape, auprc_1.shape, auprc_2.shape)\n",
    "            p_auroc = delong_roc_test(targets, pred_probs_1, pred_probs_2)[0][0]\n",
    "            p_auroc = stats.ttest_rel(auroc_1, auroc_2).pvalue\n",
    "            print(np.cov(auprc_1, auprc_2))\n",
    "            print(np.isnan(auprc_1).any(), np.isnan(auprc_2).any())\n",
    "            p_auprc = stats.ttest_rel(auprc_1, auprc_2).pvalue\n",
    "            \n",
    "            print(f'{model_pair}: p_auroc={p_auroc} , p_auprc={p_auprc}')\n",
    "        \n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "578fe99a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------\n",
      "data_simband_ecg_2400\n",
      "('deepmi_limited_labels', 'ecg_only_baseline_limited_labeled'): p_auroc=9.930811414707158e-83 , p_auprc=2.467723781036517e-151\n",
      "---------------------------------------------------------------\n",
      "---------------------------------------------------------------\n",
      "data_ucla_ecg_2400\n",
      "('deepmi_limited_labels', 'ecg_only_baseline_limited_labeled'): p_auroc=6.5387702296231955e-77 , p_auprc=3.325859118934513e-165\n",
      "---------------------------------------------------------------\n",
      "---------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "for pair in testing_data_label_pairs:\n",
    "\n",
    "    dataset = pair[0].split('.')[0]\n",
    "    print('---------------------------------------------------------------')\n",
    "    if 'ecg' in dataset:\n",
    "        print(dataset)\n",
    "        for model_pair in combs:\n",
    "\n",
    "            dataset_1 = dataset.replace('2400', dataset_lengths[folders.index(model_pair[0])])\n",
    "            dataset_2 = dataset.replace('2400', dataset_lengths[folders.index(model_pair[1])])\n",
    "            \n",
    "            targets = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_1}_{model_names[folders.index(model_pair[0])]}_targets.npy')\n",
    "            pred_probs_1 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_1}_{model_names[folders.index(model_pair[0])]}_pred_probs.npy')\n",
    "            pred_probs_2 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_2}_{model_names[folders.index(model_pair[1])]}_pred_probs.npy')\n",
    "            auroc_1 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_1}_{model_names[folders.index(model_pair[0])]}_aurocs.npy')       \n",
    "            auroc_2 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_2}_{model_names[folders.index(model_pair[1])]}_aurocs.npy')\n",
    "            auprc_1 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_1}_{model_names[folders.index(model_pair[0])]}_auprcs.npy')            \n",
    "            auprc_2 = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_2}_{model_names[folders.index(model_pair[1])]}_auprcs.npy')\n",
    "            p_auroc_wc = stats.wilcoxon(auroc_1, auroc_2).pvalue\n",
    "            p_auprc_wc = stats.wilcoxon(auprc_1, auprc_2).pvalue\n",
    "        \n",
    "            print(f'{model_pair}: p_auroc={p_auroc_wc} , p_auprc={p_auprc_wc}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "9211c5b2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data_simband_ecg_2400 ecgonlylimitedlabel\n"
     ]
    }
   ],
   "source": [
    "print(dataset_2, model_names[folders.index(model_pair[1])])\n",
    "auprc_a = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_2}_{model_names[folders.index(model_pair[1])]}_auprcs.npy')\n",
    "auprc_b = np.load(f'/labs/hulab/stark_stuff/ppg_ecg_project/data/{dataset_2}_{model_names[folders.index(model_pair[1])]}_auprcs22.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "324d0a89",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-0.12736596  0.1113902   0.097986   -0.12026541  0.10863852  0.08641139\n",
      "  0.08410832  0.13772868  0.11613212 -0.11047159  0.10411925  0.09066356\n",
      " -0.11684781 -0.14474963 -0.10132021 -0.11856603  0.1079938   0.10961868\n",
      "  0.09353498 -0.11655884  0.09695574  0.09675599  0.0954865   0.08374276\n",
      "  0.09695494  0.09538618 -0.13222351  0.08940818  0.07510125 -0.16161896\n",
      " -0.10932272 -0.13019148 -0.12763121 -0.16304358  0.08529244  0.11625032\n",
      "  0.0910924   0.10116565 -0.13310669  0.10303141  0.10516644  0.09491135\n",
      "  0.10445315  0.08400382  0.08765855  0.11400105 -0.1207005   0.1111073\n",
      " -0.13830953  0.10739836 -0.11107001  0.10330599 -0.1352469  -0.15405091\n",
      "  0.10597005 -0.1365743   0.0681474  -0.12848281 -0.14936784 -0.13283836\n",
      " -0.14560917 -0.13523861  0.11133322  0.09620434  0.09164903  0.0981501\n",
      " -0.14205062  0.09048466  0.09618169 -0.13573429  0.09120903  0.08723406\n",
      "  0.09562974  0.10111554 -0.1199902   0.10194261  0.10320011 -0.11931961\n",
      "  0.08801583 -0.12856544  0.09793015  0.09127223  0.10039747  0.08213921\n",
      " -0.13402325 -0.12046024 -0.10644285  0.09656188 -0.11927469  0.10326398\n",
      " -0.1157115   0.10120237  0.10866226  0.08271042  0.12058458  0.07881858\n",
      "  0.11154262  0.08934924  0.10123797 -0.13592907  0.09490008  0.12431603\n",
      " -0.14491149 -0.14242002  0.09485723  0.09173253 -0.17808166 -0.10646853\n",
      "  0.10601733  0.09165249  0.07949868 -0.16178404  0.07018796  0.09803946\n",
      " -0.13725072 -0.14949187  0.11223473  0.10070887  0.11479844 -0.15540059\n",
      "  0.10928021  0.08383657  0.09275953 -0.14449827  0.08824387 -0.13598121\n",
      " -0.12235151 -0.15305421 -0.13162918  0.09931677  0.0815974   0.10122475\n",
      " -0.13931217  0.09188294 -0.15563664  0.09366665  0.1005954   0.09912131\n",
      " -0.14486805  0.10457091 -0.1425189   0.104137    0.08751814 -0.13127006\n",
      " -0.15510238  0.07795354 -0.12380598  0.07909092 -0.12985232  0.10365673\n",
      " -0.11876106 -0.11579314 -0.12372238  0.09737266  0.09220327  0.09191623\n",
      " -0.13420673  0.08709468 -0.14039955 -0.14993551 -0.14448654  0.07332461\n",
      "  0.09995121  0.08523702  0.09019982 -0.11577781 -0.0892418  -0.13619929\n",
      "  0.09456116  0.09025777 -0.15445966  0.07960176  0.09508991 -0.13951104\n",
      "  0.09526544  0.10335183  0.09008426  0.0799527   0.0942093  -0.13383439\n",
      "  0.09473486  0.10453795  0.0866453   0.10296453 -0.11442398  0.0909386\n",
      " -0.11892994 -0.12074588  0.11105868  0.09212454 -0.14040577  0.10855385\n",
      "  0.0998704  -0.13870132 -0.12416759 -0.1210601   0.11214875 -0.10524959\n",
      " -0.11631562  0.10003293 -0.11557547  0.09380038 -0.14343757  0.09930893\n",
      "  0.09072019 -0.09967234  0.09462383  0.09421255  0.09037286  0.09394834\n",
      "  0.07543752 -0.15743019 -0.13571878  0.09461586  0.11403478 -0.13301566\n",
      " -0.11555895  0.12499006  0.09356126 -0.12636628  0.09185006  0.08753459\n",
      "  0.09876829  0.10329953  0.11660011  0.10074591  0.09973706 -0.0955538\n",
      " -0.14615039 -0.13502268  0.11898287 -0.13141862  0.09140446  0.08810273\n",
      "  0.08153625 -0.13886406  0.07175447  0.10939001  0.09665712  0.08280755\n",
      "  0.1036247   0.09712061 -0.14568128 -0.1416754   0.09744883  0.0770696\n",
      "  0.08667263  0.09346726  0.0967815   0.09694454  0.09983459  0.07686539\n",
      "  0.07894776  0.12022219 -0.13307133 -0.09896647  0.08151258 -0.1274829\n",
      "  0.11754145 -0.10473272  0.12078204  0.11149355  0.09883949  0.09597041\n",
      "  0.09735193  0.08574184  0.08869504  0.08207598  0.11781235  0.08621325\n",
      "  0.09414353  0.10487512 -0.12377878 -0.10253617  0.10015075  0.09754367\n",
      "  0.10397136  0.06214336 -0.12140508 -0.14061091  0.10277505 -0.13109731\n",
      "  0.08981644 -0.14764894  0.08957351  0.09761159  0.09520339  0.08565678\n",
      " -0.14187649 -0.16235248  0.11299966 -0.1077316   0.10165235  0.10801704\n",
      " -0.12737877  0.12137344 -0.12589677  0.07108035  0.09305421  0.11156837\n",
      "  0.09336447  0.11240704  0.09359399 -0.11640069  0.11015753 -0.11706341\n",
      " -0.13765431  0.08023752 -0.16945531 -0.12054646  0.10277395  0.09439931\n",
      " -0.13113398  0.10727218  0.07743704  0.08631579  0.11481435  0.09997495\n",
      "  0.10642175  0.09052341  0.12720937 -0.09591848  0.10529655  0.09758023\n",
      " -0.13761038 -0.11471164  0.10267175  0.0858024   0.09051153 -0.13968828\n",
      " -0.1435244   0.11142107 -0.15338924  0.09803593  0.11097994  0.10666853\n",
      "  0.09606637 -0.13855866 -0.12503478  0.10020558  0.08704597  0.09665897\n",
      " -0.13204399  0.07385445 -0.11696376 -0.1374621  -0.13949299  0.11724244\n",
      "  0.08803206  0.10217663  0.11615542  0.09164172 -0.13765685 -0.15746369\n",
      "  0.09364997 -0.15709849  0.09073691 -0.13519432  0.10613813 -0.11984213\n",
      "  0.08874316  0.09830791  0.0869641   0.10351273 -0.10905419 -0.14893412\n",
      "  0.10972256  0.11302753 -0.14058963 -0.10086052  0.08460134  0.08549426\n",
      " -0.15324303 -0.12102477  0.08560088  0.11064262 -0.12738348 -0.11065243\n",
      "  0.10702016  0.12102041  0.10779618 -0.13322703  0.10077024  0.084861\n",
      " -0.14298679 -0.12597298  0.09668019  0.11728738  0.10889732 -0.11035945\n",
      " -0.13498325  0.11049581 -0.15943196  0.09587903 -0.12886858  0.1052795\n",
      "  0.07987086  0.10464791  0.09618314  0.1089528   0.0775403   0.1228796\n",
      "  0.12879735  0.09168496 -0.11303719  0.10151409  0.09475525 -0.11833172\n",
      " -0.14477973  0.09594445  0.11732505  0.0906315  -0.15707589  0.10396713\n",
      "  0.09385121  0.1068136  -0.11368558 -0.14216884  0.10497213 -0.1342384\n",
      "  0.11663819 -0.10255367  0.097222    0.09041278 -0.1448159   0.08440922\n",
      "  0.10359494  0.08615013  0.10813494 -0.13144466  0.07629744  0.10203145\n",
      "  0.08139696  0.08763119 -0.11856286  0.08877761  0.10681567  0.10349463\n",
      " -0.12854711  0.07310841  0.09810788  0.11789665  0.09568294  0.08245492\n",
      "  0.09757185  0.10330516  0.09807274  0.07952851  0.10576941 -0.13249509\n",
      "  0.11110264 -0.14027558  0.10803635  0.07998717 -0.1211042   0.12137576\n",
      "  0.0903869  -0.16002624  0.08376402  0.08941368 -0.11759719  0.1103714\n",
      "  0.11660906  0.11704337  0.09825879  0.07511215 -0.14408322 -0.12905867\n",
      " -0.10842186  0.10001096  0.11217363 -0.14970094  0.09144501 -0.13847444\n",
      "  0.0953171   0.0991781   0.09306968 -0.14311139  0.11762654  0.09472874\n",
      " -0.13214767  0.09712329  0.11227749  0.09689329  0.11376039 -0.14126969\n",
      " -0.15896425  0.1273842  -0.11757756 -0.11160203 -0.12866134  0.09870634\n",
      "  0.09292176 -0.13944048  0.0998591  -0.14954528  0.10713992  0.10051277\n",
      "  0.08180095 -0.1230985  -0.11654593 -0.12589332  0.08316803 -0.09820829\n",
      " -0.11891277 -0.1359252   0.10426393  0.09030151  0.10378474  0.11610515\n",
      " -0.10985208  0.08418618 -0.13047345  0.09519014  0.09879165 -0.18211962\n",
      "  0.10256628  0.10552104  0.09772001  0.09400453  0.07681168  0.10589187\n",
      "  0.08577146  0.08661459  0.09030576  0.09433329 -0.13058234 -0.14831487\n",
      " -0.11620229  0.09200404  0.1064689   0.07262618  0.07751712  0.10206353\n",
      " -0.12680884  0.08826922  0.08160978 -0.13323734  0.09573761 -0.10134629\n",
      "  0.09699369 -0.1408126   0.08904345  0.08851735 -0.09664158 -0.1337921\n",
      "  0.10127841  0.09076326  0.09996749  0.13180682  0.10643998  0.08835939\n",
      "  0.09443274  0.0809166   0.09851489  0.09290032  0.08944412  0.10469036\n",
      "  0.10284299  0.07150214 -0.11380751 -0.11940856 -0.10058371  0.09429688\n",
      "  0.10263199 -0.1170717   0.11162003  0.0809173  -0.15417477 -0.14340186\n",
      "  0.10556666 -0.10717559  0.1018569   0.08753012  0.08903611  0.09506348\n",
      "  0.10894988 -0.14474499  0.10904162  0.11522711 -0.11237599 -0.1209672\n",
      " -0.16916073 -0.13074435  0.1146527  -0.1174616   0.07086252 -0.13218221\n",
      "  0.0977059  -0.15395809  0.09334947  0.1092841   0.08610816  0.10478529\n",
      "  0.10158142  0.08576779  0.09965347 -0.13354226  0.08747676  0.10615328\n",
      " -0.17209658  0.08741672  0.08765163  0.08611058  0.1069514   0.07831902\n",
      " -0.13405867 -0.12066212  0.08656545 -0.12224102  0.1007949   0.11476268\n",
      " -0.15040557 -0.1248121  -0.10714793 -0.12653352 -0.15434983  0.07568199\n",
      " -0.13336211 -0.15459209 -0.11951346 -0.11329901 -0.13291745 -0.12776539\n",
      " -0.1098251   0.08682689  0.10216266 -0.12931242  0.1239656   0.07477485\n",
      "  0.10421299  0.09439773  0.09767173  0.12101715  0.08582289 -0.12530171\n",
      " -0.13511635 -0.1154987   0.09897966  0.09840794 -0.15103284 -0.14284627\n",
      " -0.13317056 -0.11307848 -0.11742623  0.10366474 -0.15007229 -0.12470355\n",
      "  0.0760704   0.12644266 -0.17625646 -0.13634583 -0.13181785 -0.13485532\n",
      "  0.12096393  0.0850004   0.09831936 -0.15526989 -0.11013788 -0.1281899\n",
      "  0.1280578   0.11895452 -0.14417651  0.07734503  0.08886009 -0.10984759\n",
      " -0.1157618   0.10232897 -0.14209068  0.07909395 -0.14251135  0.0758311\n",
      "  0.11117052  0.09721196 -0.09959253 -0.14995505  0.11285443  0.10117978\n",
      "  0.10248986  0.09217854  0.09966138  0.11316444 -0.13291785 -0.14101489\n",
      "  0.12051051 -0.16822915 -0.14396232  0.08820305  0.06752236  0.10523473\n",
      " -0.12985246 -0.10929643 -0.12488578  0.12539543 -0.12365084 -0.11616536\n",
      "  0.10407647  0.09382759  0.09945353  0.09439594  0.08611686 -0.15135441\n",
      "  0.10911775  0.09516816 -0.12802963 -0.14086242 -0.11322996  0.08731177\n",
      "  0.10662878  0.09853687 -0.12029287  0.10465507  0.07780426 -0.12587705\n",
      "  0.09617362 -0.13647897  0.08706049  0.08265733  0.10124556  0.10450961\n",
      "  0.09863066  0.0905115   0.099633    0.11448744  0.09754346 -0.13455881\n",
      "  0.11693337 -0.10718133 -0.12845941 -0.13760542  0.0897662   0.11499997\n",
      " -0.15320885  0.08255407  0.12555082  0.09703246  0.11011558  0.10289046\n",
      " -0.12513826  0.1050776  -0.11456812  0.08118395 -0.1240148  -0.11311946\n",
      "  0.0972581   0.07837772  0.08840685  0.09267898 -0.15279861  0.07058638\n",
      "  0.10292675  0.09418501  0.10588428  0.10036465  0.09594957 -0.11665863\n",
      "  0.0993912   0.10157359  0.09215527  0.09251881 -0.12190064 -0.15088201\n",
      "  0.08592183 -0.12512662  0.09856718  0.11104585  0.11416185  0.0966903\n",
      " -0.17714316  0.10204778 -0.14914796  0.11279489 -0.17093307  0.09516569\n",
      "  0.08001403 -0.11177845 -0.11866444  0.10314879  0.1010559   0.0798476\n",
      "  0.09278245 -0.12136698 -0.1320027   0.0903853  -0.11411273  0.09632005\n",
      " -0.11720844 -0.13721928 -0.10693961  0.0903292  -0.1402251   0.11332449\n",
      "  0.10078607  0.09896245  0.08831977  0.11435228 -0.13191169 -0.09901224\n",
      "  0.09725064 -0.14862201  0.10222273  0.08844592  0.10022964  0.08302862\n",
      " -0.1386899  -0.15497471 -0.1119309   0.09814524  0.10764011  0.09037316\n",
      " -0.12681724  0.08442173  0.08904381  0.10820724 -0.11710475  0.09690213\n",
      "  0.11578957  0.11050113  0.08416856  0.10287318  0.11867139 -0.15505621\n",
      "  0.09989024 -0.10452559  0.11339361  0.12264172 -0.11868168  0.0867271\n",
      " -0.14385116  0.10127363 -0.13149998 -0.11621547  0.09674804 -0.13337922\n",
      " -0.15647779 -0.14825975  0.11039024  0.12210533  0.08391305  0.10167288\n",
      " -0.12727664 -0.13531752 -0.12606213  0.11897121 -0.12612775  0.11071103\n",
      "  0.1203704  -0.12077034  0.08235701  0.08064419  0.10553503  0.09948338\n",
      "  0.10775928  0.10500228  0.09958443  0.11072566 -0.12425017  0.09922821\n",
      "  0.09223318  0.09095089 -0.08488071  0.08115225 -0.14567374  0.09501085\n",
      "  0.09952642  0.09995993  0.11999583  0.12155772  0.10178928 -0.13301264\n",
      "  0.08600088  0.10537209 -0.15554607 -0.09917083  0.10498436  0.07894638\n",
      " -0.1566877  -0.11043466  0.10110942 -0.13355431 -0.12065438 -0.15024508\n",
      " -0.13625314  0.09079507  0.08795393  0.08033286  0.08550308  0.09493812\n",
      "  0.08087463 -0.13368148  0.07884703  0.09379607  0.07195529  0.10990627\n",
      "  0.10879639  0.10300481 -0.09848043  0.09779629  0.09959895 -0.14429837\n",
      " -0.12878115 -0.13052658 -0.10493899  0.0970263   0.11442668  0.09387984\n",
      "  0.12194792 -0.12971038  0.11942284  0.11351842  0.0890923  -0.14199716\n",
      "  0.10705816  0.11410302 -0.13201933  0.09469685  0.09941291  0.09107797\n",
      "  0.09788322 -0.1435275  -0.12190221  0.1220827   0.10569259  0.11806733\n",
      " -0.14908666  0.08019613 -0.10949351  0.08539787 -0.11461496  0.08786915\n",
      " -0.14810683  0.07491141 -0.11618199  0.09823832 -0.16311779  0.08403952\n",
      " -0.12801028  0.10786032  0.10674259 -0.14201515  0.09127291  0.09121501\n",
      " -0.13859288  0.08194409  0.10731713  0.12215761  0.09030587  0.08978381\n",
      "  0.09363227 -0.11807251 -0.18615913 -0.14982864  0.09161289 -0.1286681\n",
      " -0.15646825  0.11517565  0.11254936  0.10398526  0.09684138 -0.17494603\n",
      "  0.09265791  0.10203463 -0.11545064  0.10011997 -0.1102171   0.10934301\n",
      "  0.06433551 -0.12576552  0.10193497 -0.13534026  0.10065013 -0.0972538\n",
      "  0.11466539  0.11665865  0.08817279  0.06596066  0.08060166  0.11797925\n",
      "  0.09789295 -0.10284243 -0.09320178  0.11426132  0.09611698 -0.16823232\n",
      " -0.1340083   0.09444818 -0.16650491 -0.11578775  0.10532825  0.10296009\n",
      "  0.08514086 -0.13654458  0.08221966 -0.12766319  0.09270594  0.115031\n",
      "  0.10556215  0.09483803 -0.13662158  0.09543928]\n"
     ]
    }
   ],
   "source": [
    "print(np.subtract(auprc_a, auprc_b))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f2bb457e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base_pytorch",
   "language": "python",
   "name": "base_pytorch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}