0.expression-download/2A.process-pancanatlas-data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Process TCGA PanCanAtlas Data\n",
    "\n",
    "Retrieve the downloaded expression data, update gene identifiers to entrez, and curate sample IDs. The script will also identify a balanced hold-out test set to compare projection performance into learned latent spaces across algorithms."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import random\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "random.seed(1234)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read TCGA Barcode Curation Information\n",
    "\n",
    "Extract information from TCGA barcodes - `cancer-type` and `sample-type`. See https://github.com/cognoma/cancer-data for more details"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Commit from https://github.com/cognoma/cancer-data/\n",
    "sample_commit = 'da832c5edc1ca4d3f665b038d15b19fced724f4c'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TSS Code</th>\n",
       "      <th>Source Site</th>\n",
       "      <th>Study Name</th>\n",
       "      <th>BCR</th>\n",
       "      <th>acronym</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>01</td>\n",
       "      <td>International Genomics Consortium</td>\n",
       "      <td>ovarian serous cystadenocarcinoma</td>\n",
       "      <td>IGC</td>\n",
       "      <td>OV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>02</td>\n",
       "      <td>MD Anderson Cancer Center</td>\n",
       "      <td>glioblastoma multiforme</td>\n",
       "      <td>IGC</td>\n",
       "      <td>GBM</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  TSS Code                        Source Site  \\\n",
       "0       01  International Genomics Consortium   \n",
       "1       02          MD Anderson Cancer Center   \n",
       "\n",
       "                          Study Name  BCR acronym  \n",
       "0  ovarian serous cystadenocarcinoma  IGC      OV  \n",
       "1            glioblastoma multiforme  IGC     GBM  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url = 'https://raw.githubusercontent.com/cognoma/cancer-data/{}/mapping/tcga_cancertype_codes.csv'.format(sample_commit)\n",
    "cancer_types_df = pd.read_csv(url,\n",
    "                              dtype='str',\n",
    "                              keep_default_na=False)\n",
    "\n",
    "cancertype_codes_dict = dict(zip(cancer_types_df['TSS Code'],\n",
    "                                 cancer_types_df.acronym))\n",
    "cancer_types_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Code</th>\n",
       "      <th>Definition</th>\n",
       "      <th>Short Letter Code</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>01</td>\n",
       "      <td>Primary Solid Tumor</td>\n",
       "      <td>TP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>02</td>\n",
       "      <td>Recurrent Solid Tumor</td>\n",
       "      <td>TR</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Code             Definition Short Letter Code\n",
       "0   01    Primary Solid Tumor                TP\n",
       "1   02  Recurrent Solid Tumor                TR"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url = 'https://raw.githubusercontent.com/cognoma/cancer-data/{}/mapping/tcga_sampletype_codes.csv'.format(sample_commit)\n",
    "sample_types_df = pd.read_csv(url, dtype='str')\n",
    "\n",
    "sampletype_codes_dict = dict(zip(sample_types_df.Code,\n",
    "                                 sample_types_df.Definition))\n",
    "sample_types_df.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read Entrez ID Curation Information\n",
    "\n",
    "Load curated gene names from versioned resource. See https://github.com/cognoma/genes for more details"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Commit from https://github.com/cognoma/genes\n",
    "genes_commit = 'ad9631bb4e77e2cdc5413b0d77cb8f7e93fc5bee'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20395, 7)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>entrez_gene_id</th>\n",
       "      <th>symbol</th>\n",
       "      <th>description</th>\n",
       "      <th>chromosome</th>\n",
       "      <th>gene_type</th>\n",
       "      <th>synonyms</th>\n",
       "      <th>aliases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>A1BG</td>\n",
       "      <td>alpha-1-B glycoprotein</td>\n",
       "      <td>19</td>\n",
       "      <td>protein-coding</td>\n",
       "      <td>A1B|ABG|GAB|HYST2477</td>\n",
       "      <td>alpha-1B-glycoprotein|HEL-S-163pA|epididymis s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>A2M</td>\n",
       "      <td>alpha-2-macroglobulin</td>\n",
       "      <td>12</td>\n",
       "      <td>protein-coding</td>\n",
       "      <td>A2MD|CPAMD5|FWP007|S863-7</td>\n",
       "      <td>alpha-2-macroglobulin|C3 and PZP-like alpha-2-...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   entrez_gene_id symbol             description chromosome       gene_type  \\\n",
       "0               1   A1BG  alpha-1-B glycoprotein         19  protein-coding   \n",
       "1               2    A2M   alpha-2-macroglobulin         12  protein-coding   \n",
       "\n",
       "                    synonyms  \\\n",
       "0       A1B|ABG|GAB|HYST2477   \n",
       "1  A2MD|CPAMD5|FWP007|S863-7   \n",
       "\n",
       "                                             aliases  \n",
       "0  alpha-1B-glycoprotein|HEL-S-163pA|epididymis s...  \n",
       "1  alpha-2-macroglobulin|C3 and PZP-like alpha-2-...  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url = 'https://raw.githubusercontent.com/cognoma/genes/{}/data/genes.tsv'.format(genes_commit)\n",
    "gene_df = pd.read_table(url)\n",
    "\n",
    "# Only consider protein-coding genes\n",
    "gene_df = (\n",
    "    gene_df.query(\"gene_type == 'protein-coding'\")\n",
    ")\n",
    "\n",
    "print(gene_df.shape)\n",
    "gene_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load gene updater - define up to date Entrez gene identifiers where appropriate\n",
    "url = 'https://raw.githubusercontent.com/cognoma/genes/{}/data/updater.tsv'.format(genes_commit)\n",
    "updater_df = pd.read_table(url)\n",
    "\n",
    "old_to_new_entrez = dict(zip(updater_df.old_entrez_gene_id,\n",
    "                             updater_df.new_entrez_gene_id))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read Gene Expression Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20531, 11069)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TCGA-OR-A5J1-01A-11R-A29S-07</th>\n",
       "      <th>TCGA-OR-A5J2-01A-11R-A29S-07</th>\n",
       "      <th>TCGA-OR-A5J3-01A-11R-A29S-07</th>\n",
       "      <th>TCGA-OR-A5J5-01A-11R-A29S-07</th>\n",
       "      <th>TCGA-OR-A5J6-01A-31R-A29S-07</th>\n",
       "      <th>TCGA-OR-A5J7-01A-11R-A29S-07</th>\n",
       "      <th>TCGA-OR-A5J8-01A-11R-A29S-07</th>\n",
       "      <th>TCGA-OR-A5J9-01A-11R-A29S-07</th>\n",
       "      <th>TCGA-OR-A5JA-01A-11R-A29S-07</th>\n",
       "      <th>TCGA-OR-A5JB-01A-11R-A29S-07</th>\n",
       "      <th>...</th>\n",
       "      <th>TCGA-CG-4449-01A-01R-1157-13</th>\n",
       "      <th>TCGA-CG-4462-01A-01R-1157-13</th>\n",
       "      <th>TCGA-CG-4465-01A-01R-1157-13</th>\n",
       "      <th>TCGA-CG-4466-01A-01R-1157-13</th>\n",
       "      <th>TCGA-CG-4469-01A-01R-1157-13</th>\n",
       "      <th>TCGA-CG-4472-01A-01R-1157-13</th>\n",
       "      <th>TCGA-CG-4474-01A-02R-1157-13</th>\n",
       "      <th>TCGA-CG-4475-01A-01R-1157-13</th>\n",
       "      <th>TCGA-CG-4476-01A-01R-1157-13</th>\n",
       "      <th>TCGA-CG-4477-01A-01R-1157-13</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gene_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>?|100130426</th>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>?|100133144</th>\n",
       "      <td>3.2661</td>\n",
       "      <td>2.6815</td>\n",
       "      <td>1.7301</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>1.1673</td>\n",
       "      <td>1.4422</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>4.4556</td>\n",
       "      <td>7.1293</td>\n",
       "      <td>...</td>\n",
       "      <td>4.358154</td>\n",
       "      <td>5.676995</td>\n",
       "      <td>5.219350</td>\n",
       "      <td>14.846708</td>\n",
       "      <td>20.115492</td>\n",
       "      <td>6.997533</td>\n",
       "      <td>18.311906</td>\n",
       "      <td>12.057112</td>\n",
       "      <td>18.628740</td>\n",
       "      <td>17.874417</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>?|100134869</th>\n",
       "      <td>3.9385</td>\n",
       "      <td>8.9948</td>\n",
       "      <td>6.5650</td>\n",
       "      <td>1.5492</td>\n",
       "      <td>4.4709</td>\n",
       "      <td>6.0529</td>\n",
       "      <td>2.2876</td>\n",
       "      <td>1.3599</td>\n",
       "      <td>5.0581</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>...</td>\n",
       "      <td>2.656360</td>\n",
       "      <td>3.342794</td>\n",
       "      <td>2.423442</td>\n",
       "      <td>5.055287</td>\n",
       "      <td>11.626054</td>\n",
       "      <td>13.654193</td>\n",
       "      <td>7.417109</td>\n",
       "      <td>11.585177</td>\n",
       "      <td>11.482418</td>\n",
       "      <td>14.919338</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>?|10357</th>\n",
       "      <td>149.1350</td>\n",
       "      <td>81.0777</td>\n",
       "      <td>86.4879</td>\n",
       "      <td>53.9117</td>\n",
       "      <td>66.9063</td>\n",
       "      <td>103.5060</td>\n",
       "      <td>94.9316</td>\n",
       "      <td>78.1955</td>\n",
       "      <td>69.2389</td>\n",
       "      <td>155.7090</td>\n",
       "      <td>...</td>\n",
       "      <td>633.299781</td>\n",
       "      <td>294.018042</td>\n",
       "      <td>686.569179</td>\n",
       "      <td>563.573453</td>\n",
       "      <td>1039.307597</td>\n",
       "      <td>639.238135</td>\n",
       "      <td>742.479964</td>\n",
       "      <td>506.336449</td>\n",
       "      <td>712.452165</td>\n",
       "      <td>703.713324</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>?|10431</th>\n",
       "      <td>2034.1000</td>\n",
       "      <td>1304.9300</td>\n",
       "      <td>1054.6600</td>\n",
       "      <td>2350.8900</td>\n",
       "      <td>1257.9900</td>\n",
       "      <td>1866.4300</td>\n",
       "      <td>995.0270</td>\n",
       "      <td>1762.1200</td>\n",
       "      <td>1213.5300</td>\n",
       "      <td>2005.5700</td>\n",
       "      <td>...</td>\n",
       "      <td>1202.538277</td>\n",
       "      <td>644.002317</td>\n",
       "      <td>1181.884532</td>\n",
       "      <td>663.885074</td>\n",
       "      <td>647.530395</td>\n",
       "      <td>1297.152549</td>\n",
       "      <td>1152.909807</td>\n",
       "      <td>1375.495774</td>\n",
       "      <td>971.893874</td>\n",
       "      <td>1736.988111</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 11069 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             TCGA-OR-A5J1-01A-11R-A29S-07  TCGA-OR-A5J2-01A-11R-A29S-07  \\\n",
       "gene_id                                                                   \n",
       "?|100130426                        0.0000                        0.0000   \n",
       "?|100133144                        3.2661                        2.6815   \n",
       "?|100134869                        3.9385                        8.9948   \n",
       "?|10357                          149.1350                       81.0777   \n",
       "?|10431                         2034.1000                     1304.9300   \n",
       "\n",
       "             TCGA-OR-A5J3-01A-11R-A29S-07  TCGA-OR-A5J5-01A-11R-A29S-07  \\\n",
       "gene_id                                                                   \n",
       "?|100130426                        0.0000                        0.0000   \n",
       "?|100133144                        1.7301                        0.0000   \n",
       "?|100134869                        6.5650                        1.5492   \n",
       "?|10357                           86.4879                       53.9117   \n",
       "?|10431                         1054.6600                     2350.8900   \n",
       "\n",
       "             TCGA-OR-A5J6-01A-31R-A29S-07  TCGA-OR-A5J7-01A-11R-A29S-07  \\\n",
       "gene_id                                                                   \n",
       "?|100130426                        0.0000                        0.0000   \n",
       "?|100133144                        0.0000                        1.1673   \n",
       "?|100134869                        4.4709                        6.0529   \n",
       "?|10357                           66.9063                      103.5060   \n",
       "?|10431                         1257.9900                     1866.4300   \n",
       "\n",
       "             TCGA-OR-A5J8-01A-11R-A29S-07  TCGA-OR-A5J9-01A-11R-A29S-07  \\\n",
       "gene_id                                                                   \n",
       "?|100130426                        0.0000                        0.0000   \n",
       "?|100133144                        1.4422                        0.0000   \n",
       "?|100134869                        2.2876                        1.3599   \n",
       "?|10357                           94.9316                       78.1955   \n",
       "?|10431                          995.0270                     1762.1200   \n",
       "\n",
       "             TCGA-OR-A5JA-01A-11R-A29S-07  TCGA-OR-A5JB-01A-11R-A29S-07  \\\n",
       "gene_id                                                                   \n",
       "?|100130426                        0.0000                        0.0000   \n",
       "?|100133144                        4.4556                        7.1293   \n",
       "?|100134869                        5.0581                        0.0000   \n",
       "?|10357                           69.2389                      155.7090   \n",
       "?|10431                         1213.5300                     2005.5700   \n",
       "\n",
       "                         ...               TCGA-CG-4449-01A-01R-1157-13  \\\n",
       "gene_id                  ...                                              \n",
       "?|100130426              ...                                        NaN   \n",
       "?|100133144              ...                                   4.358154   \n",
       "?|100134869              ...                                   2.656360   \n",
       "?|10357                  ...                                 633.299781   \n",
       "?|10431                  ...                                1202.538277   \n",
       "\n",
       "             TCGA-CG-4462-01A-01R-1157-13  TCGA-CG-4465-01A-01R-1157-13  \\\n",
       "gene_id                                                                   \n",
       "?|100130426                           NaN                           NaN   \n",
       "?|100133144                      5.676995                      5.219350   \n",
       "?|100134869                      3.342794                      2.423442   \n",
       "?|10357                        294.018042                    686.569179   \n",
       "?|10431                        644.002317                   1181.884532   \n",
       "\n",
       "             TCGA-CG-4466-01A-01R-1157-13  TCGA-CG-4469-01A-01R-1157-13  \\\n",
       "gene_id                                                                   \n",
       "?|100130426                           NaN                           NaN   \n",
       "?|100133144                     14.846708                     20.115492   \n",
       "?|100134869                      5.055287                     11.626054   \n",
       "?|10357                        563.573453                   1039.307597   \n",
       "?|10431                        663.885074                    647.530395   \n",
       "\n",
       "             TCGA-CG-4472-01A-01R-1157-13  TCGA-CG-4474-01A-02R-1157-13  \\\n",
       "gene_id                                                                   \n",
       "?|100130426                           NaN                           NaN   \n",
       "?|100133144                      6.997533                     18.311906   \n",
       "?|100134869                     13.654193                      7.417109   \n",
       "?|10357                        639.238135                    742.479964   \n",
       "?|10431                       1297.152549                   1152.909807   \n",
       "\n",
       "             TCGA-CG-4475-01A-01R-1157-13  TCGA-CG-4476-01A-01R-1157-13  \\\n",
       "gene_id                                                                   \n",
       "?|100130426                           NaN                           NaN   \n",
       "?|100133144                     12.057112                     18.628740   \n",
       "?|100134869                     11.585177                     11.482418   \n",
       "?|10357                        506.336449                    712.452165   \n",
       "?|10431                       1375.495774                    971.893874   \n",
       "\n",
       "             TCGA-CG-4477-01A-01R-1157-13  \n",
       "gene_id                                    \n",
       "?|100130426                           NaN  \n",
       "?|100133144                     17.874417  \n",
       "?|100134869                     14.919338  \n",
       "?|10357                        703.713324  \n",
       "?|10431                       1736.988111  \n",
       "\n",
       "[5 rows x 11069 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file = os.path.join('download', 'EBPlusPlusAdjustPANCAN_IlluminaHiSeq_RNASeqV2-v2.geneExp.tsv')\n",
    "tcga_expr_df = pd.read_table(file, index_col=0)\n",
    "\n",
    "print(tcga_expr_df.shape)\n",
    "tcga_expr_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Process gene expression matrix\n",
    "\n",
    "This involves updating Entrez gene ids, sorting and subsetting."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set index as entrez_gene_id\n",
    "tcga_expr_df.index = tcga_expr_df.index.map(lambda x: x.split('|')[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "tcga_expr_df = (tcga_expr_df\n",
    "    .dropna(axis='rows')\n",
    "    .rename(index=old_to_new_entrez)\n",
    "    .groupby(level=0).mean()\n",
    "    .transpose()\n",
    "    .sort_index(axis='rows')\n",
    "    .sort_index(axis='columns')\n",
    ")\n",
    "\n",
    "tcga_expr_df.index.rename('sample_id', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Update sample IDs to remove multiple samples measured on the same tumor\n",
    "# and to map with the clinical information\n",
    "tcga_expr_df.index = tcga_expr_df.index.str.slice(start=0, stop=15)\n",
    "tcga_expr_df = tcga_expr_df.loc[~tcga_expr_df.index.duplicated(), :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filter for valid Entrez gene identifiers\n",
    "tcga_expr_df = tcga_expr_df.loc[:, tcga_expr_df.columns.isin(gene_df.entrez_gene_id.astype(str))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(11060, 16148)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>gene_id</th>\n",
       "      <th>1</th>\n",
       "      <th>10</th>\n",
       "      <th>100</th>\n",
       "      <th>1000</th>\n",
       "      <th>10000</th>\n",
       "      <th>10001</th>\n",
       "      <th>10002</th>\n",
       "      <th>10003</th>\n",
       "      <th>100037417</th>\n",
       "      <th>10004</th>\n",
       "      <th>...</th>\n",
       "      <th>9987</th>\n",
       "      <th>9988</th>\n",
       "      <th>9989</th>\n",
       "      <th>999</th>\n",
       "      <th>9990</th>\n",
       "      <th>9991</th>\n",
       "      <th>9992</th>\n",
       "      <th>9993</th>\n",
       "      <th>9994</th>\n",
       "      <th>9997</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sample_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>TCGA-02-0047-01</th>\n",
       "      <td>125.0070</td>\n",
       "      <td>10.4490</td>\n",
       "      <td>136.452</td>\n",
       "      <td>2302.47</td>\n",
       "      <td>1297.520</td>\n",
       "      <td>271.674</td>\n",
       "      <td>1.2293</td>\n",
       "      <td>8.6051</td>\n",
       "      <td>187.492</td>\n",
       "      <td>15.3662</td>\n",
       "      <td>...</td>\n",
       "      <td>4033.31</td>\n",
       "      <td>791.278</td>\n",
       "      <td>1810.14</td>\n",
       "      <td>264.9130</td>\n",
       "      <td>684.225</td>\n",
       "      <td>1097.760</td>\n",
       "      <td>11.6783</td>\n",
       "      <td>4815.14</td>\n",
       "      <td>288.269</td>\n",
       "      <td>299.948</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TCGA-02-0055-01</th>\n",
       "      <td>391.8040</td>\n",
       "      <td>1.1212</td>\n",
       "      <td>222.004</td>\n",
       "      <td>1819.76</td>\n",
       "      <td>903.154</td>\n",
       "      <td>321.233</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>38.1219</td>\n",
       "      <td>426.853</td>\n",
       "      <td>44.8493</td>\n",
       "      <td>...</td>\n",
       "      <td>4431.67</td>\n",
       "      <td>921.424</td>\n",
       "      <td>2038.96</td>\n",
       "      <td>2.2425</td>\n",
       "      <td>466.534</td>\n",
       "      <td>1399.860</td>\n",
       "      <td>16.8185</td>\n",
       "      <td>2228.45</td>\n",
       "      <td>309.460</td>\n",
       "      <td>787.106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TCGA-02-2483-01</th>\n",
       "      <td>271.8520</td>\n",
       "      <td>4.6438</td>\n",
       "      <td>255.831</td>\n",
       "      <td>2888.87</td>\n",
       "      <td>1319.680</td>\n",
       "      <td>458.048</td>\n",
       "      <td>4.6438</td>\n",
       "      <td>5.9103</td>\n",
       "      <td>260.002</td>\n",
       "      <td>29.1293</td>\n",
       "      <td>...</td>\n",
       "      <td>7839.58</td>\n",
       "      <td>846.708</td>\n",
       "      <td>2229.87</td>\n",
       "      <td>18.9974</td>\n",
       "      <td>565.184</td>\n",
       "      <td>1339.950</td>\n",
       "      <td>8.8654</td>\n",
       "      <td>2737.73</td>\n",
       "      <td>306.491</td>\n",
       "      <td>457.203</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TCGA-02-2485-01</th>\n",
       "      <td>83.9429</td>\n",
       "      <td>20.0000</td>\n",
       "      <td>129.048</td>\n",
       "      <td>6965.71</td>\n",
       "      <td>10136.200</td>\n",
       "      <td>418.571</td>\n",
       "      <td>5.2381</td>\n",
       "      <td>37.6190</td>\n",
       "      <td>539.333</td>\n",
       "      <td>19.5238</td>\n",
       "      <td>...</td>\n",
       "      <td>6560.00</td>\n",
       "      <td>1592.740</td>\n",
       "      <td>1840.95</td>\n",
       "      <td>112.3810</td>\n",
       "      <td>484.881</td>\n",
       "      <td>726.667</td>\n",
       "      <td>11.4286</td>\n",
       "      <td>2979.05</td>\n",
       "      <td>439.524</td>\n",
       "      <td>426.667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TCGA-02-2486-01</th>\n",
       "      <td>108.2560</td>\n",
       "      <td>3.6585</td>\n",
       "      <td>205.488</td>\n",
       "      <td>2250.61</td>\n",
       "      <td>873.171</td>\n",
       "      <td>441.463</td>\n",
       "      <td>1.8293</td>\n",
       "      <td>83.5366</td>\n",
       "      <td>265.061</td>\n",
       "      <td>29.8780</td>\n",
       "      <td>...</td>\n",
       "      <td>4330.49</td>\n",
       "      <td>816.335</td>\n",
       "      <td>1134.76</td>\n",
       "      <td>9.1463</td>\n",
       "      <td>397.677</td>\n",
       "      <td>1098.170</td>\n",
       "      <td>12.8049</td>\n",
       "      <td>1340.24</td>\n",
       "      <td>251.220</td>\n",
       "      <td>541.463</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 16148 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "gene_id                 1       10      100     1000      10000    10001  \\\n",
       "sample_id                                                                  \n",
       "TCGA-02-0047-01  125.0070  10.4490  136.452  2302.47   1297.520  271.674   \n",
       "TCGA-02-0055-01  391.8040   1.1212  222.004  1819.76    903.154  321.233   \n",
       "TCGA-02-2483-01  271.8520   4.6438  255.831  2888.87   1319.680  458.048   \n",
       "TCGA-02-2485-01   83.9429  20.0000  129.048  6965.71  10136.200  418.571   \n",
       "TCGA-02-2486-01  108.2560   3.6585  205.488  2250.61    873.171  441.463   \n",
       "\n",
       "gene_id           10002    10003  100037417    10004   ...        9987  \\\n",
       "sample_id                                              ...               \n",
       "TCGA-02-0047-01  1.2293   8.6051    187.492  15.3662   ...     4033.31   \n",
       "TCGA-02-0055-01  0.0000  38.1219    426.853  44.8493   ...     4431.67   \n",
       "TCGA-02-2483-01  4.6438   5.9103    260.002  29.1293   ...     7839.58   \n",
       "TCGA-02-2485-01  5.2381  37.6190    539.333  19.5238   ...     6560.00   \n",
       "TCGA-02-2486-01  1.8293  83.5366    265.061  29.8780   ...     4330.49   \n",
       "\n",
       "gene_id              9988     9989       999     9990      9991     9992  \\\n",
       "sample_id                                                                  \n",
       "TCGA-02-0047-01   791.278  1810.14  264.9130  684.225  1097.760  11.6783   \n",
       "TCGA-02-0055-01   921.424  2038.96    2.2425  466.534  1399.860  16.8185   \n",
       "TCGA-02-2483-01   846.708  2229.87   18.9974  565.184  1339.950   8.8654   \n",
       "TCGA-02-2485-01  1592.740  1840.95  112.3810  484.881   726.667  11.4286   \n",
       "TCGA-02-2486-01   816.335  1134.76    9.1463  397.677  1098.170  12.8049   \n",
       "\n",
       "gene_id             9993     9994     9997  \n",
       "sample_id                                   \n",
       "TCGA-02-0047-01  4815.14  288.269  299.948  \n",
       "TCGA-02-0055-01  2228.45  309.460  787.106  \n",
       "TCGA-02-2483-01  2737.73  306.491  457.203  \n",
       "TCGA-02-2485-01  2979.05  439.524  426.667  \n",
       "TCGA-02-2486-01  1340.24  251.220  541.463  \n",
       "\n",
       "[5 rows x 16148 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(tcga_expr_df.shape)\n",
    "tcga_expr_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Process TCGA cancer-type and sample-type info from barcodes\n",
    "\n",
    "Cancer-type includes `OV`, `BRCA`, `LUSC`, `LUAD`, etc. while sample-type includes `Primary`, `Metastatic`, `Solid Tissue Normal`, etc.\n",
    "\n",
    "See https://gdc.cancer.gov/resources-tcga-users/tcga-code-tables/tissue-source-site-codes for more details.\n",
    "\n",
    "The goal is to use this info to stratify training (90%) and testing (10%) balanced by cancer-type and sample-type. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extract sample type in the order of the gene expression matrix\n",
    "tcga_id = pd.DataFrame(tcga_expr_df.index)\n",
    "\n",
    "# Extract the last two digits of the barcode and recode sample-type\n",
    "tcga_id = tcga_id.assign(sample_type = tcga_id.sample_id.str[-2:])\n",
    "tcga_id.sample_type = tcga_id.sample_type.replace(sampletype_codes_dict)\n",
    "\n",
    "# Extract the first two ID numbers after `TCGA-` and recode cancer-type\n",
    "tcga_id = tcga_id.assign(cancer_type = tcga_id.sample_id.str[5:7])\n",
    "tcga_id.cancer_type = tcga_id.cancer_type.replace(cancertype_codes_dict)\n",
    "\n",
    "# Append cancer-type with sample-type to generate stratification variable\n",
    "tcga_id = tcga_id.assign(stratify_samples = tcga_id.cancer_type.str.cat(tcga_id.sample_type))\n",
    "\n",
    "# Get stratification counts - function cannot work with singleton strats\n",
    "stratify_counts = tcga_id.stratify_samples.value_counts().to_dict()\n",
    "\n",
    "# Recode stratification variables if they are singletons\n",
    "tcga_id = tcga_id.assign(stratify_samples_count = tcga_id.stratify_samples)\n",
    "tcga_id.stratify_samples_count = tcga_id.stratify_samples_count.replace(stratify_counts)\n",
    "tcga_id.loc[tcga_id.stratify_samples_count == 1, \"stratify_samples\"] = \"other\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(11060, 5)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sample_id</th>\n",
       "      <th>sample_type</th>\n",
       "      <th>cancer_type</th>\n",
       "      <th>stratify_samples</th>\n",
       "      <th>stratify_samples_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TCGA-02-0047-01</td>\n",
       "      <td>Primary Solid Tumor</td>\n",
       "      <td>GBM</td>\n",
       "      <td>GBMPrimary Solid Tumor</td>\n",
       "      <td>154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TCGA-02-0055-01</td>\n",
       "      <td>Primary Solid Tumor</td>\n",
       "      <td>GBM</td>\n",
       "      <td>GBMPrimary Solid Tumor</td>\n",
       "      <td>154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TCGA-02-2483-01</td>\n",
       "      <td>Primary Solid Tumor</td>\n",
       "      <td>GBM</td>\n",
       "      <td>GBMPrimary Solid Tumor</td>\n",
       "      <td>154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TCGA-02-2485-01</td>\n",
       "      <td>Primary Solid Tumor</td>\n",
       "      <td>GBM</td>\n",
       "      <td>GBMPrimary Solid Tumor</td>\n",
       "      <td>154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>TCGA-02-2486-01</td>\n",
       "      <td>Primary Solid Tumor</td>\n",
       "      <td>GBM</td>\n",
       "      <td>GBMPrimary Solid Tumor</td>\n",
       "      <td>154</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         sample_id          sample_type cancer_type        stratify_samples  \\\n",
       "0  TCGA-02-0047-01  Primary Solid Tumor         GBM  GBMPrimary Solid Tumor   \n",
       "1  TCGA-02-0055-01  Primary Solid Tumor         GBM  GBMPrimary Solid Tumor   \n",
       "2  TCGA-02-2483-01  Primary Solid Tumor         GBM  GBMPrimary Solid Tumor   \n",
       "3  TCGA-02-2485-01  Primary Solid Tumor         GBM  GBMPrimary Solid Tumor   \n",
       "4  TCGA-02-2486-01  Primary Solid Tumor         GBM  GBMPrimary Solid Tumor   \n",
       "\n",
       "   stratify_samples_count  \n",
       "0                     154  \n",
       "1                     154  \n",
       "2                     154  \n",
       "3                     154  \n",
       "4                     154  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Write out files for downstream use\n",
    "file = os.path.join('data', 'tcga_sample_identifiers.tsv')\n",
    "\n",
    "(\n",
    "    tcga_id.drop(['stratify_samples', 'stratify_samples_count'], axis='columns')\n",
    "    .to_csv(file, sep='\\t', index=False)\n",
    ")\n",
    "\n",
    "print(tcga_id.shape)\n",
    "tcga_id.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cancertype</th>\n",
       "      <th>n =</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>BRCA</td>\n",
       "      <td>1218</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>KIRC</td>\n",
       "      <td>606</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>LUAD</td>\n",
       "      <td>576</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>THCA</td>\n",
       "      <td>572</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>UCEC</td>\n",
       "      <td>567</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>HNSC</td>\n",
       "      <td>566</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>LUSC</td>\n",
       "      <td>553</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>PRAD</td>\n",
       "      <td>550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>LGG</td>\n",
       "      <td>530</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>COAD</td>\n",
       "      <td>495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>SKCM</td>\n",
       "      <td>474</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>STAD</td>\n",
       "      <td>450</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>BLCA</td>\n",
       "      <td>427</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>LIHC</td>\n",
       "      <td>423</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>KIRP</td>\n",
       "      <td>323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>CESC</td>\n",
       "      <td>310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>OV</td>\n",
       "      <td>308</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>SARC</td>\n",
       "      <td>265</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>ESCA</td>\n",
       "      <td>196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>PCPG</td>\n",
       "      <td>187</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>PAAD</td>\n",
       "      <td>183</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>LAML</td>\n",
       "      <td>173</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>GBM</td>\n",
       "      <td>172</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>READ</td>\n",
       "      <td>171</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>TGCT</td>\n",
       "      <td>156</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>THYM</td>\n",
       "      <td>122</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>KICH</td>\n",
       "      <td>91</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>MESO</td>\n",
       "      <td>87</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>UVM</td>\n",
       "      <td>80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>ACC</td>\n",
       "      <td>79</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>UCS</td>\n",
       "      <td>57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>DLBC</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>CHOL</td>\n",
       "      <td>45</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   cancertype   n =\n",
       "0        BRCA  1218\n",
       "1        KIRC   606\n",
       "2        LUAD   576\n",
       "3        THCA   572\n",
       "4        UCEC   567\n",
       "5        HNSC   566\n",
       "6        LUSC   553\n",
       "7        PRAD   550\n",
       "8         LGG   530\n",
       "9        COAD   495\n",
       "10       SKCM   474\n",
       "11       STAD   450\n",
       "12       BLCA   427\n",
       "13       LIHC   423\n",
       "14       KIRP   323\n",
       "15       CESC   310\n",
       "16         OV   308\n",
       "17       SARC   265\n",
       "18       ESCA   196\n",
       "19       PCPG   187\n",
       "20       PAAD   183\n",
       "21       LAML   173\n",
       "22        GBM   172\n",
       "23       READ   171\n",
       "24       TGCT   156\n",
       "25       THYM   122\n",
       "26       KICH    91\n",
       "27       MESO    87\n",
       "28        UVM    80\n",
       "29        ACC    79\n",
       "30        UCS    57\n",
       "31       DLBC    48\n",
       "32       CHOL    45"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cancertype_count_df = (\n",
    "    pd.DataFrame(tcga_id.cancer_type.value_counts())\n",
    "    .reset_index()\n",
    "    .rename({'index': 'cancertype', 'cancer_type': 'n ='}, axis='columns')\n",
    ")\n",
    "\n",
    "file = os.path.join('data', 'tcga_sample_counts.tsv')\n",
    "cancertype_count_df.to_csv(file, sep='\\t', index=False)\n",
    "\n",
    "cancertype_count_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stratify Balanced Training and Testing Sets in TCGA Gene Expression\n",
    "\n",
    "Output training and testing gene expression datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df, test_df = train_test_split(tcga_expr_df,\n",
    "                                     test_size=0.1,\n",
    "                                     random_state=123,\n",
    "                                     stratify=tcga_id.stratify_samples_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(9954, 16148)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1106, 16148)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(train_df.shape)\n",
    "test_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_file = os.path.join('data', 'train_tcga_expression_matrix_processed.tsv.gz')\n",
    "train_df.to_csv(train_file, sep='\\t', compression='gzip', float_format='%.3g')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_file = os.path.join('data', 'test_tcga_expression_matrix_processed.tsv.gz')\n",
    "test_df.to_csv(test_file, sep='\\t', compression='gzip', float_format='%.3g')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sort genes based on median absolute deviation and output to file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Determine most variably expressed genes and subset\n",
    "mad_genes_df = pd.DataFrame(train_df.mad(axis=0).sort_values(ascending=False)).reset_index()\n",
    "mad_genes_df.columns = ['gene_id', 'median_absolute_deviation']\n",
    "\n",
    "file = os.path.join('data', 'tcga_mad_genes.tsv')\n",
    "mad_genes_df.to_csv(file, sep='\\t', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:biobombe]",
   "language": "python",
   "name": "conda-env-biobombe-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}