inference_20191012.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# misc\n",
    "import os\n",
    "from tqdm import tqdm, tnrange, tqdm_notebook\n",
    "from pathlib import Path\n",
    "import re\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import sys\n",
    "import scipy.ndimage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.vision import *\n",
    "from fastai.metrics import *\n",
    "from fastai.callbacks import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_str = '/home/ubuntu/sfr-challenge/lungs/dataset/Inference'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_model_str = '/home/ubuntu/sfr-challenge/lungs/dataset/train'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = Path(path_str)\n",
    "os.path.exists(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_model = Path(path_model_str)\n",
    "os.path.exists(path_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define fixed random seed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Inference loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_prediction_glob(preds):\n",
    "    max_pred = 0\n",
    "    for n in preds[:, 1]:\n",
    "        if max_pred < n:\n",
    "            max_pred = n\n",
    "    return float(max_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_prediction_glob_av(preds):\n",
    "    max_1 = 0.0\n",
    "    max_2 = 0.0\n",
    "    for n in preds[:, 1]:\n",
    "        if max_1 < n:\n",
    "            max_1 = float(n)\n",
    "        elif max_1 >= n and max_2 < n:\n",
    "            max_2 = float(n)\n",
    "    return (max_1 + max_2)/2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dirs = os.listdir(path/'Patients')\n",
    "len(dirs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = []\n",
    "patients = []\n",
    "preds_glob = []\n",
    "all_preds = []\n",
    "detailed_preds = {}\n",
    "error = []\n",
    "for d in tqdm(dirs):\n",
    "    learner = load_learner(path_model, 'final-vgg16-12epc-1013-1130', test=ImageList.from_folder(path/'Patients'/d)).to_fp16()\n",
    "    try:\n",
    "        preds, y = learner.get_preds(ds_type=DatasetType.Test)\n",
    "        preds_glob.append(get_prediction_glob(preds))\n",
    "        patients.append(d)\n",
    "        detailed_preds[d] = preds\n",
    "    except:\n",
    "        error.append(d)\n",
    "        print('patient is invalid: ', d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# must 344 patients\n",
    "print(\"length patients: \", len(patients))\n",
    "print(\"length preds: \", len(preds_glob))\n",
    "print(\"samples patients: \", patients[:3])\n",
    "print(\"samples predicts: \", preds_glob[:3])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Re-generate file with predictions base on modified predictions function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "patients = []\n",
    "preds_glob = []\n",
    "for key in detailed_preds:\n",
    "    patients.append(key)\n",
    "    preds_glob.append(get_prediction_glob(detailed_preds[key]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create output .csv file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "inference_output = pd.DataFrame({'examen' : patients, 'probabilité' : preds_glob})\n",
    "inference_output.set_index('examen', inplace=True)\n",
    "inference_output.to_csv(path/'NAIS.csv',sep=';',decimal=',')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}