project/01_1_train_KNN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c7aa5cba",
   "metadata": {},
   "source": [
    "# K- Nearest Neighbors (KNN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "76e01f3e",
   "metadata": {
    "lines_to_next_cell": 2,
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "import logging\n",
    "\n",
    "import pandas as pd\n",
    "import sklearn\n",
    "import sklearn.impute\n",
    "from IPython.display import display\n",
    "\n",
    "import pimmslearn\n",
    "import pimmslearn.model\n",
    "import pimmslearn.models as models\n",
    "import pimmslearn.nb\n",
    "from pimmslearn import sampling\n",
    "from pimmslearn.io import datasplits\n",
    "from pimmslearn.models import ae\n",
    "\n",
    "logger = pimmslearn.logging.setup_logger(logging.getLogger('pimmslearn'))\n",
    "logger.info(\"Experiment 03 - Analysis of latent spaces and performance comparisions\")\n",
    "\n",
    "figures = {}  # collection of ax or figures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3fb29f3e",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "# catch passed parameters\n",
    "args = None\n",
    "args = dict(globals()).keys()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "97cabfc2",
   "metadata": {},
   "source": [
    "Papermill script parameters:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "50f19abc",
   "metadata": {
    "tags": [
     "parameters"
    ]
   },
   "outputs": [],
   "source": [
    "# files and folders\n",
    "folder_experiment: str = 'runs/example'  # Datasplit folder with data for experiment\n",
    "folder_data: str = ''  # specify data directory if needed\n",
    "file_format: str = 'csv'  # file format of create splits, default pickle (pkl)\n",
    "# Machine parsed metadata from rawfile workflow\n",
    "fn_rawfile_metadata: str = 'data/dev_datasets/HeLa_6070/files_selected_metadata_N50.csv'\n",
    "# training\n",
    "epochs_max: int = 50  # Maximum number of epochs\n",
    "# early_stopping:bool = True # Wheather to use early stopping or not\n",
    "batch_size: int = 64  # Batch size for training (and evaluation)\n",
    "cuda: bool = True  # Whether to use a GPU for training\n",
    "# model\n",
    "neighbors: int = 3  # number of neigherst neighbors to use\n",
    "force_train: bool = True  # Force training when saved model could be used. Per default re-train model\n",
    "sample_idx_position: int = 0  # position of index which is sample ID\n",
    "model: str = 'KNN'  # model name\n",
    "model_key: str = 'KNN'  # potentially alternative key for model (grid search)\n",
    "save_pred_real_na: bool = True  # Save all predictions for missing values\n",
    "# metadata -> defaults for metadata extracted from machine data\n",
    "meta_date_col: str = None  # date column in meta data\n",
    "meta_cat_col: str = None  # category column in meta data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f24fd6a",
   "metadata": {},
   "source": [
    "Some argument transformations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10dfbb95",
   "metadata": {
    "lines_to_next_cell": 2,
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "args = pimmslearn.nb.get_params(args, globals=globals())\n",
    "args = pimmslearn.nb.args_from_dict(args)\n",
    "args"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9e9b0881",
   "metadata": {},
   "source": [
    "Some naming conventions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b0f78a72",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "TEMPLATE_MODEL_PARAMS = 'model_params_{}.json'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "76172671",
   "metadata": {},
   "source": [
    "## Load data in long format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a7a2208e",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "data = datasplits.DataSplits.from_folder(args.data, file_format=args.file_format)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "74d14c1a",
   "metadata": {},
   "source": [
    "data is loaded in long format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34b79387",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "data.train_X.sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3f0bb10e",
   "metadata": {},
   "source": [
    "load meta data for splits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a61f7f70",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "if args.fn_rawfile_metadata:\n",
    "    df_meta = pd.read_csv(args.fn_rawfile_metadata, index_col=0)\n",
    "    display(df_meta.loc[data.train_X.index.levels[0]])\n",
    "else:\n",
    "    df_meta = None"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ea35dc4c",
   "metadata": {},
   "source": [
    "## Initialize Comparison"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e1a277f",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "freq_feat = sampling.frequency_by_index(data.train_X, 0)\n",
    "freq_feat.head()  # training data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "15fd461b",
   "metadata": {},
   "source": [
    "### Simulated missing values"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "382b887d",
   "metadata": {},
   "source": [
    "The validation fake NA is used to by all models to evaluate training performance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d056fb82",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "val_pred_fake_na = data.val_y.to_frame(name='observed')\n",
    "val_pred_fake_na"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ade74bab",
   "metadata": {
    "lines_to_next_cell": 2,
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "test_pred_fake_na = data.test_y.to_frame(name='observed')\n",
    "test_pred_fake_na.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e21c5c7d",
   "metadata": {},
   "source": [
    "## Data in wide format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4a75a078",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "data.to_wide_format()\n",
    "args.M = data.train_X.shape[-1]\n",
    "data.train_X.head()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "89465f1b",
   "metadata": {},
   "source": [
    "## Train\n",
    "model = 'sklearn_knn'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "58d3383c",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "knn_imputer = sklearn.impute.KNNImputer(n_neighbors=args.neighbors).fit(data.train_X)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5ed614e9",
   "metadata": {},
   "source": [
    "### Predictions\n",
    "\n",
    "- data of training data set and validation dataset to create predictions is the same as training data.\n",
    "- predictions include missing values (which are not further compared)\n",
    "\n",
    "create predictions and select for split entries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "98a1cba0",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "pred = knn_imputer.transform(data.train_X)\n",
    "pred = pd.DataFrame(pred, index=data.train_X.index, columns=data.train_X.columns).stack()\n",
    "pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b24132c",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "val_pred_fake_na[args.model_key] = pred\n",
    "val_pred_fake_na"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f8937bcc",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "test_pred_fake_na[args.model_key] = pred\n",
    "test_pred_fake_na"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a7753205",
   "metadata": {},
   "source": [
    "save missing values predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8a853343",
   "metadata": {
    "lines_to_next_cell": 2,
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "if args.save_pred_real_na:\n",
    "    pred_real_na = ae.get_missing_values(df_train_wide=data.train_X,\n",
    "                                         val_idx=val_pred_fake_na.index,\n",
    "                                         test_idx=test_pred_fake_na.index,\n",
    "                                         pred=pred)\n",
    "    display(pred_real_na)\n",
    "    pred_real_na.to_csv(args.out_preds / f\"pred_real_na_{args.model_key}.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cb3686e1",
   "metadata": {},
   "source": [
    "### Plots\n",
    "\n",
    "- validation data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e7c0d93c",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "8881f26e",
   "metadata": {},
   "source": [
    "## Comparisons\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5129e43f",
   "metadata": {},
   "source": [
    "### Validation data\n",
    "\n",
    "- all measured (identified, observed) peptides in validation data\n",
    "\n",
    "> Does not make to much sense to compare collab and AEs,\n",
    "> as the setup differs of training and validation data differs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1d3a789e",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "# papermill_description=metrics\n",
    "d_metrics = models.Metrics()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0fd9b72d",
   "metadata": {},
   "source": [
    "The fake NA for the validation step are real test data (not used for training nor early stopping)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d56eb144",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "added_metrics = d_metrics.add_metrics(val_pred_fake_na, 'valid_fake_na')\n",
    "added_metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b9b64c77",
   "metadata": {},
   "source": [
    "### Test Datasplit\n",
    "\n",
    "Fake NAs : Artificially created NAs. Some data was sampled and set\n",
    "explicitly to misssing before it was fed to the model for\n",
    "reconstruction."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d0be628",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "added_metrics = d_metrics.add_metrics(test_pred_fake_na, 'test_fake_na')\n",
    "added_metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "896c9c3d",
   "metadata": {},
   "source": [
    "Save all metrics as json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9f03ba1f",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "pimmslearn.io.dump_json(d_metrics.metrics, args.out_metrics / f'metrics_{args.model_key}.json')\n",
    "d_metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b962d322",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "metrics_df = models.get_df_from_nested_dict(d_metrics.metrics,\n",
    "                                            column_levels=['model', 'metric_name']).T\n",
    "metrics_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "27654ab9",
   "metadata": {},
   "source": [
    "## Save predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ce0fb347",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "# save simulated missing values for both splits\n",
    "val_pred_fake_na.to_csv(args.out_preds / f\"pred_val_{args.model_key}.csv\")\n",
    "test_pred_fake_na.to_csv(args.out_preds / f\"pred_test_{args.model_key}.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "954f03e4",
   "metadata": {},
   "source": [
    "## Config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fc43a4e8",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "figures  # switch to fnames?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d3bbd037",
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "args.n_params = 1  # the number of neighbors to consider\n",
    "args.dump(fname=args.out_models / f\"model_config_{args.model_key}.yaml\")\n",
    "args"
   ]
  }
 ],
 "metadata": {
  "jupytext": {
   "formats": "ipynb,py:percent"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}