project/10_1_ald_diff_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Differential Analysis - Compare model imputation with standard imputation\n",
    "\n",
    "- load missing values predictions (if specified)\n",
    "- leave all other values as they were\n",
    "- compare missing values predicition by model with baseline method\n",
    "  (default: draw from shifted normal distribution. short RSN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "import logging\n",
    "from pathlib import Path\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import njab.stats\n",
    "import pandas as pd\n",
    "from IPython.display import display\n",
    "\n",
    "import pimmslearn\n",
    "import pimmslearn.analyzers\n",
    "import pimmslearn.imputation\n",
    "import pimmslearn.io.datasplits\n",
    "import pimmslearn.nb\n",
    "\n",
    "logger = pimmslearn.logging.setup_nb_logger()\n",
    "logging.getLogger('fontTools').setLevel(logging.WARNING)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "# catch passed parameters\n",
    "args = None\n",
    "args = dict(globals()).keys()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parameters\n",
    "Default and set parameters for the notebook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "parameters"
    ]
   },
   "outputs": [],
   "source": [
    "folder_experiment = \"runs/appl_ald_data/plasma/proteinGroups\"\n",
    "folder_data: str = ''  # specify data directory if needed\n",
    "fn_clinical_data = \"data/ALD_study/processed/ald_metadata_cli.csv\"\n",
    "fn_qc_samples = ''  # 'data/ALD_study/processed/qc_plasma_proteinGroups.pkl'\n",
    "f_annotations = 'data/ALD_study/processed/ald_plasma_proteinGroups_id_mappings.csv'\n",
    "\n",
    "\n",
    "target: str = 'kleiner'\n",
    "covar: str = 'age,bmi,gender_num,nas_steatosis_ordinal,abstinent_num'\n",
    "\n",
    "file_format = \"csv\"\n",
    "model_key = 'VAE'  # model(s) to evaluate\n",
    "model = None  # default same as model_key, but could be overwritten (edge case)\n",
    "value_name = 'intensity'\n",
    "out_folder = 'diff_analysis'\n",
    "template_pred = 'pred_real_na_{}.csv'  # fixed, do not change"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "01617e36",
   "metadata": {},
   "source": [
    "Add set parameters to configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "if not model:\n",
    "    model = model_key\n",
    "params = pimmslearn.nb.get_params(args, globals=globals(), remove=True)\n",
    "params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "args = pimmslearn.nb.Config()\n",
    "args.fn_clinical_data = Path(params[\"fn_clinical_data\"])\n",
    "args.folder_experiment = Path(params[\"folder_experiment\"])\n",
    "args = pimmslearn.nb.add_default_paths(args,\n",
    "                                 out_root=(args.folder_experiment\n",
    "                                           / params[\"out_folder\"]\n",
    "                                           / params[\"target\"]\n",
    "                                           ))\n",
    "args.covar = params[\"covar\"].split(',')\n",
    "args.update_from_dict(params)\n",
    "args"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Outputs of this notebook will be stored here:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "files_out = {}\n",
    "args.out_folder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MS proteomics or specified omics data\n",
    "Aggregated from data splits of the imputation workflow run before."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "data = pimmslearn.io.datasplits.DataSplits.from_folder(\n",
    "    args.data, file_format=args.file_format)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "observed = pd.concat([data.train_X, data.val_y, data.test_y])\n",
    "observed"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Clinical data\n",
    "Describe numerical data specified for use:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "lines_to_next_cell": 2,
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "df_clinic = pd.read_csv(args.fn_clinical_data, index_col=0)\n",
    "df_clinic = df_clinic.loc[observed.index.levels[0]]\n",
    "cols_clinic = pimmslearn.pandas.get_columns_accessor(df_clinic)\n",
    "df_clinic[[args.target, *args.covar]].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "# ## Additional annotations\n",
    "# - additional annotations of features (e.g. gene names for protein groups)\n",
    "\n",
    "feat_name = observed.index.names[-1]\n",
    "if args.f_annotations:\n",
    "    gene_to_PG = pd.read_csv(args.f_annotations)\n",
    "    gene_to_PG = gene_to_PG.drop_duplicates().set_index(feat_name)\n",
    "    fname = args.out_folder / Path(args.f_annotations).name\n",
    "    gene_to_PG.to_csv(fname)\n",
    "    files_out[fname.name] = fname.as_posix()\n",
    "else:\n",
    "    gene_to_PG = None\n",
    "gene_to_PG"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Entries with missing values\n",
    "- see how many rows have one missing values (for target and covariates)\n",
    "- only complete data is used for Differential Analysis\n",
    "- covariates are not imputed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "df_clinic[[args.target, *args.covar]].isna().any(axis=1).sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Data description of data used:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "mask_sample_with_complete_clinical_data = df_clinic[[args.target, *args.covar]].notna().all(axis=1)\n",
    "fname = args.out_folder / 'mask_sample_with_complete_clinical_data.csv'\n",
    "files_out[fname.name] = fname.as_posix()\n",
    "mask_sample_with_complete_clinical_data.to_csv(fname)\n",
    "\n",
    "idx_complete_data = (mask_sample_with_complete_clinical_data\n",
    "                     .loc[mask_sample_with_complete_clinical_data]\n",
    "                     .index)\n",
    "df_clinic.loc[idx_complete_data, [args.target, *args.covar]].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "df_clinic.loc[idx_complete_data, args.target].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check which patients with kleiner score have misssing covariates:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "df_clinic.loc[(~mask_sample_with_complete_clinical_data\n",
    "               & df_clinic[args.target].notna()),\n",
    "              [args.target, *args.covar]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save feature frequency of observed data based on complete clinical data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "feat_freq_observed = observed.unstack().loc[idx_complete_data].notna().sum()\n",
    "feat_freq_observed.name = 'frequency'\n",
    "\n",
    "fname = args.folder_experiment / 'freq_features_observed.csv'\n",
    "files_out['feat_freq_observed'] = fname.as_posix()\n",
    "logger.info(fname)\n",
    "feat_freq_observed.to_csv(fname)\n",
    "ax = feat_freq_observed.sort_values().plot(marker='.', rot=90)\n",
    "_ = ax.set_xticklabels([l_.get_text().split(';')[0] for l_ in ax.get_xticklabels()])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ALD study approach using all measurements\n",
    "Use parameters as specified in [ALD study](https://github.com/RasmussenLab/pimms/tree/main/project/data/ALD_study)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "DATA_COMPLETENESS = 0.6\n",
    "# MIN_N_PROTEIN_GROUPS: int = 200\n",
    "FRAC_PROTEIN_GROUPS: int = 0.622\n",
    "CV_QC_SAMPLE: float = 0.4  # Coef. of variation on 13 QC samples\n",
    "\n",
    "ald_study, cutoffs = pimmslearn.analyzers.diff_analysis.select_raw_data(observed.unstack(\n",
    "), data_completeness=DATA_COMPLETENESS, frac_protein_groups=FRAC_PROTEIN_GROUPS)\n",
    "\n",
    "ald_study"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "if args.fn_qc_samples:\n",
    "    # Move this to data-preprocessing\n",
    "    qc_samples = pd.read_pickle(args.fn_qc_samples)\n",
    "    qc_cv_feat = qc_samples.std() / qc_samples.mean()\n",
    "    qc_cv_feat = qc_cv_feat.rename(qc_samples.columns.name)\n",
    "    fig, ax = plt.subplots(figsize=(4, 7))\n",
    "    ax = qc_cv_feat.plot.box(ax=ax)\n",
    "    ax.set_ylabel('Coefficient of Variation')\n",
    "    pimmslearn.savefig(fig, name='cv_qc_samples', folder=args.out_figures)\n",
    "    print((qc_cv_feat < CV_QC_SAMPLE).value_counts())\n",
    "    # only to ald_study data\n",
    "    ald_study = ald_study[pimmslearn.analyzers.diff_analysis.select_feat(qc_samples[ald_study.columns])]\n",
    "\n",
    "ald_study"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "lines_to_next_cell": 2,
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "fig, axes = pimmslearn.plotting.plot_cutoffs(observed.unstack(),\n",
    "                                       feat_completness_over_samples=cutoffs.feat_completness_over_samples,\n",
    "                                       min_feat_in_sample=cutoffs.min_feat_in_sample)\n",
    "pimmslearn.savefig(fig, name='tresholds_normal_imputation', folder=args.out_figures)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load model predictions for (real) missing data\n",
    "Load from:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "# available_files = list(args.out_preds.iterdir())\n",
    "template_pred = str(args.out_preds / args.template_pred)\n",
    "fname = args.out_preds / args.template_pred.format(args.model)\n",
    "fname"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e7a55383",
   "metadata": {
    "lines_to_next_cell": 0
   },
   "source": [
    "Baseline comparison:\n",
    "- in case of RSN -> use filtering as done in original [ALD study](https://github.com/RasmussenLab/pimms/tree/main/project/data/ALD_study) (Niu et al. 2022)\n",
    "- otherwise -> use all data\n",
    "\n",
    "Use columns which are provided by model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "lines_to_next_cell": 2,
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "pred_real_na = None\n",
    "if args.model_key and str(args.model_key) != 'None':\n",
    "    pred_real_na = (pimmslearn\n",
    "                    .analyzers\n",
    "                    .compare_predictions\n",
    "                    .load_single_csv_pred_file(fname)\n",
    "                    )\n",
    "else:\n",
    "    logger.info('No model key provided -> no imputation of data.')\n",
    "\n",
    "if args.model_key == 'RSN':\n",
    "    logger.info('Filtering of data as done in original paper for RSN.')\n",
    "    # Select only idx from RSN which are selected by ald study cutoffs\n",
    "    idx_to_sel = ald_study.columns.intersection(pred_real_na.index.levels[-1])\n",
    "    pred_real_na = pred_real_na.loc[pd.IndexSlice[:, idx_to_sel]]\n",
    "pred_real_na"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "source": [
    "Plot unchanged observed intensities to imputed intensity distribution (if available):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "def plot_distributions(observed: pd.Series,\n",
    "                       imputation: pd.Series = None,\n",
    "                       model_key: str = 'MODEL',\n",
    "                       figsize=(4, 3),\n",
    "                       sharex=True):\n",
    "    \"\"\"Plots distributions of intensities provided as dictionary of labels to pd.Series.\"\"\"\n",
    "    series_ = [observed, imputation] if imputation is not None else [observed]\n",
    "    min_bin, max_bin = pimmslearn.plotting.data.get_min_max_iterable([observed])\n",
    "\n",
    "    if imputation is not None:\n",
    "        fig, axes = plt.subplots(len(series_), figsize=figsize, sharex=sharex)\n",
    "        ax = axes[0]\n",
    "    else:\n",
    "        fig, ax = plt.subplots(1, figsize=figsize, sharex=sharex)\n",
    "\n",
    "    bins = range(min_bin, max_bin + 1, 1)\n",
    "\n",
    "    label = 'observed measurments'\n",
    "    ax = observed.hist(ax=ax, bins=bins, color='grey')\n",
    "    ax.set_title(f'{label} (N={len(observed):,d})')\n",
    "    ax.set_ylabel('observations')\n",
    "    ax.locator_params(axis='y', integer=True)\n",
    "    ax.yaxis.set_major_formatter(\"{x:,.0f}\")\n",
    "\n",
    "    if imputation is not None:\n",
    "        ax = axes[1]\n",
    "        label = f'Missing values imputed using {model_key.upper()}'\n",
    "        color = pimmslearn.plotting.defaults.color_model_mapping.get(model_key, None)\n",
    "        if color is None:\n",
    "            color = f'C{1}'\n",
    "        ax = imputation.hist(ax=ax, bins=bins, color=color)\n",
    "        ax.set_title(f'{label} (N={len(imputation):,d})')\n",
    "        ax.set_ylabel('observations')\n",
    "        ax.locator_params(axis='y', integer=True)\n",
    "        ax.yaxis.set_major_formatter(\"{x:,.0f}\")\n",
    "    return fig, bins\n",
    "\n",
    "\n",
    "pimmslearn.plotting.make_large_descriptors(6)\n",
    "fig, bins = plot_distributions(observed,\n",
    "                               imputation=pred_real_na,\n",
    "                               model_key=args.model_key, figsize=(2.5, 2))\n",
    "fname = args.out_folder / 'dist_plots' / f'real_na_obs_vs_{args.model_key}.pdf'\n",
    "files_out[fname.name] = fname.as_posix()\n",
    "pimmslearn.savefig(fig, name=fname)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a8394517",
   "metadata": {},
   "source": [
    "Dump frequency of histograms to file for reporting (if imputed values are used)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "if pred_real_na is not None:\n",
    "    counts_per_bin = pd.concat([\n",
    "        pimmslearn.pandas.get_counts_per_bin(observed.to_frame('observed'), bins=bins),\n",
    "        pimmslearn.pandas.get_counts_per_bin(pred_real_na.to_frame(args.model_key), bins=bins)\n",
    "    ], axis=1)\n",
    "    counts_per_bin.to_excel(fname.with_suffix('.xlsx'))\n",
    "    logger.info(\"Counts per bin saved to %s\", fname.with_suffix('.xlsx'))\n",
    "    display(counts_per_bin)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Mean shift by model\n",
    "Compare how imputed values are shifted in comparsion to overall distribution.\n",
    "\n",
    "First by using all intensities without any grouping:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "if pred_real_na is not None:\n",
    "    shifts = (pimmslearn.imputation.compute_moments_shift(observed, pred_real_na,\n",
    "                                                    names=('observed', args.model_key)))\n",
    "    display(pd.DataFrame(shifts).T)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then by averaging over the calculation by sample:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "if pred_real_na is not None:\n",
    "    index_level = 0  # per sample\n",
    "    mean_by_sample = pd.DataFrame(\n",
    "        {'observed': pimmslearn.imputation.stats_by_level(observed, index_level=index_level),\n",
    "         args.model_key: pimmslearn.imputation.stats_by_level(pred_real_na, index_level=index_level)\n",
    "         })\n",
    "    mean_by_sample.loc['mean_shift'] = (mean_by_sample.loc['mean', 'observed'] -\n",
    "                                        mean_by_sample.loc['mean']).abs() / mean_by_sample.loc['std', 'observed']\n",
    "    mean_by_sample.loc['std shrinkage'] = mean_by_sample.loc['std'] / \\\n",
    "        mean_by_sample.loc['std', 'observed']\n",
    "    display(mean_by_sample)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Differential analysis\n",
    "Combine observed and imputed data (if available) for differential analysis:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "df = pd.concat([observed, pred_real_na]).unstack()\n",
    "df.loc[idx_complete_data]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "# * if some features were not imputed -> drop them\n",
    "# ? could be changed: let a model decide if a feature should be imputed, otherwise don't.\n",
    "if pred_real_na is not None:\n",
    "    if df.isna().sum().sum():\n",
    "        logger.warning(\"DataFrame has missing entries after imputation.\")\n",
    "        logger.info(\"Drop columns with missing values.\")\n",
    "    df = df.dropna(axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Results for target and clinical variables:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "scores = njab.stats.ancova.AncovaAll(df_proteomics=df,\n",
    "                                     df_clinic=df_clinic,\n",
    "                                     target=args.target,\n",
    "                                     covar=args.covar,\n",
    "                                     value_name=args.value_name\n",
    "                                     ).ancova()\n",
    "# features are in first index position\n",
    "feat_idx = scores.index.get_level_values(0)\n",
    "if gene_to_PG is not None:\n",
    "    scores = (scores\n",
    "              .join(gene_to_PG)\n",
    "              .set_index(gene_to_PG.columns.to_list(), append=True)\n",
    "              )\n",
    "scores"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bd02d010",
   "metadata": {},
   "source": [
    "Only for target:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "scores.columns = pd.MultiIndex.from_product([[str(args.model_key)], scores.columns],\n",
    "                                            names=('model', 'var'))\n",
    "scores.loc[pd.IndexSlice[:, args.target], :]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bc8f0344",
   "metadata": {},
   "source": [
    "Save all results to file:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "fname = args.out_folder / 'scores' / f'diff_analysis_scores_{str(args.model_key)}.pkl'\n",
    "files_out[fname.name] = fname.as_posix()\n",
    "fname.parent.mkdir(exist_ok=True, parents=True)\n",
    "scores.to_pickle(fname)\n",
    "fname"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3c3db9ea",
   "metadata": {},
   "source": [
    "Saved files:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "hide-input"
    ]
   },
   "outputs": [],
   "source": [
    "files_out"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.17"
  },
  "vscode": {
   "interpreter": {
    "hash": "cf83e9cb890c7f96eb0ae04f39a82254555f56a1a0ed2f03b23a8b40fe6cd31c"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}