001_module_stats.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "50a19ce1-bb8a-40c1-8ef3-ff78f60b71d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| default_exp stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "885c918b-5721-4a0d-947e-62451a6d4c33",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ec96dcb-56ee-412f-bc63-253c5b1a19b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "#|hide\n",
    "from nbdev import *\n",
    "from fastcore.test import *\n",
    "from fastcore.utils import *"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "14f00408-3ef3-475c-8ab4-eea43088ecae",
   "metadata": {},
   "source": [
    "# stats\n",
    "> Reference API related to statistical functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "806e548c-5588-4a0a-b4d5-e744bbe1fc1f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#|export\n",
    "from typing import Callable, List, Optional\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f676398f-61c2-41cd-a042-e83d114c9e83",
   "metadata": {},
   "source": [
    "## Bootstrap estimates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d42eb6fb-e131-4603-aabd-07d947fa3aa1",
   "metadata": {},
   "outputs": [],
   "source": [
    "#|export\n",
    "def bootstrap_sampling(\n",
    "    data: pd.DataFrame,  # Data containing the columns we want to generate bootstrap estimates from. \n",
    "    estimator: Callable = np.mean,  # estimator function that accepts an array-like argument. \n",
    "    n_boot: int = 1000,  # Number of bootstrap estimates to compute.     \n",
    "    columns_to_exclude: List[str] = None  # Column names to exclude.\n",
    "):\n",
    "    \"Compute bootstrap estimates of the data distribution\"\n",
    "    if not columns_to_exclude:\n",
    "        columns_to_exclude = []\n",
    "    data = data[[x for x in data.columns if x not in columns_to_exclude]]\n",
    "    boot_dist = []\n",
    "    for i in range(int(n_boot)):\n",
    "        sample = data.sample(n=data.shape[0], replace=True)\n",
    "        boot_dist.append(estimator(sample, axis=0))\n",
    "    return pd.DataFrame(boot_dist)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f0eeb76e-1d8d-4a8a-80c9-f77d943acbad",
   "metadata": {},
   "source": [
    "Usage:"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c0bf4ccd-60e6-4082-ad25-94f15ff9d9b7",
   "metadata": {},
   "source": [
    "Generate `data` with columns containing data that we want to compute estimates from. The values in the column `a` comes from Normal distribution with mean 0 and standard deviation 1. The values from column `b` comes from Normal distribution with mean 100 and standard deviation 10."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "58df2b6d-1ad4-49ba-a0f0-03c5f1193eda",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.605639</td>\n",
       "      <td>92.817505</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.775791</td>\n",
       "      <td>92.750026</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-1.265231</td>\n",
       "      <td>107.981771</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.981306</td>\n",
       "      <td>101.388385</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.029075</td>\n",
       "      <td>122.700172</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          a           b\n",
       "0  0.605639   92.817505\n",
       "1 -0.775791   92.750026\n",
       "2 -1.265231  107.981771\n",
       "3  0.981306  101.388385\n",
       "4  0.029075  122.700172"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.DataFrame(\n",
    "    data={\n",
    "        \"a\": np.random.normal(size = 100), \n",
    "        \"b\": np.random.normal(loc=100, scale = 10, size = 100)\n",
    "    }\n",
    ")\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1fbdf7f7-8262-4eb0-8050-7279ee03a67c",
   "metadata": {},
   "source": [
    "#### Compute mean of the distribution by default"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c8d1b0f-203c-4eb1-95ec-d9d3dba45170",
   "metadata": {},
   "source": [
    "By default, the function generates the mean of each column `n_boot` times. Each value represents the mean obtained from a bootstrap sample of the original `data`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3fb07b94-8f2d-442b-b440-c28b8caa5318",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.012356</td>\n",
       "      <td>100.018394</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.143189</td>\n",
       "      <td>100.691872</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.002554</td>\n",
       "      <td>99.874399</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.079395</td>\n",
       "      <td>99.539636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.055096</td>\n",
       "      <td>100.452383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>0.063409</td>\n",
       "      <td>100.439363</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>-0.024455</td>\n",
       "      <td>98.607045</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>0.209427</td>\n",
       "      <td>99.866736</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>0.061323</td>\n",
       "      <td>98.680469</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>0.289456</td>\n",
       "      <td>99.980295</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           a           b\n",
       "0   0.012356  100.018394\n",
       "1   0.143189  100.691872\n",
       "2  -0.002554   99.874399\n",
       "3   0.079395   99.539636\n",
       "4   0.055096  100.452383\n",
       "..       ...         ...\n",
       "95  0.063409  100.439363\n",
       "96 -0.024455   98.607045\n",
       "97  0.209427   99.866736\n",
       "98  0.061323   98.680469\n",
       "99  0.289456   99.980295\n",
       "\n",
       "[100 rows x 2 columns]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "estimates = bootstrap_sampling(data, n_boot=100)\n",
    "estimates"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "33008107-0660-4ad7-b0c7-20e94e4be6bd",
   "metadata": {},
   "source": [
    "We can check if the estimates make sense by compute the mean of the bootstrap estimates and comparing with the mean of the Normal distribution they were generated from."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a9cb7d7-f10b-448a-be47-c1f95cbe5c0a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "a      0.089538\n",
       "b    100.099900\n",
       "dtype: float64"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "estimates.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca6e0595-7519-453c-84c7-dcec6112cb24",
   "metadata": {},
   "source": [
    "#### Specify function. Example: Standard deviation."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "72745cbb-9f1c-4a2a-ae31-45621c41524b",
   "metadata": {},
   "source": [
    "We can specify other functions, such as `np.std` to compute the standard deviation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "13695c63-9df8-4b7c-9cfc-ab32151f73b0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.933496</td>\n",
       "      <td>10.126658</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.929125</td>\n",
       "      <td>9.852667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.899762</td>\n",
       "      <td>10.307814</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.968039</td>\n",
       "      <td>10.416074</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.004349</td>\n",
       "      <td>10.441463</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>0.910904</td>\n",
       "      <td>10.357727</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>0.818276</td>\n",
       "      <td>12.358640</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>0.981826</td>\n",
       "      <td>9.622724</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>0.962237</td>\n",
       "      <td>10.897055</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>0.913994</td>\n",
       "      <td>11.096338</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           a          b\n",
       "0   0.933496  10.126658\n",
       "1   0.929125   9.852667\n",
       "2   0.899762  10.307814\n",
       "3   0.968039  10.416074\n",
       "4   1.004349  10.441463\n",
       "..       ...        ...\n",
       "95  0.910904  10.357727\n",
       "96  0.818276  12.358640\n",
       "97  0.981826   9.622724\n",
       "98  0.962237  10.897055\n",
       "99  0.913994  11.096338\n",
       "\n",
       "[100 rows x 2 columns]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "estimates = bootstrap_sampling(data, estimator=np.std, n_boot=100)\n",
    "estimates"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0962c4f6-e007-4ffa-9579-2755b02e3f0f",
   "metadata": {},
   "source": [
    "If we take the mean of the bootstrap estimates of the standard deviation, we should recover a value close to the standard deviation of the distribution that the data were generated from."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b10fdbbd-5c68-49c4-9d4c-eb4629cef916",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "a     0.943942\n",
       "b    10.480457\n",
       "dtype: float64"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "estimates.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ef17ab7e-5f8d-4672-b17e-e8dd1e1cd5b8",
   "metadata": {},
   "source": [
    "#### Exclude unwanted columns "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7cd8b9e5-f0c1-451d-8ad9-1786fe9c3c98",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.259128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.098232</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.087111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.131376</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.050997</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>0.129835</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>-0.004873</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>-0.046338</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>0.246239</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>0.355848</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           a\n",
       "0   0.259128\n",
       "1   0.098232\n",
       "2   0.087111\n",
       "3  -0.131376\n",
       "4   0.050997\n",
       "..       ...\n",
       "95  0.129835\n",
       "96 -0.004873\n",
       "97 -0.046338\n",
       "98  0.246239\n",
       "99  0.355848\n",
       "\n",
       "[100 rows x 1 columns]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "estimates = bootstrap_sampling(\n",
    "    data, n_boot=100, columns_to_exclude=[\"b\"]\n",
    ")\n",
    "estimates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6864599e-c090-47b9-9f8f-27fff8d52d2d",
   "metadata": {},
   "outputs": [],
   "source": [
    "#|export\n",
    "def compute_evaluation_estimates(\n",
    "    df: pd.DataFrame,  # Evaluations per query data, usually obtained pyvespa evaluate method.\n",
    "    n_boot: int = 1000,  # Number of bootstrap samples.  \n",
    "    estimator: Callable = np.mean,  # estimator function that accepts an array-like argument. \n",
    "    quantile_low: float = 0.025,  # lower quantile to compute confidence interval\n",
    "    quantile_high = 0.975  # upper quantile to compute confidence interval\n",
    "):\n",
    "    \"Compute estimate and confidence interval for evaluation per query metrics.\"\n",
    "    estimates = (\n",
    "        df\n",
    "        .groupby(\"model\")\n",
    "        .apply(bootstrap_sampling, \n",
    "               estimator = estimator, \n",
    "               n_boot = n_boot, \n",
    "               columns_to_exclude = [\"query_id\", \"model\"]\n",
    "              )\n",
    "        .reset_index(level=\"model\")\n",
    "        .groupby(\"model\")\n",
    "        .agg(\n",
    "            [\n",
    "                lambda x: np.quantile(x, q=quantile_low), \n",
    "                lambda x: np.quantile(x, q=0.5), \n",
    "                lambda x: np.quantile(x, q=quantile_high)\n",
    "            ]\n",
    "        )\n",
    "        .rename(\n",
    "            columns={\n",
    "                \"<lambda_0>\": \"low\", \n",
    "                \"<lambda_1>\": \"median\", \n",
    "                \"<lambda_2>\": \"high\"\n",
    "            }\n",
    "        )\n",
    "        .T\n",
    "        .reset_index()\n",
    "        .rename_axis(None, axis=1)\n",
    "        .rename(\n",
    "            columns={\n",
    "                \"level_0\": \"metric\", \"level_1\": \"quantile\"\n",
    "            }\n",
    "        )\n",
    "    )\n",
    "    estimates = pd.melt(estimates, id_vars=[\"metric\", \"quantile\"])    \n",
    "    estimates = (\n",
    "        pd.pivot(\n",
    "            estimates,\n",
    "            index=[\n",
    "                x for x in estimates.columns if x not in [\"quantile\", \"value\"]\n",
    "            ], \n",
    "            columns=\"quantile\", values=\"value\"\n",
    "        )\n",
    "        .reset_index()\n",
    "        .rename_axis(None, axis=1)\n",
    "        .rename(columns={\"variable\":\"model\"})[\n",
    "            [\"metric\", \"model\", \"low\", \"median\", \"high\"]\n",
    "        ]\n",
    "    )\n",
    "    return estimates"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "55f823d3-48b4-46a2-8a73-0bf9e27b9d3a",
   "metadata": {},
   "source": [
    "Usage:"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5db266f5-9f03-44ce-a2d5-0f4fdcff4b44",
   "metadata": {},
   "source": [
    "Generate sample data frame, which must contain the column `model`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e96c0f56-c3bf-4ae8-9269-1eae5f698692",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>query_id</th>\n",
       "      <th>metric_1</th>\n",
       "      <th>metric_2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>B</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>B</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1995</th>\n",
       "      <td>A</td>\n",
       "      <td>997</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1996</th>\n",
       "      <td>B</td>\n",
       "      <td>998</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1997</th>\n",
       "      <td>A</td>\n",
       "      <td>998</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1998</th>\n",
       "      <td>A</td>\n",
       "      <td>999</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1999</th>\n",
       "      <td>B</td>\n",
       "      <td>999</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2000 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     model  query_id  metric_1  metric_2\n",
       "0        A         0         0         0\n",
       "1        B         0         1         1\n",
       "2        A         1         0         1\n",
       "3        B         1         1         1\n",
       "4        A         2         0         0\n",
       "...    ...       ...       ...       ...\n",
       "1995     A       997         1         0\n",
       "1996     B       998         1         1\n",
       "1997     A       998         1         0\n",
       "1998     A       999         0         0\n",
       "1999     B       999         0         1\n",
       "\n",
       "[2000 rows x 4 columns]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "number_data_points = 1000\n",
    "data = pd.DataFrame(\n",
    "    data = {\n",
    "        \"model\": (\n",
    "            [\"A\"] * number_data_points + \n",
    "            [\"B\"] * number_data_points\n",
    "        ),\n",
    "        \"query_id\": (\n",
    "            list(range(number_data_points)) + \n",
    "            list(range(number_data_points))\n",
    "        ),\n",
    "        \"metric_1\": (\n",
    "            np.random.binomial(size=number_data_points, n=1, p=0.3).tolist() + \n",
    "            np.random.binomial(size=number_data_points, n=1, p=0.7).tolist()\n",
    "        ),\n",
    "        \"metric_2\": (\n",
    "            np.random.binomial(size=number_data_points, n=1, p=0.1).tolist() + \n",
    "            np.random.binomial(size=number_data_points, n=1, p=0.9).tolist()\n",
    "        )\n",
    "        \n",
    "    }\n",
    ").sort_values(\"query_id\").reset_index(drop=True)\n",
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7119caad-6b5c-43b9-96f0-038843e117eb",
   "metadata": {},
   "source": [
    "#### Compute the confidence interval of the mean by default"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8afda111-96dc-451d-9315-cf77c350c69c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>metric</th>\n",
       "      <th>model</th>\n",
       "      <th>low</th>\n",
       "      <th>median</th>\n",
       "      <th>high</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>metric_1</td>\n",
       "      <td>A</td>\n",
       "      <td>0.268000</td>\n",
       "      <td>0.296</td>\n",
       "      <td>0.325</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>metric_1</td>\n",
       "      <td>B</td>\n",
       "      <td>0.667000</td>\n",
       "      <td>0.696</td>\n",
       "      <td>0.724</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>metric_2</td>\n",
       "      <td>A</td>\n",
       "      <td>0.091000</td>\n",
       "      <td>0.109</td>\n",
       "      <td>0.129</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>metric_2</td>\n",
       "      <td>B</td>\n",
       "      <td>0.887975</td>\n",
       "      <td>0.907</td>\n",
       "      <td>0.924</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     metric model       low  median   high\n",
       "0  metric_1     A  0.268000   0.296  0.325\n",
       "1  metric_1     B  0.667000   0.696  0.724\n",
       "2  metric_2     A  0.091000   0.109  0.129\n",
       "3  metric_2     B  0.887975   0.907  0.924"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compute_evaluation_estimates(data)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5ae83188-8969-4d26-8f0f-9c0a73cb2e8f",
   "metadata": {},
   "source": [
    "#### Specify function. Example: Standard deviation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b70ea021-3663-41d8-bd59-107057099845",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>metric</th>\n",
       "      <th>model</th>\n",
       "      <th>low</th>\n",
       "      <th>median</th>\n",
       "      <th>high</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>metric_1</td>\n",
       "      <td>A</td>\n",
       "      <td>0.442918</td>\n",
       "      <td>0.456491</td>\n",
       "      <td>0.468375</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>metric_1</td>\n",
       "      <td>B</td>\n",
       "      <td>0.448001</td>\n",
       "      <td>0.459983</td>\n",
       "      <td>0.470931</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>metric_2</td>\n",
       "      <td>A</td>\n",
       "      <td>0.289026</td>\n",
       "      <td>0.311639</td>\n",
       "      <td>0.335200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>metric_2</td>\n",
       "      <td>B</td>\n",
       "      <td>0.264998</td>\n",
       "      <td>0.291829</td>\n",
       "      <td>0.315366</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     metric model       low    median      high\n",
       "0  metric_1     A  0.442918  0.456491  0.468375\n",
       "1  metric_1     B  0.448001  0.459983  0.470931\n",
       "2  metric_2     A  0.289026  0.311639  0.335200\n",
       "3  metric_2     B  0.264998  0.291829  0.315366"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compute_evaluation_estimates(data, estimator=np.std)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d5861611-503f-4000-8624-2d509858379f",
   "metadata": {},
   "source": [
    "#### Specify interval coverage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "718592ef-fd33-43a5-8f3d-311e0df7a5f4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>metric</th>\n",
       "      <th>model</th>\n",
       "      <th>low</th>\n",
       "      <th>median</th>\n",
       "      <th>high</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>metric_1</td>\n",
       "      <td>A</td>\n",
       "      <td>0.285</td>\n",
       "      <td>0.296</td>\n",
       "      <td>0.308</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>metric_1</td>\n",
       "      <td>B</td>\n",
       "      <td>0.684</td>\n",
       "      <td>0.696</td>\n",
       "      <td>0.708</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>metric_2</td>\n",
       "      <td>A</td>\n",
       "      <td>0.102</td>\n",
       "      <td>0.110</td>\n",
       "      <td>0.118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>metric_2</td>\n",
       "      <td>B</td>\n",
       "      <td>0.898</td>\n",
       "      <td>0.906</td>\n",
       "      <td>0.914</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     metric model    low  median   high\n",
       "0  metric_1     A  0.285   0.296  0.308\n",
       "1  metric_1     B  0.684   0.696  0.708\n",
       "2  metric_2     A  0.102   0.110  0.118\n",
       "3  metric_2     B  0.898   0.906  0.914"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compute_evaluation_estimates(\n",
    "    data, \n",
    "    quantile_low=0.2, \n",
    "    quantile_high=0.8\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5621ea61-eab9-4051-86dc-357c21e53419",
   "metadata": {},
   "outputs": [],
   "source": [
    "#|hide\n",
    "test_fail(\n",
    "    compute_evaluation_estimates, \n",
    "    kwargs={\n",
    "        \"df\":data[[\"query_id\", \"metric_1\", \"metric_2\"]]\n",
    "    },\n",
    "    contains=\"'model'\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "94bd8def-cbd2-4f6a-835f-d350221b99fe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>metric</th>\n",
       "      <th>model</th>\n",
       "      <th>low</th>\n",
       "      <th>median</th>\n",
       "      <th>high</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>metric_1</td>\n",
       "      <td>A</td>\n",
       "      <td>0.269975</td>\n",
       "      <td>0.297</td>\n",
       "      <td>0.326000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>metric_1</td>\n",
       "      <td>B</td>\n",
       "      <td>0.667975</td>\n",
       "      <td>0.696</td>\n",
       "      <td>0.726000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>metric_2</td>\n",
       "      <td>A</td>\n",
       "      <td>0.091000</td>\n",
       "      <td>0.109</td>\n",
       "      <td>0.129025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>metric_2</td>\n",
       "      <td>B</td>\n",
       "      <td>0.888000</td>\n",
       "      <td>0.907</td>\n",
       "      <td>0.923000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     metric model       low  median      high\n",
       "0  metric_1     A  0.269975   0.297  0.326000\n",
       "1  metric_1     B  0.667975   0.696  0.726000\n",
       "2  metric_2     A  0.091000   0.109  0.129025\n",
       "3  metric_2     B  0.888000   0.907  0.923000"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compute_evaluation_estimates(data[[\"model\", \"metric_1\", \"metric_2\"]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f0d77781-8f0d-46d0-92f6-22ef8b25719d",
   "metadata": {},
   "outputs": [],
   "source": [
    "#|hide\n",
    "nbdev_export()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "learntorank",
   "language": "python",
   "name": "learntorank"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}