diff --git a/apps/opik-documentation/documentation/docs/cookbook/evaluate_hallucination_metric.ipynb b/apps/opik-documentation/documentation/docs/cookbook/evaluate_hallucination_metric.ipynb
index f2ecdf6f77..90a057444a 100644
--- a/apps/opik-documentation/documentation/docs/cookbook/evaluate_hallucination_metric.ipynb
+++ b/apps/opik-documentation/documentation/docs/cookbook/evaluate_hallucination_metric.ipynb
@@ -26,7 +26,7 @@
"metadata": {},
"outputs": [],
"source": [
- "%pip install opik pyarrow fsspec huggingface_hub --upgrade"
+ "%pip install opik pyarrow fsspec huggingface_hub --upgrade --quiet"
]
},
{
@@ -51,7 +51,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
@@ -98,8 +98,8 @@
" {\n",
" \"input\": x[\"question\"],\n",
" \"context\": [x[\"passage\"]],\n",
- " \"output\": x[\"answer\"],\n",
- " \"expected_output\": x[\"label\"],\n",
+ " \"llm_output\": x[\"answer\"],\n",
+ " \"expected_hallucination_label\": x[\"label\"],\n",
" }\n",
" for x in df.to_dict(orient=\"records\")\n",
"]\n",
@@ -139,7 +139,7 @@
" metric = Hallucination()\n",
" try:\n",
" metric_score = metric.score(\n",
- " input=x[\"input\"], context=x[\"context\"], output=x[\"output\"]\n",
+ " input=x[\"input\"], context=x[\"context\"], output=x[\"llm_output\"]\n",
" )\n",
" hallucination_score = metric_score.value\n",
" hallucination_reason = metric_score.reason\n",
@@ -149,9 +149,8 @@
" hallucination_reason = str(e)\n",
"\n",
" return {\n",
- " \"output\": \"FAIL\" if hallucination_score == 1 else \"PASS\",\n",
+ " \"hallucination_score\": \"FAIL\" if hallucination_score == 1 else \"PASS\",\n",
" \"hallucination_reason\": hallucination_reason,\n",
- " \"reference\": x[\"expected_output\"],\n",
" }\n",
"\n",
"\n",
@@ -174,6 +173,10 @@
" task=evaluation_task,\n",
" scoring_metrics=[check_hallucinated_metric],\n",
" experiment_config=experiment_config,\n",
+ " scoring_key_mapping={\n",
+ " \"reference\": \"expected_hallucination_label\",\n",
+ " \"output\": \"hallucination_score\",\n",
+ " },\n",
")"
]
},
diff --git a/apps/opik-documentation/documentation/docs/cookbook/evaluate_hallucination_metric.md b/apps/opik-documentation/documentation/docs/cookbook/evaluate_hallucination_metric.md
index a5978a2894..c8c9d46902 100644
--- a/apps/opik-documentation/documentation/docs/cookbook/evaluate_hallucination_metric.md
+++ b/apps/opik-documentation/documentation/docs/cookbook/evaluate_hallucination_metric.md
@@ -10,7 +10,7 @@ For this guide we will be evaluating the Hallucination metric included in the LL
```python
-%pip install opik pyarrow fsspec huggingface_hub --upgrade
+%pip install opik pyarrow fsspec huggingface_hub --upgrade --quiet
```
@@ -60,8 +60,8 @@ dataset_records = [
{
"input": x["question"],
"context": [x["passage"]],
- "output": x["answer"],
- "expected_output": x["label"],
+ "llm_output": x["answer"],
+ "expected_hallucination_label": x["label"],
}
for x in df.to_dict(orient="records")
]
@@ -92,7 +92,7 @@ def evaluation_task(x: Dict):
metric = Hallucination()
try:
metric_score = metric.score(
- input=x["input"], context=x["context"], output=x["output"]
+ input=x["input"], context=x["context"], output=x["llm_output"]
)
hallucination_score = metric_score.value
hallucination_reason = metric_score.reason
@@ -102,9 +102,8 @@ def evaluation_task(x: Dict):
hallucination_reason = str(e)
return {
- "output": "FAIL" if hallucination_score == 1 else "PASS",
- "hallucination_reason": hallucination_reason,
- "reference": x["expected_output"],
+ "hallucination_score": "FAIL" if hallucination_score == 1 else "PASS",
+ "hallucination_reason": hallucination_reason
}
@@ -127,6 +126,7 @@ res = evaluate(
task=evaluation_task,
scoring_metrics=[check_hallucinated_metric],
experiment_config=experiment_config,
+ scoring_key_mapping={"reference": "expected_hallucination_label", "output": "hallucination_score"},
)
```
diff --git a/apps/opik-documentation/documentation/docs/cookbook/evaluate_moderation_metric.ipynb b/apps/opik-documentation/documentation/docs/cookbook/evaluate_moderation_metric.ipynb
index d866945044..526d9d2a77 100644
--- a/apps/opik-documentation/documentation/docs/cookbook/evaluate_moderation_metric.ipynb
+++ b/apps/opik-documentation/documentation/docs/cookbook/evaluate_moderation_metric.ipynb
@@ -51,7 +51,7 @@
},
{
"cell_type": "code",
- "execution_count": 1,
+ "execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
@@ -73,7 +73,7 @@
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -142,6 +142,7 @@
"from opik.evaluation.metrics.llm_judges.moderation.template import generate_query\n",
"from typing import Dict\n",
"\n",
+ "\n",
"# Define the evaluation task\n",
"def evaluation_task(x: Dict):\n",
" metric = Moderation()\n",
@@ -157,10 +158,8 @@
" moderation_score = \"moderated\" if metric_score.value > 0.5 else \"not_moderated\"\n",
"\n",
" return {\n",
- " \"output\": moderation_score,\n",
" \"moderation_score\": moderation_score,\n",
" \"moderation_reason\": moderation_reason,\n",
- " \"reference\": x[\"expected_output\"],\n",
" }\n",
"\n",
"\n",
@@ -181,6 +180,7 @@
" task=evaluation_task,\n",
" scoring_metrics=[moderation_metric],\n",
" experiment_config=experiment_config,\n",
+ " scoring_key_mapping={\"reference\": \"expected_output\", \"output\": \"moderation_score\"},\n",
")"
]
},
diff --git a/apps/opik-documentation/documentation/docs/cookbook/evaluate_moderation_metric.md b/apps/opik-documentation/documentation/docs/cookbook/evaluate_moderation_metric.md
index 8d5c23bf82..15e6b94bf8 100644
--- a/apps/opik-documentation/documentation/docs/cookbook/evaluate_moderation_metric.md
+++ b/apps/opik-documentation/documentation/docs/cookbook/evaluate_moderation_metric.md
@@ -110,10 +110,8 @@ def evaluation_task(x: Dict):
moderation_score = "moderated" if metric_score.value > 0.5 else "not_moderated"
return {
- "output": moderation_score,
"moderation_score": moderation_score,
"moderation_reason": moderation_reason,
- "reference": x["expected_output"],
}
@@ -134,6 +132,7 @@ res = evaluate(
task=evaluation_task,
scoring_metrics=[moderation_metric],
experiment_config=experiment_config,
+ scoring_key_mapping={"reference": "expected_output", "output": "moderation_score"},
)
```
diff --git a/apps/opik-documentation/documentation/docs/cookbook/langchain.ipynb b/apps/opik-documentation/documentation/docs/cookbook/langchain.ipynb
index 86d50225a3..b9798ec2c7 100644
--- a/apps/opik-documentation/documentation/docs/cookbook/langchain.ipynb
+++ b/apps/opik-documentation/documentation/docs/cookbook/langchain.ipynb
@@ -28,18 +28,37 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 1,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\n",
+ "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
+ "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
+ "Note: you may need to restart the kernel to use updated packages.\n"
+ ]
+ }
+ ],
"source": [
"%pip install --upgrade --quiet opik langchain langchain-community langchain-openai"
]
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 2,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "OPIK: Opik is already configured. You can check the settings by viewing the config file at /Users/jacquesverre/.opik.config\n"
+ ]
+ }
+ ],
"source": [
"import opik\n",
"\n",
@@ -57,7 +76,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
@@ -72,7 +91,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
@@ -113,9 +132,47 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 5,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "OPIK: Started logging traces to the \"langchain-integration-demo\" project at https://www.comet.com/opik/jacques-comet/redirect/projects?name=langchain-integration-demo.\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "{\n",
+ " \"result\": [\n",
+ " \"Which customer has made the largest total purchases in terms of dollar amount?\",\n",
+ " \"How many unique genres are available in the Chinook database?\",\n",
+ " \"What is the average total number of tracks in an album?\",\n",
+ " \"Which employee has the highest total sales amount and how does it compare to other employees?\",\n",
+ " \"What is the average revenue generated by each genre in the database?\",\n",
+ " \"Which customer has spent the most money on individual tracks instead of full albums?\",\n",
+ " \"What percentage of total sales does each customer account for?\",\n",
+ " \"How many unique artists are featured in the database?\",\n",
+ " \"Which genre has the highest average track length?\",\n",
+ " \"How does the average track length vary between different artists?\",\n",
+ " \"What is the average track length for each genre?\",\n",
+ " \"What is the average number of tracks purchased per invoice?\",\n",
+ " \"Which customer has the most diverse taste in music based on the number of unique genres purchased?\",\n",
+ " \"What is the average time between purchases for each customer?\",\n",
+ " \"How does the average purchase amount change over time?\",\n",
+ " \"Which customers have purchased from all genres available in the database?\",\n",
+ " \"What is the total revenue generated from each playlist?\",\n",
+ " \"What is the most popular track based on total number of purchases?\",\n",
+ " \"How does the average number of tracks in an album differ between different artists?\",\n",
+ " \"What is the distribution of sales by country?\"\n",
+ " ]\n",
+ "}\n"
+ ]
+ }
+ ],
"source": [
"from opik.integrations.openai import track_openai\n",
"from openai import OpenAI\n",
@@ -152,7 +209,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
@@ -180,9 +237,24 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 7,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "OPIK: Started logging traces to the \"langchain-integration-demo\" project at https://www.comet.com/opik/jacques-comet/redirect/projects?name=langchain-integration-demo.\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "SELECT COUNT(\"EmployeeId\") AS \"TotalEmployees\" FROM \"Employee\"\n"
+ ]
+ }
+ ],
"source": [
"# Use langchain to create a SQL query to answer the question\n",
"from langchain.chains import create_sql_query_chain\n",
@@ -212,9 +284,70 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 8,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Evaluation: 100%|██████████| 20/20 [00:03<00:00, 5.39it/s]\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "╭─ synthetic_questions (20 samples) ─╮\n",
+ "│ │\n",
+ "│ Total time: 00:00:03 │\n",
+ "│ Number of samples: 20 │\n",
+ "│ │\n",
+ "│ valid_sql_query: 0.9500 (avg) │\n",
+ "│ │\n",
+ "╰────────────────────────────────────╯\n",
+ "
\n"
+ ],
+ "text/plain": [
+ "╭─ synthetic_questions (20 samples) ─╮\n",
+ "│ │\n",
+ "│ \u001b[1mTotal time: \u001b[0m 00:00:03 │\n",
+ "│ \u001b[1mNumber of samples:\u001b[0m 20 │\n",
+ "│ │\n",
+ "│ \u001b[1;32mvalid_sql_query: 0.9500 (avg)\u001b[0m │\n",
+ "│ │\n",
+ "╰────────────────────────────────────╯\n"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/html": [
+ "Uploading results to Opik ... \n",
+ "
\n"
+ ],
+ "text/plain": [
+ "Uploading results to Opik \u001b[33m...\u001b[0m \n"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/html": [
+ "View the results in your Opik dashboard.\n",
+ "
\n"
+ ],
+ "text/plain": [
+ "View the results \u001b]8;id=611380;https://www.comet.com/opik/jacques-comet/experiments/0191e8b3-9c3c-7998-b5af-01e4a3e5e2c7/compare?experiments=%5B%2206740861-c1e7-7f5e-8000-b22ad440a7c3%22%5D\u001b\\in your Opik dashboard\u001b]8;;\u001b\\.\n"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
"source": [
"from opik import Opik, track\n",
"from opik.evaluation import evaluate\n",
@@ -255,7 +388,7 @@
"def evaluation_task(item):\n",
" response = llm_chain(item[\"question\"])\n",
"\n",
- " return {\"reference\": \"hello\", \"output\": response}\n",
+ " return {\"output\": response}\n",
"\n",
"\n",
"res = evaluate(\n",
diff --git a/apps/opik-documentation/documentation/docs/cookbook/langchain.md b/apps/opik-documentation/documentation/docs/cookbook/langchain.md
index 1ecb2c5eb2..e7b403c938 100644
--- a/apps/opik-documentation/documentation/docs/cookbook/langchain.md
+++ b/apps/opik-documentation/documentation/docs/cookbook/langchain.md
@@ -19,6 +19,12 @@ We will highlight three different parts of the workflow:
%pip install --upgrade --quiet opik langchain langchain-community langchain-openai
```
+
+ [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m A new release of pip is available: [0m[31;49m24.2[0m[39;49m -> [0m[32;49m24.3.1[0m
+ [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m To update, run: [0m[32;49mpip install --upgrade pip[0m
+ Note: you may need to restart the kernel to use updated packages.
+
+
```python
import opik
@@ -26,6 +32,9 @@ import opik
opik.configure(use_local=False)
```
+ OPIK: Opik is already configured. You can check the settings by viewing the config file at /Users/jacquesverre/.opik.config
+
+
## Preparing our environment
First, we will download the Chinook database and set up our different API keys.
@@ -99,6 +108,35 @@ completion = openai_client.chat.completions.create(
print(completion.choices[0].message.content)
```
+ OPIK: Started logging traces to the "langchain-integration-demo" project at https://www.comet.com/opik/jacques-comet/redirect/projects?name=langchain-integration-demo.
+
+
+ {
+ "result": [
+ "Which customer has made the largest total purchases in terms of dollar amount?",
+ "How many unique genres are available in the Chinook database?",
+ "What is the average total number of tracks in an album?",
+ "Which employee has the highest total sales amount and how does it compare to other employees?",
+ "What is the average revenue generated by each genre in the database?",
+ "Which customer has spent the most money on individual tracks instead of full albums?",
+ "What percentage of total sales does each customer account for?",
+ "How many unique artists are featured in the database?",
+ "Which genre has the highest average track length?",
+ "How does the average track length vary between different artists?",
+ "What is the average track length for each genre?",
+ "What is the average number of tracks purchased per invoice?",
+ "Which customer has the most diverse taste in music based on the number of unique genres purchased?",
+ "What is the average time between purchases for each customer?",
+ "How does the average purchase amount change over time?",
+ "Which customers have purchased from all genres available in the database?",
+ "What is the total revenue generated from each playlist?",
+ "What is the most popular track based on total number of purchases?",
+ "How does the average number of tracks in an album differ between different artists?",
+ "What is the distribution of sales by country?"
+ ]
+ }
+
+
Now that we have our synthetic dataset, we can create a dataset in Comet and insert the questions into it.
Since the insert methods in the SDK deduplicates items, we can insert 20 items and if the items already exist, Opik will automatically remove them.
@@ -139,6 +177,12 @@ response
print(response)
```
+ OPIK: Started logging traces to the "langchain-integration-demo" project at https://www.comet.com/opik/jacques-comet/redirect/projects?name=langchain-integration-demo.
+
+
+ SELECT COUNT("EmployeeId") AS "TotalEmployees" FROM "Employee"
+
+
## Automating the evaluation
In order to ensure our LLM application is working correctly, we will test it on our synthetic dataset.
@@ -186,7 +230,7 @@ def llm_chain(input: str) -> str:
def evaluation_task(item):
response = llm_chain(item["question"])
- return {"reference": "hello", "output": response}
+ return {"output": response}
res = evaluate(
@@ -198,6 +242,34 @@ res = evaluate(
)
```
+ Evaluation: 100%|██████████| 20/20 [00:03<00:00, 5.39it/s]
+
+
+
+╭─ synthetic_questions (20 samples) ─╮
+│ │
+│ Total time: 00:00:03 │
+│ Number of samples: 20 │
+│ │
+│ valid_sql_query: 0.9500 (avg) │
+│ │
+╰────────────────────────────────────╯
+
+
+
+
+
+Uploading results to Opik ...
+
+
+
+
+
+View the results in your Opik dashboard.
+
+
+
+
The evaluation results are now uploaded to the Opik platform and can be viewed in the UI.
diff --git a/apps/opik-documentation/documentation/docs/cookbook/quickstart_notebook.ipynb b/apps/opik-documentation/documentation/docs/cookbook/quickstart_notebook.ipynb
index f2fec09d63..7ee87f6658 100644
--- a/apps/opik-documentation/documentation/docs/cookbook/quickstart_notebook.ipynb
+++ b/apps/opik-documentation/documentation/docs/cookbook/quickstart_notebook.ipynb
@@ -29,19 +29,16 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "Requirement already satisfied: opik in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (1.1.4)\n",
- "Collecting opik\n",
- " Downloading opik-1.1.6-py3-none-any.whl.metadata (18 kB)\n",
- "Requirement already satisfied: openai in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (1.52.1)\n",
+ "Requirement already satisfied: opik in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (1.1.10)\n",
+ "Requirement already satisfied: openai in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (1.54.4)\n",
"Collecting openai\n",
- " Downloading openai-1.54.4-py3-none-any.whl.metadata (24 kB)\n",
+ " Downloading openai-1.55.0-py3-none-any.whl.metadata (24 kB)\n",
"Requirement already satisfied: requests in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (2.32.3)\n",
"Requirement already satisfied: PyPDF2 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (3.0.1)\n",
"Requirement already satisfied: click in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from opik) (8.1.7)\n",
"Requirement already satisfied: httpx<1.0.0 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from opik) (0.27.2)\n",
"Requirement already satisfied: levenshtein~=0.25.1 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from opik) (0.25.1)\n",
"Requirement already satisfied: litellm in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from opik) (1.51.2)\n",
- "Requirement already satisfied: pandas<3.0.0,>=2.0.0 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from opik) (2.2.2)\n",
"Requirement already satisfied: pydantic-settings<3.0.0,>=2.0.0 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from opik) (2.4.0)\n",
"Requirement already satisfied: pydantic<3.0.0,>=2.0.0 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from opik) (2.8.2)\n",
"Requirement already satisfied: pytest in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from opik) (8.3.2)\n",
@@ -60,10 +57,6 @@
"Requirement already satisfied: httpcore==1.* in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from httpx<1.0.0->opik) (1.0.5)\n",
"Requirement already satisfied: h11<0.15,>=0.13 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from httpcore==1.*->httpx<1.0.0->opik) (0.14.0)\n",
"Requirement already satisfied: rapidfuzz<4.0.0,>=3.8.0 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from levenshtein~=0.25.1->opik) (3.9.6)\n",
- "Requirement already satisfied: numpy>=1.26.0 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from pandas<3.0.0,>=2.0.0->opik) (1.26.4)\n",
- "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from pandas<3.0.0,>=2.0.0->opik) (2.9.0)\n",
- "Requirement already satisfied: pytz>=2020.1 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from pandas<3.0.0,>=2.0.0->opik) (2024.1)\n",
- "Requirement already satisfied: tzdata>=2022.7 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from pandas<3.0.0,>=2.0.0->opik) (2024.1)\n",
"Requirement already satisfied: annotated-types>=0.4.0 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.0.0->opik) (0.7.0)\n",
"Requirement already satisfied: pydantic-core==2.20.1 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.0.0->opik) (2.20.1)\n",
"Requirement already satisfied: python-dotenv>=0.21.0 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from pydantic-settings<3.0.0,>=2.0.0->opik) (1.0.1)\n",
@@ -85,7 +78,6 @@
"Requirement already satisfied: referencing>=0.28.4 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm->opik) (0.35.1)\n",
"Requirement already satisfied: rpds-py>=0.7.1 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm->opik) (0.20.0)\n",
"Requirement already satisfied: mdurl~=0.1 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from markdown-it-py>=2.2.0->rich->opik) (0.1.2)\n",
- "Requirement already satisfied: six>=1.5 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from python-dateutil>=2.8.2->pandas<3.0.0,>=2.0.0->opik) (1.16.0)\n",
"Requirement already satisfied: regex>=2022.1.18 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from tiktoken>=0.7.0->litellm->opik) (2024.7.24)\n",
"Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from aiohttp->litellm->opik) (2.3.5)\n",
"Requirement already satisfied: aiosignal>=1.1.2 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from aiohttp->litellm->opik) (1.3.1)\n",
@@ -97,20 +89,15 @@
"Requirement already satisfied: fsspec>=2023.5.0 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from huggingface-hub<1.0,>=0.16.4->tokenizers->litellm->opik) (2024.10.0)\n",
"Requirement already satisfied: pyyaml>=5.1 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from huggingface-hub<1.0,>=0.16.4->tokenizers->litellm->opik) (6.0.2)\n",
"Requirement already satisfied: propcache>=0.2.0 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from yarl<2.0,>=1.12.0->aiohttp->litellm->opik) (0.2.0)\n",
- "Downloading opik-1.1.6-py3-none-any.whl (240 kB)\n",
- "Downloading openai-1.54.4-py3-none-any.whl (389 kB)\n",
- "Installing collected packages: openai, opik\n",
+ "Downloading openai-1.55.0-py3-none-any.whl (389 kB)\n",
+ "Installing collected packages: openai\n",
" Attempting uninstall: openai\n",
- " Found existing installation: openai 1.52.1\n",
- " Uninstalling openai-1.52.1:\n",
- " Successfully uninstalled openai-1.52.1\n",
- " Attempting uninstall: opik\n",
- " Found existing installation: opik 1.1.4\n",
- " Uninstalling opik-1.1.4:\n",
- " Successfully uninstalled opik-1.1.4\n",
+ " Found existing installation: openai 1.54.4\n",
+ " Uninstalling openai-1.54.4:\n",
+ " Successfully uninstalled openai-1.54.4\n",
"\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
"predibase 2024.9.3 requires urllib3==1.26.12, but you have urllib3 2.2.3 which is incompatible.\u001b[0m\u001b[31m\n",
- "\u001b[0mSuccessfully installed openai-1.54.4 opik-1.1.6\n",
+ "\u001b[0mSuccessfully installed openai-1.55.0\n",
"\n",
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
@@ -119,7 +106,7 @@
}
],
"source": [
- "%pip install -U opik openai requests PyPDF2"
+ "%pip install -U opik openai requests PyPDF2 --quiet"
]
},
{
@@ -134,7 +121,7 @@
},
{
"cell_type": "code",
- "execution_count": 1,
+ "execution_count": 2,
"metadata": {},
"outputs": [
{
@@ -167,7 +154,7 @@
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
@@ -209,7 +196,7 @@
},
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
@@ -296,7 +283,7 @@
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": 5,
"metadata": {},
"outputs": [
{
@@ -310,8 +297,8 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "Sure! Please provide the current summary and the instruction you'd like me to\n",
- "focus on, and I'll create the concise final summary for you.\n"
+ "Please provide the current summary and the specific instruction you would like\n",
+ "me to focus on for the final summary.\n"
]
}
],
@@ -357,7 +344,7 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
@@ -430,13 +417,76 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": 20,
"metadata": {},
"outputs": [],
"source": [
"from opik.evaluation.metrics import base_metric, score_result\n",
"import json\n",
"\n",
+ "# We will define the response format so the output has the correct schema. You can also use structured outputs with Pydantic models for this.\n",
+ "json_schema = {\n",
+ " \"type\": \"json_schema\",\n",
+ " \"json_schema\": {\n",
+ " \"name\": \"summary_evaluation_schema\",\n",
+ " \"schema\": {\n",
+ " \"type\": \"object\",\n",
+ " \"properties\": {\n",
+ " \"relevance\": {\n",
+ " \"type\": \"object\",\n",
+ " \"properties\": {\n",
+ " \"score\": {\n",
+ " \"type\": \"integer\",\n",
+ " \"minimum\": 1,\n",
+ " \"maximum\": 5,\n",
+ " \"description\": \"Score between 1-5 for how well the summary addresses the instruction\",\n",
+ " },\n",
+ " \"explanation\": {\n",
+ " \"type\": \"string\",\n",
+ " \"description\": \"Brief explanation of the relevance score\",\n",
+ " },\n",
+ " },\n",
+ " \"required\": [\"score\", \"explanation\"],\n",
+ " },\n",
+ " \"conciseness\": {\n",
+ " \"type\": \"object\",\n",
+ " \"properties\": {\n",
+ " \"score\": {\n",
+ " \"type\": \"integer\",\n",
+ " \"minimum\": 1,\n",
+ " \"maximum\": 5,\n",
+ " \"description\": \"Score between 1-5 for how concise the summary is while retaining key information\",\n",
+ " },\n",
+ " \"explanation\": {\n",
+ " \"type\": \"string\",\n",
+ " \"description\": \"Brief explanation of the conciseness score\",\n",
+ " },\n",
+ " },\n",
+ " \"required\": [\"score\", \"explanation\"],\n",
+ " },\n",
+ " \"technical_accuracy\": {\n",
+ " \"type\": \"object\",\n",
+ " \"properties\": {\n",
+ " \"score\": {\n",
+ " \"type\": \"integer\",\n",
+ " \"minimum\": 1,\n",
+ " \"maximum\": 5,\n",
+ " \"description\": \"Score between 1-5 for how accurately the summary conveys technical details\",\n",
+ " },\n",
+ " \"explanation\": {\n",
+ " \"type\": \"string\",\n",
+ " \"description\": \"Brief explanation of the technical accuracy score\",\n",
+ " },\n",
+ " },\n",
+ " \"required\": [\"score\", \"explanation\"],\n",
+ " },\n",
+ " },\n",
+ " \"required\": [\"relevance\", \"conciseness\", \"technical_accuracy\"],\n",
+ " \"additionalProperties\": False,\n",
+ " },\n",
+ " },\n",
+ "}\n",
+ "\n",
"\n",
"# Custom Metric: One template/prompt to extract 4 scores/results\n",
"class EvaluateSummary(base_metric.BaseMetric):\n",
@@ -476,7 +526,10 @@
" \"\"\"\n",
"\n",
" response = openai_client.chat.completions.create(\n",
- " model=model, max_tokens=1000, messages=[{\"role\": \"user\", \"content\": prompt}]\n",
+ " model=model,\n",
+ " max_tokens=1000,\n",
+ " messages=[{\"role\": \"user\", \"content\": prompt}],\n",
+ " response_format=json_schema,\n",
" )\n",
"\n",
" eval_dict = json.loads(response.choices[0].message.content)\n",
@@ -516,7 +569,7 @@
},
{
"cell_type": "code",
- "execution_count": 7,
+ "execution_count": 21,
"metadata": {},
"outputs": [],
"source": [
@@ -559,7 +612,7 @@
" density_iterations=density_iterations,\n",
" )\n",
"\n",
- " return {\"summary\": result, \"instruction\": instruction}"
+ " return {\"summary\": result}"
]
},
{
@@ -578,82 +631,14 @@
},
{
"cell_type": "code",
- "execution_count": 8,
+ "execution_count": 22,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
- "Evaluation: 11%|█ | 2/18 [00:09<01:00, 3.77s/it]OPIK: Failed to compute metric summary-metrics. Score result will be marked as failed.\n",
- "Traceback (most recent call last):\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages/opik/evaluation/tasks_scorer.py\", line 29, in _score_test_case\n",
- " result = metric.score(**score_kwargs)\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/var/folders/rr/674mwvt12_v0k3qz2z47m8bh0000gn/T/ipykernel_67274/2409766834.py\", line 46, in score\n",
- " eval_dict = json.loads(response.choices[0].message.content)\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/__init__.py\", line 346, in loads\n",
- " return _default_decoder.decode(s)\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py\", line 337, in decode\n",
- " obj, end = self.raw_decode(s, idx=_w(s, 0).end())\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py\", line 355, in raw_decode\n",
- " raise JSONDecodeError(\"Expecting value\", s, err.value) from None\n",
- "json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)\n",
- "Evaluation: 22%|██▏ | 4/18 [00:09<00:21, 1.50s/it]OPIK: Failed to compute metric summary-metrics. Score result will be marked as failed.\n",
- "Traceback (most recent call last):\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages/opik/evaluation/tasks_scorer.py\", line 29, in _score_test_case\n",
- " result = metric.score(**score_kwargs)\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/var/folders/rr/674mwvt12_v0k3qz2z47m8bh0000gn/T/ipykernel_67274/2409766834.py\", line 46, in score\n",
- " eval_dict = json.loads(response.choices[0].message.content)\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/__init__.py\", line 346, in loads\n",
- " return _default_decoder.decode(s)\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py\", line 337, in decode\n",
- " obj, end = self.raw_decode(s, idx=_w(s, 0).end())\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py\", line 355, in raw_decode\n",
- " raise JSONDecodeError(\"Expecting value\", s, err.value) from None\n",
- "json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)\n",
- "Evaluation: 61%|██████ | 11/18 [00:12<00:03, 1.76it/s]OPIK: Failed to compute metric summary-metrics. Score result will be marked as failed.\n",
- "Traceback (most recent call last):\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages/opik/evaluation/tasks_scorer.py\", line 29, in _score_test_case\n",
- " result = metric.score(**score_kwargs)\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/var/folders/rr/674mwvt12_v0k3qz2z47m8bh0000gn/T/ipykernel_67274/2409766834.py\", line 46, in score\n",
- " eval_dict = json.loads(response.choices[0].message.content)\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/__init__.py\", line 346, in loads\n",
- " return _default_decoder.decode(s)\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py\", line 337, in decode\n",
- " obj, end = self.raw_decode(s, idx=_w(s, 0).end())\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py\", line 355, in raw_decode\n",
- " raise JSONDecodeError(\"Expecting value\", s, err.value) from None\n",
- "json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)\n",
- "Evaluation: 78%|███████▊ | 14/18 [00:13<00:01, 2.06it/s]OPIK: Failed to compute metric summary-metrics. Score result will be marked as failed.\n",
- "Traceback (most recent call last):\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages/opik/evaluation/tasks_scorer.py\", line 29, in _score_test_case\n",
- " result = metric.score(**score_kwargs)\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/var/folders/rr/674mwvt12_v0k3qz2z47m8bh0000gn/T/ipykernel_67274/2409766834.py\", line 46, in score\n",
- " eval_dict = json.loads(response.choices[0].message.content)\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/__init__.py\", line 346, in loads\n",
- " return _default_decoder.decode(s)\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py\", line 337, in decode\n",
- " obj, end = self.raw_decode(s, idx=_w(s, 0).end())\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py\", line 355, in raw_decode\n",
- " raise JSONDecodeError(\"Expecting value\", s, err.value) from None\n",
- "json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)\n",
- "Evaluation: 100%|██████████| 18/18 [00:31<00:00, 1.73s/it]\n"
+ "Evaluation: 100%|██████████| 18/18 [00:25<00:00, 1.44s/it]\n"
]
},
{
@@ -661,14 +646,13 @@
"text/html": [
"╭─ arXiv Papers (18 samples) ──────────────╮\n",
"│ │\n",
- "│ Total time: 00:00:31 │\n",
+ "│ Total time: 00:00:26 │\n",
"│ Number of samples: 18 │\n",
"│ │\n",
- "│ summary_relevance: 2.2857 (avg) │\n",
- "│ summary_conciseness: 3.8571 (avg) │\n",
- "│ summary_technical_accuracy: 1.7143 (avg) │\n",
- "│ summary_average_score: 2.6179 (avg) │\n",
- "│ summary-metrics: None (avg) - 4 failed │\n",
+ "│ summary_relevance: 1.1111 (avg) │\n",
+ "│ summary_conciseness: 3.0000 (avg) │\n",
+ "│ summary_technical_accuracy: 1.0000 (avg) │\n",
+ "│ summary_average_score: 1.7028 (avg) │\n",
"│ │\n",
"╰──────────────────────────────────────────╯\n",
"\n"
@@ -676,14 +660,13 @@
"text/plain": [
"╭─ arXiv Papers (18 samples) ──────────────╮\n",
"│ │\n",
- "│ \u001b[1mTotal time: \u001b[0m 00:00:31 │\n",
+ "│ \u001b[1mTotal time: \u001b[0m 00:00:26 │\n",
"│ \u001b[1mNumber of samples:\u001b[0m 18 │\n",
"│ │\n",
- "│ \u001b[1;32msummary_relevance: 2.2857 (avg)\u001b[0m │\n",
- "│ \u001b[1;32msummary_conciseness: 3.8571 (avg)\u001b[0m │\n",
- "│ \u001b[1;32msummary_technical_accuracy: 1.7143 (avg)\u001b[0m │\n",
- "│ \u001b[1;32msummary_average_score: 2.6179 (avg)\u001b[0m │\n",
- "│ \u001b[1;32msummary-metrics: None (avg)\u001b[0m\u001b[31m - 4 failed\u001b[0m │\n",
+ "│ \u001b[1;32msummary_relevance: 1.1111 (avg)\u001b[0m │\n",
+ "│ \u001b[1;32msummary_conciseness: 3.0000 (avg)\u001b[0m │\n",
+ "│ \u001b[1;32msummary_technical_accuracy: 1.0000 (avg)\u001b[0m │\n",
+ "│ \u001b[1;32msummary_average_score: 1.7028 (avg)\u001b[0m │\n",
"│ │\n",
"╰──────────────────────────────────────────╯\n"
]
@@ -707,11 +690,11 @@
{
"data": {
"text/html": [
- "View the results in your Opik dashboard.\n",
+ "View the results in your Opik dashboard.\n",
"\n"
],
"text/plain": [
- "View the results \u001b]8;id=101485;https://www.comet.com/opik/jacques-comet/experiments/0192ba83-4a5c-779b-bf51-48562b767f33/compare?experiments=%5B%220673a117-287d-7fa0-8000-ff425976bd5e%22%5D\u001b\\in your Opik dashboard\u001b]8;;\u001b\\.\n"
+ "View the results \u001b]8;id=136665;https://www.comet.com/opik/jacques-comet/experiments/0192ba83-4a5c-779b-bf51-48562b767f33/compare?experiments=%5B%2206740816-d7fa-7051-8000-806a033848a8%22%5D\u001b\\in your Opik dashboard\u001b]8;;\u001b\\.\n"
]
},
"metadata": {},
@@ -762,7 +745,7 @@
},
{
"cell_type": "code",
- "execution_count": 9,
+ "execution_count": 23,
"metadata": {},
"outputs": [],
"source": [
@@ -788,133 +771,14 @@
},
{
"cell_type": "code",
- "execution_count": 10,
+ "execution_count": 24,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
- "Evaluation: 11%|█ | 2/18 [00:05<00:41, 2.62s/it]OPIK: Failed to compute metric summary-metrics. Score result will be marked as failed.\n",
- "Traceback (most recent call last):\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages/opik/evaluation/tasks_scorer.py\", line 29, in _score_test_case\n",
- " result = metric.score(**score_kwargs)\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/var/folders/rr/674mwvt12_v0k3qz2z47m8bh0000gn/T/ipykernel_67274/2409766834.py\", line 46, in score\n",
- " eval_dict = json.loads(response.choices[0].message.content)\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/__init__.py\", line 346, in loads\n",
- " return _default_decoder.decode(s)\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py\", line 337, in decode\n",
- " obj, end = self.raw_decode(s, idx=_w(s, 0).end())\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py\", line 355, in raw_decode\n",
- " raise JSONDecodeError(\"Expecting value\", s, err.value) from None\n",
- "json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)\n",
- "Evaluation: 17%|█▋ | 3/18 [00:06<00:27, 1.81s/it]OPIK: Failed to compute metric summary-metrics. Score result will be marked as failed.\n",
- "Traceback (most recent call last):\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages/opik/evaluation/tasks_scorer.py\", line 29, in _score_test_case\n",
- " result = metric.score(**score_kwargs)\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/var/folders/rr/674mwvt12_v0k3qz2z47m8bh0000gn/T/ipykernel_67274/2409766834.py\", line 46, in score\n",
- " eval_dict = json.loads(response.choices[0].message.content)\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/__init__.py\", line 346, in loads\n",
- " return _default_decoder.decode(s)\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py\", line 337, in decode\n",
- " obj, end = self.raw_decode(s, idx=_w(s, 0).end())\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py\", line 355, in raw_decode\n",
- " raise JSONDecodeError(\"Expecting value\", s, err.value) from None\n",
- "json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)\n",
- "Evaluation: 22%|██▏ | 4/18 [00:07<00:17, 1.28s/it]OPIK: Failed to compute metric summary-metrics. Score result will be marked as failed.\n",
- "Traceback (most recent call last):\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages/opik/evaluation/tasks_scorer.py\", line 29, in _score_test_case\n",
- " result = metric.score(**score_kwargs)\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/var/folders/rr/674mwvt12_v0k3qz2z47m8bh0000gn/T/ipykernel_67274/2409766834.py\", line 46, in score\n",
- " eval_dict = json.loads(response.choices[0].message.content)\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/__init__.py\", line 346, in loads\n",
- " return _default_decoder.decode(s)\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py\", line 337, in decode\n",
- " obj, end = self.raw_decode(s, idx=_w(s, 0).end())\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py\", line 355, in raw_decode\n",
- " raise JSONDecodeError(\"Expecting value\", s, err.value) from None\n",
- "json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)\n",
- "Evaluation: 39%|███▉ | 7/18 [00:07<00:05, 1.94it/s]OPIK: Failed to compute metric summary-metrics. Score result will be marked as failed.\n",
- "Traceback (most recent call last):\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages/opik/evaluation/tasks_scorer.py\", line 29, in _score_test_case\n",
- " result = metric.score(**score_kwargs)\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/var/folders/rr/674mwvt12_v0k3qz2z47m8bh0000gn/T/ipykernel_67274/2409766834.py\", line 46, in score\n",
- " eval_dict = json.loads(response.choices[0].message.content)\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/__init__.py\", line 346, in loads\n",
- " return _default_decoder.decode(s)\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py\", line 337, in decode\n",
- " obj, end = self.raw_decode(s, idx=_w(s, 0).end())\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py\", line 355, in raw_decode\n",
- " raise JSONDecodeError(\"Expecting value\", s, err.value) from None\n",
- "json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)\n",
- "Evaluation: 56%|█████▌ | 10/18 [00:10<00:06, 1.22it/s]OPIK: Failed to compute metric summary-metrics. Score result will be marked as failed.\n",
- "Traceback (most recent call last):\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages/opik/evaluation/tasks_scorer.py\", line 29, in _score_test_case\n",
- " result = metric.score(**score_kwargs)\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/var/folders/rr/674mwvt12_v0k3qz2z47m8bh0000gn/T/ipykernel_67274/2409766834.py\", line 46, in score\n",
- " eval_dict = json.loads(response.choices[0].message.content)\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/__init__.py\", line 346, in loads\n",
- " return _default_decoder.decode(s)\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py\", line 337, in decode\n",
- " obj, end = self.raw_decode(s, idx=_w(s, 0).end())\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py\", line 355, in raw_decode\n",
- " raise JSONDecodeError(\"Expecting value\", s, err.value) from None\n",
- "json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)\n",
- "Evaluation: 78%|███████▊ | 14/18 [00:11<00:01, 2.58it/s]OPIK: Failed to compute metric summary-metrics. Score result will be marked as failed.\n",
- "Traceback (most recent call last):\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages/opik/evaluation/tasks_scorer.py\", line 29, in _score_test_case\n",
- " result = metric.score(**score_kwargs)\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/var/folders/rr/674mwvt12_v0k3qz2z47m8bh0000gn/T/ipykernel_67274/2409766834.py\", line 46, in score\n",
- " eval_dict = json.loads(response.choices[0].message.content)\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/__init__.py\", line 346, in loads\n",
- " return _default_decoder.decode(s)\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py\", line 337, in decode\n",
- " obj, end = self.raw_decode(s, idx=_w(s, 0).end())\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py\", line 355, in raw_decode\n",
- " raise JSONDecodeError(\"Expecting value\", s, err.value) from None\n",
- "json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)\n",
- "Evaluation: 83%|████████▎ | 15/18 [00:11<00:01, 2.45it/s]OPIK: Failed to compute metric summary-metrics. Score result will be marked as failed.\n",
- "Traceback (most recent call last):\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages/opik/evaluation/tasks_scorer.py\", line 29, in _score_test_case\n",
- " result = metric.score(**score_kwargs)\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/var/folders/rr/674mwvt12_v0k3qz2z47m8bh0000gn/T/ipykernel_67274/2409766834.py\", line 46, in score\n",
- " eval_dict = json.loads(response.choices[0].message.content)\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/__init__.py\", line 346, in loads\n",
- " return _default_decoder.decode(s)\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py\", line 337, in decode\n",
- " obj, end = self.raw_decode(s, idx=_w(s, 0).end())\n",
- " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
- " File \"/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py\", line 355, in raw_decode\n",
- " raise JSONDecodeError(\"Expecting value\", s, err.value) from None\n",
- "json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)\n",
- "Evaluation: 100%|██████████| 18/18 [00:23<00:00, 1.33s/it]\n"
+ "Evaluation: 100%|██████████| 18/18 [00:24<00:00, 1.37s/it]\n"
]
},
{
@@ -922,14 +786,13 @@
"text/html": [
"╭─ arXiv Papers (18 samples) ──────────────╮\n",
"│ │\n",
- "│ Total time: 00:00:24 │\n",
+ "│ Total time: 00:00:25 │\n",
"│ Number of samples: 18 │\n",
"│ │\n",
- "│ summary_relevance: 1.5455 (avg) │\n",
- "│ summary_conciseness: 4.5455 (avg) │\n",
- "│ summary_technical_accuracy: 1.4545 (avg) │\n",
- "│ summary_average_score: 2.5136 (avg) │\n",
- "│ summary-metrics: None (avg) - 7 failed │\n",
+ "│ summary_relevance: 1.0000 (avg) │\n",
+ "│ summary_conciseness: 2.8333 (avg) │\n",
+ "│ summary_technical_accuracy: 1.0000 (avg) │\n",
+ "│ summary_average_score: 1.6094 (avg) │\n",
"│ │\n",
"╰──────────────────────────────────────────╯\n",
"\n"
@@ -937,14 +800,13 @@
"text/plain": [
"╭─ arXiv Papers (18 samples) ──────────────╮\n",
"│ │\n",
- "│ \u001b[1mTotal time: \u001b[0m 00:00:24 │\n",
+ "│ \u001b[1mTotal time: \u001b[0m 00:00:25 │\n",
"│ \u001b[1mNumber of samples:\u001b[0m 18 │\n",
"│ │\n",
- "│ \u001b[1;32msummary_relevance: 1.5455 (avg)\u001b[0m │\n",
- "│ \u001b[1;32msummary_conciseness: 4.5455 (avg)\u001b[0m │\n",
- "│ \u001b[1;32msummary_technical_accuracy: 1.4545 (avg)\u001b[0m │\n",
- "│ \u001b[1;32msummary_average_score: 2.5136 (avg)\u001b[0m │\n",
- "│ \u001b[1;32msummary-metrics: None (avg)\u001b[0m\u001b[31m - 7 failed\u001b[0m │\n",
+ "│ \u001b[1;32msummary_relevance: 1.0000 (avg)\u001b[0m │\n",
+ "│ \u001b[1;32msummary_conciseness: 2.8333 (avg)\u001b[0m │\n",
+ "│ \u001b[1;32msummary_technical_accuracy: 1.0000 (avg)\u001b[0m │\n",
+ "│ \u001b[1;32msummary_average_score: 1.6094 (avg)\u001b[0m │\n",
"│ │\n",
"╰──────────────────────────────────────────╯\n"
]
@@ -968,11 +830,11 @@
{
"data": {
"text/html": [
- "View the results in your Opik dashboard.\n",
+ "View the results in your Opik dashboard.\n",
"\n"
],
"text/plain": [
- "View the results \u001b]8;id=452780;https://www.comet.com/opik/jacques-comet/experiments/0192ba83-4a5c-779b-bf51-48562b767f33/compare?experiments=%5B%220673a118-c1f0-7bd0-8000-f8cd077598dd%22%5D\u001b\\in your Opik dashboard\u001b]8;;\u001b\\.\n"
+ "View the results \u001b]8;id=766195;https://www.comet.com/opik/jacques-comet/experiments/0192ba83-4a5c-779b-bf51-48562b767f33/compare?experiments=%5B%2206740853-e93a-7fea-8000-c9487836fcbc%22%5D\u001b\\in your Opik dashboard\u001b]8;;\u001b\\.\n"
]
},
"metadata": {},
diff --git a/apps/opik-documentation/documentation/docs/cookbook/quickstart_notebook.md b/apps/opik-documentation/documentation/docs/cookbook/quickstart_notebook.md
index 9d1cab0c80..305e5bb1bc 100644
--- a/apps/opik-documentation/documentation/docs/cookbook/quickstart_notebook.md
+++ b/apps/opik-documentation/documentation/docs/cookbook/quickstart_notebook.md
@@ -10,22 +10,19 @@ We will first install the required dependencies and configure both Opik and Open
```python
-%pip install -U opik openai requests PyPDF2
+%pip install -U opik openai requests PyPDF2 --quiet
```
- Requirement already satisfied: opik in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (1.1.4)
- Collecting opik
- Downloading opik-1.1.6-py3-none-any.whl.metadata (18 kB)
- Requirement already satisfied: openai in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (1.52.1)
+ Requirement already satisfied: opik in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (1.1.10)
+ Requirement already satisfied: openai in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (1.54.4)
Collecting openai
- Downloading openai-1.54.4-py3-none-any.whl.metadata (24 kB)
+ Downloading openai-1.55.0-py3-none-any.whl.metadata (24 kB)
Requirement already satisfied: requests in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (2.32.3)
Requirement already satisfied: PyPDF2 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (3.0.1)
Requirement already satisfied: click in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from opik) (8.1.7)
Requirement already satisfied: httpx<1.0.0 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from opik) (0.27.2)
Requirement already satisfied: levenshtein~=0.25.1 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from opik) (0.25.1)
Requirement already satisfied: litellm in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from opik) (1.51.2)
- Requirement already satisfied: pandas<3.0.0,>=2.0.0 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from opik) (2.2.2)
Requirement already satisfied: pydantic-settings<3.0.0,>=2.0.0 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from opik) (2.4.0)
Requirement already satisfied: pydantic<3.0.0,>=2.0.0 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from opik) (2.8.2)
Requirement already satisfied: pytest in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from opik) (8.3.2)
@@ -44,10 +41,6 @@ We will first install the required dependencies and configure both Opik and Open
Requirement already satisfied: httpcore==1.* in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from httpx<1.0.0->opik) (1.0.5)
Requirement already satisfied: h11<0.15,>=0.13 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from httpcore==1.*->httpx<1.0.0->opik) (0.14.0)
Requirement already satisfied: rapidfuzz<4.0.0,>=3.8.0 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from levenshtein~=0.25.1->opik) (3.9.6)
- Requirement already satisfied: numpy>=1.26.0 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from pandas<3.0.0,>=2.0.0->opik) (1.26.4)
- Requirement already satisfied: python-dateutil>=2.8.2 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from pandas<3.0.0,>=2.0.0->opik) (2.9.0)
- Requirement already satisfied: pytz>=2020.1 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from pandas<3.0.0,>=2.0.0->opik) (2024.1)
- Requirement already satisfied: tzdata>=2022.7 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from pandas<3.0.0,>=2.0.0->opik) (2024.1)
Requirement already satisfied: annotated-types>=0.4.0 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.0.0->opik) (0.7.0)
Requirement already satisfied: pydantic-core==2.20.1 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.0.0->opik) (2.20.1)
Requirement already satisfied: python-dotenv>=0.21.0 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from pydantic-settings<3.0.0,>=2.0.0->opik) (1.0.1)
@@ -69,7 +62,6 @@ We will first install the required dependencies and configure both Opik and Open
Requirement already satisfied: referencing>=0.28.4 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm->opik) (0.35.1)
Requirement already satisfied: rpds-py>=0.7.1 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm->opik) (0.20.0)
Requirement already satisfied: mdurl~=0.1 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from markdown-it-py>=2.2.0->rich->opik) (0.1.2)
- Requirement already satisfied: six>=1.5 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from python-dateutil>=2.8.2->pandas<3.0.0,>=2.0.0->opik) (1.16.0)
Requirement already satisfied: regex>=2022.1.18 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from tiktoken>=0.7.0->litellm->opik) (2024.7.24)
Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from aiohttp->litellm->opik) (2.3.5)
Requirement already satisfied: aiosignal>=1.1.2 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from aiohttp->litellm->opik) (1.3.1)
@@ -81,20 +73,15 @@ We will first install the required dependencies and configure both Opik and Open
Requirement already satisfied: fsspec>=2023.5.0 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from huggingface-hub<1.0,>=0.16.4->tokenizers->litellm->opik) (2024.10.0)
Requirement already satisfied: pyyaml>=5.1 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from huggingface-hub<1.0,>=0.16.4->tokenizers->litellm->opik) (6.0.2)
Requirement already satisfied: propcache>=0.2.0 in /opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages (from yarl<2.0,>=1.12.0->aiohttp->litellm->opik) (0.2.0)
- Downloading opik-1.1.6-py3-none-any.whl (240 kB)
- Downloading openai-1.54.4-py3-none-any.whl (389 kB)
- Installing collected packages: openai, opik
+ Downloading openai-1.55.0-py3-none-any.whl (389 kB)
+ Installing collected packages: openai
Attempting uninstall: openai
- Found existing installation: openai 1.52.1
- Uninstalling openai-1.52.1:
- Successfully uninstalled openai-1.52.1
- Attempting uninstall: opik
- Found existing installation: opik 1.1.4
- Uninstalling opik-1.1.4:
- Successfully uninstalled opik-1.1.4
+ Found existing installation: openai 1.54.4
+ Uninstalling openai-1.54.4:
+ Successfully uninstalled openai-1.54.4
[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.
predibase 2024.9.3 requires urllib3==1.26.12, but you have urllib3 2.2.3 which is incompatible.[0m[31m
- [0mSuccessfully installed openai-1.54.4 opik-1.1.6
+ [0mSuccessfully installed openai-1.55.0
[1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m A new release of pip is available: [0m[31;49m24.2[0m[39;49m -> [0m[32;49m24.3.1[0m
[1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m To update, run: [0m[32;49mpip install --upgrade pip[0m
@@ -261,8 +248,8 @@ print("\n".join(textwrap.wrap(summary, width=80)))
OPIK: Started logging traces to the "Default Project" project at https://www.comet.com/opik/jacques-comet/redirect/projects?name=Default%20Project.
- Sure! Please provide the current summary and the instruction you'd like me to
- focus on, and I'll create the concise final summary for you.
+ Please provide the current summary and the specific instruction you would like
+ me to focus on for the final summary.
Thanks to the `@opik.track` decorator and Opik's integration with OpenAI, we can now track the entire chain and all the LLM calls in the Opik UI:
@@ -343,6 +330,68 @@ Opik includes a [library of evaluation metrics](https://www.comet.com/docs/opik/
from opik.evaluation.metrics import base_metric, score_result
import json
+# We will define the response format so the output has the correct schema. You can also use structured outputs with Pydantic models for this.
+json_schema = {
+ "type": "json_schema",
+ "json_schema": {
+ "name": "summary_evaluation_schema",
+ "schema": {
+ "type": "object",
+ "properties": {
+ "relevance": {
+ "type": "object",
+ "properties": {
+ "score": {
+ "type": "integer",
+ "minimum": 1,
+ "maximum": 5,
+ "description": "Score between 1-5 for how well the summary addresses the instruction"
+ },
+ "explanation": {
+ "type": "string",
+ "description": "Brief explanation of the relevance score"
+ }
+ },
+ "required": ["score", "explanation"]
+ },
+ "conciseness": {
+ "type": "object",
+ "properties": {
+ "score": {
+ "type": "integer",
+ "minimum": 1,
+ "maximum": 5,
+ "description": "Score between 1-5 for how concise the summary is while retaining key information"
+ },
+ "explanation": {
+ "type": "string",
+ "description": "Brief explanation of the conciseness score"
+ }
+ },
+ "required": ["score", "explanation"]
+ },
+ "technical_accuracy": {
+ "type": "object",
+ "properties": {
+ "score": {
+ "type": "integer",
+ "minimum": 1,
+ "maximum": 5,
+ "description": "Score between 1-5 for how accurately the summary conveys technical details"
+ },
+ "explanation": {
+ "type": "string",
+ "description": "Brief explanation of the technical accuracy score"
+ }
+ },
+ "required": ["score", "explanation"]
+ }
+ },
+ "required": ["relevance", "conciseness", "technical_accuracy"],
+ "additionalProperties": False
+ }
+ }
+}
# Custom Metric: One template/prompt to extract 4 scores/results
class EvaluateSummary(base_metric.BaseMetric):
@@ -382,7 +431,7 @@ class EvaluateSummary(base_metric.BaseMetric):
"""
response = openai_client.chat.completions.create(
- model=model, max_tokens=1000, messages=[{"role": "user", "content": prompt}]
+ model=model, max_tokens=1000, messages=[{"role": "user", "content": prompt}], response_format=json_schema
)
eval_dict = json.loads(response.choices[0].message.content)
@@ -456,7 +505,7 @@ def evaluation_task(x: Dict):
density_iterations=density_iterations,
)
- return {"summary": result, "instruction": instruction}
+ return {"summary": result}
```
### Run the automated evaluation
@@ -488,88 +537,19 @@ res = evaluate(
)
```
- Evaluation: 11%|█ | 2/18 [00:09<01:00, 3.77s/it]OPIK: Failed to compute metric summary-metrics. Score result will be marked as failed.
- Traceback (most recent call last):
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages/opik/evaluation/tasks_scorer.py", line 29, in _score_test_case
- result = metric.score(**score_kwargs)
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/var/folders/rr/674mwvt12_v0k3qz2z47m8bh0000gn/T/ipykernel_67274/2409766834.py", line 46, in score
- eval_dict = json.loads(response.choices[0].message.content)
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/__init__.py", line 346, in loads
- return _default_decoder.decode(s)
- ^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py", line 337, in decode
- obj, end = self.raw_decode(s, idx=_w(s, 0).end())
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py", line 355, in raw_decode
- raise JSONDecodeError("Expecting value", s, err.value) from None
- json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
- Evaluation: 22%|██▏ | 4/18 [00:09<00:21, 1.50s/it]OPIK: Failed to compute metric summary-metrics. Score result will be marked as failed.
- Traceback (most recent call last):
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages/opik/evaluation/tasks_scorer.py", line 29, in _score_test_case
- result = metric.score(**score_kwargs)
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/var/folders/rr/674mwvt12_v0k3qz2z47m8bh0000gn/T/ipykernel_67274/2409766834.py", line 46, in score
- eval_dict = json.loads(response.choices[0].message.content)
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/__init__.py", line 346, in loads
- return _default_decoder.decode(s)
- ^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py", line 337, in decode
- obj, end = self.raw_decode(s, idx=_w(s, 0).end())
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py", line 355, in raw_decode
- raise JSONDecodeError("Expecting value", s, err.value) from None
- json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
- Evaluation: 61%|██████ | 11/18 [00:12<00:03, 1.76it/s]OPIK: Failed to compute metric summary-metrics. Score result will be marked as failed.
- Traceback (most recent call last):
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages/opik/evaluation/tasks_scorer.py", line 29, in _score_test_case
- result = metric.score(**score_kwargs)
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/var/folders/rr/674mwvt12_v0k3qz2z47m8bh0000gn/T/ipykernel_67274/2409766834.py", line 46, in score
- eval_dict = json.loads(response.choices[0].message.content)
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/__init__.py", line 346, in loads
- return _default_decoder.decode(s)
- ^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py", line 337, in decode
- obj, end = self.raw_decode(s, idx=_w(s, 0).end())
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py", line 355, in raw_decode
- raise JSONDecodeError("Expecting value", s, err.value) from None
- json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
- Evaluation: 78%|███████▊ | 14/18 [00:13<00:01, 2.06it/s]OPIK: Failed to compute metric summary-metrics. Score result will be marked as failed.
- Traceback (most recent call last):
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages/opik/evaluation/tasks_scorer.py", line 29, in _score_test_case
- result = metric.score(**score_kwargs)
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/var/folders/rr/674mwvt12_v0k3qz2z47m8bh0000gn/T/ipykernel_67274/2409766834.py", line 46, in score
- eval_dict = json.loads(response.choices[0].message.content)
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/__init__.py", line 346, in loads
- return _default_decoder.decode(s)
- ^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py", line 337, in decode
- obj, end = self.raw_decode(s, idx=_w(s, 0).end())
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py", line 355, in raw_decode
- raise JSONDecodeError("Expecting value", s, err.value) from None
- json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
- Evaluation: 100%|██████████| 18/18 [00:31<00:00, 1.73s/it]
+ Evaluation: 100%|██████████| 18/18 [00:25<00:00, 1.44s/it]
╭─ arXiv Papers (18 samples) ──────────────╮
│ │
-│ Total time: 00:00:31 │
+│ Total time: 00:00:26 │
│ Number of samples: 18 │
│ │
-│ summary_relevance: 2.2857 (avg) │
-│ summary_conciseness: 3.8571 (avg) │
-│ summary_technical_accuracy: 1.7143 (avg) │
-│ summary_average_score: 2.6179 (avg) │
-│ summary-metrics: None (avg) - 4 failed │
+│ summary_relevance: 1.1111 (avg) │
+│ summary_conciseness: 3.0000 (avg) │
+│ summary_technical_accuracy: 1.0000 (avg) │
+│ summary_average_score: 1.7028 (avg) │
│ │
╰──────────────────────────────────────────╯
@@ -583,7 +563,7 @@ res = evaluate(
-View the results in your Opik dashboard.
+View the results in your Opik dashboard.
@@ -644,139 +624,19 @@ res = evaluate(
)
```
- Evaluation: 11%|█ | 2/18 [00:05<00:41, 2.62s/it]OPIK: Failed to compute metric summary-metrics. Score result will be marked as failed.
- Traceback (most recent call last):
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages/opik/evaluation/tasks_scorer.py", line 29, in _score_test_case
- result = metric.score(**score_kwargs)
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/var/folders/rr/674mwvt12_v0k3qz2z47m8bh0000gn/T/ipykernel_67274/2409766834.py", line 46, in score
- eval_dict = json.loads(response.choices[0].message.content)
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/__init__.py", line 346, in loads
- return _default_decoder.decode(s)
- ^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py", line 337, in decode
- obj, end = self.raw_decode(s, idx=_w(s, 0).end())
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py", line 355, in raw_decode
- raise JSONDecodeError("Expecting value", s, err.value) from None
- json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
- Evaluation: 17%|█▋ | 3/18 [00:06<00:27, 1.81s/it]OPIK: Failed to compute metric summary-metrics. Score result will be marked as failed.
- Traceback (most recent call last):
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages/opik/evaluation/tasks_scorer.py", line 29, in _score_test_case
- result = metric.score(**score_kwargs)
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/var/folders/rr/674mwvt12_v0k3qz2z47m8bh0000gn/T/ipykernel_67274/2409766834.py", line 46, in score
- eval_dict = json.loads(response.choices[0].message.content)
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/__init__.py", line 346, in loads
- return _default_decoder.decode(s)
- ^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py", line 337, in decode
- obj, end = self.raw_decode(s, idx=_w(s, 0).end())
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py", line 355, in raw_decode
- raise JSONDecodeError("Expecting value", s, err.value) from None
- json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
- Evaluation: 22%|██▏ | 4/18 [00:07<00:17, 1.28s/it]OPIK: Failed to compute metric summary-metrics. Score result will be marked as failed.
- Traceback (most recent call last):
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages/opik/evaluation/tasks_scorer.py", line 29, in _score_test_case
- result = metric.score(**score_kwargs)
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/var/folders/rr/674mwvt12_v0k3qz2z47m8bh0000gn/T/ipykernel_67274/2409766834.py", line 46, in score
- eval_dict = json.loads(response.choices[0].message.content)
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/__init__.py", line 346, in loads
- return _default_decoder.decode(s)
- ^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py", line 337, in decode
- obj, end = self.raw_decode(s, idx=_w(s, 0).end())
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py", line 355, in raw_decode
- raise JSONDecodeError("Expecting value", s, err.value) from None
- json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
- Evaluation: 39%|███▉ | 7/18 [00:07<00:05, 1.94it/s]OPIK: Failed to compute metric summary-metrics. Score result will be marked as failed.
- Traceback (most recent call last):
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages/opik/evaluation/tasks_scorer.py", line 29, in _score_test_case
- result = metric.score(**score_kwargs)
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/var/folders/rr/674mwvt12_v0k3qz2z47m8bh0000gn/T/ipykernel_67274/2409766834.py", line 46, in score
- eval_dict = json.loads(response.choices[0].message.content)
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/__init__.py", line 346, in loads
- return _default_decoder.decode(s)
- ^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py", line 337, in decode
- obj, end = self.raw_decode(s, idx=_w(s, 0).end())
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py", line 355, in raw_decode
- raise JSONDecodeError("Expecting value", s, err.value) from None
- json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
- Evaluation: 56%|█████▌ | 10/18 [00:10<00:06, 1.22it/s]OPIK: Failed to compute metric summary-metrics. Score result will be marked as failed.
- Traceback (most recent call last):
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages/opik/evaluation/tasks_scorer.py", line 29, in _score_test_case
- result = metric.score(**score_kwargs)
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/var/folders/rr/674mwvt12_v0k3qz2z47m8bh0000gn/T/ipykernel_67274/2409766834.py", line 46, in score
- eval_dict = json.loads(response.choices[0].message.content)
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/__init__.py", line 346, in loads
- return _default_decoder.decode(s)
- ^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py", line 337, in decode
- obj, end = self.raw_decode(s, idx=_w(s, 0).end())
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py", line 355, in raw_decode
- raise JSONDecodeError("Expecting value", s, err.value) from None
- json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
- Evaluation: 78%|███████▊ | 14/18 [00:11<00:01, 2.58it/s]OPIK: Failed to compute metric summary-metrics. Score result will be marked as failed.
- Traceback (most recent call last):
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages/opik/evaluation/tasks_scorer.py", line 29, in _score_test_case
- result = metric.score(**score_kwargs)
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/var/folders/rr/674mwvt12_v0k3qz2z47m8bh0000gn/T/ipykernel_67274/2409766834.py", line 46, in score
- eval_dict = json.loads(response.choices[0].message.content)
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/__init__.py", line 346, in loads
- return _default_decoder.decode(s)
- ^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py", line 337, in decode
- obj, end = self.raw_decode(s, idx=_w(s, 0).end())
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py", line 355, in raw_decode
- raise JSONDecodeError("Expecting value", s, err.value) from None
- json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
- Evaluation: 83%|████████▎ | 15/18 [00:11<00:01, 2.45it/s]OPIK: Failed to compute metric summary-metrics. Score result will be marked as failed.
- Traceback (most recent call last):
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/site-packages/opik/evaluation/tasks_scorer.py", line 29, in _score_test_case
- result = metric.score(**score_kwargs)
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/var/folders/rr/674mwvt12_v0k3qz2z47m8bh0000gn/T/ipykernel_67274/2409766834.py", line 46, in score
- eval_dict = json.loads(response.choices[0].message.content)
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/__init__.py", line 346, in loads
- return _default_decoder.decode(s)
- ^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py", line 337, in decode
- obj, end = self.raw_decode(s, idx=_w(s, 0).end())
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- File "/opt/homebrew/Caskroom/miniconda/base/envs/py312_llm_eval/lib/python3.12/json/decoder.py", line 355, in raw_decode
- raise JSONDecodeError("Expecting value", s, err.value) from None
- json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
- Evaluation: 100%|██████████| 18/18 [00:23<00:00, 1.33s/it]
+ Evaluation: 100%|██████████| 18/18 [00:24<00:00, 1.37s/it]
╭─ arXiv Papers (18 samples) ──────────────╮
│ │
-│ Total time: 00:00:24 │
+│ Total time: 00:00:25 │
│ Number of samples: 18 │
│ │
-│ summary_relevance: 1.5455 (avg) │
-│ summary_conciseness: 4.5455 (avg) │
-│ summary_technical_accuracy: 1.4545 (avg) │
-│ summary_average_score: 2.5136 (avg) │
-│ summary-metrics: None (avg) - 7 failed │
+│ summary_relevance: 1.0000 (avg) │
+│ summary_conciseness: 2.8333 (avg) │
+│ summary_technical_accuracy: 1.0000 (avg) │
+│ summary_average_score: 1.6094 (avg) │
│ │
╰──────────────────────────────────────────╯
@@ -790,7 +650,7 @@ res = evaluate(
-View the results in your Opik dashboard.
+View the results in your Opik dashboard.
diff --git a/apps/opik-documentation/documentation/docs/evaluation/evaluate_your_llm.md b/apps/opik-documentation/documentation/docs/evaluation/evaluate_your_llm.md
index 4466adad91..5cb3d5e858 100644
--- a/apps/opik-documentation/documentation/docs/evaluation/evaluate_your_llm.md
+++ b/apps/opik-documentation/documentation/docs/evaluation/evaluate_your_llm.md
@@ -35,10 +35,6 @@ def your_llm_application(input: str) -> str:
)
return response.choices[0].message.content
-
-@track
-def your_context_retriever(input: str) -> str:
- return ["..."]
```
:::tip
@@ -52,9 +48,7 @@ Once you have added instrumentation to your LLM application, we can define the e
```python
def evaluation_task(x):
return {
- "input": x['user_question'],
- "output": your_llm_application(x['user_question']),
- "context": your_context_retriever(x['user_question'])
+ "output": your_llm_application(x['user_question'])
}
```
@@ -66,26 +60,26 @@ If the dictionary returned does not match with the parameters expected by the me
In order to create an evaluation experiment, you will need to have a Dataset that includes all your test cases.
-If you have already created a Dataset, you can use the `Opik.get_dataset` function to fetch it:
+If you have already created a Dataset, you can use the [`Opik.get_or_create_dataset`](https://www.comet.com/docs/opik/python-sdk-reference/Opik.html#opik.Opik.get_or_create_dataset) function to fetch it:
```python
from opik import Opik
client = Opik()
-dataset = client.get_dataset(name="your-dataset-name")
+dataset = client.get_or_create_dataset(name="Example dataset")
```
-If you don't have a Dataset yet, you can create one using the `Opik.create_dataset` function:
+If you don't have a Dataset yet, you can insert dataset items using the [`Dataset.insert`](https://www.comet.com/docs/opik/python-sdk-reference/evaluation/Dataset.html#opik.Dataset.insert) method. You can call this method multiple times as Opik performs data deplication before ingestion:
```python
from opik import Opik
client = Opik()
-dataset = client.create_dataset(name="your-dataset-name")
+dataset = client.get_or_create_dataset(name="Example dataset")
dataset.insert([
- {"user_question": "Hello, world!", "expected_output": "Hello, world!"},
- {"user_question": "What is the capital of France?", "expected_output": "Paris"},
+ {"input": "Hello, world!", "expected_output": "Hello, world!"},
+ {"input": "What is the capital of France?", "expected_output": "Paris"},
])
```
@@ -99,9 +93,8 @@ Opik provides a set of built-in evaluation metrics that you can choose from. The
In the same evaluation experiment, you can use multiple metrics to evaluate your application:
```python
-from opik.evaluation.metrics import Equals, Hallucination
+from opik.evaluation.metrics import Hallucination
-equals_metric = Equals()
hallucination_metric = Hallucination()
```
@@ -131,37 +124,25 @@ def your_llm_application(input: str) -> str:
model=MODEL,
messages=[{"role": "user", "content": input}],
)
-
return response.choices[0].message.content
# Define the evaluation task
def evaluation_task(x):
return {
- "input": x['user_question'],
- "output": your_llm_application(x['user_question']),
- "context": your_context_retriever(x['user_question'])
+ "output": your_llm_application(x['user_question'])
}
-@track
-def your_context_retriever(input: str) -> str:
- return ["..."]
-
-
# Create a simple dataset
client = Opik()
-try:
- dataset = client.create_dataset(name="your-dataset-name")
- dataset.insert([
- {"input": {"user_question": "What is the capital of France?"}},
- {"input": {"user_question": "What is the capital of Germany?"}},
- ])
-except:
- dataset = client.get_dataset(name="your-dataset-name")
+dataset = client.get_or_create_dataset(name="Example dataset")
+dataset.insert([
+ {"input": "What is the capital of France?"},
+ {"input": "What is the capital of Germany?"},
+])
# Define the metrics
hallucination_metric = Hallucination()
-
evaluation = evaluate(
experiment_name="My experiment",
dataset=dataset,
@@ -179,9 +160,24 @@ You can use the `experiment_config` parameter to store information about your ev
## Advanced usage
+### Missing arguments for scoring methods
+
+When you face the `opik.exceptions.ScoreMethodMissingArguments` exception, it means that the dataset item and task output dictionaries do not contain all the arguments expected by the scoring method. The way the evaluate function works is by merging the dataset item and task output dictionaries and then passing the result to the scoring method. For example, if the dataset item contains the keys `user_question` and `context` while the evaluation task returns a dictionary with the key `output`, the scoring method will be called as `scoring_method.score(user_question='...', context= '...', output= '...')`. This can be an issue if the scoring method expects a different set of arguments.
+
+You can solve this by either updating the dataset item or evaluation task to return the missing arguments or by using the `scoring_key_mapping` parameter of the `evaluate` function. In the example above, if the scoring method expects `input` as an argument, you can map the `user_question` key to the `input` key as follows:
+
+```python
+evaluation = evaluate(
+ dataset=dataset,
+ task=evaluation_task,
+ scoring_metrics=[hallucination_metric],
+ scoring_key_mapping={"input": "user_question"},
+)
+```
+
### Linking prompts to experiments
-The [Opik prompt library](/library/prompt_management.md) can be used to version your prompt templates.
+The [Opik prompt library](/library/prompt_management.mdx) can be used to version your prompt templates.
When creating an Experiment, you can link the Experiment to a specific prompt version:
diff --git a/apps/opik-documentation/documentation/docs/reference/rest_api/add-span-feedback-score.api.mdx b/apps/opik-documentation/documentation/docs/reference/rest_api/add-span-feedback-score.api.mdx
index 1cf27c7f78..d82d314986 100644
--- a/apps/opik-documentation/documentation/docs/reference/rest_api/add-span-feedback-score.api.mdx
+++ b/apps/opik-documentation/documentation/docs/reference/rest_api/add-span-feedback-score.api.mdx
@@ -12,18 +12,13 @@ custom_edit_url: null
hide_send_button: true
---
-import ApiTabs from "@theme/ApiTabs";
-import DiscriminatorTabs from "@theme/DiscriminatorTabs";
import MethodEndpoint from "@theme/ApiExplorer/MethodEndpoint";
-import SecuritySchemes from "@theme/ApiExplorer/SecuritySchemes";
-import MimeTabs from "@theme/MimeTabs";
-import ParamsItem from "@theme/ParamsItem";
-import ResponseSamples from "@theme/ResponseSamples";
-import SchemaItem from "@theme/SchemaItem";
-import SchemaTabs from "@theme/SchemaTabs";
-import Heading from "@theme/Heading";
+import ParamsDetails from "@theme/ParamsDetails";
+import RequestSchema from "@theme/RequestSchema";
+import StatusCodes from "@theme/StatusCodes";
import OperationTabs from "@theme/OperationTabs";
import TabItem from "@theme/TabItem";
+import Heading from "@theme/Heading";
@@ -51,84 +47,26 @@ Add span feedback score
>
-
+
+
+
+
-
-
- Path Parameters
-
-
-
+
+
-
-
-
-
- Body
-
-
-
-
-
-
-
-
-
-
-
-
-
- No Content
-
-
-
-
-
-
-
-
-
+
+
+
+
-
-
- Path Parameters
-
-
-
+
+
-
-
-
-
- Body
-
-
-
-
-
-
-
-
-
-
-
-
-
- No Content
-
-
-
-
-
-
-
-
-
-
-
- Body
-
-
-
-
-
-
-
-
-
-
-
-
-
- Created
-
-
-
-
-
- Response Headers
-
-
- -
-
-
- Location
-
- string
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Body
-
-
-
-
-
-
-
-
-
- experiment_items
-
- object[]
-
-
-
- required
-
-
-
-
-
-
- **Possible values:** `>= 1`, `<= 1000`
-
-
-
-
-
- Array [
-
-
-
-
-
-
-
-
-
-
-
-
-
- -
-
- ]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- No content
-
-
-
-
-
-
-
-
-
-
-
- Body
-
-
-
-
-
-
-
-
-
-
-
-
-
- Created
-
-
-
-
-
- Response Headers
-
-
- -
-
-
- Location
-
- string
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Body
-
-
-
-
-
-
-
-
-
-
-
-
- type
-
- string
-
- required
-
-
-
-
- **Possible values:** [`numerical`, `categorical`]
-
-
-
-
-
-
-
-
-
- details
-
- object
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- details
-
- object
-
-
-
-
-
-
-
-
- categories
-
- object
-
-
-
- required
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Created
-
-
-
-
-
- Response Headers
-
-
- -
-
-
- Location
-
- string
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Body
-
-
-
-
-
-
-
-
-
-
-
-
-
- items
-
- object[]
-
-
-
- required
-
-
-
-
-
-
- **Possible values:** `>= 1`, `<= 1000`
-
-
-
-
-
- Array [
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- -
-
- ]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- No content
-
-
-
-
-
-
-
-
-
-
-
- Body
-
-
-
-
-
-
-
-
-
-
-
-
-
- Created
-
-
-
-
-
- Response Headers
-
-
- -
-
-
- Location
-
- string
-
-
-
-
-
-
-
-
-
-
-
-
-
- Bad Request
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Unprocessable Content
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Body
-
-
-
-
-
-
-
-
-
-
-
- version
-
- object
-
-
-
- required
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- OK
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Bad Request
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Conflict
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Unprocessable Content
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Body
-
-
-
-
-
-
-
-
-
-
-
-
-
- Created
-
-
-
-
-
- Response Headers
-
-
- -
-
-
- Location
-
- string
-
-
-
-
-
-
-
-
-
-
-
-
-
- Bad Request
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Conflict
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Unprocessable Content
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Body
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- usage
-
- object
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Created
-
-
-
-
-
- Response Headers
-
-
- -
-
-
- Location
-
- string
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Body
-
-
-
-
-
-
-
-
-
- spans
-
- object[]
-
-
-
- required
-
-
-
-
-
-
- **Possible values:** `>= 1`, `<= 1000`
-
-
-
-
-
- Array [
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- usage
-
- object
-
-
-
-
-
-
-
-
- -
-
- ]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- No Content
-
-
-
-
-
-
-
-
-
-
-
- Body
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Created
-
-
-
-
-
- Response Headers
-
-
- -
-
-
- Location
-
- string
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Body
-
-
-
-
-
-
-
-
-
- traces
-
- object[]
-
-
-
- required
-
-
-
-
-
-
- **Possible values:** `>= 1`, `<= 1000`
-
-
-
-
-
- Array [
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- -
-
- ]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- No Content
-
-
-
-
-
-
-
-
-
-
-
- Body
-
-
-
-
-
-
-
-
-
-
-
-
-
- No content
-
-
-
-
-
-
-
-
-
-
-
- Body
-
-
-
-
- = 1`, `<= 1000`"}
- schema={{"maxItems":1000,"minItems":1,"type":"array","items":{"type":"string","format":"uuid"}}}
- >
-
-
-
-
-
-
-
-
-
-
-
-
- No content
-
-
-
-
-
-
-
-
-
+
+
+
+
-
-
- Path Parameters
-
-
-
-
-
-
-
-
-
- No content
-
-
-
-
-
-
-
-
-
-
-
- Body
-
-
-
-
- = 1`, `<= 1000`"}
- schema={{"maxItems":1000,"minItems":1,"uniqueItems":true,"type":"array","items":{"type":"string","format":"uuid"}}}
- >
-
-
-
-
-
-
-
-
-
-
-
-
- No content
-
-
-
-
-
-
-
-
-
-
-
- Body
-
-
-
-
- = 1`, `<= 1000`"}
- schema={{"maxItems":1000,"minItems":1,"uniqueItems":true,"type":"array","items":{"type":"string","format":"uuid"}}}
- >
-
-
-
-
-
-
-
-
-
-
-
-
- No content
-
-
-
-
-
-
-
-
-
+
+
+
+
-
-
- Path Parameters
-
-
-
-
-
-
-
-
-
- No Content
-
-
-
-
-
-
-
-
-
+
+
+
+
-
-
- Path Parameters
-
-
-
-
-
-
-
-
-
- No Content
-
-
-
-
-
-
-
-
-
- Conflict
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
-
-
- Path Parameters
-
-
-
-
-
-
-
-
-
- No content
-
-
-
-
-
-
-
-
-
+
+
+
+
-
-
- Path Parameters
-
-
-
-
-
-
-
-
-
- No Content
-
-
-
-
-
-
-
-
-
- Not implemented
-
-
-
-
-
-
-
-
-
+
+
+
+
-
-
- Path Parameters
-
-
-
+
+
-
-
-
-
- Body
-
-
-
-
-
-
-
-
-
-
-
-
-
- No Content
-
-
-
-
-
-
-
-
-
+
+
+
+
-
-
- Path Parameters
-
-
-
-
-
-
-
-
-
- No Content
-
-
-
-
-
-
-
-
-
+
+
+
+
-
-
- Path Parameters
-
-
-
+
+
-
-
-
-
- Body
-
-
-
-
-
-
-
-
-
-
-
-
-
- No Content
-
-
-
-
-
-
-
-
-
-
-
- Body
-
-
-
-
- = 1`, `<= 1000`"}
- schema={{"maxItems":1000,"minItems":1,"uniqueItems":true,"type":"array","items":{"type":"string","format":"uuid"}}}
- >
-
-
-
-
-
-
-
-
-
-
-
-
- No Content
-
-
-
-
-
-
-
-
-
+
+
+
+
-
-
- Path Parameters
-
-
-
+
+
-
-
- Query Parameters
-
-
-
-
-
-
-
-
-
- Dataset item resource
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
- content
-
- object[]
-
-
-
-
-
-
- Array [
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- experiment_items
-
- object[]
-
-
-
-
-
-
- Array [
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- feedback_scores
-
- object[]
-
-
-
-
-
-
- Array [
-
-
-
-
-
- = -1000000000` and `<= 1000000000`"}
- schema={{"maximum":1000000000,"exclusiveMaximum":false,"minimum":-1000000000,"exclusiveMinimum":false,"type":"number"}}
- >
-
-
-
-
-
-
-
-
-
-
-
-
-
- -
-
- ]
-
-
-
-
-
-
-
-
-
-
-
- -
-
- ]
-
-
-
-
-
-
-
-
-
-
-
- -
-
- ]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- columns
-
- object[]
-
-
-
-
-
-
- Array [
-
-
-
-
-
- -
-
- ]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
-
-
- Query Parameters
-
-
-
-
-
-
-
-
-
- Dataset resource
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
- content
-
- object[]
-
-
-
-
-
-
- Array [
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- -
-
- ]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
-
-
- Query Parameters
-
-
-
-
-
-
-
-
-
- Experiments resource
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- content
-
- object[]
-
-
-
-
-
-
- Array [
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- feedback_scores
-
- object[]
-
-
-
-
-
-
- Array [
-
-
-
-
-
- -
-
- ]
-
-
-
-
-
-
-
-
-
-
-
-
-
- -
-
- ]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
-
-
- Query Parameters
-
-
-
-
-
-
-
-
-
- Feedback definitions resource
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- content
-
- object[]
-
-
-
-
-
-
- Array [
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- type
-
- string
-
- required
-
-
-
-
- **Possible values:** [`numerical`, `categorical`]
-
-
-
-
-
-
-
-
-
-
-
-
-
- details
-
- object
-
-
-
- required
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- type
-
- string
-
- required
-
-
-
-
- **Possible values:** [`numerical`, `categorical`]
-
-
-
-
-
-
-
- circular(NumericalFeedbackDefinition_Public)
-
-
-
-
-
-
-
-
-
- details
-
- object
-
-
-
-
-
-
-
-
- categories
-
- object
-
-
-
- required
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- details
-
- object
-
-
-
- required
-
-
-
-
-
-
-
-
- categories
-
- object
-
-
-
- required
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- type
-
- string
-
- required
-
-
-
-
- **Possible values:** [`numerical`, `categorical`]
-
-
-
-
-
-
-
-
-
- details
-
- object
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- circular(CategoricalFeedbackDefinition_Public)
-
-
-
-
-
-
-
-
-
-
-
- ]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
-
-
- Query Parameters
-
-
-
-
-
-
-
-
-
- Project resource
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- content
-
- object[]
-
-
-
-
-
-
- Array [
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- -
-
- ]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
-
-
- Path Parameters
-
-
-
-
-
-
-
-
-
- Dataset resource
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Body
-
-
-
-
-
-
-
-
-
-
-
-
-
- Dataset resource
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
-
-
- Path Parameters
-
-
-
-
-
-
-
-
-
- Dataset item resource
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- experiment_items
-
- object[]
-
-
-
-
-
-
- Array [
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- -
-
- ]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
-
-
- Path Parameters
-
-
-
+
+
-
-
- Query Parameters
-
-
-
-
-
-
-
-
-
- Dataset items resource
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
- content
-
- object[]
-
-
-
-
-
-
- Array [
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- experiment_items
-
- object[]
-
-
-
-
-
-
- Array [
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- -
-
- ]
-
-
-
-
-
-
-
-
-
-
-
- -
-
- ]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- columns
-
- object[]
-
-
-
-
-
-
- Array [
-
-
-
-
-
- -
-
- ]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
-
-
- Path Parameters
-
-
-
-
-
-
-
-
-
- Experiment resource
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- feedback_scores
-
- object[]
-
-
-
-
-
-
- Array [
-
-
-
-
-
- -
-
- ]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Not found
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
-
-
- Path Parameters
-
-
-
-
-
-
-
-
-
- Experiment item resource
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Not found
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
-
-
- Path Parameters
-
-
-
-
-
-
-
-
-
- Feedback definition resource
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- type
-
- string
-
- required
-
-
-
-
- **Possible values:** [`numerical`, `categorical`]
-
-
-
-
-
-
-
-
-
- details
-
- object
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- details
-
- object
-
-
-
-
-
-
-
-
- categories
-
- object
-
-
-
- required
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
-
-
- Path Parameters
-
-
-
-
-
-
-
-
-
- Project resource
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
-
-
- Path Parameters
-
-
-
-
-
-
-
-
-
- Prompt resource
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- latest_version
-
- object
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Not Found
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
-
-
- Path Parameters
-
-
-
-
-
-
-
-
-
- Prompt version resource
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Not Found
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
-
-
- Path Parameters
-
-
-
+
+
-
-
- Query Parameters
-
-
-
-
-
-
-
-
-
- OK
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- content
-
- object[]
-
-
-
-
-
-
- Array [
-
-
-
-
-
-
-
-
-
-
-
-
-
- -
-
- ]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
-
-
- Query Parameters
-
-
-
-
-
-
-
-
-
- OK
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- content
-
- object[]
-
-
-
-
-
-
- Array [
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- -
-
- ]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
-
-
- Path Parameters
-
-
-
-
-
-
-
-
-
- Span resource
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- usage
-
- object
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- feedback_scores
-
- object[]
-
-
-
-
-
-
- Array [
-
-
-
-
-
- = -1000000000` and `<= 1000000000`"}
- schema={{"maximum":1000000000,"exclusiveMaximum":false,"minimum":-1000000000,"exclusiveMinimum":false,"type":"number"}}
- >
-
-
-
-
-
-
-
-
-
-
-
-
-
- -
-
- ]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Not found
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- usage
-
- object
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- feedback_scores
-
- object[]
-
-
-
-
-
-
- Array [
-
-
-
-
-
- = -1000000000` and `<= 1000000000`"}
- schema={{"maximum":1000000000,"exclusiveMaximum":false,"minimum":-1000000000,"exclusiveMinimum":false,"type":"number"}}
- >
-
-
-
-
-
-
-
-
-
-
-
-
-
- -
-
- ]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
-
-
- Query Parameters
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Spans resource
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- content
-
- object[]
-
-
-
-
-
-
- Array [
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- usage
-
- object
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- feedback_scores
-
- object[]
-
-
-
-
-
-
- Array [
-
-
-
-
-
- = -1000000000` and `<= 1000000000`"}
- schema={{"maximum":1000000000,"exclusiveMaximum":false,"minimum":-1000000000,"exclusiveMinimum":false,"type":"number"}}
- >
-
-
-
-
-
-
-
-
-
-
-
-
-
- -
-
- ]
-
-
-
- -
-
- ]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
-
-
- Path Parameters
-
-
-
-
-
-
-
-
-
- Trace resource
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- usage
-
- object
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- feedback_scores
-
- object[]
-
-
-
-
-
-
- Array [
-
-
-
-
-
- = -1000000000` and `<= 1000000000`"}
- schema={{"maximum":1000000000,"exclusiveMaximum":false,"minimum":-1000000000,"exclusiveMinimum":false,"type":"number"}}
- >
-
-
-
-
-
-
-
-
-
-
-
-
-
- -
-
- ]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
-
-
- Query Parameters
-
-
-
-
-
-
-
-
-
- Trace resource
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- content
-
- object[]
-
-
-
-
-
-
- Array [
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- usage
-
- object
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- feedback_scores
-
- object[]
-
-
-
-
-
-
- Array [
-
-
-
-
-
- = -1000000000` and `<= 1000000000`"}
- schema={{"maximum":1000000000,"exclusiveMaximum":false,"minimum":-1000000000,"exclusiveMinimum":false,"type":"number"}}
- >
-
-
-
-
-
-
-
-
-
-
-
-
-
- -
-
- ]
-
-
-
- -
-
- ]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- TraceCountResponse resource
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
- workspaces_traces_count
-
- object[]
-
-
-
-
-
-
- Array [
-
-
-
-
-
- -
-
- ]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- default response
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
@@ -51,465 +47,26 @@ Retrieve prompt version
>
-
+
+
+
+
-
-
-
-
- Body
-
-
-
-
-
-
-
-
-
-
-
-
-
- OK
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Bad Request
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Not Found
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Unprocessable Content
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/apps/opik-documentation/documentation/docs/reference/rest_api/score-batch-of-spans.api.mdx b/apps/opik-documentation/documentation/docs/reference/rest_api/score-batch-of-spans.api.mdx
index f6085aedf8..394466fb19 100644
--- a/apps/opik-documentation/documentation/docs/reference/rest_api/score-batch-of-spans.api.mdx
+++ b/apps/opik-documentation/documentation/docs/reference/rest_api/score-batch-of-spans.api.mdx
@@ -5,25 +5,20 @@ description: "Batch feedback scoring for spans"
sidebar_label: "Batch feedback scoring for spans"
hide_title: true
hide_table_of_contents: true
-api: eJyNV1lv20YQ/ivTRR8SgzrsxE5LFC2co6mRNAksG0VhuvGIHIpbLXeZPSQzqv57MUtKlmKnDR947M61M98cXAmPMyfSKzFpUDtxnYiCXG5l46XRIhXP0ecVlETFFPM5uNxYqWdQGgsuciTCNGSRyc8KkQqmoMj1vpz0FJY+BXL+uSlaka5EbrQn7fkVm0bJPHKP/nascSVcXlGN/MZ80lLB9kW50UDfNiRSYaZ/U+5FIhrLFnhJruOOdOlK1Hh75ql2Ij0cj8eJqKXefG+FoLXYikTIbmNfJWkvfftRFkzAN401iUQ4E2zOLwtUgf7fJFnwvadxnh0oElEaW6MXqQhBFmKd7Kj7FurGGlb2MdqUrkSD3pPlkD36xT3+K8vcwaMsmwwPsmzyT5ZNHvPK9yK5J3k/3mcl6KBUAr4iKKjEoDz0ukA6CI6i+o3afWnrROToaWZs+/GrFJ3XugDJOtRdfLorEXSbq+Dkgn7f7JaoHMXwdd+Dh8k32z15r1aHekqW1VrCHmD3LOoD+oDbSbPIKxEkx72Yi+v1OhFeesVkv/ZpMdlingEm1v9Nw/vRHtcY7TqIHI2f8mM/Fu8MvOhThRlq8pXhFGtCxBj6SqRitDgcNVYu0NMoZuRok6yDPhUS4cguyHKar0SwSqSi8r5JRyNlclSVcT49Pnz2ZISNvAeIt0wCnQSxTnYFuHQ0Wi6Xw9zU5Pk+Mo2cPyjlfSPn8EKZUIj1dSI41c/visKrW6wbRbvpe9WljXhS4g/H5cnTwfGzw2eDp8cnR4PpkzIfHOU/njwpT06wxBOxlzvfyrKfQHch//L7C0DfbfQ4Ht9B625vgygGzvp6nQipSxMB1sMiOuT81eQCTj+c3XPXRUWwR8GplwdrSXvVgtQwJY+AugAXYtEBbyCvUM9oCGcltCZAhQsC1C1EP0ujHRh7V8pxaoKPaX764cwl0ChCR2AJ8wp4y2h4Lf1vYZrCJtoz6aswjaGOQR/UKsZ8mOlMnyoFpowSO6g6UNJ5KtheX0kHhclDTdrHeg9oKZYTmLZA0ldkI+/k5Ru2k18vz/hYUnuymHtYSl/F9eiaDpJDOHWAYMkF5ZNM72qPDpgSaTCNl7X8TEVsXL4iF1UPcnTk2Lxa6mLrODZMGTPfNDrsJYKvMBZBbfzmaDg1C9o6L7eEnjLNgZHOBbpzIp/JonQECB/Ov2OHxWNInatQkAO/NFCj1FBQo0zLfop2c9yi4u6M0Vyn5KxiJBSyLIlREUESHM4oZdEDODiYkCoHnNxUQK/KedQ5pQcH8KcJsJRKgZN1o1rQRAU72zWUy7Lt3H/+FtDBzVeLxegn0kVjpPYfuRr9fMOHdLKWCu0QLjjk0u11ku5AmygwINyws/auQuyZ95BdkZTP+4bamANx4Q9j567BnDq4EVSEBUU7CLr2uYFntwOuMkEVMO1cBnBzc8OPFd8AMvEiQnwrNxMpZKI1wQ6Wm7UBF4VMJBsWDL4yVn6OCN9hwEYO5tRmggnXW2X88tKAxzkxqNgyS1202c49aRurY9mBwlCHwx4/kf7mOaElCzfQWCrl7RAuDLil5BHOGwiOEb31YHT2xll7qZnENMgx2pcrmc8Zx0xGhfQwDd4bDYV0jcKWClhWpKEyC4qzIT97czgnLs/f3uzQ9oIsAxiqWMFkQZvI9IDgsmu0xzwOiX3hfR2LD5xTY5z0xvLctt+MvlaeuMcrmZN2tCPvtMG8IjgajvcE9U0N4+7Q2NmoZ3Wjt2cvXr2bvBocDcfDytcqTjNkXVe0D4fj4TiOZsb5GvWOqm+YovcawM6M/C28/dTi6daPGoVSsxXxRKt+TLgSi8PY8+KgwA2qZ/1yWLhOBKc4c6xWU3R0adV6zcufAtlWpFfX3PmsxCn3sSvubR0wY8+eUytS0U8tgwu2a9so7w/7PApsp5oPlxciEdP+H6E2BbNYXPL/Ay5FKuKvRqyHcVDntZVQqGcBZ0zbieTrXzXoZJI=
+api: eJyNVutv20YS/1emg/vQGtTDTuLcEUULJ01bo7k2sBwcDqYbj8ihuNVyl92HZFXV/17MkpKtOLmrPlDk7rznN48tBlp4zG9w1pHxeJthxb50qgvKGszxFYWygZq5mlO5BF9ap8wCauvAJ44MbceOhPyywhyFghPXL/VsoHD8e2QfXtlqg/kWS2sCmyCv1HValYl78psXjVv0ZcMtyZvwKceV2JfkJgPDpmPM0c5/4zJghp0TC4Ji33MnunyLLd1fBm495qfT6TTDVpn990EIOUcbzFD1F8cqVYUZGmoZM/Q2ulJeVqQj/38zVCXPgcYHCRpmWFvXUsAcY1QV7hKTsH9IWvItdhQCOwn8l9/6r34tCn/yZVHMxidFMfuzKGZfyck/MHsi+ThrlzWYqHUGoWGouKaoAwy6QHmInpP6vdpjabsMSwq8sG7z4bMUfRz6MKs2tn2U+1+GfF/q6NWK/72/rUl7Tknov0efJt9fD+SDWhPbOTtR65gGmDyxaEjRJ8LORkTeYFSSyWqJt7tdhkEFLWTfD+CeHZArMMHd/6aR+2SP76zxfdLPps/l7zgXP1t4PQBeGFoOjZVC6WJCDYUGc5ysTiedUysKPEl1NdmX3GgAdIae3YqdFOsWo9OYYxNCl08m2pakG+tD/uL05bMJdeoJIN4KCfQScJc9FuDzyWS9Xo9L23KQ58R2avlJKb90agmvtY0V7m4zlIK9eijtN/fUdpofF+FNXwj4rKZ/vqjPn49evDx9OXr+4vxsNH9Wl6Oz8l/nz+rzc6rpHD+uhof8ffz9ETofLgZQTh9w8nC3h4egYHe7y1CZ2ia0DDlO3l29mV3DxbvLJ75fNwxHFFJHZXSOTdAbUAbmHAjIVOBj6gkQLJQNmQWP4bKGjY3Q0IqBzAZS0JQ1Hqx76K40tzGkmr14d+kz6DSTZ3BMZQNyZQ38oMKPcZ7DPnULFZo4T3lLGRy1OiVwXJjCXGgNtk4Se9x50MoHrsTe0CgPlS1jyyakFgzkOPUGmG+AVWjYJd7Zdz+JnfL6/lLcUiawozLAWoUmnafQ9Pgaw4UHAsc+6pAV5rH2FIA5swHbBdWqP7hKsyQ07JPqUUmevZjXKlMdAieGaWuX+9lDg0QIDaWOZmzYu0Zzu+JD8ErHFLgwkhjlfeSHIIpPjpRnIHh39YUELLmhTKljxR7C2kJLykDFnbYbiVOyW/KWFPc+JnO9VotGkFCpumZBRQJJ9LTgXESP4ORkxroeSaVyBYMqH8iUnJ+cwH9thLXSGrxqO70Bw1xJsH3Hpao3ffiv3gJ5uPts5U++ZlN1VpnwQVrLN3fipFet0uTGcC0pV/5oLPQO7bMggPDj3tqHcj8y71N2JVLx9yfepBpIB/+xbuk7KrmHG0PDVHGyg6GfhXt49jfgGxt1BfM+ZAB3d3fyt5UHQIGvE8QPcgvMocCNjW603p+NpCkUmO1ZKIbGOvVHQvgjBurUaMmbAoVwd1AmL99ZCLRkAZVY5rjPtth5JG1vdWo7UFnucTjgJ9HfvWJy7OAOOse1uh/DtQW/VrJVBQvRC6IPEUzB3gfrqDSzVAYlJftKrcql4FjIuFIB5jEEa6BSvtO04QrWDRto7IrTuib/gzlSE++v3t49oh0EOQEwNKmDqYr3mRkAIW3XmkBl2tuGxvtDaj5wxZ31Klgnq9TxZPlce5KBrVXJxvMjeRcdlQ3D2Xh6JGiYUJRux9YtJgOrn7y9fP3m59mb0dl4Om5Cq9Nqws73Tft0PB1P055lfWjJPFL1NxbbowHwaG39O7zDChL4Pkw6TcqIFcmj7TDzb3B1mmZemvoyoAbWjyf/bYZS4sKx3c7J83undzs5/j2y22B+cyuTzymayxy7kdnWAzMN4CVvMMdhBRldi12HQfl0/5a5flhR3r2/xgznw9re2kpYHK1lpac15pi2/9QP0+4sZ1vUZBaRFkLbi5TfXziFQpk=
sidebar_class_name: "put api-method"
info_path: reference/rest_api/opik-rest-api
custom_edit_url: null
hide_send_button: true
---
-import ApiTabs from "@theme/ApiTabs";
-import DiscriminatorTabs from "@theme/DiscriminatorTabs";
import MethodEndpoint from "@theme/ApiExplorer/MethodEndpoint";
-import SecuritySchemes from "@theme/ApiExplorer/SecuritySchemes";
-import MimeTabs from "@theme/MimeTabs";
-import ParamsItem from "@theme/ParamsItem";
-import ResponseSamples from "@theme/ResponseSamples";
-import SchemaItem from "@theme/SchemaItem";
-import SchemaTabs from "@theme/SchemaTabs";
-import Heading from "@theme/Heading";
+import ParamsDetails from "@theme/ParamsDetails";
+import RequestSchema from "@theme/RequestSchema";
+import StatusCodes from "@theme/StatusCodes";
import OperationTabs from "@theme/OperationTabs";
import TabItem from "@theme/TabItem";
+import Heading from "@theme/Heading";
@@ -51,190 +47,26 @@ Batch feedback scoring for spans
>
-
+
+
+
+
-
-
-
-
- Body
-
-
-
-
-
-
-
-
-
- scores
-
- object[]
-
-
-
- required
-
-
-
-
-
-
- **Possible values:** `>= 1`, `<= 1000`
-
-
-
-
-
- Array [
-
-
-
-
-
-
-
-
-
-
-
- = -1000000000` and `<= 1000000000`"}
- schema={{"maximum":1000000000,"exclusiveMaximum":false,"minimum":-1000000000,"exclusiveMinimum":false,"type":"number"}}
- >
-
-
-
-
-
- -
-
- ]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- No Content
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/apps/opik-documentation/documentation/docs/reference/rest_api/score-batch-of-traces.api.mdx b/apps/opik-documentation/documentation/docs/reference/rest_api/score-batch-of-traces.api.mdx
index d174571417..4063621c90 100644
--- a/apps/opik-documentation/documentation/docs/reference/rest_api/score-batch-of-traces.api.mdx
+++ b/apps/opik-documentation/documentation/docs/reference/rest_api/score-batch-of-traces.api.mdx
@@ -5,25 +5,20 @@ description: "Batch feedback scoring for traces"
sidebar_label: "Batch feedback scoring for traces"
hide_title: true
hide_table_of_contents: true
-api: eJyNV+tv3DYS/1fmiH5oDe3DTuLcCYcrnDRtjaZtYDs4HCxfPEuNVuxSpMrHrtXt/u/FUNJ6N07uog96kPPizG8e2oqASy/yW3HjUJIXd5koyUun2qCsEbl4hUHWUBGVC5Qr8NI6ZZZQWQehZ8mEbckh01+WIhdMQont1+pmJHH0eyQfXtmyE/lWSGsCmcCv2LZaycQ++82zzq3wsqYG+Y35lKOSTUyCk4mha0nkwi5+IxlEJlrHJgRFvudOdPlWNPhwGajxIj+dz+eZaJQZv/dC0DnsRCZUv3GskkxQofugSibgm8GGRCa8jU7yyxp1pP9vkir5PtD4wC4UmaisazCIXMSoSrHLDtR9CXXrLCv7kGzKt6LFEMhx0L7+1n/z36LwJ18XxfX0pCiu/yyK62945SuRPZF8HPHLCkzUOoNQE5RUYdQBBl2gPERPSf2o9ljaLhMSAy2t6z58lqL3Wh8g1cSmj09/ZYIepI5erenncbdC7SmFr/+efJp83B7IB7UmNgtyrNYRDgB7YtEQ0E+4nQyLvBVRcdzLlbjb7TIRVNBM9v2QGNd70DPAxO5/0/B+sse31vgeImfz5/w4jsUvFl4PqcIMDYXaco61MWEMQy1yMVufzlqn1hho1ufkbMzXyZALmfDk1uQ41bciOi1yUYfQ5rOZthJ1bX3IX5y+fDbDVj1BxFsmgV6C2GWHAnw+m202m6m0DQW+z2yrVp+U8murVvBa21iK3V0mONevHqvCmwdsWk2H+Xvb5414VuHfX1TnzycvXp6+nDx/cX42WTyr5ORM/uP8WXV+jhWei6Pk+VKW4wx6jPnH3x8h+nFjAPL8EVuPeyOkGDm7u10mlKlsQtiAi+SQqzfXN3Dx7vKJu25qgiMKzj0ZnSMTdAfKwIICApoSfExVB4IFWaNZ0hQuK+hshBrXBGg6SH5W1niw7rGa48LGkPL84t2lz6DVhJ7AEcoaeMsa+EGFH+MihzHaSxXquEihTkGfNDrFfFqYwlxoDbZKEnusetDKByrZ3lArD6WVsSETUsEHdJTqCSw6IBVqcon3+ruf2E5+fX/Jx1ImkEMZYKNCndaTa3pITuHCA4IjH3XICnOoPTlgQWTAtkE16g8q+95Vk0+qJxI9eTavUabcO44N09auxl6Hg0QINaYqaGwYj4YLu6a986QjDFQYDozyPtKjE/lMDpUnQHh39Td2WDqGMlLHkjyEjYUGlYGSWm079lOym+OWFPdnTOZ6rZY1I6FUVUWMigSS6HFJOYuewMnJNelqwslNJQyqfEAjKT85gf/YCBulNXjVtLoDQ1Sys31LUlVd7/6rt4Ae7j9bLGb/JFO2VpnwgcvRv+75kF41SqObwg2HXPmjVtIfaIwCA8JPe2sfK8SReZ+yK5HyeX+iLuVAWvi3dSvfoqQebgQ1YUnJDoK+f47w7HfA1zbqEha9ywDu7+/5seUbQCFeJ4jv5RYih0J0NrrJZlybcFEoRDayYAy1deqPhPADBmzVZEVdIZhwt1fGL99ZCLgiBhVb5qiPNtt5JG20OpUdKC31OBzwk+jvXxE6cnAPraNKPUzhxoLfKJ7igoXoGdF7DyZnj846Ss0spYHEZJ/USq4Yx0xGpQqwiCFYA6XyrcaOStjUZKC2a0rjIT8Hczgn3l+9vT+gHQQ5BjDUqYKpksbIDIDgsmtNQJmmxKHw/pCKD1xRa70K1vHgdtyMPleeuMlrJcl4OpB30aKsCc6m8yNBQ1PDtDu1bjkbWP3s7eXrN79cv5mcTefTOjQ6jTPkfF+0T6fz6TzNZtaHBs2Bqi8ZpI86wMGU/EXMw+AS6CHMWo3KsB3pTNthUrgV69PU9dKswCwj78fzwl0mOMuZZbtdoKf3Tu92vPx7JNeJ/PaOm59TuOBWdsvtrcdmatsr6kQuhsllcsOG7Xvl04Gfp4H9ZPPu/Y3IxGL4T2hsySwON/wPgRuRi/S/kUpiGtZ5bSs0mmXEJdP2Ivn6CxOHZx0=
+api: eJyNV+tz3DQQ/1cWDR8g43skbVPwMDBpKZChQCdJh2Hi0OzJ67M4WTJ63MUc978zK9uXXNMC98GWpX1p97eP24qASy/ya3HlUJIXN5koyUun2qCsEbl4gUHWUBGVC5Qr8NI6ZZZQWQehZ8mEbckh05+XIhdMQontl+pqJHH0ZyQfXtiyE/lWSGsCmcBLbFutZGKf/eFZ51Z4WVODvGI+5ahkE5PgZGLoWhK5sIs/SAaRidaxCUGR77kTXb4VDd6dB2q8yI/n83kmGmXG770QdA47kQnVHxyqVKXIhMGGRCa8jU7yYo060n+boUp+DjQ+sNtEJirrGgwiFzGqUuwSE7O/S1ryrWgxBHLs+s++8Z//XhT+6LOiuJweFcXl30Vx+TnvfCqyR5IP43ZegYlaZxBqgpIqjDrAoAuUh+gpqR/VHkrbZUJioKV13buPUvR+6N2smtj0Xu5/maA7qaNXa/ppPK1Qe0pB6L8nHyYfjwfyQa2JzYIcq3WEA0weWTSE6ANuJ8Mir0VUHMlyJW52u0wEFTSTfTfA+3IPXYaJ2P07DZ8ne3xrje+DfjJ/yq/DWPxs4eUAeGZoKNSWM6WNCTUYapGL2fp41jq1xkCzPrNmY9ZNBkRnwpNbk+OE3YrotMhFHUKbz2baStS19SF/dvz8yQxb9QgRr5kEeglilz0U4PPZbLPZTKVtKPBzZlu1+qCUX1q1gpfaxlLsbjLBGXtxn9uv7rBpNT3Mwus+E8STCr94Vp0+nTx7fvx88vTZ6clk8aSSkxP55emT6vQUKzwV76fDfQDf/34PnvcHAyrn90C5PxvxwTDY3ewyoUxlE1yGIKfbXby6vIKzN+eP7n5VExxQcCLJ6ByZoDtQBhYUENCU4GMqChAsyBrNkqZwXkFnI9S4JkDTQXKassaDdfcFFhc2hpS0Z2/OfQatJvQEjlDWwEfWwPcq/BAXOYyhW6pQx0WKW4rgpNEpgNPCFOZMa7BVktgDz4NWPlDJ9oZaeSitjA2ZkGowoKNUHGDRAalQk0u8l9/+yHby8u05X0uZQA5lgI0KddpPrunxNYUzDwiOfNQhK8xD7ckBCyIDtg2qUX9R2beTmnxSPZHoybN5jTLl3nFsmLZ2NbYfHCRCqDGVNGPDeDVc2DXtnScdYaDCcGCU95Hunch3cqg8AcKbi0/YYekaykgdS/IQNhYaVAZKarXt2E/Jbo5bUtzfMZnrtVrWjIRSVRUxKhJIoscl5Sx6AkdHl6SrCWcqlTCo8gGNpPzoCH6zETZKa/CqaXUHhqhkZ/uWpKq63v0XrwE93H4082dfkSlbq0x4x7Xl61u+pFeN0uimcMUhV/6gL/QXGqPAgPDT3tr7dD8w70N2JVK+74/UpRxIG79at/ItSurhRlATlpTsIOib4QjP/gR8baMuYdG7DOD29pZfW34AFOJlgvhebiFyKERno5tsxr0JF4VCZCMLxlBbp/5KCH/AgK2arKgrBBPu9sp48a2FgCtiULFljvpos50H0karU9mB0lKPwwE/if72BaEjB7fQOqrU3RSuLPiN4sEqWIieEb33YHL26KyD1MxSGkhM9kmt5IpxzGRUqgCLGII1UCrfauyohE1NBmq7pjSx8Xswh3Pi7cXr2we0gyDHAIY6VTBV0hiZARBcdq0JKNPgNhTe71PxgQtqrVfBOp6lDjvLx8oTd2ytJBlPD+SdtShrgpPp/EDQ0KEwnU6tW84GVj97ff7y1c+XryYn0/m0Do1Oswk53xft4+l8Ok+DlvWhQfNA1f+ZbQ86wIPB9X8xD1NIoLswazUqw3akO22Htn8t1sep66XGzywj7/vN/yYTnOXMst0u0NNbp3c73v4zkutEfn3Dzc8pXHAru+b21mMz9eAVdSIXwxgyuWLD9r3y8QzOrX0/prx5eyUysRhG98aWzOJww2M9bkQu0l+AVBLT/Mx7W6HRLCMumbYXyb9/ABLrRSQ=
sidebar_class_name: "put api-method"
info_path: reference/rest_api/opik-rest-api
custom_edit_url: null
hide_send_button: true
---
-import ApiTabs from "@theme/ApiTabs";
-import DiscriminatorTabs from "@theme/DiscriminatorTabs";
import MethodEndpoint from "@theme/ApiExplorer/MethodEndpoint";
-import SecuritySchemes from "@theme/ApiExplorer/SecuritySchemes";
-import MimeTabs from "@theme/MimeTabs";
-import ParamsItem from "@theme/ParamsItem";
-import ResponseSamples from "@theme/ResponseSamples";
-import SchemaItem from "@theme/SchemaItem";
-import SchemaTabs from "@theme/SchemaTabs";
-import Heading from "@theme/Heading";
+import ParamsDetails from "@theme/ParamsDetails";
+import RequestSchema from "@theme/RequestSchema";
+import StatusCodes from "@theme/StatusCodes";
import OperationTabs from "@theme/OperationTabs";
import TabItem from "@theme/TabItem";
+import Heading from "@theme/Heading";
@@ -51,190 +47,26 @@ Batch feedback scoring for traces
>
-
+
+
+
+
-
-
-
-
- Body
-
-
-
-
-
-
-
-
-
- scores
-
- object[]
-
-
-
- required
-
-
-
-
-
-
- **Possible values:** `>= 1`, `<= 1000`
-
-
-
-
-
- Array [
-
-
-
-
-
-
-
-
-
-
-
- = -1000000000` and `<= 1000000000`"}
- schema={{"maximum":1000000000,"exclusiveMaximum":false,"minimum":-1000000000,"exclusiveMinimum":false,"type":"number"}}
- >
-
-
-
-
-
- -
-
- ]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- No Content
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/apps/opik-documentation/documentation/docs/reference/rest_api/stream-dataset-items.api.mdx b/apps/opik-documentation/documentation/docs/reference/rest_api/stream-dataset-items.api.mdx
index 970f8e8092..038aa3a2d1 100644
--- a/apps/opik-documentation/documentation/docs/reference/rest_api/stream-dataset-items.api.mdx
+++ b/apps/opik-documentation/documentation/docs/reference/rest_api/stream-dataset-items.api.mdx
@@ -5,25 +5,20 @@ description: "Stream dataset items"
sidebar_label: "Stream dataset items"
hide_title: true
hide_table_of_contents: true
-api: eJx1Vm1v20YM/ivcfSzklyRtugnDgLQNuqDdWiQphiEOEvpEWVef7rQ7yq5q+L8PPMmJ3Sb+YAsyyXv48OGRG8W4iCq/Ue+QMRJHdZupgqIOpmHjncrVFQfCGoreAAxTHVWmfEMBxeSiULmKyWgIcjGYBPqvpchvfNGpfKO0d0yO5RGbxhqd3CdfoxyzUVFXVKM8iZ8JVAiu4dg7hzUJNu4aUrny86+kWWWqCQKEDUXxPLDONzvryMG4hdpmymLku0AcDK2ouDPFz1aZKn2okVWu2tYU4hWZsL6zpja8Z28c04LCvoNxfHKstttMsWErRnuU9ERe9qSo7VbMAsXGu9ijP55OUxIH9L/b5x16nsEHoBB8gKIVzNAErykK58+Q7DUTj3rvQ7Jr/NbXKz+eTqcPBGMI2KlM9eV+giR03adS5TeH5Yq+DfrJQj1BiNpmm2ftziW/vyhGXJDa3m63PWE1ceVFcY2Pqf7IlcrVZHU0aYJZIdNkEEGcJOyTIelMRQorCjFhboNVuaqYm3wysV6jrXzk/NXR65MJNkb92AQfxQT6CAn2Y4CYTybr9XqsfU0s3xPfmOWTUT41ZglvrW8Ltb3NlCj/8rFHzr9h3Vj6WcePpD8hX3VS4q+vytOXo1evj16PXr46PR7NT0o9Ota/nZ6Up6dY4qn6QcPTbaaMK30q7EB4And5fnUNZ58vfoJ+XREcWICJoNsQyLHtwDiYEyOgKyC2qZbAHnSFbkFjuCih8y1UuCJA10HK2XgXRcklUTFHvQSc+5aBK5L4MYPGEkaCQKgrkL+8g/eG/2znOeyYXxiu2nmiPRVgVNvE/3jmZu7MWvBlitjrJoI1kakQvFyZCIXXbU2OU5cABoI2UgHzDshwRSH5Xr37IDjl8cuFpCWdH1AzrA1X6X2ippfHGM4iIASKreVs5vZPTwTMiRz4hk1tvlMBZR86pqNHGiNFgVcbVzwQJ8Cs90vpdbHHISJwhSyVcJ53qeHcr+iBPB0ImWZOCmNibOmRRMkpoIkECJ8vfxHCUhrGadsWFIHXHmo0DgpqrO+Ep4Rb6pYO7nNMcKM1i0qUUJiyJFFFEkkr7ZtL6BG8eHFFthxJo1EBw1GR0WnKX7yAf30La2MtRFM3tgNHVAjZsSFtyq6n//IjYIT7Zxt38ju5ovHG8Z3cDH/cS5LR1MZiGMO1lNzEFKqgElu7S2hXBRFEHPdoH7v1AN5TuJKp5PuButQD6cU/Pixjg5p6uRFUhAUlHAT9zNjJs/8HYuVbW8C8pwzg/v5efjbyBTBTb5PEH+LOVA4z1fk2jNa7dyO5M2Yq27lgy5UP5ntS+J4DNma0pG6mxHD7cJg8vPPAuCQRlSAL1FdbcB5E26FeoW2FOep1OOgn2d+/IQwU4B6aQKX5NoZrD3FtWFfCYRtF0Q8MJrJ3ZB20ZpbaQGPCp63RS9GxmFFhGOYts3dQmNhY7KiAdUUOKr+iNB7ld4AjPfHl8uP9nu0QKIiAoUo3mCloV5lBEMNoRZ1G63Avv0+XD1xS46NhH2RgHg6G566ntIsYTS7SXryzBnVFcDyeHgQaBgymf8c+LCaDa5x8vHh7/vfV+eh4PB1XXFuJK0Ouv7SPxtPxVF7JsKzR7R31zFZ3cOnvrWzP2Q/Tm+kbTxqLxslpCflmGM03anWU9rQ0nOWI3aa5Wy6yYXuUrUGaWXw2mzlG+hLsdiuv/2spdCq/uc3UCoPBuUysm9ttpnoJppm+pE7l6m0PeXQtyMTctmmf+XHjlCHee5xpTQ0/Y3uwOMnQflhAPn+6ulaZmg/bbe0L8Q24ls0X1ypXaUtO12XaauXdRll0i1Z2mlz1OOTzPzvMABw=
+api: eJzlWG1v4zYS/itz/LiQX5J0sz2hOCDdzfVy3XYXSRbFIQmSMTm2WFOkSlJ2VMP//TCU5Ngbp+fc3bfmg+1Iw+HDZx7ODLkSEWdB5DfiA0YMFIO4y4SiIL2uonZW5OIqesISVGsAOlIZRCZcRR7Z5EKJXIRk1Dm56Ew8/VZTiN871Yh8JaSzkWzkn1hVRss0fPRr4GlWIsiCSuRfPE57Uoyrm/beYkmMLTYViVy4ya8ko8hE5RlI1BR45I51vuqtQ/TazsQ6EwZDvPcUvaYFqXutnltlYup8iVHkoq614lEhEpb3Rpc6btlrG2lGfnuAtvHkWKzXmYg6GjbaoqQl8rIlRazXbOYpVM6GFv3xeJwWsUP/h23eoeUZnAfy3nlQNWOGyjtJgTl/gWQnI8VBO3qX7BIf23jlx+PxeEMweo+NyEQb7j0koW0+TUV+sxuu4GovDwjUgcRrW9XblG+89fT+Mzj7s1PExvRYkYyk7l0dXzGspIism0Pto0dJByunQnuwbcvdHlOydcnslmhrNKLDIFr3/KXm4m6didcsg9nyuiQb77+OcR97T6g+WdOIPPqatrSwb4PyS17prmcltgj7f6lid4JDRnwN8pAxr4rzq4T6On1OidQE5fw+SOfpv45TSokbkWVigaY+YKO+mEklRpo537yca9sZ2hSjS1bw0bj/4yBKUwe9oJ/6t1M0gTJRatv+P9hv3r/uzLtpbV1OyIuUUrGrKM8Q/ecdVuut7SQ9IecTjH+oAYWRBlEnendD0ReculL/s6MezKTZ5+OP5z1ozPq5kHo5/r2T4BUrMJW3Px01PRXnm9zDVfPPzcVWbyPW2fNstptJJOezQ7onLskh4Gx/VlEUUZs9LclOlLg5+qnzsr5br9tuq6RYOG5XKxcSQIyFyMVocTSqvF5gpFFXKcIoJdFR1zFlIpBfkA+p4am9EbkoYqzy0cg4iaZwIeZvj96djLDiDLLbwn1kE2g9JKKeHIR8NFoul0PpSor8OXKVnu/18qnSc3hvXK3E+i4T3DZfPjXY549YVoaeN8FPgdzT+4qTKX77dnr6zeDtu6N3g2/enh4PJidTOTiWfz09mZ6e4hRPxVcN8DjVu6lLIegIT+Auz6+u4ezzxTPo1wXBjgXoALL2nmw0DWgLE4oIaBWEOqkHogNZoJ3REC6m0LgaClwQoG0grVk7G7gN7qsj4MTVEWJB7D9kUBnCQOAJZQH8yln4Qcd/1JMceuZnOhb1JNGeAjAoTeJ/eGtv7Zkx4KbJY6ubAEaHSIrxxkIHUE7WnAlSiw3oCepACiYNkI4F+TT26sOPjJN/frngZbHwPcoISx2L9DxR08pjCGcBEDyF2sTs1m7PngiYEFlwVdSl/p0UTFvXIU09kBgoMLxSW7UhjoEZ5+Z8UGB77DxCLDByJKyL/dJw4ha0Ia9NJreWA6NDqOmJRF6TRx0IED5f/oUJS8vQVppaUYC4dFCitqCoMq5hnhJujluauF1jghuMnhWsBKWnU2JVJJHUvH1zdj2AN2+uyEwHvNFIQTdViGgl5W/ewL9cDUttDARdVqYBS6SY7FCR1NOmpf/yI2CAhxc37ug7sqpy2sZ7zgx/e+BFBl1qg34I1xxyHZIrRVOsTb+gPgosiDBs0T7t1h14+3AlU17vj9SkPZAe/OL8PFQoqZUbQUGoKOEgaFNmL8/2DYTC1UbBpKUM4OHhgb9W/AFwK94niW/83oocbkXjaj9Y9s8GnDNuRdYPwToWzuvfk8K3BmClB3NqbgUbrjeT8Y8PDiLOiUXFyDy10WacO9561KlLBOWo1WGnn2T/8D2hJw8PUHma6schXDsISx1lwRzWgRW9YTCR3ZO1szWztA0kJnzSaDlnHbMZKR1hUsfoLCgdKoMNKVgWZKFwC0pna/7u4PCe+HL58WHLtnPkWcBQpAymFfWR6QTRnctRxqeWWvyQkg9cUuWCjs5zJ79bGF5KT+kiQ0uygbb8nVUoC4Lj4XjHUVdgML0dOj8bdUPD6OPF+/Ofr84Hx8PxsIilSU07+dAm7aPheDjmR1wsS7RbU71wJbST9Lfue16y7+p3pMc4qgxqy7Ml5KuuNN+IxVFqJFJxFpuDXNjcTGTd1ROfZHgz85jVaoKBvnizXvPj32ryjchv7vhI4jVOuGLdcIPfSjDV9Dk1IhfvW8iDa0a2OcE8v67iIt6OOJOSqviC7c6tCxftTQPy+dPVtcjEpLsaK1NvJDwuueHCpchFumJL6TKd5PjZShi0szp1RqLFwX//BiuQ1CA=
sidebar_class_name: "post api-method"
info_path: reference/rest_api/opik-rest-api
custom_edit_url: null
hide_send_button: true
---
-import ApiTabs from "@theme/ApiTabs";
-import DiscriminatorTabs from "@theme/DiscriminatorTabs";
import MethodEndpoint from "@theme/ApiExplorer/MethodEndpoint";
-import SecuritySchemes from "@theme/ApiExplorer/SecuritySchemes";
-import MimeTabs from "@theme/MimeTabs";
-import ParamsItem from "@theme/ParamsItem";
-import ResponseSamples from "@theme/ResponseSamples";
-import SchemaItem from "@theme/SchemaItem";
-import SchemaTabs from "@theme/SchemaTabs";
-import Heading from "@theme/Heading";
+import ParamsDetails from "@theme/ParamsDetails";
+import RequestSchema from "@theme/RequestSchema";
+import StatusCodes from "@theme/StatusCodes";
import OperationTabs from "@theme/OperationTabs";
import TabItem from "@theme/TabItem";
+import Heading from "@theme/Heading";
@@ -51,559 +47,26 @@ Stream dataset items
>
-
+
+
+
+
-
-
-
-
- Body
-
-
-
-
-
-
-
-
-
-
-
-
-
- Dataset items stream or error during process
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
- -
-
- Array [
-
-
-
- anyOf
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- experiment_items
-
- object[]
-
-
-
-
-
-
- Array [
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- feedback_scores
-
- object[]
-
-
-
-
-
-
- Array [
-
-
-
-
-
- = -1000000000` and `<= 1000000000`"}
- schema={{"maximum":1000000000,"exclusiveMaximum":false,"minimum":-1000000000,"exclusiveMinimum":false,"type":"number"}}
- >
-
-
-
-
-
-
-
-
-
-
-
-
-
- -
-
- ]
-
-
-
-
-
-
-
-
-
-
-
- -
-
- ]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- ]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/apps/opik-documentation/documentation/docs/reference/rest_api/stream-experiment-items.api.mdx b/apps/opik-documentation/documentation/docs/reference/rest_api/stream-experiment-items.api.mdx
index 41a55374f1..29e42ce6c0 100644
--- a/apps/opik-documentation/documentation/docs/reference/rest_api/stream-experiment-items.api.mdx
+++ b/apps/opik-documentation/documentation/docs/reference/rest_api/stream-experiment-items.api.mdx
@@ -5,25 +5,20 @@ description: "Stream experiment items"
sidebar_label: "Stream experiment items"
hide_title: true
hide_table_of_contents: true
-api: eJx9Vm1v2zYQ/is3fgxky0madBOGAWkadEG7tUhSDEMcxGfqZLGmSJWk7KiG//twlPyWJssHS6Hujs8997oSAWdeZPfi6qkmpyoywYuHROTkpVN1UNaITNwGR1gBbWVABaq8SIStySFLXeciEz7K7Uxd91KOvjfkwzubtyJbCWlNIBP4FetaKxktpN88X7YSXpZUIb+xnnKUM8Dd5Y8GK2KQoa1JZMJOv5EMIhG1YzhBkWfl5wrZaqPgg1NmJtaJ0KpSYe+LMoFm5EQiCusqDN3R6UmURR8eHQWnaEH5o8p/trin1jQqF+t1IoIKmkUOWekYvel4Ees1SzrytTW+Q38yGvHjMA5XzwIAHeFgHZBz1kHeMA6onZXkmflXqLYyUBh02oeUV/jURS07GY1GW47ROWxFIrq4v+A4mvZzIbL7w6DlGNBTeGQ1ZizZj0r8PziUxK8vxPNl6sQ6Wb0uyjT8Rd7jjMT6Yb3uqK0olJYztLY+ZgqGUmQiXRyntVMLDJTugPk0epn29CTCk1uQ89G7xmmRiTKEOktTbSXq0vqQnR2/PU2xVuJ56XxiEegsROQ7Az5L0+VyOZS2osC/qa3V/EUrn2s1h0ttm1ysHxLBlXKzq6mrJ6xqTS8m/S5Cfa6PXsxkcVrgr2fF+ZvB2dvjt4M3Z+cng+lpIQcn8rfz0+L8HAs85ypQprAx/j3hEdnN1e0dXHy5/gn3XUlwIAHKg2ycIxN0C8rAlAICmhx8E2MJwYIs0cxoCNcFtLaBEhcEaFqIDitrPCd8QZRPUc4Bp7YJEEpi+z6BWhN6AkcoS+BP1sAHFf5sphlsaJ+pUDbTyHlkf1DpSP5wbMbmQmuwRbTY5Y0HrXygnPGGUnnIrWyY41hMgI6g8ZTDtAVSoSQXdW/ff2Sc/Pr1mt3i5uJQBliqUMbzSE2XG0O48IDgyDc6JGOzf3skYEpkwNZBVeoH5VB0pn28eiDRk2d4lTL5ljgGpq2dc0tgeewtQigxcCSMDRvXcGoXtCVPOsJAY8OBUd43tCORfXKoPAHCl5tfmLDohjJSNzl5CEsLFSoDOdXatrFb2bqLW7y48zHC9VrNSs6EXBUFcVbEJGm4fDM2PYCjo1vSxYCrjHLor/IBjaTs6Aj+tQ0sldbgVVXrFgxRzmT7mqQq2o7+m0+AHiavVm36O5m8tsqER+4Mf0zYSa8qpdEN4Y5Drnw0lVOBjd44tIkCJ4Qfdmh3pXoA7yVcUZT9/UhtrIF48I91c1+jpC7dCErCnCIOgm6+bNKz+wK+tI3OYdpRBjCZTPix4h+AsbiMKb61OxYZjEVrGzdYbs4G3C3GItmoYBNK69SPmOF7ClirwZzasWDB9fYyfnlvIeCcOKkYmaMu2ozzwNoG9QJ1w8xRl4d9/kT5yTtCRw4mUDsq1NMQ7iz4pQqyZA4bzxm9ZTCSvSHroDSTWAYSIz6plZxzHrMY5SrAtAnBGsiVrzW2lMOyJAOlXVCcovzs4XBNfL35NNmT7Q05TmAoYwdTOW0i0ydEP4FRxgncd+QPsfnADdXWq2Adz9XDqfBae+pWFknG0569ixplSXAyHB0Y6qcLxq9D62Zpr+rTT9eXV3/fXg1OhqNhGSrNdnnCdU37eDgajviIh2WFZu+q1xfBg76/t+L9j0o/wwM9hbTWqAzfGfGv+gF9LxbHca+LI/pge/DbZSTp107eILiqWW21mqKnr06v13z8vSHXiuz+IRELdAqnPLruH9aJ6HIxTvY5tSITlx3wwR2DY3HdxP3n+Z7Ko7zTuJCS6vCK7MGixaN7u4l8+Xx7JxIx7Xfiyuas63DJ+zIuRSbieh37ZtyF+WwlNJpZw8tNJjoc/Pcfqz4Z4A==
+api: eJzlWG1v4zYS/itTflzIL0m62Z5wOCDdpr1ct91FksXhEAfJWBxZrClS5YsdNfB/PwwlOfbG2cuhH+sPlizNDB8+80o/ioALL/Ibcf7QkFM1meDFbSYk+cKpJihrRC6ugiOsgbYyoALVXmTCNuSQpS6kyIVPck+mLnopR79H8uF7K1uRP4rCmkAm8C02jVZFsjD5zfNij8IXFdXId6ynHEkG+LT4ncGaGGRoGxK5sPPfqAgiE41jOEGRZ+UvFfLHQcEHp8xCbDKhVa3CzhtlAi3IiUyU1tUYukcnx0kWfbhzFJyiFck7JZ9b3FGLUUmx2WQiqKBZZJ+VjtHLjhex2bCkI99Y4zv0x9MpX/b9cP6FA6AjHKwDcs46kJFxQONsQZ6Zf4FqWwQKo057n/IaHzqv5cfT6XTLMTqHrchE5/cDG0fTfixFfrPvNIkBPYU7VmPGsl2vpN/BYUF8+z/9+SrCv1zgNRpfgnyNzhb2a4SVaeJulG03OMTGv7w1v1pJLGxj+D+kSyI5x2J55wvraNczg8ccofxodCvy4CLteHDXTylBMuFtdAXfrFDHV6TYi3lVYKCFde3Lmdet0AWcqmMt8qPp8GEnFjp6taJfhrclak+ZqJXpfo8Oiw+ve/F+WRPrOTmRcgz7KvMMUb/5Aw4lwyZvRFTMkVyKW96iIwwk7zB8NQYkBhoFlejdd8VQUmIj/7ShAcy8PWTj6+u+SmfzPJCGcPyxD8ErjsBU8v5y1Byu8WKTPU/i/QQqOI1f135q8h4Xh5NJUkClD9TlPXDcIX7prWxuN5uu69QUKsvNu7E+AcRQiVxMVkeTxqkVBpo8lVQ/SeVj0neOTHhyK3I+Ff7otMhFFUKTTybaFqgr60P+9ujdyQQbzp39bvaBRaCzkLh6MuDzyWS9Xo8LW1Pg74lt1PKglY+NWsJ7baMUm9tM8BBx+TRunD9g3Wg6OA88ebQfA6YHm7w4KfG7t+Xpt6O3747ejb59e3o8mp+Uxei4+NvpSXl6iiWedmW+tMkFPeEJ2eX51TWcfbp4hvu6ItiTAOWhiM6RCboFZWBOAQGNBB9T9ECwUFRoFjSGixJaG6HCFQGaFtKGlTWeZ4GhKQDObQwQKmL7PoNGE3oCR1hUwK+sgZ9U+Gec5zDQvlChivPEeWJ/VOtE/nhmZuZMa7BlstjFjQetfCDJeEOlPEhbROY4zRmAjiB6kjBvgVSoyCXdqx9+Zpx8+/mCt8WB77AIsFahSs8TNV1sjOHMA4IjH3XIZmZ39UTAnMiAbYKq1R8koexM+7T0qEBPnuHVysgtcQxMW7vkaYnlsbcIocLAnjA2DFvDuV3RlryunswMO0Z5H+mJRN6TQ+UJED5dfsOEpW0oU+goyUNYW6hRGZDUaNumQc42nd/Swt0eE1yv1aLiSJCqLImjIgVJ5PTN2fQI3ry5Il2OOMtIQr+UD2gKyt+8gf/YCGulNXhVN7oFQySZbN9Qocq2o//yA6CH+xezdvJ3MrKxyoQ7rgz/uOdNelUrjW4M1+xy5ZMpSSVGPWxo8AIHhB93aJ9SdQ/eIVxJlPf7M7UpB9KDf1u39A0W1IUbQUUoKeEg6ErmEJ7dG/CVjVrCvKMM4P7+ni+P/AUwE+9TiG/tzkQOM9Ha6Ebr4dmIq8VMZIMKxlBZp/5IEb6jgI0aLamdCRbcbBfjmx8sBFwSBxUjc9R5m3HuWRtQp+EIpKUuDvv4SfL33xM6cnAPjaNSPYzh2oJfq1BUzGH0HNFbBhPZA1l7qZmlNCgw4Su0KpYcxyxGUgWYxxCsAal8o7ElCeuKDFR2RemAwdceDufE58sP9zuyvSHHAQxVqmBK0uCZPiD6wwkW4WmSFD+l4gOX1FivgnU8wO53hZfKU3eaK8h42rF31mBRERyPp3uG+u6C6e3YusWkV/WTDxfvz3+9Oh8dj6fjKtQ6zarkfFe0j8bT8ZQfcbOs0ews9fIZea/u75x+v6LSd/FAD2HSaFSG10z4H/sGfSNWR2mcSC1672Dlt+e0rD+R8yTPWc1qj49z9PTZ6c2GH/8eybUiv7nlkdwpnHPruuEBt4vF1NmX1IpcvO+Aj64Z3HaCf36E51beaZwVBTXhBdm9Myi37u0k8unj1bXIxLz/u6BOQ5JwuObhC9ciF+mfh1Q300mGnz0KjWYR04gkOhz8+S9nutp9
sidebar_class_name: "post api-method"
info_path: reference/rest_api/opik-rest-api
custom_edit_url: null
hide_send_button: true
---
-import ApiTabs from "@theme/ApiTabs";
-import DiscriminatorTabs from "@theme/DiscriminatorTabs";
import MethodEndpoint from "@theme/ApiExplorer/MethodEndpoint";
-import SecuritySchemes from "@theme/ApiExplorer/SecuritySchemes";
-import MimeTabs from "@theme/MimeTabs";
-import ParamsItem from "@theme/ParamsItem";
-import ResponseSamples from "@theme/ResponseSamples";
-import SchemaItem from "@theme/SchemaItem";
-import SchemaTabs from "@theme/SchemaTabs";
-import Heading from "@theme/Heading";
+import ParamsDetails from "@theme/ParamsDetails";
+import RequestSchema from "@theme/RequestSchema";
+import StatusCodes from "@theme/StatusCodes";
import OperationTabs from "@theme/OperationTabs";
import TabItem from "@theme/TabItem";
+import Heading from "@theme/Heading";
@@ -51,409 +47,26 @@ Stream experiment items
>
-
+
+
+
+
-
-
-
-
- Body
-
-
-
-
-
-
-
-
-
-
-
-
-
- Experiment items stream or error during process
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
- -
-
- Array [
-
-
-
- anyOf
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- feedback_scores
-
- object[]
-
-
-
-
-
-
- Array [
-
-
-
-
-
- = -1000000000` and `<= 1000000000`"}
- schema={{"maximum":1000000000,"exclusiveMaximum":false,"minimum":-1000000000,"exclusiveMinimum":false,"type":"number"}}
- >
-
-
-
-
-
-
-
-
-
-
-
-
-
- -
-
- ]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- ]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/apps/opik-documentation/documentation/docs/reference/rest_api/update-dataset.api.mdx b/apps/opik-documentation/documentation/docs/reference/rest_api/update-dataset.api.mdx
index 4c8a619121..18a6fdc4f3 100644
--- a/apps/opik-documentation/documentation/docs/reference/rest_api/update-dataset.api.mdx
+++ b/apps/opik-documentation/documentation/docs/reference/rest_api/update-dataset.api.mdx
@@ -12,18 +12,13 @@ custom_edit_url: null
hide_send_button: true
---
-import ApiTabs from "@theme/ApiTabs";
-import DiscriminatorTabs from "@theme/DiscriminatorTabs";
import MethodEndpoint from "@theme/ApiExplorer/MethodEndpoint";
-import SecuritySchemes from "@theme/ApiExplorer/SecuritySchemes";
-import MimeTabs from "@theme/MimeTabs";
-import ParamsItem from "@theme/ParamsItem";
-import ResponseSamples from "@theme/ResponseSamples";
-import SchemaItem from "@theme/SchemaItem";
-import SchemaTabs from "@theme/SchemaTabs";
-import Heading from "@theme/Heading";
+import ParamsDetails from "@theme/ParamsDetails";
+import RequestSchema from "@theme/RequestSchema";
+import StatusCodes from "@theme/StatusCodes";
import OperationTabs from "@theme/OperationTabs";
import TabItem from "@theme/TabItem";
+import Heading from "@theme/Heading";
@@ -51,102 +47,26 @@ Update dataset by id
>
-
+
+
+
+
-
-
- Path Parameters
-
-
-
+
+
-
-
-
-
- Body
-
-
-
-
-
-
-
-
-
-
-
-
-
- No content
-
-
-
-
-
-
-
-
-
+
+
+
+
\ No newline at end of file
diff --git a/apps/opik-documentation/documentation/docs/reference/rest_api/update-feedback-definition.api.mdx b/apps/opik-documentation/documentation/docs/reference/rest_api/update-feedback-definition.api.mdx
index f8513bbf34..2f873f6337 100644
--- a/apps/opik-documentation/documentation/docs/reference/rest_api/update-feedback-definition.api.mdx
+++ b/apps/opik-documentation/documentation/docs/reference/rest_api/update-feedback-definition.api.mdx
@@ -12,18 +12,13 @@ custom_edit_url: null
hide_send_button: true
---
-import ApiTabs from "@theme/ApiTabs";
-import DiscriminatorTabs from "@theme/DiscriminatorTabs";
import MethodEndpoint from "@theme/ApiExplorer/MethodEndpoint";
-import SecuritySchemes from "@theme/ApiExplorer/SecuritySchemes";
-import MimeTabs from "@theme/MimeTabs";
-import ParamsItem from "@theme/ParamsItem";
-import ResponseSamples from "@theme/ResponseSamples";
-import SchemaItem from "@theme/SchemaItem";
-import SchemaTabs from "@theme/SchemaTabs";
-import Heading from "@theme/Heading";
+import ParamsDetails from "@theme/ParamsDetails";
+import RequestSchema from "@theme/RequestSchema";
+import StatusCodes from "@theme/StatusCodes";
import OperationTabs from "@theme/OperationTabs";
import TabItem from "@theme/TabItem";
+import Heading from "@theme/Heading";
@@ -51,272 +47,26 @@ Update feedback definition by id
>
-
+
+
+
+
-
-
- Path Parameters
-
-
-
+
+
-
-
-
-
- Body
-
-
-
-
-
-
-
-
-
-
-
-
- type
-
- string
-
- required
-
-
-
-
- **Possible values:** [`numerical`, `categorical`]
-
-
-
-
-
-
-
-
-
- details
-
- object
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- details
-
- object
-
-
-
-
-
-
-
-
- categories
-
- object
-
-
-
- required
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- No Content
-
-
-
-
-
-
-
-
-
+
+
+
+
\ No newline at end of file
diff --git a/apps/opik-documentation/documentation/docs/reference/rest_api/update-project.api.mdx b/apps/opik-documentation/documentation/docs/reference/rest_api/update-project.api.mdx
index 6e058349cb..3edd881b84 100644
--- a/apps/opik-documentation/documentation/docs/reference/rest_api/update-project.api.mdx
+++ b/apps/opik-documentation/documentation/docs/reference/rest_api/update-project.api.mdx
@@ -12,18 +12,13 @@ custom_edit_url: null
hide_send_button: true
---
-import ApiTabs from "@theme/ApiTabs";
-import DiscriminatorTabs from "@theme/DiscriminatorTabs";
import MethodEndpoint from "@theme/ApiExplorer/MethodEndpoint";
-import SecuritySchemes from "@theme/ApiExplorer/SecuritySchemes";
-import MimeTabs from "@theme/MimeTabs";
-import ParamsItem from "@theme/ParamsItem";
-import ResponseSamples from "@theme/ResponseSamples";
-import SchemaItem from "@theme/SchemaItem";
-import SchemaTabs from "@theme/SchemaTabs";
-import Heading from "@theme/Heading";
+import ParamsDetails from "@theme/ParamsDetails";
+import RequestSchema from "@theme/RequestSchema";
+import StatusCodes from "@theme/StatusCodes";
import OperationTabs from "@theme/OperationTabs";
import TabItem from "@theme/TabItem";
+import Heading from "@theme/Heading";
@@ -51,284 +47,26 @@ Update project by id
>
-
+
+
+
+
-
-
- Path Parameters
-
-
-
+
+
-
-
-
-
- Body
-
-
-
-
-
-
-
-
-
-
-
-
-
- No Content
-
-
-
-
-
-
-
-
-
- Bad Request
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Unprocessable Content
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
\ No newline at end of file
diff --git a/apps/opik-documentation/documentation/docs/reference/rest_api/update-prompt.api.mdx b/apps/opik-documentation/documentation/docs/reference/rest_api/update-prompt.api.mdx
index 8df37b830f..fdbd859678 100644
--- a/apps/opik-documentation/documentation/docs/reference/rest_api/update-prompt.api.mdx
+++ b/apps/opik-documentation/documentation/docs/reference/rest_api/update-prompt.api.mdx
@@ -12,18 +12,13 @@ custom_edit_url: null
hide_send_button: true
---
-import ApiTabs from "@theme/ApiTabs";
-import DiscriminatorTabs from "@theme/DiscriminatorTabs";
import MethodEndpoint from "@theme/ApiExplorer/MethodEndpoint";
-import SecuritySchemes from "@theme/ApiExplorer/SecuritySchemes";
-import MimeTabs from "@theme/MimeTabs";
-import ParamsItem from "@theme/ParamsItem";
-import ResponseSamples from "@theme/ResponseSamples";
-import SchemaItem from "@theme/SchemaItem";
-import SchemaTabs from "@theme/SchemaTabs";
-import Heading from "@theme/Heading";
+import ParamsDetails from "@theme/ParamsDetails";
+import RequestSchema from "@theme/RequestSchema";
+import StatusCodes from "@theme/StatusCodes";
import OperationTabs from "@theme/OperationTabs";
import TabItem from "@theme/TabItem";
+import Heading from "@theme/Heading";
@@ -51,466 +47,26 @@ Update prompt
>
-
+
+
+
+
-
-
- Path Parameters
-
-
-
+
+
-
-
-
-
- Body
-
-
-
-
-
-
-
-
-
-
-
-
-
- No content
-
-
-
-
-
-
-
-
-
- Bad Request
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Not Found
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Conflict
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Unprocessable Content
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
\ No newline at end of file
diff --git a/apps/opik-documentation/documentation/docs/reference/rest_api/update-span.api.mdx b/apps/opik-documentation/documentation/docs/reference/rest_api/update-span.api.mdx
index b65ee2f209..308cfd49c0 100644
--- a/apps/opik-documentation/documentation/docs/reference/rest_api/update-span.api.mdx
+++ b/apps/opik-documentation/documentation/docs/reference/rest_api/update-span.api.mdx
@@ -12,18 +12,13 @@ custom_edit_url: null
hide_send_button: true
---
-import ApiTabs from "@theme/ApiTabs";
-import DiscriminatorTabs from "@theme/DiscriminatorTabs";
import MethodEndpoint from "@theme/ApiExplorer/MethodEndpoint";
-import SecuritySchemes from "@theme/ApiExplorer/SecuritySchemes";
-import MimeTabs from "@theme/MimeTabs";
-import ParamsItem from "@theme/ParamsItem";
-import ResponseSamples from "@theme/ResponseSamples";
-import SchemaItem from "@theme/SchemaItem";
-import SchemaTabs from "@theme/SchemaTabs";
-import Heading from "@theme/Heading";
+import ParamsDetails from "@theme/ParamsDetails";
+import RequestSchema from "@theme/RequestSchema";
+import StatusCodes from "@theme/StatusCodes";
import OperationTabs from "@theme/OperationTabs";
import TabItem from "@theme/TabItem";
+import Heading from "@theme/Heading";
@@ -51,218 +47,26 @@ Update span by id
>
-
+
+
+
+
-
-
- Path Parameters
-
-
-
+
+
-
-
-
-
- Body
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- usage
-
- object
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- No Content
-
-
-
-
-
-
-
-
-
- Not found
-
-
-
-
-
-
-
-
-
+
+
+
+
\ No newline at end of file
diff --git a/apps/opik-documentation/documentation/docs/reference/rest_api/update-trace.api.mdx b/apps/opik-documentation/documentation/docs/reference/rest_api/update-trace.api.mdx
index 70681fee23..5c2b4e7f00 100644
--- a/apps/opik-documentation/documentation/docs/reference/rest_api/update-trace.api.mdx
+++ b/apps/opik-documentation/documentation/docs/reference/rest_api/update-trace.api.mdx
@@ -12,18 +12,13 @@ custom_edit_url: null
hide_send_button: true
---
-import ApiTabs from "@theme/ApiTabs";
-import DiscriminatorTabs from "@theme/DiscriminatorTabs";
import MethodEndpoint from "@theme/ApiExplorer/MethodEndpoint";
-import SecuritySchemes from "@theme/ApiExplorer/SecuritySchemes";
-import MimeTabs from "@theme/MimeTabs";
-import ParamsItem from "@theme/ParamsItem";
-import ResponseSamples from "@theme/ResponseSamples";
-import SchemaItem from "@theme/SchemaItem";
-import SchemaTabs from "@theme/SchemaTabs";
-import Heading from "@theme/Heading";
+import ParamsDetails from "@theme/ParamsDetails";
+import RequestSchema from "@theme/RequestSchema";
+import StatusCodes from "@theme/StatusCodes";
import OperationTabs from "@theme/OperationTabs";
import TabItem from "@theme/TabItem";
+import Heading from "@theme/Heading";
@@ -51,147 +47,26 @@ Update trace by id
>
-
+
+
+
+
-
-
- Path Parameters
-
-
-
+
+
-
-
-
-
- Body
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- No Content
-
-
-
-
-
-
-
-
-
+
+
+
+
\ No newline at end of file
diff --git a/apps/opik-documentation/documentation/docs/reference/rest_api/version.api.mdx b/apps/opik-documentation/documentation/docs/reference/rest_api/version.api.mdx
index d5db7589e0..2005cc976b 100644
--- a/apps/opik-documentation/documentation/docs/reference/rest_api/version.api.mdx
+++ b/apps/opik-documentation/documentation/docs/reference/rest_api/version.api.mdx
@@ -12,18 +12,13 @@ custom_edit_url: null
hide_send_button: true
---
-import ApiTabs from "@theme/ApiTabs";
-import DiscriminatorTabs from "@theme/DiscriminatorTabs";
import MethodEndpoint from "@theme/ApiExplorer/MethodEndpoint";
-import SecuritySchemes from "@theme/ApiExplorer/SecuritySchemes";
-import MimeTabs from "@theme/MimeTabs";
-import ParamsItem from "@theme/ParamsItem";
-import ResponseSamples from "@theme/ResponseSamples";
-import SchemaItem from "@theme/SchemaItem";
-import SchemaTabs from "@theme/SchemaTabs";
-import Heading from "@theme/Heading";
+import ParamsDetails from "@theme/ParamsDetails";
+import RequestSchema from "@theme/RequestSchema";
+import StatusCodes from "@theme/StatusCodes";
import OperationTabs from "@theme/OperationTabs";
import TabItem from "@theme/TabItem";
+import Heading from "@theme/Heading";
@@ -43,68 +39,26 @@ import TabItem from "@theme/TabItem";
version
-
-
-
-
-
-
-
- default response
-
-
-
-
-
-
-
-
-
-
- Schema
-
-
-
-
-
-
-
-
-
-
-
-