Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Multimodal evaluation #998

Merged
merged 9 commits into from
Sep 16, 2024
Merged

Multimodal evaluation #998

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
304526d
VIEScore
kritinv Sep 11, 2024
0877b01
evaluatation
kritinv Sep 11, 2024
80941be
dataset
kritinv Sep 11, 2024
eb57c08
test
kritinv Sep 11, 2024
c72ea6b
models
kritinv Sep 11, 2024
f90494b
task
kritinv Sep 11, 2024
9f9af02
updated MLLM test case
kritinv Sep 12, 2024
1ded8d5
docs for VIEScore
kritinv Sep 12, 2024
6b7a152
Merge branch 'main' into multimodal-evaluation
penguine-ip Sep 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 25 additions & 13 deletions deepeval/dataset/dataset.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,21 +23,20 @@
DatasetHttpResponse,
)
from deepeval.dataset.golden import Golden, ConversationalGolden
from deepeval.test_case import LLMTestCase, ConversationalTestCase
from deepeval.test_case import LLMTestCase, ConversationalTestCase, MLLMTestCase
from deepeval.utils import convert_keys_to_snake_case, is_confident
from deepeval.synthesizer.types import *

valid_file_types = ["csv", "json"]


def validate_test_case_type(
test_case: Union[LLMTestCase, ConversationalTestCase], subject: str
test_case: Union[LLMTestCase, ConversationalTestCase, MLLMTestCase], subject: str
):
if not isinstance(test_case, LLMTestCase) and not isinstance(
test_case, ConversationalTestCase
):
test_case, ConversationalTestCase) and not isinstance(test_case, MLLMTestCase):
raise TypeError(
f"Provided `{subject}` must be a list of LLMTestCase or ConversationalTestCase"
f"Provided `{subject}` must be a list of LLMTestCase, ConversationalTestCase, or MLLMTestCase"
)


Expand All @@ -54,7 +53,7 @@ class EvaluationDataset:

def __init__(
self,
test_cases: List[Union[LLMTestCase, ConversationalTestCase]] = [],
test_cases: List[Union[LLMTestCase, ConversationalTestCase, MLLMTestCase]] = [],
goldens: List[Golden] = [],
conversational_goldens: List[ConversationalGolden] = [],
):
Expand All @@ -67,16 +66,21 @@ def __init__(

llm_test_cases = []
conversational_test_cases = []
mllm_test_cases = []
for test_case in test_cases:
if isinstance(test_case, LLMTestCase):
test_case._dataset_rank = len(llm_test_cases)
llm_test_cases.append(test_case)
elif isinstance(test_case, ConversationalTestCase):
test_case._dataset_rank = len(conversational_test_cases)
conversational_test_cases.append(test_case)
elif isinstance(test_case, MLLMTestCase):
test_case._dataset_rank = len(mllm_test_cases)
mllm_test_cases.append(test_case)

self._llm_test_cases = llm_test_cases
self._conversational_test_cases = conversational_test_cases
self._mllm_test_cases = mllm_test_cases

def __repr__(self):
return (
Expand All @@ -87,22 +91,23 @@ def __repr__(self):
)

@property
def test_cases(self) -> List[Union[LLMTestCase, ConversationalTestCase]]:
return self._llm_test_cases + self._conversational_test_cases
def test_cases(self) -> List[Union[LLMTestCase, ConversationalTestCase, MLLMTestCase]]:
return self._llm_test_cases + self._conversational_test_cases + self._mllm_test_cases

@test_cases.setter
def test_cases(
self, test_cases: List[Union[LLMTestCase, ConversationalTestCase]]
self, test_cases: List[Union[LLMTestCase, ConversationalTestCase, MLLMTestCase]]
):
if not isinstance(test_cases, list):
raise TypeError("'test_cases' must be a list")

llm_test_cases = []
conversational_test_cases = []
mllm_test_cases = []
for test_case in test_cases:
if not isinstance(test_case, LLMTestCase) and not isinstance(
test_case, ConversationalTestCase
):
test_case, ConversationalTestCase) and not isinstance(
test_case, MLLMTestCase):
continue

test_case._dataset_alias = self._alias
Expand All @@ -113,12 +118,16 @@ def test_cases(
elif isinstance(test_case, ConversationalTestCase):
test_case._dataset_rank = len(conversational_test_cases)
conversational_test_cases.append(test_case)
elif isinstance(test_case, MLLMTestCase):
test_case._dataset_rank = len(mllm_test_cases)
mllm_test_cases.append(test_case)

self._llm_test_cases = llm_test_cases
self._conversational_test_cases = conversational_test_cases
self._mllm_test_cases = mllm_test_cases

def add_test_case(
self, test_case: Union[LLMTestCase, ConversationalTestCase]
self, test_case: Union[LLMTestCase, ConversationalTestCase, MLLMTestCase]
):
validate_test_case_type(test_case, subject="test cases")

Expand All @@ -130,7 +139,10 @@ def add_test_case(
elif isinstance(test_case, ConversationalTestCase):
test_case._dataset_rank = len(self._conversational_test_cases)
self._conversational_test_cases.append(test_case)

elif isinstance(test_case, MLLMTestCase):
test_case._dataset_rank = len(self._mllm_test_cases)
self._mllm_test_cases.append(test_case)

def __iter__(self):
return iter(self.test_cases)

Expand Down
Loading
Loading