RELEASE: 0.0.1a30

brucewlee · Mar 2, 2024 · 82b4fd0 · 82b4fd0
1 parent 1ac2bd0
commit 82b4fd0
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 2 deletions.
diff --git a/nutcracker/evaluator/engines/mcq_engine_gamma.py b/nutcracker/evaluator/engines/mcq_engine_gamma.py
@@ -15,7 +15,12 @@ def is_correct(self, instance: MCQInstance) -> bool:
         valid_options = [chr(ord('A') + i) for i in range(len(instance.options))]
 
         # Filter logprobs for valid options and ignore missing logprob values
-        filtered_logprobs = [entry for entry in logprobs if entry[0] in valid_options and len(entry) > 1]
+        try:
+            filtered_logprobs = [entry for entry in logprobs if entry[0] in valid_options and len(entry) > 1]
+        except TypeError:
+            # if only first position token was given
+            logprobs = instance.model_response_logprobs
+            filtered_logprobs = [entry for entry in logprobs if entry[0] in valid_options and len(entry) > 1]
 
         # If no valid logprobs are found, consider the response incorrect
         if not filtered_logprobs:

diff --git a/setup.py b/setup.py
@@ -1,7 +1,7 @@
 from distutils.core import setup
 from setuptools import find_packages
 
-this_version='0.0.1a29-5'
+this_version='0.0.1a30'
 
 # python setup.py sdist
 # python -m twine upload dist/*