Merge main

.github/workflows/unit_tests.yml

@@ -42,7 +42,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ "3.8", "3.9", "3.10", "3.11" ]
+        python-version: [  "3.10", "3.11" ]
     timeout-minutes: 30
     steps:
     - name: Checkout Code
@@ -75,10 +75,10 @@ jobs:
     steps:
     - name: Checkout Code
       uses: actions/checkout@v4
-    - name: Set up Python 3.8
+    - name: Set up Python 3.11
       uses: actions/setup-python@v5
       with:
-        python-version: 3.8
+        python-version: 3.11
         cache: pip
         cache-dependency-path: pyproject.toml
     - name: Install dependencies

.gitignore

@@ -1,14 +1,94 @@
-env
-*.pyc
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Ruff
+.ruff_cache/
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Pyenv version management
+.python-version
+
+# PEP 582; Dependency management
+__pypackages__/
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Output directories
 output/
+
+# Data directories
 data/
-lm_cache
-.idea
-build
-dist
-*.egg-info
-venv
+
+# IDE configuration files
+.idea/
 .vscode/
+
+# Temporary files including logs
+temp/
+logs/
+scratch/
+cache/
+slurm_logs/
+eval_results/
+votes/
+evals/
+evaluate_modules/
+outputs/
+test_logs/
+
+
+# SH files (feel free to change the condition to less strict)
+set_hf_token.sh
 temp
 __pycache__
 .ipynb_checkpoints

lm_eval/__main__.py

@@ -292,13 +292,6 @@ def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None:
             "If fewshot_as_multiturn is set, apply_chat_template must be set to True."
         )
 
-    if (
-        args.num_fewshot is None or args.num_fewshot == 0
-    ) and args.fewshot_as_multiturn:
-        raise ValueError(
-            "If fewshot_as_multiturn is set, num_fewshot must be greater than 0."
-        )
-
     if args.include_path is not None:
         eval_logger.info(f"Including path: {args.include_path}")
     task_manager = TaskManager(args.verbosity, include_path=args.include_path)

lm_eval/models/dummy.py

@@ -26,9 +26,9 @@ def loglikelihood(self, requests, disable_tqdm: bool = False):
     def generate_until(self, requests, disable_tqdm: bool = False):
         res = []
 
-        for ctx, _ in tqdm(requests, disable=disable_tqdm):
+        for request in tqdm(requests, disable=disable_tqdm):
             res.append("lol")
-            assert ctx.strip() != ""
+            assert request.arguments[0].strip() != ""
 
         return res