diff --git a/src/lighteval/tasks/lighteval_task.py b/src/lighteval/tasks/lighteval_task.py index c54afb5fe..7eb6c1f16 100644 --- a/src/lighteval/tasks/lighteval_task.py +++ b/src/lighteval/tasks/lighteval_task.py @@ -295,6 +295,11 @@ def _get_docs_from_split(self, splits: list[str], few_shots=False) -> list[Doc]: # Some tasks require to know which is the current item index in order to apply a different prompt template item["__index"] = ix doc = self.formatter(item, self.name) + + # Skip if formatter returns None (e.g., to filter out certain samples) + if doc is None: + continue + doc.id = str(ix) # Transfer task-level generation parameters to the document