add filename to stop tokens

Raymond Li · Raymond Li · commit 47adf388fe3e · 2023-10-24T13:57:27.000Z
diff --git a/lm_eval/tasks/humaneval.py b/lm_eval/tasks/humaneval.py
@@ -51,7 +51,7 @@ class GeneralHumanEval(Task):
 
     def __init__(self, strip_prompt):
         super().__init__(
-            stop_words=["\nclass", "\ndef", "\n#", "\n@", "\nprint", "\nif", "\n```"],
+            stop_words=["\nclass", "\ndef", "\n#", "\n@", "\nprint", "\nif", "\n```", "<filename>"],
             requires_execution=True,
         )
         self.strip_prompt = strip_prompt

Original file line number	Diff line number	Diff line change
`@@ -51,7 +51,7 @@ class GeneralHumanEval(Task):`
`51`	`51`
`52`	`52`	`def __init__(self, strip_prompt):`
`53`	`53`	`super().__init__(`
`54`		- stop_words=["\nclass", "\ndef", "\n#", "\n@", "\nprint", "\nif", "\n```"],
	`54`	+ stop_words=["\nclass", "\ndef", "\n#", "\n@", "\nprint", "\nif", "\n```", "<filename>"],
`55`	`55`	`requires_execution=True,`
`56`	`56`	`)`
`57`	`57`	`self.strip_prompt = strip_prompt`