updating eval script

.gitignore

@@ -132,3 +132,4 @@ launcher/
 
 
 beaker_configs/auto_created/*
+*.jsonl

beaker_configs/default_eval.yaml

@@ -34,6 +34,8 @@ tasks:
         value: true
       - name: OPENAI_API_KEY
         secret: openai_api_key
+      - name: IS_ALPACA_EVAL_2
+        value: false
     datasets:
       - mountPath: /data/
         source:

scripts/submit_open_instruct_eval.py

@@ -37,7 +37,6 @@
 d1['tasks'][0]['constraints']['cluster'] = cluster
 d1['tasks'][0]['context']['priority'] = args.priority
 d1['tasks'][0]['resources']['gpuCount'] = 1
-
 # Use a different image if requested.
 if args.beaker_image is not None:
     d1['tasks'][0]['image']['beaker'] = args.beaker_image
@@ -163,10 +162,8 @@
             --tokenizer_name_or_path /model \
             --n_shot 8 \
             --use_chat_format \
-            --chat_formatting_function eval.templates.create_prompt_with_tulu_chat_format \
+            --chat_formatting_function eval.templates.create_prompt_with_tulu_chat_format
         ''' 
-        if args.gsm_stop_at_double_newline:
-            task_spec['arguments'][0] += " --stop_at_double_newline"
     elif experiment_group == "tydiqa_goldp_1shot":
         task_spec["arguments"][0] = '''
             python -m eval.tydiqa.run_eval \
@@ -290,10 +287,10 @@
 
     if model_info[0].startswith("hf-"):  # if it's a huggingface model, load it from the model hub
         task_spec['arguments'] = [task_spec['arguments'][0].replace("--model_name_or_path /model", "--model_name_or_path "+model_info[1])]
-        task_spec['arguments'] = [task_spec['arguments'][0].replace("--tokenizer_name_or_path /model", "--tokenizer_name_or_path "+model_info[1])]
+        task_spec['arguments'] = [task_spec['arguments'][0].replace("--tokenizer_name_or_path /model", "--model_name_or_path "+model_info[1])]
     elif model_info[1].startswith("/"):  # if it's a local model, load it from the local directory
         task_spec['arguments'] = [task_spec['arguments'][0].replace("--model_name_or_path /model", "--model_name_or_path "+model_info[1])]
-        task_spec['arguments'] = [task_spec['arguments'][0].replace("--tokenizer_name_or_path /model", "--tokenizer_name_or_path "+model_info[1])]
+        task_spec['arguments'] = [task_spec['arguments'][0].replace("--tokenizer_name_or_path /model", "--model_name_or_path "+model_info[1])]
     else:  # if it's a beaker model, mount the beaker dataset to `/model`
         task_spec['datasets'][1]['source']['beaker'] = model_info[1]
 
@@ -402,4 +399,4 @@
 file.close()
 
 cmd = "beaker experiment create {} --workspace ai2/{}".format(fn, workspace)
-subprocess.Popen(cmd, shell=True)
+subprocess.Popen(cmd, shell=True)