revert to current leaderboard

Stability-AI · Aug 15, 2023 · 2f1583c · 2f1583c
1 parent fa287d6
commit 2f1583c
Show file tree

Hide file tree

Showing 45 changed files with 116 additions and 385 deletions.
diff --git a/models/abeja-gpt-neox-japanese-2.7b/harness.jaqket_v1.sh b/models/abeja-gpt-neox-japanese-2.7b/harness.jaqket_v1.sh
diff --git a/models/abeja-gpt-neox-japanese-2.7b/harness.jaqket_v2.sh b/models/abeja-gpt-neox-japanese-2.7b/harness.jaqket_v2.sh
diff --git a/models/abeja-gpt-neox-japanese-2.7b/harness.sh b/models/abeja-gpt-neox-japanese-2.7b/harness.sh
@@ -1,3 +1,3 @@
 MODEL_ARGS="pretrained=abeja/gpt-neox-japanese-2.7b"
-TASK="jaqket_v2-0.2-0.2,jcommonsenseqa-1.1-0.2,jnli-1.1-0.2,marc_ja-1.1-0.2,jsquad-1.1-0.2,xlsum_ja,jaqket_v2-0.2-0.2"
-python main.py --model hf-causal --model_args $MODEL_ARGS --tasks $TASK --num_fewshot "1,2,3,3,3,1,1" --device "cuda" --output_path "models/abeja-gpt-neox-japanese-2.7b/result.json"
+TASK="jcommonsenseqa-1.1-0.2,jnli-1.1-0.2,marc_ja-1.1-0.2,jsquad-1.1-0.2,xlsum_ja"
+python main.py --model hf-causal --model_args $MODEL_ARGS --tasks $TASK --num_fewshot "2,3,3,3,1" --device "cuda" --output_path "models/abeja-gpt-neox-japanese-2.7b/result.json"
diff --git a/models/abeja-gpt-neox-japanese-2.7b/result.jaqket_v1.json b/models/abeja-gpt-neox-japanese-2.7b/result.jaqket_v1.json
diff --git a/models/abeja-gpt-neox-japanese-2.7b/result.json b/models/abeja-gpt-neox-japanese-2.7b/result.json
@@ -28,10 +28,6 @@
     "xwinograd_ja": {
       "acc": 0.6037539103232534,
       "acc_stderr": 0.01580264261655725
-    },
-    "jaqket_v2-0.2-0.2": {
-      "exact_match": 16.237113402061855,
-      "f1": 22.56473051576145
     }
   },
   "versions": {
@@ -40,8 +36,7 @@
     "jsquad-1.1-0.2": 1.1,
     "marc_ja-1.1-0.2": 1.1,
     "xlsum_ja": 1.0,
-    "xwinograd_ja": 1.0,
-    "jaqket_v2-0.2-0.2": 0.2,
+    "xwinograd_ja": 1.0
   },
   "config": {
     "model": "hf-causal",

diff --git a/models/cyberagent/cyberagent-open-calm-1b/harness.jaqket_v1.sh b/models/cyberagent/cyberagent-open-calm-1b/harness.jaqket_v1.sh
diff --git a/models/cyberagent/cyberagent-open-calm-1b/harness.sh b/models/cyberagent/cyberagent-open-calm-1b/harness.sh
@@ -1,3 +1,3 @@
 MODEL_ARGS="pretrained=cyberagent/open-calm-1b"
-TASK="jaqket_v2-0.2-0.2,jcommonsenseqa-1.1-0.2,jnli-1.1-0.2,marc_ja-1.1-0.2,jsquad-1.1-0.2,xlsum_ja"
-python main.py --model hf-causal --model_args $MODEL_ARGS --tasks $TASK --num_fewshot "1,2,3,3,3,1" --device "cuda" --output_path "models/cyberagent-open-calm-1b/result.json"
+TASK="jcommonsenseqa-1.1-0.2,jnli-1.1-0.2,marc_ja-1.1-0.2,jsquad-1.1-0.2,xlsum_ja"
+python main.py --model hf-causal --model_args $MODEL_ARGS --tasks $TASK --num_fewshot "2,3,3,3,1" --device "cuda" --output_path "models/cyberagent-open-calm-1b/result.json"
diff --git a/models/cyberagent/cyberagent-open-calm-1b/result.jaqket_v1.json b/models/cyberagent/cyberagent-open-calm-1b/result.jaqket_v1.json
diff --git a/models/cyberagent/cyberagent-open-calm-1b/result.json b/models/cyberagent/cyberagent-open-calm-1b/result.json
@@ -1,21 +1,10 @@
 {
   "results": {
-    "jaqket_v2-0.2-0.2": {
-      "exact_match": 41.92439862542955,
-      "f1": 47.126090309079984
-    },
-    "jsquad-1.1-0.2": {
-      "exact_match": 37.12291760468258,
-      "f1": 47.16735348285718
-    },
-    "xlsum_ja": {
-      "rouge2": 2.288077088085482
-    },
     "jcommonsenseqa-1.1-0.2": {
-      "acc": 0.2725647899910634,
-      "acc_stderr": 0.01331714516405032,
-      "acc_norm": 0.2520107238605898,
-      "acc_norm_stderr": 0.01298483079891409
+      "acc": 0.26899016979445933,
+      "acc_stderr": 0.013261996572328063,
+      "acc_norm": 0.24754244861483468,
+      "acc_norm_stderr": 0.01290758346346734
     },
     "jnli-1.1-0.2": {
       "acc": 0.33566146261298274,
@@ -24,30 +13,41 @@
       "acc_norm_stderr": 0.009556042193601356
     },
     "marc_ja-1.1-0.2": {
-      "acc": 0.746904846126636,
-      "acc_stderr": 0.005782758567450378,
-      "acc_norm": 0.746904846126636,
-      "acc_norm_stderr": 0.005782758567450378
+      "acc": 0.7792117195674921,
+      "acc_stderr": 0.005478034657719626,
+      "acc_norm": 0.7792117195674921,
+      "acc_norm_stderr": 0.005478034657719626
+    },
+    "jsquad-1.1-0.2": {
+      "exact_match": 37.12291760468258,
+      "f1": 47.171446643186265
+    },
+    "xlsum_ja": {
+      "rouge2": 2.288077088085482
+    },
+    "xwinograd_ja": {
+      "acc": 0.6089676746611054,
+      "acc_stderr": 0.015765969995357912
     }
   },
   "versions": {
-    "jaqket_v2-0.2-0.2": 0.2,
     "jcommonsenseqa-1.1-0.2": 1.1,
     "jnli-1.1-0.2": 1.1,
-    "marc_ja-1.1-0.2": 1.1,
     "jsquad-1.1-0.2": 1.1,
-    "xlsum_ja": 1.0
+    "marc_ja-1.1-0.2": 1.1,
+    "xlsum_ja": 1.0,
+    "xwinograd_ja": 1.0
   },
   "config": {
     "model": "hf-causal",
     "model_args": "pretrained=cyberagent/open-calm-1b",
     "num_fewshot": [
-      1,
       2,
       3,
       3,
       3,
-      1
+      1,
+      0
     ],
     "batch_size": null,
     "device": "cuda",

diff --git a/models/cyberagent/cyberagent-open-calm-3b/harness.jaqket_v1.sh b/models/cyberagent/cyberagent-open-calm-3b/harness.jaqket_v1.sh
diff --git a/models/cyberagent/cyberagent-open-calm-3b/harness.jaqket_v2.sh b/models/cyberagent/cyberagent-open-calm-3b/harness.jaqket_v2.sh
diff --git a/models/cyberagent/cyberagent-open-calm-3b/harness.sh b/models/cyberagent/cyberagent-open-calm-3b/harness.sh
@@ -1,3 +1,3 @@
 MODEL_ARGS="pretrained=cyberagent/open-calm-3b"
-TASK="jcommonsenseqa-1.1-0.2,jnli-1.1-0.2,marc_ja-1.1-0.2,jsquad-1.1-0.2,jaqket_v2-0.2-0.2,xlsum_ja,xwinograd_ja,mgsm"
-python main.py --model hf-causal --model_args $MODEL_ARGS --tasks $TASK --num_fewshot "3,3,3,2,1,1,0,5" --device "cuda" --output_path "models/cyberagent/cyberagent-open-calm-3b/result.jaqket_v2.json"
+TASK="jcommonsenseqa-1.1-0.2,jnli-1.1-0.2,marc_ja-1.1-0.2,jsquad-1.1-0.2,jaqket_v2-0.1-0.2,xlsum_ja,xwinograd_ja,mgsm"
+python main.py --model hf-causal --model_args $MODEL_ARGS --tasks $TASK --num_fewshot "3,3,3,2,1,1,0,5" --device "cuda" --output_path "models/cyberagent/cyberagent-open-calm-3b/result.json"
diff --git a/models/cyberagent/cyberagent-open-calm-3b/result.jaqket_v1.json b/models/cyberagent/cyberagent-open-calm-3b/result.jaqket_v1.json
diff --git a/models/cyberagent/cyberagent-open-calm-3b/result.json b/models/cyberagent/cyberagent-open-calm-3b/result.json
@@ -26,9 +26,9 @@
       "exact_match": 40.45475011256191,
       "f1": 52.73709875917724
     },
-    "jaqket_v2-0.2-0.2": {
-      "exact_match": 44.41580756013746,
-      "f1": 49.390784377897766
+    "jaqket_v2-0.1-0.2": {
+      "exact_match": 46.90721649484536,
+      "f1": 51.615597556319194
     },
     "xlsum_ja": {
       "rouge2": 1.948450071736146
@@ -43,7 +43,7 @@
     "jnli-1.1-0.2": 1.1,
     "marc_ja-1.1-0.2": 1.1,
     "jsquad-1.1-0.2": 1.1,
-    "jaqket_v2-0.2-0.2": 0.2,
+    "jaqket_v2-0.1-0.2": 0.1,
     "xlsum_ja": 1.0,
     "xwinograd_ja": 1.0,
     "mgsm": 1.0

diff --git a/models/cyberagent/cyberagent-open-calm-large/harness.jaqket_v1.sh b/models/cyberagent/cyberagent-open-calm-large/harness.jaqket_v1.sh
diff --git a/models/cyberagent/cyberagent-open-calm-large/harness.sh b/models/cyberagent/cyberagent-open-calm-large/harness.sh
@@ -1,3 +1,3 @@
 MODEL_ARGS="pretrained=cyberagent/open-calm-large,use_fast=True"
-TASK="jaqket_v2-0.2-0.2,jcommonsenseqa-1.1-0.2,jnli-1.1-0.2,marc_ja-1.1-0.2,jsquad-1.1-0.2,xlsum_ja"
-python main.py --model hf-causal --model_args $MODEL_ARGS --tasks $TASK --num_fewshot "1,2,3,3,3,1" --device "cuda" --output_path "models/cyberagent-open-calm-large/result.json"
+TASK="jcommonsenseqa-1.1-0.2,jnli-1.1-0.2,marc_ja-1.1-0.2,jsquad-1.1-0.2,xlsum_ja"
+python main.py --model hf-causal --model_args $MODEL_ARGS --tasks $TASK --num_fewshot "2,3,3,3,1" --device "cuda" --output_path "models/cyberagent-open-calm-large/result.json"
diff --git a/models/cyberagent/cyberagent-open-calm-large/result.jaqket_v1.json b/models/cyberagent/cyberagent-open-calm-large/result.jaqket_v1.json
diff --git a/models/cyberagent/cyberagent-open-calm-large/result.json b/models/cyberagent/cyberagent-open-calm-large/result.json
@@ -1,16 +1,5 @@
 {
   "results": {
-    "jaqket_v2-0.2-0.2": {
-      "exact_match": 44.58762886597938,
-      "f1": 49.138445079166715
-    },
-    "jsquad-1.1-0.2": {
-      "exact_match": 37.23547951373255,
-      "f1": 48.50349592141573
-    },
-    "xlsum_ja": {
-      "rouge2": 1.9854375467671679
-    },
     "jcommonsenseqa-1.1-0.2": {
       "acc": 0.2993744414655943,
       "acc_stderr": 0.013697125864334919,
@@ -24,30 +13,41 @@
       "acc_norm_stderr": 0.009815408241248635
     },
     "marc_ja-1.1-0.2": {
-      "acc": 0.7507958967102936,
-      "acc_stderr": 0.005753061901645736,
-      "acc_norm": 0.7507958967102936,
-      "acc_norm_stderr": 0.005753061901645736
+      "acc": 0.7912452040460412,
+      "acc_stderr": 0.005367632889806105,
+      "acc_norm": 0.7912452040460412,
+      "acc_norm_stderr": 0.005367632889806105
+    },
+    "jsquad-1.1-0.2": {
+      "exact_match": 37.23547951373255,
+      "f1": 48.50349592141573
+    },
+    "xlsum_ja": {
+      "rouge2": 1.9854375467671679
+    },
+    "xwinograd_ja": {
+      "acc": 0.6152241918665277,
+      "acc_stderr": 0.015719467393137274
     }
   },
   "versions": {
-    "jaqket_v2-0.2-0.2": 0.2,
     "jcommonsenseqa-1.1-0.2": 1.1,
     "jnli-1.1-0.2": 1.1,
-    "marc_ja-1.1-0.2": 1.1,
     "jsquad-1.1-0.2": 1.1,
-    "xlsum_ja": 1.0
+    "marc_ja-1.1-0.2": 1.1,
+    "xlsum_ja": 1.0,
+    "xwinograd_ja": 1.0
   },
   "config": {
     "model": "hf-causal",
     "model_args": "pretrained=cyberagent/open-calm-large,use_fast=True",
     "num_fewshot": [
-      1,
       2,
       3,
       3,
       3,
-      1
+      1,
+      0
     ],
     "batch_size": null,
     "device": "cuda",

diff --git a/models/cyberagent/cyberagent-open-calm-medium/harness.jaqket_v1.sh b/models/cyberagent/cyberagent-open-calm-medium/harness.jaqket_v1.sh
diff --git a/models/cyberagent/cyberagent-open-calm-medium/harness.sh b/models/cyberagent/cyberagent-open-calm-medium/harness.sh
@@ -1,3 +1,3 @@
 MODEL_ARGS="pretrained=cyberagent/open-calm-medium,use_fast=True"
-TASK="jaqket_v2-0.2-0.2,jcommonsenseqa-1.1-0.2,jnli-1.1-0.2,marc_ja-1.1-0.2,jsquad-1.1-0.2,xlsum_ja"
-python main.py --model hf-causal --model_args $MODEL_ARGS --tasks $TASK --num_fewshot "1,2,3,3,3,1" --device "cuda" --output_path "models/cyberagent-open-calm-medium/result.json"
+TASK="jcommonsenseqa-1.1-0.2,jnli-1.1-0.2,marc_ja-1.1-0.2,jsquad-1.1-0.2,xlsum_ja"
+python main.py --model hf-causal --model_args $MODEL_ARGS --tasks $TASK --num_fewshot "2,3,3,3,1" --device "cuda" --output_path "models/cyberagent-open-calm-medium/result.json"
diff --git a/models/cyberagent/cyberagent-open-calm-medium/result.jaqket_v1.json b/models/cyberagent/cyberagent-open-calm-medium/result.jaqket_v1.json