Skip to content

Commit

Permalink
Merge pull request #23 from naver/feature/mt-dnn
Browse files Browse the repository at this point in the history 
Multi-Task Learning (+ ReadingComprehension, TokenClassification)
  • Loading branch information
@DongjunLee
DongjunLee committed Oct 3, 2019
2 parents cc02437 + 9eb64ef commit 56422a8
Show file tree
Hide file tree
Showing 21 changed files with 51 additions and 51 deletions.
2 changes: 1 addition & 1 deletion base_config/glue/cola_bert.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
}
},
"trainer": {
"log_dir": "logs/glue/cola_bert_base",
"log_dir": "logs/glue/cola_bert",
"num_epochs": 5,
"early_stopping_threshold": 10,
"metric_key": "matthews_corr",
Expand Down
2 changes: 1 addition & 1 deletion base_config/glue/cola_roberta.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@
}
},
"trainer": {
"log_dir": "logs/glue/cola_roberta_base",
"log_dir": "logs/glue/cola_roberta",
"num_epochs": 10,
"early_stopping_threshold": 20,
"metric_key": "matthews_corr",
Expand Down
2 changes: 1 addition & 1 deletion base_config/glue/mnlim_bert.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
}
},
"trainer": {
"log_dir": "logs/glue/mnlim_bert_base",
"log_dir": "logs/glue/mnlim_bert",
"num_epochs": 5,
"early_stopping_threshold": 10,
"metric_key": "accuracy",
Expand Down
2 changes: 1 addition & 1 deletion base_config/glue/mnlim_roberta.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
}
},
"trainer": {
"log_dir": "logs/glue/mnlim_roberta_base",
"log_dir": "logs/glue/mnlim_roberta",
"num_epochs": 10,
"early_stopping_threshold": 10,
"metric_key": "accuracy",
Expand Down
2 changes: 1 addition & 1 deletion base_config/glue/mnlimm_bert.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
}
},
"trainer": {
"log_dir": "logs/glue/mnlimm_bert_base",
"log_dir": "logs/glue/mnlimm_bert",
"num_epochs": 5,
"early_stopping_threshold": 10,
"metric_key": "accuracy",
Expand Down
2 changes: 1 addition & 1 deletion base_config/glue/mnlimm_roberta.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
}
},
"trainer": {
"log_dir": "logs/glue/mnlimm_roberta_base",
"log_dir": "logs/glue/mnlimm_roberta",
"num_epochs": 10,
"early_stopping_threshold": 10,
"metric_key": "accuracy",
Expand Down
2 changes: 1 addition & 1 deletion base_config/glue/mrpc_bert.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
}
},
"trainer": {
"log_dir": "logs/glue/mrpc_bert_base",
"log_dir": "logs/glue/mrpc_bert",
"num_epochs": 5,
"early_stopping_threshold": 10,
"metric_key": "f1",
Expand Down
2 changes: 1 addition & 1 deletion base_config/glue/mrpc_roberta.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
}
},
"trainer": {
"log_dir": "logs/glue/mrpc_roberta_base",
"log_dir": "logs/glue/mrpc_roberta",
"num_epochs": 10,
"early_stopping_threshold": 10,
"metric_key": "f1",
Expand Down
2 changes: 1 addition & 1 deletion base_config/glue/qnli_bert.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
}
},
"trainer": {
"log_dir": "logs/glue/qnli_bert_base",
"log_dir": "logs/glue/qnli_bert",
"num_epochs": 5,
"early_stopping_threshold": 10,
"metric_key": "accuracy",
Expand Down
2 changes: 1 addition & 1 deletion base_config/glue/qnli_roberta.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
}
},
"trainer": {
"log_dir": "logs/glue/qnli_roberta_base",
"log_dir": "logs/glue/qnli_roberta",
"num_epochs": 10,
"early_stopping_threshold": 10,
"metric_key": "accuracy",
Expand Down
2 changes: 1 addition & 1 deletion base_config/glue/qqp_bert.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
}
},
"trainer": {
"log_dir": "logs/glue/qqp_bert_base",
"log_dir": "logs/glue/qqp_bert",
"num_epochs": 5,
"early_stopping_threshold": 10,
"metric_key": "f1",
Expand Down
2 changes: 1 addition & 1 deletion base_config/glue/qqp_roberta.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
}
},
"trainer": {
"log_dir": "logs/glue/qqp_roberta_base",
"log_dir": "logs/glue/qqp_roberta",
"num_epochs": 10,
"early_stopping_threshold": 10,
"metric_key": "f1",
Expand Down
2 changes: 1 addition & 1 deletion base_config/glue/rte_bert.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
}
},
"trainer": {
"log_dir": "logs/glue/rte_bert_base",
"log_dir": "logs/glue/rte_bert",
"num_epochs": 5,
"early_stopping_threshold": 10,
"metric_key": "accuracy",
Expand Down
2 changes: 1 addition & 1 deletion base_config/glue/rte_roberta.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
}
},
"trainer": {
"log_dir": "logs/glue/rte_roberta_base",
"log_dir": "logs/glue/rte_roberta",
"num_epochs": 20,
"early_stopping_threshold": 10,
"metric_key": "accuracy",
Expand Down
2 changes: 1 addition & 1 deletion base_config/glue/sst_bert.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
}
},
"trainer": {
"log_dir": "logs/glue/sst_bert_base",
"log_dir": "logs/glue/sst_bert",
"num_epochs": 5,
"early_stopping_threshold": 10,
"metric_key": "accuracy",
Expand Down
2 changes: 1 addition & 1 deletion base_config/glue/sst_roberta.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
}
},
"trainer": {
"log_dir": "logs/glue/sst_roberta_base",
"log_dir": "logs/glue/sst_roberta",
"num_epochs": 10,
"early_stopping_threshold": 10,
"metric_key": "accuracy",
Expand Down
2 changes: 1 addition & 1 deletion base_config/glue/stsb_bert.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
}
},
"trainer": {
"log_dir": "logs/glue/stsb_bert_base",
"log_dir": "logs/glue/stsb_bert",
"num_epochs": 5,
"early_stopping_threshold": 10,
"metric_key": "pearson_spearman_corr",
Expand Down
2 changes: 1 addition & 1 deletion base_config/glue/stsb_roberta.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
}
},
"trainer": {
"log_dir": "logs/glue/stsb_roberta_base",
"log_dir": "logs/glue/stsb_roberta",
"num_epochs": 10,
"early_stopping_threshold": 10,
"metric_key": "pearson_spearman_corr",
Expand Down
2 changes: 1 addition & 1 deletion base_config/glue/wnli_bert.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
}
},
"trainer": {
"log_dir": "logs/glue/wnli_bert_base",
"log_dir": "logs/glue/wnli_bert",
"num_epochs": 5,
"early_stopping_threshold": 10,
"metric_key": "accuracy",
Expand Down
2 changes: 1 addition & 1 deletion base_config/glue/wnli_roberta.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
}
},
"trainer": {
"log_dir": "logs/glue/wnli_roberta_base",
"log_dir": "logs/glue/wnli_roberta",
"num_epochs": 10,
"early_stopping_threshold": 30,
"metric_key": "accuracy",
Expand Down
62 changes: 31 additions & 31 deletions docs/reports/glue.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,56 +11,56 @@
| Task (Metric) | Model | CLaF Result | Official Result | BaseConfig |
| ------------- | ----- | ----- | -------- | ---------- |
| **CoLA** (**Matthew's Corr**) | BERT-Base | 59.393 | 52.1 (Test set) | glue/cola_bert_base.json |
| | BERT-Large | - | 60.6 | - |
| | MT-DNN Base | 54.658 | - | multi_task/glue.json -> `fine-fune` |
| | MT-DNN Large | - | 63.5 | - |
| | BERT-Large | 61.151 | 60.6 | - |
| | MT-DNN (BERT) Base | 54.658 | - | multi_task/glue.json -> `fine-fune` |
| | MT-DNN (BERT) Large | - | 63.5 | - |
| | RoBERTa-Base | 64.828 | 63.6 | glue/cola_roberta_base.json |
| | RoBERTa-Large | - | 68.0 | - |
| **MNLI m/mm** (**Accuracy**) | BERT-Base | 83.923/84.306 | 84.6/83.4 (Test set) | glue/mnli{m/mm}_bert_base.json |
| | BERT-Large | - | 86.6/- | - |
| | MT-DNN Base | 84.452/84.225 | - | multi_task/glue.json -> `fine-fune` |
| | MT-DNN Large | - | 87.1/86.7 | - |
| | MT-DNN (BERT) Base | 84.452/84.225 | - | multi_task/glue.json -> `fine-fune` |
| | MT-DNN (BERT) Large | - | 87.1/86.7 | - |
| | RoBERTa-Base | 87.305/87.236 | 87.6/- | glue/mnli{m/mm}_roberta_base.json |
| | RoBERTa-Large | - | 90.2/90.2 | - |
| **MRPC** (**Accuracy/F1**) | BERT-Base | 87.5/91.282 | 88.9 (Test set) | glue/mrpc_bert_base.json |
| | BERT-Large | - | 88.0 | - |
| | MT-DNN Base | 87.5/91.005 | - | multi_task/glue.json -> `fine-fune` |
| | MT-DNN Large | - | 91.0/87.5 | - |
| | BERT-Large | 87.255/90.845 | 88.0 | - |
| | MT-DNN (BERT) Base | 87.5/91.005 | - | multi_task/glue.json -> `fine-fune` |
| | MT-DNN (BERT) Large | - | 91.0/87.5 | - |
| | RoBERTa-Base | 88.480/91.681 | 90.2 | glue/mrpc_roberta_base.json |
| | RoBERTa-Large | - | 90.9 | - |
| | RoBERTa-Large | 90.686/93.214 | 90.9 | - |
| **QNLI** (**Accuracy**) | BERT-Base | 88.521 | 90.5 (Test set) | glue/qnli_bert_base.json |
| | BERT-Large | - | 92.3 | - |
| | MT-DNN Base | - | - | multi_task/glue.json -> `fine-fune` |
| | MT-DNN Large | - | 87.1/86.7 | - |
| | BERT-Large | 90.440 | 92.3 | - |
| | MT-DNN (BERT) Base | - | - | multi_task/glue.json -> `fine-fune` |
| | MT-DNN (BERT) Large | - | 87.1/86.7 | - |
| | RoBERTa-Base | 90.823 | 92.8 | glue/qnli_roberta_base.json |
| | RoBERTa-Large | - | 94.7 | - |
| **QQP** (**Accuracy/F1**) | BERT-Base | 90.378/87.171 | 71.2 (Test set) | glue/qqp_bert_base.json |
| | BERT-Large | - | 91.3 | - |
| | MT-DNN Base | 91.261/88.219 | - | - |
| | MT-DNN Large | - | 87.1/86.7 | - |
| | BERT-Large | 91.640/88.745 | 91.3 | - |
| | MT-DNN (BERT) Base | 91.261/88.219 | - | - |
| | MT-DNN (BERT) Large | - | 87.1/86.7 | - |
| | RoBERTa-Base | 91.541/88.768 | 91.9 | glue/qqp_roberta_base.json |
| | RoBERTa-Large | - | 92.2 | - |
| | RoBERTa-Large | 91.848/89.031 | 92.2 | - |
| **RTE** (**Accuracy**) | BERT-Base | 69.314 | 66.4 (Test set) | glue/rte_bert_base.json |
| | BERT-Large | - | 70.4 | - |
| | MT-DNN Base | 79.422 | - | multi_task/glue.json -> `fine-fune` |
| | MT-DNN Large | - | 83.4 | - |
| | BERT-Large | 69.675 | 70.4 | - |
| | MT-DNN (BERT) Base | 79.422 | - | multi_task/glue.json -> `fine-fune` |
| | MT-DNN (BERT) Large | - | 83.4 | - |
| | RoBERTa-Base | 73.646 | 78.7 | glue/rte_roberta_base.json |
| | RoBERTa-Large | - | 86.6 | - |
| | RoBERTa-Large | 84.838 | 86.6 | - |
| **SST-2** (**Accuracy**) | BERT-Base | 92.546 | 93.5 (Test set) | glue/sst_bert_base.json |
| | BERT-Large | - | 93.2 | - |
| | MT-DNN Base | 93.005 | - | - |
| | MT-DNN Large | - | 94.3 | - |
| | BERT-Large | 93.349 | 93.2 | - |
| | MT-DNN (BERT) Base | 93.005 | - | - |
| | MT-DNN (BERT) Large | - | 94.3 | - |
| | RoBERTa-Base | 94.495 | 94.8 | glue/sst_roberta_base.json |
| | RoBERTa-Large | - | 96.4 | - |
| | RoBERTa-Large | 95.642 | 96.4 | - |
| **STS-B** (**Pearson/Spearman**) | BERT-Base | 88.070/87.881 | 85.8 (Test set) | glue/stsb_bert_base.json |
| | BERT-Large | - | 90.0 | - |
| | MT-DNN Base | 88.444/88.807 | - | multi_task/glue.json -> `fine-fune` |
| | MT-DNN Large | - | 90.7/90.6 | - |
| | BERT-Large | 90.041/89735 | 90.0 | - |
| | MT-DNN (BERT) Base | 88.444/88.807 | - | multi_task/glue.json -> `fine-fune` |
| | MT-DNN (BERT) Large | - | 90.7/90.6 | - |
| | RoBERTa-Base | 89.003/89.094 | 91.2 | glue/stsb_roberta_base.json |
| | RoBERTa-Large | - | 92.4 | - |
| | RoBERTa-Large | 91.980/91.764 | 92.4 | - |
| **WNLI** (**Accuracy**) | BERT-Base | 56.338 | 65.1 (Test set) | glue/wnli_bert_base.json |
| | BERT-Large | - | - | - |
| | MT-DNN Base | 57.746 | - | multi_task/glue.json -> `fine-fune` |
| | MT-DNN Large | - | - | - |
| | BERT-Large | 59.155 | - | - |
| | MT-DNN (BERT) Base | 57.746 | - | multi_task/glue.json -> `fine-fune` |
| | MT-DNN (BERT) Large | - | - | - |
| | RoBERTa-Base | 60.563 | - | glue/wnli_roberta_base.json |
| | RoBERTa-Large | - | 91.3 | - |

0 comments on commit 56422a8

Please sign in to comment.