Merge pull request huggingface#36 from stevezheng23/dev/zheng/coqa

update coqa-ensemble codalab submission pipeline

examples/run_coqa_codalab.ensemble.sh

@@ -37,14 +37,14 @@ mkdir output
 mkdir output/coqa
 mkdir output/coqa/roberta-large-coqa-ensemble
 
-wget -P output/coqa https://storage.googleapis.com/mrc_data/coqa/roberta-large-coqa-20191111.0.zip
-unzip output/coqa/roberta-large-coqa-20191111.0.zip -d output/coqa/
-mv output/coqa/roberta-large-coqa-20191111.0 output/coqa/roberta-large-coqa-0
-rm output/coqa/roberta-large-coqa-20191111.0.zip
+wget -P output/coqa https://storage.googleapis.com/mrc_data/coqa/roberta-large-coqa-20191111.1.zip
+unzip output/coqa/roberta-large-coqa-20191111.1.zip -d output/coqa/
+mv output/coqa/roberta-large-coqa-20191111.1 output/coqa/roberta-large-coqa-1
+rm output/coqa/roberta-large-coqa-20191111.1.zip
 
 CUDA_VISIBLE_DEVICES=0 python run_coqa_ensemble.py \
 --model_type=roberta \
---model_name_or_path=output/coqa/roberta-large-coqa-0 \
+--model_name_or_path=output/coqa/roberta-large-coqa-1 \
 --do_predict \
 --version_2_with_negative \
 --train_file=data/coqa/train-v1.0.json \
@@ -57,7 +57,7 @@ CUDA_VISIBLE_DEVICES=0 python run_coqa_ensemble.py \
 --max_query_length=64 \
 --max_answer_length=16 \
 --doc_stride=128 \
---output_dir=output/coqa/roberta-large-coqa-0 \
+--output_dir=output/coqa/roberta-large-coqa-1 \
 --per_gpu_eval_batch_size=12 \
 --per_gpu_train_batch_size=12 \
 --seed=1000 \
@@ -67,19 +67,22 @@ CUDA_VISIBLE_DEVICES=0 python run_coqa_ensemble.py \
 --enable_special_answer \
 --answer_score_threshold=0.5
 
-cp data/coqa/cached_dev_roberta-large-coqa-0_512 data/coqa/cached_dev_roberta-large-coqa-1_512
-cp data/coqa/cached_dev_roberta-large-coqa-0_512 data/coqa/cached_dev_roberta-large-coqa-2_512
-cp data/coqa/cached_dev_roberta-large-coqa-0_512 data/coqa/cached_dev_roberta-large-coqa-3_512
-cp data/coqa/cached_dev_roberta-large-coqa-0_512 data/coqa/cached_dev_roberta-large-coqa-4_512
+cp data/coqa/cached_dev_roberta-large-coqa-1_512 data/coqa/cached_dev_roberta-large-coqa-2_512
+cp data/coqa/cached_dev_roberta-large-coqa-1_512 data/coqa/cached_dev_roberta-large-coqa-3_512
+cp data/coqa/cached_dev_roberta-large-coqa-1_512 data/coqa/cached_dev_roberta-large-coqa-4_512
+cp data/coqa/cached_dev_roberta-large-coqa-1_512 data/coqa/cached_dev_roberta-large-coqa-5_512
+cp data/coqa/cached_dev_roberta-large-coqa-1_512 data/coqa/cached_dev_roberta-large-coqa-6_512
+cp data/coqa/cached_dev_roberta-large-coqa-1_512 data/coqa/cached_dev_roberta-large-coqa-7_512
+cp data/coqa/cached_dev_roberta-large-coqa-1_512 data/coqa/cached_dev_roberta-large-coqa-8_512
 
-wget -P output/coqa https://storage.googleapis.com/mrc_data/coqa/roberta-large-coqa-20191111.1.zip
-unzip output/coqa/roberta-large-coqa-20191111.1.zip -d output/coqa/
-mv output/coqa/roberta-large-coqa-20191111.1 output/coqa/roberta-large-coqa-1
-rm output/coqa/roberta-large-coqa-20191111.1.zip
+wget -P output/coqa https://storage.googleapis.com/mrc_data/coqa/roberta-large-coqa-20191111.2.zip
+unzip output/coqa/roberta-large-coqa-20191111.2.zip -d output/coqa/
+mv output/coqa/roberta-large-coqa-20191111.2 output/coqa/roberta-large-coqa-2
+rm output/coqa/roberta-large-coqa-20191111.2.zip
 
 CUDA_VISIBLE_DEVICES=0 python run_coqa_ensemble.py \
 --model_type=roberta \
---model_name_or_path=output/coqa/roberta-large-coqa-1 \
+--model_name_or_path=output/coqa/roberta-large-coqa-2 \
 --do_predict \
 --version_2_with_negative \
 --train_file=data/coqa/train-v1.0.json \
@@ -92,7 +95,7 @@ CUDA_VISIBLE_DEVICES=0 python run_coqa_ensemble.py \
 --max_query_length=64 \
 --max_answer_length=16 \
 --doc_stride=128 \
---output_dir=output/coqa/roberta-large-coqa-1 \
+--output_dir=output/coqa/roberta-large-coqa-2 \
 --per_gpu_eval_batch_size=12 \
 --per_gpu_train_batch_size=12 \
 --seed=1000 \
@@ -102,14 +105,14 @@ CUDA_VISIBLE_DEVICES=0 python run_coqa_ensemble.py \
 --enable_special_answer \
 --answer_score_threshold=0.5
 
-wget -P output/coqa https://storage.googleapis.com/mrc_data/coqa/roberta-large-coqa-20191111.2.zip
-unzip output/coqa/roberta-large-coqa-20191111.2.zip -d output/coqa/
-mv output/coqa/roberta-large-coqa-20191111.2 output/coqa/roberta-large-coqa-2
-rm output/coqa/roberta-large-coqa-20191111.2.zip
+wget -P output/coqa https://storage.googleapis.com/mrc_data/coqa/roberta-large-coqa-20191111.3.zip
+unzip output/coqa/roberta-large-coqa-20191111.3.zip -d output/coqa/
+mv output/coqa/roberta-large-coqa-20191111.3 output/coqa/roberta-large-coqa-3
+rm output/coqa/roberta-large-coqa-20191111.3.zip
 
 CUDA_VISIBLE_DEVICES=0 python run_coqa_ensemble.py \
 --model_type=roberta \
---model_name_or_path=output/coqa/roberta-large-coqa-2 \
+--model_name_or_path=output/coqa/roberta-large-coqa-3 \
 --do_predict \
 --version_2_with_negative \
 --train_file=data/coqa/train-v1.0.json \
@@ -122,7 +125,7 @@ CUDA_VISIBLE_DEVICES=0 python run_coqa_ensemble.py \
 --max_query_length=64 \
 --max_answer_length=16 \
 --doc_stride=128 \
---output_dir=output/coqa/roberta-large-coqa-2 \
+--output_dir=output/coqa/roberta-large-coqa-3 \
 --per_gpu_eval_batch_size=12 \
 --per_gpu_train_batch_size=12 \
 --seed=1000 \
@@ -132,14 +135,14 @@ CUDA_VISIBLE_DEVICES=0 python run_coqa_ensemble.py \
 --enable_special_answer \
 --answer_score_threshold=0.5
 
-wget -P output/coqa https://storage.googleapis.com/mrc_data/coqa/roberta-large-coqa-20191111.3.zip
-unzip output/coqa/roberta-large-coqa-20191111.3.zip -d output/coqa/
-mv output/coqa/roberta-large-coqa-20191111.3 output/coqa/roberta-large-coqa-3
-rm output/coqa/roberta-large-coqa-20191111.3.zip
+wget -P output/coqa https://storage.googleapis.com/mrc_data/coqa/roberta-large-coqa-20191111.4.zip
+unzip output/coqa/roberta-large-coqa-20191111.4.zip -d output/coqa/
+mv output/coqa/roberta-large-coqa-20191111.4 output/coqa/roberta-large-coqa-4
+rm output/coqa/roberta-large-coqa-20191111.4.zip
 
 CUDA_VISIBLE_DEVICES=0 python run_coqa_ensemble.py \
 --model_type=roberta \
---model_name_or_path=output/coqa/roberta-large-coqa-3 \
+--model_name_or_path=output/coqa/roberta-large-coqa-4 \
 --do_predict \
 --version_2_with_negative \
 --train_file=data/coqa/train-v1.0.json \
@@ -152,7 +155,7 @@ CUDA_VISIBLE_DEVICES=0 python run_coqa_ensemble.py \
 --max_query_length=64 \
 --max_answer_length=16 \
 --doc_stride=128 \
---output_dir=output/coqa/roberta-large-coqa-3 \
+--output_dir=output/coqa/roberta-large-coqa-4 \
 --per_gpu_eval_batch_size=12 \
 --per_gpu_train_batch_size=12 \
 --seed=1000 \
@@ -162,14 +165,14 @@ CUDA_VISIBLE_DEVICES=0 python run_coqa_ensemble.py \
 --enable_special_answer \
 --answer_score_threshold=0.5
 
-wget -P output/coqa https://storage.googleapis.com/mrc_data/coqa/roberta-large-coqa-20191111.4.zip
-unzip output/coqa/roberta-large-coqa-20191111.4.zip -d output/coqa/
-mv output/coqa/roberta-large-coqa-20191111.4 output/coqa/roberta-large-coqa-4
-rm output/coqa/roberta-large-coqa-20191111.4.zip
+wget -P output/coqa https://storage.googleapis.com/mrc_data/coqa/roberta-large-coqa-20191111.5.zip
+unzip output/coqa/roberta-large-coqa-20191111.5.zip -d output/coqa/
+mv output/coqa/roberta-large-coqa-20191111.5 output/coqa/roberta-large-coqa-5
+rm output/coqa/roberta-large-coqa-20191111.5.zip
 
 CUDA_VISIBLE_DEVICES=0 python run_coqa_ensemble.py \
 --model_type=roberta \
---model_name_or_path=output/coqa/roberta-large-coqa-4 \
+--model_name_or_path=output/coqa/roberta-large-coqa-5 \
 --do_predict \
 --version_2_with_negative \
 --train_file=data/coqa/train-v1.0.json \
@@ -182,7 +185,97 @@ CUDA_VISIBLE_DEVICES=0 python run_coqa_ensemble.py \
 --max_query_length=64 \
 --max_answer_length=16 \
 --doc_stride=128 \
---output_dir=output/coqa/roberta-large-coqa-4 \
+--output_dir=output/coqa/roberta-large-coqa-5 \
+--per_gpu_eval_batch_size=12 \
+--per_gpu_train_batch_size=12 \
+--seed=1000 \
+--warmup_steps=500 \
+--save_steps=1000 \
+--weight_decay=0.01 \
+--enable_special_answer \
+--answer_score_threshold=0.5
+
+wget -P output/coqa https://storage.googleapis.com/mrc_data/coqa/roberta-large-coqa-20191111.6.zip
+unzip output/coqa/roberta-large-coqa-20191111.6.zip -d output/coqa/
+mv output/coqa/roberta-large-coqa-20191111.6 output/coqa/roberta-large-coqa-6
+rm output/coqa/roberta-large-coqa-20191111.6.zip
+
+CUDA_VISIBLE_DEVICES=0 python run_coqa_ensemble.py \
+--model_type=roberta \
+--model_name_or_path=output/coqa/roberta-large-coqa-6 \
+--do_predict \
+--version_2_with_negative \
+--train_file=data/coqa/train-v1.0.json \
+--predict_file=data/coqa/dev-v1.0.json \
+--example_file=data/coqa/dev-v1.0.json \
+--feature_path=data/coqa/ensemble/ \
+--learning_rate=1.5e-5 \
+--num_train_epochs=2 \
+--max_seq_length=512 \
+--max_query_length=64 \
+--max_answer_length=16 \
+--doc_stride=128 \
+--output_dir=output/coqa/roberta-large-coqa-6 \
+--per_gpu_eval_batch_size=12 \
+--per_gpu_train_batch_size=12 \
+--seed=1000 \
+--warmup_steps=500 \
+--save_steps=1000 \
+--weight_decay=0.01 \
+--enable_special_answer \
+--answer_score_threshold=0.5
+
+wget -P output/coqa https://storage.googleapis.com/mrc_data/coqa/roberta-large-coqa-20191111.7.zip
+unzip output/coqa/roberta-large-coqa-20191111.7.zip -d output/coqa/
+mv output/coqa/roberta-large-coqa-20191111.7 output/coqa/roberta-large-coqa-7
+rm output/coqa/roberta-large-coqa-20191111.7.zip
+
+CUDA_VISIBLE_DEVICES=0 python run_coqa_ensemble.py \
+--model_type=roberta \
+--model_name_or_path=output/coqa/roberta-large-coqa-7 \
+--do_predict \
+--version_2_with_negative \
+--train_file=data/coqa/train-v1.0.json \
+--predict_file=data/coqa/dev-v1.0.json \
+--example_file=data/coqa/dev-v1.0.json \
+--feature_path=data/coqa/ensemble/ \
+--learning_rate=1.5e-5 \
+--num_train_epochs=2 \
+--max_seq_length=512 \
+--max_query_length=64 \
+--max_answer_length=16 \
+--doc_stride=128 \
+--output_dir=output/coqa/roberta-large-coqa-7 \
+--per_gpu_eval_batch_size=12 \
+--per_gpu_train_batch_size=12 \
+--seed=1000 \
+--warmup_steps=500 \
+--save_steps=1000 \
+--weight_decay=0.01 \
+--enable_special_answer \
+--answer_score_threshold=0.5
+
+wget -P output/coqa https://storage.googleapis.com/mrc_data/coqa/roberta-large-coqa-20191111.8.zip
+unzip output/coqa/roberta-large-coqa-20191111.8.zip -d output/coqa/
+mv output/coqa/roberta-large-coqa-20191111.8 output/coqa/roberta-large-coqa-8
+rm output/coqa/roberta-large-coqa-20191111.8.zip
+
+CUDA_VISIBLE_DEVICES=0 python run_coqa_ensemble.py \
+--model_type=roberta \
+--model_name_or_path=output/coqa/roberta-large-coqa-8 \
+--do_predict \
+--version_2_with_negative \
+--train_file=data/coqa/train-v1.0.json \
+--predict_file=data/coqa/dev-v1.0.json \
+--example_file=data/coqa/dev-v1.0.json \
+--feature_path=data/coqa/ensemble/ \
+--learning_rate=1.5e-5 \
+--num_train_epochs=2 \
+--max_seq_length=512 \
+--max_query_length=64 \
+--max_answer_length=16 \
+--doc_stride=128 \
+--output_dir=output/coqa/roberta-large-coqa-8 \
 --per_gpu_eval_batch_size=12 \
 --per_gpu_train_batch_size=12 \
 --seed=1000 \
@@ -192,15 +285,18 @@ CUDA_VISIBLE_DEVICES=0 python run_coqa_ensemble.py \
 --enable_special_answer \
 --answer_score_threshold=0.5
 
-mv data/coqa/updated_dev_roberta-large-coqa-0_512 data/coqa/ensemble
 mv data/coqa/updated_dev_roberta-large-coqa-1_512 data/coqa/ensemble
 mv data/coqa/updated_dev_roberta-large-coqa-2_512 data/coqa/ensemble
 mv data/coqa/updated_dev_roberta-large-coqa-3_512 data/coqa/ensemble
 mv data/coqa/updated_dev_roberta-large-coqa-4_512 data/coqa/ensemble
+mv data/coqa/updated_dev_roberta-large-coqa-5_512 data/coqa/ensemble
+mv data/coqa/updated_dev_roberta-large-coqa-6_512 data/coqa/ensemble
+mv data/coqa/updated_dev_roberta-large-coqa-7_512 data/coqa/ensemble
+mv data/coqa/updated_dev_roberta-large-coqa-8_512 data/coqa/ensemble
 
 CUDA_VISIBLE_DEVICES=0 python run_coqa_ensemble.py \
 --model_type=roberta \
---model_name_or_path=output/coqa/roberta-large-coqa-0 \
+--model_name_or_path=output/coqa/roberta-large-coqa-1 \
 --do_ensemble \
 --version_2_with_negative \
 --train_file=data/coqa/train-v1.0.json \