Merge branch 'master' into kylel/finetune_seq_tag

README.md

@@ -105,9 +105,10 @@ Where `[serialization-directory]` is the path to an output directory where the m
 If you use `SciBERT` in your research, please cite [SciBERT: Pretrained Contextualized Embeddings for Scientific Text](https://arxiv.org/abs/1903.10676).
 ```
 @inproceedings{Beltagy2019SciBERT,
-  title={SciBERT: Pretrained Contextualized Embeddings for Scientific Text},
-  author={Iz Beltagy and Arman Cohan and Kyle Lo},
+  title={SciBERT: Pretrained Language Model for Scientific Text},
+  author={Iz Beltagy and Kyle Lo and Arman Cohan},
   year={2019},
+  booktitle={EMNLP},
   Eprint={arXiv:1903.10676}
 }
 ```

scripts/cheatsheet.txt

@@ -5,6 +5,8 @@
 import sentencepiece as spm 
 spm.SentencePieceTrainer.Train('--input=combined.out --model_prefix=100B_9999_cased --vocab_size=31000 --character_coverage=0.9999 --model_type=bpe --input_sentence_size=100000000 --shuffle_input_sentence=true')
 
+# Create TPU using ctpu
+ctpu up -name ctpu-up3 -tpu-size=v3-8 -gcp-network=main -tpu-only
 
 # Run BERT training for sequences of length 128
 python3 run_pretraining.py --input_file=gs://s2-bert/s2-tfRecords/tfRecords_s2vocab_uncased_128/*.tfrecord --output_dir=gs://s2-bert/s2-models/3B-s2vocab_uncased_128  --do_train=True --do_eval=True --bert_config_file=/mnt/disk1/bert_config/s2vocab_uncased.json --train_batch_size=256 --max_seq_length=128 --max_predictions_per_seq=20 --num_train_steps=500000 --num_warmup_steps=1000 --learning_rate=1e-4 --use_tpu=True --tpu_name=node-3 --max_eval_steps=2000  --eval_batch_size 256  --init_checkpoint=gs://s2-bert/s2-models/3B-s2vocab_uncased_128 --tpu_zone=us-central1-a