Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
updated logging and saving metrics (#10436)
* updated logging and saving metrics

* space removal
  • Loading branch information
bhadreshpsavani committed Feb 27, 2021
1 parent f52a158 commit aca6288
Show file tree
Hide file tree
Showing 12 changed files with 71 additions and 191 deletions.
22 changes: 6 additions & 16 deletions examples/language-modeling/run_clm.py
Expand Up @@ -375,16 +375,11 @@ def group_texts(examples):
train_result = trainer.train(resume_from_checkpoint=checkpoint)
trainer.save_model() # Saves the tokenizer too for easy upload

output_train_file = os.path.join(training_args.output_dir, "train_results.txt")
if trainer.is_world_process_zero():
with open(output_train_file, "w") as writer:
logger.info("***** Train results *****")
for key, value in sorted(train_result.metrics.items()):
logger.info(f" {key} = {value}")
writer.write(f"{key} = {value}\n")
metrics = train_result.metrics

# Need to save the state, since Trainer.save_model saves only the tokenizer with the model
trainer.state.save_to_json(os.path.join(training_args.output_dir, "trainer_state.json"))
trainer.log_metrics("train", metrics)
trainer.save_metrics("train", metrics)
trainer.save_state()

# Evaluation
results = {}
Expand All @@ -396,13 +391,8 @@ def group_texts(examples):
perplexity = math.exp(eval_output["eval_loss"])
results["perplexity"] = perplexity

output_eval_file = os.path.join(training_args.output_dir, "eval_results_clm.txt")
if trainer.is_world_process_zero():
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key, value in sorted(results.items()):
logger.info(f" {key} = {value}")
writer.write(f"{key} = {value}\n")
trainer.log_metrics("eval", results)
trainer.save_metrics("eval", results)

return results

Expand Down
23 changes: 6 additions & 17 deletions examples/language-modeling/run_mlm.py
Expand Up @@ -411,17 +411,11 @@ def group_texts(examples):
checkpoint = None
train_result = trainer.train(resume_from_checkpoint=checkpoint)
trainer.save_model() # Saves the tokenizer too for easy upload
metrics = train_result.metrics

output_train_file = os.path.join(training_args.output_dir, "train_results.txt")
if trainer.is_world_process_zero():
with open(output_train_file, "w") as writer:
logger.info("***** Train results *****")
for key, value in sorted(train_result.metrics.items()):
logger.info(f" {key} = {value}")
writer.write(f"{key} = {value}\n")

# Need to save the state, since Trainer.save_model saves only the tokenizer with the model
trainer.state.save_to_json(os.path.join(training_args.output_dir, "trainer_state.json"))
trainer.log_metrics("train", metrics)
trainer.save_metrics("train", metrics)
trainer.save_state()

# Evaluation
results = {}
Expand All @@ -433,13 +427,8 @@ def group_texts(examples):
perplexity = math.exp(eval_output["eval_loss"])
results["perplexity"] = perplexity

output_eval_file = os.path.join(training_args.output_dir, "eval_results_mlm.txt")
if trainer.is_world_process_zero():
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key, value in sorted(results.items()):
logger.info(f" {key} = {value}")
writer.write(f"{key} = {value}\n")
trainer.log_metrics("eval", results)
trainer.save_metrics("eval", results)

return results

Expand Down
23 changes: 6 additions & 17 deletions examples/language-modeling/run_plm.py
Expand Up @@ -392,17 +392,11 @@ def group_texts(examples):
checkpoint = None
train_result = trainer.train(resume_from_checkpoint=checkpoint)
trainer.save_model() # Saves the tokenizer too for easy upload
metrics = train_result.metrics

output_train_file = os.path.join(training_args.output_dir, "train_results.txt")
if trainer.is_world_process_zero():
with open(output_train_file, "w") as writer:
logger.info("***** Train results *****")
for key, value in sorted(train_result.metrics.items()):
logger.info(f" {key} = {value}")
writer.write(f"{key} = {value}\n")

# Need to save the state, since Trainer.save_model saves only the tokenizer with the model
trainer.state.save_to_json(os.path.join(training_args.output_dir, "trainer_state.json"))
trainer.log_metrics("train", metrics)
trainer.save_metrics("train", metrics)
trainer.save_state()

# Evaluation
results = {}
Expand All @@ -414,13 +408,8 @@ def group_texts(examples):
perplexity = math.exp(eval_output["eval_loss"])
results["perplexity"] = perplexity

output_eval_file = os.path.join(training_args.output_dir, "eval_results_plm.txt")
if trainer.is_world_process_zero():
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key, value in sorted(results.items()):
logger.info(f" {key} = {value}")
writer.write(f"{key} = {value}\n")
trainer.log_metrics("eval", results)
trainer.save_metrics("eval", results)

return results

Expand Down
25 changes: 8 additions & 17 deletions examples/multiple-choice/run_swag.py
Expand Up @@ -227,6 +227,8 @@ def main():
# Set the verbosity to info of the Transformers logger (on main process only):
if is_main_process(training_args.local_rank):
transformers.utils.logging.set_verbosity_info()
transformers.utils.logging.enable_default_handler()
transformers.utils.logging.enable_explicit_format()
logger.info("Training/evaluation parameters %s", training_args)

# Set seed before initializing model.
Expand Down Expand Up @@ -367,17 +369,11 @@ def compute_metrics(eval_predictions):
checkpoint = None
train_result = trainer.train(resume_from_checkpoint=checkpoint)
trainer.save_model() # Saves the tokenizer too for easy upload
metrics = train_result.metrics

output_train_file = os.path.join(training_args.output_dir, "train_results.txt")
if trainer.is_world_process_zero():
with open(output_train_file, "w") as writer:
logger.info("***** Train results *****")
for key, value in sorted(train_result.metrics.items()):
logger.info(f" {key} = {value}")
writer.write(f"{key} = {value}\n")

# Need to save the state, since Trainer.save_model saves only the tokenizer with the model
trainer.state.save_to_json(os.path.join(training_args.output_dir, "trainer_state.json"))
trainer.log_metrics("train", metrics)
trainer.save_metrics("train", metrics)
trainer.save_state()

# Evaluation
results = {}
Expand All @@ -386,13 +382,8 @@ def compute_metrics(eval_predictions):

results = trainer.evaluate()

output_eval_file = os.path.join(training_args.output_dir, "eval_results_swag.txt")
if trainer.is_world_process_zero():
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key, value in sorted(results.items()):
logger.info(f" {key} = {value}")
writer.write(f"{key} = {value}\n")
trainer.log_metrics("eval", results)
trainer.save_metrics("eval", results)

return results

Expand Down
12 changes: 4 additions & 8 deletions examples/multiple-choice/run_tf_multiple_choice.py
Expand Up @@ -206,14 +206,10 @@ def compute_metrics(p: EvalPrediction) -> Dict:

result = trainer.evaluate()

output_eval_file = os.path.join(training_args.output_dir, "eval_results.txt")
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key, value in result.items():
logger.info(" %s = %s", key, value)
writer.write("%s = %s\n" % (key, value))

results.update(result)
trainer.log_metrics("eval", results)
trainer.save_metrics("eval", results)

results.update(result)

return results

Expand Down
25 changes: 8 additions & 17 deletions examples/question-answering/run_qa.py
Expand Up @@ -201,6 +201,8 @@ def main():
# Set the verbosity to info of the Transformers logger (on main process only):
if is_main_process(training_args.local_rank):
transformers.utils.logging.set_verbosity_info()
transformers.utils.logging.enable_default_handler()
transformers.utils.logging.enable_explicit_format()
logger.info("Training/evaluation parameters %s", training_args)

# Set seed before initializing model.
Expand Down Expand Up @@ -479,30 +481,19 @@ def compute_metrics(p: EvalPrediction):
train_result = trainer.train(resume_from_checkpoint=checkpoint)
trainer.save_model() # Saves the tokenizer too for easy upload

output_train_file = os.path.join(training_args.output_dir, "train_results.txt")
if trainer.is_world_process_zero():
with open(output_train_file, "w") as writer:
logger.info("***** Train results *****")
for key, value in sorted(train_result.metrics.items()):
logger.info(f" {key} = {value}")
writer.write(f"{key} = {value}\n")

# Need to save the state, since Trainer.save_model saves only the tokenizer with the model
trainer.state.save_to_json(os.path.join(training_args.output_dir, "trainer_state.json"))
metrics = train_result.metrics
trainer.log_metrics("train", metrics)
trainer.save_metrics("train", metrics)
trainer.save_state()

# Evaluation
results = {}
if training_args.do_eval:
logger.info("*** Evaluate ***")
results = trainer.evaluate()

output_eval_file = os.path.join(training_args.output_dir, "eval_results.txt")
if trainer.is_world_process_zero():
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key, value in sorted(results.items()):
logger.info(f" {key} = {value}")
writer.write(f"{key} = {value}\n")
trainer.log_metrics("eval", results)
trainer.save_metrics("eval", results)

return results

Expand Down
25 changes: 8 additions & 17 deletions examples/question-answering/run_qa_beam_search.py
Expand Up @@ -200,6 +200,8 @@ def main():
# Set the verbosity to info of the Transformers logger (on main process only):
if is_main_process(training_args.local_rank):
transformers.utils.logging.set_verbosity_info()
transformers.utils.logging.enable_default_handler()
transformers.utils.logging.enable_explicit_format()
logger.info("Training/evaluation parameters %s", training_args)

# Set seed before initializing model.
Expand Down Expand Up @@ -518,30 +520,19 @@ def compute_metrics(p: EvalPrediction):
train_result = trainer.train(resume_from_checkpoint=checkpoint)
trainer.save_model() # Saves the tokenizer too for easy upload

output_train_file = os.path.join(training_args.output_dir, "train_results.txt")
if trainer.is_world_process_zero():
with open(output_train_file, "w") as writer:
logger.info("***** Train results *****")
for key, value in sorted(train_result.metrics.items()):
logger.info(f" {key} = {value}")
writer.write(f"{key} = {value}\n")

# Need to save the state, since Trainer.save_model saves only the tokenizer with the model
trainer.state.save_to_json(os.path.join(training_args.output_dir, "trainer_state.json"))
metrics = train_result.metrics
trainer.log_metrics("train", metrics)
trainer.save_metrics("train", metrics)
trainer.save_state()

# Evaluation
results = {}
if training_args.do_eval:
logger.info("*** Evaluate ***")
results = trainer.evaluate()

output_eval_file = os.path.join(training_args.output_dir, "eval_results.txt")
if trainer.is_world_process_zero():
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key, value in sorted(results.items()):
logger.info(f" {key} = {value}")
writer.write(f"{key} = {value}\n")
trainer.log_metrics("eval", results)
trainer.save_metrics("eval", results)

return results

Expand Down
22 changes: 5 additions & 17 deletions examples/text-classification/run_glue.py
Expand Up @@ -417,16 +417,9 @@ def compute_metrics(p: EvalPrediction):

trainer.save_model() # Saves the tokenizer too for easy upload

output_train_file = os.path.join(training_args.output_dir, "train_results.txt")
if trainer.is_world_process_zero():
with open(output_train_file, "w") as writer:
logger.info("***** Train results *****")
for key, value in sorted(metrics.items()):
logger.info(f" {key} = {value}")
writer.write(f"{key} = {value}\n")

# Need to save the state, since Trainer.save_model saves only the tokenizer with the model
trainer.state.save_to_json(os.path.join(training_args.output_dir, "trainer_state.json"))
trainer.log_metrics("train", metrics)
trainer.save_metrics("train", metrics)
trainer.save_state()

# Evaluation
eval_results = {}
Expand All @@ -443,13 +436,8 @@ def compute_metrics(p: EvalPrediction):
for eval_dataset, task in zip(eval_datasets, tasks):
eval_result = trainer.evaluate(eval_dataset=eval_dataset)

output_eval_file = os.path.join(training_args.output_dir, f"eval_results_{task}.txt")
if trainer.is_world_process_zero():
with open(output_eval_file, "w") as writer:
logger.info(f"***** Eval results {task} *****")
for key, value in sorted(eval_result.items()):
logger.info(f" {key} = {value}")
writer.write(f"{key} = {value}\n")
trainer.log_metrics("eval", eval_result)
trainer.save_metrics("eval", eval_result)

eval_results.update(eval_result)

Expand Down
14 changes: 3 additions & 11 deletions examples/text-classification/run_tf_glue.py
Expand Up @@ -247,18 +247,10 @@ def compute_metrics(p: EvalPrediction) -> Dict:
results = {}
if training_args.do_eval:
logger.info("*** Evaluate ***")

result = trainer.evaluate()
output_eval_file = os.path.join(training_args.output_dir, "eval_results.txt")

with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")

for key, value in result.items():
logger.info(" %s = %s", key, value)
writer.write("%s = %s\n" % (key, value))

results.update(result)
trainer.log_metrics("eval", result)
trainer.save_metrics("eval", result)
results.update(result)

return results

Expand Down
14 changes: 3 additions & 11 deletions examples/text-classification/run_tf_text_classification.py
Expand Up @@ -293,18 +293,10 @@ def compute_metrics(p: EvalPrediction) -> Dict:
results = {}
if training_args.do_eval:
logger.info("*** Evaluate ***")

result = trainer.evaluate()
output_eval_file = os.path.join(training_args.output_dir, "eval_results.txt")

with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")

for key, value in result.items():
logger.info(" %s = %s", key, value)
writer.write("%s = %s\n" % (key, value))

results.update(result)
trainer.log_metrics("eval", result)
trainer.save_metrics("eval", result)
results.update(result)

return results

Expand Down
26 changes: 6 additions & 20 deletions examples/text-classification/run_xnli.py
Expand Up @@ -291,33 +291,19 @@ def compute_metrics(p: EvalPrediction):

trainer.save_model() # Saves the tokenizer too for easy upload

output_train_file = os.path.join(training_args.output_dir, "train_results.txt")
if trainer.is_world_process_zero():
with open(output_train_file, "w") as writer:
logger.info("***** Train results *****")
for key, value in sorted(metrics.items()):
logger.info(f" {key} = {value}")
writer.write(f"{key} = {value}\n")

# Need to save the state, since Trainer.save_model saves only the tokenizer with the model
trainer.state.save_to_json(os.path.join(training_args.output_dir, "trainer_state.json"))
trainer.log_metrics("train", metrics)
trainer.save_metrics("train", metrics)
trainer.save_state()

# Evaluation
eval_results = {}
if training_args.do_eval:
logger.info("*** Evaluate ***")

eval_result = trainer.evaluate(eval_dataset=eval_dataset)
output_eval_file = os.path.join(training_args.output_dir, "eval_results_xnli.txt")

if trainer.is_world_process_zero():
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results xnli *****")
for key, value in sorted(eval_result.items()):
logger.info(f" {key} = {value}")
writer.write(f"{key} = {value}\n")

trainer.log_metrics("eval", eval_result)
trainer.save_metrics("eval", eval_result)
eval_results.update(eval_result)

return eval_results


Expand Down

0 comments on commit aca6288

Please sign in to comment.