synchronize grad clip warmup between configs

allenai · Jan 9, 2024 · 8c81318 · 8c81318
1 parent b320a74
commit 8c81318
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/configs/v1_5-mix-medium-mitch-ish-s3.yaml b/configs/v1_5-mix-medium-mitch-ish-s3.yaml
@@ -53,6 +53,8 @@ scheduler:
   name: linear_with_warmup
   t_warmup: 5000
   alpha_f: 0.1
+  grad_clip_warmup_steps: 1000
+  grad_clip_warmup_factor: 10.0
 
 tokenizer:
   identifier: tokenizers/allenai_eleuther-ai-gpt-neox-20b-pii-special.json