Be explicit

allenai · Sep 12, 2023 · 8d094b6 · 8d094b6
1 parent 61004d4
commit 8d094b6
Show file tree

Hide file tree

Showing 4 changed files with 8 additions and 8 deletions.
diff --git a/configs/v1-mix-medium-mcli.yaml b/configs/v1-mix-medium-mcli.yaml
@@ -20,8 +20,8 @@ model:
   include_bias: false
   block_type: sequential
   layer_norm_type: low_precision
-  #layer_norm_with_affine: false
-  bias_for_layer_norm: true
+  layer_norm_with_affine: true  # workaround for the layer norm bug
+  bias_for_layer_norm: true  # workaround for the layer norm bug
   activation_type: swiglu
   residual_dropout: 0.0
   embedding_dropout: 0.0

diff --git a/configs/v1-mix-medium.yaml b/configs/v1-mix-medium.yaml
@@ -20,8 +20,8 @@ model:
   include_bias: false
   block_type: sequential
   layer_norm_type: low_precision
-  #layer_norm_with_affine: false
-  bias_for_layer_norm: true
+  layer_norm_with_affine: true  # workaround for the layer norm bug
+  bias_for_layer_norm: true  # workaround for the layer norm bug
   activation_type: swiglu
   residual_dropout: 0.0
   embedding_dropout: 0.0

diff --git a/configs/v1-mix-small-mcli.yaml b/configs/v1-mix-small-mcli.yaml
@@ -20,8 +20,8 @@ model:
   include_bias: false
   block_type: sequential
   layer_norm_type: low_precision
-  #layer_norm_with_affine: false
-  bias_for_layer_norm: true
+  layer_norm_with_affine: true  # workaround for the layer norm bug
+  bias_for_layer_norm: true  # workaround for the layer norm bug
   activation_type: swiglu
   residual_dropout: 0.0
   embedding_dropout: 0.0

diff --git a/configs/v1-mix-small.yaml b/configs/v1-mix-small.yaml
@@ -20,8 +20,8 @@ model:
   include_bias: false
   block_type: sequential
   layer_norm_type: low_precision
-  #layer_norm_with_affine: false
-  bias_for_layer_norm: true
+  layer_norm_with_affine: true  # workaround for the layer norm bug
+  bias_for_layer_norm: true  # workaround for the layer norm bug
   activation_type: swiglu
   residual_dropout: 0.0
   embedding_dropout: 0.0