fixes #1

trainable LayerNorms
hmohebbi · Feb 25, 2021 · 71045a2 · 71045a2
1 parent 8ddad14
commit 71045a2
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/run_tf_glue_adapter_bert.py b/run_tf_glue_adapter_bert.py
@@ -107,10 +107,10 @@ def main():
   for i in range(config.num_hidden_layers):
     model.bert.bert.encoder.layer[i].attention.self_attention.trainable = False
     model.bert.bert.encoder.layer[i].attention.dense_output.dense.trainable = False
-    model.bert.bert.encoder.layer[i].attention.dense_output.LayerNorm.trainable = False
+    #model.bert.bert.encoder.layer[i].attention.dense_output.LayerNorm.trainable = False
     model.bert.bert.encoder.layer[i].intermediate.trainable = False
     model.bert.bert.encoder.layer[i].bert_output.dense.trainable = False
-    model.bert.bert.encoder.layer[i].bert_output.LayerNorm.trainable = False
+    #model.bert.bert.encoder.layer[i].bert_output.LayerNorm.trainable = False
 
 
   # Loss & Optimizer