diff --git a/tapas/models/bert/modeling_test.py b/tapas/models/bert/modeling_test.py
index 5f74287..d592468 100644
--- a/tapas/models/bert/modeling_test.py
+++ b/tapas/models/bert/modeling_test.py
@@ -40,8 +40,8 @@ def __init__(self,
                  use_token_type_ids=True,
                  vocab_size=99,
                  hidden_size=32,
-                 num_hidden_layers=5,
-                 num_attention_heads=4,
+                 num_hidden_layers=24,
+                 num_attention_heads=5,
                  intermediate_size=37,
                  hidden_act="gelu",
                  hidden_dropout_prob=0.1,