diff --git a/src/diffusers/models/transformers/transformer_z_image.py b/src/diffusers/models/transformers/transformer_z_image.py
index 614fb0f1210c..4cea745e5ed5 100644
--- a/src/diffusers/models/transformers/transformer_z_image.py
+++ b/src/diffusers/models/transformers/transformer_z_image.py
@@ -457,14 +457,14 @@ def __init__(
                     for layer_id in range(n_refiner_layers)
                 ]
             )
-            self.siglip_pad_token = nn.Parameter(torch.empty((1, dim)))
+            self.siglip_pad_token = nn.Parameter(torch.zeros((1, dim)))
         else:
             self.siglip_embedder = None
             self.siglip_refiner = None
             self.siglip_pad_token = None
 
-        self.x_pad_token = nn.Parameter(torch.empty((1, dim)))
-        self.cap_pad_token = nn.Parameter(torch.empty((1, dim)))
+        self.x_pad_token = nn.Parameter(torch.zeros((1, dim)))
+        self.cap_pad_token = nn.Parameter(torch.zeros((1, dim)))
 
         self.layers = nn.ModuleList(
             [