diff --git a/deepspeed/runtime/engine.py b/deepspeed/runtime/engine.py
index 93af1dda458b..d24e99af488e 100755
--- a/deepspeed/runtime/engine.py
+++ b/deepspeed/runtime/engine.py
@@ -2045,7 +2045,7 @@ def get_layer_state_dict(module, prefix=""):
             # gather one layer at a time to be memory-efficient
             with deepspeed.zero.GatheredParameters(list(
                     module.parameters(recurse=False)),
-                                                   modifier_rank=0):
+                                                   modifier_rank=None):
                 if torch.distributed.get_rank() == 0:
                     for name, param in module.named_parameters(recurse=False):
                         if param is None: