diff --git a/bitsandbytes/nn/modules.py b/bitsandbytes/nn/modules.py
index c36fb68a6..f482ae6d3 100644
--- a/bitsandbytes/nn/modules.py
+++ b/bitsandbytes/nn/modules.py
@@ -697,7 +697,7 @@ def to(self, *args, **kwargs):
         if is_quantized:
             new_param.CB = new_param.data
 
-            if self.SCB is not None and device is not None:
+            if device is not None and self.SCB is not None and self.SCB.device.type != "meta":
                 new_param.SCB = self.SCB.to(device)
 
         return new_param