From 475fc1211ebe698df4ef2e6eb811adf528b6c514 Mon Sep 17 00:00:00 2001 From: KareemMusleh Date: Mon, 17 Mar 2025 02:59:02 +0700 Subject: [PATCH] fix --- unsloth/models/vision.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 24015f82fe..e4d621fba6 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -208,7 +208,11 @@ def from_pretrained( get_statistics() # For debugging - we use a download counter to see if environments are not breaking if dtype is None: - dtype = torch.float16 if not SUPPORTS_BFLOAT16 else torch.bfloat16 + # Inifite gradient bug for gemma3 if float16. https://unsloth.ai/blog/gemma3 + if "gemma-3" in model_name.lower(): + dtype = torch.bfloat16 if SUPPORTS_BFLOAT16 else torch.float32 + else: + dtype = torch.bfloat16 if SUPPORTS_BFLOAT16 else torch.float16 elif dtype == torch.bfloat16 and not SUPPORTS_BFLOAT16: logger.warning_once("Device does not support bfloat16. Will change to float16.") dtype = torch.float16