Add a check and explanation for tensor with all NaNs.

modules/devices.py

@@ -106,6 +106,33 @@ def autocast(disable=False):
     return torch.autocast("cuda")
 
 
+class NansException(Exception):
+    pass
+
+
+def test_for_nans(x, where):
+    from modules import shared
+
+    if not torch.all(torch.isnan(x)).item():
+        return
+
+    if where == "unet":
+        message = "A tensor with all NaNs was produced in Unet."
+
+        if not shared.cmd_opts.no_half:
+            message += " This could be either because there's not enough precision to represent the picture, or because your video card does not support half type. Try using --no-half commandline argument to fix this."
+
+    elif where == "vae":
+        message = "A tensor with all NaNs was produced in VAE."
+
+        if not shared.cmd_opts.no_half and not shared.cmd_opts.no_half_vae:
+            message += " This could be because there's not enough precision to represent the picture. Try adding --no-half-vae commandline argument to fix this."
+    else:
+        message = "A tensor with all NaNs was produced."
+
+    raise NansException(message)
+
+
 # MPS workaround for https://github.com/pytorch/pytorch/issues/79383
 orig_tensor_to = torch.Tensor.to
 def tensor_to_fix(self, *args, **kwargs):
@@ -156,3 +183,4 @@ def cumsum_fix(input, cumsum_func, *args, **kwargs):
             torch.Tensor.cumsum = lambda self, *args, **kwargs: ( cumsum_fix(self, orig_Tensor_cumsum, *args, **kwargs) )
         orig_narrow = torch.narrow
         torch.narrow = lambda *args, **kwargs: ( orig_narrow(*args, **kwargs).clone() )
+

modules/processing.py

@@ -608,6 +608,9 @@ def get_conds_with_caching(function, required_prompts, steps, cache):
                 samples_ddim = p.sample(conditioning=c, unconditional_conditioning=uc, seeds=seeds, subseeds=subseeds, subseed_strength=p.subseed_strength, prompts=prompts)
 
             x_samples_ddim = [decode_first_stage(p.sd_model, samples_ddim[i:i+1].to(dtype=devices.dtype_vae))[0].cpu() for i in range(samples_ddim.size(0))]
+            for x in x_samples_ddim:
+                devices.test_for_nans(x, "vae")
+
             x_samples_ddim = torch.stack(x_samples_ddim).float()
             x_samples_ddim = torch.clamp((x_samples_ddim + 1.0) / 2.0, min=0.0, max=1.0)
 

modules/sd_samplers.py

@@ -351,6 +351,8 @@ def forward(self, x, sigma, uncond, cond, cond_scale, image_cond):
 
             x_out[-uncond.shape[0]:] = self.inner_model(x_in[-uncond.shape[0]:], sigma_in[-uncond.shape[0]:], cond={"c_crossattn": [uncond], "c_concat": [image_cond_in[-uncond.shape[0]:]]})
 
+        devices.test_for_nans(x_out, "unet")
+
         if opts.live_preview_content == "Prompt":
             store_latent(x_out[0:uncond.shape[0]])
         elif opts.live_preview_content == "Negative prompt":