V2.99 r3 staging update

gimpopenvino/plugins/openvino_utils/tools/openvino_common/models_ov/stable_diffusion_engine.py

@@ -11,6 +11,7 @@
 # tokenizer
 from transformers import CLIPTokenizer
 import torch
+import random
 
 from diffusers import DiffusionPipeline
 from diffusers.schedulers import (DDIMScheduler,
@@ -101,7 +102,17 @@ def __init__(self, model="runwayml/stable-diffusion-v1-5",
 
         self.core = Core()
         self.core.set_property({'CACHE_DIR': os.path.join(model, 'cache')})
+        if "NPU" in device and "3720" not in self.core.get_property('NPU', 'DEVICE_ARCHITECTURE'):
+            try:
+                self.core.set_property(properties={'NPU_TURBO': 'YES'},device_name='NPU')
+                print_npu_turbo_art()    
+            except:
+                # do nothing. Turbo not enabled.
+                print("")
+
         print("Loading models... ")
+
+
 
         with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
             futures = {
@@ -435,14 +446,25 @@ def __init__(
 
         self.core = Core()
         self.core.set_property({'CACHE_DIR': os.path.join(model, 'cache')})
+
         self.batch_size = 2 if device[1] == device[2] and device[1] == "GPU" else 1
+
+        if "NPU" in device and "3720" not in self.core.get_property('NPU', 'DEVICE_ARCHITECTURE'):
+            try:
+                self.core.set_property(properties={'NPU_TURBO': 'YES'},device_name='NPU')
+                print_npu_turbo_art()    
+            except:
+                # do nothing. Turbo not enabled.
+                print("")
 
         try:
             self.tokenizer = CLIPTokenizer.from_pretrained(model, local_files_only=True)
         except Exception as e:
             print("Local tokenizer not found. Attempting to download...")
             self.tokenizer = self.download_tokenizer(tokenizer, model)
-
+
+        print("Loading models... ")
+
         with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
             text_future = executor.submit(self.load_model, model, "text_encoder", device[0])
             vae_de_future = executor.submit(self.load_model, model, "vae_decoder", device[3])
@@ -452,6 +474,7 @@ def __init__(
                 unet_future = executor.submit(self.load_model, model, "unet_bs1", device[1])
                 unet_neg_future = executor.submit(self.load_model, model, "unet_bs1", device[2]) if device[1] != device[2] else None
             else:
+                print("Loading BS2 model")
                 unet_future = executor.submit(self.load_model, model, "unet", device[1])
                 unet_neg_future = None
 
@@ -470,6 +493,8 @@ def __init__(
             self._vae_d_output = self.vae_decoder.output(0)
             self._vae_e_output = self.vae_encoder.output(0) if self.vae_encoder else None
 
+            self.unet_input_tensor_name = "sample" if 'sample' in self.unet.input(0).names else "latent_model_input"
+
             if self.batch_size == 1:
                 self.infer_request = self.unet.create_infer_request()
                 self.infer_request_neg = self.unet_neg.create_infer_request()
@@ -490,7 +515,7 @@ def load_model(self, model, model_name, device):
         return self.core.compile_model(os.path.join(model, f"{model_name}.xml"), device)
 
     def set_dimensions(self):
-        latent_shape = self.unet.input("latent_model_input").shape
+        latent_shape = self.unet.input(self.unet_input_tensor_name).shape
         if latent_shape[1] == 4:
             self.height = latent_shape[2] * 8
             self.width = latent_shape[3] * 8
@@ -583,21 +608,25 @@ def __call__(
                 latent_model_input_pos = latent_model_input 
                 latent_model_input_neg = latent_model_input
 
-                if self.unet.input("latent_model_input").shape[1] != 4:
+                if self.unet.input(self.unet_input_tensor_name).shape[1] != 4:
                     try:
                         latent_model_input_pos = latent_model_input_pos.permute(0,2,3,1)
                     except:
                         latent_model_input_pos = latent_model_input_pos.transpose(0,2,3,1)
 
-                if self.unet_neg.input("latent_model_input").shape[1] != 4:
+                if self.unet_neg.input(self.unet_input_tensor_name).shape[1] != 4:
                     try:
                         latent_model_input_neg = latent_model_input_neg.permute(0,2,3,1)
                     except:
                         latent_model_input_neg = latent_model_input_neg.transpose(0,2,3,1)
-
-                input_tens_neg_dict = {"latent_model_input":latent_model_input_neg, "encoder_hidden_states": np.expand_dims(text_embeddings[0], axis=0), "t": np.expand_dims(np.float32(t), axis=0)}
-                input_tens_pos_dict = {"latent_model_input":latent_model_input_pos, "encoder_hidden_states": np.expand_dims(text_embeddings[1], axis=0), "t": np.expand_dims(np.float32(t), axis=0)}
-
+
+                if "sample" in self.unet_input_tensor_name:                                        
+                    input_tens_neg_dict = {"sample" : latent_model_input_neg, "encoder_hidden_states": np.expand_dims(text_embeddings[0], axis=0), "timestep": np.expand_dims(np.float32(t), axis=0)}
+                    input_tens_pos_dict = {"sample" : latent_model_input_pos, "encoder_hidden_states": np.expand_dims(text_embeddings[1], axis=0), "timestep": np.expand_dims(np.float32(t), axis=0)}
+                else:
+                    input_tens_neg_dict = {"latent_model_input" : latent_model_input_neg, "encoder_hidden_states": np.expand_dims(text_embeddings[0], axis=0), "t": np.expand_dims(np.float32(t), axis=0)}
+                    input_tens_pos_dict = {"latent_model_input" : latent_model_input_pos, "encoder_hidden_states": np.expand_dims(text_embeddings[1], axis=0), "t": np.expand_dims(np.float32(t), axis=0)}
+
                 self.infer_request_neg.start_async(input_tens_neg_dict)
                 self.infer_request.start_async(input_tens_pos_dict)    
 
@@ -681,8 +710,6 @@ def prepare_latents(self, image: PIL.Image.Image = None, latent_timestep: torch.
 
         latents = scheduler.add_noise(torch.from_numpy(latents), torch.from_numpy(noise), latent_timestep).numpy()
         return latents, meta
-
-
 
 
     def postprocess_image(self, image: np.ndarray, meta: Dict):
@@ -765,6 +792,16 @@ def __init__(
 
         self.core = Core()
         self.core.set_property({'CACHE_DIR': os.path.join(model, 'cache')})  # adding caching to reduce init time
+
+        if "NPU" in device and "3720" not in self.core.get_property('NPU', 'DEVICE_ARCHITECTURE'):
+            try:
+                self.core.set_property(properties={'NPU_TURBO': 'YES'},device_name='NPU')
+                print_npu_turbo_art()    
+            except:
+                # do nothing. Turbo not enabled.
+                print("")
+
+
         with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
             text_future = executor.submit(self.load_model, model, "text_encoder", device[0])
             unet_future = executor.submit(self.load_model, model, "unet", device[1])    
@@ -1401,7 +1438,41 @@ def prepare_image(
 
         return image
 
-
+def print_npu_turbo_art():
+    random_number = random.randint(1, 3)
+
+    if random_number == 1:
+        print("                                                                                                                      ")
+        print("      ___           ___         ___                                ___           ___                         ___      ")
+        print("     /\  \         /\  \       /\  \                              /\  \         /\  \         _____         /\  \     ")
+        print("     \:\  \       /::\  \      \:\  \                ___          \:\  \       /::\  \       /::\  \       /::\  \    ")
+        print("      \:\  \     /:/\:\__\      \:\  \              /\__\          \:\  \     /:/\:\__\     /:/\:\  \     /:/\:\  \   ")
+        print("  _____\:\  \   /:/ /:/  /  ___  \:\  \            /:/  /      ___  \:\  \   /:/ /:/  /    /:/ /::\__\   /:/  \:\  \  ")
+        print(" /::::::::\__\ /:/_/:/  /  /\  \  \:\__\          /:/__/      /\  \  \:\__\ /:/_/:/__/___ /:/_/:/\:|__| /:/__/ \:\__\ ")
+        print(" \:\~~\~~\/__/ \:\/:/  /   \:\  \ /:/  /         /::\  \      \:\  \ /:/  / \:\/:::::/  / \:\/:/ /:/  / \:\  \ /:/  / ")
+        print("  \:\  \        \::/__/     \:\  /:/  /         /:/\:\  \      \:\  /:/  /   \::/~~/~~~~   \::/_/:/  /   \:\  /:/  /  ")
+        print("   \:\  \        \:\  \      \:\/:/  /          \/__\:\  \      \:\/:/  /     \:\~~\        \:\/:/  /     \:\/:/  /   ")
+        print("    \:\__\        \:\__\      \::/  /                \:\__\      \::/  /       \:\__\        \::/  /       \::/  /    ")
+        print("     \/__/         \/__/       \/__/                  \/__/       \/__/         \/__/         \/__/         \/__/     ")
+        print("                                                                                                                      ")
+    elif random_number == 2:
+        print(" _   _   ____    _   _     _____   _   _   ____    ____     ___  ")
+        print("| \ | | |  _ \  | | | |   |_   _| | | | | |  _ \  | __ )   / _ \ ")
+        print("|  \| | | |_) | | | | |     | |   | | | | | |_) | |  _ \  | | | |")
+        print("| |\  | |  __/  | |_| |     | |   | |_| | |  _ <  | |_) | | |_| |")
+        print("|_| \_| |_|      \___/      |_|    \___/  |_| \_\ |____/   \___/ ")
+        print("                                                                 ")
+    else:
+        print("")
+        print("    )   (                                 (                )   ")
+        print(" ( /(   )\ )              *   )           )\ )     (    ( /(   ")
+        print(" )\()) (()/(      (     ` )  /(      (   (()/(   ( )\   )\())  ")
+        print("((_)\   /(_))     )\     ( )(_))     )\   /(_))  )((_) ((_)\   ")
+        print(" _((_) (_))    _ ((_)   (_(_())   _ ((_) (_))   ((_)_    ((_)  ")
+        print("| \| | | _ \  | | | |   |_   _|  | | | | | _ \   | _ )  / _ \  ")
+        print("| .` | |  _/  | |_| |     | |    | |_| | |   /   | _ \ | (_) | ")
+        print("|_|\_| |_|     \___/      |_|     \___/  |_|_\   |___/  \___/  ")
+        print("                                                               ")
 
 
 

gimpopenvino/plugins/openvino_utils/tools/openvino_common/stable_diffusion_run_ov.py

@@ -28,7 +28,7 @@
 # utils
 import cv2
 import numpy as np
-from sys.path.extend([os.path.join(os.path.dirname(os.path.realpath(__file__)), "..","openvino_utils","tools")])
+from sys.path.extend([os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "tools")])
 from tools_utils import get_weight_path import get_weight_path
 
 

gimpopenvino/plugins/openvino_utils/tools/stable_diffusion_ov.py

@@ -6,7 +6,7 @@
 import sys
 
 sys.path.extend([os.path.join(os.path.dirname(os.path.realpath(__file__)), "openvino_common")])
-sys.path.extend([os.path.join(os.path.dirname(os.path.realpath(__file__)), "..","openvino_utils","tools")])
+sys.path.extend([os.path.join(os.path.dirname(os.path.realpath(__file__)), "..","tools")])
 
 import cv2
 from stable_diffusion_run_ov import run

gimpopenvino/plugins/openvino_utils/tools/stable_diffusion_ov_server.py

@@ -13,12 +13,14 @@
 from pathlib import Path
 import time 
 import random
+import torch
+
 from PIL import Image
 import numpy as np
 import psutil
 import threading
 sys.path.extend([os.path.join(os.path.dirname(os.path.realpath(__file__)), "openvino_common")])
-sys.path.extend([os.path.join(os.path.dirname(os.path.realpath(__file__)), "..","openvino_utils","tools")])
+sys.path.extend([os.path.join(os.path.dirname(os.path.realpath(__file__)), "..","tools")])
 from tools_utils import get_weight_path
 
 
@@ -33,6 +35,19 @@
 from models_ov.controlnet_openpose_advanced import ControlNetOpenPoseAdvanced
 from models_ov.controlnet_cannyedge_advanced import ControlNetCannyEdgeAdvanced
 
+from models_ov import (
+    stable_diffusion_engine,
+    stable_diffusion_engine_inpainting,
+    stable_diffusion_engine_inpainting_advanced,
+    stable_diffusion_3,
+    controlnet_openpose,
+    controlnet_canny_edge,
+    controlnet_scribble,
+    controlnet_openpose_advanced,
+    controlnet_cannyedge_advanced
+)
+
+
 HOST = "127.0.0.1"  # Standard loopback interface address (localhost)
 PORT = 65432  # Port to listen on (non-privileged ports are > 1023)
 
@@ -65,6 +80,7 @@ def run(model_name, available_devices, power_mode):
         "sd_1.5_inpainting_int8": ["stable-diffusion-ov", "stable-diffusion-1.5", "inpainting_int8"],
         "sd_2.1_square_base": ["stable-diffusion-ov", "stable-diffusion-2.1", "square_base"],
         "sd_2.1_square": ["stable-diffusion-ov", "stable-diffusion-2.1", "square"],
+        "sd_3.0_square": ["stable-diffusion-ov", "stable-diffusion-3.0"],
         "controlnet_referenceonly": ["stable-diffusion-ov", "controlnet-referenceonly"],
         "controlnet_openpose": ["stable-diffusion-ov", "controlnet-openpose"],
         "controlnet_canny": ["stable-diffusion-ov", "controlnet-canny"],
@@ -140,35 +156,73 @@ def run(model_name, available_devices, power_mode):
                         break
                     handle_client_data(data, conn, engine, model_name, model_path, scheduler)
 
+# def initialize_engine(model_name, model_path, device_list):
+#     if model_name == "sd_3.0_square":
+#         device_list = ["GPU"]
+#         log.info('Device list: %s', device_list)
+#         return stable_diffusion_3.StableDiffusionThreeEngine(model=model_path, device=device_list)
+#     if model_name == "sd_1.5_square_int8":
+#         log.info('Device list: %s', device_list)
+#         return StableDiffusionEngineAdvanced(model=model_path, device=device_list)
+#     if model_name == "sd_1.5_inpainting":
+#         return StableDiffusionEngineInpainting(model=model_path, device=device_list)
+#     if model_name == "sd_1.5_square_lcm":
+#         return LatentConsistencyEngine(model=model_path, device=device_list)
+#     if model_name == "sd_1.5_inpainting_int8":
+#         log.info('Advanced Inpainting Device list: %s', device_list)
+#         return StableDiffusionEngineInpaintingAdvanced(model=model_path, device=device_list)
+#     if model_name == "controlnet_openpose_int8":
+#         log.info('Device list: %s', device_list)
+#         return ControlNetOpenPoseAdvanced(model=model_path, device=device_list)
+#     if model_name == "controlnet_canny_int8":
+#         log.info('Device list: %s', device_list)
+#         return ControlNetCannyEdgeAdvanced(model=model_path, device=device_list)
+#     if model_name == "controlnet_scribble_int8":
+#         log.info('Device list: %s', device_list)
+#         return ControlNetScribbleAdvanced(model=model_path, device=device_list)
+#     if model_name == "controlnet_canny":
+#         return ControlNetCannyEdge(model=model_path, device=device_list)
+#     if model_name == "controlnet_scribble":
+#         return ControlNetScribble(model=model_path, device=device_list)
+#     if model_name == "controlnet_openpose":
+#         return ControlNetOpenPose(model=model_path, device=device_list)
+#     if model_name == "controlnet_referenceonly":
+#         return StableDiffusionEngineReferenceOnly(model=model_path, device=device_list)
+#     return StableDiffusionEngine(model=model_path, device=device_list)
+
 def initialize_engine(model_name, model_path, device_list):
     if model_name == "sd_1.5_square_int8":
         log.info('Device list: %s', device_list)
-        return StableDiffusionEngineAdvanced(model=model_path, device=device_list)
+        return stable_diffusion_engine.StableDiffusionEngineAdvanced(model=model_path, device=device_list)
+    if model_name == "sd_3.0_square":
+        device_list = ["GPU"]
+        log.info('Device list: %s', device_list)
+        return stable_diffusion_3.StableDiffusionThreeEngine(model=model_path, device=device_list)
     if model_name == "sd_1.5_inpainting":
-        return StableDiffusionEngineInpainting(model=model_path, device=device_list)
+        return stable_diffusion_engine_inpainting.StableDiffusionEngineInpainting(model=model_path, device=device_list)
     if model_name == "sd_1.5_square_lcm":
-        return LatentConsistencyEngine(model=model_path, device=device_list)
+        return stable_diffusion_engine.LatentConsistencyEngine(model=model_path, device=device_list)
     if model_name == "sd_1.5_inpainting_int8":
         log.info('Advanced Inpainting Device list: %s', device_list)
-        return StableDiffusionEngineInpaintingAdvanced(model=model_path, device=device_list)
+        return stable_diffusion_engine_inpainting_advanced.StableDiffusionEngineInpaintingAdvanced(model=model_path, device=device_list)
     if model_name == "controlnet_openpose_int8":
         log.info('Device list: %s', device_list)
-        return ControlNetOpenPoseAdvanced(model=model_path, device=device_list)
+        return controlnet_openpose_advanced.ControlNetOpenPoseAdvanced(model=model_path, device=device_list)
     if model_name == "controlnet_canny_int8":
         log.info('Device list: %s', device_list)
-        return ControlNetCannyEdgeAdvanced(model=model_path, device=device_list)
+        return controlnet_canny_edge_advanced.ControlNetCannyEdgeAdvanced(model=model_path, device=device_list)
     if model_name == "controlnet_scribble_int8":
         log.info('Device list: %s', device_list)
-        return ControlNetScribbleAdvanced(model=model_path, device=device_list)
+        return controlnet_scribble.ControlNetScribbleAdvanced(model=model_path, device=device_list)
     if model_name == "controlnet_canny":
-        return ControlNetCannyEdge(model=model_path, device=device_list)
+        return controlnet_canny_edge.ControlNetCannyEdge(model=model_path, device=device_list)
     if model_name == "controlnet_scribble":
-        return ControlNetScribble(model=model_path, device=device_list)
+        return controlnet_scribble.ControlNetScribble(model=model_path, device=device_list)
     if model_name == "controlnet_openpose":
-        return ControlNetOpenPose(model=model_path, device=device_list)
+        return controlnet_openpose.ControlNetOpenPose(model=model_path, device=device_list)
     if model_name == "controlnet_referenceonly":
-        return StableDiffusionEngineReferenceOnly(model=model_path, device=device_list)
-    return StableDiffusionEngine(model=model_path, device=device_list)
+        return stable_diffusion_engine.StableDiffusionEngineReferenceOnly(model=model_path, device=device_list)
+    return stable_diffusion_engine.StableDiffusionEngine(model=model_path, device=device_list)
 
 def handle_client_data(data, conn, engine, model_name, model_path, scheduler):
     if data.decode() == "kill":
@@ -280,6 +334,18 @@ def handle_client_data(data, conn, engine, model_name, model_path, scheduler):
                 callback_userdata=conn,
                 seed=seed
             )
+        elif "sd_3.0" in model_name:
+            output = engine(
+                    prompt = prompt,
+                    negative_prompt = negative_prompt,
+                    num_inference_steps = num_infer_steps,
+                    guidance_scale = 0,
+                    generator=torch.Generator().manual_seed(seed),
+                    callback=progress_callback,
+                    callback_userdata=conn
+                    #callback_on_step_end_tensor_inputs = conn,
+
+            ).images[0] 
 
         else:
             if model_name == "sd_2.1_square":
@@ -313,7 +379,7 @@ def handle_client_data(data, conn, engine, model_name, model_path, scheduler):
 
         image = "sd_cache.png"
 
-        if "controlnet" in model_name or model_name == "sd_1.5_square_lcm":
+        if "controlnet" in model_name or model_name == "sd_1.5_square_lcm" or "sd_3.0" in model_name:
             output.save(os.path.join(weight_path, "..", image))
             src_width, src_height = output.size
         else:

gimpopenvino/plugins/openvino_utils/tools/superresolution_ov.py

@@ -7,7 +7,7 @@
 import sys
 
 sys.path.extend([os.path.join(os.path.dirname(os.path.realpath(__file__)), "openvino_common")])
-sys.path.extend([os.path.join(os.path.dirname(os.path.realpath(__file__)), "..","openvino_utils","tools")])
+sys.path.extend([os.path.join(os.path.dirname(os.path.realpath(__file__)), "..","tools")])
 
 
 import cv2