add style and compostion transfer to SD1.5

CrossAttentionPatch.py

@@ -19,7 +19,7 @@ def __init__(self, ipadapter=None, number=0, weight=1.0, cond=None, cond_alt=Non
         self.unfold_batch = [unfold_batch]
         self.embeds_scaling = [embeds_scaling]
         self.number = number
-        self.layers = 11 if '101_to_k_ip' in ipadapter.ip_layers.to_kvs else 15 # TODO: check if this is a valid condition to detect all models
+        self.layers = 11 if '101_to_k_ip' in ipadapter.ip_layers.to_kvs else 16 # TODO: check if this is a valid condition to detect all models
 
         self.k_key = str(self.number*2+1) + "_to_k_ip"
         self.v_key = str(self.number*2+1) + "_to_v_ip"
@@ -72,12 +72,6 @@ def __call__(self, q, k, v, extra_options):
                     weight = weight * 0.2
                 elif weight_type == 'strong middle' and (block_type == 'input' or block_type == 'output'):
                     weight = weight * 0.2
-                elif weight_type.startswith('style transfer'):
-                    if t_idx != 6:
-                        continue
-                elif weight_type.startswith('composition'):
-                    if t_idx != 3:
-                        continue
                 elif isinstance(weight, dict):
                     if t_idx not in weight:
                         continue

IPAdapterPlus.py

@@ -37,7 +37,7 @@
     current_paths, _ = folder_paths.folder_names_and_paths["ipadapter"]
 folder_paths.folder_names_and_paths["ipadapter"] = (current_paths, folder_paths.supported_pt_extensions)
 
-WEIGHT_TYPES = ["linear", "ease in", "ease out", 'ease in-out', 'reverse in-out', 'weak input', 'weak output', 'weak middle', 'strong middle', 'style transfer (SDXL)', 'composition (SDXL)']
+WEIGHT_TYPES = ["linear", "ease in", "ease out", 'ease in-out', 'reverse in-out', 'weak input', 'weak output', 'weak middle', 'strong middle', 'style transfer', 'composition']
 
 """
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -174,11 +174,6 @@ def ipadapter_execute(model,
     output_cross_attention_dim = ipadapter["ip_adapter"]["1.to_k_ip.weight"].shape[1]
     is_sdxl = output_cross_attention_dim == 2048
 
-    if ('(SDXL)' in weight_type or image_composition is not None) and not is_sdxl:
-        raise Exception("Style and Composition transfer are available for SDXL models only")
-        #weight_type = "linear"
-        #print("\033[33mINFO: 'Style Transfer' weight type is only available for SDXL models, falling back to 'linear'.\033[0m")
-
     if is_faceid and not insightface:
         raise Exception("insightface model is required for FaceID models")
 
@@ -477,7 +472,8 @@ def INPUT_TYPES(s):
         }}
 
     RETURN_NAMES = ("MODEL", "ipadapter", )
-
+    CATEGORY = "ipadapter/faceid"
+
 class IPAdapterUnifiedLoaderCommunity(IPAdapterUnifiedLoader):
     @classmethod
     def INPUT_TYPES(s):
@@ -488,6 +484,8 @@ def INPUT_TYPES(s):
         "optional": {
             "ipadapter": ("IPADAPTER", ),
         }}
+
+    CATEGORY = "ipadapter/loaders"
 
 class IPAdapterModelLoader:
     @classmethod
@@ -496,7 +494,7 @@ def INPUT_TYPES(s):
 
     RETURN_TYPES = ("IPADAPTER",)
     FUNCTION = "load_ipadapter_model"
-    CATEGORY = "ipadapter"
+    CATEGORY = "ipadapter/loaders"
 
     def load_ipadapter_model(self, ipadapter_file):
         ipadapter_file = folder_paths.get_full_path("ipadapter", ipadapter_file)
@@ -513,7 +511,7 @@ def INPUT_TYPES(s):
 
     RETURN_TYPES = ("INSIGHTFACE",)
     FUNCTION = "load_insightface"
-    CATEGORY = "ipadapter"
+    CATEGORY = "ipadapter/loaders"
 
     def load_insightface(self, provider):
         return (insightface_loader(provider),)
@@ -534,7 +532,7 @@ def INPUT_TYPES(s):
                 "weight": ("FLOAT", { "default": 1.0, "min": -1, "max": 3, "step": 0.05 }),
                 "start_at": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
                 "end_at": ("FLOAT", { "default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
-                "weight_type": (['standard', 'prompt is more important', 'style transfer (SDXL only)'], ),
+                "weight_type": (['standard', 'prompt is more important', 'style transfer'], ),
             },
             "optional": {
                 "attn_mask": ("MASK",),
@@ -546,8 +544,8 @@ def INPUT_TYPES(s):
     CATEGORY = "ipadapter"
 
     def apply_ipadapter(self, model, ipadapter, image, weight, start_at, end_at, weight_type, attn_mask=None):
-        if weight_type == "style transfer (SDXL only)":
-            weight_type = "style transfer (SDXL)"
+        if weight_type.startwith("style transfer"):
+            weight_type = "style transfer"
         elif weight_type == "prompt is more important":
             weight_type = "ease out"
         else:
@@ -600,13 +598,30 @@ def INPUT_TYPES(s):
     CATEGORY = "ipadapter"
 
     def apply_ipadapter(self, model, ipadapter, start_at, end_at, weight = 1.0, weight_style=1.0, weight_composition=1.0, expand_style=False, weight_type="linear", combine_embeds="concat", weight_faceidv2=None, image=None, image_style=None, image_composition=None, image_negative=None, clip_vision=None, attn_mask=None, insightface=None, embeds_scaling='V only'):
-        if image_style is not None:
+        is_sdxl = ipadapter["ip_adapter"]["1.to_k_ip.weight"].shape[1] == 2048
+
+        if image_style is not None: # we are doing style + composition transfer
+            if not is_sdxl:
+                raise Exception("Style + Composition transfer is only available for SDXL models at the moment.") # TODO: check feasibility for SD1.5 models
+
             image = image_style
-            if image_composition is not None:
-                if expand_style:
-                    weight = { 0: weight_style, 1: weight_style, 2: weight_style, 3: weight_composition, 4: weight_style, 5: weight_style, 6: weight_style, 7: weight_style, 8: weight_style, 9: weight_style, 10: weight_style }
+            if image_composition is None:
+                image_composition = image_style
+
+            if expand_style:
+                if is_sdxl:
+                    weight = { 0:weight_style, 1:weight_style, 2:weight_style, 3:weight_composition, 4:weight_style, 5:weight_style, 6:weight_style, 7:weight_style, 8:weight_style, 9:weight_style, 10:weight_style }
+                else:
+                    weight = { 0:weight_style, 1:weight_style, 2:weight_style, 3:weight_style, 4:weight_composition, 5:weight_composition, 6:weight_style, 7:weight_style, 8:weight_style, 9:weight_style, 10:weight_style, 11:weight_style, 12:weight_style, 13:weight_style, 14:weight_style, 15:weight_style }
+            else:
+                if is_sdxl:
+                    weight = { 3:weight_composition, 6:weight_style }
                 else:
-                    weight = { 3: weight_composition, 6: weight_style }
+                    weight = { 0:weight_style, 1:weight_style, 2:weight_style, 3:weight_style, 4:weight_composition/4, 5:weight_composition, 9:weight_style, 10:weight_style, 11:weight_style, 12:weight_style, 13:weight_style, 14:weight_style, 15:weight_style }
+        elif weight_type.startswith("style transfer"):
+            weight = { 6:weight } if is_sdxl else { 0:weight, 1:weight, 2:weight, 3:weight, 9:weight, 10:weight, 11:weight, 12:weight, 13:weight, 14:weight, 15:weight }
+        elif weight_type.startswith("composition"):
+            weight = { 3:weight } if is_sdxl else { 4:weight/4, 5:weight }
 
         ipa_args = {
             "image": image,
@@ -685,7 +700,35 @@ def INPUT_TYPES(s):
                 "clip_vision": ("CLIP_VISION",),
             }
         }
+
+    CATEGORY = "ipadapter/style_composition"
+
+class IPAdapterStyleCompositionBatch(IPAdapterStyleComposition):
+    def __init__(self):
+        self.unfold_batch = True
 
+    @classmethod
+    def INPUT_TYPES(s):
+        return {
+            "required": {
+                "model": ("MODEL", ),
+                "ipadapter": ("IPADAPTER", ),
+                "image_style": ("IMAGE",),
+                "image_composition": ("IMAGE",),
+                "weight_style": ("FLOAT", { "default": 1.0, "min": -1, "max": 5, "step": 0.05 }),
+                "weight_composition": ("FLOAT", { "default": 1.0, "min": -1, "max": 5, "step": 0.05 }),
+                "expand_style": ("BOOLEAN", { "default": False }),
+                "start_at": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
+                "end_at": ("FLOAT", { "default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
+                "embeds_scaling": (['V only', 'K+V', 'K+V w/ C penalty', 'K+mean(V) w/ C penalty'], ),
+            },
+            "optional": {
+                "image_negative": ("IMAGE",),
+                "attn_mask": ("MASK",),
+                "clip_vision": ("CLIP_VISION",),
+            }
+        }
+
 class IPAdapterFaceID(IPAdapterAdvanced):
     @classmethod
     def INPUT_TYPES(s):
@@ -701,6 +744,7 @@ def INPUT_TYPES(s):
                 "start_at": ("FLOAT", { "default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
                 "end_at": ("FLOAT", { "default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001 }),
                 "embeds_scaling": (['V only', 'K+V', 'K+V w/ C penalty', 'K+mean(V) w/ C penalty'], ),
+                "layer_weights": ("STRING", { "default": "0:1.0, 1:1.0, 2:1.0, 3:1.0, 4:1.0, 5:1.0, 6:1.0, 7:1.0, 8:1.0, 9:1.0, 10:1.0", "multiline": True }),
             },
             "optional": {
                 "image_negative": ("IMAGE",),
@@ -709,6 +753,8 @@ def INPUT_TYPES(s):
                 "insightface": ("INSIGHTFACE",),
             }
         }
+
+    CATEGORY = "ipadapter/faceid"
 
 class IPAAdapterFaceIDBatch(IPAdapterFaceID):
     def __init__(self):
@@ -743,7 +789,7 @@ def INPUT_TYPES(s):
     RETURN_TYPES = ("MODEL", "IMAGE", "MASK", )
     RETURN_NAMES = ("MODEL", "tiles", "masks", )
     FUNCTION = "apply_tiled"
-    CATEGORY = "ipadapter"
+    CATEGORY = "ipadapter/tiled"
 
     def apply_tiled(self, model, ipadapter, image, weight, weight_type, start_at, end_at, sharpening, combine_embeds="concat", image_negative=None, attn_mask=None, clip_vision=None, embeds_scaling='V only'):
         # 1. Select the models
@@ -897,7 +943,7 @@ def INPUT_TYPES(s):
 
     RETURN_TYPES = ("MODEL",)
     FUNCTION = "apply_ipadapter"
-    CATEGORY = "ipadapter"
+    CATEGORY = "ipadapter/embeds"
 
     def apply_ipadapter(self, model, ipadapter, pos_embed, weight, weight_type, start_at, end_at, neg_embed=None, attn_mask=None, clip_vision=None, embeds_scaling='V only'):
         ipa_args = {
@@ -947,7 +993,7 @@ def INPUT_TYPES(s):
     RETURN_TYPES = ("EMBEDS", "EMBEDS",)
     RETURN_NAMES = ("pos_embed", "neg_embed",)
     FUNCTION = "encode"
-    CATEGORY = "ipadapter"
+    CATEGORY = "ipadapter/embeds"
 
     def encode(self, ipadapter, image, weight, mask=None, clip_vision=None):
         if 'ipadapter' in ipadapter:
@@ -1002,7 +1048,7 @@ def INPUT_TYPES(s):
 
     RETURN_TYPES = ("EMBEDS",)
     FUNCTION = "batch"
-    CATEGORY = "ipadapter"
+    CATEGORY = "ipadapter/embeds"
 
     def batch(self, embed1, method, embed2=None, embed3=None, embed4=None, embed5=None):
         if method=='concat' and embed2 is None and embed3 is None and embed4 is None and embed5 is None:
@@ -1044,7 +1090,7 @@ def INPUT_TYPES(s):
 
     RETURN_TYPES = ("IMAGE",)
     FUNCTION = "make_noise"
-    CATEGORY = "ipadapter"
+    CATEGORY = "ipadapter/utils"
 
     def make_noise(self, type, strength, blur, image_optional=None):
         if image_optional is None:
@@ -1103,7 +1149,7 @@ def INPUT_TYPES(s):
     RETURN_TYPES = ("IMAGE",)
     FUNCTION = "prep_image"
 
-    CATEGORY = "ipadapter"
+    CATEGORY = "ipadapter/utils"
 
     def prep_image(self, image, interpolation="LANCZOS", crop_position="center", sharpening=0.0):
         size = (224, 224)
@@ -1167,7 +1213,7 @@ def INPUT_TYPES(s):
     RETURN_TYPES = ()
     FUNCTION = "save"
     OUTPUT_NODE = True
-    CATEGORY = "ipadapter"
+    CATEGORY = "ipadapter/embeds"
 
     def save(self, embeds, filename_prefix):
         full_output_folder, filename, counter, subfolder, filename_prefix = folder_paths.get_save_image_path(filename_prefix, self.output_dir)
@@ -1186,7 +1232,7 @@ def INPUT_TYPES(s):
 
     RETURN_TYPES = ("EMBEDS", )
     FUNCTION = "load"
-    CATEGORY = "ipadapter"
+    CATEGORY = "ipadapter/embeds"
 
     def load(self, embeds):
         path = folder_paths.get_annotated_filepath(embeds)
@@ -1208,6 +1254,7 @@ def load(self, embeds):
     "IPAdapterTiledBatch": IPAdapterTiledBatch,
     "IPAdapterEmbeds": IPAdapterEmbeds,
     "IPAdapterStyleComposition": IPAdapterStyleComposition,
+    "IPAdapterStyleCompositionBatch": IPAdapterStyleCompositionBatch,
 
     # Loaders
     "IPAdapterUnifiedLoader": IPAdapterUnifiedLoader,
@@ -1236,6 +1283,7 @@ def load(self, embeds):
     "IPAdapterTiledBatch": "IPAdapter Tiled Batch",
     "IPAdapterEmbeds": "IPAdapter Embeds",
     "IPAdapterStyleComposition": "IPAdapter Style & Composition SDXL",
+    "IPAdapterStyleCompositionBatch": "IPAdapter Style & Composition Batch SDXL",
 
     # Loaders
     "IPAdapterUnifiedLoader": "IPAdapter Unified Loader",

README.md

@@ -23,10 +23,12 @@ Not to mention the documentation and videos tutorials. Check my **ComfyUI Advanc
 
 The only way to keep the code open and free is by sponsoring its development. The more sponsorships the more time I can dedicate to my open source projects.
 
-Please consider a [Github Sponsorship](https://github.com/sponsors/cubiq) or [PayPal donation](https://paypal.me/matt3o) (Matteo "matt3o" Spinelli). For sponsorships of $50+, let me know if you'd like to be mentioned in this readme file, you can find me on [Discord](https://latent.vision/discord) or _matt3o:snail:gmail.com_.
+Please consider a [Github Sponsorship](https://github.com/sponsors/cubiq) or [PayPal donation](https://paypal.me/matt3o) (Matteo "matt3o" Spinelli). For sponsorships of $50+, let me know if you'd like to be mentioned in this readme file, you can find me on [Discord](https://latent.vision/discord) or _matt3o :snail: gmail.com_.
 
 ## Important updates
 
+**2024/04/09**: Added experimental Style/Composition transfer for SD1.5. The results are often not as good as SDXL. Optimal weight seems to be from 0.8 to 2.0. The **Style+Composition node doesn't work for SD1.5** at the moment, you can only alter either the Style or the Composition, I need more time for testing. Old workflows will still work **but you may need to refresh the page and re-select the weight type!**
+
 **2024/04/04**: Added Style & Composition node. It's now possible to apply both Style and Composition from the same node
 
 **2024/04/01**: Added Composition only transfer weight type for SDXL
@@ -91,7 +93,7 @@ For the Unified Loader to work the files need to be named exactly as shown in th
     - [ip-adapter-faceid-plusv2_sdxl.bin](https://huggingface.co/h94/IP-Adapter-FaceID/resolve/main/ip-adapter-faceid-plusv2_sdxl.bin), SDXL plus v2
     - [ip-adapter-faceid-portrait_sdxl.bin](https://huggingface.co/h94/IP-Adapter-FaceID/resolve/main/ip-adapter-faceid-portrait_sdxl.bin), SDXL text prompt style transfer
     - **Deprecated** [ip-adapter-faceid-plus_sd15.bin](https://huggingface.co/h94/IP-Adapter-FaceID/resolve/main/ip-adapter-faceid-plus_sd15.bin), FaceID plus v1 
-    - **Deprecated** [ip-adapter-faceid-portrait_sd15.bin](https://huggingface.co/h94/IP-Adapter-FaceID/resolve/main/ip-adapter-faceid-portrait-v11_sd15.bin), v1 of the portrait model
+    - **Deprecated** [ip-adapter-faceid-portrait_sd15.bin](https://huggingface.co/h94/IP-Adapter-FaceID/resolve/main/ip-adapter-faceid-portrait_sd15.bin), v1 of the portrait model
 
 Most FaceID models require a LoRA. If you use the `IPAdapter Unified Loader FaceID` it will be loaded automatically if you follow the naming convention. Otherwise you have to load them manually, be careful each FaceID model has to be paired with its own specific LoRA.
 
@@ -136,6 +138,10 @@ It's only thanks to generous sponsors that **the whole community** can enjoy ope
 
 [![Kaiber.ai](https://f.latent.vision/imgs/kaiber.png)](https://kaiber.ai/)
 
+### Companies supporting my projects
+
+- [RunComfy](https://www.runcomfy.com/) (ComfyUI Cloud)
+
 ### Esteemed individuals
 
 - [Jack Gane](https://github.com/ganeJackS)