From 4fac14f05b7720acb955c7f873e4f962218e54cc Mon Sep 17 00:00:00 2001 From: Hezi Zisman Date: Wed, 14 Feb 2024 14:15:51 +0200 Subject: [PATCH 1/8] allow passing tokenizer & text_encoder to unload_textual_inversion --- src/diffusers/loaders/textual_inversion.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/diffusers/loaders/textual_inversion.py b/src/diffusers/loaders/textual_inversion.py index f20ad6fba667..842dfd9bc8f4 100644 --- a/src/diffusers/loaders/textual_inversion.py +++ b/src/diffusers/loaders/textual_inversion.py @@ -457,6 +457,8 @@ def load_textual_inversion( def unload_textual_inversion( self, tokens: Optional[Union[str, List[str]]] = None, + tokenizer: Optional["PreTrainedTokenizer"] = None, + text_encoder: Optional["PreTrainedModel"] = None, ): r""" Unload Textual Inversion embeddings from the text encoder of [`StableDiffusionPipeline`] @@ -484,8 +486,8 @@ def unload_textual_inversion( ``` """ - tokenizer = getattr(self, "tokenizer", None) - text_encoder = getattr(self, "text_encoder", None) + tokenizer = tokenizer or getattr(self, "tokenizer", None) + text_encoder = text_encoder or getattr(self, "text_encoder", None) # Get textual inversion tokens and ids token_ids = [] From 7567b821bba73c6b3f7fd604d71051b1892c7457 Mon Sep 17 00:00:00 2001 From: Hezi Zisman Date: Wed, 14 Feb 2024 16:03:00 +0200 Subject: [PATCH 2/8] add ysage example --- src/diffusers/loaders/textual_inversion.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/diffusers/loaders/textual_inversion.py b/src/diffusers/loaders/textual_inversion.py index 842dfd9bc8f4..d5881c4cdd62 100644 --- a/src/diffusers/loaders/textual_inversion.py +++ b/src/diffusers/loaders/textual_inversion.py @@ -483,6 +483,23 @@ def unload_textual_inversion( # Remove just one token pipeline.unload_textual_inversion("") + + # Examples 3 (based on SDXL LoRA Advanced training) + pipeline = AutoPipelineForText2Image.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0") + embedding_path = hf_hub_download(repo_id="LinoyTsaban/web_y2k_lora", filename="web_y2k_emb.safetensors", repo_type="model") + + # load embeddings to the text encoders + state_dict = load_file(embedding_path) + + # load embeddings of text_encoder 1 (CLIP ViT-L/14) + pipeline.load_textual_inversion(state_dict["clip_l"], token=["", ""], text_encoder=pipe.text_encoder, tokenizer=pipe.tokenizer) + # load embeddings of text_encoder 2 (CLIP ViT-G/14) + pipeline.load_textual_inversion(state_dict["clip_g"], token=["", ""], text_encoder=pipe.text_encoder_2, tokenizer=pipe.tokenizer_2) + + # Unload explicitly from both text encoders abd tokenizers + pipeline.unload_textual_inversion(tokens=["", ""], text_encoder=pipe.text_encoder, tokenizer=pipe.tokenizer) + pipeline.unload_textual_inversion(tokens=["", ""], text_encoder=pipe.text_encoder_2, tokenizer=pipe.tokenizer_2) + ``` """ From eb9768275b639210227fbda47d803f7e077bb3b5 Mon Sep 17 00:00:00 2001 From: Hezi Zisman Date: Sun, 18 Feb 2024 09:19:31 +0200 Subject: [PATCH 3/8] Update src/diffusers/loaders/textual_inversion.py Co-authored-by: Fabio Rigano --- src/diffusers/loaders/textual_inversion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diffusers/loaders/textual_inversion.py b/src/diffusers/loaders/textual_inversion.py index d5881c4cdd62..1df3b689dde0 100644 --- a/src/diffusers/loaders/textual_inversion.py +++ b/src/diffusers/loaders/textual_inversion.py @@ -492,7 +492,7 @@ def unload_textual_inversion( state_dict = load_file(embedding_path) # load embeddings of text_encoder 1 (CLIP ViT-L/14) - pipeline.load_textual_inversion(state_dict["clip_l"], token=["", ""], text_encoder=pipe.text_encoder, tokenizer=pipe.tokenizer) + pipeline.load_textual_inversion(state_dict["clip_l"], token=["", ""], text_encoder=pipeline.text_encoder, tokenizer=pipeline.tokenizer) # load embeddings of text_encoder 2 (CLIP ViT-G/14) pipeline.load_textual_inversion(state_dict["clip_g"], token=["", ""], text_encoder=pipe.text_encoder_2, tokenizer=pipe.tokenizer_2) From f7d1dab12527aab507d09c539f5b9145b23f0d40 Mon Sep 17 00:00:00 2001 From: Hezi Zisman Date: Sun, 18 Feb 2024 09:19:37 +0200 Subject: [PATCH 4/8] Update src/diffusers/loaders/textual_inversion.py Co-authored-by: Fabio Rigano --- src/diffusers/loaders/textual_inversion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diffusers/loaders/textual_inversion.py b/src/diffusers/loaders/textual_inversion.py index 1df3b689dde0..a689c4b9ecc7 100644 --- a/src/diffusers/loaders/textual_inversion.py +++ b/src/diffusers/loaders/textual_inversion.py @@ -494,7 +494,7 @@ def unload_textual_inversion( # load embeddings of text_encoder 1 (CLIP ViT-L/14) pipeline.load_textual_inversion(state_dict["clip_l"], token=["", ""], text_encoder=pipeline.text_encoder, tokenizer=pipeline.tokenizer) # load embeddings of text_encoder 2 (CLIP ViT-G/14) - pipeline.load_textual_inversion(state_dict["clip_g"], token=["", ""], text_encoder=pipe.text_encoder_2, tokenizer=pipe.tokenizer_2) + pipeline.load_textual_inversion(state_dict["clip_g"], token=["", ""], text_encoder=pipeline.text_encoder_2, tokenizer=pipeline.tokenizer_2) # Unload explicitly from both text encoders abd tokenizers pipeline.unload_textual_inversion(tokens=["", ""], text_encoder=pipe.text_encoder, tokenizer=pipe.tokenizer) From 56e9573ce04c716f49f05b456ebe6426a83cb494 Mon Sep 17 00:00:00 2001 From: Hezi Zisman Date: Sun, 18 Feb 2024 09:19:42 +0200 Subject: [PATCH 5/8] Update src/diffusers/loaders/textual_inversion.py Co-authored-by: Fabio Rigano --- src/diffusers/loaders/textual_inversion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diffusers/loaders/textual_inversion.py b/src/diffusers/loaders/textual_inversion.py index a689c4b9ecc7..15876fc52729 100644 --- a/src/diffusers/loaders/textual_inversion.py +++ b/src/diffusers/loaders/textual_inversion.py @@ -497,7 +497,7 @@ def unload_textual_inversion( pipeline.load_textual_inversion(state_dict["clip_g"], token=["", ""], text_encoder=pipeline.text_encoder_2, tokenizer=pipeline.tokenizer_2) # Unload explicitly from both text encoders abd tokenizers - pipeline.unload_textual_inversion(tokens=["", ""], text_encoder=pipe.text_encoder, tokenizer=pipe.tokenizer) + pipeline.unload_textual_inversion(tokens=["", ""], text_encoder=pipeline.text_encoder, tokenizer=pipeline.tokenizer) pipeline.unload_textual_inversion(tokens=["", ""], text_encoder=pipe.text_encoder_2, tokenizer=pipe.tokenizer_2) ``` From 231af68b1879d3302378ebfea949bdf7b9fc0cc8 Mon Sep 17 00:00:00 2001 From: Hezi Zisman Date: Sun, 18 Feb 2024 09:19:47 +0200 Subject: [PATCH 6/8] Update src/diffusers/loaders/textual_inversion.py Co-authored-by: Fabio Rigano --- src/diffusers/loaders/textual_inversion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diffusers/loaders/textual_inversion.py b/src/diffusers/loaders/textual_inversion.py index 15876fc52729..1171de11cbd5 100644 --- a/src/diffusers/loaders/textual_inversion.py +++ b/src/diffusers/loaders/textual_inversion.py @@ -498,7 +498,7 @@ def unload_textual_inversion( # Unload explicitly from both text encoders abd tokenizers pipeline.unload_textual_inversion(tokens=["", ""], text_encoder=pipeline.text_encoder, tokenizer=pipeline.tokenizer) - pipeline.unload_textual_inversion(tokens=["", ""], text_encoder=pipe.text_encoder_2, tokenizer=pipe.tokenizer_2) + pipeline.unload_textual_inversion(tokens=["", ""], text_encoder=pipeline.text_encoder_2, tokenizer=pipeline.tokenizer_2) ``` """ From 875fff9f5de8db7603ecad697d1455129729d907 Mon Sep 17 00:00:00 2001 From: Hezi Zisman Date: Sun, 18 Feb 2024 11:45:21 +0200 Subject: [PATCH 7/8] Update src/diffusers/loaders/textual_inversion.py Co-authored-by: Fabio Rigano --- src/diffusers/loaders/textual_inversion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diffusers/loaders/textual_inversion.py b/src/diffusers/loaders/textual_inversion.py index 1171de11cbd5..7c135ce21bc0 100644 --- a/src/diffusers/loaders/textual_inversion.py +++ b/src/diffusers/loaders/textual_inversion.py @@ -486,7 +486,7 @@ def unload_textual_inversion( # Examples 3 (based on SDXL LoRA Advanced training) pipeline = AutoPipelineForText2Image.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0") - embedding_path = hf_hub_download(repo_id="LinoyTsaban/web_y2k_lora", filename="web_y2k_emb.safetensors", repo_type="model") + embedding_path = hf_hub_download(repo_id="linoyts/web_y2k", filename="web_y2k_emb.safetensors", repo_type="model") # load embeddings to the text encoders state_dict = load_file(embedding_path) From 1096db97e106293c128e7f9de83a283c065b4c4f Mon Sep 17 00:00:00 2001 From: Hezi Zisman Date: Sun, 18 Feb 2024 12:06:09 +0200 Subject: [PATCH 8/8] change example 3 description --- src/diffusers/loaders/textual_inversion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diffusers/loaders/textual_inversion.py b/src/diffusers/loaders/textual_inversion.py index 7c135ce21bc0..e5aeea488407 100644 --- a/src/diffusers/loaders/textual_inversion.py +++ b/src/diffusers/loaders/textual_inversion.py @@ -484,7 +484,7 @@ def unload_textual_inversion( # Remove just one token pipeline.unload_textual_inversion("") - # Examples 3 (based on SDXL LoRA Advanced training) + # Example 3: unload from SDXL pipeline = AutoPipelineForText2Image.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0") embedding_path = hf_hub_download(repo_id="linoyts/web_y2k", filename="web_y2k_emb.safetensors", repo_type="model")