From 058f94e7e8114fecd13feb8d8090124a82b5d71b Mon Sep 17 00:00:00 2001 From: Sayak Paul Date: Fri, 24 Mar 2023 20:57:28 +0530 Subject: [PATCH 1/6] update docs to reflect the updated ckpts. --- .../source/en/api/pipelines/stable_unclip.mdx | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/docs/source/en/api/pipelines/stable_unclip.mdx b/docs/source/en/api/pipelines/stable_unclip.mdx index 40bc3e27af77..339d8d29547b 100644 --- a/docs/source/en/api/pipelines/stable_unclip.mdx +++ b/docs/source/en/api/pipelines/stable_unclip.mdx @@ -16,6 +16,10 @@ Stable unCLIP checkpoints are finetuned from [stable diffusion 2.1](./stable_dif Stable unCLIP also still conditions on text embeddings. Given the two separate conditionings, stable unCLIP can be used for text guided image variation. When combined with an unCLIP prior, it can also be used for full text to image generation. +To know more about the unCLIP process, check out the following paper: + +[Hierarchical Text-Conditional Image Generation with CLIP Latents](https://arxiv.org/abs/2204.06125) by Aditya Ramesh, Prafulla Dhariwal, Alex Nichol, Casey Chu, Mark Chen. + ## Tips Stable unCLIP takes a `noise_level` as input during inference. `noise_level` determines how much noise is added @@ -24,23 +28,15 @@ we do not add any additional noise to the image embeddings i.e. `noise_level = 0 ### Available checkpoints: -TODO +* Image variation + * [stabilityai/stable-diffusion-2-1-unclip](https://hf.co/stabilityai/stable-diffusion-2-1-unclip) + * [stabilityai/stable-diffusion-2-1-unclip-small](https://hf.co/stabilityai/stable-diffusion-2-1-unclip-small) +* Text-to-image + * Coming soon! ### Text-to-Image Generation -```python -import torch -from diffusers import StableUnCLIPPipeline - -pipe = StableUnCLIPPipeline.from_pretrained( - "fusing/stable-unclip-2-1-l", torch_dtype=torch.float16 -) # TODO update model path -pipe = pipe.to("cuda") - -prompt = "a photo of an astronaut riding a horse on mars" -images = pipe(prompt).images -images[0].save("astronaut_horse.png") -``` +Coming soon! ### Text guided Image-to-Image Variation @@ -54,8 +50,8 @@ from io import BytesIO from diffusers import StableUnCLIPImg2ImgPipeline pipe = StableUnCLIPImg2ImgPipeline.from_pretrained( - "fusing/stable-unclip-2-1-l-img2img", torch_dtype=torch.float16 -) # TODO update model path + "stabilityai/stable-diffusion-2-1-unclip", torch_dtype=torch.float16 +) pipe = pipe.to("cuda") url = "https://raw.githubusercontent.com/CompVis/stable-diffusion/main/assets/stable-samples/img2img/sketch-mountains-input.jpg" From 42d06a564f6466efbb9a7ab593e9a7d8199c9747 Mon Sep 17 00:00:00 2001 From: Sayak Paul Date: Fri, 24 Mar 2023 21:03:24 +0530 Subject: [PATCH 2/6] update: point about prompt. --- docs/source/en/api/pipelines/stable_unclip.mdx | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/source/en/api/pipelines/stable_unclip.mdx b/docs/source/en/api/pipelines/stable_unclip.mdx index 339d8d29547b..c66ca5839972 100644 --- a/docs/source/en/api/pipelines/stable_unclip.mdx +++ b/docs/source/en/api/pipelines/stable_unclip.mdx @@ -60,6 +60,13 @@ response = requests.get(url) init_image = Image.open(BytesIO(response.content)).convert("RGB") init_image = init_image.resize((768, 512)) +images = pipe(init_image).images +images[0].save("fantasy_landscape.png") +``` + +Optionally, you can also pass a prompt to `pipe` such as: + +```python prompt = "A fantasy landscape, trending on artstation" images = pipe(prompt, init_image).images From b959a1b1a82969dc2a6ffa6ca67cb7d88d6bd1a7 Mon Sep 17 00:00:00 2001 From: Sayak Paul Date: Fri, 24 Mar 2023 21:09:30 +0530 Subject: [PATCH 3/6] Apply suggestions from code review Co-authored-by: Patrick von Platen --- docs/source/en/api/pipelines/stable_unclip.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/en/api/pipelines/stable_unclip.mdx b/docs/source/en/api/pipelines/stable_unclip.mdx index c66ca5839972..323423fb9350 100644 --- a/docs/source/en/api/pipelines/stable_unclip.mdx +++ b/docs/source/en/api/pipelines/stable_unclip.mdx @@ -50,7 +50,7 @@ from io import BytesIO from diffusers import StableUnCLIPImg2ImgPipeline pipe = StableUnCLIPImg2ImgPipeline.from_pretrained( - "stabilityai/stable-diffusion-2-1-unclip", torch_dtype=torch.float16 + "stabilityai/stable-diffusion-2-1-unclip", torch_dtype=torch.float16, variation="fp16" ) pipe = pipe.to("cuda") From 6a0fd31e839cd325e7e83a42f92a9efcd828d1de Mon Sep 17 00:00:00 2001 From: Sayak Paul Date: Fri, 24 Mar 2023 21:11:26 +0530 Subject: [PATCH 4/6] emove image resizing. --- docs/source/en/api/pipelines/stable_unclip.mdx | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/source/en/api/pipelines/stable_unclip.mdx b/docs/source/en/api/pipelines/stable_unclip.mdx index 323423fb9350..4b9074883d12 100644 --- a/docs/source/en/api/pipelines/stable_unclip.mdx +++ b/docs/source/en/api/pipelines/stable_unclip.mdx @@ -58,7 +58,6 @@ url = "https://raw.githubusercontent.com/CompVis/stable-diffusion/main/assets/st response = requests.get(url) init_image = Image.open(BytesIO(response.content)).convert("RGB") -init_image = init_image.resize((768, 512)) images = pipe(init_image).images images[0].save("fantasy_landscape.png") From 55191d0b22a0d8aa0fdfeda18c3955de4dfe6384 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Fri, 24 Mar 2023 17:16:37 +0100 Subject: [PATCH 5/6] Apply suggestions from code review --- docs/source/en/api/pipelines/stable_unclip.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/en/api/pipelines/stable_unclip.mdx b/docs/source/en/api/pipelines/stable_unclip.mdx index 4b9074883d12..e8db94f5e593 100644 --- a/docs/source/en/api/pipelines/stable_unclip.mdx +++ b/docs/source/en/api/pipelines/stable_unclip.mdx @@ -68,7 +68,7 @@ Optionally, you can also pass a prompt to `pipe` such as: ```python prompt = "A fantasy landscape, trending on artstation" -images = pipe(prompt, init_image).images +images = pipe(init_image, prompt=prompt).images images[0].save("fantasy_landscape.png") ``` From 73afba1c81f8a8bb2c6382f9bcc8f43486f64f63 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Fri, 24 Mar 2023 17:17:55 +0100 Subject: [PATCH 6/6] Apply suggestions from code review --- docs/source/en/api/pipelines/stable_unclip.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/en/api/pipelines/stable_unclip.mdx b/docs/source/en/api/pipelines/stable_unclip.mdx index e8db94f5e593..c8b5d58705ba 100644 --- a/docs/source/en/api/pipelines/stable_unclip.mdx +++ b/docs/source/en/api/pipelines/stable_unclip.mdx @@ -54,7 +54,7 @@ pipe = StableUnCLIPImg2ImgPipeline.from_pretrained( ) pipe = pipe.to("cuda") -url = "https://raw.githubusercontent.com/CompVis/stable-diffusion/main/assets/stable-samples/img2img/sketch-mountains-input.jpg" +url = "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/stable_unclip/tarsila_do_amaral.png" response = requests.get(url) init_image = Image.open(BytesIO(response.content)).convert("RGB")