From b79474e975e2f6a942bafc59501a85be0733f822 Mon Sep 17 00:00:00 2001 From: dunkeroni Date: Thu, 11 Apr 2024 13:58:00 -0400 Subject: [PATCH 01/15] gradient mask node test for inpaint --- invokeai/app/invocations/latent.py | 53 ++++++++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/invokeai/app/invocations/latent.py b/invokeai/app/invocations/latent.py index 4534df89c18..4e9967acf4a 100644 --- a/invokeai/app/invocations/latent.py +++ b/invokeai/app/invocations/latent.py @@ -185,7 +185,7 @@ class GradientMaskOutput(BaseInvocationOutput): title="Create Gradient Mask", tags=["mask", "denoise"], category="latents", - version="1.0.0", + version="1.1.0", ) class CreateGradientMaskInvocation(BaseInvocation): """Creates mask for denoising model run.""" @@ -198,6 +198,32 @@ class CreateGradientMaskInvocation(BaseInvocation): minimum_denoise: float = InputField( default=0.0, ge=0, le=1, description="Minimum denoise level for the coherence region", ui_order=4 ) + unet: Optional[UNetField] = InputField( + description="OPTIONAL: If the Unet is a specialized Inpainting model, masked_latents will be generated from the image with the VAE", + input=Input.Connection, + title="[OPTIONAL] UNet", + ui_order=5, + ) + image: Optional[ImageField] = InputField( + default=None, + description="OPTIONAL: If the Unet is a specialized Inpainting model, masked_latents will be generated from the image with the VAE", + title="[OPTIONAL] Image", + ui_order=6 + ) + vae: Optional[VAEField] = InputField( + default=None, + description="OPTIONAL: If the Unet is a specialized Inpainting model, masked_latents will be generated from the image with the VAE", + title="[OPTIONAL] VAE", + input=Input.Connection, + ui_order=7 + ) + tiled: bool = InputField(default=False, description=FieldDescriptions.tiled, ui_order=8) + fp32: bool = InputField( + default=DEFAULT_PRECISION == "float32", + description=FieldDescriptions.fp32, + ui_order=9, + ) + @torch.no_grad() def invoke(self, context: InvocationContext) -> GradientMaskOutput: @@ -233,8 +259,31 @@ def invoke(self, context: InvocationContext) -> GradientMaskOutput: expanded_mask_image = Image.fromarray((expanded_mask.squeeze(0).numpy() * 255).astype(np.uint8), mode="L") expanded_image_dto = context.images.save(expanded_mask_image) + masked_latents_name = None + # Check for Inpaint model and generate masked_latents + if self.unet is not None and self.vae is not None and self.image is not None: + #all three fields must be present at the same time + unet_info: UNet2DConditionModel = context.models.load(self.unet.unet) + quick_info = context.models.get_config(self.unet.unet) + quick_inpaint = quick_info.variant == "inpaint" + print(f"quick_inpaint: {quick_info.variant}") + assert isinstance(unet_info.model, UNet2DConditionModel) + is_inpaint = unet_info.conv_in.in_channels == 9 + if is_inpaint: + mask = blur_tensor + vae_info: LoadedModel = context.models.load(self.vae.vae) + image = context.images.get_pil(self.image.image_name) + image_tensor = image_resized_to_grid_as_tensor(image.convert("RGB")) + if image_tensor.dim() == 3: + image_tensor = image_tensor.unsqueeze(0) + img_mask = tv_resize(mask, image_tensor.shape[-2:], T.InterpolationMode.BILINEAR, antialias=False) + masked_image = image_tensor * torch.where(img_mask < 1, 0.0, 1.0) # <1 to include gradient area + masked_latents = ImageToLatentsInvocation.vae_encode(vae_info, self.fp32, self.tiled, masked_image.clone()) + masked_latents_name = context.tensors.save(tensor=masked_latents) + + return GradientMaskOutput( - denoise_mask=DenoiseMaskField(mask_name=mask_name, masked_latents_name=None, gradient=True), + denoise_mask=DenoiseMaskField(mask_name=mask_name, masked_latents_name=masked_latents_name, gradient=True), expanded_mask_area=ImageField(image_name=expanded_image_dto.image_name), ) From a711888fec53d3f314b8167cd82d7d1285addd3e Mon Sep 17 00:00:00 2001 From: dunkeroni Date: Thu, 11 Apr 2024 18:15:00 -0400 Subject: [PATCH 02/15] add unet check in gradient mask node --- invokeai/app/invocations/latent.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/invokeai/app/invocations/latent.py b/invokeai/app/invocations/latent.py index 4e9967acf4a..aef9167c353 100644 --- a/invokeai/app/invocations/latent.py +++ b/invokeai/app/invocations/latent.py @@ -200,6 +200,7 @@ class CreateGradientMaskInvocation(BaseInvocation): ) unet: Optional[UNetField] = InputField( description="OPTIONAL: If the Unet is a specialized Inpainting model, masked_latents will be generated from the image with the VAE", + default=None, input=Input.Connection, title="[OPTIONAL] UNet", ui_order=5, @@ -263,12 +264,9 @@ def invoke(self, context: InvocationContext) -> GradientMaskOutput: # Check for Inpaint model and generate masked_latents if self.unet is not None and self.vae is not None and self.image is not None: #all three fields must be present at the same time - unet_info: UNet2DConditionModel = context.models.load(self.unet.unet) - quick_info = context.models.get_config(self.unet.unet) - quick_inpaint = quick_info.variant == "inpaint" - print(f"quick_inpaint: {quick_info.variant}") + unet_info = context.models.load(self.unet.unet) assert isinstance(unet_info.model, UNet2DConditionModel) - is_inpaint = unet_info.conv_in.in_channels == 9 + is_inpaint = unet_info.model.conv_in.in_channels == 9 if is_inpaint: mask = blur_tensor vae_info: LoadedModel = context.models.load(self.vae.vae) From f5d4a80ad58b282649dffdf5cc06a9d6e9ab441a Mon Sep 17 00:00:00 2001 From: dunkeroni Date: Thu, 11 Apr 2024 19:34:00 -0400 Subject: [PATCH 03/15] fix threshhold --- invokeai/app/invocations/latent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/invokeai/app/invocations/latent.py b/invokeai/app/invocations/latent.py index aef9167c353..5f970ab04f6 100644 --- a/invokeai/app/invocations/latent.py +++ b/invokeai/app/invocations/latent.py @@ -275,7 +275,7 @@ def invoke(self, context: InvocationContext) -> GradientMaskOutput: if image_tensor.dim() == 3: image_tensor = image_tensor.unsqueeze(0) img_mask = tv_resize(mask, image_tensor.shape[-2:], T.InterpolationMode.BILINEAR, antialias=False) - masked_image = image_tensor * torch.where(img_mask < 1, 0.0, 1.0) # <1 to include gradient area + masked_image = image_tensor * torch.where(img_mask < 0.5, 0.0, 1.0) # <1 to include gradient area masked_latents = ImageToLatentsInvocation.vae_encode(vae_info, self.fp32, self.tiled, masked_image.clone()) masked_latents_name = context.tensors.save(tensor=masked_latents) From 91508d697a2434701b714b674a7e9f35f5546754 Mon Sep 17 00:00:00 2001 From: dunkeroni Date: Thu, 11 Apr 2024 22:34:07 -0400 Subject: [PATCH 04/15] update canvas graphs --- .../nodes/util/graph/addVAEToGraph.ts | 11 +++++++++ .../util/graph/buildCanvasInpaintGraph.ts | 23 +++++++++++++++++++ .../util/graph/buildCanvasOutpaintGraph.ts | 23 +++++++++++++++++++ .../util/graph/buildCanvasSDXLInpaintGraph.ts | 23 +++++++++++++++++++ .../graph/buildCanvasSDXLOutpaintGraph.ts | 22 ++++++++++++++++++ .../frontend/web/src/services/api/schema.ts | 19 +++++++++++++++ 6 files changed, 121 insertions(+) diff --git a/invokeai/frontend/web/src/features/nodes/util/graph/addVAEToGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graph/addVAEToGraph.ts index 242fc152981..411e8860e67 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graph/addVAEToGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graph/addVAEToGraph.ts @@ -10,6 +10,7 @@ import { IMAGE_TO_IMAGE_GRAPH, IMAGE_TO_LATENTS, INPAINT_IMAGE, + INPAINT_CREATE_MASK, LATENTS_TO_IMAGE, MAIN_MODEL_LOADER, SDXL_CANVAS_IMAGE_TO_IMAGE_GRAPH, @@ -145,6 +146,16 @@ export const addVAEToGraph = async ( field: 'vae', }, }, + { + source: { + node_id: isSeamlessEnabled ? SEAMLESS : isAutoVae ? modelLoaderNodeId : VAE_LOADER, + field: 'vae', + }, + destination: { + node_id: INPAINT_CREATE_MASK, + field: 'vae', + }, + }, { source: { diff --git a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasInpaintGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasInpaintGraph.ts index 372f7d8fe5c..e4f24d531cf 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasInpaintGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasInpaintGraph.ts @@ -133,6 +133,9 @@ export const buildCanvasInpaintGraph = async ( coherence_mode: canvasCoherenceMode, minimum_denoise: canvasCoherenceMinDenoise, edge_radius: canvasCoherenceEdgeSize, + image: canvasInitImage, + tiled: false, + fp32: fp32, }, [DENOISE_LATENTS]: { type: 'denoise_latents', @@ -172,6 +175,16 @@ export const buildCanvasInpaintGraph = async ( field: 'unet', }, }, + { + source: { + node_id: modelLoaderNodeId, + field: 'unet', + }, + destination: { + node_id: INPAINT_CREATE_MASK, + field: 'unet', + }, + }, { source: { node_id: modelLoaderNodeId, @@ -331,6 +344,16 @@ export const buildCanvasInpaintGraph = async ( field: 'mask', }, }, + { + source: { + node_id: INPAINT_IMAGE_RESIZE_UP, + field: 'image', + }, + destination: { + node_id: INPAINT_CREATE_MASK, + field: 'image', + }, + }, // Resize Down { source: { diff --git a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasOutpaintGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasOutpaintGraph.ts index 6a59c51872a..af8aaa7ea25 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasOutpaintGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasOutpaintGraph.ts @@ -157,6 +157,9 @@ export const buildCanvasOutpaintGraph = async ( coherence_mode: canvasCoherenceMode, edge_radius: canvasCoherenceEdgeSize, minimum_denoise: canvasCoherenceMinDenoise, + image: canvasInitImage, + tiled: false, + fp32: fp32, }, [DENOISE_LATENTS]: { type: 'denoise_latents', @@ -197,6 +200,16 @@ export const buildCanvasOutpaintGraph = async ( field: 'unet', }, }, + { + source: { + node_id: modelLoaderNodeId, + field: 'unet', + }, + destination: { + node_id: INPAINT_CREATE_MASK, + field: 'unet', + }, + }, { source: { node_id: modelLoaderNodeId, @@ -453,6 +466,16 @@ export const buildCanvasOutpaintGraph = async ( field: 'image', }, }, + { + source: { + node_id: INPAINT_IMAGE_RESIZE_UP, + field: 'image', + }, + destination: { + node_id: INPAINT_CREATE_MASK, + field: 'image', + }, + }, // Resize Results Down { source: { diff --git a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLInpaintGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLInpaintGraph.ts index 6a648a0ea45..b6e6876503b 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLInpaintGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLInpaintGraph.ts @@ -135,6 +135,9 @@ export const buildCanvasSDXLInpaintGraph = async ( coherence_mode: canvasCoherenceMode, minimum_denoise: refinerModel ? Math.max(0.2, canvasCoherenceMinDenoise) : canvasCoherenceMinDenoise, edge_radius: canvasCoherenceEdgeSize, + image: canvasInitImage, + tiled: false, + fp32: fp32, }, [SDXL_DENOISE_LATENTS]: { type: 'denoise_latents', @@ -174,6 +177,16 @@ export const buildCanvasSDXLInpaintGraph = async ( field: 'unet', }, }, + { + source: { + node_id: modelLoaderNodeId, + field: 'unet', + }, + destination: { + node_id: INPAINT_CREATE_MASK, + field: 'unet', + }, + }, { source: { node_id: modelLoaderNodeId, @@ -342,6 +355,16 @@ export const buildCanvasSDXLInpaintGraph = async ( field: 'mask', }, }, + { + source: { + node_id: INPAINT_IMAGE_RESIZE_UP, + field: 'image', + }, + destination: { + node_id: INPAINT_CREATE_MASK, + field: 'image', + }, + }, // Resize Down { source: { diff --git a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLOutpaintGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLOutpaintGraph.ts index 87ad3e29a7d..21955905f30 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLOutpaintGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLOutpaintGraph.ts @@ -157,6 +157,8 @@ export const buildCanvasSDXLOutpaintGraph = async ( coherence_mode: canvasCoherenceMode, edge_radius: canvasCoherenceEdgeSize, minimum_denoise: refinerModel ? Math.max(0.2, canvasCoherenceMinDenoise) : canvasCoherenceMinDenoise, + tiled: false, + fp32: fp32, }, [SDXL_DENOISE_LATENTS]: { type: 'denoise_latents', @@ -197,6 +199,16 @@ export const buildCanvasSDXLOutpaintGraph = async ( field: 'unet', }, }, + { + source: { + node_id: modelLoaderNodeId, + field: 'unet', + }, + destination: { + node_id: INPAINT_CREATE_MASK, + field: 'unet', + }, + }, { source: { node_id: SDXL_MODEL_LOADER, @@ -451,6 +463,16 @@ export const buildCanvasSDXLOutpaintGraph = async ( field: 'image', }, }, + { + source: { + node_id: INPAINT_IMAGE_RESIZE_UP, + field: 'image', + }, + destination: { + node_id: INPAINT_CREATE_MASK, + field: 'image', + }, + }, // Take combined mask and resize { source: { diff --git a/invokeai/frontend/web/src/services/api/schema.ts b/invokeai/frontend/web/src/services/api/schema.ts index 834f75392fd..ec50231d033 100644 --- a/invokeai/frontend/web/src/services/api/schema.ts +++ b/invokeai/frontend/web/src/services/api/schema.ts @@ -2878,6 +2878,25 @@ export type components = { * @default 0 */ minimum_denoise?: number; + /** + * UNet + * @description The UNet input used to detect if it is an inpainting model + */ + unet?: components["schemas"]["UNetField"] | null; + /** Image */ + image?: components["schemas"]["ImageField"] | null; + /** vae */ + vae?: components["schemas"]["VAEField"] | null; + /** + * @description tiled is used by the VAE + * @default false + * */ + tiled?: boolean; + /** + * @description fp32 is used by the VAE + * @default false + */ + fp32?: boolean; /** * type * @default create_gradient_mask From b23d4926dd108672c957b306f064a56d050f0268 Mon Sep 17 00:00:00 2001 From: dunkeroni Date: Thu, 11 Apr 2024 23:12:32 -0400 Subject: [PATCH 05/15] remove unet test --- invokeai/app/invocations/latent.py | 39 +++++++------------ .../util/graph/buildCanvasInpaintGraph.ts | 10 ----- .../util/graph/buildCanvasOutpaintGraph.ts | 10 ----- .../util/graph/buildCanvasSDXLInpaintGraph.ts | 10 ----- .../graph/buildCanvasSDXLOutpaintGraph.ts | 10 ----- 5 files changed, 14 insertions(+), 65 deletions(-) diff --git a/invokeai/app/invocations/latent.py b/invokeai/app/invocations/latent.py index 5f970ab04f6..b4ab48c4f9c 100644 --- a/invokeai/app/invocations/latent.py +++ b/invokeai/app/invocations/latent.py @@ -198,22 +198,15 @@ class CreateGradientMaskInvocation(BaseInvocation): minimum_denoise: float = InputField( default=0.0, ge=0, le=1, description="Minimum denoise level for the coherence region", ui_order=4 ) - unet: Optional[UNetField] = InputField( - description="OPTIONAL: If the Unet is a specialized Inpainting model, masked_latents will be generated from the image with the VAE", - default=None, - input=Input.Connection, - title="[OPTIONAL] UNet", - ui_order=5, - ) image: Optional[ImageField] = InputField( default=None, - description="OPTIONAL: If the Unet is a specialized Inpainting model, masked_latents will be generated from the image with the VAE", + description="OPTIONAL: Only connect for specialized Inpainting models, masked_latents will be generated from the image with the VAE", title="[OPTIONAL] Image", ui_order=6 ) vae: Optional[VAEField] = InputField( default=None, - description="OPTIONAL: If the Unet is a specialized Inpainting model, masked_latents will be generated from the image with the VAE", + description="OPTIONAL: Only connect for specialized Inpainting models, masked_latents will be generated from the image with the VAE", title="[OPTIONAL] VAE", input=Input.Connection, ui_order=7 @@ -262,22 +255,18 @@ def invoke(self, context: InvocationContext) -> GradientMaskOutput: masked_latents_name = None # Check for Inpaint model and generate masked_latents - if self.unet is not None and self.vae is not None and self.image is not None: - #all three fields must be present at the same time - unet_info = context.models.load(self.unet.unet) - assert isinstance(unet_info.model, UNet2DConditionModel) - is_inpaint = unet_info.model.conv_in.in_channels == 9 - if is_inpaint: - mask = blur_tensor - vae_info: LoadedModel = context.models.load(self.vae.vae) - image = context.images.get_pil(self.image.image_name) - image_tensor = image_resized_to_grid_as_tensor(image.convert("RGB")) - if image_tensor.dim() == 3: - image_tensor = image_tensor.unsqueeze(0) - img_mask = tv_resize(mask, image_tensor.shape[-2:], T.InterpolationMode.BILINEAR, antialias=False) - masked_image = image_tensor * torch.where(img_mask < 0.5, 0.0, 1.0) # <1 to include gradient area - masked_latents = ImageToLatentsInvocation.vae_encode(vae_info, self.fp32, self.tiled, masked_image.clone()) - masked_latents_name = context.tensors.save(tensor=masked_latents) + if self.vae is not None and self.image is not None: + #both fields must be present at the same time + mask = blur_tensor + vae_info: LoadedModel = context.models.load(self.vae.vae) + image = context.images.get_pil(self.image.image_name) + image_tensor = image_resized_to_grid_as_tensor(image.convert("RGB")) + if image_tensor.dim() == 3: + image_tensor = image_tensor.unsqueeze(0) + img_mask = tv_resize(mask, image_tensor.shape[-2:], T.InterpolationMode.BILINEAR, antialias=False) + masked_image = image_tensor * torch.where(img_mask < 0.5, 0.0, 1.0) # <1 to include gradient area + masked_latents = ImageToLatentsInvocation.vae_encode(vae_info, self.fp32, self.tiled, masked_image.clone()) + masked_latents_name = context.tensors.save(tensor=masked_latents) return GradientMaskOutput( diff --git a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasInpaintGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasInpaintGraph.ts index e4f24d531cf..d5b826c43e9 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasInpaintGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasInpaintGraph.ts @@ -175,16 +175,6 @@ export const buildCanvasInpaintGraph = async ( field: 'unet', }, }, - { - source: { - node_id: modelLoaderNodeId, - field: 'unet', - }, - destination: { - node_id: INPAINT_CREATE_MASK, - field: 'unet', - }, - }, { source: { node_id: modelLoaderNodeId, diff --git a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasOutpaintGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasOutpaintGraph.ts index af8aaa7ea25..705b03e6c05 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasOutpaintGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasOutpaintGraph.ts @@ -200,16 +200,6 @@ export const buildCanvasOutpaintGraph = async ( field: 'unet', }, }, - { - source: { - node_id: modelLoaderNodeId, - field: 'unet', - }, - destination: { - node_id: INPAINT_CREATE_MASK, - field: 'unet', - }, - }, { source: { node_id: modelLoaderNodeId, diff --git a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLInpaintGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLInpaintGraph.ts index b6e6876503b..be21d1481de 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLInpaintGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLInpaintGraph.ts @@ -177,16 +177,6 @@ export const buildCanvasSDXLInpaintGraph = async ( field: 'unet', }, }, - { - source: { - node_id: modelLoaderNodeId, - field: 'unet', - }, - destination: { - node_id: INPAINT_CREATE_MASK, - field: 'unet', - }, - }, { source: { node_id: modelLoaderNodeId, diff --git a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLOutpaintGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLOutpaintGraph.ts index 21955905f30..d2bf3d3896f 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLOutpaintGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLOutpaintGraph.ts @@ -199,16 +199,6 @@ export const buildCanvasSDXLOutpaintGraph = async ( field: 'unet', }, }, - { - source: { - node_id: modelLoaderNodeId, - field: 'unet', - }, - destination: { - node_id: INPAINT_CREATE_MASK, - field: 'unet', - }, - }, { source: { node_id: SDXL_MODEL_LOADER, From bc51f9c23439a651b6ca88ad32cc607a55ae8a8a Mon Sep 17 00:00:00 2001 From: dunkeroni Date: Thu, 11 Apr 2024 23:25:04 -0400 Subject: [PATCH 06/15] chore: comments and ruff --- invokeai/app/invocations/latent.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/invokeai/app/invocations/latent.py b/invokeai/app/invocations/latent.py index b4ab48c4f9c..0f579cd9e7b 100644 --- a/invokeai/app/invocations/latent.py +++ b/invokeai/app/invocations/latent.py @@ -202,14 +202,14 @@ class CreateGradientMaskInvocation(BaseInvocation): default=None, description="OPTIONAL: Only connect for specialized Inpainting models, masked_latents will be generated from the image with the VAE", title="[OPTIONAL] Image", - ui_order=6 + ui_order=6, ) vae: Optional[VAEField] = InputField( default=None, description="OPTIONAL: Only connect for specialized Inpainting models, masked_latents will be generated from the image with the VAE", title="[OPTIONAL] VAE", input=Input.Connection, - ui_order=7 + ui_order=7, ) tiled: bool = InputField(default=False, description=FieldDescriptions.tiled, ui_order=8) fp32: bool = InputField( @@ -218,7 +218,6 @@ class CreateGradientMaskInvocation(BaseInvocation): ui_order=9, ) - @torch.no_grad() def invoke(self, context: InvocationContext) -> GradientMaskOutput: mask_image = context.images.get_pil(self.mask.image_name, mode="L") @@ -254,9 +253,8 @@ def invoke(self, context: InvocationContext) -> GradientMaskOutput: expanded_image_dto = context.images.save(expanded_mask_image) masked_latents_name = None - # Check for Inpaint model and generate masked_latents if self.vae is not None and self.image is not None: - #both fields must be present at the same time + # both fields must be present at the same time mask = blur_tensor vae_info: LoadedModel = context.models.load(self.vae.vae) image = context.images.get_pil(self.image.image_name) @@ -264,11 +262,10 @@ def invoke(self, context: InvocationContext) -> GradientMaskOutput: if image_tensor.dim() == 3: image_tensor = image_tensor.unsqueeze(0) img_mask = tv_resize(mask, image_tensor.shape[-2:], T.InterpolationMode.BILINEAR, antialias=False) - masked_image = image_tensor * torch.where(img_mask < 0.5, 0.0, 1.0) # <1 to include gradient area + masked_image = image_tensor * torch.where(img_mask < 0.5, 0.0, 1.0) masked_latents = ImageToLatentsInvocation.vae_encode(vae_info, self.fp32, self.tiled, masked_image.clone()) masked_latents_name = context.tensors.save(tensor=masked_latents) - return GradientMaskOutput( denoise_mask=DenoiseMaskField(mask_name=mask_name, masked_latents_name=masked_latents_name, gradient=True), expanded_mask_area=ImageField(image_name=expanded_image_dto.image_name), From db8ed93209e8108ad8a060dd10c5ca2e169bf6e3 Mon Sep 17 00:00:00 2001 From: dunkeroni Date: Fri, 12 Apr 2024 00:03:14 -0400 Subject: [PATCH 07/15] remove extra inputs --- .../web/src/features/nodes/util/graph/buildCanvasInpaintGraph.ts | 1 - .../src/features/nodes/util/graph/buildCanvasOutpaintGraph.ts | 1 - .../src/features/nodes/util/graph/buildCanvasSDXLInpaintGraph.ts | 1 - 3 files changed, 3 deletions(-) diff --git a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasInpaintGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasInpaintGraph.ts index d5b826c43e9..bdeeaf9ee62 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasInpaintGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasInpaintGraph.ts @@ -133,7 +133,6 @@ export const buildCanvasInpaintGraph = async ( coherence_mode: canvasCoherenceMode, minimum_denoise: canvasCoherenceMinDenoise, edge_radius: canvasCoherenceEdgeSize, - image: canvasInitImage, tiled: false, fp32: fp32, }, diff --git a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasOutpaintGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasOutpaintGraph.ts index 705b03e6c05..2d7b400c0ec 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasOutpaintGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasOutpaintGraph.ts @@ -157,7 +157,6 @@ export const buildCanvasOutpaintGraph = async ( coherence_mode: canvasCoherenceMode, edge_radius: canvasCoherenceEdgeSize, minimum_denoise: canvasCoherenceMinDenoise, - image: canvasInitImage, tiled: false, fp32: fp32, }, diff --git a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLInpaintGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLInpaintGraph.ts index be21d1481de..693c947f948 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLInpaintGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLInpaintGraph.ts @@ -135,7 +135,6 @@ export const buildCanvasSDXLInpaintGraph = async ( coherence_mode: canvasCoherenceMode, minimum_denoise: refinerModel ? Math.max(0.2, canvasCoherenceMinDenoise) : canvasCoherenceMinDenoise, edge_radius: canvasCoherenceEdgeSize, - image: canvasInitImage, tiled: false, fp32: fp32, }, From 2a1a81b0e8ba3a716ca611336021e2645f1950f0 Mon Sep 17 00:00:00 2001 From: dunkeroni Date: Fri, 12 Apr 2024 12:50:35 -0400 Subject: [PATCH 08/15] chore: frontend check error --- .../frontend/web/src/features/nodes/util/graph/addVAEToGraph.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/invokeai/frontend/web/src/features/nodes/util/graph/addVAEToGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graph/addVAEToGraph.ts index 411e8860e67..347027c539b 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graph/addVAEToGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graph/addVAEToGraph.ts @@ -9,8 +9,8 @@ import { CANVAS_TEXT_TO_IMAGE_GRAPH, IMAGE_TO_IMAGE_GRAPH, IMAGE_TO_LATENTS, - INPAINT_IMAGE, INPAINT_CREATE_MASK, + INPAINT_IMAGE, LATENTS_TO_IMAGE, MAIN_MODEL_LOADER, SDXL_CANVAS_IMAGE_TO_IMAGE_GRAPH, From 2b0ac16ca4d45f98604378ba4242828685d93c91 Mon Sep 17 00:00:00 2001 From: psychedelicious <4822129+psychedelicious@users.noreply.github.com> Date: Tue, 23 Apr 2024 19:47:58 +1000 Subject: [PATCH 09/15] tests: add test for probing diffusers model variant type --- tests/test_model_probe.py | 10 ++- .../sd-1/main/dreamshaper-8-inpainting/README | 1 + .../dreamshaper-8-inpainting/model_index.json | 34 ++++++++++ .../scheduler/scheduler_config.json | 23 +++++++ .../dreamshaper-8-inpainting/unet/config.json | 66 +++++++++++++++++++ .../diffusion_pytorch_model.fp16.safetensors | 0 6 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 tests/test_model_probe/sd-1/main/dreamshaper-8-inpainting/README create mode 100644 tests/test_model_probe/sd-1/main/dreamshaper-8-inpainting/model_index.json create mode 100644 tests/test_model_probe/sd-1/main/dreamshaper-8-inpainting/scheduler/scheduler_config.json create mode 100644 tests/test_model_probe/sd-1/main/dreamshaper-8-inpainting/unet/config.json create mode 100644 tests/test_model_probe/sd-1/main/dreamshaper-8-inpainting/unet/diffusion_pytorch_model.fp16.safetensors diff --git a/tests/test_model_probe.py b/tests/test_model_probe.py index 8be7089cf52..95929e3aa36 100644 --- a/tests/test_model_probe.py +++ b/tests/test_model_probe.py @@ -4,7 +4,7 @@ from torch import tensor from invokeai.backend.model_manager import BaseModelType, ModelRepoVariant -from invokeai.backend.model_manager.config import InvalidModelConfigException +from invokeai.backend.model_manager.config import InvalidModelConfigException, MainDiffusersConfig, ModelVariantType from invokeai.backend.model_manager.probe import ( CkptType, ModelProbe, @@ -78,3 +78,11 @@ def test_probe_handles_state_dict_with_integer_keys(): } with pytest.raises(InvalidModelConfigException): ModelProbe.get_model_type_from_checkpoint(Path("embedding.pt"), state_dict_with_integer_keys) + + +def test_probe_sd1_diffusers_inpainting(datadir: Path): + config = ModelProbe.probe(datadir / "sd-1/main/dreamshaper-8-inpainting") + assert isinstance(config, MainDiffusersConfig) + assert config.base is BaseModelType.StableDiffusion1 + assert config.variant is ModelVariantType.Inpaint + assert config.repo_variant is ModelRepoVariant.FP16 diff --git a/tests/test_model_probe/sd-1/main/dreamshaper-8-inpainting/README b/tests/test_model_probe/sd-1/main/dreamshaper-8-inpainting/README new file mode 100644 index 00000000000..15349b5f0ac --- /dev/null +++ b/tests/test_model_probe/sd-1/main/dreamshaper-8-inpainting/README @@ -0,0 +1 @@ +This folder contains config files copied from [Lykon/dreamshaper-8-inpainting](https://huggingface.co/Lykon/dreamshaper-8-inpainting). diff --git a/tests/test_model_probe/sd-1/main/dreamshaper-8-inpainting/model_index.json b/tests/test_model_probe/sd-1/main/dreamshaper-8-inpainting/model_index.json new file mode 100644 index 00000000000..e4e983f7ba6 --- /dev/null +++ b/tests/test_model_probe/sd-1/main/dreamshaper-8-inpainting/model_index.json @@ -0,0 +1,34 @@ +{ + "_class_name": "StableDiffusionInpaintPipeline", + "_diffusers_version": "0.21.0.dev0", + "_name_or_path": "lykon-models/dreamshaper-8-inpainting", + "feature_extractor": [ + "transformers", + "CLIPFeatureExtractor" + ], + "requires_safety_checker": true, + "safety_checker": [ + "stable_diffusion", + "StableDiffusionSafetyChecker" + ], + "scheduler": [ + "diffusers", + "DEISMultistepScheduler" + ], + "text_encoder": [ + "transformers", + "CLIPTextModel" + ], + "tokenizer": [ + "transformers", + "CLIPTokenizer" + ], + "unet": [ + "diffusers", + "UNet2DConditionModel" + ], + "vae": [ + "diffusers", + "AutoencoderKL" + ] +} diff --git a/tests/test_model_probe/sd-1/main/dreamshaper-8-inpainting/scheduler/scheduler_config.json b/tests/test_model_probe/sd-1/main/dreamshaper-8-inpainting/scheduler/scheduler_config.json new file mode 100644 index 00000000000..a63f334bd64 --- /dev/null +++ b/tests/test_model_probe/sd-1/main/dreamshaper-8-inpainting/scheduler/scheduler_config.json @@ -0,0 +1,23 @@ +{ + "_class_name": "DEISMultistepScheduler", + "_diffusers_version": "0.21.0.dev0", + "algorithm_type": "deis", + "beta_end": 0.012, + "beta_schedule": "scaled_linear", + "beta_start": 0.00085, + "clip_sample": false, + "dynamic_thresholding_ratio": 0.995, + "lower_order_final": true, + "num_train_timesteps": 1000, + "prediction_type": "epsilon", + "sample_max_value": 1.0, + "set_alpha_to_one": false, + "skip_prk_steps": true, + "solver_order": 2, + "solver_type": "logrho", + "steps_offset": 1, + "thresholding": false, + "timestep_spacing": "leading", + "trained_betas": null, + "use_karras_sigmas": false +} diff --git a/tests/test_model_probe/sd-1/main/dreamshaper-8-inpainting/unet/config.json b/tests/test_model_probe/sd-1/main/dreamshaper-8-inpainting/unet/config.json new file mode 100644 index 00000000000..d9f3b21d92e --- /dev/null +++ b/tests/test_model_probe/sd-1/main/dreamshaper-8-inpainting/unet/config.json @@ -0,0 +1,66 @@ +{ + "_class_name": "UNet2DConditionModel", + "_diffusers_version": "0.21.0.dev0", + "_name_or_path": "/home/patrick/.cache/huggingface/hub/models--lykon-models--dreamshaper-8-inpainting/snapshots/15dcb9dec91a39ee498e3917c9ef6174b103862d/unet", + "act_fn": "silu", + "addition_embed_type": null, + "addition_embed_type_num_heads": 64, + "addition_time_embed_dim": null, + "attention_head_dim": 8, + "attention_type": "default", + "block_out_channels": [ + 320, + 640, + 1280, + 1280 + ], + "center_input_sample": false, + "class_embed_type": null, + "class_embeddings_concat": false, + "conv_in_kernel": 3, + "conv_out_kernel": 3, + "cross_attention_dim": 768, + "cross_attention_norm": null, + "down_block_types": [ + "CrossAttnDownBlock2D", + "CrossAttnDownBlock2D", + "CrossAttnDownBlock2D", + "DownBlock2D" + ], + "downsample_padding": 1, + "dual_cross_attention": false, + "encoder_hid_dim": null, + "encoder_hid_dim_type": null, + "flip_sin_to_cos": true, + "freq_shift": 0, + "in_channels": 9, + "layers_per_block": 2, + "mid_block_only_cross_attention": null, + "mid_block_scale_factor": 1, + "mid_block_type": "UNetMidBlock2DCrossAttn", + "norm_eps": 1e-05, + "norm_num_groups": 32, + "num_attention_heads": null, + "num_class_embeds": null, + "only_cross_attention": false, + "out_channels": 4, + "projection_class_embeddings_input_dim": null, + "resnet_out_scale_factor": 1.0, + "resnet_skip_time_act": false, + "resnet_time_scale_shift": "default", + "sample_size": 64, + "time_cond_proj_dim": null, + "time_embedding_act_fn": null, + "time_embedding_dim": null, + "time_embedding_type": "positional", + "timestep_post_act": null, + "transformer_layers_per_block": 1, + "up_block_types": [ + "UpBlock2D", + "CrossAttnUpBlock2D", + "CrossAttnUpBlock2D", + "CrossAttnUpBlock2D" + ], + "upcast_attention": null, + "use_linear_projection": false +} diff --git a/tests/test_model_probe/sd-1/main/dreamshaper-8-inpainting/unet/diffusion_pytorch_model.fp16.safetensors b/tests/test_model_probe/sd-1/main/dreamshaper-8-inpainting/unet/diffusion_pytorch_model.fp16.safetensors new file mode 100644 index 00000000000..e69de29bb2d From 4c47fe66808df369a6f735df8894c303a2d070b3 Mon Sep 17 00:00:00 2001 From: psychedelicious <4822129+psychedelicious@users.noreply.github.com> Date: Tue, 23 Apr 2024 19:48:47 +1000 Subject: [PATCH 10/15] fix(mm): move `variant` to `MainConfigBase` shoulda been here all along --- invokeai/backend/model_manager/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/invokeai/backend/model_manager/config.py b/invokeai/backend/model_manager/config.py index 82f88c0e817..9fc800ccb40 100644 --- a/invokeai/backend/model_manager/config.py +++ b/invokeai/backend/model_manager/config.py @@ -301,12 +301,13 @@ class MainConfigBase(ModelConfigBase): default_settings: Optional[MainModelDefaultSettings] = Field( description="Default settings for this model", default=None ) + variant: ModelVariantType = ModelVariantType.Normal + class MainCheckpointConfig(CheckpointConfigBase, MainConfigBase): """Model config for main checkpoint models.""" - variant: ModelVariantType = ModelVariantType.Normal prediction_type: SchedulerPredictionType = SchedulerPredictionType.Epsilon upcast_attention: bool = False From f243268289398c7dfb260d3e82f1f82f71d73b6c Mon Sep 17 00:00:00 2001 From: psychedelicious <4822129+psychedelicious@users.noreply.github.com> Date: Tue, 23 Apr 2024 19:48:58 +1000 Subject: [PATCH 11/15] chore(ui): typegen --- .../frontend/web/src/services/api/schema.ts | 78 +++++++++++++++---- 1 file changed, 64 insertions(+), 14 deletions(-) diff --git a/invokeai/frontend/web/src/services/api/schema.ts b/invokeai/frontend/web/src/services/api/schema.ts index ec50231d033..727fad6f81e 100644 --- a/invokeai/frontend/web/src/services/api/schema.ts +++ b/invokeai/frontend/web/src/services/api/schema.ts @@ -2879,21 +2879,24 @@ export type components = { */ minimum_denoise?: number; /** - * UNet - * @description The UNet input used to detect if it is an inpainting model + * [OPTIONAL] Image + * @description OPTIONAL: Only connect for specialized Inpainting models, masked_latents will be generated from the image with the VAE */ - unet?: components["schemas"]["UNetField"] | null; - /** Image */ image?: components["schemas"]["ImageField"] | null; - /** vae */ + /** + * [OPTIONAL] VAE + * @description OPTIONAL: Only connect for specialized Inpainting models, masked_latents will be generated from the image with the VAE + */ vae?: components["schemas"]["VAEField"] | null; - /** - * @description tiled is used by the VAE + /** + * Tiled + * @description Processing using overlapping tiles (reduce memory consumption) * @default false - * */ + */ tiled?: boolean; - /** - * @description fp32 is used by the VAE + /** + * Fp32 + * @description Whether or not to use full float32 precision * @default false */ fp32?: boolean; @@ -4181,7 +4184,7 @@ export type components = { * @description The nodes in this graph */ nodes: { - [key: string]: components["schemas"]["CoreMetadataInvocation"] | components["schemas"]["SDXLLoRALoaderInvocation"] | components["schemas"]["ImageChannelMultiplyInvocation"] | components["schemas"]["FloatCollectionInvocation"] | components["schemas"]["LeresImageProcessorInvocation"] | components["schemas"]["BlankImageInvocation"] | components["schemas"]["ShowImageInvocation"] | components["schemas"]["TileResamplerProcessorInvocation"] | components["schemas"]["CalculateImageTilesInvocation"] | components["schemas"]["TileToPropertiesInvocation"] | components["schemas"]["LineartImageProcessorInvocation"] | components["schemas"]["HedImageProcessorInvocation"] | components["schemas"]["RectangleMaskInvocation"] | components["schemas"]["CvInpaintInvocation"] | components["schemas"]["SeamlessModeInvocation"] | components["schemas"]["StringJoinInvocation"] | components["schemas"]["ConditioningInvocation"] | components["schemas"]["StringInvocation"] | components["schemas"]["IntegerCollectionInvocation"] | components["schemas"]["MaskEdgeInvocation"] | components["schemas"]["PromptsFromFileInvocation"] | components["schemas"]["RandomFloatInvocation"] | components["schemas"]["SDXLRefinerCompelPromptInvocation"] | components["schemas"]["AddInvocation"] | components["schemas"]["SegmentAnythingProcessorInvocation"] | components["schemas"]["ColorCorrectInvocation"] | components["schemas"]["CompelInvocation"] | components["schemas"]["MaskFromIDInvocation"] | components["schemas"]["ESRGANInvocation"] | components["schemas"]["MidasDepthImageProcessorInvocation"] | components["schemas"]["SaveImageInvocation"] | components["schemas"]["T2IAdapterInvocation"] | components["schemas"]["CLIPSkipInvocation"] | components["schemas"]["SDXLCompelPromptInvocation"] | components["schemas"]["ImageCollectionInvocation"] | components["schemas"]["ZoeDepthImageProcessorInvocation"] | components["schemas"]["CV2InfillInvocation"] | components["schemas"]["RoundInvocation"] | components["schemas"]["StringSplitInvocation"] | components["schemas"]["DynamicPromptInvocation"] | components["schemas"]["FreeUInvocation"] | components["schemas"]["StringJoinThreeInvocation"] | components["schemas"]["IdealSizeInvocation"] | components["schemas"]["AlphaMaskToTensorInvocation"] | components["schemas"]["CreateGradientMaskInvocation"] | components["schemas"]["LoRALoaderInvocation"] | components["schemas"]["RandomRangeInvocation"] | components["schemas"]["MlsdImageProcessorInvocation"] | components["schemas"]["ImageHueAdjustmentInvocation"] | components["schemas"]["CalculateImageTilesMinimumOverlapInvocation"] | components["schemas"]["DenoiseLatentsInvocation"] | components["schemas"]["ImageMultiplyInvocation"] | components["schemas"]["ImageInverseLerpInvocation"] | components["schemas"]["BooleanInvocation"] | components["schemas"]["CropLatentsCoreInvocation"] | components["schemas"]["InfillPatchMatchInvocation"] | components["schemas"]["FloatToIntegerInvocation"] | components["schemas"]["ImageChannelInvocation"] | components["schemas"]["FaceMaskInvocation"] | components["schemas"]["ColorInvocation"] | components["schemas"]["CanvasPasteBackInvocation"] | components["schemas"]["MergeTilesToImageInvocation"] | components["schemas"]["SchedulerInvocation"] | components["schemas"]["CreateDenoiseMaskInvocation"] | components["schemas"]["ResizeLatentsInvocation"] | components["schemas"]["VAELoaderInvocation"] | components["schemas"]["ImageLerpInvocation"] | components["schemas"]["LineartAnimeImageProcessorInvocation"] | components["schemas"]["LatentsToImageInvocation"] | components["schemas"]["StepParamEasingInvocation"] | components["schemas"]["SDXLRefinerModelLoaderInvocation"] | components["schemas"]["FloatInvocation"] | components["schemas"]["FaceOffInvocation"] | components["schemas"]["MetadataInvocation"] | components["schemas"]["InvertTensorMaskInvocation"] | components["schemas"]["MergeMetadataInvocation"] | components["schemas"]["ImageToLatentsInvocation"] | components["schemas"]["PairTileImageInvocation"] | components["schemas"]["FaceIdentifierInvocation"] | components["schemas"]["PidiImageProcessorInvocation"] | components["schemas"]["ImageConvertInvocation"] | components["schemas"]["RangeInvocation"] | components["schemas"]["MaskFromAlphaInvocation"] | components["schemas"]["DWOpenposeImageProcessorInvocation"] | components["schemas"]["RangeOfSizeInvocation"] | components["schemas"]["LatentsInvocation"] | components["schemas"]["LatentsCollectionInvocation"] | components["schemas"]["ControlNetInvocation"] | components["schemas"]["MediapipeFaceProcessorInvocation"] | components["schemas"]["InfillColorInvocation"] | components["schemas"]["BlendLatentsInvocation"] | components["schemas"]["StringReplaceInvocation"] | components["schemas"]["ImageScaleInvocation"] | components["schemas"]["InfillTileInvocation"] | components["schemas"]["CannyImageProcessorInvocation"] | components["schemas"]["ConditioningCollectionInvocation"] | components["schemas"]["FloatMathInvocation"] | components["schemas"]["ImageBlurInvocation"] | components["schemas"]["DepthAnythingImageProcessorInvocation"] | components["schemas"]["BooleanCollectionInvocation"] | components["schemas"]["RandomIntInvocation"] | components["schemas"]["LaMaInfillInvocation"] | components["schemas"]["IterateInvocation"] | components["schemas"]["IntegerInvocation"] | components["schemas"]["IntegerMathInvocation"] | components["schemas"]["IPAdapterInvocation"] | components["schemas"]["ImagePasteInvocation"] | components["schemas"]["ImageNSFWBlurInvocation"] | components["schemas"]["CollectInvocation"] | components["schemas"]["ColorMapImageProcessorInvocation"] | components["schemas"]["SDXLModelLoaderInvocation"] | components["schemas"]["FloatLinearRangeInvocation"] | components["schemas"]["StringCollectionInvocation"] | components["schemas"]["NormalbaeImageProcessorInvocation"] | components["schemas"]["CalculateImageTilesEvenSplitInvocation"] | components["schemas"]["MaskCombineInvocation"] | components["schemas"]["CenterPadCropInvocation"] | components["schemas"]["ImageResizeInvocation"] | components["schemas"]["NoiseInvocation"] | components["schemas"]["DivideInvocation"] | components["schemas"]["ImageCropInvocation"] | components["schemas"]["ImageWatermarkInvocation"] | components["schemas"]["MainModelLoaderInvocation"] | components["schemas"]["ImageInvocation"] | components["schemas"]["StringSplitNegInvocation"] | components["schemas"]["MultiplyInvocation"] | components["schemas"]["ScaleLatentsInvocation"] | components["schemas"]["ImageChannelOffsetInvocation"] | components["schemas"]["SubtractInvocation"] | components["schemas"]["MetadataItemInvocation"] | components["schemas"]["UnsharpMaskInvocation"] | components["schemas"]["ContentShuffleImageProcessorInvocation"]; + [key: string]: components["schemas"]["HedImageProcessorInvocation"] | components["schemas"]["CropLatentsCoreInvocation"] | components["schemas"]["FaceOffInvocation"] | components["schemas"]["StepParamEasingInvocation"] | components["schemas"]["InfillPatchMatchInvocation"] | components["schemas"]["ZoeDepthImageProcessorInvocation"] | components["schemas"]["ImageConvertInvocation"] | components["schemas"]["ImageNSFWBlurInvocation"] | components["schemas"]["MetadataItemInvocation"] | components["schemas"]["MaskFromAlphaInvocation"] | components["schemas"]["MergeMetadataInvocation"] | components["schemas"]["InfillColorInvocation"] | components["schemas"]["StringInvocation"] | components["schemas"]["ImageInvocation"] | components["schemas"]["DepthAnythingImageProcessorInvocation"] | components["schemas"]["SubtractInvocation"] | components["schemas"]["RangeOfSizeInvocation"] | components["schemas"]["ImageScaleInvocation"] | components["schemas"]["ImageWatermarkInvocation"] | components["schemas"]["FloatLinearRangeInvocation"] | components["schemas"]["ImageChannelOffsetInvocation"] | components["schemas"]["MultiplyInvocation"] | components["schemas"]["RandomRangeInvocation"] | components["schemas"]["ImageBlurInvocation"] | components["schemas"]["FloatCollectionInvocation"] | components["schemas"]["SDXLRefinerModelLoaderInvocation"] | components["schemas"]["MergeTilesToImageInvocation"] | components["schemas"]["StringSplitInvocation"] | components["schemas"]["BooleanInvocation"] | components["schemas"]["MaskCombineInvocation"] | components["schemas"]["DivideInvocation"] | components["schemas"]["IterateInvocation"] | components["schemas"]["PairTileImageInvocation"] | components["schemas"]["IntegerCollectionInvocation"] | components["schemas"]["PidiImageProcessorInvocation"] | components["schemas"]["StringJoinThreeInvocation"] | components["schemas"]["MediapipeFaceProcessorInvocation"] | components["schemas"]["IPAdapterInvocation"] | components["schemas"]["CreateGradientMaskInvocation"] | components["schemas"]["IdealSizeInvocation"] | components["schemas"]["ImageMaskToTensorInvocation"] | components["schemas"]["LineartAnimeImageProcessorInvocation"] | components["schemas"]["MetadataInvocation"] | components["schemas"]["CV2InfillInvocation"] | components["schemas"]["ImageChannelMultiplyInvocation"] | components["schemas"]["MlsdImageProcessorInvocation"] | components["schemas"]["ImagePasteInvocation"] | components["schemas"]["ControlNetInvocation"] | components["schemas"]["SDXLLoRALoaderInvocation"] | components["schemas"]["ImageResizeInvocation"] | components["schemas"]["CalculateImageTilesEvenSplitInvocation"] | components["schemas"]["ColorMapImageProcessorInvocation"] | components["schemas"]["CenterPadCropInvocation"] | components["schemas"]["ConditioningInvocation"] | components["schemas"]["AddInvocation"] | components["schemas"]["StringReplaceInvocation"] | components["schemas"]["RangeInvocation"] | components["schemas"]["SaveImageInvocation"] | components["schemas"]["SDXLRefinerCompelPromptInvocation"] | components["schemas"]["ImageCropInvocation"] | components["schemas"]["UnsharpMaskInvocation"] | components["schemas"]["SeamlessModeInvocation"] | components["schemas"]["RandomFloatInvocation"] | components["schemas"]["ESRGANInvocation"] | components["schemas"]["ImageCollectionInvocation"] | components["schemas"]["MaskFromIDInvocation"] | components["schemas"]["ResizeLatentsInvocation"] | components["schemas"]["LineartImageProcessorInvocation"] | components["schemas"]["SDXLModelLoaderInvocation"] | components["schemas"]["NoiseInvocation"] | components["schemas"]["FloatMathInvocation"] | components["schemas"]["LoRALoaderInvocation"] | components["schemas"]["MaskEdgeInvocation"] | components["schemas"]["RectangleMaskInvocation"] | components["schemas"]["CanvasPasteBackInvocation"] | components["schemas"]["CreateDenoiseMaskInvocation"] | components["schemas"]["DenoiseLatentsInvocation"] | components["schemas"]["SchedulerInvocation"] | components["schemas"]["CoreMetadataInvocation"] | components["schemas"]["RoundInvocation"] | components["schemas"]["IntegerMathInvocation"] | components["schemas"]["SDXLCompelPromptInvocation"] | components["schemas"]["ImageInverseLerpInvocation"] | components["schemas"]["TileToPropertiesInvocation"] | components["schemas"]["FloatInvocation"] | components["schemas"]["ColorCorrectInvocation"] | components["schemas"]["ImageToLatentsInvocation"] | components["schemas"]["ColorInvocation"] | components["schemas"]["StringSplitNegInvocation"] | components["schemas"]["FaceMaskInvocation"] | components["schemas"]["FloatToIntegerInvocation"] | components["schemas"]["FreeUInvocation"] | components["schemas"]["CollectInvocation"] | components["schemas"]["CompelInvocation"] | components["schemas"]["TileResamplerProcessorInvocation"] | components["schemas"]["PromptsFromFileInvocation"] | components["schemas"]["CLIPSkipInvocation"] | components["schemas"]["T2IAdapterInvocation"] | components["schemas"]["VAELoaderInvocation"] | components["schemas"]["FaceIdentifierInvocation"] | components["schemas"]["BooleanCollectionInvocation"] | components["schemas"]["BlendLatentsInvocation"] | components["schemas"]["SegmentAnythingProcessorInvocation"] | components["schemas"]["LeresImageProcessorInvocation"] | components["schemas"]["ImageHueAdjustmentInvocation"] | components["schemas"]["InfillTileInvocation"] | components["schemas"]["ScaleLatentsInvocation"] | components["schemas"]["NormalbaeImageProcessorInvocation"] | components["schemas"]["IntegerInvocation"] | components["schemas"]["ContentShuffleImageProcessorInvocation"] | components["schemas"]["StringCollectionInvocation"] | components["schemas"]["LatentsToImageInvocation"] | components["schemas"]["LatentsInvocation"] | components["schemas"]["LatentsCollectionInvocation"] | components["schemas"]["CalculateImageTilesMinimumOverlapInvocation"] | components["schemas"]["ImageLerpInvocation"] | components["schemas"]["CalculateImageTilesInvocation"] | components["schemas"]["ConditioningCollectionInvocation"] | components["schemas"]["ImageMultiplyInvocation"] | components["schemas"]["ImageChannelInvocation"] | components["schemas"]["DynamicPromptInvocation"] | components["schemas"]["BlankImageInvocation"] | components["schemas"]["StringJoinInvocation"] | components["schemas"]["AlphaMaskToTensorInvocation"] | components["schemas"]["ShowImageInvocation"] | components["schemas"]["DWOpenposeImageProcessorInvocation"] | components["schemas"]["RandomIntInvocation"] | components["schemas"]["InvertTensorMaskInvocation"] | components["schemas"]["LaMaInfillInvocation"] | components["schemas"]["MainModelLoaderInvocation"] | components["schemas"]["CvInpaintInvocation"] | components["schemas"]["CannyImageProcessorInvocation"] | components["schemas"]["MidasDepthImageProcessorInvocation"]; }; /** * Edges @@ -4218,7 +4221,7 @@ export type components = { * @description The results of node executions */ results: { - [key: string]: components["schemas"]["SchedulerOutput"] | components["schemas"]["LatentsOutput"] | components["schemas"]["FaceOffOutput"] | components["schemas"]["SDXLModelLoaderOutput"] | components["schemas"]["StringPosNegOutput"] | components["schemas"]["IPAdapterOutput"] | components["schemas"]["IterateInvocationOutput"] | components["schemas"]["IntegerCollectionOutput"] | components["schemas"]["BooleanOutput"] | components["schemas"]["FloatOutput"] | components["schemas"]["PairTileImageOutput"] | components["schemas"]["SDXLLoRALoaderOutput"] | components["schemas"]["GradientMaskOutput"] | components["schemas"]["VAEOutput"] | components["schemas"]["String2Output"] | components["schemas"]["FloatCollectionOutput"] | components["schemas"]["ConditioningOutput"] | components["schemas"]["NoiseOutput"] | components["schemas"]["MetadataOutput"] | components["schemas"]["CLIPOutput"] | components["schemas"]["ColorOutput"] | components["schemas"]["ColorCollectionOutput"] | components["schemas"]["SDXLRefinerModelLoaderOutput"] | components["schemas"]["DenoiseMaskOutput"] | components["schemas"]["SeamlessModeOutput"] | components["schemas"]["T2IAdapterOutput"] | components["schemas"]["ImageOutput"] | components["schemas"]["MaskOutput"] | components["schemas"]["ConditioningCollectionOutput"] | components["schemas"]["ControlOutput"] | components["schemas"]["CollectInvocationOutput"] | components["schemas"]["IntegerOutput"] | components["schemas"]["LatentsCollectionOutput"] | components["schemas"]["TileToPropertiesOutput"] | components["schemas"]["ImageCollectionOutput"] | components["schemas"]["UNetOutput"] | components["schemas"]["StringCollectionOutput"] | components["schemas"]["ModelLoaderOutput"] | components["schemas"]["FaceMaskOutput"] | components["schemas"]["MetadataItemOutput"] | components["schemas"]["StringOutput"] | components["schemas"]["CLIPSkipInvocationOutput"] | components["schemas"]["IdealSizeOutput"] | components["schemas"]["LoRALoaderOutput"] | components["schemas"]["BooleanCollectionOutput"] | components["schemas"]["CalculateImageTilesOutput"]; + [key: string]: components["schemas"]["SDXLRefinerModelLoaderOutput"] | components["schemas"]["ImageCollectionOutput"] | components["schemas"]["ConditioningOutput"] | components["schemas"]["FaceOffOutput"] | components["schemas"]["SchedulerOutput"] | components["schemas"]["ColorOutput"] | components["schemas"]["String2Output"] | components["schemas"]["BooleanCollectionOutput"] | components["schemas"]["ColorCollectionOutput"] | components["schemas"]["DenoiseMaskOutput"] | components["schemas"]["T2IAdapterOutput"] | components["schemas"]["GradientMaskOutput"] | components["schemas"]["NoiseOutput"] | components["schemas"]["BooleanOutput"] | components["schemas"]["CalculateImageTilesOutput"] | components["schemas"]["MaskOutput"] | components["schemas"]["ConditioningCollectionOutput"] | components["schemas"]["StringCollectionOutput"] | components["schemas"]["PairTileImageOutput"] | components["schemas"]["SDXLModelLoaderOutput"] | components["schemas"]["CollectInvocationOutput"] | components["schemas"]["ControlOutput"] | components["schemas"]["IPAdapterOutput"] | components["schemas"]["LatentsCollectionOutput"] | components["schemas"]["SeamlessModeOutput"] | components["schemas"]["IdealSizeOutput"] | components["schemas"]["SDXLLoRALoaderOutput"] | components["schemas"]["ModelLoaderOutput"] | components["schemas"]["MetadataItemOutput"] | components["schemas"]["IntegerOutput"] | components["schemas"]["FloatOutput"] | components["schemas"]["StringOutput"] | components["schemas"]["FaceMaskOutput"] | components["schemas"]["VAEOutput"] | components["schemas"]["StringPosNegOutput"] | components["schemas"]["CLIPOutput"] | components["schemas"]["TileToPropertiesOutput"] | components["schemas"]["FloatCollectionOutput"] | components["schemas"]["ImageOutput"] | components["schemas"]["MetadataOutput"] | components["schemas"]["LatentsOutput"] | components["schemas"]["UNetOutput"] | components["schemas"]["CLIPSkipInvocationOutput"] | components["schemas"]["IntegerCollectionOutput"] | components["schemas"]["IterateInvocationOutput"] | components["schemas"]["LoRALoaderOutput"]; }; /** * Errors @@ -5349,6 +5352,51 @@ export type components = { */ type: "img_lerp"; }; + /** + * Image Mask to Tensor + * @description Convert a mask image to a tensor. Converts the image to grayscale and uses thresholding at the specified value. + */ + ImageMaskToTensorInvocation: { + /** @description Optional metadata to be saved with the image */ + metadata?: components["schemas"]["MetadataField"] | null; + /** + * Id + * @description The id of this instance of an invocation. Must be unique among all instances of invocations. + */ + id: string; + /** + * Is Intermediate + * @description Whether or not this is an intermediate invocation. + * @default false + */ + is_intermediate?: boolean; + /** + * Use Cache + * @description Whether or not to use the cache + * @default true + */ + use_cache?: boolean; + /** @description The mask image to convert. */ + image?: components["schemas"]["ImageField"]; + /** + * Cutoff + * @description Cutoff (<) + * @default 128 + */ + cutoff?: number; + /** + * Invert + * @description Whether to invert the mask. + * @default false + */ + invert?: boolean; + /** + * type + * @default image_mask_to_tensor + * @constant + */ + type: "image_mask_to_tensor"; + }; /** * Multiply Images * @description Multiplies two images together using `PIL.ImageChops.multiply()`. @@ -6877,6 +6925,8 @@ export type components = { trigger_phrases?: string[] | null; /** @description Default settings for this model */ default_settings?: components["schemas"]["MainModelDefaultSettings"] | null; + /** @default normal */ + variant?: components["schemas"]["ModelVariantType"]; /** * Format * @default checkpoint @@ -6893,8 +6943,6 @@ export type components = { * @description When this model was last converted to diffusers */ converted_at?: number | null; - /** @default normal */ - variant?: components["schemas"]["ModelVariantType"]; /** @default epsilon */ prediction_type?: components["schemas"]["SchedulerPredictionType"]; /** @@ -6965,6 +7013,8 @@ export type components = { trigger_phrases?: string[] | null; /** @description Default settings for this model */ default_settings?: components["schemas"]["MainModelDefaultSettings"] | null; + /** @default normal */ + variant?: components["schemas"]["ModelVariantType"]; /** * Format * @default diffusers From 07e45292a7c3f6d9b9fc65ea5c4cf25095107458 Mon Sep 17 00:00:00 2001 From: psychedelicious <4822129+psychedelicious@users.noreply.github.com> Date: Tue, 23 Apr 2024 19:55:11 +1000 Subject: [PATCH 12/15] feat(ui): add variant to model edit Also simplify the layouting for all model view/edit components. --- .../ControlNetOrT2IAdapterDefaultSettings.tsx | 12 ++-- .../MainModelDefaultSettings.tsx | 46 ++++---------- .../subpanels/ModelPanel/ModelEdit.tsx | 25 ++++---- .../subpanels/ModelPanel/ModelView.tsx | 61 ++++++------------- 4 files changed, 44 insertions(+), 100 deletions(-) diff --git a/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/ControlNetOrT2IAdapterDefaultSettings/ControlNetOrT2IAdapterDefaultSettings.tsx b/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/ControlNetOrT2IAdapterDefaultSettings/ControlNetOrT2IAdapterDefaultSettings.tsx index 93750348c0a..dcdc4e2a360 100644 --- a/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/ControlNetOrT2IAdapterDefaultSettings/ControlNetOrT2IAdapterDefaultSettings.tsx +++ b/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/ControlNetOrT2IAdapterDefaultSettings/ControlNetOrT2IAdapterDefaultSettings.tsx @@ -1,4 +1,4 @@ -import { Button, Flex, Heading, Text } from '@invoke-ai/ui-library'; +import { Button, Flex, Heading, SimpleGrid, Text } from '@invoke-ai/ui-library'; import { useAppDispatch, useAppSelector } from 'app/store/storeHooks'; import { useControlNetOrT2IAdapterDefaultSettings } from 'features/modelManagerV2/hooks/useControlNetOrT2IAdapterDefaultSettings'; import { DefaultPreprocessor } from 'features/modelManagerV2/subpanels/ModelPanel/ControlNetOrT2IAdapterDefaultSettings/DefaultPreprocessor'; @@ -92,13 +92,9 @@ export const ControlNetOrT2IAdapterDefaultSettings = () => { - - - - - - - + + + ); }; diff --git a/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/MainModelDefaultSettings/MainModelDefaultSettings.tsx b/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/MainModelDefaultSettings/MainModelDefaultSettings.tsx index 0027bd12e38..e096b11209e 100644 --- a/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/MainModelDefaultSettings/MainModelDefaultSettings.tsx +++ b/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/MainModelDefaultSettings/MainModelDefaultSettings.tsx @@ -1,4 +1,4 @@ -import { Button, Flex, Heading, Text } from '@invoke-ai/ui-library'; +import { Button, Flex, Heading, SimpleGrid, Text } from '@invoke-ai/ui-library'; import { useAppDispatch, useAppSelector } from 'app/store/storeHooks'; import { useMainModelDefaultSettings } from 'features/modelManagerV2/hooks/useMainModelDefaultSettings'; import { DefaultHeight } from 'features/modelManagerV2/subpanels/ModelPanel/MainModelDefaultSettings/DefaultHeight'; @@ -122,40 +122,16 @@ export const MainModelDefaultSettings = () => { - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + ); }; diff --git a/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/ModelEdit.tsx b/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/ModelEdit.tsx index c73b6e52ed1..1f8e50b9da0 100644 --- a/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/ModelEdit.tsx +++ b/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/ModelEdit.tsx @@ -6,6 +6,7 @@ import { FormLabel, Heading, Input, + SimpleGrid, Text, Textarea, } from '@invoke-ai/ui-library'; @@ -66,25 +67,21 @@ export const ModelEdit = ({ form }: Props) => { {t('modelManager.modelSettings')} - + {t('modelManager.baseModel')} - - {data.type === 'main' && data.format === 'checkpoint' && ( - <> - + + {t('modelManager.variant')} + + + {data.type === 'main' && data.format === 'checkpoint' && ( + <> {t('modelManager.pathToConfig')} - - {t('modelManager.variant')} - - - - {t('modelManager.predictionType')} @@ -93,9 +90,9 @@ export const ModelEdit = ({ form }: Props) => { {t('modelManager.upcastAttention')} - - - )} + + )} + diff --git a/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/ModelView.tsx b/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/ModelView.tsx index 0618af5dd04..83ae94c9bb5 100644 --- a/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/ModelView.tsx +++ b/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/ModelView.tsx @@ -1,4 +1,4 @@ -import { Box, Flex, Text } from '@invoke-ai/ui-library'; +import { Box, Flex, SimpleGrid, Text } from '@invoke-ai/ui-library'; import { skipToken } from '@reduxjs/toolkit/query'; import { useAppSelector } from 'app/store/storeHooks'; import { ControlNetOrT2IAdapterDefaultSettings } from 'features/modelManagerV2/subpanels/ModelPanel/ControlNetOrT2IAdapterDefaultSettings/ControlNetOrT2IAdapterDefaultSettings'; @@ -24,57 +24,32 @@ export const ModelView = () => { return ( - - - - - - - - - - + + + + + + {data.type === 'main' && } {data.type === 'main' && data.format === 'diffusers' && data.repo_variant && ( - - - + )} - {data.type === 'main' && data.format === 'checkpoint' && ( <> - - - - - - - - + + + )} - {data.type === 'ip_adapter' && data.format === 'invokeai' && ( - - - + )} - + + + + {data.type === 'main' && data.base !== 'sdxl-refiner' && } + {(data.type === 'controlnet' || data.type === 't2i_adapter') && } + {(data.type === 'main' || data.type === 'lora') && } - {data.type === 'main' && data.base !== 'sdxl-refiner' && ( - - - - )} - {(data.type === 'controlnet' || data.type === 't2i_adapter') && ( - - - - )} - {(data.type === 'main' || data.type === 'lora') && ( - - - - )} ); }; From 465e35da418f4809d7f699c38352abb9ce8671b8 Mon Sep 17 00:00:00 2001 From: psychedelicious <4822129+psychedelicious@users.noreply.github.com> Date: Tue, 23 Apr 2024 19:57:09 +1000 Subject: [PATCH 13/15] chore: ruff --- invokeai/backend/model_manager/config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/invokeai/backend/model_manager/config.py b/invokeai/backend/model_manager/config.py index 9fc800ccb40..b19501843cf 100644 --- a/invokeai/backend/model_manager/config.py +++ b/invokeai/backend/model_manager/config.py @@ -304,7 +304,6 @@ class MainConfigBase(ModelConfigBase): variant: ModelVariantType = ModelVariantType.Normal - class MainCheckpointConfig(CheckpointConfigBase, MainConfigBase): """Model config for main checkpoint models.""" From 409737751f9106a9f98723a4e80f24bfd9cb7c82 Mon Sep 17 00:00:00 2001 From: psychedelicious <4822129+psychedelicious@users.noreply.github.com> Date: Tue, 23 Apr 2024 20:25:12 +1000 Subject: [PATCH 14/15] feat(nodes): restore unet check on `CreateGradientMaskInvocation` Special handling for inpainting models --- invokeai/app/invocations/latent.py | 37 ++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/invokeai/app/invocations/latent.py b/invokeai/app/invocations/latent.py index 0f579cd9e7b..66d6a70e40e 100644 --- a/invokeai/app/invocations/latent.py +++ b/invokeai/app/invocations/latent.py @@ -51,6 +51,7 @@ from invokeai.backend.ip_adapter.ip_adapter import IPAdapter, IPAdapterPlus from invokeai.backend.lora import LoRAModelRaw from invokeai.backend.model_manager import BaseModelType, LoadedModel +from invokeai.backend.model_manager.config import MainConfigBase, ModelVariantType from invokeai.backend.model_patcher import ModelPatcher from invokeai.backend.stable_diffusion import PipelineIntermediateState, set_seamless from invokeai.backend.stable_diffusion.diffusion.conditioning_data import ( @@ -204,6 +205,13 @@ class CreateGradientMaskInvocation(BaseInvocation): title="[OPTIONAL] Image", ui_order=6, ) + unet: Optional[UNetField] = InputField( + description="OPTIONAL: If the Unet is a specialized Inpainting model, masked_latents will be generated from the image with the VAE", + default=None, + input=Input.Connection, + title="[OPTIONAL] UNet", + ui_order=5, + ) vae: Optional[VAEField] = InputField( default=None, description="OPTIONAL: Only connect for specialized Inpainting models, masked_latents will be generated from the image with the VAE", @@ -253,18 +261,23 @@ def invoke(self, context: InvocationContext) -> GradientMaskOutput: expanded_image_dto = context.images.save(expanded_mask_image) masked_latents_name = None - if self.vae is not None and self.image is not None: - # both fields must be present at the same time - mask = blur_tensor - vae_info: LoadedModel = context.models.load(self.vae.vae) - image = context.images.get_pil(self.image.image_name) - image_tensor = image_resized_to_grid_as_tensor(image.convert("RGB")) - if image_tensor.dim() == 3: - image_tensor = image_tensor.unsqueeze(0) - img_mask = tv_resize(mask, image_tensor.shape[-2:], T.InterpolationMode.BILINEAR, antialias=False) - masked_image = image_tensor * torch.where(img_mask < 0.5, 0.0, 1.0) - masked_latents = ImageToLatentsInvocation.vae_encode(vae_info, self.fp32, self.tiled, masked_image.clone()) - masked_latents_name = context.tensors.save(tensor=masked_latents) + if self.unet is not None and self.vae is not None and self.image is not None: + # all three fields must be present at the same time + main_model_config = context.models.get_config(self.unet.unet.key) + assert isinstance(main_model_config, MainConfigBase) + if main_model_config.variant is ModelVariantType.Inpaint: + mask = blur_tensor + vae_info: LoadedModel = context.models.load(self.vae.vae) + image = context.images.get_pil(self.image.image_name) + image_tensor = image_resized_to_grid_as_tensor(image.convert("RGB")) + if image_tensor.dim() == 3: + image_tensor = image_tensor.unsqueeze(0) + img_mask = tv_resize(mask, image_tensor.shape[-2:], T.InterpolationMode.BILINEAR, antialias=False) + masked_image = image_tensor * torch.where(img_mask < 0.5, 0.0, 1.0) + masked_latents = ImageToLatentsInvocation.vae_encode( + vae_info, self.fp32, self.tiled, masked_image.clone() + ) + masked_latents_name = context.tensors.save(tensor=masked_latents) return GradientMaskOutput( denoise_mask=DenoiseMaskField(mask_name=mask_name, masked_latents_name=masked_latents_name, gradient=True), From 5bc84835f7b260edda0838c97b474125b433121f Mon Sep 17 00:00:00 2001 From: psychedelicious <4822129+psychedelicious@users.noreply.github.com> Date: Tue, 23 Apr 2024 20:25:29 +1000 Subject: [PATCH 15/15] feat(ui): update canvas graphs to provide unet --- .../nodes/util/graph/buildCanvasInpaintGraph.ts | 10 ++++++++++ .../nodes/util/graph/buildCanvasOutpaintGraph.ts | 10 ++++++++++ .../nodes/util/graph/buildCanvasSDXLInpaintGraph.ts | 10 ++++++++++ .../nodes/util/graph/buildCanvasSDXLOutpaintGraph.ts | 10 ++++++++++ 4 files changed, 40 insertions(+) diff --git a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasInpaintGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasInpaintGraph.ts index bdeeaf9ee62..f8390b8b9a5 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasInpaintGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasInpaintGraph.ts @@ -184,6 +184,16 @@ export const buildCanvasInpaintGraph = async ( field: 'clip', }, }, + { + source: { + node_id: modelLoaderNodeId, + field: 'unet', + }, + destination: { + node_id: INPAINT_CREATE_MASK, + field: 'unet', + }, + }, // Connect CLIP Skip to Conditioning { source: { diff --git a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasOutpaintGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasOutpaintGraph.ts index 2d7b400c0ec..39eab4aa503 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasOutpaintGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasOutpaintGraph.ts @@ -209,6 +209,16 @@ export const buildCanvasOutpaintGraph = async ( field: 'clip', }, }, + { + source: { + node_id: modelLoaderNodeId, + field: 'unet', + }, + destination: { + node_id: INPAINT_CREATE_MASK, + field: 'unet', + }, + }, // Connect CLIP Skip to Conditioning { source: { diff --git a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLInpaintGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLInpaintGraph.ts index 693c947f948..6a33a6ef992 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLInpaintGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLInpaintGraph.ts @@ -216,6 +216,16 @@ export const buildCanvasSDXLInpaintGraph = async ( field: 'clip2', }, }, + { + source: { + node_id: modelLoaderNodeId, + field: 'unet', + }, + destination: { + node_id: INPAINT_CREATE_MASK, + field: 'unet', + }, + }, // Connect Everything To Inpaint Node { source: { diff --git a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLOutpaintGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLOutpaintGraph.ts index d2bf3d3896f..7cb215d5ecf 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLOutpaintGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLOutpaintGraph.ts @@ -239,6 +239,16 @@ export const buildCanvasSDXLOutpaintGraph = async ( field: 'clip2', }, }, + { + source: { + node_id: modelLoaderNodeId, + field: 'unet', + }, + destination: { + node_id: INPAINT_CREATE_MASK, + field: 'unet', + }, + }, // Connect Infill Result To Inpaint Image { source: {