diff --git a/invokeai/app/invocations/latent.py b/invokeai/app/invocations/latent.py index 4534df89c18..66d6a70e40e 100644 --- a/invokeai/app/invocations/latent.py +++ b/invokeai/app/invocations/latent.py @@ -51,6 +51,7 @@ from invokeai.backend.ip_adapter.ip_adapter import IPAdapter, IPAdapterPlus from invokeai.backend.lora import LoRAModelRaw from invokeai.backend.model_manager import BaseModelType, LoadedModel +from invokeai.backend.model_manager.config import MainConfigBase, ModelVariantType from invokeai.backend.model_patcher import ModelPatcher from invokeai.backend.stable_diffusion import PipelineIntermediateState, set_seamless from invokeai.backend.stable_diffusion.diffusion.conditioning_data import ( @@ -185,7 +186,7 @@ class GradientMaskOutput(BaseInvocationOutput): title="Create Gradient Mask", tags=["mask", "denoise"], category="latents", - version="1.0.0", + version="1.1.0", ) class CreateGradientMaskInvocation(BaseInvocation): """Creates mask for denoising model run.""" @@ -198,6 +199,32 @@ class CreateGradientMaskInvocation(BaseInvocation): minimum_denoise: float = InputField( default=0.0, ge=0, le=1, description="Minimum denoise level for the coherence region", ui_order=4 ) + image: Optional[ImageField] = InputField( + default=None, + description="OPTIONAL: Only connect for specialized Inpainting models, masked_latents will be generated from the image with the VAE", + title="[OPTIONAL] Image", + ui_order=6, + ) + unet: Optional[UNetField] = InputField( + description="OPTIONAL: If the Unet is a specialized Inpainting model, masked_latents will be generated from the image with the VAE", + default=None, + input=Input.Connection, + title="[OPTIONAL] UNet", + ui_order=5, + ) + vae: Optional[VAEField] = InputField( + default=None, + description="OPTIONAL: Only connect for specialized Inpainting models, masked_latents will be generated from the image with the VAE", + title="[OPTIONAL] VAE", + input=Input.Connection, + ui_order=7, + ) + tiled: bool = InputField(default=False, description=FieldDescriptions.tiled, ui_order=8) + fp32: bool = InputField( + default=DEFAULT_PRECISION == "float32", + description=FieldDescriptions.fp32, + ui_order=9, + ) @torch.no_grad() def invoke(self, context: InvocationContext) -> GradientMaskOutput: @@ -233,8 +260,27 @@ def invoke(self, context: InvocationContext) -> GradientMaskOutput: expanded_mask_image = Image.fromarray((expanded_mask.squeeze(0).numpy() * 255).astype(np.uint8), mode="L") expanded_image_dto = context.images.save(expanded_mask_image) + masked_latents_name = None + if self.unet is not None and self.vae is not None and self.image is not None: + # all three fields must be present at the same time + main_model_config = context.models.get_config(self.unet.unet.key) + assert isinstance(main_model_config, MainConfigBase) + if main_model_config.variant is ModelVariantType.Inpaint: + mask = blur_tensor + vae_info: LoadedModel = context.models.load(self.vae.vae) + image = context.images.get_pil(self.image.image_name) + image_tensor = image_resized_to_grid_as_tensor(image.convert("RGB")) + if image_tensor.dim() == 3: + image_tensor = image_tensor.unsqueeze(0) + img_mask = tv_resize(mask, image_tensor.shape[-2:], T.InterpolationMode.BILINEAR, antialias=False) + masked_image = image_tensor * torch.where(img_mask < 0.5, 0.0, 1.0) + masked_latents = ImageToLatentsInvocation.vae_encode( + vae_info, self.fp32, self.tiled, masked_image.clone() + ) + masked_latents_name = context.tensors.save(tensor=masked_latents) + return GradientMaskOutput( - denoise_mask=DenoiseMaskField(mask_name=mask_name, masked_latents_name=None, gradient=True), + denoise_mask=DenoiseMaskField(mask_name=mask_name, masked_latents_name=masked_latents_name, gradient=True), expanded_mask_area=ImageField(image_name=expanded_image_dto.image_name), ) diff --git a/invokeai/backend/model_manager/config.py b/invokeai/backend/model_manager/config.py index 82f88c0e817..b19501843cf 100644 --- a/invokeai/backend/model_manager/config.py +++ b/invokeai/backend/model_manager/config.py @@ -301,12 +301,12 @@ class MainConfigBase(ModelConfigBase): default_settings: Optional[MainModelDefaultSettings] = Field( description="Default settings for this model", default=None ) + variant: ModelVariantType = ModelVariantType.Normal class MainCheckpointConfig(CheckpointConfigBase, MainConfigBase): """Model config for main checkpoint models.""" - variant: ModelVariantType = ModelVariantType.Normal prediction_type: SchedulerPredictionType = SchedulerPredictionType.Epsilon upcast_attention: bool = False diff --git a/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/ControlNetOrT2IAdapterDefaultSettings/ControlNetOrT2IAdapterDefaultSettings.tsx b/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/ControlNetOrT2IAdapterDefaultSettings/ControlNetOrT2IAdapterDefaultSettings.tsx index 93750348c0a..dcdc4e2a360 100644 --- a/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/ControlNetOrT2IAdapterDefaultSettings/ControlNetOrT2IAdapterDefaultSettings.tsx +++ b/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/ControlNetOrT2IAdapterDefaultSettings/ControlNetOrT2IAdapterDefaultSettings.tsx @@ -1,4 +1,4 @@ -import { Button, Flex, Heading, Text } from '@invoke-ai/ui-library'; +import { Button, Flex, Heading, SimpleGrid, Text } from '@invoke-ai/ui-library'; import { useAppDispatch, useAppSelector } from 'app/store/storeHooks'; import { useControlNetOrT2IAdapterDefaultSettings } from 'features/modelManagerV2/hooks/useControlNetOrT2IAdapterDefaultSettings'; import { DefaultPreprocessor } from 'features/modelManagerV2/subpanels/ModelPanel/ControlNetOrT2IAdapterDefaultSettings/DefaultPreprocessor'; @@ -92,13 +92,9 @@ export const ControlNetOrT2IAdapterDefaultSettings = () => { - - - - - - - + + + ); }; diff --git a/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/MainModelDefaultSettings/MainModelDefaultSettings.tsx b/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/MainModelDefaultSettings/MainModelDefaultSettings.tsx index 0027bd12e38..e096b11209e 100644 --- a/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/MainModelDefaultSettings/MainModelDefaultSettings.tsx +++ b/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/MainModelDefaultSettings/MainModelDefaultSettings.tsx @@ -1,4 +1,4 @@ -import { Button, Flex, Heading, Text } from '@invoke-ai/ui-library'; +import { Button, Flex, Heading, SimpleGrid, Text } from '@invoke-ai/ui-library'; import { useAppDispatch, useAppSelector } from 'app/store/storeHooks'; import { useMainModelDefaultSettings } from 'features/modelManagerV2/hooks/useMainModelDefaultSettings'; import { DefaultHeight } from 'features/modelManagerV2/subpanels/ModelPanel/MainModelDefaultSettings/DefaultHeight'; @@ -122,40 +122,16 @@ export const MainModelDefaultSettings = () => { - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + ); }; diff --git a/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/ModelEdit.tsx b/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/ModelEdit.tsx index c73b6e52ed1..1f8e50b9da0 100644 --- a/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/ModelEdit.tsx +++ b/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/ModelEdit.tsx @@ -6,6 +6,7 @@ import { FormLabel, Heading, Input, + SimpleGrid, Text, Textarea, } from '@invoke-ai/ui-library'; @@ -66,25 +67,21 @@ export const ModelEdit = ({ form }: Props) => { {t('modelManager.modelSettings')} - + {t('modelManager.baseModel')} - - {data.type === 'main' && data.format === 'checkpoint' && ( - <> - + + {t('modelManager.variant')} + + + {data.type === 'main' && data.format === 'checkpoint' && ( + <> {t('modelManager.pathToConfig')} - - {t('modelManager.variant')} - - - - {t('modelManager.predictionType')} @@ -93,9 +90,9 @@ export const ModelEdit = ({ form }: Props) => { {t('modelManager.upcastAttention')} - - - )} + + )} + diff --git a/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/ModelView.tsx b/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/ModelView.tsx index 0618af5dd04..83ae94c9bb5 100644 --- a/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/ModelView.tsx +++ b/invokeai/frontend/web/src/features/modelManagerV2/subpanels/ModelPanel/ModelView.tsx @@ -1,4 +1,4 @@ -import { Box, Flex, Text } from '@invoke-ai/ui-library'; +import { Box, Flex, SimpleGrid, Text } from '@invoke-ai/ui-library'; import { skipToken } from '@reduxjs/toolkit/query'; import { useAppSelector } from 'app/store/storeHooks'; import { ControlNetOrT2IAdapterDefaultSettings } from 'features/modelManagerV2/subpanels/ModelPanel/ControlNetOrT2IAdapterDefaultSettings/ControlNetOrT2IAdapterDefaultSettings'; @@ -24,57 +24,32 @@ export const ModelView = () => { return ( - - - - - - - - - - + + + + + + {data.type === 'main' && } {data.type === 'main' && data.format === 'diffusers' && data.repo_variant && ( - - - + )} - {data.type === 'main' && data.format === 'checkpoint' && ( <> - - - - - - - - + + + )} - {data.type === 'ip_adapter' && data.format === 'invokeai' && ( - - - + )} - + + + + {data.type === 'main' && data.base !== 'sdxl-refiner' && } + {(data.type === 'controlnet' || data.type === 't2i_adapter') && } + {(data.type === 'main' || data.type === 'lora') && } - {data.type === 'main' && data.base !== 'sdxl-refiner' && ( - - - - )} - {(data.type === 'controlnet' || data.type === 't2i_adapter') && ( - - - - )} - {(data.type === 'main' || data.type === 'lora') && ( - - - - )} ); }; diff --git a/invokeai/frontend/web/src/features/nodes/util/graph/addVAEToGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graph/addVAEToGraph.ts index 242fc152981..347027c539b 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graph/addVAEToGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graph/addVAEToGraph.ts @@ -9,6 +9,7 @@ import { CANVAS_TEXT_TO_IMAGE_GRAPH, IMAGE_TO_IMAGE_GRAPH, IMAGE_TO_LATENTS, + INPAINT_CREATE_MASK, INPAINT_IMAGE, LATENTS_TO_IMAGE, MAIN_MODEL_LOADER, @@ -145,6 +146,16 @@ export const addVAEToGraph = async ( field: 'vae', }, }, + { + source: { + node_id: isSeamlessEnabled ? SEAMLESS : isAutoVae ? modelLoaderNodeId : VAE_LOADER, + field: 'vae', + }, + destination: { + node_id: INPAINT_CREATE_MASK, + field: 'vae', + }, + }, { source: { diff --git a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasInpaintGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasInpaintGraph.ts index 372f7d8fe5c..f8390b8b9a5 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasInpaintGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasInpaintGraph.ts @@ -133,6 +133,8 @@ export const buildCanvasInpaintGraph = async ( coherence_mode: canvasCoherenceMode, minimum_denoise: canvasCoherenceMinDenoise, edge_radius: canvasCoherenceEdgeSize, + tiled: false, + fp32: fp32, }, [DENOISE_LATENTS]: { type: 'denoise_latents', @@ -182,6 +184,16 @@ export const buildCanvasInpaintGraph = async ( field: 'clip', }, }, + { + source: { + node_id: modelLoaderNodeId, + field: 'unet', + }, + destination: { + node_id: INPAINT_CREATE_MASK, + field: 'unet', + }, + }, // Connect CLIP Skip to Conditioning { source: { @@ -331,6 +343,16 @@ export const buildCanvasInpaintGraph = async ( field: 'mask', }, }, + { + source: { + node_id: INPAINT_IMAGE_RESIZE_UP, + field: 'image', + }, + destination: { + node_id: INPAINT_CREATE_MASK, + field: 'image', + }, + }, // Resize Down { source: { diff --git a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasOutpaintGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasOutpaintGraph.ts index 6a59c51872a..39eab4aa503 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasOutpaintGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasOutpaintGraph.ts @@ -157,6 +157,8 @@ export const buildCanvasOutpaintGraph = async ( coherence_mode: canvasCoherenceMode, edge_radius: canvasCoherenceEdgeSize, minimum_denoise: canvasCoherenceMinDenoise, + tiled: false, + fp32: fp32, }, [DENOISE_LATENTS]: { type: 'denoise_latents', @@ -207,6 +209,16 @@ export const buildCanvasOutpaintGraph = async ( field: 'clip', }, }, + { + source: { + node_id: modelLoaderNodeId, + field: 'unet', + }, + destination: { + node_id: INPAINT_CREATE_MASK, + field: 'unet', + }, + }, // Connect CLIP Skip to Conditioning { source: { @@ -453,6 +465,16 @@ export const buildCanvasOutpaintGraph = async ( field: 'image', }, }, + { + source: { + node_id: INPAINT_IMAGE_RESIZE_UP, + field: 'image', + }, + destination: { + node_id: INPAINT_CREATE_MASK, + field: 'image', + }, + }, // Resize Results Down { source: { diff --git a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLInpaintGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLInpaintGraph.ts index 6a648a0ea45..6a33a6ef992 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLInpaintGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLInpaintGraph.ts @@ -135,6 +135,8 @@ export const buildCanvasSDXLInpaintGraph = async ( coherence_mode: canvasCoherenceMode, minimum_denoise: refinerModel ? Math.max(0.2, canvasCoherenceMinDenoise) : canvasCoherenceMinDenoise, edge_radius: canvasCoherenceEdgeSize, + tiled: false, + fp32: fp32, }, [SDXL_DENOISE_LATENTS]: { type: 'denoise_latents', @@ -214,6 +216,16 @@ export const buildCanvasSDXLInpaintGraph = async ( field: 'clip2', }, }, + { + source: { + node_id: modelLoaderNodeId, + field: 'unet', + }, + destination: { + node_id: INPAINT_CREATE_MASK, + field: 'unet', + }, + }, // Connect Everything To Inpaint Node { source: { @@ -342,6 +354,16 @@ export const buildCanvasSDXLInpaintGraph = async ( field: 'mask', }, }, + { + source: { + node_id: INPAINT_IMAGE_RESIZE_UP, + field: 'image', + }, + destination: { + node_id: INPAINT_CREATE_MASK, + field: 'image', + }, + }, // Resize Down { source: { diff --git a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLOutpaintGraph.ts b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLOutpaintGraph.ts index 87ad3e29a7d..7cb215d5ecf 100644 --- a/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLOutpaintGraph.ts +++ b/invokeai/frontend/web/src/features/nodes/util/graph/buildCanvasSDXLOutpaintGraph.ts @@ -157,6 +157,8 @@ export const buildCanvasSDXLOutpaintGraph = async ( coherence_mode: canvasCoherenceMode, edge_radius: canvasCoherenceEdgeSize, minimum_denoise: refinerModel ? Math.max(0.2, canvasCoherenceMinDenoise) : canvasCoherenceMinDenoise, + tiled: false, + fp32: fp32, }, [SDXL_DENOISE_LATENTS]: { type: 'denoise_latents', @@ -237,6 +239,16 @@ export const buildCanvasSDXLOutpaintGraph = async ( field: 'clip2', }, }, + { + source: { + node_id: modelLoaderNodeId, + field: 'unet', + }, + destination: { + node_id: INPAINT_CREATE_MASK, + field: 'unet', + }, + }, // Connect Infill Result To Inpaint Image { source: { @@ -451,6 +463,16 @@ export const buildCanvasSDXLOutpaintGraph = async ( field: 'image', }, }, + { + source: { + node_id: INPAINT_IMAGE_RESIZE_UP, + field: 'image', + }, + destination: { + node_id: INPAINT_CREATE_MASK, + field: 'image', + }, + }, // Take combined mask and resize { source: { diff --git a/invokeai/frontend/web/src/services/api/schema.ts b/invokeai/frontend/web/src/services/api/schema.ts index 834f75392fd..727fad6f81e 100644 --- a/invokeai/frontend/web/src/services/api/schema.ts +++ b/invokeai/frontend/web/src/services/api/schema.ts @@ -2878,6 +2878,28 @@ export type components = { * @default 0 */ minimum_denoise?: number; + /** + * [OPTIONAL] Image + * @description OPTIONAL: Only connect for specialized Inpainting models, masked_latents will be generated from the image with the VAE + */ + image?: components["schemas"]["ImageField"] | null; + /** + * [OPTIONAL] VAE + * @description OPTIONAL: Only connect for specialized Inpainting models, masked_latents will be generated from the image with the VAE + */ + vae?: components["schemas"]["VAEField"] | null; + /** + * Tiled + * @description Processing using overlapping tiles (reduce memory consumption) + * @default false + */ + tiled?: boolean; + /** + * Fp32 + * @description Whether or not to use full float32 precision + * @default false + */ + fp32?: boolean; /** * type * @default create_gradient_mask @@ -4162,7 +4184,7 @@ export type components = { * @description The nodes in this graph */ nodes: { - [key: string]: components["schemas"]["CoreMetadataInvocation"] | components["schemas"]["SDXLLoRALoaderInvocation"] | components["schemas"]["ImageChannelMultiplyInvocation"] | components["schemas"]["FloatCollectionInvocation"] | components["schemas"]["LeresImageProcessorInvocation"] | components["schemas"]["BlankImageInvocation"] | components["schemas"]["ShowImageInvocation"] | components["schemas"]["TileResamplerProcessorInvocation"] | components["schemas"]["CalculateImageTilesInvocation"] | components["schemas"]["TileToPropertiesInvocation"] | components["schemas"]["LineartImageProcessorInvocation"] | components["schemas"]["HedImageProcessorInvocation"] | components["schemas"]["RectangleMaskInvocation"] | components["schemas"]["CvInpaintInvocation"] | components["schemas"]["SeamlessModeInvocation"] | components["schemas"]["StringJoinInvocation"] | components["schemas"]["ConditioningInvocation"] | components["schemas"]["StringInvocation"] | components["schemas"]["IntegerCollectionInvocation"] | components["schemas"]["MaskEdgeInvocation"] | components["schemas"]["PromptsFromFileInvocation"] | components["schemas"]["RandomFloatInvocation"] | components["schemas"]["SDXLRefinerCompelPromptInvocation"] | components["schemas"]["AddInvocation"] | components["schemas"]["SegmentAnythingProcessorInvocation"] | components["schemas"]["ColorCorrectInvocation"] | components["schemas"]["CompelInvocation"] | components["schemas"]["MaskFromIDInvocation"] | components["schemas"]["ESRGANInvocation"] | components["schemas"]["MidasDepthImageProcessorInvocation"] | components["schemas"]["SaveImageInvocation"] | components["schemas"]["T2IAdapterInvocation"] | components["schemas"]["CLIPSkipInvocation"] | components["schemas"]["SDXLCompelPromptInvocation"] | components["schemas"]["ImageCollectionInvocation"] | components["schemas"]["ZoeDepthImageProcessorInvocation"] | components["schemas"]["CV2InfillInvocation"] | components["schemas"]["RoundInvocation"] | components["schemas"]["StringSplitInvocation"] | components["schemas"]["DynamicPromptInvocation"] | components["schemas"]["FreeUInvocation"] | components["schemas"]["StringJoinThreeInvocation"] | components["schemas"]["IdealSizeInvocation"] | components["schemas"]["AlphaMaskToTensorInvocation"] | components["schemas"]["CreateGradientMaskInvocation"] | components["schemas"]["LoRALoaderInvocation"] | components["schemas"]["RandomRangeInvocation"] | components["schemas"]["MlsdImageProcessorInvocation"] | components["schemas"]["ImageHueAdjustmentInvocation"] | components["schemas"]["CalculateImageTilesMinimumOverlapInvocation"] | components["schemas"]["DenoiseLatentsInvocation"] | components["schemas"]["ImageMultiplyInvocation"] | components["schemas"]["ImageInverseLerpInvocation"] | components["schemas"]["BooleanInvocation"] | components["schemas"]["CropLatentsCoreInvocation"] | components["schemas"]["InfillPatchMatchInvocation"] | components["schemas"]["FloatToIntegerInvocation"] | components["schemas"]["ImageChannelInvocation"] | components["schemas"]["FaceMaskInvocation"] | components["schemas"]["ColorInvocation"] | components["schemas"]["CanvasPasteBackInvocation"] | components["schemas"]["MergeTilesToImageInvocation"] | components["schemas"]["SchedulerInvocation"] | components["schemas"]["CreateDenoiseMaskInvocation"] | components["schemas"]["ResizeLatentsInvocation"] | components["schemas"]["VAELoaderInvocation"] | components["schemas"]["ImageLerpInvocation"] | components["schemas"]["LineartAnimeImageProcessorInvocation"] | components["schemas"]["LatentsToImageInvocation"] | components["schemas"]["StepParamEasingInvocation"] | components["schemas"]["SDXLRefinerModelLoaderInvocation"] | components["schemas"]["FloatInvocation"] | components["schemas"]["FaceOffInvocation"] | components["schemas"]["MetadataInvocation"] | components["schemas"]["InvertTensorMaskInvocation"] | components["schemas"]["MergeMetadataInvocation"] | components["schemas"]["ImageToLatentsInvocation"] | components["schemas"]["PairTileImageInvocation"] | components["schemas"]["FaceIdentifierInvocation"] | components["schemas"]["PidiImageProcessorInvocation"] | components["schemas"]["ImageConvertInvocation"] | components["schemas"]["RangeInvocation"] | components["schemas"]["MaskFromAlphaInvocation"] | components["schemas"]["DWOpenposeImageProcessorInvocation"] | components["schemas"]["RangeOfSizeInvocation"] | components["schemas"]["LatentsInvocation"] | components["schemas"]["LatentsCollectionInvocation"] | components["schemas"]["ControlNetInvocation"] | components["schemas"]["MediapipeFaceProcessorInvocation"] | components["schemas"]["InfillColorInvocation"] | components["schemas"]["BlendLatentsInvocation"] | components["schemas"]["StringReplaceInvocation"] | components["schemas"]["ImageScaleInvocation"] | components["schemas"]["InfillTileInvocation"] | components["schemas"]["CannyImageProcessorInvocation"] | components["schemas"]["ConditioningCollectionInvocation"] | components["schemas"]["FloatMathInvocation"] | components["schemas"]["ImageBlurInvocation"] | components["schemas"]["DepthAnythingImageProcessorInvocation"] | components["schemas"]["BooleanCollectionInvocation"] | components["schemas"]["RandomIntInvocation"] | components["schemas"]["LaMaInfillInvocation"] | components["schemas"]["IterateInvocation"] | components["schemas"]["IntegerInvocation"] | components["schemas"]["IntegerMathInvocation"] | components["schemas"]["IPAdapterInvocation"] | components["schemas"]["ImagePasteInvocation"] | components["schemas"]["ImageNSFWBlurInvocation"] | components["schemas"]["CollectInvocation"] | components["schemas"]["ColorMapImageProcessorInvocation"] | components["schemas"]["SDXLModelLoaderInvocation"] | components["schemas"]["FloatLinearRangeInvocation"] | components["schemas"]["StringCollectionInvocation"] | components["schemas"]["NormalbaeImageProcessorInvocation"] | components["schemas"]["CalculateImageTilesEvenSplitInvocation"] | components["schemas"]["MaskCombineInvocation"] | components["schemas"]["CenterPadCropInvocation"] | components["schemas"]["ImageResizeInvocation"] | components["schemas"]["NoiseInvocation"] | components["schemas"]["DivideInvocation"] | components["schemas"]["ImageCropInvocation"] | components["schemas"]["ImageWatermarkInvocation"] | components["schemas"]["MainModelLoaderInvocation"] | components["schemas"]["ImageInvocation"] | components["schemas"]["StringSplitNegInvocation"] | components["schemas"]["MultiplyInvocation"] | components["schemas"]["ScaleLatentsInvocation"] | components["schemas"]["ImageChannelOffsetInvocation"] | components["schemas"]["SubtractInvocation"] | components["schemas"]["MetadataItemInvocation"] | components["schemas"]["UnsharpMaskInvocation"] | components["schemas"]["ContentShuffleImageProcessorInvocation"]; + [key: string]: components["schemas"]["HedImageProcessorInvocation"] | components["schemas"]["CropLatentsCoreInvocation"] | components["schemas"]["FaceOffInvocation"] | components["schemas"]["StepParamEasingInvocation"] | components["schemas"]["InfillPatchMatchInvocation"] | components["schemas"]["ZoeDepthImageProcessorInvocation"] | components["schemas"]["ImageConvertInvocation"] | components["schemas"]["ImageNSFWBlurInvocation"] | components["schemas"]["MetadataItemInvocation"] | components["schemas"]["MaskFromAlphaInvocation"] | components["schemas"]["MergeMetadataInvocation"] | components["schemas"]["InfillColorInvocation"] | components["schemas"]["StringInvocation"] | components["schemas"]["ImageInvocation"] | components["schemas"]["DepthAnythingImageProcessorInvocation"] | components["schemas"]["SubtractInvocation"] | components["schemas"]["RangeOfSizeInvocation"] | components["schemas"]["ImageScaleInvocation"] | components["schemas"]["ImageWatermarkInvocation"] | components["schemas"]["FloatLinearRangeInvocation"] | components["schemas"]["ImageChannelOffsetInvocation"] | components["schemas"]["MultiplyInvocation"] | components["schemas"]["RandomRangeInvocation"] | components["schemas"]["ImageBlurInvocation"] | components["schemas"]["FloatCollectionInvocation"] | components["schemas"]["SDXLRefinerModelLoaderInvocation"] | components["schemas"]["MergeTilesToImageInvocation"] | components["schemas"]["StringSplitInvocation"] | components["schemas"]["BooleanInvocation"] | components["schemas"]["MaskCombineInvocation"] | components["schemas"]["DivideInvocation"] | components["schemas"]["IterateInvocation"] | components["schemas"]["PairTileImageInvocation"] | components["schemas"]["IntegerCollectionInvocation"] | components["schemas"]["PidiImageProcessorInvocation"] | components["schemas"]["StringJoinThreeInvocation"] | components["schemas"]["MediapipeFaceProcessorInvocation"] | components["schemas"]["IPAdapterInvocation"] | components["schemas"]["CreateGradientMaskInvocation"] | components["schemas"]["IdealSizeInvocation"] | components["schemas"]["ImageMaskToTensorInvocation"] | components["schemas"]["LineartAnimeImageProcessorInvocation"] | components["schemas"]["MetadataInvocation"] | components["schemas"]["CV2InfillInvocation"] | components["schemas"]["ImageChannelMultiplyInvocation"] | components["schemas"]["MlsdImageProcessorInvocation"] | components["schemas"]["ImagePasteInvocation"] | components["schemas"]["ControlNetInvocation"] | components["schemas"]["SDXLLoRALoaderInvocation"] | components["schemas"]["ImageResizeInvocation"] | components["schemas"]["CalculateImageTilesEvenSplitInvocation"] | components["schemas"]["ColorMapImageProcessorInvocation"] | components["schemas"]["CenterPadCropInvocation"] | components["schemas"]["ConditioningInvocation"] | components["schemas"]["AddInvocation"] | components["schemas"]["StringReplaceInvocation"] | components["schemas"]["RangeInvocation"] | components["schemas"]["SaveImageInvocation"] | components["schemas"]["SDXLRefinerCompelPromptInvocation"] | components["schemas"]["ImageCropInvocation"] | components["schemas"]["UnsharpMaskInvocation"] | components["schemas"]["SeamlessModeInvocation"] | components["schemas"]["RandomFloatInvocation"] | components["schemas"]["ESRGANInvocation"] | components["schemas"]["ImageCollectionInvocation"] | components["schemas"]["MaskFromIDInvocation"] | components["schemas"]["ResizeLatentsInvocation"] | components["schemas"]["LineartImageProcessorInvocation"] | components["schemas"]["SDXLModelLoaderInvocation"] | components["schemas"]["NoiseInvocation"] | components["schemas"]["FloatMathInvocation"] | components["schemas"]["LoRALoaderInvocation"] | components["schemas"]["MaskEdgeInvocation"] | components["schemas"]["RectangleMaskInvocation"] | components["schemas"]["CanvasPasteBackInvocation"] | components["schemas"]["CreateDenoiseMaskInvocation"] | components["schemas"]["DenoiseLatentsInvocation"] | components["schemas"]["SchedulerInvocation"] | components["schemas"]["CoreMetadataInvocation"] | components["schemas"]["RoundInvocation"] | components["schemas"]["IntegerMathInvocation"] | components["schemas"]["SDXLCompelPromptInvocation"] | components["schemas"]["ImageInverseLerpInvocation"] | components["schemas"]["TileToPropertiesInvocation"] | components["schemas"]["FloatInvocation"] | components["schemas"]["ColorCorrectInvocation"] | components["schemas"]["ImageToLatentsInvocation"] | components["schemas"]["ColorInvocation"] | components["schemas"]["StringSplitNegInvocation"] | components["schemas"]["FaceMaskInvocation"] | components["schemas"]["FloatToIntegerInvocation"] | components["schemas"]["FreeUInvocation"] | components["schemas"]["CollectInvocation"] | components["schemas"]["CompelInvocation"] | components["schemas"]["TileResamplerProcessorInvocation"] | components["schemas"]["PromptsFromFileInvocation"] | components["schemas"]["CLIPSkipInvocation"] | components["schemas"]["T2IAdapterInvocation"] | components["schemas"]["VAELoaderInvocation"] | components["schemas"]["FaceIdentifierInvocation"] | components["schemas"]["BooleanCollectionInvocation"] | components["schemas"]["BlendLatentsInvocation"] | components["schemas"]["SegmentAnythingProcessorInvocation"] | components["schemas"]["LeresImageProcessorInvocation"] | components["schemas"]["ImageHueAdjustmentInvocation"] | components["schemas"]["InfillTileInvocation"] | components["schemas"]["ScaleLatentsInvocation"] | components["schemas"]["NormalbaeImageProcessorInvocation"] | components["schemas"]["IntegerInvocation"] | components["schemas"]["ContentShuffleImageProcessorInvocation"] | components["schemas"]["StringCollectionInvocation"] | components["schemas"]["LatentsToImageInvocation"] | components["schemas"]["LatentsInvocation"] | components["schemas"]["LatentsCollectionInvocation"] | components["schemas"]["CalculateImageTilesMinimumOverlapInvocation"] | components["schemas"]["ImageLerpInvocation"] | components["schemas"]["CalculateImageTilesInvocation"] | components["schemas"]["ConditioningCollectionInvocation"] | components["schemas"]["ImageMultiplyInvocation"] | components["schemas"]["ImageChannelInvocation"] | components["schemas"]["DynamicPromptInvocation"] | components["schemas"]["BlankImageInvocation"] | components["schemas"]["StringJoinInvocation"] | components["schemas"]["AlphaMaskToTensorInvocation"] | components["schemas"]["ShowImageInvocation"] | components["schemas"]["DWOpenposeImageProcessorInvocation"] | components["schemas"]["RandomIntInvocation"] | components["schemas"]["InvertTensorMaskInvocation"] | components["schemas"]["LaMaInfillInvocation"] | components["schemas"]["MainModelLoaderInvocation"] | components["schemas"]["CvInpaintInvocation"] | components["schemas"]["CannyImageProcessorInvocation"] | components["schemas"]["MidasDepthImageProcessorInvocation"]; }; /** * Edges @@ -4199,7 +4221,7 @@ export type components = { * @description The results of node executions */ results: { - [key: string]: components["schemas"]["SchedulerOutput"] | components["schemas"]["LatentsOutput"] | components["schemas"]["FaceOffOutput"] | components["schemas"]["SDXLModelLoaderOutput"] | components["schemas"]["StringPosNegOutput"] | components["schemas"]["IPAdapterOutput"] | components["schemas"]["IterateInvocationOutput"] | components["schemas"]["IntegerCollectionOutput"] | components["schemas"]["BooleanOutput"] | components["schemas"]["FloatOutput"] | components["schemas"]["PairTileImageOutput"] | components["schemas"]["SDXLLoRALoaderOutput"] | components["schemas"]["GradientMaskOutput"] | components["schemas"]["VAEOutput"] | components["schemas"]["String2Output"] | components["schemas"]["FloatCollectionOutput"] | components["schemas"]["ConditioningOutput"] | components["schemas"]["NoiseOutput"] | components["schemas"]["MetadataOutput"] | components["schemas"]["CLIPOutput"] | components["schemas"]["ColorOutput"] | components["schemas"]["ColorCollectionOutput"] | components["schemas"]["SDXLRefinerModelLoaderOutput"] | components["schemas"]["DenoiseMaskOutput"] | components["schemas"]["SeamlessModeOutput"] | components["schemas"]["T2IAdapterOutput"] | components["schemas"]["ImageOutput"] | components["schemas"]["MaskOutput"] | components["schemas"]["ConditioningCollectionOutput"] | components["schemas"]["ControlOutput"] | components["schemas"]["CollectInvocationOutput"] | components["schemas"]["IntegerOutput"] | components["schemas"]["LatentsCollectionOutput"] | components["schemas"]["TileToPropertiesOutput"] | components["schemas"]["ImageCollectionOutput"] | components["schemas"]["UNetOutput"] | components["schemas"]["StringCollectionOutput"] | components["schemas"]["ModelLoaderOutput"] | components["schemas"]["FaceMaskOutput"] | components["schemas"]["MetadataItemOutput"] | components["schemas"]["StringOutput"] | components["schemas"]["CLIPSkipInvocationOutput"] | components["schemas"]["IdealSizeOutput"] | components["schemas"]["LoRALoaderOutput"] | components["schemas"]["BooleanCollectionOutput"] | components["schemas"]["CalculateImageTilesOutput"]; + [key: string]: components["schemas"]["SDXLRefinerModelLoaderOutput"] | components["schemas"]["ImageCollectionOutput"] | components["schemas"]["ConditioningOutput"] | components["schemas"]["FaceOffOutput"] | components["schemas"]["SchedulerOutput"] | components["schemas"]["ColorOutput"] | components["schemas"]["String2Output"] | components["schemas"]["BooleanCollectionOutput"] | components["schemas"]["ColorCollectionOutput"] | components["schemas"]["DenoiseMaskOutput"] | components["schemas"]["T2IAdapterOutput"] | components["schemas"]["GradientMaskOutput"] | components["schemas"]["NoiseOutput"] | components["schemas"]["BooleanOutput"] | components["schemas"]["CalculateImageTilesOutput"] | components["schemas"]["MaskOutput"] | components["schemas"]["ConditioningCollectionOutput"] | components["schemas"]["StringCollectionOutput"] | components["schemas"]["PairTileImageOutput"] | components["schemas"]["SDXLModelLoaderOutput"] | components["schemas"]["CollectInvocationOutput"] | components["schemas"]["ControlOutput"] | components["schemas"]["IPAdapterOutput"] | components["schemas"]["LatentsCollectionOutput"] | components["schemas"]["SeamlessModeOutput"] | components["schemas"]["IdealSizeOutput"] | components["schemas"]["SDXLLoRALoaderOutput"] | components["schemas"]["ModelLoaderOutput"] | components["schemas"]["MetadataItemOutput"] | components["schemas"]["IntegerOutput"] | components["schemas"]["FloatOutput"] | components["schemas"]["StringOutput"] | components["schemas"]["FaceMaskOutput"] | components["schemas"]["VAEOutput"] | components["schemas"]["StringPosNegOutput"] | components["schemas"]["CLIPOutput"] | components["schemas"]["TileToPropertiesOutput"] | components["schemas"]["FloatCollectionOutput"] | components["schemas"]["ImageOutput"] | components["schemas"]["MetadataOutput"] | components["schemas"]["LatentsOutput"] | components["schemas"]["UNetOutput"] | components["schemas"]["CLIPSkipInvocationOutput"] | components["schemas"]["IntegerCollectionOutput"] | components["schemas"]["IterateInvocationOutput"] | components["schemas"]["LoRALoaderOutput"]; }; /** * Errors @@ -5330,6 +5352,51 @@ export type components = { */ type: "img_lerp"; }; + /** + * Image Mask to Tensor + * @description Convert a mask image to a tensor. Converts the image to grayscale and uses thresholding at the specified value. + */ + ImageMaskToTensorInvocation: { + /** @description Optional metadata to be saved with the image */ + metadata?: components["schemas"]["MetadataField"] | null; + /** + * Id + * @description The id of this instance of an invocation. Must be unique among all instances of invocations. + */ + id: string; + /** + * Is Intermediate + * @description Whether or not this is an intermediate invocation. + * @default false + */ + is_intermediate?: boolean; + /** + * Use Cache + * @description Whether or not to use the cache + * @default true + */ + use_cache?: boolean; + /** @description The mask image to convert. */ + image?: components["schemas"]["ImageField"]; + /** + * Cutoff + * @description Cutoff (<) + * @default 128 + */ + cutoff?: number; + /** + * Invert + * @description Whether to invert the mask. + * @default false + */ + invert?: boolean; + /** + * type + * @default image_mask_to_tensor + * @constant + */ + type: "image_mask_to_tensor"; + }; /** * Multiply Images * @description Multiplies two images together using `PIL.ImageChops.multiply()`. @@ -6858,6 +6925,8 @@ export type components = { trigger_phrases?: string[] | null; /** @description Default settings for this model */ default_settings?: components["schemas"]["MainModelDefaultSettings"] | null; + /** @default normal */ + variant?: components["schemas"]["ModelVariantType"]; /** * Format * @default checkpoint @@ -6874,8 +6943,6 @@ export type components = { * @description When this model was last converted to diffusers */ converted_at?: number | null; - /** @default normal */ - variant?: components["schemas"]["ModelVariantType"]; /** @default epsilon */ prediction_type?: components["schemas"]["SchedulerPredictionType"]; /** @@ -6946,6 +7013,8 @@ export type components = { trigger_phrases?: string[] | null; /** @description Default settings for this model */ default_settings?: components["schemas"]["MainModelDefaultSettings"] | null; + /** @default normal */ + variant?: components["schemas"]["ModelVariantType"]; /** * Format * @default diffusers diff --git a/tests/test_model_probe.py b/tests/test_model_probe.py index 8be7089cf52..95929e3aa36 100644 --- a/tests/test_model_probe.py +++ b/tests/test_model_probe.py @@ -4,7 +4,7 @@ from torch import tensor from invokeai.backend.model_manager import BaseModelType, ModelRepoVariant -from invokeai.backend.model_manager.config import InvalidModelConfigException +from invokeai.backend.model_manager.config import InvalidModelConfigException, MainDiffusersConfig, ModelVariantType from invokeai.backend.model_manager.probe import ( CkptType, ModelProbe, @@ -78,3 +78,11 @@ def test_probe_handles_state_dict_with_integer_keys(): } with pytest.raises(InvalidModelConfigException): ModelProbe.get_model_type_from_checkpoint(Path("embedding.pt"), state_dict_with_integer_keys) + + +def test_probe_sd1_diffusers_inpainting(datadir: Path): + config = ModelProbe.probe(datadir / "sd-1/main/dreamshaper-8-inpainting") + assert isinstance(config, MainDiffusersConfig) + assert config.base is BaseModelType.StableDiffusion1 + assert config.variant is ModelVariantType.Inpaint + assert config.repo_variant is ModelRepoVariant.FP16 diff --git a/tests/test_model_probe/sd-1/main/dreamshaper-8-inpainting/README b/tests/test_model_probe/sd-1/main/dreamshaper-8-inpainting/README new file mode 100644 index 00000000000..15349b5f0ac --- /dev/null +++ b/tests/test_model_probe/sd-1/main/dreamshaper-8-inpainting/README @@ -0,0 +1 @@ +This folder contains config files copied from [Lykon/dreamshaper-8-inpainting](https://huggingface.co/Lykon/dreamshaper-8-inpainting). diff --git a/tests/test_model_probe/sd-1/main/dreamshaper-8-inpainting/model_index.json b/tests/test_model_probe/sd-1/main/dreamshaper-8-inpainting/model_index.json new file mode 100644 index 00000000000..e4e983f7ba6 --- /dev/null +++ b/tests/test_model_probe/sd-1/main/dreamshaper-8-inpainting/model_index.json @@ -0,0 +1,34 @@ +{ + "_class_name": "StableDiffusionInpaintPipeline", + "_diffusers_version": "0.21.0.dev0", + "_name_or_path": "lykon-models/dreamshaper-8-inpainting", + "feature_extractor": [ + "transformers", + "CLIPFeatureExtractor" + ], + "requires_safety_checker": true, + "safety_checker": [ + "stable_diffusion", + "StableDiffusionSafetyChecker" + ], + "scheduler": [ + "diffusers", + "DEISMultistepScheduler" + ], + "text_encoder": [ + "transformers", + "CLIPTextModel" + ], + "tokenizer": [ + "transformers", + "CLIPTokenizer" + ], + "unet": [ + "diffusers", + "UNet2DConditionModel" + ], + "vae": [ + "diffusers", + "AutoencoderKL" + ] +} diff --git a/tests/test_model_probe/sd-1/main/dreamshaper-8-inpainting/scheduler/scheduler_config.json b/tests/test_model_probe/sd-1/main/dreamshaper-8-inpainting/scheduler/scheduler_config.json new file mode 100644 index 00000000000..a63f334bd64 --- /dev/null +++ b/tests/test_model_probe/sd-1/main/dreamshaper-8-inpainting/scheduler/scheduler_config.json @@ -0,0 +1,23 @@ +{ + "_class_name": "DEISMultistepScheduler", + "_diffusers_version": "0.21.0.dev0", + "algorithm_type": "deis", + "beta_end": 0.012, + "beta_schedule": "scaled_linear", + "beta_start": 0.00085, + "clip_sample": false, + "dynamic_thresholding_ratio": 0.995, + "lower_order_final": true, + "num_train_timesteps": 1000, + "prediction_type": "epsilon", + "sample_max_value": 1.0, + "set_alpha_to_one": false, + "skip_prk_steps": true, + "solver_order": 2, + "solver_type": "logrho", + "steps_offset": 1, + "thresholding": false, + "timestep_spacing": "leading", + "trained_betas": null, + "use_karras_sigmas": false +} diff --git a/tests/test_model_probe/sd-1/main/dreamshaper-8-inpainting/unet/config.json b/tests/test_model_probe/sd-1/main/dreamshaper-8-inpainting/unet/config.json new file mode 100644 index 00000000000..d9f3b21d92e --- /dev/null +++ b/tests/test_model_probe/sd-1/main/dreamshaper-8-inpainting/unet/config.json @@ -0,0 +1,66 @@ +{ + "_class_name": "UNet2DConditionModel", + "_diffusers_version": "0.21.0.dev0", + "_name_or_path": "/home/patrick/.cache/huggingface/hub/models--lykon-models--dreamshaper-8-inpainting/snapshots/15dcb9dec91a39ee498e3917c9ef6174b103862d/unet", + "act_fn": "silu", + "addition_embed_type": null, + "addition_embed_type_num_heads": 64, + "addition_time_embed_dim": null, + "attention_head_dim": 8, + "attention_type": "default", + "block_out_channels": [ + 320, + 640, + 1280, + 1280 + ], + "center_input_sample": false, + "class_embed_type": null, + "class_embeddings_concat": false, + "conv_in_kernel": 3, + "conv_out_kernel": 3, + "cross_attention_dim": 768, + "cross_attention_norm": null, + "down_block_types": [ + "CrossAttnDownBlock2D", + "CrossAttnDownBlock2D", + "CrossAttnDownBlock2D", + "DownBlock2D" + ], + "downsample_padding": 1, + "dual_cross_attention": false, + "encoder_hid_dim": null, + "encoder_hid_dim_type": null, + "flip_sin_to_cos": true, + "freq_shift": 0, + "in_channels": 9, + "layers_per_block": 2, + "mid_block_only_cross_attention": null, + "mid_block_scale_factor": 1, + "mid_block_type": "UNetMidBlock2DCrossAttn", + "norm_eps": 1e-05, + "norm_num_groups": 32, + "num_attention_heads": null, + "num_class_embeds": null, + "only_cross_attention": false, + "out_channels": 4, + "projection_class_embeddings_input_dim": null, + "resnet_out_scale_factor": 1.0, + "resnet_skip_time_act": false, + "resnet_time_scale_shift": "default", + "sample_size": 64, + "time_cond_proj_dim": null, + "time_embedding_act_fn": null, + "time_embedding_dim": null, + "time_embedding_type": "positional", + "timestep_post_act": null, + "transformer_layers_per_block": 1, + "up_block_types": [ + "UpBlock2D", + "CrossAttnUpBlock2D", + "CrossAttnUpBlock2D", + "CrossAttnUpBlock2D" + ], + "upcast_attention": null, + "use_linear_projection": false +} diff --git a/tests/test_model_probe/sd-1/main/dreamshaper-8-inpainting/unet/diffusion_pytorch_model.fp16.safetensors b/tests/test_model_probe/sd-1/main/dreamshaper-8-inpainting/unet/diffusion_pytorch_model.fp16.safetensors new file mode 100644 index 00000000000..e69de29bb2d