feat(ui): regional prompting followups 2

invokeai/app/invocations/mask.py

@@ -88,3 +88,33 @@ def invoke(self, context: InvocationContext) -> MaskOutput:
             height=inverted.shape[1],
             width=inverted.shape[2],
         )
+
+
+@invocation(
+    "image_mask_to_tensor",
+    title="Image Mask to Tensor",
+    tags=["conditioning"],
+    category="conditioning",
+    version="1.0.0",
+)
+class ImageMaskToTensorInvocation(BaseInvocation, WithMetadata):
+    """Convert a mask image to a tensor. Converts the image to grayscale and uses thresholding at the specified value."""
+
+    image: ImageField = InputField(description="The mask image to convert.")
+    cutoff: int = InputField(ge=0, le=255, description="Cutoff (<)", default=128)
+    invert: bool = InputField(default=False, description="Whether to invert the mask.")
+
+    def invoke(self, context: InvocationContext) -> MaskOutput:
+        image = context.images.get_pil(self.image.image_name, mode="L")
+
+        mask = torch.zeros((1, image.height, image.width), dtype=torch.bool)
+        if self.invert:
+            mask[0] = torch.tensor(np.array(image)[:, :] >= self.cutoff, dtype=torch.bool)
+        else:
+            mask[0] = torch.tensor(np.array(image)[:, :] < self.cutoff, dtype=torch.bool)
+
+        return MaskOutput(
+            mask=TensorField(tensor_name=context.tensors.save(mask)),
+            height=mask.shape[1],
+            width=mask.shape[2],
+        )

invokeai/frontend/web/public/locales/en.json

@@ -1522,6 +1522,7 @@
         "autoNegative": "Auto Negative",
         "toggleVisibility": "Toggle Layer Visibility",
         "resetRegion": "Reset Region",
-        "debugLayers": "Debug Layers"
+        "debugLayers": "Debug Layers",
+        "rectangle": "Rectangle"
     }
 }

invokeai/frontend/web/src/common/components/IAIColorPicker.tsx

@@ -26,7 +26,7 @@ const sx: ChakraProps['sx'] = {
 
 const colorPickerStyles: CSSProperties = { width: '100%' };
 
-const numberInputWidth: ChakraProps['w'] = '4.2rem';
+const numberInputWidth: ChakraProps['w'] = '3.5rem';
 
 const IAIColorPicker = (props: IAIColorPickerProps) => {
   const { color, onChange, withNumberInput, ...rest } = props;
@@ -41,7 +41,7 @@ const IAIColorPicker = (props: IAIColorPickerProps) => {
       {withNumberInput && (
         <Flex gap={5}>
           <FormControl gap={0}>
-            <FormLabel>{t('common.red')}</FormLabel>
+            <FormLabel>{t('common.red')[0]}</FormLabel>
             <CompositeNumberInput
               value={color.r}
               onChange={handleChangeR}
@@ -53,7 +53,7 @@ const IAIColorPicker = (props: IAIColorPickerProps) => {
             />
           </FormControl>
           <FormControl gap={0}>
-            <FormLabel>{t('common.green')}</FormLabel>
+            <FormLabel>{t('common.green')[0]}</FormLabel>
             <CompositeNumberInput
               value={color.g}
               onChange={handleChangeG}
@@ -65,7 +65,7 @@ const IAIColorPicker = (props: IAIColorPickerProps) => {
             />
           </FormControl>
           <FormControl gap={0}>
-            <FormLabel>{t('common.blue')}</FormLabel>
+            <FormLabel>{t('common.blue')[0]}</FormLabel>
             <CompositeNumberInput
               value={color.b}
               onChange={handleChangeB}
@@ -77,7 +77,7 @@ const IAIColorPicker = (props: IAIColorPickerProps) => {
             />
           </FormControl>
           <FormControl gap={0}>
-            <FormLabel>{t('common.alpha')}</FormLabel>
+            <FormLabel>{t('common.alpha')[0]}</FormLabel>
             <CompositeNumberInput
               value={color.a}
               onChange={handleChangeA}

invokeai/frontend/web/src/features/nodes/util/graph/addRegionalPromptsToGraph.ts

@@ -11,7 +11,7 @@ import {
   PROMPT_REGION_POSITIVE_COND_INVERTED_PREFIX,
   PROMPT_REGION_POSITIVE_COND_PREFIX,
 } from 'features/nodes/util/graph/constants';
-import { isRPLayer } from 'features/regionalPrompts/store/regionalPromptsSlice';
+import { isVectorMaskLayer } from 'features/regionalPrompts/store/regionalPromptsSlice';
 import { getRegionalPromptLayerBlobs } from 'features/regionalPrompts/util/getLayerBlobs';
 import { size } from 'lodash-es';
 import { imagesApi } from 'services/api/endpoints/images';
@@ -23,12 +23,19 @@ export const addRegionalPromptsToGraph = async (state: RootState, graph: NonNull
     return;
   }
   const { dispatch } = getStore();
-  // TODO: Handle non-SDXL
   const isSDXL = state.generation.model?.base === 'sdxl';
   const layers = state.regionalPrompts.present.layers
-    .filter(isRPLayer) // We only want the prompt region layers
-    .filter((l) => l.isVisible) // Only visible layers are rendered on the canvas
-    .filter((l) => l.negativePrompt || l.positivePrompt); // Only layers with prompts get added to the graph
+    // Only support vector mask layers now
+    // TODO: Image masks
+    .filter(isVectorMaskLayer)
+    // Only visible layers are rendered on the canvas
+    .filter((l) => l.isVisible)
+    // Only layers with prompts get added to the graph
+    .filter((l) => {
+      const hasTextPrompt = l.textPrompt && (l.textPrompt.positive || l.textPrompt.negative);
+      const hasAtLeastOneImagePrompt = l.imagePrompts.length > 0;
+      return hasTextPrompt || hasAtLeastOneImagePrompt;
+    });
 
   const layerIds = layers.map((l) => l.id);
   const blobs = await getRegionalPromptLayerBlobs(layerIds);
@@ -123,19 +130,19 @@ export const addRegionalPromptsToGraph = async (state: RootState, graph: NonNull
     };
     graph.nodes[maskToTensorNode.id] = maskToTensorNode;
 
-    if (layer.positivePrompt) {
+    if (layer.textPrompt?.positive) {
       // The main positive conditioning node
       const regionalPositiveCondNode: S['SDXLCompelPromptInvocation'] | S['CompelInvocation'] = isSDXL
         ? {
             type: 'sdxl_compel_prompt',
             id: `${PROMPT_REGION_POSITIVE_COND_PREFIX}_${layer.id}`,
-            prompt: layer.positivePrompt,
-            style: layer.positivePrompt, // TODO: Should we put the positive prompt in both fields?
+            prompt: layer.textPrompt.positive,
+            style: layer.textPrompt.positive, // TODO: Should we put the positive prompt in both fields?
           }
         : {
             type: 'compel',
             id: `${PROMPT_REGION_POSITIVE_COND_PREFIX}_${layer.id}`,
-            prompt: layer.positivePrompt,
+            prompt: layer.textPrompt.positive,
           };
       graph.nodes[regionalPositiveCondNode.id] = regionalPositiveCondNode;
 
@@ -162,19 +169,19 @@ export const addRegionalPromptsToGraph = async (state: RootState, graph: NonNull
       }
     }
 
-    if (layer.negativePrompt) {
+    if (layer.textPrompt?.negative) {
       // The main negative conditioning node
       const regionalNegativeCondNode: S['SDXLCompelPromptInvocation'] | S['CompelInvocation'] = isSDXL
         ? {
             type: 'sdxl_compel_prompt',
             id: `${PROMPT_REGION_NEGATIVE_COND_PREFIX}_${layer.id}`,
-            prompt: layer.negativePrompt,
-            style: layer.negativePrompt,
+            prompt: layer.textPrompt.negative,
+            style: layer.textPrompt.negative,
           }
         : {
             type: 'compel',
             id: `${PROMPT_REGION_NEGATIVE_COND_PREFIX}_${layer.id}`,
-            prompt: layer.negativePrompt,
+            prompt: layer.textPrompt.negative,
           };
       graph.nodes[regionalNegativeCondNode.id] = regionalNegativeCondNode;
 
@@ -202,7 +209,7 @@ export const addRegionalPromptsToGraph = async (state: RootState, graph: NonNull
     }
 
     // If we are using the "invert" auto-negative setting, we need to add an additional negative conditioning node
-    if (layer.autoNegative === 'invert' && layer.positivePrompt) {
+    if (layer.autoNegative === 'invert' && layer.textPrompt?.positive) {
       // We re-use the mask image, but invert it when converting to tensor
       const invertTensorMaskNode: S['InvertTensorMaskInvocation'] = {
         id: `${PROMPT_REGION_INVERT_TENSOR_MASK_PREFIX}_${layer.id}`,
@@ -228,13 +235,13 @@ export const addRegionalPromptsToGraph = async (state: RootState, graph: NonNull
         ? {
             type: 'sdxl_compel_prompt',
             id: `${PROMPT_REGION_POSITIVE_COND_INVERTED_PREFIX}_${layer.id}`,
-            prompt: layer.positivePrompt,
-            style: layer.positivePrompt,
+            prompt: layer.textPrompt.positive,
+            style: layer.textPrompt.positive,
           }
         : {
             type: 'compel',
             id: `${PROMPT_REGION_POSITIVE_COND_INVERTED_PREFIX}_${layer.id}`,
-            prompt: layer.positivePrompt,
+            prompt: layer.textPrompt.positive,
           };
       graph.nodes[regionalPositiveCondInvertedNode.id] = regionalPositiveCondInvertedNode;
       // Connect the inverted mask to the conditioning

invokeai/frontend/web/src/features/regionalPrompts/components/AddLayerButton.tsx

@@ -8,7 +8,7 @@ export const AddLayerButton = memo(() => {
   const { t } = useTranslation();
   const dispatch = useAppDispatch();
   const onClick = useCallback(() => {
-    dispatch(layerAdded('regionalPromptLayer'));
+    dispatch(layerAdded('vector_mask_layer'));
   }, [dispatch]);
 
   return <Button onClick={onClick}>{t('regionalPrompts.addLayer')}</Button>;

invokeai/frontend/web/src/features/regionalPrompts/components/PromptLayerOpacity.tsx → invokeai/frontend/web/src/features/regionalPrompts/components/GlobalMaskLayerOpacity.tsx

@@ -1,19 +1,19 @@
 import { CompositeNumberInput, CompositeSlider, FormControl, FormLabel } from '@invoke-ai/ui-library';
 import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
 import {
+  globalMaskLayerOpacityChanged,
   initialRegionalPromptsState,
-  promptLayerOpacityChanged,
 } from 'features/regionalPrompts/store/regionalPromptsSlice';
 import { memo, useCallback } from 'react';
 import { useTranslation } from 'react-i18next';
 
-export const PromptLayerOpacity = memo(() => {
+export const GlobalMaskLayerOpacity = memo(() => {
   const dispatch = useAppDispatch();
   const { t } = useTranslation();
-  const promptLayerOpacity = useAppSelector((s) => s.regionalPrompts.present.promptLayerOpacity);
+  const globalMaskLayerOpacity = useAppSelector((s) => s.regionalPrompts.present.globalMaskLayerOpacity);
   const onChange = useCallback(
     (v: number) => {
-      dispatch(promptLayerOpacityChanged(v));
+      dispatch(globalMaskLayerOpacityChanged(v));
     },
     [dispatch]
   );
@@ -24,20 +24,20 @@ export const PromptLayerOpacity = memo(() => {
         min={0.25}
         max={1}
         step={0.01}
-        value={promptLayerOpacity}
-        defaultValue={initialRegionalPromptsState.promptLayerOpacity}
+        value={globalMaskLayerOpacity}
+        defaultValue={initialRegionalPromptsState.globalMaskLayerOpacity}
         onChange={onChange}
       />
       <CompositeNumberInput
         min={0.25}
         max={1}
         step={0.01}
-        value={promptLayerOpacity}
-        defaultValue={initialRegionalPromptsState.promptLayerOpacity}
+        value={globalMaskLayerOpacity}
+        defaultValue={initialRegionalPromptsState.globalMaskLayerOpacity}
         onChange={onChange}
       />
     </FormControl>
   );
 });
 
-PromptLayerOpacity.displayName = 'PromptLayerOpacity';
+GlobalMaskLayerOpacity.displayName = 'GlobalMaskLayerOpacity';

invokeai/frontend/web/src/features/regionalPrompts/components/RPLayerActionsButtonGroup.tsx

@@ -1,6 +1,6 @@
 import { ButtonGroup, IconButton } from '@invoke-ai/ui-library';
 import { useAppDispatch } from 'app/store/storeHooks';
-import { layerDeleted, rpLayerReset } from 'features/regionalPrompts/store/regionalPromptsSlice';
+import { layerDeleted, layerReset } from 'features/regionalPrompts/store/regionalPromptsSlice';
 import { memo, useCallback } from 'react';
 import { useTranslation } from 'react-i18next';
 import { PiArrowCounterClockwiseBold, PiTrashSimpleBold } from 'react-icons/pi';
@@ -14,7 +14,7 @@ export const RPLayerActionsButtonGroup = memo(({ layerId }: Props) => {
     dispatch(layerDeleted(layerId));
   }, [dispatch, layerId]);
   const resetLayer = useCallback(() => {
-    dispatch(rpLayerReset(layerId));
+    dispatch(layerReset(layerId));
   }, [dispatch, layerId]);
   return (
     <ButtonGroup isAttached={false}>

invokeai/frontend/web/src/features/regionalPrompts/components/RPLayerAutoNegativeCombobox.tsx

@@ -4,8 +4,8 @@ import { createSelector } from '@reduxjs/toolkit';
 import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
 import { isParameterAutoNegative } from 'features/parameters/types/parameterSchemas';
 import {
-  isRPLayer,
-  rpLayerAutoNegativeChanged,
+  isVectorMaskLayer,
+  maskLayerAutoNegativeChanged,
   selectRegionalPromptsSlice,
 } from 'features/regionalPrompts/store/regionalPromptsSlice';
 import { memo, useCallback, useMemo } from 'react';
@@ -26,7 +26,7 @@ const useAutoNegative = (layerId: string) => {
     () =>
       createSelector(selectRegionalPromptsSlice, (regionalPrompts) => {
         const layer = regionalPrompts.present.layers.find((l) => l.id === layerId);
-        assert(isRPLayer(layer), `Layer ${layerId} not found or not an RP layer`);
+        assert(isVectorMaskLayer(layer), `Layer ${layerId} not found or not an RP layer`);
         return layer.autoNegative;
       }),
     [layerId]
@@ -45,7 +45,7 @@ export const RPLayerAutoNegativeCombobox = memo(({ layerId }: Props) => {
       if (!isParameterAutoNegative(v?.value)) {
         return;
       }
-      dispatch(rpLayerAutoNegativeChanged({ layerId, autoNegative: v.value }));
+      dispatch(maskLayerAutoNegativeChanged({ layerId, autoNegative: v.value }));
     },
     [dispatch, layerId]
   );