feat(ui): regional prompting followups 3

invokeai/frontend/web/package.json

@@ -25,7 +25,7 @@
     "typegen": "node scripts/typegen.js",
     "preview": "vite preview",
     "lint:knip": "knip",
-    "lint:dpdm": "dpdm --no-warning --no-tree --transform --exit-code circular:1 src/main.tsx",
+    "lint:dpdm": "dpdm --no-warning --no-tree --transform --exit-code circular:0 src/main.tsx",
     "lint:eslint": "eslint --max-warnings=0 .",
     "lint:prettier": "prettier --check .",
     "lint:tsc": "tsc --noEmit",

invokeai/frontend/web/public/locales/en.json

@@ -84,6 +84,8 @@
         "direction": "Direction",
         "ipAdapter": "IP Adapter",
         "t2iAdapter": "T2I Adapter",
+        "positivePrompt": "Positive Prompt",
+        "negativePrompt": "Negative Prompt",
         "discordLabel": "Discord",
         "dontAskMeAgain": "Don't ask me again",
         "error": "Error",
@@ -1518,11 +1520,16 @@
         "brushSize": "Brush Size",
         "regionalPrompts": "Regional Prompts BETA",
         "enableRegionalPrompts": "Enable $t(regionalPrompts.regionalPrompts)",
-        "layerOpacity": "Layer Opacity",
+        "globalMaskOpacity": "Global Mask Opacity",
         "autoNegative": "Auto Negative",
         "toggleVisibility": "Toggle Layer Visibility",
+        "deletePrompt": "Delete Prompt",
         "resetRegion": "Reset Region",
         "debugLayers": "Debug Layers",
-        "rectangle": "Rectangle"
+        "rectangle": "Rectangle",
+        "maskPreviewColor": "Mask Preview Color",
+        "addPositivePrompt": "Add $t(common.positivePrompt)",
+        "addNegativePrompt": "Add $t(common.negativePrompt)",
+        "addIPAdapter": "Add $t(common.ipAdapter)"
     }
 }

invokeai/frontend/web/src/common/components/RgbColorPicker.tsx

@@ -0,0 +1,84 @@
+import type { ChakraProps } from '@invoke-ai/ui-library';
+import { CompositeNumberInput, Flex, FormControl, FormLabel } from '@invoke-ai/ui-library';
+import type { CSSProperties } from 'react';
+import { memo, useCallback } from 'react';
+import { RgbColorPicker as ColorfulRgbColorPicker } from 'react-colorful';
+import type { ColorPickerBaseProps, RgbColor } from 'react-colorful/dist/types';
+import { useTranslation } from 'react-i18next';
+
+type RgbColorPickerProps = ColorPickerBaseProps<RgbColor> & {
+  withNumberInput?: boolean;
+};
+
+const colorPickerPointerStyles: NonNullable<ChakraProps['sx']> = {
+  width: 6,
+  height: 6,
+  borderColor: 'base.100',
+};
+
+const sx: ChakraProps['sx'] = {
+  '.react-colorful__hue-pointer': colorPickerPointerStyles,
+  '.react-colorful__saturation-pointer': colorPickerPointerStyles,
+  '.react-colorful__alpha-pointer': colorPickerPointerStyles,
+  gap: 5,
+  flexDir: 'column',
+};
+
+const colorPickerStyles: CSSProperties = { width: '100%' };
+
+const numberInputWidth: ChakraProps['w'] = '3.5rem';
+
+const RgbColorPicker = (props: RgbColorPickerProps) => {
+  const { color, onChange, withNumberInput, ...rest } = props;
+  const { t } = useTranslation();
+  const handleChangeR = useCallback((r: number) => onChange({ ...color, r }), [color, onChange]);
+  const handleChangeG = useCallback((g: number) => onChange({ ...color, g }), [color, onChange]);
+  const handleChangeB = useCallback((b: number) => onChange({ ...color, b }), [color, onChange]);
+  return (
+    <Flex sx={sx}>
+      <ColorfulRgbColorPicker color={color} onChange={onChange} style={colorPickerStyles} {...rest} />
+      {withNumberInput && (
+        <Flex gap={5}>
+          <FormControl gap={0}>
+            <FormLabel>{t('common.red')[0]}</FormLabel>
+            <CompositeNumberInput
+              value={color.r}
+              onChange={handleChangeR}
+              min={0}
+              max={255}
+              step={1}
+              w={numberInputWidth}
+              defaultValue={90}
+            />
+          </FormControl>
+          <FormControl gap={0}>
+            <FormLabel>{t('common.green')[0]}</FormLabel>
+            <CompositeNumberInput
+              value={color.g}
+              onChange={handleChangeG}
+              min={0}
+              max={255}
+              step={1}
+              w={numberInputWidth}
+              defaultValue={90}
+            />
+          </FormControl>
+          <FormControl gap={0}>
+            <FormLabel>{t('common.blue')[0]}</FormLabel>
+            <CompositeNumberInput
+              value={color.b}
+              onChange={handleChangeB}
+              min={0}
+              max={255}
+              step={1}
+              w={numberInputWidth}
+              defaultValue={255}
+            />
+          </FormControl>
+        </Flex>
+      )}
+    </Flex>
+  );
+};
+
+export default memo(RgbColorPicker);

invokeai/frontend/web/src/features/controlAdapters/store/controlAdaptersSlice.ts

@@ -6,6 +6,7 @@ import { deepClone } from 'common/util/deepClone';
 import { buildControlAdapter } from 'features/controlAdapters/util/buildControlAdapter';
 import { buildControlAdapterProcessor } from 'features/controlAdapters/util/buildControlAdapterProcessor';
 import { zModelIdentifierField } from 'features/nodes/types/common';
+import { maskLayerIPAdapterAdded } from 'features/regionalPrompts/store/regionalPromptsSlice';
 import { merge, uniq } from 'lodash-es';
 import type { ControlNetModelConfig, IPAdapterModelConfig, T2IAdapterModelConfig } from 'services/api/types';
 import { socketInvocationError } from 'services/events/actions';
@@ -382,6 +383,10 @@ export const controlAdaptersSlice = createSlice({
     builder.addCase(socketInvocationError, (state) => {
       state.pendingControlImages = [];
     });
+
+    builder.addCase(maskLayerIPAdapterAdded, (state, action) => {
+      caAdapter.addOne(state, buildControlAdapter(action.meta.uuid, 'ip_adapter'));
+    });
   },
 });
 

invokeai/frontend/web/src/features/nodes/util/graph/addIPAdapterToLinearGraph.ts

@@ -19,12 +19,14 @@ export const addIPAdapterToLinearGraph = async (
   graph: NonNullableGraph,
   baseNodeId: string
 ): Promise<void> => {
-  const validIPAdapters = selectValidIPAdapters(state.controlAdapters).filter(({ model, controlImage, isEnabled }) => {
-    const hasModel = Boolean(model);
-    const doesBaseMatch = model?.base === state.generation.model?.base;
-    const hasControlImage = controlImage;
-    return isEnabled && hasModel && doesBaseMatch && hasControlImage;
-  });
+  const validIPAdapters = selectValidIPAdapters(state.controlAdapters)
+    .filter(({ model, controlImage, isEnabled }) => {
+      const hasModel = Boolean(model);
+      const doesBaseMatch = model?.base === state.generation.model?.base;
+      const hasControlImage = controlImage;
+      return isEnabled && hasModel && doesBaseMatch && hasControlImage;
+    })
+    .filter((ca) => !state.regionalPrompts.present.layers.some((l) => l.ipAdapterIds.includes(ca.id)));
 
   if (validIPAdapters.length) {
     // Even though denoise_latents' ip adapter input is collection or scalar, keep it simple and always use a collect

invokeai/frontend/web/src/features/nodes/util/graph/addRegionalPromptsToGraph.ts

@@ -1,6 +1,8 @@
 import { getStore } from 'app/store/nanostores/store';
 import type { RootState } from 'app/store/store';
+import { selectAllIPAdapters } from 'features/controlAdapters/store/controlAdaptersSlice';
 import {
+  IP_ADAPTER_COLLECT,
   NEGATIVE_CONDITIONING,
   NEGATIVE_CONDITIONING_COLLECT,
   POSITIVE_CONDITIONING,
@@ -13,9 +15,9 @@ import {
 } from 'features/nodes/util/graph/constants';
 import { isVectorMaskLayer } from 'features/regionalPrompts/store/regionalPromptsSlice';
 import { getRegionalPromptLayerBlobs } from 'features/regionalPrompts/util/getLayerBlobs';
-import { size } from 'lodash-es';
+import { size, sumBy } from 'lodash-es';
 import { imagesApi } from 'services/api/endpoints/images';
-import type { CollectInvocation, Edge, NonNullableGraph, S } from 'services/api/types';
+import type { CollectInvocation, Edge, IPAdapterInvocation, NonNullableGraph, S } from 'services/api/types';
 import { assert } from 'tsafe';
 
 export const addRegionalPromptsToGraph = async (state: RootState, graph: NonNullableGraph, denoiseNodeId: string) => {
@@ -32,9 +34,9 @@ export const addRegionalPromptsToGraph = async (state: RootState, graph: NonNull
     .filter((l) => l.isVisible)
     // Only layers with prompts get added to the graph
     .filter((l) => {
-      const hasTextPrompt = l.textPrompt && (l.textPrompt.positive || l.textPrompt.negative);
-      const hasAtLeastOneImagePrompt = l.imagePrompts.length > 0;
-      return hasTextPrompt || hasAtLeastOneImagePrompt;
+      const hasTextPrompt = Boolean(l.positivePrompt || l.negativePrompt);
+      const hasIPAdapter = l.ipAdapterIds.length !== 0;
+      return hasTextPrompt || hasIPAdapter;
     });
 
   const layerIds = layers.map((l) => l.id);
@@ -103,6 +105,22 @@ export const addRegionalPromptsToGraph = async (state: RootState, graph: NonNull
     },
   });
 
+  if (!graph.nodes[IP_ADAPTER_COLLECT] && sumBy(layers, (l) => l.ipAdapterIds.length) > 0) {
+    const ipAdapterCollectNode: CollectInvocation = {
+      id: IP_ADAPTER_COLLECT,
+      type: 'collect',
+      is_intermediate: true,
+    };
+    graph.nodes[IP_ADAPTER_COLLECT] = ipAdapterCollectNode;
+    graph.edges.push({
+      source: { node_id: IP_ADAPTER_COLLECT, field: 'collection' },
+      destination: {
+        node_id: denoiseNodeId,
+        field: 'ip_adapter',
+      },
+    });
+  }
+
   // Upload the blobs to the backend, add each to graph
   // TODO: Store the uploaded image names in redux to reuse them, so long as the layer hasn't otherwise changed. This
   // would be a great perf win - not only would we skip re-uploading the same image, but we'd be able to use the node
@@ -130,19 +148,19 @@ export const addRegionalPromptsToGraph = async (state: RootState, graph: NonNull
     };
     graph.nodes[maskToTensorNode.id] = maskToTensorNode;
 
-    if (layer.textPrompt?.positive) {
+    if (layer.positivePrompt) {
       // The main positive conditioning node
       const regionalPositiveCondNode: S['SDXLCompelPromptInvocation'] | S['CompelInvocation'] = isSDXL
         ? {
             type: 'sdxl_compel_prompt',
             id: `${PROMPT_REGION_POSITIVE_COND_PREFIX}_${layer.id}`,
-            prompt: layer.textPrompt.positive,
-            style: layer.textPrompt.positive, // TODO: Should we put the positive prompt in both fields?
+            prompt: layer.positivePrompt,
+            style: layer.positivePrompt, // TODO: Should we put the positive prompt in both fields?
           }
         : {
             type: 'compel',
             id: `${PROMPT_REGION_POSITIVE_COND_PREFIX}_${layer.id}`,
-            prompt: layer.textPrompt.positive,
+            prompt: layer.positivePrompt,
           };
       graph.nodes[regionalPositiveCondNode.id] = regionalPositiveCondNode;
 
@@ -169,19 +187,19 @@ export const addRegionalPromptsToGraph = async (state: RootState, graph: NonNull
       }
     }
 
-    if (layer.textPrompt?.negative) {
+    if (layer.negativePrompt) {
       // The main negative conditioning node
       const regionalNegativeCondNode: S['SDXLCompelPromptInvocation'] | S['CompelInvocation'] = isSDXL
         ? {
             type: 'sdxl_compel_prompt',
             id: `${PROMPT_REGION_NEGATIVE_COND_PREFIX}_${layer.id}`,
-            prompt: layer.textPrompt.negative,
-            style: layer.textPrompt.negative,
+            prompt: layer.negativePrompt,
+            style: layer.negativePrompt,
           }
         : {
             type: 'compel',
             id: `${PROMPT_REGION_NEGATIVE_COND_PREFIX}_${layer.id}`,
-            prompt: layer.textPrompt.negative,
+            prompt: layer.negativePrompt,
           };
       graph.nodes[regionalNegativeCondNode.id] = regionalNegativeCondNode;
 
@@ -209,7 +227,7 @@ export const addRegionalPromptsToGraph = async (state: RootState, graph: NonNull
     }
 
     // If we are using the "invert" auto-negative setting, we need to add an additional negative conditioning node
-    if (layer.autoNegative === 'invert' && layer.textPrompt?.positive) {
+    if (layer.autoNegative === 'invert' && layer.positivePrompt) {
       // We re-use the mask image, but invert it when converting to tensor
       const invertTensorMaskNode: S['InvertTensorMaskInvocation'] = {
         id: `${PROMPT_REGION_INVERT_TENSOR_MASK_PREFIX}_${layer.id}`,
@@ -235,13 +253,13 @@ export const addRegionalPromptsToGraph = async (state: RootState, graph: NonNull
         ? {
             type: 'sdxl_compel_prompt',
             id: `${PROMPT_REGION_POSITIVE_COND_INVERTED_PREFIX}_${layer.id}`,
-            prompt: layer.textPrompt.positive,
-            style: layer.textPrompt.positive,
+            prompt: layer.positivePrompt,
+            style: layer.positivePrompt,
           }
         : {
             type: 'compel',
             id: `${PROMPT_REGION_POSITIVE_COND_INVERTED_PREFIX}_${layer.id}`,
-            prompt: layer.textPrompt.positive,
+            prompt: layer.positivePrompt,
           };
       graph.nodes[regionalPositiveCondInvertedNode.id] = regionalPositiveCondInvertedNode;
       // Connect the inverted mask to the conditioning
@@ -264,5 +282,47 @@ export const addRegionalPromptsToGraph = async (state: RootState, graph: NonNull
         }
       }
     }
+
+    for (const ipAdapterId of layer.ipAdapterIds) {
+      const ipAdapter = selectAllIPAdapters(state.controlAdapters).find((ca) => ca.id === ipAdapterId);
+      console.log(ipAdapter);
+      if (!ipAdapter?.model) {
+        return;
+      }
+      const { id, weight, model, clipVisionModel, method, beginStepPct, endStepPct, controlImage } = ipAdapter;
+
+      assert(controlImage, 'IP Adapter image is required');
+
+      const ipAdapterNode: IPAdapterInvocation = {
+        id: `ip_adapter_${id}`,
+        type: 'ip_adapter',
+        is_intermediate: true,
+        weight: weight,
+        method: method,
+        ip_adapter_model: model,
+        clip_vision_model: clipVisionModel,
+        begin_step_percent: beginStepPct,
+        end_step_percent: endStepPct,
+        image: {
+          image_name: controlImage,
+        },
+      };
+
+      graph.nodes[ipAdapterNode.id] = ipAdapterNode;
+
+      // Connect the mask to the conditioning
+      graph.edges.push({
+        source: { node_id: maskToTensorNode.id, field: 'mask' },
+        destination: { node_id: ipAdapterNode.id, field: 'mask' },
+      });
+
+      graph.edges.push({
+        source: { node_id: ipAdapterNode.id, field: 'ip_adapter' },
+        destination: {
+          node_id: IP_ADAPTER_COLLECT,
+          field: 'item',
+        },
+      });
+    }
   }
 };

invokeai/frontend/web/src/features/parameters/types/parameterSchemas.ts

@@ -200,6 +200,4 @@ export const isParameterLoRAWeight = (val: unknown): val is ParameterLoRAWeight
 // #region Regional Prompts AutoNegative
 const zAutoNegative = z.enum(['off', 'invert']);
 export type ParameterAutoNegative = z.infer<typeof zAutoNegative>;
-export const isParameterAutoNegative = (val: unknown): val is ParameterAutoNegative =>
-  zAutoNegative.safeParse(val).success;
 // #endregion