feat: Add Clip Skip

invokeai/app/invocations/compel.py

@@ -95,6 +95,7 @@ def _lora_loader():
 
         with ModelPatcher.apply_lora_text_encoder(text_encoder_info.context.model, _lora_loader()),\
                 ModelPatcher.apply_ti(tokenizer_info.context.model, text_encoder_info.context.model, ti_list) as (tokenizer, ti_manager),\
+                ModelPatcher.apply_clip_skip(text_encoder_info.context.model, self.clip.skipped_layers),\
                 text_encoder_info as text_encoder:
 
             compel = Compel(
@@ -134,6 +135,24 @@ def _lora_loader():
             ),
         )
 
+class ClipSkipInvocationOutput(BaseInvocationOutput):
+    """Clip skip node output"""
+    type: Literal["clip_skip_output"] = "clip_skip_output"
+    clip: ClipField = Field(None, description="Clip with skipped layers")
+
+class ClipSkipInvocation(BaseInvocation):
+    """Skip layers in clip text_encoder model."""
+    type: Literal["clip_skip"] = "clip_skip"
+
+    clip: ClipField = Field(None, description="Clip to use")
+    skipped_layers: int = Field(0, description="Number of layers to skip in text_encoder")
+
+    def invoke(self, context: InvocationContext) -> ClipSkipInvocationOutput:
+        self.clip.skipped_layers += self.skipped_layers
+        return ClipSkipInvocationOutput(
+            clip=self.clip,
+        )
+
 
 def get_max_token_count(
         tokenizer, prompt: Union[FlattenedPrompt, Blend, Conjunction],

invokeai/app/invocations/model.py

@@ -30,6 +30,7 @@ class UNetField(BaseModel):
 class ClipField(BaseModel):
     tokenizer: ModelInfo = Field(description="Info to load tokenizer submodel")
     text_encoder: ModelInfo = Field(description="Info to load text_encoder submodel")
+    skipped_layers: int = Field(description="Number of skipped layers in text_encoder")
     loras: List[LoraInfo] = Field(description="Loras to apply on model loading")
 
 
@@ -154,6 +155,7 @@ def invoke(self, context: InvocationContext) -> ModelLoaderOutput:
                     submodel=SubModelType.TextEncoder,
                 ),
                 loras=[],
+                skipped_layers=0,
             ),
             vae=VaeField(
                 vae=ModelInfo(

invokeai/backend/model_management/lora.py

@@ -615,6 +615,24 @@ def _get_trigger(ti, index):
                 text_encoder.resize_token_embeddings(init_tokens_count)
 
 
+    @classmethod
+    @contextmanager
+    def apply_clip_skip(
+        cls,
+        text_encoder: CLIPTextModel,
+        clip_skip: int,
+    ):
+        skipped_layers = []
+        try:
+            for i in range(clip_skip):
+                skipped_layers.append(text_encoder.text_model.encoder.layers.pop(-1))
+
+            yield
+
+        finally:
+            while len(skipped_layers) > 0:
+                text_encoder.text_model.encoder.layers.append(skipped_layers.pop())
+
 class TextualInversionModel:
     name: str
     embedding: torch.Tensor # [n, 768]|[n, 1280]

invokeai/frontend/web/public/locales/en.json

@@ -527,7 +527,8 @@
         "showOptionsPanel": "Show Options Panel",
         "hidePreview": "Hide Preview",
         "showPreview": "Show Preview",
-        "controlNetControlMode": "Control Mode"
+        "controlNetControlMode": "Control Mode",
+        "clipSkip": "Clip Skip"
     },
     "settings": {
         "models": "Models",
@@ -551,7 +552,8 @@
         "generation": "Generation",
         "ui": "User Interface",
         "favoriteSchedulers": "Favorite Schedulers",
-        "favoriteSchedulersPlaceholder": "No schedulers favorited"
+        "favoriteSchedulersPlaceholder": "No schedulers favorited",
+        "showAdvancedOptions": "Show Advanced Options"
     },
     "toast": {
         "serverError": "Server Error",

invokeai/frontend/web/src/common/components/IAISlider.tsx

@@ -26,7 +26,7 @@ import {
 } from '@chakra-ui/react';
 import { clamp } from 'lodash-es';
 
-import { useTranslation } from 'react-i18next';
+import { roundDownToMultiple } from 'common/util/roundDownToMultiple';
 import {
   FocusEvent,
   memo,
@@ -36,9 +36,9 @@ import {
   useMemo,
   useState,
 } from 'react';
+import { useTranslation } from 'react-i18next';
 import { BiReset } from 'react-icons/bi';
 import IAIIconButton, { IAIIconButtonProps } from './IAIIconButton';
-import { roundDownToMultiple } from 'common/util/roundDownToMultiple';
 
 const SLIDER_MARK_STYLES: ChakraProps['sx'] = {
   mt: 1.5,

invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasImageToImageGraph.ts

@@ -12,6 +12,7 @@ import { addDynamicPromptsToGraph } from './addDynamicPromptsToGraph';
 import { addLoRAsToGraph } from './addLoRAsToGraph';
 import { addVAEToGraph } from './addVAEToGraph';
 import {
+  CLIP_SKIP,
   IMAGE_TO_IMAGE_GRAPH,
   IMAGE_TO_LATENTS,
   LATENTS_TO_IMAGE,
@@ -40,6 +41,7 @@ export const buildCanvasImageToImageGraph = (
     scheduler,
     steps,
     img2imgStrength: strength,
+    clipSkip,
     iterations,
     seed,
     shouldRandomizeSeed,
@@ -82,6 +84,11 @@ export const buildCanvasImageToImageGraph = (
         id: MAIN_MODEL_LOADER,
         model,
       },
+      [CLIP_SKIP]: {
+        type: 'clip_skip',
+        id: CLIP_SKIP,
+        skipped_layers: clipSkip,
+      },
       [LATENTS_TO_IMAGE]: {
         type: 'l2i',
         id: LATENTS_TO_IMAGE,
@@ -109,14 +116,24 @@ export const buildCanvasImageToImageGraph = (
           node_id: MAIN_MODEL_LOADER,
           field: 'clip',
         },
+        destination: {
+          node_id: CLIP_SKIP,
+          field: 'clip',
+        },
+      },
+      {
+        source: {
+          node_id: CLIP_SKIP,
+          field: 'clip',
+        },
         destination: {
           node_id: POSITIVE_CONDITIONING,
           field: 'clip',
         },
       },
       {
         source: {
-          node_id: MAIN_MODEL_LOADER,
+          node_id: CLIP_SKIP,
           field: 'clip',
         },
         destination: {

invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasInpaintGraph.ts

@@ -11,6 +11,7 @@ import { modelIdToMainModelField } from '../modelIdToMainModelField';
 import { addLoRAsToGraph } from './addLoRAsToGraph';
 import { addVAEToGraph } from './addVAEToGraph';
 import {
+  CLIP_SKIP,
   INPAINT,
   INPAINT_GRAPH,
   ITERATE,
@@ -49,6 +50,7 @@ export const buildCanvasInpaintGraph = (
     seamStrength,
     tileSize,
     infillMethod,
+    clipSkip,
   } = state.generation;
 
   // The bounding box determines width and height, not the width and height params
@@ -108,6 +110,11 @@ export const buildCanvasInpaintGraph = (
         id: MAIN_MODEL_LOADER,
         model,
       },
+      [CLIP_SKIP]: {
+        type: 'clip_skip',
+        id: CLIP_SKIP,
+        skipped_layers: clipSkip,
+      },
       [RANGE_OF_SIZE]: {
         type: 'range_of_size',
         id: RANGE_OF_SIZE,
@@ -124,27 +131,27 @@ export const buildCanvasInpaintGraph = (
     edges: [
       {
         source: {
-          node_id: NEGATIVE_CONDITIONING,
-          field: 'conditioning',
+          node_id: MAIN_MODEL_LOADER,
+          field: 'unet',
         },
         destination: {
           node_id: INPAINT,
-          field: 'negative_conditioning',
+          field: 'unet',
         },
       },
       {
         source: {
-          node_id: POSITIVE_CONDITIONING,
-          field: 'conditioning',
+          node_id: MAIN_MODEL_LOADER,
+          field: 'clip',
         },
         destination: {
-          node_id: INPAINT,
-          field: 'positive_conditioning',
+          node_id: CLIP_SKIP,
+          field: 'clip',
         },
       },
       {
         source: {
-          node_id: MAIN_MODEL_LOADER,
+          node_id: CLIP_SKIP,
           field: 'clip',
         },
         destination: {
@@ -154,7 +161,7 @@ export const buildCanvasInpaintGraph = (
       },
       {
         source: {
-          node_id: MAIN_MODEL_LOADER,
+          node_id: CLIP_SKIP,
           field: 'clip',
         },
         destination: {
@@ -164,12 +171,22 @@ export const buildCanvasInpaintGraph = (
       },
       {
         source: {
-          node_id: MAIN_MODEL_LOADER,
-          field: 'unet',
+          node_id: NEGATIVE_CONDITIONING,
+          field: 'conditioning',
         },
         destination: {
           node_id: INPAINT,
-          field: 'unet',
+          field: 'negative_conditioning',
+        },
+      },
+      {
+        source: {
+          node_id: POSITIVE_CONDITIONING,
+          field: 'conditioning',
+        },
+        destination: {
+          node_id: INPAINT,
+          field: 'positive_conditioning',
         },
       },
       {

invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasTextToImageGraph.ts

@@ -6,6 +6,7 @@ import { addDynamicPromptsToGraph } from './addDynamicPromptsToGraph';
 import { addLoRAsToGraph } from './addLoRAsToGraph';
 import { addVAEToGraph } from './addVAEToGraph';
 import {
+  CLIP_SKIP,
   LATENTS_TO_IMAGE,
   MAIN_MODEL_LOADER,
   NEGATIVE_CONDITIONING,
@@ -28,6 +29,7 @@ export const buildCanvasTextToImageGraph = (
     cfgScale: cfg_scale,
     scheduler,
     steps,
+    clipSkip,
     iterations,
     seed,
     shouldRandomizeSeed,
@@ -79,6 +81,11 @@ export const buildCanvasTextToImageGraph = (
         id: MAIN_MODEL_LOADER,
         model,
       },
+      [CLIP_SKIP]: {
+        type: 'clip_skip',
+        id: CLIP_SKIP,
+        skipped_layers: clipSkip,
+      },
       [LATENTS_TO_IMAGE]: {
         type: 'l2i',
         id: LATENTS_TO_IMAGE,
@@ -110,14 +117,24 @@ export const buildCanvasTextToImageGraph = (
           node_id: MAIN_MODEL_LOADER,
           field: 'clip',
         },
+        destination: {
+          node_id: CLIP_SKIP,
+          field: 'clip',
+        },
+      },
+      {
+        source: {
+          node_id: CLIP_SKIP,
+          field: 'clip',
+        },
         destination: {
           node_id: POSITIVE_CONDITIONING,
           field: 'clip',
         },
       },
       {
         source: {
-          node_id: MAIN_MODEL_LOADER,
+          node_id: CLIP_SKIP,
           field: 'clip',
         },
         destination: {