Write change logs

[samuraieng]

Check parts which modified by this PR

[samuraieng]

PR draft covers this. But let this keep opened.
better to understand which files must be submitted.

[samuraieng]

diff with master (9e5647a, b8840)

$ git diff --name-only master..samuraieng/sarashina22vl-00
convert_hf_to_gguf.py
gguf-py/gguf/constants.py
tools/mtmd/clip-impl.h
tools/mtmd/clip.cpp
tools/mtmd/mtmd.cpp
tools/mtmd/tests.sh

$ git diff master..samuraieng/sarashina22vl-00

diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py
index 2df5e94fe..baf2ae72f 100755
--- a/convert_hf_to_gguf.py
+++ b/convert_hf_to_gguf.py
@@ -2947,6 +2947,13 @@ class LlamaModel(TextModel):
             else:
                 return
 
+        if self.origin_hf_arch.startswith('Sarashina2VisionForCausalLM'):
+            # Remove llm. from name 
+            if name.startswith("llm."):
+                name = name[len("llm."):]
+            elif name.startswith("visual.") or name in ("norm.weight", "norm.bias"):
+                return  #Skip processing "modify_tensors"
+
         yield from super().modify_tensors(data_torch, name, bid)
 
     def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
@@ -4162,6 +4169,7 @@ class PaddleOCRVisionModel(MmprojModel):
     "Qwen2VLForConditionalGeneration",
     "Qwen2_5_VLForConditionalGeneration",
     "Qwen2_5OmniModel",
+    "Sarashina2VisionForCausalLM",
 )
 class Qwen2VLModel(TextModel):
     model_arch = gguf.MODEL_ARCH.QWEN2VL
@@ -4185,7 +4193,7 @@ class Qwen2VLModel(TextModel):
         yield from super().modify_tensors(data_torch, name, bid)
 
 
-@ModelBase.register("Qwen2VLModel", "Qwen2VLForConditionalGeneration", "Qwen2_5_VLForConditionalGeneration")
+@ModelBase.register("Qwen2VLModel", "Qwen2VLForConditionalGeneration", "Qwen2_5_VLForConditionalGeneration", "Sarashina2VisionForCausalLM")
 class Qwen2VLVisionModel(MmprojModel):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -4220,6 +4228,9 @@ class Qwen2VLVisionModel(MmprojModel):
                 if fullatt_block_indexes[i] - fullatt_block_indexes[i - 1] != n_wa_pattern:
                     raise ValueError(f"Invalid fullatt_block_indexes: {fullatt_block_indexes}")
             self.gguf_writer.add_vision_n_wa_pattern(n_wa_pattern)
+        elif model_type == "sarashina2_vision":
+            self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.SARASHINA2VL)
+            self.gguf_writer.add_vision_spatial_merge_size(2)
         else:
             raise ValueError(f"Unknown QwenVL model type: {self.global_config['model_type']}")
         # default values below are taken from HF tranformers code
@@ -13362,6 +13373,7 @@ def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> st
         arch = text_config["architectures"][0]
     elif model_type == ModelType.MMPROJ and vision_config.get("architectures") is not None:
         arch = vision_config["architectures"][0]
+
     if arch is None:
         raise ValueError("Failed to detect model architecture")
     return arch
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index c5297a2f4..2b70f2597 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -4138,6 +4138,7 @@ class VisionProjectorType:
     YOUTUVL = "youtuvl"
     NEMOTRON_V2_VL = "nemotron_v2_vl"
     HUNYUANOCR     = "hunyuanocr"
+    SARASHINA2VL = "sarashina2vl_merger"
 
 
 # Items here are (block size, type size)
diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h
index 17cb703f7..5d0ab1fed 100644
--- a/tools/mtmd/clip-impl.h
+++ b/tools/mtmd/clip-impl.h
@@ -293,6 +293,7 @@ enum projector_type {
     PROJECTOR_TYPE_KIMIK25,
     PROJECTOR_TYPE_NEMOTRON_V2_VL,
     PROJECTOR_TYPE_HUNYUANOCR,
+    PROJECTOR_TYPE_SARASHINA2VL_MERGER,
     PROJECTOR_TYPE_UNKNOWN,
 };
 
@@ -338,6 +339,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
     { PROJECTOR_TYPE_KIMIK25,   "kimik25"},
     { PROJECTOR_TYPE_NEMOTRON_V2_VL, "nemotron_v2_vl"},
     { PROJECTOR_TYPE_HUNYUANOCR, "hunyuanocr"},
+    { PROJECTOR_TYPE_SARASHINA2VL_MERGER, "sarashina2vl_merger"}
 };
 
 static projector_type clip_projector_type_from_string(const std::string & str) {
diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp
index f0e8786b6..914acbf21 100644
--- a/tools/mtmd/clip.cpp
+++ b/tools/mtmd/clip.cpp
@@ -859,6 +859,7 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32
             } break;
         case PROJECTOR_TYPE_QWEN2VL:
         case PROJECTOR_TYPE_QWEN25VL:
+        case PROJECTOR_TYPE_SARASHINA2VL_MERGER:
             {
                 builder = std::make_unique<clip_graph_qwen2vl>(ctx, img);
             } break;
@@ -1362,6 +1363,19 @@ struct clip_model_loader {
                             LOG_WRN("%s: more info: https://github.com/ggml-org/llama.cpp/issues/16842\n\n", __func__);
                         }
                     } break;
+                case PROJECTOR_TYPE_SARASHINA2VL_MERGER:
+                    {
+                        hparams.n_merge = 2; 
+                        hparams.image_resize_algo = RESIZE_ALGO_BILINEAR;
+                        get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
+                        get_u32(KEY_WIN_ATTN_PATTERN, hparams.n_wa_pattern, false);
+                        hparams.set_limit_image_tokens(8, 4096);
+                        hparams.set_warmup_n_tokens(46*46);
+                        const int warn_min_pixels = 1024 * hparams.n_merge * hparams.n_merge * hparams.patch_size * hparams.patch_size;
+                        if (hparams.image_min_pixels < warn_min_pixels) {
+                            LOG_WRN("%s: Sarashina2VL models require sufficient image tokens\n", __func__);
+                        }
+                    } break;
                 case PROJECTOR_TYPE_STEP3VL:
                     {
                         hparams.n_merge = 4; // two stride-2 downsamplers after patching
@@ -1694,6 +1708,7 @@ struct clip_model_loader {
                     || model.proj_type == PROJECTOR_TYPE_GEMMA3
                     || model.proj_type == PROJECTOR_TYPE_IDEFICS3
                     || model.proj_type == PROJECTOR_TYPE_MINICPMV
+                    || model.proj_type == PROJECTOR_TYPE_SARASHINA2VL_MERGER
                 ) && layer.ff_up_w && layer.ff_down_w && layer.ff_down_w->ne[0] == hparams.n_embd;
             if (is_ffn_swapped) {
                 // swap up and down weights
@@ -1810,6 +1825,7 @@ struct clip_model_loader {
                 } break;
             case PROJECTOR_TYPE_QWEN2VL:
             case PROJECTOR_TYPE_QWEN25VL:
+            case PROJECTOR_TYPE_SARASHINA2VL_MERGER:
                 {
                     model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"));
                     model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias"));
@@ -2798,6 +2814,7 @@ int clip_n_output_tokens_x(const struct clip_ctx * ctx, struct clip_image_f32 *
         case PROJECTOR_TYPE_PADDLEOCR:
         case PROJECTOR_TYPE_HUNYUANOCR:
         case PROJECTOR_TYPE_YOUTUVL:
+        case PROJECTOR_TYPE_SARASHINA2VL_MERGER:
             return (img->nx / params.patch_size) / 2;
         case PROJECTOR_TYPE_STEP3VL:
             return img->nx / (params.patch_size * params.n_merge);
@@ -2817,6 +2834,7 @@ int clip_n_output_tokens_y(const struct clip_ctx * ctx, struct clip_image_f32 *
         case PROJECTOR_TYPE_GLM4V:
         case PROJECTOR_TYPE_PADDLEOCR:
         case PROJECTOR_TYPE_YOUTUVL:
+        case PROJECTOR_TYPE_SARASHINA2VL_MERGER:
             return (img->ny / params.patch_size) / 2;
         case PROJECTOR_TYPE_STEP3VL:
             return img->ny / (params.patch_size * params.n_merge);
@@ -2884,6 +2902,7 @@ int clip_n_output_tokens(const struct clip_ctx * ctx, struct clip_image_f32 * im
         case PROJECTOR_TYPE_QWEN3VL:
         case PROJECTOR_TYPE_GLM4V:
         case PROJECTOR_TYPE_YOUTUVL:
+        case PROJECTOR_TYPE_SARASHINA2VL_MERGER:
             {
                 // dynamic size (2 conv, so double patch size)
                 int x_patch = img->nx / (params.patch_size * 2);
@@ -3279,6 +3298,7 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima
             } break;
         case PROJECTOR_TYPE_QWEN25VL:
         case PROJECTOR_TYPE_YOUTUVL:
+        case PROJECTOR_TYPE_SARASHINA2VL_MERGER:
             {
                 // pw * ph = number of tokens output by ViT after apply patch merger
                 // ipw * ipw = number of vision token been processed inside ViT
@@ -3656,6 +3676,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
         case PROJECTOR_TYPE_QWEN25VL:
         case PROJECTOR_TYPE_JANUS_PRO:
         case PROJECTOR_TYPE_YOUTUVL:
+        case PROJECTOR_TYPE_SARASHINA2VL_MERGER:
             return ctx->model.mm_1_b->ne[0];
         case PROJECTOR_TYPE_QWEN3VL:
             // main path + deepstack paths
diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp
index d0a0a4865..8d1d70367 100644
--- a/tools/mtmd/mtmd.cpp
+++ b/tools/mtmd/mtmd.cpp
@@ -280,6 +280,7 @@ struct mtmd_context {
             case PROJECTOR_TYPE_QWEN2VL:
             case PROJECTOR_TYPE_QWEN25VL:
             case PROJECTOR_TYPE_QWEN3VL:
+            case PROJECTOR_TYPE_SARASHINA2VL_MERGER:
                 {
                     // <|vision_start|> ... (image embeddings) ... <|vision_end|>
                     img_beg = "<|vision_start|>";
diff --git a/tools/mtmd/tests.sh b/tools/mtmd/tests.sh
index 5da48d61b..bc33a715d 100755
--- a/tools/mtmd/tests.sh
+++ b/tools/mtmd/tests.sh
@@ -92,6 +92,7 @@ add_test_vision "ggml-org/DeepSeek-OCR-GGUF:Q8_0" -p "Free OCR." --chat-template
 add_test_vision "ggml-org/dots.ocr-GGUF:Q8_0" -p "OCR"
 add_test_vision "ggml-org/HunyuanOCR-GGUF:Q8_0" -p "OCR"
 add_test_vision "ggml-org/gemma-4-E2B-it-GGUF:Q8_0" --jinja
+add_test_vision "samuraieng/sarashina2.2-vision-3b-gguf:Q4_K_M"
 
 add_test_audio  "ggml-org/ultravox-v0_5-llama-3_2-1b-GGUF:Q8_0"
 add_test_audio  "ggml-org/Qwen2.5-Omni-3B-GGUF:Q4_K_M"

[EOF]