feat(GFX1100-TG200): T25 keep ssm_out as Q5_K with runtime input permutation

CMakeLists.txt

@@ -1735,7 +1735,8 @@ if(VLLM_CPP_HIP)
     src/vt/rocm/rocm_mla_fused_norm_rope.hip
     src/vt/rocm/rocm_mla_ops.hip
     src/vt/rocm/rocm_skinny_gemm.hip
-    src/vt/rocm/rocm_ops.hip)
+    src/vt/rocm/rocm_ops.hip
+    src/vt/rocm/rocm_quant_dot.hip)
   if(VLLM_CPP_HIP_ARCHITECTURES)
     set_source_files_properties(
       src/vt/rocm/rocm_backend.hip
@@ -1762,6 +1763,7 @@ if(VLLM_CPP_HIP)
       src/vt/rocm/rocm_mla_ops.hip
       src/vt/rocm/rocm_skinny_gemm.hip
       src/vt/rocm/rocm_ops.hip
+      src/vt/rocm/rocm_quant_dot.hip
       PROPERTIES HIP_ARCHITECTURES "${VLLM_CPP_HIP_ARCHITECTURES}")
   endif()
   # Prefer the absolute path inside ${ROCM_PATH}/lib, fall back to the bare name,

include/vllm/model_executor/models/qwen3_5_weights.h

@@ -771,6 +771,7 @@ struct GdnLayerWeights {
   OwnedTensor dt_bias;        // f32  [Hv]
   OwnedTensor norm_weight;    // bf16 [Dv]           (RMSNormGated)
   OwnedTensor out_proj;       // bf16 [value_dim, H] (FP8 dequant + T)
+  bool out_proj_tiled = false;  // T25: weight kept in tiled Q5_K order; permute input at runtime
 
   // MODEL-FP8-BLOCK-WEIGHT (#1189 M3): block-wise FP8 GDN projections. The
   // target checkpoint lists the GDN small tensors under

include/vt/ops.h

@@ -170,6 +170,7 @@ enum class OpId : uint8_t {
   kCastBf16,
   kCastF32,
   kCastF16,
+  kPermuteVHeads,
   kMulColVecF32,
   kAttnGateSplit,
   kSigmoidGateBf16,
@@ -2237,6 +2238,8 @@ using MoeRelu2Fn = void (*)(Queue&, Tensor&, const Tensor&);
 // loops so the decode step can run entirely on-device (CUDA-graph capture).
 // All math in f32; dims are inferred from the tensor shapes (no args structs).
 using CastBf16Fn = void (*)(Queue&, Tensor&, const Tensor&);
+using PermuteVHeadsFn = void (*)(Queue&, Tensor&, const Tensor&, int64_t, int64_t,
+                                  int64_t, int64_t);
 using CastF32Fn = void (*)(Queue&, Tensor&, const Tensor&);
 using CastF16Fn = void (*)(Queue&, Tensor&, const Tensor&);
 using MulColVecF32Fn = void (*)(Queue&, Tensor&, const Tensor&);
@@ -5525,6 +5528,14 @@ void ApplyAllowedTokenIds(Queue& q, Tensor& logits, const Tensor& mask);
 // f32 -> bf16 activation-dtype cast used before feeding a bf16-consuming op.
 void CastBf16(Queue& q, Tensor& out, const Tensor& in);
 
+// T25: Permute V-heads from grouped (k*rpk+r) to tiled (r*num_k+k) order.
+// out[T, value_dim] = in[T, value_dim] with the last dim permuted:
+//   out[t*dv + h] = in[g*dv + h]  where  t = r*num_k + k,  g = k*rpk + r
+// Used before the K-quant GEMV when ssm_out is kept as Q5_K in tiled order.
+// value_dim = num_k * rpk * dv.  T, in, out are bf16.
+void PermuteVHeads(Queue& q, Tensor& out, const Tensor& in,
+                   int64_t T, int64_t num_k, int64_t rpk, int64_t dv);
+
 // out[i] = f32(in[i]); out f32, in bf16, same element count. The bf16 -> f32
 // upcast used to expose a bf16-only GEMM (Marlin) as an f32 result, matching the
 // value the bf16 output rounds to (mirror of the cutlass f32-output scratch cast).

src/vllm/model_executor/models/qwen3_5.cpp

@@ -1786,6 +1786,24 @@ DBuf MatmulBf16D(Dev d, const Tensor& x, const OwnedTensor& w) {
   return dout;
 }
 
+// T25: When out_proj is kept as K-quant in tiled order (out_proj_tiled), permute
+// the gated-norm output from grouped→tiled before the K-quant GEMV. The `nk`
+// flag alone is insufficient: gdn_expand_nk also sets nk=true for the bf16
+// expanded weight, but that weight has ReorderVCols applied and needs NO
+// input permutation. Only the T25 tiled Q5_K path (out_proj_tiled=true) does.
+static DBuf GdnOutProjMatmul(Dev d, const GdnLayerWeights& w,
+                              const DBuf& gated_bf16,
+                              int64_t T, int64_t Hk, int64_t Hv, int64_t Dv) {
+  if (w.out_proj_tiled) {
+    const int64_t value_dim = Hv * Dv;
+    const int64_t rpk = Hk > 0 ? Hv / Hk : 1;
+    DBuf permuted(d, DType::kBF16, {T, value_dim});
+    vt::PermuteVHeads(d.q, permuted.t(), gated_bf16.t(), T, Hk, rpk, Dv);
+    return MatmulBf16D(d, permuted.t(), w.out_proj);
+  }
+  return MatmulBf16D(d, gated_bf16.t(), w.out_proj);
+}
+
 // A tied BF16 lm_head follows torch Linear's model-dtype output, then the
 // engine exposes f32 logits to the sampler. Explicit 27B heads retain the
 // existing f32-output MatmulF32D path.
@@ -4600,7 +4618,7 @@ DBuf GdnBlock(Dev d, const GdnLayerWeights& w, const HfConfig& cfg,
              ? MatmulFp8CutlassD(d, gated_bf16.t(), w.out_proj_fp8, DType::kBF16)
          : !w.out_proj_fp4.Empty()
              ? MatmulNvfp4Bf16D(d, gated_bf16.t(), w.out_proj_fp4)
-             : MatmulBf16D(d, gated_bf16.t(), w.out_proj);  // [T,H]
+             : GdnOutProjMatmul(d, w, gated_bf16, T, Hk, Hv, Dv);  // [T,H]
 }
 
 // PERSISTENT per-step input device buffers (decode host-tax #2): the flattened
@@ -5087,7 +5105,7 @@ DBuf GdnBlockPagedMixedSpec(Dev d, const GdnLayerWeights& w, const HfConfig& cfg
              ? MatmulFp8CutlassD(d, gated_bf16.t(), w.out_proj_fp8, DType::kBF16)
          : !w.out_proj_fp4.Empty()
              ? MatmulNvfp4Bf16D(d, gated_bf16.t(), w.out_proj_fp4)
-             : MatmulBf16D(d, gated_bf16.t(), w.out_proj);  // [T,H]
+             : GdnOutProjMatmul(d, w, gated_bf16, T, Hk, Hv, Dv);  // [T,H]
 }
 
 // VT_DUMP_ACT stage probe (GDN): dump named intermediates so a layer-level
@@ -5592,7 +5610,7 @@ DBuf GdnBlockPaged(Dev d, const GdnLayerWeights& w, const HfConfig& cfg,
              ? MatmulFp8CutlassD(d, gated_bf16.t(), w.out_proj_fp8, DType::kBF16)
          : !w.out_proj_fp4.Empty()
              ? MatmulNvfp4Bf16D(d, gated_bf16.t(), w.out_proj_fp4)
-             : MatmulBf16D(d, gated_bf16.t(), w.out_proj);  // [T,H]
+             : GdnOutProjMatmul(d, w, gated_bf16, T, Hk, Hv, Dv);  // [T,H]
 }
 
 // --- Dense full_attention block. qwen36-forward-notes.md §5; pinned

src/vllm/model_executor/models/qwen3_5_gguf_weights.cpp

@@ -382,6 +382,23 @@ void ReorderVRows(std::vector<T>& buf, int64_t cols, int64_t row_off,
   }
   std::memcpy(base, seg.data(), seg.size() * sizeof(T));
 }
+// Pointer-based overload for OwnedBytes (T=uint8_t, sizeof(T)=1).
+void ReorderVRows(uint8_t* buf, int64_t cols, int64_t row_off,
+                  int64_t num_k, int64_t num_v_per_k, int64_t head_rows) {
+  const int64_t num_v = num_k * num_v_per_k;
+  const int64_t head_stride = head_rows * cols;
+  std::vector<uint8_t> seg(static_cast<size_t>(num_v) * head_stride);
+  uint8_t* base = buf + row_off * cols;
+  for (int64_t k = 0; k < num_k; ++k) {
+    for (int64_t r = 0; r < num_v_per_k; ++r) {
+      const int64_t g = k * num_v_per_k + r;
+      const int64_t t = r * num_k + k;
+      std::memcpy(seg.data() + g * head_stride, base + t * head_stride,
+                  static_cast<size_t>(head_stride));
+    }
+  }
+  std::memcpy(base, seg.data(), seg.size());
+}
 
 // Reorder the full column range [0, cols) of a [rows, cols] row-major buffer
 // (cols = num_v * head_cols) from GGUF tiled to HF grouped order (out_proj).
@@ -1072,14 +1089,38 @@ GdnLayerWeights LoadGdnGguf(const GgufFile& g, int64_t il, const HfConfig& c,
   const int64_t key_dim = num_k * c.linear_key_head_dim;
   const bool reorder = num_v != num_k && num_k > 0 && (num_v % num_k) == 0;
   const int64_t rpk = num_k > 0 ? num_v / num_k : 1;  // num_v_per_k
-  // When the V-head reorder is active these projections are LAYOUT-rewritten
-  // at load, so they are kTransformedWeight and can never keep their blocks;
-  // without it they are ordinary verbatim GEMM weights. (out_proj's reorder
-  // permutes COLUMNS, which live inside a block, so it is unconditionally
-  // block-unsafe when active — same rule, stated per tensor below.)
+  // When the V-head reorder is active, the projections are LAYOUT-rewritten at
+  // load. For COLUMN-permuted tensors (out_proj/ssm_out) the reorder cuts across
+  // quantization block boundaries, so they are kTransformedWeight and must
+  // expand to bf16. For ROW-permuted tensors (in_proj_qkv, in_proj_z) the
+  // reorder only changes row order — quantization blocks are along the K
+  // (column) dimension and are self-contained per row — so the blocks can be
+  // kept and the permutation applied to the block rows at load time (T21).
+  // Without reorder they are ordinary verbatim GEMM weights. Column-permuted
+  // tensors (out_proj/ssm_out) stay kTransformedWeight and expand to bf16.
+  // T21 env gate: VT_GDN_ROWPERM_KEEP_QUANT=0 forces the row-permuted tensors
+  // back to kTransformedWeight (bf16 expansion) for A/B isolation.
+  const char* rpkq = std::getenv("VT_GDN_ROWPERM_KEEP_QUANT");
+  const bool rowperm_keep =
+      rpkq == nullptr ||
+      !(std::strcmp(rpkq, "0") == 0 || std::strcmp(rpkq, "false") == 0 ||
+        std::strcmp(rpkq, "off") == 0);
   const GgufTensorRole proj_role = reorder
                                        ? GgufTensorRole::kTransformedWeight
                                        : GgufTensorRole::kMatmulWeight;
+  const GgufTensorRole rowperm_role =
+      (reorder && rowperm_keep) ? GgufTensorRole::kMatmulWeight : proj_role;
+  // T25: keep the COLUMN-permuted tensor (ssm_out/out_proj) as K-quant in tiled
+  // order (no ReorderVCols) and permute the GEMV input at runtime instead. The
+  // column reorder cuts across Q5_K block boundaries, so the weight cannot be
+  // permuted in place. But keeping the tiled-order weight and permuting the
+  // 4096-element activation gather before the K-quant GEMV saves ~4x weight
+  // bandwidth (Q5_K ~5 MB vs bf16 20 MB per call).
+  const char* cpkq = std::getenv("VT_GDN_COLPERM_KEEP_QUANT");
+  const bool colperm_keep =
+      cpkq != nullptr && cpkq[0] == '1' && cpkq[1] == '\0';
+  const GgufTensorRole colperm_role =
+      (reorder && colperm_keep) ? GgufTensorRole::kMatmulWeight : proj_role;
   // GdnLayerWeights carries an Nvfp4Weight ONLY for out_proj, and even that is
   // unreachable on the 27B because the V-column reorder makes ssm_out
   // kTransformedWeight. The in_proj family has no fp4 field at all. So the GDN
@@ -1092,11 +1133,27 @@ GdnLayerWeights LoadGdnGguf(const GgufFile& g, int64_t il, const HfConfig& c,
   GdnLayerWeights gdn;
 
   // in_proj_qkv <- attn_qkv [conv_dim, H]; only the trailing V rows reorder.
+  // T21: ReorderVRows is a row permutation (block-safe for K-quant). Route as
+  // kMatmulWeight to allow keep-quant, then permute the block rows in place.
+  // Saves ~661 MB/tok of bf16 read amplification (24 Q5_K tensors × 2.9x).
+  // The forward pass already dispatches quantized nk=true weights through
+  // vt::MatmulBT → matmul_bt_quant, so no forward-pass change is needed.
   {
     const std::string nm = Blk(il, "attn_qkv.weight");
-    const GgufResidency r = pol.Route(g.Get(nm), proj_role);
-    if (r != GgufResidency::kExpandBf16) {
-      const GgufTensorInfo& ti = g.Get(nm);
+    const GgufTensorInfo& ti = g.Get(nm);
+    const GgufResidency r = pol.Route(ti, rowperm_role);
+    if (r == GgufResidency::kKeepQuant) {
+      // Force a copy (not mmap) so the block rows can be permuted in place.
+      OwnedTensor qk = OwnGgufQuantBlocks(ti, ti.shape[0], ti.shape[1], 0,
+                                          /*mmap_src=*/nullptr);
+      if (reorder) {
+        const int64_t row_bytes = static_cast<int64_t>(qk.bytes.size()) /
+                                  ti.shape[0];
+        ReorderVRows(qk.bytes.data(), row_bytes, /*row_off=*/2 * key_dim,
+                     num_k, rpk, dv);
+      }
+      gdn.in_proj_qkv = std::move(qk);
+    } else if (r != GgufResidency::kExpandBf16) {
       gdn.in_proj_qkv =
           OwnGgufKeptSlice(g, pol, ti, r, ti.shape[0], ti.shape[1], 0);
     } else {
@@ -1109,11 +1166,22 @@ GdnLayerWeights LoadGdnGguf(const GgufFile& g, int64_t il, const HfConfig& c,
     }
   }
   // in_proj_z <- attn_gate [value_dim, H]; all rows are V.
+  // T21: Same row-permutation keep-quant path as in_proj_qkv above.
+  // Saves ~360 MB/tok of bf16 read amplification (24 Q4_K tensors × 2.9x).
   {
     const std::string nm = Blk(il, "attn_gate.weight");
-    const GgufResidency r = pol.Route(g.Get(nm), proj_role);
-    if (r != GgufResidency::kExpandBf16) {
-      const GgufTensorInfo& ti = g.Get(nm);
+    const GgufTensorInfo& ti = g.Get(nm);
+    const GgufResidency r = pol.Route(ti, rowperm_role);
+    if (r == GgufResidency::kKeepQuant) {
+      OwnedTensor qk = OwnGgufQuantBlocks(ti, ti.shape[0], ti.shape[1], 0,
+                                          /*mmap_src=*/nullptr);
+      if (reorder) {
+        const int64_t row_bytes = static_cast<int64_t>(qk.bytes.size()) /
+                                  ti.shape[0];
+        ReorderVRows(qk.bytes.data(), row_bytes, 0, num_k, rpk, dv);
+      }
+      gdn.in_proj_z = std::move(qk);
+    } else if (r != GgufResidency::kExpandBf16) {
       gdn.in_proj_z =
           OwnGgufKeptSlice(g, pol, ti, r, ti.shape[0], ti.shape[1], 0);
     } else {
@@ -1158,11 +1226,22 @@ GdnLayerWeights LoadGdnGguf(const GgufFile& g, int64_t il, const HfConfig& c,
   }
   // out_proj <- ssm_out [H, value_dim]; reorder V columns, then transpose.
   // The COLUMN reorder cuts across block boundaries, so when it is active this
-  // tensor is kTransformedWeight and must expand.
+  // tensor is kTransformedWeight and must expand — UNLESS T25
+  // (VT_GDN_COLPERM_KEEP_QUANT=1) keeps the tiled-order Q5_K weight and
+  // permutes the GEMV input at runtime instead.
   {
     const std::string nm = Blk(il, "ssm_out.weight");
-    const GgufResidency r = pol.Route(g.Get(nm), proj_role);
-    if (r != GgufResidency::kExpandBf16) {
+    const GgufResidency r = pol.Route(g.Get(nm), colperm_role);
+    if (r == GgufResidency::kKeepQuant && colperm_keep) {
+      // T25: keep Q5_K in tiled order (no ReorderVCols). The forward pass
+      // permutes the 4096-element activation from grouped→tiled before the
+      // K-quant GEMV, saving ~4x weight bandwidth.
+      OwnedTensor qk =
+          OwnGgufQuantBlocks(g.Get(nm), g.Get(nm).shape[0], g.Get(nm).shape[1],
+                             0, /*mmap_src=*/nullptr);
+      gdn.out_proj = std::move(qk);
+      gdn.out_proj_tiled = true;
+    } else if (r != GgufResidency::kExpandBf16) {
       const GgufTensorInfo& ti = g.Get(nm);
       gdn.out_proj =
           OwnGgufKeptSlice(g, pol, ti, r, ti.shape[0], ti.shape[1], 0);

src/vt/cpu/cpu_ops.cpp

@@ -3937,17 +3937,31 @@ void CastF32Kernel(Queue&, Tensor& out, const Tensor& in) {
   });
 }
 
-// out[i] = F32ToF16(in[i]); out f16, in f32 or bf16, same element count.
-// QUANT-EXL3 W1a (#2181). LoadF32 reads either source width as f32 and StoreF32
-// rounds once to the f16 destination (cpu_ops.cpp:44-51), so the bf16 source
-// path is "widen exactly, then round once" rather than a reinterpretation.
 void CastF16Kernel(Queue&, Tensor& out, const Tensor& in) {
   const int64_t n = out.Numel();
   ForRows(n, [&](int64_t r0, int64_t r1) {
     for (int64_t i = r0; i < r1; ++i) StoreF32(out, i, LoadF32(in, i));
   });
 }
 
+// T25: Permute V-heads from grouped (k*rpk+r) to tiled (r*num_k+k) order.
+void PermuteVHeadsKernel(Queue&, Tensor& out, const Tensor& in,
+                         int64_t T, int64_t num_k, int64_t rpk, int64_t dv) {
+  const int64_t value_dim = num_k * rpk * dv;
+  auto* out_p = out.Ptr<uint16_t>();
+  const auto* in_p = in.Ptr<uint16_t>();
+  for (int64_t row = 0; row < T; ++row) {
+    for (int64_t t = 0; t < num_k * rpk; ++t) {
+      const int64_t r = t / num_k;
+      const int64_t k = t % num_k;
+      const int64_t g = k * rpk + r;
+      for (int64_t h = 0; h < dv; ++h)
+        out_p[row * value_dim + t * dv + h] =
+            in_p[row * value_dim + g * dv + h];
+    }
+  }
+}
+
 // x[m,n] *= col[n]; x f32 OR bf16 [M,N] (inner-contiguous rows, row stride
 // x.stride[0]), col always f32 [N]. CPU sibling of the CUDA MulColVecF32 kernel,
 // and the portable reference every other backend ports FROM — so it carries the
@@ -4369,6 +4383,9 @@ struct Registrar {
     RegisterOp(OpId::kDFlashBlockAttention, DeviceType::kCPU,
                reinterpret_cast<void*>(
                    static_cast<DFlashBlockAttentionFn>(&DFlashBlockAttentionKernel)));
+    RegisterOp(OpId::kPermuteVHeads, DeviceType::kCPU,
+               reinterpret_cast<void*>(
+                   static_cast<PermuteVHeadsFn>(&PermuteVHeadsKernel)));
     RegisterOp(OpId::kDFlashPagedBlockAttention, DeviceType::kCPU,
                reinterpret_cast<void*>(
                    static_cast<DFlashPagedBlockAttentionFn>(&DFlashPagedBlockAttentionKernel)));

src/vt/op_provider.cpp

@@ -420,6 +420,8 @@ const char* OpNameImpl(OpId op) {
       return "CastF16";
     case OpId::kCastF32:
       return "CastF32";
+    case OpId::kPermuteVHeads:
+      return "PermuteVHeads";
     case OpId::kMulColVecF32:
       return "MulColVecF32";
     case OpId::kAttnGateSplit:

src/vt/ops.cpp

@@ -5182,6 +5182,18 @@ void CastF16(Queue& q, Tensor& out, const Tensor& in) {
   reinterpret_cast<CastF16Fn>(GetOp(OpId::kCastF16, q.device.type))(q, out, in);
 }
 
+void PermuteVHeads(Queue& q, Tensor& out, const Tensor& in,
+                   int64_t T, int64_t num_k, int64_t rpk, int64_t dv) {
+  VT_CHECK(out.dtype == DType::kBF16 && in.dtype == DType::kBF16,
+           "permute_v_heads: both tensors must be bf16");
+  VT_CHECK(out.Numel() == in.Numel(),
+           "permute_v_heads: out/in must have the same element count");
+  VT_CHECK(out.device == q.device && in.device == q.device,
+           "permute_v_heads: device mismatch");
+  reinterpret_cast<PermuteVHeadsFn>(GetOp(OpId::kPermuteVHeads, q.device.type))(
+      q, out, in, T, num_k, rpk, dv);
+}
+
 void CastF32(Queue& q, Tensor& out, const Tensor& in) {
   VT_CHECK(out.dtype == DType::kF32, "cast_f32: out must be f32");
   VT_CHECK(in.dtype == DType::kBF16, "cast_f32: in must be bf16");