[pull] master from mudler:master

.agents/ds4-backend.md

@@ -77,6 +77,56 @@ spectrum. **Metal (Darwin) only** - it is a no-op on CUDA/CPU. Enable with
 budget). Gallery entries built on this: `deepseek-v4-flash-q4-ssd` (153 GB Flash
 on a 128 GB Mac) and `deepseek-v4-pro-q2-ssd` (433 GB Pro, experimental).
 
+## CUDA architecture (do not build without one)
+
+`backend/cpp/ds4/Makefile` drives upstream's **object targets** directly
+(`$(MAKE) -C ds4 ds4.o ds4_cuda.o ...`), which bypasses upstream's own guard:
+its `cuda` target refuses to build unless `CUDA_ARCH` is set, and offers
+`cuda-spark` (sm_121, DGX Spark / GB10) and `cuda-generic` (native) instead.
+Built with no `-arch`, nvcc targets its default architecture and the kernels run
+as JIT'd PTX. On GB10 that silently corrupted every prefill batch of >=128
+tokens - the model emitted text unrelated to the prompt and never closed its
+thinking block, so `content` came back empty - and cost close to two orders of
+magnitude of prefill throughput (4.21 t/s vs 325.70 t/s, same box, same model).
+Short prompts stayed correct, which is why it went unnoticed.
+
+The Makefile therefore picks a gencode list from `CUDA_MAJOR_VERSION` (a build
+arg the backend matrix already declares, forwarded by `Dockerfile.ds4`) and
+`uname -m`, and passes it as `NVCC_ARCH_FLAGS` to the sub-make. Upstream's
+`CUDA_ARCH` accepts a single value, so it cannot express the fat binary the
+shipped images need; a command-line assignment beats its `:=`. An empty
+`CUDA_MAJOR_VERSION` falls back to upstream's `native` for local developer
+builds, and an unrecognised one is a hard error - no CI runner has a GPU, so a
+silent `native` there is exactly the failure mode this guards against.
+
+`DS4_CUDA_HAVE_MXF4` is deliberately unset: upstream defines it only for
+single-arch sm_120/sm_121 builds and guards it with a plain `#ifdef` rather than
+`__CUDA_ARCH__`, so it cannot be combined with older archs. It gates an optional
+MXFP4 indexer fast path whose `#ifndef` branch returns 0, so omitting it costs
+speed, not correctness.
+
+### Verifying a build
+
+Check which flags a configuration resolves to, without compiling anything:
+
+```
+make -C backend/cpp/ds4 BUILD_TYPE=cublas CUDA_MAJOR_VERSION=13 NATIVE=false \
+  --eval='show: ; @echo [$(DS4_ARCH_MAKEVARS)]' show
+```
+
+Do not use `make -n` for this: the recipe is `+$(MAKE) ...`, and the `+` prefix
+makes it run even under `-n`.
+
+Then exercise the failure mode itself against a built backend. It only appears
+above one prefill batch, so the ordinary `predict` spec cannot catch it:
+
+```
+BACKEND_BINARY=$(pwd)/backend/cpp/ds4/package/run.sh \
+BACKEND_TEST_MODEL_FILE=/path/to/ds4flash.gguf \
+BACKEND_TEST_CAPS=health,load,predict,long_prefill \
+go test -count=1 -timeout=30m -v ./tests/e2e-backends/...
+```
+
 ## Build matrix
 
 | Build | Where | Notes |

backend/Dockerfile.ds4

@@ -10,6 +10,7 @@ FROM ${BASE_IMAGE} AS builder
 ARG BUILD_TYPE
 ARG TARGETARCH
 ARG TARGETVARIANT
+ARG CUDA_MAJOR_VERSION
 
 ENV BUILD_TYPE=${BUILD_TYPE} \
     DEBIAN_FRONTEND=noninteractive \
@@ -35,7 +36,8 @@ RUN apt-get update && \
 COPY . /LocalAI
 
 RUN --mount=type=cache,target=/root/.ccache,id=ds4-ccache-${TARGETARCH}-${BUILD_TYPE},sharing=locked \
-    make -C /LocalAI/backend/cpp/ds4 BUILD_TYPE=${BUILD_TYPE} NATIVE=false grpc-server package
+    make -C /LocalAI/backend/cpp/ds4 BUILD_TYPE=${BUILD_TYPE} \
+        CUDA_MAJOR_VERSION=${CUDA_MAJOR_VERSION} NATIVE=false grpc-server package
 
 FROM scratch
 COPY --from=builder /LocalAI/backend/cpp/ds4/package/. ./

backend/cpp/ds4/Makefile

@@ -18,6 +18,67 @@ UNAME_S := $(shell uname -s)
 
 CMAKE_ARGS ?= -DCMAKE_BUILD_TYPE=Release
 
+# nvcc must be told the target architecture explicitly for a cublas build, and
+# this is not a tuning knob. Upstream's Makefile leaves CUDA_ARCH empty and its
+# `cuda` target REFUSES to build without one, offering `cuda-spark`
+# (CUDA_ARCH=sm_121) and `cuda-generic` (CUDA_ARCH=native) instead. We drive its
+# object targets directly, which bypasses that guard: nvcc then compiles with no
+# -arch at all, and the kernels run as JIT'd PTX for its default architecture.
+# On GB10 (sm_121) that silently produced corrupt inference output above a
+# ~128-token prefill batch and ~77x slower prefill (4.21 t/s vs 325.70 t/s,
+# measured on the same box with the same model). No CI runner has a GPU, so
+# `native` has nothing to enumerate there.
+#
+# Upstream's CUDA_ARCH takes a SINGLE value (see its sm_120/sm_121 special cases
+# and the `-arch=$(CUDA_ARCH)` fallback), so it cannot express the fat binary
+# these images need. NVCC_ARCH_FLAGS is overridden instead: a command-line
+# assignment wins over the `:=` in upstream's Makefile, and its NVCCFLAGS
+# expands whatever we pass.
+#
+# The architecture lists are copied from backend/go/vllm-cpp/Makefile rather
+# than invented, so the two CUDA images cover the same GPUs: amd64 datacenter +
+# consumer, and l4t/arm64 covering Orin (87), Thor (110) and GB10 (121a).
+#
+# -DDS4_CUDA_HAVE_MXF4=1 is deliberately NOT set. Upstream only defines it for
+# single-arch sm_120/sm_121 builds and guards the code with a plain #ifdef
+# rather than __CUDA_ARCH__, so it cannot be combined with older archs in one
+# fat binary. It gates an optional MXFP4 indexer fast path whose #ifndef branch
+# returns 0 and falls back to the generic path, so omitting it costs some speed
+# on GB10, not correctness. Revisit if upstream adds __CUDA_ARCH__ guards.
+#
+# An EMPTY CUDA_MAJOR_VERSION means a local developer build, not CI: fall back
+# to upstream's own `native` handling, which needs a GPU present but is what a
+# developer building on their own machine wants. Both variables are `?=` so an
+# explicit value on the command line always wins.
+UNAME_M := $(shell uname -m)
+CUDA_MAJOR_VERSION ?=
+ifeq ($(BUILD_TYPE),cublas)
+ifeq ($(CUDA_MAJOR_VERSION),13)
+ifeq ($(UNAME_M),aarch64)
+    DS4_NVCC_ARCH_FLAGS ?= -gencode arch=compute_87,code=sm_87 \
+                           -gencode arch=compute_90a,code=sm_90a \
+                           -gencode arch=compute_100a,code=sm_100a \
+                           -gencode arch=compute_110,code=sm_110 \
+                           -gencode arch=compute_121a,code=sm_121a
+else
+    DS4_NVCC_ARCH_FLAGS ?= -gencode arch=compute_80,code=sm_80 \
+                           -gencode arch=compute_86,code=sm_86 \
+                           -gencode arch=compute_89,code=sm_89 \
+                           -gencode arch=compute_90a,code=sm_90a \
+                           -gencode arch=compute_100a,code=sm_100a \
+                           -gencode arch=compute_103a,code=sm_103a \
+                           -gencode arch=compute_120a,code=sm_120a \
+                           -gencode arch=compute_121a,code=sm_121a
+endif
+    DS4_ARCH_MAKEVARS := NVCC_ARCH_FLAGS="$(DS4_NVCC_ARCH_FLAGS)"
+else ifeq ($(CUDA_MAJOR_VERSION),)
+    # Local build: let upstream resolve the host GPU.
+    DS4_ARCH_MAKEVARS := CUDA_ARCH=native
+else
+    $(error CUDA_MAJOR_VERSION=$(CUDA_MAJOR_VERSION) has no architecture list here (13 does). Leave it empty for a native build, or pass DS4_NVCC_ARCH_FLAGS explicitly.)
+endif
+endif
+
 # Upstream splits distributed inference, tensor-parallel transport, the SSD
 # expert cache, and layer placement into GPU-agnostic translation units. They
 # are shared by every GPU mode, so append them unconditionally below.
@@ -57,7 +118,7 @@ ds4:
 # the right per-platform compile flags (Objective-C/Metal on Darwin, nvcc on Linux+CUDA).
 ds4/ds4.o: ds4
 ifeq ($(BUILD_TYPE),cublas)
-	+$(MAKE) -C ds4 $(DS4_OBJ_TARGET)
+	+$(MAKE) -C ds4 $(DS4_ARCH_MAKEVARS) $(DS4_OBJ_TARGET)
 else ifeq ($(UNAME_S),Darwin)
 	+$(MAKE) -C ds4 ds4.o ds4_metal.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o
 else