fix(llama-cpp): preserve GPU layers during option passthrough

backend/cpp/llama-cpp/CMakeLists.txt

@@ -110,4 +110,9 @@ if(LLAMA_GRPC_BUILD_TESTS)
     target_link_libraries(parent_watch_test PRIVATE Threads::Threads)
     target_compile_features(parent_watch_test PRIVATE cxx_std_17)
     add_test(NAME parent_watch_test COMMAND parent_watch_test)
+
+    add_executable(passthrough_options_test passthrough_options_test.cpp passthrough_options.h)
+    target_include_directories(passthrough_options_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_compile_features(passthrough_options_test PRIVATE cxx_std_17)
+    add_test(NAME passthrough_options_test COMMAND passthrough_options_test)
 endif()

backend/cpp/llama-cpp/grpc-server.cpp

@@ -54,6 +54,7 @@
 #include "chat-auto-parser.h"
 #include "llama_compat.h" // fork-skew switches, generated by prepare.sh
 #include "message_content.h"
+#include "passthrough_options.h"
 #include <getopt.h>
 #include <grpcpp/ext/proto_server_reflection_plugin.h>
 #include <grpcpp/grpcpp.h>
@@ -579,6 +580,8 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt
     // Raw upstream llama-server flags collected from any option entry that
     // starts with '-'. Applied once after the loop via common_params_parse.
     std::vector<std::string> extra_argv;
+    bool passthrough_main_gpu_layers = false;
+    bool passthrough_draft_gpu_layers = false;
 
     // O_DIRECT intent from the `direct_io` option. Upstream folded
     // use_mmap/use_mlock/use_direct_io into a single common_params::load_mode
@@ -1186,6 +1189,17 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt
                     flag.c_str());
             } else {
                 extra_argv.push_back(flag);
+                passthrough_main_gpu_layers =
+                    passthrough_main_gpu_layers ||
+                    flag == "-ngl" ||
+                    flag == "--gpu-layers" ||
+                    flag == "--n-gpu-layers";
+                passthrough_draft_gpu_layers =
+                    passthrough_draft_gpu_layers ||
+                    flag == "--spec-draft-ngl" ||
+                    flag == "-ngld" ||
+                    flag == "--gpu-layers-draft" ||
+                    flag == "--n-gpu-layers-draft";
                 // Preserve the whole value after the first ':' so embedded
                 // colons (e.g. host:port) survive strtok's truncation of optval.
                 auto colon = opt.find(':');
@@ -1349,6 +1363,14 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt
         // (n_parallel -> -1, use_color). Snapshot n_parallel so an unrelated
         // passthrough flag can't silently clobber LocalAI's resolved value.
         const int saved_n_parallel = params.n_parallel;
+        // Newer upstream parsers assert that these fields still contain their
+        // negative initialization sentinels. LocalAI resolves them from the
+        // model request before applying passthrough options, so stage the
+        // sentinels and restore the values unless a raw flag overrides them.
+        const auto saved_gpu_layers =
+            llama_grpc::prepare_passthrough_gpu_layers(
+                params.n_gpu_layers,
+                params.speculative.draft.n_gpu_layers);
 
         std::vector<char *> argv;
         std::string prog = "llama-server";
@@ -1372,6 +1394,12 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt
         if (params.n_parallel == -1) {
             params.n_parallel = saved_n_parallel;
         }
+        llama_grpc::restore_passthrough_gpu_layers(
+            params.n_gpu_layers,
+            params.speculative.draft.n_gpu_layers,
+            saved_gpu_layers,
+            passthrough_main_gpu_layers,
+            passthrough_draft_gpu_layers);
     }
 
 #ifndef LOCALAI_LLAMA_CPP_NO_SCORE_TASK

backend/cpp/llama-cpp/passthrough_options.h

@@ -0,0 +1,38 @@
+// SPDX-License-Identifier: MIT
+
+#pragma once
+
+namespace llama_grpc {
+
+struct passthrough_gpu_layers_state {
+    int main;
+    int draft;
+};
+
+inline passthrough_gpu_layers_state prepare_passthrough_gpu_layers(
+    int & main_gpu_layers,
+    int & draft_gpu_layers) {
+    const passthrough_gpu_layers_state saved{
+        main_gpu_layers,
+        draft_gpu_layers,
+    };
+    main_gpu_layers = -1;
+    draft_gpu_layers = -1;
+    return saved;
+}
+
+inline void restore_passthrough_gpu_layers(
+    int & main_gpu_layers,
+    int & draft_gpu_layers,
+    passthrough_gpu_layers_state saved,
+    bool main_overridden = false,
+    bool draft_overridden = false) {
+    if (!main_overridden && main_gpu_layers == -1) {
+        main_gpu_layers = saved.main;
+    }
+    if (!draft_overridden && draft_gpu_layers == -1) {
+        draft_gpu_layers = saved.draft;
+    }
+}
+
+} // namespace llama_grpc

backend/cpp/llama-cpp/passthrough_options_test.cpp

@@ -0,0 +1,68 @@
+// SPDX-License-Identifier: MIT
+
+#include <cstdio>
+
+#include "passthrough_options.h"
+
+static int failures = 0;
+
+static void check(bool ok, const char * name) {
+    if (!ok) {
+        ++failures;
+        std::fprintf(stderr, "FAIL: %s\n", name);
+    }
+}
+
+static void test_stages_resolved_gpu_layers_for_upstream_parser() {
+    int main_gpu_layers = 99;
+    int draft_gpu_layers = 12;
+
+    const auto saved = llama_grpc::prepare_passthrough_gpu_layers(
+        main_gpu_layers, draft_gpu_layers);
+
+    check(main_gpu_layers == -1, "main GPU layers use parser sentinel");
+    check(draft_gpu_layers == -1, "draft GPU layers use parser sentinel");
+
+    llama_grpc::restore_passthrough_gpu_layers(
+        main_gpu_layers, draft_gpu_layers, saved);
+
+    check(main_gpu_layers == 99, "main GPU layers restored");
+    check(draft_gpu_layers == 12, "draft GPU layers restored");
+}
+
+static void test_keeps_explicit_passthrough_overrides() {
+    int main_gpu_layers = 99;
+    int draft_gpu_layers = 12;
+
+    const auto saved = llama_grpc::prepare_passthrough_gpu_layers(
+        main_gpu_layers, draft_gpu_layers);
+
+    main_gpu_layers = 4;
+    draft_gpu_layers = 2;
+    llama_grpc::restore_passthrough_gpu_layers(
+        main_gpu_layers, draft_gpu_layers, saved);
+
+    check(main_gpu_layers == 4, "main passthrough override retained");
+    check(draft_gpu_layers == 2, "draft passthrough override retained");
+}
+
+static void test_keeps_explicit_auto_passthrough_overrides() {
+    int main_gpu_layers = 99;
+    int draft_gpu_layers = 12;
+
+    const auto saved = llama_grpc::prepare_passthrough_gpu_layers(
+        main_gpu_layers, draft_gpu_layers);
+
+    llama_grpc::restore_passthrough_gpu_layers(
+        main_gpu_layers, draft_gpu_layers, saved, true, true);
+
+    check(main_gpu_layers == -1, "main auto passthrough override retained");
+    check(draft_gpu_layers == -1, "draft auto passthrough override retained");
+}
+
+int main() {
+    test_stages_resolved_gpu_layers_for_upstream_parser();
+    test_keeps_explicit_passthrough_overrides();
+    test_keeps_explicit_auto_passthrough_overrides();
+    return failures == 0 ? 0 : 1;
+}

backend/cpp/llama-cpp/prepare.sh

@@ -2,6 +2,7 @@
 
 set -e
 
+
 ## Patches
 
 ## Apply patches from the `patches` directory. Runs under set -e so a
@@ -24,6 +25,9 @@ cp -r grpc-server.cpp llama.cpp/tools/grpc-server/
 # unit test (compiled only when -DLLAMA_GRPC_BUILD_TESTS=ON).
 cp -r message_content.h llama.cpp/tools/grpc-server/
 cp -r message_content_test.cpp llama.cpp/tools/grpc-server/
+# Generic passthrough parser staging and its standalone regression test.
+cp -r passthrough_options.h llama.cpp/tools/grpc-server/
+cp -r passthrough_options_test.cpp llama.cpp/tools/grpc-server/
 # Parent-death watcher (included by grpc-server.cpp) and its standalone unit
 # test (run via backend/cpp/run-unit-tests.sh; also buildable under ctest).
 cp -r parent_watch.h llama.cpp/tools/grpc-server/
@@ -58,4 +62,3 @@ else
     echo "add_subdirectory(grpc-server)" >> llama.cpp/tools/CMakeLists.txt
 fi
 set -e
-