diff --git a/src/SharpInference.Cli/RunCommand.cs b/src/SharpInference.Cli/RunCommand.cs index 8de1cf1..9e34799 100644 --- a/src/SharpInference.Cli/RunCommand.cs +++ b/src/SharpInference.Cli/RunCommand.cs @@ -1,8 +1,11 @@ using System.ComponentModel; using System.Diagnostics; +using System.Text; +using System.Text.Json; using Spectre.Console; using Spectre.Console.Cli; using SharpInference.Core; +using SharpInference.Core.Grammar; using SharpInference.Cpu; using SharpInference.Cuda; using SharpInference.Engine; @@ -187,6 +190,16 @@ public sealed class Settings : CommandSettings [DefaultValue(0)] public int MaxThinkingTokens { get; init; } + // ── Tool calling ── + [CommandOption("--tools ")] + [Description("Path to a JSON file of OpenAI-format tool definitions ([{type:\"function\", function:{name, description, parameters}}, ...], or a {\"tools\":[...]} wrapper). Advertised to the model via its chat template; on a single-prompt (-p) run the parsed tool calls are printed after generation.")] + public string? ToolsPath { get; init; } + + [CommandOption("--tool-grammar")] + [Description("Constrain tool-call arguments to the --tools JSON Schemas (issue #374): required keys can't be dropped, only declared keys/enum values appear, value shapes match the declared type. Needs --tools and a model family with constraint support (Gemma 4 today). Default off → byte-identical to unconstrained decoding.")] + [DefaultValue(false)] + public bool ToolGrammar { get; init; } + // ── MoE expert-cache tuning (offloaded MoE models) ── // Good defaults are automatic: frequency-aware SLRU eviction, VRAM-sized cache, // and next-layer predictive prefetch are all ON without any flag. These knobs only @@ -803,6 +816,58 @@ protected override int Execute(CommandContext context, Settings settings, Cancel AnsiConsole.MarkupLine($"[dim]Model loaded in {sw.Elapsed.TotalSeconds:F1}s — " + $"{hp.NumLayers}L, {hp.EmbeddingDim}d, headDim={hp.HeadDim}, {hp.VocabSize} vocab, ctx={hp.ContextLength}[/]"); + // ── Tool calling (optional) ─────────────────────────────────────────────── + // --tools advertises OpenAI-format tool definitions to the model via its chat template; + // --tool-grammar additionally constrains the argument bytes to the supplied JSON Schemas + // (issue #374) for families with constraint support. Both are single-prompt features. + List? toolSchemas = null; + ITokenConstraint? toolConstraint = null; + int[] toolBoundaryStops = []; + s_tools = null; // reset: this run advertises tools only if --tools is given (no leak across in-process runs) + if (settings.ToolsPath is { Length: > 0 } toolsPath) + { + if (!File.Exists(toolsPath)) + { + AnsiConsole.MarkupLine($"[red]Error:[/] tools file not found: {Markup.Escape(toolsPath)}"); + return 1; + } + try + { + (s_tools, toolSchemas) = LoadTools(toolsPath); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException + or System.Security.SecurityException or NotSupportedException + or JsonException or FormatException) + { + AnsiConsole.MarkupLine($"[red]Error:[/] could not parse --tools file: {Markup.Escape(ex.Message)}"); + return 1; + } + AnsiConsole.MarkupLine($"[dim]Loaded {toolSchemas.Count} tool(s) from {Markup.Escape(Path.GetFileName(toolsPath))}.[/]"); + + var adapter = ToolCallAdapterRegistry.Get(s_arch); + + // Halt right after the tool call(s) instead of running into a hallucinated trailing + // turn (issue #304): add the adapter's tool-boundary markers (Gemma 4: <|tool_response>) + // to the stop set, resolved against the vocab. + toolBoundaryStops = adapter.ToolBoundaryStopMarkers + .Select(m => tokenizer.SpecialTokens.TryGetValue(m, out int id) ? id : -1) + .Where(id => id > 0) + .Distinct() + .ToArray(); + + if (settings.ToolGrammar) + { + toolConstraint = adapter.BuildArgumentConstraint(toolSchemas, new GrammarVocabulary(tokenizer)); + AnsiConsole.MarkupLine(toolConstraint is not null + ? "[dim]Tool-call arguments are grammar-constrained (issue #374).[/]" + : $"[yellow]Warning:[/] --tool-grammar has no effect for arch '{s_arch}' (no constraint support, or no supplied tool is constrainable); arguments generate unconstrained."); + } + } + else if (settings.ToolGrammar) + { + AnsiConsole.MarkupLine("[yellow]Warning:[/] --tool-grammar requires --tools (a schema to constrain against); ignoring."); + } + var sp = new SamplingParams { Temperature = settings.Temperature, @@ -810,12 +875,15 @@ protected override int Execute(CommandContext context, Settings settings, Cancel TopP = settings.TopP, MinP = settings.MinP, MaxNewTokens = settings.NPredict, - StopTokenIds = [.. BuildStopTokenIds(tokenizer)], + StopTokenIds = toolBoundaryStops.Length > 0 + ? [.. BuildStopTokenIds(tokenizer), .. toolBoundaryStops] + : [.. BuildStopTokenIds(tokenizer)], RepetitionPenalty = settings.RepPenalty, SpecType = ParseSpecType(settings.SpecTypeStr), SpecDraftNMax = settings.SpecDraftNMax, SpecDraftNMin = settings.SpecDraftNMin, SpecDraftPMin = settings.SpecDraftPMin, + Constraint = toolConstraint, }; var rng = settings.Seed >= 0 ? new Random(settings.Seed) : new Random(); @@ -824,7 +892,16 @@ protected override int Execute(CommandContext context, Settings settings, Cancel // packed k-token verify via CudaForwardPass.BatchVerify). Vulkan and the partial- // offload hybrids fall back to normal generation: without a batched verify, // speculation costs k sequential target forwards per step and is never a win. - if (settings.DraftModelPath is not null || settings.DraftLookup) + bool specRequested = settings.DraftModelPath is not null || settings.DraftLookup; + if (specRequested && toolConstraint is not null) + { + // The constraint masks one token at a time against the running argument state; a + // multi-token speculative verify can't honor it. Drop speculation so the constraint + // actually applies (the standard decode path below reads sp.Constraint). + AnsiConsole.MarkupLine("[yellow]Warning:[/] --tool-grammar is not applied with speculative decoding (--draft-model/--draft-lookup); generating without speculation so the argument constraint takes effect."); + specRequested = false; + } + if (specRequested) { bool cudaSpecTarget = gpuFwd is CudaForwardPass { SupportsBatchVerify: true }; // Vulkan full-offload of a dense Q4_K/Q6_K model exposes the same weight-amortized @@ -1189,6 +1266,12 @@ private static int RunSinglePrompt(Settings s, return 1; } + // When --tools is active, capture the raw token stream so the calls can be parsed and shown + // afterward, and route through the standard decode loop: the MTP fast path can't honor the + // argument-grammar constraint (sp.Constraint) and has no capture hook. + List? toolCapture = s.ToolsPath is { Length: > 0 } ? new List() : null; + if (toolCapture is not null) useMtp = false; + sw.Restart(); int generated, totalDecoded; float? acceptanceRate = null; @@ -1201,7 +1284,7 @@ private static int RunSinglePrompt(Settings s, else { (generated, totalDecoded) = - DecodeLoop(forward, logits, tokens.Count, tok, sp, rng, s.VerbosePrompt, s.HideThinking, s.MaxThinkingTokens); + DecodeLoop(forward, logits, tokens.Count, tok, sp, rng, s.VerbosePrompt, s.HideThinking, s.MaxThinkingTokens, toolCapture); } var decodeMs = sw.Elapsed.TotalMilliseconds; @@ -1211,6 +1294,9 @@ private static int RunSinglePrompt(Settings s, (totalDecoded > generated ? $" ({generated} visible, {totalDecoded - generated} thinking)" : "") + (acceptanceRate is float ar ? $" | MTP accept: {ar:P0} ({mtpAccepted}/{mtpEmitted})" : "") + "[/]"); + + if (toolCapture is not null) + PrintToolCalls(tok, toolCapture); return 0; } @@ -1525,7 +1611,8 @@ private static (int generated, int totalDecoded) DecodeLoop( Random rng, bool verbosePromptLogging = false, bool hideThinking = false, - int maxThinkingTokens = 0) + int maxThinkingTokens = 0, + List? captureTokens = null) { var logits = initialLogits; int generated = 0; @@ -1534,6 +1621,9 @@ private static (int generated, int totalDecoded) DecodeLoop( int thinkingTokenCount = 0; var recentTokens = new List(64); var streamDec = new Utf8StreamDecoder(); + // Tool-call grammar constraint (issue #374): start each response from the watching state so + // a reused instance serves this generation fresh. No-op when no constraint is attached. + sp.Constraint?.Reset(); for (int i = 0; i < sp.MaxNewTokens; i++) { var spWithHistory = sp.RepetitionPenalty != 1.0f && recentTokens.Count > 0 @@ -1548,13 +1638,20 @@ private static (int generated, int totalDecoded) DecodeLoop( } else { - next = sp.Temperature <= 0 ? Sampler.Greedy(logits) : Sampler.Sample(logits, spWithHistory, rng); + // While the grammar is restricting the vocabulary, sample from the masked logits so + // only a grammar-legal token can be chosen; otherwise sample exactly as before. + var sampleLogits = sp.Constraint is { IsConstraining: true } ctr ? ctr.Filter(logits) : logits; + next = sp.Temperature <= 0 ? Sampler.Greedy(sampleLogits) : Sampler.Sample(sampleLogits, spWithHistory, rng); } if (verbosePromptLogging) { Console.Error.WriteLine($"[DBG] tok={i} next={next}('{tok.Decode([next])}') stop={sp.StopTokenIds.Contains(next)} top5:{FormatTopLogits(logits, 5)}"); } if (sp.StopTokenIds.Contains(next)) break; + // Advance the constraint by every emitted token (so it can detect a tool call beginning + // and ending) and capture the raw stream for post-generation tool-call parsing. + sp.Constraint?.Accept(next); + captureTokens?.Add(next); // Counter resets on each open (in case the model opens multiple blocks) // and counts every token emitted while inThinking is true on entry, including // the boundary tokens themselves — that keeps the budget predictable: N tokens @@ -1684,6 +1781,10 @@ static bool SortsBefore(float a, float b) private static int s_thinkTokenId = -1; // token for any model using the / special-token convention private static int s_endThinkTokenId = -1; // token for any model using the / special-token convention private static JinjaChatTemplate? s_jinja; // parsed from GGUF tokenizer.chat_template + // Tool definitions loaded from --tools (template-facing object graph: a list of + // {type, function:{…}} dicts), rendered into the chat template's `tools` variable. Null + // unless --tools was given, in which case the prompt advertises no tools (legacy behaviour). + private static IReadOnlyList? s_tools; /// /// Builds the stop token ID list. Delegates to — @@ -1695,6 +1796,103 @@ static bool SortsBefore(float a, float b) /// private static IReadOnlyList BuildStopTokenIds(GgufTokenizer tokenizer) => tokenizer.EogTokenIds; + // ── Tool calling (--tools / --tool-grammar) ──────────────────────────────── + + /// + /// Loads OpenAI-format tool definitions from a JSON file: a bare array of + /// {type:"function", function:{name, description, parameters}} objects, or a + /// { "tools": [ … ] } wrapper. Returns the template-facing object graph (passed to the + /// chat template's tools variable) and the parsed s (used to build + /// the argument-grammar constraint). Eagerly materialised so the backing + /// can be disposed here. Throws / on a + /// malformed file. + /// + private static (IReadOnlyList Tools, List Schemas) LoadTools(string path) + { + using var doc = JsonDocument.Parse(File.ReadAllText(path)); + var root = doc.RootElement; + JsonElement arr = root.ValueKind == JsonValueKind.Object && root.TryGetProperty("tools", out var t) ? t : root; + if (arr.ValueKind != JsonValueKind.Array) + throw new FormatException("expected a JSON array of tool definitions (or a { \"tools\": [ … ] } wrapper)."); + + var tools = new List(); + var schemas = new List(); + foreach (var el in arr.EnumerateArray()) + { + tools.Add(JsonToObject(el)); // detached object graph for the template + if (el.TryGetProperty("function", out var fn) && fn.ValueKind == JsonValueKind.Object + && fn.TryGetProperty("name", out var n) && n.ValueKind == JsonValueKind.String + && n.GetString() is { Length: > 0 } name) + { + JsonElement? parameters = fn.TryGetProperty("parameters", out var p) ? p : null; + schemas.Add(ToolSchema.FromOpenAiFunction(name, parameters)); + } + } + if (schemas.Count == 0) + throw new FormatException("no tool with a function.name was found."); + return (tools, schemas); + } + + /// Recursively converts a into a detached object graph + /// ( / / scalar) the Jinja engine + /// consumes — reflection-free, so it survives NativeAOT trimming. + private static object? JsonToObject(JsonElement el) => el.ValueKind switch + { + JsonValueKind.Object => el.EnumerateObject().ToDictionary(p => p.Name, p => JsonToObject(p.Value), StringComparer.Ordinal), + JsonValueKind.Array => el.EnumerateArray().Select(JsonToObject).ToList(), + JsonValueKind.String => el.GetString(), + JsonValueKind.Number => el.TryGetInt64(out long l) ? l : el.TryGetDouble(out double d) ? d : (object?)el.GetRawText(), + JsonValueKind.True => true, + JsonValueKind.False => false, + _ => null, + }; + + /// Parses the captured raw output with the model's tool-call adapter and prints the + /// structured calls (name + JSON arguments). No-op when the model emitted no tool call. + private static void PrintToolCalls(GgufTokenizer tok, List tokens) + { + if (tokens.Count == 0) return; + var (_, calls) = ToolCallAdapterRegistry.Get(s_arch).Parse(tok.Decode(tokens)); + if (calls.Count == 0) return; + AnsiConsole.MarkupLine($"\n[green]Parsed {calls.Count} tool call(s):[/]"); + foreach (var c in calls) + AnsiConsole.MarkupLine($" [bold]{Markup.Escape(c.Name)}[/]({Markup.Escape(RenderArgs(c.Arguments))})"); + } + + /// Renders a parsed tool call's argument object as compact JSON via a manual + /// walk (no reflection-based serialization → AOT-safe). + private static string RenderArgs(IReadOnlyDictionary args) + { + var buffer = new System.Buffers.ArrayBufferWriter(); + using (var w = new Utf8JsonWriter(buffer)) + WriteJsonValue(w, args); + return Encoding.UTF8.GetString(buffer.WrittenSpan); + } + + private static void WriteJsonValue(Utf8JsonWriter w, object? value) + { + switch (value) + { + case null: w.WriteNullValue(); break; + case bool b: w.WriteBooleanValue(b); break; + case string s: w.WriteStringValue(s); break; + case long l: w.WriteNumberValue(l); break; + case int i: w.WriteNumberValue(i); break; + case double d: w.WriteNumberValue(d); break; + case IReadOnlyDictionary map: + w.WriteStartObject(); + foreach (var kv in map) { w.WritePropertyName(kv.Key); WriteJsonValue(w, kv.Value); } + w.WriteEndObject(); + break; + case System.Collections.IEnumerable seq: + w.WriteStartArray(); + foreach (var item in seq) WriteJsonValue(w, item); + w.WriteEndArray(); + break; + default: w.WriteStringValue(value.ToString() ?? ""); break; + } + } + /// /// Resolves whether reasoning ("thinking") should be OFF for this run. /// Precedence: --no-thinking forces it off and wins if both flags are passed; @@ -1753,7 +1951,7 @@ private static string FormatPrompt(string userMessage, string? systemPrompt, boo { ["messages"] = messages, ["add_generation_prompt"] = true, - ["tools"] = null, + ["tools"] = (object?)s_tools, ["enable_thinking"] = enableThinking, }); } diff --git a/src/SharpInference.Server/Endpoints/ToolGrammarHelper.cs b/src/SharpInference.Server/Endpoints/ToolGrammarHelper.cs index f95333f..c513ec0 100644 --- a/src/SharpInference.Server/Endpoints/ToolGrammarHelper.cs +++ b/src/SharpInference.Server/Endpoints/ToolGrammarHelper.cs @@ -12,16 +12,13 @@ namespace SharpInference.Server.Endpoints; internal static class ToolGrammarHelper { /// - /// Whether grammar-constrained tool decoding is enabled — via the - /// option or the - /// SHARPI_TOOL_GRAMMAR=1 environment variable (mirrors the SHARPI_* env pattern). + /// Whether grammar-constrained tool decoding is enabled — driven solely by the + /// option, the single source of truth. + /// The runnable host still honours SHARPI_TOOL_GRAMMAR=1 by mapping that environment + /// variable onto the option at startup (mirrors the SHARPI_* env pattern), so the library layer + /// doesn't re-read the environment here. /// - public static bool Enabled(SharpInferenceServerOptions opts) - { - if (opts.ToolGrammar) return true; - var env = Environment.GetEnvironmentVariable("SHARPI_TOOL_GRAMMAR"); - return env is "1" || string.Equals(env, "true", StringComparison.OrdinalIgnoreCase); - } + public static bool Enabled(SharpInferenceServerOptions opts) => opts.ToolGrammar; /// Parses (name, JSON-Schema) pairs into s, skipping nameless entries. public static List ToSchemas(IEnumerable<(string? Name, JsonElement? Parameters)> tools)