diff --git a/src/SharpInference.Cli/RunCommand.cs b/src/SharpInference.Cli/RunCommand.cs index 9e34799..9db3150 100644 --- a/src/SharpInference.Cli/RunCommand.cs +++ b/src/SharpInference.Cli/RunCommand.cs @@ -196,7 +196,7 @@ public sealed class Settings : CommandSettings public string? ToolsPath { get; init; } [CommandOption("--tool-grammar")] - [Description("Constrain tool-call arguments to the --tools JSON Schemas (issue #374): required keys can't be dropped, only declared keys/enum values appear, value shapes match the declared type. Needs --tools and a model family with constraint support (Gemma 4 today). Default off → byte-identical to unconstrained decoding.")] + [Description("Constrain tool-call arguments to the --tools JSON Schemas (issue #374): required keys can't be dropped, only declared keys/enum values appear, value shapes match the declared type. Needs --tools and a model family with constraint support (Gemma 4, Qwen/Qwen3-Coder, Llama-3, DeepSeek). Default off → byte-identical to unconstrained decoding.")] [DefaultValue(false)] public bool ToolGrammar { get; init; } diff --git a/src/SharpInference.Core/Grammar/CompositeToolArgumentConstraint.cs b/src/SharpInference.Core/Grammar/CompositeToolArgumentConstraint.cs new file mode 100644 index 0000000..c6b912f --- /dev/null +++ b/src/SharpInference.Core/Grammar/CompositeToolArgumentConstraint.cs @@ -0,0 +1,59 @@ +namespace SharpInference.Core.Grammar; + +/// +/// A tool-argument constraint that overlays several format-specific sub-constraints and lets whichever +/// one engages drive masking. It exists because a single GGUF general.architecture can host +/// more than one tool-call wire format: a qwen3moe model is Qwen3-MoE (JSON +/// <tool_call>{…}</tool_call>) OR Qwen3-Coder (XML +/// <tool_call><function=…>…</function></tool_call>) — the two are only +/// distinguishable by the chat template, which the adapter doesn't see (issue #383). +/// +/// +/// Both sub-constraints arm on the same <tool_call> token and then diverge on the first +/// body byte ({ for JSON vs < for XML), so at most one ever leaves the watching state +/// for a given call; the other disarms itself. feeds every sub, and +/// defers to the one currently constraining. When none is constraining the input +/// logits pass through untouched, so a request that never enters a constrained region is byte-identical +/// to the default path. +/// +/// +/// The per-token cost is just the extra byte-walk in the idle subs (a few +/// bytes each); only the engaged sub allocates a mask buffer / calls . +/// +public sealed class CompositeToolArgumentConstraint : ITokenConstraint +{ + private readonly ITokenConstraint[] _inner; + + public CompositeToolArgumentConstraint(IReadOnlyList inner) + { + ArgumentNullException.ThrowIfNull(inner); + if (inner.Count == 0) throw new ArgumentException("at least one inner constraint is required", nameof(inner)); + _inner = [.. inner]; + } + + public bool IsConstraining + { + get + { + foreach (var c in _inner) if (c.IsConstraining) return true; + return false; + } + } + + public ReadOnlySpan Filter(ReadOnlySpan logits) + { + // At most one sub is ever constraining (they diverge after the shared open marker); defer to it. + foreach (var c in _inner) if (c.IsConstraining) return c.Filter(logits); + return logits; + } + + public void Accept(int token) + { + foreach (var c in _inner) c.Accept(token); + } + + public void Reset() + { + foreach (var c in _inner) c.Reset(); + } +} diff --git a/src/SharpInference.Core/Grammar/QwenCoderToolArgumentConstraint.cs b/src/SharpInference.Core/Grammar/QwenCoderToolArgumentConstraint.cs new file mode 100644 index 0000000..7713128 --- /dev/null +++ b/src/SharpInference.Core/Grammar/QwenCoderToolArgumentConstraint.cs @@ -0,0 +1,696 @@ +using System.Text; + +namespace SharpInference.Core.Grammar; + +/// +/// Constrained-decoding state machine (issue #383) for Qwen3-Coder's XML tool-call wire +/// format — the one remaining family not covered by the JSON +/// () or Gemma () +/// constraints. Qwen3-Coder emits arguments as nested tags rather than JSON: +/// +/// <tool_call> +/// <function=get_weather> +/// <parameter=location> +/// Paris +/// </parameter> +/// </function> +/// </tool_call> +/// +/// +/// +/// It watches the emitted stream until a call opens (<tool_call>) and a known tool's +/// <function=NAME> tag is seen, then constrains the function body so that only declared +/// parameters appear (as <parameter=KEY>), every required parameter appears exactly once, +/// and each value region matches its declared shape where checkable: a Coder value is bare +/// text between the open/close parameter tags, so typed-string / array / object / untyped values are +/// free content, while numbers, booleans, null, and enums are constrained. The closing +/// </function> ends constraint and the machine returns to watching, so the trailing +/// </tool_call> envelope and any later text are unconstrained. +/// +/// +/// +/// Like the JSON sibling, the structural skeleton (<function=, <parameter=, +/// </parameter>, </function>) is matched entirely at the byte level — +/// these are ordinary BPE tokens the model freely merges with surrounding whitespace and content (one +/// token can carry >\n<parameter=, another </parameter>\n</function>). +/// A token is permitted iff replaying its bytes from the current state keeps the machine alive. The +/// constraint engages the instant the > closing <function=NAME> is seen — +/// BEFORE the first <parameter=> — so a merged ></function> token can't +/// drop the required parameters (the analogue of the JSON/Gemma merged-{} early-engage trick). +/// +/// +/// Default-off byte-identical: if no supplied tool is constrainable, or the +/// <tool_call> arming token isn't in the vocabulary, the constraint is inert and +/// generation is unconstrained. +/// +public sealed class QwenCoderToolArgumentConstraint : ITokenConstraint +{ + private const int MaxDepth = 8; // Func + one value frame; ample headroom (no nesting) + private const int MaxNameScan = 256; // give up watching a call whose name region runs this long + private const int MaxPreambleScan = 512; // give up watching a call whose preamble runs this long + + // Structural literals (ordinary BPE byte tokens, not special tokens). + private static readonly byte[] s_funcOpen = Encoding.UTF8.GetBytes(""); + private static readonly byte[] s_funcClose = Encoding.UTF8.GetBytes(""); + + private readonly GrammarVocabulary _vocab; + private readonly Dictionary _tools; + private readonly HashSet _forbidden; // EOG ids — never legal inside the function body + private readonly int _toolCallOpenId; // — arms watching + private readonly int _toolCallCloseId; // — disarms (optional) + + // Watching / preamble state. + private bool _armed; // saw , scanning for + private int _watchState; // preamble FSM state (W* constants) + private int _funcMatchLen; // bytes of " tools) + { + ArgumentNullException.ThrowIfNull(vocab); + ArgumentNullException.ThrowIfNull(tools); + _vocab = vocab; + + _forbidden = new HashSet(vocab.EogTokenIds); + _ = vocab.TryGetSpecialToken(QwenCoderToolCallAdapter.ArmMarker, out _toolCallOpenId); + // -1 (not the out-param's 0-on-miss) so a missing can never match a real token id + // — token 0 (pad/unk) would otherwise spuriously disarm the constraint. + if (!vocab.TryGetSpecialToken(QwenCoderToolCallAdapter.ArmCloseMarker, out _toolCallCloseId)) + _toolCallCloseId = -1; + + _tools = new Dictionary(StringComparer.Ordinal); + if (_toolCallOpenId > 0) + { + foreach (var t in tools) + { + if (t.Arguments.Open) continue; // unconstrained body — skip + var compiled = ToolSchemaCompiler.TryCompileObject(t.Arguments); + if (compiled is not null) _tools[t.Name] = compiled; + } + } + else if (tools.Count > 0) + { + // The caller asked for a constraint but this vocabulary doesn't define Qwen's + // arming token — the constraint is inert. Surface it once so an operator who + // enabled tool-grammar on a mismatched model isn't left wondering why arguments are still + // unconstrained. + WarnOnce("no-tool-call-marker", + "arming token not found in this vocabulary — tool-grammar is inert for this " + + "model (Qwen3-Coder arguments generate unconstrained)."); + } + } + + /// Whether this constraint can ever restrict anything (any tool was constrainable). + public bool HasConstrainableTools => _tools.Count > 0; + + public bool IsConstraining => _depth > 0; + + public void Reset() + { + _armed = false; + _depth = 0; + ResetPreamble(); + } + + private void ResetPreamble() + { + _watchState = WSeekTag; + _funcMatchLen = 0; + _nameBuf.Clear(); + _preambleLen = 0; + } + + // ── Frame stack ─────────────────────────────────────────────────────────── + + private enum FK : byte { Func, Free, Num, Lit } + + private struct Frame + { + public FK Kind; + public int State; + public CompiledObject? Obj; // Func frame + public CompiledNode? Node; // Lit (literals) / Num + public ulong Emitted; // Func: parameters emitted + public ulong Cand; // Func: tag-match / key-match candidates; Lit: literal candidates + public int MatchLen; // bytes into the current tag / key / literal / close-tag match + public bool SeenDigit; // Num + public bool SeenDot; // Num + public bool SeenSign; // Num + } + + // Func sub-states. + private const int FSeekTag = 0; // skip ws; '<' opens a tag () + private const int FMatchOpenTag = 1; // matching (candidate bitmask) + private const int FParamKey = 2; // matching a declared KEY up to '>' + private const int FParamClose = 3; // after a typed scalar value: ws* then + + // Open-tag candidate bits (FMatchOpenTag). + private const ulong TagParam = 1UL << 0; // + + // Free value sub-state: rolling-match over free content. + private const int FrContent = 0; + + // Num sub-states (lead ws skipped in NStart). + private const int NStart = 0; + private const int NIntDigits = 1; + private const int NFracStart = 2; + private const int NFracDigits = 3; + + // Lit sub-states. + private const int LStart = 0; // skip leading ws + private const int LMatch = 1; // match one of the bare literals + + // Preamble (watching) sub-states. + private const int WSeekTag = 0; // skip ws; '<' begins the ; typed scalars resume at + /// to match the close tag after the value. Returns false if the stack is + /// full (the caller then rejects the token); unreachable for Coder in practice since values never + /// nest, so depth never exceeds Func + one value frame — well under . + private bool PushValue(CompiledNode node, ref Frame parent) + { + if (_depth >= MaxDepth) return false; + parent.MatchLen = 0; // entering a post-value resume state — clear the key-match cursor so + // FParamClose / FSeekTag start the close tag from byte 0, not mid-key. + // Enum (any base type) / boolean / null → a bare literal from a set. Number/Integer → bare + // number. Everything else (string, array, object, Any/free) → free content until . + if (node.Literals is not null) + { + parent.State = FParamClose; + ref var f = ref _stack[_depth++]; + f = default; f.Kind = FK.Lit; f.State = LStart; f.Node = node; f.Cand = AllBits(node.Literals.Length); + return true; + } + if (node.Kind is JsonSchemaKind.Number or JsonSchemaKind.Integer) + { + parent.State = FParamClose; + ref var f = ref _stack[_depth++]; + f = default; f.Kind = FK.Num; f.State = NStart; f.Node = node; + return true; + } + parent.State = FSeekTag; + ref var ff = ref _stack[_depth++]; + ff = default; ff.Kind = FK.Free; ff.State = FrContent; ff.MatchLen = 0; + return true; + } + + private static ulong AllBits(int n) => n >= 64 ? ulong.MaxValue : (1UL << n) - 1; + + // ── Public lifecycle ────────────────────────────────────────────────────── + + public void Accept(int token) + { + if (IsConstraining) + { + bool ok = !_forbidden.Contains(token) && RunToken(token); + // The engine draws from the masked logits, so a permitted token must replay cleanly. A + // rejection here is an invariant violation (a Filter/Accept divergence); flag it once and + // stop constraining either way. A normal end-of-function is ok && _depth == 0 — re-arm so a + // second in the same still engages. + if (!ok) + { + WarnOnce("accept-divergence", + "a sampled token permitted by the mask was rejected by the grammar — constraint " + + "disabled for the rest of this call (possible Filter/Accept divergence)."); + _depth = 0; _armed = false; ResetPreamble(); + } + else if (_depth == 0) + { + _armed = true; ResetPreamble(); // function closed → keep watching the block + } + return; + } + + // Watching. + if (!_armed) + { + if (token == _toolCallOpenId) { _armed = true; ResetPreamble(); } + return; + } + if (token == _toolCallCloseId) { Disarm(); return; } // block ended without (another) call + if (token == _toolCallOpenId) { ResetPreamble(); return; } // defensive: a fresh block opened + + // Walk the token's bytes through the preamble FSM; engage at the '>' closing + // so the following token (which opens the body) is masked — a merged ">" can't slip + // the required parameters past. The '>' is consumed by the watcher; any remaining token bytes + // replay into the constrained machine. + var bytes = _vocab.TokenBytes(token); + for (int i = 0; i < bytes.Length; i++) + { + if (++_preambleLen > MaxPreambleScan) { Disarm(); return; } + var r = WatchByte(bytes[i]); + if (r == WatchResult.Disarm) { Disarm(); return; } + if (r == WatchResult.Engage) + { + _armed = false; + if (i + 1 < bytes.Length && !FeedRawBytes(bytes[(i + 1)..])) + { + _depth = 0; + WarnOnce("trailing-replay", + "could not replay body bytes after — arguments generate " + + "unconstrained for this call."); + } + return; + } + } + } + + private void Disarm() { _armed = false; ResetPreamble(); } + + public ReadOnlySpan Filter(ReadOnlySpan logits) + { + if (_depth == 0) return logits; + + var masked = _masked ??= new float[_vocab.VocabSize]; + if (masked.Length != logits.Length) return logits; // vocab mismatch — never wedge + logits.CopyTo(masked); + + int allowedCount = ComputeMask(masked); + if (allowedCount == 0) + { + ref var top = ref _stack[_depth - 1]; + WarnOnce($"dead-state:{top.Kind}:{top.State}", + $"grammar reached a dead state (no legal token) at kind={top.Kind} state={top.State} " + + "— tool arguments continue unconstrained from here."); + return logits; + } + return masked; + } + + /// Sets every forbidden token to -inf in ; returns the count kept. + private int ComputeMask(float[] buf) + { + Array.Clear(_firstByteOk); + CollectFirstBytes(_depth - 1, _firstByteOk); + + // Fast-path free content. A free value's only structural byte is '<' (the start of + // ), so CollectFree marks all 256 first-bytes — without this every step would + // SimulateToken the WHOLE vocabulary. A token carrying no '<' is pure content that can only keep + // the value open, so it's legal without simulation; only tokens that could begin the close tag + // need the full replay. + bool fastFree = _stack[_depth - 1].Kind == FK.Free; + + int kept = 0; + for (int id = 0; id < buf.Length; id++) + { + var bytes = _vocab.TokenBytes(id); + // Empty-byte tokens (EOG / control) never advance the structure — forbidding them keeps an + // end-of-generation token from truncating the call mid-body. + if (bytes.Length == 0 || !_firstByteOk[bytes[0]]) { buf[id] = float.NegativeInfinity; continue; } + bool ok = (fastFree && !ContainsLt(bytes)) || SimulateToken(id); + if (ok) kept++; + else buf[id] = float.NegativeInfinity; + } + + // Belt-and-suspenders: forbid every EOG id regardless of its bytes (a tokenizer whose EOS + // decodes to ordinary text could otherwise pass first-byte pruning inside a free value). + foreach (int id in _forbidden) + if ((uint)id < (uint)buf.Length && !float.IsNegativeInfinity(buf[id])) + { buf[id] = float.NegativeInfinity; kept--; } + + return kept; + } + + private static bool ContainsLt(ReadOnlySpan bytes) + { + for (int i = 0; i < bytes.Length; i++) + if (bytes[i] == (byte)'<') return true; + return false; + } + + private bool SimulateToken(int token) + { + int savedDepth = _depth; + for (int i = 0; i < savedDepth; i++) _scratch[i] = _stack[i]; + bool ok = FeedRawBytes(_vocab.TokenBytes(token)); + for (int i = 0; i < savedDepth; i++) _stack[i] = _scratch[i]; + _depth = savedDepth; + return ok; + } + + // ── Token execution ─────────────────────────────────────────────────────── + + private bool RunToken(int tokenId) => _depth != 0 && FeedRawBytes(_vocab.TokenBytes(tokenId)); + + /// Byte-walks a raw byte span through the structural automaton, mutating the stack. + private bool FeedRawBytes(ReadOnlySpan bytes) + { + int i = 0; + while (i < bytes.Length) + { + if (_depth == 0) return false; // closed the function but bytes remain + ref var top = ref _stack[_depth - 1]; + var r = StepByte(ref top, bytes[i]); + switch (r) + { + case Step.Consume: i++; break; + case Step.Retry: break; // frame pushed/popped; re-evaluate top + default: return false; // Reject + } + } + return true; + } + + private enum Step { Consume, Retry, Reject } + + private Step StepByte(ref Frame top, byte b) + { + switch (top.Kind) + { + case FK.Func: return StepFunc(ref top, b); + case FK.Free: return StepFree(ref top, b); + case FK.Num: return StepNum(ref top, b); + case FK.Lit: return StepLit(ref top, b); + default: return Step.Reject; + } + } + + private Step StepFunc(ref Frame f, byte b) + { + var obj = f.Obj!; + switch (f.State) + { + case FSeekTag: + if (IsWs(b)) return Step.Consume; + if (b == (byte)'<') + { + // Both open with '<'. A parameter tag is a candidate + // only while a declared key is still unemitted; the close tag only once every + // required key is emitted. (At least one always holds, so '<' is never a dead end.) + f.Cand = (UnemittedMask(f) != 0 ? TagParam : 0) | (RequiredSatisfied(f) ? TagFunc : 0); + f.MatchLen = 1; + f.State = FMatchOpenTag; + return Step.Consume; + } + return Step.Reject; + + case FMatchOpenTag: + { + ulong narrowed = 0; + if ((f.Cand & TagParam) != 0 && s_paramOpen.Length > f.MatchLen && s_paramOpen[f.MatchLen] == b) narrowed |= TagParam; + if ((f.Cand & TagFunc) != 0 && s_funcClose.Length > f.MatchLen && s_funcClose[f.MatchLen] == b) narrowed |= TagFunc; + if (narrowed == 0) return Step.Reject; + f.Cand = narrowed; f.MatchLen++; + // Exactly one candidate survives past index 1 ('p' vs '/'); act on completion. + if ((f.Cand & TagParam) != 0 && f.MatchLen == s_paramOpen.Length) + { + f.Cand = UnemittedMask(f); f.MatchLen = 0; f.State = FParamKey; + return Step.Consume; + } + if ((f.Cand & TagFunc) != 0 && f.MatchLen == s_funcClose.Length) + { + _depth--; // → the call is complete + return Step.Consume; + } + return Step.Consume; + } + + case FParamKey: + { + if (b == (byte)'>') + { + int complete = CompleteIndex(obj.KeyBytes, f.Cand, f.MatchLen); + if (complete < 0) return Step.Reject; // '>' at a non-key boundary (incl. empty key) + f.Emitted |= 1UL << complete; + return PushValue(obj.Values[complete], ref f) ? Step.Consume : Step.Reject; + } + ulong narrowed = 0; + for (int i = 0; i < obj.Count; i++) + if ((f.Cand & (1UL << i)) != 0 && obj.KeyBytes[i].Length > f.MatchLen && obj.KeyBytes[i][f.MatchLen] == b) + narrowed |= 1UL << i; + if (narrowed == 0) return Step.Reject; + f.Cand = narrowed; f.MatchLen++; + return Step.Consume; + } + + case FParamClose: + // After a typed scalar value: optional trailing ws, then . + if (f.MatchLen == 0 && IsWs(b)) return Step.Consume; + if (s_paramClose.Length > f.MatchLen && s_paramClose[f.MatchLen] == b) + { + f.MatchLen++; + if (f.MatchLen == s_paramClose.Length) { f.State = FSeekTag; f.MatchLen = 0; } + return Step.Consume; + } + return Step.Reject; + + default: + return Step.Reject; + } + } + + /// Free value: any bytes are content until the </parameter> close, matched by + /// a rolling counter. </parameter> has no repeated prefix (only index 0 is '<'), so + /// a failed match restarts cleanly. On the full match the value is complete; the parent Func resumes + /// at (the close was already consumed here). + private Step StepFree(ref Frame f, byte b) + { + if (b == s_paramClose[f.MatchLen]) + { + f.MatchLen++; + if (f.MatchLen == s_paramClose.Length) + { + _depth--; // value + consumed + return _depth == 0 ? Step.Reject : Step.Consume; // a Free value is always inside Func + } + return Step.Consume; + } + f.MatchLen = b == s_paramClose[0] ? 1 : 0; // restart the rolling match + return Step.Consume; // ordinary content byte + } + + private Step StepNum(ref Frame f, byte b) + { + bool digit = b is >= (byte)'0' and <= (byte)'9'; + switch (f.State) + { + case NStart: + if (IsWs(b)) return Step.Consume; // leading ws (the template's newline) + if (b == (byte)'-' && !f.SeenSign) { f.SeenSign = true; return Step.Consume; } + if (digit) { f.SeenDigit = true; f.State = NIntDigits; return Step.Consume; } + return Step.Reject; + case NIntDigits: + if (digit) return Step.Consume; + if (b == (byte)'.' && !f.Node!.IntegerOnly && !f.SeenDot) { f.SeenDot = true; f.State = NFracStart; return Step.Consume; } + if (f.SeenDigit) { _depth--; return Step.Retry; } // number ended — parent (FParamClose) handles b + return Step.Reject; + case NFracStart: + if (digit) { f.State = NFracDigits; return Step.Consume; } + return Step.Reject; + case NFracDigits: + if (digit) return Step.Consume; + _depth--; return Step.Retry; + default: + return Step.Reject; + } + } + + private Step StepLit(ref Frame f, byte b) + { + var lits = f.Node!.Literals!; + if (f.State == LStart) + { + if (IsWs(b)) return Step.Consume; // leading ws + f.State = LMatch; // fall through to matching + } + ulong narrowed = 0; + for (int i = 0; i < lits.Length; i++) + if ((f.Cand & (1UL << i)) != 0 && lits[i].Length > f.MatchLen && lits[i][f.MatchLen] == b) + narrowed |= 1UL << i; + if (narrowed != 0) { f.Cand = narrowed; f.MatchLen++; return Step.Consume; } + if (CompleteIndex(lits, f.Cand, f.MatchLen) >= 0) { _depth--; return Step.Retry; } // literal done + return Step.Reject; + } + + private static bool IsWs(byte b) => b is (byte)' ' or (byte)'\t' or (byte)'\n' or (byte)'\r'; + + private static bool RequiredSatisfied(in Frame f) => (f.Emitted & f.Obj!.RequiredMask) == f.Obj!.RequiredMask; + + private static ulong UnemittedMask(in Frame f) + { + ulong cand = 0; + for (int i = 0; i < f.Obj!.Count; i++) + if ((f.Emitted & (1UL << i)) == 0) cand |= 1UL << i; + return cand; + } + + // ── Preamble (watching) byte FSM ────────────────────────────────────────── + + private enum WatchResult { Continue, Engage, Disarm } + + /// Walks one preamble byte while armed: skip ws, match the literal <function=, + /// accumulate NAME up to '>', then engage on a known constrainable tool. + private WatchResult WatchByte(byte b) + { + switch (_watchState) + { + case WSeekTag: + if (IsWs(b)) return WatchResult.Continue; + if (b == s_funcOpen[0]) { _funcMatchLen = 1; _watchState = WFuncTag; return WatchResult.Continue; } + return WatchResult.Disarm; // something other than + + case WFuncTag: + if (s_funcOpen.Length > _funcMatchLen && s_funcOpen[_funcMatchLen] == b) + { + _funcMatchLen++; + if (_funcMatchLen == s_funcOpen.Length) { _nameBuf.Clear(); _watchState = WName; } + return WatchResult.Continue; + } + return WatchResult.Disarm; + + case WName: + if (b == (byte)'>') + return EngageOnTool() ? WatchResult.Engage : WatchResult.Disarm; + if (_nameBuf.Length >= MaxNameScan) return WatchResult.Disarm; + _nameBuf.Append((char)b); // function names are ASCII + return WatchResult.Continue; + + default: + return WatchResult.Disarm; + } + } + + /// Engages the constrained machine on the named tool's function body (the '>' closing + /// <function=NAME> was just consumed). Returns false (caller disarms) for an unknown / + /// non-constrainable tool. + private bool EngageOnTool() + { + string name = _nameBuf.ToString().Trim(); + if (!_tools.TryGetValue(name, out var obj)) return false; + _depth = 0; + PushFunc(obj); + return true; + } + + // ── First-byte collection (mask pruning) ────────────────────────────────── + + private void CollectFirstBytes(int d, bool[] set) + { + while (d >= 0) + { + ref var f = ref _stack[d]; + bool popThrough = false; + switch (f.Kind) + { + case FK.Func: CollectFunc(ref f, set); break; + case FK.Free: CollectFree(set); break; + case FK.Num: popThrough = CollectNum(ref f, set); break; + case FK.Lit: popThrough = CollectLit(ref f, set); break; + } + if (!popThrough) break; + d--; // a bare scalar can end here — parent bytes also start a token + } + } + + private static void CollectFunc(ref Frame f, bool[] set) + { + var obj = f.Obj!; + switch (f.State) + { + case FSeekTag: + MarkWs(set); + set['<'] = true; // + break; + case FMatchOpenTag: + if ((f.Cand & TagParam) != 0 && s_paramOpen.Length > f.MatchLen) set[s_paramOpen[f.MatchLen]] = true; + if ((f.Cand & TagFunc) != 0 && s_funcClose.Length > f.MatchLen) set[s_funcClose[f.MatchLen]] = true; + break; + case FParamKey: + for (int i = 0; i < obj.Count; i++) + if ((f.Cand & (1UL << i)) != 0 && obj.KeyBytes[i].Length > f.MatchLen) + set[obj.KeyBytes[i][f.MatchLen]] = true; + if (CompleteIndex(obj.KeyBytes, f.Cand, f.MatchLen) >= 0) set['>'] = true; + break; + case FParamClose: + if (f.MatchLen == 0) MarkWs(set); + if (s_paramClose.Length > f.MatchLen) set[s_paramClose[f.MatchLen]] = true; + break; + } + } + + private static void CollectFree(bool[] set) + { + // A free value admits any content; '<' may begin . Mark everything and let the + // (fast-pathed) simulate pass decide which '<'-bearing tokens stay alive. + for (int i = 0; i < 256; i++) set[i] = true; + } + + private static bool CollectNum(ref Frame f, bool[] set) + { + switch (f.State) + { + case NStart: + MarkWs(set); + if (!f.SeenSign) set['-'] = true; + MarkDigits(set); + return false; + case NIntDigits: + MarkDigits(set); + if (!f.SeenDot && !f.Node!.IntegerOnly) set['.'] = true; + return f.SeenDigit; // can end → parent (FParamClose) bytes too + case NFracStart: + MarkDigits(set); + return false; + case NFracDigits: + MarkDigits(set); + return true; // can end + default: + return false; + } + } + + private static bool CollectLit(ref Frame f, bool[] set) + { + var lits = f.Node!.Literals!; + if (f.State == LStart) MarkWs(set); // leading ws still allowed + for (int i = 0; i < lits.Length; i++) + if ((f.Cand & (1UL << i)) != 0 && lits[i].Length > f.MatchLen) + set[lits[i][f.MatchLen]] = true; + // A complete literal can end (parent FParamClose handles the next byte), unless we're still in + // the leading-ws state with nothing matched yet. + return f.State == LMatch && CompleteIndex(lits, f.Cand, f.MatchLen) >= 0; + } + + private static void MarkDigits(bool[] set) { for (byte b = (byte)'0'; b <= (byte)'9'; b++) set[b] = true; } + private static void MarkWs(bool[] set) { set[' '] = true; set['\t'] = true; set['\n'] = true; set['\r'] = true; } + + // ── Small helpers ───────────────────────────────────────────────────────── + + private static int CompleteIndex(byte[][] lits, ulong cand, int matchLen) + { + for (int i = 0; i < lits.Length; i++) + if ((cand & (1UL << i)) != 0 && lits[i].Length == matchLen) return i; + return -1; + } + + private static readonly System.Collections.Concurrent.ConcurrentDictionary s_warned = new(); + private static void WarnOnce(string key, string message) + { + if (s_warned.TryAdd(key, 0)) + Console.Error.WriteLine($"[SharpInference.ToolGrammar] {message}"); + } +} diff --git a/src/SharpInference.Core/ToolCallAdapter.cs b/src/SharpInference.Core/ToolCallAdapter.cs index 2af166b..f155711 100644 --- a/src/SharpInference.Core/ToolCallAdapter.cs +++ b/src/SharpInference.Core/ToolCallAdapter.cs @@ -320,18 +320,33 @@ public sealed class QwenToolCallAdapter(string architecture) : IToolCallAdapter public int MaxOpenTagLength => OpenMarker.Length; /// - /// Constrains the JSON argument object of Qwen's - /// <tool_call>{"name":"NAME","arguments":{...}}</tool_call> wire format - /// (issue #376). Inert (returns null) when no supplied tool is constrainable, when - /// <tool_call> isn't a vocabulary token, or when the model emits the Qwen3.6 XML - /// shape instead of JSON (the constraint simply never engages). + /// Constrains the argument body of Qwen's <tool_call>…</tool_call> wire + /// format. The same architecture (qwen3moe/qwen2/qwen3) hosts two formats that + /// the GGUF metadata can't tell apart — standard JSON + /// ({"name":"NAME","arguments":{...}}, issue #376) and Qwen3-Coder's XML + /// (<function=NAME><parameter=K>V</parameter>…</function>, issue + /// #383) — and they're only distinguishable by the chat template, which isn't visible here. So we + /// build BOTH and overlay them with a : both arm on + /// <tool_call> and diverge on the first body byte, so whichever matches the model's + /// actual output engages and the other stays inert. Returns null when no supplied tool is + /// constrainable or <tool_call> isn't a vocabulary token. public ITokenConstraint? BuildArgumentConstraint(IReadOnlyList tools, GrammarVocabulary vocab) { ArgumentNullException.ThrowIfNull(tools); ArgumentNullException.ThrowIfNull(vocab); - var c = new JsonToolArgumentConstraint( + var json = new JsonToolArgumentConstraint( vocab, tools, OpenMarker, JsonToolEnvelope.NameValueObject, argsKeys: ["arguments", "parameters"]); - return c.HasConstrainableTools ? c : null; + var xml = new QwenCoderToolArgumentConstraint(vocab, tools); + + var active = new List(2); + if (json.HasConstrainableTools) active.Add(json); + if (xml.HasConstrainableTools) active.Add(xml); + return active.Count switch + { + 0 => null, + 1 => active[0], + _ => new CompositeToolArgumentConstraint(active), + }; } public ToolCallParseResult Parse(string rawOutput) @@ -403,10 +418,29 @@ public sealed class QwenCoderToolCallAdapter : IToolCallAdapter { public const string OpenMarker = "Qwen's tool-call envelope tokens. Qwen3-Coder wraps each XML call in + /// <tool_call>…</tool_call>; these are single special tokens used by the + /// argument-grammar constraint as a leak-proof arming gate (the inner <function=> tags + /// are ordinary text). Mirrors . + public const string ArmMarker = ""; + public const string ArmCloseMarker = ""; public string Architecture => "qwen3coder"; public int MaxOpenTagLength => OpenMarker.Length; + /// + /// Constrains the XML argument body of Qwen3-Coder's + /// <tool_call><function=NAME><parameter=KEY>VALUE</parameter>…</function></tool_call> + /// wire format (issue #383) — the XML sibling of the JSON/Gemma constraints. Inert (returns null) + /// when no supplied tool is constrainable or <tool_call> isn't a vocabulary token. + public ITokenConstraint? BuildArgumentConstraint(IReadOnlyList tools, GrammarVocabulary vocab) + { + ArgumentNullException.ThrowIfNull(tools); + ArgumentNullException.ThrowIfNull(vocab); + var c = new QwenCoderToolArgumentConstraint(vocab, tools); + return c.HasConstrainableTools ? c : null; + } + public ToolCallParseResult Parse(string rawOutput) { var calls = new List(); diff --git a/src/SharpInference.Server/SharpInferenceServerOptions.cs b/src/SharpInference.Server/SharpInferenceServerOptions.cs index c03e7f7..6fe495c 100644 --- a/src/SharpInference.Server/SharpInferenceServerOptions.cs +++ b/src/SharpInference.Server/SharpInferenceServerOptions.cs @@ -28,11 +28,11 @@ public sealed class SharpInferenceServerOptions /// /// Enable schema/grammar-constrained decoding for tool-call arguments (issue #374). When on and - /// a tool-active request is served by a family with constraint support (Gemma 4 today), the - /// sampler is restricted to tokens that satisfy the supplied JSON Schema in the model's native - /// call syntax — required keys can't be dropped, only declared keys/enum values appear, value - /// shapes match the declared type. Default off → byte-identical to unconstrained decoding. Also - /// turned on by the SHARPI_TOOL_GRAMMAR=1 environment variable. + /// a tool-active request is served by a family with constraint support (Gemma 4, Qwen and + /// Qwen3-Coder, Llama-3, DeepSeek), the sampler is restricted to tokens that satisfy the supplied + /// JSON Schema in the model's native call syntax — required keys can't be dropped, only declared + /// keys/enum values appear, value shapes match the declared type. Default off → byte-identical to + /// unconstrained decoding. Also turned on by the SHARPI_TOOL_GRAMMAR=1 environment variable. /// public bool ToolGrammar { get; set; } diff --git a/tests/SharpInference.Tests.Core/CoderToolGrammarConstraintTests.cs b/tests/SharpInference.Tests.Core/CoderToolGrammarConstraintTests.cs new file mode 100644 index 0000000..c149acf --- /dev/null +++ b/tests/SharpInference.Tests.Core/CoderToolGrammarConstraintTests.cs @@ -0,0 +1,141 @@ +using System.Text.Json; +using SharpInference.Core; +using SharpInference.Core.Grammar; +using Xunit.Abstractions; + +namespace SharpInference.Tests.Core; + +/// +/// Decode-time conformance for the Qwen3-Coder XML tool-argument grammar constraint (issue #383) +/// against a REAL Qwen3-Coder vocabulary, so the byte-level matching is exercised over the actual BPE +/// merges (where a single token can carry a whole <parameter= tag, or close one parameter +/// and open the next). Model-gated — skips when the GGUF is absent. The grammar logic itself is covered +/// model-free by ; this asserts the same invariants survive a +/// production tokenizer. +/// +public sealed class CoderToolGrammarConstraintTests(ITestOutputHelper output) +{ + private static readonly string[] s_modelPaths = + [ + @"E:\models\Qwen3-Coder-30B-A3B-Instruct-Q4_K_M.gguf", + ]; + + private static GgufTokenizer? Tok() + { + foreach (var p in s_modelPaths) + if (File.Exists(p)) + { + var m = GgufModel.Open(p); + return GgufTokenizer.FromGgufModel(m); + } + return null; + } + + private static ToolSchema Schema(string name, string parametersJson) + { + using var doc = JsonDocument.Parse(parametersJson); + return ToolSchema.FromOpenAiFunction(name, doc.RootElement.Clone()); + } + + private static void Feed(ITokenConstraint c, GgufTokenizer tok, string text) + { + foreach (int id in tok.Encode(text)) c.Accept(id); + } + + private static bool Allowed(ITokenConstraint c, int vocab, int tokenId) + { + Span logits = new float[vocab]; + var masked = c.Filter(logits); + return !float.IsNegativeInfinity(masked[tokenId]); + } + + // The first token that emits exactly `s` (handles multi-token strings: returns the leading token). + private static int First(GgufTokenizer tok, string s) => tok.Encode(s)[0]; + + private const string Weather = + """{"type":"object","properties":{"location":{"type":"string"},"unit":{"type":"string","enum":["celsius","fahrenheit"]},"days":{"type":"integer"}},"required":["location"]}"""; + + private QwenCoderToolArgumentConstraint? Constraint(GgufTokenizer tok, out GrammarVocabulary vocab) + { + vocab = new GrammarVocabulary(tok); + return (QwenCoderToolArgumentConstraint?) + new QwenCoderToolCallAdapter().BuildArgumentConstraint([Schema("get_weather", Weather)], vocab); + } + + [Fact] + public void EngagesAtFunctionTag_BlocksImmediateClose() + { + var tok = Tok(); + if (tok is null) { output.WriteLine("missing model — skip"); return; } + var c = Constraint(tok, out var vocab); + Assert.NotNull(c); // resolved as a special token → constrainable + + Feed(c!, tok, "\n"); + Assert.True(c!.IsConstraining); // engaged at the '>' closing the function tag + + // At the body root, a '<' (tag open) and whitespace are legal; bare text is not. + Assert.True(Allowed(c, vocab.VocabSize, First(tok, "<"))); + Assert.False(Allowed(c, vocab.VocabSize, First(tok, "x"))); + + Feed(c, tok, "<"); + // Only ('/') is forbidden — required 'location' missing. + Assert.True(Allowed(c, vocab.VocabSize, First(tok, "p"))); + Assert.False(Allowed(c, vocab.VocabSize, First(tok, "/"))); + } + + [Fact] + public void RequiredKey_CannotBeOmitted_ForeignKeyRejected() + { + var tok = Tok(); + if (tok is null) { output.WriteLine("missing model — skip"); return; } + var c = Constraint(tok, out var vocab)!; + + Feed(c, tok, "\n\n\n\n\n"); + Assert.True(c.IsConstraining); + // Coder enum values are bare text — a quote is not part of the value. + Assert.False(Allowed(c, vocab.VocabSize, First(tok, "\""))); + // 'c' (celsius) / 'f' (fahrenheit) are reachable; 'x' is not. + Assert.True(Allowed(c, vocab.VocabSize, First(tok, "c"))); + Assert.True(Allowed(c, vocab.VocabSize, First(tok, "f"))); + Assert.False(Allowed(c, vocab.VocabSize, First(tok, "x"))); + } + + [Fact] + public void NumberValue_RejectsNonDigit() + { + var tok = Tok(); + if (tok is null) { output.WriteLine("missing model — skip"); return; } + var c = Constraint(tok, out var vocab)!; + + Feed(c, tok, "\n\n\n"); + Assert.True(Allowed(c, vocab.VocabSize, First(tok, "3"))); + Assert.False(Allowed(c, vocab.VocabSize, First(tok, "x"))); // a non-numeric value is illegal + } + + [Fact] + public void JsonOutput_DoesNotEngage() + { + var tok = Tok(); + if (tok is null) { output.WriteLine("missing model — skip"); return; } + var c = Constraint(tok, out _)!; + + // If the model emitted a JSON envelope instead of the XML shape, the Coder + // constraint must stay inert (it arms on but only engages on ). + Feed(c, tok, "\n{\"name\": \"get_weather\"}"); + Assert.False(c.IsConstraining); + } +} diff --git a/tests/SharpInference.Tests.Core/CoderToolGrammarMockTests.cs b/tests/SharpInference.Tests.Core/CoderToolGrammarMockTests.cs new file mode 100644 index 0000000..0f3b989 --- /dev/null +++ b/tests/SharpInference.Tests.Core/CoderToolGrammarMockTests.cs @@ -0,0 +1,319 @@ +using System.Text.Json; +using SharpInference.Core; +using SharpInference.Core.Grammar; + +namespace SharpInference.Tests.Core; + +/// +/// Model-independent decode-time conformance for the Qwen3-Coder XML tool-argument grammar constraint +/// (issue #383) using , so the byte-level masking is covered in CI +/// without the multi-gigabyte GGUF. The XML sibling of : required +/// parameter, foreign-key rejection, bare enum/number/boolean values, free-text strings, the +/// early-engage that blocks an immediate </function> when a required parameter is missing, +/// partially-typed free values, multi-parameter required-once, and re-arming across functions. +/// +public sealed class CoderToolGrammarMockTests +{ + private static (QwenCoderToolArgumentConstraint c, FakeCoderTokenizer tok, int vocab) Build( + string schemaJson, string toolName) + { + var tok = new FakeCoderTokenizer(); + var vocab = new GrammarVocabulary(tok); + using var doc = JsonDocument.Parse(schemaJson); + var schema = ToolSchema.FromOpenAiFunction(toolName, doc.RootElement.Clone()); + var c = new QwenCoderToolCallAdapter().BuildArgumentConstraint([schema], vocab); + Assert.NotNull(c); + return ((QwenCoderToolArgumentConstraint)c!, tok, vocab.VocabSize); + } + + private static void Feed(ITokenConstraint c, FakeCoderTokenizer tok, string text) + { + foreach (int id in tok.Encode(text)) c.Accept(id); + } + + private static bool Allowed(ITokenConstraint c, int vocab, int tokenId) + { + Span logits = new float[vocab]; + var masked = c.Filter(logits); + return !float.IsNegativeInfinity(masked[tokenId]); + } + + // The canonical Coder preamble up to and including the '>' closing — engage point. + private static string Preamble(string name) => $"\n"; + + [Fact] + public void EngagesAtFunctionTag_BeforeFirstParameter() + { + var (c, tok, vocab) = Build( + """{"type":"object","properties":{"location":{"type":"string"}},"required":["location"]}""", + "get_weather"); + + Feed(c, tok, Preamble("get_weather")); + Assert.True(c.IsConstraining); // engaged at the '>', BEFORE any '))); + } + + [Fact] + public void EarlyEngage_RequiredParam_BlocksImmediateFunctionClose() + { + var (c, tok, vocab) = Build( + """{"type":"object","properties":{"location":{"type":"string"}},"required":["location"]}""", + "get_weather"); + + Feed(c, tok, Preamble("get_weather") + "<"); // now matching an open tag after '<' + // Only is forbidden ('/') because the required + // 'location' hasn't been emitted — the merged-{} analogue: the call can't close empty. + Assert.True(Allowed(c, vocab, tok.Char('p'))); + Assert.False(Allowed(c, vocab, tok.Char('/'))); + } + + [Fact] + public void MergedClosingBracket_EngagesMidToken() + { + var (c, tok, vocab) = Build( + """{"type":"object","properties":{"location":{"type":"string"}},"required":["location"]}""", + "get_weather"); + + // Feed up to the name WITHOUT the '>', then deliver a merged ">\n" token (the realistic + // post-tag merge). Engagement must happen mid-token on the '>', with the trailing '\n' replayed + // into the (now constrained) function body. + Feed(c, tok, "\n\n")); + Assert.True(c.IsConstraining); + + Feed(c, tok, "<"); + Assert.True(Allowed(c, vocab, tok.Char('p'))); + Assert.False(Allowed(c, vocab, tok.Char('/'))); // still can't close — required missing + } + + [Fact] + public void OnlyDeclaredKeys_AreReachable() + { + var (c, tok, vocab) = Build( + """{"type":"object","properties":{"query":{"type":"string"}},"required":["query"]}""", + "web_search"); + + Feed(c, tok, Preamble("web_search") + "'))); // 'query' complete → close the key tag + Assert.False(Allowed(c, vocab, tok.Char('s'))); // can't extend past a declared key + } + + [Fact] + public void FreeStringValue_AcceptsContent_ClosesOnParamTag() + { + var (c, tok, vocab) = Build( + """{"type":"object","properties":{"location":{"type":"string"}},"required":["location"]}""", + "get_weather"); + + Feed(c, tok, Preamble("get_weather") + ""); + // A string value is free content: any byte stays, EOS forbidden mid-call. + Assert.True(Allowed(c, vocab, tok.Char('B'))); + Assert.True(Allowed(c, vocab, tok.Char('\n'))); + Assert.True(Allowed(c, vocab, tok.Char('<'))); // may begin the close + Assert.False(Allowed(c, vocab, FakeCoderTokenizer.Eos)); + + Feed(c, tok, "\nParis\n"); + // Back at the function body: required 'location' satisfied → may close, or open another tag. + Assert.True(Allowed(c, vocab, tok.Char('<'))); + Feed(c, tok, ""); + Assert.False(c.IsConstraining); // function closed → back to watching + } + + [Fact] + public void Enum_RestrictsToDeclaredValues_BareNotQuoted() + { + var (c, tok, vocab) = Build( + """{"type":"object","properties":{"unit":{"type":"string","enum":["celsius","fahrenheit"]}},"required":["unit"]}""", + "get_weather"); + + Feed(c, tok, Preamble("get_weather") + "\n"); + // Coder enum values are BARE text (no quotes): only declared prefixes after the leading newline. + Assert.True(Allowed(c, vocab, tok.Char('c'))); // celsius + Assert.True(Allowed(c, vocab, tok.Char('f'))); // fahrenheit + Assert.False(Allowed(c, vocab, tok.Char('x'))); // neither + Assert.False(Allowed(c, vocab, tok.Char('"'))); // not quoted + + Feed(c, tok, "celsius"); + Assert.False(Allowed(c, vocab, tok.Char('z'))); // can't extend past the enum + Assert.True(Allowed(c, vocab, tok.Char('<'))); // value complete → start + Assert.True(Allowed(c, vocab, tok.Char('\n'))); // …or trailing whitespace first + } + + [Fact] + public void NumberValue_AcceptsDigits_ThenCloses() + { + var (c, tok, vocab) = Build( + """{"type":"object","properties":{"days":{"type":"integer"}},"required":["days"]}""", + "get_weather"); + + Feed(c, tok, Preamble("get_weather") + "\n"); + Assert.True(Allowed(c, vocab, tok.Char('3'))); + Assert.False(Allowed(c, vocab, tok.Char('.'))); // integer → no decimal point + Assert.False(Allowed(c, vocab, tok.Char('x'))); + + Feed(c, tok, "3"); + Assert.True(Allowed(c, vocab, tok.Char('0'))); // more digits + Assert.True(Allowed(c, vocab, tok.Char('<'))); // …or begin the close + } + + [Fact] + public void BooleanValue_RestrictedToTrueFalse() + { + var (c, tok, vocab) = Build( + """{"type":"object","properties":{"metric":{"type":"boolean"}},"required":["metric"]}""", + "get_weather"); + + Feed(c, tok, Preamble("get_weather") + "\n"); + Assert.True(Allowed(c, vocab, tok.Char('t'))); // true + Assert.True(Allowed(c, vocab, tok.Char('f'))); // false + Assert.False(Allowed(c, vocab, tok.Char('y'))); // not a boolean literal + } + + [Fact] + public void MultipleParams_RequiredOnce_NoRepeat() + { + var (c, tok, vocab) = Build( + """{"type":"object","properties":{"location":{"type":"string"},"unit":{"type":"string","enum":["celsius","fahrenheit"]}},"required":["location","unit"]}""", + "get_weather"); + + Feed(c, tok, Preamble("get_weather") + "\nParis\n"); + Assert.True(c.IsConstraining); // close tag consumed cleanly, still in body + // 'location' done but 'unit' still required → cannot close the function yet. + Feed(c, tok, "<"); + Assert.True(Allowed(c, vocab, tok.Char('p'))); // another still forbidden ('unit' missing) + + Feed(c, tok, "parameter="); + // The already-emitted 'location' must be unreachable; only 'unit' remains. + Assert.True(Allowed(c, vocab, tok.Char('u'))); + Assert.False(Allowed(c, vocab, tok.Char('l'))); // 'location' can't repeat + + Feed(c, tok, "unit>\ncelsius\n"); + Assert.True(c.IsConstraining); // both values + close tags consumed cleanly + Feed(c, tok, "<"); + Assert.True(Allowed(c, vocab, tok.Char('/'))); // all required emitted → may close now + } + + [Fact] + public void AllParamsEmitted_OnlyCloseRemains() + { + var (c, tok, vocab) = Build( + """{"type":"object","properties":{"location":{"type":"string"}},"required":["location"]}""", + "get_weather"); + + // The single declared parameter is emitted; another remains legal (prevents a dead-state from opening a keyless parameter). + Feed(c, tok, Preamble("get_weather") + "\nParis\n"); + Assert.True(c.IsConstraining); + Feed(c, tok, "<"); + Assert.True(Allowed(c, vocab, tok.Char('/'))); // + Assert.False(Allowed(c, vocab, tok.Char('p'))); // no \n{\"any\":[1,2],\"k\":\"v\"}\n"); + Assert.True(c.IsConstraining); + Feed(c, tok, "<"); + Assert.True(Allowed(c, vocab, tok.Char('p'))); // another \nParis\n"); + Feed(c, tok, "<"); + Assert.True(Allowed(c, vocab, tok.Char('/'))); // required satisfied → may close + Feed(c, tok, "/function>"); + Assert.False(c.IsConstraining); + } + + [Fact] + public void ClosesAndReArms_ForSecondFunctionInBlock() + { + var (c, tok, vocab) = Build( + """{"type":"object","properties":{"location":{"type":"string"}},"required":["location"]}""", + "get_weather"); + + // First call closes; without a the constraint stays armed for a second function. + Feed(c, tok, Preamble("get_weather") + "\nParis\n\n"); + Assert.False(c.IsConstraining); // function closed + + Feed(c, tok, "\n"); + Assert.True(c.IsConstraining); // re-engaged on the second function + Feed(c, tok, "<"); + Assert.False(Allowed(c, vocab, tok.Char('/'))); // required enforced again + + // The envelope token disarms entirely. + c.Reset(); + Feed(c, tok, ""); + Feed(c, tok, ""); + Feed(c, tok, ""); + Assert.False(c.IsConstraining); // disarmed by → no engage + } + + [Fact] + public void UnknownTool_IsNotConstrained() + { + var (c, tok, _) = Build( + """{"type":"object","properties":{"location":{"type":"string"}},"required":["location"]}""", + "get_weather"); + + Feed(c, tok, "\n"); + Assert.False(c.IsConstraining); // name not in the constrainable set → passive + } + + [Fact] + public void Reset_ReturnsToWatching() + { + var (c, tok, _) = Build( + """{"type":"object","properties":{"location":{"type":"string"}},"required":["location"]}""", + "get_weather"); + + Feed(c, tok, Preamble("get_weather") + " with no arming token must stay inert (the constraint never + // arms on raw text, so non-tool generation is byte-identical to unconstrained). + Feed(c, tok, "here is some code: "); + Assert.False(c.IsConstraining); + } +} diff --git a/tests/SharpInference.Tests.Core/FakeCoderTokenizer.cs b/tests/SharpInference.Tests.Core/FakeCoderTokenizer.cs new file mode 100644 index 0000000..47aa85b --- /dev/null +++ b/tests/SharpInference.Tests.Core/FakeCoderTokenizer.cs @@ -0,0 +1,95 @@ +using System.Collections.Immutable; +using System.Text; +using SharpInference.Core; + +namespace SharpInference.Tests.Core; + +/// +/// A tiny deterministic tokenizer that mimics the facets of a real Qwen3-Coder BPE vocabulary the XML +/// tool-argument grammar depends on (issue #383): the <tool_call>/</tool_call> +/// envelope tokens are single specials (the constraint's arming gate), every byte is also a single-char +/// token, AND a handful of merged structural tokens (<function=, <parameter=, +/// </parameter>, </function>, >\n, …) mirror how a real BPE fuses the +/// XML tags and trailing newlines into single tokens. Those merges are why byte-level matching is +/// mandatory — a single token can carry a whole tag, or close one parameter and open the next. Unlike +/// Gemma/Qwen-JSON, the Coder tags are NOT special tokens; they are ordinary text the constraint +/// byte-walks, so they live in the merged table here. +/// +public sealed class FakeCoderTokenizer : ITokenizer +{ + public const int Pad = 0, Bos = 1, Eos = 2; + + private readonly Dictionary _specials = new(StringComparer.Ordinal); + private readonly Dictionary _merged = new(StringComparer.Ordinal); + private readonly int _singleBase; + + // Realistic BPE-style merges: whole XML tags plus tags fused with the template's newlines. The + // post-tag ">\n" merge is what exercises the early-engage (the '>' that closes + // arrives merged with the following newline, so engagement happens mid-token). + private static readonly string[] MergedPieces = + [ + "", "", + ">\n", "\n\n", + ]; + + public FakeCoderTokenizer() + { + int id = 3; + foreach (var s in new[] { "", "" }) + _specials[s] = id++; + foreach (var s in MergedPieces) + _merged[s] = id++; + _singleBase = id; + } + + public int VocabSize => _singleBase + 256; + public int BosTokenId => Bos; + public int EosTokenId => Eos; + public int UnknownTokenId => Pad; + public int PadTokenId => Pad; + public bool AddBosToken => false; + public ImmutableArray EogTokenIds => [Eos]; + public IReadOnlyDictionary SpecialTokens => _specials; + + /// Token id for a single ASCII byte. + public int Char(char c) => _singleBase + (byte)c; + + /// Token id for a registered merged structural piece (e.g. <parameter=). + public int Merged(string piece) => _merged[piece]; + + public byte[] DecodeBytes(int token) + { + foreach (var (s, id) in _specials) if (id == token) return Encoding.UTF8.GetBytes(s); + foreach (var (s, id) in _merged) if (id == token) return Encoding.UTF8.GetBytes(s); + if (token is Bos or Eos or Pad) return []; + if (token >= _singleBase && token < _singleBase + 256) return [(byte)(token - _singleBase)]; + return []; + } + + /// Greedy longest-match tokenization over specials ∪ merged pieces, else single chars. + public IReadOnlyList Encode(string text) + { + var ids = new List(); + int i = 0; + while (i < text.Length) + { + int matched = -1, matchLen = 0; + foreach (var table in new[] { _specials, _merged }) + foreach (var (s, id) in table) + if (s.Length > matchLen && i + s.Length <= text.Length + && text.AsSpan(i, s.Length).SequenceEqual(s)) + { matched = id; matchLen = s.Length; } + + if (matched >= 0) { ids.Add(matched); i += matchLen; } + else { ids.Add(Char(text[i])); i++; } + } + return ids; + } + + public string Decode(IEnumerable tokens) + { + var sb = new StringBuilder(); + foreach (int t in tokens) sb.Append(Encoding.UTF8.GetString(DecodeBytes(t))); + return sb.ToString(); + } +} diff --git a/tests/SharpInference.Tests.Core/JsonToolGrammarConstraintTests.cs b/tests/SharpInference.Tests.Core/JsonToolGrammarConstraintTests.cs index 210bd3b..2ef0707 100644 --- a/tests/SharpInference.Tests.Core/JsonToolGrammarConstraintTests.cs +++ b/tests/SharpInference.Tests.Core/JsonToolGrammarConstraintTests.cs @@ -113,15 +113,35 @@ public void EnumValue_RestrictedToDeclaredSet() } [Fact] - public void NonJsonOutput_DoesNotEngage() + public void XmlOutput_EngagesViaComposite() { var tok = Tok(); if (tok is null) { output.WriteLine("missing model — skip"); return; } var vocab = new GrammarVocabulary(tok); var c = new QwenToolCallAdapter("qwen3").BuildArgumentConstraint([Schema("get_weather", Weather)], vocab)!; - // Qwen3.6 also emits an XML shape — the JSON constraint must stay inert on it. + // The Qwen adapter now overlays the JSON (#376) and Qwen3-Coder XML (#383) constraints (the two + // can't be told apart from the GGUF architecture alone). Feeding the XML shape + // engages the XML sub-constraint through the composite — the previously-uncovered case. Feed(c, tok, "\n"); - Assert.False(c.IsConstraining); + Assert.True(c.IsConstraining); + // Required 'location' missing → the function can't close yet. + Feed(c, tok, "<"); + Assert.True(Allowed(c, vocab.VocabSize, tok.Encode("p")[0])); // forbidden + } + + [Fact] + public void JsonOutput_StillEngages_ViaComposite() + { + var tok = Tok(); + if (tok is null) { output.WriteLine("missing model — skip"); return; } + var vocab = new GrammarVocabulary(tok); + var c = new QwenToolCallAdapter("qwen3").BuildArgumentConstraint([Schema("get_weather", Weather)], vocab)!; + + // The JSON path is unchanged: the JSON sub-constraint still engages at the args-key colon. + Feed(c, tok, "\n{\"name\": \"get_weather\", \"arguments\": "); + Assert.True(c.IsConstraining); + Assert.False(Allowed(c, vocab.VocabSize, Single(tok, "{}"))); // merged empty-object still rejected } } diff --git a/tests/SharpInference.Tests.Core/JsonToolGrammarMockTests.cs b/tests/SharpInference.Tests.Core/JsonToolGrammarMockTests.cs index 1c254f9..47ad42c 100644 --- a/tests/SharpInference.Tests.Core/JsonToolGrammarMockTests.cs +++ b/tests/SharpInference.Tests.Core/JsonToolGrammarMockTests.cs @@ -343,4 +343,43 @@ public void NonConstrainableTool_BuildsNoConstraint() var schema = ToolSchema.FromOpenAiFunction("noop", doc.RootElement.Clone()); Assert.Null(new QwenToolCallAdapter("qwen3").BuildArgumentConstraint([schema], vocab)); } + + // The Qwen adapter overlays the JSON (#376) and Qwen3-Coder XML (#383) constraints in a + // CompositeToolArgumentConstraint, because the same architecture hosts both formats (#383). These + // exercise the composite's format dispatch model-free, so a CI runner without the GGUFs covers it. + + [Fact] + public void Composite_DispatchesToXmlSub_OnCoderOutput() + { + var (c, tok, vocab) = Build( + """{"type":"object","properties":{"location":{"type":"string"}},"required":["location"]}""", + "get_weather"); + Assert.IsType(c); + + // Coder XML output engages the XML sub through the composite and enforces the required param. + Feed(c, tok, "\n"); + Assert.True(c.IsConstraining); + Feed(c, tok, "<"); + Assert.True(Allowed(c, vocab, tok.Char('p'))); // forbidden — 'location' missing + } + + [Fact] + public void Composite_DispatchesToJsonSub_OnJsonOutput() + { + var (c, tok, vocab) = Build( + """{"type":"object","properties":{"location":{"type":"string"}},"required":["location"]}""", + "get_weather"); + Assert.IsType(c); + + // JSON output engages the JSON sub through the composite — unchanged from #376. + Feed(c, tok, QwenPreamble); + Assert.True(c.IsConstraining); + Assert.False(Allowed(c, vocab, tok.Merged("{}"))); // merged empty-object still rejected + Assert.True(Allowed(c, vocab, tok.Char('{'))); + + // Reset returns the whole composite to the watching (pass-through) state. + c.Reset(); + Assert.False(c.IsConstraining); + } }