diff --git a/CHANGELOG.md b/CHANGELOG.md index 560cacde..78257144 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed +- [ask sb] Fixed issue where reasoning tokens would appear in `text` content for openai compatible models. [#582](https://github.com/sourcebot-dev/sourcebot/pull/582) + ## [4.8.1] - 2025-10-29 ### Fixed diff --git a/docs/docs/configuration/language-model-providers.mdx b/docs/docs/configuration/language-model-providers.mdx index 606f4d31..5873e9d4 100644 --- a/docs/docs/configuration/language-model-providers.mdx +++ b/docs/docs/configuration/language-model-providers.mdx @@ -292,6 +292,7 @@ The OpenAI compatible provider allows you to use any model that is compatible wi - When using [llama.cpp](https://github.com/ggml-org/llama.cpp), if you hit "Failed after 3 attempts. Last error: tools param requires --jinja flag", add the `--jinja` flag to your `llama-server` command. +- If you're seeing the LLM outputing reasoning tokens wrapped in XML tags (e.g., ``, ``, etc.), you can configure the `reasoningTag` parameter to the name of the tag (without angle brackets). This parameter defaults to `think`. ### OpenRouter diff --git a/docs/snippets/schemas/v3/index.schema.mdx b/docs/snippets/schemas/v3/index.schema.mdx index 84c73bc5..89ed6be2 100644 --- a/docs/snippets/schemas/v3/index.schema.mdx +++ b/docs/snippets/schemas/v3/index.schema.mdx @@ -2633,6 +2633,16 @@ } }, "additionalProperties": false + }, + "reasoningTag": { + "type": "string", + "description": "The name of the XML tag to extract reasoning from (without angle brackets). Defaults to `think`.", + "default": "think", + "examples": [ + "think", + "thinking", + "reasoning" + ] } }, "required": [ @@ -4052,6 +4062,16 @@ } }, "additionalProperties": false + }, + "reasoningTag": { + "type": "string", + "description": "The name of the XML tag to extract reasoning from (without angle brackets). Defaults to `think`.", + "default": "think", + "examples": [ + "think", + "thinking", + "reasoning" + ] } }, "required": [ diff --git a/docs/snippets/schemas/v3/languageModel.schema.mdx b/docs/snippets/schemas/v3/languageModel.schema.mdx index d90c4a76..accfc95c 100644 --- a/docs/snippets/schemas/v3/languageModel.schema.mdx +++ b/docs/snippets/schemas/v3/languageModel.schema.mdx @@ -1202,6 +1202,16 @@ } }, "additionalProperties": false + }, + "reasoningTag": { + "type": "string", + "description": "The name of the XML tag to extract reasoning from (without angle brackets). Defaults to `think`.", + "default": "think", + "examples": [ + "think", + "thinking", + "reasoning" + ] } }, "required": [ @@ -2621,6 +2631,16 @@ } }, "additionalProperties": false + }, + "reasoningTag": { + "type": "string", + "description": "The name of the XML tag to extract reasoning from (without angle brackets). Defaults to `think`.", + "default": "think", + "examples": [ + "think", + "thinking", + "reasoning" + ] } }, "required": [ diff --git a/packages/schemas/src/v3/index.schema.ts b/packages/schemas/src/v3/index.schema.ts index d5125c69..e49d998b 100644 --- a/packages/schemas/src/v3/index.schema.ts +++ b/packages/schemas/src/v3/index.schema.ts @@ -2632,6 +2632,16 @@ const schema = { } }, "additionalProperties": false + }, + "reasoningTag": { + "type": "string", + "description": "The name of the XML tag to extract reasoning from (without angle brackets). Defaults to `think`.", + "default": "think", + "examples": [ + "think", + "thinking", + "reasoning" + ] } }, "required": [ @@ -4051,6 +4061,16 @@ const schema = { } }, "additionalProperties": false + }, + "reasoningTag": { + "type": "string", + "description": "The name of the XML tag to extract reasoning from (without angle brackets). Defaults to `think`.", + "default": "think", + "examples": [ + "think", + "thinking", + "reasoning" + ] } }, "required": [ diff --git a/packages/schemas/src/v3/index.type.ts b/packages/schemas/src/v3/index.type.ts index 53ae9533..eb2b412c 100644 --- a/packages/schemas/src/v3/index.type.ts +++ b/packages/schemas/src/v3/index.type.ts @@ -974,6 +974,10 @@ export interface OpenAICompatibleLanguageModel { baseUrl: string; headers?: LanguageModelHeaders; queryParams?: LanguageModelQueryParams; + /** + * The name of the XML tag to extract reasoning from (without angle brackets). Defaults to `think`. + */ + reasoningTag?: string; } /** * Optional query parameters to include in the request url. diff --git a/packages/schemas/src/v3/languageModel.schema.ts b/packages/schemas/src/v3/languageModel.schema.ts index cd879e59..3bdb4c00 100644 --- a/packages/schemas/src/v3/languageModel.schema.ts +++ b/packages/schemas/src/v3/languageModel.schema.ts @@ -1201,6 +1201,16 @@ const schema = { } }, "additionalProperties": false + }, + "reasoningTag": { + "type": "string", + "description": "The name of the XML tag to extract reasoning from (without angle brackets). Defaults to `think`.", + "default": "think", + "examples": [ + "think", + "thinking", + "reasoning" + ] } }, "required": [ @@ -2620,6 +2630,16 @@ const schema = { } }, "additionalProperties": false + }, + "reasoningTag": { + "type": "string", + "description": "The name of the XML tag to extract reasoning from (without angle brackets). Defaults to `think`.", + "default": "think", + "examples": [ + "think", + "thinking", + "reasoning" + ] } }, "required": [ diff --git a/packages/schemas/src/v3/languageModel.type.ts b/packages/schemas/src/v3/languageModel.type.ts index 803a6c17..58b9aac6 100644 --- a/packages/schemas/src/v3/languageModel.type.ts +++ b/packages/schemas/src/v3/languageModel.type.ts @@ -453,6 +453,10 @@ export interface OpenAICompatibleLanguageModel { baseUrl: string; headers?: LanguageModelHeaders; queryParams?: LanguageModelQueryParams; + /** + * The name of the XML tag to extract reasoning from (without angle brackets). Defaults to `think`. + */ + reasoningTag?: string; } /** * Optional query parameters to include in the request url. diff --git a/packages/web/src/features/chat/actions.ts b/packages/web/src/features/chat/actions.ts index 4f93ac1d..0e9638c4 100644 --- a/packages/web/src/features/chat/actions.ts +++ b/packages/web/src/features/chat/actions.ts @@ -25,7 +25,7 @@ import { ChatVisibility, OrgRole, Prisma, PrismaClient } from "@sourcebot/db"; import { LanguageModel } from "@sourcebot/schemas/v3/languageModel.type"; import { Token } from "@sourcebot/schemas/v3/shared.type"; import { loadConfig } from "@sourcebot/shared"; -import { generateText, JSONValue } from "ai"; +import { generateText, JSONValue, extractReasoningMiddleware, wrapLanguageModel } from "ai"; import fs from 'fs'; import { StatusCodes } from "http-status-codes"; import path from 'path'; @@ -568,8 +568,17 @@ export const _getAISDKLanguageModelAndOptions = async (config: LanguageModel, or : undefined, }); - return { + const model = wrapLanguageModel({ model: openai.chatModel(modelId), + middleware: [ + extractReasoningMiddleware({ + tagName: config.reasoningTag ?? 'think', + }), + ] + }); + + return { + model, } } case 'openrouter': { diff --git a/schemas/v3/languageModel.json b/schemas/v3/languageModel.json index 613eb87b..e56df4ce 100644 --- a/schemas/v3/languageModel.json +++ b/schemas/v3/languageModel.json @@ -425,6 +425,16 @@ }, "queryParams": { "$ref": "./shared.json#/definitions/LanguageModelQueryParams" + }, + "reasoningTag": { + "type": "string", + "description": "The name of the XML tag to extract reasoning from (without angle brackets). Defaults to `think`.", + "default": "think", + "examples": [ + "think", + "thinking", + "reasoning" + ] } }, "required": [