From 50f9c2f9fd7213508b19b9087d451914d91696cc Mon Sep 17 00:00:00 2001 From: Christopher Petito Date: Fri, 10 Oct 2025 19:46:54 +0200 Subject: [PATCH] Don't use structured outputs in title creation and summarization Signed-off-by: Christopher Petito --- pkg/model/provider/anthropic/client.go | 11 +++++-- pkg/model/provider/clone.go | 42 ++++++++++++++++++++++++++ pkg/model/provider/dmr/client.go | 5 +++ pkg/model/provider/gemini/client.go | 5 +++ pkg/model/provider/openai/client.go | 5 +++ pkg/model/provider/options/options.go | 13 ++++++++ pkg/model/provider/provider.go | 2 ++ pkg/runtime/runtime.go | 10 ++++-- pkg/runtime/runtime_test.go | 5 +++ 9 files changed, 93 insertions(+), 5 deletions(-) create mode 100644 pkg/model/provider/clone.go diff --git a/pkg/model/provider/anthropic/client.go b/pkg/model/provider/anthropic/client.go index 3f3d186d3..17204d208 100644 --- a/pkg/model/provider/anthropic/client.go +++ b/pkg/model/provider/anthropic/client.go @@ -21,8 +21,9 @@ import ( // Client represents an Anthropic client wrapper implementing provider.Provider // It holds the anthropic client and model config type Client struct { - client anthropic.Client - config *latest.ModelConfig + client anthropic.Client + config *latest.ModelConfig + modelOptions options.ModelOptions // When using the Docker AI Gateway, tokens are short-lived. We rebuild // the client per request when in gateway mode. useGateway bool @@ -114,6 +115,7 @@ func NewClient(ctx context.Context, cfg *latest.ModelConfig, env environment.Pro return &Client{ client: client, config: cfg, + modelOptions: globalOptions, useGateway: useGateway, gatewayBaseURL: gatewayBaseURL, }, nil @@ -402,3 +404,8 @@ func ConvertParametersToSchema(params tools.FunctionParameters) anthropic.ToolIn func (c *Client) ID() string { return c.config.Provider + "/" + c.config.Model } + +// Options returns the effective model options used by this client. +func (c *Client) Options() options.ModelOptions { + return c.modelOptions +} diff --git a/pkg/model/provider/clone.go b/pkg/model/provider/clone.go new file mode 100644 index 000000000..de53dbad2 --- /dev/null +++ b/pkg/model/provider/clone.go @@ -0,0 +1,42 @@ +package provider + +import ( + "context" + "log/slog" + "strings" + + latest "github.com/docker/cagent/pkg/config/v2" + "github.com/docker/cagent/pkg/environment" + "github.com/docker/cagent/pkg/model/provider/options" +) + +// CloneWithOptions returns a new Provider instance using the same provider/model +// as the base provider, applying the provided options. If cloning fails, the +// original base provider is returned. +func CloneWithOptions(ctx context.Context, base Provider, env environment.Provider, opts ...options.Opt) Provider { + if base == nil { + return nil + } + + id := strings.TrimSpace(base.ID()) + parts := strings.SplitN(id, "/", 2) + if len(parts) != 2 { + return base + } + + cfg := &latest.ModelConfig{Provider: parts[0], Model: parts[1]} + if env == nil { + env = environment.NewDefaultProvider(ctx) + } + + // Preserve existing options, then apply overrides. Later opts take precedence. + baseOpts := options.FromModelOptions(base.Options()) + mergedOpts := append(baseOpts, opts...) + + cloned, err := New(ctx, cfg, env, mergedOpts...) + if err != nil { + slog.Debug("Failed to clone provider; using base provider", "error", err, "id", id) + return base + } + return cloned +} diff --git a/pkg/model/provider/dmr/client.go b/pkg/model/provider/dmr/client.go index 056f8e8d8..df0293cd7 100644 --- a/pkg/model/provider/dmr/client.go +++ b/pkg/model/provider/dmr/client.go @@ -442,6 +442,11 @@ func (c *Client) ID() string { return c.config.Provider + "/" + c.config.Model } +// Options returns the effective model options used by this client. +func (c *Client) Options() options.ModelOptions { + return c.modelOptions +} + func parseDMRProviderOpts(cfg *latest.ModelConfig) (contextSize int, runtimeFlags []string) { if cfg == nil { return 0, nil diff --git a/pkg/model/provider/gemini/client.go b/pkg/model/provider/gemini/client.go index c17f47ee5..d982e2ad0 100644 --- a/pkg/model/provider/gemini/client.go +++ b/pkg/model/provider/gemini/client.go @@ -414,3 +414,8 @@ func (c *Client) CreateChatCompletionStream( func (c *Client) ID() string { return c.config.Provider + "/" + c.config.Model } + +// Options returns the effective model options used by this client. +func (c *Client) Options() options.ModelOptions { + return c.modelOptions +} diff --git a/pkg/model/provider/openai/client.go b/pkg/model/provider/openai/client.go index 6dd0aa16b..3312a3628 100644 --- a/pkg/model/provider/openai/client.go +++ b/pkg/model/provider/openai/client.go @@ -367,6 +367,11 @@ func (c *Client) ID() string { return c.config.Provider + "/" + c.config.Model } +// Options returns the effective model options used by this client. +func (c *Client) Options() options.ModelOptions { + return c.modelOptions +} + // getOpenAIReasoningEffort resolves the reasoning effort value from the // model configuration's ThinkingBudget. Returns the effort (minimal|low|medium|high) or an error func getOpenAIReasoningEffort(cfg *latest.ModelConfig) (effort string, err error) { diff --git a/pkg/model/provider/options/options.go b/pkg/model/provider/options/options.go index 4df95e4ac..5cb17e77e 100644 --- a/pkg/model/provider/options/options.go +++ b/pkg/model/provider/options/options.go @@ -26,3 +26,16 @@ func WithStructuredOutput(output *latest.StructuredOutput) Opt { cfg.StructuredOutput = output } } + +// FromModelOptions converts a concrete ModelOptions value into a slice of +// Opt configuration functions. Later Opts override earlier ones when applied. +func FromModelOptions(m ModelOptions) []Opt { + var out []Opt + if g := m.Gateway(); g != "" { + out = append(out, WithGateway(g)) + } + if m.StructuredOutput != nil { + out = append(out, WithStructuredOutput(m.StructuredOutput)) + } + return out +} diff --git a/pkg/model/provider/provider.go b/pkg/model/provider/provider.go index d0c277b35..cd228a0b4 100644 --- a/pkg/model/provider/provider.go +++ b/pkg/model/provider/provider.go @@ -47,6 +47,8 @@ type Provider interface { messages []chat.Message, tools []tools.Tool, ) (chat.MessageStream, error) + // Options returns the effective model options used by this provider + Options() options.ModelOptions } func New(ctx context.Context, cfg *latest.ModelConfig, env environment.Provider, opts ...options.Opt) (Provider, error) { diff --git a/pkg/runtime/runtime.go b/pkg/runtime/runtime.go index bf37e2ca0..175ed1d0b 100644 --- a/pkg/runtime/runtime.go +++ b/pkg/runtime/runtime.go @@ -18,6 +18,8 @@ import ( "github.com/docker/cagent/pkg/agent" "github.com/docker/cagent/pkg/chat" + "github.com/docker/cagent/pkg/model/provider" + "github.com/docker/cagent/pkg/model/provider/options" "github.com/docker/cagent/pkg/modelsdev" "github.com/docker/cagent/pkg/session" "github.com/docker/cagent/pkg/team" @@ -959,9 +961,11 @@ func (r *runtime) generateSessionTitle(ctx context.Context, sess *session.Sessio systemPrompt := "You are a helpful AI assistant that generates concise, descriptive titles for conversations. You will be given a conversation history and asked to create a title that captures the main topic." userPrompt := fmt.Sprintf("Based on the following conversation between a user and an AI assistant, generate a short, descriptive title (maximum 50 characters) that captures the main topic or purpose of the conversation. Return ONLY the title text, nothing else.\n\nConversation history:%s\n\nGenerate a title for this conversation:", conversationHistory.String()) + titleModel := provider.CloneWithOptions(ctx, r.CurrentAgent().Model(), nil, options.WithStructuredOutput(nil)) + newTeam := team.New( team.WithID("title-generator"), - team.WithAgents(agent.New("root", systemPrompt, agent.WithModel(r.CurrentAgent().Model()))), + team.WithAgents(agent.New("root", systemPrompt, agent.WithModel(titleModel))), ) titleSession := session.New(session.WithSystemMessage(systemPrompt)) @@ -1019,10 +1023,10 @@ func (r *runtime) Summarize(ctx context.Context, sess *session.Session, events c // Create a new session for summary generation systemPrompt := "You are a helpful AI assistant that creates comprehensive summaries of conversations. You will be given a conversation history and asked to create a concise yet thorough summary that captures the key points, decisions made, and outcomes." userPrompt := fmt.Sprintf("Based on the following conversation between a user and an AI assistant, create a comprehensive summary that captures:\n- The main topics discussed\n- Key information exchanged\n- Decisions made or conclusions reached\n- Important outcomes or results\n\nProvide a well-structured summary (2-4 paragraphs) that someone could read to understand what happened in this conversation. Return ONLY the summary text, nothing else.\n\nConversation history:%s\n\nGenerate a summary for this conversation:", conversationHistory.String()) - + newModel := provider.CloneWithOptions(ctx, r.CurrentAgent().Model(), nil, options.WithStructuredOutput(nil)) newTeam := team.New( team.WithID("summary-generator"), - team.WithAgents(agent.New("root", systemPrompt, agent.WithModel(r.CurrentAgent().Model()))), + team.WithAgents(agent.New("root", systemPrompt, agent.WithModel(newModel))), ) summarySession := session.New(session.WithSystemMessage(systemPrompt)) diff --git a/pkg/runtime/runtime_test.go b/pkg/runtime/runtime_test.go index a65380405..99fe2926e 100644 --- a/pkg/runtime/runtime_test.go +++ b/pkg/runtime/runtime_test.go @@ -11,6 +11,7 @@ import ( "github.com/docker/cagent/pkg/agent" "github.com/docker/cagent/pkg/chat" + "github.com/docker/cagent/pkg/model/provider/options" "github.com/docker/cagent/pkg/modelsdev" "github.com/docker/cagent/pkg/session" "github.com/docker/cagent/pkg/team" @@ -112,6 +113,8 @@ func (m *mockProvider) CreateChatCompletionStream(ctx context.Context, messages return m.stream, nil } +func (m *mockProvider) Options() options.ModelOptions { return options.ModelOptions{} } + type mockProviderWithError struct { id string } @@ -122,6 +125,8 @@ func (m *mockProviderWithError) CreateChatCompletionStream(ctx context.Context, return nil, fmt.Errorf("simulated error creating chat completion stream") } +func (m *mockProviderWithError) Options() options.ModelOptions { return options.ModelOptions{} } + type mockModelStore struct{} func (m mockModelStore) GetModel(ctx context.Context, id string) (*modelsdev.Model, error) {