Skip to content

test: make models-list tests hermetic and parallelizable #3388

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Sayt-0 merged 1 commit into from
Jul 1, 2026
Merged

test: make models-list tests hermetic and parallelizable #3388

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 13 additions & 3 deletions cmd/root/models.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,14 @@ type modelRow struct {
Default bool `json:"default,omitempty"`
}

func newModelsCmd() *cobra.Command {
// modelsCmdOption pre-seeds the RuntimeConfig backing the models command
// before its flags are wired. It is the seam tests use to inject a hermetic
// env provider and an in-memory models.dev store, so listing models never
// shells out to the OS keychain or reaches the network. Production calls
// newModelsCmd with no options, leaving behavior unchanged.
type modelsCmdOption func(*config.RuntimeConfig)

func newModelsCmd(opts ...modelsCmdOption) *cobra.Command {
cmd := &cobra.Command{
Use: "models",
Short: "List available models",
Expand All @@ -66,7 +73,7 @@ you have credentials for. Use --all to include all providers.`,
GroupID: "core",
}

listCmd := newModelsListCmd()
listCmd := newModelsListCmd(opts...)
cmd.AddCommand(listCmd)

// Default to "list" when no subcommand given.
Expand All @@ -79,8 +86,11 @@ you have credentials for. Use --all to include all providers.`,
return cmd
}

func newModelsListCmd() *cobra.Command {
func newModelsListCmd(opts ...modelsCmdOption) *cobra.Command {
var flags modelsListFlags
for _, opt := range opts {
opt(&flags.runConfig)
}

cmd := &cobra.Command{
Use: "list",
Expand Down
156 changes: 80 additions & 76 deletions cmd/root/models_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,55 +14,86 @@ import (
"github.com/stretchr/testify/require"

"github.com/docker/docker-agent/pkg/config"
"github.com/docker/docker-agent/pkg/paths"
"github.com/docker/docker-agent/pkg/environment"
"github.com/docker/docker-agent/pkg/modelsdev"
)

func TestModelsListCommand_DefaultOutput(t *testing.T) {
// With ANTHROPIC_API_KEY set, the default output should include
// at least the anthropic default model.
t.Setenv("ANTHROPIC_API_KEY", "test-key")
t.Setenv("DOCKER_AGENT_MODELS_GATEWAY", "")
t.Setenv("DOCKER_AGENT_DEFAULT_MODEL", "")
// catalogOnlyModel is a text model present only in the in-memory test catalog
// (not in config.DefaultModels). Asserting it surfaces proves the command
// actually reads the injected models.dev store rather than relying solely on
// the per-provider defaults, which would be added even with an empty catalog.
const catalogOnlyModel = "claude-catalog-only"

// testCatalog is a tiny in-memory models.dev database used by the models-list
// tests so they never fetch the real catalog over the network or read the
// developer's on-disk cache.
func testCatalog() *modelsdev.Database {
return &modelsdev.Database{
Providers: map[string]modelsdev.Provider{
"anthropic": {Models: map[string]modelsdev.Model{
"claude-sonnet-4-6": {
Name: "Claude Sonnet 4.6",
Modalities: modelsdev.Modalities{Output: []string{"text"}},
},
catalogOnlyModel: {
Name: "Claude Catalog Only",
Modalities: modelsdev.Modalities{Output: []string{"text"}},
},
}},
"openai": {Models: map[string]modelsdev.Model{
"gpt-5": {
Name: "GPT-5",
Modalities: modelsdev.Modalities{Output: []string{"text"}},
},
}},
},
}
}

// Point at an empty config dir so userconfig.Load returns an empty
// config without reading the developer's real ~/.config/cagent.
paths.SetConfigDir(t.TempDir())
t.Cleanup(func() { paths.SetConfigDir("") })
// withTestConfig injects a hermetic env provider and an in-memory models.dev
// store into the models command. It keeps listing side-effect-free: without it
// the real env provider chain shells out to the OS keychain / pass / 1Password
// for every missing API key and the store fetches https://models.dev, making
// the tests slow and non-parallelizable.
func withTestConfig(env map[string]string) modelsCmdOption {
return func(rc *config.RuntimeConfig) {
rc.EnvProviderForTests = environment.NewMapEnvProvider(env)
rc.ModelsDevStoreOverride = modelsdev.NewDatabaseStore(testCatalog())
}
}

func TestModelsListCommand_DefaultOutput(t *testing.T) {
t.Parallel()

var buf bytes.Buffer
cmd := newModelsCmd()
cmd := newModelsCmd(withTestConfig(map[string]string{"ANTHROPIC_API_KEY": "test-key"}))
cmd.SetOut(&buf)
cmd.SetErr(&buf)
cmd.SetArgs(nil)

err := cmd.Execute()
require.NoError(t, err)
require.NoError(t, cmd.Execute())

output := buf.String()
assert.Contains(t, output, "PROVIDER")
assert.Contains(t, output, "MODEL")
assert.Contains(t, output, "anthropic")
// A catalog-only model must appear, proving the injected store was read.
assert.Contains(t, output, catalogOnlyModel)
}

func TestModelsListCommand_ProviderFilter(t *testing.T) {
t.Setenv("ANTHROPIC_API_KEY", "test-key")
t.Setenv("OPENAI_API_KEY", "test-key")
t.Setenv("DOCKER_AGENT_MODELS_GATEWAY", "")
t.Setenv("DOCKER_AGENT_DEFAULT_MODEL", "")

// Point at an empty config dir so userconfig.Load returns an empty
// config without reading the developer's real ~/.config/cagent.
paths.SetConfigDir(t.TempDir())
t.Cleanup(func() { paths.SetConfigDir("") })
t.Parallel()

var buf bytes.Buffer
cmd := newModelsCmd()
cmd := newModelsCmd(withTestConfig(map[string]string{
"ANTHROPIC_API_KEY": "test-key",
"OPENAI_API_KEY": "test-key",
}))
cmd.SetOut(&buf)
cmd.SetErr(&buf)
cmd.SetArgs([]string{"--provider", "anthropic"})

err := cmd.Execute()
require.NoError(t, err)
require.NoError(t, cmd.Execute())

output := buf.String()
// Every non-header line should be anthropic
Expand All @@ -77,27 +108,18 @@ func TestModelsListCommand_ProviderFilter(t *testing.T) {
}

func TestModelsListCommand_JSONFormat(t *testing.T) {
t.Setenv("ANTHROPIC_API_KEY", "test-key")
t.Setenv("DOCKER_AGENT_MODELS_GATEWAY", "")
t.Setenv("DOCKER_AGENT_DEFAULT_MODEL", "")

// Point at an empty config dir so userconfig.Load returns an empty
// config without reading the developer's real ~/.config/cagent.
paths.SetConfigDir(t.TempDir())
t.Cleanup(func() { paths.SetConfigDir("") })
t.Parallel()

var buf bytes.Buffer
cmd := newModelsCmd()
cmd := newModelsCmd(withTestConfig(map[string]string{"ANTHROPIC_API_KEY": "test-key"}))
cmd.SetOut(&buf)
cmd.SetErr(&buf)
cmd.SetArgs([]string{"--format", "json"})

err := cmd.Execute()
require.NoError(t, err)
require.NoError(t, cmd.Execute())

var rows []modelRow
err = json.Unmarshal(buf.Bytes(), &rows)
require.NoError(t, err)
require.NoError(t, json.Unmarshal(buf.Bytes(), &rows))
assert.NotEmpty(t, rows)

// At least one should be the default
Expand All @@ -112,36 +134,33 @@ func TestModelsListCommand_JSONFormat(t *testing.T) {
}

func TestModelsListCommand_DefaultMarker(t *testing.T) {
// When a default model is configured via env, it should be marked.
t.Setenv("ANTHROPIC_API_KEY", "test-key")
t.Setenv("DOCKER_AGENT_MODELS_GATEWAY", "")
t.Setenv("DOCKER_AGENT_DEFAULT_MODEL", "")
t.Parallel()

// Point at an empty config dir so userconfig.Load returns an empty
// config without reading the developer's real ~/.config/cagent.
paths.SetConfigDir(t.TempDir())
t.Cleanup(func() { paths.SetConfigDir("") })
env := map[string]string{"ANTHROPIC_API_KEY": "test-key"}

var buf bytes.Buffer
cmd := newModelsCmd()
cmd := newModelsCmd(withTestConfig(env))
cmd.SetOut(&buf)
cmd.SetErr(&buf)
cmd.SetArgs([]string{"--format", "json"})

err := cmd.Execute()
require.NoError(t, err)
require.NoError(t, cmd.Execute())

var rows []modelRow
require.NoError(t, json.Unmarshal(buf.Bytes(), &rows))

// The auto-selected model should be marked as default
rc := config.RuntimeConfig{}
autoModel := config.AutoModelConfig(t.Context(), "", rc.EnvProvider(), nil, nil)
// Exactly one row should be marked default, and it must be the
// auto-selected model for this environment.
autoModel := config.AutoModelConfig(t.Context(), "", environment.NewMapEnvProvider(env), nil, nil)
var defaults []modelRow
for _, r := range rows {
if r.Provider == autoModel.Provider && r.Model == autoModel.Model {
assert.True(t, r.Default, "auto-selected model %s/%s should be marked as default", r.Provider, r.Model)
if r.Default {
defaults = append(defaults, r)
}
}
require.Len(t, defaults, 1, "expected exactly one default model")
assert.Equal(t, autoModel.Provider, defaults[0].Provider)
assert.Equal(t, autoModel.Model, defaults[0].Model)
}

func TestFetchModelsFromURL_Success(t *testing.T) {
Expand Down Expand Up @@ -277,6 +296,7 @@ func TestFetchModelsFromURL_ContextCanceled(t *testing.T) {

models := fetchModelsFromURL(ctx, server.URL+"/v1/models", server.Client())
assert.Empty(t, models)
assert.False(t, called.Load(), "server must not be reached with an already-canceled context")
}

func TestFetchModelsFromURL_SkipsEmbeddingModels(t *testing.T) {
Expand All @@ -299,33 +319,17 @@ func TestFetchModelsFromURL_SkipsEmbeddingModels(t *testing.T) {
}

func TestModelsListCommand_NoCredentials(t *testing.T) {
// Clear all provider keys — only DMR should remain as fallback.
t.Setenv("ANTHROPIC_API_KEY", "")
t.Setenv("OPENAI_API_KEY", "")
t.Setenv("GOOGLE_API_KEY", "")
t.Setenv("GEMINI_API_KEY", "")
t.Setenv("MISTRAL_API_KEY", "")
t.Setenv("AWS_ACCESS_KEY_ID", "")
t.Setenv("AWS_PROFILE", "")
t.Setenv("AWS_ROLE_ARN", "")
t.Setenv("DOCKER_AGENT_MODELS_GATEWAY", "")
t.Setenv("DOCKER_AGENT_DEFAULT_MODEL", "")

// Point at an empty config dir so userconfig.Load returns an empty
// config without reading the developer's real ~/.config/cagent.
paths.SetConfigDir(t.TempDir())
t.Cleanup(func() { paths.SetConfigDir("") })
t.Parallel()

// No provider keys — only DMR should remain as fallback.
var buf bytes.Buffer
cmd := newModelsCmd()
cmd := newModelsCmd(withTestConfig(map[string]string{}))
cmd.SetOut(&buf)
cmd.SetErr(&buf)
cmd.SetArgs(nil)

err := cmd.Execute()
require.NoError(t, err)
require.NoError(t, cmd.Execute())

output := buf.String()
// DMR is always available as fallback
assert.Contains(t, output, "dmr")
assert.Contains(t, buf.String(), "dmr")
}
Loading