Make secure token storage the default storage mode

NEXT_CHANGELOG.md

@@ -4,6 +4,8 @@
 
 ### Notable Changes
 
+* Breaking change: U2M (`auth_type = databricks-cli`) tokens are now stored in the OS-native secure store by default (Keychain on macOS, Credential Manager on Windows, Secret Service on Linux) instead of `~/.databricks/token-cache.json`. After upgrading, run `databricks auth login` once per profile to re-authenticate; cached tokens from older versions are not migrated. To keep the previous file-backed storage, set `DATABRICKS_AUTH_STORAGE=plaintext` or add `auth_storage = plaintext` under `[__settings__]` in `~/.databrickscfg` (the env var takes precedence over the config setting), then re-run `databricks auth login`.
+
 ### CLI
 
 * Added `databricks aitools` command group for installing Databricks skills into your coding agents (Claude Code, Cursor, Codex CLI, OpenCode, GitHub Copilot, Antigravity). Skills are fetched from [github.com/databricks/databricks-agent-skills](https://github.com/databricks/databricks-agent-skills) and either symlinked into each agent's skills directory or copied into the current project. Use `databricks aitools install` to set up, `update` to pull newer versions, `list` to see what's available, and `uninstall` to remove them.

acceptance/cmd/auth/describe/u2m-json-output/output.txt

@@ -2,7 +2,7 @@
 >>> [CLI] auth describe --profile u2m-profile --output json
 Warn: [hostmetadata] failed to fetch host metadata for https://u2m-profile.databricks.test, will skip for 1m0s
 {
-  "mode": "plaintext",
-  "location": "~/.databricks/token-cache.json",
+  "mode": "secure",
+  "location": "OS keyring (service: databricks-cli)",
   "source": "default"
 }

acceptance/cmd/auth/describe/u2m-plaintext-default/output.txt → acceptance/cmd/auth/describe/u2m-secure-default/output.txt

@@ -1,8 +1,8 @@
 
 >>> [CLI] auth describe --profile u2m-profile
 Warn: [hostmetadata] failed to fetch host metadata for https://u2m-profile.databricks.test, will skip for 1m0s
-Unable to authenticate: error getting token: cache: token not found
-Token storage: plaintext, ~/.databricks/token-cache.json (from default)
+Unable to authenticate: error getting token: [KEYRING_LOOKUP_ERROR]
+Token storage: secure, OS keyring (service: databricks-cli) (from default)
 -----
 Current configuration:
   ✓ host: https://u2m-profile.databricks.test (from [TEST_TMP_DIR]/home/.databrickscfg config file)

acceptance/cmd/auth/describe/u2m-secure-default/test.toml

@@ -0,0 +1,11 @@
+Ignore = [
+    "home"
+]
+
+# This test runs against the real OS keyring at Lookup time (no writes).
+# macOS produces a clean miss; Linux without a usable D-Bus session bus
+# produces a backend error. Normalize both so the assertion stays on the
+# resolved storage mode, not the lookup outcome.
+[[Repls]]
+Old = 'Unable to authenticate: error getting token: .*'
+New = 'Unable to authenticate: error getting token: [KEYRING_LOOKUP_ERROR]'

acceptance/script.prepare

@@ -1,3 +1,9 @@
+# Force plaintext token storage so acceptance tests exercise the file-backed
+# cache rather than the OS keyring, which is not reliably reachable in CI.
+# Tests that want to exercise the secure path or the resolver default unset
+# or override this in their own script.prepare or script.
+export DATABRICKS_AUTH_STORAGE=plaintext
+
 errcode() {
     # Temporarily disable 'set -e' to prevent the script from exiting on error
     set +e

cmd/auth/auth_test.go

@@ -4,6 +4,7 @@ import (
 	"testing"
 
 	"github.com/databricks/cli/cmd/root"
+	"github.com/databricks/cli/libs/auth/storage"
 	"github.com/databricks/cli/libs/cmdctx"
 	"github.com/databricks/cli/libs/databrickscfg/profile"
 	"github.com/stretchr/testify/assert"
@@ -114,6 +115,10 @@ func TestProfileHostConflictTokenViaCobra(t *testing.T) {
 // NOT with a conflict error).
 func TestProfileHostCompatibleViaCobra(t *testing.T) {
 	t.Setenv("DATABRICKS_CONFIG_FILE", "./testdata/.databrickscfg")
+	// Force plaintext so the storage resolver does not probe the OS keyring
+	// and silently persist auth_storage = plaintext to the checked-in fixture
+	// on CI runners without a usable keyring.
+	t.Setenv(storage.EnvVar, "plaintext")
 
 	ctx := cmdctx.GenerateExecId(t.Context())
 	cli := root.New(ctx)

cmd/auth/describe_test.go

@@ -240,21 +240,21 @@ func TestResolveTokenStorageInfo(t *testing.T) {
 			want:     nil,
 		},
 		{
-			name:     "databricks-cli with default plaintext",
+			name:     "databricks-cli with default secure",
 			authType: authTypeDatabricksCLI,
 			want: &tokenStorageInfo{
-				Mode:     "plaintext",
-				Location: plaintextLocation,
+				Mode:     "secure",
+				Location: secureLocation,
 				Source:   "default",
 			},
 		},
 		{
-			name:     "databricks-cli with secure from env",
+			name:     "databricks-cli with plaintext from env",
 			authType: authTypeDatabricksCLI,
-			envValue: "secure",
+			envValue: "plaintext",
 			want: &tokenStorageInfo{
-				Mode:     "secure",
-				Location: secureLocation,
+				Mode:     "plaintext",
+				Location: plaintextLocation,
 				Source:   "DATABRICKS_AUTH_STORAGE environment variable",
 			},
 		},

cmd/auth/login.go

@@ -144,15 +144,6 @@ a new profile is created.
 		ctx := cmd.Context()
 		profileName := cmd.Flag("profile").Value.String()
 
-		// Resolve the cache before the browser step so an unavailable
-		// keyring surfaces here rather than after OAuth. The probe also
-		// triggers the OS unlock prompt, which the user can answer during
-		// OAuth.
-		tokenCache, mode, err := storage.ResolveCacheForLogin(ctx, "")
-		if err != nil {
-			return err
-		}
-
 		// Cluster and Serverless are mutually exclusive.
 		if configureCluster && configureServerless {
 			return errors.New("please either configure serverless or cluster, not both")
@@ -178,6 +169,16 @@ a new profile is created.
 			}
 		}
 
+		// Resolve the cache before the browser step so an unavailable
+		// keyring surfaces here rather than after OAuth. The probe also
+		// triggers the OS unlock prompt, which the user can answer during
+		// OAuth. Run after input validation so trivially-invalid commands
+		// fail without probing.
+		tokenCache, mode, err := storage.ResolveCacheForLogin(ctx, "")
+		if err != nil {
+			return err
+		}
+
 		// When interactive and nothing was specified, show a picker that lets
 		// the user re-login to an existing profile, create a new one, or enter
 		// a host URL. With no profiles configured the picker still shows the
@@ -294,6 +295,11 @@ a new profile is created.
 		if err = persistentAuth.Challenge(); err != nil {
 			return err
 		}
+		// Lock secure mode in after a successful keyring write so a later
+		// transient keyring probe failure cannot silently demote this user
+		// to plaintext.
+		storage.PinSecureMode(ctx, mode)
+
 		// At this point, an OAuth token has been successfully minted and stored
 		// in the CLI cache. The rest of the command focuses on:
 		// 1. Workspace selection for SPOG hosts (best-effort);
@@ -633,6 +639,7 @@ func discoveryLogin(ctx context.Context, in discoveryLoginInputs) error {
 	if err := persistentAuth.Challenge(); err != nil {
 		return discoveryErr("login via login.databricks.com failed", err)
 	}
+	storage.PinSecureMode(ctx, in.mode)
 
 	discoveredHost := arg.GetDiscoveredHost()
 	if discoveredHost == "" {

cmd/auth/token.go

@@ -433,6 +433,7 @@ func runInlineLogin(ctx context.Context, profiler profile.Profiler, tokenCache c
 	if err = persistentAuth.Challenge(); err != nil {
 		return "", nil, err
 	}
+	storage.PinSecureMode(ctx, mode)
 
 	clearKeys := oauthLoginClearKeys()
 	clearKeys = append(clearKeys, databrickscfg.ExperimentalIsUnifiedHostKey)

libs/auth/storage/cache.go

@@ -59,10 +59,6 @@ func ResolveCache(ctx context.Context, override StorageMode) (cache.TokenCache,
 //     The timeout is ambiguous (locked vs hung); a misdiagnosis fails
 //     the final Store rather than silently downgrading to plaintext.
 //
-// Rules 1 and 2 are dormant today: the resolver default is plaintext, so
-// (mode=Secure, explicit=false) is unreachable. They activate when the
-// default flips to secure at GA.
-//
 // Login-specific. Read paths (auth token, bundle commands) keep the original
 // keyring error so they don't silently mint plaintext copies of tokens that
 // were stored in the keyring on another machine.
@@ -120,12 +116,6 @@ func resolveCacheForLoginWith(ctx context.Context, override StorageMode, f cache
 // resolved mode and whether the user explicitly asked for it. Split out so
 // tests can drive the (mode, explicit) input space directly without depending
 // on whatever the resolver's default mode happens to be at any point in time.
-//
-// Pin-on-success across modes (locking in the first working behavior to
-// insulate users from keyring flakiness) is intentionally not implemented
-// here. It lands at GA alongside the default flip; pinning before the
-// flip would freeze every default user into plaintext and make the flip a
-// no-op for them.
 func applyLoginFallback(ctx context.Context, mode StorageMode, explicit bool, f cacheFactories) (cache.TokenCache, StorageMode, error) {
 	switch mode {
 	case StorageModePlaintext:
@@ -168,12 +158,38 @@ func applyLoginFallback(ctx context.Context, mode StorageMode, explicit bool, f
 // in .databrickscfg so subsequent commands skip the (slow/blocking) keyring
 // probe and route straight to the file cache.
 //
-// Only called on the (mode=Secure, explicit=false) probe-failure branch. That
-// branch is unreachable today (resolver default is plaintext), so this is
-// dormant infrastructure: it activates when the default flips to secure
-// at GA and lets default-on-broken-keyring users avoid a 3s probe on every
-// command.
+// Only called on the (mode=Secure, explicit=false) probe-failure branch.
+// Persisting the fallback lets default-on-broken-keyring users avoid a 3s
+// probe on every command.
 func persistPlaintextFallback(ctx context.Context) error {
 	configPath := env.Get(ctx, "DATABRICKS_CONFIG_FILE")
 	return databrickscfg.SetConfiguredAuthStorage(ctx, string(StorageModePlaintext), configPath)
 }
+
+// PinSecureMode persists auth_storage = secure to [__settings__] when the
+// user is currently on the secure-from-default path. Once pinned, subsequent
+// invocations see source=Config (explicit), so applyLoginFallback returns an
+// error on a transient keyring probe failure instead of silently demoting
+// the user to plaintext.
+//
+// No-op when mode is not secure or when the user already chose a mode
+// explicitly. Best-effort: persistence failures are logged at debug and
+// never block login. Concurrent logins racing this write is benign because
+// both write the same value.
+func PinSecureMode(ctx context.Context, mode StorageMode) {
+	if mode != StorageModeSecure {
+		return
+	}
+	_, source, err := ResolveStorageModeWithSource(ctx, "")
+	if err != nil {
+		log.Debugf(ctx, "pin secure mode: resolve: %v", err)
+		return
+	}
+	if source.Explicit() {
+		return
+	}
+	configPath := env.Get(ctx, "DATABRICKS_CONFIG_FILE")
+	if err := databrickscfg.SetConfiguredAuthStorage(ctx, string(StorageModeSecure), configPath); err != nil {
+		log.Debugf(ctx, "pin secure mode: persist: %v", err)
+	}
+}

libs/auth/storage/cache_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"testing"
@@ -41,15 +42,15 @@ func hermetic(t *testing.T) {
 	t.Setenv("DATABRICKS_CONFIG_FILE", filepath.Join(t.TempDir(), "databrickscfg"))
 }
 
-func TestResolveCache_DefaultsToPlaintextFile(t *testing.T) {
+func TestResolveCache_DefaultsToSecureKeyring(t *testing.T) {
 	hermetic(t)
 	ctx := t.Context()
 
 	got, mode, err := resolveCacheWith(ctx, "", fakeFactories(t))
 
 	require.NoError(t, err)
-	assert.Equal(t, StorageModePlaintext, mode)
-	assert.Equal(t, "file", got.(stubCache).source)
+	assert.Equal(t, StorageModeSecure, mode)
+	assert.Equal(t, "keyring", got.(stubCache).source)
 }
 
 func TestResolveCache_OverrideSecureUsesKeyring(t *testing.T) {
@@ -272,6 +273,93 @@ func TestApplyLoginFallback_ProbeTimeout_StaysOnKeyring(t *testing.T) {
 	}
 }
 
+func TestPinSecureMode(t *testing.T) {
+	cases := []struct {
+		name        string
+		mode        StorageMode
+		envValue    string
+		configBody  string
+		wantWritten string
+	}{
+		{
+			name:        "secure from default persists secure",
+			mode:        StorageModeSecure,
+			wantWritten: "secure",
+		},
+		{
+			name:        "plaintext mode is a no-op",
+			mode:        StorageModePlaintext,
+			wantWritten: "",
+		},
+		{
+			name:        "secure from env is a no-op",
+			mode:        StorageModeSecure,
+			envValue:    "secure",
+			wantWritten: "",
+		},
+		{
+			name:        "secure from config is a no-op (already pinned)",
+			mode:        StorageModeSecure,
+			configBody:  "[__settings__]\nauth_storage = secure\n",
+			wantWritten: "secure",
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			hermetic(t)
+			ctx := t.Context()
+			configPath := env.Get(ctx, "DATABRICKS_CONFIG_FILE")
+			if tc.configBody != "" {
+				require.NoError(t, os.WriteFile(configPath, []byte(tc.configBody), 0o600))
+			}
+			if tc.envValue != "" {
+				ctx = env.Set(ctx, EnvVar, tc.envValue)
+			}
+
+			PinSecureMode(ctx, tc.mode)
+
+			got, err := databrickscfg.GetConfiguredAuthStorage(ctx, configPath)
+			require.NoError(t, err)
+			assert.Equal(t, tc.wantWritten, got)
+		})
+	}
+}
+
+func TestPinSecureMode_IsIdempotent(t *testing.T) {
+	hermetic(t)
+	ctx := t.Context()
+	configPath := env.Get(ctx, "DATABRICKS_CONFIG_FILE")
+
+	PinSecureMode(ctx, StorageModeSecure)
+	first, err := databrickscfg.GetConfiguredAuthStorage(ctx, configPath)
+	require.NoError(t, err)
+	require.Equal(t, "secure", first)
+
+	// Second call should see source=Config and skip the write.
+	PinSecureMode(ctx, StorageModeSecure)
+	second, err := databrickscfg.GetConfiguredAuthStorage(ctx, configPath)
+	require.NoError(t, err)
+	assert.Equal(t, "secure", second)
+}
+
+func TestPinSecureMode_PersistFailureIsSwallowed(t *testing.T) {
+	hermetic(t)
+	ctx := t.Context()
+	// Point DATABRICKS_CONFIG_FILE at a path whose parent does not exist.
+	// loadOrCreateConfigFile does not mkdir, so the underlying os.OpenFile
+	// fails and SetConfiguredAuthStorage returns an error.
+	configPath := filepath.Join(t.TempDir(), "no-such-dir", ".databrickscfg")
+	t.Setenv("DATABRICKS_CONFIG_FILE", configPath)
+
+	// Must not panic or block; failures are logged at debug.
+	PinSecureMode(ctx, StorageModeSecure)
+
+	// The persist failure must not have produced any file.
+	_, err := os.Stat(configPath)
+	assert.ErrorIs(t, err, fs.ErrNotExist, "no file should have been written")
+}
+
 func TestWrapForOAuthArgument(t *testing.T) {
 	const (
 		host       = "https://example.com"

libs/auth/storage/keyring.go

@@ -149,7 +149,7 @@ func (k *keyringCache) Lookup(key string) (*oauth2.Token, error) {
 		return nil, cache.ErrNotFound
 	}
 	if err != nil {
-		return nil, err
+		return nil, wrapKeyringUnreachable(err)
 	}
 
 	var entry keyringEntry
@@ -159,6 +159,24 @@ func (k *keyringCache) Lookup(key string) (*oauth2.Token, error) {
 	return entry.Token, nil
 }
 
+// wrapKeyringUnreachable wraps a non-ErrNotFound keyring error with
+// actionable guidance for users whose system has no usable keyring
+// backend (Linux without a Secret Service / D-Bus session bus, headless
+// containers, certain SSH sessions). Surfaces on the read path, where
+// the resolver does not silently fall back to plaintext: a missing
+// token might actually be reachable from the keyring on another machine,
+// so we surface the unreachability instead of minting a fresh plaintext
+// copy.
+//
+// ErrNotFound passes through unchanged because a clean miss is not an
+// availability problem.
+func wrapKeyringUnreachable(err error) error {
+	if err == nil || errors.Is(err, keyring.ErrNotFound) {
+		return err
+	}
+	return fmt.Errorf("OS keyring unreachable: %w (set DATABRICKS_AUTH_STORAGE=plaintext or run `databricks auth login` to use file-based token storage)", err)
+}
+
 // Compile-time confirmation that keyringCache satisfies the SDK interface.
 var _ cache.TokenCache = (*keyringCache)(nil)