diff --git a/acceptance/localenv/cluster-name-ambiguous-json/test.toml b/acceptance/localenv/cluster-name-ambiguous-json/test.toml index be79d13bed8..2f2e3a9ac28 100644 --- a/acceptance/localenv/cluster-name-ambiguous-json/test.toml +++ b/acceptance/localenv/cluster-name-ambiguous-json/test.toml @@ -1,7 +1,7 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] [Env] -DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST" +DATABRICKS_LOCALENV_CONSTRAINT_SOURCE_URL_TEST_OVERRIDE = "$DATABRICKS_HOST" # Two active clusters share the same name, so name→ID resolution is genuinely # ambiguous and errors as an actionable E_RESOLVE at the resolve phase. diff --git a/acceptance/localenv/cluster-name-ambiguous/test.toml b/acceptance/localenv/cluster-name-ambiguous/test.toml index be79d13bed8..2f2e3a9ac28 100644 --- a/acceptance/localenv/cluster-name-ambiguous/test.toml +++ b/acceptance/localenv/cluster-name-ambiguous/test.toml @@ -1,7 +1,7 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] [Env] -DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST" +DATABRICKS_LOCALENV_CONSTRAINT_SOURCE_URL_TEST_OVERRIDE = "$DATABRICKS_HOST" # Two active clusters share the same name, so name→ID resolution is genuinely # ambiguous and errors as an actionable E_RESOLVE at the resolve phase. diff --git a/acceptance/localenv/cluster-name-check/test.toml b/acceptance/localenv/cluster-name-check/test.toml index 0a49bc20a41..c8d10129521 100644 --- a/acceptance/localenv/cluster-name-check/test.toml +++ b/acceptance/localenv/cluster-name-check/test.toml @@ -1,7 +1,7 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] [Env] -DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST" +DATABRICKS_LOCALENV_CONSTRAINT_SOURCE_URL_TEST_OVERRIDE = "$DATABRICKS_HOST" # --cluster-name resolves the name to an ID via the Clusters API (which lists # clusters and matches on name), then resolves identically to --cluster-id. diff --git a/acceptance/localenv/cluster-name-unknown/test.toml b/acceptance/localenv/cluster-name-unknown/test.toml index 154e747bbd5..d8a630f4c96 100644 --- a/acceptance/localenv/cluster-name-unknown/test.toml +++ b/acceptance/localenv/cluster-name-unknown/test.toml @@ -1,7 +1,7 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] [Env] -DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST" +DATABRICKS_LOCALENV_CONSTRAINT_SOURCE_URL_TEST_OVERRIDE = "$DATABRICKS_HOST" # No active cluster matches the requested name, so name→ID resolution finds # nothing and errors as an actionable E_RESOLVE at the resolve phase. This is diff --git a/acceptance/localenv/constraints-only/test.toml b/acceptance/localenv/constraints-only/test.toml index c636f7a9cef..30b11dc3826 100644 --- a/acceptance/localenv/constraints-only/test.toml +++ b/acceptance/localenv/constraints-only/test.toml @@ -1,7 +1,7 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] [Env] -DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST" +DATABRICKS_LOCALENV_CONSTRAINT_SOURCE_URL_TEST_OVERRIDE = "$DATABRICKS_HOST" [[Server]] Pattern = "GET /serverless/serverless-v4/pyproject.toml" diff --git a/acceptance/localenv/env-unsupported/test.toml b/acceptance/localenv/env-unsupported/test.toml index c6bb9f30340..a5a66481801 100644 --- a/acceptance/localenv/env-unsupported/test.toml +++ b/acceptance/localenv/env-unsupported/test.toml @@ -1,7 +1,7 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] [Env] -DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST" +DATABRICKS_LOCALENV_CONSTRAINT_SOURCE_URL_TEST_OVERRIDE = "$DATABRICKS_HOST" [[Server]] Pattern = "GET /api/2.1/clusters/get" diff --git a/acceptance/localenv/job-classic-check/test.toml b/acceptance/localenv/job-classic-check/test.toml index 33fc90a5e37..032dd1f3438 100644 --- a/acceptance/localenv/job-classic-check/test.toml +++ b/acceptance/localenv/job-classic-check/test.toml @@ -1,7 +1,7 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] [Env] -DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST" +DATABRICKS_LOCALENV_CONSTRAINT_SOURCE_URL_TEST_OVERRIDE = "$DATABRICKS_HOST" # A classic task carries its own new_cluster, so --job-task . # resolves to that cluster's DBR-derived environment key. diff --git a/acceptance/localenv/job-serverless-check/test.toml b/acceptance/localenv/job-serverless-check/test.toml index 89ef2ce5794..5c2262b52a9 100644 --- a/acceptance/localenv/job-serverless-check/test.toml +++ b/acceptance/localenv/job-serverless-check/test.toml @@ -1,7 +1,7 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] [Env] -DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST" +DATABRICKS_LOCALENV_CONSTRAINT_SOURCE_URL_TEST_OVERRIDE = "$DATABRICKS_HOST" # A serverless task binds an environment_key to one of the job's environments; # that environment pins the version, so --job-task . resolves to diff --git a/acceptance/localenv/job-task-foreach/test.toml b/acceptance/localenv/job-task-foreach/test.toml index 8c7e836acca..d7cd2999c37 100644 --- a/acceptance/localenv/job-task-foreach/test.toml +++ b/acceptance/localenv/job-task-foreach/test.toml @@ -1,7 +1,7 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] [Env] -DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST" +DATABRICKS_LOCALENV_CONSTRAINT_SOURCE_URL_TEST_OVERRIDE = "$DATABRICKS_HOST" # A for_each_task wraps the real per-iteration task; its compute lives on the # nested task. --job-task resolves against that nested task (here a serverless diff --git a/acceptance/localenv/job-task-jobcluster/test.toml b/acceptance/localenv/job-task-jobcluster/test.toml index 7eb76463420..1b03950d9f7 100644 --- a/acceptance/localenv/job-task-jobcluster/test.toml +++ b/acceptance/localenv/job-task-jobcluster/test.toml @@ -1,7 +1,7 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] [Env] -DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST" +DATABRICKS_LOCALENV_CONSTRAINT_SOURCE_URL_TEST_OVERRIDE = "$DATABRICKS_HOST" # A classic task references a shared job cluster by job_cluster_key (rather than # an inline new_cluster), so --job-task resolves the version from the job's diff --git a/acceptance/localenv/job-task-missing-key/test.toml b/acceptance/localenv/job-task-missing-key/test.toml index 048d3a6f478..abd5382e1a5 100644 --- a/acceptance/localenv/job-task-missing-key/test.toml +++ b/acceptance/localenv/job-task-missing-key/test.toml @@ -1,7 +1,7 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] [Env] -DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST" +DATABRICKS_LOCALENV_CONSTRAINT_SOURCE_URL_TEST_OVERRIDE = "$DATABRICKS_HOST" # A bare --job-task (no .) is a usage error: a job can bind # tasks to different environment versions, so the CLI will not guess which one. diff --git a/acceptance/localenv/job-task-unknown/test.toml b/acceptance/localenv/job-task-unknown/test.toml index 2a5cd68d54c..7a11d3fa1f3 100644 --- a/acceptance/localenv/job-task-unknown/test.toml +++ b/acceptance/localenv/job-task-unknown/test.toml @@ -1,7 +1,7 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] [Env] -DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST" +DATABRICKS_LOCALENV_CONSTRAINT_SOURCE_URL_TEST_OVERRIDE = "$DATABRICKS_HOST" # A --job-task naming a task key that the job does not define fails at resolve # (E_RESOLVE); the message lists the job's actual task keys. diff --git a/acceptance/localenv/job-task-unpinned/test.toml b/acceptance/localenv/job-task-unpinned/test.toml index 292dae64b5a..16a67400948 100644 --- a/acceptance/localenv/job-task-unpinned/test.toml +++ b/acceptance/localenv/job-task-unpinned/test.toml @@ -1,7 +1,7 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] [Env] -DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST" +DATABRICKS_LOCALENV_CONSTRAINT_SOURCE_URL_TEST_OVERRIDE = "$DATABRICKS_HOST" # A task whose bound serverless environment records no version cannot be resolved: # the job-task path reads the version directly and never falls back to a default diff --git a/acceptance/localenv/serverless-check/test.toml b/acceptance/localenv/serverless-check/test.toml index c636f7a9cef..30b11dc3826 100644 --- a/acceptance/localenv/serverless-check/test.toml +++ b/acceptance/localenv/serverless-check/test.toml @@ -1,7 +1,7 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] [Env] -DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST" +DATABRICKS_LOCALENV_CONSTRAINT_SOURCE_URL_TEST_OVERRIDE = "$DATABRICKS_HOST" [[Server]] Pattern = "GET /serverless/serverless-v4/pyproject.toml" diff --git a/acceptance/localenv/serverless-json/test.toml b/acceptance/localenv/serverless-json/test.toml index c636f7a9cef..30b11dc3826 100644 --- a/acceptance/localenv/serverless-json/test.toml +++ b/acceptance/localenv/serverless-json/test.toml @@ -1,7 +1,7 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] [Env] -DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST" +DATABRICKS_LOCALENV_CONSTRAINT_SOURCE_URL_TEST_OVERRIDE = "$DATABRICKS_HOST" [[Server]] Pattern = "GET /serverless/serverless-v4/pyproject.toml" diff --git a/cmd/environments/sync.go b/cmd/environments/sync.go index 5f534f8605a..2e6c5dc2567 100644 --- a/cmd/environments/sync.go +++ b/cmd/environments/sync.go @@ -1,23 +1,15 @@ package environments import ( - "context" "os" "path/filepath" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdctx" - "github.com/databricks/cli/libs/env" libslocalenv "github.com/databricks/cli/libs/localenv" "github.com/spf13/cobra" ) -// envConstraintSource is the environment variable that overrides the constraint -// source with a full base URL (used e.g. by tests pointing at a local server). -// When unset, the base URL is derived from the hosting repo via -// libslocalenv.RepoConstraintBaseURL (which reads its own repo env var). -const envConstraintSource = "DATABRICKS_LOCALENV_CONSTRAINT_SOURCE" - func newSetupLocalCommand() *cobra.Command { cmd := &cobra.Command{ Use: libslocalenv.CommandVerb, @@ -53,9 +45,6 @@ func addComputeFlags(cmd *cobra.Command) { cmd.Flags().String("job-task", "", "job task to use as the compute target, as . (the task key is required)") cmd.Flags().Bool("constraints-only", false, "apply the Python version and constraints without adding the databricks-connect dependency") cmd.Flags().Bool("dry-run", false, "compute the plan without writing files or provisioning") - cmd.Flags().String("constraint-source-url", "", "URL for the constraint source (overrides "+envConstraintSource+")") - // Hide constraint-source-url from casual --help output; it is a power-user escape hatch. - _ = cmd.Flags().MarkHidden("constraint-source-url") // The mutual exclusivity of the target flags is enforced in the pipeline's // preflight (as E_USAGE) rather than via cmd.MarkFlagsMutuallyExclusive, so // the conflict is reported through the phase/JSON contract the --output json @@ -72,7 +61,6 @@ func runPipeline(cmd *cobra.Command) error { jobTask, _ := cmd.Flags().GetString("job-task") constraintsOnly, _ := cmd.Flags().GetBool("constraints-only") check, _ := cmd.Flags().GetBool("dry-run") - constraintSource, _ := cmd.Flags().GetString("constraint-source-url") computeFlags := libslocalenv.ComputeFlags{ Cluster: cluster, @@ -89,7 +77,7 @@ func runPipeline(cmd *cobra.Command) error { mode = libslocalenv.ModeConstraintsOnly } - constraintBaseURL := resolveConstraintBaseURL(ctx, constraintSource) + constraintBaseURL := libslocalenv.ConstraintBaseURL(ctx) projectDir, err := os.Getwd() if err != nil { @@ -128,21 +116,6 @@ func runPipeline(cmd *cobra.Command) error { return renderResult(ctx, cmd, res, pipelineErr) } -// resolveConstraintBaseURL returns the constraint base URL using ordered precedence: -// an explicit --constraint-source-url flag, then a full-URL override from -// DATABRICKS_LOCALENV_CONSTRAINT_SOURCE, then the URL derived from the hosting repo -// (libslocalenv.RepoConstraintBaseURL). All three may be unset, in which case it -// returns "" and the pipeline reports the missing source at the fetch phase. -func resolveConstraintBaseURL(ctx context.Context, flagValue string) string { - if flagValue != "" { - return flagValue - } - if v, ok := env.Lookup(ctx, envConstraintSource); ok && v != "" { - return v - } - return libslocalenv.RepoConstraintBaseURL(ctx) -} - // bundleTarget reads the active bundle (if any) and maps its compute configuration // to a libslocalenv.BundleTarget. // diff --git a/libs/localenv/constraints.go b/libs/localenv/constraints.go index 5dff23a67dd..eae55263a6e 100644 --- a/libs/localenv/constraints.go +++ b/libs/localenv/constraints.go @@ -19,40 +19,30 @@ import ( "github.com/databricks/cli/libs/log" ) -// EnvConstraintRepo names the environment variable that supplies the GitHub repo -// ("owner/name") hosting the environment constraint artifacts. -const EnvConstraintRepo = "DATABRICKS_LOCALENV_CONSTRAINT_REPO" +// EnvConstraintSourceURLTestOverride names the environment variable that +// overrides the constraint source with a full base URL. It exists so tests can +// point the fetch at a local server (see the acceptance suite) and for power-user +// debugging; normal runs use defaultConstraintBaseURL. The TEST_OVERRIDE naming +// signals it is not a supported user-facing knob. +const EnvConstraintSourceURLTestOverride = "DATABRICKS_LOCALENV_CONSTRAINT_SOURCE_URL_TEST_OVERRIDE" -// defaultConstraintRepo is the GitHub repo that hosts the constraint artifacts. -// It is intentionally empty for now: the artifacts must move to a -// Databricks-owned repo (databricks/environments) before this ships, but that -// repo can't publish them yet (its GitHub Actions are disabled). Until then the -// repo is supplied via the EnvConstraintRepo environment variable rather than -// hardcoding a personal repo, so no untrusted default controls what the CLI -// installs. Once databricks/environments is ready this becomes that constant. -const defaultConstraintRepo = "" +// defaultConstraintBaseURL is the base URL of the published constraint artifacts. +// +// The databricks/environments repo nests its language ecosystems under a +// top-level directory, so the Python artifacts live at python// +// pyproject.toml (e.g. python/serverless/serverless-v5, python/dbr/), +// not at the repo root. The base URL is anchored at that python/ subtree so an +// env key of "serverless/serverless-v5" resolves to the real path. +const defaultConstraintBaseURL = "https://raw.githubusercontent.com/databricks/environments/main/python" -// RepoConstraintBaseURL resolves the base URL for constraint artifacts from the -// hosting GitHub repo: EnvConstraintRepo overrides the (currently empty) built-in -// default, and the repo is turned into a raw.githubusercontent.com main-branch -// URL. It returns "" when no repo is configured; the caller must not fall back to -// any other source, and FetchConstraints reports the missing configuration as a -// fetch-phase error so it surfaces through the normal phase/JSON reporting rather -// than aborting the command before the phase table is rendered. -func RepoConstraintBaseURL(ctx context.Context) string { - repo := defaultConstraintRepo - if v, ok := env.Lookup(ctx, EnvConstraintRepo); ok && strings.TrimSpace(v) != "" { - repo = strings.TrimSpace(v) - } - if repo == "" { - return "" +// ConstraintBaseURL returns the base URL for constraint artifacts: the +// EnvConstraintSourceURLTestOverride value when set (a full base URL), otherwise +// the built-in defaultConstraintBaseURL. +func ConstraintBaseURL(ctx context.Context) string { + if v, ok := env.Lookup(ctx, EnvConstraintSourceURLTestOverride); ok && strings.TrimSpace(v) != "" { + return strings.TrimSpace(v) } - // The databricks/environments repo nests its language ecosystems under a - // top-level directory, so the Python artifacts live at python// - // pyproject.toml (e.g. python/serverless/serverless-v5, python/dbr/), - // not at the repo root. Anchor the base URL at that python/ subtree so an - // env key of "serverless/serverless-v5" resolves to the real path. - return "https://raw.githubusercontent.com/" + repo + "/main/python" + return defaultConstraintBaseURL } // errEnvKeyNotFound is returned by fetchURL when the constraint artifact does @@ -139,9 +129,7 @@ func writeCacheAtomic(path string, data []byte) error { // and does not fall back to cache — a resolvable target with no environment is a distinct, // non-transient condition. // -// baseURL points at the repo hosting the constraint artifacts (see -// RepoConstraintBaseURL); it is empty when no source is configured, which is -// reported below as a fetch-phase error. +// baseURL points at the base of the constraint artifacts (see ConstraintBaseURL). // // writeCache controls whether a successful live fetch populates the on-disk // cache. Callers pass false for a dry run (--dry-run), which must not mutate @@ -149,11 +137,10 @@ func writeCacheAtomic(path string, data []byte) error { // not a mutation. func FetchConstraints(ctx context.Context, baseURL, envKey, cacheDir string, writeCache bool) (*Constraints, error) { if baseURL == "" { - // No constraint host is configured. This is reported at the fetch phase (as - // E_FETCH) rather than aborting earlier, so the failure flows through the - // same phase/JSON reporting as any other fetch error. - return nil, NewError(ErrFetch, nil, - "no constraint source configured: set %s to the GitHub repo (owner/name) that hosts the environment constraints", EnvConstraintRepo) + // ConstraintBaseURL always returns a non-empty default, so an empty baseURL + // here means the resolver was bypassed. Report it at the fetch phase so it + // still flows through the same phase/JSON reporting as any other fetch error. + return nil, NewError(ErrFetch, nil, "no constraint source configured") } url := baseURL + "/" + envKey + "/pyproject.toml" cachePath := filepath.Join(cacheDir, cacheFileName(envKey)) diff --git a/libs/localenv/constraints_test.go b/libs/localenv/constraints_test.go index 472078749e3..2590ce0c643 100644 --- a/libs/localenv/constraints_test.go +++ b/libs/localenv/constraints_test.go @@ -13,29 +13,28 @@ import ( "github.com/stretchr/testify/require" ) -func TestRepoConstraintBaseURL(t *testing.T) { - // With no repo configured (empty built-in default), it returns "" so the caller - // can report the missing source at the fetch phase rather than aborting early. - assert.Empty(t, RepoConstraintBaseURL(t.Context())) - - // The env var supplies the repo and is turned into a raw main-branch URL - // anchored at the python/ subtree where the Python artifacts live. - ctx := env.Set(t.Context(), EnvConstraintRepo, "databricks/environments") - assert.Equal(t, "https://raw.githubusercontent.com/databricks/environments/main/python", RepoConstraintBaseURL(ctx)) - - // Whitespace-only is treated as unset. - ctx = env.Set(t.Context(), EnvConstraintRepo, " ") - assert.Empty(t, RepoConstraintBaseURL(ctx)) +func TestConstraintBaseURL(t *testing.T) { + // With nothing set it returns the built-in default, anchored at the python/ + // subtree of databricks/environments where the Python artifacts live. + assert.Equal(t, "https://raw.githubusercontent.com/databricks/environments/main/python", ConstraintBaseURL(t.Context())) + + // The override env var supplies a full base URL verbatim. + ctx := env.Set(t.Context(), EnvConstraintSourceURLTestOverride, "http://localhost:8477") + assert.Equal(t, "http://localhost:8477", ConstraintBaseURL(ctx)) + + // Whitespace-only is treated as unset and falls back to the default. + ctx = env.Set(t.Context(), EnvConstraintSourceURLTestOverride, " ") + assert.Equal(t, "https://raw.githubusercontent.com/databricks/environments/main/python", ConstraintBaseURL(ctx)) } func TestFetchConstraintsNoSourceConfigured(t *testing.T) { - // An empty base URL means no constraint host is configured; it must classify as - // E_FETCH (surfaced at the fetch phase) and name the env var to set. + // An empty base URL should never reach FetchConstraints (ConstraintBaseURL + // always returns a non-empty default), but if it does it must classify as + // E_FETCH so it surfaces at the fetch phase rather than as a bare error. _, err := FetchConstraints(t.Context(), "", "serverless/serverless-v4", t.TempDir(), true) var pe *PipelineError require.ErrorAs(t, err, &pe) assert.Equal(t, ErrFetch, pe.Code) - assert.Contains(t, pe.Error(), EnvConstraintRepo) } const sampleToml = `[project]