Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion acceptance/localenv/cluster-name-ambiguous-json/test.toml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]

[Env]
DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST"
DATABRICKS_LOCALENV_CONSTRAINT_SOURCE_URL_TEST_OVERRIDE = "$DATABRICKS_HOST"

# Two active clusters share the same name, so name→ID resolution is genuinely
# ambiguous and errors as an actionable E_RESOLVE at the resolve phase.
Expand Down
2 changes: 1 addition & 1 deletion acceptance/localenv/cluster-name-ambiguous/test.toml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]

[Env]
DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST"
DATABRICKS_LOCALENV_CONSTRAINT_SOURCE_URL_TEST_OVERRIDE = "$DATABRICKS_HOST"

# Two active clusters share the same name, so name→ID resolution is genuinely
# ambiguous and errors as an actionable E_RESOLVE at the resolve phase.
Expand Down
2 changes: 1 addition & 1 deletion acceptance/localenv/cluster-name-check/test.toml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]

[Env]
DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST"
DATABRICKS_LOCALENV_CONSTRAINT_SOURCE_URL_TEST_OVERRIDE = "$DATABRICKS_HOST"

# --cluster-name resolves the name to an ID via the Clusters API (which lists
# clusters and matches on name), then resolves identically to --cluster-id.
Expand Down
2 changes: 1 addition & 1 deletion acceptance/localenv/cluster-name-unknown/test.toml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]

[Env]
DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST"
DATABRICKS_LOCALENV_CONSTRAINT_SOURCE_URL_TEST_OVERRIDE = "$DATABRICKS_HOST"

# No active cluster matches the requested name, so name→ID resolution finds
# nothing and errors as an actionable E_RESOLVE at the resolve phase. This is
Expand Down
2 changes: 1 addition & 1 deletion acceptance/localenv/constraints-only/test.toml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]

[Env]
DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST"
DATABRICKS_LOCALENV_CONSTRAINT_SOURCE_URL_TEST_OVERRIDE = "$DATABRICKS_HOST"

[[Server]]
Pattern = "GET /serverless/serverless-v4/pyproject.toml"
Expand Down
2 changes: 1 addition & 1 deletion acceptance/localenv/env-unsupported/test.toml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]

[Env]
DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST"
DATABRICKS_LOCALENV_CONSTRAINT_SOURCE_URL_TEST_OVERRIDE = "$DATABRICKS_HOST"

[[Server]]
Pattern = "GET /api/2.1/clusters/get"
Expand Down
2 changes: 1 addition & 1 deletion acceptance/localenv/job-classic-check/test.toml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]

[Env]
DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST"
DATABRICKS_LOCALENV_CONSTRAINT_SOURCE_URL_TEST_OVERRIDE = "$DATABRICKS_HOST"

# A classic task carries its own new_cluster, so --job-task <id>.<task-key>
# resolves to that cluster's DBR-derived environment key.
Expand Down
2 changes: 1 addition & 1 deletion acceptance/localenv/job-serverless-check/test.toml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]

[Env]
DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST"
DATABRICKS_LOCALENV_CONSTRAINT_SOURCE_URL_TEST_OVERRIDE = "$DATABRICKS_HOST"

# A serverless task binds an environment_key to one of the job's environments;
# that environment pins the version, so --job-task <id>.<task-key> resolves to
Expand Down
2 changes: 1 addition & 1 deletion acceptance/localenv/job-task-foreach/test.toml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]

[Env]
DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST"
DATABRICKS_LOCALENV_CONSTRAINT_SOURCE_URL_TEST_OVERRIDE = "$DATABRICKS_HOST"

# A for_each_task wraps the real per-iteration task; its compute lives on the
# nested task. --job-task resolves against that nested task (here a serverless
Expand Down
2 changes: 1 addition & 1 deletion acceptance/localenv/job-task-jobcluster/test.toml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]

[Env]
DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST"
DATABRICKS_LOCALENV_CONSTRAINT_SOURCE_URL_TEST_OVERRIDE = "$DATABRICKS_HOST"

# A classic task references a shared job cluster by job_cluster_key (rather than
# an inline new_cluster), so --job-task resolves the version from the job's
Expand Down
2 changes: 1 addition & 1 deletion acceptance/localenv/job-task-missing-key/test.toml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]

[Env]
DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST"
DATABRICKS_LOCALENV_CONSTRAINT_SOURCE_URL_TEST_OVERRIDE = "$DATABRICKS_HOST"

# A bare --job-task <job-id> (no .<task-key>) is a usage error: a job can bind
# tasks to different environment versions, so the CLI will not guess which one.
Expand Down
2 changes: 1 addition & 1 deletion acceptance/localenv/job-task-unknown/test.toml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]

[Env]
DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST"
DATABRICKS_LOCALENV_CONSTRAINT_SOURCE_URL_TEST_OVERRIDE = "$DATABRICKS_HOST"

# A --job-task naming a task key that the job does not define fails at resolve
# (E_RESOLVE); the message lists the job's actual task keys.
Expand Down
2 changes: 1 addition & 1 deletion acceptance/localenv/job-task-unpinned/test.toml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]

[Env]
DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST"
DATABRICKS_LOCALENV_CONSTRAINT_SOURCE_URL_TEST_OVERRIDE = "$DATABRICKS_HOST"

# A task whose bound serverless environment records no version cannot be resolved:
# the job-task path reads the version directly and never falls back to a default
Expand Down
2 changes: 1 addition & 1 deletion acceptance/localenv/serverless-check/test.toml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]

[Env]
DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST"
DATABRICKS_LOCALENV_CONSTRAINT_SOURCE_URL_TEST_OVERRIDE = "$DATABRICKS_HOST"

[[Server]]
Pattern = "GET /serverless/serverless-v4/pyproject.toml"
Expand Down
2 changes: 1 addition & 1 deletion acceptance/localenv/serverless-json/test.toml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]

[Env]
DATABRICKS_LOCALENV_CONSTRAINT_SOURCE = "$DATABRICKS_HOST"
DATABRICKS_LOCALENV_CONSTRAINT_SOURCE_URL_TEST_OVERRIDE = "$DATABRICKS_HOST"

[[Server]]
Pattern = "GET /serverless/serverless-v4/pyproject.toml"
Expand Down
29 changes: 1 addition & 28 deletions cmd/environments/sync.go
Original file line number Diff line number Diff line change
@@ -1,23 +1,15 @@
package environments

import (
"context"
"os"
"path/filepath"

"github.com/databricks/cli/cmd/root"
"github.com/databricks/cli/libs/cmdctx"
"github.com/databricks/cli/libs/env"
libslocalenv "github.com/databricks/cli/libs/localenv"
"github.com/spf13/cobra"
)

// envConstraintSource is the environment variable that overrides the constraint
// source with a full base URL (used e.g. by tests pointing at a local server).
// When unset, the base URL is derived from the hosting repo via
// libslocalenv.RepoConstraintBaseURL (which reads its own repo env var).
const envConstraintSource = "DATABRICKS_LOCALENV_CONSTRAINT_SOURCE"

func newSetupLocalCommand() *cobra.Command {
cmd := &cobra.Command{
Use: libslocalenv.CommandVerb,
Expand Down Expand Up @@ -53,9 +45,6 @@ func addComputeFlags(cmd *cobra.Command) {
cmd.Flags().String("job-task", "", "job task to use as the compute target, as <job-id>.<task-key> (the task key is required)")
cmd.Flags().Bool("constraints-only", false, "apply the Python version and constraints without adding the databricks-connect dependency")
cmd.Flags().Bool("dry-run", false, "compute the plan without writing files or provisioning")
cmd.Flags().String("constraint-source-url", "", "URL for the constraint source (overrides "+envConstraintSource+")")
// Hide constraint-source-url from casual --help output; it is a power-user escape hatch.
_ = cmd.Flags().MarkHidden("constraint-source-url")
// The mutual exclusivity of the target flags is enforced in the pipeline's
// preflight (as E_USAGE) rather than via cmd.MarkFlagsMutuallyExclusive, so
// the conflict is reported through the phase/JSON contract the --output json
Expand All @@ -72,7 +61,6 @@ func runPipeline(cmd *cobra.Command) error {
jobTask, _ := cmd.Flags().GetString("job-task")
constraintsOnly, _ := cmd.Flags().GetBool("constraints-only")
check, _ := cmd.Flags().GetBool("dry-run")
constraintSource, _ := cmd.Flags().GetString("constraint-source-url")

computeFlags := libslocalenv.ComputeFlags{
Cluster: cluster,
Expand All @@ -89,7 +77,7 @@ func runPipeline(cmd *cobra.Command) error {
mode = libslocalenv.ModeConstraintsOnly
}

constraintBaseURL := resolveConstraintBaseURL(ctx, constraintSource)
constraintBaseURL := libslocalenv.ConstraintBaseURL(ctx)

projectDir, err := os.Getwd()
if err != nil {
Expand Down Expand Up @@ -128,21 +116,6 @@ func runPipeline(cmd *cobra.Command) error {
return renderResult(ctx, cmd, res, pipelineErr)
}

// resolveConstraintBaseURL returns the constraint base URL using ordered precedence:
// an explicit --constraint-source-url flag, then a full-URL override from
// DATABRICKS_LOCALENV_CONSTRAINT_SOURCE, then the URL derived from the hosting repo
// (libslocalenv.RepoConstraintBaseURL). All three may be unset, in which case it
// returns "" and the pipeline reports the missing source at the fetch phase.
func resolveConstraintBaseURL(ctx context.Context, flagValue string) string {
if flagValue != "" {
return flagValue
}
if v, ok := env.Lookup(ctx, envConstraintSource); ok && v != "" {
return v
}
return libslocalenv.RepoConstraintBaseURL(ctx)
}

// bundleTarget reads the active bundle (if any) and maps its compute configuration
// to a libslocalenv.BundleTarget.
//
Expand Down
65 changes: 26 additions & 39 deletions libs/localenv/constraints.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,40 +19,30 @@ import (
"github.com/databricks/cli/libs/log"
)

// EnvConstraintRepo names the environment variable that supplies the GitHub repo
// ("owner/name") hosting the environment constraint artifacts.
const EnvConstraintRepo = "DATABRICKS_LOCALENV_CONSTRAINT_REPO"
// EnvConstraintSourceURLTestOverride names the environment variable that
// overrides the constraint source with a full base URL. It exists so tests can
// point the fetch at a local server (see the acceptance suite) and for power-user
// debugging; normal runs use defaultConstraintBaseURL. The TEST_OVERRIDE naming
// signals it is not a supported user-facing knob.
const EnvConstraintSourceURLTestOverride = "DATABRICKS_LOCALENV_CONSTRAINT_SOURCE_URL_TEST_OVERRIDE"

// defaultConstraintRepo is the GitHub repo that hosts the constraint artifacts.
// It is intentionally empty for now: the artifacts must move to a
// Databricks-owned repo (databricks/environments) before this ships, but that
// repo can't publish them yet (its GitHub Actions are disabled). Until then the
// repo is supplied via the EnvConstraintRepo environment variable rather than
// hardcoding a personal repo, so no untrusted default controls what the CLI
// installs. Once databricks/environments is ready this becomes that constant.
const defaultConstraintRepo = ""
// defaultConstraintBaseURL is the base URL of the published constraint artifacts.
//
// The databricks/environments repo nests its language ecosystems under a
// top-level directory, so the Python artifacts live at python/<env key>/
// pyproject.toml (e.g. python/serverless/serverless-v5, python/dbr/<spark>),
// not at the repo root. The base URL is anchored at that python/ subtree so an
// env key of "serverless/serverless-v5" resolves to the real path.
const defaultConstraintBaseURL = "https://raw.githubusercontent.com/databricks/environments/main/python"

// RepoConstraintBaseURL resolves the base URL for constraint artifacts from the
// hosting GitHub repo: EnvConstraintRepo overrides the (currently empty) built-in
// default, and the repo is turned into a raw.githubusercontent.com main-branch
// URL. It returns "" when no repo is configured; the caller must not fall back to
// any other source, and FetchConstraints reports the missing configuration as a
// fetch-phase error so it surfaces through the normal phase/JSON reporting rather
// than aborting the command before the phase table is rendered.
func RepoConstraintBaseURL(ctx context.Context) string {
repo := defaultConstraintRepo
if v, ok := env.Lookup(ctx, EnvConstraintRepo); ok && strings.TrimSpace(v) != "" {
repo = strings.TrimSpace(v)
}
if repo == "" {
return ""
// ConstraintBaseURL returns the base URL for constraint artifacts: the
// EnvConstraintSourceURLTestOverride value when set (a full base URL), otherwise
// the built-in defaultConstraintBaseURL.
func ConstraintBaseURL(ctx context.Context) string {
if v, ok := env.Lookup(ctx, EnvConstraintSourceURLTestOverride); ok && strings.TrimSpace(v) != "" {
return strings.TrimSpace(v)
}
// The databricks/environments repo nests its language ecosystems under a
// top-level directory, so the Python artifacts live at python/<env key>/
// pyproject.toml (e.g. python/serverless/serverless-v5, python/dbr/<spark>),
// not at the repo root. Anchor the base URL at that python/ subtree so an
// env key of "serverless/serverless-v5" resolves to the real path.
return "https://raw.githubusercontent.com/" + repo + "/main/python"
return defaultConstraintBaseURL
}

// errEnvKeyNotFound is returned by fetchURL when the constraint artifact does
Expand Down Expand Up @@ -139,21 +129,18 @@ func writeCacheAtomic(path string, data []byte) error {
// and does not fall back to cache — a resolvable target with no environment is a distinct,
// non-transient condition.
//
// baseURL points at the repo hosting the constraint artifacts (see
// RepoConstraintBaseURL); it is empty when no source is configured, which is
// reported below as a fetch-phase error.
// baseURL points at the base of the constraint artifacts (see ConstraintBaseURL).
//
// writeCache controls whether a successful live fetch populates the on-disk
// cache. Callers pass false for a dry run (--dry-run), which must not mutate
// disk; an existing cache is still read for offline fallback, since reading is
// not a mutation.
func FetchConstraints(ctx context.Context, baseURL, envKey, cacheDir string, writeCache bool) (*Constraints, error) {
if baseURL == "" {
// No constraint host is configured. This is reported at the fetch phase (as
// E_FETCH) rather than aborting earlier, so the failure flows through the
// same phase/JSON reporting as any other fetch error.
return nil, NewError(ErrFetch, nil,
"no constraint source configured: set %s to the GitHub repo (owner/name) that hosts the environment constraints", EnvConstraintRepo)
// ConstraintBaseURL always returns a non-empty default, so an empty baseURL
// here means the resolver was bypassed. Report it at the fetch phase so it
// still flows through the same phase/JSON reporting as any other fetch error.
return nil, NewError(ErrFetch, nil, "no constraint source configured")
}
url := baseURL + "/" + envKey + "/pyproject.toml"
cachePath := filepath.Join(cacheDir, cacheFileName(envKey))
Expand Down
31 changes: 15 additions & 16 deletions libs/localenv/constraints_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,29 +13,28 @@ import (
"github.com/stretchr/testify/require"
)

func TestRepoConstraintBaseURL(t *testing.T) {
// With no repo configured (empty built-in default), it returns "" so the caller
// can report the missing source at the fetch phase rather than aborting early.
assert.Empty(t, RepoConstraintBaseURL(t.Context()))

// The env var supplies the repo and is turned into a raw main-branch URL
// anchored at the python/ subtree where the Python artifacts live.
ctx := env.Set(t.Context(), EnvConstraintRepo, "databricks/environments")
assert.Equal(t, "https://raw.githubusercontent.com/databricks/environments/main/python", RepoConstraintBaseURL(ctx))

// Whitespace-only is treated as unset.
ctx = env.Set(t.Context(), EnvConstraintRepo, " ")
assert.Empty(t, RepoConstraintBaseURL(ctx))
func TestConstraintBaseURL(t *testing.T) {
// With nothing set it returns the built-in default, anchored at the python/
// subtree of databricks/environments where the Python artifacts live.
assert.Equal(t, "https://raw.githubusercontent.com/databricks/environments/main/python", ConstraintBaseURL(t.Context()))

// The override env var supplies a full base URL verbatim.
ctx := env.Set(t.Context(), EnvConstraintSourceURLTestOverride, "http://localhost:8477")
assert.Equal(t, "http://localhost:8477", ConstraintBaseURL(ctx))

// Whitespace-only is treated as unset and falls back to the default.
ctx = env.Set(t.Context(), EnvConstraintSourceURLTestOverride, " ")
assert.Equal(t, "https://raw.githubusercontent.com/databricks/environments/main/python", ConstraintBaseURL(ctx))
}

func TestFetchConstraintsNoSourceConfigured(t *testing.T) {
// An empty base URL means no constraint host is configured; it must classify as
// E_FETCH (surfaced at the fetch phase) and name the env var to set.
// An empty base URL should never reach FetchConstraints (ConstraintBaseURL
// always returns a non-empty default), but if it does it must classify as
// E_FETCH so it surfaces at the fetch phase rather than as a bare error.
_, err := FetchConstraints(t.Context(), "", "serverless/serverless-v4", t.TempDir(), true)
var pe *PipelineError
require.ErrorAs(t, err, &pe)
assert.Equal(t, ErrFetch, pe.Code)
assert.Contains(t, pe.Error(), EnvConstraintRepo)
}

const sampleToml = `[project]
Expand Down
Loading