diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index df070a324a5..24be6b2d716 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -11,5 +11,6 @@ * Retry transient HTTP 5xx and 408 errors in direct deployment engine ([#5349](https://github.com/databricks/cli/pull/5349), [#5364](https://github.com/databricks/cli/pull/5364)). * Preserve `.designer.ipynb` suffix when translating notebook task paths so Lakeflow Designer files referenced from a `notebook_task` resolve correctly in the workspace ([#5370](https://github.com/databricks/cli/pull/5370)). * Fix script output dropping last line without trailing newline ([#4995](https://github.com/databricks/cli/pull/4995)). +* Add `--select` flag to `bundle plan` and `bundle deploy` to plan/deploy a subset of resources (e.g. `--select my_job` or `--select jobs.my_job`); resources referenced by the selection are included transitively. Direct engine only ([#5413](https://github.com/databricks/cli/pull/5413)). ### Dependency updates diff --git a/acceptance/bundle/help/bundle-deploy/output.txt b/acceptance/bundle/help/bundle-deploy/output.txt index d9c5b1f14c2..7e9e3ff34dd 100644 --- a/acceptance/bundle/help/bundle-deploy/output.txt +++ b/acceptance/bundle/help/bundle-deploy/output.txt @@ -20,6 +20,7 @@ Flags: --force-lock Force acquisition of deployment lock. -h, --help help for deploy --plan string Path to a JSON plan file to apply instead of planning (direct engine only). + --select strings Deploy only the specified resource (e.g. 'my_job' or 'jobs.my_job'). Can be repeated or comma-separated. Global Flags: --debug enable debug logging diff --git a/acceptance/bundle/select/ambiguous/databricks.yml b/acceptance/bundle/select/ambiguous/databricks.yml new file mode 100644 index 00000000000..27962d00d55 --- /dev/null +++ b/acceptance/bundle/select/ambiguous/databricks.yml @@ -0,0 +1,13 @@ +bundle: + name: select-ambiguous + +# "thing" is defined as both a job and a pipeline. This is what an ambiguous +# --select selector would resolve to, but UniqueResourceKeys rejects it at load +# time, so the bundle never reaches selector resolution. +resources: + jobs: + thing: + name: job-thing + pipelines: + thing: + name: pipeline-thing diff --git a/acceptance/bundle/select/ambiguous/out.test.toml b/acceptance/bundle/select/ambiguous/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/select/ambiguous/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/select/ambiguous/output.txt b/acceptance/bundle/select/ambiguous/output.txt new file mode 100644 index 00000000000..7dfcae53017 --- /dev/null +++ b/acceptance/bundle/select/ambiguous/output.txt @@ -0,0 +1,16 @@ + +>>> musterr [CLI] bundle plan --select thing +Error: multiple resources or scripts have been defined with the same key: thing + at resources.jobs.thing + resources.pipelines.thing + in databricks.yml:10:7 + databricks.yml:13:7 + + +>>> musterr [CLI] bundle deploy --select thing +Error: multiple resources or scripts have been defined with the same key: thing + at resources.jobs.thing + resources.pipelines.thing + in databricks.yml:10:7 + databricks.yml:13:7 + diff --git a/acceptance/bundle/select/ambiguous/script b/acceptance/bundle/select/ambiguous/script new file mode 100644 index 00000000000..6f5afbc33d6 --- /dev/null +++ b/acceptance/bundle/select/ambiguous/script @@ -0,0 +1,2 @@ +trace musterr $CLI bundle plan --select thing +trace musterr $CLI bundle deploy --select thing diff --git a/acceptance/bundle/select/basic/databricks.yml.tmpl b/acceptance/bundle/select/basic/databricks.yml.tmpl new file mode 100644 index 00000000000..228b695e072 --- /dev/null +++ b/acceptance/bundle/select/basic/databricks.yml.tmpl @@ -0,0 +1,17 @@ +bundle: + name: select-$UNIQUE_NAME + +resources: + jobs: + bar: + name: bar-$UNIQUE_NAME + + foo: + name: foo-$UNIQUE_NAME + tasks: + - task_key: run_bar + run_job_task: + job_id: ${resources.jobs.bar.id} + + baz: + name: baz-$UNIQUE_NAME diff --git a/acceptance/bundle/select/basic/out.test.toml b/acceptance/bundle/select/basic/out.test.toml new file mode 100644 index 00000000000..8b995e4d177 --- /dev/null +++ b/acceptance/bundle/select/basic/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/select/basic/output.txt b/acceptance/bundle/select/basic/output.txt new file mode 100644 index 00000000000..56c20a404d4 --- /dev/null +++ b/acceptance/bundle/select/basic/output.txt @@ -0,0 +1,141 @@ + +>>> [CLI] bundle plan --select bar +create jobs.bar + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle plan --select jobs.bar --select jobs.baz +create jobs.bar +create jobs.baz + +Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle plan --select jobs.bar,jobs.baz +create jobs.bar +create jobs.baz + +Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle plan --select jobs.foo +create jobs.bar +create jobs.foo + +Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged + +=== bundle deploy --select jobs.foo +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/select-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Telemetry: +select_used true + +>>> print_requests.py --sort //jobs +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/select-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "bar-[UNIQUE_NAME]", + "queue": { + "enabled": true + } + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/select-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo-[UNIQUE_NAME]", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [NUMID] + }, + "task_key": "run_bar" + } + ] + } +} + +>>> [CLI] bundle summary +Name: select-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/select-[UNIQUE_NAME]/default +Resources: + Jobs: + bar: + Name: bar-[UNIQUE_NAME] + URL: [DATABRICKS_URL]/jobs/[NUMID]?w=[NUMID] + baz: + Name: baz-[UNIQUE_NAME] + URL: (not deployed) + foo: + Name: foo-[UNIQUE_NAME] + URL: [DATABRICKS_URL]/jobs/[NUMID]?w=[NUMID] + +=== Full plan after partial deploy +>>> [CLI] bundle plan +create jobs.baz + +Plan: 1 to add, 0 to change, 0 to delete, 2 unchanged + +=== Full deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/select-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py --sort //jobs +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/select-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "baz-[UNIQUE_NAME]", + "queue": { + "enabled": true + } + } +} + +=== Full plan again +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged + +=== Destroy +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.bar + delete resources.jobs.baz + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/select-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/select/basic/script b/acceptance/bundle/select/basic/script new file mode 100644 index 00000000000..88c61a537b7 --- /dev/null +++ b/acceptance/bundle/select/basic/script @@ -0,0 +1,47 @@ +envsubst '$UNIQUE_NAME' < databricks.yml.tmpl > databricks.yml + +cleanup() { + title "Destroy" + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +# --- selector resolution --- +# Unqualified name, unique across resource types. +trace $CLI bundle plan --select bar +# Repeated --select. +trace $CLI bundle plan --select jobs.bar --select jobs.baz +# Comma-separated --select (equivalent to repeating the flag). +trace $CLI bundle plan --select jobs.bar,jobs.baz +# Qualified name; foo pulls in its dependency bar but not the independent baz. +trace $CLI bundle plan --select jobs.foo + +# --- partial deploy --- +# Serialize the filtered plan to a temp file (kept out of the recorded output: the +# JSON embeds remote state that differs between local and cloud), then deploy it: +# inline, or via --plan (READPLAN=1). The deploy is not traced because readplanarg +# varies the command line between READPLAN variants, which must produce identical output. +$CLI bundle plan --select jobs.foo -o json > plan.json +title "bundle deploy --select jobs.foo\n" +$CLI bundle deploy --select jobs.foo $(readplanarg plan.json) +# The deploy reports that --select was used via telemetry. +title "Telemetry:\n" +print_telemetry_bool_values | grep '^select_used ' +# Only bar and foo were created, never baz. +trace print_requests.py --sort //jobs +# Summary after the partial deploy: foo and bar are deployed, baz is not. +trace $CLI bundle summary + +# --- full plan/deploy after the partial deploy --- +# foo and bar are already deployed, so only baz remains to create. +title "Full plan after partial deploy" +trace $CLI bundle plan +$CLI bundle plan -o json > plan-full.json +title "Full deploy\n" +$CLI bundle deploy $(readplanarg plan-full.json) +# Only baz is created this time. +trace print_requests.py --sort //jobs +# Everything is deployed now: no changes. +title "Full plan again" +trace $CLI bundle plan diff --git a/acceptance/bundle/select/basic/test.toml b/acceptance/bundle/select/basic/test.toml new file mode 100644 index 00000000000..93ebc572acf --- /dev/null +++ b/acceptance/bundle/select/basic/test.toml @@ -0,0 +1,11 @@ +Local = true +Cloud = true +RecordRequests = true +# --select is only supported by the direct engine, so this happy-path test runs on +# direct only (the rejection on terraform is covered by bundle/select/rejected). +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +# Also run a variant that deploys a serialized (already filtered) plan via --plan. +EnvMatrix.READPLAN = ["", "1"] +# databricks.yml and the serialized plans are generated at runtime; the plan JSON +# is kept out of the recorded output because it embeds local/cloud-specific state. +Ignore = [".databricks", ".gitignore", "databricks.yml", "plan.json", "plan-full.json"] diff --git a/acceptance/bundle/select/missing/databricks.yml b/acceptance/bundle/select/missing/databricks.yml new file mode 100644 index 00000000000..e093f64dd7e --- /dev/null +++ b/acceptance/bundle/select/missing/databricks.yml @@ -0,0 +1,7 @@ +bundle: + name: select-errors + +resources: + jobs: + my_job: + name: my-job diff --git a/acceptance/bundle/select/missing/out.test.toml b/acceptance/bundle/select/missing/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/select/missing/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/select/missing/output.txt b/acceptance/bundle/select/missing/output.txt new file mode 100644 index 00000000000..f70c74476ba --- /dev/null +++ b/acceptance/bundle/select/missing/output.txt @@ -0,0 +1,12 @@ + +>>> [CLI] bundle plan --select no_such_resource +Error: no such resource: no_such_resource + + +Exit code: 1 + +>>> [CLI] bundle plan --select jobs.no_such_job +Error: no such resource: jobs.no_such_job + + +Exit code: 1 diff --git a/acceptance/bundle/select/missing/script b/acceptance/bundle/select/missing/script new file mode 100644 index 00000000000..0c3193764ef --- /dev/null +++ b/acceptance/bundle/select/missing/script @@ -0,0 +1,8 @@ +# Selectors are resolved by the ResolveSelect mutator during initialize, before the +# engine is known, so these errors are identical on both engines. + +# Unqualified name that matches no resource. +errcode trace $CLI bundle plan --select no_such_resource + +# Qualified name whose resource does not exist. +errcode trace $CLI bundle plan --select jobs.no_such_job diff --git a/acceptance/bundle/select/rejected/databricks.yml b/acceptance/bundle/select/rejected/databricks.yml new file mode 100644 index 00000000000..2c5ff850e63 --- /dev/null +++ b/acceptance/bundle/select/rejected/databricks.yml @@ -0,0 +1,7 @@ +bundle: + name: select-rejected + +resources: + jobs: + my_job: + name: my-job diff --git a/acceptance/bundle/select/rejected/out.test.toml b/acceptance/bundle/select/rejected/out.test.toml new file mode 100644 index 00000000000..65156e0457c --- /dev/null +++ b/acceptance/bundle/select/rejected/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/select/rejected/output.txt b/acceptance/bundle/select/rejected/output.txt new file mode 100644 index 00000000000..9f4417461f6 --- /dev/null +++ b/acceptance/bundle/select/rejected/output.txt @@ -0,0 +1,12 @@ + +>>> [CLI] bundle plan --select my_job +Error: --select is only supported with the direct engine. See https://docs.databricks.com/aws/en/dev-tools/bundles/direct + + +Exit code: 1 + +>>> [CLI] bundle deploy --select my_job +Error: --select is only supported with the direct engine. See https://docs.databricks.com/aws/en/dev-tools/bundles/direct + + +Exit code: 1 diff --git a/acceptance/bundle/select/rejected/script b/acceptance/bundle/select/rejected/script new file mode 100644 index 00000000000..8e5acc82b87 --- /dev/null +++ b/acceptance/bundle/select/rejected/script @@ -0,0 +1,3 @@ +# --select is only supported by the direct engine; both plan and deploy reject it. +errcode trace $CLI bundle plan --select my_job +errcode trace $CLI bundle deploy --select my_job diff --git a/acceptance/bundle/select/rejected/test.toml b/acceptance/bundle/select/rejected/test.toml new file mode 100644 index 00000000000..7d8b7840fab --- /dev/null +++ b/acceptance/bundle/select/rejected/test.toml @@ -0,0 +1,2 @@ +# --select is rejected on the terraform engine with an actionable error. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/select/test.toml b/acceptance/bundle/select/test.toml new file mode 100644 index 00000000000..85ce448afd3 --- /dev/null +++ b/acceptance/bundle/select/test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +Ignore = [".databricks", ".gitignore"] diff --git a/bundle/bundle.go b/bundle/bundle.go index e7eef14b907..868510b15e6 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -145,6 +145,10 @@ type Bundle struct { // files AutoApprove bool + // Select contains resource selectors passed via --select flag. + // When non-empty, only the specified resources are included in deployment. + Select []string + // SkipLocalFileValidation makes path translation tolerant of missing local files. // When set, TranslatePaths computes workspace paths without verifying files exist. // Used by config-remote-sync: a user may modify resource paths remotely (e.g., diff --git a/bundle/config/mutator/resolve_select.go b/bundle/config/mutator/resolve_select.go new file mode 100644 index 00000000000..28d9751c322 --- /dev/null +++ b/bundle/config/mutator/resolve_select.go @@ -0,0 +1,76 @@ +package mutator + +import ( + "context" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/metrics" + "github.com/databricks/cli/libs/diag" +) + +type resolveSelect struct{} + +// ResolveSelect returns a mutator that resolves and validates the selectors in +// b.Select, normalizing each to its qualified "type.name" form. Selectors may be +// "type.name" (e.g. "jobs.myjob") or just "name" if unique across all resource +// types. The mutator does not filter the config; the direct engine selects against +// the resolved keys later via plan.FilterToSelected. +// If b.Select is empty, this is a no-op. +func ResolveSelect() bundle.Mutator { + return &resolveSelect{} +} + +func (m *resolveSelect) Name() string { + return "ResolveSelect" +} + +func (m *resolveSelect) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + if len(b.Select) == 0 { + return nil + } + + b.Metrics.SetBoolValue(metrics.SelectUsed, true) + + // Build reverse index: unqualified name → []"type.name" matches. + byName := map[string][]string{} + for _, group := range b.Config.Resources.AllResources() { + typeName := group.Description.PluralName + for name := range group.Resources { + byName[name] = append(byName[name], typeName+"."+name) + } + } + + resolved := make([]string, 0, len(b.Select)) + for _, selector := range b.Select { + if strings.Contains(selector, ".") { + typeName, name, _ := strings.Cut(selector, ".") + found := false + for _, group := range b.Config.Resources.AllResources() { + if group.Description.PluralName == typeName { + if _, ok := group.Resources[name]; ok { + found = true + } + break + } + } + if !found { + return diag.Errorf("no such resource: %s", selector) + } + resolved = append(resolved, selector) + } else { + matches := byName[selector] + switch len(matches) { + case 0: + return diag.Errorf("no such resource: %s", selector) + case 1: + resolved = append(resolved, matches[0]) + default: + return diag.Errorf("ambiguous resource: %s (can resolve to %s); use a qualified name to disambiguate", selector, strings.Join(matches, ", ")) + } + } + } + + b.Select = resolved + return nil +} diff --git a/bundle/config/mutator/resolve_select_test.go b/bundle/config/mutator/resolve_select_test.go new file mode 100644 index 00000000000..d116d9ca601 --- /dev/null +++ b/bundle/config/mutator/resolve_select_test.go @@ -0,0 +1,53 @@ +package mutator_test + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/bundle/config/resources" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestResolveSelect_Normalizes checks that selectors are normalized to their +// qualified "type.name" form in place, without filtering the config. End-to-end +// selection behavior is covered by acceptance/bundle/select. +func TestResolveSelect_Normalizes(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{"my_job": {}}, + Pipelines: map[string]*resources.Pipeline{"my_pipeline": {}}, + }, + }, + } + b.Select = []string{"my_job", "pipelines.my_pipeline"} + diags := bundle.Apply(t.Context(), b, mutator.ResolveSelect()) + require.NoError(t, diags.Error()) + assert.Equal(t, []string{"jobs.my_job", "pipelines.my_pipeline"}, b.Select) + // Config is left untouched; the mutator only resolves selectors. + assert.Len(t, b.Config.Resources.Jobs, 1) + assert.Len(t, b.Config.Resources.Pipelines, 1) +} + +// TestResolveSelect_Ambiguous covers the ambiguous-selector error in isolation. +// In a real bundle this is unreachable — UniqueResourceKeys rejects two resources +// sharing a key across types before this mutator runs (see the acceptance test +// bundle/select_errors/ambiguous for the real, load-time error). +func TestResolveSelect_Ambiguous(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{"thing": {}}, + Pipelines: map[string]*resources.Pipeline{"thing": {}}, + }, + }, + } + b.Select = []string{"thing"} + diags := bundle.Apply(t.Context(), b, mutator.ResolveSelect()) + require.Error(t, diags.Error()) + assert.ErrorContains(t, diags.Error(), "ambiguous resource: thing") + assert.ErrorContains(t, diags.Error(), "use a qualified name to disambiguate") +} diff --git a/bundle/deployplan/plan.go b/bundle/deployplan/plan.go index 2d06b5fdd25..2fb5d38c806 100644 --- a/bundle/deployplan/plan.go +++ b/bundle/deployplan/plan.go @@ -202,6 +202,42 @@ func (p *Plan) RemoveEntry(resourceKey string) { delete(p.Plan, resourceKey) } +// FilterToSelected reduces the plan to the nodes in selected (format "type.name", +// e.g. "jobs.my_job") plus their transitive dependencies as recorded in each +// entry's DependsOn field. Nodes not reachable from the selected set are removed. +func (p *Plan) FilterToSelected(selected []string) { + // Convert "type.name" → "resources.type.name" (plan key format). + queue := make([]string, 0, len(selected)) + reachable := make(map[string]struct{}, len(selected)) + for _, s := range selected { + key := "resources." + s + if _, ok := p.Plan[key]; ok { + reachable[key] = struct{}{} + queue = append(queue, key) + } + } + + // BFS following DependsOn edges to include transitive dependencies. + for len(queue) > 0 { + key := queue[0] + queue = queue[1:] + for _, dep := range p.Plan[key].DependsOn { + if _, seen := reachable[dep.Node]; !seen { + if _, ok := p.Plan[dep.Node]; ok { + reachable[dep.Node] = struct{}{} + queue = append(queue, dep.Node) + } + } + } + } + + for key := range p.Plan { + if _, ok := reachable[key]; !ok { + delete(p.Plan, key) + } + } +} + type lockmap struct { state map[string]int } diff --git a/bundle/deployplan/plan_filter_test.go b/bundle/deployplan/plan_filter_test.go new file mode 100644 index 00000000000..b16d78eb43d --- /dev/null +++ b/bundle/deployplan/plan_filter_test.go @@ -0,0 +1,48 @@ +package deployplan_test + +import ( + "testing" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/stretchr/testify/assert" +) + +func planWithDeps() *deployplan.Plan { + p := deployplan.NewPlanDirect() + p.Plan["resources.jobs.foo"] = &deployplan.PlanEntry{ + DependsOn: []deployplan.DependsOnEntry{{Node: "resources.jobs.bar"}}, + } + p.Plan["resources.jobs.bar"] = &deployplan.PlanEntry{ + DependsOn: []deployplan.DependsOnEntry{{Node: "resources.jobs.baz"}}, + } + p.Plan["resources.jobs.baz"] = &deployplan.PlanEntry{} + p.Plan["resources.jobs.independent"] = &deployplan.PlanEntry{} + return p +} + +func TestFilterToSelected_Direct(t *testing.T) { + p := planWithDeps() + p.FilterToSelected([]string{"jobs.foo"}) + assert.Contains(t, p.Plan, "resources.jobs.foo") + assert.Contains(t, p.Plan, "resources.jobs.bar") + assert.Contains(t, p.Plan, "resources.jobs.baz") + assert.NotContains(t, p.Plan, "resources.jobs.independent") +} + +func TestFilterToSelected_NoDeps(t *testing.T) { + p := planWithDeps() + p.FilterToSelected([]string{"jobs.baz"}) + assert.Contains(t, p.Plan, "resources.jobs.baz") + assert.NotContains(t, p.Plan, "resources.jobs.foo") + assert.NotContains(t, p.Plan, "resources.jobs.bar") + assert.NotContains(t, p.Plan, "resources.jobs.independent") +} + +func TestFilterToSelected_Multiple(t *testing.T) { + p := planWithDeps() + p.FilterToSelected([]string{"jobs.baz", "jobs.independent"}) + assert.Contains(t, p.Plan, "resources.jobs.baz") + assert.Contains(t, p.Plan, "resources.jobs.independent") + assert.NotContains(t, p.Plan, "resources.jobs.foo") + assert.NotContains(t, p.Plan, "resources.jobs.bar") +} diff --git a/bundle/metrics/metrics.go b/bundle/metrics/metrics.go index c3c0599789c..01d7166e23c 100644 --- a/bundle/metrics/metrics.go +++ b/bundle/metrics/metrics.go @@ -8,4 +8,5 @@ const ( PresetsNamePrefixIsSet = "presets_name_prefix_is_set" AppLifecycleStarted = "app_lifecycle_started" ClusterLifecycleStarted = "cluster_lifecycle_started" + SelectUsed = "select_used" ) diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 15546880b9a..3cac322f9e3 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -212,9 +212,15 @@ func RunPlan(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) *d logdiag.LogError(ctx, err) return nil } + if len(b.Select) > 0 { + plan.FilterToSelected(b.Select) + } return plan } + // b.Select is rejected for the terraform engine in ProcessBundleRet, so it is + // never set here. + bundle.ApplySeqContext(ctx, b, terraform.Interpolate(), terraform.Write(), diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 440bf6c6652..a40506ebb18 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -142,6 +142,12 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // After PythonMutator, mutators must not change bundle resources, or such changes are not // going to be visible in Python code. + // Resolve --select selectors against the materialized resources: normalize + // each to its "type.name" form and validate it exists. Runs after all resource + // mutations so that dynamically added resources are visible. This does not + // filter resources; the direct engine selects against the resolved keys later. + mutator.ResolveSelect(), + // Validate all required fields are set. This is run after variable interpolation and PyDABs mutators // since they can also set and modify resources. validate.Required(), diff --git a/cmd/bundle/deploy.go b/cmd/bundle/deploy.go index 31ffe7090d8..95776eb25f4 100644 --- a/cmd/bundle/deploy.go +++ b/cmd/bundle/deploy.go @@ -31,6 +31,7 @@ See https://docs.databricks.com/en/dev-tools/bundles/index.html for more informa var autoApprove bool var verbose bool var readPlanPath string + var selectResources []string cmd.Flags().BoolVar(&force, "force", false, "Force-override Git branch validation.") cmd.Flags().BoolVar(&forceLock, "force-lock", false, "Force acquisition of deployment lock.") cmd.Flags().BoolVar(&failOnActiveRuns, "fail-on-active-runs", false, "Fail if there are running jobs or pipelines in the deployment.") @@ -40,6 +41,7 @@ See https://docs.databricks.com/en/dev-tools/bundles/index.html for more informa cmd.Flags().MarkDeprecated("compute-id", "use --cluster-id instead") cmd.Flags().BoolVar(&verbose, "verbose", false, "Enable verbose output.") cmd.Flags().StringVar(&readPlanPath, "plan", "", "Path to a JSON plan file to apply instead of planning (direct engine only).") + cmd.Flags().StringSliceVar(&selectResources, "select", nil, "Deploy only the specified resource (e.g. 'my_job' or 'jobs.my_job'). Can be repeated or comma-separated.") // Verbose flag currently only affects file sync output, it's used by the vscode extension cmd.Flags().MarkHidden("verbose") @@ -49,6 +51,7 @@ See https://docs.databricks.com/en/dev-tools/bundles/index.html for more informa b.Config.Bundle.Force = force b.Config.Bundle.Deployment.Lock.Force = forceLock b.AutoApprove = autoApprove + b.Select = selectResources if cmd.Flag("compute-id").Changed { b.Config.Bundle.ClusterId = clusterId diff --git a/cmd/bundle/plan.go b/cmd/bundle/plan.go index e3dd63929ed..20df8cb5f0f 100644 --- a/cmd/bundle/plan.go +++ b/cmd/bundle/plan.go @@ -28,10 +28,12 @@ It is useful for previewing changes before running 'bundle deploy'.`, var force bool var clusterId string + var selectResources []string cmd.Flags().BoolVar(&force, "force", false, "Force-override Git branch validation.") cmd.Flags().StringVar(&clusterId, "compute-id", "", "Override cluster in the deployment with the given compute ID.") cmd.Flags().StringVarP(&clusterId, "cluster-id", "c", "", "Override cluster in the deployment with the given cluster ID.") cmd.Flags().MarkDeprecated("compute-id", "use --cluster-id instead") + cmd.Flags().StringSliceVar(&selectResources, "select", nil, "Plan only the specified resource (e.g. 'my_job' or 'jobs.my_job'). Can be repeated or comma-separated.") cmd.RunE = func(cmd *cobra.Command, args []string) error { opts := utils.ProcessOptions{ @@ -39,12 +41,9 @@ It is useful for previewing changes before running 'bundle deploy'.`, FastValidate: true, Build: true, PreDeployChecks: true, - } - - // Only add InitFunc if we need to set force or cluster ID - if force || cmd.Flag("compute-id").Changed || cmd.Flag("cluster-id").Changed { - opts.InitFunc = func(b *bundle.Bundle) { + InitFunc: func(b *bundle.Bundle) { b.Config.Bundle.Force = force + b.Select = selectResources if cmd.Flag("compute-id").Changed { b.Config.Bundle.ClusterId = clusterId @@ -53,7 +52,7 @@ It is useful for previewing changes before running 'bundle deploy'.`, if cmd.Flag("cluster-id").Changed { b.Config.Bundle.ClusterId = clusterId } - } + }, } b, stateDesc, err := utils.ProcessBundleRet(cmd, opts) diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 5f43cff6acd..d61c4525530 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -185,6 +185,15 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle } cmd.SetContext(ctx) + // --select is only supported by the direct engine, which tracks resource + // dependencies in the plan graph (used to expand the selection transitively). + // The engine is only known for certain after the state is pulled, so reject it + // here rather than silently planning/deploying every resource on terraform. + if len(b.Select) > 0 && !stateDesc.Engine.IsDirect() { + logdiag.LogError(ctx, errors.New("--select is only supported with the direct engine. See https://docs.databricks.com/aws/en/dev-tools/bundles/direct")) + return b, stateDesc, root.ErrAlreadyPrinted + } + // Open direct engine state once for all subsequent operations (ExportState, CalculatePlan, Apply, etc.) needDirectState := stateDesc.Engine.IsDirect() && (opts.InitIDs || opts.ErrorOnEmptyState || opts.Deploy || opts.ReadPlanPath != "" || opts.PreDeployChecks || opts.PostStateFunc != nil) if needDirectState {