From a3ef7861ca75c32627543235d0741a4b8f8ca230 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 1 Jul 2026 11:29:54 +0200 Subject: [PATCH 1/4] Dry-run migration to direct engine after terraform deploy After a successful terraform deploy, convert the state to the direct engine in memory (writing nothing locally or remotely) and report the outcome via telemetry. Surface the error to the user as a warning if it fails; internal warnings are downgraded to info level and reported through telemetry only. Co-authored-by: Isaac --- NEXT_CHANGELOG.md | 1 + .../url_ref/out.deploy.terraform.txt | 1 + .../telemetry/deploy/out.migration.direct.txt | 1 + .../deploy/out.migration.terraform.txt | 3 + acceptance/bundle/telemetry/deploy/output.txt | 2 + acceptance/bundle/telemetry/deploy/script | 9 +- bundle/bundle.go | 5 + bundle/direct/dstate/state.go | 49 ++++++- bundle/migrate/build_state.go | 56 +++++--- bundle/migrate/build_state_test.go | 2 +- bundle/migrate/resolve.go | 11 +- bundle/migrate/resolve_test.go | 13 +- bundle/phases/deploy.go | 7 + bundle/phases/telemetry.go | 1 + bundle/statemgmt/check_direct_migration.go | 123 ++++++++++++++++++ .../statemgmt/upload_state_for_yaml_sync.go | 2 +- cmd/bundle/deployment/migrate.go | 2 +- libs/telemetry/protos/bundle_deploy.go | 20 +++ 18 files changed, 272 insertions(+), 36 deletions(-) create mode 100644 acceptance/bundle/telemetry/deploy/out.migration.direct.txt create mode 100644 acceptance/bundle/telemetry/deploy/out.migration.terraform.txt create mode 100644 bundle/statemgmt/check_direct_migration.go diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index a6cccd2a5cb..55bc8be24f4 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -16,6 +16,7 @@ * direct: Fix spurious update when `apply_policy_default_values: true` is set on job task, for-each-task, or job cluster new_cluster ([#5731](https://github.com/databricks/cli/pull/5731)). Also fix spurious updates for for-each-task clusters due to missing backend defaults for `data_security_mode`, `node_type_id`, `driver_node_type_id`, `driver_instance_pool_id`, `enable_elastic_disk`, and `enable_local_disk_encryption`. * direct: Cluster resize now falls back to regular update if resize fails due to `INVALID_STATE` ([#5716](https://github.com/databricks/cli/pull/5716)). * Fixed `bundle deployment migrate` failing on `model_serving_endpoints`/`database_instances` with permissions (regression since v1.5.0) ([#5775](https://github.com/databricks/cli/pull/5775)). + * After a terraform deploy, the CLI now dry-runs an in-memory migration to the direct engine (writing nothing locally or remotely) and reports the outcome via telemetry, warning if the migration could not be completed. ### Dependency updates diff --git a/acceptance/bundle/bundle_tag/url_ref/out.deploy.terraform.txt b/acceptance/bundle/bundle_tag/url_ref/out.deploy.terraform.txt index 92039fc564a..702ee4022f4 100644 --- a/acceptance/bundle/bundle_tag/url_ref/out.deploy.terraform.txt +++ b/acceptance/bundle/bundle_tag/url_ref/out.deploy.terraform.txt @@ -4,4 +4,5 @@ Warning: "resources.jobs.foo.url": Terraform-only field; this reference will not Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Deploying resources... Updating deployment state... +Warn: Dry-run migration to direct engine failed: resources.jobs.bar: cannot set resolved value for field "url": field "url" not found in jobs.JobSettings Deployment complete! diff --git a/acceptance/bundle/telemetry/deploy/out.migration.direct.txt b/acceptance/bundle/telemetry/deploy/out.migration.direct.txt new file mode 100644 index 00000000000..19765bd501b --- /dev/null +++ b/acceptance/bundle/telemetry/deploy/out.migration.direct.txt @@ -0,0 +1 @@ +null diff --git a/acceptance/bundle/telemetry/deploy/out.migration.terraform.txt b/acceptance/bundle/telemetry/deploy/out.migration.terraform.txt new file mode 100644 index 00000000000..5550c6db40a --- /dev/null +++ b/acceptance/bundle/telemetry/deploy/out.migration.terraform.txt @@ -0,0 +1,3 @@ +{ + "success": true +} diff --git a/acceptance/bundle/telemetry/deploy/output.txt b/acceptance/bundle/telemetry/deploy/output.txt index cf5dd1e434e..98e1d643081 100644 --- a/acceptance/bundle/telemetry/deploy/output.txt +++ b/acceptance/bundle/telemetry/deploy/output.txt @@ -10,3 +10,5 @@ Deployment complete! === Assert that mutator execution times are being recorded >>> cat telemetry.json true + +=== Assert the dry-run migration to the direct engine is reported \ No newline at end of file diff --git a/acceptance/bundle/telemetry/deploy/script b/acceptance/bundle/telemetry/deploy/script index 83705b0ea41..b258147a2c2 100644 --- a/acceptance/bundle/telemetry/deploy/script +++ b/acceptance/bundle/telemetry/deploy/script @@ -18,12 +18,19 @@ trace cat telemetry.json | jq ' .entry.databricks_cli_log.bundle_deploy_event.ex # 0.0.0-dev on windows) — it is still emitted in real telemetry. cat telemetry.json | jq '.entry.databricks_cli_log.bundle_deploy_event.resources_metadata | if . then del(.state_file_size_bytes) else . end' > out.resources_metadata.$DATABRICKS_BUNDLE_ENGINE.txt +# terraform_to_direct_migration is the result of the in-memory dry-run migration +# to the direct engine, attempted only after a terraform deploy. It therefore +# diverges across the DATABRICKS_BUNDLE_ENGINE matrix (direct produces "null"), +# so capture it per-engine and omit it from the engine-agnostic out.telemetry.txt. +title "Assert the dry-run migration to the direct engine is reported" +cat telemetry.json | jq '.entry.databricks_cli_log.bundle_deploy_event.experimental.terraform_to_direct_migration' > out.migration.$DATABRICKS_BUNDLE_ENGINE.txt + # bundle_mutator_execution_time_ms can have variable number of entries depending upon the runtime of the mutators. Thus we omit it from # being asserted here. # # upload_file_count and upload_file_sizes are asserted here and kept deterministic by # the sync.paths restriction in databricks.yml (see the comment there). -cat telemetry.json | jq 'del(.entry.databricks_cli_log.bundle_deploy_event.experimental.bundle_mutator_execution_time_ms, .entry.databricks_cli_log.bundle_deploy_event.resources_metadata)' > out.telemetry.txt +cat telemetry.json | jq 'del(.entry.databricks_cli_log.bundle_deploy_event.experimental.bundle_mutator_execution_time_ms, .entry.databricks_cli_log.bundle_deploy_event.experimental.terraform_to_direct_migration, .entry.databricks_cli_log.bundle_deploy_event.resources_metadata)' > out.telemetry.txt cmd_exec_id=$(extract_command_exec_id.py) deployment_id=$(cat .databricks/bundle/default/deployment.json | jq -r .id) diff --git a/bundle/bundle.go b/bundle/bundle.go index 53332f73bdb..4bdf818dc36 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -62,6 +62,11 @@ type Metrics struct { // bytes so deploy telemetry can be derived without re-reading or re-parsing // the state file. Nil for terraform deploys. ResourceState resourcestate.ExportedResourcesMap + + // TerraformToDirectMigration holds the result of the in-memory dry-run + // migration to the direct engine attempted after a successful terraform + // deploy. Nil unless a terraform deploy attempted the migration. + TerraformToDirectMigration *protos.BundleDirectMigration } // SetBoolValue sets the value of a boolean metric. diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index eb41217f61d..034a861b4af 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -31,12 +31,21 @@ const ( // The caller should delete the stale WAL and proceed normally. var errStaleWAL = errors.New("stale WAL") +// inMemoryPath is the placeholder Path for a state opened in in-memory write +// mode. It is never used for I/O; it only keeps the "opened" asserts happy. +const inMemoryPath = "(in-memory)" + type DeploymentState struct { Path string Data Database mu sync.Mutex walFile *os.File + // When true, SaveState/DeleteState mutate Data.State directly instead of + // appending to an on-disk WAL, so nothing is written to disk. Used for + // dry-run migrations (see OpenInMemoryForWrite). + inMemory bool + // Maps resource key to ID. Unlike Data.State, this is up to date during writes (deploys). stateIDs map[string]string } @@ -101,6 +110,12 @@ func (db *DeploymentState) SaveState(key, newID string, state any, dependsOn []d DependsOn: dependsOn, } + if db.inMemory { + db.Data.State[key] = entry + db.stateIDs[key] = newID + return nil + } + err = appendJSONLine(db.walFile, WALEntry{Key: key, Value: &entry}) if err == nil { db.stateIDs[key] = newID @@ -117,6 +132,12 @@ func (db *DeploymentState) DeleteState(key string) error { return nil } + if db.inMemory { + delete(db.Data.State, key) + delete(db.stateIDs, key) + return nil + } + err := appendJSONLine(db.walFile, WALEntry{Key: key}) if err == nil { delete(db.stateIDs, key) @@ -263,6 +284,31 @@ func (db *DeploymentState) OpenWithData(path string, data Database) { } } +// OpenInMemoryForWrite initializes the state from an in-memory database in write +// mode without touching disk. SaveState and DeleteState mutate Data.State +// directly instead of appending to an on-disk WAL, and no state file is ever +// written. It backs dry-run migrations that must not persist any state locally +// or remotely. +func (db *DeploymentState) OpenInMemoryForWrite(data Database) { + db.mu.Lock() + defer db.mu.Unlock() + + if db.Path != "" { + panic(fmt.Sprintf("state already opened: %v, cannot open in memory", db.Path)) + } + + db.Path = inMemoryPath + db.inMemory = true + db.Data = data + if db.Data.State == nil { + db.Data.State = make(map[string]ResourceEntry) + } + db.stateIDs = make(map[string]string) + for key, entry := range db.Data.State { + db.stateIDs[key] = entry.ID + } +} + func (db *DeploymentState) replayWAL(ctx context.Context) error { walPath := db.Path + walSuffix hasEntries, err := db.mergeWalIntoState(ctx) @@ -402,6 +448,7 @@ func (db *DeploymentState) Finalize(ctx context.Context) (resourcestate.Exported db.Path = "" db.Data = Database{} db.stateIDs = nil + db.inMemory = false return state, err } @@ -453,7 +500,7 @@ func (db *DeploymentState) AssertOpenedForRead() { func (db *DeploymentState) AssertOpenedForWrite() { db.AssertOpenedForReadOrWrite() - if db.walFile == nil { + if db.walFile == nil && !db.inMemory { panic("internal error: DeploymentState must be opened in write mode") } } diff --git a/bundle/migrate/build_state.go b/bundle/migrate/build_state.go index 06638a84458..6e057523ea4 100644 --- a/bundle/migrate/build_state.go +++ b/bundle/migrate/build_state.go @@ -24,6 +24,12 @@ import ( // BuildStateFromTF iterates over bundle resources, resolves cross-resource // references using TF state attributes, and writes each resource's state entry. // configRoot should be an un-interpolated config (with ${resources.*} references). +// +// When downgradeWarnings is true, warnings (e.g. skipped unsupported resource +// types) are logged at info level instead of warn level. This is used by the +// dry-run migration that runs after a terraform deploy, where these conditions +// are reported via telemetry rather than surfaced to the user. The returned bool +// reports whether any such warning was emitted, regardless of its log level. func BuildStateFromTF( ctx context.Context, configRoot *config.Root, @@ -31,7 +37,17 @@ func BuildStateFromTF( stateDB *dstate.DeploymentState, tfAttrs TFStateAttrs, tfIDs map[string]string, -) error { + downgradeWarnings bool, +) (bool, error) { + warningsSeen := false + warnf := func(format string, args ...any) { + warningsSeen = true + if downgradeWarnings { + log.Infof(ctx, format, args...) + } else { + log.Warnf(ctx, format, args...) + } + } // Collect all resource nodes (same patterns as makePlan). var nodes []string patterns := []dyn.Pattern{ @@ -49,7 +65,7 @@ func BuildStateFromTF( }, ) if err != nil { - return err + return warningsSeen, err } } @@ -63,18 +79,18 @@ func BuildStateFromTF( group := config.GetResourceTypeFromKey(node) if group == "" { - return fmt.Errorf("cannot determine resource type for %q", node) + return warningsSeen, fmt.Errorf("cannot determine resource type for %q", node) } adapter, ok := adapters[group] if !ok { - log.Warnf(ctx, "unsupported resource type %q for %s, skipping", group, node) + warnf("unsupported resource type %q for %s, skipping", group, node) continue } inputConfig, err := configRoot.GetResourceConfig(node) if err != nil { - return fmt.Errorf("%s: getting config: %w", node, err) + return warningsSeen, fmt.Errorf("%s: getting config: %w", node, err) } baseRefs := map[string]string{} @@ -85,16 +101,16 @@ func BuildStateFromTF( if strings.HasPrefix(node, "resources.secret_scopes.") { typedConfig, ok := inputConfig.(*[]resources.SecretScopePermission) if !ok { - return fmt.Errorf("%s: expected *[]resources.SecretScopePermission, got %T", node, inputConfig) + return warningsSeen, fmt.Errorf("%s: expected *[]resources.SecretScopePermission, got %T", node, inputConfig) } sv, err = dresources.PrepareSecretScopeAclsInputConfig(*typedConfig, node) if err != nil { - return fmt.Errorf("%s: preparing secret scope ACLs config: %w", node, err) + return warningsSeen, fmt.Errorf("%s: preparing secret scope ACLs config: %w", node, err) } } else { sv, err = dresources.PreparePermissionsInputConfig(inputConfig, node) if err != nil { - return fmt.Errorf("%s: preparing permissions config: %w", node, err) + return warningsSeen, fmt.Errorf("%s: preparing permissions config: %w", node, err) } } inputConfig = sv.Value @@ -103,7 +119,7 @@ func BuildStateFromTF( case strings.HasSuffix(node, ".grants"): sv, err := dresources.PrepareGrantsInputConfig(inputConfig, node) if err != nil { - return fmt.Errorf("%s: preparing grants config: %w", node, err) + return warningsSeen, fmt.Errorf("%s: preparing grants config: %w", node, err) } inputConfig = sv.Value baseRefs = sv.Refs @@ -111,12 +127,12 @@ func BuildStateFromTF( newStateValue, err := adapter.PrepareState(inputConfig) if err != nil { - return fmt.Errorf("%s: PrepareState: %w", node, err) + return warningsSeen, fmt.Errorf("%s: PrepareState: %w", node, err) } refs, err := direct.ExtractReferences(configRoot.Value(), node) if err != nil { - return fmt.Errorf("%s: extracting references: %w", node, err) + return warningsSeen, fmt.Errorf("%s: extracting references: %w", node, err) } maps.Copy(refs, baseRefs) @@ -167,7 +183,7 @@ func BuildStateFromTF( // is absent there (model_serving_endpoints, database_instances). if _, ok := sv.Refs["object_id"]; ok { if err := structaccess.Set(sv.Value, structpath.NewStringKey(nil, "object_id"), id); err != nil { - return fmt.Errorf("%s: setting object_id: %w", node, err) + return warningsSeen, fmt.Errorf("%s: setting object_id: %w", node, err) } delete(sv.Refs, "object_id") } @@ -194,25 +210,25 @@ func BuildStateFromTF( for _, pending := range pendingRefs { fieldPath, err := structpath.ParsePath(pending.fieldPathStr) if err != nil { - return fmt.Errorf("%s: parsing field path %q: %w", node, pending.fieldPathStr, err) + return warningsSeen, fmt.Errorf("%s: parsing field path %q: %w", node, pending.fieldPathStr, err) } // ResolveFieldRef returns the fully resolved value for this field, // using either Method A (TF state lookup) or Method B (template evaluation). - value, err := ResolveFieldRef(ctx, tfAttrs, srcGroup, srcName, fieldPath, pending.refTemplate) + value, err := ResolveFieldRef(tfAttrs, srcGroup, srcName, fieldPath, pending.refTemplate, warnf) if err != nil { - return fmt.Errorf("%s: cannot resolve field %q (template %q): %w", node, pending.fieldPathStr, pending.refTemplate, err) + return warningsSeen, fmt.Errorf("%s: cannot resolve field %q (template %q): %w", node, pending.fieldPathStr, pending.refTemplate, err) } // Set the resolved value directly and remove the ref entry. if err := structaccess.Set(sv.Value, fieldPath, value); err != nil { - return fmt.Errorf("%s: cannot set resolved value for field %q: %w", node, pending.fieldPathStr, err) + return warningsSeen, fmt.Errorf("%s: cannot set resolved value for field %q: %w", node, pending.fieldPathStr, err) } delete(sv.Refs, pending.fieldPathStr) } if len(sv.Refs) > 0 { - return fmt.Errorf("%s: unresolved references: %v", node, sv.Refs) + return warningsSeen, fmt.Errorf("%s: unresolved references: %v", node, sv.Refs) } // Handle etag for dashboards: read it directly from TF state attributes. @@ -222,15 +238,15 @@ func BuildStateFromTF( if v, err := LookupTFField(tfAttrs, group, srcName, structpath.NewStringKey(nil, "etag")); err == nil { if etag, ok := v.(string); ok && etag != "" { if err := structaccess.Set(sv.Value, structpath.NewStringKey(nil, "etag"), etag); err != nil { - return fmt.Errorf("%s: cannot set etag: %w", node, err) + return warningsSeen, fmt.Errorf("%s: cannot set etag: %w", node, err) } } } if err := stateDB.SaveState(node, id, sv.Value, dependsOn); err != nil { - return fmt.Errorf("%s: SaveState: %w", node, err) + return warningsSeen, fmt.Errorf("%s: SaveState: %w", node, err) } } - return nil + return warningsSeen, nil } diff --git a/bundle/migrate/build_state_test.go b/bundle/migrate/build_state_test.go index 338954a330e..ba924c0f9be 100644 --- a/bundle/migrate/build_state_test.go +++ b/bundle/migrate/build_state_test.go @@ -50,7 +50,7 @@ func runBuildStateFromTF( db.OpenWithData(statePath, dstate.NewDatabase("lineage", 1)) require.NoError(t, db.UpgradeToWrite()) - err = migrate.BuildStateFromTF(t.Context(), &root, adapters, &db, tfAttrs, tfIDs) + _, err = migrate.BuildStateFromTF(t.Context(), &root, adapters, &db, tfAttrs, tfIDs, false) require.NoError(t, err) _, err = db.Finalize(t.Context()) diff --git a/bundle/migrate/resolve.go b/bundle/migrate/resolve.go index 8d5f6bc6b35..6d953bd1f7b 100644 --- a/bundle/migrate/resolve.go +++ b/bundle/migrate/resolve.go @@ -1,13 +1,11 @@ package migrate import ( - "context" "fmt" "strings" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/dynvar" - "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/structs/structpath" ) @@ -58,7 +56,10 @@ func evaluateTemplate(state TFStateAttrs, template string) (string, error) { // - Method B: evaluate the template by reading each referenced field from TF state. // // Returns the reconciled value or an error if both methods fail. -func ResolveFieldRef(ctx context.Context, state TFStateAttrs, srcGroup, srcName string, fieldPath *structpath.PathNode, refTemplate string) (any, error) { +// +// warnf reports a warning; the caller controls its severity (the post-deploy +// dry-run downgrades these to info level) and tracks whether any warning fired. +func ResolveFieldRef(state TFStateAttrs, srcGroup, srcName string, fieldPath *structpath.PathNode, refTemplate string, warnf func(format string, args ...any)) (any, error) { // Method A: read field from source resource's TF state. valueA, errA := LookupTFField(state, srcGroup, srcName, fieldPath) @@ -73,11 +74,11 @@ func ResolveFieldRef(ctx context.Context, state TFStateAttrs, srcGroup, srcName } // Both succeeded but disagree: prefer longer string and warn. if len(valueB) > len(aStr) { - log.Warnf(ctx, "resource %s.%s field %s: method A value %q and method B value %q disagree; using longer (method B)", + warnf("resource %s.%s field %s: method A value %q and method B value %q disagree; using longer (method B)", srcGroup, srcName, fieldPath, aStr, valueB) return valueB, nil } - log.Warnf(ctx, "resource %s.%s field %s: method A value %q and method B value %q disagree; using longer (method A)", + warnf("resource %s.%s field %s: method A value %q and method B value %q disagree; using longer (method A)", srcGroup, srcName, fieldPath, aStr, valueB) return valueA, nil case errA == nil: diff --git a/bundle/migrate/resolve_test.go b/bundle/migrate/resolve_test.go index 221c873a73d..2eda596d68e 100644 --- a/bundle/migrate/resolve_test.go +++ b/bundle/migrate/resolve_test.go @@ -11,6 +11,9 @@ import ( "github.com/stretchr/testify/require" ) +// noWarn is a warnf that drops messages; these cases never hit the warning path. +func noWarn(string, ...any) {} + // state with src job having int and bool fields set. func testState() migrate.TFStateAttrs { return migrate.TFStateAttrs{ @@ -37,12 +40,11 @@ func TestResolveFieldRefInt(t *testing.T) { // Remove dst from state so Method A fails and Method B must be used. delete(state["databricks_job"], "dst") - ctx := t.Context() fieldPath, err := structpath.ParsePath("max_concurrent_runs") require.NoError(t, err) - value, err := migrate.ResolveFieldRef(ctx, state, "jobs", "dst", fieldPath, - "${resources.jobs.src.max_concurrent_runs}") + value, err := migrate.ResolveFieldRef(state, "jobs", "dst", fieldPath, + "${resources.jobs.src.max_concurrent_runs}", noWarn) require.NoError(t, err) // Method B succeeds: returns string "4". Verify Set converts it to int. @@ -60,12 +62,11 @@ func TestResolveFieldRefBool(t *testing.T) { state := testState() delete(state["databricks_job"], "dst") - ctx := t.Context() fieldPath, err := structpath.ParsePath("always_running") require.NoError(t, err) - value, err := migrate.ResolveFieldRef(ctx, state, "jobs", "dst", fieldPath, - "${resources.jobs.src.always_running}") + value, err := migrate.ResolveFieldRef(state, "jobs", "dst", fieldPath, + "${resources.jobs.src.always_running}", noWarn) require.NoError(t, err) type target struct { diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 2edbfd1bf1c..ff1107ad567 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -111,6 +111,13 @@ func deployCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, ta statemgmt.UploadStateForYamlSync(targetEngine), ) + // After a successful terraform deploy, dry-run the migration to the direct + // engine in memory and record the outcome in telemetry. It writes nothing and + // never fails the deploy. + if !targetEngine.IsDirect() && !logdiag.HasError(ctx) { + statemgmt.CheckDirectMigration(ctx, b) + } + if !logdiag.HasError(ctx) { cmdio.LogString(ctx, "Deployment complete!") } diff --git a/bundle/phases/telemetry.go b/bundle/phases/telemetry.go index 1a0123edd14..c5456414236 100644 --- a/bundle/phases/telemetry.go +++ b/bundle/phases/telemetry.go @@ -292,6 +292,7 @@ func LogDeployTelemetry(ctx context.Context, b *bundle.Bundle, errMsg string) { ComplexVariableCount: complexVariableCount, LookupVariableCount: lookupVariableCount, BundleMutatorExecutionTimeMs: getExecutionTimes(b), + TerraformToDirectMigration: b.Metrics.TerraformToDirectMigration, }, }, }) diff --git a/bundle/statemgmt/check_direct_migration.go b/bundle/statemgmt/check_direct_migration.go new file mode 100644 index 00000000000..e3dc9493848 --- /dev/null +++ b/bundle/statemgmt/check_direct_migration.go @@ -0,0 +1,123 @@ +package statemgmt + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/engine" + "github.com/databricks/cli/bundle/config/mutator/resourcemutator" + "github.com/databricks/cli/bundle/direct/dresources" + "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/cli/bundle/migrate" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/telemetry" + "github.com/databricks/cli/libs/telemetry/protos" +) + +// CheckDirectMigration performs an in-memory dry-run migration of the just-deployed +// terraform state to the direct engine and records the outcome in deploy telemetry. +// +// It writes nothing to local disk or the workspace; its only purpose is to measure, +// across the fleet, how many terraform deploys could migrate to the direct engine +// cleanly. Any error is surfaced to the user as a warning so it never fails a deploy +// that already succeeded; warnings raised during the migration are downgraded to +// info level and reported only via telemetry. +func CheckDirectMigration(ctx context.Context, b *bundle.Bundle) { + result := &protos.BundleDirectMigration{} + b.Metrics.TerraformToDirectMigration = result + + hasWarnings, err := migrateInMemory(ctx, b) + if err != nil { + result.ErrorMessage = telemetry.ScrubErrorMessage(err.Error()) + log.Warnf(ctx, "Dry-run migration to direct engine failed: %v", err) + return + } + + result.Success = true + result.HasWarnings = hasWarnings +} + +// migrateInMemory converts the local terraform state to the direct engine state +// entirely in memory, returning whether any warnings were emitted. It mirrors the +// `bundle migrate` command but never persists or uploads the resulting state. +func migrateInMemory(ctx context.Context, b *bundle.Bundle) (bool, error) { + _, localTerraformPath := b.StateFilenameTerraform(ctx) + tfState, err := migrate.ParseTFStateFull(ctx, localTerraformPath) + if err != nil { + return false, fmt.Errorf("failed to parse terraform state: %w", err) + } + + // ParseTFStateFull returns nil when the terraform state file doesn't exist + // (e.g. first deploy with no resources); nothing to migrate, trivially OK. + if tfState == nil { + return false, nil + } + + // SecretScopeFixups and the direct-engine state builder report failures via + // logdiag. Run them in an isolated context so a dry-run failure never affects + // the deploy's own diagnostics or exit code. + ctx = logdiag.IsolatedContext(ctx) + + state := make(map[string]dstate.ResourceEntry) + for key, id := range tfState.IDs { + state[key] = dstate.ResourceEntry{ + ID: id, + State: json.RawMessage("{}"), + } + } + + migratedDB := dstate.NewDatabase(tfState.Lineage, tfState.Serial+1) + migratedDB.State = state + + var stateDB dstate.DeploymentState + stateDB.OpenInMemoryForWrite(migratedDB) + + // Apply SecretScopeFixups so the config matches what the direct engine expects. + // This adds MANAGE ACL for the current user to all secret scopes, ensuring + // the migrated state and config agree on .permissions entries. + bundle.ApplyContext(ctx, b, resourcemutator.SecretScopeFixups(engine.EngineDirect)) + if logdiag.HasError(ctx) { + return false, errors.New("failed to apply secret scope fixups") + } + + // b.Config has been modified by terraform.Interpolate which converts bundle-style + // references (${resources.pipelines.x.id}) to terraform-style (${databricks_pipeline.x.id}). + // BuildStateFromTF expects ${resources.*} references, so reverse the interpolation first. + uninterpolatedRoot, err := reverseInterpolate(b.Config.Value()) + if err != nil { + return false, fmt.Errorf("failed to reverse interpolation: %w", err) + } + + var uninterpolatedConfig config.Root + err = uninterpolatedConfig.Mutate(func(_ dyn.Value) (dyn.Value, error) { + return uninterpolatedRoot, nil + }) + if err != nil { + return false, fmt.Errorf("failed to create uninterpolated config: %w", err) + } + + adapters, err := dresources.InitAll(nil) + if err != nil { + return false, err + } + + // downgradeWarnings=true: warnings are reported via telemetry, not surfaced to + // the user, since this is a background dry run on a deploy that already succeeded. + hasWarnings, err := migrate.BuildStateFromTF(ctx, &uninterpolatedConfig, adapters, &stateDB, tfState.Attrs, tfState.IDs, true) + if err != nil { + return hasWarnings, err + } + + // BuildStateFromTF reports some failures via logdiag instead of returning an error. + if logdiag.HasError(ctx) { + return hasWarnings, errors.New("state conversion failed") + } + + return hasWarnings, nil +} diff --git a/bundle/statemgmt/upload_state_for_yaml_sync.go b/bundle/statemgmt/upload_state_for_yaml_sync.go index 163c9fb4fdb..b43f477b69b 100644 --- a/bundle/statemgmt/upload_state_for_yaml_sync.go +++ b/bundle/statemgmt/upload_state_for_yaml_sync.go @@ -172,7 +172,7 @@ func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bun return false, fmt.Errorf("upgrading state for apply: %w", err) } - if err := migrate.BuildStateFromTF(ctx, &uninterpolatedConfig, adapters, &stateDB, tfState.Attrs, tfState.IDs); err != nil { + if _, err := migrate.BuildStateFromTF(ctx, &uninterpolatedConfig, adapters, &stateDB, tfState.Attrs, tfState.IDs, false); err != nil { return false, err } diff --git a/cmd/bundle/deployment/migrate.go b/cmd/bundle/deployment/migrate.go index 39d9a0454d5..1a28d4d6153 100644 --- a/cmd/bundle/deployment/migrate.go +++ b/cmd/bundle/deployment/migrate.go @@ -172,7 +172,7 @@ To start using direct engine, set "engine: direct" under bundle in your databric return fmt.Errorf("upgrading state for apply: %w", err) } - if err := migrate.BuildStateFromTF(ctx, &b.Config, adapters, &stateDB, tfState.Attrs, tfState.IDs); err != nil { + if _, err := migrate.BuildStateFromTF(ctx, &b.Config, adapters, &stateDB, tfState.Attrs, tfState.IDs, false); err != nil { return err } diff --git a/libs/telemetry/protos/bundle_deploy.go b/libs/telemetry/protos/bundle_deploy.go index 06521160c41..0ba77898216 100644 --- a/libs/telemetry/protos/bundle_deploy.go +++ b/libs/telemetry/protos/bundle_deploy.go @@ -110,6 +110,26 @@ type BundleDeployExperimental struct { // Local cache measurements in milliseconds (compute duration, potential savings, etc.) LocalCacheMeasurementsMs []IntMapEntry `json:"local_cache_measurements_ms,omitempty"` + + // Result of the in-memory dry-run migration from the terraform engine to the + // direct engine, attempted after a successful terraform deploy to measure how + // many bundles could migrate cleanly. Nil for direct deploys and for terraform + // deploys that had nothing to migrate. + TerraformToDirectMigration *BundleDirectMigration `json:"terraform_to_direct_migration,omitempty"` +} + +// BundleDirectMigration reports the outcome of the in-memory dry-run migration +// from the terraform engine to the direct engine. +type BundleDirectMigration struct { + // Whether the migration completed without error. + Success bool `json:"success"` + + // Scrubbed error message if the migration failed; empty on success. + ErrorMessage string `json:"error_message,omitempty"` + + // Whether the migration emitted any warnings (e.g. resources skipped because + // their type is not supported by the direct engine). + HasWarnings bool `json:"has_warnings,omitempty"` } // BundleResourcesMetadata mirrors the universe proto. Per-resource-type From aa1a93fd2f39ad1e63ce7d0524dd07c93c31f561 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 2 Jul 2026 10:51:32 +0200 Subject: [PATCH 2/4] Address review: temp-file dry-run, prefixed warnings, bool telemetry - Revert dstate changes; write the throwaway direct state to a temp file that is deleted afterwards instead of an in-memory write mode. - Keep migration warnings user-visible with a "post-deploy dry-run migration to direct: " prefix instead of downgrading them. - Record the outcome as direct_drymigrate_success / direct_drymigrate_warnings bool telemetry instead of a dedicated proto field. - Run the dry-run after "Deployment complete!". Co-authored-by: Isaac --- .../telemetry/deploy/out.migration.direct.txt | 2 +- .../deploy/out.migration.terraform.txt | 13 +++- acceptance/bundle/telemetry/deploy/script | 12 ++-- bundle/bundle.go | 5 -- bundle/direct/dstate/state.go | 49 +------------ bundle/metrics/metrics.go | 8 +++ bundle/migrate/build_state.go | 17 ++--- bundle/migrate/build_state_test.go | 2 +- bundle/phases/deploy.go | 14 ++-- bundle/phases/telemetry.go | 1 - bundle/statemgmt/check_direct_migration.go | 71 ++++++++++++------- .../statemgmt/upload_state_for_yaml_sync.go | 2 +- cmd/bundle/deployment/migrate.go | 2 +- libs/telemetry/protos/bundle_deploy.go | 20 ------ 14 files changed, 86 insertions(+), 132 deletions(-) diff --git a/acceptance/bundle/telemetry/deploy/out.migration.direct.txt b/acceptance/bundle/telemetry/deploy/out.migration.direct.txt index 19765bd501b..fe51488c706 100644 --- a/acceptance/bundle/telemetry/deploy/out.migration.direct.txt +++ b/acceptance/bundle/telemetry/deploy/out.migration.direct.txt @@ -1 +1 @@ -null +[] diff --git a/acceptance/bundle/telemetry/deploy/out.migration.terraform.txt b/acceptance/bundle/telemetry/deploy/out.migration.terraform.txt index 5550c6db40a..8e38b3f7595 100644 --- a/acceptance/bundle/telemetry/deploy/out.migration.terraform.txt +++ b/acceptance/bundle/telemetry/deploy/out.migration.terraform.txt @@ -1,3 +1,10 @@ -{ - "success": true -} +[ + { + "key": "direct_migration_success", + "value": true + }, + { + "key": "direct_migration_has_warnings", + "value": false + } +] diff --git a/acceptance/bundle/telemetry/deploy/script b/acceptance/bundle/telemetry/deploy/script index b258147a2c2..feb65287206 100644 --- a/acceptance/bundle/telemetry/deploy/script +++ b/acceptance/bundle/telemetry/deploy/script @@ -18,19 +18,19 @@ trace cat telemetry.json | jq ' .entry.databricks_cli_log.bundle_deploy_event.ex # 0.0.0-dev on windows) — it is still emitted in real telemetry. cat telemetry.json | jq '.entry.databricks_cli_log.bundle_deploy_event.resources_metadata | if . then del(.state_file_size_bytes) else . end' > out.resources_metadata.$DATABRICKS_BUNDLE_ENGINE.txt -# terraform_to_direct_migration is the result of the in-memory dry-run migration -# to the direct engine, attempted only after a terraform deploy. It therefore -# diverges across the DATABRICKS_BUNDLE_ENGINE matrix (direct produces "null"), -# so capture it per-engine and omit it from the engine-agnostic out.telemetry.txt. +# The dry-run migration to the direct engine runs only after a terraform deploy, +# so its bool_values entries (direct_drymigrate_*) diverge across the +# DATABRICKS_BUNDLE_ENGINE matrix (absent for direct). Capture them per-engine and +# drop them from the engine-agnostic out.telemetry.txt below. title "Assert the dry-run migration to the direct engine is reported" -cat telemetry.json | jq '.entry.databricks_cli_log.bundle_deploy_event.experimental.terraform_to_direct_migration' > out.migration.$DATABRICKS_BUNDLE_ENGINE.txt +cat telemetry.json | jq '[.entry.databricks_cli_log.bundle_deploy_event.experimental.bool_values[] | select(.key | startswith("direct_drymigrate_"))]' > out.migration.$DATABRICKS_BUNDLE_ENGINE.txt # bundle_mutator_execution_time_ms can have variable number of entries depending upon the runtime of the mutators. Thus we omit it from # being asserted here. # # upload_file_count and upload_file_sizes are asserted here and kept deterministic by # the sync.paths restriction in databricks.yml (see the comment there). -cat telemetry.json | jq 'del(.entry.databricks_cli_log.bundle_deploy_event.experimental.bundle_mutator_execution_time_ms, .entry.databricks_cli_log.bundle_deploy_event.experimental.terraform_to_direct_migration, .entry.databricks_cli_log.bundle_deploy_event.resources_metadata)' > out.telemetry.txt +cat telemetry.json | jq 'del(.entry.databricks_cli_log.bundle_deploy_event.experimental.bundle_mutator_execution_time_ms, .entry.databricks_cli_log.bundle_deploy_event.resources_metadata) | .entry.databricks_cli_log.bundle_deploy_event.experimental.bool_values |= map(select(.key | startswith("direct_drymigrate_") | not))' > out.telemetry.txt cmd_exec_id=$(extract_command_exec_id.py) deployment_id=$(cat .databricks/bundle/default/deployment.json | jq -r .id) diff --git a/bundle/bundle.go b/bundle/bundle.go index 4bdf818dc36..53332f73bdb 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -62,11 +62,6 @@ type Metrics struct { // bytes so deploy telemetry can be derived without re-reading or re-parsing // the state file. Nil for terraform deploys. ResourceState resourcestate.ExportedResourcesMap - - // TerraformToDirectMigration holds the result of the in-memory dry-run - // migration to the direct engine attempted after a successful terraform - // deploy. Nil unless a terraform deploy attempted the migration. - TerraformToDirectMigration *protos.BundleDirectMigration } // SetBoolValue sets the value of a boolean metric. diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 034a861b4af..eb41217f61d 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -31,21 +31,12 @@ const ( // The caller should delete the stale WAL and proceed normally. var errStaleWAL = errors.New("stale WAL") -// inMemoryPath is the placeholder Path for a state opened in in-memory write -// mode. It is never used for I/O; it only keeps the "opened" asserts happy. -const inMemoryPath = "(in-memory)" - type DeploymentState struct { Path string Data Database mu sync.Mutex walFile *os.File - // When true, SaveState/DeleteState mutate Data.State directly instead of - // appending to an on-disk WAL, so nothing is written to disk. Used for - // dry-run migrations (see OpenInMemoryForWrite). - inMemory bool - // Maps resource key to ID. Unlike Data.State, this is up to date during writes (deploys). stateIDs map[string]string } @@ -110,12 +101,6 @@ func (db *DeploymentState) SaveState(key, newID string, state any, dependsOn []d DependsOn: dependsOn, } - if db.inMemory { - db.Data.State[key] = entry - db.stateIDs[key] = newID - return nil - } - err = appendJSONLine(db.walFile, WALEntry{Key: key, Value: &entry}) if err == nil { db.stateIDs[key] = newID @@ -132,12 +117,6 @@ func (db *DeploymentState) DeleteState(key string) error { return nil } - if db.inMemory { - delete(db.Data.State, key) - delete(db.stateIDs, key) - return nil - } - err := appendJSONLine(db.walFile, WALEntry{Key: key}) if err == nil { delete(db.stateIDs, key) @@ -284,31 +263,6 @@ func (db *DeploymentState) OpenWithData(path string, data Database) { } } -// OpenInMemoryForWrite initializes the state from an in-memory database in write -// mode without touching disk. SaveState and DeleteState mutate Data.State -// directly instead of appending to an on-disk WAL, and no state file is ever -// written. It backs dry-run migrations that must not persist any state locally -// or remotely. -func (db *DeploymentState) OpenInMemoryForWrite(data Database) { - db.mu.Lock() - defer db.mu.Unlock() - - if db.Path != "" { - panic(fmt.Sprintf("state already opened: %v, cannot open in memory", db.Path)) - } - - db.Path = inMemoryPath - db.inMemory = true - db.Data = data - if db.Data.State == nil { - db.Data.State = make(map[string]ResourceEntry) - } - db.stateIDs = make(map[string]string) - for key, entry := range db.Data.State { - db.stateIDs[key] = entry.ID - } -} - func (db *DeploymentState) replayWAL(ctx context.Context) error { walPath := db.Path + walSuffix hasEntries, err := db.mergeWalIntoState(ctx) @@ -448,7 +402,6 @@ func (db *DeploymentState) Finalize(ctx context.Context) (resourcestate.Exported db.Path = "" db.Data = Database{} db.stateIDs = nil - db.inMemory = false return state, err } @@ -500,7 +453,7 @@ func (db *DeploymentState) AssertOpenedForRead() { func (db *DeploymentState) AssertOpenedForWrite() { db.AssertOpenedForReadOrWrite() - if db.walFile == nil && !db.inMemory { + if db.walFile == nil { panic("internal error: DeploymentState must be opened in write mode") } } diff --git a/bundle/metrics/metrics.go b/bundle/metrics/metrics.go index f5238e85202..618be8c5e4a 100644 --- a/bundle/metrics/metrics.go +++ b/bundle/metrics/metrics.go @@ -11,6 +11,14 @@ const ( SqlWarehouseLifecycleStarted = "sql_warehouse_lifecycle_started" SelectUsed = "select_used" + // Outcome of the dry-run migration to the direct engine attempted after a + // successful terraform deploy. DirectDryMigrateSuccess is false when the state + // could not be converted; DirectDryMigrateWarnings is true when the conversion + // emitted warnings (e.g. resources the direct engine can't represent). + // Only recorded on terraform deploys. + DirectDryMigrateSuccess = "direct_drymigrate_success" + DirectDryMigrateWarnings = "direct_drymigrate_warnings" + // Whether workspace.state_path is under /Workspace/Shared. StatePathIsShared = "state_path_is_shared" diff --git a/bundle/migrate/build_state.go b/bundle/migrate/build_state.go index 6e057523ea4..f66e1a3f35a 100644 --- a/bundle/migrate/build_state.go +++ b/bundle/migrate/build_state.go @@ -25,11 +25,10 @@ import ( // references using TF state attributes, and writes each resource's state entry. // configRoot should be an un-interpolated config (with ${resources.*} references). // -// When downgradeWarnings is true, warnings (e.g. skipped unsupported resource -// types) are logged at info level instead of warn level. This is used by the -// dry-run migration that runs after a terraform deploy, where these conditions -// are reported via telemetry rather than surfaced to the user. The returned bool -// reports whether any such warning was emitted, regardless of its log level. +// warnPrefix is prepended to every warning (e.g. skipped unsupported resource +// types). Callers that run the conversion in the background (the post-deploy +// dry-run) set it so their warnings are attributable and not confused with the +// user-invoked migration. The returned bool reports whether any warning fired. func BuildStateFromTF( ctx context.Context, configRoot *config.Root, @@ -37,16 +36,12 @@ func BuildStateFromTF( stateDB *dstate.DeploymentState, tfAttrs TFStateAttrs, tfIDs map[string]string, - downgradeWarnings bool, + warnPrefix string, ) (bool, error) { warningsSeen := false warnf := func(format string, args ...any) { warningsSeen = true - if downgradeWarnings { - log.Infof(ctx, format, args...) - } else { - log.Warnf(ctx, format, args...) - } + log.Warnf(ctx, warnPrefix+format, args...) } // Collect all resource nodes (same patterns as makePlan). var nodes []string diff --git a/bundle/migrate/build_state_test.go b/bundle/migrate/build_state_test.go index ba924c0f9be..a495436008c 100644 --- a/bundle/migrate/build_state_test.go +++ b/bundle/migrate/build_state_test.go @@ -50,7 +50,7 @@ func runBuildStateFromTF( db.OpenWithData(statePath, dstate.NewDatabase("lineage", 1)) require.NoError(t, db.UpgradeToWrite()) - _, err = migrate.BuildStateFromTF(t.Context(), &root, adapters, &db, tfAttrs, tfIDs, false) + _, err = migrate.BuildStateFromTF(t.Context(), &root, adapters, &db, tfAttrs, tfIDs, "") require.NoError(t, err) _, err = db.Finalize(t.Context()) diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index ff1107ad567..b9c595a9702 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -111,16 +111,16 @@ func deployCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, ta statemgmt.UploadStateForYamlSync(targetEngine), ) - // After a successful terraform deploy, dry-run the migration to the direct - // engine in memory and record the outcome in telemetry. It writes nothing and - // never fails the deploy. - if !targetEngine.IsDirect() && !logdiag.HasError(ctx) { - statemgmt.CheckDirectMigration(ctx, b) - } - if !logdiag.HasError(ctx) { cmdio.LogString(ctx, "Deployment complete!") } + + // Once the deploy is complete, dry-run the migration to the direct engine in + // memory and record the outcome in telemetry. It writes nothing and never + // fails the deploy. + if !targetEngine.IsDirect() && !logdiag.HasError(ctx) { + statemgmt.CheckDirectMigration(ctx, b) + } } // uploadLibraries uploads libraries to the workspace. diff --git a/bundle/phases/telemetry.go b/bundle/phases/telemetry.go index c5456414236..1a0123edd14 100644 --- a/bundle/phases/telemetry.go +++ b/bundle/phases/telemetry.go @@ -292,7 +292,6 @@ func LogDeployTelemetry(ctx context.Context, b *bundle.Bundle, errMsg string) { ComplexVariableCount: complexVariableCount, LookupVariableCount: lookupVariableCount, BundleMutatorExecutionTimeMs: getExecutionTimes(b), - TerraformToDirectMigration: b.Metrics.TerraformToDirectMigration, }, }, }) diff --git a/bundle/statemgmt/check_direct_migration.go b/bundle/statemgmt/check_direct_migration.go index e3dc9493848..2e20c65bc8a 100644 --- a/bundle/statemgmt/check_direct_migration.go +++ b/bundle/statemgmt/check_direct_migration.go @@ -5,6 +5,8 @@ import ( "encoding/json" "errors" "fmt" + "os" + "path/filepath" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" @@ -12,41 +14,39 @@ import ( "github.com/databricks/cli/bundle/config/mutator/resourcemutator" "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/cli/bundle/metrics" "github.com/databricks/cli/bundle/migrate" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" - "github.com/databricks/cli/libs/telemetry" - "github.com/databricks/cli/libs/telemetry/protos" ) -// CheckDirectMigration performs an in-memory dry-run migration of the just-deployed -// terraform state to the direct engine and records the outcome in deploy telemetry. +// warnPrefix labels warnings emitted by the post-deploy dry-run so they are not +// confused with warnings from the user-invoked `bundle migrate` command. +const warnPrefix = "post-deploy dry-run migration to direct: " + +// CheckDirectMigration performs a dry-run migration of the just-deployed terraform +// state to the direct engine and records the outcome in deploy telemetry. // -// It writes nothing to local disk or the workspace; its only purpose is to measure, -// across the fleet, how many terraform deploys could migrate to the direct engine -// cleanly. Any error is surfaced to the user as a warning so it never fails a deploy -// that already succeeded; warnings raised during the migration are downgraded to -// info level and reported only via telemetry. +// The converted state is written to a temporary file that is deleted before +// returning, so nothing is persisted locally or uploaded to the workspace; its +// only purpose is to measure, across the fleet, how many terraform deploys could +// migrate to the direct engine cleanly. Any error is surfaced to the user as a +// warning so it never fails a deploy that already succeeded. func CheckDirectMigration(ctx context.Context, b *bundle.Bundle) { - result := &protos.BundleDirectMigration{} - b.Metrics.TerraformToDirectMigration = result - - hasWarnings, err := migrateInMemory(ctx, b) + hasWarnings, err := dryRunMigrate(ctx, b) + b.Metrics.SetBoolValue(metrics.DirectDryMigrateSuccess, err == nil) + b.Metrics.SetBoolValue(metrics.DirectDryMigrateWarnings, hasWarnings) if err != nil { - result.ErrorMessage = telemetry.ScrubErrorMessage(err.Error()) - log.Warnf(ctx, "Dry-run migration to direct engine failed: %v", err) - return + log.Warnf(ctx, "%s%v", warnPrefix, err) } - - result.Success = true - result.HasWarnings = hasWarnings } -// migrateInMemory converts the local terraform state to the direct engine state -// entirely in memory, returning whether any warnings were emitted. It mirrors the -// `bundle migrate` command but never persists or uploads the resulting state. -func migrateInMemory(ctx context.Context, b *bundle.Bundle) (bool, error) { +// dryRunMigrate converts the local terraform state to the direct engine state, +// returning whether any warnings were emitted. It mirrors the `bundle migrate` +// command but writes the result to a throwaway temp file that is deleted before +// returning, and never uploads anything. +func dryRunMigrate(ctx context.Context, b *bundle.Bundle) (bool, error) { _, localTerraformPath := b.StateFilenameTerraform(ctx) tfState, err := migrate.ParseTFStateFull(ctx, localTerraformPath) if err != nil { @@ -59,6 +59,16 @@ func migrateInMemory(ctx context.Context, b *bundle.Bundle) (bool, error) { return false, nil } + // The converted state is a throwaway: write it to a temp dir that is removed + // (along with the WAL the state DB creates) before returning, so the dry run + // leaves nothing behind on disk. + tempDir, err := os.MkdirTemp("", "databricks-direct-migration-") + if err != nil { + return false, fmt.Errorf("failed to create temp dir: %w", err) + } + defer os.RemoveAll(tempDir) + tempStatePath := filepath.Join(tempDir, "resources.json") + // SecretScopeFixups and the direct-engine state builder report failures via // logdiag. Run them in an isolated context so a dry-run failure never affects // the deploy's own diagnostics or exit code. @@ -76,7 +86,7 @@ func migrateInMemory(ctx context.Context, b *bundle.Bundle) (bool, error) { migratedDB.State = state var stateDB dstate.DeploymentState - stateDB.OpenInMemoryForWrite(migratedDB) + stateDB.OpenWithData(tempStatePath, migratedDB) // Apply SecretScopeFixups so the config matches what the direct engine expects. // This adds MANAGE ACL for the current user to all secret scopes, ensuring @@ -107,13 +117,20 @@ func migrateInMemory(ctx context.Context, b *bundle.Bundle) (bool, error) { return false, err } - // downgradeWarnings=true: warnings are reported via telemetry, not surfaced to - // the user, since this is a background dry run on a deploy that already succeeded. - hasWarnings, err := migrate.BuildStateFromTF(ctx, &uninterpolatedConfig, adapters, &stateDB, tfState.Attrs, tfState.IDs, true) + if err := stateDB.UpgradeToWrite(); err != nil { + return false, fmt.Errorf("upgrading state for apply: %w", err) + } + + // warnPrefix labels the conversion's warnings as coming from the background dry run. + hasWarnings, err := migrate.BuildStateFromTF(ctx, &uninterpolatedConfig, adapters, &stateDB, tfState.Attrs, tfState.IDs, warnPrefix) if err != nil { return hasWarnings, err } + if _, err := stateDB.Finalize(ctx); err != nil { + return hasWarnings, err + } + // BuildStateFromTF reports some failures via logdiag instead of returning an error. if logdiag.HasError(ctx) { return hasWarnings, errors.New("state conversion failed") diff --git a/bundle/statemgmt/upload_state_for_yaml_sync.go b/bundle/statemgmt/upload_state_for_yaml_sync.go index b43f477b69b..80d4a3dbd12 100644 --- a/bundle/statemgmt/upload_state_for_yaml_sync.go +++ b/bundle/statemgmt/upload_state_for_yaml_sync.go @@ -172,7 +172,7 @@ func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bun return false, fmt.Errorf("upgrading state for apply: %w", err) } - if _, err := migrate.BuildStateFromTF(ctx, &uninterpolatedConfig, adapters, &stateDB, tfState.Attrs, tfState.IDs, false); err != nil { + if _, err := migrate.BuildStateFromTF(ctx, &uninterpolatedConfig, adapters, &stateDB, tfState.Attrs, tfState.IDs, ""); err != nil { return false, err } diff --git a/cmd/bundle/deployment/migrate.go b/cmd/bundle/deployment/migrate.go index 1a28d4d6153..93e229f71e8 100644 --- a/cmd/bundle/deployment/migrate.go +++ b/cmd/bundle/deployment/migrate.go @@ -172,7 +172,7 @@ To start using direct engine, set "engine: direct" under bundle in your databric return fmt.Errorf("upgrading state for apply: %w", err) } - if _, err := migrate.BuildStateFromTF(ctx, &b.Config, adapters, &stateDB, tfState.Attrs, tfState.IDs, false); err != nil { + if _, err := migrate.BuildStateFromTF(ctx, &b.Config, adapters, &stateDB, tfState.Attrs, tfState.IDs, ""); err != nil { return err } diff --git a/libs/telemetry/protos/bundle_deploy.go b/libs/telemetry/protos/bundle_deploy.go index 0ba77898216..06521160c41 100644 --- a/libs/telemetry/protos/bundle_deploy.go +++ b/libs/telemetry/protos/bundle_deploy.go @@ -110,26 +110,6 @@ type BundleDeployExperimental struct { // Local cache measurements in milliseconds (compute duration, potential savings, etc.) LocalCacheMeasurementsMs []IntMapEntry `json:"local_cache_measurements_ms,omitempty"` - - // Result of the in-memory dry-run migration from the terraform engine to the - // direct engine, attempted after a successful terraform deploy to measure how - // many bundles could migrate cleanly. Nil for direct deploys and for terraform - // deploys that had nothing to migrate. - TerraformToDirectMigration *BundleDirectMigration `json:"terraform_to_direct_migration,omitempty"` -} - -// BundleDirectMigration reports the outcome of the in-memory dry-run migration -// from the terraform engine to the direct engine. -type BundleDirectMigration struct { - // Whether the migration completed without error. - Success bool `json:"success"` - - // Scrubbed error message if the migration failed; empty on success. - ErrorMessage string `json:"error_message,omitempty"` - - // Whether the migration emitted any warnings (e.g. resources skipped because - // their type is not supported by the direct engine). - HasWarnings bool `json:"has_warnings,omitempty"` } // BundleResourcesMetadata mirrors the universe proto. Per-resource-type From d4c595eaed07caa6579ee7e9db9c0e0e7276eed6 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 2 Jul 2026 11:06:55 +0200 Subject: [PATCH 3/4] acceptance: capture dry-run migration telemetry per-engine Redirect the terraform-only direct_drymigrate_* bool_values snapshots (and the migration warning output) to per-engine files so the shared output stays engine-agnostic. Co-authored-by: Isaac --- .../url_ref/out.deploy.terraform.txt | 3 +- .../job_tasks/out.telemetry.terraform.txt | 2 + .../resources_var/out.telemetry.direct.txt | 20 ++ .../resources_var/out.telemetry.terraform.txt | 22 +++ .../resource_deps/resources_var/output.txt | 22 --- .../bundle/resource_deps/resources_var/script | 2 +- .../out.telemetry.terraform.txt | 2 + .../out.bool_values.direct.txt | 164 ++++++++++++++++ .../out.bool_values.terraform.txt | 180 ++++++++++++++++++ .../telemetry/deploy-compute-type/output.txt | 166 ---------------- .../telemetry/deploy-compute-type/script | 2 +- .../out.bool_values.direct.txt | 84 ++++++++ .../out.bool_values.terraform.txt | 92 +++++++++ .../telemetry/deploy-experimental/output.txt | 86 --------- .../telemetry/deploy-experimental/script | 2 +- .../custom/out.bool_values.direct.txt | 84 ++++++++ .../custom/out.bool_values.terraform.txt | 92 +++++++++ .../deploy-name-prefix/custom/output.txt | 86 --------- .../deploy-name-prefix/custom/script | 2 +- .../out.bool_values.direct.txt | 84 ++++++++ .../out.bool_values.terraform.txt | 92 +++++++++ .../mode-development/output.txt | 86 --------- .../mode-development/script | 2 +- .../out.bool_values.direct.txt | 164 ++++++++++++++++ .../out.bool_values.terraform.txt | 180 ++++++++++++++++++ .../telemetry/deploy-whl-artifacts/output.txt | 166 ---------------- .../telemetry/deploy-whl-artifacts/script | 2 +- .../deploy/out.migration.terraform.txt | 4 +- 28 files changed, 1272 insertions(+), 621 deletions(-) create mode 100644 acceptance/bundle/resource_deps/resources_var/out.telemetry.direct.txt create mode 100644 acceptance/bundle/resource_deps/resources_var/out.telemetry.terraform.txt create mode 100644 acceptance/bundle/telemetry/deploy-compute-type/out.bool_values.direct.txt create mode 100644 acceptance/bundle/telemetry/deploy-compute-type/out.bool_values.terraform.txt create mode 100644 acceptance/bundle/telemetry/deploy-experimental/out.bool_values.direct.txt create mode 100644 acceptance/bundle/telemetry/deploy-experimental/out.bool_values.terraform.txt create mode 100644 acceptance/bundle/telemetry/deploy-name-prefix/custom/out.bool_values.direct.txt create mode 100644 acceptance/bundle/telemetry/deploy-name-prefix/custom/out.bool_values.terraform.txt create mode 100644 acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.bool_values.direct.txt create mode 100644 acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.bool_values.terraform.txt create mode 100644 acceptance/bundle/telemetry/deploy-whl-artifacts/out.bool_values.direct.txt create mode 100644 acceptance/bundle/telemetry/deploy-whl-artifacts/out.bool_values.terraform.txt diff --git a/acceptance/bundle/bundle_tag/url_ref/out.deploy.terraform.txt b/acceptance/bundle/bundle_tag/url_ref/out.deploy.terraform.txt index 702ee4022f4..3959376eb13 100644 --- a/acceptance/bundle/bundle_tag/url_ref/out.deploy.terraform.txt +++ b/acceptance/bundle/bundle_tag/url_ref/out.deploy.terraform.txt @@ -4,5 +4,6 @@ Warning: "resources.jobs.foo.url": Terraform-only field; this reference will not Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Deploying resources... Updating deployment state... -Warn: Dry-run migration to direct engine failed: resources.jobs.bar: cannot set resolved value for field "url": field "url" not found in jobs.JobSettings Deployment complete! +Warn: post-deploy dry-run migration to direct: resource jobs.bar field url: method A value "[DATABRICKS_URL]/#job/[NUMID]" and method B value "[DATABRICKS_URL]/#job/[NUMID]" disagree; using longer (method A) +Warn: post-deploy dry-run migration to direct: resources.jobs.bar: cannot set resolved value for field "url": field "url" not found in jobs.JobSettings diff --git a/acceptance/bundle/resource_deps/job_tasks/out.telemetry.terraform.txt b/acceptance/bundle/resource_deps/job_tasks/out.telemetry.terraform.txt index bbd857524ea..15159258fa4 100644 --- a/acceptance/bundle/resource_deps/job_tasks/out.telemetry.terraform.txt +++ b/acceptance/bundle/resource_deps/job_tasks/out.telemetry.terraform.txt @@ -1,3 +1,5 @@ +direct_drymigrate_success true +direct_drymigrate_warnings false dms_compat_auto true dms_undeclared_deploying_user false dms_undeclared_group false diff --git a/acceptance/bundle/resource_deps/resources_var/out.telemetry.direct.txt b/acceptance/bundle/resource_deps/resources_var/out.telemetry.direct.txt new file mode 100644 index 00000000000..b4e373f349f --- /dev/null +++ b/acceptance/bundle/resource_deps/resources_var/out.telemetry.direct.txt @@ -0,0 +1,20 @@ +dms_compat_auto true +dms_undeclared_deploying_user false +dms_undeclared_group false +dms_undeclared_other_user false +dms_undeclared_service_principal false +experimental.use_legacy_run_as false +has_classic_interactive_compute false +has_classic_job_compute false +has_serverless_compute false +local.cache.attempt true +local.cache.hit true +permissions_section_set false +presets_name_prefix_is_set true +python_wheel_wrapper_is_set false +run_as_set false +skip_artifact_cleanup false +state_path_in_deployer_home true +state_path_in_other_user_home false +state_path_is_shared false +state_path_other false diff --git a/acceptance/bundle/resource_deps/resources_var/out.telemetry.terraform.txt b/acceptance/bundle/resource_deps/resources_var/out.telemetry.terraform.txt new file mode 100644 index 00000000000..5f14d4825ad --- /dev/null +++ b/acceptance/bundle/resource_deps/resources_var/out.telemetry.terraform.txt @@ -0,0 +1,22 @@ +direct_drymigrate_success true +direct_drymigrate_warnings false +dms_compat_auto true +dms_undeclared_deploying_user false +dms_undeclared_group false +dms_undeclared_other_user false +dms_undeclared_service_principal false +experimental.use_legacy_run_as false +has_classic_interactive_compute false +has_classic_job_compute false +has_serverless_compute false +local.cache.attempt true +local.cache.hit true +permissions_section_set false +presets_name_prefix_is_set true +python_wheel_wrapper_is_set false +run_as_set false +skip_artifact_cleanup false +state_path_in_deployer_home true +state_path_in_other_user_home false +state_path_is_shared false +state_path_other false diff --git a/acceptance/bundle/resource_deps/resources_var/output.txt b/acceptance/bundle/resource_deps/resources_var/output.txt index 34e6a69b57a..0b469d562db 100644 --- a/acceptance/bundle/resource_deps/resources_var/output.txt +++ b/acceptance/bundle/resource_deps/resources_var/output.txt @@ -34,25 +34,3 @@ >>> jq -s .[] | select(.path=="/api/2.0/pipelines") | .body.name out.requests.txt "[dev [USERNAME]] pipeline for mycatalog.myschema.myname" - ->>> print_telemetry_bool_values -dms_compat_auto true -dms_undeclared_deploying_user false -dms_undeclared_group false -dms_undeclared_other_user false -dms_undeclared_service_principal false -experimental.use_legacy_run_as false -has_classic_interactive_compute false -has_classic_job_compute false -has_serverless_compute false -local.cache.attempt true -local.cache.hit true -permissions_section_set false -presets_name_prefix_is_set true -python_wheel_wrapper_is_set false -run_as_set false -skip_artifact_cleanup false -state_path_in_deployer_home true -state_path_in_other_user_home false -state_path_is_shared false -state_path_other false diff --git a/acceptance/bundle/resource_deps/resources_var/script b/acceptance/bundle/resource_deps/resources_var/script index 5b195f9b2df..25154d36479 100644 --- a/acceptance/bundle/resource_deps/resources_var/script +++ b/acceptance/bundle/resource_deps/resources_var/script @@ -2,5 +2,5 @@ trace $CLI bundle validate -t dev -o json | jq .resources $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace errcode $CLI bundle deploy -t dev &> out.deploy.txt trace jq -s '.[] | select(.path=="/api/2.0/pipelines") | .body.name' out.requests.txt -trace print_telemetry_bool_values +print_telemetry_bool_values > out.telemetry.$DATABRICKS_BUNDLE_ENGINE.txt rm out.requests.txt diff --git a/acceptance/bundle/resource_deps/tf_path_only_error/out.telemetry.terraform.txt b/acceptance/bundle/resource_deps/tf_path_only_error/out.telemetry.terraform.txt index 9249391ce43..6075651fdde 100644 --- a/acceptance/bundle/resource_deps/tf_path_only_error/out.telemetry.terraform.txt +++ b/acceptance/bundle/resource_deps/tf_path_only_error/out.telemetry.terraform.txt @@ -1,3 +1,5 @@ +direct_drymigrate_success true +direct_drymigrate_warnings false dms_compat_auto true dms_undeclared_deploying_user false dms_undeclared_group false diff --git a/acceptance/bundle/telemetry/deploy-compute-type/out.bool_values.direct.txt b/acceptance/bundle/telemetry/deploy-compute-type/out.bool_values.direct.txt new file mode 100644 index 00000000000..9859175e208 --- /dev/null +++ b/acceptance/bundle/telemetry/deploy-compute-type/out.bool_values.direct.txt @@ -0,0 +1,164 @@ +[ + { + "key": "local.cache.attempt", + "value": true + }, + { + "key": "local.cache.miss", + "value": true + }, + { + "key": "experimental.use_legacy_run_as", + "value": false + }, + { + "key": "run_as_set", + "value": false + }, + { + "key": "presets_name_prefix_is_set", + "value": false + }, + { + "key": "python_wheel_wrapper_is_set", + "value": false + }, + { + "key": "skip_artifact_cleanup", + "value": false + }, + { + "key": "state_path_is_shared", + "value": false + }, + { + "key": "permissions_section_set", + "value": false + }, + { + "key": "state_path_in_deployer_home", + "value": true + }, + { + "key": "state_path_in_other_user_home", + "value": false + }, + { + "key": "state_path_other", + "value": false + }, + { + "key": "dms_undeclared_deploying_user", + "value": false + }, + { + "key": "dms_undeclared_other_user", + "value": false + }, + { + "key": "dms_undeclared_service_principal", + "value": false + }, + { + "key": "dms_undeclared_group", + "value": false + }, + { + "key": "dms_compat_auto", + "value": true + }, + { + "key": "has_serverless_compute", + "value": true + }, + { + "key": "has_classic_job_compute", + "value": true + }, + { + "key": "has_classic_interactive_compute", + "value": true + } +] +[ + { + "key": "local.cache.attempt", + "value": true + }, + { + "key": "local.cache.hit", + "value": true + }, + { + "key": "experimental.use_legacy_run_as", + "value": false + }, + { + "key": "run_as_set", + "value": false + }, + { + "key": "presets_name_prefix_is_set", + "value": false + }, + { + "key": "python_wheel_wrapper_is_set", + "value": false + }, + { + "key": "skip_artifact_cleanup", + "value": false + }, + { + "key": "state_path_is_shared", + "value": false + }, + { + "key": "permissions_section_set", + "value": false + }, + { + "key": "state_path_in_deployer_home", + "value": true + }, + { + "key": "state_path_in_other_user_home", + "value": false + }, + { + "key": "state_path_other", + "value": false + }, + { + "key": "dms_undeclared_deploying_user", + "value": false + }, + { + "key": "dms_undeclared_other_user", + "value": false + }, + { + "key": "dms_undeclared_service_principal", + "value": false + }, + { + "key": "dms_undeclared_group", + "value": false + }, + { + "key": "dms_compat_auto", + "value": true + }, + { + "key": "has_serverless_compute", + "value": true + }, + { + "key": "has_classic_job_compute", + "value": false + }, + { + "key": "has_classic_interactive_compute", + "value": false + } +] diff --git a/acceptance/bundle/telemetry/deploy-compute-type/out.bool_values.terraform.txt b/acceptance/bundle/telemetry/deploy-compute-type/out.bool_values.terraform.txt new file mode 100644 index 00000000000..8411b34e884 --- /dev/null +++ b/acceptance/bundle/telemetry/deploy-compute-type/out.bool_values.terraform.txt @@ -0,0 +1,180 @@ +[ + { + "key": "local.cache.attempt", + "value": true + }, + { + "key": "local.cache.miss", + "value": true + }, + { + "key": "experimental.use_legacy_run_as", + "value": false + }, + { + "key": "run_as_set", + "value": false + }, + { + "key": "presets_name_prefix_is_set", + "value": false + }, + { + "key": "python_wheel_wrapper_is_set", + "value": false + }, + { + "key": "skip_artifact_cleanup", + "value": false + }, + { + "key": "state_path_is_shared", + "value": false + }, + { + "key": "permissions_section_set", + "value": false + }, + { + "key": "state_path_in_deployer_home", + "value": true + }, + { + "key": "state_path_in_other_user_home", + "value": false + }, + { + "key": "state_path_other", + "value": false + }, + { + "key": "dms_undeclared_deploying_user", + "value": false + }, + { + "key": "dms_undeclared_other_user", + "value": false + }, + { + "key": "dms_undeclared_service_principal", + "value": false + }, + { + "key": "dms_undeclared_group", + "value": false + }, + { + "key": "dms_compat_auto", + "value": true + }, + { + "key": "has_serverless_compute", + "value": true + }, + { + "key": "has_classic_job_compute", + "value": true + }, + { + "key": "has_classic_interactive_compute", + "value": true + }, + { + "key": "direct_drymigrate_success", + "value": true + }, + { + "key": "direct_drymigrate_warnings", + "value": false + } +] +[ + { + "key": "local.cache.attempt", + "value": true + }, + { + "key": "local.cache.hit", + "value": true + }, + { + "key": "experimental.use_legacy_run_as", + "value": false + }, + { + "key": "run_as_set", + "value": false + }, + { + "key": "presets_name_prefix_is_set", + "value": false + }, + { + "key": "python_wheel_wrapper_is_set", + "value": false + }, + { + "key": "skip_artifact_cleanup", + "value": false + }, + { + "key": "state_path_is_shared", + "value": false + }, + { + "key": "permissions_section_set", + "value": false + }, + { + "key": "state_path_in_deployer_home", + "value": true + }, + { + "key": "state_path_in_other_user_home", + "value": false + }, + { + "key": "state_path_other", + "value": false + }, + { + "key": "dms_undeclared_deploying_user", + "value": false + }, + { + "key": "dms_undeclared_other_user", + "value": false + }, + { + "key": "dms_undeclared_service_principal", + "value": false + }, + { + "key": "dms_undeclared_group", + "value": false + }, + { + "key": "dms_compat_auto", + "value": true + }, + { + "key": "has_serverless_compute", + "value": true + }, + { + "key": "has_classic_job_compute", + "value": false + }, + { + "key": "has_classic_interactive_compute", + "value": false + }, + { + "key": "direct_drymigrate_success", + "value": true + }, + { + "key": "direct_drymigrate_warnings", + "value": false + } +] diff --git a/acceptance/bundle/telemetry/deploy-compute-type/output.txt b/acceptance/bundle/telemetry/deploy-compute-type/output.txt index 9c59aba8e71..3af4477bf55 100644 --- a/acceptance/bundle/telemetry/deploy-compute-type/output.txt +++ b/acceptance/bundle/telemetry/deploy-compute-type/output.txt @@ -10,169 +10,3 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-compute-typ Deploying resources... Updating deployment state... Deployment complete! - ->>> cat out.requests.txt -[ - { - "key": "local.cache.attempt", - "value": true - }, - { - "key": "local.cache.miss", - "value": true - }, - { - "key": "experimental.use_legacy_run_as", - "value": false - }, - { - "key": "run_as_set", - "value": false - }, - { - "key": "presets_name_prefix_is_set", - "value": false - }, - { - "key": "python_wheel_wrapper_is_set", - "value": false - }, - { - "key": "skip_artifact_cleanup", - "value": false - }, - { - "key": "state_path_is_shared", - "value": false - }, - { - "key": "permissions_section_set", - "value": false - }, - { - "key": "state_path_in_deployer_home", - "value": true - }, - { - "key": "state_path_in_other_user_home", - "value": false - }, - { - "key": "state_path_other", - "value": false - }, - { - "key": "dms_undeclared_deploying_user", - "value": false - }, - { - "key": "dms_undeclared_other_user", - "value": false - }, - { - "key": "dms_undeclared_service_principal", - "value": false - }, - { - "key": "dms_undeclared_group", - "value": false - }, - { - "key": "dms_compat_auto", - "value": true - }, - { - "key": "has_serverless_compute", - "value": true - }, - { - "key": "has_classic_job_compute", - "value": true - }, - { - "key": "has_classic_interactive_compute", - "value": true - } -] -[ - { - "key": "local.cache.attempt", - "value": true - }, - { - "key": "local.cache.hit", - "value": true - }, - { - "key": "experimental.use_legacy_run_as", - "value": false - }, - { - "key": "run_as_set", - "value": false - }, - { - "key": "presets_name_prefix_is_set", - "value": false - }, - { - "key": "python_wheel_wrapper_is_set", - "value": false - }, - { - "key": "skip_artifact_cleanup", - "value": false - }, - { - "key": "state_path_is_shared", - "value": false - }, - { - "key": "permissions_section_set", - "value": false - }, - { - "key": "state_path_in_deployer_home", - "value": true - }, - { - "key": "state_path_in_other_user_home", - "value": false - }, - { - "key": "state_path_other", - "value": false - }, - { - "key": "dms_undeclared_deploying_user", - "value": false - }, - { - "key": "dms_undeclared_other_user", - "value": false - }, - { - "key": "dms_undeclared_service_principal", - "value": false - }, - { - "key": "dms_undeclared_group", - "value": false - }, - { - "key": "dms_compat_auto", - "value": true - }, - { - "key": "has_serverless_compute", - "value": true - }, - { - "key": "has_classic_job_compute", - "value": false - }, - { - "key": "has_classic_interactive_compute", - "value": false - } -] diff --git a/acceptance/bundle/telemetry/deploy-compute-type/script b/acceptance/bundle/telemetry/deploy-compute-type/script index cfe8cba93be..1613750ef10 100644 --- a/acceptance/bundle/telemetry/deploy-compute-type/script +++ b/acceptance/bundle/telemetry/deploy-compute-type/script @@ -1,6 +1,6 @@ trace $CLI bundle deploy -t one trace $CLI bundle deploy -t two -trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.experimental.bool_values' +cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.experimental.bool_values' > out.bool_values.$DATABRICKS_BUNDLE_ENGINE.txt rm out.requests.txt diff --git a/acceptance/bundle/telemetry/deploy-experimental/out.bool_values.direct.txt b/acceptance/bundle/telemetry/deploy-experimental/out.bool_values.direct.txt new file mode 100644 index 00000000000..7c9748ed5f5 --- /dev/null +++ b/acceptance/bundle/telemetry/deploy-experimental/out.bool_values.direct.txt @@ -0,0 +1,84 @@ +{ + "bool_values": [ + { + "key": "local.cache.attempt", + "value": true + }, + { + "key": "local.cache.miss", + "value": true + }, + { + "key": "experimental.use_legacy_run_as", + "value": true + }, + { + "key": "run_as_set", + "value": true + }, + { + "key": "presets_name_prefix_is_set", + "value": false + }, + { + "key": "python_wheel_wrapper_is_set", + "value": false + }, + { + "key": "skip_artifact_cleanup", + "value": false + }, + { + "key": "state_path_is_shared", + "value": false + }, + { + "key": "permissions_section_set", + "value": false + }, + { + "key": "state_path_in_deployer_home", + "value": true + }, + { + "key": "state_path_in_other_user_home", + "value": false + }, + { + "key": "state_path_other", + "value": false + }, + { + "key": "dms_undeclared_deploying_user", + "value": false + }, + { + "key": "dms_undeclared_other_user", + "value": false + }, + { + "key": "dms_undeclared_service_principal", + "value": false + }, + { + "key": "dms_undeclared_group", + "value": false + }, + { + "key": "dms_compat_auto", + "value": true + }, + { + "key": "has_serverless_compute", + "value": false + }, + { + "key": "has_classic_job_compute", + "value": false + }, + { + "key": "has_classic_interactive_compute", + "value": true + } + ] +} diff --git a/acceptance/bundle/telemetry/deploy-experimental/out.bool_values.terraform.txt b/acceptance/bundle/telemetry/deploy-experimental/out.bool_values.terraform.txt new file mode 100644 index 00000000000..216fbabc677 --- /dev/null +++ b/acceptance/bundle/telemetry/deploy-experimental/out.bool_values.terraform.txt @@ -0,0 +1,92 @@ +{ + "bool_values": [ + { + "key": "local.cache.attempt", + "value": true + }, + { + "key": "local.cache.miss", + "value": true + }, + { + "key": "experimental.use_legacy_run_as", + "value": true + }, + { + "key": "run_as_set", + "value": true + }, + { + "key": "presets_name_prefix_is_set", + "value": false + }, + { + "key": "python_wheel_wrapper_is_set", + "value": false + }, + { + "key": "skip_artifact_cleanup", + "value": false + }, + { + "key": "state_path_is_shared", + "value": false + }, + { + "key": "permissions_section_set", + "value": false + }, + { + "key": "state_path_in_deployer_home", + "value": true + }, + { + "key": "state_path_in_other_user_home", + "value": false + }, + { + "key": "state_path_other", + "value": false + }, + { + "key": "dms_undeclared_deploying_user", + "value": false + }, + { + "key": "dms_undeclared_other_user", + "value": false + }, + { + "key": "dms_undeclared_service_principal", + "value": false + }, + { + "key": "dms_undeclared_group", + "value": false + }, + { + "key": "dms_compat_auto", + "value": true + }, + { + "key": "has_serverless_compute", + "value": false + }, + { + "key": "has_classic_job_compute", + "value": false + }, + { + "key": "has_classic_interactive_compute", + "value": true + }, + { + "key": "direct_drymigrate_success", + "value": true + }, + { + "key": "direct_drymigrate_warnings", + "value": false + } + ] +} diff --git a/acceptance/bundle/telemetry/deploy-experimental/output.txt b/acceptance/bundle/telemetry/deploy-experimental/output.txt index df24e19a74d..94db4beaf78 100644 --- a/acceptance/bundle/telemetry/deploy-experimental/output.txt +++ b/acceptance/bundle/telemetry/deploy-experimental/output.txt @@ -8,89 +8,3 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/defaul Deploying resources... Updating deployment state... Deployment complete! - ->>> cat out.requests.txt -{ - "bool_values": [ - { - "key": "local.cache.attempt", - "value": true - }, - { - "key": "local.cache.miss", - "value": true - }, - { - "key": "experimental.use_legacy_run_as", - "value": true - }, - { - "key": "run_as_set", - "value": true - }, - { - "key": "presets_name_prefix_is_set", - "value": false - }, - { - "key": "python_wheel_wrapper_is_set", - "value": false - }, - { - "key": "skip_artifact_cleanup", - "value": false - }, - { - "key": "state_path_is_shared", - "value": false - }, - { - "key": "permissions_section_set", - "value": false - }, - { - "key": "state_path_in_deployer_home", - "value": true - }, - { - "key": "state_path_in_other_user_home", - "value": false - }, - { - "key": "state_path_other", - "value": false - }, - { - "key": "dms_undeclared_deploying_user", - "value": false - }, - { - "key": "dms_undeclared_other_user", - "value": false - }, - { - "key": "dms_undeclared_service_principal", - "value": false - }, - { - "key": "dms_undeclared_group", - "value": false - }, - { - "key": "dms_compat_auto", - "value": true - }, - { - "key": "has_serverless_compute", - "value": false - }, - { - "key": "has_classic_job_compute", - "value": false - }, - { - "key": "has_classic_interactive_compute", - "value": true - } - ] -} diff --git a/acceptance/bundle/telemetry/deploy-experimental/script b/acceptance/bundle/telemetry/deploy-experimental/script index 67a3ba6299e..97b32292a48 100644 --- a/acceptance/bundle/telemetry/deploy-experimental/script +++ b/acceptance/bundle/telemetry/deploy-experimental/script @@ -1,5 +1,5 @@ trace $CLI bundle deploy -trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.experimental | {bool_values}' +cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.experimental | {bool_values}' > out.bool_values.$DATABRICKS_BUNDLE_ENGINE.txt rm out.requests.txt diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.bool_values.direct.txt b/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.bool_values.direct.txt new file mode 100644 index 00000000000..1cd8c04b1d0 --- /dev/null +++ b/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.bool_values.direct.txt @@ -0,0 +1,84 @@ +{ + "bool_values": [ + { + "key": "local.cache.attempt", + "value": true + }, + { + "key": "local.cache.miss", + "value": true + }, + { + "key": "experimental.use_legacy_run_as", + "value": false + }, + { + "key": "run_as_set", + "value": false + }, + { + "key": "presets_name_prefix_is_set", + "value": true + }, + { + "key": "python_wheel_wrapper_is_set", + "value": false + }, + { + "key": "skip_artifact_cleanup", + "value": false + }, + { + "key": "state_path_is_shared", + "value": false + }, + { + "key": "permissions_section_set", + "value": false + }, + { + "key": "state_path_in_deployer_home", + "value": true + }, + { + "key": "state_path_in_other_user_home", + "value": false + }, + { + "key": "state_path_other", + "value": false + }, + { + "key": "dms_undeclared_deploying_user", + "value": false + }, + { + "key": "dms_undeclared_other_user", + "value": false + }, + { + "key": "dms_undeclared_service_principal", + "value": false + }, + { + "key": "dms_undeclared_group", + "value": false + }, + { + "key": "dms_compat_auto", + "value": true + }, + { + "key": "has_serverless_compute", + "value": false + }, + { + "key": "has_classic_job_compute", + "value": false + }, + { + "key": "has_classic_interactive_compute", + "value": false + } + ] +} diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.bool_values.terraform.txt b/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.bool_values.terraform.txt new file mode 100644 index 00000000000..1f88db8f72f --- /dev/null +++ b/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.bool_values.terraform.txt @@ -0,0 +1,92 @@ +{ + "bool_values": [ + { + "key": "local.cache.attempt", + "value": true + }, + { + "key": "local.cache.miss", + "value": true + }, + { + "key": "experimental.use_legacy_run_as", + "value": false + }, + { + "key": "run_as_set", + "value": false + }, + { + "key": "presets_name_prefix_is_set", + "value": true + }, + { + "key": "python_wheel_wrapper_is_set", + "value": false + }, + { + "key": "skip_artifact_cleanup", + "value": false + }, + { + "key": "state_path_is_shared", + "value": false + }, + { + "key": "permissions_section_set", + "value": false + }, + { + "key": "state_path_in_deployer_home", + "value": true + }, + { + "key": "state_path_in_other_user_home", + "value": false + }, + { + "key": "state_path_other", + "value": false + }, + { + "key": "dms_undeclared_deploying_user", + "value": false + }, + { + "key": "dms_undeclared_other_user", + "value": false + }, + { + "key": "dms_undeclared_service_principal", + "value": false + }, + { + "key": "dms_undeclared_group", + "value": false + }, + { + "key": "dms_compat_auto", + "value": true + }, + { + "key": "has_serverless_compute", + "value": false + }, + { + "key": "has_classic_job_compute", + "value": false + }, + { + "key": "has_classic_interactive_compute", + "value": false + }, + { + "key": "direct_drymigrate_success", + "value": true + }, + { + "key": "direct_drymigrate_warnings", + "value": false + } + ] +} diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt b/acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt index f68b3f24975..d6ec8ecb9a7 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt +++ b/acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt @@ -4,89 +4,3 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/custom-prefix/defa Deploying resources... Updating deployment state... Deployment complete! - ->>> cat out.requests.txt -{ - "bool_values": [ - { - "key": "local.cache.attempt", - "value": true - }, - { - "key": "local.cache.miss", - "value": true - }, - { - "key": "experimental.use_legacy_run_as", - "value": false - }, - { - "key": "run_as_set", - "value": false - }, - { - "key": "presets_name_prefix_is_set", - "value": true - }, - { - "key": "python_wheel_wrapper_is_set", - "value": false - }, - { - "key": "skip_artifact_cleanup", - "value": false - }, - { - "key": "state_path_is_shared", - "value": false - }, - { - "key": "permissions_section_set", - "value": false - }, - { - "key": "state_path_in_deployer_home", - "value": true - }, - { - "key": "state_path_in_other_user_home", - "value": false - }, - { - "key": "state_path_other", - "value": false - }, - { - "key": "dms_undeclared_deploying_user", - "value": false - }, - { - "key": "dms_undeclared_other_user", - "value": false - }, - { - "key": "dms_undeclared_service_principal", - "value": false - }, - { - "key": "dms_undeclared_group", - "value": false - }, - { - "key": "dms_compat_auto", - "value": true - }, - { - "key": "has_serverless_compute", - "value": false - }, - { - "key": "has_classic_job_compute", - "value": false - }, - { - "key": "has_classic_interactive_compute", - "value": false - } - ] -} diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/custom/script b/acceptance/bundle/telemetry/deploy-name-prefix/custom/script index 67a3ba6299e..97b32292a48 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/custom/script +++ b/acceptance/bundle/telemetry/deploy-name-prefix/custom/script @@ -1,5 +1,5 @@ trace $CLI bundle deploy -trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.experimental | {bool_values}' +cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.experimental | {bool_values}' > out.bool_values.$DATABRICKS_BUNDLE_ENGINE.txt rm out.requests.txt diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.bool_values.direct.txt b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.bool_values.direct.txt new file mode 100644 index 00000000000..1cd8c04b1d0 --- /dev/null +++ b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.bool_values.direct.txt @@ -0,0 +1,84 @@ +{ + "bool_values": [ + { + "key": "local.cache.attempt", + "value": true + }, + { + "key": "local.cache.miss", + "value": true + }, + { + "key": "experimental.use_legacy_run_as", + "value": false + }, + { + "key": "run_as_set", + "value": false + }, + { + "key": "presets_name_prefix_is_set", + "value": true + }, + { + "key": "python_wheel_wrapper_is_set", + "value": false + }, + { + "key": "skip_artifact_cleanup", + "value": false + }, + { + "key": "state_path_is_shared", + "value": false + }, + { + "key": "permissions_section_set", + "value": false + }, + { + "key": "state_path_in_deployer_home", + "value": true + }, + { + "key": "state_path_in_other_user_home", + "value": false + }, + { + "key": "state_path_other", + "value": false + }, + { + "key": "dms_undeclared_deploying_user", + "value": false + }, + { + "key": "dms_undeclared_other_user", + "value": false + }, + { + "key": "dms_undeclared_service_principal", + "value": false + }, + { + "key": "dms_undeclared_group", + "value": false + }, + { + "key": "dms_compat_auto", + "value": true + }, + { + "key": "has_serverless_compute", + "value": false + }, + { + "key": "has_classic_job_compute", + "value": false + }, + { + "key": "has_classic_interactive_compute", + "value": false + } + ] +} diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.bool_values.terraform.txt b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.bool_values.terraform.txt new file mode 100644 index 00000000000..1f88db8f72f --- /dev/null +++ b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.bool_values.terraform.txt @@ -0,0 +1,92 @@ +{ + "bool_values": [ + { + "key": "local.cache.attempt", + "value": true + }, + { + "key": "local.cache.miss", + "value": true + }, + { + "key": "experimental.use_legacy_run_as", + "value": false + }, + { + "key": "run_as_set", + "value": false + }, + { + "key": "presets_name_prefix_is_set", + "value": true + }, + { + "key": "python_wheel_wrapper_is_set", + "value": false + }, + { + "key": "skip_artifact_cleanup", + "value": false + }, + { + "key": "state_path_is_shared", + "value": false + }, + { + "key": "permissions_section_set", + "value": false + }, + { + "key": "state_path_in_deployer_home", + "value": true + }, + { + "key": "state_path_in_other_user_home", + "value": false + }, + { + "key": "state_path_other", + "value": false + }, + { + "key": "dms_undeclared_deploying_user", + "value": false + }, + { + "key": "dms_undeclared_other_user", + "value": false + }, + { + "key": "dms_undeclared_service_principal", + "value": false + }, + { + "key": "dms_undeclared_group", + "value": false + }, + { + "key": "dms_compat_auto", + "value": true + }, + { + "key": "has_serverless_compute", + "value": false + }, + { + "key": "has_classic_job_compute", + "value": false + }, + { + "key": "has_classic_interactive_compute", + "value": false + }, + { + "key": "direct_drymigrate_success", + "value": true + }, + { + "key": "direct_drymigrate_warnings", + "value": false + } + ] +} diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt index da51756db48..d660ff4298a 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt +++ b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt @@ -4,89 +4,3 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/development-prefix Deploying resources... Updating deployment state... Deployment complete! - ->>> cat out.requests.txt -{ - "bool_values": [ - { - "key": "local.cache.attempt", - "value": true - }, - { - "key": "local.cache.miss", - "value": true - }, - { - "key": "experimental.use_legacy_run_as", - "value": false - }, - { - "key": "run_as_set", - "value": false - }, - { - "key": "presets_name_prefix_is_set", - "value": true - }, - { - "key": "python_wheel_wrapper_is_set", - "value": false - }, - { - "key": "skip_artifact_cleanup", - "value": false - }, - { - "key": "state_path_is_shared", - "value": false - }, - { - "key": "permissions_section_set", - "value": false - }, - { - "key": "state_path_in_deployer_home", - "value": true - }, - { - "key": "state_path_in_other_user_home", - "value": false - }, - { - "key": "state_path_other", - "value": false - }, - { - "key": "dms_undeclared_deploying_user", - "value": false - }, - { - "key": "dms_undeclared_other_user", - "value": false - }, - { - "key": "dms_undeclared_service_principal", - "value": false - }, - { - "key": "dms_undeclared_group", - "value": false - }, - { - "key": "dms_compat_auto", - "value": true - }, - { - "key": "has_serverless_compute", - "value": false - }, - { - "key": "has_classic_job_compute", - "value": false - }, - { - "key": "has_classic_interactive_compute", - "value": false - } - ] -} diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/script b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/script index 67a3ba6299e..97b32292a48 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/script +++ b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/script @@ -1,5 +1,5 @@ trace $CLI bundle deploy -trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.experimental | {bool_values}' +cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.experimental | {bool_values}' > out.bool_values.$DATABRICKS_BUNDLE_ENGINE.txt rm out.requests.txt diff --git a/acceptance/bundle/telemetry/deploy-whl-artifacts/out.bool_values.direct.txt b/acceptance/bundle/telemetry/deploy-whl-artifacts/out.bool_values.direct.txt new file mode 100644 index 00000000000..8468783d306 --- /dev/null +++ b/acceptance/bundle/telemetry/deploy-whl-artifacts/out.bool_values.direct.txt @@ -0,0 +1,164 @@ +{ + "bool_values": [ + { + "key": "local.cache.attempt", + "value": true + }, + { + "key": "local.cache.miss", + "value": true + }, + { + "key": "artifact_build_command_is_set", + "value": false + }, + { + "key": "artifact_files_is_set", + "value": false + }, + { + "key": "python_wheel_wrapper_is_set", + "value": false + }, + { + "key": "skip_artifact_cleanup", + "value": false + }, + { + "key": "state_path_is_shared", + "value": false + }, + { + "key": "permissions_section_set", + "value": false + }, + { + "key": "state_path_in_deployer_home", + "value": true + }, + { + "key": "state_path_in_other_user_home", + "value": false + }, + { + "key": "state_path_other", + "value": false + }, + { + "key": "dms_undeclared_deploying_user", + "value": false + }, + { + "key": "dms_undeclared_other_user", + "value": false + }, + { + "key": "dms_undeclared_service_principal", + "value": false + }, + { + "key": "dms_undeclared_group", + "value": false + }, + { + "key": "dms_compat_auto", + "value": true + }, + { + "key": "has_serverless_compute", + "value": false + }, + { + "key": "has_classic_job_compute", + "value": false + }, + { + "key": "has_classic_interactive_compute", + "value": false + } + ] +} +{ + "bool_values": [ + { + "key": "local.cache.attempt", + "value": true + }, + { + "key": "local.cache.hit", + "value": true + }, + { + "key": "artifact_build_command_is_set", + "value": true + }, + { + "key": "artifact_files_is_set", + "value": true + }, + { + "key": "artifact_dynamic_version_is_set", + "value": true + }, + { + "key": "python_wheel_wrapper_is_set", + "value": true + }, + { + "key": "skip_artifact_cleanup", + "value": true + }, + { + "key": "state_path_is_shared", + "value": false + }, + { + "key": "permissions_section_set", + "value": false + }, + { + "key": "state_path_in_deployer_home", + "value": true + }, + { + "key": "state_path_in_other_user_home", + "value": false + }, + { + "key": "state_path_other", + "value": false + }, + { + "key": "dms_undeclared_deploying_user", + "value": false + }, + { + "key": "dms_undeclared_other_user", + "value": false + }, + { + "key": "dms_undeclared_service_principal", + "value": false + }, + { + "key": "dms_undeclared_group", + "value": false + }, + { + "key": "dms_compat_auto", + "value": true + }, + { + "key": "has_serverless_compute", + "value": false + }, + { + "key": "has_classic_job_compute", + "value": false + }, + { + "key": "has_classic_interactive_compute", + "value": false + } + ] +} diff --git a/acceptance/bundle/telemetry/deploy-whl-artifacts/out.bool_values.terraform.txt b/acceptance/bundle/telemetry/deploy-whl-artifacts/out.bool_values.terraform.txt new file mode 100644 index 00000000000..e85c0030303 --- /dev/null +++ b/acceptance/bundle/telemetry/deploy-whl-artifacts/out.bool_values.terraform.txt @@ -0,0 +1,180 @@ +{ + "bool_values": [ + { + "key": "local.cache.attempt", + "value": true + }, + { + "key": "local.cache.miss", + "value": true + }, + { + "key": "artifact_build_command_is_set", + "value": false + }, + { + "key": "artifact_files_is_set", + "value": false + }, + { + "key": "python_wheel_wrapper_is_set", + "value": false + }, + { + "key": "skip_artifact_cleanup", + "value": false + }, + { + "key": "state_path_is_shared", + "value": false + }, + { + "key": "permissions_section_set", + "value": false + }, + { + "key": "state_path_in_deployer_home", + "value": true + }, + { + "key": "state_path_in_other_user_home", + "value": false + }, + { + "key": "state_path_other", + "value": false + }, + { + "key": "dms_undeclared_deploying_user", + "value": false + }, + { + "key": "dms_undeclared_other_user", + "value": false + }, + { + "key": "dms_undeclared_service_principal", + "value": false + }, + { + "key": "dms_undeclared_group", + "value": false + }, + { + "key": "dms_compat_auto", + "value": true + }, + { + "key": "has_serverless_compute", + "value": false + }, + { + "key": "has_classic_job_compute", + "value": false + }, + { + "key": "has_classic_interactive_compute", + "value": false + }, + { + "key": "direct_drymigrate_success", + "value": true + }, + { + "key": "direct_drymigrate_warnings", + "value": false + } + ] +} +{ + "bool_values": [ + { + "key": "local.cache.attempt", + "value": true + }, + { + "key": "local.cache.hit", + "value": true + }, + { + "key": "artifact_build_command_is_set", + "value": true + }, + { + "key": "artifact_files_is_set", + "value": true + }, + { + "key": "artifact_dynamic_version_is_set", + "value": true + }, + { + "key": "python_wheel_wrapper_is_set", + "value": true + }, + { + "key": "skip_artifact_cleanup", + "value": true + }, + { + "key": "state_path_is_shared", + "value": false + }, + { + "key": "permissions_section_set", + "value": false + }, + { + "key": "state_path_in_deployer_home", + "value": true + }, + { + "key": "state_path_in_other_user_home", + "value": false + }, + { + "key": "state_path_other", + "value": false + }, + { + "key": "dms_undeclared_deploying_user", + "value": false + }, + { + "key": "dms_undeclared_other_user", + "value": false + }, + { + "key": "dms_undeclared_service_principal", + "value": false + }, + { + "key": "dms_undeclared_group", + "value": false + }, + { + "key": "dms_compat_auto", + "value": true + }, + { + "key": "has_serverless_compute", + "value": false + }, + { + "key": "has_classic_job_compute", + "value": false + }, + { + "key": "has_classic_interactive_compute", + "value": false + }, + { + "key": "direct_drymigrate_success", + "value": true + }, + { + "key": "direct_drymigrate_warnings", + "value": false + } + ] +} diff --git a/acceptance/bundle/telemetry/deploy-whl-artifacts/output.txt b/acceptance/bundle/telemetry/deploy-whl-artifacts/output.txt index 7b8678cf5b7..8013f9cf782 100644 --- a/acceptance/bundle/telemetry/deploy-whl-artifacts/output.txt +++ b/acceptance/bundle/telemetry/deploy-whl-artifacts/output.txt @@ -12,169 +12,3 @@ Uploading .databricks/bundle/two/patched_wheels/test_my_test_code/my_test_code-0 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/two/files... Deploying resources... Deployment complete! - ->>> cat out.requests.txt -{ - "bool_values": [ - { - "key": "local.cache.attempt", - "value": true - }, - { - "key": "local.cache.miss", - "value": true - }, - { - "key": "artifact_build_command_is_set", - "value": false - }, - { - "key": "artifact_files_is_set", - "value": false - }, - { - "key": "python_wheel_wrapper_is_set", - "value": false - }, - { - "key": "skip_artifact_cleanup", - "value": false - }, - { - "key": "state_path_is_shared", - "value": false - }, - { - "key": "permissions_section_set", - "value": false - }, - { - "key": "state_path_in_deployer_home", - "value": true - }, - { - "key": "state_path_in_other_user_home", - "value": false - }, - { - "key": "state_path_other", - "value": false - }, - { - "key": "dms_undeclared_deploying_user", - "value": false - }, - { - "key": "dms_undeclared_other_user", - "value": false - }, - { - "key": "dms_undeclared_service_principal", - "value": false - }, - { - "key": "dms_undeclared_group", - "value": false - }, - { - "key": "dms_compat_auto", - "value": true - }, - { - "key": "has_serverless_compute", - "value": false - }, - { - "key": "has_classic_job_compute", - "value": false - }, - { - "key": "has_classic_interactive_compute", - "value": false - } - ] -} -{ - "bool_values": [ - { - "key": "local.cache.attempt", - "value": true - }, - { - "key": "local.cache.hit", - "value": true - }, - { - "key": "artifact_build_command_is_set", - "value": true - }, - { - "key": "artifact_files_is_set", - "value": true - }, - { - "key": "artifact_dynamic_version_is_set", - "value": true - }, - { - "key": "python_wheel_wrapper_is_set", - "value": true - }, - { - "key": "skip_artifact_cleanup", - "value": true - }, - { - "key": "state_path_is_shared", - "value": false - }, - { - "key": "permissions_section_set", - "value": false - }, - { - "key": "state_path_in_deployer_home", - "value": true - }, - { - "key": "state_path_in_other_user_home", - "value": false - }, - { - "key": "state_path_other", - "value": false - }, - { - "key": "dms_undeclared_deploying_user", - "value": false - }, - { - "key": "dms_undeclared_other_user", - "value": false - }, - { - "key": "dms_undeclared_service_principal", - "value": false - }, - { - "key": "dms_undeclared_group", - "value": false - }, - { - "key": "dms_compat_auto", - "value": true - }, - { - "key": "has_serverless_compute", - "value": false - }, - { - "key": "has_classic_job_compute", - "value": false - }, - { - "key": "has_classic_interactive_compute", - "value": false - } - ] -} diff --git a/acceptance/bundle/telemetry/deploy-whl-artifacts/script b/acceptance/bundle/telemetry/deploy-whl-artifacts/script index 078fa94cdd3..adab2fd4ae8 100644 --- a/acceptance/bundle/telemetry/deploy-whl-artifacts/script +++ b/acceptance/bundle/telemetry/deploy-whl-artifacts/script @@ -6,6 +6,6 @@ trace $CLI bundle deploy -t one trace $CLI bundle deploy -t two -trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.experimental | {bool_values}' +cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.experimental | {bool_values}' > out.bool_values.$DATABRICKS_BUNDLE_ENGINE.txt rm out.requests.txt diff --git a/acceptance/bundle/telemetry/deploy/out.migration.terraform.txt b/acceptance/bundle/telemetry/deploy/out.migration.terraform.txt index 8e38b3f7595..c503d0fb40b 100644 --- a/acceptance/bundle/telemetry/deploy/out.migration.terraform.txt +++ b/acceptance/bundle/telemetry/deploy/out.migration.terraform.txt @@ -1,10 +1,10 @@ [ { - "key": "direct_migration_success", + "key": "direct_drymigrate_success", "value": true }, { - "key": "direct_migration_has_warnings", + "key": "direct_drymigrate_warnings", "value": false } ] From f1e4c674c0dd39fb136ac7268646d8f7c601c56e Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 2 Jul 2026 13:11:40 +0200 Subject: [PATCH 4/4] Add feedback notice, compact per-engine telemetry snapshots - Append a notice after dry-run migration warnings pointing users to dabs-feedback@databricks.com and clarifying the deploy is unaffected. - Snapshot common bool_values in shared output and only the terraform-only direct_drymigrate_* entries per-engine, using print_telemetry_bool_values. - Drop obvious doc comment; changelog: drop "in-memory", add PR link. Co-authored-by: Isaac --- NEXT_CHANGELOG.md | 2 +- .../url_ref/out.deploy.terraform.txt | 3 + .../job_tasks/out.drymigrate.direct.txt | 0 .../job_tasks/out.drymigrate.terraform.txt | 2 + .../job_tasks/out.telemetry.direct.txt | 20 -- .../job_tasks/out.telemetry.terraform.txt | 22 --- .../bundle/resource_deps/job_tasks/output.txt | 20 ++ .../bundle/resource_deps/job_tasks/script | 3 +- .../resources_var/out.drymigrate.direct.txt | 0 .../out.drymigrate.terraform.txt | 2 + .../resources_var/out.telemetry.direct.txt | 20 -- .../resources_var/out.telemetry.terraform.txt | 22 --- .../resource_deps/resources_var/output.txt | 22 +++ .../bundle/resource_deps/resources_var/script | 3 +- .../out.bool_values.direct.txt | 164 ---------------- .../out.bool_values.terraform.txt | 180 ------------------ .../out.drymigrate.direct.txt | 0 .../out.drymigrate.terraform.txt | 4 + .../telemetry/deploy-compute-type/output.txt | 42 ++++ .../telemetry/deploy-compute-type/script | 5 +- .../out.bool_values.direct.txt | 84 -------- .../out.bool_values.terraform.txt | 92 --------- .../out.drymigrate.direct.txt | 0 .../out.drymigrate.terraform.txt | 2 + .../telemetry/deploy-experimental/output.txt | 22 +++ .../telemetry/deploy-experimental/script | 5 +- .../custom/out.bool_values.direct.txt | 84 -------- .../custom/out.bool_values.terraform.txt | 92 --------- .../custom/out.drymigrate.direct.txt | 0 .../custom/out.drymigrate.terraform.txt | 2 + .../deploy-name-prefix/custom/output.txt | 22 +++ .../deploy-name-prefix/custom/script | 5 +- .../out.bool_values.direct.txt | 84 -------- .../out.bool_values.terraform.txt | 92 --------- .../out.drymigrate.direct.txt | 0 .../out.drymigrate.terraform.txt | 2 + .../mode-development/output.txt | 22 +++ .../mode-development/script | 5 +- .../out.bool_values.direct.txt | 164 ---------------- .../out.bool_values.terraform.txt | 180 ------------------ .../out.drymigrate.direct.txt | 0 .../out.drymigrate.terraform.txt | 4 + .../telemetry/deploy-whl-artifacts/output.txt | 41 ++++ .../telemetry/deploy-whl-artifacts/script | 5 +- bundle/migrate/build_state.go | 17 +- bundle/migrate/resolve.go | 28 +-- bundle/migrate/resolve_test.go | 11 +- bundle/statemgmt/check_direct_migration.go | 9 + 48 files changed, 271 insertions(+), 1339 deletions(-) create mode 100644 acceptance/bundle/resource_deps/job_tasks/out.drymigrate.direct.txt create mode 100644 acceptance/bundle/resource_deps/job_tasks/out.drymigrate.terraform.txt delete mode 100644 acceptance/bundle/resource_deps/job_tasks/out.telemetry.direct.txt delete mode 100644 acceptance/bundle/resource_deps/job_tasks/out.telemetry.terraform.txt create mode 100644 acceptance/bundle/resource_deps/resources_var/out.drymigrate.direct.txt create mode 100644 acceptance/bundle/resource_deps/resources_var/out.drymigrate.terraform.txt delete mode 100644 acceptance/bundle/resource_deps/resources_var/out.telemetry.direct.txt delete mode 100644 acceptance/bundle/resource_deps/resources_var/out.telemetry.terraform.txt delete mode 100644 acceptance/bundle/telemetry/deploy-compute-type/out.bool_values.direct.txt delete mode 100644 acceptance/bundle/telemetry/deploy-compute-type/out.bool_values.terraform.txt create mode 100644 acceptance/bundle/telemetry/deploy-compute-type/out.drymigrate.direct.txt create mode 100644 acceptance/bundle/telemetry/deploy-compute-type/out.drymigrate.terraform.txt delete mode 100644 acceptance/bundle/telemetry/deploy-experimental/out.bool_values.direct.txt delete mode 100644 acceptance/bundle/telemetry/deploy-experimental/out.bool_values.terraform.txt create mode 100644 acceptance/bundle/telemetry/deploy-experimental/out.drymigrate.direct.txt create mode 100644 acceptance/bundle/telemetry/deploy-experimental/out.drymigrate.terraform.txt delete mode 100644 acceptance/bundle/telemetry/deploy-name-prefix/custom/out.bool_values.direct.txt delete mode 100644 acceptance/bundle/telemetry/deploy-name-prefix/custom/out.bool_values.terraform.txt create mode 100644 acceptance/bundle/telemetry/deploy-name-prefix/custom/out.drymigrate.direct.txt create mode 100644 acceptance/bundle/telemetry/deploy-name-prefix/custom/out.drymigrate.terraform.txt delete mode 100644 acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.bool_values.direct.txt delete mode 100644 acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.bool_values.terraform.txt create mode 100644 acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.drymigrate.direct.txt create mode 100644 acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.drymigrate.terraform.txt delete mode 100644 acceptance/bundle/telemetry/deploy-whl-artifacts/out.bool_values.direct.txt delete mode 100644 acceptance/bundle/telemetry/deploy-whl-artifacts/out.bool_values.terraform.txt create mode 100644 acceptance/bundle/telemetry/deploy-whl-artifacts/out.drymigrate.direct.txt create mode 100644 acceptance/bundle/telemetry/deploy-whl-artifacts/out.drymigrate.terraform.txt diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 55bc8be24f4..30b0b09e148 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -16,7 +16,7 @@ * direct: Fix spurious update when `apply_policy_default_values: true` is set on job task, for-each-task, or job cluster new_cluster ([#5731](https://github.com/databricks/cli/pull/5731)). Also fix spurious updates for for-each-task clusters due to missing backend defaults for `data_security_mode`, `node_type_id`, `driver_node_type_id`, `driver_instance_pool_id`, `enable_elastic_disk`, and `enable_local_disk_encryption`. * direct: Cluster resize now falls back to regular update if resize fails due to `INVALID_STATE` ([#5716](https://github.com/databricks/cli/pull/5716)). * Fixed `bundle deployment migrate` failing on `model_serving_endpoints`/`database_instances` with permissions (regression since v1.5.0) ([#5775](https://github.com/databricks/cli/pull/5775)). - * After a terraform deploy, the CLI now dry-runs an in-memory migration to the direct engine (writing nothing locally or remotely) and reports the outcome via telemetry, warning if the migration could not be completed. + * After a terraform deploy, the CLI now dry-runs a migration to the direct engine (writing nothing locally or remotely) and reports the outcome via telemetry, warning if the migration could not be completed ([#5797](https://github.com/databricks/cli/pull/5797)). ### Dependency updates diff --git a/acceptance/bundle/bundle_tag/url_ref/out.deploy.terraform.txt b/acceptance/bundle/bundle_tag/url_ref/out.deploy.terraform.txt index 3959376eb13..4533587ea7d 100644 --- a/acceptance/bundle/bundle_tag/url_ref/out.deploy.terraform.txt +++ b/acceptance/bundle/bundle_tag/url_ref/out.deploy.terraform.txt @@ -7,3 +7,6 @@ Updating deployment state... Deployment complete! Warn: post-deploy dry-run migration to direct: resource jobs.bar field url: method A value "[DATABRICKS_URL]/#job/[NUMID]" and method B value "[DATABRICKS_URL]/#job/[NUMID]" disagree; using longer (method A) Warn: post-deploy dry-run migration to direct: resources.jobs.bar: cannot set resolved value for field "url": field "url" not found in jobs.JobSettings +Warn: The warnings above are from a dry-run migration to the direct deployment engine (https://docs.databricks.com/aws/en/dev-tools/bundles/direct). +Your deployment is not affected and works normally, but you may experience these issues when migrating to the direct deployment engine. +Please forward these warnings to dabs-feedback@databricks.com diff --git a/acceptance/bundle/resource_deps/job_tasks/out.drymigrate.direct.txt b/acceptance/bundle/resource_deps/job_tasks/out.drymigrate.direct.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/resource_deps/job_tasks/out.drymigrate.terraform.txt b/acceptance/bundle/resource_deps/job_tasks/out.drymigrate.terraform.txt new file mode 100644 index 00000000000..d7fac6e7778 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_tasks/out.drymigrate.terraform.txt @@ -0,0 +1,2 @@ +direct_drymigrate_success true +direct_drymigrate_warnings false diff --git a/acceptance/bundle/resource_deps/job_tasks/out.telemetry.direct.txt b/acceptance/bundle/resource_deps/job_tasks/out.telemetry.direct.txt deleted file mode 100644 index bbd857524ea..00000000000 --- a/acceptance/bundle/resource_deps/job_tasks/out.telemetry.direct.txt +++ /dev/null @@ -1,20 +0,0 @@ -dms_compat_auto true -dms_undeclared_deploying_user false -dms_undeclared_group false -dms_undeclared_other_user false -dms_undeclared_service_principal false -experimental.use_legacy_run_as false -has_classic_interactive_compute false -has_classic_job_compute false -has_serverless_compute true -local.cache.attempt true -local.cache.miss true -permissions_section_set false -presets_name_prefix_is_set false -python_wheel_wrapper_is_set false -run_as_set false -skip_artifact_cleanup false -state_path_in_deployer_home true -state_path_in_other_user_home false -state_path_is_shared false -state_path_other false diff --git a/acceptance/bundle/resource_deps/job_tasks/out.telemetry.terraform.txt b/acceptance/bundle/resource_deps/job_tasks/out.telemetry.terraform.txt deleted file mode 100644 index 15159258fa4..00000000000 --- a/acceptance/bundle/resource_deps/job_tasks/out.telemetry.terraform.txt +++ /dev/null @@ -1,22 +0,0 @@ -direct_drymigrate_success true -direct_drymigrate_warnings false -dms_compat_auto true -dms_undeclared_deploying_user false -dms_undeclared_group false -dms_undeclared_other_user false -dms_undeclared_service_principal false -experimental.use_legacy_run_as false -has_classic_interactive_compute false -has_classic_job_compute false -has_serverless_compute true -local.cache.attempt true -local.cache.miss true -permissions_section_set false -presets_name_prefix_is_set false -python_wheel_wrapper_is_set false -run_as_set false -skip_artifact_cleanup false -state_path_in_deployer_home true -state_path_in_other_user_home false -state_path_is_shared false -state_path_other false diff --git a/acceptance/bundle/resource_deps/job_tasks/output.txt b/acceptance/bundle/resource_deps/job_tasks/output.txt index 3ff91361fb4..a7a53ff621c 100644 --- a/acceptance/bundle/resource_deps/job_tasks/output.txt +++ b/acceptance/bundle/resource_deps/job_tasks/output.txt @@ -7,3 +7,23 @@ Updating deployment state... Deployment complete! >>> print_telemetry_bool_values +dms_compat_auto true +dms_undeclared_deploying_user false +dms_undeclared_group false +dms_undeclared_other_user false +dms_undeclared_service_principal false +experimental.use_legacy_run_as false +has_classic_interactive_compute false +has_classic_job_compute false +has_serverless_compute true +local.cache.attempt true +local.cache.miss true +permissions_section_set false +presets_name_prefix_is_set false +python_wheel_wrapper_is_set false +run_as_set false +skip_artifact_cleanup false +state_path_in_deployer_home true +state_path_in_other_user_home false +state_path_is_shared false +state_path_other false diff --git a/acceptance/bundle/resource_deps/job_tasks/script b/acceptance/bundle/resource_deps/job_tasks/script index cbfe356121b..1634d9f381b 100644 --- a/acceptance/bundle/resource_deps/job_tasks/script +++ b/acceptance/bundle/resource_deps/job_tasks/script @@ -1,4 +1,5 @@ touch hello.whl trace $CLI bundle deploy -trace print_telemetry_bool_values > out.telemetry.$DATABRICKS_BUNDLE_ENGINE.txt +trace print_telemetry_bool_values | grep -v direct_drymigrate +print_telemetry_bool_values | grep direct_drymigrate > out.drymigrate.$DATABRICKS_BUNDLE_ENGINE.txt || true rm out.requests.txt hello.whl diff --git a/acceptance/bundle/resource_deps/resources_var/out.drymigrate.direct.txt b/acceptance/bundle/resource_deps/resources_var/out.drymigrate.direct.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/resource_deps/resources_var/out.drymigrate.terraform.txt b/acceptance/bundle/resource_deps/resources_var/out.drymigrate.terraform.txt new file mode 100644 index 00000000000..d7fac6e7778 --- /dev/null +++ b/acceptance/bundle/resource_deps/resources_var/out.drymigrate.terraform.txt @@ -0,0 +1,2 @@ +direct_drymigrate_success true +direct_drymigrate_warnings false diff --git a/acceptance/bundle/resource_deps/resources_var/out.telemetry.direct.txt b/acceptance/bundle/resource_deps/resources_var/out.telemetry.direct.txt deleted file mode 100644 index b4e373f349f..00000000000 --- a/acceptance/bundle/resource_deps/resources_var/out.telemetry.direct.txt +++ /dev/null @@ -1,20 +0,0 @@ -dms_compat_auto true -dms_undeclared_deploying_user false -dms_undeclared_group false -dms_undeclared_other_user false -dms_undeclared_service_principal false -experimental.use_legacy_run_as false -has_classic_interactive_compute false -has_classic_job_compute false -has_serverless_compute false -local.cache.attempt true -local.cache.hit true -permissions_section_set false -presets_name_prefix_is_set true -python_wheel_wrapper_is_set false -run_as_set false -skip_artifact_cleanup false -state_path_in_deployer_home true -state_path_in_other_user_home false -state_path_is_shared false -state_path_other false diff --git a/acceptance/bundle/resource_deps/resources_var/out.telemetry.terraform.txt b/acceptance/bundle/resource_deps/resources_var/out.telemetry.terraform.txt deleted file mode 100644 index 5f14d4825ad..00000000000 --- a/acceptance/bundle/resource_deps/resources_var/out.telemetry.terraform.txt +++ /dev/null @@ -1,22 +0,0 @@ -direct_drymigrate_success true -direct_drymigrate_warnings false -dms_compat_auto true -dms_undeclared_deploying_user false -dms_undeclared_group false -dms_undeclared_other_user false -dms_undeclared_service_principal false -experimental.use_legacy_run_as false -has_classic_interactive_compute false -has_classic_job_compute false -has_serverless_compute false -local.cache.attempt true -local.cache.hit true -permissions_section_set false -presets_name_prefix_is_set true -python_wheel_wrapper_is_set false -run_as_set false -skip_artifact_cleanup false -state_path_in_deployer_home true -state_path_in_other_user_home false -state_path_is_shared false -state_path_other false diff --git a/acceptance/bundle/resource_deps/resources_var/output.txt b/acceptance/bundle/resource_deps/resources_var/output.txt index 0b469d562db..34e6a69b57a 100644 --- a/acceptance/bundle/resource_deps/resources_var/output.txt +++ b/acceptance/bundle/resource_deps/resources_var/output.txt @@ -34,3 +34,25 @@ >>> jq -s .[] | select(.path=="/api/2.0/pipelines") | .body.name out.requests.txt "[dev [USERNAME]] pipeline for mycatalog.myschema.myname" + +>>> print_telemetry_bool_values +dms_compat_auto true +dms_undeclared_deploying_user false +dms_undeclared_group false +dms_undeclared_other_user false +dms_undeclared_service_principal false +experimental.use_legacy_run_as false +has_classic_interactive_compute false +has_classic_job_compute false +has_serverless_compute false +local.cache.attempt true +local.cache.hit true +permissions_section_set false +presets_name_prefix_is_set true +python_wheel_wrapper_is_set false +run_as_set false +skip_artifact_cleanup false +state_path_in_deployer_home true +state_path_in_other_user_home false +state_path_is_shared false +state_path_other false diff --git a/acceptance/bundle/resource_deps/resources_var/script b/acceptance/bundle/resource_deps/resources_var/script index 25154d36479..cc0db790b5c 100644 --- a/acceptance/bundle/resource_deps/resources_var/script +++ b/acceptance/bundle/resource_deps/resources_var/script @@ -2,5 +2,6 @@ trace $CLI bundle validate -t dev -o json | jq .resources $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace errcode $CLI bundle deploy -t dev &> out.deploy.txt trace jq -s '.[] | select(.path=="/api/2.0/pipelines") | .body.name' out.requests.txt -print_telemetry_bool_values > out.telemetry.$DATABRICKS_BUNDLE_ENGINE.txt +trace print_telemetry_bool_values | grep -v direct_drymigrate +print_telemetry_bool_values | grep direct_drymigrate > out.drymigrate.$DATABRICKS_BUNDLE_ENGINE.txt || true rm out.requests.txt diff --git a/acceptance/bundle/telemetry/deploy-compute-type/out.bool_values.direct.txt b/acceptance/bundle/telemetry/deploy-compute-type/out.bool_values.direct.txt deleted file mode 100644 index 9859175e208..00000000000 --- a/acceptance/bundle/telemetry/deploy-compute-type/out.bool_values.direct.txt +++ /dev/null @@ -1,164 +0,0 @@ -[ - { - "key": "local.cache.attempt", - "value": true - }, - { - "key": "local.cache.miss", - "value": true - }, - { - "key": "experimental.use_legacy_run_as", - "value": false - }, - { - "key": "run_as_set", - "value": false - }, - { - "key": "presets_name_prefix_is_set", - "value": false - }, - { - "key": "python_wheel_wrapper_is_set", - "value": false - }, - { - "key": "skip_artifact_cleanup", - "value": false - }, - { - "key": "state_path_is_shared", - "value": false - }, - { - "key": "permissions_section_set", - "value": false - }, - { - "key": "state_path_in_deployer_home", - "value": true - }, - { - "key": "state_path_in_other_user_home", - "value": false - }, - { - "key": "state_path_other", - "value": false - }, - { - "key": "dms_undeclared_deploying_user", - "value": false - }, - { - "key": "dms_undeclared_other_user", - "value": false - }, - { - "key": "dms_undeclared_service_principal", - "value": false - }, - { - "key": "dms_undeclared_group", - "value": false - }, - { - "key": "dms_compat_auto", - "value": true - }, - { - "key": "has_serverless_compute", - "value": true - }, - { - "key": "has_classic_job_compute", - "value": true - }, - { - "key": "has_classic_interactive_compute", - "value": true - } -] -[ - { - "key": "local.cache.attempt", - "value": true - }, - { - "key": "local.cache.hit", - "value": true - }, - { - "key": "experimental.use_legacy_run_as", - "value": false - }, - { - "key": "run_as_set", - "value": false - }, - { - "key": "presets_name_prefix_is_set", - "value": false - }, - { - "key": "python_wheel_wrapper_is_set", - "value": false - }, - { - "key": "skip_artifact_cleanup", - "value": false - }, - { - "key": "state_path_is_shared", - "value": false - }, - { - "key": "permissions_section_set", - "value": false - }, - { - "key": "state_path_in_deployer_home", - "value": true - }, - { - "key": "state_path_in_other_user_home", - "value": false - }, - { - "key": "state_path_other", - "value": false - }, - { - "key": "dms_undeclared_deploying_user", - "value": false - }, - { - "key": "dms_undeclared_other_user", - "value": false - }, - { - "key": "dms_undeclared_service_principal", - "value": false - }, - { - "key": "dms_undeclared_group", - "value": false - }, - { - "key": "dms_compat_auto", - "value": true - }, - { - "key": "has_serverless_compute", - "value": true - }, - { - "key": "has_classic_job_compute", - "value": false - }, - { - "key": "has_classic_interactive_compute", - "value": false - } -] diff --git a/acceptance/bundle/telemetry/deploy-compute-type/out.bool_values.terraform.txt b/acceptance/bundle/telemetry/deploy-compute-type/out.bool_values.terraform.txt deleted file mode 100644 index 8411b34e884..00000000000 --- a/acceptance/bundle/telemetry/deploy-compute-type/out.bool_values.terraform.txt +++ /dev/null @@ -1,180 +0,0 @@ -[ - { - "key": "local.cache.attempt", - "value": true - }, - { - "key": "local.cache.miss", - "value": true - }, - { - "key": "experimental.use_legacy_run_as", - "value": false - }, - { - "key": "run_as_set", - "value": false - }, - { - "key": "presets_name_prefix_is_set", - "value": false - }, - { - "key": "python_wheel_wrapper_is_set", - "value": false - }, - { - "key": "skip_artifact_cleanup", - "value": false - }, - { - "key": "state_path_is_shared", - "value": false - }, - { - "key": "permissions_section_set", - "value": false - }, - { - "key": "state_path_in_deployer_home", - "value": true - }, - { - "key": "state_path_in_other_user_home", - "value": false - }, - { - "key": "state_path_other", - "value": false - }, - { - "key": "dms_undeclared_deploying_user", - "value": false - }, - { - "key": "dms_undeclared_other_user", - "value": false - }, - { - "key": "dms_undeclared_service_principal", - "value": false - }, - { - "key": "dms_undeclared_group", - "value": false - }, - { - "key": "dms_compat_auto", - "value": true - }, - { - "key": "has_serverless_compute", - "value": true - }, - { - "key": "has_classic_job_compute", - "value": true - }, - { - "key": "has_classic_interactive_compute", - "value": true - }, - { - "key": "direct_drymigrate_success", - "value": true - }, - { - "key": "direct_drymigrate_warnings", - "value": false - } -] -[ - { - "key": "local.cache.attempt", - "value": true - }, - { - "key": "local.cache.hit", - "value": true - }, - { - "key": "experimental.use_legacy_run_as", - "value": false - }, - { - "key": "run_as_set", - "value": false - }, - { - "key": "presets_name_prefix_is_set", - "value": false - }, - { - "key": "python_wheel_wrapper_is_set", - "value": false - }, - { - "key": "skip_artifact_cleanup", - "value": false - }, - { - "key": "state_path_is_shared", - "value": false - }, - { - "key": "permissions_section_set", - "value": false - }, - { - "key": "state_path_in_deployer_home", - "value": true - }, - { - "key": "state_path_in_other_user_home", - "value": false - }, - { - "key": "state_path_other", - "value": false - }, - { - "key": "dms_undeclared_deploying_user", - "value": false - }, - { - "key": "dms_undeclared_other_user", - "value": false - }, - { - "key": "dms_undeclared_service_principal", - "value": false - }, - { - "key": "dms_undeclared_group", - "value": false - }, - { - "key": "dms_compat_auto", - "value": true - }, - { - "key": "has_serverless_compute", - "value": true - }, - { - "key": "has_classic_job_compute", - "value": false - }, - { - "key": "has_classic_interactive_compute", - "value": false - }, - { - "key": "direct_drymigrate_success", - "value": true - }, - { - "key": "direct_drymigrate_warnings", - "value": false - } -] diff --git a/acceptance/bundle/telemetry/deploy-compute-type/out.drymigrate.direct.txt b/acceptance/bundle/telemetry/deploy-compute-type/out.drymigrate.direct.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/telemetry/deploy-compute-type/out.drymigrate.terraform.txt b/acceptance/bundle/telemetry/deploy-compute-type/out.drymigrate.terraform.txt new file mode 100644 index 00000000000..d788ea96e0b --- /dev/null +++ b/acceptance/bundle/telemetry/deploy-compute-type/out.drymigrate.terraform.txt @@ -0,0 +1,4 @@ +direct_drymigrate_success true +direct_drymigrate_success true +direct_drymigrate_warnings false +direct_drymigrate_warnings false diff --git a/acceptance/bundle/telemetry/deploy-compute-type/output.txt b/acceptance/bundle/telemetry/deploy-compute-type/output.txt index 3af4477bf55..754948595ff 100644 --- a/acceptance/bundle/telemetry/deploy-compute-type/output.txt +++ b/acceptance/bundle/telemetry/deploy-compute-type/output.txt @@ -10,3 +10,45 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-compute-typ Deploying resources... Updating deployment state... Deployment complete! + +>>> print_telemetry_bool_values +dms_compat_auto true +dms_compat_auto true +dms_undeclared_deploying_user false +dms_undeclared_deploying_user false +dms_undeclared_group false +dms_undeclared_group false +dms_undeclared_other_user false +dms_undeclared_other_user false +dms_undeclared_service_principal false +dms_undeclared_service_principal false +experimental.use_legacy_run_as false +experimental.use_legacy_run_as false +has_classic_interactive_compute false +has_classic_interactive_compute true +has_classic_job_compute false +has_classic_job_compute true +has_serverless_compute true +has_serverless_compute true +local.cache.attempt true +local.cache.attempt true +local.cache.hit true +local.cache.miss true +permissions_section_set false +permissions_section_set false +presets_name_prefix_is_set false +presets_name_prefix_is_set false +python_wheel_wrapper_is_set false +python_wheel_wrapper_is_set false +run_as_set false +run_as_set false +skip_artifact_cleanup false +skip_artifact_cleanup false +state_path_in_deployer_home true +state_path_in_deployer_home true +state_path_in_other_user_home false +state_path_in_other_user_home false +state_path_is_shared false +state_path_is_shared false +state_path_other false +state_path_other false diff --git a/acceptance/bundle/telemetry/deploy-compute-type/script b/acceptance/bundle/telemetry/deploy-compute-type/script index 1613750ef10..6d2a0de0b05 100644 --- a/acceptance/bundle/telemetry/deploy-compute-type/script +++ b/acceptance/bundle/telemetry/deploy-compute-type/script @@ -1,6 +1,9 @@ trace $CLI bundle deploy -t one trace $CLI bundle deploy -t two -cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.experimental.bool_values' > out.bool_values.$DATABRICKS_BUNDLE_ENGINE.txt +# Common bool_values are engine-agnostic; the terraform-only direct_drymigrate_* +# entries go to a per-engine file so output.txt stays shared. +trace print_telemetry_bool_values | grep -v direct_drymigrate +print_telemetry_bool_values | grep direct_drymigrate > out.drymigrate.$DATABRICKS_BUNDLE_ENGINE.txt || true rm out.requests.txt diff --git a/acceptance/bundle/telemetry/deploy-experimental/out.bool_values.direct.txt b/acceptance/bundle/telemetry/deploy-experimental/out.bool_values.direct.txt deleted file mode 100644 index 7c9748ed5f5..00000000000 --- a/acceptance/bundle/telemetry/deploy-experimental/out.bool_values.direct.txt +++ /dev/null @@ -1,84 +0,0 @@ -{ - "bool_values": [ - { - "key": "local.cache.attempt", - "value": true - }, - { - "key": "local.cache.miss", - "value": true - }, - { - "key": "experimental.use_legacy_run_as", - "value": true - }, - { - "key": "run_as_set", - "value": true - }, - { - "key": "presets_name_prefix_is_set", - "value": false - }, - { - "key": "python_wheel_wrapper_is_set", - "value": false - }, - { - "key": "skip_artifact_cleanup", - "value": false - }, - { - "key": "state_path_is_shared", - "value": false - }, - { - "key": "permissions_section_set", - "value": false - }, - { - "key": "state_path_in_deployer_home", - "value": true - }, - { - "key": "state_path_in_other_user_home", - "value": false - }, - { - "key": "state_path_other", - "value": false - }, - { - "key": "dms_undeclared_deploying_user", - "value": false - }, - { - "key": "dms_undeclared_other_user", - "value": false - }, - { - "key": "dms_undeclared_service_principal", - "value": false - }, - { - "key": "dms_undeclared_group", - "value": false - }, - { - "key": "dms_compat_auto", - "value": true - }, - { - "key": "has_serverless_compute", - "value": false - }, - { - "key": "has_classic_job_compute", - "value": false - }, - { - "key": "has_classic_interactive_compute", - "value": true - } - ] -} diff --git a/acceptance/bundle/telemetry/deploy-experimental/out.bool_values.terraform.txt b/acceptance/bundle/telemetry/deploy-experimental/out.bool_values.terraform.txt deleted file mode 100644 index 216fbabc677..00000000000 --- a/acceptance/bundle/telemetry/deploy-experimental/out.bool_values.terraform.txt +++ /dev/null @@ -1,92 +0,0 @@ -{ - "bool_values": [ - { - "key": "local.cache.attempt", - "value": true - }, - { - "key": "local.cache.miss", - "value": true - }, - { - "key": "experimental.use_legacy_run_as", - "value": true - }, - { - "key": "run_as_set", - "value": true - }, - { - "key": "presets_name_prefix_is_set", - "value": false - }, - { - "key": "python_wheel_wrapper_is_set", - "value": false - }, - { - "key": "skip_artifact_cleanup", - "value": false - }, - { - "key": "state_path_is_shared", - "value": false - }, - { - "key": "permissions_section_set", - "value": false - }, - { - "key": "state_path_in_deployer_home", - "value": true - }, - { - "key": "state_path_in_other_user_home", - "value": false - }, - { - "key": "state_path_other", - "value": false - }, - { - "key": "dms_undeclared_deploying_user", - "value": false - }, - { - "key": "dms_undeclared_other_user", - "value": false - }, - { - "key": "dms_undeclared_service_principal", - "value": false - }, - { - "key": "dms_undeclared_group", - "value": false - }, - { - "key": "dms_compat_auto", - "value": true - }, - { - "key": "has_serverless_compute", - "value": false - }, - { - "key": "has_classic_job_compute", - "value": false - }, - { - "key": "has_classic_interactive_compute", - "value": true - }, - { - "key": "direct_drymigrate_success", - "value": true - }, - { - "key": "direct_drymigrate_warnings", - "value": false - } - ] -} diff --git a/acceptance/bundle/telemetry/deploy-experimental/out.drymigrate.direct.txt b/acceptance/bundle/telemetry/deploy-experimental/out.drymigrate.direct.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/telemetry/deploy-experimental/out.drymigrate.terraform.txt b/acceptance/bundle/telemetry/deploy-experimental/out.drymigrate.terraform.txt new file mode 100644 index 00000000000..d7fac6e7778 --- /dev/null +++ b/acceptance/bundle/telemetry/deploy-experimental/out.drymigrate.terraform.txt @@ -0,0 +1,2 @@ +direct_drymigrate_success true +direct_drymigrate_warnings false diff --git a/acceptance/bundle/telemetry/deploy-experimental/output.txt b/acceptance/bundle/telemetry/deploy-experimental/output.txt index 94db4beaf78..e8c847cd617 100644 --- a/acceptance/bundle/telemetry/deploy-experimental/output.txt +++ b/acceptance/bundle/telemetry/deploy-experimental/output.txt @@ -8,3 +8,25 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/defaul Deploying resources... Updating deployment state... Deployment complete! + +>>> print_telemetry_bool_values +dms_compat_auto true +dms_undeclared_deploying_user false +dms_undeclared_group false +dms_undeclared_other_user false +dms_undeclared_service_principal false +experimental.use_legacy_run_as true +has_classic_interactive_compute true +has_classic_job_compute false +has_serverless_compute false +local.cache.attempt true +local.cache.miss true +permissions_section_set false +presets_name_prefix_is_set false +python_wheel_wrapper_is_set false +run_as_set true +skip_artifact_cleanup false +state_path_in_deployer_home true +state_path_in_other_user_home false +state_path_is_shared false +state_path_other false diff --git a/acceptance/bundle/telemetry/deploy-experimental/script b/acceptance/bundle/telemetry/deploy-experimental/script index 97b32292a48..1e32b6c98ea 100644 --- a/acceptance/bundle/telemetry/deploy-experimental/script +++ b/acceptance/bundle/telemetry/deploy-experimental/script @@ -1,5 +1,8 @@ trace $CLI bundle deploy -cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.experimental | {bool_values}' > out.bool_values.$DATABRICKS_BUNDLE_ENGINE.txt +# Common bool_values are engine-agnostic; the terraform-only direct_drymigrate_* +# entries go to a per-engine file so output.txt stays shared. +trace print_telemetry_bool_values | grep -v direct_drymigrate +print_telemetry_bool_values | grep direct_drymigrate > out.drymigrate.$DATABRICKS_BUNDLE_ENGINE.txt || true rm out.requests.txt diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.bool_values.direct.txt b/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.bool_values.direct.txt deleted file mode 100644 index 1cd8c04b1d0..00000000000 --- a/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.bool_values.direct.txt +++ /dev/null @@ -1,84 +0,0 @@ -{ - "bool_values": [ - { - "key": "local.cache.attempt", - "value": true - }, - { - "key": "local.cache.miss", - "value": true - }, - { - "key": "experimental.use_legacy_run_as", - "value": false - }, - { - "key": "run_as_set", - "value": false - }, - { - "key": "presets_name_prefix_is_set", - "value": true - }, - { - "key": "python_wheel_wrapper_is_set", - "value": false - }, - { - "key": "skip_artifact_cleanup", - "value": false - }, - { - "key": "state_path_is_shared", - "value": false - }, - { - "key": "permissions_section_set", - "value": false - }, - { - "key": "state_path_in_deployer_home", - "value": true - }, - { - "key": "state_path_in_other_user_home", - "value": false - }, - { - "key": "state_path_other", - "value": false - }, - { - "key": "dms_undeclared_deploying_user", - "value": false - }, - { - "key": "dms_undeclared_other_user", - "value": false - }, - { - "key": "dms_undeclared_service_principal", - "value": false - }, - { - "key": "dms_undeclared_group", - "value": false - }, - { - "key": "dms_compat_auto", - "value": true - }, - { - "key": "has_serverless_compute", - "value": false - }, - { - "key": "has_classic_job_compute", - "value": false - }, - { - "key": "has_classic_interactive_compute", - "value": false - } - ] -} diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.bool_values.terraform.txt b/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.bool_values.terraform.txt deleted file mode 100644 index 1f88db8f72f..00000000000 --- a/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.bool_values.terraform.txt +++ /dev/null @@ -1,92 +0,0 @@ -{ - "bool_values": [ - { - "key": "local.cache.attempt", - "value": true - }, - { - "key": "local.cache.miss", - "value": true - }, - { - "key": "experimental.use_legacy_run_as", - "value": false - }, - { - "key": "run_as_set", - "value": false - }, - { - "key": "presets_name_prefix_is_set", - "value": true - }, - { - "key": "python_wheel_wrapper_is_set", - "value": false - }, - { - "key": "skip_artifact_cleanup", - "value": false - }, - { - "key": "state_path_is_shared", - "value": false - }, - { - "key": "permissions_section_set", - "value": false - }, - { - "key": "state_path_in_deployer_home", - "value": true - }, - { - "key": "state_path_in_other_user_home", - "value": false - }, - { - "key": "state_path_other", - "value": false - }, - { - "key": "dms_undeclared_deploying_user", - "value": false - }, - { - "key": "dms_undeclared_other_user", - "value": false - }, - { - "key": "dms_undeclared_service_principal", - "value": false - }, - { - "key": "dms_undeclared_group", - "value": false - }, - { - "key": "dms_compat_auto", - "value": true - }, - { - "key": "has_serverless_compute", - "value": false - }, - { - "key": "has_classic_job_compute", - "value": false - }, - { - "key": "has_classic_interactive_compute", - "value": false - }, - { - "key": "direct_drymigrate_success", - "value": true - }, - { - "key": "direct_drymigrate_warnings", - "value": false - } - ] -} diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.drymigrate.direct.txt b/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.drymigrate.direct.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.drymigrate.terraform.txt b/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.drymigrate.terraform.txt new file mode 100644 index 00000000000..d7fac6e7778 --- /dev/null +++ b/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.drymigrate.terraform.txt @@ -0,0 +1,2 @@ +direct_drymigrate_success true +direct_drymigrate_warnings false diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt b/acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt index d6ec8ecb9a7..4f5630089f2 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt +++ b/acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt @@ -4,3 +4,25 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/custom-prefix/defa Deploying resources... Updating deployment state... Deployment complete! + +>>> print_telemetry_bool_values +dms_compat_auto true +dms_undeclared_deploying_user false +dms_undeclared_group false +dms_undeclared_other_user false +dms_undeclared_service_principal false +experimental.use_legacy_run_as false +has_classic_interactive_compute false +has_classic_job_compute false +has_serverless_compute false +local.cache.attempt true +local.cache.miss true +permissions_section_set false +presets_name_prefix_is_set true +python_wheel_wrapper_is_set false +run_as_set false +skip_artifact_cleanup false +state_path_in_deployer_home true +state_path_in_other_user_home false +state_path_is_shared false +state_path_other false diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/custom/script b/acceptance/bundle/telemetry/deploy-name-prefix/custom/script index 97b32292a48..1e32b6c98ea 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/custom/script +++ b/acceptance/bundle/telemetry/deploy-name-prefix/custom/script @@ -1,5 +1,8 @@ trace $CLI bundle deploy -cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.experimental | {bool_values}' > out.bool_values.$DATABRICKS_BUNDLE_ENGINE.txt +# Common bool_values are engine-agnostic; the terraform-only direct_drymigrate_* +# entries go to a per-engine file so output.txt stays shared. +trace print_telemetry_bool_values | grep -v direct_drymigrate +print_telemetry_bool_values | grep direct_drymigrate > out.drymigrate.$DATABRICKS_BUNDLE_ENGINE.txt || true rm out.requests.txt diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.bool_values.direct.txt b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.bool_values.direct.txt deleted file mode 100644 index 1cd8c04b1d0..00000000000 --- a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.bool_values.direct.txt +++ /dev/null @@ -1,84 +0,0 @@ -{ - "bool_values": [ - { - "key": "local.cache.attempt", - "value": true - }, - { - "key": "local.cache.miss", - "value": true - }, - { - "key": "experimental.use_legacy_run_as", - "value": false - }, - { - "key": "run_as_set", - "value": false - }, - { - "key": "presets_name_prefix_is_set", - "value": true - }, - { - "key": "python_wheel_wrapper_is_set", - "value": false - }, - { - "key": "skip_artifact_cleanup", - "value": false - }, - { - "key": "state_path_is_shared", - "value": false - }, - { - "key": "permissions_section_set", - "value": false - }, - { - "key": "state_path_in_deployer_home", - "value": true - }, - { - "key": "state_path_in_other_user_home", - "value": false - }, - { - "key": "state_path_other", - "value": false - }, - { - "key": "dms_undeclared_deploying_user", - "value": false - }, - { - "key": "dms_undeclared_other_user", - "value": false - }, - { - "key": "dms_undeclared_service_principal", - "value": false - }, - { - "key": "dms_undeclared_group", - "value": false - }, - { - "key": "dms_compat_auto", - "value": true - }, - { - "key": "has_serverless_compute", - "value": false - }, - { - "key": "has_classic_job_compute", - "value": false - }, - { - "key": "has_classic_interactive_compute", - "value": false - } - ] -} diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.bool_values.terraform.txt b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.bool_values.terraform.txt deleted file mode 100644 index 1f88db8f72f..00000000000 --- a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.bool_values.terraform.txt +++ /dev/null @@ -1,92 +0,0 @@ -{ - "bool_values": [ - { - "key": "local.cache.attempt", - "value": true - }, - { - "key": "local.cache.miss", - "value": true - }, - { - "key": "experimental.use_legacy_run_as", - "value": false - }, - { - "key": "run_as_set", - "value": false - }, - { - "key": "presets_name_prefix_is_set", - "value": true - }, - { - "key": "python_wheel_wrapper_is_set", - "value": false - }, - { - "key": "skip_artifact_cleanup", - "value": false - }, - { - "key": "state_path_is_shared", - "value": false - }, - { - "key": "permissions_section_set", - "value": false - }, - { - "key": "state_path_in_deployer_home", - "value": true - }, - { - "key": "state_path_in_other_user_home", - "value": false - }, - { - "key": "state_path_other", - "value": false - }, - { - "key": "dms_undeclared_deploying_user", - "value": false - }, - { - "key": "dms_undeclared_other_user", - "value": false - }, - { - "key": "dms_undeclared_service_principal", - "value": false - }, - { - "key": "dms_undeclared_group", - "value": false - }, - { - "key": "dms_compat_auto", - "value": true - }, - { - "key": "has_serverless_compute", - "value": false - }, - { - "key": "has_classic_job_compute", - "value": false - }, - { - "key": "has_classic_interactive_compute", - "value": false - }, - { - "key": "direct_drymigrate_success", - "value": true - }, - { - "key": "direct_drymigrate_warnings", - "value": false - } - ] -} diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.drymigrate.direct.txt b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.drymigrate.direct.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.drymigrate.terraform.txt b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.drymigrate.terraform.txt new file mode 100644 index 00000000000..d7fac6e7778 --- /dev/null +++ b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.drymigrate.terraform.txt @@ -0,0 +1,2 @@ +direct_drymigrate_success true +direct_drymigrate_warnings false diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt index d660ff4298a..3122ad1ee9a 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt +++ b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt @@ -4,3 +4,25 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/development-prefix Deploying resources... Updating deployment state... Deployment complete! + +>>> print_telemetry_bool_values +dms_compat_auto true +dms_undeclared_deploying_user false +dms_undeclared_group false +dms_undeclared_other_user false +dms_undeclared_service_principal false +experimental.use_legacy_run_as false +has_classic_interactive_compute false +has_classic_job_compute false +has_serverless_compute false +local.cache.attempt true +local.cache.miss true +permissions_section_set false +presets_name_prefix_is_set true +python_wheel_wrapper_is_set false +run_as_set false +skip_artifact_cleanup false +state_path_in_deployer_home true +state_path_in_other_user_home false +state_path_is_shared false +state_path_other false diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/script b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/script index 97b32292a48..1e32b6c98ea 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/script +++ b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/script @@ -1,5 +1,8 @@ trace $CLI bundle deploy -cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.experimental | {bool_values}' > out.bool_values.$DATABRICKS_BUNDLE_ENGINE.txt +# Common bool_values are engine-agnostic; the terraform-only direct_drymigrate_* +# entries go to a per-engine file so output.txt stays shared. +trace print_telemetry_bool_values | grep -v direct_drymigrate +print_telemetry_bool_values | grep direct_drymigrate > out.drymigrate.$DATABRICKS_BUNDLE_ENGINE.txt || true rm out.requests.txt diff --git a/acceptance/bundle/telemetry/deploy-whl-artifacts/out.bool_values.direct.txt b/acceptance/bundle/telemetry/deploy-whl-artifacts/out.bool_values.direct.txt deleted file mode 100644 index 8468783d306..00000000000 --- a/acceptance/bundle/telemetry/deploy-whl-artifacts/out.bool_values.direct.txt +++ /dev/null @@ -1,164 +0,0 @@ -{ - "bool_values": [ - { - "key": "local.cache.attempt", - "value": true - }, - { - "key": "local.cache.miss", - "value": true - }, - { - "key": "artifact_build_command_is_set", - "value": false - }, - { - "key": "artifact_files_is_set", - "value": false - }, - { - "key": "python_wheel_wrapper_is_set", - "value": false - }, - { - "key": "skip_artifact_cleanup", - "value": false - }, - { - "key": "state_path_is_shared", - "value": false - }, - { - "key": "permissions_section_set", - "value": false - }, - { - "key": "state_path_in_deployer_home", - "value": true - }, - { - "key": "state_path_in_other_user_home", - "value": false - }, - { - "key": "state_path_other", - "value": false - }, - { - "key": "dms_undeclared_deploying_user", - "value": false - }, - { - "key": "dms_undeclared_other_user", - "value": false - }, - { - "key": "dms_undeclared_service_principal", - "value": false - }, - { - "key": "dms_undeclared_group", - "value": false - }, - { - "key": "dms_compat_auto", - "value": true - }, - { - "key": "has_serverless_compute", - "value": false - }, - { - "key": "has_classic_job_compute", - "value": false - }, - { - "key": "has_classic_interactive_compute", - "value": false - } - ] -} -{ - "bool_values": [ - { - "key": "local.cache.attempt", - "value": true - }, - { - "key": "local.cache.hit", - "value": true - }, - { - "key": "artifact_build_command_is_set", - "value": true - }, - { - "key": "artifact_files_is_set", - "value": true - }, - { - "key": "artifact_dynamic_version_is_set", - "value": true - }, - { - "key": "python_wheel_wrapper_is_set", - "value": true - }, - { - "key": "skip_artifact_cleanup", - "value": true - }, - { - "key": "state_path_is_shared", - "value": false - }, - { - "key": "permissions_section_set", - "value": false - }, - { - "key": "state_path_in_deployer_home", - "value": true - }, - { - "key": "state_path_in_other_user_home", - "value": false - }, - { - "key": "state_path_other", - "value": false - }, - { - "key": "dms_undeclared_deploying_user", - "value": false - }, - { - "key": "dms_undeclared_other_user", - "value": false - }, - { - "key": "dms_undeclared_service_principal", - "value": false - }, - { - "key": "dms_undeclared_group", - "value": false - }, - { - "key": "dms_compat_auto", - "value": true - }, - { - "key": "has_serverless_compute", - "value": false - }, - { - "key": "has_classic_job_compute", - "value": false - }, - { - "key": "has_classic_interactive_compute", - "value": false - } - ] -} diff --git a/acceptance/bundle/telemetry/deploy-whl-artifacts/out.bool_values.terraform.txt b/acceptance/bundle/telemetry/deploy-whl-artifacts/out.bool_values.terraform.txt deleted file mode 100644 index e85c0030303..00000000000 --- a/acceptance/bundle/telemetry/deploy-whl-artifacts/out.bool_values.terraform.txt +++ /dev/null @@ -1,180 +0,0 @@ -{ - "bool_values": [ - { - "key": "local.cache.attempt", - "value": true - }, - { - "key": "local.cache.miss", - "value": true - }, - { - "key": "artifact_build_command_is_set", - "value": false - }, - { - "key": "artifact_files_is_set", - "value": false - }, - { - "key": "python_wheel_wrapper_is_set", - "value": false - }, - { - "key": "skip_artifact_cleanup", - "value": false - }, - { - "key": "state_path_is_shared", - "value": false - }, - { - "key": "permissions_section_set", - "value": false - }, - { - "key": "state_path_in_deployer_home", - "value": true - }, - { - "key": "state_path_in_other_user_home", - "value": false - }, - { - "key": "state_path_other", - "value": false - }, - { - "key": "dms_undeclared_deploying_user", - "value": false - }, - { - "key": "dms_undeclared_other_user", - "value": false - }, - { - "key": "dms_undeclared_service_principal", - "value": false - }, - { - "key": "dms_undeclared_group", - "value": false - }, - { - "key": "dms_compat_auto", - "value": true - }, - { - "key": "has_serverless_compute", - "value": false - }, - { - "key": "has_classic_job_compute", - "value": false - }, - { - "key": "has_classic_interactive_compute", - "value": false - }, - { - "key": "direct_drymigrate_success", - "value": true - }, - { - "key": "direct_drymigrate_warnings", - "value": false - } - ] -} -{ - "bool_values": [ - { - "key": "local.cache.attempt", - "value": true - }, - { - "key": "local.cache.hit", - "value": true - }, - { - "key": "artifact_build_command_is_set", - "value": true - }, - { - "key": "artifact_files_is_set", - "value": true - }, - { - "key": "artifact_dynamic_version_is_set", - "value": true - }, - { - "key": "python_wheel_wrapper_is_set", - "value": true - }, - { - "key": "skip_artifact_cleanup", - "value": true - }, - { - "key": "state_path_is_shared", - "value": false - }, - { - "key": "permissions_section_set", - "value": false - }, - { - "key": "state_path_in_deployer_home", - "value": true - }, - { - "key": "state_path_in_other_user_home", - "value": false - }, - { - "key": "state_path_other", - "value": false - }, - { - "key": "dms_undeclared_deploying_user", - "value": false - }, - { - "key": "dms_undeclared_other_user", - "value": false - }, - { - "key": "dms_undeclared_service_principal", - "value": false - }, - { - "key": "dms_undeclared_group", - "value": false - }, - { - "key": "dms_compat_auto", - "value": true - }, - { - "key": "has_serverless_compute", - "value": false - }, - { - "key": "has_classic_job_compute", - "value": false - }, - { - "key": "has_classic_interactive_compute", - "value": false - }, - { - "key": "direct_drymigrate_success", - "value": true - }, - { - "key": "direct_drymigrate_warnings", - "value": false - } - ] -} diff --git a/acceptance/bundle/telemetry/deploy-whl-artifacts/out.drymigrate.direct.txt b/acceptance/bundle/telemetry/deploy-whl-artifacts/out.drymigrate.direct.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/telemetry/deploy-whl-artifacts/out.drymigrate.terraform.txt b/acceptance/bundle/telemetry/deploy-whl-artifacts/out.drymigrate.terraform.txt new file mode 100644 index 00000000000..d788ea96e0b --- /dev/null +++ b/acceptance/bundle/telemetry/deploy-whl-artifacts/out.drymigrate.terraform.txt @@ -0,0 +1,4 @@ +direct_drymigrate_success true +direct_drymigrate_success true +direct_drymigrate_warnings false +direct_drymigrate_warnings false diff --git a/acceptance/bundle/telemetry/deploy-whl-artifacts/output.txt b/acceptance/bundle/telemetry/deploy-whl-artifacts/output.txt index 8013f9cf782..95776306d7e 100644 --- a/acceptance/bundle/telemetry/deploy-whl-artifacts/output.txt +++ b/acceptance/bundle/telemetry/deploy-whl-artifacts/output.txt @@ -12,3 +12,44 @@ Uploading .databricks/bundle/two/patched_wheels/test_my_test_code/my_test_code-0 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/two/files... Deploying resources... Deployment complete! + +>>> print_telemetry_bool_values +artifact_build_command_is_set false +artifact_build_command_is_set true +artifact_dynamic_version_is_set true +artifact_files_is_set false +artifact_files_is_set true +dms_compat_auto true +dms_compat_auto true +dms_undeclared_deploying_user false +dms_undeclared_deploying_user false +dms_undeclared_group false +dms_undeclared_group false +dms_undeclared_other_user false +dms_undeclared_other_user false +dms_undeclared_service_principal false +dms_undeclared_service_principal false +has_classic_interactive_compute false +has_classic_interactive_compute false +has_classic_job_compute false +has_classic_job_compute false +has_serverless_compute false +has_serverless_compute false +local.cache.attempt true +local.cache.attempt true +local.cache.hit true +local.cache.miss true +permissions_section_set false +permissions_section_set false +python_wheel_wrapper_is_set false +python_wheel_wrapper_is_set true +skip_artifact_cleanup false +skip_artifact_cleanup true +state_path_in_deployer_home true +state_path_in_deployer_home true +state_path_in_other_user_home false +state_path_in_other_user_home false +state_path_is_shared false +state_path_is_shared false +state_path_other false +state_path_other false diff --git a/acceptance/bundle/telemetry/deploy-whl-artifacts/script b/acceptance/bundle/telemetry/deploy-whl-artifacts/script index adab2fd4ae8..b1ee8772a64 100644 --- a/acceptance/bundle/telemetry/deploy-whl-artifacts/script +++ b/acceptance/bundle/telemetry/deploy-whl-artifacts/script @@ -6,6 +6,9 @@ trace $CLI bundle deploy -t one trace $CLI bundle deploy -t two -cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.experimental | {bool_values}' > out.bool_values.$DATABRICKS_BUNDLE_ENGINE.txt +# Common bool_values are engine-agnostic; the terraform-only direct_drymigrate_* +# entries go to a per-engine file so output.txt stays shared. +trace print_telemetry_bool_values | grep -v direct_drymigrate +print_telemetry_bool_values | grep direct_drymigrate > out.drymigrate.$DATABRICKS_BUNDLE_ENGINE.txt || true rm out.requests.txt diff --git a/bundle/migrate/build_state.go b/bundle/migrate/build_state.go index f66e1a3f35a..c27602802cb 100644 --- a/bundle/migrate/build_state.go +++ b/bundle/migrate/build_state.go @@ -24,11 +24,6 @@ import ( // BuildStateFromTF iterates over bundle resources, resolves cross-resource // references using TF state attributes, and writes each resource's state entry. // configRoot should be an un-interpolated config (with ${resources.*} references). -// -// warnPrefix is prepended to every warning (e.g. skipped unsupported resource -// types). Callers that run the conversion in the background (the post-deploy -// dry-run) set it so their warnings are attributable and not confused with the -// user-invoked migration. The returned bool reports whether any warning fired. func BuildStateFromTF( ctx context.Context, configRoot *config.Root, @@ -39,10 +34,6 @@ func BuildStateFromTF( warnPrefix string, ) (bool, error) { warningsSeen := false - warnf := func(format string, args ...any) { - warningsSeen = true - log.Warnf(ctx, warnPrefix+format, args...) - } // Collect all resource nodes (same patterns as makePlan). var nodes []string patterns := []dyn.Pattern{ @@ -79,7 +70,8 @@ func BuildStateFromTF( adapter, ok := adapters[group] if !ok { - warnf("unsupported resource type %q for %s, skipping", group, node) + warningsSeen = true + log.Warnf(ctx, warnPrefix+"unsupported resource type %q for %s, skipping", group, node) continue } @@ -210,10 +202,13 @@ func BuildStateFromTF( // ResolveFieldRef returns the fully resolved value for this field, // using either Method A (TF state lookup) or Method B (template evaluation). - value, err := ResolveFieldRef(tfAttrs, srcGroup, srcName, fieldPath, pending.refTemplate, warnf) + value, warned, err := ResolveFieldRef(ctx, tfAttrs, srcGroup, srcName, fieldPath, pending.refTemplate, warnPrefix) if err != nil { return warningsSeen, fmt.Errorf("%s: cannot resolve field %q (template %q): %w", node, pending.fieldPathStr, pending.refTemplate, err) } + if warned { + warningsSeen = true + } // Set the resolved value directly and remove the ref entry. if err := structaccess.Set(sv.Value, fieldPath, value); err != nil { diff --git a/bundle/migrate/resolve.go b/bundle/migrate/resolve.go index 6d953bd1f7b..23d0180e3c6 100644 --- a/bundle/migrate/resolve.go +++ b/bundle/migrate/resolve.go @@ -1,11 +1,13 @@ package migrate import ( + "context" "fmt" "strings" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/structs/structpath" ) @@ -55,11 +57,11 @@ func evaluateTemplate(state TFStateAttrs, template string) (string, error) { // - Method A: read the field from the source resource's own TF state. // - Method B: evaluate the template by reading each referenced field from TF state. // -// Returns the reconciled value or an error if both methods fail. -// -// warnf reports a warning; the caller controls its severity (the post-deploy -// dry-run downgrades these to info level) and tracks whether any warning fired. -func ResolveFieldRef(state TFStateAttrs, srcGroup, srcName string, fieldPath *structpath.PathNode, refTemplate string, warnf func(format string, args ...any)) (any, error) { +// Returns the reconciled value or an error if both methods fail. The bool return +// reports whether a warning was logged (methods disagreed); warnPrefix is +// prepended to that warning so background callers (the post-deploy dry-run) can +// attribute it. +func ResolveFieldRef(ctx context.Context, state TFStateAttrs, srcGroup, srcName string, fieldPath *structpath.PathNode, refTemplate, warnPrefix string) (any, bool, error) { // Method A: read field from source resource's TF state. valueA, errA := LookupTFField(state, srcGroup, srcName, fieldPath) @@ -70,23 +72,23 @@ func ResolveFieldRef(state TFStateAttrs, srcGroup, srcName string, fieldPath *st case errA == nil && errB == nil: aStr := fmt.Sprintf("%v", valueA) if aStr == valueB { - return valueA, nil + return valueA, false, nil } // Both succeeded but disagree: prefer longer string and warn. if len(valueB) > len(aStr) { - warnf("resource %s.%s field %s: method A value %q and method B value %q disagree; using longer (method B)", + log.Warnf(ctx, warnPrefix+"resource %s.%s field %s: method A value %q and method B value %q disagree; using longer (method B)", srcGroup, srcName, fieldPath, aStr, valueB) - return valueB, nil + return valueB, true, nil } - warnf("resource %s.%s field %s: method A value %q and method B value %q disagree; using longer (method A)", + log.Warnf(ctx, warnPrefix+"resource %s.%s field %s: method A value %q and method B value %q disagree; using longer (method A)", srcGroup, srcName, fieldPath, aStr, valueB) - return valueA, nil + return valueA, true, nil case errA == nil: - return valueA, nil + return valueA, false, nil case errB == nil: - return valueB, nil + return valueB, false, nil default: - return nil, fmt.Errorf("%s.%s field %s: method A: %w; method B: %w", + return nil, false, fmt.Errorf("%s.%s field %s: method A: %w; method B: %w", srcGroup, srcName, fieldPath, errA, errB) } } diff --git a/bundle/migrate/resolve_test.go b/bundle/migrate/resolve_test.go index 2eda596d68e..1f6ee0d7533 100644 --- a/bundle/migrate/resolve_test.go +++ b/bundle/migrate/resolve_test.go @@ -11,9 +11,6 @@ import ( "github.com/stretchr/testify/require" ) -// noWarn is a warnf that drops messages; these cases never hit the warning path. -func noWarn(string, ...any) {} - // state with src job having int and bool fields set. func testState() migrate.TFStateAttrs { return migrate.TFStateAttrs{ @@ -43,8 +40,8 @@ func TestResolveFieldRefInt(t *testing.T) { fieldPath, err := structpath.ParsePath("max_concurrent_runs") require.NoError(t, err) - value, err := migrate.ResolveFieldRef(state, "jobs", "dst", fieldPath, - "${resources.jobs.src.max_concurrent_runs}", noWarn) + value, _, err := migrate.ResolveFieldRef(t.Context(), state, "jobs", "dst", fieldPath, + "${resources.jobs.src.max_concurrent_runs}", "") require.NoError(t, err) // Method B succeeds: returns string "4". Verify Set converts it to int. @@ -65,8 +62,8 @@ func TestResolveFieldRefBool(t *testing.T) { fieldPath, err := structpath.ParsePath("always_running") require.NoError(t, err) - value, err := migrate.ResolveFieldRef(state, "jobs", "dst", fieldPath, - "${resources.jobs.src.always_running}", noWarn) + value, _, err := migrate.ResolveFieldRef(t.Context(), state, "jobs", "dst", fieldPath, + "${resources.jobs.src.always_running}", "") require.NoError(t, err) type target struct { diff --git a/bundle/statemgmt/check_direct_migration.go b/bundle/statemgmt/check_direct_migration.go index 2e20c65bc8a..789213cf5c5 100644 --- a/bundle/statemgmt/check_direct_migration.go +++ b/bundle/statemgmt/check_direct_migration.go @@ -25,6 +25,12 @@ import ( // confused with warnings from the user-invoked `bundle migrate` command. const warnPrefix = "post-deploy dry-run migration to direct: " +// feedbackNotice is appended after the dry-run warnings to reassure the user the +// deploy is unaffected and to ask them to report the warnings. +const feedbackNotice = `The warnings above are from a dry-run migration to the direct deployment engine (https://docs.databricks.com/aws/en/dev-tools/bundles/direct). +Your deployment is not affected and works normally, but you may experience these issues when migrating to the direct deployment engine. +Please forward these warnings to dabs-feedback@databricks.com` + // CheckDirectMigration performs a dry-run migration of the just-deployed terraform // state to the direct engine and records the outcome in deploy telemetry. // @@ -40,6 +46,9 @@ func CheckDirectMigration(ctx context.Context, b *bundle.Bundle) { if err != nil { log.Warnf(ctx, "%s%v", warnPrefix, err) } + if hasWarnings || err != nil { + log.Warnf(ctx, "%s", feedbackNotice) + } } // dryRunMigrate converts the local terraform state to the direct engine state,