Dry-run migration to direct engine after terraform deploy

NEXT_CHANGELOG.md

@@ -16,6 +16,7 @@
  * direct: Fix spurious update when `apply_policy_default_values: true` is set on job task, for-each-task, or job cluster new_cluster ([#5731](https://github.com/databricks/cli/pull/5731)). Also fix spurious updates for for-each-task clusters due to missing backend defaults for `data_security_mode`, `node_type_id`, `driver_node_type_id`, `driver_instance_pool_id`, `enable_elastic_disk`, and `enable_local_disk_encryption`.
  * direct: Cluster resize now falls back to regular update if resize fails due to `INVALID_STATE` ([#5716](https://github.com/databricks/cli/pull/5716)).
  * Fixed `bundle deployment migrate` failing on `model_serving_endpoints`/`database_instances` with permissions (regression since v1.5.0) ([#5775](https://github.com/databricks/cli/pull/5775)).
+ * After a terraform deploy, the CLI now dry-runs a migration to the direct engine (writing nothing locally or remotely) and reports the outcome via telemetry, warning if the migration could not be completed ([#5797](https://github.com/databricks/cli/pull/5797)).
 
 ### Dependency updates
 

acceptance/bundle/bundle_tag/url_ref/out.deploy.terraform.txt

@@ -5,3 +5,8 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/defaul
 Deploying resources...
 Updating deployment state...
 Deployment complete!
+Warn: post-deploy dry-run migration to direct: resource jobs.bar field url: method A value "[DATABRICKS_URL]/#job/[NUMID]" and method B value "[DATABRICKS_URL]/#job/[NUMID]" disagree; using longer (method A)
+Warn: post-deploy dry-run migration to direct: resources.jobs.bar: cannot set resolved value for field "url": field "url" not found in jobs.JobSettings
+Warn: The warnings above are from a dry-run migration to the direct deployment engine (https://docs.databricks.com/aws/en/dev-tools/bundles/direct).
+Your deployment is not affected and works normally, but you may experience these issues when migrating to the direct deployment engine.
+Please forward these warnings to dabs-feedback@databricks.com

acceptance/bundle/resource_deps/job_tasks/out.drymigrate.terraform.txt

@@ -0,0 +1,2 @@
+direct_drymigrate_success true
+direct_drymigrate_warnings false

acceptance/bundle/resource_deps/job_tasks/output.txt

@@ -7,3 +7,23 @@ Updating deployment state...
 Deployment complete!
 
 >>> print_telemetry_bool_values
+dms_compat_auto true
+dms_undeclared_deploying_user false
+dms_undeclared_group false
+dms_undeclared_other_user false
+dms_undeclared_service_principal false
+experimental.use_legacy_run_as false
+has_classic_interactive_compute false
+has_classic_job_compute false
+has_serverless_compute true
+local.cache.attempt true
+local.cache.miss true
+permissions_section_set false
+presets_name_prefix_is_set false
+python_wheel_wrapper_is_set false
+run_as_set false
+skip_artifact_cleanup false
+state_path_in_deployer_home true
+state_path_in_other_user_home false
+state_path_is_shared false
+state_path_other false

acceptance/bundle/resource_deps/job_tasks/script

@@ -1,4 +1,5 @@
 touch hello.whl
 trace $CLI bundle deploy
-trace print_telemetry_bool_values > out.telemetry.$DATABRICKS_BUNDLE_ENGINE.txt
+trace print_telemetry_bool_values | grep -v direct_drymigrate
+print_telemetry_bool_values | grep direct_drymigrate > out.drymigrate.$DATABRICKS_BUNDLE_ENGINE.txt || true
 rm out.requests.txt hello.whl

acceptance/bundle/resource_deps/resources_var/out.drymigrate.terraform.txt

@@ -0,0 +1,2 @@
+direct_drymigrate_success true
+direct_drymigrate_warnings false

acceptance/bundle/resource_deps/resources_var/script

@@ -2,5 +2,6 @@ trace $CLI bundle validate -t dev -o json | jq .resources
 $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json
 trace errcode $CLI bundle deploy -t dev &> out.deploy.txt
 trace jq -s '.[] | select(.path=="/api/2.0/pipelines") | .body.name' out.requests.txt
-trace print_telemetry_bool_values
+trace print_telemetry_bool_values | grep -v direct_drymigrate
+print_telemetry_bool_values | grep direct_drymigrate > out.drymigrate.$DATABRICKS_BUNDLE_ENGINE.txt || true
 rm out.requests.txt

acceptance/bundle/resource_deps/tf_path_only_error/out.telemetry.terraform.txt

@@ -1,3 +1,5 @@
+direct_drymigrate_success true
+direct_drymigrate_warnings false
 dms_compat_auto true
 dms_undeclared_deploying_user false
 dms_undeclared_group false

acceptance/bundle/telemetry/deploy-compute-type/out.drymigrate.terraform.txt

@@ -0,0 +1,4 @@
+direct_drymigrate_success true
+direct_drymigrate_success true
+direct_drymigrate_warnings false
+direct_drymigrate_warnings false

acceptance/bundle/telemetry/deploy-compute-type/output.txt

@@ -11,168 +11,44 @@ Deploying resources...
 Updating deployment state...
 Deployment complete!
 
->>> cat out.requests.txt
-[
-  {
-    "key": "local.cache.attempt",
-    "value": true
-  },
-  {
-    "key": "local.cache.miss",
-    "value": true
-  },
-  {
-    "key": "experimental.use_legacy_run_as",
-    "value": false
-  },
-  {
-    "key": "run_as_set",
-    "value": false
-  },
-  {
-    "key": "presets_name_prefix_is_set",
-    "value": false
-  },
-  {
-    "key": "python_wheel_wrapper_is_set",
-    "value": false
-  },
-  {
-    "key": "skip_artifact_cleanup",
-    "value": false
-  },
-  {
-    "key": "state_path_is_shared",
-    "value": false
-  },
-  {
-    "key": "permissions_section_set",
-    "value": false
-  },
-  {
-    "key": "state_path_in_deployer_home",
-    "value": true
-  },
-  {
-    "key": "state_path_in_other_user_home",
-    "value": false
-  },
-  {
-    "key": "state_path_other",
-    "value": false
-  },
-  {
-    "key": "dms_undeclared_deploying_user",
-    "value": false
-  },
-  {
-    "key": "dms_undeclared_other_user",
-    "value": false
-  },
-  {
-    "key": "dms_undeclared_service_principal",
-    "value": false
-  },
-  {
-    "key": "dms_undeclared_group",
-    "value": false
-  },
-  {
-    "key": "dms_compat_auto",
-    "value": true
-  },
-  {
-    "key": "has_serverless_compute",
-    "value": true
-  },
-  {
-    "key": "has_classic_job_compute",
-    "value": true
-  },
-  {
-    "key": "has_classic_interactive_compute",
-    "value": true
-  }
-]
-[
-  {
-    "key": "local.cache.attempt",
-    "value": true
-  },
-  {
-    "key": "local.cache.hit",
-    "value": true
-  },
-  {
-    "key": "experimental.use_legacy_run_as",
-    "value": false
-  },
-  {
-    "key": "run_as_set",
-    "value": false
-  },
-  {
-    "key": "presets_name_prefix_is_set",
-    "value": false
-  },
-  {
-    "key": "python_wheel_wrapper_is_set",
-    "value": false
-  },
-  {
-    "key": "skip_artifact_cleanup",
-    "value": false
-  },
-  {
-    "key": "state_path_is_shared",
-    "value": false
-  },
-  {
-    "key": "permissions_section_set",
-    "value": false
-  },
-  {
-    "key": "state_path_in_deployer_home",
-    "value": true
-  },
-  {
-    "key": "state_path_in_other_user_home",
-    "value": false
-  },
-  {
-    "key": "state_path_other",
-    "value": false
-  },
-  {
-    "key": "dms_undeclared_deploying_user",
-    "value": false
-  },
-  {
-    "key": "dms_undeclared_other_user",
-    "value": false
-  },
-  {
-    "key": "dms_undeclared_service_principal",
-    "value": false
-  },
-  {
-    "key": "dms_undeclared_group",
-    "value": false
-  },
-  {
-    "key": "dms_compat_auto",
-    "value": true
-  },
-  {
-    "key": "has_serverless_compute",
-    "value": true
-  },
-  {
-    "key": "has_classic_job_compute",
-    "value": false
-  },
-  {
-    "key": "has_classic_interactive_compute",
-    "value": false
-  }
-]
+>>> print_telemetry_bool_values
+dms_compat_auto true
+dms_compat_auto true
+dms_undeclared_deploying_user false
+dms_undeclared_deploying_user false
+dms_undeclared_group false
+dms_undeclared_group false
+dms_undeclared_other_user false
+dms_undeclared_other_user false
+dms_undeclared_service_principal false
+dms_undeclared_service_principal false
+experimental.use_legacy_run_as false
+experimental.use_legacy_run_as false
+has_classic_interactive_compute false
+has_classic_interactive_compute true
+has_classic_job_compute false
+has_classic_job_compute true
+has_serverless_compute true
+has_serverless_compute true
+local.cache.attempt true
+local.cache.attempt true
+local.cache.hit true
+local.cache.miss true
+permissions_section_set false
+permissions_section_set false
+presets_name_prefix_is_set false
+presets_name_prefix_is_set false
+python_wheel_wrapper_is_set false
+python_wheel_wrapper_is_set false
+run_as_set false
+run_as_set false
+skip_artifact_cleanup false
+skip_artifact_cleanup false
+state_path_in_deployer_home true
+state_path_in_deployer_home true
+state_path_in_other_user_home false
+state_path_in_other_user_home false
+state_path_is_shared false
+state_path_is_shared false
+state_path_other false
+state_path_other false

acceptance/bundle/telemetry/deploy-compute-type/script

@@ -1,6 +1,9 @@
 trace $CLI bundle deploy -t one
 trace $CLI bundle deploy -t two
 
-trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.experimental.bool_values'
+# Common bool_values are engine-agnostic; the terraform-only direct_drymigrate_*
+# entries go to a per-engine file so output.txt stays shared.
+trace print_telemetry_bool_values | grep -v direct_drymigrate
+print_telemetry_bool_values | grep direct_drymigrate > out.drymigrate.$DATABRICKS_BUNDLE_ENGINE.txt || true
 
 rm out.requests.txt

acceptance/bundle/telemetry/deploy-experimental/out.drymigrate.terraform.txt

@@ -0,0 +1,2 @@
+direct_drymigrate_success true
+direct_drymigrate_warnings false