diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index a6cccd2a5cb..30b0b09e148 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -16,6 +16,7 @@ * direct: Fix spurious update when `apply_policy_default_values: true` is set on job task, for-each-task, or job cluster new_cluster ([#5731](https://github.com/databricks/cli/pull/5731)). Also fix spurious updates for for-each-task clusters due to missing backend defaults for `data_security_mode`, `node_type_id`, `driver_node_type_id`, `driver_instance_pool_id`, `enable_elastic_disk`, and `enable_local_disk_encryption`. * direct: Cluster resize now falls back to regular update if resize fails due to `INVALID_STATE` ([#5716](https://github.com/databricks/cli/pull/5716)). * Fixed `bundle deployment migrate` failing on `model_serving_endpoints`/`database_instances` with permissions (regression since v1.5.0) ([#5775](https://github.com/databricks/cli/pull/5775)). + * After a terraform deploy, the CLI now dry-runs a migration to the direct engine (writing nothing locally or remotely) and reports the outcome via telemetry, warning if the migration could not be completed ([#5797](https://github.com/databricks/cli/pull/5797)). ### Dependency updates diff --git a/acceptance/bundle/bundle_tag/url_ref/out.deploy.terraform.txt b/acceptance/bundle/bundle_tag/url_ref/out.deploy.terraform.txt index 92039fc564a..4533587ea7d 100644 --- a/acceptance/bundle/bundle_tag/url_ref/out.deploy.terraform.txt +++ b/acceptance/bundle/bundle_tag/url_ref/out.deploy.terraform.txt @@ -5,3 +5,8 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/defaul Deploying resources... Updating deployment state... Deployment complete! +Warn: post-deploy dry-run migration to direct: resource jobs.bar field url: method A value "[DATABRICKS_URL]/#job/[NUMID]" and method B value "[DATABRICKS_URL]/#job/[NUMID]" disagree; using longer (method A) +Warn: post-deploy dry-run migration to direct: resources.jobs.bar: cannot set resolved value for field "url": field "url" not found in jobs.JobSettings +Warn: The warnings above are from a dry-run migration to the direct deployment engine (https://docs.databricks.com/aws/en/dev-tools/bundles/direct). +Your deployment is not affected and works normally, but you may experience these issues when migrating to the direct deployment engine. +Please forward these warnings to dabs-feedback@databricks.com diff --git a/acceptance/bundle/resource_deps/job_tasks/out.drymigrate.direct.txt b/acceptance/bundle/resource_deps/job_tasks/out.drymigrate.direct.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/resource_deps/job_tasks/out.drymigrate.terraform.txt b/acceptance/bundle/resource_deps/job_tasks/out.drymigrate.terraform.txt new file mode 100644 index 00000000000..d7fac6e7778 --- /dev/null +++ b/acceptance/bundle/resource_deps/job_tasks/out.drymigrate.terraform.txt @@ -0,0 +1,2 @@ +direct_drymigrate_success true +direct_drymigrate_warnings false diff --git a/acceptance/bundle/resource_deps/job_tasks/out.telemetry.direct.txt b/acceptance/bundle/resource_deps/job_tasks/out.telemetry.direct.txt deleted file mode 100644 index bbd857524ea..00000000000 --- a/acceptance/bundle/resource_deps/job_tasks/out.telemetry.direct.txt +++ /dev/null @@ -1,20 +0,0 @@ -dms_compat_auto true -dms_undeclared_deploying_user false -dms_undeclared_group false -dms_undeclared_other_user false -dms_undeclared_service_principal false -experimental.use_legacy_run_as false -has_classic_interactive_compute false -has_classic_job_compute false -has_serverless_compute true -local.cache.attempt true -local.cache.miss true -permissions_section_set false -presets_name_prefix_is_set false -python_wheel_wrapper_is_set false -run_as_set false -skip_artifact_cleanup false -state_path_in_deployer_home true -state_path_in_other_user_home false -state_path_is_shared false -state_path_other false diff --git a/acceptance/bundle/resource_deps/job_tasks/out.telemetry.terraform.txt b/acceptance/bundle/resource_deps/job_tasks/out.telemetry.terraform.txt deleted file mode 100644 index bbd857524ea..00000000000 --- a/acceptance/bundle/resource_deps/job_tasks/out.telemetry.terraform.txt +++ /dev/null @@ -1,20 +0,0 @@ -dms_compat_auto true -dms_undeclared_deploying_user false -dms_undeclared_group false -dms_undeclared_other_user false -dms_undeclared_service_principal false -experimental.use_legacy_run_as false -has_classic_interactive_compute false -has_classic_job_compute false -has_serverless_compute true -local.cache.attempt true -local.cache.miss true -permissions_section_set false -presets_name_prefix_is_set false -python_wheel_wrapper_is_set false -run_as_set false -skip_artifact_cleanup false -state_path_in_deployer_home true -state_path_in_other_user_home false -state_path_is_shared false -state_path_other false diff --git a/acceptance/bundle/resource_deps/job_tasks/output.txt b/acceptance/bundle/resource_deps/job_tasks/output.txt index 3ff91361fb4..a7a53ff621c 100644 --- a/acceptance/bundle/resource_deps/job_tasks/output.txt +++ b/acceptance/bundle/resource_deps/job_tasks/output.txt @@ -7,3 +7,23 @@ Updating deployment state... Deployment complete! >>> print_telemetry_bool_values +dms_compat_auto true +dms_undeclared_deploying_user false +dms_undeclared_group false +dms_undeclared_other_user false +dms_undeclared_service_principal false +experimental.use_legacy_run_as false +has_classic_interactive_compute false +has_classic_job_compute false +has_serverless_compute true +local.cache.attempt true +local.cache.miss true +permissions_section_set false +presets_name_prefix_is_set false +python_wheel_wrapper_is_set false +run_as_set false +skip_artifact_cleanup false +state_path_in_deployer_home true +state_path_in_other_user_home false +state_path_is_shared false +state_path_other false diff --git a/acceptance/bundle/resource_deps/job_tasks/script b/acceptance/bundle/resource_deps/job_tasks/script index cbfe356121b..1634d9f381b 100644 --- a/acceptance/bundle/resource_deps/job_tasks/script +++ b/acceptance/bundle/resource_deps/job_tasks/script @@ -1,4 +1,5 @@ touch hello.whl trace $CLI bundle deploy -trace print_telemetry_bool_values > out.telemetry.$DATABRICKS_BUNDLE_ENGINE.txt +trace print_telemetry_bool_values | grep -v direct_drymigrate +print_telemetry_bool_values | grep direct_drymigrate > out.drymigrate.$DATABRICKS_BUNDLE_ENGINE.txt || true rm out.requests.txt hello.whl diff --git a/acceptance/bundle/resource_deps/resources_var/out.drymigrate.direct.txt b/acceptance/bundle/resource_deps/resources_var/out.drymigrate.direct.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/resource_deps/resources_var/out.drymigrate.terraform.txt b/acceptance/bundle/resource_deps/resources_var/out.drymigrate.terraform.txt new file mode 100644 index 00000000000..d7fac6e7778 --- /dev/null +++ b/acceptance/bundle/resource_deps/resources_var/out.drymigrate.terraform.txt @@ -0,0 +1,2 @@ +direct_drymigrate_success true +direct_drymigrate_warnings false diff --git a/acceptance/bundle/resource_deps/resources_var/script b/acceptance/bundle/resource_deps/resources_var/script index 5b195f9b2df..cc0db790b5c 100644 --- a/acceptance/bundle/resource_deps/resources_var/script +++ b/acceptance/bundle/resource_deps/resources_var/script @@ -2,5 +2,6 @@ trace $CLI bundle validate -t dev -o json | jq .resources $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace errcode $CLI bundle deploy -t dev &> out.deploy.txt trace jq -s '.[] | select(.path=="/api/2.0/pipelines") | .body.name' out.requests.txt -trace print_telemetry_bool_values +trace print_telemetry_bool_values | grep -v direct_drymigrate +print_telemetry_bool_values | grep direct_drymigrate > out.drymigrate.$DATABRICKS_BUNDLE_ENGINE.txt || true rm out.requests.txt diff --git a/acceptance/bundle/resource_deps/tf_path_only_error/out.telemetry.terraform.txt b/acceptance/bundle/resource_deps/tf_path_only_error/out.telemetry.terraform.txt index 9249391ce43..6075651fdde 100644 --- a/acceptance/bundle/resource_deps/tf_path_only_error/out.telemetry.terraform.txt +++ b/acceptance/bundle/resource_deps/tf_path_only_error/out.telemetry.terraform.txt @@ -1,3 +1,5 @@ +direct_drymigrate_success true +direct_drymigrate_warnings false dms_compat_auto true dms_undeclared_deploying_user false dms_undeclared_group false diff --git a/acceptance/bundle/telemetry/deploy-compute-type/out.drymigrate.direct.txt b/acceptance/bundle/telemetry/deploy-compute-type/out.drymigrate.direct.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/telemetry/deploy-compute-type/out.drymigrate.terraform.txt b/acceptance/bundle/telemetry/deploy-compute-type/out.drymigrate.terraform.txt new file mode 100644 index 00000000000..d788ea96e0b --- /dev/null +++ b/acceptance/bundle/telemetry/deploy-compute-type/out.drymigrate.terraform.txt @@ -0,0 +1,4 @@ +direct_drymigrate_success true +direct_drymigrate_success true +direct_drymigrate_warnings false +direct_drymigrate_warnings false diff --git a/acceptance/bundle/telemetry/deploy-compute-type/output.txt b/acceptance/bundle/telemetry/deploy-compute-type/output.txt index 9c59aba8e71..754948595ff 100644 --- a/acceptance/bundle/telemetry/deploy-compute-type/output.txt +++ b/acceptance/bundle/telemetry/deploy-compute-type/output.txt @@ -11,168 +11,44 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> cat out.requests.txt -[ - { - "key": "local.cache.attempt", - "value": true - }, - { - "key": "local.cache.miss", - "value": true - }, - { - "key": "experimental.use_legacy_run_as", - "value": false - }, - { - "key": "run_as_set", - "value": false - }, - { - "key": "presets_name_prefix_is_set", - "value": false - }, - { - "key": "python_wheel_wrapper_is_set", - "value": false - }, - { - "key": "skip_artifact_cleanup", - "value": false - }, - { - "key": "state_path_is_shared", - "value": false - }, - { - "key": "permissions_section_set", - "value": false - }, - { - "key": "state_path_in_deployer_home", - "value": true - }, - { - "key": "state_path_in_other_user_home", - "value": false - }, - { - "key": "state_path_other", - "value": false - }, - { - "key": "dms_undeclared_deploying_user", - "value": false - }, - { - "key": "dms_undeclared_other_user", - "value": false - }, - { - "key": "dms_undeclared_service_principal", - "value": false - }, - { - "key": "dms_undeclared_group", - "value": false - }, - { - "key": "dms_compat_auto", - "value": true - }, - { - "key": "has_serverless_compute", - "value": true - }, - { - "key": "has_classic_job_compute", - "value": true - }, - { - "key": "has_classic_interactive_compute", - "value": true - } -] -[ - { - "key": "local.cache.attempt", - "value": true - }, - { - "key": "local.cache.hit", - "value": true - }, - { - "key": "experimental.use_legacy_run_as", - "value": false - }, - { - "key": "run_as_set", - "value": false - }, - { - "key": "presets_name_prefix_is_set", - "value": false - }, - { - "key": "python_wheel_wrapper_is_set", - "value": false - }, - { - "key": "skip_artifact_cleanup", - "value": false - }, - { - "key": "state_path_is_shared", - "value": false - }, - { - "key": "permissions_section_set", - "value": false - }, - { - "key": "state_path_in_deployer_home", - "value": true - }, - { - "key": "state_path_in_other_user_home", - "value": false - }, - { - "key": "state_path_other", - "value": false - }, - { - "key": "dms_undeclared_deploying_user", - "value": false - }, - { - "key": "dms_undeclared_other_user", - "value": false - }, - { - "key": "dms_undeclared_service_principal", - "value": false - }, - { - "key": "dms_undeclared_group", - "value": false - }, - { - "key": "dms_compat_auto", - "value": true - }, - { - "key": "has_serverless_compute", - "value": true - }, - { - "key": "has_classic_job_compute", - "value": false - }, - { - "key": "has_classic_interactive_compute", - "value": false - } -] +>>> print_telemetry_bool_values +dms_compat_auto true +dms_compat_auto true +dms_undeclared_deploying_user false +dms_undeclared_deploying_user false +dms_undeclared_group false +dms_undeclared_group false +dms_undeclared_other_user false +dms_undeclared_other_user false +dms_undeclared_service_principal false +dms_undeclared_service_principal false +experimental.use_legacy_run_as false +experimental.use_legacy_run_as false +has_classic_interactive_compute false +has_classic_interactive_compute true +has_classic_job_compute false +has_classic_job_compute true +has_serverless_compute true +has_serverless_compute true +local.cache.attempt true +local.cache.attempt true +local.cache.hit true +local.cache.miss true +permissions_section_set false +permissions_section_set false +presets_name_prefix_is_set false +presets_name_prefix_is_set false +python_wheel_wrapper_is_set false +python_wheel_wrapper_is_set false +run_as_set false +run_as_set false +skip_artifact_cleanup false +skip_artifact_cleanup false +state_path_in_deployer_home true +state_path_in_deployer_home true +state_path_in_other_user_home false +state_path_in_other_user_home false +state_path_is_shared false +state_path_is_shared false +state_path_other false +state_path_other false diff --git a/acceptance/bundle/telemetry/deploy-compute-type/script b/acceptance/bundle/telemetry/deploy-compute-type/script index cfe8cba93be..6d2a0de0b05 100644 --- a/acceptance/bundle/telemetry/deploy-compute-type/script +++ b/acceptance/bundle/telemetry/deploy-compute-type/script @@ -1,6 +1,9 @@ trace $CLI bundle deploy -t one trace $CLI bundle deploy -t two -trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.experimental.bool_values' +# Common bool_values are engine-agnostic; the terraform-only direct_drymigrate_* +# entries go to a per-engine file so output.txt stays shared. +trace print_telemetry_bool_values | grep -v direct_drymigrate +print_telemetry_bool_values | grep direct_drymigrate > out.drymigrate.$DATABRICKS_BUNDLE_ENGINE.txt || true rm out.requests.txt diff --git a/acceptance/bundle/telemetry/deploy-experimental/out.drymigrate.direct.txt b/acceptance/bundle/telemetry/deploy-experimental/out.drymigrate.direct.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/telemetry/deploy-experimental/out.drymigrate.terraform.txt b/acceptance/bundle/telemetry/deploy-experimental/out.drymigrate.terraform.txt new file mode 100644 index 00000000000..d7fac6e7778 --- /dev/null +++ b/acceptance/bundle/telemetry/deploy-experimental/out.drymigrate.terraform.txt @@ -0,0 +1,2 @@ +direct_drymigrate_success true +direct_drymigrate_warnings false diff --git a/acceptance/bundle/telemetry/deploy-experimental/output.txt b/acceptance/bundle/telemetry/deploy-experimental/output.txt index df24e19a74d..e8c847cd617 100644 --- a/acceptance/bundle/telemetry/deploy-experimental/output.txt +++ b/acceptance/bundle/telemetry/deploy-experimental/output.txt @@ -9,88 +9,24 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> cat out.requests.txt -{ - "bool_values": [ - { - "key": "local.cache.attempt", - "value": true - }, - { - "key": "local.cache.miss", - "value": true - }, - { - "key": "experimental.use_legacy_run_as", - "value": true - }, - { - "key": "run_as_set", - "value": true - }, - { - "key": "presets_name_prefix_is_set", - "value": false - }, - { - "key": "python_wheel_wrapper_is_set", - "value": false - }, - { - "key": "skip_artifact_cleanup", - "value": false - }, - { - "key": "state_path_is_shared", - "value": false - }, - { - "key": "permissions_section_set", - "value": false - }, - { - "key": "state_path_in_deployer_home", - "value": true - }, - { - "key": "state_path_in_other_user_home", - "value": false - }, - { - "key": "state_path_other", - "value": false - }, - { - "key": "dms_undeclared_deploying_user", - "value": false - }, - { - "key": "dms_undeclared_other_user", - "value": false - }, - { - "key": "dms_undeclared_service_principal", - "value": false - }, - { - "key": "dms_undeclared_group", - "value": false - }, - { - "key": "dms_compat_auto", - "value": true - }, - { - "key": "has_serverless_compute", - "value": false - }, - { - "key": "has_classic_job_compute", - "value": false - }, - { - "key": "has_classic_interactive_compute", - "value": true - } - ] -} +>>> print_telemetry_bool_values +dms_compat_auto true +dms_undeclared_deploying_user false +dms_undeclared_group false +dms_undeclared_other_user false +dms_undeclared_service_principal false +experimental.use_legacy_run_as true +has_classic_interactive_compute true +has_classic_job_compute false +has_serverless_compute false +local.cache.attempt true +local.cache.miss true +permissions_section_set false +presets_name_prefix_is_set false +python_wheel_wrapper_is_set false +run_as_set true +skip_artifact_cleanup false +state_path_in_deployer_home true +state_path_in_other_user_home false +state_path_is_shared false +state_path_other false diff --git a/acceptance/bundle/telemetry/deploy-experimental/script b/acceptance/bundle/telemetry/deploy-experimental/script index 67a3ba6299e..1e32b6c98ea 100644 --- a/acceptance/bundle/telemetry/deploy-experimental/script +++ b/acceptance/bundle/telemetry/deploy-experimental/script @@ -1,5 +1,8 @@ trace $CLI bundle deploy -trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.experimental | {bool_values}' +# Common bool_values are engine-agnostic; the terraform-only direct_drymigrate_* +# entries go to a per-engine file so output.txt stays shared. +trace print_telemetry_bool_values | grep -v direct_drymigrate +print_telemetry_bool_values | grep direct_drymigrate > out.drymigrate.$DATABRICKS_BUNDLE_ENGINE.txt || true rm out.requests.txt diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.drymigrate.direct.txt b/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.drymigrate.direct.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.drymigrate.terraform.txt b/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.drymigrate.terraform.txt new file mode 100644 index 00000000000..d7fac6e7778 --- /dev/null +++ b/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.drymigrate.terraform.txt @@ -0,0 +1,2 @@ +direct_drymigrate_success true +direct_drymigrate_warnings false diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt b/acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt index f68b3f24975..4f5630089f2 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt +++ b/acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt @@ -5,88 +5,24 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> cat out.requests.txt -{ - "bool_values": [ - { - "key": "local.cache.attempt", - "value": true - }, - { - "key": "local.cache.miss", - "value": true - }, - { - "key": "experimental.use_legacy_run_as", - "value": false - }, - { - "key": "run_as_set", - "value": false - }, - { - "key": "presets_name_prefix_is_set", - "value": true - }, - { - "key": "python_wheel_wrapper_is_set", - "value": false - }, - { - "key": "skip_artifact_cleanup", - "value": false - }, - { - "key": "state_path_is_shared", - "value": false - }, - { - "key": "permissions_section_set", - "value": false - }, - { - "key": "state_path_in_deployer_home", - "value": true - }, - { - "key": "state_path_in_other_user_home", - "value": false - }, - { - "key": "state_path_other", - "value": false - }, - { - "key": "dms_undeclared_deploying_user", - "value": false - }, - { - "key": "dms_undeclared_other_user", - "value": false - }, - { - "key": "dms_undeclared_service_principal", - "value": false - }, - { - "key": "dms_undeclared_group", - "value": false - }, - { - "key": "dms_compat_auto", - "value": true - }, - { - "key": "has_serverless_compute", - "value": false - }, - { - "key": "has_classic_job_compute", - "value": false - }, - { - "key": "has_classic_interactive_compute", - "value": false - } - ] -} +>>> print_telemetry_bool_values +dms_compat_auto true +dms_undeclared_deploying_user false +dms_undeclared_group false +dms_undeclared_other_user false +dms_undeclared_service_principal false +experimental.use_legacy_run_as false +has_classic_interactive_compute false +has_classic_job_compute false +has_serverless_compute false +local.cache.attempt true +local.cache.miss true +permissions_section_set false +presets_name_prefix_is_set true +python_wheel_wrapper_is_set false +run_as_set false +skip_artifact_cleanup false +state_path_in_deployer_home true +state_path_in_other_user_home false +state_path_is_shared false +state_path_other false diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/custom/script b/acceptance/bundle/telemetry/deploy-name-prefix/custom/script index 67a3ba6299e..1e32b6c98ea 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/custom/script +++ b/acceptance/bundle/telemetry/deploy-name-prefix/custom/script @@ -1,5 +1,8 @@ trace $CLI bundle deploy -trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.experimental | {bool_values}' +# Common bool_values are engine-agnostic; the terraform-only direct_drymigrate_* +# entries go to a per-engine file so output.txt stays shared. +trace print_telemetry_bool_values | grep -v direct_drymigrate +print_telemetry_bool_values | grep direct_drymigrate > out.drymigrate.$DATABRICKS_BUNDLE_ENGINE.txt || true rm out.requests.txt diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.drymigrate.direct.txt b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.drymigrate.direct.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.drymigrate.terraform.txt b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.drymigrate.terraform.txt new file mode 100644 index 00000000000..d7fac6e7778 --- /dev/null +++ b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.drymigrate.terraform.txt @@ -0,0 +1,2 @@ +direct_drymigrate_success true +direct_drymigrate_warnings false diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt index da51756db48..3122ad1ee9a 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt +++ b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt @@ -5,88 +5,24 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> cat out.requests.txt -{ - "bool_values": [ - { - "key": "local.cache.attempt", - "value": true - }, - { - "key": "local.cache.miss", - "value": true - }, - { - "key": "experimental.use_legacy_run_as", - "value": false - }, - { - "key": "run_as_set", - "value": false - }, - { - "key": "presets_name_prefix_is_set", - "value": true - }, - { - "key": "python_wheel_wrapper_is_set", - "value": false - }, - { - "key": "skip_artifact_cleanup", - "value": false - }, - { - "key": "state_path_is_shared", - "value": false - }, - { - "key": "permissions_section_set", - "value": false - }, - { - "key": "state_path_in_deployer_home", - "value": true - }, - { - "key": "state_path_in_other_user_home", - "value": false - }, - { - "key": "state_path_other", - "value": false - }, - { - "key": "dms_undeclared_deploying_user", - "value": false - }, - { - "key": "dms_undeclared_other_user", - "value": false - }, - { - "key": "dms_undeclared_service_principal", - "value": false - }, - { - "key": "dms_undeclared_group", - "value": false - }, - { - "key": "dms_compat_auto", - "value": true - }, - { - "key": "has_serverless_compute", - "value": false - }, - { - "key": "has_classic_job_compute", - "value": false - }, - { - "key": "has_classic_interactive_compute", - "value": false - } - ] -} +>>> print_telemetry_bool_values +dms_compat_auto true +dms_undeclared_deploying_user false +dms_undeclared_group false +dms_undeclared_other_user false +dms_undeclared_service_principal false +experimental.use_legacy_run_as false +has_classic_interactive_compute false +has_classic_job_compute false +has_serverless_compute false +local.cache.attempt true +local.cache.miss true +permissions_section_set false +presets_name_prefix_is_set true +python_wheel_wrapper_is_set false +run_as_set false +skip_artifact_cleanup false +state_path_in_deployer_home true +state_path_in_other_user_home false +state_path_is_shared false +state_path_other false diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/script b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/script index 67a3ba6299e..1e32b6c98ea 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/script +++ b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/script @@ -1,5 +1,8 @@ trace $CLI bundle deploy -trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.experimental | {bool_values}' +# Common bool_values are engine-agnostic; the terraform-only direct_drymigrate_* +# entries go to a per-engine file so output.txt stays shared. +trace print_telemetry_bool_values | grep -v direct_drymigrate +print_telemetry_bool_values | grep direct_drymigrate > out.drymigrate.$DATABRICKS_BUNDLE_ENGINE.txt || true rm out.requests.txt diff --git a/acceptance/bundle/telemetry/deploy-whl-artifacts/out.drymigrate.direct.txt b/acceptance/bundle/telemetry/deploy-whl-artifacts/out.drymigrate.direct.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/telemetry/deploy-whl-artifacts/out.drymigrate.terraform.txt b/acceptance/bundle/telemetry/deploy-whl-artifacts/out.drymigrate.terraform.txt new file mode 100644 index 00000000000..d788ea96e0b --- /dev/null +++ b/acceptance/bundle/telemetry/deploy-whl-artifacts/out.drymigrate.terraform.txt @@ -0,0 +1,4 @@ +direct_drymigrate_success true +direct_drymigrate_success true +direct_drymigrate_warnings false +direct_drymigrate_warnings false diff --git a/acceptance/bundle/telemetry/deploy-whl-artifacts/output.txt b/acceptance/bundle/telemetry/deploy-whl-artifacts/output.txt index 7b8678cf5b7..95776306d7e 100644 --- a/acceptance/bundle/telemetry/deploy-whl-artifacts/output.txt +++ b/acceptance/bundle/telemetry/deploy-whl-artifacts/output.txt @@ -13,168 +13,43 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/two/fi Deploying resources... Deployment complete! ->>> cat out.requests.txt -{ - "bool_values": [ - { - "key": "local.cache.attempt", - "value": true - }, - { - "key": "local.cache.miss", - "value": true - }, - { - "key": "artifact_build_command_is_set", - "value": false - }, - { - "key": "artifact_files_is_set", - "value": false - }, - { - "key": "python_wheel_wrapper_is_set", - "value": false - }, - { - "key": "skip_artifact_cleanup", - "value": false - }, - { - "key": "state_path_is_shared", - "value": false - }, - { - "key": "permissions_section_set", - "value": false - }, - { - "key": "state_path_in_deployer_home", - "value": true - }, - { - "key": "state_path_in_other_user_home", - "value": false - }, - { - "key": "state_path_other", - "value": false - }, - { - "key": "dms_undeclared_deploying_user", - "value": false - }, - { - "key": "dms_undeclared_other_user", - "value": false - }, - { - "key": "dms_undeclared_service_principal", - "value": false - }, - { - "key": "dms_undeclared_group", - "value": false - }, - { - "key": "dms_compat_auto", - "value": true - }, - { - "key": "has_serverless_compute", - "value": false - }, - { - "key": "has_classic_job_compute", - "value": false - }, - { - "key": "has_classic_interactive_compute", - "value": false - } - ] -} -{ - "bool_values": [ - { - "key": "local.cache.attempt", - "value": true - }, - { - "key": "local.cache.hit", - "value": true - }, - { - "key": "artifact_build_command_is_set", - "value": true - }, - { - "key": "artifact_files_is_set", - "value": true - }, - { - "key": "artifact_dynamic_version_is_set", - "value": true - }, - { - "key": "python_wheel_wrapper_is_set", - "value": true - }, - { - "key": "skip_artifact_cleanup", - "value": true - }, - { - "key": "state_path_is_shared", - "value": false - }, - { - "key": "permissions_section_set", - "value": false - }, - { - "key": "state_path_in_deployer_home", - "value": true - }, - { - "key": "state_path_in_other_user_home", - "value": false - }, - { - "key": "state_path_other", - "value": false - }, - { - "key": "dms_undeclared_deploying_user", - "value": false - }, - { - "key": "dms_undeclared_other_user", - "value": false - }, - { - "key": "dms_undeclared_service_principal", - "value": false - }, - { - "key": "dms_undeclared_group", - "value": false - }, - { - "key": "dms_compat_auto", - "value": true - }, - { - "key": "has_serverless_compute", - "value": false - }, - { - "key": "has_classic_job_compute", - "value": false - }, - { - "key": "has_classic_interactive_compute", - "value": false - } - ] -} +>>> print_telemetry_bool_values +artifact_build_command_is_set false +artifact_build_command_is_set true +artifact_dynamic_version_is_set true +artifact_files_is_set false +artifact_files_is_set true +dms_compat_auto true +dms_compat_auto true +dms_undeclared_deploying_user false +dms_undeclared_deploying_user false +dms_undeclared_group false +dms_undeclared_group false +dms_undeclared_other_user false +dms_undeclared_other_user false +dms_undeclared_service_principal false +dms_undeclared_service_principal false +has_classic_interactive_compute false +has_classic_interactive_compute false +has_classic_job_compute false +has_classic_job_compute false +has_serverless_compute false +has_serverless_compute false +local.cache.attempt true +local.cache.attempt true +local.cache.hit true +local.cache.miss true +permissions_section_set false +permissions_section_set false +python_wheel_wrapper_is_set false +python_wheel_wrapper_is_set true +skip_artifact_cleanup false +skip_artifact_cleanup true +state_path_in_deployer_home true +state_path_in_deployer_home true +state_path_in_other_user_home false +state_path_in_other_user_home false +state_path_is_shared false +state_path_is_shared false +state_path_other false +state_path_other false diff --git a/acceptance/bundle/telemetry/deploy-whl-artifacts/script b/acceptance/bundle/telemetry/deploy-whl-artifacts/script index 078fa94cdd3..b1ee8772a64 100644 --- a/acceptance/bundle/telemetry/deploy-whl-artifacts/script +++ b/acceptance/bundle/telemetry/deploy-whl-artifacts/script @@ -6,6 +6,9 @@ trace $CLI bundle deploy -t one trace $CLI bundle deploy -t two -trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.experimental | {bool_values}' +# Common bool_values are engine-agnostic; the terraform-only direct_drymigrate_* +# entries go to a per-engine file so output.txt stays shared. +trace print_telemetry_bool_values | grep -v direct_drymigrate +print_telemetry_bool_values | grep direct_drymigrate > out.drymigrate.$DATABRICKS_BUNDLE_ENGINE.txt || true rm out.requests.txt diff --git a/acceptance/bundle/telemetry/deploy/out.migration.direct.txt b/acceptance/bundle/telemetry/deploy/out.migration.direct.txt new file mode 100644 index 00000000000..fe51488c706 --- /dev/null +++ b/acceptance/bundle/telemetry/deploy/out.migration.direct.txt @@ -0,0 +1 @@ +[] diff --git a/acceptance/bundle/telemetry/deploy/out.migration.terraform.txt b/acceptance/bundle/telemetry/deploy/out.migration.terraform.txt new file mode 100644 index 00000000000..c503d0fb40b --- /dev/null +++ b/acceptance/bundle/telemetry/deploy/out.migration.terraform.txt @@ -0,0 +1,10 @@ +[ + { + "key": "direct_drymigrate_success", + "value": true + }, + { + "key": "direct_drymigrate_warnings", + "value": false + } +] diff --git a/acceptance/bundle/telemetry/deploy/output.txt b/acceptance/bundle/telemetry/deploy/output.txt index cf5dd1e434e..98e1d643081 100644 --- a/acceptance/bundle/telemetry/deploy/output.txt +++ b/acceptance/bundle/telemetry/deploy/output.txt @@ -10,3 +10,5 @@ Deployment complete! === Assert that mutator execution times are being recorded >>> cat telemetry.json true + +=== Assert the dry-run migration to the direct engine is reported \ No newline at end of file diff --git a/acceptance/bundle/telemetry/deploy/script b/acceptance/bundle/telemetry/deploy/script index 83705b0ea41..feb65287206 100644 --- a/acceptance/bundle/telemetry/deploy/script +++ b/acceptance/bundle/telemetry/deploy/script @@ -18,12 +18,19 @@ trace cat telemetry.json | jq ' .entry.databricks_cli_log.bundle_deploy_event.ex # 0.0.0-dev on windows) — it is still emitted in real telemetry. cat telemetry.json | jq '.entry.databricks_cli_log.bundle_deploy_event.resources_metadata | if . then del(.state_file_size_bytes) else . end' > out.resources_metadata.$DATABRICKS_BUNDLE_ENGINE.txt +# The dry-run migration to the direct engine runs only after a terraform deploy, +# so its bool_values entries (direct_drymigrate_*) diverge across the +# DATABRICKS_BUNDLE_ENGINE matrix (absent for direct). Capture them per-engine and +# drop them from the engine-agnostic out.telemetry.txt below. +title "Assert the dry-run migration to the direct engine is reported" +cat telemetry.json | jq '[.entry.databricks_cli_log.bundle_deploy_event.experimental.bool_values[] | select(.key | startswith("direct_drymigrate_"))]' > out.migration.$DATABRICKS_BUNDLE_ENGINE.txt + # bundle_mutator_execution_time_ms can have variable number of entries depending upon the runtime of the mutators. Thus we omit it from # being asserted here. # # upload_file_count and upload_file_sizes are asserted here and kept deterministic by # the sync.paths restriction in databricks.yml (see the comment there). -cat telemetry.json | jq 'del(.entry.databricks_cli_log.bundle_deploy_event.experimental.bundle_mutator_execution_time_ms, .entry.databricks_cli_log.bundle_deploy_event.resources_metadata)' > out.telemetry.txt +cat telemetry.json | jq 'del(.entry.databricks_cli_log.bundle_deploy_event.experimental.bundle_mutator_execution_time_ms, .entry.databricks_cli_log.bundle_deploy_event.resources_metadata) | .entry.databricks_cli_log.bundle_deploy_event.experimental.bool_values |= map(select(.key | startswith("direct_drymigrate_") | not))' > out.telemetry.txt cmd_exec_id=$(extract_command_exec_id.py) deployment_id=$(cat .databricks/bundle/default/deployment.json | jq -r .id) diff --git a/bundle/metrics/metrics.go b/bundle/metrics/metrics.go index f5238e85202..618be8c5e4a 100644 --- a/bundle/metrics/metrics.go +++ b/bundle/metrics/metrics.go @@ -11,6 +11,14 @@ const ( SqlWarehouseLifecycleStarted = "sql_warehouse_lifecycle_started" SelectUsed = "select_used" + // Outcome of the dry-run migration to the direct engine attempted after a + // successful terraform deploy. DirectDryMigrateSuccess is false when the state + // could not be converted; DirectDryMigrateWarnings is true when the conversion + // emitted warnings (e.g. resources the direct engine can't represent). + // Only recorded on terraform deploys. + DirectDryMigrateSuccess = "direct_drymigrate_success" + DirectDryMigrateWarnings = "direct_drymigrate_warnings" + // Whether workspace.state_path is under /Workspace/Shared. StatePathIsShared = "state_path_is_shared" diff --git a/bundle/migrate/build_state.go b/bundle/migrate/build_state.go index 06638a84458..c27602802cb 100644 --- a/bundle/migrate/build_state.go +++ b/bundle/migrate/build_state.go @@ -31,7 +31,9 @@ func BuildStateFromTF( stateDB *dstate.DeploymentState, tfAttrs TFStateAttrs, tfIDs map[string]string, -) error { + warnPrefix string, +) (bool, error) { + warningsSeen := false // Collect all resource nodes (same patterns as makePlan). var nodes []string patterns := []dyn.Pattern{ @@ -49,7 +51,7 @@ func BuildStateFromTF( }, ) if err != nil { - return err + return warningsSeen, err } } @@ -63,18 +65,19 @@ func BuildStateFromTF( group := config.GetResourceTypeFromKey(node) if group == "" { - return fmt.Errorf("cannot determine resource type for %q", node) + return warningsSeen, fmt.Errorf("cannot determine resource type for %q", node) } adapter, ok := adapters[group] if !ok { - log.Warnf(ctx, "unsupported resource type %q for %s, skipping", group, node) + warningsSeen = true + log.Warnf(ctx, warnPrefix+"unsupported resource type %q for %s, skipping", group, node) continue } inputConfig, err := configRoot.GetResourceConfig(node) if err != nil { - return fmt.Errorf("%s: getting config: %w", node, err) + return warningsSeen, fmt.Errorf("%s: getting config: %w", node, err) } baseRefs := map[string]string{} @@ -85,16 +88,16 @@ func BuildStateFromTF( if strings.HasPrefix(node, "resources.secret_scopes.") { typedConfig, ok := inputConfig.(*[]resources.SecretScopePermission) if !ok { - return fmt.Errorf("%s: expected *[]resources.SecretScopePermission, got %T", node, inputConfig) + return warningsSeen, fmt.Errorf("%s: expected *[]resources.SecretScopePermission, got %T", node, inputConfig) } sv, err = dresources.PrepareSecretScopeAclsInputConfig(*typedConfig, node) if err != nil { - return fmt.Errorf("%s: preparing secret scope ACLs config: %w", node, err) + return warningsSeen, fmt.Errorf("%s: preparing secret scope ACLs config: %w", node, err) } } else { sv, err = dresources.PreparePermissionsInputConfig(inputConfig, node) if err != nil { - return fmt.Errorf("%s: preparing permissions config: %w", node, err) + return warningsSeen, fmt.Errorf("%s: preparing permissions config: %w", node, err) } } inputConfig = sv.Value @@ -103,7 +106,7 @@ func BuildStateFromTF( case strings.HasSuffix(node, ".grants"): sv, err := dresources.PrepareGrantsInputConfig(inputConfig, node) if err != nil { - return fmt.Errorf("%s: preparing grants config: %w", node, err) + return warningsSeen, fmt.Errorf("%s: preparing grants config: %w", node, err) } inputConfig = sv.Value baseRefs = sv.Refs @@ -111,12 +114,12 @@ func BuildStateFromTF( newStateValue, err := adapter.PrepareState(inputConfig) if err != nil { - return fmt.Errorf("%s: PrepareState: %w", node, err) + return warningsSeen, fmt.Errorf("%s: PrepareState: %w", node, err) } refs, err := direct.ExtractReferences(configRoot.Value(), node) if err != nil { - return fmt.Errorf("%s: extracting references: %w", node, err) + return warningsSeen, fmt.Errorf("%s: extracting references: %w", node, err) } maps.Copy(refs, baseRefs) @@ -167,7 +170,7 @@ func BuildStateFromTF( // is absent there (model_serving_endpoints, database_instances). if _, ok := sv.Refs["object_id"]; ok { if err := structaccess.Set(sv.Value, structpath.NewStringKey(nil, "object_id"), id); err != nil { - return fmt.Errorf("%s: setting object_id: %w", node, err) + return warningsSeen, fmt.Errorf("%s: setting object_id: %w", node, err) } delete(sv.Refs, "object_id") } @@ -194,25 +197,28 @@ func BuildStateFromTF( for _, pending := range pendingRefs { fieldPath, err := structpath.ParsePath(pending.fieldPathStr) if err != nil { - return fmt.Errorf("%s: parsing field path %q: %w", node, pending.fieldPathStr, err) + return warningsSeen, fmt.Errorf("%s: parsing field path %q: %w", node, pending.fieldPathStr, err) } // ResolveFieldRef returns the fully resolved value for this field, // using either Method A (TF state lookup) or Method B (template evaluation). - value, err := ResolveFieldRef(ctx, tfAttrs, srcGroup, srcName, fieldPath, pending.refTemplate) + value, warned, err := ResolveFieldRef(ctx, tfAttrs, srcGroup, srcName, fieldPath, pending.refTemplate, warnPrefix) if err != nil { - return fmt.Errorf("%s: cannot resolve field %q (template %q): %w", node, pending.fieldPathStr, pending.refTemplate, err) + return warningsSeen, fmt.Errorf("%s: cannot resolve field %q (template %q): %w", node, pending.fieldPathStr, pending.refTemplate, err) + } + if warned { + warningsSeen = true } // Set the resolved value directly and remove the ref entry. if err := structaccess.Set(sv.Value, fieldPath, value); err != nil { - return fmt.Errorf("%s: cannot set resolved value for field %q: %w", node, pending.fieldPathStr, err) + return warningsSeen, fmt.Errorf("%s: cannot set resolved value for field %q: %w", node, pending.fieldPathStr, err) } delete(sv.Refs, pending.fieldPathStr) } if len(sv.Refs) > 0 { - return fmt.Errorf("%s: unresolved references: %v", node, sv.Refs) + return warningsSeen, fmt.Errorf("%s: unresolved references: %v", node, sv.Refs) } // Handle etag for dashboards: read it directly from TF state attributes. @@ -222,15 +228,15 @@ func BuildStateFromTF( if v, err := LookupTFField(tfAttrs, group, srcName, structpath.NewStringKey(nil, "etag")); err == nil { if etag, ok := v.(string); ok && etag != "" { if err := structaccess.Set(sv.Value, structpath.NewStringKey(nil, "etag"), etag); err != nil { - return fmt.Errorf("%s: cannot set etag: %w", node, err) + return warningsSeen, fmt.Errorf("%s: cannot set etag: %w", node, err) } } } if err := stateDB.SaveState(node, id, sv.Value, dependsOn); err != nil { - return fmt.Errorf("%s: SaveState: %w", node, err) + return warningsSeen, fmt.Errorf("%s: SaveState: %w", node, err) } } - return nil + return warningsSeen, nil } diff --git a/bundle/migrate/build_state_test.go b/bundle/migrate/build_state_test.go index 338954a330e..a495436008c 100644 --- a/bundle/migrate/build_state_test.go +++ b/bundle/migrate/build_state_test.go @@ -50,7 +50,7 @@ func runBuildStateFromTF( db.OpenWithData(statePath, dstate.NewDatabase("lineage", 1)) require.NoError(t, db.UpgradeToWrite()) - err = migrate.BuildStateFromTF(t.Context(), &root, adapters, &db, tfAttrs, tfIDs) + _, err = migrate.BuildStateFromTF(t.Context(), &root, adapters, &db, tfAttrs, tfIDs, "") require.NoError(t, err) _, err = db.Finalize(t.Context()) diff --git a/bundle/migrate/resolve.go b/bundle/migrate/resolve.go index 8d5f6bc6b35..23d0180e3c6 100644 --- a/bundle/migrate/resolve.go +++ b/bundle/migrate/resolve.go @@ -57,8 +57,11 @@ func evaluateTemplate(state TFStateAttrs, template string) (string, error) { // - Method A: read the field from the source resource's own TF state. // - Method B: evaluate the template by reading each referenced field from TF state. // -// Returns the reconciled value or an error if both methods fail. -func ResolveFieldRef(ctx context.Context, state TFStateAttrs, srcGroup, srcName string, fieldPath *structpath.PathNode, refTemplate string) (any, error) { +// Returns the reconciled value or an error if both methods fail. The bool return +// reports whether a warning was logged (methods disagreed); warnPrefix is +// prepended to that warning so background callers (the post-deploy dry-run) can +// attribute it. +func ResolveFieldRef(ctx context.Context, state TFStateAttrs, srcGroup, srcName string, fieldPath *structpath.PathNode, refTemplate, warnPrefix string) (any, bool, error) { // Method A: read field from source resource's TF state. valueA, errA := LookupTFField(state, srcGroup, srcName, fieldPath) @@ -69,23 +72,23 @@ func ResolveFieldRef(ctx context.Context, state TFStateAttrs, srcGroup, srcName case errA == nil && errB == nil: aStr := fmt.Sprintf("%v", valueA) if aStr == valueB { - return valueA, nil + return valueA, false, nil } // Both succeeded but disagree: prefer longer string and warn. if len(valueB) > len(aStr) { - log.Warnf(ctx, "resource %s.%s field %s: method A value %q and method B value %q disagree; using longer (method B)", + log.Warnf(ctx, warnPrefix+"resource %s.%s field %s: method A value %q and method B value %q disagree; using longer (method B)", srcGroup, srcName, fieldPath, aStr, valueB) - return valueB, nil + return valueB, true, nil } - log.Warnf(ctx, "resource %s.%s field %s: method A value %q and method B value %q disagree; using longer (method A)", + log.Warnf(ctx, warnPrefix+"resource %s.%s field %s: method A value %q and method B value %q disagree; using longer (method A)", srcGroup, srcName, fieldPath, aStr, valueB) - return valueA, nil + return valueA, true, nil case errA == nil: - return valueA, nil + return valueA, false, nil case errB == nil: - return valueB, nil + return valueB, false, nil default: - return nil, fmt.Errorf("%s.%s field %s: method A: %w; method B: %w", + return nil, false, fmt.Errorf("%s.%s field %s: method A: %w; method B: %w", srcGroup, srcName, fieldPath, errA, errB) } } diff --git a/bundle/migrate/resolve_test.go b/bundle/migrate/resolve_test.go index 221c873a73d..1f6ee0d7533 100644 --- a/bundle/migrate/resolve_test.go +++ b/bundle/migrate/resolve_test.go @@ -37,12 +37,11 @@ func TestResolveFieldRefInt(t *testing.T) { // Remove dst from state so Method A fails and Method B must be used. delete(state["databricks_job"], "dst") - ctx := t.Context() fieldPath, err := structpath.ParsePath("max_concurrent_runs") require.NoError(t, err) - value, err := migrate.ResolveFieldRef(ctx, state, "jobs", "dst", fieldPath, - "${resources.jobs.src.max_concurrent_runs}") + value, _, err := migrate.ResolveFieldRef(t.Context(), state, "jobs", "dst", fieldPath, + "${resources.jobs.src.max_concurrent_runs}", "") require.NoError(t, err) // Method B succeeds: returns string "4". Verify Set converts it to int. @@ -60,12 +59,11 @@ func TestResolveFieldRefBool(t *testing.T) { state := testState() delete(state["databricks_job"], "dst") - ctx := t.Context() fieldPath, err := structpath.ParsePath("always_running") require.NoError(t, err) - value, err := migrate.ResolveFieldRef(ctx, state, "jobs", "dst", fieldPath, - "${resources.jobs.src.always_running}") + value, _, err := migrate.ResolveFieldRef(t.Context(), state, "jobs", "dst", fieldPath, + "${resources.jobs.src.always_running}", "") require.NoError(t, err) type target struct { diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 2edbfd1bf1c..b9c595a9702 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -114,6 +114,13 @@ func deployCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, ta if !logdiag.HasError(ctx) { cmdio.LogString(ctx, "Deployment complete!") } + + // Once the deploy is complete, dry-run the migration to the direct engine in + // memory and record the outcome in telemetry. It writes nothing and never + // fails the deploy. + if !targetEngine.IsDirect() && !logdiag.HasError(ctx) { + statemgmt.CheckDirectMigration(ctx, b) + } } // uploadLibraries uploads libraries to the workspace. diff --git a/bundle/statemgmt/check_direct_migration.go b/bundle/statemgmt/check_direct_migration.go new file mode 100644 index 00000000000..789213cf5c5 --- /dev/null +++ b/bundle/statemgmt/check_direct_migration.go @@ -0,0 +1,149 @@ +package statemgmt + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/engine" + "github.com/databricks/cli/bundle/config/mutator/resourcemutator" + "github.com/databricks/cli/bundle/direct/dresources" + "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/cli/bundle/metrics" + "github.com/databricks/cli/bundle/migrate" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/logdiag" +) + +// warnPrefix labels warnings emitted by the post-deploy dry-run so they are not +// confused with warnings from the user-invoked `bundle migrate` command. +const warnPrefix = "post-deploy dry-run migration to direct: " + +// feedbackNotice is appended after the dry-run warnings to reassure the user the +// deploy is unaffected and to ask them to report the warnings. +const feedbackNotice = `The warnings above are from a dry-run migration to the direct deployment engine (https://docs.databricks.com/aws/en/dev-tools/bundles/direct). +Your deployment is not affected and works normally, but you may experience these issues when migrating to the direct deployment engine. +Please forward these warnings to dabs-feedback@databricks.com` + +// CheckDirectMigration performs a dry-run migration of the just-deployed terraform +// state to the direct engine and records the outcome in deploy telemetry. +// +// The converted state is written to a temporary file that is deleted before +// returning, so nothing is persisted locally or uploaded to the workspace; its +// only purpose is to measure, across the fleet, how many terraform deploys could +// migrate to the direct engine cleanly. Any error is surfaced to the user as a +// warning so it never fails a deploy that already succeeded. +func CheckDirectMigration(ctx context.Context, b *bundle.Bundle) { + hasWarnings, err := dryRunMigrate(ctx, b) + b.Metrics.SetBoolValue(metrics.DirectDryMigrateSuccess, err == nil) + b.Metrics.SetBoolValue(metrics.DirectDryMigrateWarnings, hasWarnings) + if err != nil { + log.Warnf(ctx, "%s%v", warnPrefix, err) + } + if hasWarnings || err != nil { + log.Warnf(ctx, "%s", feedbackNotice) + } +} + +// dryRunMigrate converts the local terraform state to the direct engine state, +// returning whether any warnings were emitted. It mirrors the `bundle migrate` +// command but writes the result to a throwaway temp file that is deleted before +// returning, and never uploads anything. +func dryRunMigrate(ctx context.Context, b *bundle.Bundle) (bool, error) { + _, localTerraformPath := b.StateFilenameTerraform(ctx) + tfState, err := migrate.ParseTFStateFull(ctx, localTerraformPath) + if err != nil { + return false, fmt.Errorf("failed to parse terraform state: %w", err) + } + + // ParseTFStateFull returns nil when the terraform state file doesn't exist + // (e.g. first deploy with no resources); nothing to migrate, trivially OK. + if tfState == nil { + return false, nil + } + + // The converted state is a throwaway: write it to a temp dir that is removed + // (along with the WAL the state DB creates) before returning, so the dry run + // leaves nothing behind on disk. + tempDir, err := os.MkdirTemp("", "databricks-direct-migration-") + if err != nil { + return false, fmt.Errorf("failed to create temp dir: %w", err) + } + defer os.RemoveAll(tempDir) + tempStatePath := filepath.Join(tempDir, "resources.json") + + // SecretScopeFixups and the direct-engine state builder report failures via + // logdiag. Run them in an isolated context so a dry-run failure never affects + // the deploy's own diagnostics or exit code. + ctx = logdiag.IsolatedContext(ctx) + + state := make(map[string]dstate.ResourceEntry) + for key, id := range tfState.IDs { + state[key] = dstate.ResourceEntry{ + ID: id, + State: json.RawMessage("{}"), + } + } + + migratedDB := dstate.NewDatabase(tfState.Lineage, tfState.Serial+1) + migratedDB.State = state + + var stateDB dstate.DeploymentState + stateDB.OpenWithData(tempStatePath, migratedDB) + + // Apply SecretScopeFixups so the config matches what the direct engine expects. + // This adds MANAGE ACL for the current user to all secret scopes, ensuring + // the migrated state and config agree on .permissions entries. + bundle.ApplyContext(ctx, b, resourcemutator.SecretScopeFixups(engine.EngineDirect)) + if logdiag.HasError(ctx) { + return false, errors.New("failed to apply secret scope fixups") + } + + // b.Config has been modified by terraform.Interpolate which converts bundle-style + // references (${resources.pipelines.x.id}) to terraform-style (${databricks_pipeline.x.id}). + // BuildStateFromTF expects ${resources.*} references, so reverse the interpolation first. + uninterpolatedRoot, err := reverseInterpolate(b.Config.Value()) + if err != nil { + return false, fmt.Errorf("failed to reverse interpolation: %w", err) + } + + var uninterpolatedConfig config.Root + err = uninterpolatedConfig.Mutate(func(_ dyn.Value) (dyn.Value, error) { + return uninterpolatedRoot, nil + }) + if err != nil { + return false, fmt.Errorf("failed to create uninterpolated config: %w", err) + } + + adapters, err := dresources.InitAll(nil) + if err != nil { + return false, err + } + + if err := stateDB.UpgradeToWrite(); err != nil { + return false, fmt.Errorf("upgrading state for apply: %w", err) + } + + // warnPrefix labels the conversion's warnings as coming from the background dry run. + hasWarnings, err := migrate.BuildStateFromTF(ctx, &uninterpolatedConfig, adapters, &stateDB, tfState.Attrs, tfState.IDs, warnPrefix) + if err != nil { + return hasWarnings, err + } + + if _, err := stateDB.Finalize(ctx); err != nil { + return hasWarnings, err + } + + // BuildStateFromTF reports some failures via logdiag instead of returning an error. + if logdiag.HasError(ctx) { + return hasWarnings, errors.New("state conversion failed") + } + + return hasWarnings, nil +} diff --git a/bundle/statemgmt/upload_state_for_yaml_sync.go b/bundle/statemgmt/upload_state_for_yaml_sync.go index 163c9fb4fdb..80d4a3dbd12 100644 --- a/bundle/statemgmt/upload_state_for_yaml_sync.go +++ b/bundle/statemgmt/upload_state_for_yaml_sync.go @@ -172,7 +172,7 @@ func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bun return false, fmt.Errorf("upgrading state for apply: %w", err) } - if err := migrate.BuildStateFromTF(ctx, &uninterpolatedConfig, adapters, &stateDB, tfState.Attrs, tfState.IDs); err != nil { + if _, err := migrate.BuildStateFromTF(ctx, &uninterpolatedConfig, adapters, &stateDB, tfState.Attrs, tfState.IDs, ""); err != nil { return false, err } diff --git a/cmd/bundle/deployment/migrate.go b/cmd/bundle/deployment/migrate.go index 39d9a0454d5..93e229f71e8 100644 --- a/cmd/bundle/deployment/migrate.go +++ b/cmd/bundle/deployment/migrate.go @@ -172,7 +172,7 @@ To start using direct engine, set "engine: direct" under bundle in your databric return fmt.Errorf("upgrading state for apply: %w", err) } - if err := migrate.BuildStateFromTF(ctx, &b.Config, adapters, &stateDB, tfState.Attrs, tfState.IDs); err != nil { + if _, err := migrate.BuildStateFromTF(ctx, &b.Config, adapters, &stateDB, tfState.Attrs, tfState.IDs, ""); err != nil { return err }