From 2dcf99c5cf5071947794e08cc5eff3a8875a5d97 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Fri, 5 Jun 2026 16:06:57 +0200 Subject: [PATCH 1/3] Reproduce #1976 --- .../files/out-of-band-delete/databricks.yml | 15 ++++++ .../files/out-of-band-delete/hello_world.py | 1 + .../files/out-of-band-delete/out.test.toml | 3 ++ .../files/out-of-band-delete/output.txt | 47 +++++++++++++++++++ .../deploy/files/out-of-band-delete/script | 31 ++++++++++++ 5 files changed, 97 insertions(+) create mode 100644 acceptance/bundle/deploy/files/out-of-band-delete/databricks.yml create mode 100644 acceptance/bundle/deploy/files/out-of-band-delete/hello_world.py create mode 100644 acceptance/bundle/deploy/files/out-of-band-delete/out.test.toml create mode 100644 acceptance/bundle/deploy/files/out-of-band-delete/output.txt create mode 100644 acceptance/bundle/deploy/files/out-of-band-delete/script diff --git a/acceptance/bundle/deploy/files/out-of-band-delete/databricks.yml b/acceptance/bundle/deploy/files/out-of-band-delete/databricks.yml new file mode 100644 index 00000000000..a4c66846c58 --- /dev/null +++ b/acceptance/bundle/deploy/files/out-of-band-delete/databricks.yml @@ -0,0 +1,15 @@ +bundle: + name: out-of-band-delete + +resources: + jobs: + foo: + name: test-job + tasks: + - task_key: my_task + spark_python_task: + python_file: ./hello_world.py + new_cluster: + num_workers: 1 + spark_version: 13.3.x-snapshot-scala2.12 + node_type_id: i3.xlarge diff --git a/acceptance/bundle/deploy/files/out-of-band-delete/hello_world.py b/acceptance/bundle/deploy/files/out-of-band-delete/hello_world.py new file mode 100644 index 00000000000..f301245e242 --- /dev/null +++ b/acceptance/bundle/deploy/files/out-of-band-delete/hello_world.py @@ -0,0 +1 @@ +print("Hello World!") diff --git a/acceptance/bundle/deploy/files/out-of-band-delete/out.test.toml b/acceptance/bundle/deploy/files/out-of-band-delete/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/deploy/files/out-of-band-delete/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deploy/files/out-of-band-delete/output.txt b/acceptance/bundle/deploy/files/out-of-band-delete/output.txt new file mode 100644 index 00000000000..c46bbfb2f89 --- /dev/null +++ b/acceptance/bundle/deploy/files/out-of-band-delete/output.txt @@ -0,0 +1,47 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/out-of-band-delete/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== After the initial deploy the job's python_file and the deployment state exist +>>> [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/out-of-band-delete/default/files/hello_world.py +{ + "object_type": "FILE", + "path": "/Workspace/Users/[USERNAME]/.bundle/out-of-band-delete/default/files/hello_world.py" +} + +>>> [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/out-of-band-delete/default/state/deployment.json +{ + "object_type": "FILE", + "path": "/Workspace/Users/[USERNAME]/.bundle/out-of-band-delete/default/state/deployment.json" +} + +=== Delete the remote bundle directory out-of-band (simulates deleting it in the UI) +>>> [CLI] workspace delete --recursive /Workspace/Users/[USERNAME]/.bundle/out-of-band-delete + +=== Redeploy without changing any local file, so the stale sync snapshot is reused +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/out-of-band-delete/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== After redeploy: the files dir and state are recreated, but the python_file is NOT re-uploaded +>>> errcode [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/out-of-band-delete/default/files/hello_world.py +Error: Workspace path not found + +Exit code: 1 + +>>> [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/out-of-band-delete/default/files +{ + "object_type": "DIRECTORY", + "path": "/Workspace/Users/[USERNAME]/.bundle/out-of-band-delete/default/files" +} + +>>> [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/out-of-band-delete/default/state/deployment.json +{ + "object_type": "FILE", + "path": "/Workspace/Users/[USERNAME]/.bundle/out-of-band-delete/default/state/deployment.json" +} diff --git a/acceptance/bundle/deploy/files/out-of-band-delete/script b/acceptance/bundle/deploy/files/out-of-band-delete/script new file mode 100644 index 00000000000..46b95a26528 --- /dev/null +++ b/acceptance/bundle/deploy/files/out-of-band-delete/script @@ -0,0 +1,31 @@ +# Redeploy after the remote bundle directory is deleted out-of-band (e.g. a user +# deletes it in the workspace UI). +# +# After such a deletion the next "bundle deploy" does not re-upload the bundle files: +# the local sync snapshot still records them as present remotely, and the remote +# deployment state that would otherwise reset the snapshot was deleted too, so the +# file sync computes an empty diff and uploads nothing. +# +# This test pins down the current behaviour: hello_world.py is missing from the +# workspace after the redeploy even though the deploy reports success. If the file +# sync is changed to re-upload in this case, the final get-status will succeed and +# this output is expected to change. + +BUNDLE_PATH="/Workspace/Users/${CURRENT_USER_NAME}/.bundle/out-of-band-delete/default" + +trace $CLI bundle deploy + +title "After the initial deploy the job's python_file and the deployment state exist" +trace $CLI workspace get-status "$BUNDLE_PATH/files/hello_world.py" | jq '{object_type,path}' +trace $CLI workspace get-status "$BUNDLE_PATH/state/deployment.json" | jq '{object_type,path}' + +title "Delete the remote bundle directory out-of-band (simulates deleting it in the UI)" +trace $CLI workspace delete --recursive "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/out-of-band-delete" + +title "Redeploy without changing any local file, so the stale sync snapshot is reused" +trace $CLI bundle deploy + +title "After redeploy: the files dir and state are recreated, but the python_file is NOT re-uploaded" +trace errcode $CLI workspace get-status "$BUNDLE_PATH/files/hello_world.py" +trace $CLI workspace get-status "$BUNDLE_PATH/files" | jq '{object_type,path}' +trace $CLI workspace get-status "$BUNDLE_PATH/state/deployment.json" | jq '{object_type,path}' From 36c791cd1657fca9dc39a585268ddd2f5043dfa4 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Mon, 8 Jun 2026 09:49:42 +0200 Subject: [PATCH 2/3] Add MSYS_NO_PATHCONV to test.toml --- .../bundle/deploy/files/out-of-band-delete/test.toml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 acceptance/bundle/deploy/files/out-of-band-delete/test.toml diff --git a/acceptance/bundle/deploy/files/out-of-band-delete/test.toml b/acceptance/bundle/deploy/files/out-of-band-delete/test.toml new file mode 100644 index 00000000000..e39491d91a4 --- /dev/null +++ b/acceptance/bundle/deploy/files/out-of-band-delete/test.toml @@ -0,0 +1,7 @@ +[Env] +# This test passes absolute workspace paths like /Workspace/Users/.../hello_world.py +# as CLI arguments. On Windows the script runs under MSYS2, which rewrites such +# leading-slash arguments to Windows paths (e.g. C:/Program Files/Git/Workspace/...) +# before the CLI sees them, so workspace get-status would query the wrong path. +# Setting this environment variable disables that conversion. +MSYS_NO_PATHCONV = "1" From 616e2e3e074baedf1b223863f314d6fe6fd71154 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Mon, 8 Jun 2026 10:01:47 +0200 Subject: [PATCH 3/3] Address denik's comments --- .../files/out-of-band-delete/output.txt | 30 ++++++++----------- .../deploy/files/out-of-band-delete/script | 28 ++++++----------- .../deploy/files/out-of-band-delete/test.toml | 2 ++ 3 files changed, 23 insertions(+), 37 deletions(-) diff --git a/acceptance/bundle/deploy/files/out-of-band-delete/output.txt b/acceptance/bundle/deploy/files/out-of-band-delete/output.txt index c46bbfb2f89..8ac61e622b7 100644 --- a/acceptance/bundle/deploy/files/out-of-band-delete/output.txt +++ b/acceptance/bundle/deploy/files/out-of-band-delete/output.txt @@ -5,43 +5,37 @@ Deploying resources... Updating deployment state... Deployment complete! -=== After the initial deploy the job's python_file and the deployment state exist +=== After the initial deploy the python_file exists >>> [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/out-of-band-delete/default/files/hello_world.py { "object_type": "FILE", "path": "/Workspace/Users/[USERNAME]/.bundle/out-of-band-delete/default/files/hello_world.py" } ->>> [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/out-of-band-delete/default/state/deployment.json -{ - "object_type": "FILE", - "path": "/Workspace/Users/[USERNAME]/.bundle/out-of-band-delete/default/state/deployment.json" -} - === Delete the remote bundle directory out-of-band (simulates deleting it in the UI) >>> [CLI] workspace delete --recursive /Workspace/Users/[USERNAME]/.bundle/out-of-band-delete -=== Redeploy without changing any local file, so the stale sync snapshot is reused +=== Redeploy reuses the stale snapshot, so the python_file is NOT re-uploaded >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/out-of-band-delete/default/files... Deploying resources... Updating deployment state... Deployment complete! -=== After redeploy: the files dir and state are recreated, but the python_file is NOT re-uploaded ->>> errcode [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/out-of-band-delete/default/files/hello_world.py +>>> musterr [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/out-of-band-delete/default/files/hello_world.py Error: Workspace path not found -Exit code: 1 +=== Removing the sync snapshot forces a full re-upload, restoring the python_file +>>> rm -rf .databricks/bundle/default/sync-snapshots ->>> [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/out-of-band-delete/default/files -{ - "object_type": "DIRECTORY", - "path": "/Workspace/Users/[USERNAME]/.bundle/out-of-band-delete/default/files" -} +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/out-of-band-delete/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! ->>> [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/out-of-band-delete/default/state/deployment.json +>>> [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/out-of-band-delete/default/files/hello_world.py { "object_type": "FILE", - "path": "/Workspace/Users/[USERNAME]/.bundle/out-of-band-delete/default/state/deployment.json" + "path": "/Workspace/Users/[USERNAME]/.bundle/out-of-band-delete/default/files/hello_world.py" } diff --git a/acceptance/bundle/deploy/files/out-of-band-delete/script b/acceptance/bundle/deploy/files/out-of-band-delete/script index 46b95a26528..183e58d04a2 100644 --- a/acceptance/bundle/deploy/files/out-of-band-delete/script +++ b/acceptance/bundle/deploy/files/out-of-band-delete/script @@ -1,31 +1,21 @@ -# Redeploy after the remote bundle directory is deleted out-of-band (e.g. a user -# deletes it in the workspace UI). -# -# After such a deletion the next "bundle deploy" does not re-upload the bundle files: -# the local sync snapshot still records them as present remotely, and the remote -# deployment state that would otherwise reset the snapshot was deleted too, so the -# file sync computes an empty diff and uploads nothing. -# -# This test pins down the current behaviour: hello_world.py is missing from the -# workspace after the redeploy even though the deploy reports success. If the file -# sync is changed to re-upload in this case, the final get-status will succeed and -# this output is expected to change. +# Reproduces https://github.com/databricks/cli/issues/1976: after the remote bundle +# files are deleted out-of-band, a plain redeploy does not re-upload them. BUNDLE_PATH="/Workspace/Users/${CURRENT_USER_NAME}/.bundle/out-of-band-delete/default" trace $CLI bundle deploy -title "After the initial deploy the job's python_file and the deployment state exist" +title "After the initial deploy the python_file exists" trace $CLI workspace get-status "$BUNDLE_PATH/files/hello_world.py" | jq '{object_type,path}' -trace $CLI workspace get-status "$BUNDLE_PATH/state/deployment.json" | jq '{object_type,path}' title "Delete the remote bundle directory out-of-band (simulates deleting it in the UI)" trace $CLI workspace delete --recursive "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/out-of-band-delete" -title "Redeploy without changing any local file, so the stale sync snapshot is reused" +title "Redeploy reuses the stale snapshot, so the python_file is NOT re-uploaded" trace $CLI bundle deploy +trace musterr $CLI workspace get-status "$BUNDLE_PATH/files/hello_world.py" -title "After redeploy: the files dir and state are recreated, but the python_file is NOT re-uploaded" -trace errcode $CLI workspace get-status "$BUNDLE_PATH/files/hello_world.py" -trace $CLI workspace get-status "$BUNDLE_PATH/files" | jq '{object_type,path}' -trace $CLI workspace get-status "$BUNDLE_PATH/state/deployment.json" | jq '{object_type,path}' +title "Removing the sync snapshot forces a full re-upload, restoring the python_file" +trace rm -rf .databricks/bundle/default/sync-snapshots +trace $CLI bundle deploy +trace $CLI workspace get-status "$BUNDLE_PATH/files/hello_world.py" | jq '{object_type,path}' diff --git a/acceptance/bundle/deploy/files/out-of-band-delete/test.toml b/acceptance/bundle/deploy/files/out-of-band-delete/test.toml index e39491d91a4..6a62c44e94d 100644 --- a/acceptance/bundle/deploy/files/out-of-band-delete/test.toml +++ b/acceptance/bundle/deploy/files/out-of-band-delete/test.toml @@ -1,3 +1,5 @@ +Badness = "After the remote bundle files are deleted out-of-band, the next deploy does not re-upload them until the local sync snapshot is removed." + [Env] # This test passes absolute workspace paths like /Workspace/Users/.../hello_world.py # as CLI arguments. On Windows the script runs under MSYS2, which rewrites such