From 39f18147d19dddc437ce901a4a7bced068804c76 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Mon, 10 Aug 2026 09:57:04 +0200 Subject: [PATCH] acc: Reproduce linear mismatch error after destroy and oob redeploy --- .../databricks.yml | 14 +++++ .../out.test.toml | 2 + .../output.txt | 51 +++++++++++++++++++ .../lineage-mismatch-after-redeploy/script | 36 +++++++++++++ .../lineage-mismatch-after-redeploy/test.py | 1 + .../lineage-mismatch-after-redeploy/test.toml | 6 +++ 6 files changed, 110 insertions(+) create mode 100644 acceptance/bundle/destroy/lineage-mismatch-after-redeploy/databricks.yml create mode 100644 acceptance/bundle/destroy/lineage-mismatch-after-redeploy/out.test.toml create mode 100644 acceptance/bundle/destroy/lineage-mismatch-after-redeploy/output.txt create mode 100644 acceptance/bundle/destroy/lineage-mismatch-after-redeploy/script create mode 100644 acceptance/bundle/destroy/lineage-mismatch-after-redeploy/test.py create mode 100644 acceptance/bundle/destroy/lineage-mismatch-after-redeploy/test.toml diff --git a/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/databricks.yml b/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/databricks.yml new file mode 100644 index 00000000000..6f7e2364f0e --- /dev/null +++ b/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/databricks.yml @@ -0,0 +1,14 @@ +bundle: + name: lineage-mismatch-after-redeploy + +resources: + jobs: + test_job: + name: "test-job" + tasks: + - task_key: "test-task" + spark_python_task: + python_file: ./test.py + new_cluster: + spark_version: 15.4.x-scala2.12 + node_type_id: i3.xlarge diff --git a/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/out.test.toml b/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/output.txt b/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/output.txt new file mode 100644 index 00000000000..8deb98e5931 --- /dev/null +++ b/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/output.txt @@ -0,0 +1,51 @@ + +=== Machine A: deploy, then destroy (leaves local state with lineage L1 behind) + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lineage-mismatch-after-redeploy/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.test_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/lineage-mismatch-after-redeploy/default + +Deleting files... +Destroy complete! + +=== Machine B: fresh deploy of the same bundle (mints a new lineage L2 remotely) + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lineage-mismatch-after-redeploy/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Machine A: any command that pulls remote state now fails + +>>> musterr [CLI] bundle plan +Error: lineage mismatch in state files + +Available state files: +- resources.json: remote direct state serial=1 lineage="[LINEAGE_B]" +- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=2 lineage="[LINEAGE_A]" + + +>>> musterr [CLI] bundle deploy +Error: lineage mismatch in state files + +Available state files: +- resources.json: remote direct state serial=1 lineage="[LINEAGE_B]" +- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=2 lineage="[LINEAGE_A]" + + +>>> musterr [CLI] bundle destroy --auto-approve +Error: lineage mismatch in state files + +Available state files: +- resources.json: remote direct state serial=1 lineage="[LINEAGE_B]" +- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=2 lineage="[LINEAGE_A]" + diff --git a/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/script b/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/script new file mode 100644 index 00000000000..62a64358329 --- /dev/null +++ b/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/script @@ -0,0 +1,36 @@ +# Reproduces the lineage mismatch a user hits when one machine destroys a bundle +# and another machine (with no local state) redeploys it under the same name. +# +# Local state is scoped by target only (.databricks/bundle/), while remote +# state lives under ~/.bundle///state. Destroy deletes the +# remote state but leaves the local file (with its lineage) behind, so the next +# fresh deploy elsewhere mints a brand-new lineage that no longer matches the +# lingering local one. + +title "Machine A: deploy, then destroy (leaves local state with lineage L1 behind)\n" +trace $CLI bundle deploy +trace $CLI bundle destroy --auto-approve + +# Mask machine A's (local) lineage so the golden output is stable. +LINEAGE_A=$(print_state.py | jq -r .lineage) +add_repl.py "$LINEAGE_A" LINEAGE_A + +# Stash A's local state and wipe it so the next deploy behaves like a fresh machine. +cp -r .databricks .databricks.machineA +rm -rf .databricks + +title "Machine B: fresh deploy of the same bundle (mints a new lineage L2 remotely)\n" +trace $CLI bundle deploy + +# Mask machine B's (remote) lineage. +LINEAGE_B=$(print_state.py | jq -r .lineage) +add_repl.py "$LINEAGE_B" LINEAGE_B + +# Back on machine A: restore the stale local state that still carries L1. +rm -rf .databricks +mv .databricks.machineA .databricks + +title "Machine A: any command that pulls remote state now fails\n" +trace musterr $CLI bundle plan +trace musterr $CLI bundle deploy +trace musterr $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/test.py b/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/test.py new file mode 100644 index 00000000000..1ff8e07c707 --- /dev/null +++ b/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/test.py @@ -0,0 +1 @@ +print("test") diff --git a/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/test.toml b/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/test.toml new file mode 100644 index 00000000000..e2d630064c8 --- /dev/null +++ b/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/test.toml @@ -0,0 +1,6 @@ +# Lineage is a direct-engine concept (a UUID minted per deployment history), +# so this scenario only applies to the direct engine. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +# .databricks.machineA is the stashed local state; may linger if the test fails mid-run. +Ignore = [".databricks.machineA"]