Fix config-remote-sync error when a remote rename is reverted before redeploy

acceptance/bundle/config-remote-sync/task_rename_revert/databricks.yml.tmpl

@@ -0,0 +1,18 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  jobs:
+    sample_job:
+      tasks:
+        - task_key: new_task
+          notebook_task:
+            notebook_path: /Users/{{workspace_user_name}}/new_task
+          new_cluster:
+            spark_version: $DEFAULT_SPARK_VERSION
+            node_type_id: $NODE_TYPE_ID
+            num_workers: 1
+
+targets:
+  default:
+    mode: development

acceptance/bundle/config-remote-sync/task_rename_revert/output.txt

@@ -0,0 +1,67 @@
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Rename task to new_task_2 remotely
+
+=== Sync the rename into config (no redeploy)
+Detected changes in 1 resource(s):
+
+Resource: resources.jobs.sample_job
+  tasks[task_key='new_task']: remove
+  tasks[task_key='new_task_2']: add
+
+
+
+>>> diff.py databricks.yml.backup databricks.yml
+--- databricks.yml.backup
++++ databricks.yml
+@@ -6,11 +6,11 @@
+     sample_job:
+       tasks:
+-        - task_key: new_task
+-          notebook_task:
+-            notebook_path: /Users/{{workspace_user_name}}/new_task
+-          new_cluster:
+-            spark_version: 13.3.x-snapshot-scala2.12
++        - new_cluster:
+             node_type_id: [NODE_TYPE_ID]
+             num_workers: 1
++            spark_version: 13.3.x-snapshot-scala2.12
++          notebook_task:
++            notebook_path: '/Users/{{workspace_user_name}}/new_task'
++          task_key: new_task_2
+
+ targets:
+
+=== Rename task back to new_task remotely
+
+=== Sync the revert into config
+Detected changes in 1 resource(s):
+
+Resource: resources.jobs.sample_job
+  tasks[task_key='new_task']: add
+  tasks[task_key='new_task_2']: remove
+
+
+
+>>> diff.py databricks.yml.backup databricks.yml
+--- databricks.yml.backup
++++ databricks.yml
+@@ -12,5 +12,5 @@
+           notebook_task:
+             notebook_path: '/Users/{{workspace_user_name}}/new_task'
+-          task_key: new_task_2
++          task_key: new_task
+
+ targets:
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.jobs.sample_job
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default
+
+Deleting files...
+Destroy complete!

acceptance/bundle/config-remote-sync/task_rename_revert/script

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+  trace $CLI bundle destroy --auto-approve
+}
+trap cleanup EXIT
+
+$CLI bundle deploy
+job_id="$(read_id.py sample_job)"
+
+title "Rename task to new_task_2 remotely"
+echo
+edit_resource.py jobs $job_id <<EOF
+for task in r["tasks"]:
+    if task["task_key"] == "new_task":
+        task["task_key"] = "new_task_2"
+EOF
+
+title "Sync the rename into config (no redeploy)"
+echo
+cp databricks.yml databricks.yml.backup
+$CLI bundle config-remote-sync --save
+trace diff.py databricks.yml.backup databricks.yml
+rm databricks.yml.backup
+
+title "Rename task back to new_task remotely"
+echo
+edit_resource.py jobs $job_id <<EOF
+for task in r["tasks"]:
+    if task["task_key"] == "new_task_2":
+        task["task_key"] = "new_task"
+EOF
+
+# Without redeploy, state still holds task_key='new_task' while config holds
+# 'new_task_2' from the first sync. Before the fix, sync errored with
+# "no array element found with task_key='new_task'" because the change was
+# classified as Replace on a key that the YAML no longer contains.
+title "Sync the revert into config"
+echo
+cp databricks.yml databricks.yml.backup
+$CLI bundle config-remote-sync --save
+trace diff.py databricks.yml.backup databricks.yml
+rm databricks.yml.backup

acceptance/bundle/config-remote-sync/task_rename_revert/test.toml

@@ -0,0 +1,10 @@
+Cloud = true
+
+RecordRequests = false
+Ignore = [".databricks", "databricks.yml", "databricks.yml.backup"]
+
+[Env]
+DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_YAML_SYNC = "true"
+
+[EnvMatrix]
+DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"]

bundle/configsync/diff.go

@@ -85,7 +85,12 @@ func filterEntityDefaults(basePath string, value any) any {
 }
 
 func convertChangeDesc(path string, cd *deployplan.ChangeDesc) (*ConfigChangeDesc, error) {
-	hasConfigValue := cd.Old != nil || cd.New != nil
+	// Use cd.New (current config) to decide whether the field exists "on the config side".
+	// cd.Old (saved state) must not be considered: when the user has already synced a rename
+	// locally (cd.New == nil for the old key) but state still holds the prior key, including
+	// cd.Old in this check would classify the change as Replace and fail later in
+	// resolveSelectors because the old key no longer exists in the YAML.
+	hasConfigValue := cd.New != nil
 	normalizedValue, err := normalizeValue(cd.Remote)
 	if err != nil {
 		return nil, fmt.Errorf("failed to normalize remote value: %w", err)

bundle/configsync/diff_test.go

@@ -3,9 +3,86 @@ package configsync
 import (
 	"testing"
 
+	"github.com/databricks/cli/bundle/deployplan"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
+func TestConvertChangeDesc(t *testing.T) {
+	tests := []struct {
+		name    string
+		path    string
+		cd      *deployplan.ChangeDesc
+		wantOp  OperationType
+		wantVal any
+	}{
+		{
+			name:    "add: new in remote only",
+			path:    "resources.jobs.my_job.description",
+			cd:      &deployplan.ChangeDesc{Old: nil, New: nil, Remote: "remote-desc"},
+			wantOp:  OperationAdd,
+			wantVal: "remote-desc",
+		},
+		{
+			name:    "remove: in config, missing from remote",
+			path:    "resources.jobs.my_job.description",
+			cd:      &deployplan.ChangeDesc{Old: "state-desc", New: "config-desc", Remote: nil},
+			wantOp:  OperationRemove,
+			wantVal: nil,
+		},
+		{
+			name:    "replace: differs between config and remote",
+			path:    "resources.jobs.my_job.description",
+			cd:      &deployplan.ChangeDesc{Old: "state-desc", New: "config-desc", Remote: "remote-desc"},
+			wantOp:  OperationReplace,
+			wantVal: "remote-desc",
+		},
+		{
+			name:    "skip: absent everywhere",
+			path:    "resources.jobs.my_job.description",
+			cd:      &deployplan.ChangeDesc{Old: nil, New: nil, Remote: nil},
+			wantOp:  OperationSkip,
+			wantVal: nil,
+		},
+		// Regression: rename-back-and-forth. State holds the old key (user did not
+		// redeploy after the first sync), config holds the intermediate key, and
+		// remote now matches the original. The element at this path is missing from
+		// config, so the change must be Add — Replace would error in resolveSelectors
+		// because the keyed element no longer exists in the YAML.
+		{
+			name: "add: rename-back path, state has it but config does not",
+			path: "resources.jobs.my_job.tasks[task_key='new_task']",
+			cd: &deployplan.ChangeDesc{
+				Old:    map[string]any{"task_key": "new_task"},
+				New:    nil,
+				Remote: map[string]any{"task_key": "new_task"},
+			},
+			wantOp:  OperationAdd,
+			wantVal: map[string]any{"task_key": "new_task"},
+		},
+		{
+			name: "skip: state has it, config and remote do not",
+			path: "resources.jobs.my_job.tasks[task_key='gone']",
+			cd: &deployplan.ChangeDesc{
+				Old:    map[string]any{"task_key": "gone"},
+				New:    nil,
+				Remote: nil,
+			},
+			wantOp:  OperationSkip,
+			wantVal: nil,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := convertChangeDesc(tt.path, tt.cd)
+			require.NoError(t, err)
+			assert.Equal(t, tt.wantOp, got.Operation)
+			assert.Equal(t, tt.wantVal, got.Value)
+		})
+	}
+}
+
 func TestStripNamePrefix(t *testing.T) {
 	tests := []struct {
 		name   string