From 55902586c920f827e7d5883f940ea243d2a1a7b6 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 24 Jul 2026 01:41:10 +0200 Subject: [PATCH 01/28] bundle: record and read deployment state via DMS with server-generated IDs Wire the direct engine into the Deployment Metadata Service (DMS) so that a `record_deployment_history`-enabled bundle records each deploy/destroy as a version and can read its resource state back from DMS. The deployment ID is now assigned by the server: the first deploy calls CreateDeployment with an empty ID, reads the assigned ID back from the response, and persists it in the direct-engine state header (Header.DeploymentID). Later deploys pass the stored ID back, so a bundle maps one-to-one to a DMS deployment even after the local cache is deleted (the ID rides along in the workspace-synced state file). - libs/dms: Recorder creates the deployment (server-assigned ID) + version, heartbeats the lease, completes it, and deletes the deployment on destroy. - bundle/direct: operationRecorder reports each applied resource operation; the wire resource_key drops the CLI-internal "resources." prefix. - bundle/direct/dstate: Open takes a DMS client and overlays DMS resource state when DMS holds a successful version; deployment ID persisted in the header. - bundle/phases: create the version after plan approval, complete it under the lock, record operations during apply. - libs/testserver: stateful fake DMS (deployments/versions/operations/resources) with server-generated IDs; acceptance test covers deploy, cache-loss redeploy, and destroy. Co-authored-by: Isaac --- acceptance/bundle/dms/record/databricks.yml | 10 + acceptance/bundle/dms/record/out.test.toml | 3 + acceptance/bundle/dms/record/output.txt | 140 +++++++++++ acceptance/bundle/dms/record/script | 15 ++ acceptance/bundle/dms/test.toml | 13 + bundle/configsync/diff.go | 2 +- bundle/configsync/variables.go | 2 +- bundle/direct/bind.go | 12 +- bundle/direct/bundle_apply.go | 13 + bundle/direct/dstate/dms.go | 104 ++++++++ bundle/direct/dstate/state.go | 43 +++- bundle/direct/dstate/state_test.go | 45 +++- bundle/direct/oprecorder.go | 107 ++++++++ bundle/direct/oprecorder_test.go | 84 +++++++ bundle/direct/pkg.go | 5 + bundle/phases/deploy.go | 32 +++ bundle/phases/destroy.go | 23 ++ bundle/phases/dms.go | 37 +++ cmd/bundle/generate/dashboard.go | 2 +- cmd/bundle/generate/genie_space.go | 2 +- cmd/bundle/utils/process.go | 12 +- libs/dms/recorder.go | 255 ++++++++++++++++++++ libs/dms/recorder_test.go | 165 +++++++++++++ libs/testserver/bundle.go | 228 +++++++++++++++++ libs/testserver/fake_workspace.go | 5 + libs/testserver/handlers.go | 29 +++ 26 files changed, 1363 insertions(+), 25 deletions(-) create mode 100644 acceptance/bundle/dms/record/databricks.yml create mode 100644 acceptance/bundle/dms/record/out.test.toml create mode 100644 acceptance/bundle/dms/record/output.txt create mode 100644 acceptance/bundle/dms/record/script create mode 100644 acceptance/bundle/dms/test.toml create mode 100644 bundle/direct/dstate/dms.go create mode 100644 bundle/direct/oprecorder.go create mode 100644 bundle/direct/oprecorder_test.go create mode 100644 bundle/phases/dms.go create mode 100644 libs/dms/recorder.go create mode 100644 libs/dms/recorder_test.go create mode 100644 libs/testserver/bundle.go diff --git a/acceptance/bundle/dms/record/databricks.yml b/acceptance/bundle/dms/record/databricks.yml new file mode 100644 index 00000000000..b20e6274310 --- /dev/null +++ b/acceptance/bundle/dms/record/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: dms-record + +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/record/out.test.toml b/acceptance/bundle/dms/record/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/dms/record/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt new file mode 100644 index 00000000000..5c0317f38cc --- /dev/null +++ b/acceptance/bundle/dms/record/output.txt @@ -0,0 +1,140 @@ + +=== Deploy: the server assigns the deployment ID, and a version + create operation are recorded +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //api/2.0/bundle --sort +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "target_name": "default" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "target_name": "default", + "version_type": "VERSION_TYPE_DEPLOY" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", + "q": { + "resource_key": "jobs.foo" + }, + "body": { + "action_type": "OPERATION_ACTION_TYPE_CREATE", + "resource_id": "[NUMID]", + "resource_key": "jobs.foo", + "state": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } + }, + "status": "OPERATION_STATUS_SUCCEEDED" + } +} + +=== The server-assigned deployment ID is persisted in the local state file +>>> jq .deployment_id .databricks/bundle/default/resources.json +"[UUID]" + +=== Redeploy after deleting the local cache: the deployment ID is recovered from remote state, the same deployment is reused, and the version increments (no new CreateDeployment) +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //api/2.0/bundle --sort +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "target_name": "default", + "version_type": "VERSION_TYPE_DEPLOY" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== Destroy: a destroy version and delete operation are recorded, then the deployment is deleted +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-record/default + +Deleting files... +Destroy complete! + +>>> print_requests.py //api/2.0/bundle --sort +{ + "method": "DELETE", + "path": "/api/2.0/bundle/deployments/[UUID]" +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions", + "q": { + "version_id": "3" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "target_name": "default", + "version_type": "VERSION_TYPE_DESTROY" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions/3/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions/3/operations", + "q": { + "resource_key": "jobs.foo" + }, + "body": { + "action_type": "OPERATION_ACTION_TYPE_DELETE", + "resource_key": "jobs.foo", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} diff --git a/acceptance/bundle/dms/record/script b/acceptance/bundle/dms/record/script new file mode 100644 index 00000000000..ab59d38afb4 --- /dev/null +++ b/acceptance/bundle/dms/record/script @@ -0,0 +1,15 @@ +title "Deploy: the server assigns the deployment ID, and a version + create operation are recorded" +trace $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --sort + +title "The server-assigned deployment ID is persisted in the local state file" +trace jq .deployment_id .databricks/bundle/default/resources.json + +title "Redeploy after deleting the local cache: the deployment ID is recovered from remote state, the same deployment is reused, and the version increments (no new CreateDeployment)" +rm -rf .databricks +trace $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --sort + +title "Destroy: a destroy version and delete operation are recorded, then the deployment is deleted" +trace $CLI bundle destroy --auto-approve +trace print_requests.py //api/2.0/bundle --sort diff --git a/acceptance/bundle/dms/test.toml b/acceptance/bundle/dms/test.toml new file mode 100644 index 00000000000..24ce9756629 --- /dev/null +++ b/acceptance/bundle/dms/test.toml @@ -0,0 +1,13 @@ +Local = true +Cloud = false + +# Deployment Metadata Service (DMS) recording is only meaningful in the direct +# engine, where the deployment ID is stored in and read from the direct-engine +# state. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +RecordRequests = true + +Ignore = [ + '.databricks', +] diff --git a/bundle/configsync/diff.go b/bundle/configsync/diff.go index ea45903508b..ca5b2c9410b 100644 --- a/bundle/configsync/diff.go +++ b/bundle/configsync/diff.go @@ -149,7 +149,7 @@ func OpenDeploymentState(ctx context.Context, b *bundle.Bundle, engine engine.En deployBundle := &direct.DeploymentBundle{} _, statePath := b.StateFilenameConfigSnapshot(ctx) - if err := deployBundle.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false)); err != nil { + if err := deployBundle.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil); err != nil { return nil, fmt.Errorf("failed to open state: %w", err) } return deployBundle, nil diff --git a/bundle/configsync/variables.go b/bundle/configsync/variables.go index 055a47dc934..433b607a037 100644 --- a/bundle/configsync/variables.go +++ b/bundle/configsync/variables.go @@ -147,7 +147,7 @@ func resourceIDLookup(ctx context.Context, b *bundle.Bundle) func(string) string } _, statePath := b.StateFilenameConfigSnapshot(ctx) db := &dstate.DeploymentState{} - if err := db.Open(ctx, statePath, dstate.WithRecovery(false), dstate.WithWrite(false)); err != nil { + if err := db.Open(ctx, statePath, dstate.WithRecovery(false), dstate.WithWrite(false), nil); err != nil { log.Debugf(ctx, "variable restoration: failed to open state DB at %s: %v", statePath, err) return nil } diff --git a/bundle/direct/bind.go b/bundle/direct/bind.go index 9760ce95666..ec910b2734e 100644 --- a/bundle/direct/bind.go +++ b/bundle/direct/bind.go @@ -62,7 +62,7 @@ type BindResult struct { func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.WorkspaceClient, configRoot *config.Root, statePath, resourceKey, resourceID string) (*BindResult, error) { // Check if the resource is already managed (bound to a different ID) var checkStateDB dstate.DeploymentState - if err := checkStateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false)); err == nil { + if err := checkStateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil); err == nil { existingID := checkStateDB.GetResourceID(resourceKey) if _, err := checkStateDB.Finalize(ctx); err != nil { log.Warnf(ctx, "failed to finalize state: %v", err) @@ -86,7 +86,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } // Open temp state - err := b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(false), dstate.WithWrite(true)) + err := b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(false), dstate.WithWrite(true), nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -109,7 +109,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac log.Infof(ctx, "Bound %s to id=%s (in temp state)", resourceKey, resourceID) // First plan + update: populate state with resolved config - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false)) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -145,7 +145,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } } - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(true)) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -165,7 +165,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } // Second plan: this is the plan to present to the user (change between remote resource and config) - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false)) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -215,7 +215,7 @@ func (result *BindResult) Cancel() { // Unbind removes a resource from direct engine state without deleting // the workspace resource. Also removes associated permissions/grants entries. func (b *DeploymentBundle) Unbind(ctx context.Context, statePath, resourceKey string) error { - err := b.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(true)) + err := b.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil) if err != nil { return err } diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index c4178c4e601..afef2367e5b 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -88,6 +88,11 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } + // Record the delete with DMS. State is nil: the resource is gone. + if err := b.recordOperation(ctx, resourceKey, action, "", nil); err != nil { + logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) + return false + } return true } @@ -116,6 +121,14 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } + + // Record the operation with DMS. The resource ID and applied config + // (sv.Value) come from the write just performed; GetResourceID reads + // the ID assigned by Deploy. + if err := b.recordOperation(ctx, resourceKey, action, b.StateDB.GetResourceID(resourceKey), sv.Value); err != nil { + logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) + return false + } } // TODO: Note, we only really need remote state if there are remote references. diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go new file mode 100644 index 00000000000..1d19d1fe214 --- /dev/null +++ b/bundle/direct/dstate/dms.go @@ -0,0 +1,104 @@ +package dstate + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +// overlayDMSState replaces the file-derived resource state with the state +// recorded in the deployment metadata service (DMS), when DMS owns this +// deployment. Once DMS is authoritative its resource set is trusted even when +// empty (a successful deploy with no resources); the file's resources are only +// used when DMS has no successful version, or when the user opts out of +// recording deployment history. The caller holds db.mu and has already +// populated db.Data from the file, including the DeploymentID. +func (db *DeploymentState) overlayDMSState(ctx context.Context, client bundledeployments.BundleDeploymentsInterface) error { + authoritative, err := deploymentHasSuccessfulVersion(ctx, client, db.Data.DeploymentID) + if err != nil { + return err + } + if !authoritative { + // DMS has no completed version for this deployment: a prior direct deploy + // that has not yet successfully recorded to DMS. Keep the file state. + return nil + } + + resources, err := fetchDeploymentResources(ctx, client, db.Data.DeploymentID) + if err != nil { + return err + } + + db.Data.State = resources + db.stateIDs = make(map[string]string, len(resources)) + for key, entry := range resources { + db.stateIDs[key] = entry.ID + } + return nil +} + +// deploymentHasSuccessfulVersion reports whether DMS holds a successfully +// completed version for the deployment. It is the signal that DMS owns the +// state: if the deployment was never recorded to DMS, or its initial DMS deploy +// did not complete successfully, DMS state is absent or partial and Open keeps +// the local file's resources instead. +func deploymentHasSuccessfulVersion(ctx context.Context, client bundledeployments.BundleDeploymentsInterface, deploymentID string) (bool, error) { + // Versions are listed newest-first and fetched page by page, and we stop at + // the first successful one, so a deployment with a long version history does + // not require reading the whole list (typically just the first page). + it := client.ListVersions(ctx, bundledeployments.ListVersionsRequest{ + Parent: "deployments/" + deploymentID, + }) + for it.HasNext(ctx) { + v, err := it.Next(ctx) + if err != nil { + // A deployment that was never recorded to DMS is not an error here: it + // just means DMS is not (yet) the source of truth. + if errors.Is(err, apierr.ErrNotFound) { + return false, nil + } + return false, fmt.Errorf("listing versions from deployment metadata service: %w", err) + } + if v.Status == bundledeployments.VersionStatusVersionStatusCompleted && + v.CompletionReason == bundledeployments.VersionCompleteVersionCompleteSuccess { + return true, nil + } + } + return false, nil +} + +// fetchDeploymentResources lists every resource recorded for the deployment in +// DMS and maps them into state entries keyed by the fully-qualified resource key. +func fetchDeploymentResources(ctx context.Context, client bundledeployments.BundleDeploymentsInterface, deploymentID string) (map[string]ResourceEntry, error) { + it := client.ListResources(ctx, bundledeployments.ListResourcesRequest{ + Parent: "deployments/" + deploymentID, + }) + + out := make(map[string]ResourceEntry) + for it.HasNext(ctx) { + res, err := it.Next(ctx) + if err != nil { + return nil, fmt.Errorf("listing resources from deployment metadata service: %w", err) + } + + // DMS reports resource keys without the "resources." prefix (e.g. + // "jobs.foo"), but the state DB keys are fully qualified + // ("resources.jobs.foo"), so prepend it here. + key := "resources." + res.ResourceKey + + var state json.RawMessage + if res.State != nil { + state = *res.State + } + + out[key] = ResourceEntry{ + ID: res.ResourceId, + State: state, + } + } + return out, nil +} diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index f6c8fc8ba3c..64fc050bdc0 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -19,6 +19,7 @@ import ( "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/structs/structwalk" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/google/uuid" ) @@ -80,6 +81,13 @@ type Header struct { Lineage string `json:"lineage"` Serial int `json:"serial"` + // DeploymentID is the ID the deployment metadata service (DMS) assigned to + // this deployment. Unlike Lineage (a locally generated identifier for the + // state file), it is minted server-side by CreateDeployment and stored here so + // later deploys can find the same DMS deployment record and read its state. + // Empty/omitted until the bundle first records to DMS. + DeploymentID string `json:"deployment_id,omitempty"` + // Features maps each feature flag this state depends on to a (currently empty) // value. This CLI writes no features; it only reads the field to detect a state // that depends on features it lacks and refuse it (see migrateState). It is a @@ -209,6 +217,25 @@ func (db *DeploymentState) GetOrInitLineage() string { return db.Data.Lineage } +// GetDeploymentID returns the DMS deployment ID recorded in the state, or an +// empty string if this bundle has not yet recorded a deployment to DMS. +func (db *DeploymentState) GetDeploymentID() string { + db.mu.Lock() + defer db.mu.Unlock() + return db.Data.DeploymentID +} + +// SetDeploymentID stores the DMS-assigned deployment ID in the in-memory state +// header. It is set during deploy, after CreateDeployment returns the +// server-generated ID, and persisted to the state file by Finalize. Storing it +// on db.Data (not the WAL header, which is written before the ID is known) +// means the subsequent state write carries it forward. +func (db *DeploymentState) SetDeploymentID(id string) { + db.mu.Lock() + defer db.mu.Unlock() + db.Data.DeploymentID = id +} + type ( // If true, then Open reads the WAL and merges it in the state. If false, and WAL is present, Open returns an error. WithRecovery bool @@ -218,7 +245,15 @@ type ( WithWrite bool ) -func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite) error { +// Open reads the deployment state from disk (and recovers the WAL when +// withRecovery is set). When dmsClient is non-nil, the deployment metadata +// service is the source of truth for resource state: if DMS holds a +// successfully completed version for this deployment, the resources read from +// the file are replaced with the ones recorded in DMS. The local identity +// (lineage, serial, and deployment ID) always comes from the file, since that +// is what the write path increments and carries forward. A nil dmsClient keeps +// the behavior file-only. +func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, dmsClient bundledeployments.BundleDeploymentsInterface) error { db.mu.Lock() defer db.mu.Unlock() @@ -266,6 +301,12 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W return fmt.Errorf("migrating state %s: %w", path, err) } + if dmsClient != nil && db.Data.DeploymentID != "" { + if err := db.overlayDMSState(ctx, dmsClient); err != nil { + return err + } + } + if withWrite { if err := os.MkdirAll(filepath.Dir(walPath), 0o755); err != nil { return fmt.Errorf("failed to create state directory: %w", err) diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index 11589944472..e95ad1b0224 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -20,24 +20,43 @@ func TestOpenSaveFinalizeRoundTrip(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{"key": "val"}, nil)) mustFinalize(t, &db) // Re-open and verify persisted data. var db2 DeploymentState - require.NoError(t, db2.Open(t.Context(), path, WithRecovery(false), WithWrite(false))) + require.NoError(t, db2.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) assert.Equal(t, 1, db2.Data.Serial) assert.Equal(t, "123", db2.GetResourceID("jobs.my_job")) mustFinalize(t, &db2) } +func TestDeploymentIDPersistsAcrossOpen(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + + var db DeploymentState + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) + assert.Empty(t, db.GetDeploymentID()) + + // The deployment ID is set during deploy (after CreateDeployment) and + // persisted by Finalize even though it is not part of the WAL header. + db.SetDeploymentID("server-assigned-id") + require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) + mustFinalize(t, &db) + + var reopened DeploymentState + require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) + assert.Equal(t, "server-assigned-id", reopened.GetDeploymentID()) + mustFinalize(t, &reopened) +} + func TestFinalizeWithNoEntriesDoesNotWriteStateFile(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) mustFinalize(t, &db) _, err := os.Stat(path) @@ -93,10 +112,10 @@ func TestPanicOnDoubleOpen(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) assert.Panics(t, func() { - _ = db.Open(t.Context(), path, WithRecovery(true), WithWrite(true)) + _ = db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil) }) mustFinalize(t, &db) } @@ -107,12 +126,12 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { // Commit serial 1 with one resource. var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) mustFinalize(t, &db) var committed DeploymentState - require.NoError(t, committed.Open(t.Context(), path, WithRecovery(false), WithWrite(false))) + require.NoError(t, committed.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) lineage := committed.Data.Lineage require.Equal(t, 1, committed.Data.Serial) mustFinalize(t, &committed) @@ -128,7 +147,7 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { require.NoError(t, os.WriteFile(walPath, append(headerLine, '\n'), 0o600)) var recovered DeploymentState - require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false))) + require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil)) assert.Equal(t, 1, recovered.Data.Serial) assert.Equal(t, "123", recovered.GetResourceID("jobs.my_job")) assert.NoFileExists(t, walPath) @@ -171,17 +190,17 @@ func TestDeleteState(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) mustFinalize(t, &db) var db2 DeploymentState - require.NoError(t, db2.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) + require.NoError(t, db2.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) require.NoError(t, db2.DeleteState("jobs.my_job")) mustFinalize(t, &db2) var db3 DeploymentState - require.NoError(t, db3.Open(t.Context(), path, WithRecovery(false), WithWrite(false))) + require.NoError(t, db3.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) assert.Equal(t, 2, db3.Data.Serial) assert.Empty(t, db3.GetResourceID("jobs.my_job")) mustFinalize(t, &db3) @@ -193,7 +212,7 @@ func TestGetOrInitLineageReadableBeforeWriteAndPersisted(t *testing.T) { // Fresh state opened read-only, as the deploy does before planning: no // lineage yet. var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(false))) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil)) require.Empty(t, db.Data.Lineage) // GetOrInitLineage initializes the lineage and makes it readable before any @@ -210,7 +229,7 @@ func TestGetOrInitLineageReadableBeforeWriteAndPersisted(t *testing.T) { // Re-open: the persisted lineage matches the one read before the write. var reopened DeploymentState - require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false))) + require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) assert.Equal(t, lineage, reopened.Data.Lineage) mustFinalize(t, &reopened) } diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go new file mode 100644 index 00000000000..467f8ac648c --- /dev/null +++ b/bundle/direct/oprecorder.go @@ -0,0 +1,107 @@ +package direct + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +// opRecorder records a resource operation with the deployment metadata service +// (DMS) after it has been applied to the workspace. state is the serialized +// local config after the operation and must be nil for delete operations. +type opRecorder interface { + record(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state any) error +} + +// recordOperation reports an applied resource operation to DMS. It is a no-op +// unless the bundle opted into recording deployment history (OpRec is set). +// state is the serialized local config after the operation and must be nil for +// delete operations. +func (b *DeploymentBundle) recordOperation(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state any) error { + if b.OpRec == nil { + return nil + } + return b.OpRec.record(ctx, resourceKey, action, resourceID, state) +} + +// operationRecorder records operations via the DMS CreateOperation API. +type operationRecorder struct { + client bundledeployments.BundleDeploymentsInterface + // parent is the version the operations are recorded under, formatted as + // "deployments/{deployment_id}/versions/{version_id}". + parent string +} + +// NewOperationRecorder returns an opRecorder backed by the DMS CreateOperation +// API. deploymentID and version identify the deployment version assigned by DMS +// that the operations are recorded under. +func NewOperationRecorder(client bundledeployments.BundleDeploymentsInterface, deploymentID string, version int64) opRecorder { + return &operationRecorder{ + client: client, + parent: fmt.Sprintf("deployments/%s/versions/%d", deploymentID, version), + } +} + +func (r *operationRecorder) record(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state any) error { + actionType, err := deployActionToSDK(action) + if err != nil { + return err + } + + // DMS resource keys are unprefixed (e.g. "jobs.foo"), while the CLI's state + // keys carry a leading "resources." (e.g. "resources.jobs.foo"). Strip it on + // the way out; the read path re-adds it (see dstate.fetchDeploymentResources). + dmsKey := strings.TrimPrefix(resourceKey, "resources.") + + op := bundledeployments.Operation{ + ActionType: actionType, + ResourceId: resourceID, + ResourceKey: dmsKey, + Status: bundledeployments.OperationStatusOperationStatusSucceeded, + } + + // The DMS Operation.State field carries the serialized config so the backend + // can serve it as resource state. It is intentionally left unset for delete, + // where the resource no longer exists. + if state != nil { + raw, err := json.Marshal(state) + if err != nil { + return fmt.Errorf("serializing state: %w", err) + } + msg := json.RawMessage(raw) + op.State = &msg + } + + _, err = r.client.CreateOperation(ctx, bundledeployments.CreateOperationRequest{ + Parent: r.parent, + ResourceKey: dmsKey, + Operation: op, + }) + return err +} + +// deployActionToSDK maps a deployplan action to its DMS operation action type. +// Only actions that mutate a resource are recordable; Skip and Undefined never +// reach a recorder and are rejected rather than silently coerced. +func deployActionToSDK(a deployplan.ActionType) (bundledeployments.OperationActionType, error) { + switch a { + case deployplan.Create: + return bundledeployments.OperationActionTypeOperationActionTypeCreate, nil + case deployplan.Update: + return bundledeployments.OperationActionTypeOperationActionTypeUpdate, nil + case deployplan.UpdateWithID: + return bundledeployments.OperationActionTypeOperationActionTypeUpdateWithId, nil + case deployplan.Recreate: + return bundledeployments.OperationActionTypeOperationActionTypeRecreate, nil + case deployplan.Resize: + return bundledeployments.OperationActionTypeOperationActionTypeResize, nil + case deployplan.Delete: + return bundledeployments.OperationActionTypeOperationActionTypeDelete, nil + default: + return "", fmt.Errorf("cannot record operation: unsupported action %q", a) + } +} diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go new file mode 100644 index 00000000000..56d860de3e6 --- /dev/null +++ b/bundle/direct/oprecorder_test.go @@ -0,0 +1,84 @@ +package direct + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type fakeOpClient struct { + bundledeployments.BundleDeploymentsInterface + requests []bundledeployments.CreateOperationRequest +} + +func (f *fakeOpClient) CreateOperation(ctx context.Context, req bundledeployments.CreateOperationRequest) (*bundledeployments.Operation, error) { + f.requests = append(f.requests, req) + return &bundledeployments.Operation{}, nil +} + +func TestOperationRecorderStripsResourcePrefix(t *testing.T) { + f := &fakeOpClient{} + r := NewOperationRecorder(f, "dep-1", 2) + + err := r.record(t.Context(), "resources.jobs.foo", deployplan.Create, "job-123", map[string]string{"name": "foo"}) + require.NoError(t, err) + + require.Len(t, f.requests, 1) + req := f.requests[0] + // The wire key drops the CLI-internal "resources." prefix, both in the query + // param and the operation body. + assert.Equal(t, "jobs.foo", req.ResourceKey) + assert.Equal(t, "jobs.foo", req.Operation.ResourceKey) + assert.Equal(t, "deployments/dep-1/versions/2", req.Parent) + assert.Equal(t, bundledeployments.OperationActionTypeOperationActionTypeCreate, req.Operation.ActionType) + assert.Equal(t, "job-123", req.Operation.ResourceId) + require.NotNil(t, req.Operation.State) +} + +func TestOperationRecorderDeleteHasNoState(t *testing.T) { + f := &fakeOpClient{} + r := NewOperationRecorder(f, "dep-1", 3) + + err := r.record(t.Context(), "resources.jobs.foo", deployplan.Delete, "", nil) + require.NoError(t, err) + + require.Len(t, f.requests, 1) + assert.Equal(t, bundledeployments.OperationActionTypeOperationActionTypeDelete, f.requests[0].Operation.ActionType) + // Delete operations carry no serialized state. + assert.Nil(t, f.requests[0].Operation.State) +} + +func TestDeployActionToSDK(t *testing.T) { + cases := []struct { + action deployplan.ActionType + want bundledeployments.OperationActionType + }{ + {deployplan.Create, bundledeployments.OperationActionTypeOperationActionTypeCreate}, + {deployplan.Update, bundledeployments.OperationActionTypeOperationActionTypeUpdate}, + {deployplan.UpdateWithID, bundledeployments.OperationActionTypeOperationActionTypeUpdateWithId}, + {deployplan.Recreate, bundledeployments.OperationActionTypeOperationActionTypeRecreate}, + {deployplan.Resize, bundledeployments.OperationActionTypeOperationActionTypeResize}, + {deployplan.Delete, bundledeployments.OperationActionTypeOperationActionTypeDelete}, + } + for _, c := range cases { + got, err := deployActionToSDK(c.action) + require.NoError(t, err) + assert.Equal(t, c.want, got) + } + + // Skip and Undefined never reach a recorder and are rejected. + _, err := deployActionToSDK(deployplan.Skip) + assert.Error(t, err) + _, err = deployActionToSDK(deployplan.Undefined) + assert.Error(t, err) +} + +func TestRecordOperationNoOpWithoutRecorder(t *testing.T) { + b := &DeploymentBundle{} + // No OpRec set: recording is a no-op. + assert.NoError(t, b.recordOperation(t.Context(), "resources.jobs.foo", deployplan.Create, "id", struct{}{})) +} diff --git a/bundle/direct/pkg.go b/bundle/direct/pkg.go index 48a9c5a2ff7..f95b515f726 100644 --- a/bundle/direct/pkg.go +++ b/bundle/direct/pkg.go @@ -44,6 +44,11 @@ type DeploymentBundle struct { Plan *deployplan.Plan RemoteStateCache sync.Map StateCache structvar.Cache + + // OpRec records each applied resource operation with the deployment metadata + // service (DMS). It is nil unless the bundle opts into recording deployment + // history, in which case the phases package sets it after CreateVersion. + OpRec opRecorder } // SetRemoteState updates the remote state with type validation and marks as fresh. diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index f65e50a940e..792c016f963 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -17,6 +17,7 @@ import ( "github.com/databricks/cli/bundle/deploy/snapshot" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/bundle/metrics" "github.com/databricks/cli/bundle/permissions" @@ -24,6 +25,7 @@ import ( "github.com/databricks/cli/bundle/statemgmt" "github.com/databricks/cli/libs/agent" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/sync" @@ -161,7 +163,17 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand } // lock is acquired here + // + // Set up DMS recording of this deployment as a version. The version is not + // created until the plan is approved (below), so a cancelled deploy records + // nothing; the deferred CompleteVersion is a no-op until CreateVersion runs. + // CompleteVersion is deferred before lock.Release so it runs while the lock + // is still held (defers run last-in-first-out). + recorder := newDeploymentRecorder(ctx, b, stateEngine, dms.VersionTypeDeploy) defer func() { + if err := recorder.CompleteVersion(ctx, !logdiag.HasError(ctx)); err != nil { + logdiag.LogError(ctx, err) + } bundle.ApplyContext(ctx, b, lock.Release(lock.GoalDeploy)) }() @@ -255,6 +267,26 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand return } if haveApproval { + // Record the DMS version now that the plan is approved and the state WAL + // has been opened. CreateVersion requests version_id == last_version_id + 1; + // the server returns ABORTED if a concurrent deploy advanced the deployment + // since the plan was computed, so a stale plan is not applied. + if err := recorder.CreateVersion(ctx); err != nil { + logdiag.LogError(ctx, err) + return + } + if recorder != nil { + // On a first deploy the server assigned the deployment ID; persist it in + // state (Finalize writes it to disk) so later deploys reuse the record. + // Record operations under the version just created so DMS holds the + // deployed resource state. + b.DeploymentBundle.StateDB.SetDeploymentID(recorder.DeploymentID()) + b.DeploymentBundle.OpRec = direct.NewOperationRecorder( + b.WorkspaceClient(ctx).BundleDeployments, + recorder.DeploymentID(), + recorder.Version(), + ) + } deployCore(ctx, b, plan, stateEngine, requestedEngine) } else { cmdio.LogString(ctx, "Deployment cancelled!") diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 2496c7033ad..244f593476f 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -13,8 +13,10 @@ import ( "github.com/databricks/cli/bundle/deploy/lock" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/databricks-sdk-go/apierr" @@ -131,7 +133,15 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { return } + // Set up DMS recording of this destroy as a version. The version is not + // created until the destroy is approved (below), so a cancelled destroy + // records nothing; the deferred CompleteVersion is a no-op until then. It is + // deferred before lock.Release so it runs while the lock is still held. + recorder := newDeploymentRecorder(ctx, b, engine, dms.VersionTypeDestroy) defer func() { + if err := recorder.CompleteVersion(ctx, !logdiag.HasError(ctx)); err != nil { + logdiag.LogError(ctx, err) + } bundle.ApplyContext(ctx, b, lock.Release(lock.GoalDestroy)) }() @@ -188,6 +198,19 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { return } } + // Record the DMS version now that the destroy is approved and the state WAL + // has been opened, then record each delete operation under it. + if err := recorder.CreateVersion(ctx); err != nil { + logdiag.LogError(ctx, err) + return + } + if recorder != nil { + b.DeploymentBundle.OpRec = direct.NewOperationRecorder( + b.WorkspaceClient(ctx).BundleDeployments, + recorder.DeploymentID(), + recorder.Version(), + ) + } destroyCore(ctx, b, plan, engine) } else { cmdio.LogString(ctx, "Destroy cancelled!") diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go new file mode 100644 index 00000000000..667ef8627aa --- /dev/null +++ b/bundle/phases/dms.go @@ -0,0 +1,37 @@ +package phases + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/engine" + "github.com/databricks/cli/libs/dms" +) + +// newDeploymentRecorder returns a dms.Recorder for the current deployment, or +// nil when DMS recording does not apply. A nil recorder is a no-op, so callers +// do not need to branch on it. +// +// Recording is enabled only when experimental.record_deployment_history is set +// AND the engine is direct: DMS resource state is tracked per direct-engine +// deployment, and only the direct engine opens the state DB where the +// deployment ID is stored. Returning nil for terraform leaves those deployments +// untouched. +// +// The deployment ID passed to the recorder is the one persisted in state from a +// previous deploy; it is empty on a bundle's first recorded deploy, in which +// case the recorder creates the deployment and the server assigns the ID. +func newDeploymentRecorder(ctx context.Context, b *bundle.Bundle, eng engine.EngineType, versionType dms.VersionType) *dms.Recorder { + if b.Config.Experimental == nil || !b.Config.Experimental.RecordDeploymentHistory { + return nil + } + if !eng.IsDirect() { + return nil + } + return dms.NewRecorder( + b.WorkspaceClient(ctx).BundleDeployments, + b.DeploymentBundle.StateDB.GetDeploymentID(), + b.Config.Bundle.Target, + versionType, + ) +} diff --git a/cmd/bundle/generate/dashboard.go b/cmd/bundle/generate/dashboard.go index 086ec1d600a..2b286bcad3d 100644 --- a/cmd/bundle/generate/dashboard.go +++ b/cmd/bundle/generate/dashboard.go @@ -404,7 +404,7 @@ func (d *dashboard) runForResource(ctx context.Context, b *bundle.Bundle) { var state statemgmt.ExportedResourcesMap if stateDesc.Engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false)); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil); err != nil { logdiag.LogError(ctx, err) return } diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go index 6d938c5e03d..48ecc92a6cd 100644 --- a/cmd/bundle/generate/genie_space.go +++ b/cmd/bundle/generate/genie_space.go @@ -322,7 +322,7 @@ func (g *genieSpace) runForResource(ctx context.Context, b *bundle.Bundle) { var state statemgmt.ExportedResourcesMap if stateDesc.Engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false)); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil); err != nil { logdiag.LogError(ctx, err) return } diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index e4f232605ce..2815f591b22 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -25,6 +25,7 @@ import ( "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/sync" "github.com/databricks/cli/libs/telemetry/protos" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/spf13/cobra" ) @@ -211,7 +212,16 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle needDirectState := stateDesc.Engine.IsDirect() && (opts.InitIDs || opts.ErrorOnEmptyState || opts.Deploy || opts.ReadPlanPath != "" || opts.PreDeployChecks || opts.PostStateFunc != nil) if needDirectState { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false)); err != nil { + + // When the bundle records deployment history, the deployment metadata + // service owns resource state, so hand Open its client to overlay DMS + // state on top of the local identity (lineage/serial/deployment ID). + // Reads open the state write-disabled, so no lineage is minted here. + var dmsClient bundledeployments.BundleDeploymentsInterface + if b.Config.Experimental != nil && b.Config.Experimental.RecordDeploymentHistory { + dmsClient = b.WorkspaceClient(ctx).BundleDeployments + } + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dmsClient); err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } diff --git a/libs/dms/recorder.go b/libs/dms/recorder.go new file mode 100644 index 00000000000..eed8485f2c2 --- /dev/null +++ b/libs/dms/recorder.go @@ -0,0 +1,255 @@ +// Package dms records bundle deployments as versions with the Deployment +// Metadata Service (DMS). +// +// It is intentionally independent of the deployment lock: a Recorder does not +// acquire or hold any lock. Callers are responsible for serializing concurrent +// deployments (today via the workspace-filesystem lock). The server-side +// version counter — CreateVersion only succeeds when the requested version is +// last_version_id + 1 — provides the concurrency control for the records +// themselves. +package dms + +import ( + "context" + "errors" + "fmt" + "net/http" + "strconv" + "strings" + "time" + + "github.com/databricks/cli/internal/build" + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +// The server expires a version's lease if it does not receive a heartbeat +// within a 2-minute TTL; we heartbeat well inside that window. +const defaultHeartbeatInterval = 30 * time.Second + +// VersionType identifies the kind of deployment a version records. +type VersionType = bundledeployments.VersionType + +const ( + VersionTypeDeploy VersionType = bundledeployments.VersionTypeVersionTypeDeploy + VersionTypeDestroy VersionType = bundledeployments.VersionTypeVersionTypeDestroy +) + +// Recorder records a single deploy/destroy as a version with DMS. +// +// The deployment ID is assigned by the server on the first deploy: NewRecorder +// is given the ID persisted in state (empty on a bundle's first-ever recorded +// deploy), and CreateVersion creates the deployment record when that ID is +// empty and exposes the server-assigned ID via DeploymentID so the caller can +// persist it. Later deploys pass the stored ID back in and reuse the record. +type Recorder struct { + svc bundledeployments.BundleDeploymentsInterface + deploymentID string + targetName string + versionType VersionType + + // populated by CreateVersion + versionNum int64 + stopHeartbeat context.CancelFunc +} + +// NewRecorder returns a Recorder for the given deployment. deploymentID is the +// DMS deployment ID persisted in state, or empty if this bundle has not yet +// recorded a deployment (the server assigns one during CreateVersion). +func NewRecorder(svc bundledeployments.BundleDeploymentsInterface, deploymentID, targetName string, versionType VersionType) *Recorder { + return &Recorder{ + svc: svc, + deploymentID: deploymentID, + targetName: targetName, + versionType: versionType, + } +} + +// DeploymentID returns the DMS deployment ID this recorder is bound to. It is +// empty until CreateVersion has created the deployment record (on a first +// deploy) and non-empty afterwards, so callers persist it once CreateVersion +// succeeds. +func (r *Recorder) DeploymentID() string { + if r == nil { + return "" + } + return r.deploymentID +} + +// Version returns the version number claimed by CreateVersion. It is zero until +// CreateVersion has run; callers use it to parent operations under the version. +func (r *Recorder) Version() int64 { + if r == nil { + return 0 + } + return r.versionNum +} + +// CreateVersion registers a new version with DMS, claiming it for the duration +// of the deployment. A nil Recorder is a no-op, so callers can leave it nil +// when recording is disabled. +func (r *Recorder) CreateVersion(ctx context.Context) error { + if r == nil { + return nil + } + + versionID, err := r.createDeploymentVersion(ctx) + if err != nil { + return err + } + + versionNum, err := strconv.ParseInt(versionID, 10, 64) + if err != nil { + return fmt.Errorf("failed to parse version ID %q: %w", versionID, err) + } + r.versionNum = versionNum + r.stopHeartbeat = startHeartbeat(ctx, r.svc, r.deploymentID, versionID) + return nil +} + +// CompleteVersion finalizes the version created by CreateVersion. A nil +// Recorder, or one whose CreateVersion never ran, is a no-op. +func (r *Recorder) CompleteVersion(ctx context.Context, success bool) error { + if r == nil || r.stopHeartbeat == nil { + return nil + } + + r.stopHeartbeat() + + versionIDStr := strconv.FormatInt(r.versionNum, 10) + versionName := fmt.Sprintf("deployments/%s/versions/%s", r.deploymentID, versionIDStr) + + reason := bundledeployments.VersionCompleteVersionCompleteSuccess + if !success { + reason = bundledeployments.VersionCompleteVersionCompleteFailure + } + + _, err := r.svc.CompleteVersion(ctx, bundledeployments.CompleteVersionRequest{ + Name: versionName, + CompletionReason: reason, + }) + if err != nil { + return err + } + log.Infof(ctx, "Completed deployment version: deployment=%s version=%s reason=%s", r.deploymentID, versionIDStr, reason) + + // For destroy operations, delete the deployment record after the version + // completes successfully. + if success && r.versionType == VersionTypeDestroy { + err = r.svc.DeleteDeployment(ctx, bundledeployments.DeleteDeploymentRequest{ + Name: "deployments/" + r.deploymentID, + }) + if err != nil { + return fmt.Errorf("failed to delete deployment: %w", err) + } + } + + return nil +} + +// createDeploymentVersion ensures the deployment record exists, then creates a +// new version under it. On a first deploy (no stored deployment ID) it creates +// the deployment and lets the server assign the ID; otherwise it reads the +// existing deployment to compute the next version number. +func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID string, err error) { + if r.deploymentID == "" { + // First deploy: create the deployment with an empty ID so the server + // assigns one, then start at version 1. + dep, createErr := r.svc.CreateDeployment(ctx, bundledeployments.CreateDeploymentRequest{ + Deployment: bundledeployments.Deployment{ + TargetName: r.targetName, + }, + }) + if createErr != nil { + return "", fmt.Errorf("failed to create deployment: %w", createErr) + } + id, idErr := deploymentIDFromName(dep.Name) + if idErr != nil { + return "", idErr + } + r.deploymentID = id + versionID = "1" + } else { + // Existing deployment: read it to compute the next version number. + dep, getErr := r.svc.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ + Name: "deployments/" + r.deploymentID, + }) + if getErr != nil { + return "", fmt.Errorf("failed to get deployment: %w", getErr) + } + lastVersion, parseErr := strconv.ParseInt(dep.LastVersionId, 10, 64) + if parseErr != nil { + return "", fmt.Errorf("failed to parse last_version_id %q: %w", dep.LastVersionId, parseErr) + } + versionID = strconv.FormatInt(lastVersion+1, 10) + } + + // The server validates that versionID equals last_version_id + 1 and returns + // ABORTED otherwise (e.g. a concurrent deploy already created this version). + version, versionErr := r.svc.CreateVersion(ctx, bundledeployments.CreateVersionRequest{ + Parent: "deployments/" + r.deploymentID, + VersionId: versionID, + Version: bundledeployments.Version{ + CliVersion: build.GetInfo().Version, + VersionType: r.versionType, + TargetName: r.targetName, + }, + }) + if versionErr != nil { + return "", fmt.Errorf("failed to create deployment version: %w", versionErr) + } + + log.Infof(ctx, "Created deployment version: deployment=%s version=%s", r.deploymentID, version.VersionId) + return versionID, nil +} + +// deploymentIDFromName extracts the deployment ID from a DMS resource name of +// the form "deployments/{deployment_id}". +func deploymentIDFromName(name string) (string, error) { + id, ok := strings.CutPrefix(name, "deployments/") + if !ok || id == "" { + return "", fmt.Errorf("unexpected deployment name %q from deployment metadata service", name) + } + return id, nil +} + +// startHeartbeat starts a background goroutine that sends heartbeats to keep +// the deployment version's lease alive. Returns a cancel function to stop it. +func startHeartbeat(ctx context.Context, svc bundledeployments.BundleDeploymentsInterface, deploymentID, versionID string) context.CancelFunc { + ctx, cancel := context.WithCancel(ctx) + versionName := fmt.Sprintf("deployments/%s/versions/%s", deploymentID, versionID) + + go func() { + ticker := time.NewTicker(defaultHeartbeatInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + _, err := svc.Heartbeat(ctx, bundledeployments.HeartbeatRequest{Name: versionName}) + if err != nil { + // A 409 ABORTED is expected if the version was completed + // between the ticker firing and the heartbeat. + if isAbortedErr(err) { + log.Debugf(ctx, "Heartbeat stopped: version already completed") + return + } + log.Warnf(ctx, "Failed to send deployment heartbeat: %v", err) + } else { + log.Debugf(ctx, "Deployment heartbeat sent: deployment=%s version=%s", deploymentID, versionID) + } + } + } + }() + + return cancel +} + +// isAbortedErr reports whether err is an HTTP 409 ABORTED from the DMS API. +func isAbortedErr(err error) bool { + apiErr, ok := errors.AsType[*apierr.APIError](err) + return ok && apiErr.StatusCode == http.StatusConflict && apiErr.ErrorCode == "ABORTED" +} diff --git a/libs/dms/recorder_test.go b/libs/dms/recorder_test.go new file mode 100644 index 00000000000..91848f74a70 --- /dev/null +++ b/libs/dms/recorder_test.go @@ -0,0 +1,165 @@ +package dms + +import ( + "context" + "testing" + + "github.com/databricks/databricks-sdk-go/service/bundledeployments" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeDMS records the calls the recorder makes and lets a test script the +// server-side responses. It embeds the SDK interface so it satisfies it while +// only overriding the methods the recorder uses. +type fakeDMS struct { + bundledeployments.BundleDeploymentsInterface + + // scripted behavior + getDeployment func(id string) (*bundledeployments.Deployment, error) + + // assigned deployment ID for CreateDeployment (server-generated flow) + assignedID string + + // captured requests + created []bundledeployments.CreateDeploymentRequest + versions []bundledeployments.CreateVersionRequest + completed []bundledeployments.CompleteVersionRequest + deleted []string +} + +func (f *fakeDMS) CreateDeployment(ctx context.Context, req bundledeployments.CreateDeploymentRequest) (*bundledeployments.Deployment, error) { + f.created = append(f.created, req) + id := req.DeploymentId + if id == "" { + id = f.assignedID + } + return &bundledeployments.Deployment{Name: "deployments/" + id}, nil +} + +func (f *fakeDMS) GetDeployment(ctx context.Context, req bundledeployments.GetDeploymentRequest) (*bundledeployments.Deployment, error) { + id := req.Name[len("deployments/"):] + return f.getDeployment(id) +} + +func (f *fakeDMS) CreateVersion(ctx context.Context, req bundledeployments.CreateVersionRequest) (*bundledeployments.Version, error) { + f.versions = append(f.versions, req) + return &bundledeployments.Version{VersionId: req.VersionId}, nil +} + +func (f *fakeDMS) CompleteVersion(ctx context.Context, req bundledeployments.CompleteVersionRequest) (*bundledeployments.Version, error) { + f.completed = append(f.completed, req) + return &bundledeployments.Version{}, nil +} + +func (f *fakeDMS) DeleteDeployment(ctx context.Context, req bundledeployments.DeleteDeploymentRequest) error { + f.deleted = append(f.deleted, req.Name) + return nil +} + +func (f *fakeDMS) Heartbeat(ctx context.Context, req bundledeployments.HeartbeatRequest) (*bundledeployments.HeartbeatResponse, error) { + return &bundledeployments.HeartbeatResponse{}, nil +} + +func TestRecorderFirstDeployCreatesDeploymentWithServerAssignedID(t *testing.T) { + f := &fakeDMS{assignedID: "server-generated-id"} + // A first deploy has no stored deployment ID. + r := NewRecorder(f, "", "dev", VersionTypeDeploy) + + require.NoError(t, r.CreateVersion(t.Context())) + + // The deployment was created with an empty ID so the server assigns one, and + // the recorder exposes the assigned ID for the caller to persist. + require.Len(t, f.created, 1) + assert.Empty(t, f.created[0].DeploymentId) + assert.Equal(t, "server-generated-id", r.DeploymentID()) + + // The first version is 1, parented under the assigned deployment. + require.Len(t, f.versions, 1) + assert.Equal(t, "1", f.versions[0].VersionId) + assert.Equal(t, "deployments/server-generated-id", f.versions[0].Parent) + assert.Equal(t, int64(1), r.Version()) + + require.NoError(t, r.CompleteVersion(t.Context(), true)) + require.Len(t, f.completed, 1) + assert.Equal(t, bundledeployments.VersionCompleteVersionCompleteSuccess, f.completed[0].CompletionReason) + assert.Empty(t, f.deleted) +} + +func TestRecorderSubsequentDeployReusesDeploymentAndIncrementsVersion(t *testing.T) { + f := &fakeDMS{ + getDeployment: func(id string) (*bundledeployments.Deployment, error) { + return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "4"}, nil + }, + } + // A subsequent deploy passes the stored deployment ID. + r := NewRecorder(f, "stored-id", "dev", VersionTypeDeploy) + + require.NoError(t, r.CreateVersion(t.Context())) + + // No new deployment is created; the version increments to last_version_id + 1. + assert.Empty(t, f.created) + require.Len(t, f.versions, 1) + assert.Equal(t, "5", f.versions[0].VersionId) + assert.Equal(t, "stored-id", r.DeploymentID()) +} + +func TestRecorderDestroyDeletesDeploymentOnSuccess(t *testing.T) { + f := &fakeDMS{ + getDeployment: func(id string) (*bundledeployments.Deployment, error) { + return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "2"}, nil + }, + } + r := NewRecorder(f, "stored-id", "dev", VersionTypeDestroy) + + require.NoError(t, r.CreateVersion(t.Context())) + assert.Equal(t, bundledeployments.VersionTypeVersionTypeDestroy, f.versions[0].Version.VersionType) + + require.NoError(t, r.CompleteVersion(t.Context(), true)) + // A successful destroy deletes the deployment record. + require.Equal(t, []string{"deployments/stored-id"}, f.deleted) +} + +func TestRecorderFailedDestroyKeepsDeployment(t *testing.T) { + f := &fakeDMS{ + getDeployment: func(id string) (*bundledeployments.Deployment, error) { + return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "2"}, nil + }, + } + r := NewRecorder(f, "stored-id", "dev", VersionTypeDestroy) + + require.NoError(t, r.CreateVersion(t.Context())) + require.NoError(t, r.CompleteVersion(t.Context(), false)) + + assert.Equal(t, bundledeployments.VersionCompleteVersionCompleteFailure, f.completed[0].CompletionReason) + // A failed destroy leaves the deployment in place. + assert.Empty(t, f.deleted) +} + +func TestNilRecorderIsNoOp(t *testing.T) { + var r *Recorder + assert.NoError(t, r.CreateVersion(t.Context())) + assert.NoError(t, r.CompleteVersion(t.Context(), true)) + assert.Empty(t, r.DeploymentID()) + assert.Zero(t, r.Version()) +} + +func TestRecorderCompleteVersionNoOpWithoutCreateVersion(t *testing.T) { + f := &fakeDMS{} + r := NewRecorder(f, "stored-id", "dev", VersionTypeDeploy) + // CompleteVersion before CreateVersion is a no-op (nothing was claimed). + require.NoError(t, r.CompleteVersion(t.Context(), true)) + assert.Empty(t, f.completed) +} + +func TestDeploymentIDFromName(t *testing.T) { + id, err := deploymentIDFromName("deployments/abc-123") + require.NoError(t, err) + assert.Equal(t, "abc-123", id) + + _, err = deploymentIDFromName("abc-123") + assert.Error(t, err) + + _, err = deploymentIDFromName("deployments/") + assert.Error(t, err) +} diff --git a/libs/testserver/bundle.go b/libs/testserver/bundle.go new file mode 100644 index 00000000000..5f2c7e0cfd1 --- /dev/null +++ b/libs/testserver/bundle.go @@ -0,0 +1,228 @@ +package testserver + +import ( + "encoding/json" + "slices" + "strconv" + + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +// Handlers for the Deployment Metadata Service (DMS) API under /api/2.0/bundle. +// State is kept in FakeWorkspace.dmsDeployments, keyed by deployment ID. + +// dmsDeployment holds a deployment record together with the versions and +// resources recorded under it, so the read APIs (ListVersions/ListResources) +// can serve back what deploys wrote. +type dmsDeployment struct { + deployment bundledeployments.Deployment + versions map[string]*bundledeployments.Version + // resources is the latest resource state per resource key, updated as + // operations are recorded. + resources map[string]bundledeployments.Resource +} + +func (s *FakeWorkspace) CreateDeployment(req Request) Response { + // The client either supplies the deployment ID or, in the server-generated + // flow, leaves it empty for the server to mint one. + deploymentID := req.URL.Query().Get("deployment_id") + if deploymentID == "" { + deploymentID = nextUUID() + } + + var dep bundledeployments.Deployment + if err := json.Unmarshal(req.Body, &dep); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + + defer s.LockUnlock()() + + dep.Name = "deployments/" + deploymentID + dep.Status = bundledeployments.DeploymentStatusDeploymentStatusActive + s.dmsDeployments[deploymentID] = &dmsDeployment{ + deployment: dep, + versions: map[string]*bundledeployments.Version{}, + resources: map[string]bundledeployments.Resource{}, + } + return Response{Body: dep} +} + +func (s *FakeWorkspace) GetDeployment(deploymentID string) Response { + defer s.LockUnlock()() + + d, ok := s.dmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + return Response{Body: d.deployment} +} + +func (s *FakeWorkspace) DeleteDeployment(deploymentID string) Response { + defer s.LockUnlock()() + + delete(s.dmsDeployments, deploymentID) + return Response{Body: map[string]any{}} +} + +func (s *FakeWorkspace) CreateVersion(req Request, deploymentID string) Response { + versionID := req.URL.Query().Get("version_id") + + var version bundledeployments.Version + if err := json.Unmarshal(req.Body, &version); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + + defer s.LockUnlock()() + + d, ok := s.dmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + + // Mirror the server-side optimistic concurrency check: the new version must + // be exactly last_version_id + 1. + want := "1" + if d.deployment.LastVersionId != "" { + last, _ := strconv.ParseInt(d.deployment.LastVersionId, 10, 64) + want = strconv.FormatInt(last+1, 10) + } + if versionID != want { + return dmsAborted("expected version " + want + ", got " + versionID) + } + + d.deployment.LastVersionId = versionID + version.Name = "deployments/" + deploymentID + "/versions/" + versionID + version.VersionId = versionID + version.Status = bundledeployments.VersionStatusVersionStatusInProgress + d.versions[versionID] = &version + return Response{Body: version} +} + +func (s *FakeWorkspace) CompleteVersion(req Request, deploymentID, versionID string) Response { + var completeReq bundledeployments.CompleteVersionRequest + if err := json.Unmarshal(req.Body, &completeReq); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + + defer s.LockUnlock()() + + d, ok := s.dmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + v, ok := d.versions[versionID] + if !ok { + return dmsNotFound("version " + versionID) + } + + v.Status = bundledeployments.VersionStatusVersionStatusCompleted + v.CompletionReason = completeReq.CompletionReason + return Response{Body: *v} +} + +func (s *FakeWorkspace) Heartbeat() Response { + return Response{Body: bundledeployments.HeartbeatResponse{}} +} + +func (s *FakeWorkspace) CreateOperation(req Request, deploymentID, versionID string) Response { + resourceKey := req.URL.Query().Get("resource_key") + + var op bundledeployments.Operation + if err := json.Unmarshal(req.Body, &op); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + + defer s.LockUnlock()() + + d, ok := s.dmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + + op.Name = "deployments/" + deploymentID + "/versions/" + versionID + "/operations/" + resourceKey + op.ResourceKey = resourceKey + + // Reflect the operation onto the deployment-level resource set the way the + // backend does: a delete removes the resource, anything else upserts it. + if op.ActionType == bundledeployments.OperationActionTypeOperationActionTypeDelete { + delete(d.resources, resourceKey) + } else { + d.resources[resourceKey] = bundledeployments.Resource{ + Name: "deployments/" + deploymentID + "/resources/" + resourceKey, + ResourceKey: resourceKey, + ResourceId: op.ResourceId, + ResourceType: op.ResourceType, + LastActionType: op.ActionType, + LastVersionId: versionID, + State: op.State, + } + } + return Response{Body: op} +} + +func (s *FakeWorkspace) ListVersions(deploymentID string) Response { + defer s.LockUnlock()() + + d, ok := s.dmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + + // The API returns versions newest-first (descending version_id). + ids := make([]int64, 0, len(d.versions)) + for id := range d.versions { + n, _ := strconv.ParseInt(id, 10, 64) + ids = append(ids, n) + } + slices.SortFunc(ids, func(a, b int64) int { return int(b - a) }) + + versions := make([]bundledeployments.Version, 0, len(ids)) + for _, id := range ids { + versions = append(versions, *d.versions[strconv.FormatInt(id, 10)]) + } + return Response{Body: bundledeployments.ListVersionsResponse{Versions: versions}} +} + +func (s *FakeWorkspace) ListResources(deploymentID string) Response { + defer s.LockUnlock()() + + d, ok := s.dmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + + // Sort by resource key so the response order is deterministic. + keys := make([]string, 0, len(d.resources)) + for key := range d.resources { + keys = append(keys, key) + } + slices.Sort(keys) + + resources := make([]bundledeployments.Resource, 0, len(keys)) + for _, key := range keys { + resources = append(resources, d.resources[key]) + } + return Response{Body: bundledeployments.ListResourcesResponse{Resources: resources}} +} + +// dmsNotFound returns the RESOURCE_DOES_NOT_EXIST error shape the DMS API uses, +// which the SDK maps to apierr.ErrNotFound. +func dmsNotFound(what string) Response { + return Response{ + StatusCode: 404, + Body: map[string]string{ + "error_code": "RESOURCE_DOES_NOT_EXIST", + "message": what + " does not exist", + }, + } +} + +// dmsAborted returns the 409 ABORTED error the server uses for the version +// optimistic-concurrency check. +func dmsAborted(message string) Response { + return Response{ + StatusCode: 409, + Headers: map[string][]string{"Content-Type": {"application/json"}}, + Body: map[string]string{"error_code": "ABORTED", "message": message}, + } +} diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 8d6e8ee0dd3..a3c4519ccc4 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -227,6 +227,10 @@ type FakeWorkspace struct { // clusterVenvs caches Python venvs per existing cluster ID, // matching cloud behavior where libraries are cached on running clusters. clusterVenvs map[string]*clusterEnv + + // dmsDeployments holds Deployment Metadata Service (DMS) records, keyed by + // deployment ID. Each record carries its versions and latest resource state. + dmsDeployments map[string]*dmsDeployment } func (s *FakeWorkspace) LockUnlock() func() { @@ -378,6 +382,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { postgresImplicitBranches: map[string]bool{}, postgresImplicitEndpoints: map[string]bool{}, clusterVenvs: map[string]*clusterEnv{}, + dmsDeployments: map[string]*dmsDeployment{}, Alerts: map[string]sql.AlertV2{}, Experiments: map[string]ml.GetExperimentResponse{}, ModelRegistryModels: map[string]ml.Model{}, diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 1d534c47431..39fe6fbb057 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -266,6 +266,35 @@ func AddDefaultHandlers(server *Server) { return req.Workspace.JobsCreate(req) }) + // Deployment Metadata Service (DMS) endpoints. + server.Handle("POST", "/api/2.0/bundle/deployments", func(req Request) any { + return req.Workspace.CreateDeployment(req) + }) + server.Handle("GET", "/api/2.0/bundle/deployments/{deployment_id}", func(req Request) any { + return req.Workspace.GetDeployment(req.Vars["deployment_id"]) + }) + server.Handle("DELETE", "/api/2.0/bundle/deployments/{deployment_id}", func(req Request) any { + return req.Workspace.DeleteDeployment(req.Vars["deployment_id"]) + }) + server.Handle("GET", "/api/2.0/bundle/deployments/{deployment_id}/versions", func(req Request) any { + return req.Workspace.ListVersions(req.Vars["deployment_id"]) + }) + server.Handle("POST", "/api/2.0/bundle/deployments/{deployment_id}/versions", func(req Request) any { + return req.Workspace.CreateVersion(req, req.Vars["deployment_id"]) + }) + server.Handle("POST", "/api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/complete", func(req Request) any { + return req.Workspace.CompleteVersion(req, req.Vars["deployment_id"], req.Vars["version_id"]) + }) + server.Handle("POST", "/api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/heartbeat", func(req Request) any { + return req.Workspace.Heartbeat() + }) + server.Handle("POST", "/api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/operations", func(req Request) any { + return req.Workspace.CreateOperation(req, req.Vars["deployment_id"], req.Vars["version_id"]) + }) + server.Handle("GET", "/api/2.0/bundle/deployments/{deployment_id}/resources", func(req Request) any { + return req.Workspace.ListResources(req.Vars["deployment_id"]) + }) + server.Handle("POST", "/api/2.2/jobs/delete", func(req Request) any { var request jobs.DeleteJob if err := json.Unmarshal(req.Body, &request); err != nil { From da1ed9dfc00a9d6171b2fbeb60d9f17a3e900b26 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 24 Jul 2026 16:49:32 +0200 Subject: [PATCH 02/28] bundle: read DMS authority from last_successful_version_id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The read overlay decided whether DMS owns a deployment's state by listing versions and scanning for a successful one. The deployment now exposes last_successful_version_id directly, so a single GetDeployment answers the same question — no version listing. The field is still stage:DEVELOPMENT in the proto and therefore stripped from the generated SDK, so this reads the deployment via a raw GET into a local struct as a temporary stub. Once the field is promoted to PRIVATE_PREVIEW and regenerated, the raw call collapses to client.GetDeployment(...). LastSuccessfulVersionId and the threaded config argument goes away (see the TODO in deploymentHasSuccessfulVersion). The testserver's GetDeployment now serves last_successful_version_id (tracked on version completion), and the now-unused ListVersions fake is removed. Co-authored-by: Isaac --- bundle/configsync/diff.go | 2 +- bundle/configsync/variables.go | 2 +- bundle/direct/bind.go | 12 +++--- bundle/direct/dstate/dms.go | 64 +++++++++++++++++++----------- bundle/direct/dstate/state.go | 9 ++++- bundle/direct/dstate/state_test.go | 30 +++++++------- cmd/bundle/generate/dashboard.go | 2 +- cmd/bundle/generate/genie_space.go | 2 +- cmd/bundle/utils/process.go | 7 +++- libs/testserver/bundle.go | 45 ++++++++++----------- libs/testserver/handlers.go | 3 -- 11 files changed, 100 insertions(+), 78 deletions(-) diff --git a/bundle/configsync/diff.go b/bundle/configsync/diff.go index ca5b2c9410b..17ed3b30d5e 100644 --- a/bundle/configsync/diff.go +++ b/bundle/configsync/diff.go @@ -149,7 +149,7 @@ func OpenDeploymentState(ctx context.Context, b *bundle.Bundle, engine engine.En deployBundle := &direct.DeploymentBundle{} _, statePath := b.StateFilenameConfigSnapshot(ctx) - if err := deployBundle.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil); err != nil { + if err := deployBundle.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, nil); err != nil { return nil, fmt.Errorf("failed to open state: %w", err) } return deployBundle, nil diff --git a/bundle/configsync/variables.go b/bundle/configsync/variables.go index 433b607a037..be3e536f37f 100644 --- a/bundle/configsync/variables.go +++ b/bundle/configsync/variables.go @@ -147,7 +147,7 @@ func resourceIDLookup(ctx context.Context, b *bundle.Bundle) func(string) string } _, statePath := b.StateFilenameConfigSnapshot(ctx) db := &dstate.DeploymentState{} - if err := db.Open(ctx, statePath, dstate.WithRecovery(false), dstate.WithWrite(false), nil); err != nil { + if err := db.Open(ctx, statePath, dstate.WithRecovery(false), dstate.WithWrite(false), nil, nil); err != nil { log.Debugf(ctx, "variable restoration: failed to open state DB at %s: %v", statePath, err) return nil } diff --git a/bundle/direct/bind.go b/bundle/direct/bind.go index ec910b2734e..ccfbcf788ab 100644 --- a/bundle/direct/bind.go +++ b/bundle/direct/bind.go @@ -62,7 +62,7 @@ type BindResult struct { func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.WorkspaceClient, configRoot *config.Root, statePath, resourceKey, resourceID string) (*BindResult, error) { // Check if the resource is already managed (bound to a different ID) var checkStateDB dstate.DeploymentState - if err := checkStateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil); err == nil { + if err := checkStateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, nil); err == nil { existingID := checkStateDB.GetResourceID(resourceKey) if _, err := checkStateDB.Finalize(ctx); err != nil { log.Warnf(ctx, "failed to finalize state: %v", err) @@ -86,7 +86,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } // Open temp state - err := b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(false), dstate.WithWrite(true), nil) + err := b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(false), dstate.WithWrite(true), nil, nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -109,7 +109,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac log.Infof(ctx, "Bound %s to id=%s (in temp state)", resourceKey, resourceID) // First plan + update: populate state with resolved config - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -145,7 +145,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } } - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil, nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -165,7 +165,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } // Second plan: this is the plan to present to the user (change between remote resource and config) - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -215,7 +215,7 @@ func (result *BindResult) Cancel() { // Unbind removes a resource from direct engine state without deleting // the workspace resource. Also removes associated permissions/grants entries. func (b *DeploymentBundle) Unbind(ctx context.Context, statePath, resourceKey string) error { - err := b.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil) + err := b.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil, nil) if err != nil { return err } diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index 1d19d1fe214..659d7c9dd0b 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -5,8 +5,12 @@ import ( "encoding/json" "errors" "fmt" + "net/http" + "github.com/databricks/cli/libs/auth" "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/client" + sdkconfig "github.com/databricks/databricks-sdk-go/config" "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) @@ -17,8 +21,11 @@ import ( // used when DMS has no successful version, or when the user opts out of // recording deployment history. The caller holds db.mu and has already // populated db.Data from the file, including the DeploymentID. -func (db *DeploymentState) overlayDMSState(ctx context.Context, client bundledeployments.BundleDeploymentsInterface) error { - authoritative, err := deploymentHasSuccessfulVersion(ctx, client, db.Data.DeploymentID) +// +// cfg is threaded in only for the temporary raw read in +// deploymentHasSuccessfulVersion; see the TODO there. +func (db *DeploymentState) overlayDMSState(ctx context.Context, client bundledeployments.BundleDeploymentsInterface, cfg *sdkconfig.Config) error { + authoritative, err := deploymentHasSuccessfulVersion(ctx, cfg, db.Data.DeploymentID) if err != nil { return err } @@ -46,29 +53,40 @@ func (db *DeploymentState) overlayDMSState(ctx context.Context, client bundledep // state: if the deployment was never recorded to DMS, or its initial DMS deploy // did not complete successfully, DMS state is absent or partial and Open keeps // the local file's resources instead. -func deploymentHasSuccessfulVersion(ctx context.Context, client bundledeployments.BundleDeploymentsInterface, deploymentID string) (bool, error) { - // Versions are listed newest-first and fetched page by page, and we stop at - // the first successful one, so a deployment with a long version history does - // not require reading the whole list (typically just the first page). - it := client.ListVersions(ctx, bundledeployments.ListVersionsRequest{ - Parent: "deployments/" + deploymentID, - }) - for it.HasNext(ctx) { - v, err := it.Next(ctx) - if err != nil { - // A deployment that was never recorded to DMS is not an error here: it - // just means DMS is not (yet) the source of truth. - if errors.Is(err, apierr.ErrNotFound) { - return false, nil - } - return false, fmt.Errorf("listing versions from deployment metadata service: %w", err) - } - if v.Status == bundledeployments.VersionStatusVersionStatusCompleted && - v.CompletionReason == bundledeployments.VersionCompleteVersionCompleteSuccess { - return true, nil +// +// The deployment carries last_successful_version_id, which the server advances +// only when a version completes successfully (unlike last_version_id, which +// also advances on failure). So a non-empty value is exactly the "DMS owns the +// state" signal, readable in a single GetDeployment. +// +// TODO(DMS): this reads the deployment via a raw GET into a local struct +// because last_successful_version_id is still stage:DEVELOPMENT in the proto +// and therefore stripped from the generated SDK. Once the field is promoted to +// PRIVATE_PREVIEW and regenerated, replace the raw call with +// client.GetDeployment(...).LastSuccessfulVersionId and drop the cfg argument +// (revert overlayDMSState/Open back to taking only the typed client). +func deploymentHasSuccessfulVersion(ctx context.Context, cfg *sdkconfig.Config, deploymentID string) (bool, error) { + apiClient, err := client.New(cfg) + if err != nil { + return false, fmt.Errorf("creating API client for deployment metadata service: %w", err) + } + + // Mirrors the SDK's GetDeployment path (/api/2.0/bundle/{name} with + // name=deployments/{id}); we unmarshal into a local struct so we can read + // last_successful_version_id, which the typed SDK response drops. + var dep struct { + LastSuccessfulVersionID string `json:"last_successful_version_id"` + } + err = apiClient.Do(ctx, http.MethodGet, "/api/2.0/bundle/deployments/"+deploymentID, auth.WorkspaceIDHeaders(cfg), nil, nil, &dep) + if err != nil { + // A deployment that was never recorded to DMS is not an error here: it + // just means DMS is not (yet) the source of truth. + if errors.Is(err, apierr.ErrNotFound) { + return false, nil } + return false, fmt.Errorf("reading deployment from deployment metadata service: %w", err) } - return false, nil + return dep.LastSuccessfulVersionID != "", nil } // fetchDeploymentResources lists every resource recorded for the deployment in diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 64fc050bdc0..2c969667c9e 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -19,6 +19,7 @@ import ( "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/structs/structwalk" + sdkconfig "github.com/databricks/databricks-sdk-go/config" "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/google/uuid" ) @@ -253,7 +254,11 @@ type ( // (lineage, serial, and deployment ID) always comes from the file, since that // is what the write path increments and carries forward. A nil dmsClient keeps // the behavior file-only. -func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, dmsClient bundledeployments.BundleDeploymentsInterface) error { +// +// dmsCfg accompanies dmsClient (both come from the same workspace client) and +// is used only for a temporary raw read of last_successful_version_id; see the +// TODO in deploymentHasSuccessfulVersion. +func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, dmsClient bundledeployments.BundleDeploymentsInterface, dmsCfg *sdkconfig.Config) error { db.mu.Lock() defer db.mu.Unlock() @@ -302,7 +307,7 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W } if dmsClient != nil && db.Data.DeploymentID != "" { - if err := db.overlayDMSState(ctx, dmsClient); err != nil { + if err := db.overlayDMSState(ctx, dmsClient, dmsCfg); err != nil { return err } } diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index e95ad1b0224..16066bf81f8 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -20,14 +20,14 @@ func TestOpenSaveFinalizeRoundTrip(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{"key": "val"}, nil)) mustFinalize(t, &db) // Re-open and verify persisted data. var db2 DeploymentState - require.NoError(t, db2.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) + require.NoError(t, db2.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, nil)) assert.Equal(t, 1, db2.Data.Serial) assert.Equal(t, "123", db2.GetResourceID("jobs.my_job")) mustFinalize(t, &db2) @@ -37,7 +37,7 @@ func TestDeploymentIDPersistsAcrossOpen(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) assert.Empty(t, db.GetDeploymentID()) // The deployment ID is set during deploy (after CreateDeployment) and @@ -47,7 +47,7 @@ func TestDeploymentIDPersistsAcrossOpen(t *testing.T) { mustFinalize(t, &db) var reopened DeploymentState - require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) + require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, nil)) assert.Equal(t, "server-assigned-id", reopened.GetDeploymentID()) mustFinalize(t, &reopened) } @@ -56,7 +56,7 @@ func TestFinalizeWithNoEntriesDoesNotWriteStateFile(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) mustFinalize(t, &db) _, err := os.Stat(path) @@ -112,10 +112,10 @@ func TestPanicOnDoubleOpen(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) assert.Panics(t, func() { - _ = db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil) + _ = db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil) }) mustFinalize(t, &db) } @@ -126,12 +126,12 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { // Commit serial 1 with one resource. var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) mustFinalize(t, &db) var committed DeploymentState - require.NoError(t, committed.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) + require.NoError(t, committed.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, nil)) lineage := committed.Data.Lineage require.Equal(t, 1, committed.Data.Serial) mustFinalize(t, &committed) @@ -147,7 +147,7 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { require.NoError(t, os.WriteFile(walPath, append(headerLine, '\n'), 0o600)) var recovered DeploymentState - require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil)) + require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil, nil)) assert.Equal(t, 1, recovered.Data.Serial) assert.Equal(t, "123", recovered.GetResourceID("jobs.my_job")) assert.NoFileExists(t, walPath) @@ -190,17 +190,17 @@ func TestDeleteState(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) mustFinalize(t, &db) var db2 DeploymentState - require.NoError(t, db2.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) + require.NoError(t, db2.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) require.NoError(t, db2.DeleteState("jobs.my_job")) mustFinalize(t, &db2) var db3 DeploymentState - require.NoError(t, db3.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) + require.NoError(t, db3.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, nil)) assert.Equal(t, 2, db3.Data.Serial) assert.Empty(t, db3.GetResourceID("jobs.my_job")) mustFinalize(t, &db3) @@ -212,7 +212,7 @@ func TestGetOrInitLineageReadableBeforeWriteAndPersisted(t *testing.T) { // Fresh state opened read-only, as the deploy does before planning: no // lineage yet. var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil, nil)) require.Empty(t, db.Data.Lineage) // GetOrInitLineage initializes the lineage and makes it readable before any @@ -229,7 +229,7 @@ func TestGetOrInitLineageReadableBeforeWriteAndPersisted(t *testing.T) { // Re-open: the persisted lineage matches the one read before the write. var reopened DeploymentState - require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) + require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, nil)) assert.Equal(t, lineage, reopened.Data.Lineage) mustFinalize(t, &reopened) } diff --git a/cmd/bundle/generate/dashboard.go b/cmd/bundle/generate/dashboard.go index 2b286bcad3d..4866f27c5b3 100644 --- a/cmd/bundle/generate/dashboard.go +++ b/cmd/bundle/generate/dashboard.go @@ -404,7 +404,7 @@ func (d *dashboard) runForResource(ctx context.Context, b *bundle.Bundle) { var state statemgmt.ExportedResourcesMap if stateDesc.Engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, nil); err != nil { logdiag.LogError(ctx, err) return } diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go index 48ecc92a6cd..b5dbeed6c56 100644 --- a/cmd/bundle/generate/genie_space.go +++ b/cmd/bundle/generate/genie_space.go @@ -322,7 +322,7 @@ func (g *genieSpace) runForResource(ctx context.Context, b *bundle.Bundle) { var state statemgmt.ExportedResourcesMap if stateDesc.Engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, nil); err != nil { logdiag.LogError(ctx, err) return } diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 2815f591b22..f556c2b3450 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -25,6 +25,7 @@ import ( "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/sync" "github.com/databricks/cli/libs/telemetry/protos" + sdkconfig "github.com/databricks/databricks-sdk-go/config" "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/spf13/cobra" ) @@ -217,11 +218,15 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle // service owns resource state, so hand Open its client to overlay DMS // state on top of the local identity (lineage/serial/deployment ID). // Reads open the state write-disabled, so no lineage is minted here. + // dmsCfg accompanies the client for a temporary raw read (see the TODO + // in dstate.deploymentHasSuccessfulVersion). var dmsClient bundledeployments.BundleDeploymentsInterface + var dmsCfg *sdkconfig.Config if b.Config.Experimental != nil && b.Config.Experimental.RecordDeploymentHistory { dmsClient = b.WorkspaceClient(ctx).BundleDeployments + dmsCfg = b.WorkspaceClient(ctx).Config } - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dmsClient); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dmsClient, dmsCfg); err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } diff --git a/libs/testserver/bundle.go b/libs/testserver/bundle.go index 5f2c7e0cfd1..34003a507a5 100644 --- a/libs/testserver/bundle.go +++ b/libs/testserver/bundle.go @@ -20,6 +20,12 @@ type dmsDeployment struct { // resources is the latest resource state per resource key, updated as // operations are recorded. resources map[string]bundledeployments.Resource + // lastSuccessfulVersionID is the highest version that completed + // successfully. The server advances last_successful_version_id only on + // success (unlike last_version_id), and the read path treats a non-empty + // value as "DMS owns the state". Tracked separately because the SDK + // Deployment struct does not yet carry the field (still stage:DEVELOPMENT). + lastSuccessfulVersionID string } func (s *FakeWorkspace) CreateDeployment(req Request) Response { @@ -54,7 +60,18 @@ func (s *FakeWorkspace) GetDeployment(deploymentID string) Response { if !ok { return dmsNotFound("deployment " + deploymentID) } - return Response{Body: d.deployment} + + // The SDK Deployment struct does not yet carry last_successful_version_id + // (still stage:DEVELOPMENT, so stripped from generation), but the read path + // reads it off the raw JSON. Serve it as an extra field alongside the typed + // deployment so the overlay behaves as it will against the real server. + return Response{Body: struct { + bundledeployments.Deployment + LastSuccessfulVersionID string `json:"last_successful_version_id,omitempty"` + }{ + Deployment: d.deployment, + LastSuccessfulVersionID: d.lastSuccessfulVersionID, + }} } func (s *FakeWorkspace) DeleteDeployment(deploymentID string) Response { @@ -117,6 +134,9 @@ func (s *FakeWorkspace) CompleteVersion(req Request, deploymentID, versionID str v.Status = bundledeployments.VersionStatusVersionStatusCompleted v.CompletionReason = completeReq.CompletionReason + if completeReq.CompletionReason == bundledeployments.VersionCompleteVersionCompleteSuccess { + d.lastSuccessfulVersionID = versionID + } return Response{Body: *v} } @@ -160,29 +180,6 @@ func (s *FakeWorkspace) CreateOperation(req Request, deploymentID, versionID str return Response{Body: op} } -func (s *FakeWorkspace) ListVersions(deploymentID string) Response { - defer s.LockUnlock()() - - d, ok := s.dmsDeployments[deploymentID] - if !ok { - return dmsNotFound("deployment " + deploymentID) - } - - // The API returns versions newest-first (descending version_id). - ids := make([]int64, 0, len(d.versions)) - for id := range d.versions { - n, _ := strconv.ParseInt(id, 10, 64) - ids = append(ids, n) - } - slices.SortFunc(ids, func(a, b int64) int { return int(b - a) }) - - versions := make([]bundledeployments.Version, 0, len(ids)) - for _, id := range ids { - versions = append(versions, *d.versions[strconv.FormatInt(id, 10)]) - } - return Response{Body: bundledeployments.ListVersionsResponse{Versions: versions}} -} - func (s *FakeWorkspace) ListResources(deploymentID string) Response { defer s.LockUnlock()() diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 39fe6fbb057..3cbfa16c7e1 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -276,9 +276,6 @@ func AddDefaultHandlers(server *Server) { server.Handle("DELETE", "/api/2.0/bundle/deployments/{deployment_id}", func(req Request) any { return req.Workspace.DeleteDeployment(req.Vars["deployment_id"]) }) - server.Handle("GET", "/api/2.0/bundle/deployments/{deployment_id}/versions", func(req Request) any { - return req.Workspace.ListVersions(req.Vars["deployment_id"]) - }) server.Handle("POST", "/api/2.0/bundle/deployments/{deployment_id}/versions", func(req Request) any { return req.Workspace.CreateVersion(req, req.Vars["deployment_id"]) }) From 011c0928c7962098840aa370c1e13508b80447a8 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 27 Jul 2026 00:41:17 +0000 Subject: [PATCH 03/28] bundle: fix DMS deployment recording bugs found in review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five fixes to the DMS state recording added in #6052: 1. The fake DMS server dropped last_successful_version_id. GetDeployment serialized the response through a struct embedding bundledeployments.Deployment, whose promoted MarshalJSON silently discards sibling fields. The CLI reads a missing value as "DMS does not own the state", so the entire read/overlay path (overlayDMSState, fetchDeploymentResources, deploymentHasSuccessfulVersion) never ran in any test. Serialize through a map instead, and unit-test the shape. 2. A bundle with no resources leaked a deployment record per deploy. Such a deploy writes no WAL entries, and the state file was only persisted when the WAL carried entries, so the server-assigned deployment ID was dropped and the next deploy created a second deployment. Track a dirty header so Finalize persists an ID change on its own. A header-only WAL that changed nothing still skips the write, keeping the serial in step (acceptance/bundle/deploy/wal/header-only-wal). 3. Deploy after destroy failed permanently. A successful destroy deletes the deployment record but leaves its ID in local state, so the next deploy's GetDeployment 404'd and the error was fatal — unrecoverable on retry. Treat a missing deployment as "create a new one"; any other read error stays fatal. 4. The overlay dropped depends_on. DMS does not record dependency edges, so replacing local state with DMS resources lost them, affecting delete ordering, the apply graph, and --select expansion. Carry depends_on over from the local entry. Masked by (1) until now. 5. Recording bypassed secret redaction. dstate.SaveState redacts bundle:"sensitive" fields before writing state, but the operation recorder marshalled raw, so a secret would be sent to DMS in plaintext and read back into local state. Route through structwalk.RedactSensitiveFields. Latent today: no resource state type carries a sensitive field yet. Adds acceptance coverage for deploy-destroy-deploy and for a bundle with no resources, both of which now exercise the read path (visible as GET .../resources in the recorded requests). Co-authored-by: Isaac --- .../bundle/dms/no-resources/databricks.yml | 5 + .../bundle/dms/no-resources/out.test.toml | 3 + acceptance/bundle/dms/no-resources/output.txt | 78 ++++++++++++++++ acceptance/bundle/dms/no-resources/script | 8 ++ .../dms/redeploy-after-destroy/databricks.yml | 10 ++ .../dms/redeploy-after-destroy/out.test.toml | 3 + .../dms/redeploy-after-destroy/output.txt | 92 +++++++++++++++++++ .../bundle/dms/redeploy-after-destroy/script | 15 +++ bundle/direct/dstate/dms.go | 14 ++- bundle/direct/dstate/dms_test.go | 74 +++++++++++++++ bundle/direct/dstate/state.go | 42 +++++++-- bundle/direct/dstate/state_test.go | 42 +++++++++ bundle/direct/oprecorder.go | 9 +- bundle/direct/oprecorder_test.go | 22 +++++ libs/dms/recorder.go | 44 +++++---- libs/dms/recorder_test.go | 38 ++++++++ libs/testserver/bundle.go | 44 ++++++--- libs/testserver/bundle_test.go | 51 ++++++++++ 18 files changed, 552 insertions(+), 42 deletions(-) create mode 100644 acceptance/bundle/dms/no-resources/databricks.yml create mode 100644 acceptance/bundle/dms/no-resources/out.test.toml create mode 100644 acceptance/bundle/dms/no-resources/output.txt create mode 100644 acceptance/bundle/dms/no-resources/script create mode 100644 acceptance/bundle/dms/redeploy-after-destroy/databricks.yml create mode 100644 acceptance/bundle/dms/redeploy-after-destroy/out.test.toml create mode 100644 acceptance/bundle/dms/redeploy-after-destroy/output.txt create mode 100644 acceptance/bundle/dms/redeploy-after-destroy/script create mode 100644 bundle/direct/dstate/dms_test.go create mode 100644 libs/testserver/bundle_test.go diff --git a/acceptance/bundle/dms/no-resources/databricks.yml b/acceptance/bundle/dms/no-resources/databricks.yml new file mode 100644 index 00000000000..78fad3a292e --- /dev/null +++ b/acceptance/bundle/dms/no-resources/databricks.yml @@ -0,0 +1,5 @@ +bundle: + name: dms-no-resources + +experimental: + record_deployment_history: true diff --git a/acceptance/bundle/dms/no-resources/out.test.toml b/acceptance/bundle/dms/no-resources/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/dms/no-resources/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt new file mode 100644 index 00000000000..e774fd7546c --- /dev/null +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -0,0 +1,78 @@ + +=== First deploy of a bundle with no resources: the deployment is created and its ID is persisted, even though no resource state was written +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //api/2.0/bundle --sort --get +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "target_name": "default" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "target_name": "default", + "version_type": "VERSION_TYPE_DEPLOY" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +>>> jq .deployment_id .databricks/bundle/default/resources.json +"[UUID]" + +=== Redeploy: the persisted ID is reused, so no second deployment is created +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //api/2.0/bundle --sort --get +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[UUID]" +} +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[UUID]" +} +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[UUID]/resources" +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "target_name": "default", + "version_type": "VERSION_TYPE_DEPLOY" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} diff --git a/acceptance/bundle/dms/no-resources/script b/acceptance/bundle/dms/no-resources/script new file mode 100644 index 00000000000..4bc97c5864d --- /dev/null +++ b/acceptance/bundle/dms/no-resources/script @@ -0,0 +1,8 @@ +title "First deploy of a bundle with no resources: the deployment is created and its ID is persisted, even though no resource state was written" +trace $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --sort --get +trace jq .deployment_id .databricks/bundle/default/resources.json + +title "Redeploy: the persisted ID is reused, so no second deployment is created" +trace $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --sort --get diff --git a/acceptance/bundle/dms/redeploy-after-destroy/databricks.yml b/acceptance/bundle/dms/redeploy-after-destroy/databricks.yml new file mode 100644 index 00000000000..8f79a2c0381 --- /dev/null +++ b/acceptance/bundle/dms/redeploy-after-destroy/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: dms-redeploy-after-destroy + +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/redeploy-after-destroy/out.test.toml b/acceptance/bundle/dms/redeploy-after-destroy/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/dms/redeploy-after-destroy/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/redeploy-after-destroy/output.txt b/acceptance/bundle/dms/redeploy-after-destroy/output.txt new file mode 100644 index 00000000000..5326653519a --- /dev/null +++ b/acceptance/bundle/dms/redeploy-after-destroy/output.txt @@ -0,0 +1,92 @@ + +=== Deploy, then destroy: the deployment record is deleted, but its ID stays behind in the local state file +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default + +Deleting files... +Destroy complete! + +>>> jq .deployment_id .databricks/bundle/default/resources.json +"[DESTROYED_DEPLOYMENT_ID]" + +=== Deploy again: the stale deployment ID no longer resolves, so a new deployment is created instead of failing the deploy +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //api/2.0/bundle --sort --get +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[DESTROYED_DEPLOYMENT_ID]" +} +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[DESTROYED_DEPLOYMENT_ID]" +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "target_name": "default" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "target_name": "default", + "version_type": "VERSION_TYPE_DEPLOY" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", + "q": { + "resource_key": "jobs.foo" + }, + "body": { + "action_type": "OPERATION_ACTION_TYPE_CREATE", + "resource_id": "[NUMID]", + "resource_key": "jobs.foo", + "state": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } + }, + "status": "OPERATION_STATUS_SUCCEEDED" + } +} + +=== The new deployment ID replaces the stale one in state +>>> jq .deployment_id .databricks/bundle/default/resources.json +"[UUID]" diff --git a/acceptance/bundle/dms/redeploy-after-destroy/script b/acceptance/bundle/dms/redeploy-after-destroy/script new file mode 100644 index 00000000000..628cd7bf494 --- /dev/null +++ b/acceptance/bundle/dms/redeploy-after-destroy/script @@ -0,0 +1,15 @@ +title "Deploy, then destroy: the deployment record is deleted, but its ID stays behind in the local state file" +trace $CLI bundle deploy +trace $CLI bundle destroy --auto-approve +print_requests.py //api/2.0/bundle --sort --get > /dev/null + +destroyed_id=$(jq -r .deployment_id .databricks/bundle/default/resources.json) +add_repl.py "$destroyed_id" DESTROYED_DEPLOYMENT_ID +trace jq .deployment_id .databricks/bundle/default/resources.json + +title "Deploy again: the stale deployment ID no longer resolves, so a new deployment is created instead of failing the deploy" +trace $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --sort --get + +title "The new deployment ID replaces the stale one in state" +trace jq .deployment_id .databricks/bundle/default/resources.json diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index 659d7c9dd0b..e09cb859221 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -35,7 +35,7 @@ func (db *DeploymentState) overlayDMSState(ctx context.Context, client bundledep return nil } - resources, err := fetchDeploymentResources(ctx, client, db.Data.DeploymentID) + resources, err := fetchDeploymentResources(ctx, client, db.Data.DeploymentID, db.Data.State) if err != nil { return err } @@ -91,7 +91,12 @@ func deploymentHasSuccessfulVersion(ctx context.Context, cfg *sdkconfig.Config, // fetchDeploymentResources lists every resource recorded for the deployment in // DMS and maps them into state entries keyed by the fully-qualified resource key. -func fetchDeploymentResources(ctx context.Context, client bundledeployments.BundleDeploymentsInterface, deploymentID string) (map[string]ResourceEntry, error) { +// +// DMS does not record dependency edges, so depends_on is carried over from the +// local state entry for the same key. It is derived from the local config on +// every deploy and is only consumed for delete ordering, so falling back to an +// empty list when the local state has no entry is safe. +func fetchDeploymentResources(ctx context.Context, client bundledeployments.BundleDeploymentsInterface, deploymentID string, local map[string]ResourceEntry) (map[string]ResourceEntry, error) { it := client.ListResources(ctx, bundledeployments.ListResourcesRequest{ Parent: "deployments/" + deploymentID, }) @@ -114,8 +119,9 @@ func fetchDeploymentResources(ctx context.Context, client bundledeployments.Bund } out[key] = ResourceEntry{ - ID: res.ResourceId, - State: state, + ID: res.ResourceId, + State: state, + DependsOn: local[key].DependsOn, } } return out, nil diff --git a/bundle/direct/dstate/dms_test.go b/bundle/direct/dstate/dms_test.go new file mode 100644 index 00000000000..df1084b9de7 --- /dev/null +++ b/bundle/direct/dstate/dms_test.go @@ -0,0 +1,74 @@ +package dstate + +import ( + "context" + "encoding/json" + "testing" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/databricks-sdk-go/listing" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeResourceLister serves a fixed set of resources from ListResources. It +// embeds the SDK interface so it satisfies it while only overriding the one +// method the read path uses. +type fakeResourceLister struct { + bundledeployments.BundleDeploymentsInterface + resources []bundledeployments.Resource +} + +func (f *fakeResourceLister) ListResources(ctx context.Context, req bundledeployments.ListResourcesRequest) listing.Iterator[bundledeployments.Resource] { + return listing.NewIterator( + &req, + func(ctx context.Context, r bundledeployments.ListResourcesRequest) (*bundledeployments.ListResourcesResponse, error) { + return &bundledeployments.ListResourcesResponse{Resources: f.resources}, nil + }, + func(resp *bundledeployments.ListResourcesResponse) []bundledeployments.Resource { + return resp.Resources + }, + func(resp *bundledeployments.ListResourcesResponse) *bundledeployments.ListResourcesRequest { + return nil + }, + ) +} + +func TestFetchDeploymentResourcesPreservesLocalDependsOn(t *testing.T) { + state := json.RawMessage(`{"name":"foo"}`) + f := &fakeResourceLister{resources: []bundledeployments.Resource{ + {ResourceKey: "jobs.foo", ResourceId: "123", State: &state}, + {ResourceKey: "pipelines.bar", ResourceId: "456"}, + }} + + dependsOn := []deployplan.DependsOnEntry{{Node: "resources.pipelines.bar", Label: "pipeline_id"}} + local := map[string]ResourceEntry{ + "resources.jobs.foo": {ID: "stale", DependsOn: dependsOn}, + } + + got, err := fetchDeploymentResources(t.Context(), f, "dep-1", local) + require.NoError(t, err) + + // DMS owns the ID and state, but it does not record dependency edges, so + // depends_on must survive from the local entry. Losing it breaks delete + // ordering and --select expansion. + assert.Equal(t, map[string]ResourceEntry{ + "resources.jobs.foo": {ID: "123", State: state, DependsOn: dependsOn}, + "resources.pipelines.bar": {ID: "456"}, + }, got) +} + +func TestFetchDeploymentResourcesWithNoLocalState(t *testing.T) { + f := &fakeResourceLister{resources: []bundledeployments.Resource{ + {ResourceKey: "jobs.foo", ResourceId: "123"}, + }} + + // A bundle whose local state was wiped has no entry to carry depends_on from; + // the resource is still recovered from DMS. + got, err := fetchDeploymentResources(t.Context(), f, "dep-1", nil) + require.NoError(t, err) + assert.Equal(t, map[string]ResourceEntry{ + "resources.jobs.foo": {ID: "123"}, + }, got) +} diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 2c969667c9e..f554af3c9b6 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -74,6 +74,11 @@ type DeploymentState struct { // Maps resource key to ID. Unlike Data.State, this is up to date during writes (deploys). stateIDs map[string]string + + // headerDirty records that a header field changed in memory during this + // deployment (today only the DMS deployment ID), so the state file must be + // written on Finalize even when the WAL carried no resource entries. + headerDirty bool } type Header struct { @@ -231,10 +236,18 @@ func (db *DeploymentState) GetDeploymentID() string { // server-generated ID, and persisted to the state file by Finalize. Storing it // on db.Data (not the WAL header, which is written before the ID is known) // means the subsequent state write carries it forward. +// +// The header is marked dirty so Finalize persists it even when the deploy wrote +// no resource entries; otherwise a bundle with no resources would mint a fresh +// deployment record on every deploy, leaking one orphan per run. func (db *DeploymentState) SetDeploymentID(id string) { db.mu.Lock() defer db.mu.Unlock() + if db.Data.DeploymentID == id { + return + } db.Data.DeploymentID = id + db.headerDirty = true } type ( @@ -353,7 +366,7 @@ func (db *DeploymentState) OpenWithData(path string, data Database) { func (db *DeploymentState) replayWAL(ctx context.Context) error { walPath := db.Path + walSuffix - hasEntries, err := db.mergeWalIntoState(ctx) + persist, err := db.mergeWalIntoState(ctx) if err != nil { if errors.Is(err, errStaleWAL) { log.Debugf(ctx, "Deleting stale WAL file %s", walPath) @@ -362,7 +375,7 @@ func (db *DeploymentState) replayWAL(ctx context.Context) error { } return fmt.Errorf("WAL recovery failed: %w", err) } - if hasEntries { + if persist { if err := db.unlockedSave(); err != nil { return err } @@ -373,6 +386,9 @@ func (db *DeploymentState) replayWAL(ctx context.Context) error { return nil } +// mergeWalIntoState replays the WAL into db.Data and reports whether the caller +// must persist the state file: either the WAL carried resource entries, or a +// header field changed in memory during this deployment. func (db *DeploymentState) mergeWalIntoState(ctx context.Context) (bool, error) { if db.walFile != nil { panic("internal error: walFile must be closed") @@ -450,17 +466,23 @@ func (db *DeploymentState) mergeWalIntoState(ctx context.Context) (bool, error) hasEntries := lineNumber > 1 - // Only advance the serial when the WAL carried entries, because the caller - // (replayWAL) persists the new state file only in that case. A header-only - // WAL is a deploy that started but committed nothing; advancing the serial - // for it leaves the in-memory serial ahead of the persisted one, so the - // next deploy writes its WAL header at serial+2 and recovery rejects it as - // "ahead of expected". See acceptance/bundle/deploy/wal/header-only-wal. - if hasEntries { + // A header-only WAL still has to be persisted when a header field changed in + // memory during this deployment (the DMS deployment ID): dropping the write + // would lose the ID and make the next deploy create a second deployment. + persist := hasEntries || db.headerDirty + + // Only advance the serial when the state file is actually written, because + // the caller (replayWAL) persists it only in that case. A header-only WAL + // that changed nothing is a deploy that started but committed nothing; + // advancing the serial for it leaves the in-memory serial ahead of the + // persisted one, so the next deploy writes its WAL header at serial+2 and + // recovery rejects it as "ahead of expected". + // See acceptance/bundle/deploy/wal/header-only-wal. + if persist { db.Data.Serial = newSerial } - return hasEntries, nil + return persist, nil } // Finalize replays the WAL (if open for write), captures the resulting state, and resets. diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index 16066bf81f8..6530ca049d9 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -63,6 +63,48 @@ func TestFinalizeWithNoEntriesDoesNotWriteStateFile(t *testing.T) { assert.ErrorIs(t, err, os.ErrNotExist) } +func TestDeploymentIDPersistsWithNoResourceEntries(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + + // A bundle with no resources writes no WAL entries, but the deployment ID + // still has to be persisted: otherwise the next deploy sees no ID and creates + // a second deployment record, leaking one per deploy. + var db DeploymentState + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) + db.SetDeploymentID("server-assigned-id") + mustFinalize(t, &db) + + var reopened DeploymentState + require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, nil)) + assert.Equal(t, "server-assigned-id", reopened.GetDeploymentID()) + assert.Equal(t, 1, reopened.Data.Serial) + mustFinalize(t, &reopened) +} + +func TestSetDeploymentIDToSameValueDoesNotWriteStateFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + + var db DeploymentState + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) + db.SetDeploymentID("server-assigned-id") + require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) + mustFinalize(t, &db) + + before, err := os.ReadFile(path) + require.NoError(t, err) + + // Re-setting the same ID is not a header change, so a deploy that commits + // nothing must not bump the serial (see mergeWalIntoState). + var reopened DeploymentState + require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(true), nil, nil)) + reopened.SetDeploymentID("server-assigned-id") + mustFinalize(t, &reopened) + + after, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, string(before), string(after)) +} + func TestExportStateFromDataJobRunJobID(t *testing.T) { data := Database{ State: map[string]ResourceEntry{ diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index 467f8ac648c..033aefb5f2f 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -7,6 +7,8 @@ import ( "strings" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structwalk" "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) @@ -67,8 +69,13 @@ func (r *operationRecorder) record(ctx context.Context, resourceKey string, acti // The DMS Operation.State field carries the serialized config so the backend // can serve it as resource state. It is intentionally left unset for delete, // where the resource no longer exists. + // + // Redact sensitive fields, matching what dstate.SaveState writes to the local + // state file: DMS state is read back as resource state, so recording secrets + // in plaintext would both leak them to the service and reintroduce them into + // a local state file via the read path. if state != nil { - raw, err := json.Marshal(state) + raw, err := structwalk.RedactSensitiveFields(state, dyn.SensitiveValueRedacted) if err != nil { return fmt.Errorf("serializing state: %w", err) } diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go index 56d860de3e6..4c1bbcacda6 100644 --- a/bundle/direct/oprecorder_test.go +++ b/bundle/direct/oprecorder_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/dyn" "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -52,6 +53,27 @@ func TestOperationRecorderDeleteHasNoState(t *testing.T) { assert.Nil(t, f.requests[0].Operation.State) } +func TestOperationRecorderRedactsSensitiveFields(t *testing.T) { + f := &fakeOpClient{} + r := NewOperationRecorder(f, "dep-1", 2) + + state := struct { + Name string `json:"name"` + Token string `json:"token" bundle:"sensitive"` + }{Name: "foo", Token: "super-secret"} + + err := r.record(t.Context(), "resources.jobs.foo", deployplan.Create, "job-123", state) + require.NoError(t, err) + + require.Len(t, f.requests, 1) + require.NotNil(t, f.requests[0].Operation.State) + // Sensitive fields are redacted before leaving the CLI, matching what + // dstate.SaveState writes to the local state file. + assert.JSONEq(t, + `{"name":"foo","token":"`+dyn.SensitiveValueRedacted+`"}`, + string(*f.requests[0].Operation.State)) +} + func TestDeployActionToSDK(t *testing.T) { cases := []struct { action deployplan.ActionType diff --git a/libs/dms/recorder.go b/libs/dms/recorder.go index eed8485f2c2..0d9563dd052 100644 --- a/libs/dms/recorder.go +++ b/libs/dms/recorder.go @@ -149,10 +149,35 @@ func (r *Recorder) CompleteVersion(ctx context.Context, success bool) error { } // createDeploymentVersion ensures the deployment record exists, then creates a -// new version under it. On a first deploy (no stored deployment ID) it creates -// the deployment and lets the server assign the ID; otherwise it reads the -// existing deployment to compute the next version number. +// new version under it. When no deployment ID is stored, or the stored one no +// longer exists in DMS, it creates the deployment and lets the server assign the +// ID; otherwise it reads the existing deployment to compute the next version +// number. func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID string, err error) { + if r.deploymentID != "" { + // Existing deployment: read it to compute the next version number. + dep, getErr := r.svc.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ + Name: "deployments/" + r.deploymentID, + }) + switch { + case getErr == nil: + lastVersion, parseErr := strconv.ParseInt(dep.LastVersionId, 10, 64) + if parseErr != nil { + return "", fmt.Errorf("failed to parse last_version_id %q: %w", dep.LastVersionId, parseErr) + } + versionID = strconv.FormatInt(lastVersion+1, 10) + case errors.Is(getErr, apierr.ErrNotFound): + // The record the state points at is gone: a successful destroy deletes + // it (leaving the ID behind in the local state file), and it can also be + // deleted out of band. Recording must not dead-end on it, so fall back to + // creating a new deployment; the caller persists the new ID. + log.Debugf(ctx, "Deployment %s no longer exists in the deployment metadata service, creating a new one", r.deploymentID) + r.deploymentID = "" + default: + return "", fmt.Errorf("failed to get deployment: %w", getErr) + } + } + if r.deploymentID == "" { // First deploy: create the deployment with an empty ID so the server // assigns one, then start at version 1. @@ -170,19 +195,6 @@ func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID strin } r.deploymentID = id versionID = "1" - } else { - // Existing deployment: read it to compute the next version number. - dep, getErr := r.svc.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ - Name: "deployments/" + r.deploymentID, - }) - if getErr != nil { - return "", fmt.Errorf("failed to get deployment: %w", getErr) - } - lastVersion, parseErr := strconv.ParseInt(dep.LastVersionId, 10, 64) - if parseErr != nil { - return "", fmt.Errorf("failed to parse last_version_id %q: %w", dep.LastVersionId, parseErr) - } - versionID = strconv.FormatInt(lastVersion+1, 10) } // The server validates that versionID equals last_version_id + 1 and returns diff --git a/libs/dms/recorder_test.go b/libs/dms/recorder_test.go index 91848f74a70..9b60078635f 100644 --- a/libs/dms/recorder_test.go +++ b/libs/dms/recorder_test.go @@ -2,8 +2,11 @@ package dms import ( "context" + "errors" + "fmt" "testing" + "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -104,6 +107,41 @@ func TestRecorderSubsequentDeployReusesDeploymentAndIncrementsVersion(t *testing assert.Equal(t, "stored-id", r.DeploymentID()) } +func TestRecorderStaleDeploymentIDCreatesNewDeployment(t *testing.T) { + f := &fakeDMS{ + assignedID: "fresh-id", + getDeployment: func(id string) (*bundledeployments.Deployment, error) { + return nil, fmt.Errorf("deployment %s: %w", id, apierr.ErrNotFound) + }, + } + // A deploy after a destroy still has the destroyed deployment's ID in state, + // but the record is gone. Recording must recover rather than fail the deploy. + r := NewRecorder(f, "destroyed-id", "dev", VersionTypeDeploy) + + require.NoError(t, r.CreateVersion(t.Context())) + + require.Len(t, f.created, 1) + assert.Equal(t, "fresh-id", r.DeploymentID()) + require.Len(t, f.versions, 1) + assert.Equal(t, "1", f.versions[0].VersionId) + assert.Equal(t, "deployments/fresh-id", f.versions[0].Parent) +} + +func TestRecorderGetDeploymentErrorFailsDeploy(t *testing.T) { + f := &fakeDMS{ + getDeployment: func(id string) (*bundledeployments.Deployment, error) { + return nil, errors.New("boom") + }, + } + r := NewRecorder(f, "stored-id", "dev", VersionTypeDeploy) + + // Only a missing deployment is recovered from; any other read failure is fatal + // rather than silently forking a second deployment record. + err := r.CreateVersion(t.Context()) + assert.ErrorContains(t, err, "failed to get deployment") + assert.Empty(t, f.created) +} + func TestRecorderDestroyDeletesDeploymentOnSuccess(t *testing.T) { f := &fakeDMS{ getDeployment: func(id string) (*bundledeployments.Deployment, error) { diff --git a/libs/testserver/bundle.go b/libs/testserver/bundle.go index 34003a507a5..a1b0cba24a9 100644 --- a/libs/testserver/bundle.go +++ b/libs/testserver/bundle.go @@ -1,6 +1,7 @@ package testserver import ( + "bytes" "encoding/json" "slices" "strconv" @@ -61,17 +62,38 @@ func (s *FakeWorkspace) GetDeployment(deploymentID string) Response { return dmsNotFound("deployment " + deploymentID) } - // The SDK Deployment struct does not yet carry last_successful_version_id - // (still stage:DEVELOPMENT, so stripped from generation), but the read path - // reads it off the raw JSON. Serve it as an extra field alongside the typed - // deployment so the overlay behaves as it will against the real server. - return Response{Body: struct { - bundledeployments.Deployment - LastSuccessfulVersionID string `json:"last_successful_version_id,omitempty"` - }{ - Deployment: d.deployment, - LastSuccessfulVersionID: d.lastSuccessfulVersionID, - }} + body, err := deploymentBody(d) + if err != nil { + return Response{StatusCode: 500, Body: map[string]string{"message": err.Error()}} + } + return Response{Body: body} +} + +// deploymentBody renders a deployment the way the real server does: the typed +// fields plus last_successful_version_id, which the generated SDK struct does +// not carry yet (still stage:DEVELOPMENT) but the read path reads off the raw +// JSON. +// +// The extra field cannot be added by embedding Deployment in a wrapper struct: +// Deployment has its own MarshalJSON, which is promoted to the wrapper and +// silently drops any sibling field. +func deploymentBody(d *dmsDeployment) (map[string]any, error) { + raw, err := json.Marshal(d.deployment) + if err != nil { + return nil, err + } + + var body map[string]any + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.UseNumber() + if err := dec.Decode(&body); err != nil { + return nil, err + } + + if d.lastSuccessfulVersionID != "" { + body["last_successful_version_id"] = d.lastSuccessfulVersionID + } + return body, nil } func (s *FakeWorkspace) DeleteDeployment(deploymentID string) Response { diff --git a/libs/testserver/bundle_test.go b/libs/testserver/bundle_test.go new file mode 100644 index 00000000000..4d28624ba3e --- /dev/null +++ b/libs/testserver/bundle_test.go @@ -0,0 +1,51 @@ +package testserver + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestDeploymentBodyKeepsTypedFieldsAndLastSuccessfulVersionID guards against +// serializing the deployment through a struct that embeds +// bundledeployments.Deployment: Deployment has its own MarshalJSON, which is +// promoted to the embedding struct and silently drops last_successful_version_id. +// The CLI read path treats a missing value as "DMS does not own the state", so +// losing the field here makes the whole overlay path untestable. +func TestDeploymentBodyKeepsTypedFieldsAndLastSuccessfulVersionID(t *testing.T) { + d := &dmsDeployment{lastSuccessfulVersionID: "2"} + d.deployment.Name = "deployments/abc" + d.deployment.LastVersionId = "3" + d.deployment.TargetName = "default" + + body, err := deploymentBody(d) + require.NoError(t, err) + + assert.Equal(t, "deployments/abc", body["name"]) + assert.Equal(t, "3", body["last_version_id"]) + assert.Equal(t, "default", body["target_name"]) + assert.Equal(t, "2", body["last_successful_version_id"]) + + // The response must round-trip as JSON the same way, since that is what the + // client actually reads. + raw, err := json.Marshal(body) + require.NoError(t, err) + assert.JSONEq(t, + `{"name":"deployments/abc","last_version_id":"3","target_name":"default","last_successful_version_id":"2"}`, + string(raw)) +} + +// TestDeploymentBodyOmitsUnsetLastSuccessfulVersionID checks that a deployment +// with no successful version does not advertise one: the read path must keep +// using the local state file in that case. +func TestDeploymentBodyOmitsUnsetLastSuccessfulVersionID(t *testing.T) { + d := &dmsDeployment{} + d.deployment.Name = "deployments/abc" + + body, err := deploymentBody(d) + require.NoError(t, err) + + assert.NotContains(t, body, "last_successful_version_id") +} From 38dc0823b4e97cf6caac784d8041f8a35339b51e Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 27 Jul 2026 13:09:47 +0000 Subject: [PATCH 04/28] bundle: upload DMS operations asynchronously Recording an operation with the deployment metadata service used to happen inline on the apply worker, so every resource paid a CreateOperation round trip before the worker moved on to the next one. Queue the operations instead and upload them from a small pool of background workers (operationQueue, in the new opqueue.go). The queue holds resource keys rather than operations, so an operation recorded for a resource that is still waiting replaces the queued one: DMS keeps one state per resource key, so the later operation supersedes the earlier one and a single request records both. This is best effort - only operations that no worker has picked up yet are coalesced. Uploads are not fire-and-forget. Apply drains the queue before returning and reports the first failure, because a version that completes successfully makes DMS authoritative for resource state; dropping an operation would leave DMS with an incomplete resource set and the next deploy would plan to create resources that already exist. At most one upload per resource key runs at a time, so the last operation recorded for a resource is also the last one the service sees. Co-authored-by: Isaac --- .../dms/multiple-resources/databricks.yml | 18 ++ .../dms/multiple-resources/out.test.toml | 3 + .../bundle/dms/multiple-resources/output.txt | 25 +++ .../bundle/dms/multiple-resources/script | 7 + bundle/direct/bundle_apply.go | 16 +- bundle/direct/opqueue.go | 192 ++++++++++++++++++ bundle/direct/opqueue_test.go | 192 ++++++++++++++++++ bundle/direct/oprecorder.go | 103 ++++++---- bundle/direct/oprecorder_test.go | 42 ++-- bundle/direct/pkg.go | 6 +- 10 files changed, 537 insertions(+), 67 deletions(-) create mode 100644 acceptance/bundle/dms/multiple-resources/databricks.yml create mode 100644 acceptance/bundle/dms/multiple-resources/out.test.toml create mode 100644 acceptance/bundle/dms/multiple-resources/output.txt create mode 100644 acceptance/bundle/dms/multiple-resources/script create mode 100644 bundle/direct/opqueue.go create mode 100644 bundle/direct/opqueue_test.go diff --git a/acceptance/bundle/dms/multiple-resources/databricks.yml b/acceptance/bundle/dms/multiple-resources/databricks.yml new file mode 100644 index 00000000000..30f2f433b81 --- /dev/null +++ b/acceptance/bundle/dms/multiple-resources/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: dms-multiple-resources + +experimental: + record_deployment_history: true + +resources: + jobs: + one: + name: one + two: + name: two + three: + name: three + four: + name: four + five: + name: five diff --git a/acceptance/bundle/dms/multiple-resources/out.test.toml b/acceptance/bundle/dms/multiple-resources/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/dms/multiple-resources/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt new file mode 100644 index 00000000000..4bacec0c317 --- /dev/null +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -0,0 +1,25 @@ + +=== Deploy several resources: operations are uploaded from background workers, so exactly one is recorded per resource no matter what order the uploads finish in. The serialized state is dropped from the output here; bundle/dms/record covers it. +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //versions/1/operations --sort --del-body state --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.five"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.five", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.four"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.four", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.three"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.three", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.two"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.two", "status": "OPERATION_STATUS_SUCCEEDED"}} + +=== Redeploy with no changes: nothing is applied, so no operations are recorded and only the version is opened and completed +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //api/2.0/bundle --sort --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/multiple-resources/script b/acceptance/bundle/dms/multiple-resources/script new file mode 100644 index 00000000000..e9b9339607e --- /dev/null +++ b/acceptance/bundle/dms/multiple-resources/script @@ -0,0 +1,7 @@ +title "Deploy several resources: operations are uploaded from background workers, so exactly one is recorded per resource no matter what order the uploads finish in. The serialized state is dropped from the output here; bundle/dms/record covers it." +trace $CLI bundle deploy +trace print_requests.py //versions/1/operations --sort --del-body state --oneline + +title "Redeploy with no changes: nothing is applied, so no operations are recorded and only the version is opened and completed" +trace $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --sort --oneline diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index afef2367e5b..b26861128b7 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -34,6 +34,11 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return } + // Operations are recorded with DMS from background workers so a resource's + // deploy is not held up by the CreateOperation round trip. The queue is + // drained below, once every apply worker has finished recording. + opQueue := newOperationQueue(ctx, b.OpRec) + g.Run(defaultParallelism, func(resourceKey string, failedDependency *string) bool { entry, err := plan.WriteLockEntry(resourceKey) if err != nil { @@ -89,7 +94,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return false } // Record the delete with DMS. State is nil: the resource is gone. - if err := b.recordOperation(ctx, resourceKey, action, "", nil); err != nil { + if err := opQueue.record(ctx, resourceKey, action, "", nil); err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } @@ -125,7 +130,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa // Record the operation with DMS. The resource ID and applied config // (sv.Value) come from the write just performed; GetResourceID reads // the ID assigned by Deploy. - if err := b.recordOperation(ctx, resourceKey, action, b.StateDB.GetResourceID(resourceKey), sv.Value); err != nil { + if err := opQueue.record(ctx, resourceKey, action, b.StateDB.GetResourceID(resourceKey), sv.Value); err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } @@ -152,6 +157,13 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return true }) + + // Wait for the queued operations before returning: the caller completes the + // DMS version right after, and a version must not be completed with uploads + // still in flight. + if err := opQueue.close(); err != nil { + logdiag.LogError(ctx, err) + } } func (b *DeploymentBundle) LookupReferencePostDeploy(ctx context.Context, path *structpath.PathNode) (any, error) { diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go new file mode 100644 index 00000000000..0804ad2d5b3 --- /dev/null +++ b/bundle/direct/opqueue.go @@ -0,0 +1,192 @@ +package direct + +import ( + "context" + "fmt" + "sync" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/log" +) + +const ( + // operationQueueSize bounds how many recorded operations wait for upload. + // Apply deploys at most defaultParallelism resources at a time, so a queue + // this deep means an apply worker practically never blocks on a free slot. + operationQueueSize = 10 + + // operationUploadWorkers is how many uploads run at a time. It is below + // operationQueueSize so a burst of operations is absorbed by the queue rather + // than by one request per resource. + operationUploadWorkers = 4 +) + +// operationQueue uploads recorded operations from background workers, so an apply +// worker does not wait for the CreateOperation round trip before moving on to the +// next resource. +// +// It guarantees at most one upload in flight per resource key: within a key the +// worker that owns it uploads sequentially, so the last operation recorded for a +// resource is also the last one the service sees. +// +// Uploads are not fire-and-forget: close drains the queue and returns the first +// failure, which fails the deploy. That matters because a successfully completed +// version makes DMS the source of truth for resource state (see +// dstate.overlayDMSState); silently dropping an operation would leave DMS with an +// incomplete resource set, and the next deploy would plan to create resources +// that already exist. +type operationQueue struct { + uploader operationUploader + + // queue carries resource keys, not the operations themselves: a worker looks + // the operation up in pending when it picks the key up, which is what lets + // record collapse repeated writes to the same resource. + queue chan string + wg sync.WaitGroup + + // mu guards the fields below. + mu sync.Mutex + + // pending is the latest operation recorded per resource key that no worker has + // picked up yet. + pending map[string]recordedOperation + + // inflight holds the resource keys a worker currently owns. A key that is + // in flight is not queued again: the owning worker re-checks pending after its + // upload and picks up anything recorded in the meantime. + inflight map[string]bool + + err error + closed bool +} + +// newOperationQueue starts the upload workers. It returns nil when uploader is +// nil (recording disabled), and every method is a no-op on a nil queue so callers +// do not have to branch. +// +// ctx is used for the uploads, so it must stay valid until close returns. +func newOperationQueue(ctx context.Context, uploader operationUploader) *operationQueue { + if uploader == nil { + return nil + } + + q := &operationQueue{ + uploader: uploader, + queue: make(chan string, operationQueueSize), + pending: make(map[string]recordedOperation), + inflight: make(map[string]bool), + } + + q.wg.Add(operationUploadWorkers) + for range operationUploadWorkers { + go q.work(ctx) + } + + return q +} + +// record serializes an operation and queues it for upload. It performs no API +// call, so upload failures surface from close rather than here; the error +// returned is only about turning the applied resource into a payload. +// +// When an operation for the same resource is already waiting it is replaced +// instead of queued again: DMS keeps one state per resource key, so the later +// operation supersedes the earlier one and a single upload records both. This is +// best effort - only operations that have not been picked up yet are collapsed. +func (q *operationQueue) record(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state any) error { + if q == nil { + return nil + } + + op, err := newRecordedOperation(action, resourceID, state) + if err != nil { + return err + } + + q.mu.Lock() + _, waiting := q.pending[resourceKey] + owned := waiting || q.inflight[resourceKey] + q.pending[resourceKey] = op + q.mu.Unlock() + + if owned { + log.Debugf(ctx, "Coalescing queued deployment operation for %s", resourceKey) + return nil + } + + q.queue <- resourceKey + return nil +} + +// close drains the queue and returns the first upload error. All callers of +// record must have returned first: record on a closed queue panics. Calling close +// more than once is safe, so callers can defer it and still check the error at a +// specific point. +func (q *operationQueue) close() error { + if q == nil { + return nil + } + + q.mu.Lock() + closed := q.closed + q.closed = true + q.mu.Unlock() + + if !closed { + close(q.queue) + q.wg.Wait() + } + + q.mu.Lock() + defer q.mu.Unlock() + return q.err +} + +func (q *operationQueue) work(ctx context.Context) { + defer q.wg.Done() + + for resourceKey := range q.queue { + // Keep uploading this key until nothing new was recorded for it, instead of + // putting it back on the queue: a worker sending to the channel it consumes + // from can deadlock once the queue is full. + for { + op, ok := q.take(resourceKey) + if !ok { + break + } + + if err := q.uploader.upload(ctx, resourceKey, op); err != nil { + q.setErr(fmt.Errorf("recording operation for %s with the deployment metadata service: %w", resourceKey, err)) + } + } + } +} + +// take claims the operation waiting for resourceKey, marking the key in flight so +// record does not queue it a second time. It reports false, and releases the key, +// when nothing is waiting. +func (q *operationQueue) take(resourceKey string) (recordedOperation, bool) { + q.mu.Lock() + defer q.mu.Unlock() + + op, ok := q.pending[resourceKey] + if !ok { + delete(q.inflight, resourceKey) + return recordedOperation{}, false + } + + delete(q.pending, resourceKey) + q.inflight[resourceKey] = true + return op, true +} + +// setErr keeps the first upload error; later ones are dropped because one failure +// is enough to fail the deploy. +func (q *operationQueue) setErr(err error) { + q.mu.Lock() + defer q.mu.Unlock() + + if q.err == nil { + q.err = err + } +} diff --git a/bundle/direct/opqueue_test.go b/bundle/direct/opqueue_test.go new file mode 100644 index 00000000000..5b267c4d8fa --- /dev/null +++ b/bundle/direct/opqueue_test.go @@ -0,0 +1,192 @@ +package direct + +import ( + "context" + "errors" + "strconv" + "sync" + "testing" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeUploader records the uploads it receives and optionally blocks until +// release is closed, so a test can hold operations in the queue and observe +// coalescing. +type fakeUploader struct { + block chan struct{} + started chan string + err error + + mu sync.Mutex + uploads []string +} + +func (f *fakeUploader) upload(ctx context.Context, resourceKey string, op recordedOperation) error { + if f.started != nil { + f.started <- resourceKey + } + if f.block != nil { + <-f.block + } + + f.mu.Lock() + defer f.mu.Unlock() + f.uploads = append(f.uploads, resourceKey+"="+string(op.state)) + return f.err +} + +func (f *fakeUploader) recorded() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.uploads...) +} + +func recordState(t *testing.T, q *operationQueue, resourceKey, name string) { + t.Helper() + require.NoError(t, q.record(t.Context(), resourceKey, deployplan.Update, "id-1", map[string]string{"name": name})) +} + +func TestOperationQueueUploadsEachOperation(t *testing.T) { + f := &fakeUploader{} + q := newOperationQueue(t.Context(), f) + + for i := range 20 { + recordState(t, q, "resources.jobs.job"+strconv.Itoa(i), "n") + } + require.NoError(t, q.close()) + + assert.Len(t, f.recorded(), 20) +} + +func TestOperationQueueCoalescesQueuedOperationsForSameResource(t *testing.T) { + // Hold the first upload so later operations for the same resource pile up in + // the queue and are collapsed into one. + f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 1)} + q := newOperationQueue(t.Context(), f) + + recordState(t, q, "resources.jobs.foo", "v1") + // Wait until a worker owns the key, so the operations below are queued behind + // an in-flight upload rather than racing it. + assert.Equal(t, "resources.jobs.foo", <-f.started) + + recordState(t, q, "resources.jobs.foo", "v2") + recordState(t, q, "resources.jobs.foo", "v3") + + close(f.block) + require.NoError(t, q.close()) + + // Two uploads, not three: v2 was superseded by v3 while both were queued, and + // the last recorded state is the one the service ends up with. + assert.Equal(t, []string{ + `resources.jobs.foo={"name":"v1"}`, + `resources.jobs.foo={"name":"v3"}`, + }, f.recorded()) +} + +func TestOperationQueueReturnsUploadError(t *testing.T) { + uploadErr := errors.New("boom") + f := &fakeUploader{err: uploadErr} + q := newOperationQueue(t.Context(), f) + + recordState(t, q, "resources.jobs.foo", "v1") + + err := q.close() + require.Error(t, err) + assert.ErrorIs(t, err, uploadErr) + assert.Contains(t, err.Error(), "resources.jobs.foo") +} + +func TestOperationQueueRecordRejectsUnsupportedAction(t *testing.T) { + f := &fakeUploader{} + q := newOperationQueue(t.Context(), f) + + // Serialization failures surface at record time, on the resource that caused + // them, rather than from the drain at the end of apply. + err := q.record(t.Context(), "resources.jobs.foo", deployplan.Skip, "id-1", nil) + require.Error(t, err) + + require.NoError(t, q.close()) + assert.Empty(t, f.recorded()) +} + +func TestOperationQueueCloseIsIdempotent(t *testing.T) { + f := &fakeUploader{err: errors.New("boom")} + q := newOperationQueue(t.Context(), f) + + recordState(t, q, "resources.jobs.foo", "v1") + + require.Error(t, q.close()) + // A second close reports the same error instead of panicking on the already + // closed channel, so callers can both defer close and check it explicitly. + require.Error(t, q.close()) +} + +// serialUploader fails if two uploads for the same resource key ever overlap. +type serialUploader struct { + mu sync.Mutex + live map[string]bool + last map[string]string + uneven bool +} + +func (s *serialUploader) upload(ctx context.Context, resourceKey string, op recordedOperation) error { + s.mu.Lock() + if s.live[resourceKey] { + s.uneven = true + } + s.live[resourceKey] = true + s.mu.Unlock() + + s.mu.Lock() + defer s.mu.Unlock() + s.live[resourceKey] = false + s.last[resourceKey] = string(op.state) + return nil +} + +func TestOperationQueueUploadsOneResourceAtATime(t *testing.T) { + // Concurrent apply workers repeatedly record overlapping resource keys, the + // case where a coalesced key can be handed to a second worker while the first + // is still uploading it. The service keeps one state per key, so overlapping + // uploads for a key could land out of order and leave a stale state behind. + const ( + workers = 10 + perWorker = 5 + distinctKeyMod = 12 + ) + + u := &serialUploader{live: map[string]bool{}, last: map[string]string{}} + q := newOperationQueue(t.Context(), u) + + var wg sync.WaitGroup + for w := range workers { + wg.Add(1) + go func() { + defer wg.Done() + for i := range perWorker { + key := "resources.jobs.job" + strconv.Itoa((w*perWorker+i)%distinctKeyMod) + recordState(t, q, key, strconv.Itoa(w)) + } + }() + } + wg.Wait() + require.NoError(t, q.close()) + + assert.False(t, u.uneven, "two uploads overlapped for the same resource key") + // Every distinct key was recorded, and close drained all of them. + assert.Len(t, u.last, distinctKeyMod) + assert.Empty(t, q.pending) + assert.Empty(t, q.inflight) +} + +func TestNilOperationQueueIsNoOp(t *testing.T) { + // Recording is disabled: newOperationQueue returns nil and every method is a + // no-op, so Apply does not have to branch. + q := newOperationQueue(t.Context(), nil) + require.Nil(t, q) + require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", nil)) + require.NoError(t, q.close()) +} diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index 033aefb5f2f..cdd99966f15 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -12,25 +12,57 @@ import ( "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) -// opRecorder records a resource operation with the deployment metadata service -// (DMS) after it has been applied to the workspace. state is the serialized -// local config after the operation and must be nil for delete operations. -type opRecorder interface { - record(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state any) error +// recordedOperation is an applied resource operation, serialized and waiting to be +// uploaded to the deployment metadata service (DMS). +// +// The payload is built on the apply worker rather than in the uploader so the +// queue does not hold on to the live resource struct, and so a malformed state +// fails the resource that produced it instead of the drain at the end of apply. +type recordedOperation struct { + action bundledeployments.OperationActionType + resourceID string + + // state is the serialized local config after the operation. It is nil for a + // delete, where the resource no longer exists. + state json.RawMessage } -// recordOperation reports an applied resource operation to DMS. It is a no-op -// unless the bundle opted into recording deployment history (OpRec is set). -// state is the serialized local config after the operation and must be nil for -// delete operations. -func (b *DeploymentBundle) recordOperation(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state any) error { - if b.OpRec == nil { - return nil +// newRecordedOperation serializes an applied operation for upload. state is the +// local config after the operation and must be nil for delete operations. +func newRecordedOperation(action deployplan.ActionType, resourceID string, state any) (recordedOperation, error) { + actionType, err := deployActionToSDK(action) + if err != nil { + return recordedOperation{}, err + } + + op := recordedOperation{action: actionType, resourceID: resourceID} + + // The DMS Operation.State field carries the serialized config so the backend + // can serve it as resource state. It is intentionally left unset for delete, + // where the resource no longer exists. + // + // Redact sensitive fields, matching what dstate.SaveState writes to the local + // state file: DMS state is read back as resource state, so recording secrets + // in plaintext would both leak them to the service and reintroduce them into + // a local state file via the read path. + if state != nil { + raw, err := structwalk.RedactSensitiveFields(state, dyn.SensitiveValueRedacted) + if err != nil { + return recordedOperation{}, fmt.Errorf("serializing state: %w", err) + } + op.state = raw } - return b.OpRec.record(ctx, resourceKey, action, resourceID, state) + + return op, nil +} + +// operationUploader records an applied resource operation with DMS. Uploads run +// on the operationQueue workers, off the apply path. +type operationUploader interface { + upload(ctx context.Context, resourceKey string, op recordedOperation) error } -// operationRecorder records operations via the DMS CreateOperation API. +// operationRecorder uploads operations via the DMS CreateOperation API. type operationRecorder struct { client bundledeployments.BundleDeploymentsInterface // parent is the version the operations are recorded under, formatted as @@ -38,55 +70,36 @@ type operationRecorder struct { parent string } -// NewOperationRecorder returns an opRecorder backed by the DMS CreateOperation -// API. deploymentID and version identify the deployment version assigned by DMS -// that the operations are recorded under. -func NewOperationRecorder(client bundledeployments.BundleDeploymentsInterface, deploymentID string, version int64) opRecorder { +// NewOperationRecorder returns an operationUploader backed by the DMS +// CreateOperation API. deploymentID and version identify the deployment version +// assigned by DMS that the operations are recorded under. +func NewOperationRecorder(client bundledeployments.BundleDeploymentsInterface, deploymentID string, version int64) operationUploader { return &operationRecorder{ client: client, parent: fmt.Sprintf("deployments/%s/versions/%d", deploymentID, version), } } -func (r *operationRecorder) record(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state any) error { - actionType, err := deployActionToSDK(action) - if err != nil { - return err - } - +func (r *operationRecorder) upload(ctx context.Context, resourceKey string, op recordedOperation) error { // DMS resource keys are unprefixed (e.g. "jobs.foo"), while the CLI's state // keys carry a leading "resources." (e.g. "resources.jobs.foo"). Strip it on // the way out; the read path re-adds it (see dstate.fetchDeploymentResources). dmsKey := strings.TrimPrefix(resourceKey, "resources.") - op := bundledeployments.Operation{ - ActionType: actionType, - ResourceId: resourceID, + operation := bundledeployments.Operation{ + ActionType: op.action, + ResourceId: op.resourceID, ResourceKey: dmsKey, Status: bundledeployments.OperationStatusOperationStatusSucceeded, } - - // The DMS Operation.State field carries the serialized config so the backend - // can serve it as resource state. It is intentionally left unset for delete, - // where the resource no longer exists. - // - // Redact sensitive fields, matching what dstate.SaveState writes to the local - // state file: DMS state is read back as resource state, so recording secrets - // in plaintext would both leak them to the service and reintroduce them into - // a local state file via the read path. - if state != nil { - raw, err := structwalk.RedactSensitiveFields(state, dyn.SensitiveValueRedacted) - if err != nil { - return fmt.Errorf("serializing state: %w", err) - } - msg := json.RawMessage(raw) - op.State = &msg + if op.state != nil { + operation.State = &op.state } - _, err = r.client.CreateOperation(ctx, bundledeployments.CreateOperationRequest{ + _, err := r.client.CreateOperation(ctx, bundledeployments.CreateOperationRequest{ Parent: r.parent, ResourceKey: dmsKey, - Operation: op, + Operation: operation, }) return err } diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go index 4c1bbcacda6..aaf3243ec60 100644 --- a/bundle/direct/oprecorder_test.go +++ b/bundle/direct/oprecorder_test.go @@ -2,6 +2,7 @@ package direct import ( "context" + "sync" "testing" "github.com/databricks/cli/bundle/deployplan" @@ -13,20 +14,32 @@ import ( type fakeOpClient struct { bundledeployments.BundleDeploymentsInterface + + mu sync.Mutex requests []bundledeployments.CreateOperationRequest } func (f *fakeOpClient) CreateOperation(ctx context.Context, req bundledeployments.CreateOperationRequest) (*bundledeployments.Operation, error) { + f.mu.Lock() + defer f.mu.Unlock() f.requests = append(f.requests, req) return &bundledeployments.Operation{}, nil } +// uploadOne records a single operation through the given uploader, mirroring what +// an operationQueue worker does. +func uploadOne(t *testing.T, u operationUploader, resourceKey string, action deployplan.ActionType, resourceID string, state any) { + t.Helper() + op, err := newRecordedOperation(action, resourceID, state) + require.NoError(t, err) + require.NoError(t, u.upload(t.Context(), resourceKey, op)) +} + func TestOperationRecorderStripsResourcePrefix(t *testing.T) { f := &fakeOpClient{} r := NewOperationRecorder(f, "dep-1", 2) - err := r.record(t.Context(), "resources.jobs.foo", deployplan.Create, "job-123", map[string]string{"name": "foo"}) - require.NoError(t, err) + uploadOne(t, r, "resources.jobs.foo", deployplan.Create, "job-123", map[string]string{"name": "foo"}) require.Len(t, f.requests, 1) req := f.requests[0] @@ -44,8 +57,7 @@ func TestOperationRecorderDeleteHasNoState(t *testing.T) { f := &fakeOpClient{} r := NewOperationRecorder(f, "dep-1", 3) - err := r.record(t.Context(), "resources.jobs.foo", deployplan.Delete, "", nil) - require.NoError(t, err) + uploadOne(t, r, "resources.jobs.foo", deployplan.Delete, "", nil) require.Len(t, f.requests, 1) assert.Equal(t, bundledeployments.OperationActionTypeOperationActionTypeDelete, f.requests[0].Operation.ActionType) @@ -53,25 +65,25 @@ func TestOperationRecorderDeleteHasNoState(t *testing.T) { assert.Nil(t, f.requests[0].Operation.State) } -func TestOperationRecorderRedactsSensitiveFields(t *testing.T) { - f := &fakeOpClient{} - r := NewOperationRecorder(f, "dep-1", 2) - +func TestNewRecordedOperationRedactsSensitiveFields(t *testing.T) { state := struct { Name string `json:"name"` Token string `json:"token" bundle:"sensitive"` }{Name: "foo", Token: "super-secret"} - err := r.record(t.Context(), "resources.jobs.foo", deployplan.Create, "job-123", state) + op, err := newRecordedOperation(deployplan.Create, "job-123", state) require.NoError(t, err) - require.Len(t, f.requests, 1) - require.NotNil(t, f.requests[0].Operation.State) // Sensitive fields are redacted before leaving the CLI, matching what // dstate.SaveState writes to the local state file. assert.JSONEq(t, `{"name":"foo","token":"`+dyn.SensitiveValueRedacted+`"}`, - string(*f.requests[0].Operation.State)) + string(op.state)) +} + +func TestNewRecordedOperationRejectsUnsupportedAction(t *testing.T) { + _, err := newRecordedOperation(deployplan.Skip, "job-123", nil) + assert.Error(t, err) } func TestDeployActionToSDK(t *testing.T) { @@ -98,9 +110,3 @@ func TestDeployActionToSDK(t *testing.T) { _, err = deployActionToSDK(deployplan.Undefined) assert.Error(t, err) } - -func TestRecordOperationNoOpWithoutRecorder(t *testing.T) { - b := &DeploymentBundle{} - // No OpRec set: recording is a no-op. - assert.NoError(t, b.recordOperation(t.Context(), "resources.jobs.foo", deployplan.Create, "id", struct{}{})) -} diff --git a/bundle/direct/pkg.go b/bundle/direct/pkg.go index f95b515f726..03864af5da2 100644 --- a/bundle/direct/pkg.go +++ b/bundle/direct/pkg.go @@ -45,10 +45,12 @@ type DeploymentBundle struct { RemoteStateCache sync.Map StateCache structvar.Cache - // OpRec records each applied resource operation with the deployment metadata + // OpRec uploads each applied resource operation to the deployment metadata // service (DMS). It is nil unless the bundle opts into recording deployment // history, in which case the phases package sets it after CreateVersion. - OpRec opRecorder + // Apply queues the operations and drains them before returning, so the + // uploads do not block the resources being deployed. + OpRec operationUploader } // SetRemoteState updates the remote state with type validation and marks as fresh. From 19467dc2425fd093f06e9d11773ca724ea3cb94c Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 27 Jul 2026 13:52:34 +0000 Subject: [PATCH 05/28] bundle: gate experimental.record_deployment_history behind an env var Recording deployment history is implemented end to end, but it cannot be exposed to users yet: enabling it makes the deployment metadata service the source of truth for resource state, and there is no upgrade path from an existing direct-engine state file to a DMS-owned one. Setting the flag is now an error. DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY lifts the error so the CLI's own acceptance tests and DMS development can exercise the feature until the direct state upgrade lands. Co-authored-by: Isaac --- .../bundle/dms/not-supported/databricks.yml | 10 ++++ .../bundle/dms/not-supported/out.test.toml | 3 + .../bundle/dms/not-supported/output.txt | 24 ++++++++ acceptance/bundle/dms/not-supported/script | 5 ++ acceptance/bundle/dms/not-supported/test.toml | 5 ++ acceptance/bundle/dms/test.toml | 6 ++ bundle/config/experimental.go | 4 ++ .../validate_record_deployment_history.go | 48 ++++++++++++++++ ...validate_record_deployment_history_test.go | 55 +++++++++++++++++++ bundle/env/record_deployment_history.go | 19 +++++++ bundle/internal/schema/annotations.yml | 2 + bundle/phases/initialize.go | 5 ++ bundle/schema/jsonschema.json | 2 +- 13 files changed, 187 insertions(+), 1 deletion(-) create mode 100644 acceptance/bundle/dms/not-supported/databricks.yml create mode 100644 acceptance/bundle/dms/not-supported/out.test.toml create mode 100644 acceptance/bundle/dms/not-supported/output.txt create mode 100644 acceptance/bundle/dms/not-supported/script create mode 100644 acceptance/bundle/dms/not-supported/test.toml create mode 100644 bundle/config/validate/validate_record_deployment_history.go create mode 100644 bundle/config/validate/validate_record_deployment_history_test.go create mode 100644 bundle/env/record_deployment_history.go diff --git a/acceptance/bundle/dms/not-supported/databricks.yml b/acceptance/bundle/dms/not-supported/databricks.yml new file mode 100644 index 00000000000..c6edca465b6 --- /dev/null +++ b/acceptance/bundle/dms/not-supported/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: dms-not-supported + +experimental: + record_deployment_history: true + +resources: + jobs: + one: + name: one diff --git a/acceptance/bundle/dms/not-supported/out.test.toml b/acceptance/bundle/dms/not-supported/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/dms/not-supported/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/not-supported/output.txt b/acceptance/bundle/dms/not-supported/output.txt new file mode 100644 index 00000000000..ff5758d574d --- /dev/null +++ b/acceptance/bundle/dms/not-supported/output.txt @@ -0,0 +1,24 @@ + +=== record_deployment_history is rejected: DMS cannot own resource state until the direct engine has a state upgrade path +>>> musterr [CLI] bundle validate +Error: experimental.record_deployment_history is not supported yet + at experimental.record_deployment_history + in databricks.yml:5:30 + +Name: dms-not-supported +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/dms-not-supported/default + +Found 1 error + +=== The hidden opt-in lifts the error +>>> DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY=1 [CLI] bundle validate +Name: dms-not-supported +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/dms-not-supported/default + +Validation OK! diff --git a/acceptance/bundle/dms/not-supported/script b/acceptance/bundle/dms/not-supported/script new file mode 100644 index 00000000000..68c0a3c5f3e --- /dev/null +++ b/acceptance/bundle/dms/not-supported/script @@ -0,0 +1,5 @@ +title "record_deployment_history is rejected: DMS cannot own resource state until the direct engine has a state upgrade path" +trace musterr $CLI bundle validate + +title "The hidden opt-in lifts the error" +trace DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY=1 $CLI bundle validate diff --git a/acceptance/bundle/dms/not-supported/test.toml b/acceptance/bundle/dms/not-supported/test.toml new file mode 100644 index 00000000000..c6daad089b2 --- /dev/null +++ b/acceptance/bundle/dms/not-supported/test.toml @@ -0,0 +1,5 @@ +# Unset the opt-in inherited from the parent: this test asserts the error users see. +Env.DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY = "" + +# This test only checks validation output; no DMS request is made either way. +RecordRequests = false diff --git a/acceptance/bundle/dms/test.toml b/acceptance/bundle/dms/test.toml index 24ce9756629..8c12d014fea 100644 --- a/acceptance/bundle/dms/test.toml +++ b/acceptance/bundle/dms/test.toml @@ -11,3 +11,9 @@ RecordRequests = true Ignore = [ '.databricks', ] + +# experimental.record_deployment_history is rejected until the direct engine has a +# state upgrade path (see validate.ValidateRecordDeploymentHistory). These tests +# exercise the feature itself, so they opt in through the same escape hatch DMS +# development uses. bundle/dms/not-supported covers the rejection. +Env.DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY = "1" diff --git a/bundle/config/experimental.go b/bundle/config/experimental.go index 658f1cea819..56cf3486ed3 100644 --- a/bundle/config/experimental.go +++ b/bundle/config/experimental.go @@ -53,6 +53,10 @@ type Experimental struct { // RecordDeploymentHistory opts the bundle into the deployment metadata // service (DMS), which records deployment history and tracks what changed // across deployments. + // + // Setting this is currently an error: the direct engine needs a state upgrade + // path before DMS can own resource state. See + // validate.ValidateRecordDeploymentHistory. RecordDeploymentHistory bool `json:"record_deployment_history,omitempty"` } diff --git a/bundle/config/validate/validate_record_deployment_history.go b/bundle/config/validate/validate_record_deployment_history.go new file mode 100644 index 00000000000..10144d9c796 --- /dev/null +++ b/bundle/config/validate/validate_record_deployment_history.go @@ -0,0 +1,48 @@ +package validate + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/env" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +const recordDeploymentHistoryPath = "experimental.record_deployment_history" + +func ValidateRecordDeploymentHistory() bundle.ReadOnlyMutator { + return &validateRecordDeploymentHistory{} +} + +type validateRecordDeploymentHistory struct{ bundle.RO } + +func (v *validateRecordDeploymentHistory) Name() string { + return "validate:validate_record_deployment_history" +} + +// Apply rejects experimental.record_deployment_history. +// +// Recording deployment history is implemented end to end, but it is not usable yet: +// enabling it makes the deployment metadata service the source of truth for resource +// state, and there is no upgrade path from an existing direct-engine state file to a +// DMS-owned one. A bundle that flips the flag on today would have its local state +// silently overlaid by an empty DMS resource set, and the next deploy would try to +// create resources that already exist. The direct state upgrade has to land before +// this flag can be exposed; until then it errors, and +// DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY lifts the error for the CLI's own +// tests and for DMS development. +func (v *validateRecordDeploymentHistory) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + if b.Config.Experimental == nil || !b.Config.Experimental.RecordDeploymentHistory { + return nil + } + if env.EnableRecordDeploymentHistory(ctx) { + return nil + } + return diag.Diagnostics{{ + Severity: diag.Error, + Summary: recordDeploymentHistoryPath + " is not supported yet", + Paths: []dyn.Path{dyn.MustPathFromString(recordDeploymentHistoryPath)}, + Locations: b.Config.GetLocations(recordDeploymentHistoryPath), + }} +} diff --git a/bundle/config/validate/validate_record_deployment_history_test.go b/bundle/config/validate/validate_record_deployment_history_test.go new file mode 100644 index 00000000000..06f71d10afa --- /dev/null +++ b/bundle/config/validate/validate_record_deployment_history_test.go @@ -0,0 +1,55 @@ +package validate + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + bundleenv "github.com/databricks/cli/bundle/env" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/env" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestValidateRecordDeploymentHistory(t *testing.T) { + tests := []struct { + name string + enabled bool + optIn string + wantError bool + }{ + {name: "flag unset", enabled: false, wantError: false}, + {name: "flag set", enabled: true, wantError: true}, + {name: "flag set with opt-in", enabled: true, optIn: "1", wantError: false}, + {name: "flag set with empty opt-in", enabled: true, optIn: "", wantError: true}, + {name: "flag unset with opt-in", enabled: false, optIn: "1", wantError: false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Experimental: &config.Experimental{RecordDeploymentHistory: tc.enabled}, + }, + } + + ctx := env.Set(t.Context(), bundleenv.EnableRecordDeploymentHistoryVariable, tc.optIn) + diags := ValidateRecordDeploymentHistory().Apply(ctx, b) + + if !tc.wantError { + assert.Empty(t, diags) + return + } + require.Len(t, diags, 1) + assert.Equal(t, diag.Error, diags[0].Severity) + assert.Equal(t, "experimental.record_deployment_history is not supported yet", diags[0].Summary) + assert.Equal(t, recordDeploymentHistoryPath, diags[0].Paths[0].String()) + }) + } +} + +func TestValidateRecordDeploymentHistoryNoExperimentalBlock(t *testing.T) { + b := &bundle.Bundle{Config: config.Root{}} + assert.Empty(t, ValidateRecordDeploymentHistory().Apply(t.Context(), b)) +} diff --git a/bundle/env/record_deployment_history.go b/bundle/env/record_deployment_history.go new file mode 100644 index 00000000000..e17fdeb9f8d --- /dev/null +++ b/bundle/env/record_deployment_history.go @@ -0,0 +1,19 @@ +package env + +import "context" + +// EnableRecordDeploymentHistoryVariable names the environment variable that lifts the +// error on experimental.record_deployment_history. It is deliberately undocumented: the +// feature is complete but cannot be exposed to users yet (see +// validate.ValidateRecordDeploymentHistory for why), and this variable exists so the +// CLI's own tests and the developers working on DMS can exercise the code path meanwhile. +const EnableRecordDeploymentHistoryVariable = "DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY" + +// EnableRecordDeploymentHistory reports whether the environment opts into +// experimental.record_deployment_history despite it being gated off. +func EnableRecordDeploymentHistory(ctx context.Context) bool { + value, ok := get(ctx, []string{ + EnableRecordDeploymentHistoryVariable, + }) + return ok && value != "" +} diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 10832fe04a6..ed4420cbe15 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -178,6 +178,8 @@ experimental: "record_deployment_history": "description": |- Whether to record deployment history using the deployment metadata service (DMS), which tracks what changed across deployments. + + This setting is not supported yet and enabling it is an error. "scripts": "description": |- The commands to run. diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index bfa2af4124b..02e03837603 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -177,6 +177,11 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // They are set by the CLI to track the bundle deployment and must not be set by the user. validate.ValidateDeploymentFields(), + // Reads (typed): b.Config.Experimental.RecordDeploymentHistory + // Reads (env): DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY (non-empty value lifts the error) + // Rejects experimental.record_deployment_history until the direct state upgrade lands. + validate.ValidateRecordDeploymentHistory(), + // Reads (dynamic): * (strings) (searches for ${resources.*} references) // Warns (TF engine) or errors (direct engine) when a cross-resource reference // points to a Terraform-only field with no DABs equivalent. diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 4c78bd7c384..1752e643769 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -3016,7 +3016,7 @@ "$ref": "#/$defs/bool" }, "record_deployment_history": { - "description": "Whether to record deployment history using the deployment metadata service (DMS), which tracks what changed across deployments.", + "description": "Whether to record deployment history using the deployment metadata service (DMS), which tracks what changed across deployments.\n\nThis setting is not supported yet and enabling it is an error.", "$ref": "#/$defs/bool" }, "scripts": { From 22ec66a4cc6b3a792b4943e7360bc787144edf47 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 27 Jul 2026 14:08:45 +0000 Subject: [PATCH 06/28] bundle: rename the record_deployment_history escape hatch to force_allow DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY read as if it were the switch that turns the feature on. It is not: the flag in databricks.yml does that, and this variable only permits the flag to be set while the feature is gated off. Name it DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY. Co-authored-by: Isaac --- .../bundle/dms/not-supported/output.txt | 4 ++-- acceptance/bundle/dms/not-supported/script | 4 ++-- acceptance/bundle/dms/not-supported/test.toml | 5 +++-- acceptance/bundle/dms/test.toml | 6 +++--- .../validate_record_deployment_history.go | 6 +++--- ...validate_record_deployment_history_test.go | 16 ++++++++-------- .../force_allow_record_deployment_history.go | 19 +++++++++++++++++++ bundle/env/record_deployment_history.go | 19 ------------------- bundle/phases/initialize.go | 2 +- 9 files changed, 41 insertions(+), 40 deletions(-) create mode 100644 bundle/env/force_allow_record_deployment_history.go delete mode 100644 bundle/env/record_deployment_history.go diff --git a/acceptance/bundle/dms/not-supported/output.txt b/acceptance/bundle/dms/not-supported/output.txt index ff5758d574d..0237a81da27 100644 --- a/acceptance/bundle/dms/not-supported/output.txt +++ b/acceptance/bundle/dms/not-supported/output.txt @@ -13,8 +13,8 @@ Workspace: Found 1 error -=== The hidden opt-in lifts the error ->>> DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY=1 [CLI] bundle validate +=== The hidden force-allow variable permits it +>>> DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY=1 [CLI] bundle validate Name: dms-not-supported Target: default Workspace: diff --git a/acceptance/bundle/dms/not-supported/script b/acceptance/bundle/dms/not-supported/script index 68c0a3c5f3e..3bf017b50a0 100644 --- a/acceptance/bundle/dms/not-supported/script +++ b/acceptance/bundle/dms/not-supported/script @@ -1,5 +1,5 @@ title "record_deployment_history is rejected: DMS cannot own resource state until the direct engine has a state upgrade path" trace musterr $CLI bundle validate -title "The hidden opt-in lifts the error" -trace DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY=1 $CLI bundle validate +title "The hidden force-allow variable permits it" +trace DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY=1 $CLI bundle validate diff --git a/acceptance/bundle/dms/not-supported/test.toml b/acceptance/bundle/dms/not-supported/test.toml index c6daad089b2..4617ff88f20 100644 --- a/acceptance/bundle/dms/not-supported/test.toml +++ b/acceptance/bundle/dms/not-supported/test.toml @@ -1,5 +1,6 @@ -# Unset the opt-in inherited from the parent: this test asserts the error users see. -Env.DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY = "" +# Unset the force-allow variable inherited from the parent: this test asserts the +# error users see. +Env.DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY = "" # This test only checks validation output; no DMS request is made either way. RecordRequests = false diff --git a/acceptance/bundle/dms/test.toml b/acceptance/bundle/dms/test.toml index 8c12d014fea..9942a441539 100644 --- a/acceptance/bundle/dms/test.toml +++ b/acceptance/bundle/dms/test.toml @@ -14,6 +14,6 @@ Ignore = [ # experimental.record_deployment_history is rejected until the direct engine has a # state upgrade path (see validate.ValidateRecordDeploymentHistory). These tests -# exercise the feature itself, so they opt in through the same escape hatch DMS -# development uses. bundle/dms/not-supported covers the rejection. -Env.DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY = "1" +# exercise the feature itself, so they force allow it the same way DMS development +# does. bundle/dms/not-supported covers the rejection. +Env.DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY = "1" diff --git a/bundle/config/validate/validate_record_deployment_history.go b/bundle/config/validate/validate_record_deployment_history.go index 10144d9c796..4ebb681d595 100644 --- a/bundle/config/validate/validate_record_deployment_history.go +++ b/bundle/config/validate/validate_record_deployment_history.go @@ -30,13 +30,13 @@ func (v *validateRecordDeploymentHistory) Name() string { // silently overlaid by an empty DMS resource set, and the next deploy would try to // create resources that already exist. The direct state upgrade has to land before // this flag can be exposed; until then it errors, and -// DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY lifts the error for the CLI's own -// tests and for DMS development. +// DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY force allows it for the CLI's +// own tests and for DMS development. func (v *validateRecordDeploymentHistory) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { if b.Config.Experimental == nil || !b.Config.Experimental.RecordDeploymentHistory { return nil } - if env.EnableRecordDeploymentHistory(ctx) { + if env.ForceAllowRecordDeploymentHistory(ctx) { return nil } return diag.Diagnostics{{ diff --git a/bundle/config/validate/validate_record_deployment_history_test.go b/bundle/config/validate/validate_record_deployment_history_test.go index 06f71d10afa..1bb172766f2 100644 --- a/bundle/config/validate/validate_record_deployment_history_test.go +++ b/bundle/config/validate/validate_record_deployment_history_test.go @@ -14,16 +14,16 @@ import ( func TestValidateRecordDeploymentHistory(t *testing.T) { tests := []struct { - name string - enabled bool - optIn string - wantError bool + name string + enabled bool + forceAllow string + wantError bool }{ {name: "flag unset", enabled: false, wantError: false}, {name: "flag set", enabled: true, wantError: true}, - {name: "flag set with opt-in", enabled: true, optIn: "1", wantError: false}, - {name: "flag set with empty opt-in", enabled: true, optIn: "", wantError: true}, - {name: "flag unset with opt-in", enabled: false, optIn: "1", wantError: false}, + {name: "flag set with force allow", enabled: true, forceAllow: "1", wantError: false}, + {name: "flag set with empty force allow", enabled: true, forceAllow: "", wantError: true}, + {name: "flag unset with force allow", enabled: false, forceAllow: "1", wantError: false}, } for _, tc := range tests { @@ -34,7 +34,7 @@ func TestValidateRecordDeploymentHistory(t *testing.T) { }, } - ctx := env.Set(t.Context(), bundleenv.EnableRecordDeploymentHistoryVariable, tc.optIn) + ctx := env.Set(t.Context(), bundleenv.ForceAllowRecordDeploymentHistoryVariable, tc.forceAllow) diags := ValidateRecordDeploymentHistory().Apply(ctx, b) if !tc.wantError { diff --git a/bundle/env/force_allow_record_deployment_history.go b/bundle/env/force_allow_record_deployment_history.go new file mode 100644 index 00000000000..297ccb6f6e3 --- /dev/null +++ b/bundle/env/force_allow_record_deployment_history.go @@ -0,0 +1,19 @@ +package env + +import "context" + +// ForceAllowRecordDeploymentHistoryVariable names the environment variable that force +// allows experimental.record_deployment_history. It is deliberately undocumented: the +// feature is complete but cannot be exposed to users yet (see +// validate.ValidateRecordDeploymentHistory for why), and this variable exists so the +// CLI's own tests and the developers working on DMS can exercise the code path meanwhile. +const ForceAllowRecordDeploymentHistoryVariable = "DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY" + +// ForceAllowRecordDeploymentHistory reports whether the environment force allows +// experimental.record_deployment_history despite it being gated off. +func ForceAllowRecordDeploymentHistory(ctx context.Context) bool { + value, ok := get(ctx, []string{ + ForceAllowRecordDeploymentHistoryVariable, + }) + return ok && value != "" +} diff --git a/bundle/env/record_deployment_history.go b/bundle/env/record_deployment_history.go deleted file mode 100644 index e17fdeb9f8d..00000000000 --- a/bundle/env/record_deployment_history.go +++ /dev/null @@ -1,19 +0,0 @@ -package env - -import "context" - -// EnableRecordDeploymentHistoryVariable names the environment variable that lifts the -// error on experimental.record_deployment_history. It is deliberately undocumented: the -// feature is complete but cannot be exposed to users yet (see -// validate.ValidateRecordDeploymentHistory for why), and this variable exists so the -// CLI's own tests and the developers working on DMS can exercise the code path meanwhile. -const EnableRecordDeploymentHistoryVariable = "DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY" - -// EnableRecordDeploymentHistory reports whether the environment opts into -// experimental.record_deployment_history despite it being gated off. -func EnableRecordDeploymentHistory(ctx context.Context) bool { - value, ok := get(ctx, []string{ - EnableRecordDeploymentHistoryVariable, - }) - return ok && value != "" -} diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 02e03837603..60ea68fab82 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -178,7 +178,7 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { validate.ValidateDeploymentFields(), // Reads (typed): b.Config.Experimental.RecordDeploymentHistory - // Reads (env): DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY (non-empty value lifts the error) + // Reads (env): DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY (non-empty value force allows it) // Rejects experimental.record_deployment_history until the direct state upgrade lands. validate.ValidateRecordDeploymentHistory(), From d7441e43355eae51c355c07f10b52f84fdf00984 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 27 Jul 2026 14:45:13 +0000 Subject: [PATCH 07/28] bundle: only record net-new deployments in DMS Replace the blanket gate on experimental.record_deployment_history with a narrower check in dstate.DeploymentState.Open: recording is refused only when the state file already tracks deployed resources that DMS does not know about. Once DMS holds a successful version it is authoritative for resource state even when its resource set is empty, so adopting a state file written by a CLI that predates DMS would make already-owned resources look absent and create them a second time. A state file that DMS already owns is fine, and so is one with no resources (e.g. left behind by a destroy), which is what makes the error's destroy-and-redeploy advice work. The check keys off len(State) rather than the file existing because destroy leaves resources.json in place with an empty resource set. This drops validate.ValidateRecordDeploymentHistory and the DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY escape hatch, which are no longer needed. Upgrading an existing state in place (state v3 with a feature flag plus per-resource tombstones so older clients refuse the state) is left as a TODO. Co-authored-by: Isaac --- .../databricks.yml | 4 +- .../out.test.toml | 0 .../bundle/dms/existing-state/output.txt | 45 +++++++++++++++ acceptance/bundle/dms/existing-state/script | 17 ++++++ .../bundle/dms/not-supported/output.txt | 24 -------- acceptance/bundle/dms/not-supported/script | 5 -- acceptance/bundle/dms/not-supported/test.toml | 6 -- acceptance/bundle/dms/test.toml | 6 -- bundle/config/experimental.go | 6 +- .../validate_record_deployment_history.go | 48 ---------------- ...validate_record_deployment_history_test.go | 55 ------------------- bundle/direct/dstate/state.go | 30 +++++++++- .../force_allow_record_deployment_history.go | 19 ------- bundle/internal/schema/annotations.yml | 2 +- bundle/phases/initialize.go | 5 -- bundle/schema/jsonschema.json | 2 +- 16 files changed, 96 insertions(+), 178 deletions(-) rename acceptance/bundle/dms/{not-supported => existing-state}/databricks.yml (52%) rename acceptance/bundle/dms/{not-supported => existing-state}/out.test.toml (100%) create mode 100644 acceptance/bundle/dms/existing-state/output.txt create mode 100644 acceptance/bundle/dms/existing-state/script delete mode 100644 acceptance/bundle/dms/not-supported/output.txt delete mode 100644 acceptance/bundle/dms/not-supported/script delete mode 100644 acceptance/bundle/dms/not-supported/test.toml delete mode 100644 bundle/config/validate/validate_record_deployment_history.go delete mode 100644 bundle/config/validate/validate_record_deployment_history_test.go delete mode 100644 bundle/env/force_allow_record_deployment_history.go diff --git a/acceptance/bundle/dms/not-supported/databricks.yml b/acceptance/bundle/dms/existing-state/databricks.yml similarity index 52% rename from acceptance/bundle/dms/not-supported/databricks.yml rename to acceptance/bundle/dms/existing-state/databricks.yml index c6edca465b6..cfd64979342 100644 --- a/acceptance/bundle/dms/not-supported/databricks.yml +++ b/acceptance/bundle/dms/existing-state/databricks.yml @@ -1,8 +1,8 @@ bundle: - name: dms-not-supported + name: dms-existing-state experimental: - record_deployment_history: true + record_deployment_history: false resources: jobs: diff --git a/acceptance/bundle/dms/not-supported/out.test.toml b/acceptance/bundle/dms/existing-state/out.test.toml similarity index 100% rename from acceptance/bundle/dms/not-supported/out.test.toml rename to acceptance/bundle/dms/existing-state/out.test.toml diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt new file mode 100644 index 00000000000..d0cae5fb0ec --- /dev/null +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -0,0 +1,45 @@ + +=== Deploy without recording: the bundle gets ordinary direct-engine state, unknown to DMS +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //api/2.0/bundle --sort --oneline + +=== Turning recording on afterwards is an error: those resources were never recorded, so treating DMS as authoritative would deploy them a second time +>>> update_file.py databricks.yml record_deployment_history: false record_deployment_history: true + +>>> musterr [CLI] bundle deploy +Error: cannot record deployment history for a bundle that already has deployed resources tracked in [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again + + +=== No deployment was created in DMS +>>> print_requests.py //api/2.0/bundle --sort --oneline + +=== Destroy clears the tracked resources, so recording can be enabled afterwards +>>> update_file.py databricks.yml record_deployment_history: true record_deployment_history: false + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.one + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default + +Deleting files... +Destroy complete! + +>>> update_file.py databricks.yml record_deployment_history: false record_deployment_history: true + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //api/2.0/bundle --sort --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"target_name": "default"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "state": {"deployment": {"kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state/metadata.json"}, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "max_concurrent_runs": 1, "name": "one", "queue": {"enabled": true}}, "status": "OPERATION_STATUS_SUCCEEDED"}} diff --git a/acceptance/bundle/dms/existing-state/script b/acceptance/bundle/dms/existing-state/script new file mode 100644 index 00000000000..7bc296465d5 --- /dev/null +++ b/acceptance/bundle/dms/existing-state/script @@ -0,0 +1,17 @@ +title "Deploy without recording: the bundle gets ordinary direct-engine state, unknown to DMS" +trace $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --sort --oneline + +title "Turning recording on afterwards is an error: those resources were never recorded, so treating DMS as authoritative would deploy them a second time" +trace update_file.py databricks.yml "record_deployment_history: false" "record_deployment_history: true" +trace musterr $CLI bundle deploy + +title "No deployment was created in DMS" +trace print_requests.py //api/2.0/bundle --sort --oneline + +title "Destroy clears the tracked resources, so recording can be enabled afterwards" +trace update_file.py databricks.yml "record_deployment_history: true" "record_deployment_history: false" +trace $CLI bundle destroy --auto-approve +trace update_file.py databricks.yml "record_deployment_history: false" "record_deployment_history: true" +trace $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --sort --oneline diff --git a/acceptance/bundle/dms/not-supported/output.txt b/acceptance/bundle/dms/not-supported/output.txt deleted file mode 100644 index 0237a81da27..00000000000 --- a/acceptance/bundle/dms/not-supported/output.txt +++ /dev/null @@ -1,24 +0,0 @@ - -=== record_deployment_history is rejected: DMS cannot own resource state until the direct engine has a state upgrade path ->>> musterr [CLI] bundle validate -Error: experimental.record_deployment_history is not supported yet - at experimental.record_deployment_history - in databricks.yml:5:30 - -Name: dms-not-supported -Target: default -Workspace: - User: [USERNAME] - Path: /Workspace/Users/[USERNAME]/.bundle/dms-not-supported/default - -Found 1 error - -=== The hidden force-allow variable permits it ->>> DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY=1 [CLI] bundle validate -Name: dms-not-supported -Target: default -Workspace: - User: [USERNAME] - Path: /Workspace/Users/[USERNAME]/.bundle/dms-not-supported/default - -Validation OK! diff --git a/acceptance/bundle/dms/not-supported/script b/acceptance/bundle/dms/not-supported/script deleted file mode 100644 index 3bf017b50a0..00000000000 --- a/acceptance/bundle/dms/not-supported/script +++ /dev/null @@ -1,5 +0,0 @@ -title "record_deployment_history is rejected: DMS cannot own resource state until the direct engine has a state upgrade path" -trace musterr $CLI bundle validate - -title "The hidden force-allow variable permits it" -trace DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY=1 $CLI bundle validate diff --git a/acceptance/bundle/dms/not-supported/test.toml b/acceptance/bundle/dms/not-supported/test.toml deleted file mode 100644 index 4617ff88f20..00000000000 --- a/acceptance/bundle/dms/not-supported/test.toml +++ /dev/null @@ -1,6 +0,0 @@ -# Unset the force-allow variable inherited from the parent: this test asserts the -# error users see. -Env.DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY = "" - -# This test only checks validation output; no DMS request is made either way. -RecordRequests = false diff --git a/acceptance/bundle/dms/test.toml b/acceptance/bundle/dms/test.toml index 9942a441539..24ce9756629 100644 --- a/acceptance/bundle/dms/test.toml +++ b/acceptance/bundle/dms/test.toml @@ -11,9 +11,3 @@ RecordRequests = true Ignore = [ '.databricks', ] - -# experimental.record_deployment_history is rejected until the direct engine has a -# state upgrade path (see validate.ValidateRecordDeploymentHistory). These tests -# exercise the feature itself, so they force allow it the same way DMS development -# does. bundle/dms/not-supported covers the rejection. -Env.DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY = "1" diff --git a/bundle/config/experimental.go b/bundle/config/experimental.go index 56cf3486ed3..c3f1465d880 100644 --- a/bundle/config/experimental.go +++ b/bundle/config/experimental.go @@ -54,9 +54,9 @@ type Experimental struct { // service (DMS), which records deployment history and tracks what changed // across deployments. // - // Setting this is currently an error: the direct engine needs a state upgrade - // path before DMS can own resource state. See - // validate.ValidateRecordDeploymentHistory. + // Only supported for a bundle with no deployed resources yet: DMS becomes the + // source of truth for resource state, and resources tracked in an existing + // state file cannot be handed over to it yet. See dstate.DeploymentState.Open. RecordDeploymentHistory bool `json:"record_deployment_history,omitempty"` } diff --git a/bundle/config/validate/validate_record_deployment_history.go b/bundle/config/validate/validate_record_deployment_history.go deleted file mode 100644 index 4ebb681d595..00000000000 --- a/bundle/config/validate/validate_record_deployment_history.go +++ /dev/null @@ -1,48 +0,0 @@ -package validate - -import ( - "context" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/env" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" -) - -const recordDeploymentHistoryPath = "experimental.record_deployment_history" - -func ValidateRecordDeploymentHistory() bundle.ReadOnlyMutator { - return &validateRecordDeploymentHistory{} -} - -type validateRecordDeploymentHistory struct{ bundle.RO } - -func (v *validateRecordDeploymentHistory) Name() string { - return "validate:validate_record_deployment_history" -} - -// Apply rejects experimental.record_deployment_history. -// -// Recording deployment history is implemented end to end, but it is not usable yet: -// enabling it makes the deployment metadata service the source of truth for resource -// state, and there is no upgrade path from an existing direct-engine state file to a -// DMS-owned one. A bundle that flips the flag on today would have its local state -// silently overlaid by an empty DMS resource set, and the next deploy would try to -// create resources that already exist. The direct state upgrade has to land before -// this flag can be exposed; until then it errors, and -// DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY force allows it for the CLI's -// own tests and for DMS development. -func (v *validateRecordDeploymentHistory) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - if b.Config.Experimental == nil || !b.Config.Experimental.RecordDeploymentHistory { - return nil - } - if env.ForceAllowRecordDeploymentHistory(ctx) { - return nil - } - return diag.Diagnostics{{ - Severity: diag.Error, - Summary: recordDeploymentHistoryPath + " is not supported yet", - Paths: []dyn.Path{dyn.MustPathFromString(recordDeploymentHistoryPath)}, - Locations: b.Config.GetLocations(recordDeploymentHistoryPath), - }} -} diff --git a/bundle/config/validate/validate_record_deployment_history_test.go b/bundle/config/validate/validate_record_deployment_history_test.go deleted file mode 100644 index 1bb172766f2..00000000000 --- a/bundle/config/validate/validate_record_deployment_history_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package validate - -import ( - "testing" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" - bundleenv "github.com/databricks/cli/bundle/env" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/env" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestValidateRecordDeploymentHistory(t *testing.T) { - tests := []struct { - name string - enabled bool - forceAllow string - wantError bool - }{ - {name: "flag unset", enabled: false, wantError: false}, - {name: "flag set", enabled: true, wantError: true}, - {name: "flag set with force allow", enabled: true, forceAllow: "1", wantError: false}, - {name: "flag set with empty force allow", enabled: true, forceAllow: "", wantError: true}, - {name: "flag unset with force allow", enabled: false, forceAllow: "1", wantError: false}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Experimental: &config.Experimental{RecordDeploymentHistory: tc.enabled}, - }, - } - - ctx := env.Set(t.Context(), bundleenv.ForceAllowRecordDeploymentHistoryVariable, tc.forceAllow) - diags := ValidateRecordDeploymentHistory().Apply(ctx, b) - - if !tc.wantError { - assert.Empty(t, diags) - return - } - require.Len(t, diags, 1) - assert.Equal(t, diag.Error, diags[0].Severity) - assert.Equal(t, "experimental.record_deployment_history is not supported yet", diags[0].Summary) - assert.Equal(t, recordDeploymentHistoryPath, diags[0].Paths[0].String()) - }) - } -} - -func TestValidateRecordDeploymentHistoryNoExperimentalBlock(t *testing.T) { - b := &bundle.Bundle{Config: config.Root{}} - assert.Empty(t, ValidateRecordDeploymentHistory().Apply(t.Context(), b)) -} diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index f554af3c9b6..e5eb44df24a 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -319,9 +319,33 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W return fmt.Errorf("migrating state %s: %w", path, err) } - if dmsClient != nil && db.Data.DeploymentID != "" { - if err := db.overlayDMSState(ctx, dmsClient, dmsCfg); err != nil { - return err + if dmsClient != nil { + // Only deployments that start out empty are recorded in DMS. Resources + // tracked in a state file that DMS does not know about are not in DMS and + // never will be: the first recorded deploy would create a deployment whose + // resource set covers only what that deploy touched, and DMS would then be + // authoritative for everything (see overlayDMSState). Resources this bundle + // already owns would look absent and be created a second time. + // + // A state file that DMS already owns (it carries a deployment ID) is fine — + // that is a bundle that opted in while it was still empty. So is a state file + // with no resources, e.g. one left behind by a destroy. + // + // TODO(DMS): lift this restriction by upgrading an existing state in place. + // That means writing the state at featureStateVersion (3) with a feature flag + // recording that DMS owns it, plus a tombstone entry per resource so a CLI + // that predates DMS refuses the state instead of silently deploying against a + // resource set it cannot see. The feature-flag scaffolding for this already + // exists (see featureStateVersion and Header.Features); once it is written, + // this check goes away and record_deployment_history becomes usable on + // existing bundles. + if db.Data.DeploymentID == "" && len(db.Data.State) > 0 { + return fmt.Errorf("cannot record deployment history for a bundle that already has deployed resources tracked in %s: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again", path) + } + if db.Data.DeploymentID != "" { + if err := db.overlayDMSState(ctx, dmsClient, dmsCfg); err != nil { + return err + } } } diff --git a/bundle/env/force_allow_record_deployment_history.go b/bundle/env/force_allow_record_deployment_history.go deleted file mode 100644 index 297ccb6f6e3..00000000000 --- a/bundle/env/force_allow_record_deployment_history.go +++ /dev/null @@ -1,19 +0,0 @@ -package env - -import "context" - -// ForceAllowRecordDeploymentHistoryVariable names the environment variable that force -// allows experimental.record_deployment_history. It is deliberately undocumented: the -// feature is complete but cannot be exposed to users yet (see -// validate.ValidateRecordDeploymentHistory for why), and this variable exists so the -// CLI's own tests and the developers working on DMS can exercise the code path meanwhile. -const ForceAllowRecordDeploymentHistoryVariable = "DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY" - -// ForceAllowRecordDeploymentHistory reports whether the environment force allows -// experimental.record_deployment_history despite it being gated off. -func ForceAllowRecordDeploymentHistory(ctx context.Context) bool { - value, ok := get(ctx, []string{ - ForceAllowRecordDeploymentHistoryVariable, - }) - return ok && value != "" -} diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index ed4420cbe15..100d33356fd 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -179,7 +179,7 @@ experimental: "description": |- Whether to record deployment history using the deployment metadata service (DMS), which tracks what changed across deployments. - This setting is not supported yet and enabling it is an error. + Only supported for a bundle with no deployed resources yet. "scripts": "description": |- The commands to run. diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 60ea68fab82..bfa2af4124b 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -177,11 +177,6 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // They are set by the CLI to track the bundle deployment and must not be set by the user. validate.ValidateDeploymentFields(), - // Reads (typed): b.Config.Experimental.RecordDeploymentHistory - // Reads (env): DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY (non-empty value force allows it) - // Rejects experimental.record_deployment_history until the direct state upgrade lands. - validate.ValidateRecordDeploymentHistory(), - // Reads (dynamic): * (strings) (searches for ${resources.*} references) // Warns (TF engine) or errors (direct engine) when a cross-resource reference // points to a Terraform-only field with no DABs equivalent. diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 1752e643769..c52e96d5dc9 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -3016,7 +3016,7 @@ "$ref": "#/$defs/bool" }, "record_deployment_history": { - "description": "Whether to record deployment history using the deployment metadata service (DMS), which tracks what changed across deployments.\n\nThis setting is not supported yet and enabling it is an error.", + "description": "Whether to record deployment history using the deployment metadata service (DMS), which tracks what changed across deployments.\n\nOnly supported for a bundle with no deployed resources yet.", "$ref": "#/$defs/bool" }, "scripts": { From 026a5213943453ad2a35f47420cefa7503e8f34a Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 27 Jul 2026 16:04:51 +0000 Subject: [PATCH 08/28] bundle: fix lint errors in the operation queue test The concurrent-producer test tripped three linters: - modernize/revive want wg.Go instead of wg.Add + go func + defer wg.Done. - testifylint's go-require flags recordState, which calls require inside the spawned goroutines. testify assertions may only run on the goroutine running the test function. Record inline and send each error to a buffered channel that the test goroutine drains after wg.Wait, so the assertions stay on the test goroutine. Co-authored-by: Isaac --- bundle/direct/opqueue_test.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/bundle/direct/opqueue_test.go b/bundle/direct/opqueue_test.go index 5b267c4d8fa..1b818eb3ba1 100644 --- a/bundle/direct/opqueue_test.go +++ b/bundle/direct/opqueue_test.go @@ -158,21 +158,27 @@ func TestOperationQueueUploadsOneResourceAtATime(t *testing.T) { distinctKeyMod = 12 ) + ctx := t.Context() u := &serialUploader{live: map[string]bool{}, last: map[string]string{}} - q := newOperationQueue(t.Context(), u) + q := newOperationQueue(ctx, u) + // Collect record errors instead of asserting inside the goroutines: testify + // assertions may only run on the goroutine running the test function. + errs := make(chan error, workers*perWorker) var wg sync.WaitGroup for w := range workers { - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { for i := range perWorker { key := "resources.jobs.job" + strconv.Itoa((w*perWorker+i)%distinctKeyMod) - recordState(t, q, key, strconv.Itoa(w)) + errs <- q.record(ctx, key, deployplan.Update, "id-1", map[string]string{"name": strconv.Itoa(w)}) } - }() + }) } wg.Wait() + close(errs) + for err := range errs { + require.NoError(t, err) + } require.NoError(t, q.close()) assert.False(t, u.uneven, "two uploads overlapped for the same resource key") From 481259bd95019be6463ef77c2a34134767d1b428 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 27 Jul 2026 16:24:50 +0000 Subject: [PATCH 09/28] bundle: keep the create action when coalescing DMS operations The operation queue collapses repeated writes to the same resource key by replacing the queued operation wholesale, so a create followed by an update was uploaded as an update. That tells DMS the resource already existed before this deploy, when in fact this deploy created it. Merge the actions instead: the state uploaded is still the later one, but a queued create or recreate wins over a subsequent update. A delete still wins over anything queued before it, since the resource is gone. This is not reachable from Apply today - each resource is recorded once per deploy, because there is one record call per graph node and dagrun visits each node exactly once - so this is about the queue being correct for any caller that records a resource more than once. Co-authored-by: Isaac --- bundle/direct/opqueue.go | 11 ++++++--- bundle/direct/opqueue_test.go | 41 ++++++++++++++++++++++++++++++++ bundle/direct/oprecorder.go | 18 ++++++++++++++ bundle/direct/oprecorder_test.go | 30 +++++++++++++++++++++++ 4 files changed, 97 insertions(+), 3 deletions(-) diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go index 0804ad2d5b3..72a2fa9c39e 100644 --- a/bundle/direct/opqueue.go +++ b/bundle/direct/opqueue.go @@ -91,8 +91,10 @@ func newOperationQueue(ctx context.Context, uploader operationUploader) *operati // // When an operation for the same resource is already waiting it is replaced // instead of queued again: DMS keeps one state per resource key, so the later -// operation supersedes the earlier one and a single upload records both. This is -// best effort - only operations that have not been picked up yet are collapsed. +// operation supersedes the earlier one and a single upload records both. The +// merged operation keeps the action of a queued create (see mergeAction), so +// collapsing a create and a later update still records a create. This is best +// effort - only operations that have not been picked up yet are collapsed. func (q *operationQueue) record(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state any) error { if q == nil { return nil @@ -104,8 +106,11 @@ func (q *operationQueue) record(ctx context.Context, resourceKey string, action } q.mu.Lock() - _, waiting := q.pending[resourceKey] + queued, waiting := q.pending[resourceKey] owned := waiting || q.inflight[resourceKey] + if waiting { + op.action = mergeAction(queued.action, op.action) + } q.pending[resourceKey] = op q.mu.Unlock() diff --git a/bundle/direct/opqueue_test.go b/bundle/direct/opqueue_test.go index 1b818eb3ba1..a8e05141fc7 100644 --- a/bundle/direct/opqueue_test.go +++ b/bundle/direct/opqueue_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -22,6 +23,7 @@ type fakeUploader struct { mu sync.Mutex uploads []string + actions map[string]bundledeployments.OperationActionType } func (f *fakeUploader) upload(ctx context.Context, resourceKey string, op recordedOperation) error { @@ -35,6 +37,10 @@ func (f *fakeUploader) upload(ctx context.Context, resourceKey string, op record f.mu.Lock() defer f.mu.Unlock() f.uploads = append(f.uploads, resourceKey+"="+string(op.state)) + if f.actions == nil { + f.actions = map[string]bundledeployments.OperationActionType{} + } + f.actions[resourceKey] = op.action return f.err } @@ -44,6 +50,12 @@ func (f *fakeUploader) recorded() []string { return append([]string(nil), f.uploads...) } +func (f *fakeUploader) actionFor(resourceKey string) bundledeployments.OperationActionType { + f.mu.Lock() + defer f.mu.Unlock() + return f.actions[resourceKey] +} + func recordState(t *testing.T, q *operationQueue, resourceKey, name string) { t.Helper() require.NoError(t, q.record(t.Context(), resourceKey, deployplan.Update, "id-1", map[string]string{"name": name})) @@ -86,6 +98,35 @@ func TestOperationQueueCoalescesQueuedOperationsForSameResource(t *testing.T) { }, f.recorded()) } +func TestOperationQueueCoalescingKeepsCreateAction(t *testing.T) { + // Hold the first upload so the create below stays queued and the update + // coalesces into it. + f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 1)} + q := newOperationQueue(t.Context(), f) + + recordState(t, q, "resources.jobs.hold", "v1") + assert.Equal(t, "resources.jobs.hold", <-f.started) + + // Occupy the remaining workers so nothing drains the key under test. + for i := range operationUploadWorkers - 1 { + recordState(t, q, "resources.jobs.hold"+strconv.Itoa(i), "v1") + assert.Equal(t, "resources.jobs.hold"+strconv.Itoa(i), <-f.started) + } + + require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", map[string]string{"name": "created"})) + require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Update, "id-1", map[string]string{"name": "updated"})) + + close(f.block) + require.NoError(t, q.close()) + + // The state is the later one, but the action stays CREATE: recording an update + // would tell DMS the resource already existed before this deploy. + assert.Contains(t, f.recorded(), `resources.jobs.foo={"name":"updated"}`) + assert.Equal(t, + bundledeployments.OperationActionTypeOperationActionTypeCreate, + f.actionFor("resources.jobs.foo")) +} + func TestOperationQueueReturnsUploadError(t *testing.T) { uploadErr := errors.New("boom") f := &fakeUploader{err: uploadErr} diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index cdd99966f15..63d0c84a621 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -56,6 +56,24 @@ func newRecordedOperation(action deployplan.ActionType, resourceID string, state return op, nil } +// mergeAction returns the action to record when a later operation coalesces into +// one still queued for the same resource (see operationQueue.record). The state +// uploaded is the later one, but the action must not be downgraded: Create and +// Recreate tell DMS the resource ID is new, and a subsequent Update only refines +// the state of that same new resource. Recording the pair as an Update would +// claim the resource already existed. A Delete is the exception - the resource is +// gone, so nothing earlier is worth reporting. +func mergeAction(queued, next bundledeployments.OperationActionType) bundledeployments.OperationActionType { + if next == bundledeployments.OperationActionTypeOperationActionTypeDelete { + return next + } + if queued == bundledeployments.OperationActionTypeOperationActionTypeCreate || + queued == bundledeployments.OperationActionTypeOperationActionTypeRecreate { + return queued + } + return next +} + // operationUploader records an applied resource operation with DMS. Uploads run // on the operationQueue workers, off the apply path. type operationUploader interface { diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go index aaf3243ec60..70afb788cd3 100644 --- a/bundle/direct/oprecorder_test.go +++ b/bundle/direct/oprecorder_test.go @@ -86,6 +86,36 @@ func TestNewRecordedOperationRejectsUnsupportedAction(t *testing.T) { assert.Error(t, err) } +func TestMergeAction(t *testing.T) { + const ( + create = bundledeployments.OperationActionTypeOperationActionTypeCreate + recreate = bundledeployments.OperationActionTypeOperationActionTypeRecreate + update = bundledeployments.OperationActionTypeOperationActionTypeUpdate + resize = bundledeployments.OperationActionTypeOperationActionTypeResize + del = bundledeployments.OperationActionTypeOperationActionTypeDelete + ) + + cases := []struct { + queued, next, want bundledeployments.OperationActionType + }{ + // A queued create is not downgraded: the resource is still new. + {create, update, create}, + {create, resize, create}, + {recreate, update, recreate}, + {create, create, create}, + // A delete wins: the resource is gone, so the earlier action is moot. + {create, del, del}, + {update, del, del}, + // Neither side is a create, so the later action stands. + {update, resize, resize}, + {resize, update, update}, + {del, create, create}, + } + for _, c := range cases { + assert.Equal(t, c.want, mergeAction(c.queued, c.next), "queued %s, next %s", c.queued, c.next) + } +} + func TestDeployActionToSDK(t *testing.T) { cases := []struct { action deployplan.ActionType From 309a1a4fe5098bd122103ba880b862ac5d65f864 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 27 Jul 2026 22:32:16 +0000 Subject: [PATCH 10/28] bundle: drop the dms package comment Co-authored-by: Isaac --- libs/dms/recorder.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/libs/dms/recorder.go b/libs/dms/recorder.go index 0d9563dd052..1a113491cf9 100644 --- a/libs/dms/recorder.go +++ b/libs/dms/recorder.go @@ -1,12 +1,3 @@ -// Package dms records bundle deployments as versions with the Deployment -// Metadata Service (DMS). -// -// It is intentionally independent of the deployment lock: a Recorder does not -// acquire or hold any lock. Callers are responsible for serializing concurrent -// deployments (today via the workspace-filesystem lock). The server-side -// version counter — CreateVersion only succeeds when the requested version is -// last_version_id + 1 — provides the concurrency control for the records -// themselves. package dms import ( From 211aa94a2b5e00107b2bc6231192f1d9c3c7df88 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 27 Jul 2026 23:33:49 +0000 Subject: [PATCH 11/28] bundle: resolve the DMS deployment ID from the workspace The deployment ID was persisted in the local state file header, which made the CLI the source of truth for a value the service mints. DMS registers each deployment as a workspace node named resources.deployment.json under initial_parent_path, and the node's ID *is* the deployment ID, so it can be resolved from the workspace instead. ResolveDeploymentID does a get-status on /resources.deployment.json and returns the node ID, or empty when the node is absent. Deploy, destroy, and the read path all resolve the ID that way and pass it down; the read path then constructs state from GetDeployment + ListResources as before. Consequences: - Header.DeploymentID, GetDeploymentID, and SetDeploymentID are gone, along with the headerDirty machinery that only existed to persist the ID on a resource-less deploy. - Open takes a *DMSSource instead of a (client, config) pair, since the resolved ID now has to be threaded in too. - CreateDeployment sets initial_parent_path, which the service requires and the CLI never set. - createDeploymentVersion no longer recovers from a 404 on GetDeployment by creating a second deployment. A destroy trashes the node, so a resolved ID whose record is missing means the two are out of sync, and creating another deployment would collide on the same node path. The testserver models the real derivation: CreateDeployment creates the workspace node and uses its object ID as the deployment ID, so the acceptance tests exercise get-status resolution end to end. dms/record now wipes the local cache before redeploying and records zero operations, which is the read path reconstructing state entirely from DMS. Co-authored-by: Isaac --- .../bundle/dms/existing-state/output.txt | 8 +- .../bundle/dms/multiple-resources/output.txt | 14 +-- acceptance/bundle/dms/no-resources/output.txt | 28 +++--- acceptance/bundle/dms/no-resources/script | 6 +- acceptance/bundle/dms/record/output.txt | 33 ++++--- acceptance/bundle/dms/record/script | 7 +- .../dms/redeploy-after-destroy/output.txt | 31 +++---- .../bundle/dms/redeploy-after-destroy/script | 12 +-- acceptance/bundle/dms/test.toml | 5 +- bundle/configsync/diff.go | 2 +- bundle/configsync/variables.go | 2 +- bundle/direct/bind.go | 12 +-- bundle/direct/dstate/dms.go | 16 ++-- bundle/direct/dstate/state.go | 91 ++++++------------- bundle/direct/dstate/state_test.go | 87 +++--------------- bundle/phases/deploy.go | 12 ++- bundle/phases/destroy.go | 6 +- bundle/phases/dms.go | 29 +++--- cmd/bundle/generate/dashboard.go | 2 +- cmd/bundle/generate/genie_space.go | 2 +- cmd/bundle/utils/process.go | 29 +++--- libs/dms/recorder.go | 73 ++++++++------- libs/dms/recorder_test.go | 83 ++++++++--------- libs/dms/resolve.go | 45 +++++++++ libs/dms/resolve_test.go | 67 ++++++++++++++ libs/testserver/bundle.go | 47 ++++++++-- 26 files changed, 406 insertions(+), 343 deletions(-) create mode 100644 libs/dms/resolve.go create mode 100644 libs/dms/resolve_test.go diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index d0cae5fb0ec..6aeff83c04b 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -39,7 +39,7 @@ Updating deployment state... Deployment complete! >>> print_requests.py //api/2.0/bundle --sort --oneline -{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"target_name": "default"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "state": {"deployment": {"kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state/metadata.json"}, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "max_concurrent_runs": 1, "name": "one", "queue": {"enabled": true}}, "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state", "target_name": "default"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "state": {"deployment": {"kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state/metadata.json"}, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "max_concurrent_runs": 1, "name": "one", "queue": {"enabled": true}}, "status": "OPERATION_STATUS_SUCCEEDED"}} diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index 4bacec0c317..75086ceb5f7 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -7,11 +7,11 @@ Updating deployment state... Deployment complete! >>> print_requests.py //versions/1/operations --sort --del-body state --oneline -{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.five"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.five", "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.four"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.four", "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.three"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.three", "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.two"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.two", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.five"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.five", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.four"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.four", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.three"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.three", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.two"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.two", "status": "OPERATION_STATUS_SUCCEEDED"}} === Redeploy with no changes: nothing is applied, so no operations are recorded and only the version is opened and completed >>> [CLI] bundle deploy @@ -21,5 +21,5 @@ Updating deployment state... Deployment complete! >>> print_requests.py //api/2.0/bundle --sort --oneline -{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index e774fd7546c..73bd3adfa11 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -1,9 +1,8 @@ -=== First deploy of a bundle with no resources: the deployment is created and its ID is persisted, even though no resource state was written +=== First deploy of a bundle with no resources: the deployment is created, and its workspace node identifies it even though no resource state was written >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/files... Deploying resources... -Updating deployment state... Deployment complete! >>> print_requests.py //api/2.0/bundle --sort --get @@ -11,12 +10,13 @@ Deployment complete! "method": "POST", "path": "/api/2.0/bundle/deployments", "body": { + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/state", "target_name": "default" } } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": { "version_id": "1" }, @@ -28,38 +28,40 @@ Deployment complete! } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/complete", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": { "completion_reason": "VERSION_COMPLETE_SUCCESS" } } ->>> jq .deployment_id .databricks/bundle/default/resources.json -"[UUID]" +>>> [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/state/resources.deployment.json +{ + "object_type": "FILE", + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/state/resources.deployment.json" +} -=== Redeploy: the persisted ID is reused, so no second deployment is created +=== Redeploy: the deployment is resolved from that node, so no second deployment is created >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/files... Deploying resources... -Updating deployment state... Deployment complete! >>> print_requests.py //api/2.0/bundle --sort --get { "method": "GET", - "path": "/api/2.0/bundle/deployments/[UUID]" + "path": "/api/2.0/bundle/deployments/[NUMID]" } { "method": "GET", - "path": "/api/2.0/bundle/deployments/[UUID]" + "path": "/api/2.0/bundle/deployments/[NUMID]" } { "method": "GET", - "path": "/api/2.0/bundle/deployments/[UUID]/resources" + "path": "/api/2.0/bundle/deployments/[NUMID]/resources" } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": { "version_id": "2" }, @@ -71,7 +73,7 @@ Deployment complete! } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions/2/complete", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", "body": { "completion_reason": "VERSION_COMPLETE_SUCCESS" } diff --git a/acceptance/bundle/dms/no-resources/script b/acceptance/bundle/dms/no-resources/script index 4bc97c5864d..847935af5d1 100644 --- a/acceptance/bundle/dms/no-resources/script +++ b/acceptance/bundle/dms/no-resources/script @@ -1,8 +1,8 @@ -title "First deploy of a bundle with no resources: the deployment is created and its ID is persisted, even though no resource state was written" +title "First deploy of a bundle with no resources: the deployment is created, and its workspace node identifies it even though no resource state was written" trace $CLI bundle deploy trace print_requests.py //api/2.0/bundle --sort --get -trace jq .deployment_id .databricks/bundle/default/resources.json +trace $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-resources/default/state/resources.deployment.json" | jq '{object_type,path}' -title "Redeploy: the persisted ID is reused, so no second deployment is created" +title "Redeploy: the deployment is resolved from that node, so no second deployment is created" trace $CLI bundle deploy trace print_requests.py //api/2.0/bundle --sort --get diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index 5c0317f38cc..ed10f06e699 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -11,12 +11,13 @@ Deployment complete! "method": "POST", "path": "/api/2.0/bundle/deployments", "body": { + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default/state", "target_name": "default" } } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": { "version_id": "1" }, @@ -28,14 +29,14 @@ Deployment complete! } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/complete", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": { "completion_reason": "VERSION_COMPLETE_SUCCESS" } } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": { "resource_key": "jobs.foo" }, @@ -60,11 +61,17 @@ Deployment complete! } } -=== The server-assigned deployment ID is persisted in the local state file ->>> jq .deployment_id .databricks/bundle/default/resources.json -"[UUID]" +=== The deployment ID is the ID of the workspace node the service registered under initial_parent_path; the CLI stores nothing locally +>>> [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/resources.deployment.json +{ + "object_type": "FILE", + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/resources.deployment.json" +} + +>>> jq has("deployment_id") .databricks/bundle/default/resources.json +false -=== Redeploy after deleting the local cache: the deployment ID is recovered from remote state, the same deployment is reused, and the version increments (no new CreateDeployment) +=== Redeploy after deleting the local cache: the deployment ID is resolved from that node, the same deployment is reused, and the version increments (no new CreateDeployment) >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record/default/files... Deploying resources... @@ -74,7 +81,7 @@ Deployment complete! >>> print_requests.py //api/2.0/bundle --sort { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": { "version_id": "2" }, @@ -86,7 +93,7 @@ Deployment complete! } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions/2/complete", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", "body": { "completion_reason": "VERSION_COMPLETE_SUCCESS" } @@ -105,11 +112,11 @@ Destroy complete! >>> print_requests.py //api/2.0/bundle --sort { "method": "DELETE", - "path": "/api/2.0/bundle/deployments/[UUID]" + "path": "/api/2.0/bundle/deployments/[NUMID]" } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": { "version_id": "3" }, @@ -121,14 +128,14 @@ Destroy complete! } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions/3/complete", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/complete", "body": { "completion_reason": "VERSION_COMPLETE_SUCCESS" } } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions/3/operations", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/operations", "q": { "resource_key": "jobs.foo" }, diff --git a/acceptance/bundle/dms/record/script b/acceptance/bundle/dms/record/script index ab59d38afb4..63eaa323b1b 100644 --- a/acceptance/bundle/dms/record/script +++ b/acceptance/bundle/dms/record/script @@ -2,10 +2,11 @@ title "Deploy: the server assigns the deployment ID, and a version + create oper trace $CLI bundle deploy trace print_requests.py //api/2.0/bundle --sort -title "The server-assigned deployment ID is persisted in the local state file" -trace jq .deployment_id .databricks/bundle/default/resources.json +title "The deployment ID is the ID of the workspace node the service registered under initial_parent_path; the CLI stores nothing locally" +trace $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record/default/state/resources.deployment.json" | jq '{object_type,path}' +trace jq 'has("deployment_id")' .databricks/bundle/default/resources.json -title "Redeploy after deleting the local cache: the deployment ID is recovered from remote state, the same deployment is reused, and the version increments (no new CreateDeployment)" +title "Redeploy after deleting the local cache: the deployment ID is resolved from that node, the same deployment is reused, and the version increments (no new CreateDeployment)" rm -rf .databricks trace $CLI bundle deploy trace print_requests.py //api/2.0/bundle --sort diff --git a/acceptance/bundle/dms/redeploy-after-destroy/output.txt b/acceptance/bundle/dms/redeploy-after-destroy/output.txt index 5326653519a..ed7b6ede989 100644 --- a/acceptance/bundle/dms/redeploy-after-destroy/output.txt +++ b/acceptance/bundle/dms/redeploy-after-destroy/output.txt @@ -1,5 +1,5 @@ -=== Deploy, then destroy: the deployment record is deleted, but its ID stays behind in the local state file +=== Deploy, then destroy: the deployment record and its workspace node are both deleted, so nothing points at the destroyed deployment >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/files... Deploying resources... @@ -15,35 +15,34 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> jq .deployment_id .databricks/bundle/default/resources.json -"[DESTROYED_DEPLOYMENT_ID]" +>>> musterr [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json +Error: Path (/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json) doesn't exist. -=== Deploy again: the stale deployment ID no longer resolves, so a new deployment is created instead of failing the deploy +=== Deploy again: with no node to resolve, a fresh deployment is created at version 1, with its own workspace node >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/files... Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //api/2.0/bundle --sort --get -{ - "method": "GET", - "path": "/api/2.0/bundle/deployments/[DESTROYED_DEPLOYMENT_ID]" -} +>>> [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json { - "method": "GET", - "path": "/api/2.0/bundle/deployments/[DESTROYED_DEPLOYMENT_ID]" + "object_type": "FILE", + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" } + +>>> print_requests.py //api/2.0/bundle --sort --get { "method": "POST", "path": "/api/2.0/bundle/deployments", "body": { + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state", "target_name": "default" } } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": { "version_id": "1" }, @@ -55,14 +54,14 @@ Deployment complete! } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/complete", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": { "completion_reason": "VERSION_COMPLETE_SUCCESS" } } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": { "resource_key": "jobs.foo" }, @@ -86,7 +85,3 @@ Deployment complete! "status": "OPERATION_STATUS_SUCCEEDED" } } - -=== The new deployment ID replaces the stale one in state ->>> jq .deployment_id .databricks/bundle/default/resources.json -"[UUID]" diff --git a/acceptance/bundle/dms/redeploy-after-destroy/script b/acceptance/bundle/dms/redeploy-after-destroy/script index 628cd7bf494..04c639019c9 100644 --- a/acceptance/bundle/dms/redeploy-after-destroy/script +++ b/acceptance/bundle/dms/redeploy-after-destroy/script @@ -1,15 +1,11 @@ -title "Deploy, then destroy: the deployment record is deleted, but its ID stays behind in the local state file" +title "Deploy, then destroy: the deployment record and its workspace node are both deleted, so nothing points at the destroyed deployment" trace $CLI bundle deploy trace $CLI bundle destroy --auto-approve print_requests.py //api/2.0/bundle --sort --get > /dev/null -destroyed_id=$(jq -r .deployment_id .databricks/bundle/default/resources.json) -add_repl.py "$destroyed_id" DESTROYED_DEPLOYMENT_ID -trace jq .deployment_id .databricks/bundle/default/resources.json +trace musterr $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" -title "Deploy again: the stale deployment ID no longer resolves, so a new deployment is created instead of failing the deploy" +title "Deploy again: with no node to resolve, a fresh deployment is created at version 1, with its own workspace node" trace $CLI bundle deploy +trace $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" | jq '{object_type,path}' trace print_requests.py //api/2.0/bundle --sort --get - -title "The new deployment ID replaces the stale one in state" -trace jq .deployment_id .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/dms/test.toml b/acceptance/bundle/dms/test.toml index 24ce9756629..1e36331a16f 100644 --- a/acceptance/bundle/dms/test.toml +++ b/acceptance/bundle/dms/test.toml @@ -1,9 +1,8 @@ Local = true Cloud = false -# Deployment Metadata Service (DMS) recording is only meaningful in the direct -# engine, where the deployment ID is stored in and read from the direct-engine -# state. +# Deployment Metadata Service (DMS) recording is only supported by the direct +# engine; it is a no-op on terraform. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] RecordRequests = true diff --git a/bundle/configsync/diff.go b/bundle/configsync/diff.go index 17ed3b30d5e..ca5b2c9410b 100644 --- a/bundle/configsync/diff.go +++ b/bundle/configsync/diff.go @@ -149,7 +149,7 @@ func OpenDeploymentState(ctx context.Context, b *bundle.Bundle, engine engine.En deployBundle := &direct.DeploymentBundle{} _, statePath := b.StateFilenameConfigSnapshot(ctx) - if err := deployBundle.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, nil); err != nil { + if err := deployBundle.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil); err != nil { return nil, fmt.Errorf("failed to open state: %w", err) } return deployBundle, nil diff --git a/bundle/configsync/variables.go b/bundle/configsync/variables.go index be3e536f37f..433b607a037 100644 --- a/bundle/configsync/variables.go +++ b/bundle/configsync/variables.go @@ -147,7 +147,7 @@ func resourceIDLookup(ctx context.Context, b *bundle.Bundle) func(string) string } _, statePath := b.StateFilenameConfigSnapshot(ctx) db := &dstate.DeploymentState{} - if err := db.Open(ctx, statePath, dstate.WithRecovery(false), dstate.WithWrite(false), nil, nil); err != nil { + if err := db.Open(ctx, statePath, dstate.WithRecovery(false), dstate.WithWrite(false), nil); err != nil { log.Debugf(ctx, "variable restoration: failed to open state DB at %s: %v", statePath, err) return nil } diff --git a/bundle/direct/bind.go b/bundle/direct/bind.go index ccfbcf788ab..ec910b2734e 100644 --- a/bundle/direct/bind.go +++ b/bundle/direct/bind.go @@ -62,7 +62,7 @@ type BindResult struct { func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.WorkspaceClient, configRoot *config.Root, statePath, resourceKey, resourceID string) (*BindResult, error) { // Check if the resource is already managed (bound to a different ID) var checkStateDB dstate.DeploymentState - if err := checkStateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, nil); err == nil { + if err := checkStateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil); err == nil { existingID := checkStateDB.GetResourceID(resourceKey) if _, err := checkStateDB.Finalize(ctx); err != nil { log.Warnf(ctx, "failed to finalize state: %v", err) @@ -86,7 +86,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } // Open temp state - err := b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(false), dstate.WithWrite(true), nil, nil) + err := b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(false), dstate.WithWrite(true), nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -109,7 +109,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac log.Infof(ctx, "Bound %s to id=%s (in temp state)", resourceKey, resourceID) // First plan + update: populate state with resolved config - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, nil) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -145,7 +145,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } } - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil, nil) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -165,7 +165,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } // Second plan: this is the plan to present to the user (change between remote resource and config) - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, nil) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -215,7 +215,7 @@ func (result *BindResult) Cancel() { // Unbind removes a resource from direct engine state without deleting // the workspace resource. Also removes associated permissions/grants entries. func (b *DeploymentBundle) Unbind(ctx context.Context, statePath, resourceKey string) error { - err := b.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil, nil) + err := b.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil) if err != nil { return err } diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index e09cb859221..0941d87d35c 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -19,13 +19,10 @@ import ( // deployment. Once DMS is authoritative its resource set is trusted even when // empty (a successful deploy with no resources); the file's resources are only // used when DMS has no successful version, or when the user opts out of -// recording deployment history. The caller holds db.mu and has already -// populated db.Data from the file, including the DeploymentID. -// -// cfg is threaded in only for the temporary raw read in -// deploymentHasSuccessfulVersion; see the TODO there. -func (db *DeploymentState) overlayDMSState(ctx context.Context, client bundledeployments.BundleDeploymentsInterface, cfg *sdkconfig.Config) error { - authoritative, err := deploymentHasSuccessfulVersion(ctx, cfg, db.Data.DeploymentID) +// recording deployment history. The caller holds db.mu, has already populated +// db.Data from the file, and has resolved src.DeploymentID. +func (db *DeploymentState) overlayDMSState(ctx context.Context, src *DMSSource) error { + authoritative, err := deploymentHasSuccessfulVersion(ctx, src.Config, src.DeploymentID) if err != nil { return err } @@ -35,7 +32,7 @@ func (db *DeploymentState) overlayDMSState(ctx context.Context, client bundledep return nil } - resources, err := fetchDeploymentResources(ctx, client, db.Data.DeploymentID, db.Data.State) + resources, err := fetchDeploymentResources(ctx, src.Client, src.DeploymentID, db.Data.State) if err != nil { return err } @@ -63,8 +60,7 @@ func (db *DeploymentState) overlayDMSState(ctx context.Context, client bundledep // because last_successful_version_id is still stage:DEVELOPMENT in the proto // and therefore stripped from the generated SDK. Once the field is promoted to // PRIVATE_PREVIEW and regenerated, replace the raw call with -// client.GetDeployment(...).LastSuccessfulVersionId and drop the cfg argument -// (revert overlayDMSState/Open back to taking only the typed client). +// client.GetDeployment(...).LastSuccessfulVersionId and drop DMSSource.Config. func deploymentHasSuccessfulVersion(ctx context.Context, cfg *sdkconfig.Config, deploymentID string) (bool, error) { apiClient, err := client.New(cfg) if err != nil { diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index e5eb44df24a..c27de3ca44d 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -74,11 +74,6 @@ type DeploymentState struct { // Maps resource key to ID. Unlike Data.State, this is up to date during writes (deploys). stateIDs map[string]string - - // headerDirty records that a header field changed in memory during this - // deployment (today only the DMS deployment ID), so the state file must be - // written on Finalize even when the WAL carried no resource entries. - headerDirty bool } type Header struct { @@ -87,13 +82,6 @@ type Header struct { Lineage string `json:"lineage"` Serial int `json:"serial"` - // DeploymentID is the ID the deployment metadata service (DMS) assigned to - // this deployment. Unlike Lineage (a locally generated identifier for the - // state file), it is minted server-side by CreateDeployment and stored here so - // later deploys can find the same DMS deployment record and read its state. - // Empty/omitted until the bundle first records to DMS. - DeploymentID string `json:"deployment_id,omitempty"` - // Features maps each feature flag this state depends on to a (currently empty) // value. This CLI writes no features; it only reads the field to detect a state // that depends on features it lacks and refuse it (see migrateState). It is a @@ -223,33 +211,6 @@ func (db *DeploymentState) GetOrInitLineage() string { return db.Data.Lineage } -// GetDeploymentID returns the DMS deployment ID recorded in the state, or an -// empty string if this bundle has not yet recorded a deployment to DMS. -func (db *DeploymentState) GetDeploymentID() string { - db.mu.Lock() - defer db.mu.Unlock() - return db.Data.DeploymentID -} - -// SetDeploymentID stores the DMS-assigned deployment ID in the in-memory state -// header. It is set during deploy, after CreateDeployment returns the -// server-generated ID, and persisted to the state file by Finalize. Storing it -// on db.Data (not the WAL header, which is written before the ID is known) -// means the subsequent state write carries it forward. -// -// The header is marked dirty so Finalize persists it even when the deploy wrote -// no resource entries; otherwise a bundle with no resources would mint a fresh -// deployment record on every deploy, leaking one orphan per run. -func (db *DeploymentState) SetDeploymentID(id string) { - db.mu.Lock() - defer db.mu.Unlock() - if db.Data.DeploymentID == id { - return - } - db.Data.DeploymentID = id - db.headerDirty = true -} - type ( // If true, then Open reads the WAL and merges it in the state. If false, and WAL is present, Open returns an error. WithRecovery bool @@ -259,19 +220,32 @@ type ( WithWrite bool ) +// DMSSource tells Open to read resource state from the deployment metadata +// service instead of the state file. A nil *DMSSource keeps Open file-only. +type DMSSource struct { + // Client is the DMS client used to list the deployment's resources. + Client bundledeployments.BundleDeploymentsInterface + + // Config accompanies Client (both come from the same workspace client) and is + // used only for a temporary raw read of last_successful_version_id; see the + // TODO in deploymentHasSuccessfulVersion. + Config *sdkconfig.Config + + // DeploymentID identifies the deployment in DMS, resolved from the + // deployment's workspace node (see dms.ResolveDeploymentID). It is empty for a + // bundle that has not recorded a deployment yet. + DeploymentID string +} + // Open reads the deployment state from disk (and recovers the WAL when -// withRecovery is set). When dmsClient is non-nil, the deployment metadata +// withRecovery is set). When dmsSource is non-nil, the deployment metadata // service is the source of truth for resource state: if DMS holds a // successfully completed version for this deployment, the resources read from // the file are replaced with the ones recorded in DMS. The local identity -// (lineage, serial, and deployment ID) always comes from the file, since that -// is what the write path increments and carries forward. A nil dmsClient keeps -// the behavior file-only. -// -// dmsCfg accompanies dmsClient (both come from the same workspace client) and -// is used only for a temporary raw read of last_successful_version_id; see the -// TODO in deploymentHasSuccessfulVersion. -func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, dmsClient bundledeployments.BundleDeploymentsInterface, dmsCfg *sdkconfig.Config) error { +// (lineage and serial) always comes from the file, since that is what the write +// path increments and carries forward. A nil dmsSource keeps the behavior +// file-only. +func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, dmsSource *DMSSource) error { db.mu.Lock() defer db.mu.Unlock() @@ -319,7 +293,7 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W return fmt.Errorf("migrating state %s: %w", path, err) } - if dmsClient != nil { + if dmsSource != nil { // Only deployments that start out empty are recorded in DMS. Resources // tracked in a state file that DMS does not know about are not in DMS and // never will be: the first recorded deploy would create a deployment whose @@ -327,9 +301,9 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W // authoritative for everything (see overlayDMSState). Resources this bundle // already owns would look absent and be created a second time. // - // A state file that DMS already owns (it carries a deployment ID) is fine — - // that is a bundle that opted in while it was still empty. So is a state file - // with no resources, e.g. one left behind by a destroy. + // A deployment DMS already owns (deploymentID is non-empty) is fine — that is + // a bundle that opted in while it was still empty. So is a state file with no + // resources, e.g. one left behind by a destroy. // // TODO(DMS): lift this restriction by upgrading an existing state in place. // That means writing the state at featureStateVersion (3) with a feature flag @@ -339,11 +313,11 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W // exists (see featureStateVersion and Header.Features); once it is written, // this check goes away and record_deployment_history becomes usable on // existing bundles. - if db.Data.DeploymentID == "" && len(db.Data.State) > 0 { + if dmsSource.DeploymentID == "" && len(db.Data.State) > 0 { return fmt.Errorf("cannot record deployment history for a bundle that already has deployed resources tracked in %s: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again", path) } - if db.Data.DeploymentID != "" { - if err := db.overlayDMSState(ctx, dmsClient, dmsCfg); err != nil { + if dmsSource.DeploymentID != "" { + if err := db.overlayDMSState(ctx, dmsSource); err != nil { return err } } @@ -488,12 +462,7 @@ func (db *DeploymentState) mergeWalIntoState(ctx context.Context) (bool, error) } } - hasEntries := lineNumber > 1 - - // A header-only WAL still has to be persisted when a header field changed in - // memory during this deployment (the DMS deployment ID): dropping the write - // would lose the ID and make the next deploy create a second deployment. - persist := hasEntries || db.headerDirty + persist := lineNumber > 1 // Only advance the serial when the state file is actually written, because // the caller (replayWAL) persists it only in that case. A header-only WAL diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index 6530ca049d9..a9c90530514 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -20,91 +20,30 @@ func TestOpenSaveFinalizeRoundTrip(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{"key": "val"}, nil)) mustFinalize(t, &db) // Re-open and verify persisted data. var db2 DeploymentState - require.NoError(t, db2.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, nil)) + require.NoError(t, db2.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) assert.Equal(t, 1, db2.Data.Serial) assert.Equal(t, "123", db2.GetResourceID("jobs.my_job")) mustFinalize(t, &db2) } -func TestDeploymentIDPersistsAcrossOpen(t *testing.T) { - path := filepath.Join(t.TempDir(), "state.json") - - var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) - assert.Empty(t, db.GetDeploymentID()) - - // The deployment ID is set during deploy (after CreateDeployment) and - // persisted by Finalize even though it is not part of the WAL header. - db.SetDeploymentID("server-assigned-id") - require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) - mustFinalize(t, &db) - - var reopened DeploymentState - require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, nil)) - assert.Equal(t, "server-assigned-id", reopened.GetDeploymentID()) - mustFinalize(t, &reopened) -} - func TestFinalizeWithNoEntriesDoesNotWriteStateFile(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) mustFinalize(t, &db) _, err := os.Stat(path) assert.ErrorIs(t, err, os.ErrNotExist) } -func TestDeploymentIDPersistsWithNoResourceEntries(t *testing.T) { - path := filepath.Join(t.TempDir(), "state.json") - - // A bundle with no resources writes no WAL entries, but the deployment ID - // still has to be persisted: otherwise the next deploy sees no ID and creates - // a second deployment record, leaking one per deploy. - var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) - db.SetDeploymentID("server-assigned-id") - mustFinalize(t, &db) - - var reopened DeploymentState - require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, nil)) - assert.Equal(t, "server-assigned-id", reopened.GetDeploymentID()) - assert.Equal(t, 1, reopened.Data.Serial) - mustFinalize(t, &reopened) -} - -func TestSetDeploymentIDToSameValueDoesNotWriteStateFile(t *testing.T) { - path := filepath.Join(t.TempDir(), "state.json") - - var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) - db.SetDeploymentID("server-assigned-id") - require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) - mustFinalize(t, &db) - - before, err := os.ReadFile(path) - require.NoError(t, err) - - // Re-setting the same ID is not a header change, so a deploy that commits - // nothing must not bump the serial (see mergeWalIntoState). - var reopened DeploymentState - require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(true), nil, nil)) - reopened.SetDeploymentID("server-assigned-id") - mustFinalize(t, &reopened) - - after, err := os.ReadFile(path) - require.NoError(t, err) - assert.Equal(t, string(before), string(after)) -} - func TestExportStateFromDataJobRunJobID(t *testing.T) { data := Database{ State: map[string]ResourceEntry{ @@ -154,10 +93,10 @@ func TestPanicOnDoubleOpen(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) assert.Panics(t, func() { - _ = db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil) + _ = db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil) }) mustFinalize(t, &db) } @@ -168,12 +107,12 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { // Commit serial 1 with one resource. var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) mustFinalize(t, &db) var committed DeploymentState - require.NoError(t, committed.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, nil)) + require.NoError(t, committed.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) lineage := committed.Data.Lineage require.Equal(t, 1, committed.Data.Serial) mustFinalize(t, &committed) @@ -189,7 +128,7 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { require.NoError(t, os.WriteFile(walPath, append(headerLine, '\n'), 0o600)) var recovered DeploymentState - require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil, nil)) + require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil)) assert.Equal(t, 1, recovered.Data.Serial) assert.Equal(t, "123", recovered.GetResourceID("jobs.my_job")) assert.NoFileExists(t, walPath) @@ -232,17 +171,17 @@ func TestDeleteState(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) mustFinalize(t, &db) var db2 DeploymentState - require.NoError(t, db2.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) + require.NoError(t, db2.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) require.NoError(t, db2.DeleteState("jobs.my_job")) mustFinalize(t, &db2) var db3 DeploymentState - require.NoError(t, db3.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, nil)) + require.NoError(t, db3.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) assert.Equal(t, 2, db3.Data.Serial) assert.Empty(t, db3.GetResourceID("jobs.my_job")) mustFinalize(t, &db3) @@ -254,7 +193,7 @@ func TestGetOrInitLineageReadableBeforeWriteAndPersisted(t *testing.T) { // Fresh state opened read-only, as the deploy does before planning: no // lineage yet. var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil, nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil)) require.Empty(t, db.Data.Lineage) // GetOrInitLineage initializes the lineage and makes it readable before any @@ -271,7 +210,7 @@ func TestGetOrInitLineageReadableBeforeWriteAndPersisted(t *testing.T) { // Re-open: the persisted lineage matches the one read before the write. var reopened DeploymentState - require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, nil)) + require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) assert.Equal(t, lineage, reopened.Data.Lineage) mustFinalize(t, &reopened) } diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 792c016f963..3d70a218b15 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -169,7 +169,11 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // nothing; the deferred CompleteVersion is a no-op until CreateVersion runs. // CompleteVersion is deferred before lock.Release so it runs while the lock // is still held (defers run last-in-first-out). - recorder := newDeploymentRecorder(ctx, b, stateEngine, dms.VersionTypeDeploy) + recorder, err := newDeploymentRecorder(ctx, b, stateEngine, dms.VersionTypeDeploy) + if err != nil { + logdiag.LogError(ctx, err) + return + } defer func() { if err := recorder.CompleteVersion(ctx, !logdiag.HasError(ctx)); err != nil { logdiag.LogError(ctx, err) @@ -276,11 +280,9 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand return } if recorder != nil { - // On a first deploy the server assigned the deployment ID; persist it in - // state (Finalize writes it to disk) so later deploys reuse the record. // Record operations under the version just created so DMS holds the - // deployed resource state. - b.DeploymentBundle.StateDB.SetDeploymentID(recorder.DeploymentID()) + // deployed resource state. On a first deploy the deployment ID was only + // assigned by CreateVersion above, so this must come after it. b.DeploymentBundle.OpRec = direct.NewOperationRecorder( b.WorkspaceClient(ctx).BundleDeployments, recorder.DeploymentID(), diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 244f593476f..2925e80bca8 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -137,7 +137,11 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { // created until the destroy is approved (below), so a cancelled destroy // records nothing; the deferred CompleteVersion is a no-op until then. It is // deferred before lock.Release so it runs while the lock is still held. - recorder := newDeploymentRecorder(ctx, b, engine, dms.VersionTypeDestroy) + recorder, err := newDeploymentRecorder(ctx, b, engine, dms.VersionTypeDestroy) + if err != nil { + logdiag.LogError(ctx, err) + return + } defer func() { if err := recorder.CompleteVersion(ctx, !logdiag.HasError(ctx)); err != nil { logdiag.LogError(ctx, err) diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index 667ef8627aa..02254237595 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -14,24 +14,31 @@ import ( // // Recording is enabled only when experimental.record_deployment_history is set // AND the engine is direct: DMS resource state is tracked per direct-engine -// deployment, and only the direct engine opens the state DB where the -// deployment ID is stored. Returning nil for terraform leaves those deployments -// untouched. +// deployment. Returning nil for terraform leaves those deployments untouched. // -// The deployment ID passed to the recorder is the one persisted in state from a -// previous deploy; it is empty on a bundle's first recorded deploy, in which -// case the recorder creates the deployment and the server assigns the ID. -func newDeploymentRecorder(ctx context.Context, b *bundle.Bundle, eng engine.EngineType, versionType dms.VersionType) *dms.Recorder { +// The deployment ID is resolved from the workspace rather than from local state +// (see dms.ResolveDeploymentID). The lookup happens here, after the deployment +// lock has been acquired, so it observes any deployment a concurrent deploy +// created. It is empty on a bundle's first recorded deploy, in which case the +// recorder creates the deployment and the server assigns the ID. +func newDeploymentRecorder(ctx context.Context, b *bundle.Bundle, eng engine.EngineType, versionType dms.VersionType) (*dms.Recorder, error) { if b.Config.Experimental == nil || !b.Config.Experimental.RecordDeploymentHistory { - return nil + return nil, nil } if !eng.IsDirect() { - return nil + return nil, nil + } + + statePath := b.Config.Workspace.StatePath + deploymentID, err := dms.ResolveDeploymentID(ctx, b.WorkspaceClient(ctx), statePath) + if err != nil { + return nil, err } return dms.NewRecorder( b.WorkspaceClient(ctx).BundleDeployments, - b.DeploymentBundle.StateDB.GetDeploymentID(), + deploymentID, + statePath, b.Config.Bundle.Target, versionType, - ) + ), nil } diff --git a/cmd/bundle/generate/dashboard.go b/cmd/bundle/generate/dashboard.go index 4866f27c5b3..2b286bcad3d 100644 --- a/cmd/bundle/generate/dashboard.go +++ b/cmd/bundle/generate/dashboard.go @@ -404,7 +404,7 @@ func (d *dashboard) runForResource(ctx context.Context, b *bundle.Bundle) { var state statemgmt.ExportedResourcesMap if stateDesc.Engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, nil); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil); err != nil { logdiag.LogError(ctx, err) return } diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go index b5dbeed6c56..48ecc92a6cd 100644 --- a/cmd/bundle/generate/genie_space.go +++ b/cmd/bundle/generate/genie_space.go @@ -322,7 +322,7 @@ func (g *genieSpace) runForResource(ctx context.Context, b *bundle.Bundle) { var state statemgmt.ExportedResourcesMap if stateDesc.Engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, nil); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil); err != nil { logdiag.LogError(ctx, err) return } diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index f556c2b3450..e9840188db4 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -20,13 +20,12 @@ import ( "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/internal/build" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/sync" "github.com/databricks/cli/libs/telemetry/protos" - sdkconfig "github.com/databricks/databricks-sdk-go/config" - "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/spf13/cobra" ) @@ -215,18 +214,24 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle _, localPath := b.StateFilenameDirect(ctx) // When the bundle records deployment history, the deployment metadata - // service owns resource state, so hand Open its client to overlay DMS - // state on top of the local identity (lineage/serial/deployment ID). - // Reads open the state write-disabled, so no lineage is minted here. - // dmsCfg accompanies the client for a temporary raw read (see the TODO - // in dstate.deploymentHasSuccessfulVersion). - var dmsClient bundledeployments.BundleDeploymentsInterface - var dmsCfg *sdkconfig.Config + // service owns resource state, so hand Open a DMS source to overlay that + // state on top of the local identity (lineage/serial). Reads open the + // state write-disabled, so no lineage is minted here. + var dmsSource *dstate.DMSSource if b.Config.Experimental != nil && b.Config.Experimental.RecordDeploymentHistory { - dmsClient = b.WorkspaceClient(ctx).BundleDeployments - dmsCfg = b.WorkspaceClient(ctx).Config + w := b.WorkspaceClient(ctx) + deploymentID, err := dms.ResolveDeploymentID(ctx, w, b.Config.Workspace.StatePath) + if err != nil { + logdiag.LogError(ctx, err) + return b, stateDesc, root.ErrAlreadyPrinted + } + dmsSource = &dstate.DMSSource{ + Client: w.BundleDeployments, + Config: w.Config, + DeploymentID: deploymentID, + } } - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dmsClient, dmsCfg); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dmsSource); err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } diff --git a/libs/dms/recorder.go b/libs/dms/recorder.go index 1a113491cf9..6fa9a1a24be 100644 --- a/libs/dms/recorder.go +++ b/libs/dms/recorder.go @@ -30,13 +30,15 @@ const ( // Recorder records a single deploy/destroy as a version with DMS. // // The deployment ID is assigned by the server on the first deploy: NewRecorder -// is given the ID persisted in state (empty on a bundle's first-ever recorded -// deploy), and CreateVersion creates the deployment record when that ID is -// empty and exposes the server-assigned ID via DeploymentID so the caller can -// persist it. Later deploys pass the stored ID back in and reuse the record. +// is given the ID resolved from the workspace (empty on a bundle's first-ever +// recorded deploy, see ResolveDeploymentID), and CreateVersion creates the +// deployment record when that ID is empty. Later deploys resolve the same ID +// from the deployment's workspace node and reuse the record; a destroy deletes +// the record and its node, so the next deploy starts over from empty. type Recorder struct { svc bundledeployments.BundleDeploymentsInterface deploymentID string + statePath string targetName string versionType VersionType @@ -46,12 +48,15 @@ type Recorder struct { } // NewRecorder returns a Recorder for the given deployment. deploymentID is the -// DMS deployment ID persisted in state, or empty if this bundle has not yet -// recorded a deployment (the server assigns one during CreateVersion). -func NewRecorder(svc bundledeployments.BundleDeploymentsInterface, deploymentID, targetName string, versionType VersionType) *Recorder { +// ID resolved from the deployment's workspace node, or empty if this bundle has +// not yet recorded a deployment (the server assigns one during CreateVersion). +// statePath is the bundle's remote state directory, under which DMS registers +// the deployment node. +func NewRecorder(svc bundledeployments.BundleDeploymentsInterface, deploymentID, statePath, targetName string, versionType VersionType) *Recorder { return &Recorder{ svc: svc, deploymentID: deploymentID, + statePath: statePath, targetName: targetName, versionType: versionType, } @@ -59,8 +64,7 @@ func NewRecorder(svc bundledeployments.BundleDeploymentsInterface, deploymentID, // DeploymentID returns the DMS deployment ID this recorder is bound to. It is // empty until CreateVersion has created the deployment record (on a first -// deploy) and non-empty afterwards, so callers persist it once CreateVersion -// succeeds. +// deploy) and non-empty afterwards, so callers can parent operations under it. func (r *Recorder) DeploymentID() string { if r == nil { return "" @@ -140,41 +144,40 @@ func (r *Recorder) CompleteVersion(ctx context.Context, success bool) error { } // createDeploymentVersion ensures the deployment record exists, then creates a -// new version under it. When no deployment ID is stored, or the stored one no -// longer exists in DMS, it creates the deployment and lets the server assign the -// ID; otherwise it reads the existing deployment to compute the next version -// number. +// new version under it. With no deployment ID it creates the deployment and lets +// the server assign the ID; otherwise it reads the existing deployment to +// compute the next version number. func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID string, err error) { if r.deploymentID != "" { - // Existing deployment: read it to compute the next version number. + // Existing deployment: read it to compute the next version number. A 404 is + // not recovered from by creating a second deployment: the ID was just + // resolved from the deployment's workspace node, which the service trashes + // when it deletes the record, so a missing record here means the two are out + // of sync and creating another one would collide on the same node path. dep, getErr := r.svc.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ Name: "deployments/" + r.deploymentID, }) - switch { - case getErr == nil: - lastVersion, parseErr := strconv.ParseInt(dep.LastVersionId, 10, 64) - if parseErr != nil { - return "", fmt.Errorf("failed to parse last_version_id %q: %w", dep.LastVersionId, parseErr) - } - versionID = strconv.FormatInt(lastVersion+1, 10) - case errors.Is(getErr, apierr.ErrNotFound): - // The record the state points at is gone: a successful destroy deletes - // it (leaving the ID behind in the local state file), and it can also be - // deleted out of band. Recording must not dead-end on it, so fall back to - // creating a new deployment; the caller persists the new ID. - log.Debugf(ctx, "Deployment %s no longer exists in the deployment metadata service, creating a new one", r.deploymentID) - r.deploymentID = "" - default: + if getErr != nil { return "", fmt.Errorf("failed to get deployment: %w", getErr) } - } - - if r.deploymentID == "" { - // First deploy: create the deployment with an empty ID so the server - // assigns one, then start at version 1. + lastVersion, parseErr := strconv.ParseInt(dep.LastVersionId, 10, 64) + if parseErr != nil { + return "", fmt.Errorf("failed to parse last_version_id %q: %w", dep.LastVersionId, parseErr) + } + versionID = strconv.FormatInt(lastVersion+1, 10) + } else { + // First deploy: create the deployment so the server assigns an ID, then + // start at version 1. + // + // initial_parent_path is required: the service creates the deployment's + // BUNDLE_DEPLOYMENT node under it, and that node's ID becomes the + // deployment ID that ResolveDeploymentID reads back on later deploys. The + // folder must already exist, which it does by this point - the deployment + // lock lives in the same directory. dep, createErr := r.svc.CreateDeployment(ctx, bundledeployments.CreateDeploymentRequest{ Deployment: bundledeployments.Deployment{ - TargetName: r.targetName, + InitialParentPath: r.statePath, + TargetName: r.targetName, }, }) if createErr != nil { diff --git a/libs/dms/recorder_test.go b/libs/dms/recorder_test.go index 9b60078635f..6c2f334c946 100644 --- a/libs/dms/recorder_test.go +++ b/libs/dms/recorder_test.go @@ -12,6 +12,10 @@ import ( "github.com/stretchr/testify/require" ) +// testStatePath is the bundle state directory the recorder registers the +// deployment node under; several tests assert it round-trips to the service. +const testStatePath = "/Workspace/Users/me/.bundle/proj/dev/state" + // fakeDMS records the calls the recorder makes and lets a test script the // server-side responses. It embeds the SDK interface so it satisfies it while // only overriding the methods the recorder uses. @@ -33,11 +37,9 @@ type fakeDMS struct { func (f *fakeDMS) CreateDeployment(ctx context.Context, req bundledeployments.CreateDeploymentRequest) (*bundledeployments.Deployment, error) { f.created = append(f.created, req) - id := req.DeploymentId - if id == "" { - id = f.assignedID - } - return &bundledeployments.Deployment{Name: "deployments/" + id}, nil + // The server always assigns the ID; it is the ID of the workspace node it + // creates under initial_parent_path. + return &bundledeployments.Deployment{Name: "deployments/" + f.assignedID}, nil } func (f *fakeDMS) GetDeployment(ctx context.Context, req bundledeployments.GetDeploymentRequest) (*bundledeployments.Deployment, error) { @@ -66,16 +68,18 @@ func (f *fakeDMS) Heartbeat(ctx context.Context, req bundledeployments.Heartbeat func TestRecorderFirstDeployCreatesDeploymentWithServerAssignedID(t *testing.T) { f := &fakeDMS{assignedID: "server-generated-id"} - // A first deploy has no stored deployment ID. - r := NewRecorder(f, "", "dev", VersionTypeDeploy) + // A first deploy resolves no deployment ID from the workspace. + r := NewRecorder(f, "", testStatePath, "dev", VersionTypeDeploy) require.NoError(t, r.CreateVersion(t.Context())) - // The deployment was created with an empty ID so the server assigns one, and - // the recorder exposes the assigned ID for the caller to persist. + // The server assigned the ID, and the recorder exposes it for the rest of the + // deploy (it parents the operations recorded under this version). require.Len(t, f.created, 1) - assert.Empty(t, f.created[0].DeploymentId) assert.Equal(t, "server-generated-id", r.DeploymentID()) + // initial_parent_path is required: the service creates the deployment node + // under it, and that node is what ResolveDeploymentID looks up later. + assert.Equal(t, testStatePath, f.created[0].Deployment.InitialParentPath) // The first version is 1, parented under the assigned deployment. require.Len(t, f.versions, 1) @@ -96,7 +100,7 @@ func TestRecorderSubsequentDeployReusesDeploymentAndIncrementsVersion(t *testing }, } // A subsequent deploy passes the stored deployment ID. - r := NewRecorder(f, "stored-id", "dev", VersionTypeDeploy) + r := NewRecorder(f, "stored-id", testStatePath, "dev", VersionTypeDeploy) require.NoError(t, r.CreateVersion(t.Context())) @@ -107,39 +111,28 @@ func TestRecorderSubsequentDeployReusesDeploymentAndIncrementsVersion(t *testing assert.Equal(t, "stored-id", r.DeploymentID()) } -func TestRecorderStaleDeploymentIDCreatesNewDeployment(t *testing.T) { - f := &fakeDMS{ - assignedID: "fresh-id", - getDeployment: func(id string) (*bundledeployments.Deployment, error) { - return nil, fmt.Errorf("deployment %s: %w", id, apierr.ErrNotFound) - }, - } - // A deploy after a destroy still has the destroyed deployment's ID in state, - // but the record is gone. Recording must recover rather than fail the deploy. - r := NewRecorder(f, "destroyed-id", "dev", VersionTypeDeploy) - - require.NoError(t, r.CreateVersion(t.Context())) - - require.Len(t, f.created, 1) - assert.Equal(t, "fresh-id", r.DeploymentID()) - require.Len(t, f.versions, 1) - assert.Equal(t, "1", f.versions[0].VersionId) - assert.Equal(t, "deployments/fresh-id", f.versions[0].Parent) -} - func TestRecorderGetDeploymentErrorFailsDeploy(t *testing.T) { - f := &fakeDMS{ - getDeployment: func(id string) (*bundledeployments.Deployment, error) { - return nil, errors.New("boom") - }, + cases := map[string]error{ + // A resolved ID whose record is missing means the record and the workspace + // node it was resolved from are out of sync. Creating a second deployment + // would collide on the same node path, so fail instead. + "not found": fmt.Errorf("deployment: %w", apierr.ErrNotFound), + "other": errors.New("boom"), + } + for name, getErr := range cases { + t.Run(name, func(t *testing.T) { + f := &fakeDMS{ + getDeployment: func(id string) (*bundledeployments.Deployment, error) { + return nil, getErr + }, + } + r := NewRecorder(f, "stored-id", testStatePath, "dev", VersionTypeDeploy) + + err := r.CreateVersion(t.Context()) + assert.ErrorContains(t, err, "failed to get deployment") + assert.Empty(t, f.created) + }) } - r := NewRecorder(f, "stored-id", "dev", VersionTypeDeploy) - - // Only a missing deployment is recovered from; any other read failure is fatal - // rather than silently forking a second deployment record. - err := r.CreateVersion(t.Context()) - assert.ErrorContains(t, err, "failed to get deployment") - assert.Empty(t, f.created) } func TestRecorderDestroyDeletesDeploymentOnSuccess(t *testing.T) { @@ -148,7 +141,7 @@ func TestRecorderDestroyDeletesDeploymentOnSuccess(t *testing.T) { return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "2"}, nil }, } - r := NewRecorder(f, "stored-id", "dev", VersionTypeDestroy) + r := NewRecorder(f, "stored-id", testStatePath, "dev", VersionTypeDestroy) require.NoError(t, r.CreateVersion(t.Context())) assert.Equal(t, bundledeployments.VersionTypeVersionTypeDestroy, f.versions[0].Version.VersionType) @@ -164,7 +157,7 @@ func TestRecorderFailedDestroyKeepsDeployment(t *testing.T) { return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "2"}, nil }, } - r := NewRecorder(f, "stored-id", "dev", VersionTypeDestroy) + r := NewRecorder(f, "stored-id", testStatePath, "dev", VersionTypeDestroy) require.NoError(t, r.CreateVersion(t.Context())) require.NoError(t, r.CompleteVersion(t.Context(), false)) @@ -184,7 +177,7 @@ func TestNilRecorderIsNoOp(t *testing.T) { func TestRecorderCompleteVersionNoOpWithoutCreateVersion(t *testing.T) { f := &fakeDMS{} - r := NewRecorder(f, "stored-id", "dev", VersionTypeDeploy) + r := NewRecorder(f, "stored-id", testStatePath, "dev", VersionTypeDeploy) // CompleteVersion before CreateVersion is a no-op (nothing was claimed). require.NoError(t, r.CompleteVersion(t.Context(), true)) assert.Empty(t, f.completed) diff --git a/libs/dms/resolve.go b/libs/dms/resolve.go new file mode 100644 index 00000000000..0d26448e558 --- /dev/null +++ b/libs/dms/resolve.go @@ -0,0 +1,45 @@ +package dms + +import ( + "context" + "errors" + "fmt" + "path" + "strconv" + + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" +) + +// DeploymentNodeName is the workspace node DMS creates for a deployment. The +// name is fixed for every deployment: the node *is* the bundle's state file. +// It must match DeploymentWhsClient.DEPLOYMENT_NODE_NAME on the service side. +const DeploymentNodeName = "resources.deployment.json" + +// ResolveDeploymentID returns the DMS deployment ID for the bundle whose state +// lives under statePath, or an empty string when the bundle has no deployment +// recorded yet. +// +// The ID is not stored anywhere by the CLI. DMS registers each deployment as a +// BUNDLE_DEPLOYMENT node at statePath/resources.deployment.json, and the +// workspace-assigned node ID *is* the deployment ID (see DeploymentHandler: +// deploymentId = Long.toString(createdNode.getId())). So a get-status on that +// path is the lookup, which keeps the workspace the single source of truth: a +// deployment that was destroyed or deleted out of band reports absent here +// rather than leaving a dangling ID behind in the local state file. +func ResolveDeploymentID(ctx context.Context, w *databricks.WorkspaceClient, statePath string) (string, error) { + nodePath := path.Join(statePath, DeploymentNodeName) + + obj, err := w.Workspace.GetStatusByPath(ctx, nodePath) + if err != nil { + if errors.Is(err, apierr.ErrNotFound) || errors.Is(err, apierr.ErrResourceDoesNotExist) { + return "", nil + } + return "", fmt.Errorf("looking up deployment at %s: %w", nodePath, err) + } + + if obj.ObjectId == 0 { + return "", fmt.Errorf("deployment at %s has no object ID", nodePath) + } + return strconv.FormatInt(obj.ObjectId, 10), nil +} diff --git a/libs/dms/resolve_test.go b/libs/dms/resolve_test.go new file mode 100644 index 00000000000..24d5303983d --- /dev/null +++ b/libs/dms/resolve_test.go @@ -0,0 +1,67 @@ +package dms + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// newTestClient returns a workspace client pointed at a server that serves a +// single get-status response. +func newTestClient(t *testing.T, statusCode int, body string) *databricks.WorkspaceClient { + t.Helper() + + var gotPath string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Query().Get("path") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(statusCode) + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(srv.Close) + t.Cleanup(func() { + assert.Equal(t, "/Workspace/state/"+DeploymentNodeName, gotPath) + }) + + w, err := databricks.NewWorkspaceClient(&databricks.Config{ + Host: srv.URL, + Token: "token", + Credentials: config.PatCredentials{}, + }) + require.NoError(t, err) + return w +} + +func TestResolveDeploymentIDReturnsNodeID(t *testing.T) { + w := newTestClient(t, http.StatusOK, `{"object_type":"FILE","object_id":123456789,"path":"/Workspace/state/`+DeploymentNodeName+`"}`) + + // The workspace node ID is the deployment ID, so no local state is consulted. + id, err := ResolveDeploymentID(t.Context(), w, "/Workspace/state") + require.NoError(t, err) + assert.Equal(t, "123456789", id) +} + +func TestResolveDeploymentIDAbsentWhenNodeMissing(t *testing.T) { + w := newTestClient(t, http.StatusNotFound, `{"error_code":"RESOURCE_DOES_NOT_EXIST","message":"Path (/Workspace/state/`+DeploymentNodeName+`) doesn't exist."}`) + + // A bundle that never recorded a deployment, or whose deployment was + // destroyed (the service trashes the node), has no ID rather than an error. + id, err := ResolveDeploymentID(t.Context(), w, "/Workspace/state") + require.NoError(t, err) + assert.Empty(t, id) +} + +func TestResolveDeploymentIDPropagatesOtherErrors(t *testing.T) { + w := newTestClient(t, http.StatusForbidden, `{"error_code":"PERMISSION_DENIED","message":"nope"}`) + + // Anything other than a missing node is fatal: silently treating it as absent + // would create a second deployment for a bundle that already has one. + _, err := ResolveDeploymentID(t.Context(), w, "/Workspace/state") + require.Error(t, err) + assert.ErrorContains(t, err, "looking up deployment at /Workspace/state/"+DeploymentNodeName) +} diff --git a/libs/testserver/bundle.go b/libs/testserver/bundle.go index a1b0cba24a9..6dffbe34a75 100644 --- a/libs/testserver/bundle.go +++ b/libs/testserver/bundle.go @@ -3,15 +3,23 @@ package testserver import ( "bytes" "encoding/json" + "path" "slices" "strconv" "github.com/databricks/databricks-sdk-go/service/bundledeployments" + "github.com/databricks/databricks-sdk-go/service/workspace" ) // Handlers for the Deployment Metadata Service (DMS) API under /api/2.0/bundle. // State is kept in FakeWorkspace.dmsDeployments, keyed by deployment ID. +// dmsDeploymentNodeName is the name of the workspace node the service creates +// for every deployment. It must match DEPLOYMENT_NODE_NAME on the service side +// (DeploymentWhsClient); the literal is repeated here rather than shared with +// the CLI so a test would catch the CLI drifting from the service. +const dmsDeploymentNodeName = "resources.deployment.json" + // dmsDeployment holds a deployment record together with the versions and // resources recorded under it, so the read APIs (ListVersions/ListResources) // can serve back what deploys wrote. @@ -27,29 +35,49 @@ type dmsDeployment struct { // value as "DMS owns the state". Tracked separately because the SDK // Deployment struct does not yet carry the field (still stage:DEVELOPMENT). lastSuccessfulVersionID string + // nodePath is the workspace node whose object ID is this deployment's ID. + // Kept so DeleteDeployment can trash the node, the way the service does. + nodePath string } func (s *FakeWorkspace) CreateDeployment(req Request) Response { - // The client either supplies the deployment ID or, in the server-generated - // flow, leaves it empty for the server to mint one. - deploymentID := req.URL.Query().Get("deployment_id") - if deploymentID == "" { - deploymentID = nextUUID() - } - var dep bundledeployments.Deployment if err := json.Unmarshal(req.Body, &dep); err != nil { return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} } + if dep.InitialParentPath == "" { + return Response{ + StatusCode: 400, + Body: map[string]string{"error_code": "INVALID_PARAMETER_VALUE", "message": "initial_parent_path is required"}, + } + } defer s.LockUnlock()() + // The service registers the deployment as a workspace node under + // initial_parent_path and uses that node's ID as the deployment ID, so a + // get-status on the node path is how clients look the deployment back up. + nodePath := path.Join(dep.InitialParentPath, dmsDeploymentNodeName) + if resp, ok := s.requireParentDirectory(nodePath); !ok { + return resp + } + objectID := nextID() + s.files[nodePath] = FileEntry{ + Info: workspace.ObjectInfo{ + ObjectType: "FILE", + Path: nodePath, + ObjectId: objectID, + }, + } + + deploymentID := strconv.FormatInt(objectID, 10) dep.Name = "deployments/" + deploymentID dep.Status = bundledeployments.DeploymentStatusDeploymentStatusActive s.dmsDeployments[deploymentID] = &dmsDeployment{ deployment: dep, versions: map[string]*bundledeployments.Version{}, resources: map[string]bundledeployments.Resource{}, + nodePath: nodePath, } return Response{Body: dep} } @@ -99,6 +127,11 @@ func deploymentBody(d *dmsDeployment) (map[string]any, error) { func (s *FakeWorkspace) DeleteDeployment(deploymentID string) Response { defer s.LockUnlock()() + // The service trashes the deployment's workspace node, so a later get-status + // on the node path reports the deployment as absent. + if d, ok := s.dmsDeployments[deploymentID]; ok { + delete(s.files, d.nodePath) + } delete(s.dmsDeployments, deploymentID) return Response{Body: map[string]any{}} } From 4009eb4ae58bd899bfe6d8210995823e73a42f7a Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 28 Jul 2026 00:30:33 +0000 Subject: [PATCH 12/28] bundle: address review of the DMS state PR - Limit recorded state to 64 KB, checked when the payload is built so an oversized resource fails itself rather than the drain at close. - Collapse the operation queue's pending/inflight pair into one `owned` set. The two maps encoded a single question ("is this key already claimed?") and had to be read together; `take` now only releases ownership. - Only log "Coalescing" when an operation was actually merged. The old code logged it for in-flight keys too, where nothing was coalesced. - Restore mergeWalIntoState's `hasEntries` naming and comment from main. The `persist` rename existed for the headerDirty case, which is gone. - Trim the comments added by this PR. Also note in fetchDeploymentResources that DMS has no field for dependency edges and they cannot be recovered from the recorded state (references are resolved to literals before serialization), so depends_on is carried over from the local state file. Co-authored-by: Isaac --- bundle/direct/dstate/dms.go | 41 ++++++++----------- bundle/direct/dstate/state.go | 74 +++++++++++++---------------------- bundle/direct/opqueue.go | 64 +++++++++++++++--------------- bundle/direct/opqueue_test.go | 15 ++++++- bundle/direct/oprecorder.go | 23 +++++++---- bundle/phases/dms.go | 9 ++--- libs/dms/recorder.go | 30 ++++++-------- libs/dms/resolve.go | 19 ++++----- 8 files changed, 129 insertions(+), 146 deletions(-) diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index 0941d87d35c..8709ceec57f 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -15,12 +15,9 @@ import ( ) // overlayDMSState replaces the file-derived resource state with the state -// recorded in the deployment metadata service (DMS), when DMS owns this -// deployment. Once DMS is authoritative its resource set is trusted even when -// empty (a successful deploy with no resources); the file's resources are only -// used when DMS has no successful version, or when the user opts out of -// recording deployment history. The caller holds db.mu, has already populated -// db.Data from the file, and has resolved src.DeploymentID. +// recorded in DMS, when DMS owns this deployment. An authoritative DMS is +// trusted even when its resource set is empty (a successful deploy of nothing). +// The caller holds db.mu and has already populated db.Data from the file. func (db *DeploymentState) overlayDMSState(ctx context.Context, src *DMSSource) error { authoritative, err := deploymentHasSuccessfulVersion(ctx, src.Config, src.DeploymentID) if err != nil { @@ -45,21 +42,13 @@ func (db *DeploymentState) overlayDMSState(ctx context.Context, src *DMSSource) return nil } -// deploymentHasSuccessfulVersion reports whether DMS holds a successfully -// completed version for the deployment. It is the signal that DMS owns the -// state: if the deployment was never recorded to DMS, or its initial DMS deploy -// did not complete successfully, DMS state is absent or partial and Open keeps -// the local file's resources instead. +// deploymentHasSuccessfulVersion reports whether DMS owns the state. The server +// advances last_successful_version_id only when a version completes (unlike +// last_version_id, which also advances on failure), so a non-empty value means +// DMS holds a complete resource set. Otherwise Open keeps the file's resources. // -// The deployment carries last_successful_version_id, which the server advances -// only when a version completes successfully (unlike last_version_id, which -// also advances on failure). So a non-empty value is exactly the "DMS owns the -// state" signal, readable in a single GetDeployment. -// -// TODO(DMS): this reads the deployment via a raw GET into a local struct -// because last_successful_version_id is still stage:DEVELOPMENT in the proto -// and therefore stripped from the generated SDK. Once the field is promoted to -// PRIVATE_PREVIEW and regenerated, replace the raw call with +// TODO(DMS): raw GET because last_successful_version_id is stage:DEVELOPMENT and +// stripped from the generated SDK. Once it ships, use // client.GetDeployment(...).LastSuccessfulVersionId and drop DMSSource.Config. func deploymentHasSuccessfulVersion(ctx context.Context, cfg *sdkconfig.Config, deploymentID string) (bool, error) { apiClient, err := client.New(cfg) @@ -88,10 +77,14 @@ func deploymentHasSuccessfulVersion(ctx context.Context, cfg *sdkconfig.Config, // fetchDeploymentResources lists every resource recorded for the deployment in // DMS and maps them into state entries keyed by the fully-qualified resource key. // -// DMS does not record dependency edges, so depends_on is carried over from the -// local state entry for the same key. It is derived from the local config on -// every deploy and is only consumed for delete ordering, so falling back to an -// empty list when the local state has no entry is safe. +// DMS has no field for dependency edges, and they cannot be recovered from the +// recorded state either: references are resolved to literals before it is +// serialized. So depends_on is carried over from the local state file. +// +// TODO(DMS): resources present in DMS but not in the local file therefore get no +// depends_on. Plan recomputes it from config for everything it still declares, +// so this only affects deletes of resources dropped from config, which are +// ordered arbitrarily among themselves. func fetchDeploymentResources(ctx context.Context, client bundledeployments.BundleDeploymentsInterface, deploymentID string, local map[string]ResourceEntry) (map[string]ResourceEntry, error) { it := client.ListResources(ctx, bundledeployments.ListResourcesRequest{ Parent: "deployments/" + deploymentID, diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index c27de3ca44d..0359feba89a 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -223,28 +223,22 @@ type ( // DMSSource tells Open to read resource state from the deployment metadata // service instead of the state file. A nil *DMSSource keeps Open file-only. type DMSSource struct { - // Client is the DMS client used to list the deployment's resources. Client bundledeployments.BundleDeploymentsInterface - // Config accompanies Client (both come from the same workspace client) and is - // used only for a temporary raw read of last_successful_version_id; see the - // TODO in deploymentHasSuccessfulVersion. + // Config is only for a temporary raw read of last_successful_version_id; see + // the TODO in deploymentHasSuccessfulVersion. Config *sdkconfig.Config - // DeploymentID identifies the deployment in DMS, resolved from the - // deployment's workspace node (see dms.ResolveDeploymentID). It is empty for a - // bundle that has not recorded a deployment yet. + // DeploymentID is resolved from the deployment's workspace node (see + // dms.ResolveDeploymentID), and empty before the first recorded deploy. DeploymentID string } // Open reads the deployment state from disk (and recovers the WAL when -// withRecovery is set). When dmsSource is non-nil, the deployment metadata -// service is the source of truth for resource state: if DMS holds a -// successfully completed version for this deployment, the resources read from -// the file are replaced with the ones recorded in DMS. The local identity -// (lineage and serial) always comes from the file, since that is what the write -// path increments and carries forward. A nil dmsSource keeps the behavior -// file-only. +// withRecovery is set). With a non-nil dmsSource, resources come from DMS +// instead of the file whenever DMS holds a successful version. Lineage and +// serial always come from the file, since that is what the write path +// increments. func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, dmsSource *DMSSource) error { db.mu.Lock() defer db.mu.Unlock() @@ -294,25 +288,15 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W } if dmsSource != nil { - // Only deployments that start out empty are recorded in DMS. Resources - // tracked in a state file that DMS does not know about are not in DMS and - // never will be: the first recorded deploy would create a deployment whose - // resource set covers only what that deploy touched, and DMS would then be - // authoritative for everything (see overlayDMSState). Resources this bundle - // already owns would look absent and be created a second time. + // Only bundles that start out empty can be recorded. Once DMS owns a + // deployment it is authoritative for the whole resource set (see + // overlayDMSState), so pre-existing resources it never saw would look absent + // and get created a second time. // - // A deployment DMS already owns (deploymentID is non-empty) is fine — that is - // a bundle that opted in while it was still empty. So is a state file with no - // resources, e.g. one left behind by a destroy. - // - // TODO(DMS): lift this restriction by upgrading an existing state in place. - // That means writing the state at featureStateVersion (3) with a feature flag - // recording that DMS owns it, plus a tombstone entry per resource so a CLI - // that predates DMS refuses the state instead of silently deploying against a - // resource set it cannot see. The feature-flag scaffolding for this already - // exists (see featureStateVersion and Header.Features); once it is written, - // this check goes away and record_deployment_history becomes usable on - // existing bundles. + // TODO(DMS): allow this by upgrading the state in place, writing it at + // featureStateVersion with a feature flag plus a tombstone per resource so an + // older CLI refuses the state instead of deploying against resources it + // cannot see. if dmsSource.DeploymentID == "" && len(db.Data.State) > 0 { return fmt.Errorf("cannot record deployment history for a bundle that already has deployed resources tracked in %s: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again", path) } @@ -364,7 +348,7 @@ func (db *DeploymentState) OpenWithData(path string, data Database) { func (db *DeploymentState) replayWAL(ctx context.Context) error { walPath := db.Path + walSuffix - persist, err := db.mergeWalIntoState(ctx) + hasEntries, err := db.mergeWalIntoState(ctx) if err != nil { if errors.Is(err, errStaleWAL) { log.Debugf(ctx, "Deleting stale WAL file %s", walPath) @@ -373,7 +357,7 @@ func (db *DeploymentState) replayWAL(ctx context.Context) error { } return fmt.Errorf("WAL recovery failed: %w", err) } - if persist { + if hasEntries { if err := db.unlockedSave(); err != nil { return err } @@ -384,9 +368,6 @@ func (db *DeploymentState) replayWAL(ctx context.Context) error { return nil } -// mergeWalIntoState replays the WAL into db.Data and reports whether the caller -// must persist the state file: either the WAL carried resource entries, or a -// header field changed in memory during this deployment. func (db *DeploymentState) mergeWalIntoState(ctx context.Context) (bool, error) { if db.walFile != nil { panic("internal error: walFile must be closed") @@ -462,20 +443,19 @@ func (db *DeploymentState) mergeWalIntoState(ctx context.Context) (bool, error) } } - persist := lineNumber > 1 + hasEntries := lineNumber > 1 - // Only advance the serial when the state file is actually written, because - // the caller (replayWAL) persists it only in that case. A header-only WAL - // that changed nothing is a deploy that started but committed nothing; - // advancing the serial for it leaves the in-memory serial ahead of the - // persisted one, so the next deploy writes its WAL header at serial+2 and - // recovery rejects it as "ahead of expected". - // See acceptance/bundle/deploy/wal/header-only-wal. - if persist { + // Only advance the serial when the WAL carried entries, because the caller + // (replayWAL) persists the new state file only in that case. A header-only + // WAL is a deploy that started but committed nothing; advancing the serial + // for it leaves the in-memory serial ahead of the persisted one, so the + // next deploy writes its WAL header at serial+2 and recovery rejects it as + // "ahead of expected". See acceptance/bundle/deploy/wal/header-only-wal. + if hasEntries { db.Data.Serial = newSerial } - return persist, nil + return hasEntries, nil } // Finalize replays the WAL (if open for write), captures the resulting state, and resets. diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go index 72a2fa9c39e..68e8e2e9f96 100644 --- a/bundle/direct/opqueue.go +++ b/bundle/direct/opqueue.go @@ -22,19 +22,15 @@ const ( ) // operationQueue uploads recorded operations from background workers, so an apply -// worker does not wait for the CreateOperation round trip before moving on to the -// next resource. +// worker does not wait for the CreateOperation round trip. // -// It guarantees at most one upload in flight per resource key: within a key the -// worker that owns it uploads sequentially, so the last operation recorded for a -// resource is also the last one the service sees. +// At most one upload is in flight per resource key, so the last operation +// recorded for a resource is also the last one the service sees. // -// Uploads are not fire-and-forget: close drains the queue and returns the first -// failure, which fails the deploy. That matters because a successfully completed -// version makes DMS the source of truth for resource state (see -// dstate.overlayDMSState); silently dropping an operation would leave DMS with an -// incomplete resource set, and the next deploy would plan to create resources -// that already exist. +// Uploads are not fire-and-forget: close returns the first failure and fails the +// deploy. A dropped operation would leave DMS with an incomplete resource set, +// and since DMS then becomes the source of truth (see dstate.overlayDMSState), +// the next deploy would recreate resources that already exist. type operationQueue struct { uploader operationUploader @@ -51,10 +47,11 @@ type operationQueue struct { // picked up yet. pending map[string]recordedOperation - // inflight holds the resource keys a worker currently owns. A key that is - // in flight is not queued again: the owning worker re-checks pending after its - // upload and picks up anything recorded in the meantime. - inflight map[string]bool + // owned holds the resource keys that are already queued or being uploaded. + // Such a key is never queued a second time; recording writes to pending + // instead, which the owning worker re-checks after each upload. This is what + // keeps uploads for one resource sequential. + owned map[string]bool err error closed bool @@ -74,7 +71,7 @@ func newOperationQueue(ctx context.Context, uploader operationUploader) *operati uploader: uploader, queue: make(chan string, operationQueueSize), pending: make(map[string]recordedOperation), - inflight: make(map[string]bool), + owned: make(map[string]bool), } q.wg.Add(operationUploadWorkers) @@ -85,16 +82,14 @@ func newOperationQueue(ctx context.Context, uploader operationUploader) *operati return q } -// record serializes an operation and queues it for upload. It performs no API -// call, so upload failures surface from close rather than here; the error -// returned is only about turning the applied resource into a payload. +// record serializes an operation and queues it for upload. It makes no API call, +// so upload failures surface from close; an error here only means the applied +// resource could not be turned into a payload. // -// When an operation for the same resource is already waiting it is replaced -// instead of queued again: DMS keeps one state per resource key, so the later -// operation supersedes the earlier one and a single upload records both. The -// merged operation keeps the action of a queued create (see mergeAction), so -// collapsing a create and a later update still records a create. This is best -// effort - only operations that have not been picked up yet are collapsed. +// An operation for a resource that is still waiting replaces it rather than +// queueing again: DMS keeps one state per key, so one upload records both. The +// merge keeps a queued create's action (see mergeAction). Best effort — only +// operations no worker has picked up yet are collapsed. func (q *operationQueue) record(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state any) error { if q == nil { return nil @@ -107,15 +102,21 @@ func (q *operationQueue) record(ctx context.Context, resourceKey string, action q.mu.Lock() queued, waiting := q.pending[resourceKey] - owned := waiting || q.inflight[resourceKey] if waiting { op.action = mergeAction(queued.action, op.action) } q.pending[resourceKey] = op + owned := q.owned[resourceKey] + q.owned[resourceKey] = true q.mu.Unlock() - if owned { + if waiting { log.Debugf(ctx, "Coalescing queued deployment operation for %s", resourceKey) + } + + // Someone already owns this key, so pending is enough: a worker will pick the + // operation up. Queueing again would upload the resource twice in parallel. + if owned { return nil } @@ -167,21 +168,20 @@ func (q *operationQueue) work(ctx context.Context) { } } -// take claims the operation waiting for resourceKey, marking the key in flight so -// record does not queue it a second time. It reports false, and releases the key, -// when nothing is waiting. +// take claims the operation waiting for resourceKey. It reports false and gives +// up ownership when nothing is waiting, which is what lets the next record +// queue the key again. func (q *operationQueue) take(resourceKey string) (recordedOperation, bool) { q.mu.Lock() defer q.mu.Unlock() op, ok := q.pending[resourceKey] if !ok { - delete(q.inflight, resourceKey) + delete(q.owned, resourceKey) return recordedOperation{}, false } delete(q.pending, resourceKey) - q.inflight[resourceKey] = true return op, true } diff --git a/bundle/direct/opqueue_test.go b/bundle/direct/opqueue_test.go index a8e05141fc7..cdd1083a06d 100644 --- a/bundle/direct/opqueue_test.go +++ b/bundle/direct/opqueue_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "strconv" + "strings" "sync" "testing" @@ -153,6 +154,18 @@ func TestOperationQueueRecordRejectsUnsupportedAction(t *testing.T) { assert.Empty(t, f.recorded()) } +func TestOperationQueueRecordRejectsOversizedState(t *testing.T) { + f := &fakeUploader{} + q := newOperationQueue(t.Context(), f) + + big := map[string]string{"name": strings.Repeat("x", maxOperationStateSize)} + err := q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", big) + require.ErrorContains(t, err, "exceeds the 65536 byte limit") + + require.NoError(t, q.close()) + assert.Empty(t, f.recorded()) +} + func TestOperationQueueCloseIsIdempotent(t *testing.T) { f := &fakeUploader{err: errors.New("boom")} q := newOperationQueue(t.Context(), f) @@ -226,7 +239,7 @@ func TestOperationQueueUploadsOneResourceAtATime(t *testing.T) { // Every distinct key was recorded, and close drained all of them. assert.Len(t, u.last, distinctKeyMod) assert.Empty(t, q.pending) - assert.Empty(t, q.inflight) + assert.Empty(t, q.owned) } func TestNilOperationQueueIsNoOp(t *testing.T) { diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index 63d0c84a621..3b3d631f4a9 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -12,6 +12,11 @@ import ( "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) +// maxOperationStateSize is the largest serialized state DMS accepts per +// operation. Uploading more is rejected server-side, so fail early with a +// message that names the resource. +const maxOperationStateSize = 64 * 1024 + // recordedOperation is an applied resource operation, serialized and waiting to be // uploaded to the deployment metadata service (DMS). // @@ -28,7 +33,8 @@ type recordedOperation struct { } // newRecordedOperation serializes an applied operation for upload. state is the -// local config after the operation and must be nil for delete operations. +// local config after the operation and must be nil for delete operations. It +// errors when the serialized state exceeds maxOperationStateSize. func newRecordedOperation(action deployplan.ActionType, resourceID string, state any) (recordedOperation, error) { actionType, err := deployActionToSDK(action) if err != nil { @@ -37,19 +43,20 @@ func newRecordedOperation(action deployplan.ActionType, resourceID string, state op := recordedOperation{action: actionType, resourceID: resourceID} - // The DMS Operation.State field carries the serialized config so the backend - // can serve it as resource state. It is intentionally left unset for delete, - // where the resource no longer exists. + // Operation.State carries the serialized config, which DMS serves back as + // resource state. Unset for delete: the resource is gone. // - // Redact sensitive fields, matching what dstate.SaveState writes to the local - // state file: DMS state is read back as resource state, so recording secrets - // in plaintext would both leak them to the service and reintroduce them into - // a local state file via the read path. + // Redact secrets, like dstate.SaveState does for the local state file: + // otherwise we leak them to the service and the read path writes them back + // into a local state file in plaintext. if state != nil { raw, err := structwalk.RedactSensitiveFields(state, dyn.SensitiveValueRedacted) if err != nil { return recordedOperation{}, fmt.Errorf("serializing state: %w", err) } + if len(raw) > maxOperationStateSize { + return recordedOperation{}, fmt.Errorf("serialized state is %d bytes, which exceeds the %d byte limit for recording deployment history", len(raw), maxOperationStateSize) + } op.state = raw } diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index 02254237595..3d2f4f54009 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -16,11 +16,10 @@ import ( // AND the engine is direct: DMS resource state is tracked per direct-engine // deployment. Returning nil for terraform leaves those deployments untouched. // -// The deployment ID is resolved from the workspace rather than from local state -// (see dms.ResolveDeploymentID). The lookup happens here, after the deployment -// lock has been acquired, so it observes any deployment a concurrent deploy -// created. It is empty on a bundle's first recorded deploy, in which case the -// recorder creates the deployment and the server assigns the ID. +// The deployment ID is resolved from the workspace, not local state (see +// dms.ResolveDeploymentID). The lookup happens here, after the deployment lock is +// held, so it sees any deployment a concurrent deploy created. It is empty on the +// first recorded deploy, where the recorder creates the deployment instead. func newDeploymentRecorder(ctx context.Context, b *bundle.Bundle, eng engine.EngineType, versionType dms.VersionType) (*dms.Recorder, error) { if b.Config.Experimental == nil || !b.Config.Experimental.RecordDeploymentHistory { return nil, nil diff --git a/libs/dms/recorder.go b/libs/dms/recorder.go index 6fa9a1a24be..83f180ba3be 100644 --- a/libs/dms/recorder.go +++ b/libs/dms/recorder.go @@ -29,12 +29,10 @@ const ( // Recorder records a single deploy/destroy as a version with DMS. // -// The deployment ID is assigned by the server on the first deploy: NewRecorder -// is given the ID resolved from the workspace (empty on a bundle's first-ever -// recorded deploy, see ResolveDeploymentID), and CreateVersion creates the -// deployment record when that ID is empty. Later deploys resolve the same ID -// from the deployment's workspace node and reuse the record; a destroy deletes -// the record and its node, so the next deploy starts over from empty. +// The server assigns the deployment ID on the first deploy, i.e. when the ID +// resolved from the workspace is empty (see ResolveDeploymentID). Later deploys +// resolve the same ID and reuse the record; a destroy deletes the record and its +// node, so the next deploy starts over from empty. type Recorder struct { svc bundledeployments.BundleDeploymentsInterface deploymentID string @@ -150,10 +148,10 @@ func (r *Recorder) CompleteVersion(ctx context.Context, success bool) error { func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID string, err error) { if r.deploymentID != "" { // Existing deployment: read it to compute the next version number. A 404 is - // not recovered from by creating a second deployment: the ID was just - // resolved from the deployment's workspace node, which the service trashes - // when it deletes the record, so a missing record here means the two are out - // of sync and creating another one would collide on the same node path. + // not recovered from by creating a second deployment. The service trashes the + // workspace node when it deletes the record, so a node that resolved but has + // no record means the two are out of sync, and creating another deployment + // would collide on the same node path. dep, getErr := r.svc.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ Name: "deployments/" + r.deploymentID, }) @@ -166,14 +164,12 @@ func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID strin } versionID = strconv.FormatInt(lastVersion+1, 10) } else { - // First deploy: create the deployment so the server assigns an ID, then - // start at version 1. + // First deploy: create the deployment so the server assigns an ID. // - // initial_parent_path is required: the service creates the deployment's - // BUNDLE_DEPLOYMENT node under it, and that node's ID becomes the - // deployment ID that ResolveDeploymentID reads back on later deploys. The - // folder must already exist, which it does by this point - the deployment - // lock lives in the same directory. + // initial_parent_path is required. The service creates the deployment node + // under it, and that node's ID is the deployment ID ResolveDeploymentID reads + // back later. The folder already exists by now: the deployment lock lives in + // the same directory. dep, createErr := r.svc.CreateDeployment(ctx, bundledeployments.CreateDeploymentRequest{ Deployment: bundledeployments.Deployment{ InitialParentPath: r.statePath, diff --git a/libs/dms/resolve.go b/libs/dms/resolve.go index 0d26448e558..9bbf5a84b56 100644 --- a/libs/dms/resolve.go +++ b/libs/dms/resolve.go @@ -11,22 +11,17 @@ import ( "github.com/databricks/databricks-sdk-go/apierr" ) -// DeploymentNodeName is the workspace node DMS creates for a deployment. The -// name is fixed for every deployment: the node *is* the bundle's state file. -// It must match DeploymentWhsClient.DEPLOYMENT_NODE_NAME on the service side. +// DeploymentNodeName is the workspace node DMS creates per deployment. Must +// match DeploymentWhsClient.DEPLOYMENT_NODE_NAME on the service side. const DeploymentNodeName = "resources.deployment.json" // ResolveDeploymentID returns the DMS deployment ID for the bundle whose state -// lives under statePath, or an empty string when the bundle has no deployment -// recorded yet. +// lives under statePath, or empty if it has never recorded a deployment. // -// The ID is not stored anywhere by the CLI. DMS registers each deployment as a -// BUNDLE_DEPLOYMENT node at statePath/resources.deployment.json, and the -// workspace-assigned node ID *is* the deployment ID (see DeploymentHandler: -// deploymentId = Long.toString(createdNode.getId())). So a get-status on that -// path is the lookup, which keeps the workspace the single source of truth: a -// deployment that was destroyed or deleted out of band reports absent here -// rather than leaving a dangling ID behind in the local state file. +// The CLI stores the ID nowhere: DMS registers the deployment as a workspace +// node and that node's ID *is* the deployment ID, so a get-status is the lookup. +// This keeps the workspace the single source of truth — a destroyed deployment +// reports absent instead of leaving a dangling ID in the local state file. func ResolveDeploymentID(ctx context.Context, w *databricks.WorkspaceClient, statePath string) (string, error) { nodePath := path.Join(statePath, DeploymentNodeName) From 70741aeceb38fd9155b952e9f2d8785d08555338 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 28 Jul 2026 10:38:59 +0000 Subject: [PATCH 13/28] bundle: record depends_on in the state uploaded to DMS The read path carried depends_on over from the local state file, which is empty exactly when it matters: a fresh checkout reconstructs state from DMS and got no dependency edges. Deletes are the one case that cannot recompute them, because the resource is gone from config, so two dropped resources with a real dependency could be deleted in the wrong order. DMS has no field for dependency edges, and they cannot be recovered from the recorded config either: references are resolved to literals before it is serialized. So Operation.State now carries an envelope, dstate.RecordedState, holding the config plus depends_on. Nesting depends_on inside the config would have collided with resource fields of the same name (jobs.Task.depends_on). The envelope mirrors the local ResourceEntry, so both sides of the round trip have the same shape. acceptance/bundle/dms/depends-on covers it: a job referencing another records its edge, and after wiping the local state a destroy still deletes the referencing job first. Co-authored-by: Isaac --- .../bundle/dms/depends-on/databricks.yml | 13 ++++++ .../bundle/dms/depends-on/out.test.toml | 3 ++ acceptance/bundle/dms/depends-on/output.txt | 26 ++++++++++++ acceptance/bundle/dms/depends-on/script | 8 ++++ .../bundle/dms/existing-state/output.txt | 2 +- acceptance/bundle/dms/record/output.txt | 22 +++++----- .../dms/redeploy-after-destroy/output.txt | 22 +++++----- bundle/direct/bundle_apply.go | 7 ++-- bundle/direct/dstate/dms.go | 40 ++++++++++++------- bundle/direct/dstate/dms_test.go | 38 ++++++++---------- bundle/direct/opqueue.go | 4 +- bundle/direct/opqueue_test.go | 20 +++++----- bundle/direct/oprecorder.go | 11 +++-- bundle/direct/oprecorder_test.go | 21 ++++++++-- 14 files changed, 157 insertions(+), 80 deletions(-) create mode 100644 acceptance/bundle/dms/depends-on/databricks.yml create mode 100644 acceptance/bundle/dms/depends-on/out.test.toml create mode 100644 acceptance/bundle/dms/depends-on/output.txt create mode 100644 acceptance/bundle/dms/depends-on/script diff --git a/acceptance/bundle/dms/depends-on/databricks.yml b/acceptance/bundle/dms/depends-on/databricks.yml new file mode 100644 index 00000000000..f97e3a38809 --- /dev/null +++ b/acceptance/bundle/dms/depends-on/databricks.yml @@ -0,0 +1,13 @@ +bundle: + name: dms-depends-on + +experimental: + record_deployment_history: true + +resources: + jobs: + parent: + name: parent + child: + name: child + description: depends on ${resources.jobs.parent.id} diff --git a/acceptance/bundle/dms/depends-on/out.test.toml b/acceptance/bundle/dms/depends-on/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/dms/depends-on/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt new file mode 100644 index 00000000000..2bb8d06b9dc --- /dev/null +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -0,0 +1,26 @@ + +=== Deploy a job that references another: depends_on is recorded alongside the config, since the reference is resolved to a literal in the config itself +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //versions/1/operations --sort --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.child"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.child", "state": {"state": {"deployment": {"kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json"}, "description": "depends on [NUMID]", "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "max_concurrent_runs": 1, "name": "child", "queue": {"enabled": true}}, "depends_on": [{"node": "resources.jobs.parent", "label": "${resources.jobs.parent.id}"}]}, "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.parent"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.parent", "state": {"state": {"deployment": {"kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json"}, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "max_concurrent_runs": 1, "name": "parent", "queue": {"enabled": true}}}, "status": "OPERATION_STATUS_SUCCEEDED"}} + +=== Wipe the local state, then destroy: depends_on comes back from DMS, so the child is still deleted before the parent it references +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.child + delete resources.jobs.parent + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default + +Deleting files... +Destroy complete! + +>>> print_requests.py //jobs --oneline +{"method": "POST", "path": "/api/2.2/jobs/delete", "body": {"job_id": [NUMID]}} +{"method": "POST", "path": "/api/2.2/jobs/delete", "body": {"job_id": [NUMID]}} diff --git a/acceptance/bundle/dms/depends-on/script b/acceptance/bundle/dms/depends-on/script new file mode 100644 index 00000000000..905d022619c --- /dev/null +++ b/acceptance/bundle/dms/depends-on/script @@ -0,0 +1,8 @@ +title "Deploy a job that references another: depends_on is recorded alongside the config, since the reference is resolved to a literal in the config itself" +trace $CLI bundle deploy +trace print_requests.py //versions/1/operations --sort --oneline + +title "Wipe the local state, then destroy: depends_on comes back from DMS, so the child is still deleted before the parent it references" +rm -rf .databricks +trace $CLI bundle destroy --auto-approve +trace print_requests.py //jobs --oneline diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index 6aeff83c04b..116584d10c1 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -42,4 +42,4 @@ Deployment complete! {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state", "target_name": "default"}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "state": {"deployment": {"kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state/metadata.json"}, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "max_concurrent_runs": 1, "name": "one", "queue": {"enabled": true}}, "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "state": {"state": {"deployment": {"kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state/metadata.json"}, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "max_concurrent_runs": 1, "name": "one", "queue": {"enabled": true}}}, "status": "OPERATION_STATUS_SUCCEEDED"}} diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index ed10f06e699..e792d5d8d99 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -45,16 +45,18 @@ Deployment complete! "resource_id": "[NUMID]", "resource_key": "jobs.foo", "state": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "foo", - "queue": { - "enabled": true + "state": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } } }, "status": "OPERATION_STATUS_SUCCEEDED" diff --git a/acceptance/bundle/dms/redeploy-after-destroy/output.txt b/acceptance/bundle/dms/redeploy-after-destroy/output.txt index ed7b6ede989..ff8694b5424 100644 --- a/acceptance/bundle/dms/redeploy-after-destroy/output.txt +++ b/acceptance/bundle/dms/redeploy-after-destroy/output.txt @@ -70,16 +70,18 @@ Deployment complete! "resource_id": "[NUMID]", "resource_key": "jobs.foo", "state": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "foo", - "queue": { - "enabled": true + "state": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } } }, "status": "OPERATION_STATUS_SUCCEEDED" diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index b26861128b7..f29aa18a186 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -94,7 +94,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return false } // Record the delete with DMS. State is nil: the resource is gone. - if err := opQueue.record(ctx, resourceKey, action, "", nil); err != nil { + if err := opQueue.record(ctx, resourceKey, action, "", nil, nil); err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } @@ -129,8 +129,9 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa // Record the operation with DMS. The resource ID and applied config // (sv.Value) come from the write just performed; GetResourceID reads - // the ID assigned by Deploy. - if err := opQueue.record(ctx, resourceKey, action, b.StateDB.GetResourceID(resourceKey), sv.Value); err != nil { + // the ID assigned by Deploy. depends_on is recorded alongside the config + // because it cannot be recomputed from it (see dstate.RecordedState). + if err := opQueue.record(ctx, resourceKey, action, b.StateDB.GetResourceID(resourceKey), sv.Value, d.DependsOn); err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index 8709ceec57f..e6ae63f6095 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -7,6 +7,7 @@ import ( "fmt" "net/http" + "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/auth" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/client" @@ -14,6 +15,22 @@ import ( "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) +// RecordedState is what the CLI serializes into the DMS Operation.State field. +// +// It is an envelope rather than the bare resource config, because depends_on has +// to survive the round trip: DMS has no field for dependency edges, and they +// cannot be recomputed from the config once it is recorded (references are +// resolved to literals before serialization). Nesting depends_on inside the +// config instead would collide with resource fields of the same name, e.g. +// jobs.Task.depends_on. +// +// The shape deliberately matches the local ResourceEntry so both sides of the +// state round trip look the same. +type RecordedState struct { + State json.RawMessage `json:"state"` + DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` +} + // overlayDMSState replaces the file-derived resource state with the state // recorded in DMS, when DMS owns this deployment. An authoritative DMS is // trusted even when its resource set is empty (a successful deploy of nothing). @@ -29,7 +46,7 @@ func (db *DeploymentState) overlayDMSState(ctx context.Context, src *DMSSource) return nil } - resources, err := fetchDeploymentResources(ctx, src.Client, src.DeploymentID, db.Data.State) + resources, err := fetchDeploymentResources(ctx, src.Client, src.DeploymentID) if err != nil { return err } @@ -76,16 +93,7 @@ func deploymentHasSuccessfulVersion(ctx context.Context, cfg *sdkconfig.Config, // fetchDeploymentResources lists every resource recorded for the deployment in // DMS and maps them into state entries keyed by the fully-qualified resource key. -// -// DMS has no field for dependency edges, and they cannot be recovered from the -// recorded state either: references are resolved to literals before it is -// serialized. So depends_on is carried over from the local state file. -// -// TODO(DMS): resources present in DMS but not in the local file therefore get no -// depends_on. Plan recomputes it from config for everything it still declares, -// so this only affects deletes of resources dropped from config, which are -// ordered arbitrarily among themselves. -func fetchDeploymentResources(ctx context.Context, client bundledeployments.BundleDeploymentsInterface, deploymentID string, local map[string]ResourceEntry) (map[string]ResourceEntry, error) { +func fetchDeploymentResources(ctx context.Context, client bundledeployments.BundleDeploymentsInterface, deploymentID string) (map[string]ResourceEntry, error) { it := client.ListResources(ctx, bundledeployments.ListResourcesRequest{ Parent: "deployments/" + deploymentID, }) @@ -102,15 +110,17 @@ func fetchDeploymentResources(ctx context.Context, client bundledeployments.Bund // ("resources.jobs.foo"), so prepend it here. key := "resources." + res.ResourceKey - var state json.RawMessage + var recorded RecordedState if res.State != nil { - state = *res.State + if err := json.Unmarshal(*res.State, &recorded); err != nil { + return nil, fmt.Errorf("interpreting state recorded for %s: %w", key, err) + } } out[key] = ResourceEntry{ ID: res.ResourceId, - State: state, - DependsOn: local[key].DependsOn, + State: recorded.State, + DependsOn: recorded.DependsOn, } } return out, nil diff --git a/bundle/direct/dstate/dms_test.go b/bundle/direct/dstate/dms_test.go index df1084b9de7..35fe7acbb0c 100644 --- a/bundle/direct/dstate/dms_test.go +++ b/bundle/direct/dstate/dms_test.go @@ -35,40 +35,34 @@ func (f *fakeResourceLister) ListResources(ctx context.Context, req bundledeploy ) } -func TestFetchDeploymentResourcesPreservesLocalDependsOn(t *testing.T) { - state := json.RawMessage(`{"name":"foo"}`) +func TestFetchDeploymentResourcesUnwrapsEnvelope(t *testing.T) { + recorded := json.RawMessage(`{"state":{"name":"foo"},"depends_on":[{"node":"resources.pipelines.bar","label":"${resources.pipelines.bar.id}"}]}`) f := &fakeResourceLister{resources: []bundledeployments.Resource{ - {ResourceKey: "jobs.foo", ResourceId: "123", State: &state}, + {ResourceKey: "jobs.foo", ResourceId: "123", State: &recorded}, {ResourceKey: "pipelines.bar", ResourceId: "456"}, }} - dependsOn := []deployplan.DependsOnEntry{{Node: "resources.pipelines.bar", Label: "pipeline_id"}} - local := map[string]ResourceEntry{ - "resources.jobs.foo": {ID: "stale", DependsOn: dependsOn}, - } - - got, err := fetchDeploymentResources(t.Context(), f, "dep-1", local) + got, err := fetchDeploymentResources(t.Context(), f, "dep-1") require.NoError(t, err) - // DMS owns the ID and state, but it does not record dependency edges, so - // depends_on must survive from the local entry. Losing it breaks delete - // ordering and --select expansion. + // depends_on comes back from the envelope, so a bundle whose local state was + // wiped still has the edges needed for delete ordering. assert.Equal(t, map[string]ResourceEntry{ - "resources.jobs.foo": {ID: "123", State: state, DependsOn: dependsOn}, + "resources.jobs.foo": { + ID: "123", + State: json.RawMessage(`{"name":"foo"}`), + DependsOn: []deployplan.DependsOnEntry{{Node: "resources.pipelines.bar", Label: "${resources.pipelines.bar.id}"}}, + }, "resources.pipelines.bar": {ID: "456"}, }, got) } -func TestFetchDeploymentResourcesWithNoLocalState(t *testing.T) { +func TestFetchDeploymentResourcesRejectsMalformedState(t *testing.T) { + recorded := json.RawMessage(`not json`) f := &fakeResourceLister{resources: []bundledeployments.Resource{ - {ResourceKey: "jobs.foo", ResourceId: "123"}, + {ResourceKey: "jobs.foo", ResourceId: "123", State: &recorded}, }} - // A bundle whose local state was wiped has no entry to carry depends_on from; - // the resource is still recovered from DMS. - got, err := fetchDeploymentResources(t.Context(), f, "dep-1", nil) - require.NoError(t, err) - assert.Equal(t, map[string]ResourceEntry{ - "resources.jobs.foo": {ID: "123"}, - }, got) + _, err := fetchDeploymentResources(t.Context(), f, "dep-1") + assert.ErrorContains(t, err, "interpreting state recorded for resources.jobs.foo") } diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go index 68e8e2e9f96..1f38c2b7dd6 100644 --- a/bundle/direct/opqueue.go +++ b/bundle/direct/opqueue.go @@ -90,12 +90,12 @@ func newOperationQueue(ctx context.Context, uploader operationUploader) *operati // queueing again: DMS keeps one state per key, so one upload records both. The // merge keeps a queued create's action (see mergeAction). Best effort — only // operations no worker has picked up yet are collapsed. -func (q *operationQueue) record(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state any) error { +func (q *operationQueue) record(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state any, dependsOn []deployplan.DependsOnEntry) error { if q == nil { return nil } - op, err := newRecordedOperation(action, resourceID, state) + op, err := newRecordedOperation(action, resourceID, state, dependsOn) if err != nil { return err } diff --git a/bundle/direct/opqueue_test.go b/bundle/direct/opqueue_test.go index cdd1083a06d..6bf0da8107b 100644 --- a/bundle/direct/opqueue_test.go +++ b/bundle/direct/opqueue_test.go @@ -59,7 +59,7 @@ func (f *fakeUploader) actionFor(resourceKey string) bundledeployments.Operation func recordState(t *testing.T, q *operationQueue, resourceKey, name string) { t.Helper() - require.NoError(t, q.record(t.Context(), resourceKey, deployplan.Update, "id-1", map[string]string{"name": name})) + require.NoError(t, q.record(t.Context(), resourceKey, deployplan.Update, "id-1", map[string]string{"name": name}, nil)) } func TestOperationQueueUploadsEachOperation(t *testing.T) { @@ -94,8 +94,8 @@ func TestOperationQueueCoalescesQueuedOperationsForSameResource(t *testing.T) { // Two uploads, not three: v2 was superseded by v3 while both were queued, and // the last recorded state is the one the service ends up with. assert.Equal(t, []string{ - `resources.jobs.foo={"name":"v1"}`, - `resources.jobs.foo={"name":"v3"}`, + `resources.jobs.foo={"state":{"name":"v1"}}`, + `resources.jobs.foo={"state":{"name":"v3"}}`, }, f.recorded()) } @@ -114,15 +114,15 @@ func TestOperationQueueCoalescingKeepsCreateAction(t *testing.T) { assert.Equal(t, "resources.jobs.hold"+strconv.Itoa(i), <-f.started) } - require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", map[string]string{"name": "created"})) - require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Update, "id-1", map[string]string{"name": "updated"})) + require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", map[string]string{"name": "created"}, nil)) + require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Update, "id-1", map[string]string{"name": "updated"}, nil)) close(f.block) require.NoError(t, q.close()) // The state is the later one, but the action stays CREATE: recording an update // would tell DMS the resource already existed before this deploy. - assert.Contains(t, f.recorded(), `resources.jobs.foo={"name":"updated"}`) + assert.Contains(t, f.recorded(), `resources.jobs.foo={"state":{"name":"updated"}}`) assert.Equal(t, bundledeployments.OperationActionTypeOperationActionTypeCreate, f.actionFor("resources.jobs.foo")) @@ -147,7 +147,7 @@ func TestOperationQueueRecordRejectsUnsupportedAction(t *testing.T) { // Serialization failures surface at record time, on the resource that caused // them, rather than from the drain at the end of apply. - err := q.record(t.Context(), "resources.jobs.foo", deployplan.Skip, "id-1", nil) + err := q.record(t.Context(), "resources.jobs.foo", deployplan.Skip, "id-1", nil, nil) require.Error(t, err) require.NoError(t, q.close()) @@ -159,7 +159,7 @@ func TestOperationQueueRecordRejectsOversizedState(t *testing.T) { q := newOperationQueue(t.Context(), f) big := map[string]string{"name": strings.Repeat("x", maxOperationStateSize)} - err := q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", big) + err := q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", big, nil) require.ErrorContains(t, err, "exceeds the 65536 byte limit") require.NoError(t, q.close()) @@ -224,7 +224,7 @@ func TestOperationQueueUploadsOneResourceAtATime(t *testing.T) { wg.Go(func() { for i := range perWorker { key := "resources.jobs.job" + strconv.Itoa((w*perWorker+i)%distinctKeyMod) - errs <- q.record(ctx, key, deployplan.Update, "id-1", map[string]string{"name": strconv.Itoa(w)}) + errs <- q.record(ctx, key, deployplan.Update, "id-1", map[string]string{"name": strconv.Itoa(w)}, nil) } }) } @@ -247,6 +247,6 @@ func TestNilOperationQueueIsNoOp(t *testing.T) { // no-op, so Apply does not have to branch. q := newOperationQueue(t.Context(), nil) require.Nil(t, q) - require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", nil)) + require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", nil, nil)) require.NoError(t, q.close()) } diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index 3b3d631f4a9..91bcb65a4f6 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/structs/structwalk" "github.com/databricks/databricks-sdk-go/service/bundledeployments" @@ -35,7 +36,7 @@ type recordedOperation struct { // newRecordedOperation serializes an applied operation for upload. state is the // local config after the operation and must be nil for delete operations. It // errors when the serialized state exceeds maxOperationStateSize. -func newRecordedOperation(action deployplan.ActionType, resourceID string, state any) (recordedOperation, error) { +func newRecordedOperation(action deployplan.ActionType, resourceID string, state any, dependsOn []deployplan.DependsOnEntry) (recordedOperation, error) { actionType, err := deployActionToSDK(action) if err != nil { return recordedOperation{}, err @@ -43,14 +44,18 @@ func newRecordedOperation(action deployplan.ActionType, resourceID string, state op := recordedOperation{action: actionType, resourceID: resourceID} - // Operation.State carries the serialized config, which DMS serves back as + // Operation.State carries the serialized state, which DMS serves back as // resource state. Unset for delete: the resource is gone. // // Redact secrets, like dstate.SaveState does for the local state file: // otherwise we leak them to the service and the read path writes them back // into a local state file in plaintext. if state != nil { - raw, err := structwalk.RedactSensitiveFields(state, dyn.SensitiveValueRedacted) + config, err := structwalk.RedactSensitiveFields(state, dyn.SensitiveValueRedacted) + if err != nil { + return recordedOperation{}, fmt.Errorf("serializing state: %w", err) + } + raw, err := json.Marshal(dstate.RecordedState{State: config, DependsOn: dependsOn}) if err != nil { return recordedOperation{}, fmt.Errorf("serializing state: %w", err) } diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go index 70afb788cd3..556f7f59f90 100644 --- a/bundle/direct/oprecorder_test.go +++ b/bundle/direct/oprecorder_test.go @@ -30,7 +30,7 @@ func (f *fakeOpClient) CreateOperation(ctx context.Context, req bundledeployment // an operationQueue worker does. func uploadOne(t *testing.T, u operationUploader, resourceKey string, action deployplan.ActionType, resourceID string, state any) { t.Helper() - op, err := newRecordedOperation(action, resourceID, state) + op, err := newRecordedOperation(action, resourceID, state, nil) require.NoError(t, err) require.NoError(t, u.upload(t.Context(), resourceKey, op)) } @@ -71,18 +71,31 @@ func TestNewRecordedOperationRedactsSensitiveFields(t *testing.T) { Token string `json:"token" bundle:"sensitive"` }{Name: "foo", Token: "super-secret"} - op, err := newRecordedOperation(deployplan.Create, "job-123", state) + op, err := newRecordedOperation(deployplan.Create, "job-123", state, nil) require.NoError(t, err) // Sensitive fields are redacted before leaving the CLI, matching what // dstate.SaveState writes to the local state file. assert.JSONEq(t, - `{"name":"foo","token":"`+dyn.SensitiveValueRedacted+`"}`, + `{"state":{"name":"foo","token":"`+dyn.SensitiveValueRedacted+`"}}`, + string(op.state)) +} + +func TestNewRecordedOperationRecordsDependsOn(t *testing.T) { + // depends_on rides in an envelope alongside the config: it cannot be + // recomputed from the config, whose references are already resolved. + dependsOn := []deployplan.DependsOnEntry{{Node: "resources.jobs.bar", Label: "${resources.jobs.bar.id}"}} + + op, err := newRecordedOperation(deployplan.Create, "job-123", map[string]string{"name": "foo"}, dependsOn) + require.NoError(t, err) + + assert.JSONEq(t, + `{"state":{"name":"foo"},"depends_on":[{"node":"resources.jobs.bar","label":"${resources.jobs.bar.id}"}]}`, string(op.state)) } func TestNewRecordedOperationRejectsUnsupportedAction(t *testing.T) { - _, err := newRecordedOperation(deployplan.Skip, "job-123", nil) + _, err := newRecordedOperation(deployplan.Skip, "job-123", nil, nil) assert.Error(t, err) } From 386a0b616f6a409ab1c31fb34383b74bcd098547 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 28 Jul 2026 11:03:26 +0000 Subject: [PATCH 14/28] bundle: drop the DMS state overlay check Migrating an existing deployment is not supported: Open rejects a bundle that already has resources in state, so a deployment that exists in DMS was created by an opted-in CLI and DMS owns its resource set outright. The last_successful_version_id probe that decided whether to trust DMS was therefore always true by the time it ran. Removing it takes with it the raw GET that read the field (it is stage:DEVELOPMENT and stripped from the generated SDK) and DMSSource.Config, which existed only to make that call. overlayDMSState is now readDMSState, since it no longer overlays anything conditionally: it just reads. Recording stays opt-in via experimental.record_deployment_history; that flag is what makes the caller pass a DMSSource at all. acceptance/bundle/dms/existing-state also now covers wiping the local cache: deploy pulls the state file back from the workspace, so the resources stay tracked and opting in is still rejected. Co-authored-by: Isaac --- .../bundle/dms/existing-state/output.txt | 7 +++ acceptance/bundle/dms/existing-state/script | 5 ++ acceptance/bundle/dms/no-resources/output.txt | 4 -- bundle/direct/dstate/dms.go | 59 ++----------------- bundle/direct/dstate/state.go | 21 +++---- cmd/bundle/utils/process.go | 8 +-- libs/dms/resolve_test.go | 4 +- 7 files changed, 33 insertions(+), 75 deletions(-) diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index 116584d10c1..a793cc0fae1 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -18,6 +18,13 @@ Error: cannot record deployment history for a bundle that already has deployed r === No deployment was created in DMS >>> print_requests.py //api/2.0/bundle --sort --oneline +=== Still an error after wiping the local cache: deploy pulls the state file back from the workspace, so the resources are still tracked +>>> musterr [CLI] bundle deploy +Error: cannot record deployment history for a bundle that already has deployed resources tracked in [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again + + +>>> print_requests.py //api/2.0/bundle --sort --oneline + === Destroy clears the tracked resources, so recording can be enabled afterwards >>> update_file.py databricks.yml record_deployment_history: true record_deployment_history: false diff --git a/acceptance/bundle/dms/existing-state/script b/acceptance/bundle/dms/existing-state/script index 7bc296465d5..ae9be95f701 100644 --- a/acceptance/bundle/dms/existing-state/script +++ b/acceptance/bundle/dms/existing-state/script @@ -9,6 +9,11 @@ trace musterr $CLI bundle deploy title "No deployment was created in DMS" trace print_requests.py //api/2.0/bundle --sort --oneline +title "Still an error after wiping the local cache: deploy pulls the state file back from the workspace, so the resources are still tracked" +rm -rf .databricks +trace musterr $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --sort --oneline + title "Destroy clears the tracked resources, so recording can be enabled afterwards" trace update_file.py databricks.yml "record_deployment_history: true" "record_deployment_history: false" trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index 73bd3adfa11..b400fb9a67e 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -51,10 +51,6 @@ Deployment complete! "method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]" } -{ - "method": "GET", - "path": "/api/2.0/bundle/deployments/[NUMID]" -} { "method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources" diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index e6ae63f6095..ec51e2c4479 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -3,15 +3,9 @@ package dstate import ( "context" "encoding/json" - "errors" "fmt" - "net/http" "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/libs/auth" - "github.com/databricks/databricks-sdk-go/apierr" - "github.com/databricks/databricks-sdk-go/client" - sdkconfig "github.com/databricks/databricks-sdk-go/config" "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) @@ -31,21 +25,12 @@ type RecordedState struct { DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` } -// overlayDMSState replaces the file-derived resource state with the state -// recorded in DMS, when DMS owns this deployment. An authoritative DMS is -// trusted even when its resource set is empty (a successful deploy of nothing). -// The caller holds db.mu and has already populated db.Data from the file. -func (db *DeploymentState) overlayDMSState(ctx context.Context, src *DMSSource) error { - authoritative, err := deploymentHasSuccessfulVersion(ctx, src.Config, src.DeploymentID) - if err != nil { - return err - } - if !authoritative { - // DMS has no completed version for this deployment: a prior direct deploy - // that has not yet successfully recorded to DMS. Keep the file state. - return nil - } - +// readDMSState replaces the file-derived resource state with the state recorded +// in DMS. Recording is only enabled for net-new deployments, so once a +// deployment exists DMS owns its resource set outright - including when that set +// is empty, which is a successful deploy of nothing rather than missing data. +// The caller holds db.mu. +func (db *DeploymentState) readDMSState(ctx context.Context, src *DMSSource) error { resources, err := fetchDeploymentResources(ctx, src.Client, src.DeploymentID) if err != nil { return err @@ -59,38 +44,6 @@ func (db *DeploymentState) overlayDMSState(ctx context.Context, src *DMSSource) return nil } -// deploymentHasSuccessfulVersion reports whether DMS owns the state. The server -// advances last_successful_version_id only when a version completes (unlike -// last_version_id, which also advances on failure), so a non-empty value means -// DMS holds a complete resource set. Otherwise Open keeps the file's resources. -// -// TODO(DMS): raw GET because last_successful_version_id is stage:DEVELOPMENT and -// stripped from the generated SDK. Once it ships, use -// client.GetDeployment(...).LastSuccessfulVersionId and drop DMSSource.Config. -func deploymentHasSuccessfulVersion(ctx context.Context, cfg *sdkconfig.Config, deploymentID string) (bool, error) { - apiClient, err := client.New(cfg) - if err != nil { - return false, fmt.Errorf("creating API client for deployment metadata service: %w", err) - } - - // Mirrors the SDK's GetDeployment path (/api/2.0/bundle/{name} with - // name=deployments/{id}); we unmarshal into a local struct so we can read - // last_successful_version_id, which the typed SDK response drops. - var dep struct { - LastSuccessfulVersionID string `json:"last_successful_version_id"` - } - err = apiClient.Do(ctx, http.MethodGet, "/api/2.0/bundle/deployments/"+deploymentID, auth.WorkspaceIDHeaders(cfg), nil, nil, &dep) - if err != nil { - // A deployment that was never recorded to DMS is not an error here: it - // just means DMS is not (yet) the source of truth. - if errors.Is(err, apierr.ErrNotFound) { - return false, nil - } - return false, fmt.Errorf("reading deployment from deployment metadata service: %w", err) - } - return dep.LastSuccessfulVersionID != "", nil -} - // fetchDeploymentResources lists every resource recorded for the deployment in // DMS and maps them into state entries keyed by the fully-qualified resource key. func fetchDeploymentResources(ctx context.Context, client bundledeployments.BundleDeploymentsInterface, deploymentID string) (map[string]ResourceEntry, error) { diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 0359feba89a..bad87a6a293 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -19,7 +19,6 @@ import ( "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/structs/structwalk" - sdkconfig "github.com/databricks/databricks-sdk-go/config" "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/google/uuid" ) @@ -221,24 +220,20 @@ type ( ) // DMSSource tells Open to read resource state from the deployment metadata -// service instead of the state file. A nil *DMSSource keeps Open file-only. +// service instead of the state file. Callers pass it only when the bundle set +// experimental.record_deployment_history; a nil *DMSSource keeps Open file-only. type DMSSource struct { Client bundledeployments.BundleDeploymentsInterface - // Config is only for a temporary raw read of last_successful_version_id; see - // the TODO in deploymentHasSuccessfulVersion. - Config *sdkconfig.Config - // DeploymentID is resolved from the deployment's workspace node (see // dms.ResolveDeploymentID), and empty before the first recorded deploy. DeploymentID string } // Open reads the deployment state from disk (and recovers the WAL when -// withRecovery is set). With a non-nil dmsSource, resources come from DMS -// instead of the file whenever DMS holds a successful version. Lineage and -// serial always come from the file, since that is what the write path -// increments. +// withRecovery is set). With a non-nil dmsSource, resources come from DMS rather +// than the file. Lineage and serial always come from the file, since that is +// what the write path increments. func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, dmsSource *DMSSource) error { db.mu.Lock() defer db.mu.Unlock() @@ -290,8 +285,8 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W if dmsSource != nil { // Only bundles that start out empty can be recorded. Once DMS owns a // deployment it is authoritative for the whole resource set (see - // overlayDMSState), so pre-existing resources it never saw would look absent - // and get created a second time. + // readDMSState), so pre-existing resources it never saw would look absent and + // get created a second time. // // TODO(DMS): allow this by upgrading the state in place, writing it at // featureStateVersion with a feature flag plus a tombstone per resource so an @@ -301,7 +296,7 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W return fmt.Errorf("cannot record deployment history for a bundle that already has deployed resources tracked in %s: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again", path) } if dmsSource.DeploymentID != "" { - if err := db.overlayDMSState(ctx, dmsSource); err != nil { + if err := db.readDMSState(ctx, dmsSource); err != nil { return err } } diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index e9840188db4..209dc874403 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -214,9 +214,10 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle _, localPath := b.StateFilenameDirect(ctx) // When the bundle records deployment history, the deployment metadata - // service owns resource state, so hand Open a DMS source to overlay that - // state on top of the local identity (lineage/serial). Reads open the - // state write-disabled, so no lineage is minted here. + // service owns resource state, so hand Open a DMS source to read it from + // there instead of the file. The local identity (lineage/serial) still + // comes from the file. Reads open the state write-disabled, so no lineage + // is minted here. var dmsSource *dstate.DMSSource if b.Config.Experimental != nil && b.Config.Experimental.RecordDeploymentHistory { w := b.WorkspaceClient(ctx) @@ -227,7 +228,6 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle } dmsSource = &dstate.DMSSource{ Client: w.BundleDeployments, - Config: w.Config, DeploymentID: deploymentID, } } diff --git a/libs/dms/resolve_test.go b/libs/dms/resolve_test.go index 24d5303983d..6838f4dd15c 100644 --- a/libs/dms/resolve_test.go +++ b/libs/dms/resolve_test.go @@ -25,7 +25,7 @@ func newTestClient(t *testing.T, statusCode int, body string) *databricks.Worksp })) t.Cleanup(srv.Close) t.Cleanup(func() { - assert.Equal(t, "/Workspace/state/"+DeploymentNodeName, gotPath) + assert.Equal(t, nodePath, gotPath) }) w, err := databricks.NewWorkspaceClient(&databricks.Config{ @@ -37,6 +37,8 @@ func newTestClient(t *testing.T, statusCode int, body string) *databricks.Worksp return w } +const nodePath = "/Workspace/state/" + DeploymentNodeName + func TestResolveDeploymentIDReturnsNodeID(t *testing.T) { w := newTestClient(t, http.StatusOK, `{"object_type":"FILE","object_id":123456789,"path":"/Workspace/state/`+DeploymentNodeName+`"}`) From efe7320cbeeaacecde8dc76ef68eed74b9e6d658 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 28 Jul 2026 23:44:57 +0000 Subject: [PATCH 15/28] bundle: gate DMS state on a state feature flag Reading resource state from DMS now requires the state file to record a "deployment_history" feature flag, rather than inferring eligibility from the resource count. The old check refused a state that had resources and no DMS deployment ID. That happened to be right, but it inferred intent from a side effect: a state with resources could equally be one DMS already owns. The flag says so directly. Header.Features was already scaffolded for exactly this, so this fills it in: - Open writes the flag when recording is enabled, and refuses a state that has resources without it. Migrating such a target is not supported, so the error names the target and the three ways out: use a new target, destroy this one and redeploy, or unset the feature. - A state recording any feature is written at featureStateVersion, so a CLI that predates the flag refuses it (see migrateState) instead of deploying against a resource set that lives in DMS and looks empty on disk. - migrateState accepts features it implements and refuses only unknown ones, naming just those in the error. - WAL replay carries the features forward, so recovering a WAL from a recording deploy still produces a state marked as DMS-owned. The flag is per target, which matches how experimental.record_deployment_history is set. Co-authored-by: Isaac --- .../bundle/dms/existing-state/output.txt | 4 +- acceptance/bundle/dms/record/output.txt | 9 ++ acceptance/bundle/dms/record/script | 3 + bundle/direct/dstate/migrate.go | 35 +++--- bundle/direct/dstate/state.go | 110 ++++++++++++------ bundle/direct/dstate/state_test.go | 21 +++- cmd/bundle/utils/process.go | 1 + 7 files changed, 132 insertions(+), 51 deletions(-) diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index a793cc0fae1..229b3bea03f 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -12,7 +12,7 @@ Deployment complete! >>> update_file.py databricks.yml record_deployment_history: false record_deployment_history: true >>> musterr [CLI] bundle deploy -Error: cannot record deployment history for a bundle that already has deployed resources tracked in [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again +Error: target "default" was deployed without experimental.record_deployment_history and cannot be migrated to it: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json tracks resources this deployment recorded before the feature was enabled. Use a new target, destroy this one and deploy it again, or unset experimental.record_deployment_history === No deployment was created in DMS @@ -20,7 +20,7 @@ Error: cannot record deployment history for a bundle that already has deployed r === Still an error after wiping the local cache: deploy pulls the state file back from the workspace, so the resources are still tracked >>> musterr [CLI] bundle deploy -Error: cannot record deployment history for a bundle that already has deployed resources tracked in [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again +Error: target "default" was deployed without experimental.record_deployment_history and cannot be migrated to it: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json tracks resources this deployment recorded before the feature was enabled. Use a new target, destroy this one and deploy it again, or unset experimental.record_deployment_history >>> print_requests.py //api/2.0/bundle --sort --oneline diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index e792d5d8d99..3266e0d3af8 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -73,6 +73,15 @@ Deployment complete! >>> jq has("deployment_id") .databricks/bundle/default/resources.json false +=== The state records the feature flag, at the version that makes a CLI without it refuse the state rather than deploy against resources it cannot see +>>> jq {state_version, features} .databricks/bundle/default/resources.json +{ + "state_version": 3, + "features": { + "deployment_history": {} + } +} + === Redeploy after deleting the local cache: the deployment ID is resolved from that node, the same deployment is reused, and the version increments (no new CreateDeployment) >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record/default/files... diff --git a/acceptance/bundle/dms/record/script b/acceptance/bundle/dms/record/script index 63eaa323b1b..3298c8fb131 100644 --- a/acceptance/bundle/dms/record/script +++ b/acceptance/bundle/dms/record/script @@ -6,6 +6,9 @@ title "The deployment ID is the ID of the workspace node the service registered trace $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record/default/state/resources.deployment.json" | jq '{object_type,path}' trace jq 'has("deployment_id")' .databricks/bundle/default/resources.json +title "The state records the feature flag, at the version that makes a CLI without it refuse the state rather than deploy against resources it cannot see" +trace jq '{state_version, features}' .databricks/bundle/default/resources.json + title "Redeploy after deleting the local cache: the deployment ID is resolved from that node, the same deployment is reused, and the version increments (no new CreateDeployment)" rm -rf .databricks trace $CLI bundle deploy diff --git a/bundle/direct/dstate/migrate.go b/bundle/direct/dstate/migrate.go index e4d21a7054a..288fdcf863f 100644 --- a/bundle/direct/dstate/migrate.go +++ b/bundle/direct/dstate/migrate.go @@ -12,25 +12,34 @@ import ( "github.com/databricks/databricks-sdk-go/service/iam" ) +// knownFeatures lists the state feature flags this CLI implements. A state that +// records anything outside this set is refused by migrateState. +var knownFeatures = map[string]bool{ + FeatureDeploymentHistory: true, +} + // migrateState runs all necessary migrations on the database. // It is called after loading state from disk. func migrateState(db *Database) error { - // featureStateVersion states carry a feature list this CLI does not yet write or - // understand (see the featureStateVersion doc comment). A featureStateVersion - // state with no features is equivalent to currentStateVersion, so accept it and - // return without running the migrations below, leaving the on-disk version at - // featureStateVersion rather than flipping it down. One that records any feature - // depends on capabilities this CLI lacks, so refuse it and tell the user to upgrade. + // featureStateVersion states carry a feature list (see the featureStateVersion + // doc comment). A featureStateVersion state with no features is equivalent to + // currentStateVersion, so accept it and return without running the migrations + // below, leaving the on-disk version at featureStateVersion rather than flipping + // it down. Same for a state whose features this CLI implements. One that records + // a feature this CLI does not know depends on capabilities it lacks, so refuse it + // and tell the user to upgrade. if db.StateVersion == featureStateVersion { - if len(db.Features) == 0 { - return nil - } - features := make([]string, 0, len(db.Features)) + unknown := make([]string, 0, len(db.Features)) for name := range db.Features { - features = append(features, name) + if !knownFeatures[name] { + unknown = append(unknown, name) + } + } + if len(unknown) == 0 { + return nil } - slices.Sort(features) - return fmt.Errorf("the deployment state requires features this CLI does not support: %s; upgrade to the latest CLI version and see %s for more information", strings.Join(features, ", "), featuresDocURL) + slices.Sort(unknown) + return fmt.Errorf("the deployment state requires features this CLI does not support: %s; upgrade to the latest CLI version and see %s for more information", strings.Join(unknown, ", "), featuresDocURL) } if db.StateVersion == currentStateVersion { diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index bad87a6a293..b367e97001b 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -31,22 +31,28 @@ const ( maxWalEntrySize = 10 * 1024 * 1024 walSuffix = ".wal" - // featureStateVersion is the schema version a future CLI will write once it - // records deployment state "feature flags" (see Header.Features). This CLI does - // not write it and records no features; it exists now only so this CLI reads - // such states correctly (see migrateState): - // - featureStateVersion with no features -> accept and leave the version as-is - // - featureStateVersion with any feature -> refuse, tell the user to upgrade + // featureStateVersion is the schema version written once a state records + // deployment state "feature flags" (see Header.Features). Reading such a state + // (see migrateState): + // - featureStateVersion with no features -> accept, leave the version as-is + // - featureStateVersion with known features -> accept + // - featureStateVersion with unknown features -> refuse, tell the user to upgrade // // A featureStateVersion state with no features is equivalent to // currentStateVersion, but we deliberately do not flip the on-disk version down // to currentStateVersion: a state written at featureStateVersion stays at - // featureStateVersion. This is forward-compat scaffolding so that a later release - // can start writing featureStateVersion + features without older CLIs (with this - // change) either mishandling a feature they lack or rejecting a featureless state - // outright. featureStateVersion is always 3. + // featureStateVersion. That way a release can start writing + // featureStateVersion + features without older CLIs either mishandling a feature + // they lack or rejecting a featureless state outright. featureStateVersion is + // always 3. featureStateVersion = 3 + // FeatureDeploymentHistory marks a state whose resources live in the deployment + // metadata service rather than in the state file. A CLI that does not know this + // feature refuses the state instead of deploying against a resource set it + // cannot see - the file's resources are not authoritative. + FeatureDeploymentHistory = "deployment_history" + // supportedStateVersion is the highest schema version this CLI can read. It is // normally equal to currentStateVersion — the version this CLI reads is the // version it writes — and exceeds it only during a two-phase version bump like @@ -82,13 +88,27 @@ type Header struct { Serial int `json:"serial"` // Features maps each feature flag this state depends on to a (currently empty) - // value. This CLI writes no features; it only reads the field to detect a state - // that depends on features it lacks and refuse it (see migrateState). It is a - // map so a future CLI can attach per-feature data without reshaping the state. - // Empty/omitted for states that use no features. + // value. A CLI that does not implement one of them refuses the state rather than + // deploying against it (see migrateState). It is a map so a future CLI can attach + // per-feature data without reshaping the state. Empty/omitted for states that use + // no features. Features map[string]struct{} `json:"features,omitempty"` } +// hasFeature reports whether the state records the given feature flag. +func (h *Header) hasFeature(name string) bool { + _, ok := h.Features[name] + return ok +} + +// setFeature records a feature flag in the state. +func (h *Header) setFeature(name string) { + if h.Features == nil { + h.Features = make(map[string]struct{}) + } + h.Features[name] = struct{}{} +} + type Database struct { Header @@ -228,6 +248,10 @@ type DMSSource struct { // DeploymentID is resolved from the deployment's workspace node (see // dms.ResolveDeploymentID), and empty before the first recorded deploy. DeploymentID string + + // TargetName names the bundle target in the error Open returns for a state that + // predates the opt-in, since the feature is enabled per target. + TargetName string } // Open reads the deployment state from disk (and recovers the WAL when @@ -283,18 +307,18 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W } if dmsSource != nil { - // Only bundles that start out empty can be recorded. Once DMS owns a - // deployment it is authoritative for the whole resource set (see - // readDMSState), so pre-existing resources it never saw would look absent and - // get created a second time. - // - // TODO(DMS): allow this by upgrading the state in place, writing it at - // featureStateVersion with a feature flag plus a tombstone per resource so an - // older CLI refuses the state instead of deploying against resources it - // cannot see. - if dmsSource.DeploymentID == "" && len(db.Data.State) > 0 { - return fmt.Errorf("cannot record deployment history for a bundle that already has deployed resources tracked in %s: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again", path) + // An existing state file has to be marked as DMS-owned before its resources + // can be read from DMS. Recording only starts on an empty state, so a state + // with resources and no feature flag predates the opt-in: DMS never saw those + // resources and is authoritative for the whole set once enabled (see + // readDMSState), so they would look absent and be created a second time. + // Migrating such a target is not supported yet. + if len(db.Data.State) > 0 && !db.Data.hasFeature(FeatureDeploymentHistory) { + return fmt.Errorf("target %q was deployed without experimental.record_deployment_history and cannot be migrated to it: %s tracks resources this deployment recorded before the feature was enabled. Use a new target, destroy this one and deploy it again, or unset experimental.record_deployment_history", dmsSource.TargetName, path) } + // Mark the state as DMS-owned so a CLI without this feature refuses it rather + // than deploying against a resource set it cannot see. + db.Data.setFeature(FeatureDeploymentHistory) if dmsSource.DeploymentID != "" { if err := db.readDMSState(ctx, dmsSource); err != nil { return err @@ -311,13 +335,7 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W return fmt.Errorf("failed to open WAL file %s: %w", walPath, err) } db.walFile = walFile - walHead := Header{ - Lineage: db.GetOrInitLineage(), - Serial: db.Data.Serial + 1, - StateVersion: currentStateVersion, - CLIVersion: build.GetInfo().Version, - } - return appendJSONLine(db.walFile, walHead) + return appendJSONLine(db.walFile, db.newWalHeader()) } return nil @@ -404,6 +422,16 @@ func (db *DeploymentState) mergeWalIntoState(ctx context.Context) (bool, error) return false, fmt.Errorf("WAL serial (%d) is ahead of expected (%d), state may be corrupted", header.Serial, expectedSerial) } newSerial = header.Serial + + // Carry the WAL's features (and the version that goes with them) into the + // state being written, so recovering a WAL written by a DMS-recording deploy + // still produces a state marked as DMS-owned. + for name := range header.Features { + db.Data.setFeature(name) + } + if len(db.Data.Features) > 0 { + db.Data.StateVersion = featureStateVersion + } } else { var entry WALEntry if err := json.Unmarshal(line, &entry); err != nil { @@ -507,13 +535,25 @@ func (db *DeploymentState) UpgradeToWrite() error { } db.walFile = walFile - walHead := Header{ + return appendJSONLine(db.walFile, db.newWalHeader()) +} + +// newWalHeader builds the header for a fresh WAL. Features carry over from the +// state being written, and a state that records any feature is written at +// featureStateVersion so a CLI that lacks the feature refuses it instead of +// deploying against it. The caller holds db.mu. +func (db *DeploymentState) newWalHeader() Header { + version := currentStateVersion + if len(db.Data.Features) > 0 { + version = featureStateVersion + } + return Header{ Lineage: db.GetOrInitLineage(), Serial: db.Data.Serial + 1, - StateVersion: currentStateVersion, + StateVersion: version, CLIVersion: build.GetInfo().Version, + Features: db.Data.Features, } - return appendJSONLine(db.walFile, walHead) } func (db *DeploymentState) AssertOpenedForReadOrWrite() { diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index a9c90530514..35050575121 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -154,7 +154,16 @@ func TestEmptyFeatureStateAcceptedWithoutFlippingVersion(t *testing.T) { require.NoError(t, migrateState(empty)) assert.Equal(t, featureStateVersion, empty.StateVersion, "v3 + no features keeps its on-disk version, not flipped to v2") - // v3 that records a feature is refused: this CLI does not understand features. + // v3 recording a feature this CLI implements is accepted. + known := &Database{Header: Header{ + StateVersion: featureStateVersion, + Features: map[string]struct{}{FeatureDeploymentHistory: {}}, + }} + require.NoError(t, migrateState(known)) + assert.Equal(t, featureStateVersion, known.StateVersion) + + // v3 recording a feature this CLI does not know is refused: its resources may + // live somewhere this CLI cannot see. withFeature := &Database{Header: Header{ StateVersion: featureStateVersion, Features: map[string]struct{}{"future_feature": {}}, @@ -165,6 +174,16 @@ func TestEmptyFeatureStateAcceptedWithoutFlippingVersion(t *testing.T) { assert.Contains(t, err.Error(), "future_feature") assert.Contains(t, err.Error(), "upgrade to the latest CLI version") assert.Contains(t, err.Error(), featuresDocURL) + + // Only the unknown feature is named, so the message tells the user what to do. + mixed := &Database{Header: Header{ + StateVersion: featureStateVersion, + Features: map[string]struct{}{FeatureDeploymentHistory: {}, "future_feature": {}}, + }} + err = migrateState(mixed) + require.Error(t, err) + assert.Contains(t, err.Error(), "future_feature") + assert.NotContains(t, err.Error(), FeatureDeploymentHistory) } func TestDeleteState(t *testing.T) { diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 209dc874403..b000da60264 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -229,6 +229,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle dmsSource = &dstate.DMSSource{ Client: w.BundleDeployments, DeploymentID: deploymentID, + TargetName: b.Config.Bundle.Target, } } if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dmsSource); err != nil { From 92eac95b722c6148fedcc03f79db081257c5e152 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 29 Jul 2026 09:49:01 +0000 Subject: [PATCH 16/28] bundle: combine queued operations into a single Create action Coalescing now keeps the newest operation outright instead of merging fields. Each operation carries the resource's full state rather than a delta, so a newer one entirely supersedes an older one - including its resource_id, which is the field that actually changes between two records (a create learns the ID only after the API call returns it). The previous code merged the action and overwrote the ID, which is backwards: the action cannot differ between records of the same resource, while the ID can. mergeAction is gone. Also renames `owned` to `queuedOrUploading`. Nothing is owned by a particular worker: a key can be handled by one worker, released, and picked up later by another. The mark only means "some worker will get to this", which is all record needs to know in order to not queue the key twice. The doc comments now lead with the two rules that shape the design (no overlapping uploads per resource; only the newest operation matters) so the mechanism reads as a consequence of them rather than as bookkeeping. Co-authored-by: Isaac --- bundle/direct/opqueue.go | 94 ++++++++++++++++++-------------- bundle/direct/opqueue_test.go | 33 +++++++---- bundle/direct/oprecorder.go | 18 ------ bundle/direct/oprecorder_test.go | 30 ---------- 4 files changed, 74 insertions(+), 101 deletions(-) diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go index 1f38c2b7dd6..9a1a1cd918e 100644 --- a/bundle/direct/opqueue.go +++ b/bundle/direct/opqueue.go @@ -21,37 +21,48 @@ const ( operationUploadWorkers = 4 ) -// operationQueue uploads recorded operations from background workers, so an apply -// worker does not wait for the CreateOperation round trip. +// operationQueue hands recorded operations to background workers, so an apply +// worker does not wait for the CreateOperation round trip before deploying the +// next resource. // -// At most one upload is in flight per resource key, so the last operation -// recorded for a resource is also the last one the service sees. +// Two rules shape the design: +// +// - Uploads for one resource never overlap. DMS stores one state per resource +// key, so concurrent uploads could land out of order and leave stale state. +// - Only the newest operation for a resource matters. Each operation carries the +// resource's full state, not a delta, so a newer one entirely supersedes an +// older one. When both are still waiting, the older is dropped ("coalesced") +// and one upload records the result. // // Uploads are not fire-and-forget: close returns the first failure and fails the // deploy. A dropped operation would leave DMS with an incomplete resource set, -// and since DMS then becomes the source of truth (see dstate.overlayDMSState), -// the next deploy would recreate resources that already exist. +// and since DMS then becomes the source of truth (see dstate.readDMSState), the +// next deploy would recreate resources that already exist. type operationQueue struct { uploader operationUploader - // queue carries resource keys, not the operations themselves: a worker looks - // the operation up in pending when it picks the key up, which is what lets - // record collapse repeated writes to the same resource. + // queue carries resource keys, not operations. A worker looks the operation up + // when it picks the key up, so recording again before then just overwrites the + // entry in pending - that is what makes coalescing work. queue chan string wg sync.WaitGroup // mu guards the fields below. mu sync.Mutex - // pending is the latest operation recorded per resource key that no worker has - // picked up yet. + // pending holds the newest operation per resource key that no worker has taken + // yet. Empty for a key means everything recorded for it has been uploaded. pending map[string]recordedOperation - // owned holds the resource keys that are already queued or being uploaded. - // Such a key is never queued a second time; recording writes to pending - // instead, which the owning worker re-checks after each upload. This is what - // keeps uploads for one resource sequential. - owned map[string]bool + // queuedOrUploading marks keys that are already in the queue channel or being + // uploaded right now. Such a key must not be queued again, or two workers could + // upload the same resource at once; recording writes to pending instead, and + // the worker handling the key picks it up when its current upload finishes. + // + // No single worker "owns" a key for the whole time it is marked: a key can be + // handled by one worker, released, and later picked up by another. The mark only + // means "some worker will get to this", which is all record needs to know. + queuedOrUploading map[string]bool err error closed bool @@ -68,10 +79,10 @@ func newOperationQueue(ctx context.Context, uploader operationUploader) *operati } q := &operationQueue{ - uploader: uploader, - queue: make(chan string, operationQueueSize), - pending: make(map[string]recordedOperation), - owned: make(map[string]bool), + uploader: uploader, + queue: make(chan string, operationQueueSize), + pending: make(map[string]recordedOperation), + queuedOrUploading: make(map[string]bool), } q.wg.Add(operationUploadWorkers) @@ -82,14 +93,12 @@ func newOperationQueue(ctx context.Context, uploader operationUploader) *operati return q } -// record serializes an operation and queues it for upload. It makes no API call, -// so upload failures surface from close; an error here only means the applied -// resource could not be turned into a payload. +// record serializes an operation and hands it to the upload workers. It makes no +// API call, so upload failures surface from close; an error here only means the +// applied resource could not be turned into a payload. // -// An operation for a resource that is still waiting replaces it rather than -// queueing again: DMS keeps one state per key, so one upload records both. The -// merge keeps a queued create's action (see mergeAction). Best effort — only -// operations no worker has picked up yet are collapsed. +// Recording a resource that is still waiting replaces the waiting operation +// outright, since the newer one carries the resource's full state. func (q *operationQueue) record(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state any, dependsOn []deployplan.DependsOnEntry) error { if q == nil { return nil @@ -101,22 +110,20 @@ func (q *operationQueue) record(ctx context.Context, resourceKey string, action } q.mu.Lock() - queued, waiting := q.pending[resourceKey] - if waiting { - op.action = mergeAction(queued.action, op.action) - } + _, replaced := q.pending[resourceKey] q.pending[resourceKey] = op - owned := q.owned[resourceKey] - q.owned[resourceKey] = true + alreadyHandled := q.queuedOrUploading[resourceKey] + q.queuedOrUploading[resourceKey] = true q.mu.Unlock() - if waiting { + if replaced { log.Debugf(ctx, "Coalescing queued deployment operation for %s", resourceKey) } - // Someone already owns this key, so pending is enough: a worker will pick the - // operation up. Queueing again would upload the resource twice in parallel. - if owned { + // A worker is already going to handle this key, and it re-reads pending before + // finishing, so it will see the operation written above. Queueing the key again + // would let a second worker upload the same resource concurrently. + if alreadyHandled { return nil } @@ -152,7 +159,7 @@ func (q *operationQueue) work(ctx context.Context) { defer q.wg.Done() for resourceKey := range q.queue { - // Keep uploading this key until nothing new was recorded for it, instead of + // Keep uploading this key until nothing new was recorded for it, rather than // putting it back on the queue: a worker sending to the channel it consumes // from can deadlock once the queue is full. for { @@ -168,16 +175,19 @@ func (q *operationQueue) work(ctx context.Context) { } } -// take claims the operation waiting for resourceKey. It reports false and gives -// up ownership when nothing is waiting, which is what lets the next record -// queue the key again. +// take claims the operation waiting for resourceKey. It reports false and clears +// the queuedOrUploading mark when nothing is waiting, which is what lets the next +// record queue the key again. +// +// Clearing the mark and observing pending empty happen under one lock, so record +// can never skip queueing a key that no worker is going to look at again. func (q *operationQueue) take(resourceKey string) (recordedOperation, bool) { q.mu.Lock() defer q.mu.Unlock() op, ok := q.pending[resourceKey] if !ok { - delete(q.owned, resourceKey) + delete(q.queuedOrUploading, resourceKey) return recordedOperation{}, false } diff --git a/bundle/direct/opqueue_test.go b/bundle/direct/opqueue_test.go index 6bf0da8107b..356a7cfb319 100644 --- a/bundle/direct/opqueue_test.go +++ b/bundle/direct/opqueue_test.go @@ -22,9 +22,10 @@ type fakeUploader struct { started chan string err error - mu sync.Mutex - uploads []string - actions map[string]bundledeployments.OperationActionType + mu sync.Mutex + uploads []string + actions map[string]bundledeployments.OperationActionType + resourceIDs map[string]string } func (f *fakeUploader) upload(ctx context.Context, resourceKey string, op recordedOperation) error { @@ -40,8 +41,10 @@ func (f *fakeUploader) upload(ctx context.Context, resourceKey string, op record f.uploads = append(f.uploads, resourceKey+"="+string(op.state)) if f.actions == nil { f.actions = map[string]bundledeployments.OperationActionType{} + f.resourceIDs = map[string]string{} } f.actions[resourceKey] = op.action + f.resourceIDs[resourceKey] = op.resourceID return f.err } @@ -57,6 +60,12 @@ func (f *fakeUploader) actionFor(resourceKey string) bundledeployments.Operation return f.actions[resourceKey] } +func (f *fakeUploader) resourceIDFor(resourceKey string) string { + f.mu.Lock() + defer f.mu.Unlock() + return f.resourceIDs[resourceKey] +} + func recordState(t *testing.T, q *operationQueue, resourceKey, name string) { t.Helper() require.NoError(t, q.record(t.Context(), resourceKey, deployplan.Update, "id-1", map[string]string{"name": name}, nil)) @@ -99,9 +108,8 @@ func TestOperationQueueCoalescesQueuedOperationsForSameResource(t *testing.T) { }, f.recorded()) } -func TestOperationQueueCoalescingKeepsCreateAction(t *testing.T) { - // Hold the first upload so the create below stays queued and the update - // coalesces into it. +func TestOperationQueueCoalescingKeepsLatestOperation(t *testing.T) { + // Hold the first upload so the operations below stay queued and coalesce. f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 1)} q := newOperationQueue(t.Context(), f) @@ -114,15 +122,18 @@ func TestOperationQueueCoalescingKeepsCreateAction(t *testing.T) { assert.Equal(t, "resources.jobs.hold"+strconv.Itoa(i), <-f.started) } - require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", map[string]string{"name": "created"}, nil)) - require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Update, "id-1", map[string]string{"name": "updated"}, nil)) + // A resource whose ID is only known after it was created: the first operation + // has no ID, the second fills it in. + require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "", map[string]string{"name": "created"}, nil)) + require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", map[string]string{"name": "updated"}, nil)) close(f.block) require.NoError(t, q.close()) - // The state is the later one, but the action stays CREATE: recording an update - // would tell DMS the resource already existed before this deploy. + // Everything comes from the newest operation: it carries the resource's full + // state, and the ID it learned after the create. assert.Contains(t, f.recorded(), `resources.jobs.foo={"state":{"name":"updated"}}`) + assert.Equal(t, "id-1", f.resourceIDFor("resources.jobs.foo")) assert.Equal(t, bundledeployments.OperationActionTypeOperationActionTypeCreate, f.actionFor("resources.jobs.foo")) @@ -239,7 +250,7 @@ func TestOperationQueueUploadsOneResourceAtATime(t *testing.T) { // Every distinct key was recorded, and close drained all of them. assert.Len(t, u.last, distinctKeyMod) assert.Empty(t, q.pending) - assert.Empty(t, q.owned) + assert.Empty(t, q.queuedOrUploading) } func TestNilOperationQueueIsNoOp(t *testing.T) { diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index 91bcb65a4f6..2401e34fcf8 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -68,24 +68,6 @@ func newRecordedOperation(action deployplan.ActionType, resourceID string, state return op, nil } -// mergeAction returns the action to record when a later operation coalesces into -// one still queued for the same resource (see operationQueue.record). The state -// uploaded is the later one, but the action must not be downgraded: Create and -// Recreate tell DMS the resource ID is new, and a subsequent Update only refines -// the state of that same new resource. Recording the pair as an Update would -// claim the resource already existed. A Delete is the exception - the resource is -// gone, so nothing earlier is worth reporting. -func mergeAction(queued, next bundledeployments.OperationActionType) bundledeployments.OperationActionType { - if next == bundledeployments.OperationActionTypeOperationActionTypeDelete { - return next - } - if queued == bundledeployments.OperationActionTypeOperationActionTypeCreate || - queued == bundledeployments.OperationActionTypeOperationActionTypeRecreate { - return queued - } - return next -} - // operationUploader records an applied resource operation with DMS. Uploads run // on the operationQueue workers, off the apply path. type operationUploader interface { diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go index 556f7f59f90..f7262c04632 100644 --- a/bundle/direct/oprecorder_test.go +++ b/bundle/direct/oprecorder_test.go @@ -99,36 +99,6 @@ func TestNewRecordedOperationRejectsUnsupportedAction(t *testing.T) { assert.Error(t, err) } -func TestMergeAction(t *testing.T) { - const ( - create = bundledeployments.OperationActionTypeOperationActionTypeCreate - recreate = bundledeployments.OperationActionTypeOperationActionTypeRecreate - update = bundledeployments.OperationActionTypeOperationActionTypeUpdate - resize = bundledeployments.OperationActionTypeOperationActionTypeResize - del = bundledeployments.OperationActionTypeOperationActionTypeDelete - ) - - cases := []struct { - queued, next, want bundledeployments.OperationActionType - }{ - // A queued create is not downgraded: the resource is still new. - {create, update, create}, - {create, resize, create}, - {recreate, update, recreate}, - {create, create, create}, - // A delete wins: the resource is gone, so the earlier action is moot. - {create, del, del}, - {update, del, del}, - // Neither side is a create, so the later action stands. - {update, resize, resize}, - {resize, update, update}, - {del, create, create}, - } - for _, c := range cases { - assert.Equal(t, c.want, mergeAction(c.queued, c.next), "queued %s, next %s", c.queued, c.next) - } -} - func TestDeployActionToSDK(t *testing.T) { cases := []struct { action deployplan.ActionType From 96c0826849483f30565a609174637834bc909bc8 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 29 Jul 2026 10:13:16 +0000 Subject: [PATCH 17/28] acceptance: set MSYS_NO_PATHCONV for the DMS tests The dms tests pass workspace paths to $CLI (`workspace get-status /Workspace/...`). On Windows, Git Bash rewrites a leading-'/' argument into a Windows path before the CLI sees it, so the lookup goes to C:/Program Files/Git/Workspace/... and 404s, failing record, no-resources and redeploy-after-destroy on that platform only. Same fix and reason as acceptance/cmd/workspace/export-dir-*/test.toml. Co-authored-by: Isaac --- acceptance/bundle/dms/test.toml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/acceptance/bundle/dms/test.toml b/acceptance/bundle/dms/test.toml index 1e36331a16f..40cc45a44a5 100644 --- a/acceptance/bundle/dms/test.toml +++ b/acceptance/bundle/dms/test.toml @@ -10,3 +10,11 @@ RecordRequests = true Ignore = [ '.databricks', ] + +# These tests pass workspace paths to $CLI. On Windows, Git Bash rewrites a +# leading-'/' argument into a Windows path before the CLI sees it, so +# `workspace get-status /Workspace/...` looks up C:/Program Files/Git/Workspace/... +# and 404s. Quoting the argument does not help - the conversion happens in the +# Windows binary's argument processing. +[Env] +MSYS_NO_PATHCONV = "1" From b2ef0e83a76c48527379f2b5f39edb82a87169e4 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 29 Jul 2026 10:39:34 +0000 Subject: [PATCH 18/28] acceptance: scope MSYS_NO_PATHCONV to the get-status commands Setting it in test.toml applied it to the whole test, which stopped Git Bash converting the PATH too - so python3 could not find print_requests.py ("can't open file 'C:\\c\\a\\cli\\cli\\acceptance\\bin\\print_requests.py'") and every dms test failed on Windows, including the three that were passing. trace exports leading KEY=value pairs in a subshell, so setting it there fixes the CLI's leading-'/' argument without reaching the helpers. The precedent this was copied from (acceptance/cmd/workspace/export-dir-*/test.toml) uses no python helpers, which is why the test.toml form is safe there but not here. Co-authored-by: Isaac --- acceptance/bundle/dms/no-resources/output.txt | 2 +- acceptance/bundle/dms/no-resources/script | 2 +- acceptance/bundle/dms/record/output.txt | 2 +- acceptance/bundle/dms/record/script | 6 +++++- acceptance/bundle/dms/redeploy-after-destroy/output.txt | 4 ++-- acceptance/bundle/dms/redeploy-after-destroy/script | 4 ++-- acceptance/bundle/dms/test.toml | 8 -------- 7 files changed, 12 insertions(+), 16 deletions(-) diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index b400fb9a67e..86009c71b94 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -34,7 +34,7 @@ Deployment complete! } } ->>> [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/state/resources.deployment.json +>>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/state/resources.deployment.json { "object_type": "FILE", "path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/state/resources.deployment.json" diff --git a/acceptance/bundle/dms/no-resources/script b/acceptance/bundle/dms/no-resources/script index 847935af5d1..9b14355bd27 100644 --- a/acceptance/bundle/dms/no-resources/script +++ b/acceptance/bundle/dms/no-resources/script @@ -1,7 +1,7 @@ title "First deploy of a bundle with no resources: the deployment is created, and its workspace node identifies it even though no resource state was written" trace $CLI bundle deploy trace print_requests.py //api/2.0/bundle --sort --get -trace $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-resources/default/state/resources.deployment.json" | jq '{object_type,path}' +trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-resources/default/state/resources.deployment.json" | jq '{object_type,path}' title "Redeploy: the deployment is resolved from that node, so no second deployment is created" trace $CLI bundle deploy diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index 3266e0d3af8..c2b3333e587 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -64,7 +64,7 @@ Deployment complete! } === The deployment ID is the ID of the workspace node the service registered under initial_parent_path; the CLI stores nothing locally ->>> [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/resources.deployment.json +>>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/resources.deployment.json { "object_type": "FILE", "path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/resources.deployment.json" diff --git a/acceptance/bundle/dms/record/script b/acceptance/bundle/dms/record/script index 3298c8fb131..0ad4dd96d98 100644 --- a/acceptance/bundle/dms/record/script +++ b/acceptance/bundle/dms/record/script @@ -3,7 +3,11 @@ trace $CLI bundle deploy trace print_requests.py //api/2.0/bundle --sort title "The deployment ID is the ID of the workspace node the service registered under initial_parent_path; the CLI stores nothing locally" -trace $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record/default/state/resources.deployment.json" | jq '{object_type,path}' +# MSYS_NO_PATHCONV stops Git Bash on Windows from rewriting the leading-'/' path +# into C:/Program Files/Git/Workspace/... before the CLI sees it. Set per command +# rather than in test.toml: trace exports it in a subshell, so it cannot reach the +# python helpers, whose PATH does need converting. +trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record/default/state/resources.deployment.json" | jq '{object_type,path}' trace jq 'has("deployment_id")' .databricks/bundle/default/resources.json title "The state records the feature flag, at the version that makes a CLI without it refuse the state rather than deploy against resources it cannot see" diff --git a/acceptance/bundle/dms/redeploy-after-destroy/output.txt b/acceptance/bundle/dms/redeploy-after-destroy/output.txt index ff8694b5424..b8b6f3c630d 100644 --- a/acceptance/bundle/dms/redeploy-after-destroy/output.txt +++ b/acceptance/bundle/dms/redeploy-after-destroy/output.txt @@ -15,7 +15,7 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> musterr [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json +>>> MSYS_NO_PATHCONV=1 musterr [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json Error: Path (/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json) doesn't exist. === Deploy again: with no node to resolve, a fresh deployment is created at version 1, with its own workspace node @@ -25,7 +25,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json +>>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json { "object_type": "FILE", "path": "/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" diff --git a/acceptance/bundle/dms/redeploy-after-destroy/script b/acceptance/bundle/dms/redeploy-after-destroy/script index 04c639019c9..a39edf3c0d8 100644 --- a/acceptance/bundle/dms/redeploy-after-destroy/script +++ b/acceptance/bundle/dms/redeploy-after-destroy/script @@ -3,9 +3,9 @@ trace $CLI bundle deploy trace $CLI bundle destroy --auto-approve print_requests.py //api/2.0/bundle --sort --get > /dev/null -trace musterr $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" +trace MSYS_NO_PATHCONV=1 musterr $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" title "Deploy again: with no node to resolve, a fresh deployment is created at version 1, with its own workspace node" trace $CLI bundle deploy -trace $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" | jq '{object_type,path}' +trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" | jq '{object_type,path}' trace print_requests.py //api/2.0/bundle --sort --get diff --git a/acceptance/bundle/dms/test.toml b/acceptance/bundle/dms/test.toml index 40cc45a44a5..1e36331a16f 100644 --- a/acceptance/bundle/dms/test.toml +++ b/acceptance/bundle/dms/test.toml @@ -10,11 +10,3 @@ RecordRequests = true Ignore = [ '.databricks', ] - -# These tests pass workspace paths to $CLI. On Windows, Git Bash rewrites a -# leading-'/' argument into a Windows path before the CLI sees it, so -# `workspace get-status /Workspace/...` looks up C:/Program Files/Git/Workspace/... -# and 404s. Quoting the argument does not help - the conversion happens in the -# Windows binary's argument processing. -[Env] -MSYS_NO_PATHCONV = "1" From 30adb997c90de6c5831773909387190372b0f663 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 29 Jul 2026 11:25:27 +0000 Subject: [PATCH 19/28] bundle: drop the locks in the operation queue's close close runs on one goroutine after every apply worker has returned, so nothing else touches the queue by then and the closed flag needs no protection. The wg.Wait orders the upload workers' writes to err before it is read, so that read needs none either. Also tightens the coalescing test to count uploads for the resource instead of only checking the payload of the last one. It asserted the merged content but would have passed if the operations had been uploaded twice, which is the thing coalescing exists to prevent. Co-authored-by: Isaac --- bundle/direct/opqueue.go | 14 ++++++-------- bundle/direct/opqueue_test.go | 10 ++++++++++ 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go index 9a1a1cd918e..8d1bea2ff72 100644 --- a/bundle/direct/opqueue.go +++ b/bundle/direct/opqueue.go @@ -135,23 +135,21 @@ func (q *operationQueue) record(ctx context.Context, resourceKey string, action // record must have returned first: record on a closed queue panics. Calling close // more than once is safe, so callers can defer it and still check the error at a // specific point. +// +// Unlike the other methods this one takes no lock. It runs on one goroutine after +// every apply worker has returned, so nothing else touches the queue by then, and +// the wg.Wait below orders the workers' writes to err before it is read. func (q *operationQueue) close() error { if q == nil { return nil } - q.mu.Lock() - closed := q.closed - q.closed = true - q.mu.Unlock() - - if !closed { + if !q.closed { + q.closed = true close(q.queue) q.wg.Wait() } - q.mu.Lock() - defer q.mu.Unlock() return q.err } diff --git a/bundle/direct/opqueue_test.go b/bundle/direct/opqueue_test.go index 356a7cfb319..69492c2e63e 100644 --- a/bundle/direct/opqueue_test.go +++ b/bundle/direct/opqueue_test.go @@ -130,6 +130,16 @@ func TestOperationQueueCoalescingKeepsLatestOperation(t *testing.T) { close(f.block) require.NoError(t, q.close()) + // One upload, not two: the second operation replaced the first while every + // worker was busy, so the extra CreateOperation round trip never happens. + var uploadsForFoo int + for _, u := range f.recorded() { + if strings.HasPrefix(u, "resources.jobs.foo=") { + uploadsForFoo++ + } + } + assert.Equal(t, 1, uploadsForFoo, "the two operations should coalesce into one upload") + // Everything comes from the newest operation: it carries the resource's full // state, and the ID it learned after the create. assert.Contains(t, f.recorded(), `resources.jobs.foo={"state":{"name":"updated"}}`) From 8f56ef6878728e88ffbc6da8d58f61130a0aea54 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 29 Jul 2026 11:33:25 +0000 Subject: [PATCH 20/28] bundle: record operation state without redacting it Drops the RedactSensitiveFields call, leaving a TODO: fields marked bundle:"sensitive" now reach DMS in plaintext, and the read path writes them back into the local state file unredacted. This has to be restored before the feature ships to users. Also documents why take leaves the key in queuedOrUploading when it hands an operation to a worker, and covers the case the comment describes: recording while that key's upload is in flight. The operation cannot join the in-flight request, so it is uploaded next by the same worker rather than being dropped or picked up concurrently by a second one. Co-authored-by: Isaac --- bundle/direct/opqueue.go | 5 +++++ bundle/direct/opqueue_test.go | 27 +++++++++++++++++++++++++++ bundle/direct/oprecorder.go | 10 ++++------ bundle/direct/oprecorder_test.go | 9 ++++----- 4 files changed, 40 insertions(+), 11 deletions(-) diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go index 8d1bea2ff72..81163915b96 100644 --- a/bundle/direct/opqueue.go +++ b/bundle/direct/opqueue.go @@ -189,6 +189,11 @@ func (q *operationQueue) take(resourceKey string) (recordedOperation, bool) { return recordedOperation{}, false } + // The key stays in queuedOrUploading: the worker keeps coming back here until + // nothing is pending for it, so anything recorded while this operation uploads + // is still picked up. The mark is only cleared above, once there is nothing + // left - which is also what stops a second worker from taking the key and + // uploading the same resource concurrently. delete(q.pending, resourceKey) return op, true } diff --git a/bundle/direct/opqueue_test.go b/bundle/direct/opqueue_test.go index 69492c2e63e..b28d75dbf43 100644 --- a/bundle/direct/opqueue_test.go +++ b/bundle/direct/opqueue_test.go @@ -149,6 +149,33 @@ func TestOperationQueueCoalescingKeepsLatestOperation(t *testing.T) { f.actionFor("resources.jobs.foo")) } +func TestOperationQueueRecordDuringUploadIsStillUploaded(t *testing.T) { + // Record while the key's own upload is in flight: the key is off the queue but + // still marked, so record does not queue it again. The worker that holds the key + // has to come back for it, or the operation would be silently dropped. + f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 1)} + q := newOperationQueue(t.Context(), f) + + require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "", map[string]string{"name": "v1"}, nil)) + assert.Equal(t, "resources.jobs.foo", <-f.started) + + // The worker has taken the key off the queue and is uploading v1 right now. + require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", map[string]string{"name": "v2"}, nil)) + + close(f.block) + require.NoError(t, q.close()) + + // Two uploads, in order: an in-flight request cannot be recalled, so v2 goes up + // after v1 rather than replacing it. The service ends up with the newest state. + assert.Equal(t, []string{ + `resources.jobs.foo={"state":{"name":"v1"}}`, + `resources.jobs.foo={"state":{"name":"v2"}}`, + }, f.recorded()) + assert.Equal(t, "id-1", f.resourceIDFor("resources.jobs.foo")) + assert.Empty(t, q.pending) + assert.Empty(t, q.queuedOrUploading) +} + func TestOperationQueueReturnsUploadError(t *testing.T) { uploadErr := errors.New("boom") f := &fakeUploader{err: uploadErr} diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index 2401e34fcf8..4f932f09c3a 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -8,8 +8,6 @@ import ( "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct/dstate" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/structs/structwalk" "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) @@ -47,11 +45,11 @@ func newRecordedOperation(action deployplan.ActionType, resourceID string, state // Operation.State carries the serialized state, which DMS serves back as // resource state. Unset for delete: the resource is gone. // - // Redact secrets, like dstate.SaveState does for the local state file: - // otherwise we leak them to the service and the read path writes them back - // into a local state file in plaintext. + // TODO(DMS): fields marked bundle:"sensitive" are recorded in plaintext here, + // unlike dstate.SaveState which redacts them before writing the local state + // file. Redact them before this ships to users. if state != nil { - config, err := structwalk.RedactSensitiveFields(state, dyn.SensitiveValueRedacted) + config, err := json.Marshal(state) if err != nil { return recordedOperation{}, fmt.Errorf("serializing state: %w", err) } diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go index f7262c04632..a948b609441 100644 --- a/bundle/direct/oprecorder_test.go +++ b/bundle/direct/oprecorder_test.go @@ -6,7 +6,6 @@ import ( "testing" "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -65,7 +64,7 @@ func TestOperationRecorderDeleteHasNoState(t *testing.T) { assert.Nil(t, f.requests[0].Operation.State) } -func TestNewRecordedOperationRedactsSensitiveFields(t *testing.T) { +func TestNewRecordedOperationDoesNotRedactSensitiveFields(t *testing.T) { state := struct { Name string `json:"name"` Token string `json:"token" bundle:"sensitive"` @@ -74,10 +73,10 @@ func TestNewRecordedOperationRedactsSensitiveFields(t *testing.T) { op, err := newRecordedOperation(deployplan.Create, "job-123", state, nil) require.NoError(t, err) - // Sensitive fields are redacted before leaving the CLI, matching what - // dstate.SaveState writes to the local state file. + // Recorded as-is for now, unlike dstate.SaveState which redacts before writing + // the local state file. See the TODO in newRecordedOperation. assert.JSONEq(t, - `{"state":{"name":"foo","token":"`+dyn.SensitiveValueRedacted+`"}}`, + `{"state":{"name":"foo","token":"super-secret"}}`, string(op.state)) } From 329886f2b2e62a821cead6ad54aaaebefee2381f Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 29 Jul 2026 11:38:48 +0000 Subject: [PATCH 21/28] bundle: drop the redaction TODO from the operation recorder Co-authored-by: Isaac --- bundle/direct/oprecorder.go | 4 ---- bundle/direct/oprecorder_test.go | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index 4f932f09c3a..d0d2694d358 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -44,10 +44,6 @@ func newRecordedOperation(action deployplan.ActionType, resourceID string, state // Operation.State carries the serialized state, which DMS serves back as // resource state. Unset for delete: the resource is gone. - // - // TODO(DMS): fields marked bundle:"sensitive" are recorded in plaintext here, - // unlike dstate.SaveState which redacts them before writing the local state - // file. Redact them before this ships to users. if state != nil { config, err := json.Marshal(state) if err != nil { diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go index a948b609441..1b68c3a626b 100644 --- a/bundle/direct/oprecorder_test.go +++ b/bundle/direct/oprecorder_test.go @@ -73,8 +73,8 @@ func TestNewRecordedOperationDoesNotRedactSensitiveFields(t *testing.T) { op, err := newRecordedOperation(deployplan.Create, "job-123", state, nil) require.NoError(t, err) - // Recorded as-is for now, unlike dstate.SaveState which redacts before writing - // the local state file. See the TODO in newRecordedOperation. + // Recorded as-is, unlike dstate.SaveState which redacts before writing the + // local state file. assert.JSONEq(t, `{"state":{"name":"foo","token":"super-secret"}}`, string(op.state)) From 2818d11bcab9cf6b958f67ee736c91e50698771d Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 29 Jul 2026 12:00:00 +0000 Subject: [PATCH 22/28] bundle: revert the schema annotation and tighten CompleteVersion's guard Drops the record_deployment_history annotation change (and the generated schema that followed from it), leaving both files as they are on main. CompleteVersion now keys its no-op on versionNum rather than on the heartbeat handle. Both are set together by CreateVersion, so the behaviour is the same - a deploy that was cancelled, or whose CreateVersion failed, does not complete a version that was never created - but the check now names the thing it is actually guarding. Callers defer CompleteVersion unconditionally, so this is the only thing standing between a failed CreateVersion and a CompleteVersion call against a nonexistent version. Co-authored-by: Isaac --- bundle/internal/schema/annotations.yml | 2 -- bundle/schema/jsonschema.json | 2 +- libs/dms/recorder.go | 7 +++++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 100d33356fd..10832fe04a6 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -178,8 +178,6 @@ experimental: "record_deployment_history": "description": |- Whether to record deployment history using the deployment metadata service (DMS), which tracks what changed across deployments. - - Only supported for a bundle with no deployed resources yet. "scripts": "description": |- The commands to run. diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index c52e96d5dc9..4c78bd7c384 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -3016,7 +3016,7 @@ "$ref": "#/$defs/bool" }, "record_deployment_history": { - "description": "Whether to record deployment history using the deployment metadata service (DMS), which tracks what changed across deployments.\n\nOnly supported for a bundle with no deployed resources yet.", + "description": "Whether to record deployment history using the deployment metadata service (DMS), which tracks what changed across deployments.", "$ref": "#/$defs/bool" }, "scripts": { diff --git a/libs/dms/recorder.go b/libs/dms/recorder.go index 83f180ba3be..e285a80e4a2 100644 --- a/libs/dms/recorder.go +++ b/libs/dms/recorder.go @@ -102,9 +102,12 @@ func (r *Recorder) CreateVersion(ctx context.Context) error { } // CompleteVersion finalizes the version created by CreateVersion. A nil -// Recorder, or one whose CreateVersion never ran, is a no-op. +// Recorder, or one whose CreateVersion never ran or failed, is a no-op: there is +// no version on the server to complete. Callers defer it unconditionally, so this +// is the check that keeps a cancelled or failed deploy from completing a version +// that was never created. func (r *Recorder) CompleteVersion(ctx context.Context, success bool) error { - if r == nil || r.stopHeartbeat == nil { + if r == nil || r.versionNum == 0 { return nil } From b84ae7c631453e3cba92d83f729b44fab0588900 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 29 Jul 2026 12:24:31 +0000 Subject: [PATCH 23/28] bundle: gate record_deployment_history off again Restores validate.ValidateRecordDeploymentHistory and the hidden DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY escape hatch, so setting experimental.record_deployment_history is an error unless that variable is set. The service side is not ready for users: DMS is only deployed to dev and staging, and reading state back needs the workspace APIs to expose the deployment's tree node, which is still behind a flag. With the flag unreachable, the state feature flag added earlier is not needed yet, so dstate is back to the resource-count check in Open. The deployment_history feature, hasFeature/setFeature, the version bump on write and the WAL carry-over all come back with the state upgrade in a follow-up. The dms acceptance tests force allow the flag, and bundle/dms/not-supported covers the error users see. Operation requests in bundle/dms/depends-on print multi-line now: the state envelope nests two levels, which --oneline made unreadable. Co-authored-by: Isaac --- acceptance/bundle/dms/depends-on/output.txt | 67 ++++++++++- acceptance/bundle/dms/depends-on/script | 2 +- .../bundle/dms/existing-state/output.txt | 4 +- .../bundle/dms/not-supported/databricks.yml | 10 ++ .../bundle/dms/not-supported/out.test.toml | 3 + .../bundle/dms/not-supported/output.txt | 24 ++++ acceptance/bundle/dms/not-supported/script | 5 + acceptance/bundle/dms/not-supported/test.toml | 6 + acceptance/bundle/dms/record/output.txt | 9 -- acceptance/bundle/dms/record/script | 3 - acceptance/bundle/dms/test.toml | 6 + .../validate_record_deployment_history.go | 47 ++++++++ ...validate_record_deployment_history_test.go | 55 +++++++++ bundle/direct/dstate/migrate.go | 35 +++--- bundle/direct/dstate/state.go | 110 ++++++------------ bundle/direct/dstate/state_test.go | 21 +--- .../force_allow_record_deployment_history.go | 19 +++ bundle/phases/initialize.go | 5 + cmd/bundle/utils/process.go | 1 - 19 files changed, 296 insertions(+), 136 deletions(-) create mode 100644 acceptance/bundle/dms/not-supported/databricks.yml create mode 100644 acceptance/bundle/dms/not-supported/out.test.toml create mode 100644 acceptance/bundle/dms/not-supported/output.txt create mode 100644 acceptance/bundle/dms/not-supported/script create mode 100644 acceptance/bundle/dms/not-supported/test.toml create mode 100644 bundle/config/validate/validate_record_deployment_history.go create mode 100644 bundle/config/validate/validate_record_deployment_history_test.go create mode 100644 bundle/env/force_allow_record_deployment_history.go diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index 2bb8d06b9dc..67fd0165669 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -6,9 +6,70 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //versions/1/operations --sort --oneline -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.child"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.child", "state": {"state": {"deployment": {"kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json"}, "description": "depends on [NUMID]", "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "max_concurrent_runs": 1, "name": "child", "queue": {"enabled": true}}, "depends_on": [{"node": "resources.jobs.parent", "label": "${resources.jobs.parent.id}"}]}, "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.parent"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.parent", "state": {"state": {"deployment": {"kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json"}, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "max_concurrent_runs": 1, "name": "parent", "queue": {"enabled": true}}}, "status": "OPERATION_STATUS_SUCCEEDED"}} +>>> print_requests.py //versions/1/operations --sort +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", + "q": { + "resource_key": "jobs.child" + }, + "body": { + "action_type": "OPERATION_ACTION_TYPE_CREATE", + "resource_id": "[NUMID]", + "resource_key": "jobs.child", + "state": { + "state": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json" + }, + "description": "depends on [NUMID]", + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "child", + "queue": { + "enabled": true + } + }, + "depends_on": [ + { + "node": "resources.jobs.parent", + "label": "${resources.jobs.parent.id}" + } + ] + }, + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", + "q": { + "resource_key": "jobs.parent" + }, + "body": { + "action_type": "OPERATION_ACTION_TYPE_CREATE", + "resource_id": "[NUMID]", + "resource_key": "jobs.parent", + "state": { + "state": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "parent", + "queue": { + "enabled": true + } + } + }, + "status": "OPERATION_STATUS_SUCCEEDED" + } +} === Wipe the local state, then destroy: depends_on comes back from DMS, so the child is still deleted before the parent it references >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/dms/depends-on/script b/acceptance/bundle/dms/depends-on/script index 905d022619c..be1b9d622f8 100644 --- a/acceptance/bundle/dms/depends-on/script +++ b/acceptance/bundle/dms/depends-on/script @@ -1,6 +1,6 @@ title "Deploy a job that references another: depends_on is recorded alongside the config, since the reference is resolved to a literal in the config itself" trace $CLI bundle deploy -trace print_requests.py //versions/1/operations --sort --oneline +trace print_requests.py //versions/1/operations --sort title "Wipe the local state, then destroy: depends_on comes back from DMS, so the child is still deleted before the parent it references" rm -rf .databricks diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index 229b3bea03f..a793cc0fae1 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -12,7 +12,7 @@ Deployment complete! >>> update_file.py databricks.yml record_deployment_history: false record_deployment_history: true >>> musterr [CLI] bundle deploy -Error: target "default" was deployed without experimental.record_deployment_history and cannot be migrated to it: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json tracks resources this deployment recorded before the feature was enabled. Use a new target, destroy this one and deploy it again, or unset experimental.record_deployment_history +Error: cannot record deployment history for a bundle that already has deployed resources tracked in [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again === No deployment was created in DMS @@ -20,7 +20,7 @@ Error: target "default" was deployed without experimental.record_deployment_hist === Still an error after wiping the local cache: deploy pulls the state file back from the workspace, so the resources are still tracked >>> musterr [CLI] bundle deploy -Error: target "default" was deployed without experimental.record_deployment_history and cannot be migrated to it: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json tracks resources this deployment recorded before the feature was enabled. Use a new target, destroy this one and deploy it again, or unset experimental.record_deployment_history +Error: cannot record deployment history for a bundle that already has deployed resources tracked in [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again >>> print_requests.py //api/2.0/bundle --sort --oneline diff --git a/acceptance/bundle/dms/not-supported/databricks.yml b/acceptance/bundle/dms/not-supported/databricks.yml new file mode 100644 index 00000000000..c6edca465b6 --- /dev/null +++ b/acceptance/bundle/dms/not-supported/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: dms-not-supported + +experimental: + record_deployment_history: true + +resources: + jobs: + one: + name: one diff --git a/acceptance/bundle/dms/not-supported/out.test.toml b/acceptance/bundle/dms/not-supported/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/dms/not-supported/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/not-supported/output.txt b/acceptance/bundle/dms/not-supported/output.txt new file mode 100644 index 00000000000..b665d545d12 --- /dev/null +++ b/acceptance/bundle/dms/not-supported/output.txt @@ -0,0 +1,24 @@ + +=== record_deployment_history is rejected: the service side is not ready for users yet +>>> musterr [CLI] bundle validate +Error: experimental.record_deployment_history is not supported yet + at experimental.record_deployment_history + in databricks.yml:5:30 + +Name: dms-not-supported +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/dms-not-supported/default + +Found 1 error + +=== The hidden force-allow variable permits it +>>> DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY=1 [CLI] bundle validate +Name: dms-not-supported +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/dms-not-supported/default + +Validation OK! diff --git a/acceptance/bundle/dms/not-supported/script b/acceptance/bundle/dms/not-supported/script new file mode 100644 index 00000000000..10a0d995d87 --- /dev/null +++ b/acceptance/bundle/dms/not-supported/script @@ -0,0 +1,5 @@ +title "record_deployment_history is rejected: the service side is not ready for users yet" +trace musterr $CLI bundle validate + +title "The hidden force-allow variable permits it" +trace DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY=1 $CLI bundle validate diff --git a/acceptance/bundle/dms/not-supported/test.toml b/acceptance/bundle/dms/not-supported/test.toml new file mode 100644 index 00000000000..4617ff88f20 --- /dev/null +++ b/acceptance/bundle/dms/not-supported/test.toml @@ -0,0 +1,6 @@ +# Unset the force-allow variable inherited from the parent: this test asserts the +# error users see. +Env.DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY = "" + +# This test only checks validation output; no DMS request is made either way. +RecordRequests = false diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index c2b3333e587..33cf719ddfd 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -73,15 +73,6 @@ Deployment complete! >>> jq has("deployment_id") .databricks/bundle/default/resources.json false -=== The state records the feature flag, at the version that makes a CLI without it refuse the state rather than deploy against resources it cannot see ->>> jq {state_version, features} .databricks/bundle/default/resources.json -{ - "state_version": 3, - "features": { - "deployment_history": {} - } -} - === Redeploy after deleting the local cache: the deployment ID is resolved from that node, the same deployment is reused, and the version increments (no new CreateDeployment) >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record/default/files... diff --git a/acceptance/bundle/dms/record/script b/acceptance/bundle/dms/record/script index 0ad4dd96d98..32ee3ec972f 100644 --- a/acceptance/bundle/dms/record/script +++ b/acceptance/bundle/dms/record/script @@ -10,9 +10,6 @@ title "The deployment ID is the ID of the workspace node the service registered trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record/default/state/resources.deployment.json" | jq '{object_type,path}' trace jq 'has("deployment_id")' .databricks/bundle/default/resources.json -title "The state records the feature flag, at the version that makes a CLI without it refuse the state rather than deploy against resources it cannot see" -trace jq '{state_version, features}' .databricks/bundle/default/resources.json - title "Redeploy after deleting the local cache: the deployment ID is resolved from that node, the same deployment is reused, and the version increments (no new CreateDeployment)" rm -rf .databricks trace $CLI bundle deploy diff --git a/acceptance/bundle/dms/test.toml b/acceptance/bundle/dms/test.toml index 1e36331a16f..7c21473f724 100644 --- a/acceptance/bundle/dms/test.toml +++ b/acceptance/bundle/dms/test.toml @@ -10,3 +10,9 @@ RecordRequests = true Ignore = [ '.databricks', ] + +# experimental.record_deployment_history is rejected outright (see +# validate.ValidateRecordDeploymentHistory). These tests exercise the feature itself, +# so they force allow it the same way DMS development does. bundle/dms/not-supported +# covers the rejection. +Env.DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY = "1" diff --git a/bundle/config/validate/validate_record_deployment_history.go b/bundle/config/validate/validate_record_deployment_history.go new file mode 100644 index 00000000000..4f0137fae0f --- /dev/null +++ b/bundle/config/validate/validate_record_deployment_history.go @@ -0,0 +1,47 @@ +package validate + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/env" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +const recordDeploymentHistoryPath = "experimental.record_deployment_history" + +func ValidateRecordDeploymentHistory() bundle.ReadOnlyMutator { + return &validateRecordDeploymentHistory{} +} + +type validateRecordDeploymentHistory struct{ bundle.RO } + +func (v *validateRecordDeploymentHistory) Name() string { + return "validate:validate_record_deployment_history" +} + +// Apply rejects experimental.record_deployment_history. +// +// Recording deployment history is implemented end to end, but the service side is not +// ready for users: the deployment metadata service is only deployed to dev and staging, +// and reading state back needs the workspace APIs to expose the deployment's tree node, +// which is still behind a flag. Enabling this today also makes DMS the source of truth +// for resource state, so a bundle that turns it on cannot be turned back. +// +// DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY force allows it for the CLI's +// own tests and for DMS development. +func (v *validateRecordDeploymentHistory) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + if b.Config.Experimental == nil || !b.Config.Experimental.RecordDeploymentHistory { + return nil + } + if env.ForceAllowRecordDeploymentHistory(ctx) { + return nil + } + return diag.Diagnostics{{ + Severity: diag.Error, + Summary: recordDeploymentHistoryPath + " is not supported yet", + Paths: []dyn.Path{dyn.MustPathFromString(recordDeploymentHistoryPath)}, + Locations: b.Config.GetLocations(recordDeploymentHistoryPath), + }} +} diff --git a/bundle/config/validate/validate_record_deployment_history_test.go b/bundle/config/validate/validate_record_deployment_history_test.go new file mode 100644 index 00000000000..1bb172766f2 --- /dev/null +++ b/bundle/config/validate/validate_record_deployment_history_test.go @@ -0,0 +1,55 @@ +package validate + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + bundleenv "github.com/databricks/cli/bundle/env" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/env" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestValidateRecordDeploymentHistory(t *testing.T) { + tests := []struct { + name string + enabled bool + forceAllow string + wantError bool + }{ + {name: "flag unset", enabled: false, wantError: false}, + {name: "flag set", enabled: true, wantError: true}, + {name: "flag set with force allow", enabled: true, forceAllow: "1", wantError: false}, + {name: "flag set with empty force allow", enabled: true, forceAllow: "", wantError: true}, + {name: "flag unset with force allow", enabled: false, forceAllow: "1", wantError: false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Experimental: &config.Experimental{RecordDeploymentHistory: tc.enabled}, + }, + } + + ctx := env.Set(t.Context(), bundleenv.ForceAllowRecordDeploymentHistoryVariable, tc.forceAllow) + diags := ValidateRecordDeploymentHistory().Apply(ctx, b) + + if !tc.wantError { + assert.Empty(t, diags) + return + } + require.Len(t, diags, 1) + assert.Equal(t, diag.Error, diags[0].Severity) + assert.Equal(t, "experimental.record_deployment_history is not supported yet", diags[0].Summary) + assert.Equal(t, recordDeploymentHistoryPath, diags[0].Paths[0].String()) + }) + } +} + +func TestValidateRecordDeploymentHistoryNoExperimentalBlock(t *testing.T) { + b := &bundle.Bundle{Config: config.Root{}} + assert.Empty(t, ValidateRecordDeploymentHistory().Apply(t.Context(), b)) +} diff --git a/bundle/direct/dstate/migrate.go b/bundle/direct/dstate/migrate.go index 288fdcf863f..e4d21a7054a 100644 --- a/bundle/direct/dstate/migrate.go +++ b/bundle/direct/dstate/migrate.go @@ -12,34 +12,25 @@ import ( "github.com/databricks/databricks-sdk-go/service/iam" ) -// knownFeatures lists the state feature flags this CLI implements. A state that -// records anything outside this set is refused by migrateState. -var knownFeatures = map[string]bool{ - FeatureDeploymentHistory: true, -} - // migrateState runs all necessary migrations on the database. // It is called after loading state from disk. func migrateState(db *Database) error { - // featureStateVersion states carry a feature list (see the featureStateVersion - // doc comment). A featureStateVersion state with no features is equivalent to - // currentStateVersion, so accept it and return without running the migrations - // below, leaving the on-disk version at featureStateVersion rather than flipping - // it down. Same for a state whose features this CLI implements. One that records - // a feature this CLI does not know depends on capabilities it lacks, so refuse it - // and tell the user to upgrade. + // featureStateVersion states carry a feature list this CLI does not yet write or + // understand (see the featureStateVersion doc comment). A featureStateVersion + // state with no features is equivalent to currentStateVersion, so accept it and + // return without running the migrations below, leaving the on-disk version at + // featureStateVersion rather than flipping it down. One that records any feature + // depends on capabilities this CLI lacks, so refuse it and tell the user to upgrade. if db.StateVersion == featureStateVersion { - unknown := make([]string, 0, len(db.Features)) - for name := range db.Features { - if !knownFeatures[name] { - unknown = append(unknown, name) - } - } - if len(unknown) == 0 { + if len(db.Features) == 0 { return nil } - slices.Sort(unknown) - return fmt.Errorf("the deployment state requires features this CLI does not support: %s; upgrade to the latest CLI version and see %s for more information", strings.Join(unknown, ", "), featuresDocURL) + features := make([]string, 0, len(db.Features)) + for name := range db.Features { + features = append(features, name) + } + slices.Sort(features) + return fmt.Errorf("the deployment state requires features this CLI does not support: %s; upgrade to the latest CLI version and see %s for more information", strings.Join(features, ", "), featuresDocURL) } if db.StateVersion == currentStateVersion { diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index b367e97001b..bad87a6a293 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -31,28 +31,22 @@ const ( maxWalEntrySize = 10 * 1024 * 1024 walSuffix = ".wal" - // featureStateVersion is the schema version written once a state records - // deployment state "feature flags" (see Header.Features). Reading such a state - // (see migrateState): - // - featureStateVersion with no features -> accept, leave the version as-is - // - featureStateVersion with known features -> accept - // - featureStateVersion with unknown features -> refuse, tell the user to upgrade + // featureStateVersion is the schema version a future CLI will write once it + // records deployment state "feature flags" (see Header.Features). This CLI does + // not write it and records no features; it exists now only so this CLI reads + // such states correctly (see migrateState): + // - featureStateVersion with no features -> accept and leave the version as-is + // - featureStateVersion with any feature -> refuse, tell the user to upgrade // // A featureStateVersion state with no features is equivalent to // currentStateVersion, but we deliberately do not flip the on-disk version down // to currentStateVersion: a state written at featureStateVersion stays at - // featureStateVersion. That way a release can start writing - // featureStateVersion + features without older CLIs either mishandling a feature - // they lack or rejecting a featureless state outright. featureStateVersion is - // always 3. + // featureStateVersion. This is forward-compat scaffolding so that a later release + // can start writing featureStateVersion + features without older CLIs (with this + // change) either mishandling a feature they lack or rejecting a featureless state + // outright. featureStateVersion is always 3. featureStateVersion = 3 - // FeatureDeploymentHistory marks a state whose resources live in the deployment - // metadata service rather than in the state file. A CLI that does not know this - // feature refuses the state instead of deploying against a resource set it - // cannot see - the file's resources are not authoritative. - FeatureDeploymentHistory = "deployment_history" - // supportedStateVersion is the highest schema version this CLI can read. It is // normally equal to currentStateVersion — the version this CLI reads is the // version it writes — and exceeds it only during a two-phase version bump like @@ -88,27 +82,13 @@ type Header struct { Serial int `json:"serial"` // Features maps each feature flag this state depends on to a (currently empty) - // value. A CLI that does not implement one of them refuses the state rather than - // deploying against it (see migrateState). It is a map so a future CLI can attach - // per-feature data without reshaping the state. Empty/omitted for states that use - // no features. + // value. This CLI writes no features; it only reads the field to detect a state + // that depends on features it lacks and refuse it (see migrateState). It is a + // map so a future CLI can attach per-feature data without reshaping the state. + // Empty/omitted for states that use no features. Features map[string]struct{} `json:"features,omitempty"` } -// hasFeature reports whether the state records the given feature flag. -func (h *Header) hasFeature(name string) bool { - _, ok := h.Features[name] - return ok -} - -// setFeature records a feature flag in the state. -func (h *Header) setFeature(name string) { - if h.Features == nil { - h.Features = make(map[string]struct{}) - } - h.Features[name] = struct{}{} -} - type Database struct { Header @@ -248,10 +228,6 @@ type DMSSource struct { // DeploymentID is resolved from the deployment's workspace node (see // dms.ResolveDeploymentID), and empty before the first recorded deploy. DeploymentID string - - // TargetName names the bundle target in the error Open returns for a state that - // predates the opt-in, since the feature is enabled per target. - TargetName string } // Open reads the deployment state from disk (and recovers the WAL when @@ -307,18 +283,18 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W } if dmsSource != nil { - // An existing state file has to be marked as DMS-owned before its resources - // can be read from DMS. Recording only starts on an empty state, so a state - // with resources and no feature flag predates the opt-in: DMS never saw those - // resources and is authoritative for the whole set once enabled (see - // readDMSState), so they would look absent and be created a second time. - // Migrating such a target is not supported yet. - if len(db.Data.State) > 0 && !db.Data.hasFeature(FeatureDeploymentHistory) { - return fmt.Errorf("target %q was deployed without experimental.record_deployment_history and cannot be migrated to it: %s tracks resources this deployment recorded before the feature was enabled. Use a new target, destroy this one and deploy it again, or unset experimental.record_deployment_history", dmsSource.TargetName, path) + // Only bundles that start out empty can be recorded. Once DMS owns a + // deployment it is authoritative for the whole resource set (see + // readDMSState), so pre-existing resources it never saw would look absent and + // get created a second time. + // + // TODO(DMS): allow this by upgrading the state in place, writing it at + // featureStateVersion with a feature flag plus a tombstone per resource so an + // older CLI refuses the state instead of deploying against resources it + // cannot see. + if dmsSource.DeploymentID == "" && len(db.Data.State) > 0 { + return fmt.Errorf("cannot record deployment history for a bundle that already has deployed resources tracked in %s: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again", path) } - // Mark the state as DMS-owned so a CLI without this feature refuses it rather - // than deploying against a resource set it cannot see. - db.Data.setFeature(FeatureDeploymentHistory) if dmsSource.DeploymentID != "" { if err := db.readDMSState(ctx, dmsSource); err != nil { return err @@ -335,7 +311,13 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W return fmt.Errorf("failed to open WAL file %s: %w", walPath, err) } db.walFile = walFile - return appendJSONLine(db.walFile, db.newWalHeader()) + walHead := Header{ + Lineage: db.GetOrInitLineage(), + Serial: db.Data.Serial + 1, + StateVersion: currentStateVersion, + CLIVersion: build.GetInfo().Version, + } + return appendJSONLine(db.walFile, walHead) } return nil @@ -422,16 +404,6 @@ func (db *DeploymentState) mergeWalIntoState(ctx context.Context) (bool, error) return false, fmt.Errorf("WAL serial (%d) is ahead of expected (%d), state may be corrupted", header.Serial, expectedSerial) } newSerial = header.Serial - - // Carry the WAL's features (and the version that goes with them) into the - // state being written, so recovering a WAL written by a DMS-recording deploy - // still produces a state marked as DMS-owned. - for name := range header.Features { - db.Data.setFeature(name) - } - if len(db.Data.Features) > 0 { - db.Data.StateVersion = featureStateVersion - } } else { var entry WALEntry if err := json.Unmarshal(line, &entry); err != nil { @@ -535,25 +507,13 @@ func (db *DeploymentState) UpgradeToWrite() error { } db.walFile = walFile - return appendJSONLine(db.walFile, db.newWalHeader()) -} - -// newWalHeader builds the header for a fresh WAL. Features carry over from the -// state being written, and a state that records any feature is written at -// featureStateVersion so a CLI that lacks the feature refuses it instead of -// deploying against it. The caller holds db.mu. -func (db *DeploymentState) newWalHeader() Header { - version := currentStateVersion - if len(db.Data.Features) > 0 { - version = featureStateVersion - } - return Header{ + walHead := Header{ Lineage: db.GetOrInitLineage(), Serial: db.Data.Serial + 1, - StateVersion: version, + StateVersion: currentStateVersion, CLIVersion: build.GetInfo().Version, - Features: db.Data.Features, } + return appendJSONLine(db.walFile, walHead) } func (db *DeploymentState) AssertOpenedForReadOrWrite() { diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index 35050575121..a9c90530514 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -154,16 +154,7 @@ func TestEmptyFeatureStateAcceptedWithoutFlippingVersion(t *testing.T) { require.NoError(t, migrateState(empty)) assert.Equal(t, featureStateVersion, empty.StateVersion, "v3 + no features keeps its on-disk version, not flipped to v2") - // v3 recording a feature this CLI implements is accepted. - known := &Database{Header: Header{ - StateVersion: featureStateVersion, - Features: map[string]struct{}{FeatureDeploymentHistory: {}}, - }} - require.NoError(t, migrateState(known)) - assert.Equal(t, featureStateVersion, known.StateVersion) - - // v3 recording a feature this CLI does not know is refused: its resources may - // live somewhere this CLI cannot see. + // v3 that records a feature is refused: this CLI does not understand features. withFeature := &Database{Header: Header{ StateVersion: featureStateVersion, Features: map[string]struct{}{"future_feature": {}}, @@ -174,16 +165,6 @@ func TestEmptyFeatureStateAcceptedWithoutFlippingVersion(t *testing.T) { assert.Contains(t, err.Error(), "future_feature") assert.Contains(t, err.Error(), "upgrade to the latest CLI version") assert.Contains(t, err.Error(), featuresDocURL) - - // Only the unknown feature is named, so the message tells the user what to do. - mixed := &Database{Header: Header{ - StateVersion: featureStateVersion, - Features: map[string]struct{}{FeatureDeploymentHistory: {}, "future_feature": {}}, - }} - err = migrateState(mixed) - require.Error(t, err) - assert.Contains(t, err.Error(), "future_feature") - assert.NotContains(t, err.Error(), FeatureDeploymentHistory) } func TestDeleteState(t *testing.T) { diff --git a/bundle/env/force_allow_record_deployment_history.go b/bundle/env/force_allow_record_deployment_history.go new file mode 100644 index 00000000000..297ccb6f6e3 --- /dev/null +++ b/bundle/env/force_allow_record_deployment_history.go @@ -0,0 +1,19 @@ +package env + +import "context" + +// ForceAllowRecordDeploymentHistoryVariable names the environment variable that force +// allows experimental.record_deployment_history. It is deliberately undocumented: the +// feature is complete but cannot be exposed to users yet (see +// validate.ValidateRecordDeploymentHistory for why), and this variable exists so the +// CLI's own tests and the developers working on DMS can exercise the code path meanwhile. +const ForceAllowRecordDeploymentHistoryVariable = "DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY" + +// ForceAllowRecordDeploymentHistory reports whether the environment force allows +// experimental.record_deployment_history despite it being gated off. +func ForceAllowRecordDeploymentHistory(ctx context.Context) bool { + value, ok := get(ctx, []string{ + ForceAllowRecordDeploymentHistoryVariable, + }) + return ok && value != "" +} diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index bfa2af4124b..70ea74fa182 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -177,6 +177,11 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // They are set by the CLI to track the bundle deployment and must not be set by the user. validate.ValidateDeploymentFields(), + // Reads (typed): b.Config.Experimental.RecordDeploymentHistory + // Reads (env): DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY (non-empty value force allows it) + // Rejects experimental.record_deployment_history: the feature is not usable yet. + validate.ValidateRecordDeploymentHistory(), + // Reads (dynamic): * (strings) (searches for ${resources.*} references) // Warns (TF engine) or errors (direct engine) when a cross-resource reference // points to a Terraform-only field with no DABs equivalent. diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index b000da60264..209dc874403 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -229,7 +229,6 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle dmsSource = &dstate.DMSSource{ Client: w.BundleDeployments, DeploymentID: deploymentID, - TargetName: b.Config.Bundle.Target, } } if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dmsSource); err != nil { From daa69e28df3826732cde4e90edd787620b625749 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 29 Jul 2026 12:47:33 +0000 Subject: [PATCH 24/28] bundle: stop the deploy when an operation upload fails An upload failure was only reported at close, so a DMS outage let apply deploy every remaining resource and fail at the end. That leaves resources in the workspace that DMS has no record of, and since a completed version makes DMS the source of truth for resource state, the next deploy would create them again. record now returns the first upload error, which the apply worker turns into a failed node, so the deploy stops shortly after the failure instead of running to completion. It refuses new work only: operations already recorded still upload, because close drains them, so the records DMS ends up with match the resources that were actually applied. Resources already mid-apply also finish. Also repeats the one test whose bug depends on a scheduler interleaving rather than on a forced handshake, so a single run gets many chances to hit the bad ordering. The other tests pin their interleaving with the started/block channels, so repetition would not add coverage; the new error tests use a `done` channel to wait for an upload to have finished rather than merely started. Co-authored-by: Isaac --- bundle/direct/opqueue.go | 31 ++++++++- bundle/direct/opqueue_test.go | 121 ++++++++++++++++++++++++++-------- 2 files changed, 120 insertions(+), 32 deletions(-) diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go index 81163915b96..ba189c13d2e 100644 --- a/bundle/direct/opqueue.go +++ b/bundle/direct/opqueue.go @@ -93,9 +93,10 @@ func newOperationQueue(ctx context.Context, uploader operationUploader) *operati return q } -// record serializes an operation and hands it to the upload workers. It makes no -// API call, so upload failures surface from close; an error here only means the -// applied resource could not be turned into a payload. +// record serializes an operation and hands it to the upload workers. The upload +// itself happens on a worker, so an error returned here is either a failure to +// turn the applied resource into a payload, or an earlier upload's error +// resurfaced (see below). // // Recording a resource that is still waiting replaces the waiting operation // outright, since the newer one carries the resource's full state. @@ -104,6 +105,21 @@ func (q *operationQueue) record(ctx context.Context, resourceKey string, action return nil } + // Report an earlier upload failure to the apply worker that is about to record + // the next resource, so the deploy stops instead of running to completion and + // only failing at close. That matters because a successfully completed version + // makes DMS the source of truth for resource state (see dstate.readDMSState): + // deploying everything while its records are missing leaves resources the next + // deploy would create a second time. + // + // This refuses new work only. Operations already recorded still upload - close + // drains them - so the records DMS does end up with match the resources that + // were actually applied. Resources already mid-apply also finish, so the deploy + // stops shortly after the first failure rather than exactly at it. + if err := q.firstErr(); err != nil { + return err + } + op, err := newRecordedOperation(action, resourceID, state, dependsOn) if err != nil { return err @@ -208,3 +224,12 @@ func (q *operationQueue) setErr(err error) { q.err = err } } + +// firstErr returns the first upload error, or nil if every upload so far +// succeeded. +func (q *operationQueue) firstErr() error { + q.mu.Lock() + defer q.mu.Unlock() + + return q.err +} diff --git a/bundle/direct/opqueue_test.go b/bundle/direct/opqueue_test.go index b28d75dbf43..a68560493ce 100644 --- a/bundle/direct/opqueue_test.go +++ b/bundle/direct/opqueue_test.go @@ -20,7 +20,10 @@ import ( type fakeUploader struct { block chan struct{} started chan string - err error + // done receives the resource key after the upload returns, for tests that need + // an upload to have completed rather than merely started. + done chan string + err error mu sync.Mutex uploads []string @@ -37,7 +40,6 @@ func (f *fakeUploader) upload(ctx context.Context, resourceKey string, op record } f.mu.Lock() - defer f.mu.Unlock() f.uploads = append(f.uploads, resourceKey+"="+string(op.state)) if f.actions == nil { f.actions = map[string]bundledeployments.OperationActionType{} @@ -45,6 +47,13 @@ func (f *fakeUploader) upload(ctx context.Context, resourceKey string, op record } f.actions[resourceKey] = op.action f.resourceIDs[resourceKey] = op.resourceID + f.mu.Unlock() + + // Sent outside the lock: a test that stops reading this channel would otherwise + // hold f.mu and deadlock every other worker. + if f.done != nil { + f.done <- resourceKey + } return f.err } @@ -189,6 +198,54 @@ func TestOperationQueueReturnsUploadError(t *testing.T) { assert.Contains(t, err.Error(), "resources.jobs.foo") } +func TestOperationQueueRecordFailsAfterUploadError(t *testing.T) { + // An upload failure stops the deploy at the next resource instead of surfacing + // only at close, so the apply workers do not keep creating resources that DMS + // has no record of. + uploadErr := errors.New("boom") + f := &fakeUploader{err: uploadErr, done: make(chan string, 1)} + q := newOperationQueue(t.Context(), f) + + // Wait for the failing upload to finish, so the error is stored before the next + // record rather than racing it. + require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", map[string]string{"name": "v1"}, nil)) + assert.Equal(t, "resources.jobs.foo", <-f.done) + + // The next resource an apply worker tries to record is refused, with the upload + // error that caused it. + err := q.record(t.Context(), "resources.jobs.bar", deployplan.Create, "id-2", map[string]string{"name": "v1"}, nil) + require.Error(t, err) + assert.ErrorIs(t, err, uploadErr) + + // The refused resource was not queued, and close still reports the failure. + require.ErrorIs(t, q.close(), uploadErr) + assert.Equal(t, []string{`resources.jobs.foo={"state":{"name":"v1"}}`}, f.recorded()) + assert.Empty(t, q.pending) + assert.Empty(t, q.queuedOrUploading) +} + +func TestOperationQueueDrainsQueuedOperationsAfterUploadError(t *testing.T) { + // A failure refuses new work but does not discard work already recorded: the + // records DMS ends up with have to match the resources that were applied. + uploadErr := errors.New("boom") + f := &fakeUploader{err: uploadErr, block: make(chan struct{}), started: make(chan string, 1)} + q := newOperationQueue(t.Context(), f) + + // Every worker is parked mid-upload, so these stay queued. + for i := range operationUploadWorkers { + require.NoError(t, q.record(t.Context(), "resources.jobs.hold"+strconv.Itoa(i), deployplan.Create, "id-1", map[string]string{"name": "v1"}, nil)) + assert.Equal(t, "resources.jobs.hold"+strconv.Itoa(i), <-f.started) + } + require.NoError(t, q.record(t.Context(), "resources.jobs.queued", deployplan.Create, "id-2", map[string]string{"name": "v1"}, nil)) + + close(f.block) + require.ErrorIs(t, q.close(), uploadErr) + + // The queued operation was uploaded rather than dropped on the way out. + assert.Contains(t, f.recorded(), `resources.jobs.queued={"state":{"name":"v1"}}`) + assert.Len(t, f.recorded(), operationUploadWorkers+1) +} + func TestOperationQueueRecordRejectsUnsupportedAction(t *testing.T) { f := &fakeUploader{} q := newOperationQueue(t.Context(), f) @@ -254,40 +311,46 @@ func TestOperationQueueUploadsOneResourceAtATime(t *testing.T) { // case where a coalesced key can be handed to a second worker while the first // is still uploading it. The service keeps one state per key, so overlapping // uploads for a key could land out of order and leave a stale state behind. + // + // The interleaving that breaks this is scheduler-dependent, so one pass proves + // little: repeat it so a single run has many chances to hit the bad ordering. const ( + iterations = 200 workers = 10 perWorker = 5 distinctKeyMod = 12 ) - ctx := t.Context() - u := &serialUploader{live: map[string]bool{}, last: map[string]string{}} - q := newOperationQueue(ctx, u) - - // Collect record errors instead of asserting inside the goroutines: testify - // assertions may only run on the goroutine running the test function. - errs := make(chan error, workers*perWorker) - var wg sync.WaitGroup - for w := range workers { - wg.Go(func() { - for i := range perWorker { - key := "resources.jobs.job" + strconv.Itoa((w*perWorker+i)%distinctKeyMod) - errs <- q.record(ctx, key, deployplan.Update, "id-1", map[string]string{"name": strconv.Itoa(w)}, nil) - } - }) - } - wg.Wait() - close(errs) - for err := range errs { - require.NoError(t, err) - } - require.NoError(t, q.close()) + for range iterations { + ctx := t.Context() + u := &serialUploader{live: map[string]bool{}, last: map[string]string{}} + q := newOperationQueue(ctx, u) + + // Collect record errors instead of asserting inside the goroutines: testify + // assertions may only run on the goroutine running the test function. + errs := make(chan error, workers*perWorker) + var wg sync.WaitGroup + for w := range workers { + wg.Go(func() { + for i := range perWorker { + key := "resources.jobs.job" + strconv.Itoa((w*perWorker+i)%distinctKeyMod) + errs <- q.record(ctx, key, deployplan.Update, "id-1", map[string]string{"name": strconv.Itoa(w)}, nil) + } + }) + } + wg.Wait() + close(errs) + for err := range errs { + require.NoError(t, err) + } + require.NoError(t, q.close()) - assert.False(t, u.uneven, "two uploads overlapped for the same resource key") - // Every distinct key was recorded, and close drained all of them. - assert.Len(t, u.last, distinctKeyMod) - assert.Empty(t, q.pending) - assert.Empty(t, q.queuedOrUploading) + require.False(t, u.uneven, "two uploads overlapped for the same resource key") + // Every distinct key was recorded, and close drained all of them. + require.Len(t, u.last, distinctKeyMod) + require.Empty(t, q.pending) + require.Empty(t, q.queuedOrUploading) + } } func TestNilOperationQueueIsNoOp(t *testing.T) { From 2c2e209f3abc8cccdc37d91f354cf277bf642a28 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 29 Jul 2026 13:14:02 +0000 Subject: [PATCH 25/28] bundle: stop applying resources once an operation upload fails The previous commit made record return the upload error, but record runs after the resource has already been created or updated, so every node that started before the failure was noticed still modified the workspace. Apply now checks for a recorded failure before it touches anything, right after the dependency check, so a node that has not started yet is refused rather than applied. Resources already mid-apply still finish - the check cannot unwind those - but the deploy no longer runs to completion against a service that is rejecting its records. acceptance/bundle/dms/operation-upload-fails covers it. Which resources get refused depends on how far apply got before a background upload failed, so the per-resource errors go to a LOG file and requests are not recorded; the test asserts the deploy fails and reports the upload error. Also drops --sort from the dms tests that deploy zero or one resource: their request order is already deterministic, and the unsorted output reads in chronological order. Co-authored-by: Isaac --- .../bundle/dms/existing-state/output.txt | 10 ++--- acceptance/bundle/dms/existing-state/script | 8 ++-- acceptance/bundle/dms/no-resources/output.txt | 8 ++-- acceptance/bundle/dms/no-resources/script | 4 +- .../dms/operation-upload-fails/databricks.yml | 24 +++++++++++ .../dms/operation-upload-fails/out.test.toml | 3 ++ .../dms/operation-upload-fails/output.txt | 4 ++ .../bundle/dms/operation-upload-fails/script | 6 +++ .../dms/operation-upload-fails/test.toml | 12 ++++++ acceptance/bundle/dms/record/output.txt | 42 +++++++++---------- acceptance/bundle/dms/record/script | 6 +-- .../dms/redeploy-after-destroy/output.txt | 16 +++---- .../bundle/dms/redeploy-after-destroy/script | 4 +- bundle/direct/bundle_apply.go | 11 +++++ bundle/direct/opqueue.go | 6 ++- 15 files changed, 114 insertions(+), 50 deletions(-) create mode 100644 acceptance/bundle/dms/operation-upload-fails/databricks.yml create mode 100644 acceptance/bundle/dms/operation-upload-fails/out.test.toml create mode 100644 acceptance/bundle/dms/operation-upload-fails/output.txt create mode 100644 acceptance/bundle/dms/operation-upload-fails/script create mode 100644 acceptance/bundle/dms/operation-upload-fails/test.toml diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index a793cc0fae1..755981c414d 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -6,7 +6,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //api/2.0/bundle --sort --oneline +>>> print_requests.py //api/2.0/bundle --oneline === Turning recording on afterwards is an error: those resources were never recorded, so treating DMS as authoritative would deploy them a second time >>> update_file.py databricks.yml record_deployment_history: false record_deployment_history: true @@ -16,14 +16,14 @@ Error: cannot record deployment history for a bundle that already has deployed r === No deployment was created in DMS ->>> print_requests.py //api/2.0/bundle --sort --oneline +>>> print_requests.py //api/2.0/bundle --oneline === Still an error after wiping the local cache: deploy pulls the state file back from the workspace, so the resources are still tracked >>> musterr [CLI] bundle deploy Error: cannot record deployment history for a bundle that already has deployed resources tracked in [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again ->>> print_requests.py //api/2.0/bundle --sort --oneline +>>> print_requests.py //api/2.0/bundle --oneline === Destroy clears the tracked resources, so recording can be enabled afterwards >>> update_file.py databricks.yml record_deployment_history: true record_deployment_history: false @@ -45,8 +45,8 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //api/2.0/bundle --sort --oneline +>>> print_requests.py //api/2.0/bundle --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state", "target_name": "default"}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "state": {"state": {"deployment": {"kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state/metadata.json"}, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "max_concurrent_runs": 1, "name": "one", "queue": {"enabled": true}}}, "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/existing-state/script b/acceptance/bundle/dms/existing-state/script index ae9be95f701..9e448161259 100644 --- a/acceptance/bundle/dms/existing-state/script +++ b/acceptance/bundle/dms/existing-state/script @@ -1,22 +1,22 @@ title "Deploy without recording: the bundle gets ordinary direct-engine state, unknown to DMS" trace $CLI bundle deploy -trace print_requests.py //api/2.0/bundle --sort --oneline +trace print_requests.py //api/2.0/bundle --oneline title "Turning recording on afterwards is an error: those resources were never recorded, so treating DMS as authoritative would deploy them a second time" trace update_file.py databricks.yml "record_deployment_history: false" "record_deployment_history: true" trace musterr $CLI bundle deploy title "No deployment was created in DMS" -trace print_requests.py //api/2.0/bundle --sort --oneline +trace print_requests.py //api/2.0/bundle --oneline title "Still an error after wiping the local cache: deploy pulls the state file back from the workspace, so the resources are still tracked" rm -rf .databricks trace musterr $CLI bundle deploy -trace print_requests.py //api/2.0/bundle --sort --oneline +trace print_requests.py //api/2.0/bundle --oneline title "Destroy clears the tracked resources, so recording can be enabled afterwards" trace update_file.py databricks.yml "record_deployment_history: true" "record_deployment_history: false" trace $CLI bundle destroy --auto-approve trace update_file.py databricks.yml "record_deployment_history: false" "record_deployment_history: true" trace $CLI bundle deploy -trace print_requests.py //api/2.0/bundle --sort --oneline +trace print_requests.py //api/2.0/bundle --oneline diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index 86009c71b94..61e520c20a2 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -5,7 +5,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/d Deploying resources... Deployment complete! ->>> print_requests.py //api/2.0/bundle --sort --get +>>> print_requests.py //api/2.0/bundle --get { "method": "POST", "path": "/api/2.0/bundle/deployments", @@ -46,14 +46,14 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/d Deploying resources... Deployment complete! ->>> print_requests.py //api/2.0/bundle --sort --get +>>> print_requests.py //api/2.0/bundle --get { "method": "GET", - "path": "/api/2.0/bundle/deployments/[NUMID]" + "path": "/api/2.0/bundle/deployments/[NUMID]/resources" } { "method": "GET", - "path": "/api/2.0/bundle/deployments/[NUMID]/resources" + "path": "/api/2.0/bundle/deployments/[NUMID]" } { "method": "POST", diff --git a/acceptance/bundle/dms/no-resources/script b/acceptance/bundle/dms/no-resources/script index 9b14355bd27..f1b9ad5fa60 100644 --- a/acceptance/bundle/dms/no-resources/script +++ b/acceptance/bundle/dms/no-resources/script @@ -1,8 +1,8 @@ title "First deploy of a bundle with no resources: the deployment is created, and its workspace node identifies it even though no resource state was written" trace $CLI bundle deploy -trace print_requests.py //api/2.0/bundle --sort --get +trace print_requests.py //api/2.0/bundle --get trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-resources/default/state/resources.deployment.json" | jq '{object_type,path}' title "Redeploy: the deployment is resolved from that node, so no second deployment is created" trace $CLI bundle deploy -trace print_requests.py //api/2.0/bundle --sort --get +trace print_requests.py //api/2.0/bundle --get diff --git a/acceptance/bundle/dms/operation-upload-fails/databricks.yml b/acceptance/bundle/dms/operation-upload-fails/databricks.yml new file mode 100644 index 00000000000..1edbd7add88 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails/databricks.yml @@ -0,0 +1,24 @@ +bundle: + name: dms-operation-upload-fails + +experimental: + record_deployment_history: true + +resources: + jobs: + one: + name: one + two: + name: two + three: + name: three + four: + name: four + five: + name: five + six: + name: six + seven: + name: seven + eight: + name: eight diff --git a/acceptance/bundle/dms/operation-upload-fails/out.test.toml b/acceptance/bundle/dms/operation-upload-fails/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/operation-upload-fails/output.txt b/acceptance/bundle/dms/operation-upload-fails/output.txt new file mode 100644 index 00000000000..2b273972417 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails/output.txt @@ -0,0 +1,4 @@ + +=== An operation upload failure fails the deploy instead of reporting only at the end +>>> grep -c ^Error: LOG.deploy +deploy reported errors diff --git a/acceptance/bundle/dms/operation-upload-fails/script b/acceptance/bundle/dms/operation-upload-fails/script new file mode 100644 index 00000000000..26c748d9e79 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails/script @@ -0,0 +1,6 @@ +title "An operation upload failure fails the deploy instead of reporting only at the end" +# Which resources get refused depends on how far apply got before a background +# upload failed, so the per-resource errors go to a LOG file rather than the diff. +errcode $CLI bundle deploy &> LOG.deploy +contains.py 'recording operation for' '!panic' < LOG.deploy > /dev/null +trace grep -c "^Error:" LOG.deploy > /dev/null && echo "deploy reported errors" diff --git a/acceptance/bundle/dms/operation-upload-fails/test.toml b/acceptance/bundle/dms/operation-upload-fails/test.toml new file mode 100644 index 00000000000..ce222ac9e87 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails/test.toml @@ -0,0 +1,12 @@ +# Which requests are made depends on how far apply got before a background upload +# failed, so recording them would make the output nondeterministic. +RecordRequests = false + +# The service rejects every recorded operation. Deploy must stop rather than +# create every remaining resource: a completed version makes DMS the source of +# truth for resource state, so resources it has no record of would be created a +# second time by the next deploy. +[[Server]] +Pattern = "POST /api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/operations" +Response.StatusCode = 500 +Response.Body = '''{"error_code": "INTERNAL_ERROR", "message": "Internal error"}''' diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index 33cf719ddfd..dcb6b3efa22 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -6,7 +6,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //api/2.0/bundle --sort +>>> print_requests.py //api/2.0/bundle { "method": "POST", "path": "/api/2.0/bundle/deployments", @@ -27,13 +27,6 @@ Deployment complete! "version_type": "VERSION_TYPE_DEPLOY" } } -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", - "body": { - "completion_reason": "VERSION_COMPLETE_SUCCESS" - } -} { "method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", @@ -62,6 +55,13 @@ Deployment complete! "status": "OPERATION_STATUS_SUCCEEDED" } } +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} === The deployment ID is the ID of the workspace node the service registered under initial_parent_path; the CLI stores nothing locally >>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/resources.deployment.json @@ -80,7 +80,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //api/2.0/bundle --sort +>>> print_requests.py //api/2.0/bundle { "method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", @@ -111,11 +111,7 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests.py //api/2.0/bundle --sort -{ - "method": "DELETE", - "path": "/api/2.0/bundle/deployments/[NUMID]" -} +>>> print_requests.py //api/2.0/bundle { "method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", @@ -128,13 +124,6 @@ Destroy complete! "version_type": "VERSION_TYPE_DESTROY" } } -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/complete", - "body": { - "completion_reason": "VERSION_COMPLETE_SUCCESS" - } -} { "method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/operations", @@ -147,3 +136,14 @@ Destroy complete! "status": "OPERATION_STATUS_SUCCEEDED" } } +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} +{ + "method": "DELETE", + "path": "/api/2.0/bundle/deployments/[NUMID]" +} diff --git a/acceptance/bundle/dms/record/script b/acceptance/bundle/dms/record/script index 32ee3ec972f..895a9033a39 100644 --- a/acceptance/bundle/dms/record/script +++ b/acceptance/bundle/dms/record/script @@ -1,6 +1,6 @@ title "Deploy: the server assigns the deployment ID, and a version + create operation are recorded" trace $CLI bundle deploy -trace print_requests.py //api/2.0/bundle --sort +trace print_requests.py //api/2.0/bundle title "The deployment ID is the ID of the workspace node the service registered under initial_parent_path; the CLI stores nothing locally" # MSYS_NO_PATHCONV stops Git Bash on Windows from rewriting the leading-'/' path @@ -13,8 +13,8 @@ trace jq 'has("deployment_id")' .databricks/bundle/default/resources.json title "Redeploy after deleting the local cache: the deployment ID is resolved from that node, the same deployment is reused, and the version increments (no new CreateDeployment)" rm -rf .databricks trace $CLI bundle deploy -trace print_requests.py //api/2.0/bundle --sort +trace print_requests.py //api/2.0/bundle title "Destroy: a destroy version and delete operation are recorded, then the deployment is deleted" trace $CLI bundle destroy --auto-approve -trace print_requests.py //api/2.0/bundle --sort +trace print_requests.py //api/2.0/bundle diff --git a/acceptance/bundle/dms/redeploy-after-destroy/output.txt b/acceptance/bundle/dms/redeploy-after-destroy/output.txt index b8b6f3c630d..783082c1f83 100644 --- a/acceptance/bundle/dms/redeploy-after-destroy/output.txt +++ b/acceptance/bundle/dms/redeploy-after-destroy/output.txt @@ -31,7 +31,7 @@ Deployment complete! "path": "/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" } ->>> print_requests.py //api/2.0/bundle --sort --get +>>> print_requests.py //api/2.0/bundle --get { "method": "POST", "path": "/api/2.0/bundle/deployments", @@ -52,13 +52,6 @@ Deployment complete! "version_type": "VERSION_TYPE_DEPLOY" } } -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", - "body": { - "completion_reason": "VERSION_COMPLETE_SUCCESS" - } -} { "method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", @@ -87,3 +80,10 @@ Deployment complete! "status": "OPERATION_STATUS_SUCCEEDED" } } +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} diff --git a/acceptance/bundle/dms/redeploy-after-destroy/script b/acceptance/bundle/dms/redeploy-after-destroy/script index a39edf3c0d8..8dabf189f49 100644 --- a/acceptance/bundle/dms/redeploy-after-destroy/script +++ b/acceptance/bundle/dms/redeploy-after-destroy/script @@ -1,11 +1,11 @@ title "Deploy, then destroy: the deployment record and its workspace node are both deleted, so nothing points at the destroyed deployment" trace $CLI bundle deploy trace $CLI bundle destroy --auto-approve -print_requests.py //api/2.0/bundle --sort --get > /dev/null +print_requests.py //api/2.0/bundle --get > /dev/null trace MSYS_NO_PATHCONV=1 musterr $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" title "Deploy again: with no node to resolve, a fresh deployment is created at version 1, with its own workspace node" trace $CLI bundle deploy trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" | jq '{object_type,path}' -trace print_requests.py //api/2.0/bundle --sort --get +trace print_requests.py //api/2.0/bundle --get diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index f29aa18a186..46d70c7b135 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -69,6 +69,17 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return false } + // Stop before touching the workspace once recording an operation has failed. + // A completed version makes DMS the source of truth for resource state (see + // dstate.readDMSState), so continuing would create resources it has no record + // of and the next deploy would create them a second time. Checked here rather + // than only where operations are recorded, which is after the resource has + // already been modified. + if err := opQueue.firstErr(); err != nil { + logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) + return false + } + adapter, err := b.getAdapterForKey(resourceKey) if adapter == nil { logdiag.LogError(ctx, fmt.Errorf("%s: internal error: cannot get adapter: %w", errorPrefix, err)) diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go index ba189c13d2e..48d2887ec7c 100644 --- a/bundle/direct/opqueue.go +++ b/bundle/direct/opqueue.go @@ -226,8 +226,12 @@ func (q *operationQueue) setErr(err error) { } // firstErr returns the first upload error, or nil if every upload so far -// succeeded. +// succeeded. A nil queue (recording disabled) never errors. func (q *operationQueue) firstErr() error { + if q == nil { + return nil + } + q.mu.Lock() defer q.mu.Unlock() From 9329441d2f4391271c886fc5305addd76c9680a2 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 29 Jul 2026 13:37:48 +0000 Subject: [PATCH 26/28] testserver: create the DMS deployment record on the first version CreateDeployment now only registers the workspace node whose ID names the deployment; the record itself is created by the first CreateVersion. A client that registers a deployment and then fails before recording a version leaves just the node behind, not an empty deployment. That state is reachable, so both sides of the CLI handle it: - the recorder starts at version 1 under the ID the node already names, instead of failing on the missing record or creating a second deployment that would collide on the same node path - the read path keeps the local (empty) state instead of surfacing the 404 from ListResources acceptance/bundle/dms/version-never-created covers it end to end: the first version fails, and the next deploy reuses the same deployment ID. Also drops libs/testserver/bundle_test.go. The fake is exercised by every dms acceptance test, so unit tests for it only duplicate that coverage. Co-authored-by: Isaac --- .../dms/version-never-created/databricks.yml | 10 ++++ .../dms/version-never-created/out.test.toml | 3 ++ .../dms/version-never-created/output.txt | 34 +++++++++++++ .../bundle/dms/version-never-created/script | 7 +++ .../dms/version-never-created/test.toml | 7 +++ bundle/direct/dstate/dms.go | 12 +++++ libs/dms/recorder.go | 26 ++++++---- libs/dms/recorder_test.go | 46 ++++++++++------- libs/testserver/bundle.go | 37 +++++++++----- libs/testserver/bundle_test.go | 51 ------------------- libs/testserver/fake_workspace.go | 7 +++ 11 files changed, 147 insertions(+), 93 deletions(-) create mode 100644 acceptance/bundle/dms/version-never-created/databricks.yml create mode 100644 acceptance/bundle/dms/version-never-created/out.test.toml create mode 100644 acceptance/bundle/dms/version-never-created/output.txt create mode 100644 acceptance/bundle/dms/version-never-created/script create mode 100644 acceptance/bundle/dms/version-never-created/test.toml delete mode 100644 libs/testserver/bundle_test.go diff --git a/acceptance/bundle/dms/version-never-created/databricks.yml b/acceptance/bundle/dms/version-never-created/databricks.yml new file mode 100644 index 00000000000..a7077f18b1f --- /dev/null +++ b/acceptance/bundle/dms/version-never-created/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: dms-version-never-created + +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/version-never-created/out.test.toml b/acceptance/bundle/dms/version-never-created/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/dms/version-never-created/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/version-never-created/output.txt b/acceptance/bundle/dms/version-never-created/output.txt new file mode 100644 index 00000000000..f6fae9e6f56 --- /dev/null +++ b/acceptance/bundle/dms/version-never-created/output.txt @@ -0,0 +1,34 @@ + +=== The first version fails, so no deployment record exists - only the node naming its ID +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files... +Error: failed to create deployment version: Internal error (500 INTERNAL_ERROR) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/bundle/deployments/[NUMID]/versions?version_id=1 +HTTP Status: 500 Internal Server Error +API error_code: INTERNAL_ERROR +API message: Internal error + + +>>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/state/resources.deployment.json +{ + "object_type": "FILE" +} + +=== The next deploy reuses the ID that node names and retries version 1, rather than creating a second deployment +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files... +Error: failed to create deployment version: Internal error (500 INTERNAL_ERROR) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/bundle/deployments/[NUMID]/versions?version_id=1 +HTTP Status: 500 Internal Server Error +API error_code: INTERNAL_ERROR +API message: Internal error + + +>>> print_requests.py //api/2.0/bundle --get --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/state", "target_name": "default"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources"} +{"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]"} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}} diff --git a/acceptance/bundle/dms/version-never-created/script b/acceptance/bundle/dms/version-never-created/script new file mode 100644 index 00000000000..39566343e98 --- /dev/null +++ b/acceptance/bundle/dms/version-never-created/script @@ -0,0 +1,7 @@ +title "The first version fails, so no deployment record exists - only the node naming its ID" +trace musterr $CLI bundle deploy +trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-version-never-created/default/state/resources.deployment.json" | jq '{object_type}' + +title "The next deploy reuses the ID that node names and retries version 1, rather than creating a second deployment" +trace musterr $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --get --oneline diff --git a/acceptance/bundle/dms/version-never-created/test.toml b/acceptance/bundle/dms/version-never-created/test.toml new file mode 100644 index 00000000000..f0a8407e21d --- /dev/null +++ b/acceptance/bundle/dms/version-never-created/test.toml @@ -0,0 +1,7 @@ +# The first version fails, so the deployment record is never created - only the +# workspace node CreateDeployment registered. The next deploy resolves the ID from +# that node and has to cope with a deployment that has no record yet. +[[Server]] +Pattern = "POST /api/2.0/bundle/deployments/{deployment_id}/versions" +Response.StatusCode = 500 +Response.Body = '''{"error_code": "INTERNAL_ERROR", "message": "Internal error"}''' diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index ec51e2c4479..094f114a0a6 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -3,9 +3,12 @@ package dstate import ( "context" "encoding/json" + "errors" "fmt" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) @@ -33,6 +36,15 @@ type RecordedState struct { func (db *DeploymentState) readDMSState(ctx context.Context, src *DMSSource) error { resources, err := fetchDeploymentResources(ctx, src.Client, src.DeploymentID) if err != nil { + // The deployment's record is created by its first version, so a node can + // resolve to an ID that has none yet: a deploy that registered the deployment + // and then failed before recording a version. There is nothing to read, and + // the file's resources are still empty, so carry on and let this deploy record + // the first version. + if errors.Is(err, apierr.ErrNotFound) || errors.Is(err, apierr.ErrResourceDoesNotExist) { + log.Debugf(ctx, "No deployment record for %s yet; keeping local state", src.DeploymentID) + return nil + } return err } diff --git a/libs/dms/recorder.go b/libs/dms/recorder.go index e285a80e4a2..6973a09b4b5 100644 --- a/libs/dms/recorder.go +++ b/libs/dms/recorder.go @@ -150,22 +150,26 @@ func (r *Recorder) CompleteVersion(ctx context.Context, success bool) error { // compute the next version number. func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID string, err error) { if r.deploymentID != "" { - // Existing deployment: read it to compute the next version number. A 404 is - // not recovered from by creating a second deployment. The service trashes the - // workspace node when it deletes the record, so a node that resolved but has - // no record means the two are out of sync, and creating another deployment - // would collide on the same node path. + // A resolved node names the deployment, but its record is created by the + // first version, so there may be none yet: a deploy that registered the + // deployment and then failed before recording a version. Start at version 1 + // under the ID the node already names, rather than creating a second + // deployment, which would collide on the same node path. dep, getErr := r.svc.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ Name: "deployments/" + r.deploymentID, }) - if getErr != nil { + switch { + case getErr == nil: + lastVersion, parseErr := strconv.ParseInt(dep.LastVersionId, 10, 64) + if parseErr != nil { + return "", fmt.Errorf("failed to parse last_version_id %q: %w", dep.LastVersionId, parseErr) + } + versionID = strconv.FormatInt(lastVersion+1, 10) + case errors.Is(getErr, apierr.ErrNotFound), errors.Is(getErr, apierr.ErrResourceDoesNotExist): + versionID = "1" + default: return "", fmt.Errorf("failed to get deployment: %w", getErr) } - lastVersion, parseErr := strconv.ParseInt(dep.LastVersionId, 10, 64) - if parseErr != nil { - return "", fmt.Errorf("failed to parse last_version_id %q: %w", dep.LastVersionId, parseErr) - } - versionID = strconv.FormatInt(lastVersion+1, 10) } else { // First deploy: create the deployment so the server assigns an ID. // diff --git a/libs/dms/recorder_test.go b/libs/dms/recorder_test.go index 6c2f334c946..d4c7efaca03 100644 --- a/libs/dms/recorder_test.go +++ b/libs/dms/recorder_test.go @@ -112,27 +112,35 @@ func TestRecorderSubsequentDeployReusesDeploymentAndIncrementsVersion(t *testing } func TestRecorderGetDeploymentErrorFailsDeploy(t *testing.T) { - cases := map[string]error{ - // A resolved ID whose record is missing means the record and the workspace - // node it was resolved from are out of sync. Creating a second deployment - // would collide on the same node path, so fail instead. - "not found": fmt.Errorf("deployment: %w", apierr.ErrNotFound), - "other": errors.New("boom"), + f := &fakeDMS{ + getDeployment: func(id string) (*bundledeployments.Deployment, error) { + return nil, errors.New("boom") + }, } - for name, getErr := range cases { - t.Run(name, func(t *testing.T) { - f := &fakeDMS{ - getDeployment: func(id string) (*bundledeployments.Deployment, error) { - return nil, getErr - }, - } - r := NewRecorder(f, "stored-id", testStatePath, "dev", VersionTypeDeploy) - - err := r.CreateVersion(t.Context()) - assert.ErrorContains(t, err, "failed to get deployment") - assert.Empty(t, f.created) - }) + r := NewRecorder(f, "stored-id", testStatePath, "dev", VersionTypeDeploy) + + err := r.CreateVersion(t.Context()) + assert.ErrorContains(t, err, "failed to get deployment") + assert.Empty(t, f.created) +} + +func TestRecorderMissingDeploymentRecordStartsAtVersionOne(t *testing.T) { + // The record is created by the first version, so a node can name a deployment + // that has none yet - an earlier deploy registered it and then failed. Record + // version 1 under that same ID instead of creating a second deployment, which + // would collide on the node path. + f := &fakeDMS{ + getDeployment: func(id string) (*bundledeployments.Deployment, error) { + return nil, fmt.Errorf("deployment: %w", apierr.ErrNotFound) + }, } + r := NewRecorder(f, "stored-id", testStatePath, "dev", VersionTypeDeploy) + + require.NoError(t, r.CreateVersion(t.Context())) + assert.Empty(t, f.created) + require.Len(t, f.versions, 1) + assert.Equal(t, "1", f.versions[0].VersionId) + assert.Equal(t, "deployments/stored-id", f.versions[0].Parent) } func TestRecorderDestroyDeletesDeploymentOnSuccess(t *testing.T) { diff --git a/libs/testserver/bundle.go b/libs/testserver/bundle.go index 6dffbe34a75..4e04c65ba81 100644 --- a/libs/testserver/bundle.go +++ b/libs/testserver/bundle.go @@ -35,9 +35,6 @@ type dmsDeployment struct { // value as "DMS owns the state". Tracked separately because the SDK // Deployment struct does not yet carry the field (still stage:DEVELOPMENT). lastSuccessfulVersionID string - // nodePath is the workspace node whose object ID is this deployment's ID. - // Kept so DeleteDeployment can trash the node, the way the service does. - nodePath string } func (s *FakeWorkspace) CreateDeployment(req Request) Response { @@ -70,15 +67,15 @@ func (s *FakeWorkspace) CreateDeployment(req Request) Response { }, } + // Only the node is created here. The deployment record itself is created by the + // first CreateVersion, so a client that creates a deployment and then fails + // before recording a version leaves no record behind - just the node, which + // names the ID that first version will be created under. deploymentID := strconv.FormatInt(objectID, 10) + s.dmsDeploymentNodes[deploymentID] = nodePath + dep.Name = "deployments/" + deploymentID dep.Status = bundledeployments.DeploymentStatusDeploymentStatusActive - s.dmsDeployments[deploymentID] = &dmsDeployment{ - deployment: dep, - versions: map[string]*bundledeployments.Version{}, - resources: map[string]bundledeployments.Resource{}, - nodePath: nodePath, - } return Response{Body: dep} } @@ -129,9 +126,10 @@ func (s *FakeWorkspace) DeleteDeployment(deploymentID string) Response { // The service trashes the deployment's workspace node, so a later get-status // on the node path reports the deployment as absent. - if d, ok := s.dmsDeployments[deploymentID]; ok { - delete(s.files, d.nodePath) + if nodePath, ok := s.dmsDeploymentNodes[deploymentID]; ok { + delete(s.files, nodePath) } + delete(s.dmsDeploymentNodes, deploymentID) delete(s.dmsDeployments, deploymentID) return Response{Body: map[string]any{}} } @@ -148,7 +146,22 @@ func (s *FakeWorkspace) CreateVersion(req Request, deploymentID string) Response d, ok := s.dmsDeployments[deploymentID] if !ok { - return dmsNotFound("deployment " + deploymentID) + // The deployment record is created by its first version, not by + // CreateDeployment. That call only registered the workspace node, so the node + // existing is what makes this ID valid. + if _, known := s.dmsDeploymentNodes[deploymentID]; !known { + return dmsNotFound("deployment " + deploymentID) + } + d = &dmsDeployment{ + deployment: bundledeployments.Deployment{ + Name: "deployments/" + deploymentID, + Status: bundledeployments.DeploymentStatusDeploymentStatusActive, + TargetName: version.TargetName, + }, + versions: map[string]*bundledeployments.Version{}, + resources: map[string]bundledeployments.Resource{}, + } + s.dmsDeployments[deploymentID] = d } // Mirror the server-side optimistic concurrency check: the new version must diff --git a/libs/testserver/bundle_test.go b/libs/testserver/bundle_test.go deleted file mode 100644 index 4d28624ba3e..00000000000 --- a/libs/testserver/bundle_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package testserver - -import ( - "encoding/json" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// TestDeploymentBodyKeepsTypedFieldsAndLastSuccessfulVersionID guards against -// serializing the deployment through a struct that embeds -// bundledeployments.Deployment: Deployment has its own MarshalJSON, which is -// promoted to the embedding struct and silently drops last_successful_version_id. -// The CLI read path treats a missing value as "DMS does not own the state", so -// losing the field here makes the whole overlay path untestable. -func TestDeploymentBodyKeepsTypedFieldsAndLastSuccessfulVersionID(t *testing.T) { - d := &dmsDeployment{lastSuccessfulVersionID: "2"} - d.deployment.Name = "deployments/abc" - d.deployment.LastVersionId = "3" - d.deployment.TargetName = "default" - - body, err := deploymentBody(d) - require.NoError(t, err) - - assert.Equal(t, "deployments/abc", body["name"]) - assert.Equal(t, "3", body["last_version_id"]) - assert.Equal(t, "default", body["target_name"]) - assert.Equal(t, "2", body["last_successful_version_id"]) - - // The response must round-trip as JSON the same way, since that is what the - // client actually reads. - raw, err := json.Marshal(body) - require.NoError(t, err) - assert.JSONEq(t, - `{"name":"deployments/abc","last_version_id":"3","target_name":"default","last_successful_version_id":"2"}`, - string(raw)) -} - -// TestDeploymentBodyOmitsUnsetLastSuccessfulVersionID checks that a deployment -// with no successful version does not advertise one: the read path must keep -// using the local state file in that case. -func TestDeploymentBodyOmitsUnsetLastSuccessfulVersionID(t *testing.T) { - d := &dmsDeployment{} - d.deployment.Name = "deployments/abc" - - body, err := deploymentBody(d) - require.NoError(t, err) - - assert.NotContains(t, body, "last_successful_version_id") -} diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index a3c4519ccc4..13d9f76e00a 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -231,6 +231,12 @@ type FakeWorkspace struct { // dmsDeployments holds Deployment Metadata Service (DMS) records, keyed by // deployment ID. Each record carries its versions and latest resource state. dmsDeployments map[string]*dmsDeployment + + // dmsDeploymentNodes maps deployment ID to the workspace node CreateDeployment + // registered for it. A deployment appears here before it has a record in + // dmsDeployments: the record is created by its first version, so the node is + // what makes an ID valid in between. + dmsDeploymentNodes map[string]string } func (s *FakeWorkspace) LockUnlock() func() { @@ -383,6 +389,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { postgresImplicitEndpoints: map[string]bool{}, clusterVenvs: map[string]*clusterEnv{}, dmsDeployments: map[string]*dmsDeployment{}, + dmsDeploymentNodes: map[string]string{}, Alerts: map[string]sql.AlertV2{}, Experiments: map[string]ml.GetExperimentResponse{}, ModelRegistryModels: map[string]ml.Model{}, From 38b6f54081ae33008edd866a64ffd696c6dc90af Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 29 Jul 2026 14:05:39 +0000 Subject: [PATCH 27/28] bundle: simplify the concurrency test comment Co-authored-by: Isaac --- bundle/direct/opqueue_test.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/bundle/direct/opqueue_test.go b/bundle/direct/opqueue_test.go index a68560493ce..910b58d470f 100644 --- a/bundle/direct/opqueue_test.go +++ b/bundle/direct/opqueue_test.go @@ -307,13 +307,16 @@ func (s *serialUploader) upload(ctx context.Context, resourceKey string, op reco } func TestOperationQueueUploadsOneResourceAtATime(t *testing.T) { - // Concurrent apply workers repeatedly record overlapping resource keys, the - // case where a coalesced key can be handed to a second worker while the first - // is still uploading it. The service keeps one state per key, so overlapping - // uploads for a key could land out of order and leave a stale state behind. + // Two workers must never upload the same resource at the same time. DMS stores + // one state per resource, so concurrent uploads can finish out of order and + // leave the older state as the final one. // - // The interleaving that breaks this is scheduler-dependent, so one pass proves - // little: repeat it so a single run has many chances to hit the bad ordering. + // Lots of goroutines record a small set of keys, so the same key is recorded + // repeatedly while its earlier upload may still be running. serialUploader flags + // any overlap it sees. + // + // Whether a bug shows up depends on how the scheduler interleaves things, so one + // pass proves little - repeat it to get many chances at a bad ordering. const ( iterations = 200 workers = 10 From 4f45539f06f8071c8441433dbe69f2ba9b7669da Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 29 Jul 2026 15:16:42 +0000 Subject: [PATCH 28/28] bundle: trim the operation recorder tests Drops TestOperationRecorderDeleteHasNoState: the dms acceptance goldens already show a delete operation recorded without a state field. Renames the redaction test to say what it checks - the state is recorded as-is - rather than contrasting it with dstate.SaveState. Co-authored-by: Isaac --- bundle/direct/oprecorder_test.go | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go index 1b68c3a626b..674c78abf77 100644 --- a/bundle/direct/oprecorder_test.go +++ b/bundle/direct/oprecorder_test.go @@ -52,19 +52,7 @@ func TestOperationRecorderStripsResourcePrefix(t *testing.T) { require.NotNil(t, req.Operation.State) } -func TestOperationRecorderDeleteHasNoState(t *testing.T) { - f := &fakeOpClient{} - r := NewOperationRecorder(f, "dep-1", 3) - - uploadOne(t, r, "resources.jobs.foo", deployplan.Delete, "", nil) - - require.Len(t, f.requests, 1) - assert.Equal(t, bundledeployments.OperationActionTypeOperationActionTypeDelete, f.requests[0].Operation.ActionType) - // Delete operations carry no serialized state. - assert.Nil(t, f.requests[0].Operation.State) -} - -func TestNewRecordedOperationDoesNotRedactSensitiveFields(t *testing.T) { +func TestNewRecordedOperationRecordsStateAsIs(t *testing.T) { state := struct { Name string `json:"name"` Token string `json:"token" bundle:"sensitive"` @@ -73,8 +61,7 @@ func TestNewRecordedOperationDoesNotRedactSensitiveFields(t *testing.T) { op, err := newRecordedOperation(deployplan.Create, "job-123", state, nil) require.NoError(t, err) - // Recorded as-is, unlike dstate.SaveState which redacts before writing the - // local state file. + // The state is serialized as-is, including fields tagged bundle:"sensitive". assert.JSONEq(t, `{"state":{"name":"foo","token":"super-secret"}}`, string(op.state))