postgres: Support purge_on_delete on postgres_branches

NEXT_CHANGELOG.md

@@ -18,6 +18,8 @@
  * `bundle generate dashboard` now honors the `--key` flag when naming the generated resource, and rejects combining `--existing-path`, `--existing-id`, and `--resource` instead of silently ignoring all but one ([#5492](https://github.com/databricks/cli/pull/5492)).
  * Fixed `bundle deployment migrate` failing on `model_serving_endpoints`/`database_instances` with permissions (regression since v1.5.0) ([#5775](https://github.com/databricks/cli/pull/5775)).
  * After a terraform deploy, the CLI now dry-runs a migration to the direct engine (writing nothing locally or remotely) and reports the outcome via telemetry, warning if the migration could not be completed ([#5797](https://github.com/databricks/cli/pull/5797)).
+ * direct: Fix deploy bug when a `postgres_projects`, `postgres_branches`, or `postgres_endpoints` field is set to its zero value (e.g. `enable_pg_native_login: false`, `replace_existing: false`).
+ * Support `purge_on_delete: true` on `postgres_branches` so bundles can hard-delete a Lakebase branch on destroy (skipping the soft-delete retention window) ([#5801](https://github.com/databricks/cli/pull/5801)).
 
 ### Dependency updates
  * Bump `github.com/databricks/databricks-sdk-go` from v0.147.0 to v0.152.0 ([#5773](https://github.com/databricks/cli/pull/5773)).

acceptance/bundle/refschema/out.fields.txt

@@ -2892,6 +2892,7 @@ resources.postgres_branches.*.modified_status	string	INPUT
 resources.postgres_branches.*.name	string	REMOTE
 resources.postgres_branches.*.no_expiry	bool	ALL
 resources.postgres_branches.*.parent	string	ALL
+resources.postgres_branches.*.purge_on_delete	bool	INPUT	STATE
 resources.postgres_branches.*.replace_existing	bool	INPUT	STATE
 resources.postgres_branches.*.source_branch	string	ALL
 resources.postgres_branches.*.source_branch_lsn	string	ALL

acceptance/bundle/resources/postgres_branches/purge_on_delete/databricks.yml.tmpl

@@ -0,0 +1,28 @@
+bundle:
+  name: deploy-postgres-branch-purge-$UNIQUE_NAME
+
+sync:
+  paths: []
+
+# A project with two branches side by side so the recorded destroy requests
+# make the purge_on_delete contrast obvious:
+#   - hard_delete:  purge_on_delete: true   -> DELETE …?purge=true
+#   - soft_delete:  field omitted (default) -> DELETE …    (no purge query)
+resources:
+  postgres_projects:
+    my_project:
+      project_id: test-pg-proj-$UNIQUE_NAME
+      display_name: "Test Project for Branch purge_on_delete"
+      pg_version: 16
+
+  postgres_branches:
+    hard_delete:
+      parent: ${resources.postgres_projects.my_project.id}
+      branch_id: test-pg-branch-hard-$UNIQUE_NAME
+      no_expiry: true
+      purge_on_delete: true
+
+    soft_delete:
+      parent: ${resources.postgres_projects.my_project.id}
+      branch_id: test-pg-branch-soft-$UNIQUE_NAME
+      no_expiry: true

acceptance/bundle/resources/postgres_branches/purge_on_delete/out.destroy.txt

@@ -0,0 +1,18 @@
+The following resources will be deleted:
+  delete resources.postgres_branches.hard_delete
+  delete resources.postgres_branches.soft_delete
+  delete resources.postgres_projects.my_project
+
+This action will result in the deletion of the following Lakebase projects along with
+all their branches, databases, and endpoints. All data stored in them will be permanently lost:
+  delete resources.postgres_projects.my_project
+
+This action will result in the deletion of the following Lakebase branches.
+All data stored in them will be permanently lost:
+  delete resources.postgres_branches.hard_delete
+  delete resources.postgres_branches.soft_delete
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-branch-purge-[UNIQUE_NAME]/default
+
+Deleting files...
+Destroy complete!

acceptance/bundle/resources/postgres_branches/purge_on_delete/out.requests.deploy.direct.json

@@ -0,0 +1,45 @@
+{
+  "method": "GET",
+  "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/test-pg-branch-hard-[UNIQUE_NAME]"
+}
+{
+  "method": "GET",
+  "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/test-pg-branch-soft-[UNIQUE_NAME]"
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/postgres/projects",
+  "q": {
+    "project_id": "test-pg-proj-[UNIQUE_NAME]"
+  },
+  "body": {
+    "spec": {
+      "display_name": "Test Project for Branch purge_on_delete",
+      "pg_version": 16
+    }
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches",
+  "q": {
+    "branch_id": "test-pg-branch-hard-[UNIQUE_NAME]"
+  },
+  "body": {
+    "spec": {
+      "no_expiry": true
+    }
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches",
+  "q": {
+    "branch_id": "test-pg-branch-soft-[UNIQUE_NAME]"
+  },
+  "body": {
+    "spec": {
+      "no_expiry": true
+    }
+  }
+}

acceptance/bundle/resources/postgres_branches/purge_on_delete/out.requests.deploy.terraform.json

@@ -0,0 +1,47 @@
+{
+  "method": "GET",
+  "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/test-pg-branch-hard-[UNIQUE_NAME]"
+}
+{
+  "method": "GET",
+  "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/test-pg-branch-soft-[UNIQUE_NAME]"
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/postgres/projects",
+  "q": {
+    "project_id": "test-pg-proj-[UNIQUE_NAME]"
+  },
+  "body": {
+    "spec": {
+      "display_name": "Test Project for Branch purge_on_delete",
+      "pg_version": 16
+    }
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches",
+  "q": {
+    "branch_id": "test-pg-branch-hard-[UNIQUE_NAME]"
+  },
+  "body": {
+    "parent": "projects/test-pg-proj-[UNIQUE_NAME]",
+    "spec": {
+      "no_expiry": true
+    }
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches",
+  "q": {
+    "branch_id": "test-pg-branch-soft-[UNIQUE_NAME]"
+  },
+  "body": {
+    "parent": "projects/test-pg-proj-[UNIQUE_NAME]",
+    "spec": {
+      "no_expiry": true
+    }
+  }
+}

acceptance/bundle/resources/postgres_branches/purge_on_delete/out.requests.destroy.json

@@ -0,0 +1,27 @@
+{
+  "method": "DELETE",
+  "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]"
+}
+{
+  "method": "DELETE",
+  "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/test-pg-branch-hard-[UNIQUE_NAME]",
+  "q": {
+    "purge": "true"
+  }
+}
+{
+  "method": "DELETE",
+  "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/test-pg-branch-soft-[UNIQUE_NAME]"
+}
+{
+  "method": "GET",
+  "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]"
+}
+{
+  "method": "GET",
+  "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/test-pg-branch-hard-[UNIQUE_NAME]"
+}
+{
+  "method": "GET",
+  "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/test-pg-branch-soft-[UNIQUE_NAME]"
+}

acceptance/bundle/resources/postgres_branches/purge_on_delete/output.txt

@@ -0,0 +1,56 @@
+
+>>> [CLI] bundle validate
+Name: deploy-postgres-branch-purge-[UNIQUE_NAME]
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-branch-purge-[UNIQUE_NAME]/default
+
+Validation OK!
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-branch-purge-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> [CLI] postgres get-branch projects/test-pg-proj-[UNIQUE_NAME]/branches/test-pg-branch-hard-[UNIQUE_NAME]
+{
+  "branch_id": "test-pg-branch-hard-[UNIQUE_NAME]",
+  "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/test-pg-branch-hard-[UNIQUE_NAME]",
+  "parent": "projects/test-pg-proj-[UNIQUE_NAME]",
+  "status": {
+    "branch_id": "test-pg-branch-hard-[UNIQUE_NAME]",
+    "current_state": "READY",
+    "default": false,
+    "is_protected": false,
+    "source_branch": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production",
+    "source_branch_lsn": "[LSN]",
+    "source_branch_time": "[TIMESTAMP]",
+    "state_change_time": "[TIMESTAMP]"
+  },
+  "uid": "[BRANCH_UID]"
+}
+
+>>> [CLI] postgres get-branch projects/test-pg-proj-[UNIQUE_NAME]/branches/test-pg-branch-soft-[UNIQUE_NAME]
+{
+  "branch_id": "test-pg-branch-soft-[UNIQUE_NAME]",
+  "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/test-pg-branch-soft-[UNIQUE_NAME]",
+  "parent": "projects/test-pg-proj-[UNIQUE_NAME]",
+  "status": {
+    "branch_id": "test-pg-branch-soft-[UNIQUE_NAME]",
+    "current_state": "READY",
+    "default": false,
+    "is_protected": false,
+    "source_branch": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production",
+    "source_branch_lsn": "[LSN]",
+    "source_branch_time": "[TIMESTAMP]",
+    "state_change_time": "[TIMESTAMP]"
+  },
+  "uid": "[BRANCH_UID]"
+}
+
+>>> print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --sort --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/
+
+=== bundle destroy
+>>> print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --sort --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/

acceptance/bundle/resources/postgres_branches/purge_on_delete/script

@@ -0,0 +1,35 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+project_name="projects/test-pg-proj-${UNIQUE_NAME}"
+hard_branch="${project_name}/branches/test-pg-branch-hard-${UNIQUE_NAME}"
+soft_branch="${project_name}/branches/test-pg-branch-soft-${UNIQUE_NAME}"
+
+cleanup() {
+   # Belt-and-braces in case bundle destroy was skipped or partially failed.
+   # The soft-delete case leaves a record in the trash; --purge clears it.
+   $CLI postgres delete-branch --purge "${hard_branch}" 2>>LOG.delete-branch || true
+   $CLI postgres delete-branch --purge "${soft_branch}" 2>>LOG.delete-branch || true
+   $CLI postgres delete-project --purge "${project_name}" 2>>LOG.delete-project || true
+   rm -f out.requests.txt
+}
+trap cleanup EXIT
+
+trace $CLI bundle validate
+
+rm -f out.requests.txt
+trace $CLI bundle deploy
+
+trace $CLI postgres get-branch "${hard_branch}" | jq 'del(.create_time, .update_time, .status.logical_size_bytes)'
+trace $CLI postgres get-branch "${soft_branch}" | jq 'del(.create_time, .update_time, .status.logical_size_bytes)'
+
+# Deploy requests are split per-engine: Terraform sends "parent" in the branch
+# create body, the direct engine sends it only as a query parameter.
+trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --sort --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.deploy.$DATABRICKS_BUNDLE_ENGINE.json
+
+# bundle destroy should send ?purge=true on the hard_delete branch and no
+# purge query on the soft_delete branch. Both DELETEs land in the recorded
+# requests so the contrast is visible in the diff.
+title "bundle destroy"
+$CLI bundle destroy --auto-approve > out.destroy.txt 2>&1 || true
+
+trace print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --sort --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.destroy.json

acceptance/bundle/resources/postgres_branches/purge_on_delete_transitions/databricks.yml.tmpl

@@ -0,0 +1,23 @@
+bundle:
+  name: pg-branch-purge-transitions-$UNIQUE_NAME
+
+sync:
+  paths: []
+
+# Walks purge_on_delete through unset -> true -> false -> unset, deploying
+# at each step and inspecting the persisted state so reviewers can see that
+# state tracks the user's latest intent. The final destroy is a soft delete
+# (state has purge_on_delete unset), recorded for regression coverage.
+resources:
+  postgres_projects:
+    proj:
+      project_id: test-pg-proj-$UNIQUE_NAME
+      display_name: "Transitions test"
+      pg_version: 16
+
+  postgres_branches:
+    branch:
+      parent: ${resources.postgres_projects.proj.id}
+      branch_id: test-pg-branch-$UNIQUE_NAME
+      no_expiry: true
+      # PURGE_ON_DELETE

acceptance/bundle/resources/postgres_branches/purge_on_delete_transitions/out.destroy.txt

@@ -0,0 +1,16 @@
+The following resources will be deleted:
+  delete resources.postgres_branches.branch
+  delete resources.postgres_projects.proj
+
+This action will result in the deletion of the following Lakebase projects along with
+all their branches, databases, and endpoints. All data stored in them will be permanently lost:
+  delete resources.postgres_projects.proj
+
+This action will result in the deletion of the following Lakebase branches.
+All data stored in them will be permanently lost:
+  delete resources.postgres_branches.branch
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/pg-branch-purge-transitions-[UNIQUE_NAME]/default
+
+Deleting files...
+Destroy complete!

acceptance/bundle/resources/postgres_branches/purge_on_delete_transitions/out.requests.destroy.json

@@ -0,0 +1,16 @@
+{
+  "method": "DELETE",
+  "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]"
+}
+{
+  "method": "DELETE",
+  "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/test-pg-branch-[UNIQUE_NAME]"
+}
+{
+  "method": "GET",
+  "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]"
+}
+{
+  "method": "GET",
+  "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/test-pg-branch-[UNIQUE_NAME]"
+}

acceptance/bundle/resources/postgres_branches/purge_on_delete_transitions/output.txt

@@ -0,0 +1,43 @@
+
+=== Step 1: deploy with purge_on_delete unset
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pg-branch-purge-transitions-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> get_purge
+(unset)
+
+=== Step 2: set purge_on_delete: true
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pg-branch-purge-transitions-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> get_purge
+json.state.resources.postgres_branches.branch.state.purge_on_delete = true;
+
+=== Step 3: flip to purge_on_delete: false
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pg-branch-purge-transitions-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> get_purge
+json.state.resources.postgres_branches.branch.state.purge_on_delete = false;
+
+=== Step 4: remove the line again
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pg-branch-purge-transitions-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> get_purge
+json.state.resources.postgres_branches.branch.state.purge_on_delete = false;
+
+=== bundle destroy (branch DELETE must be a plain DELETE, no ?purge=true)
+>>> print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --sort --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/