direct: Retry vector search index create while name is pending deletion

.nextchanges/bundles/vector-search-index-pending-deletion.md

@@ -0,0 +1 @@
+direct: Recreating a `vector_search_indexes` resource no longer fails with "Index ... is currently pending deletion" when the backend has not yet released the index name. The create is now retried until the name becomes available.

acceptance/bundle/resources/vector_search_indexes/recreate/pending_deletion/databricks.yml.tmpl

@@ -0,0 +1,24 @@
+bundle:
+  name: deploy-vs-index-$UNIQUE_NAME
+
+sync:
+  paths: []
+
+resources:
+  vector_search_endpoints:
+    my_endpoint:
+      name: vs-endpoint-$UNIQUE_NAME
+      endpoint_type: STANDARD
+  vector_search_indexes:
+    my_index:
+      # The testserver simulates the two-phase backend deletion only for index
+      # names containing vs_index_pending_deletion (indexNamePendingDeletion).
+      name: main.default.vs_index_pending_deletion_$UNIQUE_NAME
+      endpoint_name: ${resources.vector_search_endpoints.my_endpoint.name}
+      primary_key: id
+      index_type: DIRECT_ACCESS
+      direct_access_index_spec:
+        schema_json: '{"id":"integer","vector":"array<float>"}'
+        embedding_vector_columns:
+          - name: vector
+            embedding_dimension: 768

acceptance/bundle/resources/vector_search_indexes/recreate/pending_deletion/output.txt

@@ -0,0 +1,92 @@
+
+=== Initial deployment
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-vs-index-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Change embedding_dimension (recreate: DELETE then CREATE the same name)
+>>> update_file.py databricks.yml embedding_dimension: 768 embedding_dimension: 384
+
+>>> [CLI] bundle deploy --auto-approve
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-vs-index-[UNIQUE_NAME]/default/files...
+
+This action will result in the deletion or recreation of the following Vector Search indexes.
+Recreating a Delta Sync index re-runs the full embedding pipeline; recreating a Direct Access
+index drops all upserted vectors. Both can be expensive to rebuild:
+  recreate resources.vector_search_indexes.my_index
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py --get //vector-search/indexes
+{
+  "method": "GET",
+  "path": "/api/2.0/vector-search/indexes/main.default.vs_index_pending_deletion_[UNIQUE_NAME]"
+}
+{
+  "method": "DELETE",
+  "path": "/api/2.0/vector-search/indexes/main.default.vs_index_pending_deletion_[UNIQUE_NAME]"
+}
+{
+  "method": "GET",
+  "path": "/api/2.0/vector-search/indexes/main.default.vs_index_pending_deletion_[UNIQUE_NAME]"
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/vector-search/indexes",
+  "body": {
+    "direct_access_index_spec": {
+      "embedding_vector_columns": [
+        {
+          "embedding_dimension": 384,
+          "name": "vector"
+        }
+      ],
+      "schema_json": "{\"id\":\"integer\",\"vector\":\"array<float>\"}"
+    },
+    "endpoint_name": "vs-endpoint-[UNIQUE_NAME]",
+    "index_type": "DIRECT_ACCESS",
+    "name": "main.default.vs_index_pending_deletion_[UNIQUE_NAME]",
+    "primary_key": "id"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/vector-search/indexes",
+  "body": {
+    "direct_access_index_spec": {
+      "embedding_vector_columns": [
+        {
+          "embedding_dimension": 384,
+          "name": "vector"
+        }
+      ],
+      "schema_json": "{\"id\":\"integer\",\"vector\":\"array<float>\"}"
+    },
+    "endpoint_name": "vs-endpoint-[UNIQUE_NAME]",
+    "index_type": "DIRECT_ACCESS",
+    "name": "main.default.vs_index_pending_deletion_[UNIQUE_NAME]",
+    "primary_key": "id"
+  }
+}
+{
+  "method": "GET",
+  "path": "/api/2.0/vector-search/indexes/main.default.vs_index_pending_deletion_[UNIQUE_NAME]"
+}
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.vector_search_endpoints.my_endpoint
+  delete resources.vector_search_indexes.my_index
+
+This action will result in the deletion of the following Vector Search indexes.
+For Delta Sync indexes, the source Delta Table is preserved but the embedding pipeline is removed.
+For Direct Access indexes, all upserted vectors are permanently lost:
+  delete resources.vector_search_indexes.my_index
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-vs-index-[UNIQUE_NAME]/default
+
+Deleting files...
+Destroy complete!

acceptance/bundle/resources/vector_search_indexes/recreate/pending_deletion/script

@@ -0,0 +1,23 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+    rm -f out.requests.txt
+}
+trap cleanup EXIT
+
+title "Initial deployment"
+trace $CLI bundle deploy
+
+title "Change embedding_dimension (recreate: DELETE then CREATE the same name)"
+trace update_file.py databricks.yml "embedding_dimension: 768" "embedding_dimension: 384"
+
+rm -f out.requests.txt
+trace $CLI bundle deploy --auto-approve
+
+# The name is released only after the index stops being visible on GET, so
+# WaitAfterDelete's poll returns before CREATE is allowed. The first POST here is
+# the rejected one; the second, identical POST is the assertion that createIndex
+# retries instead of failing the deploy — hence no --unique, which would collapse
+# the two into one. --get shows the WaitAfterDelete poll too.
+trace print_requests.py --get //vector-search/indexes

acceptance/bundle/resources/vector_search_indexes/recreate/pending_deletion/test.toml

@@ -0,0 +1,4 @@
+# The two-phase deletion race is simulated by the testserver, so this only runs locally.
+Cloud = false
+# The parent sets CloudSlow=true; disable it here too, otherwise CloudSlow would imply Cloud=true.
+CloudSlow = false

bundle/direct/dresources/vector_search_index.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/databricks/cli/bundle/config/resources"
@@ -21,6 +22,12 @@ import (
 // embedding pipeline shutdown can stretch closer to ten minutes.
 const deleteIndexTimeout = 15 * time.Minute
 
+// pendingDeletionTimeout caps how long a create is retried while the backend
+// still holds the index name from a preceding delete (see createIndex). Kept
+// deliberately short so we fail fast rather than hang for the full
+// deleteIndexTimeout: if nightlies still hit the race, raise it then.
+const pendingDeletionTimeout = time.Minute
+
 // createIndexTimeout caps the wait for an index to become ready after creation.
 // Delta sync indexes do an initial sync from the source table, which can stretch
 // out for large tables. Matches the terraform provider's defaultIndexProvisionTimeout.
@@ -122,7 +129,7 @@ func (r *ResourceVectorSearchIndex) DoRead(ctx context.Context, id string) (*Vec
 }
 
 func (r *ResourceVectorSearchIndex) DoCreate(ctx context.Context, config *VectorSearchIndexState) (string, *VectorSearchIndexRemote, error) {
-	index, err := r.client.VectorSearchIndexes.CreateIndex(ctx, config.CreateVectorIndexRequest)
+	index, err := r.createIndex(ctx, config.CreateVectorIndexRequest)
 	if err != nil {
 		return "", nil, err
 	}
@@ -138,6 +145,42 @@ func (r *ResourceVectorSearchIndex) DoCreate(ctx context.Context, config *Vector
 	return config.Name, &VectorSearchIndexRemote{VectorIndex: *index, EndpointUuid: endpointUuid}, nil
 }
 
+// createIndex calls CreateIndex, retrying while the backend still reports the
+// name as pending deletion.
+//
+// Deleting an index completes in two backend phases: the index first disappears
+// from GET, and only later is the name released for reuse. WaitAfterDelete polls
+// GET, so it can return as soon as phase one is done and a follow-up CREATE for
+// the same name is still rejected with 400 INVALID_PARAMETER_VALUE "currently
+// pending deletion". Polling GET cannot close that race because the two phases
+// are observed on different endpoints, so the CREATE itself has to be retried.
+// The API exposes no DELETING state to poll for instead; remove this once it
+// does, or once CREATE queues behind the pending delete rather than failing.
+func (r *ResourceVectorSearchIndex) createIndex(ctx context.Context, req vectorsearch.CreateVectorIndexRequest) (*vectorsearch.VectorIndex, error) {
+	return retries.Poll(ctx, pendingDeletionTimeout, func() (*vectorsearch.VectorIndex, *retries.Err) {
+		index, err := r.client.VectorSearchIndexes.CreateIndex(ctx, req)
+		if err == nil {
+			return index, nil
+		}
+		if isIndexPendingDeletion(err) {
+			return nil, retries.Continues("index name is still pending deletion, waiting to recreate it")
+		}
+		return nil, retries.Halt(err)
+	})
+}
+
+// isIndexPendingDeletion reports whether err is the backend's rejection of an
+// operation on an index whose deletion has not fully completed. There is no
+// distinct error code for it — the backend returns a generic
+// INVALID_PARAMETER_VALUE — so the message has to be matched.
+func isIndexPendingDeletion(err error) bool {
+	if !errors.Is(err, apierr.ErrInvalidParameterValue) {
+		return false
+	}
+	apiErr, ok := errors.AsType[*apierr.APIError](err)
+	return ok && strings.Contains(apiErr.Message, "pending deletion")
+}
+
 // No DoUpdate: vector search indexes have no update API. All SDK fields are
 // declared in resources.yml under recreate_on_changes or ignore_remote_changes.
 // If a future SDK bump adds a new field that isn't classified, the framework
@@ -174,10 +217,12 @@ func (r *ResourceVectorSearchIndex) WaitAfterCreate(ctx context.Context, id stri
 }
 
 // WaitAfterDelete polls GetIndex until it returns 404. The DELETE call is
-// asynchronous: a follow-up CREATE for the same name (e.g. during recreate) is
-// rejected with "index is currently pending deletion" until the backend finishes
-// tearing down the embedding pipeline. The framework calls this after dropping
+// asynchronous, so without this a `bundle destroy` would report success while
+// the index is still being torn down. The framework calls this after dropping
 // state so a wait-time failure leaves the bundle consistent.
+//
+// This does NOT on its own make a recreate safe: the name is released after the
+// index disappears from GET, so createIndex retries the CREATE for the rest.
 func (r *ResourceVectorSearchIndex) WaitAfterDelete(ctx context.Context, id string) error {
 	_, err := retries.Poll[struct{}](ctx, deleteIndexTimeout, func() (*struct{}, *retries.Err) {
 		_, getErr := r.client.VectorSearchIndexes.GetIndexByIndexName(ctx, id)

libs/testserver/fake_workspace.go

@@ -193,6 +193,11 @@ type FakeWorkspace struct {
 	VectorSearchEndpoints map[string]vectorsearch.EndpointInfo
 	VectorSearchIndexes   map[string]fakeVectorSearchIndex
 
+	// VectorSearchIndexesPendingDeletion counts how many further CREATEs an
+	// already-deleted index name must reject with "pending deletion". See
+	// VectorSearchIndexDelete.
+	VectorSearchIndexesPendingDeletion map[string]int
+
 	SecretScopes map[string]workspace.SecretScope
 	Secrets      map[string]map[string]string // scope -> key -> value
 	Acls         map[string][]workspace.AclItem
@@ -394,7 +399,8 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace {
 				SingleUserName:   TestUser.UserName,
 			},
 		},
-		InstancePools: map[string]compute.GetInstancePool{},
+		InstancePools:                      map[string]compute.GetInstancePool{},
+		VectorSearchIndexesPendingDeletion: map[string]int{},
 	}
 }
 

libs/testserver/handlers.go

@@ -993,7 +993,7 @@ func AddDefaultHandlers(server *Server) {
 	})
 
 	server.Handle("DELETE", "/api/2.0/vector-search/indexes/{index_name}", func(req Request) any {
-		return MapDelete(req.Workspace, req.Workspace.VectorSearchIndexes, req.Vars["index_name"])
+		return req.Workspace.VectorSearchIndexDelete(req.Vars["index_name"])
 	})
 
 	// Generic permissions endpoints

libs/testserver/vector_search_indexes.go

@@ -15,6 +15,13 @@ import (
 // accepts: only alphanumerics and underscores.
 var indexNamePart = regexp.MustCompile(`^[A-Za-z0-9_]+$`)
 
+// indexNamePendingDeletion scopes the two-phase deletion simulation to the
+// pending-deletion recreate test. On the real backend every delete goes through
+// both phases, but the second phase usually completes before the CLI gets to
+// CREATE, so modelling it unconditionally would add a spurious retry to every
+// recreate test. Mirrors catalogNameManagedDefaults.
+const indexNamePendingDeletion = "vs_index_pending_deletion"
+
 // fakeVectorSearchIndex captures the endpoint's UUID at index creation time.
 // On the real backend an index is bound to a specific endpoint instance, not
 // just the name: deleting and recreating an endpoint with the same name yields
@@ -54,6 +61,16 @@ func (s *FakeWorkspace) VectorSearchIndexCreate(req Request) Response {
 			Body:       map[string]string{"error_code": "RESOURCE_ALREADY_EXISTS", "message": fmt.Sprintf("Vector search index with name %s already exists", createReq.Name)},
 		}
 	}
+	if s.VectorSearchIndexesPendingDeletion[createReq.Name] > 0 {
+		s.VectorSearchIndexesPendingDeletion[createReq.Name]--
+		return Response{
+			StatusCode: http.StatusBadRequest,
+			Body: map[string]string{
+				"error_code": "INVALID_PARAMETER_VALUE",
+				"message":    fmt.Sprintf("Index %s is currently pending deletion. Operations on the index are not permitted while the index is being deleted.", createReq.Name),
+			},
+		}
+	}
 	endpoint, exists := s.VectorSearchEndpoints[createReq.EndpointName]
 	if !exists {
 		return Response{
@@ -103,6 +120,24 @@ func (s *FakeWorkspace) VectorSearchIndexCreate(req Request) Response {
 	}
 }
 
+// VectorSearchIndexDelete removes the index and, for the pending-deletion test
+// name, records that the next CREATE for the same name must still be rejected.
+// The real backend releases the name only after the index has already stopped
+// being visible on GET, so a CLI that polls GET for absence can observe the
+// index as gone and still lose the race on CREATE.
+func (s *FakeWorkspace) VectorSearchIndexDelete(indexName string) Response {
+	defer s.LockUnlock()()
+
+	if _, ok := s.VectorSearchIndexes[indexName]; !ok {
+		return Response{StatusCode: http.StatusNotFound}
+	}
+	delete(s.VectorSearchIndexes, indexName)
+	if strings.Contains(indexName, indexNamePendingDeletion) {
+		s.VectorSearchIndexesPendingDeletion[indexName] = 1
+	}
+	return Response{}
+}
+
 // isValidIndexName checks that name is in catalog.schema.table form with
 // only alphanumerics and underscores per UC, mirroring the backend's
 // validation rejection at create time.