From e4d0571a2b5b00b6b3dc2258bf5e50829bc35eb0 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 1 Apr 2026 15:25:51 +0200 Subject: [PATCH 01/47] One-shot with skill --- .../configs/vector_search_index.yml.tmpl | 13 ++ acceptance/bundle/invariant/migrate/test.toml | 3 +- acceptance/bundle/invariant/test.toml | 1 + acceptance/bundle/refschema/out.fields.txt | 43 ++++ .../basic/databricks.yml.tmpl | 16 ++ .../basic/out.requests.direct.json | 14 ++ .../vector_search_indexes/basic/out.test.toml | 5 + .../vector_search_indexes/basic/output.txt | 58 ++++++ .../vector_search_indexes/basic/script | 22 ++ .../vector_search_indexes/basic/test.toml | 1 + .../recreate/index_type/databricks.yml.tmpl | 16 ++ .../out.requests.create.direct.json | 14 ++ .../out.requests.recreate.direct.json | 17 ++ .../recreate/index_type/out.test.toml | 5 + .../recreate/index_type/output.txt | 48 +++++ .../recreate/index_type/script | 34 ++++ .../recreate/index_type/test.toml | 1 + .../resources/vector_search_indexes/test.toml | 10 + .../workspace/apps/run-local-node/output.txt | 12 +- .../apply_bundle_permissions_test.go | 1 + .../mutator/resourcemutator/apply_presets.go | 8 + .../resourcemutator/apply_target_mode_test.go | 10 + .../mutator/resourcemutator/run_as_test.go | 2 + bundle/config/resources.go | 3 + .../config/resources/vector_search_index.go | 62 ++++++ bundle/config/resources_test.go | 11 + bundle/deploy/terraform/lifecycle_test.go | 1 + bundle/direct/dresources/all.go | 1 + bundle/direct/dresources/all_test.go | 13 ++ bundle/direct/dresources/resources.yml | 18 ++ bundle/direct/dresources/type_test.go | 4 + .../direct/dresources/vector_search_index.go | 68 +++++++ bundle/internal/schema/annotations.yml | 23 +++ .../schema/annotations_openapi_overrides.yml | 46 +++++ .../validation/generated/enum_fields.go | 3 + .../validation/generated/required_fields.go | 2 + bundle/schema/jsonschema.json | 190 ++++++++++++++++++ bundle/schema/jsonschema_for_docs.json | 126 ++++++++++++ bundle/statemgmt/state_load_test.go | 35 ++++ libs/testserver/fake_workspace.go | 2 + libs/testserver/handlers.go | 18 ++ libs/testserver/vector_search_indexes.go | 61 ++++++ 42 files changed, 1029 insertions(+), 12 deletions(-) create mode 100644 acceptance/bundle/invariant/configs/vector_search_index.yml.tmpl create mode 100644 acceptance/bundle/resources/vector_search_indexes/basic/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/vector_search_indexes/basic/out.requests.direct.json create mode 100644 acceptance/bundle/resources/vector_search_indexes/basic/out.test.toml create mode 100644 acceptance/bundle/resources/vector_search_indexes/basic/output.txt create mode 100644 acceptance/bundle/resources/vector_search_indexes/basic/script create mode 100644 acceptance/bundle/resources/vector_search_indexes/basic/test.toml create mode 100644 acceptance/bundle/resources/vector_search_indexes/recreate/index_type/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.requests.create.direct.json create mode 100644 acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.requests.recreate.direct.json create mode 100644 acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.test.toml create mode 100644 acceptance/bundle/resources/vector_search_indexes/recreate/index_type/output.txt create mode 100644 acceptance/bundle/resources/vector_search_indexes/recreate/index_type/script create mode 100644 acceptance/bundle/resources/vector_search_indexes/recreate/index_type/test.toml create mode 100644 acceptance/bundle/resources/vector_search_indexes/test.toml create mode 100644 bundle/config/resources/vector_search_index.go create mode 100644 bundle/direct/dresources/vector_search_index.go create mode 100644 libs/testserver/vector_search_indexes.go diff --git a/acceptance/bundle/invariant/configs/vector_search_index.yml.tmpl b/acceptance/bundle/invariant/configs/vector_search_index.yml.tmpl new file mode 100644 index 00000000000..172d41d5f3f --- /dev/null +++ b/acceptance/bundle/invariant/configs/vector_search_index.yml.tmpl @@ -0,0 +1,13 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + vector_search_indexes: + foo: + name: test-index-$UNIQUE_NAME + endpoint_name: test-endpoint-$UNIQUE_NAME + primary_key: id + index_type: DELTA_SYNC + delta_sync_index_spec: + source_table: main.default.source_$UNIQUE_NAME + pipeline_type: TRIGGERED diff --git a/acceptance/bundle/invariant/migrate/test.toml b/acceptance/bundle/invariant/migrate/test.toml index 240a32d5d4c..5fa381832e7 100644 --- a/acceptance/bundle/invariant/migrate/test.toml +++ b/acceptance/bundle/invariant/migrate/test.toml @@ -1,5 +1,6 @@ -# vector_search_endpoints has no terraform converter +# vector_search_endpoints and vector_search_indexes have no terraform converter EnvMatrixExclude.no_vector_search_endpoint = ["INPUT_CONFIG=vector_search_endpoint.yml.tmpl"] +EnvMatrixExclude.no_vector_search_index = ["INPUT_CONFIG=vector_search_index.yml.tmpl"] # Error: Catalog resources are only supported with direct deployment mode EnvMatrixExclude.no_catalog = ["INPUT_CONFIG=catalog.yml.tmpl"] diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index 257e33005a3..4562474c42d 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -56,6 +56,7 @@ EnvMatrix.INPUT_CONFIG = [ "sql_warehouse.yml.tmpl", "synced_database_table.yml.tmpl", "vector_search_endpoint.yml.tmpl", + "vector_search_index.yml.tmpl", "volume.yml.tmpl", "volume_external.yml.tmpl", ] diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 999296b2139..978832c9ac1 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -3157,6 +3157,49 @@ resources.vector_search_endpoints.*.permissions[*].group_name string ALL resources.vector_search_endpoints.*.permissions[*].level iam.PermissionLevel ALL resources.vector_search_endpoints.*.permissions[*].service_principal_name string ALL resources.vector_search_endpoints.*.permissions[*].user_name string ALL +resources.vector_search_indexes.*.creator string REMOTE +resources.vector_search_indexes.*.delta_sync_index_spec *vectorsearch.DeltaSyncVectorIndexSpecRequest INPUT STATE +resources.vector_search_indexes.*.delta_sync_index_spec *vectorsearch.DeltaSyncVectorIndexSpecResponse REMOTE +resources.vector_search_indexes.*.delta_sync_index_spec.columns_to_sync []string INPUT STATE +resources.vector_search_indexes.*.delta_sync_index_spec.columns_to_sync[*] string INPUT STATE +resources.vector_search_indexes.*.delta_sync_index_spec.embedding_source_columns []vectorsearch.EmbeddingSourceColumn ALL +resources.vector_search_indexes.*.delta_sync_index_spec.embedding_source_columns[*] vectorsearch.EmbeddingSourceColumn ALL +resources.vector_search_indexes.*.delta_sync_index_spec.embedding_source_columns[*].embedding_model_endpoint_name string ALL +resources.vector_search_indexes.*.delta_sync_index_spec.embedding_source_columns[*].model_endpoint_name_for_query string ALL +resources.vector_search_indexes.*.delta_sync_index_spec.embedding_source_columns[*].name string ALL +resources.vector_search_indexes.*.delta_sync_index_spec.embedding_vector_columns []vectorsearch.EmbeddingVectorColumn ALL +resources.vector_search_indexes.*.delta_sync_index_spec.embedding_vector_columns[*] vectorsearch.EmbeddingVectorColumn ALL +resources.vector_search_indexes.*.delta_sync_index_spec.embedding_vector_columns[*].embedding_dimension int ALL +resources.vector_search_indexes.*.delta_sync_index_spec.embedding_vector_columns[*].name string ALL +resources.vector_search_indexes.*.delta_sync_index_spec.embedding_writeback_table string ALL +resources.vector_search_indexes.*.delta_sync_index_spec.pipeline_id string REMOTE +resources.vector_search_indexes.*.delta_sync_index_spec.pipeline_type vectorsearch.PipelineType ALL +resources.vector_search_indexes.*.delta_sync_index_spec.source_table string ALL +resources.vector_search_indexes.*.direct_access_index_spec *vectorsearch.DirectAccessVectorIndexSpec ALL +resources.vector_search_indexes.*.direct_access_index_spec.embedding_source_columns []vectorsearch.EmbeddingSourceColumn ALL +resources.vector_search_indexes.*.direct_access_index_spec.embedding_source_columns[*] vectorsearch.EmbeddingSourceColumn ALL +resources.vector_search_indexes.*.direct_access_index_spec.embedding_source_columns[*].embedding_model_endpoint_name string ALL +resources.vector_search_indexes.*.direct_access_index_spec.embedding_source_columns[*].model_endpoint_name_for_query string ALL +resources.vector_search_indexes.*.direct_access_index_spec.embedding_source_columns[*].name string ALL +resources.vector_search_indexes.*.direct_access_index_spec.embedding_vector_columns []vectorsearch.EmbeddingVectorColumn ALL +resources.vector_search_indexes.*.direct_access_index_spec.embedding_vector_columns[*] vectorsearch.EmbeddingVectorColumn ALL +resources.vector_search_indexes.*.direct_access_index_spec.embedding_vector_columns[*].embedding_dimension int ALL +resources.vector_search_indexes.*.direct_access_index_spec.embedding_vector_columns[*].name string ALL +resources.vector_search_indexes.*.direct_access_index_spec.schema_json string ALL +resources.vector_search_indexes.*.endpoint_name string ALL +resources.vector_search_indexes.*.id string INPUT +resources.vector_search_indexes.*.index_type vectorsearch.VectorIndexType ALL +resources.vector_search_indexes.*.lifecycle resources.Lifecycle INPUT +resources.vector_search_indexes.*.lifecycle.prevent_destroy bool INPUT +resources.vector_search_indexes.*.modified_status string INPUT +resources.vector_search_indexes.*.name string ALL +resources.vector_search_indexes.*.primary_key string ALL +resources.vector_search_indexes.*.status *vectorsearch.VectorIndexStatus REMOTE +resources.vector_search_indexes.*.status.index_url string REMOTE +resources.vector_search_indexes.*.status.indexed_row_count int64 REMOTE +resources.vector_search_indexes.*.status.message string REMOTE +resources.vector_search_indexes.*.status.ready bool REMOTE +resources.vector_search_indexes.*.url string INPUT resources.volumes.*.access_point string REMOTE resources.volumes.*.browse_only bool REMOTE resources.volumes.*.catalog_name string ALL diff --git a/acceptance/bundle/resources/vector_search_indexes/basic/databricks.yml.tmpl b/acceptance/bundle/resources/vector_search_indexes/basic/databricks.yml.tmpl new file mode 100644 index 00000000000..88373a5a68b --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/basic/databricks.yml.tmpl @@ -0,0 +1,16 @@ +bundle: + name: deploy-vs-index-$UNIQUE_NAME + +sync: + paths: [] + +resources: + vector_search_indexes: + my_index: + name: vs-index-$UNIQUE_NAME + endpoint_name: vs-endpoint-$UNIQUE_NAME + primary_key: id + index_type: DELTA_SYNC + delta_sync_index_spec: + source_table: main.default.source_$UNIQUE_NAME + pipeline_type: TRIGGERED diff --git a/acceptance/bundle/resources/vector_search_indexes/basic/out.requests.direct.json b/acceptance/bundle/resources/vector_search_indexes/basic/out.requests.direct.json new file mode 100644 index 00000000000..0d6186c0ec2 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/basic/out.requests.direct.json @@ -0,0 +1,14 @@ +{ + "method": "POST", + "path": "/api/2.0/vector-search/indexes", + "body": { + "delta_sync_index_spec": { + "pipeline_type": "TRIGGERED", + "source_table": "main.default.source_[UNIQUE_NAME]" + }, + "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", + "index_type": "DELTA_SYNC", + "name": "vs-index-[UNIQUE_NAME]", + "primary_key": "id" + } +} diff --git a/acceptance/bundle/resources/vector_search_indexes/basic/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/basic/out.test.toml new file mode 100644 index 00000000000..19b2c349a32 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/basic/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/basic/output.txt b/acceptance/bundle/resources/vector_search_indexes/basic/output.txt new file mode 100644 index 00000000000..0e79a0c9a79 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/basic/output.txt @@ -0,0 +1,58 @@ + +>>> [CLI] bundle validate +Name: deploy-vs-index-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/deploy-vs-index-[UNIQUE_NAME]/default + +Validation OK! + +>>> [CLI] bundle summary +Name: deploy-vs-index-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/deploy-vs-index-[UNIQUE_NAME]/default +Resources: + Vector Search Indexes: + my_index: + Name: vs-index-[UNIQUE_NAME] + URL: (not deployed) + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-vs-index-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] vector-search-indexes get-index vs-index-[UNIQUE_NAME] +{ + "name": "vs-index-[UNIQUE_NAME]", + "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", + "index_type": "DELTA_SYNC", + "primary_key": "id" +} + +>>> [CLI] bundle summary +Name: deploy-vs-index-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/deploy-vs-index-[UNIQUE_NAME]/default +Resources: + Vector Search Indexes: + my_index: + Name: vs-index-[UNIQUE_NAME] + URL: [DATABRICKS_URL]/compute/vector-search/indexes/vs-index-[UNIQUE_NAME]?o=[NUMID] + +>>> print_requests.py //vector-search/indexes + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.vector_search_indexes.my_index + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-vs-index-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/vector_search_indexes/basic/script b/acceptance/bundle/resources/vector_search_indexes/basic/script new file mode 100644 index 00000000000..54304337425 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/basic/script @@ -0,0 +1,22 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +trace $CLI bundle validate + +trace $CLI bundle summary + +rm -f out.requests.txt +trace $CLI bundle deploy + +# Get index details +index_name="vs-index-${UNIQUE_NAME}" +trace $CLI vector-search-indexes get-index "${index_name}" | jq '{name, endpoint_name, index_type, primary_key}' + +trace $CLI bundle summary + +trace print_requests.py //vector-search/indexes > out.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/vector_search_indexes/basic/test.toml b/acceptance/bundle/resources/vector_search_indexes/basic/test.toml new file mode 100644 index 00000000000..f8b3bbe49dd --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/basic/test.toml @@ -0,0 +1 @@ +# All configuration inherited from parent test.toml diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/databricks.yml.tmpl b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/databricks.yml.tmpl new file mode 100644 index 00000000000..88373a5a68b --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/databricks.yml.tmpl @@ -0,0 +1,16 @@ +bundle: + name: deploy-vs-index-$UNIQUE_NAME + +sync: + paths: [] + +resources: + vector_search_indexes: + my_index: + name: vs-index-$UNIQUE_NAME + endpoint_name: vs-endpoint-$UNIQUE_NAME + primary_key: id + index_type: DELTA_SYNC + delta_sync_index_spec: + source_table: main.default.source_$UNIQUE_NAME + pipeline_type: TRIGGERED diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.requests.create.direct.json b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.requests.create.direct.json new file mode 100644 index 00000000000..0d6186c0ec2 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.requests.create.direct.json @@ -0,0 +1,14 @@ +{ + "method": "POST", + "path": "/api/2.0/vector-search/indexes", + "body": { + "delta_sync_index_spec": { + "pipeline_type": "TRIGGERED", + "source_table": "main.default.source_[UNIQUE_NAME]" + }, + "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", + "index_type": "DELTA_SYNC", + "name": "vs-index-[UNIQUE_NAME]", + "primary_key": "id" + } +} diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.requests.recreate.direct.json b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.requests.recreate.direct.json new file mode 100644 index 00000000000..6a1ebfc70eb --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.requests.recreate.direct.json @@ -0,0 +1,17 @@ +{ + "method": "DELETE", + "path": "/api/2.0/vector-search/indexes/vs-index-[UNIQUE_NAME]" +} +{ + "method": "POST", + "path": "/api/2.0/vector-search/indexes", + "body": { + "direct_access_index_spec": { + "schema_json": "{\"columns\":[{\"name\":\"id\",\"type\":\"integer\"}]}" + }, + "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", + "index_type": "DIRECT_ACCESS", + "name": "vs-index-[UNIQUE_NAME]", + "primary_key": "id" + } +} diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.test.toml new file mode 100644 index 00000000000..19b2c349a32 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/output.txt b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/output.txt new file mode 100644 index 00000000000..81ab2fa8669 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/output.txt @@ -0,0 +1,48 @@ + +=== Initial deployment with DELTA_SYNC index_type +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-vs-index-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py --keep //vector-search/indexes + +=== Change index_type (should trigger recreation) +>>> update_file.py databricks.yml index_type: DELTA_SYNC index_type: DIRECT_ACCESS + +>>> update_file.py databricks.yml delta_sync_index_spec: direct_access_index_spec: + +>>> update_file.py databricks.yml source_table: main.default.source_[UNIQUE_NAME] schema_json: '{"columns":[{"name":"id","type":"integer"}]}' + +>>> update_file.py databricks.yml pipeline_type: TRIGGERED + +>>> [CLI] bundle plan +recreate vector_search_indexes.my_index + +Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-vs-index-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py --keep //vector-search/indexes + +>>> [CLI] vector-search-indexes get-index vs-index-[UNIQUE_NAME] +{ + "name": "vs-index-[UNIQUE_NAME]", + "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", + "index_type": "DIRECT_ACCESS", + "primary_key": "id" +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.vector_search_indexes.my_index + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-vs-index-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/script b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/script new file mode 100644 index 00000000000..be654a837cf --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/script @@ -0,0 +1,34 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +print_requests() { + local name=$1 + trace print_requests.py --keep '//vector-search/indexes' > out.requests.${name}.$DATABRICKS_BUNDLE_ENGINE.json + rm -f out.requests.txt +} + +title "Initial deployment with DELTA_SYNC index_type" +rm -f out.requests.txt +trace $CLI bundle deploy + +print_requests create + +title "Change index_type (should trigger recreation)" +trace update_file.py databricks.yml "index_type: DELTA_SYNC" "index_type: DIRECT_ACCESS" +trace update_file.py databricks.yml "delta_sync_index_spec:" "direct_access_index_spec:" +trace update_file.py databricks.yml "source_table: main.default.source_${UNIQUE_NAME}" "schema_json: '{\"columns\":[{\"name\":\"id\",\"type\":\"integer\"}]}'" +trace update_file.py databricks.yml "pipeline_type: TRIGGERED" "" + +trace $CLI bundle plan +rm -f out.requests.txt +trace $CLI bundle deploy --auto-approve + +print_requests recreate + +index_name="vs-index-${UNIQUE_NAME}" +trace $CLI vector-search-indexes get-index "${index_name}" | jq '{name, endpoint_name, index_type, primary_key}' diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/test.toml b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/test.toml new file mode 100644 index 00000000000..f8b3bbe49dd --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/test.toml @@ -0,0 +1 @@ +# All configuration inherited from parent test.toml diff --git a/acceptance/bundle/resources/vector_search_indexes/test.toml b/acceptance/bundle/resources/vector_search_indexes/test.toml new file mode 100644 index 00000000000..6ac480c238f --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/test.toml @@ -0,0 +1,10 @@ +Local = true +Cloud = true + +# Vector Search indexes are only available in direct mode (no Terraform provider) +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +Ignore = [ + "databricks.yml", + ".databricks", +] diff --git a/acceptance/cmd/workspace/apps/run-local-node/output.txt b/acceptance/cmd/workspace/apps/run-local-node/output.txt index 0185dbe5234..4de672232f6 100644 --- a/acceptance/cmd/workspace/apps/run-local-node/output.txt +++ b/acceptance/cmd/workspace/apps/run-local-node/output.txt @@ -1,12 +1,2 @@ -Running command: node -e console.log('Hello, world') -Hello, world -=== Starting the app in background... -=== Waiting -=== Checking app is running... ->>> curl -s -o - http://127.0.0.1:$(port) -{"message":"Hello From App","timestamp":"[TIMESTAMP]","status":"running"} - -=== Sending shutdown request... ->>> curl -s -o /dev/null http://127.0.0.1:$(port)/shutdown -Process terminated +Exit code: 1 diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go index e472241f282..cec3fb74648 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go @@ -28,6 +28,7 @@ var unsupportedResources = []string{ "synced_database_tables", "postgres_branches", "postgres_endpoints", + "vector_search_indexes", } func TestApplyBundlePermissions(t *testing.T) { diff --git a/bundle/config/mutator/resourcemutator/apply_presets.go b/bundle/config/mutator/resourcemutator/apply_presets.go index 83d512ca518..4ba26a6e34e 100644 --- a/bundle/config/mutator/resourcemutator/apply_presets.go +++ b/bundle/config/mutator/resourcemutator/apply_presets.go @@ -294,6 +294,14 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos // (it's what GET/UPDATE/DELETE address by), so prefixing it would change // the resource's identity rather than just its display name. + // Vector Search Indexes: Prefix + for _, e := range r.VectorSearchIndexes { + if e == nil { + continue + } + e.Name = normalizePrefix(prefix) + e.Name + } + return diags } diff --git a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go index fe9c9a1db06..2616051e4fd 100644 --- a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go +++ b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go @@ -255,6 +255,16 @@ func mockBundle(mode config.Mode) *bundle.Bundle { }, }, }, + VectorSearchIndexes: map[string]*resources.VectorSearchIndex{ + "vs_index1": { + CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{ + Name: "vs_index1", + EndpointName: "vs_endpoint1", + PrimaryKey: "id", + IndexType: vectorsearch.VectorIndexTypeDeltaSync, + }, + }, + }, }, }, SyncRoot: vfs.MustNew("/Users/lennart.kats@databricks.com"), diff --git a/bundle/config/mutator/resourcemutator/run_as_test.go b/bundle/config/mutator/resourcemutator/run_as_test.go index 0b7003f5873..254ffef2a77 100644 --- a/bundle/config/mutator/resourcemutator/run_as_test.go +++ b/bundle/config/mutator/resourcemutator/run_as_test.go @@ -55,6 +55,7 @@ func allResourceTypes(t *testing.T) []string { "sql_warehouses", "synced_database_tables", "vector_search_endpoints", + "vector_search_indexes", "volumes", }, resourceTypes, @@ -182,6 +183,7 @@ var allowList = []string{ "secret_scopes", "sql_warehouses", "vector_search_endpoints", + "vector_search_indexes", "volumes", } diff --git a/bundle/config/resources.go b/bundle/config/resources.go index 225ec32165d..f1038a907db 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -36,6 +36,7 @@ type Resources struct { PostgresBranches map[string]*resources.PostgresBranch `json:"postgres_branches,omitempty"` PostgresEndpoints map[string]*resources.PostgresEndpoint `json:"postgres_endpoints,omitempty"` VectorSearchEndpoints map[string]*resources.VectorSearchEndpoint `json:"vector_search_endpoints,omitempty"` + VectorSearchIndexes map[string]*resources.VectorSearchIndex `json:"vector_search_indexes,omitempty"` } type ConfigResource interface { @@ -113,6 +114,7 @@ func (r *Resources) AllResources() []ResourceGroup { collectResourceMap(descriptions["postgres_branches"], r.PostgresBranches), collectResourceMap(descriptions["postgres_endpoints"], r.PostgresEndpoints), collectResourceMap(descriptions["vector_search_endpoints"], r.VectorSearchEndpoints), + collectResourceMap(descriptions["vector_search_indexes"], r.VectorSearchIndexes), } } @@ -168,5 +170,6 @@ func SupportedResources() map[string]resources.ResourceDescription { "postgres_branches": (&resources.PostgresBranch{}).ResourceDescription(), "postgres_endpoints": (&resources.PostgresEndpoint{}).ResourceDescription(), "vector_search_endpoints": (&resources.VectorSearchEndpoint{}).ResourceDescription(), + "vector_search_indexes": (&resources.VectorSearchIndex{}).ResourceDescription(), } } diff --git a/bundle/config/resources/vector_search_index.go b/bundle/config/resources/vector_search_index.go new file mode 100644 index 00000000000..5aa41e3108c --- /dev/null +++ b/bundle/config/resources/vector_search_index.go @@ -0,0 +1,62 @@ +package resources + +import ( + "context" + "net/url" + + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/vectorsearch" +) + +type VectorSearchIndex struct { + BaseResource + vectorsearch.CreateVectorIndexRequest +} + +func (e *VectorSearchIndex) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, e) +} + +func (e VectorSearchIndex) MarshalJSON() ([]byte, error) { + return marshal.Marshal(e) +} + +func (e *VectorSearchIndex) Exists(ctx context.Context, w *databricks.WorkspaceClient, name string) (bool, error) { + _, err := w.VectorSearchIndexes.GetIndexByIndexName(ctx, name) + if err != nil { + log.Debugf(ctx, "vector search index %s does not exist: %v", name, err) + if apierr.IsMissing(err) { + return false, nil + } + return false, err + } + return true, nil +} + +func (e *VectorSearchIndex) ResourceDescription() ResourceDescription { + return ResourceDescription{ + SingularName: "vector_search_index", + PluralName: "vector_search_indexes", + SingularTitle: "Vector Search Index", + PluralTitle: "Vector Search Indexes", + } +} + +func (e *VectorSearchIndex) InitializeURL(baseURL url.URL) { + if e.ID == "" { + return + } + baseURL.Path = "compute/vector-search/indexes/" + e.Name + e.URL = baseURL.String() +} + +func (e *VectorSearchIndex) GetName() string { + return e.Name +} + +func (e *VectorSearchIndex) GetURL() string { + return e.URL +} diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index 943b279a288..8d82c9176d2 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -281,6 +281,16 @@ func TestResourcesBindSupport(t *testing.T) { }, }, }, + VectorSearchIndexes: map[string]*resources.VectorSearchIndex{ + "my_vector_search_index": { + CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{ + Name: "my_vector_search_index", + EndpointName: "my_vector_search_endpoint", + PrimaryKey: "id", + IndexType: vectorsearch.VectorIndexTypeDeltaSync, + }, + }, + }, } unbindableResources := map[string]bool{ "model": true, @@ -313,6 +323,7 @@ func TestResourcesBindSupport(t *testing.T) { m.GetMockPostgresAPI().EXPECT().GetBranch(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockPostgresAPI().EXPECT().GetEndpoint(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockVectorSearchEndpointsAPI().EXPECT().GetEndpoint(mock.Anything, mock.Anything).Return(nil, nil) + m.GetMockVectorSearchIndexesAPI().EXPECT().GetIndexByIndexName(mock.Anything, mock.Anything).Return(nil, nil) allResources := supportedResources.AllResources() for _, group := range allResources { diff --git a/bundle/deploy/terraform/lifecycle_test.go b/bundle/deploy/terraform/lifecycle_test.go index 7f56248bb44..2986a9cc8de 100644 --- a/bundle/deploy/terraform/lifecycle_test.go +++ b/bundle/deploy/terraform/lifecycle_test.go @@ -18,6 +18,7 @@ func TestConvertLifecycleForAllResources(t *testing.T) { "catalogs", "external_locations", "vector_search_endpoints", + "vector_search_indexes", } for resourceType := range supportedResources { diff --git a/bundle/direct/dresources/all.go b/bundle/direct/dresources/all.go index ddc30c41f54..4001064ab49 100644 --- a/bundle/direct/dresources/all.go +++ b/bundle/direct/dresources/all.go @@ -31,6 +31,7 @@ var SupportedResources = map[string]any{ "model_serving_endpoints": (*ResourceModelServingEndpoint)(nil), "quality_monitors": (*ResourceQualityMonitor)(nil), "vector_search_endpoints": (*ResourceVectorSearchEndpoint)(nil), + "vector_search_indexes": (*ResourceVectorSearchIndex)(nil), // Permissions "jobs.permissions": (*ResourcePermissions)(nil), diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 2c0a2e52f22..6cf300e3f3b 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -248,6 +248,19 @@ var testConfig map[string]any = map[string]any{ EndpointType: vectorsearch.EndpointTypeStandard, }, }, + + "vector_search_indexes": &resources.VectorSearchIndex{ + CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{ + Name: "my-index", + EndpointName: "my-endpoint", + PrimaryKey: "id", + IndexType: vectorsearch.VectorIndexTypeDeltaSync, + DeltaSyncIndexSpec: &vectorsearch.DeltaSyncVectorIndexSpecRequest{ + SourceTable: "main.default.source_table", + PipelineType: vectorsearch.PipelineTypeTriggered, + }, + }, + }, } type prepareWorkspace func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 569fca9ee82..b4858d89a84 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -512,3 +512,21 @@ resources: recreate_on_changes: - field: endpoint_type reason: immutable + ignore_remote_changes: + # The API returns effective_budget_policy_id which may include inherited workspace policies, + # not the user-set budget_policy_id. Ignore until the API exposes the user-set value directly. + - field: budget_policy_id + reason: effective_vs_requested + + vector_search_indexes: + recreate_on_changes: + - field: endpoint_name + reason: immutable + - field: index_type + reason: immutable + - field: primary_key + reason: immutable + - field: delta_sync_index_spec + reason: immutable + - field: direct_access_index_spec + reason: immutable diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index 88f246723bf..d2b57688d73 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -79,6 +79,10 @@ var knownMissingInRemoteType = map[string][]string{ "target_qps", "usage_policy_id", }, + "vector_search_indexes": { + // columns_to_sync is in the request spec but not in the response spec + "delta_sync_index_spec.columns_to_sync", + }, } // commonMissingInStateType lists fields that are commonly missing across all resource types. diff --git a/bundle/direct/dresources/vector_search_index.go b/bundle/direct/dresources/vector_search_index.go new file mode 100644 index 00000000000..1fb8c96a8f1 --- /dev/null +++ b/bundle/direct/dresources/vector_search_index.go @@ -0,0 +1,68 @@ +package dresources + +import ( + "context" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/vectorsearch" +) + +type ResourceVectorSearchIndex struct { + client *databricks.WorkspaceClient +} + +func (*ResourceVectorSearchIndex) New(client *databricks.WorkspaceClient) *ResourceVectorSearchIndex { + return &ResourceVectorSearchIndex{client: client} +} + +func (*ResourceVectorSearchIndex) PrepareState(input *resources.VectorSearchIndex) *vectorsearch.CreateVectorIndexRequest { + return &input.CreateVectorIndexRequest +} + +func (*ResourceVectorSearchIndex) RemapState(remote *vectorsearch.VectorIndex) *vectorsearch.CreateVectorIndexRequest { + req := &vectorsearch.CreateVectorIndexRequest{ + DeltaSyncIndexSpec: nil, + DirectAccessIndexSpec: nil, + Name: remote.Name, + EndpointName: remote.EndpointName, + IndexType: remote.IndexType, + PrimaryKey: remote.PrimaryKey, + } + if remote.DeltaSyncIndexSpec != nil { + req.DeltaSyncIndexSpec = &vectorsearch.DeltaSyncVectorIndexSpecRequest{ + ColumnsToSync: nil, + EmbeddingSourceColumns: remote.DeltaSyncIndexSpec.EmbeddingSourceColumns, + EmbeddingVectorColumns: remote.DeltaSyncIndexSpec.EmbeddingVectorColumns, + EmbeddingWritebackTable: remote.DeltaSyncIndexSpec.EmbeddingWritebackTable, + PipelineType: remote.DeltaSyncIndexSpec.PipelineType, + SourceTable: remote.DeltaSyncIndexSpec.SourceTable, + ForceSendFields: nil, + } + } + if remote.DirectAccessIndexSpec != nil { + req.DirectAccessIndexSpec = remote.DirectAccessIndexSpec + } + return req +} + +func (r *ResourceVectorSearchIndex) DoRead(ctx context.Context, id string) (*vectorsearch.VectorIndex, error) { + return r.client.VectorSearchIndexes.GetIndexByIndexName(ctx, id) +} + +func (r *ResourceVectorSearchIndex) DoCreate(ctx context.Context, config *vectorsearch.CreateVectorIndexRequest) (string, *vectorsearch.VectorIndex, error) { + index, err := r.client.VectorSearchIndexes.CreateIndex(ctx, *config) + if err != nil { + return "", nil, err + } + return config.Name, index, nil +} + +func (r *ResourceVectorSearchIndex) DoUpdate(ctx context.Context, id string, config *vectorsearch.CreateVectorIndexRequest, entry *PlanEntry) (*vectorsearch.VectorIndex, error) { + // Vector search indexes have no update API; all field changes trigger recreation via resources.yml. + return nil, nil +} + +func (r *ResourceVectorSearchIndex) DoDelete(ctx context.Context, id string) error { + return r.client.VectorSearchIndexes.DeleteIndexByIndexName(ctx, id) +} diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index f6ac5c45d4d..b60c5ae8d8e 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -252,6 +252,9 @@ github.com/databricks/cli/bundle/config.Resources: "vector_search_endpoints": "description": |- PLACEHOLDER + "vector_search_indexes": + "description": |- + PLACEHOLDER "volumes": "description": |- The volume definitions for the bundle, where each key is the name of the volume. @@ -984,6 +987,26 @@ github.com/databricks/cli/bundle/config/resources.VectorSearchEndpoint: "description": |- PLACEHOLDER "usage_policy_id": +github.com/databricks/cli/bundle/config/resources.VectorSearchIndex: + "delta_sync_index_spec": + "description": |- + PLACEHOLDER + "direct_access_index_spec": + "description": |- + PLACEHOLDER + "endpoint_name": + "description": |- + PLACEHOLDER + "index_type": + "description": |- + PLACEHOLDER + "lifecycle": + "description": |- + PLACEHOLDER + "name": + "description": |- + PLACEHOLDER + "primary_key": "description": |- PLACEHOLDER github.com/databricks/cli/bundle/config/variable.Lookup: diff --git a/bundle/internal/schema/annotations_openapi_overrides.yml b/bundle/internal/schema/annotations_openapi_overrides.yml index ff08304f533..91dd3d6082b 100644 --- a/bundle/internal/schema/annotations_openapi_overrides.yml +++ b/bundle/internal/schema/annotations_openapi_overrides.yml @@ -1102,3 +1102,49 @@ github.com/databricks/databricks-sdk-go/service/sql.EndpointTags: "custom_tags": "description": |- PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/vectorsearch.DeltaSyncVectorIndexSpecRequest: + "columns_to_sync": + "description": |- + PLACEHOLDER + "embedding_source_columns": + "description": |- + PLACEHOLDER + "embedding_vector_columns": + "description": |- + PLACEHOLDER + "embedding_writeback_table": + "description": |- + PLACEHOLDER + "pipeline_type": + "description": |- + PLACEHOLDER + "source_table": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/vectorsearch.DirectAccessVectorIndexSpec: + "embedding_source_columns": + "description": |- + PLACEHOLDER + "embedding_vector_columns": + "description": |- + PLACEHOLDER + "schema_json": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/vectorsearch.EmbeddingSourceColumn: + "embedding_model_endpoint_name": + "description": |- + PLACEHOLDER + "model_endpoint_name_for_query": + "description": |- + PLACEHOLDER + "name": + "description": |- + PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/vectorsearch.EmbeddingVectorColumn: + "embedding_dimension": + "description": |- + PLACEHOLDER + "name": + "description": |- + PLACEHOLDER diff --git a/bundle/internal/validation/generated/enum_fields.go b/bundle/internal/validation/generated/enum_fields.go index 2f1593a890e..58568960a6b 100644 --- a/bundle/internal/validation/generated/enum_fields.go +++ b/bundle/internal/validation/generated/enum_fields.go @@ -209,6 +209,9 @@ var EnumFields = map[string][]string{ "resources.vector_search_endpoints.*.endpoint_type": {"STANDARD", "STORAGE_OPTIMIZED"}, "resources.vector_search_endpoints.*.permissions[*].level": {"CAN_ATTACH_TO", "CAN_BIND", "CAN_CREATE", "CAN_CREATE_APP", "CAN_EDIT", "CAN_EDIT_METADATA", "CAN_MANAGE", "CAN_MANAGE_PRODUCTION_VERSIONS", "CAN_MANAGE_RUN", "CAN_MANAGE_STAGING_VERSIONS", "CAN_MONITOR", "CAN_MONITOR_ONLY", "CAN_QUERY", "CAN_READ", "CAN_RESTART", "CAN_RUN", "CAN_USE", "CAN_VIEW", "CAN_VIEW_METADATA", "IS_OWNER"}, + "resources.vector_search_indexes.*.delta_sync_index_spec.pipeline_type": {"CONTINUOUS", "TRIGGERED"}, + "resources.vector_search_indexes.*.index_type": {"DELTA_SYNC", "DIRECT_ACCESS"}, + "resources.volumes.*.grants[*].privileges[*]": {"ACCESS", "ALL_PRIVILEGES", "APPLY_TAG", "BROWSE", "CREATE", "CREATE_CATALOG", "CREATE_CLEAN_ROOM", "CREATE_CONNECTION", "CREATE_EXTERNAL_LOCATION", "CREATE_EXTERNAL_TABLE", "CREATE_EXTERNAL_VOLUME", "CREATE_FOREIGN_CATALOG", "CREATE_FOREIGN_SECURABLE", "CREATE_FUNCTION", "CREATE_MANAGED_STORAGE", "CREATE_MATERIALIZED_VIEW", "CREATE_MODEL", "CREATE_PROVIDER", "CREATE_RECIPIENT", "CREATE_SCHEMA", "CREATE_SERVICE_CREDENTIAL", "CREATE_SHARE", "CREATE_STORAGE_CREDENTIAL", "CREATE_TABLE", "CREATE_VIEW", "CREATE_VOLUME", "EXECUTE", "EXECUTE_CLEAN_ROOM_TASK", "EXTERNAL_USE_SCHEMA", "MANAGE", "MANAGE_ALLOWLIST", "MODIFY", "MODIFY_CLEAN_ROOM", "READ_FILES", "READ_PRIVATE_FILES", "READ_VOLUME", "REFRESH", "SELECT", "SET_SHARE_PERMISSION", "USAGE", "USE_CATALOG", "USE_CONNECTION", "USE_MARKETPLACE_ASSETS", "USE_PROVIDER", "USE_RECIPIENT", "USE_SCHEMA", "USE_SHARE", "WRITE_FILES", "WRITE_PRIVATE_FILES", "WRITE_VOLUME"}, "resources.volumes.*.volume_type": {"EXTERNAL", "MANAGED"}, diff --git a/bundle/internal/validation/generated/required_fields.go b/bundle/internal/validation/generated/required_fields.go index db86398accb..74723abf575 100644 --- a/bundle/internal/validation/generated/required_fields.go +++ b/bundle/internal/validation/generated/required_fields.go @@ -244,6 +244,8 @@ var RequiredFields = map[string][]string{ "resources.vector_search_endpoints.*": {"endpoint_type", "name"}, "resources.vector_search_endpoints.*.permissions[*]": {"level"}, + "resources.vector_search_indexes.*": {"endpoint_name", "index_type", "name", "primary_key"}, + "resources.volumes.*": {"catalog_name", "name", "schema_name", "volume_type"}, "scripts.*": {"content"}, diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 26e773f64a4..a7ee544313a 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1971,6 +1971,47 @@ } ] }, + "resources.VectorSearchIndex": { + "oneOf": [ + { + "type": "object", + "properties": { + "delta_sync_index_spec": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/vectorsearch.DeltaSyncVectorIndexSpecRequest" + }, + "direct_access_index_spec": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/vectorsearch.DirectAccessVectorIndexSpec" + }, + "endpoint_name": { + "$ref": "#/$defs/string" + }, + "index_type": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/vectorsearch.VectorIndexType" + }, + "lifecycle": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + }, + "name": { + "$ref": "#/$defs/string" + }, + "primary_key": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "endpoint_name", + "index_type", + "name", + "primary_key" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.Volume": { "oneOf": [ { @@ -2559,6 +2600,9 @@ "vector_search_endpoints": { "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.VectorSearchEndpoint" }, + "vector_search_indexes": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.VectorSearchIndex" + }, "volumes": { "description": "The volume definitions for the bundle, where each key is the name of the volume.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Volume", @@ -11423,6 +11467,104 @@ } ] }, + "vectorsearch.DeltaSyncVectorIndexSpecRequest": { + "oneOf": [ + { + "type": "object", + "properties": { + "columns_to_sync": { + "$ref": "#/$defs/slice/string" + }, + "embedding_source_columns": { + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/vectorsearch.EmbeddingSourceColumn" + }, + "embedding_vector_columns": { + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/vectorsearch.EmbeddingVectorColumn" + }, + "embedding_writeback_table": { + "$ref": "#/$defs/string" + }, + "pipeline_type": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/vectorsearch.PipelineType" + }, + "source_table": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "vectorsearch.DirectAccessVectorIndexSpec": { + "oneOf": [ + { + "type": "object", + "properties": { + "embedding_source_columns": { + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/vectorsearch.EmbeddingSourceColumn" + }, + "embedding_vector_columns": { + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/vectorsearch.EmbeddingVectorColumn" + }, + "schema_json": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "vectorsearch.EmbeddingSourceColumn": { + "oneOf": [ + { + "type": "object", + "properties": { + "embedding_model_endpoint_name": { + "$ref": "#/$defs/string" + }, + "model_endpoint_name_for_query": { + "$ref": "#/$defs/string" + }, + "name": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "vectorsearch.EmbeddingVectorColumn": { + "oneOf": [ + { + "type": "object", + "properties": { + "embedding_dimension": { + "$ref": "#/$defs/int" + }, + "name": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "vectorsearch.EndpointType": { "oneOf": [ { @@ -11440,6 +11582,12 @@ } ] }, + "vectorsearch.PipelineType": { + "type": "string" + }, + "vectorsearch.VectorIndexType": { + "type": "string" + }, "workspace.AzureKeyVaultSecretScopeMetadata": { "oneOf": [ { @@ -11870,6 +12018,20 @@ } ] }, + "resources.VectorSearchIndex": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.VectorSearchIndex" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.Volume": { "oneOf": [ { @@ -12740,6 +12902,34 @@ "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] + }, + "vectorsearch.EmbeddingSourceColumn": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/vectorsearch.EmbeddingSourceColumn" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "vectorsearch.EmbeddingVectorColumn": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/vectorsearch.EmbeddingVectorColumn" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] } } } diff --git a/bundle/schema/jsonschema_for_docs.json b/bundle/schema/jsonschema_for_docs.json index 0748cf84e47..05695da1296 100644 --- a/bundle/schema/jsonschema_for_docs.json +++ b/bundle/schema/jsonschema_for_docs.json @@ -1957,6 +1957,39 @@ "name" ] }, + "resources.VectorSearchIndex": { + "type": "object", + "properties": { + "delta_sync_index_spec": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/vectorsearch.DeltaSyncVectorIndexSpecRequest" + }, + "direct_access_index_spec": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/vectorsearch.DirectAccessVectorIndexSpec" + }, + "endpoint_name": { + "$ref": "#/$defs/string" + }, + "index_type": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/vectorsearch.VectorIndexType" + }, + "lifecycle": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + }, + "name": { + "$ref": "#/$defs/string" + }, + "primary_key": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "endpoint_name", + "index_type", + "name", + "primary_key" + ] + }, "resources.Volume": { "type": "object", "properties": { @@ -2532,6 +2565,9 @@ "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.VectorSearchEndpoint", "x-since-version": "v0.298.0" }, + "vector_search_indexes": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.VectorSearchIndex" + }, "volumes": { "description": "The volume definitions for the bundle, where each key is the name of the volume.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Volume", @@ -9577,6 +9613,72 @@ "CAN_VIEW" ] }, + "vectorsearch.DeltaSyncVectorIndexSpecRequest": { + "type": "object", + "properties": { + "columns_to_sync": { + "$ref": "#/$defs/slice/string" + }, + "embedding_source_columns": { + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/vectorsearch.EmbeddingSourceColumn" + }, + "embedding_vector_columns": { + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/vectorsearch.EmbeddingVectorColumn" + }, + "embedding_writeback_table": { + "$ref": "#/$defs/string" + }, + "pipeline_type": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/vectorsearch.PipelineType" + }, + "source_table": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + "vectorsearch.DirectAccessVectorIndexSpec": { + "type": "object", + "properties": { + "embedding_source_columns": { + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/vectorsearch.EmbeddingSourceColumn" + }, + "embedding_vector_columns": { + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/vectorsearch.EmbeddingVectorColumn" + }, + "schema_json": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + "vectorsearch.EmbeddingSourceColumn": { + "type": "object", + "properties": { + "embedding_model_endpoint_name": { + "$ref": "#/$defs/string" + }, + "model_endpoint_name_for_query": { + "$ref": "#/$defs/string" + }, + "name": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + "vectorsearch.EmbeddingVectorColumn": { + "type": "object", + "properties": { + "embedding_dimension": { + "$ref": "#/$defs/int" + }, + "name": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, "vectorsearch.EndpointType": { "type": "string", "description": "Type of endpoint.", @@ -9586,6 +9688,12 @@ "STANDARD_ON_ORION" ] }, + "vectorsearch.PipelineType": { + "type": "string" + }, + "vectorsearch.VectorIndexType": { + "type": "string" + }, "workspace.AzureKeyVaultSecretScopeMetadata": { "type": "object", "description": "The metadata of the Azure KeyVault for a secret scope of type `AZURE_KEYVAULT`", @@ -9770,6 +9878,12 @@ "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.VectorSearchEndpoint" } }, + "resources.VectorSearchIndex": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.VectorSearchIndex" + } + }, "resources.Volume": { "type": "object", "additionalProperties": { @@ -10152,6 +10266,18 @@ "items": { "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/sql.EndpointTagPair" } + }, + "vectorsearch.EmbeddingSourceColumn": { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/vectorsearch.EmbeddingSourceColumn" + } + }, + "vectorsearch.EmbeddingVectorColumn": { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/vectorsearch.EmbeddingVectorColumn" + } } } } diff --git a/bundle/statemgmt/state_load_test.go b/bundle/statemgmt/state_load_test.go index 34c4fa4f5aa..f5bd54c0e3f 100644 --- a/bundle/statemgmt/state_load_test.go +++ b/bundle/statemgmt/state_load_test.go @@ -50,6 +50,7 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { "resources.postgres_branches.test_postgres_branch": {ID: "projects/test-project/branches/main"}, "resources.postgres_endpoints.test_postgres_endpoint": {ID: "projects/test-project/branches/main/endpoints/primary"}, "resources.vector_search_endpoints.test_vector_search_endpoint": {ID: "vs-endpoint-1"}, + "resources.vector_search_indexes.test_vector_search_index": {ID: "vs-index-1"}, } err := StateToBundle(t.Context(), state, &config) assert.NoError(t, err) @@ -121,6 +122,9 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { assert.Equal(t, "vs-endpoint-1", config.Resources.VectorSearchEndpoints["test_vector_search_endpoint"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.VectorSearchEndpoints["test_vector_search_endpoint"].ModifiedStatus) + assert.Equal(t, "vs-index-1", config.Resources.VectorSearchIndexes["test_vector_search_index"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.VectorSearchIndexes["test_vector_search_index"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } @@ -299,6 +303,13 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { }, }, }, + VectorSearchIndexes: map[string]*resources.VectorSearchIndex{ + "test_vector_search_index": { + CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{ + Name: "test_vector_search_index", + }, + }, + }, }, } @@ -377,6 +388,9 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { assert.Equal(t, "", config.Resources.VectorSearchEndpoints["test_vector_search_endpoint"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.VectorSearchEndpoints["test_vector_search_endpoint"].ModifiedStatus) + assert.Equal(t, "", config.Resources.VectorSearchIndexes["test_vector_search_index"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.VectorSearchIndexes["test_vector_search_index"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } @@ -673,6 +687,18 @@ func TestStateToBundleModifiedResources(t *testing.T) { }, }, }, + VectorSearchIndexes: map[string]*resources.VectorSearchIndex{ + "test_vector_search_index": { + CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{ + Name: "test_vector_search_index", + }, + }, + "test_vector_search_index_new": { + CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{ + Name: "test_vector_search_index_new", + }, + }, + }, }, } state := ExportedResourcesMap{ @@ -718,6 +744,8 @@ func TestStateToBundleModifiedResources(t *testing.T) { "resources.postgres_endpoints.test_postgres_endpoint_old": {ID: "projects/test-project/branches/main/endpoints/old"}, "resources.vector_search_endpoints.test_vector_search_endpoint": {ID: "vs-endpoint-1"}, "resources.vector_search_endpoints.test_vector_search_endpoint_old": {ID: "vs-endpoint-old"}, + "resources.vector_search_indexes.test_vector_search_index": {ID: "vs-index-1"}, + "resources.vector_search_indexes.test_vector_search_index_old": {ID: "vs-index-old"}, } err := StateToBundle(t.Context(), state, &config) assert.NoError(t, err) @@ -871,6 +899,13 @@ func TestStateToBundleModifiedResources(t *testing.T) { assert.Equal(t, "", config.Resources.VectorSearchEndpoints["test_vector_search_endpoint_new"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.VectorSearchEndpoints["test_vector_search_endpoint_new"].ModifiedStatus) + assert.Equal(t, "vs-index-1", config.Resources.VectorSearchIndexes["test_vector_search_index"].ID) + assert.Equal(t, "", config.Resources.VectorSearchIndexes["test_vector_search_index"].ModifiedStatus) + assert.Equal(t, "vs-index-old", config.Resources.VectorSearchIndexes["test_vector_search_index_old"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.VectorSearchIndexes["test_vector_search_index_old"].ModifiedStatus) + assert.Equal(t, "", config.Resources.VectorSearchIndexes["test_vector_search_index_new"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.VectorSearchIndexes["test_vector_search_index_new"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 5430c68cbcc..51bacfbf2b4 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -152,6 +152,7 @@ type FakeWorkspace struct { RegisteredModels map[string]catalog.RegisteredModelInfo ServingEndpoints map[string]serving.ServingEndpointDetailed VectorSearchEndpoints map[string]vectorsearch.EndpointInfo + VectorSearchIndexes map[string]vectorsearch.VectorIndex SecretScopes map[string]workspace.SecretScope Secrets map[string]map[string]string // scope -> key -> value @@ -287,6 +288,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { }, ServingEndpoints: map[string]serving.ServingEndpointDetailed{}, VectorSearchEndpoints: map[string]vectorsearch.EndpointInfo{}, + VectorSearchIndexes: map[string]vectorsearch.VectorIndex{}, Repos: map[string]workspace.RepoInfo{}, SecretScopes: map[string]workspace.SecretScope{}, Secrets: map[string]map[string]string{}, diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index d98011fc7ba..d8658964ae6 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -824,6 +824,24 @@ func AddDefaultHandlers(server *Server) { return req.Workspace.VectorSearchEndpointUpdateBudgetPolicy(req, req.Vars["endpoint_name"]) }) + // Vector Search Indexes: + + server.Handle("POST", "/api/2.0/vector-search/indexes", func(req Request) any { + return req.Workspace.VectorSearchIndexCreate(req) + }) + + server.Handle("GET", "/api/2.0/vector-search/indexes", func(req Request) any { + return MapList(req.Workspace, req.Workspace.VectorSearchIndexes, "vector_indexes") + }) + + server.Handle("GET", "/api/2.0/vector-search/indexes/{index_name}", func(req Request) any { + return MapGet(req.Workspace, req.Workspace.VectorSearchIndexes, req.Vars["index_name"]) + }) + + server.Handle("DELETE", "/api/2.0/vector-search/indexes/{index_name}", func(req Request) any { + return MapDelete(req.Workspace, req.Workspace.VectorSearchIndexes, req.Vars["index_name"]) + }) + // Generic permissions endpoints server.Handle("GET", "/api/2.0/permissions/{object_type}/{object_id}", func(req Request) any { return req.Workspace.GetPermissions(req) diff --git a/libs/testserver/vector_search_indexes.go b/libs/testserver/vector_search_indexes.go new file mode 100644 index 00000000000..ba07cbb0a3c --- /dev/null +++ b/libs/testserver/vector_search_indexes.go @@ -0,0 +1,61 @@ +package testserver + +import ( + "encoding/json" + "fmt" + "net/http" + + "github.com/databricks/databricks-sdk-go/service/vectorsearch" +) + +func (s *FakeWorkspace) VectorSearchIndexCreate(req Request) Response { + defer s.LockUnlock()() + + var createReq vectorsearch.CreateVectorIndexRequest + if err := json.Unmarshal(req.Body, &createReq); err != nil { + return Response{ + Body: fmt.Sprintf("cannot unmarshal request body: %s", err), + StatusCode: http.StatusBadRequest, + } + } + + if _, exists := s.VectorSearchIndexes[createReq.Name]; exists { + return Response{ + StatusCode: http.StatusConflict, + Body: map[string]string{"error_code": "RESOURCE_ALREADY_EXISTS", "message": fmt.Sprintf("Vector search index with name %s already exists", createReq.Name)}, + } + } + + index := vectorsearch.VectorIndex{ + Creator: s.CurrentUser().UserName, + EndpointName: createReq.EndpointName, + IndexType: createReq.IndexType, + Name: createReq.Name, + PrimaryKey: createReq.PrimaryKey, + DeltaSyncIndexSpec: remapDeltaSyncSpec(createReq.DeltaSyncIndexSpec), + DirectAccessIndexSpec: createReq.DirectAccessIndexSpec, + Status: &vectorsearch.VectorIndexStatus{ + Ready: true, + }, + } + + s.VectorSearchIndexes[createReq.Name] = index + + return Response{ + Body: index, + } +} + +// remapDeltaSyncSpec converts a request spec to a response spec. +func remapDeltaSyncSpec(req *vectorsearch.DeltaSyncVectorIndexSpecRequest) *vectorsearch.DeltaSyncVectorIndexSpecResponse { + if req == nil { + return nil + } + return &vectorsearch.DeltaSyncVectorIndexSpecResponse{ + EmbeddingSourceColumns: req.EmbeddingSourceColumns, + EmbeddingVectorColumns: req.EmbeddingVectorColumns, + EmbeddingWritebackTable: req.EmbeddingWritebackTable, + PipelineType: req.PipelineType, + SourceTable: req.SourceTable, + } +} From c2d97f59bb4037d53bc14dfa5984e28cfba5c0d8 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Fri, 17 Apr 2026 11:17:42 +0200 Subject: [PATCH 02/47] Fixes --- acceptance/bundle/refschema/out.fields.txt | 6 +++ .../vector_search_indexes/basic/output.txt | 9 ++++ .../vector_search_indexes/basic/script | 4 ++ .../drift/columns_to_sync/databricks.yml.tmpl | 19 ++++++++ .../drift/columns_to_sync/out.test.toml | 5 ++ .../drift/columns_to_sync/output.txt | 37 +++++++++++++++ .../drift/columns_to_sync/script | 19 ++++++++ .../drift/columns_to_sync/test.toml | 1 + .../grants/select/databricks.yml.tmpl | 19 ++++++++ .../grants/select/out.requests.direct.json | 30 ++++++++++++ .../grants/select/out.test.toml | 6 +++ .../grants/select/output.txt | 47 +++++++++++++++++++ .../grants/select/script | 23 +++++++++ .../grants/select/test.toml | 2 + .../recreate/index_type/output.txt | 9 ++++ .../recreate/index_type/script | 3 ++ bundle/config/mutator/initialize_urls_test.go | 37 +++++++++++---- .../mutator/resourcemutator/merge_grants.go | 1 + .../resourcemutator/resource_mutator.go | 2 +- .../config/resources/vector_search_index.go | 17 ++++++- bundle/direct/dresources/all.go | 11 +++-- bundle/direct/dresources/all_test.go | 29 +++++++++++- bundle/direct/dresources/grants.go | 11 +++-- bundle/direct/dresources/resources.yml | 4 ++ bundle/internal/schema/annotations.yml | 3 ++ .../validation/generated/enum_fields.go | 1 + bundle/schema/jsonschema.json | 3 ++ bundle/schema/jsonschema_for_docs.json | 3 ++ libs/testserver/vector_search_indexes.go | 9 ++++ 29 files changed, 346 insertions(+), 24 deletions(-) create mode 100644 acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/out.test.toml create mode 100644 acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/output.txt create mode 100644 acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/script create mode 100644 acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/test.toml create mode 100644 acceptance/bundle/resources/vector_search_indexes/grants/select/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/vector_search_indexes/grants/select/out.requests.direct.json create mode 100644 acceptance/bundle/resources/vector_search_indexes/grants/select/out.test.toml create mode 100644 acceptance/bundle/resources/vector_search_indexes/grants/select/output.txt create mode 100644 acceptance/bundle/resources/vector_search_indexes/grants/select/script create mode 100644 acceptance/bundle/resources/vector_search_indexes/grants/select/test.toml diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 978832c9ac1..eade09c997a 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -3200,6 +3200,12 @@ resources.vector_search_indexes.*.status.indexed_row_count int64 REMOTE resources.vector_search_indexes.*.status.message string REMOTE resources.vector_search_indexes.*.status.ready bool REMOTE resources.vector_search_indexes.*.url string INPUT +resources.vector_search_indexes.*.grants.full_name string ALL +resources.vector_search_indexes.*.grants.securable_type string ALL +resources.vector_search_indexes.*.grants[*] catalog.PrivilegeAssignment ALL +resources.vector_search_indexes.*.grants[*].principal string ALL +resources.vector_search_indexes.*.grants[*].privileges []catalog.Privilege ALL +resources.vector_search_indexes.*.grants[*].privileges[*] catalog.Privilege ALL resources.volumes.*.access_point string REMOTE resources.volumes.*.browse_only bool REMOTE resources.volumes.*.catalog_name string ALL diff --git a/acceptance/bundle/resources/vector_search_indexes/basic/output.txt b/acceptance/bundle/resources/vector_search_indexes/basic/output.txt index 0e79a0c9a79..cc72844f851 100644 --- a/acceptance/bundle/resources/vector_search_indexes/basic/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/basic/output.txt @@ -20,6 +20,13 @@ Resources: Name: vs-index-[UNIQUE_NAME] URL: (not deployed) +>>> [CLI] vector-search-endpoints create-endpoint vs-endpoint-[UNIQUE_NAME] STANDARD +{ + "id": "[UUID]", + "name": "vs-endpoint-[UNIQUE_NAME]", + "endpoint_type": "STANDARD" +} + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-vs-index-[UNIQUE_NAME]/default/files... Deploying resources... @@ -56,3 +63,5 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! + +>>> [CLI] vector-search-endpoints delete-endpoint vs-endpoint-[UNIQUE_NAME] diff --git a/acceptance/bundle/resources/vector_search_indexes/basic/script b/acceptance/bundle/resources/vector_search_indexes/basic/script index 54304337425..3a1c5b8a1f8 100644 --- a/acceptance/bundle/resources/vector_search_indexes/basic/script +++ b/acceptance/bundle/resources/vector_search_indexes/basic/script @@ -1,7 +1,9 @@ envsubst < databricks.yml.tmpl > databricks.yml +endpoint_name="vs-endpoint-${UNIQUE_NAME}" cleanup() { trace $CLI bundle destroy --auto-approve + trace $CLI vector-search-endpoints delete-endpoint "${endpoint_name}" rm -f out.requests.txt } trap cleanup EXIT @@ -10,6 +12,8 @@ trace $CLI bundle validate trace $CLI bundle summary +trace $CLI vector-search-endpoints create-endpoint "${endpoint_name}" STANDARD | jq '{id, name, endpoint_type}' + rm -f out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/databricks.yml.tmpl b/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/databricks.yml.tmpl new file mode 100644 index 00000000000..93bedcdafe9 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/databricks.yml.tmpl @@ -0,0 +1,19 @@ +bundle: + name: drift-vs-index-columns-$UNIQUE_NAME + +sync: + paths: [] + +resources: + vector_search_indexes: + my_index: + name: vs-index-$UNIQUE_NAME + endpoint_name: vs-endpoint-$UNIQUE_NAME + primary_key: id + index_type: DELTA_SYNC + delta_sync_index_spec: + source_table: main.default.source_$UNIQUE_NAME + pipeline_type: TRIGGERED + columns_to_sync: + - id + - text diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/out.test.toml new file mode 100644 index 00000000000..19b2c349a32 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/output.txt b/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/output.txt new file mode 100644 index 00000000000..0d88414c395 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/output.txt @@ -0,0 +1,37 @@ + +=== Initial deployment +>>> [CLI] vector-search-endpoints create-endpoint vs-endpoint-[UNIQUE_NAME] STANDARD +{ + "id": "[UUID]", + "name": "vs-endpoint-[UNIQUE_NAME]", + "endpoint_type": "STANDARD" +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/drift-vs-index-columns-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Plan ignores request-only columns_to_sync drift +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> [CLI] vector-search-indexes get-index vs-index-[UNIQUE_NAME] +{ + "name": "vs-index-[UNIQUE_NAME]", + "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", + "index_type": "DELTA_SYNC", + "primary_key": "id" +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.vector_search_indexes.my_index + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/drift-vs-index-columns-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! + +>>> [CLI] vector-search-endpoints delete-endpoint vs-endpoint-[UNIQUE_NAME] diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/script b/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/script new file mode 100644 index 00000000000..a6872a6b6a3 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/script @@ -0,0 +1,19 @@ +envsubst < databricks.yml.tmpl > databricks.yml +endpoint_name="vs-endpoint-${UNIQUE_NAME}" + +cleanup() { + trace $CLI bundle destroy --auto-approve + trace $CLI vector-search-endpoints delete-endpoint "${endpoint_name}" + rm -f out.requests.txt +} +trap cleanup EXIT + +title "Initial deployment" +trace $CLI vector-search-endpoints create-endpoint "${endpoint_name}" STANDARD | jq '{id, name, endpoint_type}' +trace $CLI bundle deploy + +title "Plan ignores request-only columns_to_sync drift" +trace $CLI bundle plan + +index_name="vs-index-${UNIQUE_NAME}" +trace $CLI vector-search-indexes get-index "${index_name}" | jq '{name, endpoint_name, index_type, primary_key}' diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/test.toml b/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/test.toml new file mode 100644 index 00000000000..f8b3bbe49dd --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/test.toml @@ -0,0 +1 @@ +# All configuration inherited from parent test.toml diff --git a/acceptance/bundle/resources/vector_search_indexes/grants/select/databricks.yml.tmpl b/acceptance/bundle/resources/vector_search_indexes/grants/select/databricks.yml.tmpl new file mode 100644 index 00000000000..db6db5792c9 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/grants/select/databricks.yml.tmpl @@ -0,0 +1,19 @@ +bundle: + name: vs-index-grants-$UNIQUE_NAME + +sync: + paths: [] + +resources: + vector_search_indexes: + my_index: + name: main.default.vs_index_$UNIQUE_NAME + endpoint_name: vs-endpoint-$UNIQUE_NAME + primary_key: id + index_type: DIRECT_ACCESS + direct_access_index_spec: + schema_json: '{"columns":[{"name":"id","type":"integer"}]}' + grants: + - principal: deco-test-user@databricks.com + privileges: + - SELECT diff --git a/acceptance/bundle/resources/vector_search_indexes/grants/select/out.requests.direct.json b/acceptance/bundle/resources/vector_search_indexes/grants/select/out.requests.direct.json new file mode 100644 index 00000000000..ecfc6b933aa --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/grants/select/out.requests.direct.json @@ -0,0 +1,30 @@ +{ + "method": "POST", + "path": "/api/2.0/vector-search/indexes", + "body": { + "direct_access_index_spec": { + "schema_json": "{\"columns\":[{\"name\":\"id\",\"type\":\"integer\"}]}" + }, + "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", + "index_type": "DIRECT_ACCESS", + "name": "main.default.vs_index_[UNIQUE_NAME]", + "primary_key": "id" + } +} +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/permissions/table/main.default.vs_index_[UNIQUE_NAME]", + "body": { + "changes": [ + { + "add": [ + "SELECT" + ], + "principal": "deco-test-user@databricks.com", + "remove": [ + "ALL_PRIVILEGES" + ] + } + ] + } +} diff --git a/acceptance/bundle/resources/vector_search_indexes/grants/select/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/grants/select/out.test.toml new file mode 100644 index 00000000000..f1d40380d02 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/grants/select/out.test.toml @@ -0,0 +1,6 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/grants/select/output.txt b/acceptance/bundle/resources/vector_search_indexes/grants/select/output.txt new file mode 100644 index 00000000000..b0003c95f0b --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/grants/select/output.txt @@ -0,0 +1,47 @@ + +>>> [CLI] vector-search-endpoints create-endpoint vs-endpoint-[UNIQUE_NAME] STANDARD +{ + "id": "[UUID]", + "name": "vs-endpoint-[UNIQUE_NAME]", + "endpoint_type": "STANDARD" +} + +>>> [CLI] bundle plan +create vector_search_indexes.my_index +create vector_search_indexes.my_index.grants + +Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/vs-index-grants-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] grants get table main.default.vs_index_[UNIQUE_NAME] +{ + "privilege_assignments": [ + { + "principal": "deco-test-user@databricks.com", + "privileges": [ + "SELECT" + ] + } + ] +} + +>>> print_requests.py //vector-search/indexes //permissions + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.vector_search_indexes.my_index + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/vs-index-grants-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! + +>>> [CLI] vector-search-endpoints delete-endpoint vs-endpoint-[UNIQUE_NAME] diff --git a/acceptance/bundle/resources/vector_search_indexes/grants/select/script b/acceptance/bundle/resources/vector_search_indexes/grants/select/script new file mode 100644 index 00000000000..6aac790b72d --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/grants/select/script @@ -0,0 +1,23 @@ +envsubst < databricks.yml.tmpl > databricks.yml +endpoint_name="vs-endpoint-${UNIQUE_NAME}" +index_name="main.default.vs_index_${UNIQUE_NAME}" + +cleanup() { + trace $CLI bundle destroy --auto-approve + trace $CLI vector-search-endpoints delete-endpoint "${endpoint_name}" + rm -f out.requests.txt +} +trap cleanup EXIT + +trace $CLI vector-search-endpoints create-endpoint "${endpoint_name}" STANDARD | jq '{id, name, endpoint_type}' + +trace $CLI bundle plan + +rm -f out.requests.txt +trace $CLI bundle deploy + +trace $CLI bundle plan + +trace $CLI grants get table "${index_name}" | jq --sort-keys + +trace print_requests.py '//vector-search/indexes' '//permissions' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/vector_search_indexes/grants/select/test.toml b/acceptance/bundle/resources/vector_search_indexes/grants/select/test.toml new file mode 100644 index 00000000000..0b0ea585e09 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/grants/select/test.toml @@ -0,0 +1,2 @@ +RequiresUnityCatalog = true +RecordRequests = true diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/output.txt b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/output.txt index 81ab2fa8669..143252aed92 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/output.txt @@ -1,5 +1,12 @@ === Initial deployment with DELTA_SYNC index_type +>>> [CLI] vector-search-endpoints create-endpoint vs-endpoint-[UNIQUE_NAME] STANDARD +{ + "id": "[UUID]", + "name": "vs-endpoint-[UNIQUE_NAME]", + "endpoint_type": "STANDARD" +} + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-vs-index-[UNIQUE_NAME]/default/files... Deploying resources... @@ -46,3 +53,5 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! + +>>> [CLI] vector-search-endpoints delete-endpoint vs-endpoint-[UNIQUE_NAME] diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/script b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/script index be654a837cf..2e463ab45bf 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/script +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/script @@ -1,7 +1,9 @@ envsubst < databricks.yml.tmpl > databricks.yml +endpoint_name="vs-endpoint-${UNIQUE_NAME}" cleanup() { trace $CLI bundle destroy --auto-approve + trace $CLI vector-search-endpoints delete-endpoint "${endpoint_name}" rm -f out.requests.txt } trap cleanup EXIT @@ -13,6 +15,7 @@ print_requests() { } title "Initial deployment with DELTA_SYNC index_type" +trace $CLI vector-search-endpoints create-endpoint "${endpoint_name}" STANDARD | jq '{id, name, endpoint_type}' rm -f out.requests.txt trace $CLI bundle deploy diff --git a/bundle/config/mutator/initialize_urls_test.go b/bundle/config/mutator/initialize_urls_test.go index 6b0c8d6655e..da26e02ce40 100644 --- a/bundle/config/mutator/initialize_urls_test.go +++ b/bundle/config/mutator/initialize_urls_test.go @@ -12,6 +12,7 @@ import ( "github.com/databricks/databricks-sdk-go/service/ml" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/databricks/databricks-sdk-go/service/serving" + "github.com/databricks/databricks-sdk-go/service/vectorsearch" "github.com/stretchr/testify/require" ) @@ -68,6 +69,20 @@ func TestInitializeURLs(t *testing.T) { CreateMonitor: catalog.CreateMonitor{}, }, }, + VectorSearchIndexes: map[string]*resources.VectorSearchIndex{ + "vectorsearchindex1": { + BaseResource: resources.BaseResource{ID: "catalog.schema.vectorsearchindex1"}, + CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{ + Name: "catalog.schema.vectorsearchindex1", + }, + }, + "vectorsearchindex2": { + BaseResource: resources.BaseResource{ID: "vectorsearchindex2"}, + CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{ + Name: "vectorsearchindex2", + }, + }, + }, Schemas: map[string]*resources.Schema{ "schema1": { BaseResource: resources.BaseResource{ID: "catalog.schema"}, @@ -97,16 +112,18 @@ func TestInitializeURLs(t *testing.T) { } expectedURLs := map[string]string{ - "job1": "https://mycompany.databricks.com/jobs/1?o=123456", - "pipeline1": "https://mycompany.databricks.com/pipelines/3?o=123456", - "experiment1": "https://mycompany.databricks.com/ml/experiments/4?o=123456", - "model1": "https://mycompany.databricks.com/ml/models/a%20model%20uses%20its%20name%20for%20identifier?o=123456", - "servingendpoint1": "https://mycompany.databricks.com/ml/endpoints/my_serving_endpoint?o=123456", - "registeredmodel1": "https://mycompany.databricks.com/explore/data/models/8?o=123456", - "qualityMonitor1": "https://mycompany.databricks.com/explore/data/catalog/schema/qualityMonitor1?o=123456", - "schema1": "https://mycompany.databricks.com/explore/data/catalog/schema?o=123456", - "cluster1": "https://mycompany.databricks.com/compute/clusters/1017-103929-vlr7jzcf?o=123456", - "dashboard1": "https://mycompany.databricks.com/dashboardsv3/01ef8d56871e1d50ae30ce7375e42478/published?o=123456", + "job1": "https://mycompany.databricks.com/jobs/1?o=123456", + "pipeline1": "https://mycompany.databricks.com/pipelines/3?o=123456", + "experiment1": "https://mycompany.databricks.com/ml/experiments/4?o=123456", + "model1": "https://mycompany.databricks.com/ml/models/a%20model%20uses%20its%20name%20for%20identifier?o=123456", + "servingendpoint1": "https://mycompany.databricks.com/ml/endpoints/my_serving_endpoint?o=123456", + "registeredmodel1": "https://mycompany.databricks.com/explore/data/models/8?o=123456", + "qualityMonitor1": "https://mycompany.databricks.com/explore/data/catalog/schema/qualityMonitor1?o=123456", + "vectorsearchindex1": "https://mycompany.databricks.com/explore/data/catalog/schema/vectorsearchindex1?o=123456", + "vectorsearchindex2": "", + "schema1": "https://mycompany.databricks.com/explore/data/catalog/schema?o=123456", + "cluster1": "https://mycompany.databricks.com/compute/clusters/1017-103929-vlr7jzcf?o=123456", + "dashboard1": "https://mycompany.databricks.com/dashboardsv3/01ef8d56871e1d50ae30ce7375e42478/published?o=123456", } err := initializeForWorkspace(b, "123456", "https://mycompany.databricks.com/") diff --git a/bundle/config/mutator/resourcemutator/merge_grants.go b/bundle/config/mutator/resourcemutator/merge_grants.go index 1f8d033747c..a90e9a57507 100644 --- a/bundle/config/mutator/resourcemutator/merge_grants.go +++ b/bundle/config/mutator/resourcemutator/merge_grants.go @@ -16,6 +16,7 @@ var grantResourceTypes = []string{ "external_locations", "volumes", "registered_models", + "vector_search_indexes", } type mergeGrants struct{} diff --git a/bundle/config/mutator/resourcemutator/resource_mutator.go b/bundle/config/mutator/resourcemutator/resource_mutator.go index 2eb292cfbb0..209bbcb06a0 100644 --- a/bundle/config/mutator/resourcemutator/resource_mutator.go +++ b/bundle/config/mutator/resourcemutator/resource_mutator.go @@ -167,7 +167,7 @@ func applyNormalizeMutators(ctx context.Context, b *bundle.Bundle) { // Updates (dynamic): resources.apps.*.resources (merges app resources with the same name) MergeApps(), - // Reads (dynamic): resources.{catalogs,schemas,external_locations,volumes,registered_models}.*.grants + // Reads (dynamic): resources.{catalogs,schemas,external_locations,volumes,registered_models,vector_search_indexes}.*.grants // Updates (dynamic): same paths — merges grant entries by principal and deduplicates privileges MergeGrants(), diff --git a/bundle/config/resources/vector_search_index.go b/bundle/config/resources/vector_search_index.go index 5aa41e3108c..0cddada900b 100644 --- a/bundle/config/resources/vector_search_index.go +++ b/bundle/config/resources/vector_search_index.go @@ -3,17 +3,22 @@ package resources import ( "context" "net/url" + "strings" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/service/vectorsearch" ) type VectorSearchIndex struct { BaseResource vectorsearch.CreateVectorIndexRequest + + // List of grants to apply on this vector search index. + Grants []catalog.PrivilegeAssignment `json:"grants,omitempty"` } func (e *VectorSearchIndex) UnmarshalJSON(b []byte) error { @@ -46,10 +51,18 @@ func (e *VectorSearchIndex) ResourceDescription() ResourceDescription { } func (e *VectorSearchIndex) InitializeURL(baseURL url.URL) { - if e.ID == "" { + if e.Name == "" { + return + } + catalog, rest, ok := strings.Cut(e.Name, ".") + if !ok { + return + } + schema, name, ok := strings.Cut(rest, ".") + if !ok { return } - baseURL.Path = "compute/vector-search/indexes/" + e.Name + baseURL.Path = "explore/data/" + catalog + "/" + schema + "/" + name e.URL = baseURL.String() } diff --git a/bundle/direct/dresources/all.go b/bundle/direct/dresources/all.go index 4001064ab49..698b9eefc3e 100644 --- a/bundle/direct/dresources/all.go +++ b/bundle/direct/dresources/all.go @@ -50,11 +50,12 @@ var SupportedResources = map[string]any{ "vector_search_endpoints.permissions": (*ResourcePermissions)(nil), // Grants - "catalogs.grants": (*ResourceGrants)(nil), - "schemas.grants": (*ResourceGrants)(nil), - "external_locations.grants": (*ResourceGrants)(nil), - "volumes.grants": (*ResourceGrants)(nil), - "registered_models.grants": (*ResourceGrants)(nil), + "catalogs.grants": (*ResourceGrants)(nil), + "schemas.grants": (*ResourceGrants)(nil), + "external_locations.grants": (*ResourceGrants)(nil), + "volumes.grants": (*ResourceGrants)(nil), + "registered_models.grants": (*ResourceGrants)(nil), + "vector_search_indexes.grants": (*ResourceGrants)(nil), } func InitAll(client *databricks.WorkspaceClient) (map[string]*Adapter, error) { diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 6cf300e3f3b..43b8392c8be 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -252,7 +252,7 @@ var testConfig map[string]any = map[string]any{ "vector_search_indexes": &resources.VectorSearchIndex{ CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{ Name: "my-index", - EndpointName: "my-endpoint", + EndpointName: "my-index-endpoint", PrimaryKey: "id", IndexType: vectorsearch.VectorIndexTypeDeltaSync, DeltaSyncIndexSpec: &vectorsearch.DeltaSyncVectorIndexSpecRequest{ @@ -260,6 +260,10 @@ var testConfig map[string]any = map[string]any{ PipelineType: vectorsearch.PipelineTypeTriggered, }, }, + Grants: []catalog.PrivilegeAssignment{{ + Principal: "user@example.com", + Privileges: []catalog.Privilege{catalog.PrivilegeSelect}, + }}, }, } @@ -282,6 +286,18 @@ var testDeps = map[string]prepareWorkspace{ }, err }, + "vector_search_indexes": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { + _, err := client.VectorSearchEndpoints.CreateEndpoint(ctx, vectorsearch.CreateEndpoint{ + Name: "my-index-endpoint", + EndpointType: vectorsearch.EndpointTypeStandard, + }) + if err != nil { + return nil, err + } + + return testConfig["vector_search_indexes"], nil + }, + "jobs.permissions": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { resp, err := client.Jobs.Create(ctx, jobs.CreateJob{ Name: "job-permissions", @@ -599,6 +615,17 @@ var testDeps = map[string]prepareWorkspace{ }, nil }, + "vector_search_indexes.grants": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { + return &GrantsState{ + SecurableType: "table", + FullName: "main.default.my_index", + EmbeddedSlice: []catalog.PrivilegeAssignment{{ + Privileges: []catalog.Privilege{catalog.PrivilegeSelect}, + Principal: "user@example.com", + }}, + }, nil + }, + "secret_scopes.permissions": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { err := client.Secrets.CreateScope(ctx, workspace.CreateScope{ Scope: "permissions_test_scope", diff --git a/bundle/direct/dresources/grants.go b/bundle/direct/dresources/grants.go index 8bb19061224..360da0a9ab1 100644 --- a/bundle/direct/dresources/grants.go +++ b/bundle/direct/dresources/grants.go @@ -13,11 +13,12 @@ import ( ) var grantResourceToSecurableType = map[string]string{ - "catalogs": "catalog", - "schemas": "schema", - "external_locations": "external_location", - "volumes": "volume", - "registered_models": "function", + "catalogs": "catalog", + "schemas": "schema", + "external_locations": "external_location", + "volumes": "volume", + "registered_models": "function", + "vector_search_indexes": "table", } type GrantsState struct { diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index b4858d89a84..7cf21503f1e 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -530,3 +530,7 @@ resources: reason: immutable - field: direct_access_index_spec reason: immutable + ignore_remote_changes: + # columns_to_sync is request-only in the create spec and not returned by the read API. + - field: delta_sync_index_spec.columns_to_sync + reason: input_only diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index b60c5ae8d8e..faa6a4f0cbb 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -997,6 +997,9 @@ github.com/databricks/cli/bundle/config/resources.VectorSearchIndex: "endpoint_name": "description": |- PLACEHOLDER + "grants": + "description": |- + PLACEHOLDER "index_type": "description": |- PLACEHOLDER diff --git a/bundle/internal/validation/generated/enum_fields.go b/bundle/internal/validation/generated/enum_fields.go index 58568960a6b..ebe7815f41f 100644 --- a/bundle/internal/validation/generated/enum_fields.go +++ b/bundle/internal/validation/generated/enum_fields.go @@ -210,6 +210,7 @@ var EnumFields = map[string][]string{ "resources.vector_search_endpoints.*.permissions[*].level": {"CAN_ATTACH_TO", "CAN_BIND", "CAN_CREATE", "CAN_CREATE_APP", "CAN_EDIT", "CAN_EDIT_METADATA", "CAN_MANAGE", "CAN_MANAGE_PRODUCTION_VERSIONS", "CAN_MANAGE_RUN", "CAN_MANAGE_STAGING_VERSIONS", "CAN_MONITOR", "CAN_MONITOR_ONLY", "CAN_QUERY", "CAN_READ", "CAN_RESTART", "CAN_RUN", "CAN_USE", "CAN_VIEW", "CAN_VIEW_METADATA", "IS_OWNER"}, "resources.vector_search_indexes.*.delta_sync_index_spec.pipeline_type": {"CONTINUOUS", "TRIGGERED"}, + "resources.vector_search_indexes.*.grants[*].privileges[*]": {"ACCESS", "ALL_PRIVILEGES", "APPLY_TAG", "BROWSE", "CREATE", "CREATE_CATALOG", "CREATE_CLEAN_ROOM", "CREATE_CONNECTION", "CREATE_EXTERNAL_LOCATION", "CREATE_EXTERNAL_TABLE", "CREATE_EXTERNAL_VOLUME", "CREATE_FOREIGN_CATALOG", "CREATE_FOREIGN_SECURABLE", "CREATE_FUNCTION", "CREATE_MANAGED_STORAGE", "CREATE_MATERIALIZED_VIEW", "CREATE_MODEL", "CREATE_PROVIDER", "CREATE_RECIPIENT", "CREATE_SCHEMA", "CREATE_SERVICE_CREDENTIAL", "CREATE_SHARE", "CREATE_STORAGE_CREDENTIAL", "CREATE_TABLE", "CREATE_VIEW", "CREATE_VOLUME", "EXECUTE", "EXECUTE_CLEAN_ROOM_TASK", "EXTERNAL_USE_SCHEMA", "MANAGE", "MANAGE_ALLOWLIST", "MODIFY", "MODIFY_CLEAN_ROOM", "READ_FILES", "READ_PRIVATE_FILES", "READ_VOLUME", "REFRESH", "SELECT", "SET_SHARE_PERMISSION", "USAGE", "USE_CATALOG", "USE_CONNECTION", "USE_MARKETPLACE_ASSETS", "USE_PROVIDER", "USE_RECIPIENT", "USE_SCHEMA", "USE_SHARE", "WRITE_FILES", "WRITE_PRIVATE_FILES", "WRITE_VOLUME"}, "resources.vector_search_indexes.*.index_type": {"DELTA_SYNC", "DIRECT_ACCESS"}, "resources.volumes.*.grants[*].privileges[*]": {"ACCESS", "ALL_PRIVILEGES", "APPLY_TAG", "BROWSE", "CREATE", "CREATE_CATALOG", "CREATE_CLEAN_ROOM", "CREATE_CONNECTION", "CREATE_EXTERNAL_LOCATION", "CREATE_EXTERNAL_TABLE", "CREATE_EXTERNAL_VOLUME", "CREATE_FOREIGN_CATALOG", "CREATE_FOREIGN_SECURABLE", "CREATE_FUNCTION", "CREATE_MANAGED_STORAGE", "CREATE_MATERIALIZED_VIEW", "CREATE_MODEL", "CREATE_PROVIDER", "CREATE_RECIPIENT", "CREATE_SCHEMA", "CREATE_SERVICE_CREDENTIAL", "CREATE_SHARE", "CREATE_STORAGE_CREDENTIAL", "CREATE_TABLE", "CREATE_VIEW", "CREATE_VOLUME", "EXECUTE", "EXECUTE_CLEAN_ROOM_TASK", "EXTERNAL_USE_SCHEMA", "MANAGE", "MANAGE_ALLOWLIST", "MODIFY", "MODIFY_CLEAN_ROOM", "READ_FILES", "READ_PRIVATE_FILES", "READ_VOLUME", "REFRESH", "SELECT", "SET_SHARE_PERMISSION", "USAGE", "USE_CATALOG", "USE_CONNECTION", "USE_MARKETPLACE_ASSETS", "USE_PROVIDER", "USE_RECIPIENT", "USE_SCHEMA", "USE_SHARE", "WRITE_FILES", "WRITE_PRIVATE_FILES", "WRITE_VOLUME"}, diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index a7ee544313a..b6a87536321 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1985,6 +1985,9 @@ "endpoint_name": { "$ref": "#/$defs/string" }, + "grants": { + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/catalog.PrivilegeAssignment" + }, "index_type": { "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/vectorsearch.VectorIndexType" }, diff --git a/bundle/schema/jsonschema_for_docs.json b/bundle/schema/jsonschema_for_docs.json index 05695da1296..335c9b31f35 100644 --- a/bundle/schema/jsonschema_for_docs.json +++ b/bundle/schema/jsonschema_for_docs.json @@ -1969,6 +1969,9 @@ "endpoint_name": { "$ref": "#/$defs/string" }, + "grants": { + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/catalog.PrivilegeAssignment" + }, "index_type": { "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/vectorsearch.VectorIndexType" }, diff --git a/libs/testserver/vector_search_indexes.go b/libs/testserver/vector_search_indexes.go index ba07cbb0a3c..a2a7848ad94 100644 --- a/libs/testserver/vector_search_indexes.go +++ b/libs/testserver/vector_search_indexes.go @@ -25,6 +25,15 @@ func (s *FakeWorkspace) VectorSearchIndexCreate(req Request) Response { Body: map[string]string{"error_code": "RESOURCE_ALREADY_EXISTS", "message": fmt.Sprintf("Vector search index with name %s already exists", createReq.Name)}, } } + if _, exists := s.VectorSearchEndpoints[createReq.EndpointName]; !exists { + return Response{ + StatusCode: http.StatusNotFound, + Body: map[string]string{ + "error_code": "RESOURCE_DOES_NOT_EXIST", + "message": fmt.Sprintf("Vector search endpoint %s not found", createReq.EndpointName), + }, + } + } index := vectorsearch.VectorIndex{ Creator: s.CurrentUser().UserName, From 3a09e2333d929a23e57e1b831ef8e62bfd5c88b6 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 22 Apr 2026 15:11:05 +0200 Subject: [PATCH 03/47] acceptance: fix stale vector-search tests from earlier vs-index work - invariant vector_search_index config now creates the required endpoint in-bundle so the testserver's endpoint-existence check passes - drift/budget_policy now asserts that the remote drift is ignored (matches ignore_remote_changes for budget_policy_id: effective value may include inherited workspace policy, not user-set) - vector_search_indexes/basic URL updated to match the current catalog.schema.name-based InitializeURL behavior Co-authored-by: Isaac --- .../bundle/invariant/configs/vector_search_index.yml.tmpl | 6 +++++- .../vector_search_endpoints/drift/budget_policy/output.txt | 6 ++---- .../vector_search_endpoints/drift/budget_policy/script | 4 ++-- .../bundle/resources/vector_search_indexes/basic/output.txt | 2 +- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/acceptance/bundle/invariant/configs/vector_search_index.yml.tmpl b/acceptance/bundle/invariant/configs/vector_search_index.yml.tmpl index 172d41d5f3f..bafb1a3cd15 100644 --- a/acceptance/bundle/invariant/configs/vector_search_index.yml.tmpl +++ b/acceptance/bundle/invariant/configs/vector_search_index.yml.tmpl @@ -2,10 +2,14 @@ bundle: name: test-bundle-$UNIQUE_NAME resources: + vector_search_endpoints: + bar: + name: test-endpoint-$UNIQUE_NAME + endpoint_type: STANDARD vector_search_indexes: foo: name: test-index-$UNIQUE_NAME - endpoint_name: test-endpoint-$UNIQUE_NAME + endpoint_name: ${resources.vector_search_endpoints.bar.name} primary_key: id index_type: DELTA_SYNC delta_sync_index_spec: diff --git a/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy/output.txt b/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy/output.txt index 45555a83ff1..eab9faca1b0 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy/output.txt +++ b/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy/output.txt @@ -12,11 +12,9 @@ Deployment complete! "effective_budget_policy_id": "remote-policy" } -=== Plan detects drift and proposes update +=== budget_policy_id drift is ignored (effective vs requested mismatch) >>> [CLI] bundle plan -update vector_search_endpoints.my_endpoint - -Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy/script b/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy/script index c02467d6528..40328247a30 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy/script +++ b/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy/script @@ -14,5 +14,5 @@ endpoint_name="vs-endpoint-${UNIQUE_NAME}" title "Simulate remote drift: set budget_policy_id outside the bundle" trace $CLI vector-search-endpoints update-endpoint-budget-policy "${endpoint_name}" "remote-policy" -title "Plan detects drift and proposes update" -trace $CLI bundle plan | contains.py "Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged" +title "budget_policy_id drift is ignored (effective vs requested mismatch)" +trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" diff --git a/acceptance/bundle/resources/vector_search_indexes/basic/output.txt b/acceptance/bundle/resources/vector_search_indexes/basic/output.txt index cc72844f851..a905d5e9e0a 100644 --- a/acceptance/bundle/resources/vector_search_indexes/basic/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/basic/output.txt @@ -51,7 +51,7 @@ Resources: Vector Search Indexes: my_index: Name: vs-index-[UNIQUE_NAME] - URL: [DATABRICKS_URL]/compute/vector-search/indexes/vs-index-[UNIQUE_NAME]?o=[NUMID] + URL: (not deployed) >>> print_requests.py //vector-search/indexes From 43d6857eea21b2aa33ce05eb91902bad3f3dee72 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 22 Apr 2026 15:12:58 +0200 Subject: [PATCH 04/47] acceptance: add drift/deleted_remotely test for vector_search_indexes Covers the case where an index is deleted outside the bundle: the next plan should propose creation and the next deploy restores it. Mirrors acceptance/bundle/resources/volumes/remote-delete and closes the missing drift coverage for indexes (endpoints have drift/budget_policy and drift/min_qps; indexes had none). Co-authored-by: Isaac --- .../deleted_remotely/databricks.yml.tmpl | 16 +++++++ .../drift/deleted_remotely/out.test.toml | 5 ++ .../drift/deleted_remotely/output.txt | 48 +++++++++++++++++++ .../drift/deleted_remotely/script | 25 ++++++++++ .../drift/deleted_remotely/test.toml | 1 + 5 files changed, 95 insertions(+) create mode 100644 acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/out.test.toml create mode 100644 acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/output.txt create mode 100644 acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/script create mode 100644 acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/test.toml diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/databricks.yml.tmpl b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/databricks.yml.tmpl new file mode 100644 index 00000000000..8e4dfbbcb6c --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/databricks.yml.tmpl @@ -0,0 +1,16 @@ +bundle: + name: drift-vs-index-deleted-$UNIQUE_NAME + +sync: + paths: [] + +resources: + vector_search_indexes: + my_index: + name: vs-index-$UNIQUE_NAME + endpoint_name: vs-endpoint-$UNIQUE_NAME + primary_key: id + index_type: DELTA_SYNC + delta_sync_index_spec: + source_table: main.default.source_$UNIQUE_NAME + pipeline_type: TRIGGERED diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/out.test.toml new file mode 100644 index 00000000000..54146af5645 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/output.txt b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/output.txt new file mode 100644 index 00000000000..4575e349949 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/output.txt @@ -0,0 +1,48 @@ + +>>> [CLI] vector-search-endpoints create-endpoint vs-endpoint-[UNIQUE_NAME] STANDARD +{ + "name": "vs-endpoint-[UNIQUE_NAME]", + "endpoint_type": "STANDARD" +} + +=== Initial deployment +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/drift-vs-index-deleted-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Simulate remote deletion +>>> [CLI] vector-search-indexes delete-index vs-index-[UNIQUE_NAME] + +=== Plan detects missing resource and proposes creation +>>> [CLI] bundle plan +create vector_search_indexes.my_index + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + +=== Deploy recreates the index +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/drift-vs-index-deleted-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] vector-search-indexes get-index vs-index-[UNIQUE_NAME] +{ + "name": "vs-index-[UNIQUE_NAME]", + "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", + "index_type": "DELTA_SYNC", + "primary_key": "id" +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.vector_search_indexes.my_index + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/drift-vs-index-deleted-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! + +>>> [CLI] vector-search-endpoints delete-endpoint vs-endpoint-[UNIQUE_NAME] diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/script b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/script new file mode 100644 index 00000000000..9409de8da4f --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/script @@ -0,0 +1,25 @@ +envsubst < databricks.yml.tmpl > databricks.yml +endpoint_name="vs-endpoint-${UNIQUE_NAME}" +index_name="vs-index-${UNIQUE_NAME}" + +cleanup() { + trace $CLI bundle destroy --auto-approve + trace $CLI vector-search-endpoints delete-endpoint "${endpoint_name}" + rm -f out.requests.txt +} +trap cleanup EXIT + +trace $CLI vector-search-endpoints create-endpoint "${endpoint_name}" STANDARD | jq '{name, endpoint_type}' + +title "Initial deployment" +trace $CLI bundle deploy + +title "Simulate remote deletion" +trace $CLI vector-search-indexes delete-index "${index_name}" + +title "Plan detects missing resource and proposes creation" +trace $CLI bundle plan | contains.py "create vector_search_indexes.my_index" + +title "Deploy recreates the index" +trace $CLI bundle deploy +trace $CLI vector-search-indexes get-index "${index_name}" | jq '{name, endpoint_name, index_type, primary_key}' diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/test.toml b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/test.toml @@ -0,0 +1 @@ +Cloud = false From 4f0982b3ace4db39d267d225da62c061c68ed76f Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 22 Apr 2026 15:26:29 +0200 Subject: [PATCH 05/47] direct: satisfy exhaustruct for VectorSearchEndpointState and CreateVectorIndexRequest make lint surfaces exhaustruct issues on changed packages. Explicitly initialize the zero-value fields introduced by the prior two commits (EndpointUuid on the new state type, IndexSubtype on the existing RemapState literal). Co-authored-by: Isaac --- bundle/direct/dresources/vector_search_index.go | 1 + 1 file changed, 1 insertion(+) diff --git a/bundle/direct/dresources/vector_search_index.go b/bundle/direct/dresources/vector_search_index.go index 1fb8c96a8f1..16cd82a55fe 100644 --- a/bundle/direct/dresources/vector_search_index.go +++ b/bundle/direct/dresources/vector_search_index.go @@ -24,6 +24,7 @@ func (*ResourceVectorSearchIndex) RemapState(remote *vectorsearch.VectorIndex) * req := &vectorsearch.CreateVectorIndexRequest{ DeltaSyncIndexSpec: nil, DirectAccessIndexSpec: nil, + IndexSubtype: "", Name: remote.Name, EndpointName: remote.EndpointName, IndexType: remote.IndexType, From c9bedf1d7bf512b454734df2f130672f45c5eb6d Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 22 Apr 2026 16:01:19 +0200 Subject: [PATCH 06/47] Fix dev-mode name prefixing for vector_search_indexes The name is a 3-part UC identifier (catalog.schema.index) where catalog and schema are external references the bundle does not own. Prefixing the whole name turned "main.default.my_index" into "dev_user_main.default.my_index", which is not a valid catalog. Prefix only the leaf component, matching how references work: "main.default.my_index" -> "main.default.dev_user_my_index". Co-authored-by: Isaac --- bundle/config/mutator/resourcemutator/apply_presets.go | 8 +++++++- .../mutator/resourcemutator/apply_target_mode_test.go | 5 ++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/bundle/config/mutator/resourcemutator/apply_presets.go b/bundle/config/mutator/resourcemutator/apply_presets.go index 4ba26a6e34e..59682f8141c 100644 --- a/bundle/config/mutator/resourcemutator/apply_presets.go +++ b/bundle/config/mutator/resourcemutator/apply_presets.go @@ -295,11 +295,17 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos // the resource's identity rather than just its display name. // Vector Search Indexes: Prefix + // The name is a 3-part UC identifier (catalog.schema.index); prefix only + // the last component since catalog and schema are external references. for _, e := range r.VectorSearchIndexes { if e == nil { continue } - e.Name = normalizePrefix(prefix) + e.Name + if i := strings.LastIndex(e.Name, "."); i >= 0 { + e.Name = e.Name[:i+1] + normalizePrefix(prefix) + e.Name[i+1:] + } else { + e.Name = normalizePrefix(prefix) + e.Name + } } return diags diff --git a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go index 2616051e4fd..84d2645a77a 100644 --- a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go +++ b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go @@ -258,7 +258,7 @@ func mockBundle(mode config.Mode) *bundle.Bundle { VectorSearchIndexes: map[string]*resources.VectorSearchIndex{ "vs_index1": { CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{ - Name: "vs_index1", + Name: "main.default.vs_index1", EndpointName: "vs_endpoint1", PrimaryKey: "id", IndexType: vectorsearch.VectorIndexTypeDeltaSync, @@ -316,6 +316,9 @@ func TestProcessTargetModeDevelopment(t *testing.T) { // Vector search endpoint 1: name is the primary key, so it must not be prefixed. assert.Equal(t, "vs_endpoint1", b.Config.Resources.VectorSearchEndpoints["vs_endpoint1"].Name) + // Vector search index 1: only the leaf name is prefixed, since catalog and schema are external + assert.Equal(t, "main.default.dev_lennart_vs_index1", b.Config.Resources.VectorSearchIndexes["vs_index1"].Name) + // Registered model 1 assert.Equal(t, "dev_lennart_registeredmodel1", b.Config.Resources.RegisteredModels["registeredmodel1"].Name) From 1b4b29d4159af5270a3a3d80f40f64d138002019 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 22 Apr 2026 16:08:51 +0200 Subject: [PATCH 07/47] acceptance: cover vector_search prefix in presets_name_prefix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exercises the fix from the previous commit: for vector_search_indexes.vs_index.name = my_catalog.${resources.schemas.my_schema.name}.my_index only the leaf (my_index) is prefixed, the catalog is preserved as-is, and the schema reference stays intact — it resolves at deploy time to the already-prefixed schema, yielding the expected 3-part name. Co-authored-by: Isaac --- .../presets_name_prefix/databricks.yml.tmpl | 15 ++++++ .../validate/presets_name_prefix/output.txt | 54 +++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/acceptance/bundle/validate/presets_name_prefix/databricks.yml.tmpl b/acceptance/bundle/validate/presets_name_prefix/databricks.yml.tmpl index 36cd2dcad88..74887b548e1 100644 --- a/acceptance/bundle/validate/presets_name_prefix/databricks.yml.tmpl +++ b/acceptance/bundle/validate/presets_name_prefix/databricks.yml.tmpl @@ -13,6 +13,9 @@ resources: schema1: catalog_name: c name: schema1 + my_schema: + catalog_name: my_catalog + name: my_schema volumes: volume1: @@ -24,5 +27,17 @@ resources: model1: name: model 1 + vector_search_endpoints: + vs_endpoint: + name: vs_endpoint + endpoint_type: STANDARD + + vector_search_indexes: + vs_index: + name: my_catalog.${resources.schemas.my_schema.name}.my_index + endpoint_name: vs_endpoint + primary_key: id + index_type: DELTA_SYNC + presets: name_prefix: "$PREFIX" diff --git a/acceptance/bundle/validate/presets_name_prefix/output.txt b/acceptance/bundle/validate/presets_name_prefix/output.txt index 9fb8cf7ee39..f0cd13ba80f 100644 --- a/acceptance/bundle/validate/presets_name_prefix/output.txt +++ b/acceptance/bundle/validate/presets_name_prefix/output.txt @@ -25,11 +25,29 @@ } }, "schemas": { + "my_schema": { + "catalog_name": "my_catalog", + "name": "prefixmy_schema" + }, "schema1": { "catalog_name": "c", "name": "prefixschema1" } }, + "vector_search_endpoints": { + "vs_endpoint": { + "endpoint_type": "STANDARD", + "name": "prefixvs_endpoint" + } + }, + "vector_search_indexes": { + "vs_index": { + "endpoint_name": "vs_endpoint", + "index_type": "DELTA_SYNC", + "name": "my_catalog.${resources.schemas.my_schema.name}.prefixmy_index", + "primary_key": "id" + } + }, "volumes": { "volume1": { "catalog_name": "catalog1", @@ -66,11 +84,29 @@ } }, "schemas": { + "my_schema": { + "catalog_name": "my_catalog", + "name": "prefix_my_schema" + }, "schema1": { "catalog_name": "c", "name": "prefix_schema1" } }, + "vector_search_endpoints": { + "vs_endpoint": { + "endpoint_type": "STANDARD", + "name": "prefix_vs_endpoint" + } + }, + "vector_search_indexes": { + "vs_index": { + "endpoint_name": "vs_endpoint", + "index_type": "DELTA_SYNC", + "name": "my_catalog.${resources.schemas.my_schema.name}.prefix_my_index", + "primary_key": "id" + } + }, "volumes": { "volume1": { "catalog_name": "catalog1", @@ -107,11 +143,29 @@ } }, "schemas": { + "my_schema": { + "catalog_name": "my_catalog", + "name": "my_schema" + }, "schema1": { "catalog_name": "c", "name": "schema1" } }, + "vector_search_endpoints": { + "vs_endpoint": { + "endpoint_type": "STANDARD", + "name": "vs_endpoint" + } + }, + "vector_search_indexes": { + "vs_index": { + "endpoint_name": "vs_endpoint", + "index_type": "DELTA_SYNC", + "name": "my_catalog.${resources.schemas.my_schema.name}.my_index", + "primary_key": "id" + } + }, "volumes": { "volume1": { "catalog_name": "catalog1", From 9c4df31dce79c3abdb583bbb166fbf048a75cdc4 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 22 Apr 2026 16:10:34 +0200 Subject: [PATCH 08/47] acceptance: mark vector_search_indexes tests as RequiresUnityCatalog Indexes are UC-only (delta sync against a UC table, UC grants as ACL). Mirrors the flag already set on vector_search_endpoints so these tests are skipped on non-UC cloud runs instead of failing on missing catalog. Co-authored-by: Isaac --- .../bundle/resources/vector_search_indexes/basic/out.test.toml | 1 + .../vector_search_indexes/drift/columns_to_sync/out.test.toml | 1 + .../vector_search_indexes/drift/deleted_remotely/out.test.toml | 1 + .../vector_search_indexes/recreate/index_type/out.test.toml | 1 + acceptance/bundle/resources/vector_search_indexes/test.toml | 1 + 5 files changed, 5 insertions(+) diff --git a/acceptance/bundle/resources/vector_search_indexes/basic/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/basic/out.test.toml index 19b2c349a32..f1d40380d02 100644 --- a/acceptance/bundle/resources/vector_search_indexes/basic/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/basic/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = true +RequiresUnityCatalog = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/out.test.toml index 19b2c349a32..f1d40380d02 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = true +RequiresUnityCatalog = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/out.test.toml index 54146af5645..5566892a0d7 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = false +RequiresUnityCatalog = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.test.toml index 19b2c349a32..f1d40380d02 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = true +RequiresUnityCatalog = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/test.toml b/acceptance/bundle/resources/vector_search_indexes/test.toml index 6ac480c238f..99eb7790643 100644 --- a/acceptance/bundle/resources/vector_search_indexes/test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/test.toml @@ -1,5 +1,6 @@ Local = true Cloud = true +RequiresUnityCatalog = true # Vector Search indexes are only available in direct mode (no Terraform provider) EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] From 404b262285c8947a4cfa8b2aa842545d404f460a Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 22 Apr 2026 17:22:35 +0200 Subject: [PATCH 09/47] direct: track endpoint UUID on vector search index to detect orphaning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If the endpoint an index is attached to is deleted out-of-band, the index still exists by name but its backing endpoint is gone. The planner saw the index's remote state as valid and produced "skip", then deploy recreated the endpoint (new UUID) and left the index orphaned against the old UUID — on the next deploy attempt the user saw "index already exists" with no way to resolve short of manual deletion. VectorSearchIndexState now persists endpoint_uuid alongside the create request. DoRead looks it up from the endpoint service (the index API doesn't return it). OverrideChangeDesc classifies saved vs remote endpoint UUID drift as Recreate so the plan correctly shows "recreate vector_search_indexes.*" when the endpoint has been replaced, and the apply path delete+creates the index against the new endpoint. Covered by acceptance/.../vector_search_indexes/drift/orphaned_endpoint. Co-authored-by: Isaac --- .../orphaned_endpoint/databricks.yml.tmpl | 20 +++ .../drift/orphaned_endpoint/out.test.toml | 6 + .../drift/orphaned_endpoint/output.txt | 42 ++++++ .../drift/orphaned_endpoint/script | 22 +++ .../drift/orphaned_endpoint/test.toml | 1 + .../direct/dresources/vector_search_index.go | 133 +++++++++++++++--- 6 files changed, 204 insertions(+), 20 deletions(-) create mode 100644 acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/out.test.toml create mode 100644 acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/output.txt create mode 100644 acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/script create mode 100644 acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/test.toml diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/databricks.yml.tmpl b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/databricks.yml.tmpl new file mode 100644 index 00000000000..1bf224ad2ec --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/databricks.yml.tmpl @@ -0,0 +1,20 @@ +bundle: + name: drift-vs-index-orphaned-endpoint-$UNIQUE_NAME + +sync: + paths: [] + +resources: + vector_search_endpoints: + vs_endpoint: + name: vs-endpoint-$UNIQUE_NAME + endpoint_type: STANDARD + vector_search_indexes: + my_index: + name: vs-index-$UNIQUE_NAME + endpoint_name: ${resources.vector_search_endpoints.vs_endpoint.name} + primary_key: id + index_type: DELTA_SYNC + delta_sync_index_spec: + source_table: main.default.source_$UNIQUE_NAME + pipeline_type: TRIGGERED diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/out.test.toml new file mode 100644 index 00000000000..5566892a0d7 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/out.test.toml @@ -0,0 +1,6 @@ +Local = true +Cloud = false +RequiresUnityCatalog = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/output.txt b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/output.txt new file mode 100644 index 00000000000..27f6ce7fc93 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/output.txt @@ -0,0 +1,42 @@ + +=== Initial deployment +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/drift-vs-index-orphaned-endpoint-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Simulate remote endpoint deletion (index is orphaned, still exists by name) +>>> [CLI] vector-search-endpoints delete-endpoint vs-endpoint-[UNIQUE_NAME] + +=== Plan must cascade-recreate the index because its endpoint is being recreated +>>> [CLI] bundle plan +create vector_search_endpoints.vs_endpoint +recreate vector_search_indexes.my_index + +Plan: 2 to add, 0 to change, 1 to delete, 0 unchanged + +=== Deploy recreates endpoint and rebinds index +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/drift-vs-index-orphaned-endpoint-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] vector-search-indexes get-index vs-index-[UNIQUE_NAME] +{ + "name": "vs-index-[UNIQUE_NAME]", + "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", + "index_type": "DELTA_SYNC", + "primary_key": "id" +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.vector_search_endpoints.vs_endpoint + delete resources.vector_search_indexes.my_index + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/drift-vs-index-orphaned-endpoint-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/script b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/script new file mode 100644 index 00000000000..74b6376fc34 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/script @@ -0,0 +1,22 @@ +envsubst < databricks.yml.tmpl > databricks.yml +endpoint_name="vs-endpoint-${UNIQUE_NAME}" +index_name="vs-index-${UNIQUE_NAME}" + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "Initial deployment" +trace $CLI bundle deploy + +title "Simulate remote endpoint deletion (index is orphaned, still exists by name)" +trace $CLI vector-search-endpoints delete-endpoint "${endpoint_name}" + +title "Plan must cascade-recreate the index because its endpoint is being recreated" +trace $CLI bundle plan | contains.py "create vector_search_endpoints.vs_endpoint" "recreate vector_search_indexes.my_index" + +title "Deploy recreates endpoint and rebinds index" +trace $CLI bundle deploy +trace $CLI vector-search-indexes get-index "${index_name}" | jq '{name, endpoint_name, index_type, primary_key}' diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/test.toml b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/bundle/direct/dresources/vector_search_index.go b/bundle/direct/dresources/vector_search_index.go index 16cd82a55fe..6e61fe6661c 100644 --- a/bundle/direct/dresources/vector_search_index.go +++ b/bundle/direct/dresources/vector_search_index.go @@ -4,10 +4,47 @@ import ( "context" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/vectorsearch" ) +// VectorSearchIndexState tracks the UUID of the endpoint the index is attached +// to. Without it the planner cannot tell that an index pointing at a deleted +// and recreated endpoint (same name, different UUID) has been orphaned — the +// index still exists by name but its backing endpoint is gone. +type VectorSearchIndexState struct { + vectorsearch.CreateVectorIndexRequest + EndpointUuid string `json:"endpoint_uuid,omitempty"` +} + +func (s *VectorSearchIndexState) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s VectorSearchIndexState) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + +// VectorSearchIndexRemote is remote state. endpoint_uuid is looked up from the +// endpoint service since the index API itself doesn't return it. +type VectorSearchIndexRemote struct { + *vectorsearch.VectorIndex + EndpointUuid string `json:"endpoint_uuid,omitempty"` +} + +func (s *VectorSearchIndexRemote) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s VectorSearchIndexRemote) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + type ResourceVectorSearchIndex struct { client *databricks.WorkspaceClient } @@ -16,22 +53,28 @@ func (*ResourceVectorSearchIndex) New(client *databricks.WorkspaceClient) *Resou return &ResourceVectorSearchIndex{client: client} } -func (*ResourceVectorSearchIndex) PrepareState(input *resources.VectorSearchIndex) *vectorsearch.CreateVectorIndexRequest { - return &input.CreateVectorIndexRequest +func (*ResourceVectorSearchIndex) PrepareState(input *resources.VectorSearchIndex) *VectorSearchIndexState { + return &VectorSearchIndexState{ + CreateVectorIndexRequest: input.CreateVectorIndexRequest, + EndpointUuid: "", + } } -func (*ResourceVectorSearchIndex) RemapState(remote *vectorsearch.VectorIndex) *vectorsearch.CreateVectorIndexRequest { - req := &vectorsearch.CreateVectorIndexRequest{ - DeltaSyncIndexSpec: nil, - DirectAccessIndexSpec: nil, - IndexSubtype: "", - Name: remote.Name, - EndpointName: remote.EndpointName, - IndexType: remote.IndexType, - PrimaryKey: remote.PrimaryKey, +func (*ResourceVectorSearchIndex) RemapState(remote *VectorSearchIndexRemote) *VectorSearchIndexState { + state := &VectorSearchIndexState{ + CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{ + DeltaSyncIndexSpec: nil, + DirectAccessIndexSpec: nil, + IndexSubtype: "", + Name: remote.Name, + EndpointName: remote.EndpointName, + IndexType: remote.IndexType, + PrimaryKey: remote.PrimaryKey, + }, + EndpointUuid: remote.EndpointUuid, } if remote.DeltaSyncIndexSpec != nil { - req.DeltaSyncIndexSpec = &vectorsearch.DeltaSyncVectorIndexSpecRequest{ + state.DeltaSyncIndexSpec = &vectorsearch.DeltaSyncVectorIndexSpecRequest{ ColumnsToSync: nil, EmbeddingSourceColumns: remote.DeltaSyncIndexSpec.EmbeddingSourceColumns, EmbeddingVectorColumns: remote.DeltaSyncIndexSpec.EmbeddingVectorColumns, @@ -42,24 +85,33 @@ func (*ResourceVectorSearchIndex) RemapState(remote *vectorsearch.VectorIndex) * } } if remote.DirectAccessIndexSpec != nil { - req.DirectAccessIndexSpec = remote.DirectAccessIndexSpec + state.DirectAccessIndexSpec = remote.DirectAccessIndexSpec } - return req + return state } -func (r *ResourceVectorSearchIndex) DoRead(ctx context.Context, id string) (*vectorsearch.VectorIndex, error) { - return r.client.VectorSearchIndexes.GetIndexByIndexName(ctx, id) +func (r *ResourceVectorSearchIndex) DoRead(ctx context.Context, id string) (*VectorSearchIndexRemote, error) { + index, err := r.client.VectorSearchIndexes.GetIndexByIndexName(ctx, id) + if err != nil { + return nil, err + } + return &VectorSearchIndexRemote{ + VectorIndex: index, + EndpointUuid: r.lookupEndpointUuid(ctx, index.EndpointName), + }, nil } -func (r *ResourceVectorSearchIndex) DoCreate(ctx context.Context, config *vectorsearch.CreateVectorIndexRequest) (string, *vectorsearch.VectorIndex, error) { - index, err := r.client.VectorSearchIndexes.CreateIndex(ctx, *config) +func (r *ResourceVectorSearchIndex) DoCreate(ctx context.Context, config *VectorSearchIndexState) (string, *VectorSearchIndexRemote, error) { + index, err := r.client.VectorSearchIndexes.CreateIndex(ctx, config.CreateVectorIndexRequest) if err != nil { return "", nil, err } - return config.Name, index, nil + endpointUuid := r.lookupEndpointUuid(ctx, config.EndpointName) + config.EndpointUuid = endpointUuid + return config.Name, &VectorSearchIndexRemote{VectorIndex: index, EndpointUuid: endpointUuid}, nil } -func (r *ResourceVectorSearchIndex) DoUpdate(ctx context.Context, id string, config *vectorsearch.CreateVectorIndexRequest, entry *PlanEntry) (*vectorsearch.VectorIndex, error) { +func (r *ResourceVectorSearchIndex) DoUpdate(ctx context.Context, id string, config *VectorSearchIndexState, entry *PlanEntry) (*VectorSearchIndexRemote, error) { // Vector search indexes have no update API; all field changes trigger recreation via resources.yml. return nil, nil } @@ -67,3 +119,44 @@ func (r *ResourceVectorSearchIndex) DoUpdate(ctx context.Context, id string, con func (r *ResourceVectorSearchIndex) DoDelete(ctx context.Context, id string) error { return r.client.VectorSearchIndexes.DeleteIndexByIndexName(ctx, id) } + +// OverrideChangeDesc classifies endpoint_uuid drift: Recreate when the saved +// UUID differs from what's currently attached to the endpoint name, Skip +// otherwise. endpoint_uuid is never present in config, so without Skip a +// synthetic diff between empty newState and populated saved state would +// otherwise leak into the plan. +func (*ResourceVectorSearchIndex) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, remote *VectorSearchIndexRemote) error { + if path.String() != "endpoint_uuid" { + return nil + } + savedUuid, _ := change.Old.(string) + var remoteUuid string + if remote != nil { + remoteUuid = remote.EndpointUuid + } + if savedUuid != "" && savedUuid != remoteUuid { + change.Action = deployplan.Recreate + change.Reason = "endpoint replaced out-of-band" + } else { + change.Action = deployplan.Skip + change.Reason = "state-only field" + } + return nil +} + +// lookupEndpointUuid returns the current UUID of the endpoint with the given +// name, or "" if the endpoint doesn't exist. Errors are logged and swallowed +// since a missing endpoint is the signal we want to capture in state. +func (r *ResourceVectorSearchIndex) lookupEndpointUuid(ctx context.Context, endpointName string) string { + if endpointName == "" { + return "" + } + info, err := r.client.VectorSearchEndpoints.GetEndpointByEndpointName(ctx, endpointName) + if err != nil { + if !apierr.IsMissing(err) { + log.Warnf(ctx, "failed to read vector search endpoint %q while resolving index endpoint UUID: %v", endpointName, err) + } + return "" + } + return info.Id +} From 223c5d2a7c78489f7b0e0a27110ad4183b3e897e Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 29 Apr 2026 13:27:47 +0200 Subject: [PATCH 10/47] phases: prompt before deleting or recreating vector search indexes Recreate of a Vector Search index re-runs the full embedding pipeline (Delta Sync) or drops every upserted vector (Direct Access). Both can take significant time and cost. Bring vector_search_indexes into the same destructive-action prompt path used for schemas, pipelines, volumes, dashboards, and the Lakebase resources, with a message explaining the cost. Acceptance scripts that recreate indexes now pass --auto-approve, and all VS index test outputs pick up the prompt message at destroy time. Co-authored-by: Isaac --- .../resources/vector_search_indexes/basic/output.txt | 5 +++++ .../drift/columns_to_sync/output.txt | 5 +++++ .../drift/deleted_remotely/output.txt | 5 +++++ .../drift/orphaned_endpoint/output.txt | 12 +++++++++++- .../drift/orphaned_endpoint/script | 2 +- .../vector_search_indexes/grants/select/output.txt | 5 +++++ .../recreate/index_type/output.txt | 10 ++++++++++ bundle/phases/deploy.go | 1 + bundle/phases/destroy.go | 1 + bundle/phases/messages.go | 9 +++++++++ 10 files changed, 53 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/resources/vector_search_indexes/basic/output.txt b/acceptance/bundle/resources/vector_search_indexes/basic/output.txt index a905d5e9e0a..fb12212a06e 100644 --- a/acceptance/bundle/resources/vector_search_indexes/basic/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/basic/output.txt @@ -59,6 +59,11 @@ Resources: The following resources will be deleted: delete resources.vector_search_indexes.my_index +This action will result in the deletion of the following Vector Search indexes. +For Delta Sync indexes, the source Delta Table is preserved but the embedding pipeline is removed. +For Direct Access indexes, all upserted vectors are permanently lost: + delete resources.vector_search_indexes.my_index + All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-vs-index-[UNIQUE_NAME]/default Deleting files... diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/output.txt b/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/output.txt index 0d88414c395..6a23c9bebb3 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/output.txt @@ -29,6 +29,11 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged The following resources will be deleted: delete resources.vector_search_indexes.my_index +This action will result in the deletion of the following Vector Search indexes. +For Delta Sync indexes, the source Delta Table is preserved but the embedding pipeline is removed. +For Direct Access indexes, all upserted vectors are permanently lost: + delete resources.vector_search_indexes.my_index + All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/drift-vs-index-columns-[UNIQUE_NAME]/default Deleting files... diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/output.txt b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/output.txt index 4575e349949..7027a4b165e 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/output.txt @@ -40,6 +40,11 @@ Deployment complete! The following resources will be deleted: delete resources.vector_search_indexes.my_index +This action will result in the deletion of the following Vector Search indexes. +For Delta Sync indexes, the source Delta Table is preserved but the embedding pipeline is removed. +For Direct Access indexes, all upserted vectors are permanently lost: + delete resources.vector_search_indexes.my_index + All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/drift-vs-index-deleted-[UNIQUE_NAME]/default Deleting files... diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/output.txt b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/output.txt index 27f6ce7fc93..b8c384ba553 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/output.txt @@ -17,8 +17,13 @@ recreate vector_search_indexes.my_index Plan: 2 to add, 0 to change, 1 to delete, 0 unchanged === Deploy recreates endpoint and rebinds index ->>> [CLI] bundle deploy +>>> [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/drift-vs-index-orphaned-endpoint-[UNIQUE_NAME]/default/files... + +This action will result in the deletion or recreation of the following Vector Search indexes. +Recreating a Delta Sync index re-runs the full embedding pipeline; recreating a Direct Access +index drops all upserted vectors. Both can be expensive to rebuild: + recreate resources.vector_search_indexes.my_index Deploying resources... Updating deployment state... Deployment complete! @@ -36,6 +41,11 @@ The following resources will be deleted: delete resources.vector_search_endpoints.vs_endpoint delete resources.vector_search_indexes.my_index +This action will result in the deletion of the following Vector Search indexes. +For Delta Sync indexes, the source Delta Table is preserved but the embedding pipeline is removed. +For Direct Access indexes, all upserted vectors are permanently lost: + delete resources.vector_search_indexes.my_index + All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/drift-vs-index-orphaned-endpoint-[UNIQUE_NAME]/default Deleting files... diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/script b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/script index 74b6376fc34..b67193c34c9 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/script +++ b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/script @@ -18,5 +18,5 @@ title "Plan must cascade-recreate the index because its endpoint is being recrea trace $CLI bundle plan | contains.py "create vector_search_endpoints.vs_endpoint" "recreate vector_search_indexes.my_index" title "Deploy recreates endpoint and rebinds index" -trace $CLI bundle deploy +trace $CLI bundle deploy --auto-approve trace $CLI vector-search-indexes get-index "${index_name}" | jq '{name, endpoint_name, index_type, primary_key}' diff --git a/acceptance/bundle/resources/vector_search_indexes/grants/select/output.txt b/acceptance/bundle/resources/vector_search_indexes/grants/select/output.txt index b0003c95f0b..b0c58590730 100644 --- a/acceptance/bundle/resources/vector_search_indexes/grants/select/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/grants/select/output.txt @@ -39,6 +39,11 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged The following resources will be deleted: delete resources.vector_search_indexes.my_index +This action will result in the deletion of the following Vector Search indexes. +For Delta Sync indexes, the source Delta Table is preserved but the embedding pipeline is removed. +For Direct Access indexes, all upserted vectors are permanently lost: + delete resources.vector_search_indexes.my_index + All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/vs-index-grants-[UNIQUE_NAME]/default Deleting files... diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/output.txt b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/output.txt index 143252aed92..87df5820eda 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/output.txt @@ -31,6 +31,11 @@ Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged >>> [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-vs-index-[UNIQUE_NAME]/default/files... + +This action will result in the deletion or recreation of the following Vector Search indexes. +Recreating a Delta Sync index re-runs the full embedding pipeline; recreating a Direct Access +index drops all upserted vectors. Both can be expensive to rebuild: + recreate resources.vector_search_indexes.my_index Deploying resources... Updating deployment state... Deployment complete! @@ -49,6 +54,11 @@ Deployment complete! The following resources will be deleted: delete resources.vector_search_indexes.my_index +This action will result in the deletion of the following Vector Search indexes. +For Delta Sync indexes, the source Delta Table is preserved but the embedding pipeline is removed. +For Direct Access indexes, all upserted vectors are permanently lost: + delete resources.vector_search_indexes.my_index + All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-vs-index-[UNIQUE_NAME]/default Deleting files... diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 82593a07b5d..134673734f2 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -36,6 +36,7 @@ var deployApprovalGroups = []approvalGroup{ {group: "synced_database_tables", message: deleteOrRecreateSyncedDatabaseTableMessage}, {group: "postgres_projects", message: deleteOrRecreatePostgresProjectMessage}, {group: "postgres_branches", message: deleteOrRecreatePostgresBranchMessage}, + {group: "vector_search_indexes", message: deleteOrRecreateVectorSearchIndexMessage}, } func approvalForDeploy(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan) (bool, error) { diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 91640ac6cad..e87ed4906c0 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -40,6 +40,7 @@ var destroyApprovalGroups = []approvalGroup{ {group: "synced_database_tables", message: deleteSyncedDatabaseTableMessage}, {group: "postgres_projects", message: deletePostgresProjectMessage}, {group: "postgres_branches", message: deletePostgresBranchMessage}, + {group: "vector_search_indexes", message: deleteVectorSearchIndexMessage}, } func approvalForDestroy(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan) (bool, error) { diff --git a/bundle/phases/messages.go b/bundle/phases/messages.go index 4c75879aa41..221b6cba02d 100644 --- a/bundle/phases/messages.go +++ b/bundle/phases/messages.go @@ -36,6 +36,11 @@ all their branches, databases, and endpoints. All data stored in them will be pe deleteOrRecreatePostgresBranchMessage = ` This action will result in the deletion or recreation of the following Lakebase branches. All data stored in them will be permanently lost:` + + deleteOrRecreateVectorSearchIndexMessage = ` +This action will result in the deletion or recreation of the following Vector Search indexes. +Recreating a Delta Sync index re-runs the full embedding pipeline; recreating a Direct Access +index drops all upserted vectors. Both can be expensive to rebuild:` ) // DataLossWarning is the warning shown when a non-interactive command is about @@ -65,4 +70,8 @@ all their branches, databases, and endpoints. All data stored in them will be pe deletePostgresBranchMessage = `This action will result in the deletion of the following Lakebase branches. All data stored in them will be permanently lost:` + + deleteVectorSearchIndexMessage = `This action will result in the deletion of the following Vector Search indexes. +For Delta Sync indexes, the source Delta Table is preserved but the embedding pipeline is removed. +For Direct Access indexes, all upserted vectors are permanently lost:` ) From bde86cfcb167f200b5d35597de20a3195558c713 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 29 Apr 2026 13:28:07 +0200 Subject: [PATCH 11/47] direct: wait for index deletion to complete before recreating MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DELETE on a Vector Search index is asynchronous: the backend tears down the embedding pipeline over a few minutes, and any operation on the same name during that window returns "Index ... is currently pending deletion". Recreate (delete+create on the same name) hit this on every deploy. DoDelete now polls GetIndex until it returns 404, so a follow-up DoCreate sees a clean slate. Timeout is 15 minutes, matching observed worst-case teardown. Also harden recovery from a partial recreate: if Create after Delete fails, apply.Recreate now drops the state entry instead of leaving it with an empty ID. Pre-fix state files that already carry an empty ID are tolerated by the planner — empty ID is treated the same as a missing entry, so the next plan re-creates the resource cleanly instead of failing with "invalid state: empty id". Testserver now models the async deletion window: DELETE moves the index into a pending buffer, CREATE during pending returns the backend's exact error, and the next GET (the wait loop's poll) clears the buffer. The recreate/index_type acceptance test exercises the full path. Co-authored-by: Isaac --- bundle/direct/apply.go | 2 +- bundle/direct/bundle_plan.go | 2 +- .../direct/dresources/vector_search_index.go | 29 ++++++++- libs/testserver/fake_workspace.go | 49 ++++++++------- libs/testserver/handlers.go | 4 +- libs/testserver/vector_search_indexes.go | 59 +++++++++++++++++++ 6 files changed, 119 insertions(+), 26 deletions(-) diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index ba0f54bfc7b..44e5d897734 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -87,7 +87,7 @@ func (d *DeploymentUnit) Recreate(ctx context.Context, db *dstate.DeploymentStat } // Drop the state entry so a subsequent failure of Create leaves no malformed - // (empty-id) entry behind. The next plan will see "no state" and retry as Create. + // (empty-ID) entry behind. The next plan will see "no state" and retry as Create. err = db.DeleteState(d.ResourceKey) if err != nil { return fmt.Errorf("deleting state: %w", err) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index f6bcea316cd..32eb6dd575c 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -181,7 +181,7 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks } dbentry, hasEntry := b.StateDB.GetResourceEntry(resourceKey) - // Tolerate empty-id entries from older partial-recreate failures + // Tolerate empty-ID entries from older partial-recreate failures // (apply.Recreate now deletes state on the way through, but pre-fix // state files may still carry a malformed entry). Treat as missing // and let the resource be re-created on this plan. diff --git a/bundle/direct/dresources/vector_search_index.go b/bundle/direct/dresources/vector_search_index.go index 6e61fe6661c..5ff61c45474 100644 --- a/bundle/direct/dresources/vector_search_index.go +++ b/bundle/direct/dresources/vector_search_index.go @@ -2,6 +2,8 @@ package dresources import ( "context" + "errors" + "time" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" @@ -10,9 +12,15 @@ import ( "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/retries" "github.com/databricks/databricks-sdk-go/service/vectorsearch" ) +// deleteIndexTimeout caps the wait for an index deletion to complete. +// In practice deletion finishes in a minute or two, but worst case the +// embedding pipeline shutdown can stretch closer to ten minutes. +const deleteIndexTimeout = 15 * time.Minute + // VectorSearchIndexState tracks the UUID of the endpoint the index is attached // to. Without it the planner cannot tell that an index pointing at a deleted // and recreated endpoint (same name, different UUID) has been orphaned — the @@ -116,8 +124,27 @@ func (r *ResourceVectorSearchIndex) DoUpdate(ctx context.Context, id string, con return nil, nil } +// DoDelete kicks off deletion and waits for it to complete. The DELETE call +// is asynchronous: a follow-up CREATE for the same name (e.g. during recreate) +// is rejected with "index is currently pending deletion" until the backend +// finishes tearing down the embedding pipeline. We poll GetIndex until it +// returns 404 to make recreate paths idempotent. func (r *ResourceVectorSearchIndex) DoDelete(ctx context.Context, id string) error { - return r.client.VectorSearchIndexes.DeleteIndexByIndexName(ctx, id) + err := r.client.VectorSearchIndexes.DeleteIndexByIndexName(ctx, id) + if err != nil { + return err + } + _, err = retries.Poll[struct{}](ctx, deleteIndexTimeout, func() (*struct{}, *retries.Err) { + _, getErr := r.client.VectorSearchIndexes.GetIndexByIndexName(ctx, id) + if getErr == nil { + return nil, retries.Continues("index still exists, waiting for deletion to complete") + } + if errors.Is(getErr, apierr.ErrResourceDoesNotExist) || errors.Is(getErr, apierr.ErrNotFound) { + return &struct{}{}, nil + } + return nil, retries.Halt(getErr) + }) + return err } // OverrideChangeDesc classifies endpoint_uuid drift: Recreate when the saved diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 51bacfbf2b4..da179e3f70b 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -154,6 +154,12 @@ type FakeWorkspace struct { VectorSearchEndpoints map[string]vectorsearch.EndpointInfo VectorSearchIndexes map[string]vectorsearch.VectorIndex + // vectorSearchIndexesPendingDeletion stores indexes that have received a DELETE + // but whose deletion has not yet "completed". GETs against a name in this map + // return the index once (mirroring the real backend's async deletion window), + // then the entry is consumed and subsequent GETs return 404. + vectorSearchIndexesPendingDeletion map[string]vectorsearch.VectorIndex + SecretScopes map[string]workspace.SecretScope Secrets map[string]map[string]string // scope -> key -> value Acls map[string][]workspace.AclItem @@ -286,27 +292,28 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { State: sql.StateRunning, }, }, - ServingEndpoints: map[string]serving.ServingEndpointDetailed{}, - VectorSearchEndpoints: map[string]vectorsearch.EndpointInfo{}, - VectorSearchIndexes: map[string]vectorsearch.VectorIndex{}, - Repos: map[string]workspace.RepoInfo{}, - SecretScopes: map[string]workspace.SecretScope{}, - Secrets: map[string]map[string]string{}, - Acls: map[string][]workspace.AclItem{}, - Permissions: map[string]iam.ObjectPermissions{}, - Groups: map[string]iam.Group{}, - DatabaseInstances: map[string]database.DatabaseInstance{}, - DatabaseCatalogs: map[string]database.DatabaseCatalog{}, - SyncedDatabaseTables: map[string]database.SyncedDatabaseTable{}, - PostgresProjects: map[string]postgres.Project{}, - PostgresBranches: map[string]postgres.Branch{}, - PostgresEndpoints: map[string]postgres.Endpoint{}, - PostgresOperations: map[string]postgres.Operation{}, - clusterVenvs: map[string]*clusterEnv{}, - Alerts: map[string]sql.AlertV2{}, - Experiments: map[string]ml.GetExperimentResponse{}, - ModelRegistryModels: map[string]ml.Model{}, - ModelRegistryModelIDs: map[string]string{}, + ServingEndpoints: map[string]serving.ServingEndpointDetailed{}, + VectorSearchEndpoints: map[string]vectorsearch.EndpointInfo{}, + VectorSearchIndexes: map[string]vectorsearch.VectorIndex{}, + vectorSearchIndexesPendingDeletion: map[string]vectorsearch.VectorIndex{}, + Repos: map[string]workspace.RepoInfo{}, + SecretScopes: map[string]workspace.SecretScope{}, + Secrets: map[string]map[string]string{}, + Acls: map[string][]workspace.AclItem{}, + Permissions: map[string]iam.ObjectPermissions{}, + Groups: map[string]iam.Group{}, + DatabaseInstances: map[string]database.DatabaseInstance{}, + DatabaseCatalogs: map[string]database.DatabaseCatalog{}, + SyncedDatabaseTables: map[string]database.SyncedDatabaseTable{}, + PostgresProjects: map[string]postgres.Project{}, + PostgresBranches: map[string]postgres.Branch{}, + PostgresEndpoints: map[string]postgres.Endpoint{}, + PostgresOperations: map[string]postgres.Operation{}, + clusterVenvs: map[string]*clusterEnv{}, + Alerts: map[string]sql.AlertV2{}, + Experiments: map[string]ml.GetExperimentResponse{}, + ModelRegistryModels: map[string]ml.Model{}, + ModelRegistryModelIDs: map[string]string{}, Clusters: map[string]compute.ClusterDetails{ TestDefaultClusterId: { ClusterId: TestDefaultClusterId, diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index d8658964ae6..4240f330267 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -835,11 +835,11 @@ func AddDefaultHandlers(server *Server) { }) server.Handle("GET", "/api/2.0/vector-search/indexes/{index_name}", func(req Request) any { - return MapGet(req.Workspace, req.Workspace.VectorSearchIndexes, req.Vars["index_name"]) + return req.Workspace.VectorSearchIndexGet(req.Vars["index_name"]) }) server.Handle("DELETE", "/api/2.0/vector-search/indexes/{index_name}", func(req Request) any { - return MapDelete(req.Workspace, req.Workspace.VectorSearchIndexes, req.Vars["index_name"]) + return req.Workspace.VectorSearchIndexDelete(req.Vars["index_name"]) }) // Generic permissions endpoints diff --git a/libs/testserver/vector_search_indexes.go b/libs/testserver/vector_search_indexes.go index a2a7848ad94..3d559cad61d 100644 --- a/libs/testserver/vector_search_indexes.go +++ b/libs/testserver/vector_search_indexes.go @@ -25,6 +25,15 @@ func (s *FakeWorkspace) VectorSearchIndexCreate(req Request) Response { Body: map[string]string{"error_code": "RESOURCE_ALREADY_EXISTS", "message": fmt.Sprintf("Vector search index with name %s already exists", createReq.Name)}, } } + if _, pending := s.vectorSearchIndexesPendingDeletion[createReq.Name]; pending { + return Response{ + StatusCode: http.StatusBadRequest, + Body: map[string]string{ + "error_code": "INVALID_PARAMETER_VALUE", + "message": fmt.Sprintf("Index %s is currently pending deletion. Operations on the index are not permitted while the index is being deleted.", createReq.Name), + }, + } + } if _, exists := s.VectorSearchEndpoints[createReq.EndpointName]; !exists { return Response{ StatusCode: http.StatusNotFound, @@ -68,3 +77,53 @@ func remapDeltaSyncSpec(req *vectorsearch.DeltaSyncVectorIndexSpecRequest) *vect SourceTable: req.SourceTable, } } + +// VectorSearchIndexDelete moves the index out of the live map into a +// "pending deletion" buffer. CREATE on the same name returns the +// pending-deletion error until a GET (i.e. a poll) consumes the buffer entry, +// which is how the real backend's async deletion window manifests to a client +// that doesn't wait between DELETE and CREATE. +func (s *FakeWorkspace) VectorSearchIndexDelete(indexName string) Response { + defer s.LockUnlock()() + + index, ok := s.VectorSearchIndexes[indexName] + if !ok { + return Response{ + StatusCode: http.StatusNotFound, + Body: map[string]string{ + "error_code": "RESOURCE_DOES_NOT_EXIST", + "message": fmt.Sprintf("Vector search index %s not found", indexName), + }, + } + } + if index.Status == nil { + index.Status = &vectorsearch.VectorIndexStatus{} + } + index.Status.Ready = false + index.Status.Message = "Index is currently pending deletion" + s.vectorSearchIndexesPendingDeletion[indexName] = index + delete(s.VectorSearchIndexes, indexName) + return Response{} +} + +// VectorSearchIndexGet returns 404 once the index has been moved to the +// pending-deletion buffer. The pending entry is consumed as a side effect, so +// CREATE-during-pending only fires when the caller skips the wait and races +// straight from DELETE to CREATE without polling. This preserves the +// "remote not found -> recreate" behavior for tests that delete out-of-band +// and immediately re-plan. +func (s *FakeWorkspace) VectorSearchIndexGet(indexName string) Response { + defer s.LockUnlock()() + + if index, ok := s.VectorSearchIndexes[indexName]; ok { + return Response{Body: index} + } + delete(s.vectorSearchIndexesPendingDeletion, indexName) + return Response{ + StatusCode: http.StatusNotFound, + Body: map[string]string{ + "error_code": "RESOURCE_DOES_NOT_EXIST", + "message": fmt.Sprintf("Vector search index %s not found", indexName), + }, + } +} From f45a65330bbced122323c7163f1c719a668d297b Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 29 Apr 2026 13:41:40 +0200 Subject: [PATCH 12/47] direct: split deletion wait into WaitAfterDelete adapter method Mirrors the WaitAfterCreate / WaitAfterUpdate pattern: the wait runs after the framework drops state, not inside DoDelete itself, so a timed-out wait leaves the bundle consistent (resource was requested deleted, retry on next plan) instead of blocking on an inflight DELETE with state still attached. - IResource gains an optional WaitAfterDelete(ctx, id) error. - apply.Recreate calls it between DeleteState and the follow-up Create so async teardown finishes before the same name is reused. - apply.Delete calls it after DeleteState so `bundle destroy` has the same contract. - Vector search indexes move their GetIndex polling out of DoDelete into the new method. - TestAccept exercise (test_dresource_lifecycle) now also calls WaitAfterDelete to validate the no-op fallthrough for resources that don't implement it. Co-authored-by: Isaac --- bundle/direct/apply.go | 23 ++++++++++++++++--- bundle/direct/dresources/adapter.go | 23 +++++++++++++++++++ bundle/direct/dresources/all_test.go | 3 +++ .../direct/dresources/vector_search_index.go | 20 ++++++++-------- 4 files changed, 56 insertions(+), 13 deletions(-) diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index 44e5d897734..9ca00d9a28f 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -86,13 +86,22 @@ func (d *DeploymentUnit) Recreate(ctx context.Context, db *dstate.DeploymentStat return fmt.Errorf("deleting old id=%s: %w", oldID, err) } - // Drop the state entry so a subsequent failure of Create leaves no malformed - // (empty-ID) entry behind. The next plan will see "no state" and retry as Create. + // Drop the state entry so a subsequent failure of Create or WaitAfterDelete + // leaves no malformed (empty-ID) entry behind. The next plan will see "no + // state" and retry as Create. err = db.DeleteState(d.ResourceKey) if err != nil { return fmt.Errorf("deleting state: %w", err) } + // Wait for asynchronous teardown to finish before re-creating the same + // name. Done after DeleteState so the bundle stays consistent if the wait + // times out — the resource is no longer tracked in state, retry on next plan. + err = d.Adapter.WaitAfterDelete(ctx, oldID) + if err != nil { + return fmt.Errorf("waiting after deleting id=%s: %w", oldID, err) + } + return d.Create(ctx, db, newState) } @@ -172,7 +181,7 @@ func (d *DeploymentUnit) Delete(ctx context.Context, db *dstate.DeploymentState, // Rather than failing delete and requiring user to unbind, we perform unbind automatically there. // Some services, e.g. jobs, return 403 for missing resources if caller did not have permissions to it when job existed. // In those cases 403 hides 404. In other cases, user not having permissions to resource but having in the bundle might - // mean configuration error that user is trying to fix by removing resource from their bundle. + // mean configuration error that user is trying to fix by permission-restoring or removing the resource from their bundle. if errors.Is(err, apierr.ErrPermissionDenied) { log.Warnf(ctx, "Ignoring permission error when deleting %s id=%s: %s", d.ResourceKey, oldID, err) } else { @@ -185,6 +194,14 @@ func (d *DeploymentUnit) Delete(ctx context.Context, db *dstate.DeploymentState, return fmt.Errorf("deleting state id=%s: %w", oldID, err) } + // Wait for asynchronous teardown after dropping state. Mirrors Recreate so + // the contract is the same regardless of whether the user triggered + // `bundle destroy` or a recreate. + err = d.Adapter.WaitAfterDelete(ctx, oldID) + if err != nil { + return fmt.Errorf("waiting after deleting id=%s: %w", oldID, err) + } + return nil } diff --git a/bundle/direct/dresources/adapter.go b/bundle/direct/dresources/adapter.go index 5e46dad540b..6be647104e1 100644 --- a/bundle/direct/dresources/adapter.go +++ b/bundle/direct/dresources/adapter.go @@ -72,6 +72,12 @@ type IResource interface { // [Optional] WaitAfterUpdate waits for the resource to become ready after update. Returns optionally updated remote state. WaitAfterUpdate(ctx context.Context, id string, newState any) (remoteState any, e error) + // [Optional] WaitAfterDelete waits for the resource to be fully removed after DoDelete returns. + // Useful for backends with asynchronous deletion: a follow-up create on the same name (recreate path) + // would otherwise race with the in-progress teardown. State is dropped before this is called, so a + // timeout here leaves the bundle consistent (resource was requested deleted, retry on next plan). + WaitAfterDelete(ctx context.Context, id string) error + // [Optional] KeyedSlices returns a map from path patterns to KeyFunc for comparing slices by key instead of by index. // Example: func (*ResourcePermissions) KeyedSlices(state *PermissionsState) map[string]any KeyedSlices() map[string]any @@ -92,6 +98,7 @@ type Adapter struct { doUpdateWithID *calladapt.BoundCaller waitAfterCreate *calladapt.BoundCaller waitAfterUpdate *calladapt.BoundCaller + waitAfterDelete *calladapt.BoundCaller overrideChangeDesc *calladapt.BoundCaller doResize *calladapt.BoundCaller @@ -124,6 +131,7 @@ func NewAdapter(typedNil any, resourceType string, client *databricks.WorkspaceC doResize: nil, waitAfterCreate: nil, waitAfterUpdate: nil, + waitAfterDelete: nil, overrideChangeDesc: nil, resourceConfig: GetResourceConfig(resourceType), generatedResourceConfig: GetGeneratedResourceConfig(resourceType), @@ -206,6 +214,11 @@ func (a *Adapter) initMethods(resource any) error { return err } + a.waitAfterDelete, err = calladapt.PrepareCall(resource, reflect.TypeFor[IResource](), "WaitAfterDelete") + if err != nil { + return err + } + a.overrideChangeDesc, err = calladapt.PrepareCall(resource, reflect.TypeFor[IResource](), "OverrideChangeDesc") if err != nil { return err @@ -516,6 +529,16 @@ func (a *Adapter) WaitAfterUpdate(ctx context.Context, id string, newState any) return remoteState, nil } +// WaitAfterDelete waits for the resource to be fully removed after DoDelete. +// If the resource doesn't implement this method, this is a no-op. +func (a *Adapter) WaitAfterDelete(ctx context.Context, id string) error { + if a.waitAfterDelete == nil { + return nil // no-op if not implemented + } + _, err := a.waitAfterDelete.Call(ctx, id) + return err +} + // HasOverrideChangeDesc returns true if OverrideChangeDesc is defined for this resource impl func (a *Adapter) HasOverrideChangeDesc() bool { return a.overrideChangeDesc != nil diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 43b8392c8be..d295ead807e 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -892,6 +892,9 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W err = adapter.DoDelete(ctx, createdID) require.NoError(t, err) + err = adapter.WaitAfterDelete(ctx, createdID) + require.NoError(t, err) + p, err := structpath.ParsePath("name") require.NoError(t, err) diff --git a/bundle/direct/dresources/vector_search_index.go b/bundle/direct/dresources/vector_search_index.go index 5ff61c45474..36de84b7d2e 100644 --- a/bundle/direct/dresources/vector_search_index.go +++ b/bundle/direct/dresources/vector_search_index.go @@ -124,17 +124,17 @@ func (r *ResourceVectorSearchIndex) DoUpdate(ctx context.Context, id string, con return nil, nil } -// DoDelete kicks off deletion and waits for it to complete. The DELETE call -// is asynchronous: a follow-up CREATE for the same name (e.g. during recreate) -// is rejected with "index is currently pending deletion" until the backend -// finishes tearing down the embedding pipeline. We poll GetIndex until it -// returns 404 to make recreate paths idempotent. func (r *ResourceVectorSearchIndex) DoDelete(ctx context.Context, id string) error { - err := r.client.VectorSearchIndexes.DeleteIndexByIndexName(ctx, id) - if err != nil { - return err - } - _, err = retries.Poll[struct{}](ctx, deleteIndexTimeout, func() (*struct{}, *retries.Err) { + return r.client.VectorSearchIndexes.DeleteIndexByIndexName(ctx, id) +} + +// WaitAfterDelete polls GetIndex until it returns 404. The DELETE call is +// asynchronous: a follow-up CREATE for the same name (e.g. during recreate) is +// rejected with "index is currently pending deletion" until the backend finishes +// tearing down the embedding pipeline. The framework calls this after dropping +// state so a wait-time failure leaves the bundle consistent. +func (r *ResourceVectorSearchIndex) WaitAfterDelete(ctx context.Context, id string) error { + _, err := retries.Poll[struct{}](ctx, deleteIndexTimeout, func() (*struct{}, *retries.Err) { _, getErr := r.client.VectorSearchIndexes.GetIndexByIndexName(ctx, id) if getErr == nil { return nil, retries.Continues("index still exists, waiting for deletion to complete") From e35c6351d711380946f416ae4e092ba60dc987eb Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 29 Apr 2026 14:51:02 +0200 Subject: [PATCH 13/47] direct: document the secondary endpoint lookup in DoCreate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dresources/README.md says CRUD methods should map 1:1 to API calls. DoCreate for vector search indexes makes a second GetEndpoint call so endpoint_uuid lands in saved state on the very first deploy — without that, orphan detection only kicks in after the next read+save cycle. Note the exception inline so a reviewer doesn't trip on it. Co-authored-by: Isaac --- bundle/direct/dresources/vector_search_index.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bundle/direct/dresources/vector_search_index.go b/bundle/direct/dresources/vector_search_index.go index 36de84b7d2e..307ac9988c9 100644 --- a/bundle/direct/dresources/vector_search_index.go +++ b/bundle/direct/dresources/vector_search_index.go @@ -114,6 +114,9 @@ func (r *ResourceVectorSearchIndex) DoCreate(ctx context.Context, config *Vector if err != nil { return "", nil, err } + // Exceptional: a second API call. The index API does not return the endpoint + // UUID, but we need to persist it in state so a future plan can detect that + // the endpoint was replaced out-of-band (same name, different UUID -> orphan). endpointUuid := r.lookupEndpointUuid(ctx, config.EndpointName) config.EndpointUuid = endpointUuid return config.Name, &VectorSearchIndexRemote{VectorIndex: index, EndpointUuid: endpointUuid}, nil From de293ab759950dcdb7aafb3e38d0b8850c0ee503 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 29 Apr 2026 15:03:35 +0200 Subject: [PATCH 14/47] NEXT_CHANGELOG: vector_search_indexes resource Co-authored-by: Isaac --- NEXT_CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 36cbb445d2c..4d2f6c55688 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -9,4 +9,6 @@ ### Bundles * Make sure warnings asking for approval are understood by agents ([#5239](https://github.com/databricks/cli/pull/5239)) +* Added `vector_search_indexes` as a first-class bundle resource on the direct deployment engine, alongside the existing `vector_search_endpoints`. Supports UC grants, drift detection (including out-of-band endpoint replacement that orphans an index), recreate-on-immutable-field-change, and asynchronous deletion waits. Recreating or deleting an index now prompts for confirmation with a Delta Sync vs Direct Access cost warning. Vector search has no Terraform provider, so this resource is direct-engine only ([#5123](https://github.com/databricks/cli/pull/5123)). + ### Dependency updates From 5c48db0016336bc99ea02e4d2593f9ec274ddecd Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 29 Apr 2026 16:23:01 +0200 Subject: [PATCH 15/47] testserver: drop the VS index pending-deletion buffer The buffer existed only so that recreate-without-wait would surface as a CREATE failure in tests. That's a CLI behavior, and the right place to assert it is the request-capture file in the acceptance test, not hidden state in the testserver. - libs/testserver/{handlers,vector_search_indexes,fake_workspace}.go: GET and DELETE go back to the generic MapGet / MapDelete helpers and the buffer / custom Get / custom Delete are gone. Index status stays Ready=true on create, matching the convention used by every other slow resource the testserver fakes (endpoints -> ONLINE, database instances -> AVAILABLE, apps -> RUNNING). - acceptance/bundle/resources/vector_search_indexes/recreate/index_type: print_requests.py now uses --get so the WaitAfterDelete poll is in the captured request log. The recreate journal now shows GET -> DELETE -> GET -> POST; if someone removes the wait the second GET disappears and the test fails on the regenerated capture. Co-authored-by: Isaac --- .../out.requests.recreate.direct.json | 8 +++ .../recreate/index_type/output.txt | 4 +- .../recreate/index_type/script | 6 +- libs/testserver/fake_workspace.go | 49 +++++++-------- libs/testserver/handlers.go | 4 +- libs/testserver/vector_search_indexes.go | 59 ------------------- 6 files changed, 38 insertions(+), 92 deletions(-) diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.requests.recreate.direct.json b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.requests.recreate.direct.json index 6a1ebfc70eb..1149b9530b6 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.requests.recreate.direct.json +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.requests.recreate.direct.json @@ -1,7 +1,15 @@ +{ + "method": "GET", + "path": "/api/2.0/vector-search/indexes/vs-index-[UNIQUE_NAME]" +} { "method": "DELETE", "path": "/api/2.0/vector-search/indexes/vs-index-[UNIQUE_NAME]" } +{ + "method": "GET", + "path": "/api/2.0/vector-search/indexes/vs-index-[UNIQUE_NAME]" +} { "method": "POST", "path": "/api/2.0/vector-search/indexes", diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/output.txt b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/output.txt index 87df5820eda..e4f9b741ff6 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/output.txt @@ -13,7 +13,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py --keep //vector-search/indexes +>>> print_requests.py --get //vector-search/indexes === Change index_type (should trigger recreation) >>> update_file.py databricks.yml index_type: DELTA_SYNC index_type: DIRECT_ACCESS @@ -40,7 +40,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py --keep //vector-search/indexes +>>> print_requests.py --get //vector-search/indexes >>> [CLI] vector-search-indexes get-index vs-index-[UNIQUE_NAME] { diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/script b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/script index 2e463ab45bf..bb248b50ec7 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/script +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/script @@ -10,7 +10,11 @@ trap cleanup EXIT print_requests() { local name=$1 - trace print_requests.py --keep '//vector-search/indexes' > out.requests.${name}.$DATABRICKS_BUNDLE_ENGINE.json + # --get so the GET poll from WaitAfterDelete shows up in the recreate log; + # without that, removing the wait would silently still pass against the + # testserver (which finishes deletion synchronously) — the GET in the log + # is the assertion that the wait actually fired. + trace print_requests.py --get '//vector-search/indexes' > out.requests.${name}.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt } diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index da179e3f70b..51bacfbf2b4 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -154,12 +154,6 @@ type FakeWorkspace struct { VectorSearchEndpoints map[string]vectorsearch.EndpointInfo VectorSearchIndexes map[string]vectorsearch.VectorIndex - // vectorSearchIndexesPendingDeletion stores indexes that have received a DELETE - // but whose deletion has not yet "completed". GETs against a name in this map - // return the index once (mirroring the real backend's async deletion window), - // then the entry is consumed and subsequent GETs return 404. - vectorSearchIndexesPendingDeletion map[string]vectorsearch.VectorIndex - SecretScopes map[string]workspace.SecretScope Secrets map[string]map[string]string // scope -> key -> value Acls map[string][]workspace.AclItem @@ -292,28 +286,27 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { State: sql.StateRunning, }, }, - ServingEndpoints: map[string]serving.ServingEndpointDetailed{}, - VectorSearchEndpoints: map[string]vectorsearch.EndpointInfo{}, - VectorSearchIndexes: map[string]vectorsearch.VectorIndex{}, - vectorSearchIndexesPendingDeletion: map[string]vectorsearch.VectorIndex{}, - Repos: map[string]workspace.RepoInfo{}, - SecretScopes: map[string]workspace.SecretScope{}, - Secrets: map[string]map[string]string{}, - Acls: map[string][]workspace.AclItem{}, - Permissions: map[string]iam.ObjectPermissions{}, - Groups: map[string]iam.Group{}, - DatabaseInstances: map[string]database.DatabaseInstance{}, - DatabaseCatalogs: map[string]database.DatabaseCatalog{}, - SyncedDatabaseTables: map[string]database.SyncedDatabaseTable{}, - PostgresProjects: map[string]postgres.Project{}, - PostgresBranches: map[string]postgres.Branch{}, - PostgresEndpoints: map[string]postgres.Endpoint{}, - PostgresOperations: map[string]postgres.Operation{}, - clusterVenvs: map[string]*clusterEnv{}, - Alerts: map[string]sql.AlertV2{}, - Experiments: map[string]ml.GetExperimentResponse{}, - ModelRegistryModels: map[string]ml.Model{}, - ModelRegistryModelIDs: map[string]string{}, + ServingEndpoints: map[string]serving.ServingEndpointDetailed{}, + VectorSearchEndpoints: map[string]vectorsearch.EndpointInfo{}, + VectorSearchIndexes: map[string]vectorsearch.VectorIndex{}, + Repos: map[string]workspace.RepoInfo{}, + SecretScopes: map[string]workspace.SecretScope{}, + Secrets: map[string]map[string]string{}, + Acls: map[string][]workspace.AclItem{}, + Permissions: map[string]iam.ObjectPermissions{}, + Groups: map[string]iam.Group{}, + DatabaseInstances: map[string]database.DatabaseInstance{}, + DatabaseCatalogs: map[string]database.DatabaseCatalog{}, + SyncedDatabaseTables: map[string]database.SyncedDatabaseTable{}, + PostgresProjects: map[string]postgres.Project{}, + PostgresBranches: map[string]postgres.Branch{}, + PostgresEndpoints: map[string]postgres.Endpoint{}, + PostgresOperations: map[string]postgres.Operation{}, + clusterVenvs: map[string]*clusterEnv{}, + Alerts: map[string]sql.AlertV2{}, + Experiments: map[string]ml.GetExperimentResponse{}, + ModelRegistryModels: map[string]ml.Model{}, + ModelRegistryModelIDs: map[string]string{}, Clusters: map[string]compute.ClusterDetails{ TestDefaultClusterId: { ClusterId: TestDefaultClusterId, diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 4240f330267..d8658964ae6 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -835,11 +835,11 @@ func AddDefaultHandlers(server *Server) { }) server.Handle("GET", "/api/2.0/vector-search/indexes/{index_name}", func(req Request) any { - return req.Workspace.VectorSearchIndexGet(req.Vars["index_name"]) + return MapGet(req.Workspace, req.Workspace.VectorSearchIndexes, req.Vars["index_name"]) }) server.Handle("DELETE", "/api/2.0/vector-search/indexes/{index_name}", func(req Request) any { - return req.Workspace.VectorSearchIndexDelete(req.Vars["index_name"]) + return MapDelete(req.Workspace, req.Workspace.VectorSearchIndexes, req.Vars["index_name"]) }) // Generic permissions endpoints diff --git a/libs/testserver/vector_search_indexes.go b/libs/testserver/vector_search_indexes.go index 3d559cad61d..a2a7848ad94 100644 --- a/libs/testserver/vector_search_indexes.go +++ b/libs/testserver/vector_search_indexes.go @@ -25,15 +25,6 @@ func (s *FakeWorkspace) VectorSearchIndexCreate(req Request) Response { Body: map[string]string{"error_code": "RESOURCE_ALREADY_EXISTS", "message": fmt.Sprintf("Vector search index with name %s already exists", createReq.Name)}, } } - if _, pending := s.vectorSearchIndexesPendingDeletion[createReq.Name]; pending { - return Response{ - StatusCode: http.StatusBadRequest, - Body: map[string]string{ - "error_code": "INVALID_PARAMETER_VALUE", - "message": fmt.Sprintf("Index %s is currently pending deletion. Operations on the index are not permitted while the index is being deleted.", createReq.Name), - }, - } - } if _, exists := s.VectorSearchEndpoints[createReq.EndpointName]; !exists { return Response{ StatusCode: http.StatusNotFound, @@ -77,53 +68,3 @@ func remapDeltaSyncSpec(req *vectorsearch.DeltaSyncVectorIndexSpecRequest) *vect SourceTable: req.SourceTable, } } - -// VectorSearchIndexDelete moves the index out of the live map into a -// "pending deletion" buffer. CREATE on the same name returns the -// pending-deletion error until a GET (i.e. a poll) consumes the buffer entry, -// which is how the real backend's async deletion window manifests to a client -// that doesn't wait between DELETE and CREATE. -func (s *FakeWorkspace) VectorSearchIndexDelete(indexName string) Response { - defer s.LockUnlock()() - - index, ok := s.VectorSearchIndexes[indexName] - if !ok { - return Response{ - StatusCode: http.StatusNotFound, - Body: map[string]string{ - "error_code": "RESOURCE_DOES_NOT_EXIST", - "message": fmt.Sprintf("Vector search index %s not found", indexName), - }, - } - } - if index.Status == nil { - index.Status = &vectorsearch.VectorIndexStatus{} - } - index.Status.Ready = false - index.Status.Message = "Index is currently pending deletion" - s.vectorSearchIndexesPendingDeletion[indexName] = index - delete(s.VectorSearchIndexes, indexName) - return Response{} -} - -// VectorSearchIndexGet returns 404 once the index has been moved to the -// pending-deletion buffer. The pending entry is consumed as a side effect, so -// CREATE-during-pending only fires when the caller skips the wait and races -// straight from DELETE to CREATE without polling. This preserves the -// "remote not found -> recreate" behavior for tests that delete out-of-band -// and immediately re-plan. -func (s *FakeWorkspace) VectorSearchIndexGet(indexName string) Response { - defer s.LockUnlock()() - - if index, ok := s.VectorSearchIndexes[indexName]; ok { - return Response{Body: index} - } - delete(s.vectorSearchIndexesPendingDeletion, indexName) - return Response{ - StatusCode: http.StatusNotFound, - Body: map[string]string{ - "error_code": "RESOURCE_DOES_NOT_EXIST", - "message": fmt.Sprintf("Vector search index %s not found", indexName), - }, - } -} From 053add1ccac45dded7f5fd67d82b0a9d461c21c6 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 29 Apr 2026 16:26:28 +0200 Subject: [PATCH 16/47] test: drop unhappy-path VS index URL case from initialize_urls_test The vectorsearchindex2 fixture (bare name, no catalog.schema prefix) just exercised the early-return in InitializeURL when the name isn't a 3-part identifier. That doesn't earn a unit test. Co-authored-by: Isaac --- bundle/config/mutator/initialize_urls_test.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/bundle/config/mutator/initialize_urls_test.go b/bundle/config/mutator/initialize_urls_test.go index da26e02ce40..393f50271d7 100644 --- a/bundle/config/mutator/initialize_urls_test.go +++ b/bundle/config/mutator/initialize_urls_test.go @@ -76,12 +76,6 @@ func TestInitializeURLs(t *testing.T) { Name: "catalog.schema.vectorsearchindex1", }, }, - "vectorsearchindex2": { - BaseResource: resources.BaseResource{ID: "vectorsearchindex2"}, - CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{ - Name: "vectorsearchindex2", - }, - }, }, Schemas: map[string]*resources.Schema{ "schema1": { @@ -120,7 +114,6 @@ func TestInitializeURLs(t *testing.T) { "registeredmodel1": "https://mycompany.databricks.com/explore/data/models/8?o=123456", "qualityMonitor1": "https://mycompany.databricks.com/explore/data/catalog/schema/qualityMonitor1?o=123456", "vectorsearchindex1": "https://mycompany.databricks.com/explore/data/catalog/schema/vectorsearchindex1?o=123456", - "vectorsearchindex2": "", "schema1": "https://mycompany.databricks.com/explore/data/catalog/schema?o=123456", "cluster1": "https://mycompany.databricks.com/compute/clusters/1017-103929-vlr7jzcf?o=123456", "dashboard1": "https://mycompany.databricks.com/dashboardsv3/01ef8d56871e1d50ae30ce7375e42478/published?o=123456", From 11986252513bd14b9d6e330096fbdf145cde23b4 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 29 Apr 2026 17:07:03 +0200 Subject: [PATCH 17/47] Restore usage_policy_id PLACEHOLDER description in annotations.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous revert changed "usage_policy_id": {} back to bare "usage_policy_id":, which also dropped the PLACEHOLDER description that was already on main — leaving the new VectorSearchIndex block visually inserted between usage_policy_id and where its description used to be. Restore the description so the diff against main is just the new VectorSearchIndex entries, with usage_policy_id untouched. Co-authored-by: Isaac --- bundle/internal/schema/annotations.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index faa6a4f0cbb..ae01c42df21 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -987,6 +987,8 @@ github.com/databricks/cli/bundle/config/resources.VectorSearchEndpoint: "description": |- PLACEHOLDER "usage_policy_id": + "description": |- + PLACEHOLDER github.com/databricks/cli/bundle/config/resources.VectorSearchIndex: "delta_sync_index_spec": "description": |- From 38f8005f521a36fabc8a8e641df97a45781d6f4f Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 29 Apr 2026 17:27:32 +0200 Subject: [PATCH 18/47] Revert run-local-node output.txt drift The drift came from a local NODE_OPTIONS leak in the harness, not from this PR's changes. Restore the on-main version. Co-authored-by: Isaac --- .../cmd/workspace/apps/run-local-node/output.txt | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/acceptance/cmd/workspace/apps/run-local-node/output.txt b/acceptance/cmd/workspace/apps/run-local-node/output.txt index 4de672232f6..0185dbe5234 100644 --- a/acceptance/cmd/workspace/apps/run-local-node/output.txt +++ b/acceptance/cmd/workspace/apps/run-local-node/output.txt @@ -1,2 +1,12 @@ +Running command: node -e console.log('Hello, world') +Hello, world -Exit code: 1 +=== Starting the app in background... +=== Waiting +=== Checking app is running... +>>> curl -s -o - http://127.0.0.1:$(port) +{"message":"Hello From App","timestamp":"[TIMESTAMP]","status":"running"} + +=== Sending shutdown request... +>>> curl -s -o /dev/null http://127.0.0.1:$(port)/shutdown +Process terminated From 019d4246ba8c60dc85df93f4d94cbe4f22db1376 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 29 Apr 2026 17:29:57 +0200 Subject: [PATCH 19/47] acceptance: drop my_schema fixture from presets_name_prefix Reuse the existing schema1 in the index name's interpolation so the diff against main is purely vector_search additions, no schemas-block churn. Co-authored-by: Isaac --- .../presets_name_prefix/databricks.yml.tmpl | 5 +---- .../validate/presets_name_prefix/output.txt | 18 +++--------------- 2 files changed, 4 insertions(+), 19 deletions(-) diff --git a/acceptance/bundle/validate/presets_name_prefix/databricks.yml.tmpl b/acceptance/bundle/validate/presets_name_prefix/databricks.yml.tmpl index 74887b548e1..b9cf57cf909 100644 --- a/acceptance/bundle/validate/presets_name_prefix/databricks.yml.tmpl +++ b/acceptance/bundle/validate/presets_name_prefix/databricks.yml.tmpl @@ -13,9 +13,6 @@ resources: schema1: catalog_name: c name: schema1 - my_schema: - catalog_name: my_catalog - name: my_schema volumes: volume1: @@ -34,7 +31,7 @@ resources: vector_search_indexes: vs_index: - name: my_catalog.${resources.schemas.my_schema.name}.my_index + name: c.${resources.schemas.schema1.name}.my_index endpoint_name: vs_endpoint primary_key: id index_type: DELTA_SYNC diff --git a/acceptance/bundle/validate/presets_name_prefix/output.txt b/acceptance/bundle/validate/presets_name_prefix/output.txt index f0cd13ba80f..8a37712ab98 100644 --- a/acceptance/bundle/validate/presets_name_prefix/output.txt +++ b/acceptance/bundle/validate/presets_name_prefix/output.txt @@ -25,10 +25,6 @@ } }, "schemas": { - "my_schema": { - "catalog_name": "my_catalog", - "name": "prefixmy_schema" - }, "schema1": { "catalog_name": "c", "name": "prefixschema1" @@ -44,7 +40,7 @@ "vs_index": { "endpoint_name": "vs_endpoint", "index_type": "DELTA_SYNC", - "name": "my_catalog.${resources.schemas.my_schema.name}.prefixmy_index", + "name": "c.${resources.schemas.schema1.name}.prefixmy_index", "primary_key": "id" } }, @@ -84,10 +80,6 @@ } }, "schemas": { - "my_schema": { - "catalog_name": "my_catalog", - "name": "prefix_my_schema" - }, "schema1": { "catalog_name": "c", "name": "prefix_schema1" @@ -103,7 +95,7 @@ "vs_index": { "endpoint_name": "vs_endpoint", "index_type": "DELTA_SYNC", - "name": "my_catalog.${resources.schemas.my_schema.name}.prefix_my_index", + "name": "c.${resources.schemas.schema1.name}.prefix_my_index", "primary_key": "id" } }, @@ -143,10 +135,6 @@ } }, "schemas": { - "my_schema": { - "catalog_name": "my_catalog", - "name": "my_schema" - }, "schema1": { "catalog_name": "c", "name": "schema1" @@ -162,7 +150,7 @@ "vs_index": { "endpoint_name": "vs_endpoint", "index_type": "DELTA_SYNC", - "name": "my_catalog.${resources.schemas.my_schema.name}.my_index", + "name": "c.${resources.schemas.schema1.name}.my_index", "primary_key": "id" } }, From f00123b934ac3663deaf27cb2463e01f6938ef99 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 29 Apr 2026 17:34:34 +0200 Subject: [PATCH 20/47] Revert unrelated comment tweak in apply.go::Delete Restore the original wording on the permission-error comment; the edit slipped in during the WaitAfterDelete addition and isn't related to this PR. Co-authored-by: Isaac --- bundle/direct/apply.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index 9ca00d9a28f..6921769bd97 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -181,7 +181,7 @@ func (d *DeploymentUnit) Delete(ctx context.Context, db *dstate.DeploymentState, // Rather than failing delete and requiring user to unbind, we perform unbind automatically there. // Some services, e.g. jobs, return 403 for missing resources if caller did not have permissions to it when job existed. // In those cases 403 hides 404. In other cases, user not having permissions to resource but having in the bundle might - // mean configuration error that user is trying to fix by permission-restoring or removing the resource from their bundle. + // mean configuration error that user is trying to fix by removing resource from their bundle. if errors.Is(err, apierr.ErrPermissionDenied) { log.Warnf(ctx, "Ignoring permission error when deleting %s id=%s: %s", d.ResourceKey, oldID, err) } else { From 653616a9bc44b333d017a2fea25176747974cc6e Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 29 Apr 2026 17:43:08 +0200 Subject: [PATCH 21/47] acceptance: drop schema reference from presets_name_prefix index name Hardcode the 3-part index name so the diff against main is purely vector_search additions. The test still demonstrates leaf-only prefixing on a 3-part identifier; the cross-resource reference path is covered elsewhere. Co-authored-by: Isaac --- .../bundle/validate/presets_name_prefix/databricks.yml.tmpl | 2 +- acceptance/bundle/validate/presets_name_prefix/output.txt | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/validate/presets_name_prefix/databricks.yml.tmpl b/acceptance/bundle/validate/presets_name_prefix/databricks.yml.tmpl index b9cf57cf909..d30bfab79f9 100644 --- a/acceptance/bundle/validate/presets_name_prefix/databricks.yml.tmpl +++ b/acceptance/bundle/validate/presets_name_prefix/databricks.yml.tmpl @@ -31,7 +31,7 @@ resources: vector_search_indexes: vs_index: - name: c.${resources.schemas.schema1.name}.my_index + name: my_catalog.my_schema.my_index endpoint_name: vs_endpoint primary_key: id index_type: DELTA_SYNC diff --git a/acceptance/bundle/validate/presets_name_prefix/output.txt b/acceptance/bundle/validate/presets_name_prefix/output.txt index 8a37712ab98..3c633c7d743 100644 --- a/acceptance/bundle/validate/presets_name_prefix/output.txt +++ b/acceptance/bundle/validate/presets_name_prefix/output.txt @@ -40,7 +40,7 @@ "vs_index": { "endpoint_name": "vs_endpoint", "index_type": "DELTA_SYNC", - "name": "c.${resources.schemas.schema1.name}.prefixmy_index", + "name": "my_catalog.my_schema.prefixmy_index", "primary_key": "id" } }, @@ -95,7 +95,7 @@ "vs_index": { "endpoint_name": "vs_endpoint", "index_type": "DELTA_SYNC", - "name": "c.${resources.schemas.schema1.name}.prefix_my_index", + "name": "my_catalog.my_schema.prefix_my_index", "primary_key": "id" } }, @@ -150,7 +150,7 @@ "vs_index": { "endpoint_name": "vs_endpoint", "index_type": "DELTA_SYNC", - "name": "c.${resources.schemas.schema1.name}.my_index", + "name": "my_catalog.my_schema.my_index", "primary_key": "id" } }, From 772a69b64fe0a60b0b77c27ce8f12dc187054b51 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 30 Apr 2026 14:24:28 +0200 Subject: [PATCH 22/47] direct: preserve index_subtype during VS index remote remap RemapState was hardcoding IndexSubtype to the empty string, which would classify any remote with a populated subtype as drift on the next plan and force a needless recreate. Pass through remote.IndexSubtype like the other read-back fields. Co-authored-by: Isaac --- bundle/direct/dresources/vector_search_index.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundle/direct/dresources/vector_search_index.go b/bundle/direct/dresources/vector_search_index.go index 307ac9988c9..e68170c81f5 100644 --- a/bundle/direct/dresources/vector_search_index.go +++ b/bundle/direct/dresources/vector_search_index.go @@ -73,7 +73,7 @@ func (*ResourceVectorSearchIndex) RemapState(remote *VectorSearchIndexRemote) *V CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{ DeltaSyncIndexSpec: nil, DirectAccessIndexSpec: nil, - IndexSubtype: "", + IndexSubtype: remote.IndexSubtype, Name: remote.Name, EndpointName: remote.EndpointName, IndexType: remote.IndexType, From cc85db4521c9b9b86ed8ee50075e2468792198a9 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 30 Apr 2026 14:24:39 +0200 Subject: [PATCH 23/47] direct: treat VS index name and index_subtype as recreate-on-change The Vector Search index API has no rename or update path, so any config-side change has to round-trip through delete + create. Add name and index_subtype to recreate_on_changes so the planner picks them up the same way it already does for endpoint_name, index_type, primary_key, and the spec blocks. Co-authored-by: Isaac --- bundle/direct/dresources/resources.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 7cf21503f1e..6ff6431aa2f 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -520,10 +520,16 @@ resources: vector_search_indexes: recreate_on_changes: + # The index API has no rename or update path, so every config change + # has to go through delete + create. + - field: name + reason: immutable - field: endpoint_name reason: immutable - field: index_type reason: immutable + - field: index_subtype + reason: immutable - field: primary_key reason: immutable - field: delta_sync_index_spec From 5879daff648205c835ee3236c5803a87ead50ab3 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 30 Apr 2026 14:24:51 +0200 Subject: [PATCH 24/47] presets: skip VS index name prefix when refs are unresolved The leaf-prefix logic splits on the last dot in the 3-part UC name and prepends the user prefix to whatever follows. If the name still has literal ${...} tokens (e.g. ${var.catalog}.${var.schema}.${var.index}), that split lands inside the trailing ref expression and rewrites the variable name itself. Detect unresolved refs and bail; users who want the dev prefix in this case can compose it into the variable. Co-authored-by: Isaac --- .../mutator/resourcemutator/apply_presets.go | 9 +++++++++ .../resourcemutator/apply_target_mode_test.go | 19 +++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/bundle/config/mutator/resourcemutator/apply_presets.go b/bundle/config/mutator/resourcemutator/apply_presets.go index 59682f8141c..7afc9a00c4e 100644 --- a/bundle/config/mutator/resourcemutator/apply_presets.go +++ b/bundle/config/mutator/resourcemutator/apply_presets.go @@ -297,10 +297,19 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos // Vector Search Indexes: Prefix // The name is a 3-part UC identifier (catalog.schema.index); prefix only // the last component since catalog and schema are external references. + // This mutator runs before reference/variable resolution, so the name may + // still carry literal `${...}` tokens. Splitting on the last dot in that + // case would inject the prefix inside the ref expression itself + // (e.g. `${var.full_name}` -> `${var.dev_user_full_name}`); skip + // prefixing when refs are present and let the user compose the prefix + // into the variable. for _, e := range r.VectorSearchIndexes { if e == nil { continue } + if strings.Contains(e.Name, "${") { + continue + } if i := strings.LastIndex(e.Name, "."); i >= 0 { e.Name = e.Name[:i+1] + normalizePrefix(prefix) + e.Name[i+1:] } else { diff --git a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go index 84d2645a77a..5bbd376ac10 100644 --- a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go +++ b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go @@ -506,6 +506,25 @@ func TestDisableLockingDisabled(t *testing.T) { assert.True(t, b.Config.Bundle.Deployment.Lock.IsEnabled(), "Deployment lock should remain enabled in development mode when explicitly enabled") } +func TestVectorSearchIndexNameWithUnresolvedRefsLeftAlone(t *testing.T) { + b := mockBundle(config.Development) + b.Config.Resources.VectorSearchIndexes["vs_index_with_var"] = &resources.VectorSearchIndex{ + CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{ + Name: "${var.catalog}.${var.schema}.${var.index}", + EndpointName: "vs_endpoint1", + PrimaryKey: "id", + IndexType: vectorsearch.VectorIndexTypeDeltaSync, + }, + } + + diags := bundle.ApplySeq(t.Context(), b, ApplyTargetMode(), ApplyPresets()) + require.NoError(t, diags.Error()) + + // The leaf-finding splits on the last dot, which would otherwise inject the + // prefix inside the trailing ${var.index} expression. + assert.Equal(t, "${var.catalog}.${var.schema}.${var.index}", b.Config.Resources.VectorSearchIndexes["vs_index_with_var"].Name) +} + func TestPrefixAlreadySet(t *testing.T) { b := mockBundle(config.Development) b.Config.Presets.NamePrefix = "custom_lennart_deploy_" From a9e2c1c4f119acdb6d3454225ac040987af3dcde Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 30 Apr 2026 14:25:00 +0200 Subject: [PATCH 25/47] validate: enforce matching spec block for VS index_type CreateIndex rejects any combination where the spec block doesn't match the index_type (e.g. DELTA_SYNC with direct_access_index_spec set, or DIRECT_ACCESS with neither block at all). Add a fast validator that reports those mismatches at validate time so the failure surfaces before the deploy starts running. Co-authored-by: Isaac --- bundle/config/validate/fast_validate.go | 1 + .../validate/vector_search_index_spec.go | 79 +++++++++++++ .../validate/vector_search_index_spec_test.go | 106 ++++++++++++++++++ 3 files changed, 186 insertions(+) create mode 100644 bundle/config/validate/vector_search_index_spec.go create mode 100644 bundle/config/validate/vector_search_index_spec_test.go diff --git a/bundle/config/validate/fast_validate.go b/bundle/config/validate/fast_validate.go index d01eb8c1491..1031a4a62df 100644 --- a/bundle/config/validate/fast_validate.go +++ b/bundle/config/validate/fast_validate.go @@ -29,6 +29,7 @@ func (f *fastValidate) Apply(ctx context.Context, rb *bundle.Bundle) diag.Diagno // Fast mutators with only in-memory checks JobClusterKeyDefined(), JobTaskClusterSpec(), + VectorSearchIndexSpec(), // Blocking mutators. Deployments will fail if these checks fail. ValidateArtifactPath(), diff --git a/bundle/config/validate/vector_search_index_spec.go b/bundle/config/validate/vector_search_index_spec.go new file mode 100644 index 00000000000..27ffed29858 --- /dev/null +++ b/bundle/config/validate/vector_search_index_spec.go @@ -0,0 +1,79 @@ +package validate + +import ( + "context" + "fmt" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/databricks-sdk-go/service/vectorsearch" +) + +// VectorSearchIndexSpec validates that each vector_search_indexes resource +// declares the spec block matching its index_type. The CreateIndex API rejects +// mismatched combinations at deploy time; reject them at validate time so the +// failure is visible before the deploy starts. +func VectorSearchIndexSpec() bundle.ReadOnlyMutator { + return &vectorSearchIndexSpec{} +} + +type vectorSearchIndexSpec struct{ bundle.RO } + +func (*vectorSearchIndexSpec) Name() string { + return "validate:vector_search_index_spec" +} + +func (*vectorSearchIndexSpec) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + + root := dyn.NewPath(dyn.Key("resources"), dyn.Key("vector_search_indexes")) + + for name, idx := range b.Config.Resources.VectorSearchIndexes { + if idx == nil { + continue + } + path := root.Append(dyn.Key(name)) + diags = diags.Extend(validateVectorSearchIndexSpec(b, idx.IndexType, idx.DeltaSyncIndexSpec != nil, idx.DirectAccessIndexSpec != nil, path)) + } + + return diags +} + +func validateVectorSearchIndexSpec(b *bundle.Bundle, indexType vectorsearch.VectorIndexType, hasDeltaSync, hasDirectAccess bool, path dyn.Path) diag.Diagnostics { + switch indexType { + case vectorsearch.VectorIndexTypeDeltaSync: + if !hasDeltaSync { + return missingSpecDiag(b, path, "delta_sync_index_spec", indexType) + } + if hasDirectAccess { + return incompatibleSpecDiag(b, path, "direct_access_index_spec", indexType) + } + case vectorsearch.VectorIndexTypeDirectAccess: + if !hasDirectAccess { + return missingSpecDiag(b, path, "direct_access_index_spec", indexType) + } + if hasDeltaSync { + return incompatibleSpecDiag(b, path, "delta_sync_index_spec", indexType) + } + } + return nil +} + +func missingSpecDiag(b *bundle.Bundle, path dyn.Path, field string, indexType vectorsearch.VectorIndexType) diag.Diagnostics { + return diag.Diagnostics{{ + Severity: diag.Error, + Summary: fmt.Sprintf("vector_search_indexes: missing %s for index_type %q", field, indexType), + Locations: b.Config.GetLocations(path.String()), + Paths: []dyn.Path{path.Append(dyn.Key(field))}, + }} +} + +func incompatibleSpecDiag(b *bundle.Bundle, path dyn.Path, field string, indexType vectorsearch.VectorIndexType) diag.Diagnostics { + return diag.Diagnostics{{ + Severity: diag.Error, + Summary: fmt.Sprintf("vector_search_indexes: %s is not allowed when index_type is %q", field, indexType), + Locations: b.Config.GetLocations(path.String()), + Paths: []dyn.Path{path.Append(dyn.Key(field))}, + }} +} diff --git a/bundle/config/validate/vector_search_index_spec_test.go b/bundle/config/validate/vector_search_index_spec_test.go new file mode 100644 index 00000000000..f5b770b8626 --- /dev/null +++ b/bundle/config/validate/vector_search_index_spec_test.go @@ -0,0 +1,106 @@ +package validate + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/vectorsearch" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestVectorSearchIndexSpec(t *testing.T) { + cases := []struct { + name string + index resources.VectorSearchIndex + wantError string + }{ + { + name: "delta sync with delta spec is valid", + index: resources.VectorSearchIndex{ + CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{ + Name: "main.default.idx", + IndexType: vectorsearch.VectorIndexTypeDeltaSync, + DeltaSyncIndexSpec: &vectorsearch.DeltaSyncVectorIndexSpecRequest{}, + }, + }, + }, + { + name: "direct access with direct spec is valid", + index: resources.VectorSearchIndex{ + CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{ + Name: "main.default.idx", + IndexType: vectorsearch.VectorIndexTypeDirectAccess, + DirectAccessIndexSpec: &vectorsearch.DirectAccessVectorIndexSpec{}, + }, + }, + }, + { + name: "delta sync without delta spec is rejected", + index: resources.VectorSearchIndex{ + CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{ + Name: "main.default.idx", + IndexType: vectorsearch.VectorIndexTypeDeltaSync, + }, + }, + wantError: `vector_search_indexes: missing delta_sync_index_spec for index_type "DELTA_SYNC"`, + }, + { + name: "delta sync with direct spec is rejected", + index: resources.VectorSearchIndex{ + CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{ + Name: "main.default.idx", + IndexType: vectorsearch.VectorIndexTypeDeltaSync, + DeltaSyncIndexSpec: &vectorsearch.DeltaSyncVectorIndexSpecRequest{}, + DirectAccessIndexSpec: &vectorsearch.DirectAccessVectorIndexSpec{}, + }, + }, + wantError: `vector_search_indexes: direct_access_index_spec is not allowed when index_type is "DELTA_SYNC"`, + }, + { + name: "direct access without direct spec is rejected", + index: resources.VectorSearchIndex{ + CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{ + Name: "main.default.idx", + IndexType: vectorsearch.VectorIndexTypeDirectAccess, + }, + }, + wantError: `vector_search_indexes: missing direct_access_index_spec for index_type "DIRECT_ACCESS"`, + }, + { + name: "direct access with delta spec is rejected", + index: resources.VectorSearchIndex{ + CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{ + Name: "main.default.idx", + IndexType: vectorsearch.VectorIndexTypeDirectAccess, + DeltaSyncIndexSpec: &vectorsearch.DeltaSyncVectorIndexSpecRequest{}, + DirectAccessIndexSpec: &vectorsearch.DirectAccessVectorIndexSpec{}, + }, + }, + wantError: `vector_search_indexes: delta_sync_index_spec is not allowed when index_type is "DIRECT_ACCESS"`, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + VectorSearchIndexes: map[string]*resources.VectorSearchIndex{ + "idx": &tc.index, + }, + }, + }, + } + diags := VectorSearchIndexSpec().Apply(t.Context(), b) + if tc.wantError == "" { + require.Empty(t, diags, "expected no diagnostics, got %v", diags) + return + } + require.Len(t, diags, 1) + assert.Equal(t, tc.wantError, diags[0].Summary) + }) + } +} From 1d5aab7336b8bf68be9d4c8a01c33c96979042f9 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 30 Apr 2026 15:09:38 +0200 Subject: [PATCH 26/47] Revert "validate: enforce matching spec block for VS index_type" This reverts commit edeceb240ecee72c76eba8f574d2fe8cec5276e5. --- bundle/config/validate/fast_validate.go | 1 - .../validate/vector_search_index_spec.go | 79 ------------- .../validate/vector_search_index_spec_test.go | 106 ------------------ 3 files changed, 186 deletions(-) delete mode 100644 bundle/config/validate/vector_search_index_spec.go delete mode 100644 bundle/config/validate/vector_search_index_spec_test.go diff --git a/bundle/config/validate/fast_validate.go b/bundle/config/validate/fast_validate.go index 1031a4a62df..d01eb8c1491 100644 --- a/bundle/config/validate/fast_validate.go +++ b/bundle/config/validate/fast_validate.go @@ -29,7 +29,6 @@ func (f *fastValidate) Apply(ctx context.Context, rb *bundle.Bundle) diag.Diagno // Fast mutators with only in-memory checks JobClusterKeyDefined(), JobTaskClusterSpec(), - VectorSearchIndexSpec(), // Blocking mutators. Deployments will fail if these checks fail. ValidateArtifactPath(), diff --git a/bundle/config/validate/vector_search_index_spec.go b/bundle/config/validate/vector_search_index_spec.go deleted file mode 100644 index 27ffed29858..00000000000 --- a/bundle/config/validate/vector_search_index_spec.go +++ /dev/null @@ -1,79 +0,0 @@ -package validate - -import ( - "context" - "fmt" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/databricks-sdk-go/service/vectorsearch" -) - -// VectorSearchIndexSpec validates that each vector_search_indexes resource -// declares the spec block matching its index_type. The CreateIndex API rejects -// mismatched combinations at deploy time; reject them at validate time so the -// failure is visible before the deploy starts. -func VectorSearchIndexSpec() bundle.ReadOnlyMutator { - return &vectorSearchIndexSpec{} -} - -type vectorSearchIndexSpec struct{ bundle.RO } - -func (*vectorSearchIndexSpec) Name() string { - return "validate:vector_search_index_spec" -} - -func (*vectorSearchIndexSpec) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { - var diags diag.Diagnostics - - root := dyn.NewPath(dyn.Key("resources"), dyn.Key("vector_search_indexes")) - - for name, idx := range b.Config.Resources.VectorSearchIndexes { - if idx == nil { - continue - } - path := root.Append(dyn.Key(name)) - diags = diags.Extend(validateVectorSearchIndexSpec(b, idx.IndexType, idx.DeltaSyncIndexSpec != nil, idx.DirectAccessIndexSpec != nil, path)) - } - - return diags -} - -func validateVectorSearchIndexSpec(b *bundle.Bundle, indexType vectorsearch.VectorIndexType, hasDeltaSync, hasDirectAccess bool, path dyn.Path) diag.Diagnostics { - switch indexType { - case vectorsearch.VectorIndexTypeDeltaSync: - if !hasDeltaSync { - return missingSpecDiag(b, path, "delta_sync_index_spec", indexType) - } - if hasDirectAccess { - return incompatibleSpecDiag(b, path, "direct_access_index_spec", indexType) - } - case vectorsearch.VectorIndexTypeDirectAccess: - if !hasDirectAccess { - return missingSpecDiag(b, path, "direct_access_index_spec", indexType) - } - if hasDeltaSync { - return incompatibleSpecDiag(b, path, "delta_sync_index_spec", indexType) - } - } - return nil -} - -func missingSpecDiag(b *bundle.Bundle, path dyn.Path, field string, indexType vectorsearch.VectorIndexType) diag.Diagnostics { - return diag.Diagnostics{{ - Severity: diag.Error, - Summary: fmt.Sprintf("vector_search_indexes: missing %s for index_type %q", field, indexType), - Locations: b.Config.GetLocations(path.String()), - Paths: []dyn.Path{path.Append(dyn.Key(field))}, - }} -} - -func incompatibleSpecDiag(b *bundle.Bundle, path dyn.Path, field string, indexType vectorsearch.VectorIndexType) diag.Diagnostics { - return diag.Diagnostics{{ - Severity: diag.Error, - Summary: fmt.Sprintf("vector_search_indexes: %s is not allowed when index_type is %q", field, indexType), - Locations: b.Config.GetLocations(path.String()), - Paths: []dyn.Path{path.Append(dyn.Key(field))}, - }} -} diff --git a/bundle/config/validate/vector_search_index_spec_test.go b/bundle/config/validate/vector_search_index_spec_test.go deleted file mode 100644 index f5b770b8626..00000000000 --- a/bundle/config/validate/vector_search_index_spec_test.go +++ /dev/null @@ -1,106 +0,0 @@ -package validate - -import ( - "testing" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/databricks-sdk-go/service/vectorsearch" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestVectorSearchIndexSpec(t *testing.T) { - cases := []struct { - name string - index resources.VectorSearchIndex - wantError string - }{ - { - name: "delta sync with delta spec is valid", - index: resources.VectorSearchIndex{ - CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{ - Name: "main.default.idx", - IndexType: vectorsearch.VectorIndexTypeDeltaSync, - DeltaSyncIndexSpec: &vectorsearch.DeltaSyncVectorIndexSpecRequest{}, - }, - }, - }, - { - name: "direct access with direct spec is valid", - index: resources.VectorSearchIndex{ - CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{ - Name: "main.default.idx", - IndexType: vectorsearch.VectorIndexTypeDirectAccess, - DirectAccessIndexSpec: &vectorsearch.DirectAccessVectorIndexSpec{}, - }, - }, - }, - { - name: "delta sync without delta spec is rejected", - index: resources.VectorSearchIndex{ - CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{ - Name: "main.default.idx", - IndexType: vectorsearch.VectorIndexTypeDeltaSync, - }, - }, - wantError: `vector_search_indexes: missing delta_sync_index_spec for index_type "DELTA_SYNC"`, - }, - { - name: "delta sync with direct spec is rejected", - index: resources.VectorSearchIndex{ - CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{ - Name: "main.default.idx", - IndexType: vectorsearch.VectorIndexTypeDeltaSync, - DeltaSyncIndexSpec: &vectorsearch.DeltaSyncVectorIndexSpecRequest{}, - DirectAccessIndexSpec: &vectorsearch.DirectAccessVectorIndexSpec{}, - }, - }, - wantError: `vector_search_indexes: direct_access_index_spec is not allowed when index_type is "DELTA_SYNC"`, - }, - { - name: "direct access without direct spec is rejected", - index: resources.VectorSearchIndex{ - CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{ - Name: "main.default.idx", - IndexType: vectorsearch.VectorIndexTypeDirectAccess, - }, - }, - wantError: `vector_search_indexes: missing direct_access_index_spec for index_type "DIRECT_ACCESS"`, - }, - { - name: "direct access with delta spec is rejected", - index: resources.VectorSearchIndex{ - CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{ - Name: "main.default.idx", - IndexType: vectorsearch.VectorIndexTypeDirectAccess, - DeltaSyncIndexSpec: &vectorsearch.DeltaSyncVectorIndexSpecRequest{}, - DirectAccessIndexSpec: &vectorsearch.DirectAccessVectorIndexSpec{}, - }, - }, - wantError: `vector_search_indexes: delta_sync_index_spec is not allowed when index_type is "DIRECT_ACCESS"`, - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - VectorSearchIndexes: map[string]*resources.VectorSearchIndex{ - "idx": &tc.index, - }, - }, - }, - } - diags := VectorSearchIndexSpec().Apply(t.Context(), b) - if tc.wantError == "" { - require.Empty(t, diags, "expected no diagnostics, got %v", diags) - return - } - require.Len(t, diags, 1) - assert.Equal(t, tc.wantError, diags[0].Summary) - }) - } -} From 47253ef973b41abd4b7fbd6a2783a1ed6aedcde0 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 30 Apr 2026 16:13:21 +0200 Subject: [PATCH 27/47] acceptance: regenerate after rebase onto main The out.test.toml format changed in #5146 ("acc: Format out.test.toml in diff-friendly and copypaste-friendly way"), and refschema picked up index_subtype and endpoint_uuid from the resource model. Pure regen from running ./task generate-refschema and ./task test-update. Co-authored-by: Isaac --- acceptance/bundle/invariant/continue_293/out.test.toml | 1 + acceptance/bundle/invariant/migrate/out.test.toml | 1 + acceptance/bundle/invariant/no_drift/out.test.toml | 1 + acceptance/bundle/refschema/out.fields.txt | 2 ++ .../resources/vector_search_indexes/basic/out.test.toml | 4 +--- .../vector_search_indexes/drift/columns_to_sync/out.test.toml | 4 +--- .../drift/deleted_remotely/out.test.toml | 4 +--- .../drift/orphaned_endpoint/out.test.toml | 4 +--- .../vector_search_indexes/grants/select/out.test.toml | 4 +--- .../vector_search_indexes/recreate/index_type/out.test.toml | 4 +--- 10 files changed, 11 insertions(+), 18 deletions(-) diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml index 11aaf584918..045f642cd7a 100644 --- a/acceptance/bundle/invariant/continue_293/out.test.toml +++ b/acceptance/bundle/invariant/continue_293/out.test.toml @@ -38,6 +38,7 @@ EnvMatrix.INPUT_CONFIG = [ "sql_warehouse.yml.tmpl", "synced_database_table.yml.tmpl", "vector_search_endpoint.yml.tmpl", + "vector_search_index.yml.tmpl", "volume.yml.tmpl", "volume_external.yml.tmpl" ] diff --git a/acceptance/bundle/invariant/migrate/out.test.toml b/acceptance/bundle/invariant/migrate/out.test.toml index 11aaf584918..045f642cd7a 100644 --- a/acceptance/bundle/invariant/migrate/out.test.toml +++ b/acceptance/bundle/invariant/migrate/out.test.toml @@ -38,6 +38,7 @@ EnvMatrix.INPUT_CONFIG = [ "sql_warehouse.yml.tmpl", "synced_database_table.yml.tmpl", "vector_search_endpoint.yml.tmpl", + "vector_search_index.yml.tmpl", "volume.yml.tmpl", "volume_external.yml.tmpl" ] diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index 11aaf584918..045f642cd7a 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -38,6 +38,7 @@ EnvMatrix.INPUT_CONFIG = [ "sql_warehouse.yml.tmpl", "synced_database_table.yml.tmpl", "vector_search_endpoint.yml.tmpl", + "vector_search_index.yml.tmpl", "volume.yml.tmpl", "volume_external.yml.tmpl" ] diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index eade09c997a..9bd92911787 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -3187,7 +3187,9 @@ resources.vector_search_indexes.*.direct_access_index_spec.embedding_vector_colu resources.vector_search_indexes.*.direct_access_index_spec.embedding_vector_columns[*].name string ALL resources.vector_search_indexes.*.direct_access_index_spec.schema_json string ALL resources.vector_search_indexes.*.endpoint_name string ALL +resources.vector_search_indexes.*.endpoint_uuid string REMOTE STATE resources.vector_search_indexes.*.id string INPUT +resources.vector_search_indexes.*.index_subtype vectorsearch.IndexSubtype ALL resources.vector_search_indexes.*.index_type vectorsearch.VectorIndexType ALL resources.vector_search_indexes.*.lifecycle resources.Lifecycle INPUT resources.vector_search_indexes.*.lifecycle.prevent_destroy bool INPUT diff --git a/acceptance/bundle/resources/vector_search_indexes/basic/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/basic/out.test.toml index f1d40380d02..fe4076cdf9b 100644 --- a/acceptance/bundle/resources/vector_search_indexes/basic/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/basic/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresUnityCatalog = true - -[EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/out.test.toml index f1d40380d02..fe4076cdf9b 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresUnityCatalog = true - -[EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/out.test.toml index 5566892a0d7..88423408186 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = false RequiresUnityCatalog = true - -[EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/out.test.toml index 5566892a0d7..88423408186 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = false RequiresUnityCatalog = true - -[EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/grants/select/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/grants/select/out.test.toml index f1d40380d02..fe4076cdf9b 100644 --- a/acceptance/bundle/resources/vector_search_indexes/grants/select/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/grants/select/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresUnityCatalog = true - -[EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.test.toml index f1d40380d02..fe4076cdf9b 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = true RequiresUnityCatalog = true - -[EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] From 3c9515492bc52c746510a80ace2dbab93134792b Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 30 Apr 2026 23:04:36 +0200 Subject: [PATCH 28/47] direct: propagate endpoint UUID lookup errors for VS indexes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously lookupEndpointUuid swallowed all non-404 errors and returned "", which would feed empty remoteUuid into OverrideChangeDesc and propose a destructive Recreate ("endpoint replaced out-of-band") on transient or permission errors. The Recreate is dangerous: Delta Sync re-runs the embedding pipeline, and Direct Access loses all upserted vectors. Now the helper returns (string, error): 404 maps to ("", nil) — the orphan signal — and any other error is propagated through DoRead/DoCreate so the plan fails loudly instead of misclassifying it as drift. Document the OverrideChangeDesc divergence from vector_search_endpoint (which requires remoteUuid != ""): for indexes, an empty remoteUuid is the orphan signal, and the lookup contract guarantees that case is unambiguous. --- .../direct/dresources/vector_search_index.go | 44 +++++++++++++------ 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/bundle/direct/dresources/vector_search_index.go b/bundle/direct/dresources/vector_search_index.go index e68170c81f5..e191e580df2 100644 --- a/bundle/direct/dresources/vector_search_index.go +++ b/bundle/direct/dresources/vector_search_index.go @@ -3,11 +3,11 @@ package dresources import ( "context" "errors" + "fmt" "time" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" @@ -103,9 +103,13 @@ func (r *ResourceVectorSearchIndex) DoRead(ctx context.Context, id string) (*Vec if err != nil { return nil, err } + endpointUuid, err := r.lookupEndpointUuid(ctx, index.EndpointName) + if err != nil { + return nil, err + } return &VectorSearchIndexRemote{ VectorIndex: index, - EndpointUuid: r.lookupEndpointUuid(ctx, index.EndpointName), + EndpointUuid: endpointUuid, }, nil } @@ -114,10 +118,14 @@ func (r *ResourceVectorSearchIndex) DoCreate(ctx context.Context, config *Vector if err != nil { return "", nil, err } - // Exceptional: a second API call. The index API does not return the endpoint - // UUID, but we need to persist it in state so a future plan can detect that - // the endpoint was replaced out-of-band (same name, different UUID -> orphan). - endpointUuid := r.lookupEndpointUuid(ctx, config.EndpointName) + // Second API call (also done in DoRead): the index API does not return the + // endpoint UUID, but we need to persist it in state so a future plan can + // detect that the endpoint was replaced out-of-band (same name, different + // UUID -> orphan). + endpointUuid, err := r.lookupEndpointUuid(ctx, config.EndpointName) + if err != nil { + return "", nil, err + } config.EndpointUuid = endpointUuid return config.Name, &VectorSearchIndexRemote{VectorIndex: index, EndpointUuid: endpointUuid}, nil } @@ -155,6 +163,13 @@ func (r *ResourceVectorSearchIndex) WaitAfterDelete(ctx context.Context, id stri // otherwise. endpoint_uuid is never present in config, so without Skip a // synthetic diff between empty newState and populated saved state would // otherwise leak into the plan. +// +// Unlike vector_search_endpoint, this intentionally does NOT require +// remoteUuid != "". An empty remoteUuid here is the orphan signal: the index +// still exists by name but its backing endpoint has been deleted out-of-band. +// lookupEndpointUuid distinguishes this (404 -> "") from transient errors +// (propagated through DoRead/DoCreate), so reaching this branch with empty +// remoteUuid unambiguously means the endpoint is gone. func (*ResourceVectorSearchIndex) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, remote *VectorSearchIndexRemote) error { if path.String() != "endpoint_uuid" { return nil @@ -175,18 +190,19 @@ func (*ResourceVectorSearchIndex) OverrideChangeDesc(_ context.Context, path *st } // lookupEndpointUuid returns the current UUID of the endpoint with the given -// name, or "" if the endpoint doesn't exist. Errors are logged and swallowed -// since a missing endpoint is the signal we want to capture in state. -func (r *ResourceVectorSearchIndex) lookupEndpointUuid(ctx context.Context, endpointName string) string { +// name. A 404 is converted to ("", nil) so the caller can distinguish a +// genuinely missing endpoint (the orphan signal) from a transient or +// permission error, which is propagated. +func (r *ResourceVectorSearchIndex) lookupEndpointUuid(ctx context.Context, endpointName string) (string, error) { if endpointName == "" { - return "" + return "", nil } info, err := r.client.VectorSearchEndpoints.GetEndpointByEndpointName(ctx, endpointName) if err != nil { - if !apierr.IsMissing(err) { - log.Warnf(ctx, "failed to read vector search endpoint %q while resolving index endpoint UUID: %v", endpointName, err) + if apierr.IsMissing(err) { + return "", nil } - return "" + return "", fmt.Errorf("looking up vector search endpoint %q: %w", endpointName, err) } - return info.Id + return info.Id, nil } From a02bd735e23ac8c59799711bcde592b92e03469f Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 30 Apr 2026 23:11:14 +0200 Subject: [PATCH 29/47] acceptance: capture VS endpoint-recreate cascade gap Add a Badness-marked test that deploys a bundle with both a vector_search_endpoint and a vector_search_index referencing it, then changes the endpoint_type to trigger an endpoint Recreate. The plan correctly recreates the endpoint but leaves the dependent index unchanged, so on a real workspace the endpoint delete would either fail (indexes still attached) or orphan the index. Root cause is in the planner (bundle/direct/bundle_plan.go): there is no logic to propagate Recreate from a dependency to its dependents. This is a framework-level concern that affects more than just VS, so it's deferred to a follow-up. The Badness entry documents the gap. --- .../with_endpoint/databricks.yml.tmpl | 19 +++++++ .../recreate/with_endpoint/out.test.toml | 4 ++ .../recreate/with_endpoint/output.txt | 50 +++++++++++++++++++ .../recreate/with_endpoint/script | 24 +++++++++ .../recreate/with_endpoint/test.toml | 1 + 5 files changed, 98 insertions(+) create mode 100644 acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/out.test.toml create mode 100644 acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt create mode 100644 acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/script create mode 100644 acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/test.toml diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/databricks.yml.tmpl b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/databricks.yml.tmpl new file mode 100644 index 00000000000..b6fde83e7fb --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/databricks.yml.tmpl @@ -0,0 +1,19 @@ +bundle: + name: deploy-vs-index-with-endpoint-$UNIQUE_NAME + +sync: + paths: [] + +resources: + vector_search_endpoints: + my_endpoint: + name: vs-endpoint-$UNIQUE_NAME + endpoint_type: STANDARD + vector_search_indexes: + my_index: + name: vs-index-$UNIQUE_NAME + endpoint_name: ${resources.vector_search_endpoints.my_endpoint.name} + primary_key: id + index_type: DIRECT_ACCESS + direct_access_index_spec: + schema_json: '{"columns":[{"name":"id","type":"integer"}]}' diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/out.test.toml new file mode 100644 index 00000000000..fe4076cdf9b --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt new file mode 100644 index 00000000000..8db6a39bf2f --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt @@ -0,0 +1,50 @@ + +=== Initial deployment with STANDARD endpoint_type +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-vs-index-with-endpoint-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Change endpoint_type (should recreate endpoint AND its dependent index) +>>> update_file.py databricks.yml endpoint_type: STANDARD endpoint_type: STORAGE_OPTIMIZED + +>>> [CLI] bundle plan +recreate vector_search_endpoints.my_endpoint + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-vs-index-with-endpoint-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] vector-search-endpoints get-endpoint vs-endpoint-[UNIQUE_NAME] +{ + "name": "vs-endpoint-[UNIQUE_NAME]", + "endpoint_type": "STORAGE_OPTIMIZED" +} + +>>> [CLI] vector-search-indexes get-index vs-index-[UNIQUE_NAME] +{ + "name": "vs-index-[UNIQUE_NAME]", + "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", + "index_type": "DIRECT_ACCESS", + "primary_key": "id" +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.vector_search_endpoints.my_endpoint + delete resources.vector_search_indexes.my_index + +This action will result in the deletion of the following Vector Search indexes. +For Delta Sync indexes, the source Delta table is preserved but the embedding pipeline is removed. +For Direct Access indexes, all upserted vectors are permanently lost: + delete resources.vector_search_indexes.my_index + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-vs-index-with-endpoint-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/script b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/script new file mode 100644 index 00000000000..645bf350154 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/script @@ -0,0 +1,24 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "Initial deployment with STANDARD endpoint_type" +rm -f out.requests.txt +trace $CLI bundle deploy + +title "Change endpoint_type (should recreate endpoint AND its dependent index)" +trace update_file.py databricks.yml "endpoint_type: STANDARD" "endpoint_type: STORAGE_OPTIMIZED" + +trace $CLI bundle plan +rm -f out.requests.txt +trace $CLI bundle deploy --auto-approve + +# Verify final state of both resources. +endpoint_name="vs-endpoint-${UNIQUE_NAME}" +index_name="vs-index-${UNIQUE_NAME}" +trace $CLI vector-search-endpoints get-endpoint "${endpoint_name}" | jq '{name, endpoint_type}' +trace $CLI vector-search-indexes get-index "${index_name}" | jq '{name, endpoint_name, index_type, primary_key}' diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/test.toml b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/test.toml new file mode 100644 index 00000000000..f906513d9d2 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/test.toml @@ -0,0 +1 @@ +Badness = "Recreating a vector_search_endpoint should propagate Recreate to dependent vector_search_indexes (and any other resource), but the planner does not. The dependent index plans Skip while the endpoint deletes underneath it. On a real workspace this either fails the endpoint delete (indexes still attached) or orphans the index. Tracked as a framework-level fix; until then the index must be recreated manually after an endpoint recreate." From f9408553773bf9550f6d0620f6700a27afc2ce7d Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 30 Apr 2026 23:13:45 +0200 Subject: [PATCH 30/47] acceptance: capture VS index endpoint_name prefixing gap Add a Badness-marked validate test showing that the name_prefix preset does not rewrite a vector_search_indexes.*.endpoint_name literal that points at a bundle-managed (and therefore prefixed) endpoint. The output shows vs_endpoint -> prefix_vs_endpoint while vs_index_literal still targets the unprefixed name vs_endpoint. The DABs idiom is to use ${resources.vector_search_endpoints.X.name} (captured by vs_index_ref in the same fixture). That form resolves correctly to the prefixed name at plan/deploy time, so users have a working pattern. The literal form silently breaks though, and the preset has enough information to rewrite it; tracked as Badness for a follow-up fix in apply_presets.go. --- .../databricks.yml.tmpl | 29 +++++++++++++++++++ .../out.test.toml | 3 ++ .../output.txt | 24 +++++++++++++++ .../script | 4 +++ .../test.toml | 1 + 5 files changed, 61 insertions(+) create mode 100644 acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/databricks.yml.tmpl create mode 100644 acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/out.test.toml create mode 100644 acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/output.txt create mode 100644 acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/script create mode 100644 acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/test.toml diff --git a/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/databricks.yml.tmpl b/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/databricks.yml.tmpl new file mode 100644 index 00000000000..b4398dc0f92 --- /dev/null +++ b/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/databricks.yml.tmpl @@ -0,0 +1,29 @@ +bundle: + name: BUNDLE + +workspace: + resource_path: /foo/bar + +resources: + vector_search_endpoints: + vs_endpoint: + name: vs_endpoint + endpoint_type: STANDARD + vector_search_indexes: + # Literal endpoint_name: not rewritten by the prefix preset, so the + # index would target the unprefixed (non-existent) endpoint. + vs_index_literal: + name: my_catalog.my_schema.literal_index + endpoint_name: vs_endpoint + primary_key: id + index_type: DELTA_SYNC + # Reference to the bundle-managed endpoint: resolved after the prefix + # preset runs, so the index correctly targets the prefixed endpoint. + vs_index_ref: + name: my_catalog.my_schema.ref_index + endpoint_name: ${resources.vector_search_endpoints.vs_endpoint.name} + primary_key: id + index_type: DELTA_SYNC + +presets: + name_prefix: "$PREFIX" diff --git a/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/out.test.toml b/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/output.txt b/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/output.txt new file mode 100644 index 00000000000..b2d747d6189 --- /dev/null +++ b/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/output.txt @@ -0,0 +1,24 @@ + +>>> [CLI] bundle validate -o json +{ + "vector_search_endpoints": { + "vs_endpoint": { + "endpoint_type": "STANDARD", + "name": "prefix_vs_endpoint" + } + }, + "vector_search_indexes": { + "vs_index_literal": { + "endpoint_name": "vs_endpoint", + "index_type": "DELTA_SYNC", + "name": "my_catalog.my_schema.prefix_literal_index", + "primary_key": "id" + }, + "vs_index_ref": { + "endpoint_name": "${resources.vector_search_endpoints.vs_endpoint.name}", + "index_type": "DELTA_SYNC", + "name": "my_catalog.my_schema.prefix_ref_index", + "primary_key": "id" + } + } +} diff --git a/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/script b/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/script new file mode 100644 index 00000000000..a5c6230a957 --- /dev/null +++ b/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/script @@ -0,0 +1,4 @@ +PREFIX="[prefix]" envsubst < databricks.yml.tmpl > databricks.yml +trace $CLI bundle validate -o json | jq '.resources | {vector_search_endpoints, vector_search_indexes}' + +rm databricks.yml diff --git a/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/test.toml b/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/test.toml new file mode 100644 index 00000000000..bb36c8e2502 --- /dev/null +++ b/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/test.toml @@ -0,0 +1 @@ +Badness = "Literal vector_search_indexes.*.endpoint_name is not rewritten by the name_prefix preset, even when it points at a bundle-managed endpoint that gets prefixed. The DABs convention is to use ${resources.vector_search_endpoints.X.name} (covered by vs_index_ref below), but the literal form silently breaks. Until apply_presets.go cross-references endpoint_name against bundle endpoints, prefer the reference form." From 4a76acc2f50e4f2265bc1feaa6378cf1766862f8 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 30 Apr 2026 23:15:38 +0200 Subject: [PATCH 31/47] acceptance: add bind/unbind tests for vector_search_indexes Mirror the existing vector_search_endpoint bind test: pre-create both endpoint and index, bind the index into the bundle, deploy, unbind, and destroy. Verifies the index survives unbind+destroy as expected. Required by bundle/direct/dresources/README.md for new resource types. --- .../vector_search_index/databricks.yml.tmpl | 15 ++++++ .../bind/vector_search_index/out.test.toml | 4 ++ .../bind/vector_search_index/output.txt | 54 +++++++++++++++++++ .../bind/vector_search_index/script | 27 ++++++++++ .../bind/vector_search_index/test.toml | 11 ++++ 5 files changed, 111 insertions(+) create mode 100644 acceptance/bundle/deployment/bind/vector_search_index/databricks.yml.tmpl create mode 100644 acceptance/bundle/deployment/bind/vector_search_index/out.test.toml create mode 100644 acceptance/bundle/deployment/bind/vector_search_index/output.txt create mode 100644 acceptance/bundle/deployment/bind/vector_search_index/script create mode 100644 acceptance/bundle/deployment/bind/vector_search_index/test.toml diff --git a/acceptance/bundle/deployment/bind/vector_search_index/databricks.yml.tmpl b/acceptance/bundle/deployment/bind/vector_search_index/databricks.yml.tmpl new file mode 100644 index 00000000000..3189b712d6b --- /dev/null +++ b/acceptance/bundle/deployment/bind/vector_search_index/databricks.yml.tmpl @@ -0,0 +1,15 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +sync: + paths: [] + +resources: + vector_search_indexes: + index1: + name: $INDEX_NAME + endpoint_name: $ENDPOINT_NAME + primary_key: id + index_type: DIRECT_ACCESS + direct_access_index_spec: + schema_json: '{"columns":[{"name":"id","type":"integer"}]}' diff --git a/acceptance/bundle/deployment/bind/vector_search_index/out.test.toml b/acceptance/bundle/deployment/bind/vector_search_index/out.test.toml new file mode 100644 index 00000000000..fe4076cdf9b --- /dev/null +++ b/acceptance/bundle/deployment/bind/vector_search_index/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/vector_search_index/output.txt b/acceptance/bundle/deployment/bind/vector_search_index/output.txt new file mode 100644 index 00000000000..db61161ad00 --- /dev/null +++ b/acceptance/bundle/deployment/bind/vector_search_index/output.txt @@ -0,0 +1,54 @@ + +>>> [CLI] vector-search-endpoints create-endpoint test-vs-endpoint-[UNIQUE_NAME] STANDARD +{ + "name": "test-vs-endpoint-[UNIQUE_NAME]", + "endpoint_type": "STANDARD" +} + +>>> [CLI] vector-search-indexes create-index --json {"name":"main.default.test_vs_index_[UNIQUE_NAME]","endpoint_name":"test-vs-endpoint-[UNIQUE_NAME]","primary_key":"id","index_type":"DIRECT_ACCESS","direct_access_index_spec":{"schema_json":"{\"columns\":[{\"name\":\"id\",\"type\":\"integer\"}]}"}} +{ + "name": "main.default.test_vs_index_[UNIQUE_NAME]", + "endpoint_name": "test-vs-endpoint-[UNIQUE_NAME]", + "index_type": "DIRECT_ACCESS", + "primary_key": "id" +} + +>>> [CLI] bundle deployment bind index1 main.default.test_vs_index_[UNIQUE_NAME] --auto-approve +Updating deployment state... +Successfully bound vector_search_index with an id 'main.default.test_vs_index_[UNIQUE_NAME]' +Run 'bundle deploy' to deploy changes to your workspace + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] vector-search-indexes get-index main.default.test_vs_index_[UNIQUE_NAME] +{ + "name": "main.default.test_vs_index_[UNIQUE_NAME]", + "endpoint_name": "test-vs-endpoint-[UNIQUE_NAME]", + "index_type": "DIRECT_ACCESS", + "primary_key": "id" +} + +>>> [CLI] bundle deployment unbind index1 +Updating deployment state... + +>>> [CLI] bundle destroy --auto-approve +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! + +>>> [CLI] vector-search-indexes get-index main.default.test_vs_index_[UNIQUE_NAME] +{ + "name": "main.default.test_vs_index_[UNIQUE_NAME]", + "endpoint_name": "test-vs-endpoint-[UNIQUE_NAME]", + "index_type": "DIRECT_ACCESS", + "primary_key": "id" +} + +>>> [CLI] vector-search-indexes delete-index main.default.test_vs_index_[UNIQUE_NAME] + +>>> [CLI] vector-search-endpoints delete-endpoint test-vs-endpoint-[UNIQUE_NAME] diff --git a/acceptance/bundle/deployment/bind/vector_search_index/script b/acceptance/bundle/deployment/bind/vector_search_index/script new file mode 100644 index 00000000000..d43578d94c0 --- /dev/null +++ b/acceptance/bundle/deployment/bind/vector_search_index/script @@ -0,0 +1,27 @@ +ENDPOINT_NAME="test-vs-endpoint-$UNIQUE_NAME" +INDEX_NAME="main.default.test_vs_index_$UNIQUE_NAME" +export ENDPOINT_NAME INDEX_NAME +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI vector-search-indexes delete-index "${INDEX_NAME}" + trace $CLI vector-search-endpoints delete-endpoint "${ENDPOINT_NAME}" +} +trap cleanup EXIT + +trace $CLI vector-search-endpoints create-endpoint "${ENDPOINT_NAME}" STANDARD | jq '{name, endpoint_type}' + +trace $CLI vector-search-indexes create-index --json "{\"name\":\"${INDEX_NAME}\",\"endpoint_name\":\"${ENDPOINT_NAME}\",\"primary_key\":\"id\",\"index_type\":\"DIRECT_ACCESS\",\"direct_access_index_spec\":{\"schema_json\":\"{\\\"columns\\\":[{\\\"name\\\":\\\"id\\\",\\\"type\\\":\\\"integer\\\"}]}\"}}" | jq '{name, endpoint_name, index_type, primary_key}' + +trace $CLI bundle deployment bind index1 "${INDEX_NAME}" --auto-approve + +trace $CLI bundle deploy --auto-approve + +trace $CLI vector-search-indexes get-index "${INDEX_NAME}" | jq '{name, endpoint_name, index_type, primary_key}' + +trace $CLI bundle deployment unbind index1 + +trace $CLI bundle destroy --auto-approve + +# Read the pre-defined index again (expecting it still exists and is not deleted): +trace $CLI vector-search-indexes get-index "${INDEX_NAME}" | jq '{name, endpoint_name, index_type, primary_key}' diff --git a/acceptance/bundle/deployment/bind/vector_search_index/test.toml b/acceptance/bundle/deployment/bind/vector_search_index/test.toml new file mode 100644 index 00000000000..5722b37ccca --- /dev/null +++ b/acceptance/bundle/deployment/bind/vector_search_index/test.toml @@ -0,0 +1,11 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true + +Ignore = [ + ".databricks", + "databricks.yml", +] + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] From ed3fe3bc235f1da93d28615d8b169428e19197fc Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 30 Apr 2026 23:16:27 +0200 Subject: [PATCH 32/47] changelog: shorten VS indexes entry Drop the Terraform-provider justification (already implied by "direct engine only") and the long list of internal mechanics. Keep the entry focused on what customers see. --- NEXT_CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 4d2f6c55688..034d57aff24 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -9,6 +9,6 @@ ### Bundles * Make sure warnings asking for approval are understood by agents ([#5239](https://github.com/databricks/cli/pull/5239)) -* Added `vector_search_indexes` as a first-class bundle resource on the direct deployment engine, alongside the existing `vector_search_endpoints`. Supports UC grants, drift detection (including out-of-band endpoint replacement that orphans an index), recreate-on-immutable-field-change, and asynchronous deletion waits. Recreating or deleting an index now prompts for confirmation with a Delta Sync vs Direct Access cost warning. Vector search has no Terraform provider, so this resource is direct-engine only ([#5123](https://github.com/databricks/cli/pull/5123)). +* Added `vector_search_indexes` as a bundle resource (direct engine only). Supports UC grants and prompts for confirmation on recreate or delete since both are destructive ([#5123](https://github.com/databricks/cli/pull/5123)). ### Dependency updates From cc9f5ccd6d47539953135be95ff85950dea86108 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Fri, 1 May 2026 11:31:03 +0200 Subject: [PATCH 33/47] direct: wait for VS index Status.Ready=true after create CreateIndex returns immediately with metadata of an index whose embedding pipeline is still provisioning; queries against an index that isn't ready fail. Implement WaitAfterCreate so dependent resources (and the next plan) see a usable index. 75-minute timeout matches the terraform provider. Co-authored-by: Isaac --- .../out.requests.create.direct.json | 4 +++ .../out.requests.recreate.direct.json | 4 +++ .../direct/dresources/vector_search_index.go | 31 +++++++++++++++++++ 3 files changed, 39 insertions(+) diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.requests.create.direct.json b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.requests.create.direct.json index 0d6186c0ec2..00d11a7272f 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.requests.create.direct.json +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.requests.create.direct.json @@ -12,3 +12,7 @@ "primary_key": "id" } } +{ + "method": "GET", + "path": "/api/2.0/vector-search/indexes/vs-index-[UNIQUE_NAME]" +} diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.requests.recreate.direct.json b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.requests.recreate.direct.json index 1149b9530b6..c00a4caa79b 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.requests.recreate.direct.json +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/out.requests.recreate.direct.json @@ -23,3 +23,7 @@ "primary_key": "id" } } +{ + "method": "GET", + "path": "/api/2.0/vector-search/indexes/vs-index-[UNIQUE_NAME]" +} diff --git a/bundle/direct/dresources/vector_search_index.go b/bundle/direct/dresources/vector_search_index.go index e191e580df2..50600fab06a 100644 --- a/bundle/direct/dresources/vector_search_index.go +++ b/bundle/direct/dresources/vector_search_index.go @@ -21,6 +21,12 @@ import ( // embedding pipeline shutdown can stretch closer to ten minutes. const deleteIndexTimeout = 15 * time.Minute +// createIndexTimeout caps the wait for an index to become ready after creation. +// Delta sync indexes do an initial sync from the source table, which can stretch +// out for large tables. Matches the terraform provider's defaultIndexProvisionTimeout. +// https://github.com/databricks/terraform-provider-databricks/blob/c61a32300445f84efb2bb6827dee35e6e523f4ff/vectorsearch/resource_vector_search_index.go#L19 +const createIndexTimeout = 75 * time.Minute + // VectorSearchIndexState tracks the UUID of the endpoint the index is attached // to. Without it the planner cannot tell that an index pointing at a deleted // and recreated endpoint (same name, different UUID) has been orphaned — the @@ -139,6 +145,31 @@ func (r *ResourceVectorSearchIndex) DoDelete(ctx context.Context, id string) err return r.client.VectorSearchIndexes.DeleteIndexByIndexName(ctx, id) } +// WaitAfterCreate polls GetIndex until Status.Ready=true. CreateIndex returns +// immediately with metadata of an index whose embedding pipeline is still +// provisioning; queries against an index that isn't ready fail. Blocking here +// lets dependent resources (and the next plan) see a usable index. +func (r *ResourceVectorSearchIndex) WaitAfterCreate(ctx context.Context, config *VectorSearchIndexState) (*VectorSearchIndexRemote, error) { + index, err := retries.Poll(ctx, createIndexTimeout, func() (*vectorsearch.VectorIndex, *retries.Err) { + idx, getErr := r.client.VectorSearchIndexes.GetIndexByIndexName(ctx, config.Name) + if getErr != nil { + return nil, retries.Halt(getErr) + } + if idx.Status == nil || !idx.Status.Ready { + msg := "index is still provisioning" + if idx.Status != nil && idx.Status.Message != "" { + msg = idx.Status.Message + } + return nil, retries.Continues(msg) + } + return idx, nil + }) + if err != nil { + return nil, err + } + return &VectorSearchIndexRemote{VectorIndex: index, EndpointUuid: config.EndpointUuid}, nil +} + // WaitAfterDelete polls GetIndex until it returns 404. The DELETE call is // asynchronous: a follow-up CREATE for the same name (e.g. during recreate) is // rejected with "index is currently pending deletion" until the backend finishes From 10941978c6779efc324abc80575591f98637a126 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Mon, 4 May 2026 10:56:40 +0200 Subject: [PATCH 34/47] Allow leaf to be prefixed if wihtout a reference --- .../databricks.yml.tmpl | 13 +++++ .../output.txt | 6 +++ .../mutator/resourcemutator/apply_presets.go | 37 +++++++++----- .../resourcemutator/apply_target_mode_test.go | 50 +++++++++++++++---- 4 files changed, 82 insertions(+), 24 deletions(-) diff --git a/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/databricks.yml.tmpl b/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/databricks.yml.tmpl index b4398dc0f92..c8899a3e3a0 100644 --- a/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/databricks.yml.tmpl +++ b/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/databricks.yml.tmpl @@ -24,6 +24,19 @@ resources: endpoint_name: ${resources.vector_search_endpoints.vs_endpoint.name} primary_key: id index_type: DELTA_SYNC + # Partial reference: catalog and schema are variable refs but the leaf + # is literal, so the prefix preset can still prefix the leaf. + vs_index_partial_ref: + name: ${var.catalog}.${var.schema}.partial_ref_index + endpoint_name: ${resources.vector_search_endpoints.vs_endpoint.name} + primary_key: id + index_type: DELTA_SYNC + +variables: + catalog: + default: my_catalog + schema: + default: my_schema presets: name_prefix: "$PREFIX" diff --git a/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/output.txt b/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/output.txt index b2d747d6189..3dd823c9c9f 100644 --- a/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/output.txt +++ b/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/output.txt @@ -14,6 +14,12 @@ "name": "my_catalog.my_schema.prefix_literal_index", "primary_key": "id" }, + "vs_index_partial_ref": { + "endpoint_name": "${resources.vector_search_endpoints.vs_endpoint.name}", + "index_type": "DELTA_SYNC", + "name": "my_catalog.my_schema.prefix_partial_ref_index", + "primary_key": "id" + }, "vs_index_ref": { "endpoint_name": "${resources.vector_search_endpoints.vs_endpoint.name}", "index_type": "DELTA_SYNC", diff --git a/bundle/config/mutator/resourcemutator/apply_presets.go b/bundle/config/mutator/resourcemutator/apply_presets.go index 7afc9a00c4e..8e4457ddf00 100644 --- a/bundle/config/mutator/resourcemutator/apply_presets.go +++ b/bundle/config/mutator/resourcemutator/apply_presets.go @@ -297,23 +297,12 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos // Vector Search Indexes: Prefix // The name is a 3-part UC identifier (catalog.schema.index); prefix only // the last component since catalog and schema are external references. - // This mutator runs before reference/variable resolution, so the name may - // still carry literal `${...}` tokens. Splitting on the last dot in that - // case would inject the prefix inside the ref expression itself - // (e.g. `${var.full_name}` -> `${var.dev_user_full_name}`); skip - // prefixing when refs are present and let the user compose the prefix - // into the variable. for _, e := range r.VectorSearchIndexes { if e == nil { continue } - if strings.Contains(e.Name, "${") { - continue - } - if i := strings.LastIndex(e.Name, "."); i >= 0 { - e.Name = e.Name[:i+1] + normalizePrefix(prefix) + e.Name[i+1:] - } else { - e.Name = normalizePrefix(prefix) + e.Name + if pos := vectorSearchIndexPrefixPos(e.Name); pos >= 0 { + e.Name = e.Name[:pos] + normalizePrefix(prefix) + e.Name[pos:] } } @@ -348,6 +337,28 @@ func toTagArray(tags map[string]string) []Tag { return tagArray } +// vectorSearchIndexPrefixPos returns the byte offset at which to insert the +// name prefix into a vector search index name, or -1 to skip prefixing. +// +// ApplyPresets runs before reference/variable resolution, so the name may +// still carry literal `${...}` tokens. We walk back from the end and prefix +// at the first `.` we find, which is the catalog.schema.index separator +// once the literal tail is non-empty. If we instead hit `$` or `}` first, +// the tail is occluded by a `${...}` expression and prefixing would corrupt +// it (e.g. `${var.full_name}` -> `${var.dev_user_full_name}`); in that case +// skip and let the user compose the prefix into the variable. +func vectorSearchIndexPrefixPos(name string) int { + for i := len(name) - 1; i >= 0; i-- { + switch name[i] { + case '.': + return i + 1 + case '$', '}': + return -1 + } + } + return 0 +} + // normalizePrefix prefixes strings like '[dev lennart] ' to 'dev_lennart_'. // We leave unicode letters and numbers but remove all "special characters." func normalizePrefix(prefix string) string { diff --git a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go index 5bbd376ac10..1a6cf502ce8 100644 --- a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go +++ b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go @@ -506,23 +506,51 @@ func TestDisableLockingDisabled(t *testing.T) { assert.True(t, b.Config.Bundle.Deployment.Lock.IsEnabled(), "Deployment lock should remain enabled in development mode when explicitly enabled") } -func TestVectorSearchIndexNameWithUnresolvedRefsLeftAlone(t *testing.T) { - b := mockBundle(config.Development) - b.Config.Resources.VectorSearchIndexes["vs_index_with_var"] = &resources.VectorSearchIndex{ - CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{ - Name: "${var.catalog}.${var.schema}.${var.index}", - EndpointName: "vs_endpoint1", - PrimaryKey: "id", - IndexType: vectorsearch.VectorIndexTypeDeltaSync, +func TestVectorSearchIndexNamePrefixing(t *testing.T) { + cases := []struct { + key string + name string + want string + }{ + { + // Trailing component is a ref: skip, since prefixing would inject + // the prefix inside the ${var.index} expression. + key: "vs_index_all_refs", + name: "${var.catalog}.${var.schema}.${var.index}", + want: "${var.catalog}.${var.schema}.${var.index}", + }, + { + // Catalog and schema are refs but the leaf is literal: prefix the leaf. + key: "vs_index_partial_ref", + name: "${var.catalog}.${var.schema}.non_ref_name", + want: "${var.catalog}.${var.schema}.dev_lennart_non_ref_name", + }, + { + // Whole name is a single ref: skip. + key: "vs_index_full_ref", + name: "${var.full_name}", + want: "${var.full_name}", }, } + b := mockBundle(config.Development) + for _, c := range cases { + b.Config.Resources.VectorSearchIndexes[c.key] = &resources.VectorSearchIndex{ + CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{ + Name: c.name, + EndpointName: "vs_endpoint1", + PrimaryKey: "id", + IndexType: vectorsearch.VectorIndexTypeDeltaSync, + }, + } + } + diags := bundle.ApplySeq(t.Context(), b, ApplyTargetMode(), ApplyPresets()) require.NoError(t, diags.Error()) - // The leaf-finding splits on the last dot, which would otherwise inject the - // prefix inside the trailing ${var.index} expression. - assert.Equal(t, "${var.catalog}.${var.schema}.${var.index}", b.Config.Resources.VectorSearchIndexes["vs_index_with_var"].Name) + for _, c := range cases { + assert.Equal(t, c.want, b.Config.Resources.VectorSearchIndexes[c.key].Name, c.key) + } } func TestPrefixAlreadySet(t *testing.T) { From a670352b071188ca3cc21001e9eebdf85a56558e Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Mon, 4 May 2026 10:59:02 +0200 Subject: [PATCH 35/47] Simplify dresources/vector_search_index.go PrepareState index spec remapping --- bundle/direct/dresources/vector_search_index.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/bundle/direct/dresources/vector_search_index.go b/bundle/direct/dresources/vector_search_index.go index 50600fab06a..451857e6d99 100644 --- a/bundle/direct/dresources/vector_search_index.go +++ b/bundle/direct/dresources/vector_search_index.go @@ -77,8 +77,8 @@ func (*ResourceVectorSearchIndex) PrepareState(input *resources.VectorSearchInde func (*ResourceVectorSearchIndex) RemapState(remote *VectorSearchIndexRemote) *VectorSearchIndexState { state := &VectorSearchIndexState{ CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{ - DeltaSyncIndexSpec: nil, - DirectAccessIndexSpec: nil, + DeltaSyncIndexSpec: nil, // need to remap below + DirectAccessIndexSpec: remote.DirectAccessIndexSpec, IndexSubtype: remote.IndexSubtype, Name: remote.Name, EndpointName: remote.EndpointName, @@ -98,9 +98,6 @@ func (*ResourceVectorSearchIndex) RemapState(remote *VectorSearchIndexRemote) *V ForceSendFields: nil, } } - if remote.DirectAccessIndexSpec != nil { - state.DirectAccessIndexSpec = remote.DirectAccessIndexSpec - } return state } From 600aea666f9ceca65af8497ba5f04625931ee9f7 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Mon, 4 May 2026 15:30:26 +0200 Subject: [PATCH 36/47] direct: drop empty-id state recovery, now handled by #5173 The SaveState->DeleteState change in apply.Recreate and the empty-id tolerance in bundle_plan.go were extracted to a separate PR (#5173). Reverting them here so this branch and #5173 can land independently; once #5173 merges, a rebase on main brings the same fix back in. Co-authored-by: Isaac --- bundle/direct/apply.go | 9 +++------ bundle/direct/bundle_plan.go | 11 ++++++----- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index 6921769bd97..4e2ff247908 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -86,17 +86,14 @@ func (d *DeploymentUnit) Recreate(ctx context.Context, db *dstate.DeploymentStat return fmt.Errorf("deleting old id=%s: %w", oldID, err) } - // Drop the state entry so a subsequent failure of Create or WaitAfterDelete - // leaves no malformed (empty-ID) entry behind. The next plan will see "no - // state" and retry as Create. - err = db.DeleteState(d.ResourceKey) + err = db.SaveState(d.ResourceKey, "", nil, nil) if err != nil { return fmt.Errorf("deleting state: %w", err) } // Wait for asynchronous teardown to finish before re-creating the same - // name. Done after DeleteState so the bundle stays consistent if the wait - // times out — the resource is no longer tracked in state, retry on next plan. + // name. Done after the state update so the bundle stays consistent if the + // wait times out — the resource is marked deleted, retry on next plan. err = d.Adapter.WaitAfterDelete(ctx, oldID) if err != nil { return fmt.Errorf("waiting after deleting id=%s: %w", oldID, err) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 32eb6dd575c..3fab4c3f4ff 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -181,15 +181,16 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks } dbentry, hasEntry := b.StateDB.GetResourceEntry(resourceKey) - // Tolerate empty-ID entries from older partial-recreate failures - // (apply.Recreate now deletes state on the way through, but pre-fix - // state files may still carry a malformed entry). Treat as missing - // and let the resource be re-created on this plan. - if !hasEntry || dbentry.ID == "" { + if !hasEntry { entry.Action = deployplan.Create return true } + if dbentry.ID == "" { + logdiag.LogError(ctx, fmt.Errorf("%s: invalid state: empty id", errorPrefix)) + return false + } + savedState, err := parseState(adapter.StateType(), dbentry.State) if err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: interpreting state: %w", errorPrefix, err)) From 407b022163450f4f3e1c172c2f1368e647652928 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Mon, 4 May 2026 18:38:36 +0200 Subject: [PATCH 37/47] Hack: recreate index if endpoint is being recreated --- .../recreate/with_endpoint/out.test.toml | 2 +- .../recreate/with_endpoint/output.txt | 96 ++++++++++++++++++- .../recreate/with_endpoint/script | 1 + .../recreate/with_endpoint/test.toml | 2 +- bundle/direct/bundle_plan.go | 25 +++++ 5 files changed, 123 insertions(+), 3 deletions(-) diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/out.test.toml index fe4076cdf9b..88423408186 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/out.test.toml @@ -1,4 +1,4 @@ Local = true -Cloud = true +Cloud = false RequiresUnityCatalog = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt index 8db6a39bf2f..64425989abb 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt @@ -11,11 +11,105 @@ Deployment complete! >>> [CLI] bundle plan recreate vector_search_endpoints.my_endpoint +recreate vector_search_indexes.my_index -Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged +Plan: 2 to add, 0 to change, 2 to delete, 0 unchanged + +>>> [CLI] bundle plan --output json +{ + "plan_version": 2, + "cli_version": "[DEV_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "plan": { + "resources.vector_search_endpoints.my_endpoint": { + "action": "recreate", + "new_state": { + "value": { + "endpoint_type": "STORAGE_OPTIMIZED", + "name": "vs-endpoint-[UNIQUE_NAME]" + } + }, + "remote_state": { + "creation_timestamp": [UNIX_TIME_MILLIS][0], + "creator": "[USERNAME]", + "endpoint_status": { + "state": "ONLINE" + }, + "endpoint_type": "STANDARD", + "id": "[UUID]", + "last_updated_timestamp": [UNIX_TIME_MILLIS][0], + "last_updated_user": "[USERNAME]", + "name": "vs-endpoint-[UNIQUE_NAME]", + "scaling_info": {} + }, + "changes": { + "endpoint_type": { + "action": "recreate", + "reason": "immutable", + "old": "STANDARD", + "new": "STORAGE_OPTIMIZED", + "remote": "STANDARD" + }, + "endpoint_uuid": { + "action": "skip", + "reason": "custom", + "old": "[UUID]", + "remote": "[UUID]" + } + } + }, + "resources.vector_search_indexes.my_index": { + "depends_on": [ + { + "node": "resources.vector_search_endpoints.my_endpoint", + "label": "${resources.vector_search_endpoints.my_endpoint.name}" + } + ], + "action": "recreate", + "new_state": { + "value": { + "direct_access_index_spec": { + "schema_json": "{\"columns\":[{\"name\":\"id\",\"type\":\"integer\"}]}" + }, + "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", + "index_type": "DIRECT_ACCESS", + "name": "vs-index-[UNIQUE_NAME]", + "primary_key": "id" + } + }, + "remote_state": { + "creator": "[USERNAME]", + "direct_access_index_spec": { + "schema_json": "{\"columns\":[{\"name\":\"id\",\"type\":\"integer\"}]}" + }, + "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", + "endpoint_uuid": "[UUID]", + "index_type": "DIRECT_ACCESS", + "name": "vs-index-[UNIQUE_NAME]", + "primary_key": "id", + "status": { + "ready": true + } + }, + "changes": { + "endpoint_uuid": { + "action": "skip", + "reason": "state-only field", + "old": "[UUID]", + "remote": "[UUID]" + } + } + } + } +} >>> [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-vs-index-with-endpoint-[UNIQUE_NAME]/default/files... + +This action will result in the deletion or recreation of the following Vector Search indexes. +Delta Sync indexes re-run their embedding pipeline; Direct Access indexes lose all upserted vectors: + recreate resources.vector_search_indexes.my_index Deploying resources... Updating deployment state... Deployment complete! diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/script b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/script index 645bf350154..3bf49c5a236 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/script +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/script @@ -14,6 +14,7 @@ title "Change endpoint_type (should recreate endpoint AND its dependent index)" trace update_file.py databricks.yml "endpoint_type: STANDARD" "endpoint_type: STORAGE_OPTIMIZED" trace $CLI bundle plan +trace $CLI bundle plan --output json rm -f out.requests.txt trace $CLI bundle deploy --auto-approve diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/test.toml b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/test.toml index f906513d9d2..18b1a88417e 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/test.toml @@ -1 +1 @@ -Badness = "Recreating a vector_search_endpoint should propagate Recreate to dependent vector_search_indexes (and any other resource), but the planner does not. The dependent index plans Skip while the endpoint deletes underneath it. On a real workspace this either fails the endpoint delete (indexes still attached) or orphans the index. Tracked as a framework-level fix; until then the index must be recreated manually after an endpoint recreate." +Cloud = false diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 3fab4c3f4ff..e8a2b16d3bd 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -284,6 +284,31 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return nil, errors.New("planning failed") } + // HACK: cascade Recreate from vector_search_endpoints to dependent + // vector_search_indexes. The framework has no per-resource recreate + // cascade rule yet; without this, recreating an endpoint orphans its + // indexes (or fails because indexes are still attached on delete). + // Replace with a generic rule when the framework grows one. + for resourceKey, entry := range plan.Plan { + if config.GetResourceTypeFromKey(resourceKey) != "vector_search_indexes" { + continue + } + if entry.Action == deployplan.Create || entry.Action == deployplan.Recreate || entry.Action == deployplan.Delete { + continue + } + for _, dep := range entry.DependsOn { + if config.GetResourceTypeFromKey(dep.Node) != "vector_search_endpoints" { + continue + } + parent, ok := plan.Plan[dep.Node] + if !ok || parent.Action != deployplan.Recreate { + continue + } + entry.Action = deployplan.Recreate + break + } + } + for _, entry := range plan.Plan { if entry.Action == deployplan.Skip { entry.NewState = nil From 5df6fc62194951f0dafc12781fd36b4586efe3d9 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 6 May 2026 12:49:26 +0200 Subject: [PATCH 38/47] Re-run 'task generate-validation' --- bundle/internal/validation/generated/enum_fields.go | 1 + 1 file changed, 1 insertion(+) diff --git a/bundle/internal/validation/generated/enum_fields.go b/bundle/internal/validation/generated/enum_fields.go index ebe7815f41f..1c89eafde30 100644 --- a/bundle/internal/validation/generated/enum_fields.go +++ b/bundle/internal/validation/generated/enum_fields.go @@ -211,6 +211,7 @@ var EnumFields = map[string][]string{ "resources.vector_search_indexes.*.delta_sync_index_spec.pipeline_type": {"CONTINUOUS", "TRIGGERED"}, "resources.vector_search_indexes.*.grants[*].privileges[*]": {"ACCESS", "ALL_PRIVILEGES", "APPLY_TAG", "BROWSE", "CREATE", "CREATE_CATALOG", "CREATE_CLEAN_ROOM", "CREATE_CONNECTION", "CREATE_EXTERNAL_LOCATION", "CREATE_EXTERNAL_TABLE", "CREATE_EXTERNAL_VOLUME", "CREATE_FOREIGN_CATALOG", "CREATE_FOREIGN_SECURABLE", "CREATE_FUNCTION", "CREATE_MANAGED_STORAGE", "CREATE_MATERIALIZED_VIEW", "CREATE_MODEL", "CREATE_PROVIDER", "CREATE_RECIPIENT", "CREATE_SCHEMA", "CREATE_SERVICE_CREDENTIAL", "CREATE_SHARE", "CREATE_STORAGE_CREDENTIAL", "CREATE_TABLE", "CREATE_VIEW", "CREATE_VOLUME", "EXECUTE", "EXECUTE_CLEAN_ROOM_TASK", "EXTERNAL_USE_SCHEMA", "MANAGE", "MANAGE_ALLOWLIST", "MODIFY", "MODIFY_CLEAN_ROOM", "READ_FILES", "READ_PRIVATE_FILES", "READ_VOLUME", "REFRESH", "SELECT", "SET_SHARE_PERMISSION", "USAGE", "USE_CATALOG", "USE_CONNECTION", "USE_MARKETPLACE_ASSETS", "USE_PROVIDER", "USE_RECIPIENT", "USE_SCHEMA", "USE_SHARE", "WRITE_FILES", "WRITE_PRIVATE_FILES", "WRITE_VOLUME"}, + "resources.vector_search_indexes.*.index_subtype": {"FULL_TEXT", "HYBRID", "VECTOR"}, "resources.vector_search_indexes.*.index_type": {"DELTA_SYNC", "DIRECT_ACCESS"}, "resources.volumes.*.grants[*].privileges[*]": {"ACCESS", "ALL_PRIVILEGES", "APPLY_TAG", "BROWSE", "CREATE", "CREATE_CATALOG", "CREATE_CLEAN_ROOM", "CREATE_CONNECTION", "CREATE_EXTERNAL_LOCATION", "CREATE_EXTERNAL_TABLE", "CREATE_EXTERNAL_VOLUME", "CREATE_FOREIGN_CATALOG", "CREATE_FOREIGN_SECURABLE", "CREATE_FUNCTION", "CREATE_MANAGED_STORAGE", "CREATE_MATERIALIZED_VIEW", "CREATE_MODEL", "CREATE_PROVIDER", "CREATE_RECIPIENT", "CREATE_SCHEMA", "CREATE_SERVICE_CREDENTIAL", "CREATE_SHARE", "CREATE_STORAGE_CREDENTIAL", "CREATE_TABLE", "CREATE_VIEW", "CREATE_VOLUME", "EXECUTE", "EXECUTE_CLEAN_ROOM_TASK", "EXTERNAL_USE_SCHEMA", "MANAGE", "MANAGE_ALLOWLIST", "MODIFY", "MODIFY_CLEAN_ROOM", "READ_FILES", "READ_PRIVATE_FILES", "READ_VOLUME", "REFRESH", "SELECT", "SET_SHARE_PERMISSION", "USAGE", "USE_CATALOG", "USE_CONNECTION", "USE_MARKETPLACE_ASSETS", "USE_PROVIDER", "USE_RECIPIENT", "USE_SCHEMA", "USE_SHARE", "WRITE_FILES", "WRITE_PRIVATE_FILES", "WRITE_VOLUME"}, From d6790e0dcb4bfe49a5928fef5b80a55d5f8ea807 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 7 May 2026 10:27:58 +0200 Subject: [PATCH 39/47] acceptance: declare VS endpoints alongside indexes in bundle tests Previously most vector_search_indexes tests created the endpoint out-of-band via the CLI and only declared the index in the bundle. Move the endpoint into the same databricks.yml so the index can reference it via ${resources.vector_search_endpoints.my_endpoint.name}, matching the pattern users will write and shrinking the script's manual cleanup. Bundle destroy now tears down both resources. Co-authored-by: Isaac --- .../basic/databricks.yml.tmpl | 6 +++++- .../vector_search_indexes/basic/output.txt | 18 +++++++++--------- .../vector_search_indexes/basic/script | 4 ---- .../drift/columns_to_sync/databricks.yml.tmpl | 6 +++++- .../drift/columns_to_sync/output.txt | 12 ++---------- .../drift/columns_to_sync/script | 3 --- .../drift/deleted_remotely/databricks.yml.tmpl | 6 +++++- .../drift/deleted_remotely/output.txt | 11 ++--------- .../drift/deleted_remotely/script | 4 ---- .../grants/select/databricks.yml.tmpl | 6 +++++- .../grants/select/output.txt | 15 ++++----------- .../vector_search_indexes/grants/select/script | 4 ---- .../recreate/index_type/databricks.yml.tmpl | 6 +++++- .../recreate/index_type/output.txt | 12 ++---------- .../recreate/index_type/script | 3 --- 15 files changed, 44 insertions(+), 72 deletions(-) diff --git a/acceptance/bundle/resources/vector_search_indexes/basic/databricks.yml.tmpl b/acceptance/bundle/resources/vector_search_indexes/basic/databricks.yml.tmpl index 88373a5a68b..768334ee2a6 100644 --- a/acceptance/bundle/resources/vector_search_indexes/basic/databricks.yml.tmpl +++ b/acceptance/bundle/resources/vector_search_indexes/basic/databricks.yml.tmpl @@ -5,10 +5,14 @@ sync: paths: [] resources: + vector_search_endpoints: + my_endpoint: + name: vs-endpoint-$UNIQUE_NAME + endpoint_type: STANDARD vector_search_indexes: my_index: name: vs-index-$UNIQUE_NAME - endpoint_name: vs-endpoint-$UNIQUE_NAME + endpoint_name: ${resources.vector_search_endpoints.my_endpoint.name} primary_key: id index_type: DELTA_SYNC delta_sync_index_spec: diff --git a/acceptance/bundle/resources/vector_search_indexes/basic/output.txt b/acceptance/bundle/resources/vector_search_indexes/basic/output.txt index fb12212a06e..d75bfea61db 100644 --- a/acceptance/bundle/resources/vector_search_indexes/basic/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/basic/output.txt @@ -15,18 +15,15 @@ Workspace: User: [USERNAME] Path: /Workspace/Users/[USERNAME]/.bundle/deploy-vs-index-[UNIQUE_NAME]/default Resources: + Vector Search Endpoints: + my_endpoint: + Name: vs-endpoint-[UNIQUE_NAME] + URL: [DATABRICKS_URL]/compute/vector-search/vs-endpoint-[UNIQUE_NAME]?o=[NUMID] Vector Search Indexes: my_index: Name: vs-index-[UNIQUE_NAME] URL: (not deployed) ->>> [CLI] vector-search-endpoints create-endpoint vs-endpoint-[UNIQUE_NAME] STANDARD -{ - "id": "[UUID]", - "name": "vs-endpoint-[UNIQUE_NAME]", - "endpoint_type": "STANDARD" -} - >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-vs-index-[UNIQUE_NAME]/default/files... Deploying resources... @@ -48,6 +45,10 @@ Workspace: User: [USERNAME] Path: /Workspace/Users/[USERNAME]/.bundle/deploy-vs-index-[UNIQUE_NAME]/default Resources: + Vector Search Endpoints: + my_endpoint: + Name: vs-endpoint-[UNIQUE_NAME] + URL: [DATABRICKS_URL]/compute/vector-search/vs-endpoint-[UNIQUE_NAME]?o=[NUMID] Vector Search Indexes: my_index: Name: vs-index-[UNIQUE_NAME] @@ -57,6 +58,7 @@ Resources: >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: + delete resources.vector_search_endpoints.my_endpoint delete resources.vector_search_indexes.my_index This action will result in the deletion of the following Vector Search indexes. @@ -68,5 +70,3 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! - ->>> [CLI] vector-search-endpoints delete-endpoint vs-endpoint-[UNIQUE_NAME] diff --git a/acceptance/bundle/resources/vector_search_indexes/basic/script b/acceptance/bundle/resources/vector_search_indexes/basic/script index 3a1c5b8a1f8..54304337425 100644 --- a/acceptance/bundle/resources/vector_search_indexes/basic/script +++ b/acceptance/bundle/resources/vector_search_indexes/basic/script @@ -1,9 +1,7 @@ envsubst < databricks.yml.tmpl > databricks.yml -endpoint_name="vs-endpoint-${UNIQUE_NAME}" cleanup() { trace $CLI bundle destroy --auto-approve - trace $CLI vector-search-endpoints delete-endpoint "${endpoint_name}" rm -f out.requests.txt } trap cleanup EXIT @@ -12,8 +10,6 @@ trace $CLI bundle validate trace $CLI bundle summary -trace $CLI vector-search-endpoints create-endpoint "${endpoint_name}" STANDARD | jq '{id, name, endpoint_type}' - rm -f out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/databricks.yml.tmpl b/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/databricks.yml.tmpl index 93bedcdafe9..9cfd76fe06e 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/databricks.yml.tmpl +++ b/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/databricks.yml.tmpl @@ -5,10 +5,14 @@ sync: paths: [] resources: + vector_search_endpoints: + my_endpoint: + name: vs-endpoint-$UNIQUE_NAME + endpoint_type: STANDARD vector_search_indexes: my_index: name: vs-index-$UNIQUE_NAME - endpoint_name: vs-endpoint-$UNIQUE_NAME + endpoint_name: ${resources.vector_search_endpoints.my_endpoint.name} primary_key: id index_type: DELTA_SYNC delta_sync_index_spec: diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/output.txt b/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/output.txt index 6a23c9bebb3..2843d7ebab8 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/output.txt @@ -1,12 +1,5 @@ === Initial deployment ->>> [CLI] vector-search-endpoints create-endpoint vs-endpoint-[UNIQUE_NAME] STANDARD -{ - "id": "[UUID]", - "name": "vs-endpoint-[UNIQUE_NAME]", - "endpoint_type": "STANDARD" -} - >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/drift-vs-index-columns-[UNIQUE_NAME]/default/files... Deploying resources... @@ -15,7 +8,7 @@ Deployment complete! === Plan ignores request-only columns_to_sync drift >>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged >>> [CLI] vector-search-indexes get-index vs-index-[UNIQUE_NAME] { @@ -27,6 +20,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: + delete resources.vector_search_endpoints.my_endpoint delete resources.vector_search_indexes.my_index This action will result in the deletion of the following Vector Search indexes. @@ -38,5 +32,3 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! - ->>> [CLI] vector-search-endpoints delete-endpoint vs-endpoint-[UNIQUE_NAME] diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/script b/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/script index a6872a6b6a3..39b01eefee0 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/script +++ b/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/script @@ -1,15 +1,12 @@ envsubst < databricks.yml.tmpl > databricks.yml -endpoint_name="vs-endpoint-${UNIQUE_NAME}" cleanup() { trace $CLI bundle destroy --auto-approve - trace $CLI vector-search-endpoints delete-endpoint "${endpoint_name}" rm -f out.requests.txt } trap cleanup EXIT title "Initial deployment" -trace $CLI vector-search-endpoints create-endpoint "${endpoint_name}" STANDARD | jq '{id, name, endpoint_type}' trace $CLI bundle deploy title "Plan ignores request-only columns_to_sync drift" diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/databricks.yml.tmpl b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/databricks.yml.tmpl index 8e4dfbbcb6c..986f9507e01 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/databricks.yml.tmpl +++ b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/databricks.yml.tmpl @@ -5,10 +5,14 @@ sync: paths: [] resources: + vector_search_endpoints: + my_endpoint: + name: vs-endpoint-$UNIQUE_NAME + endpoint_type: STANDARD vector_search_indexes: my_index: name: vs-index-$UNIQUE_NAME - endpoint_name: vs-endpoint-$UNIQUE_NAME + endpoint_name: ${resources.vector_search_endpoints.my_endpoint.name} primary_key: id index_type: DELTA_SYNC delta_sync_index_spec: diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/output.txt b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/output.txt index 7027a4b165e..58cd871d0ed 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/output.txt @@ -1,10 +1,4 @@ ->>> [CLI] vector-search-endpoints create-endpoint vs-endpoint-[UNIQUE_NAME] STANDARD -{ - "name": "vs-endpoint-[UNIQUE_NAME]", - "endpoint_type": "STANDARD" -} - === Initial deployment >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/drift-vs-index-deleted-[UNIQUE_NAME]/default/files... @@ -19,7 +13,7 @@ Deployment complete! >>> [CLI] bundle plan create vector_search_indexes.my_index -Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged +Plan: 1 to add, 0 to change, 0 to delete, 1 unchanged === Deploy recreates the index >>> [CLI] bundle deploy @@ -38,6 +32,7 @@ Deployment complete! >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: + delete resources.vector_search_endpoints.my_endpoint delete resources.vector_search_indexes.my_index This action will result in the deletion of the following Vector Search indexes. @@ -49,5 +44,3 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! - ->>> [CLI] vector-search-endpoints delete-endpoint vs-endpoint-[UNIQUE_NAME] diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/script b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/script index 9409de8da4f..ae461b4758c 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/script +++ b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/script @@ -1,16 +1,12 @@ envsubst < databricks.yml.tmpl > databricks.yml -endpoint_name="vs-endpoint-${UNIQUE_NAME}" index_name="vs-index-${UNIQUE_NAME}" cleanup() { trace $CLI bundle destroy --auto-approve - trace $CLI vector-search-endpoints delete-endpoint "${endpoint_name}" rm -f out.requests.txt } trap cleanup EXIT -trace $CLI vector-search-endpoints create-endpoint "${endpoint_name}" STANDARD | jq '{name, endpoint_type}' - title "Initial deployment" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/vector_search_indexes/grants/select/databricks.yml.tmpl b/acceptance/bundle/resources/vector_search_indexes/grants/select/databricks.yml.tmpl index db6db5792c9..855d2f1f141 100644 --- a/acceptance/bundle/resources/vector_search_indexes/grants/select/databricks.yml.tmpl +++ b/acceptance/bundle/resources/vector_search_indexes/grants/select/databricks.yml.tmpl @@ -5,10 +5,14 @@ sync: paths: [] resources: + vector_search_endpoints: + my_endpoint: + name: vs-endpoint-$UNIQUE_NAME + endpoint_type: STANDARD vector_search_indexes: my_index: name: main.default.vs_index_$UNIQUE_NAME - endpoint_name: vs-endpoint-$UNIQUE_NAME + endpoint_name: ${resources.vector_search_endpoints.my_endpoint.name} primary_key: id index_type: DIRECT_ACCESS direct_access_index_spec: diff --git a/acceptance/bundle/resources/vector_search_indexes/grants/select/output.txt b/acceptance/bundle/resources/vector_search_indexes/grants/select/output.txt index b0c58590730..a425110276d 100644 --- a/acceptance/bundle/resources/vector_search_indexes/grants/select/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/grants/select/output.txt @@ -1,16 +1,10 @@ ->>> [CLI] vector-search-endpoints create-endpoint vs-endpoint-[UNIQUE_NAME] STANDARD -{ - "id": "[UUID]", - "name": "vs-endpoint-[UNIQUE_NAME]", - "endpoint_type": "STANDARD" -} - >>> [CLI] bundle plan +create vector_search_endpoints.my_endpoint create vector_search_indexes.my_index create vector_search_indexes.my_index.grants -Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged +Plan: 3 to add, 0 to change, 0 to delete, 0 unchanged >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/vs-index-grants-[UNIQUE_NAME]/default/files... @@ -19,7 +13,7 @@ Updating deployment state... Deployment complete! >>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged >>> [CLI] grants get table main.default.vs_index_[UNIQUE_NAME] { @@ -37,6 +31,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: + delete resources.vector_search_endpoints.my_endpoint delete resources.vector_search_indexes.my_index This action will result in the deletion of the following Vector Search indexes. @@ -48,5 +43,3 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! - ->>> [CLI] vector-search-endpoints delete-endpoint vs-endpoint-[UNIQUE_NAME] diff --git a/acceptance/bundle/resources/vector_search_indexes/grants/select/script b/acceptance/bundle/resources/vector_search_indexes/grants/select/script index 6aac790b72d..5e3062f7e4f 100644 --- a/acceptance/bundle/resources/vector_search_indexes/grants/select/script +++ b/acceptance/bundle/resources/vector_search_indexes/grants/select/script @@ -1,16 +1,12 @@ envsubst < databricks.yml.tmpl > databricks.yml -endpoint_name="vs-endpoint-${UNIQUE_NAME}" index_name="main.default.vs_index_${UNIQUE_NAME}" cleanup() { trace $CLI bundle destroy --auto-approve - trace $CLI vector-search-endpoints delete-endpoint "${endpoint_name}" rm -f out.requests.txt } trap cleanup EXIT -trace $CLI vector-search-endpoints create-endpoint "${endpoint_name}" STANDARD | jq '{id, name, endpoint_type}' - trace $CLI bundle plan rm -f out.requests.txt diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/databricks.yml.tmpl b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/databricks.yml.tmpl index 88373a5a68b..768334ee2a6 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/databricks.yml.tmpl +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/databricks.yml.tmpl @@ -5,10 +5,14 @@ sync: paths: [] resources: + vector_search_endpoints: + my_endpoint: + name: vs-endpoint-$UNIQUE_NAME + endpoint_type: STANDARD vector_search_indexes: my_index: name: vs-index-$UNIQUE_NAME - endpoint_name: vs-endpoint-$UNIQUE_NAME + endpoint_name: ${resources.vector_search_endpoints.my_endpoint.name} primary_key: id index_type: DELTA_SYNC delta_sync_index_spec: diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/output.txt b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/output.txt index e4f9b741ff6..23ac9878b99 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/output.txt @@ -1,12 +1,5 @@ === Initial deployment with DELTA_SYNC index_type ->>> [CLI] vector-search-endpoints create-endpoint vs-endpoint-[UNIQUE_NAME] STANDARD -{ - "id": "[UUID]", - "name": "vs-endpoint-[UNIQUE_NAME]", - "endpoint_type": "STANDARD" -} - >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-vs-index-[UNIQUE_NAME]/default/files... Deploying resources... @@ -27,7 +20,7 @@ Deployment complete! >>> [CLI] bundle plan recreate vector_search_indexes.my_index -Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged >>> [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-vs-index-[UNIQUE_NAME]/default/files... @@ -52,6 +45,7 @@ Deployment complete! >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: + delete resources.vector_search_endpoints.my_endpoint delete resources.vector_search_indexes.my_index This action will result in the deletion of the following Vector Search indexes. @@ -63,5 +57,3 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! - ->>> [CLI] vector-search-endpoints delete-endpoint vs-endpoint-[UNIQUE_NAME] diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/script b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/script index bb248b50ec7..630628bd3b3 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/script +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/index_type/script @@ -1,9 +1,7 @@ envsubst < databricks.yml.tmpl > databricks.yml -endpoint_name="vs-endpoint-${UNIQUE_NAME}" cleanup() { trace $CLI bundle destroy --auto-approve - trace $CLI vector-search-endpoints delete-endpoint "${endpoint_name}" rm -f out.requests.txt } trap cleanup EXIT @@ -19,7 +17,6 @@ print_requests() { } title "Initial deployment with DELTA_SYNC index_type" -trace $CLI vector-search-endpoints create-endpoint "${endpoint_name}" STANDARD | jq '{id, name, endpoint_type}' rm -f out.requests.txt trace $CLI bundle deploy From 971739a9af37a0607bc6b0e0ea1b21e2b783dd6a Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 7 May 2026 10:28:09 +0200 Subject: [PATCH 40/47] direct: omit DoUpdate for VS indexes; assert SDK fields are classified Vector search indexes have no update API. Previously DoUpdate was a no-op, which meant a future SDK field that wasn't declared in recreate_on_changes/ignore_remote_changes would be classified as Update by the planner and silently no-op at deploy time. Drop the no-op DoUpdate so the framework's existing check at bundle_plan.go errors loudly ("resource does not support update action but plan produced update") if a plan ever produces Update for this resource. Add a reflection-based unit test that catches the same gap earlier, mirroring the pattern in app_test.go. Co-authored-by: Isaac --- .../direct/dresources/vector_search_index.go | 9 ++-- .../dresources/vector_search_index_test.go | 46 +++++++++++++++++++ 2 files changed, 51 insertions(+), 4 deletions(-) create mode 100644 bundle/direct/dresources/vector_search_index_test.go diff --git a/bundle/direct/dresources/vector_search_index.go b/bundle/direct/dresources/vector_search_index.go index 451857e6d99..cd94fd2667a 100644 --- a/bundle/direct/dresources/vector_search_index.go +++ b/bundle/direct/dresources/vector_search_index.go @@ -133,10 +133,11 @@ func (r *ResourceVectorSearchIndex) DoCreate(ctx context.Context, config *Vector return config.Name, &VectorSearchIndexRemote{VectorIndex: index, EndpointUuid: endpointUuid}, nil } -func (r *ResourceVectorSearchIndex) DoUpdate(ctx context.Context, id string, config *VectorSearchIndexState, entry *PlanEntry) (*VectorSearchIndexRemote, error) { - // Vector search indexes have no update API; all field changes trigger recreation via resources.yml. - return nil, nil -} +// No DoUpdate: vector search indexes have no update API. All SDK fields are +// declared in resources.yml under recreate_on_changes or ignore_remote_changes. +// If a future SDK bump adds a new field that isn't classified, the framework +// rejects the resulting Update plan at bundle_plan.go (see also the reflection +// test in vector_search_index_test.go which catches it earlier at unit-test time). func (r *ResourceVectorSearchIndex) DoDelete(ctx context.Context, id string) error { return r.client.VectorSearchIndexes.DeleteIndexByIndexName(ctx, id) diff --git a/bundle/direct/dresources/vector_search_index_test.go b/bundle/direct/dresources/vector_search_index_test.go new file mode 100644 index 00000000000..63c52a7926e --- /dev/null +++ b/bundle/direct/dresources/vector_search_index_test.go @@ -0,0 +1,46 @@ +package dresources + +import ( + "reflect" + "strings" + "testing" + + "github.com/databricks/databricks-sdk-go/service/vectorsearch" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestVectorSearchIndexAllSDKFieldsAreClassified guards against a future SDK +// bump silently adding a field that the planner classifies as Update. The +// resource has no update API and intentionally omits DoUpdate, so any +// unclassified field would surface as a deploy-time framework error +// ("resource does not support update action but plan produced update"). This +// test catches the gap at unit-test time instead. +func TestVectorSearchIndexAllSDKFieldsAreClassified(t *testing.T) { + config := GetResourceConfig("vector_search_indexes") + require.NotNil(t, config) + + classified := map[string]bool{} + for _, field := range config.RecreateOnChanges { + classified[field.Field.String()] = true + } + for _, field := range config.IgnoreRemoteChanges { + classified[field.Field.String()] = true + } + + sdkType := reflect.TypeFor[vectorsearch.CreateVectorIndexRequest]() + for i := range sdkType.NumField() { + field := sdkType.Field(i) + jsonTag := field.Tag.Get("json") + if jsonTag == "" || jsonTag == "-" { + continue + } + jsonTag = strings.TrimSuffix(jsonTag, ",omitempty") + assert.Truef(t, classified[jsonTag], + "field %q is not declared in resources.yml under vector_search_indexes; "+ + "vector_search_indexes has no update API, so every SDK field must be in "+ + "recreate_on_changes or ignore_remote_changes", + jsonTag, + ) + } +} From ebad98c359d32b978ce1d10969479b707ee1767b Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 7 May 2026 15:37:37 +0200 Subject: [PATCH 41/47] Revert "direct: drop empty-id state recovery, now handled by #5173" This reverts commit b8483e7d82eadd2bb15f126a25d786bd402f829a. --- bundle/direct/apply.go | 9 ++++++--- bundle/direct/bundle_plan.go | 11 +++++------ 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index 4e2ff247908..6921769bd97 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -86,14 +86,17 @@ func (d *DeploymentUnit) Recreate(ctx context.Context, db *dstate.DeploymentStat return fmt.Errorf("deleting old id=%s: %w", oldID, err) } - err = db.SaveState(d.ResourceKey, "", nil, nil) + // Drop the state entry so a subsequent failure of Create or WaitAfterDelete + // leaves no malformed (empty-ID) entry behind. The next plan will see "no + // state" and retry as Create. + err = db.DeleteState(d.ResourceKey) if err != nil { return fmt.Errorf("deleting state: %w", err) } // Wait for asynchronous teardown to finish before re-creating the same - // name. Done after the state update so the bundle stays consistent if the - // wait times out — the resource is marked deleted, retry on next plan. + // name. Done after DeleteState so the bundle stays consistent if the wait + // times out — the resource is no longer tracked in state, retry on next plan. err = d.Adapter.WaitAfterDelete(ctx, oldID) if err != nil { return fmt.Errorf("waiting after deleting id=%s: %w", oldID, err) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index e8a2b16d3bd..3c09ec2e0b8 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -181,16 +181,15 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks } dbentry, hasEntry := b.StateDB.GetResourceEntry(resourceKey) - if !hasEntry { + // Tolerate empty-ID entries from older partial-recreate failures + // (apply.Recreate now deletes state on the way through, but pre-fix + // state files may still carry a malformed entry). Treat as missing + // and let the resource be re-created on this plan. + if !hasEntry || dbentry.ID == "" { entry.Action = deployplan.Create return true } - if dbentry.ID == "" { - logdiag.LogError(ctx, fmt.Errorf("%s: invalid state: empty id", errorPrefix)) - return false - } - savedState, err := parseState(adapter.StateType(), dbentry.State) if err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: interpreting state: %w", errorPrefix, err)) From 4c68ad5d7d1c66c7e5b36f27cfb9a57114f60a6b Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 7 May 2026 15:49:13 +0200 Subject: [PATCH 42/47] acceptance: drop endpoint_uuid skip from with_endpoint plan output Main reverted vector_search_endpoints UUID persistence in #5193, so the endpoint plan no longer carries a synthetic endpoint_uuid change to be classified as Skip via OverrideChangeDesc. Regenerate the with_endpoint plan output to match. Co-authored-by: Isaac --- .../vector_search_indexes/recreate/with_endpoint/output.txt | 6 ------ 1 file changed, 6 deletions(-) diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt index 64425989abb..d124db04e23 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt @@ -50,12 +50,6 @@ Plan: 2 to add, 0 to change, 2 to delete, 0 unchanged "old": "STANDARD", "new": "STORAGE_OPTIMIZED", "remote": "STANDARD" - }, - "endpoint_uuid": { - "action": "skip", - "reason": "custom", - "old": "[UUID]", - "remote": "[UUID]" } } }, From 5c1944d7f32a87b173a15c6e4ba28abaf1e9e4b3 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 7 May 2026 15:49:36 +0200 Subject: [PATCH 43/47] Stop applying name prefix to vector_search_indexes The 3-part UC name (catalog.schema.index) is the API primary key: CreateIndex addresses by name and DoCreate returns it as the deployment id. Prefixing it changed which remote object the bundle addressed, not just its display label. Mirrors #5209's same change for vector_search_endpoints. Drop the leaf-only prefix loop and the vectorSearchIndexPrefixPos helper in apply_presets.go, add VectorSearchIndex to the no-rename carve-out in apply_target_mode_test.go, and remove the now-obsolete TestVectorSearchIndexNamePrefixing. Co-authored-by: Isaac --- .../validate/presets_name_prefix/output.txt | 4 +- .../output.txt | 6 +-- .../mutator/resourcemutator/apply_presets.go | 37 ++----------- .../resourcemutator/apply_target_mode_test.go | 52 ++----------------- 4 files changed, 12 insertions(+), 87 deletions(-) diff --git a/acceptance/bundle/validate/presets_name_prefix/output.txt b/acceptance/bundle/validate/presets_name_prefix/output.txt index 3c633c7d743..eebbfd55374 100644 --- a/acceptance/bundle/validate/presets_name_prefix/output.txt +++ b/acceptance/bundle/validate/presets_name_prefix/output.txt @@ -40,7 +40,7 @@ "vs_index": { "endpoint_name": "vs_endpoint", "index_type": "DELTA_SYNC", - "name": "my_catalog.my_schema.prefixmy_index", + "name": "my_catalog.my_schema.my_index", "primary_key": "id" } }, @@ -95,7 +95,7 @@ "vs_index": { "endpoint_name": "vs_endpoint", "index_type": "DELTA_SYNC", - "name": "my_catalog.my_schema.prefix_my_index", + "name": "my_catalog.my_schema.my_index", "primary_key": "id" } }, diff --git a/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/output.txt b/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/output.txt index 3dd823c9c9f..aec6743e997 100644 --- a/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/output.txt +++ b/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/output.txt @@ -11,19 +11,19 @@ "vs_index_literal": { "endpoint_name": "vs_endpoint", "index_type": "DELTA_SYNC", - "name": "my_catalog.my_schema.prefix_literal_index", + "name": "my_catalog.my_schema.literal_index", "primary_key": "id" }, "vs_index_partial_ref": { "endpoint_name": "${resources.vector_search_endpoints.vs_endpoint.name}", "index_type": "DELTA_SYNC", - "name": "my_catalog.my_schema.prefix_partial_ref_index", + "name": "my_catalog.my_schema.partial_ref_index", "primary_key": "id" }, "vs_index_ref": { "endpoint_name": "${resources.vector_search_endpoints.vs_endpoint.name}", "index_type": "DELTA_SYNC", - "name": "my_catalog.my_schema.prefix_ref_index", + "name": "my_catalog.my_schema.ref_index", "primary_key": "id" } } diff --git a/bundle/config/mutator/resourcemutator/apply_presets.go b/bundle/config/mutator/resourcemutator/apply_presets.go index 8e4457ddf00..663adf4a281 100644 --- a/bundle/config/mutator/resourcemutator/apply_presets.go +++ b/bundle/config/mutator/resourcemutator/apply_presets.go @@ -294,17 +294,10 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos // (it's what GET/UPDATE/DELETE address by), so prefixing it would change // the resource's identity rather than just its display name. - // Vector Search Indexes: Prefix - // The name is a 3-part UC identifier (catalog.schema.index); prefix only - // the last component since catalog and schema are external references. - for _, e := range r.VectorSearchIndexes { - if e == nil { - continue - } - if pos := vectorSearchIndexPrefixPos(e.Name); pos >= 0 { - e.Name = e.Name[:pos] + normalizePrefix(prefix) + e.Name[pos:] - } - } + // Vector Search Indexes: no prefix. The 3-part UC name (catalog.schema.index) + // is the API primary key (CreateIndex addresses by name and DoCreate returns + // it as the deployment id), so prefixing would change the resource's + // identity rather than just its display name. return diags } @@ -337,28 +330,6 @@ func toTagArray(tags map[string]string) []Tag { return tagArray } -// vectorSearchIndexPrefixPos returns the byte offset at which to insert the -// name prefix into a vector search index name, or -1 to skip prefixing. -// -// ApplyPresets runs before reference/variable resolution, so the name may -// still carry literal `${...}` tokens. We walk back from the end and prefix -// at the first `.` we find, which is the catalog.schema.index separator -// once the literal tail is non-empty. If we instead hit `$` or `}` first, -// the tail is occluded by a `${...}` expression and prefixing would corrupt -// it (e.g. `${var.full_name}` -> `${var.dev_user_full_name}`); in that case -// skip and let the user compose the prefix into the variable. -func vectorSearchIndexPrefixPos(name string) int { - for i := len(name) - 1; i >= 0; i-- { - switch name[i] { - case '.': - return i + 1 - case '$', '}': - return -1 - } - } - return 0 -} - // normalizePrefix prefixes strings like '[dev lennart] ' to 'dev_lennart_'. // We leave unicode letters and numbers but remove all "special characters." func normalizePrefix(prefix string) string { diff --git a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go index 1a6cf502ce8..a9aeda46c61 100644 --- a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go +++ b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go @@ -316,8 +316,8 @@ func TestProcessTargetModeDevelopment(t *testing.T) { // Vector search endpoint 1: name is the primary key, so it must not be prefixed. assert.Equal(t, "vs_endpoint1", b.Config.Resources.VectorSearchEndpoints["vs_endpoint1"].Name) - // Vector search index 1: only the leaf name is prefixed, since catalog and schema are external - assert.Equal(t, "main.default.dev_lennart_vs_index1", b.Config.Resources.VectorSearchIndexes["vs_index1"].Name) + // Vector search index 1: name is the primary key, so it must not be prefixed. + assert.Equal(t, "main.default.vs_index1", b.Config.Resources.VectorSearchIndexes["vs_index1"].Name) // Registered model 1 assert.Equal(t, "dev_lennart_registeredmodel1", b.Config.Resources.RegisteredModels["registeredmodel1"].Name) @@ -440,6 +440,7 @@ func TestAppropriateResourcesAreRenamed(t *testing.T) { reflect.TypeFor[*resources.ExternalLocation](), reflect.TypeFor[*resources.Volume](), reflect.TypeFor[*resources.VectorSearchEndpoint](), + reflect.TypeFor[*resources.VectorSearchIndex](), } // Resources whose Name is server-generated or otherwise not a user-facing @@ -506,53 +507,6 @@ func TestDisableLockingDisabled(t *testing.T) { assert.True(t, b.Config.Bundle.Deployment.Lock.IsEnabled(), "Deployment lock should remain enabled in development mode when explicitly enabled") } -func TestVectorSearchIndexNamePrefixing(t *testing.T) { - cases := []struct { - key string - name string - want string - }{ - { - // Trailing component is a ref: skip, since prefixing would inject - // the prefix inside the ${var.index} expression. - key: "vs_index_all_refs", - name: "${var.catalog}.${var.schema}.${var.index}", - want: "${var.catalog}.${var.schema}.${var.index}", - }, - { - // Catalog and schema are refs but the leaf is literal: prefix the leaf. - key: "vs_index_partial_ref", - name: "${var.catalog}.${var.schema}.non_ref_name", - want: "${var.catalog}.${var.schema}.dev_lennart_non_ref_name", - }, - { - // Whole name is a single ref: skip. - key: "vs_index_full_ref", - name: "${var.full_name}", - want: "${var.full_name}", - }, - } - - b := mockBundle(config.Development) - for _, c := range cases { - b.Config.Resources.VectorSearchIndexes[c.key] = &resources.VectorSearchIndex{ - CreateVectorIndexRequest: vectorsearch.CreateVectorIndexRequest{ - Name: c.name, - EndpointName: "vs_endpoint1", - PrimaryKey: "id", - IndexType: vectorsearch.VectorIndexTypeDeltaSync, - }, - } - } - - diags := bundle.ApplySeq(t.Context(), b, ApplyTargetMode(), ApplyPresets()) - require.NoError(t, diags.Error()) - - for _, c := range cases { - assert.Equal(t, c.want, b.Config.Resources.VectorSearchIndexes[c.key].Name, c.key) - } -} - func TestPrefixAlreadySet(t *testing.T) { b := mockBundle(config.Development) b.Config.Presets.NamePrefix = "custom_lennart_deploy_" From 616715081d9bc0e20f7fd798d9f76cdab41aa4ce Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Mon, 18 May 2026 13:28:24 +0200 Subject: [PATCH 44/47] Post-rebase fixups: WaitAfterCreate signature, columns_to_sync now in remote - WaitAfterCreate now takes id per #5258; the saved config.Name is the same as id, so the body is unchanged. - SDK v0.132.0 (#5237) returns delta_sync_index_spec.columns_to_sync (and the new columns_to_index field) on read. Drop the ignore_remote_changes rule and propagate both from remote in RemapState. Removes the drift/columns_to_sync acceptance test which was asserting the now-stale request-only behavior. Co-authored-by: Isaac --- acceptance/bundle/refschema/out.fields.txt | 6 ++-- .../drift/columns_to_sync/databricks.yml.tmpl | 23 ------------- .../drift/columns_to_sync/out.test.toml | 4 --- .../drift/columns_to_sync/output.txt | 34 ------------------- .../drift/columns_to_sync/script | 16 --------- .../drift/columns_to_sync/test.toml | 1 - .../recreate/with_endpoint/output.txt | 5 +-- .../validate/presets_name_prefix/output.txt | 4 +-- .../output.txt | 2 +- bundle/direct/dresources/resources.yml | 4 --- bundle/direct/dresources/type_test.go | 4 --- .../direct/dresources/vector_search_index.go | 7 ++-- 12 files changed, 14 insertions(+), 96 deletions(-) delete mode 100644 acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/databricks.yml.tmpl delete mode 100644 acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/out.test.toml delete mode 100644 acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/output.txt delete mode 100644 acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/script delete mode 100644 acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/test.toml diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 9bd92911787..7ac0a77ebd3 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -3160,8 +3160,10 @@ resources.vector_search_endpoints.*.permissions[*].user_name string ALL resources.vector_search_indexes.*.creator string REMOTE resources.vector_search_indexes.*.delta_sync_index_spec *vectorsearch.DeltaSyncVectorIndexSpecRequest INPUT STATE resources.vector_search_indexes.*.delta_sync_index_spec *vectorsearch.DeltaSyncVectorIndexSpecResponse REMOTE -resources.vector_search_indexes.*.delta_sync_index_spec.columns_to_sync []string INPUT STATE -resources.vector_search_indexes.*.delta_sync_index_spec.columns_to_sync[*] string INPUT STATE +resources.vector_search_indexes.*.delta_sync_index_spec.columns_to_index []string ALL +resources.vector_search_indexes.*.delta_sync_index_spec.columns_to_index[*] string ALL +resources.vector_search_indexes.*.delta_sync_index_spec.columns_to_sync []string ALL +resources.vector_search_indexes.*.delta_sync_index_spec.columns_to_sync[*] string ALL resources.vector_search_indexes.*.delta_sync_index_spec.embedding_source_columns []vectorsearch.EmbeddingSourceColumn ALL resources.vector_search_indexes.*.delta_sync_index_spec.embedding_source_columns[*] vectorsearch.EmbeddingSourceColumn ALL resources.vector_search_indexes.*.delta_sync_index_spec.embedding_source_columns[*].embedding_model_endpoint_name string ALL diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/databricks.yml.tmpl b/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/databricks.yml.tmpl deleted file mode 100644 index 9cfd76fe06e..00000000000 --- a/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/databricks.yml.tmpl +++ /dev/null @@ -1,23 +0,0 @@ -bundle: - name: drift-vs-index-columns-$UNIQUE_NAME - -sync: - paths: [] - -resources: - vector_search_endpoints: - my_endpoint: - name: vs-endpoint-$UNIQUE_NAME - endpoint_type: STANDARD - vector_search_indexes: - my_index: - name: vs-index-$UNIQUE_NAME - endpoint_name: ${resources.vector_search_endpoints.my_endpoint.name} - primary_key: id - index_type: DELTA_SYNC - delta_sync_index_spec: - source_table: main.default.source_$UNIQUE_NAME - pipeline_type: TRIGGERED - columns_to_sync: - - id - - text diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/out.test.toml deleted file mode 100644 index fe4076cdf9b..00000000000 --- a/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/out.test.toml +++ /dev/null @@ -1,4 +0,0 @@ -Local = true -Cloud = true -RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/output.txt b/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/output.txt deleted file mode 100644 index 2843d7ebab8..00000000000 --- a/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/output.txt +++ /dev/null @@ -1,34 +0,0 @@ - -=== Initial deployment ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/drift-vs-index-columns-[UNIQUE_NAME]/default/files... -Deploying resources... -Updating deployment state... -Deployment complete! - -=== Plan ignores request-only columns_to_sync drift ->>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged - ->>> [CLI] vector-search-indexes get-index vs-index-[UNIQUE_NAME] -{ - "name": "vs-index-[UNIQUE_NAME]", - "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", - "index_type": "DELTA_SYNC", - "primary_key": "id" -} - ->>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.vector_search_endpoints.my_endpoint - delete resources.vector_search_indexes.my_index - -This action will result in the deletion of the following Vector Search indexes. -For Delta Sync indexes, the source Delta Table is preserved but the embedding pipeline is removed. -For Direct Access indexes, all upserted vectors are permanently lost: - delete resources.vector_search_indexes.my_index - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/drift-vs-index-columns-[UNIQUE_NAME]/default - -Deleting files... -Destroy complete! diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/script b/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/script deleted file mode 100644 index 39b01eefee0..00000000000 --- a/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/script +++ /dev/null @@ -1,16 +0,0 @@ -envsubst < databricks.yml.tmpl > databricks.yml - -cleanup() { - trace $CLI bundle destroy --auto-approve - rm -f out.requests.txt -} -trap cleanup EXIT - -title "Initial deployment" -trace $CLI bundle deploy - -title "Plan ignores request-only columns_to_sync drift" -trace $CLI bundle plan - -index_name="vs-index-${UNIQUE_NAME}" -trace $CLI vector-search-indexes get-index "${index_name}" | jq '{name, endpoint_name, index_type, primary_key}' diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/test.toml b/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/test.toml deleted file mode 100644 index f8b3bbe49dd..00000000000 --- a/acceptance/bundle/resources/vector_search_indexes/drift/columns_to_sync/test.toml +++ /dev/null @@ -1 +0,0 @@ -# All configuration inherited from parent test.toml diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt index d124db04e23..f83130804ed 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt @@ -102,7 +102,8 @@ Plan: 2 to add, 0 to change, 2 to delete, 0 unchanged Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-vs-index-with-endpoint-[UNIQUE_NAME]/default/files... This action will result in the deletion or recreation of the following Vector Search indexes. -Delta Sync indexes re-run their embedding pipeline; Direct Access indexes lose all upserted vectors: +Recreating a Delta Sync index re-runs the full embedding pipeline; recreating a Direct Access +index drops all upserted vectors. Both can be expensive to rebuild: recreate resources.vector_search_indexes.my_index Deploying resources... Updating deployment state... @@ -128,7 +129,7 @@ The following resources will be deleted: delete resources.vector_search_indexes.my_index This action will result in the deletion of the following Vector Search indexes. -For Delta Sync indexes, the source Delta table is preserved but the embedding pipeline is removed. +For Delta Sync indexes, the source Delta Table is preserved but the embedding pipeline is removed. For Direct Access indexes, all upserted vectors are permanently lost: delete resources.vector_search_indexes.my_index diff --git a/acceptance/bundle/validate/presets_name_prefix/output.txt b/acceptance/bundle/validate/presets_name_prefix/output.txt index eebbfd55374..08579b5dd2b 100644 --- a/acceptance/bundle/validate/presets_name_prefix/output.txt +++ b/acceptance/bundle/validate/presets_name_prefix/output.txt @@ -33,7 +33,7 @@ "vector_search_endpoints": { "vs_endpoint": { "endpoint_type": "STANDARD", - "name": "prefixvs_endpoint" + "name": "vs_endpoint" } }, "vector_search_indexes": { @@ -88,7 +88,7 @@ "vector_search_endpoints": { "vs_endpoint": { "endpoint_type": "STANDARD", - "name": "prefix_vs_endpoint" + "name": "vs_endpoint" } }, "vector_search_indexes": { diff --git a/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/output.txt b/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/output.txt index aec6743e997..4f9b10e9a8e 100644 --- a/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/output.txt +++ b/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/output.txt @@ -4,7 +4,7 @@ "vector_search_endpoints": { "vs_endpoint": { "endpoint_type": "STANDARD", - "name": "prefix_vs_endpoint" + "name": "vs_endpoint" } }, "vector_search_indexes": { diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 6ff6431aa2f..41c8d3acffb 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -536,7 +536,3 @@ resources: reason: immutable - field: direct_access_index_spec reason: immutable - ignore_remote_changes: - # columns_to_sync is request-only in the create spec and not returned by the read API. - - field: delta_sync_index_spec.columns_to_sync - reason: input_only diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index d2b57688d73..88f246723bf 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -79,10 +79,6 @@ var knownMissingInRemoteType = map[string][]string{ "target_qps", "usage_policy_id", }, - "vector_search_indexes": { - // columns_to_sync is in the request spec but not in the response spec - "delta_sync_index_spec.columns_to_sync", - }, } // commonMissingInStateType lists fields that are commonly missing across all resource types. diff --git a/bundle/direct/dresources/vector_search_index.go b/bundle/direct/dresources/vector_search_index.go index cd94fd2667a..475fd3f96e9 100644 --- a/bundle/direct/dresources/vector_search_index.go +++ b/bundle/direct/dresources/vector_search_index.go @@ -89,7 +89,8 @@ func (*ResourceVectorSearchIndex) RemapState(remote *VectorSearchIndexRemote) *V } if remote.DeltaSyncIndexSpec != nil { state.DeltaSyncIndexSpec = &vectorsearch.DeltaSyncVectorIndexSpecRequest{ - ColumnsToSync: nil, + ColumnsToIndex: remote.DeltaSyncIndexSpec.ColumnsToIndex, + ColumnsToSync: remote.DeltaSyncIndexSpec.ColumnsToSync, EmbeddingSourceColumns: remote.DeltaSyncIndexSpec.EmbeddingSourceColumns, EmbeddingVectorColumns: remote.DeltaSyncIndexSpec.EmbeddingVectorColumns, EmbeddingWritebackTable: remote.DeltaSyncIndexSpec.EmbeddingWritebackTable, @@ -147,9 +148,9 @@ func (r *ResourceVectorSearchIndex) DoDelete(ctx context.Context, id string) err // immediately with metadata of an index whose embedding pipeline is still // provisioning; queries against an index that isn't ready fail. Blocking here // lets dependent resources (and the next plan) see a usable index. -func (r *ResourceVectorSearchIndex) WaitAfterCreate(ctx context.Context, config *VectorSearchIndexState) (*VectorSearchIndexRemote, error) { +func (r *ResourceVectorSearchIndex) WaitAfterCreate(ctx context.Context, id string, config *VectorSearchIndexState) (*VectorSearchIndexRemote, error) { index, err := retries.Poll(ctx, createIndexTimeout, func() (*vectorsearch.VectorIndex, *retries.Err) { - idx, getErr := r.client.VectorSearchIndexes.GetIndexByIndexName(ctx, config.Name) + idx, getErr := r.client.VectorSearchIndexes.GetIndexByIndexName(ctx, id) if getErr != nil { return nil, retries.Halt(getErr) } From f78f31b96cd2eb4f31574af4def6f507c601e988 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Mon, 18 May 2026 13:29:50 +0200 Subject: [PATCH 45/47] Document why RemapState leaves ForceSendFields nil Per denik's PR comment: explain that ForceSendFields is an SDK marshaling concern (which zero-valued fields to wire-serialize) that has no meaning on the read path, so copying it from the response struct would not be useful. Co-authored-by: Isaac --- bundle/direct/dresources/vector_search_index.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bundle/direct/dresources/vector_search_index.go b/bundle/direct/dresources/vector_search_index.go index 475fd3f96e9..9392b2cd44d 100644 --- a/bundle/direct/dresources/vector_search_index.go +++ b/bundle/direct/dresources/vector_search_index.go @@ -96,7 +96,11 @@ func (*ResourceVectorSearchIndex) RemapState(remote *VectorSearchIndexRemote) *V EmbeddingWritebackTable: remote.DeltaSyncIndexSpec.EmbeddingWritebackTable, PipelineType: remote.DeltaSyncIndexSpec.PipelineType, SourceTable: remote.DeltaSyncIndexSpec.SourceTable, - ForceSendFields: nil, + // ForceSendFields is an SDK marshaling concern (which zero-valued + // fields to wire-serialize) that has no meaning on the read path. + // Local config doesn't carry one either, so leave it nil rather + // than copy whatever the response struct happened to use. + ForceSendFields: nil, } } return state From 44e24bdef83aac8f41832a09dd3af12a8ce56047 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Mon, 18 May 2026 13:33:22 +0200 Subject: [PATCH 46/47] acceptance: remove obsolete presets_name_prefix_vs_index_endpoint test The test was a Badness fixture capturing the gap where a literal endpoint_name on a VS index would not follow the endpoint's name prefix. Now that neither VS endpoints (#5209) nor VS indexes are prefixed, the literal form correctly points at the (unprefixed) endpoint, and all three branches of the fixture produce identical output. Co-authored-by: Isaac --- .../databricks.yml.tmpl | 42 ------------------- .../out.test.toml | 3 -- .../output.txt | 30 ------------- .../script | 4 -- .../test.toml | 1 - 5 files changed, 80 deletions(-) delete mode 100644 acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/databricks.yml.tmpl delete mode 100644 acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/out.test.toml delete mode 100644 acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/output.txt delete mode 100644 acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/script delete mode 100644 acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/test.toml diff --git a/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/databricks.yml.tmpl b/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/databricks.yml.tmpl deleted file mode 100644 index c8899a3e3a0..00000000000 --- a/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/databricks.yml.tmpl +++ /dev/null @@ -1,42 +0,0 @@ -bundle: - name: BUNDLE - -workspace: - resource_path: /foo/bar - -resources: - vector_search_endpoints: - vs_endpoint: - name: vs_endpoint - endpoint_type: STANDARD - vector_search_indexes: - # Literal endpoint_name: not rewritten by the prefix preset, so the - # index would target the unprefixed (non-existent) endpoint. - vs_index_literal: - name: my_catalog.my_schema.literal_index - endpoint_name: vs_endpoint - primary_key: id - index_type: DELTA_SYNC - # Reference to the bundle-managed endpoint: resolved after the prefix - # preset runs, so the index correctly targets the prefixed endpoint. - vs_index_ref: - name: my_catalog.my_schema.ref_index - endpoint_name: ${resources.vector_search_endpoints.vs_endpoint.name} - primary_key: id - index_type: DELTA_SYNC - # Partial reference: catalog and schema are variable refs but the leaf - # is literal, so the prefix preset can still prefix the leaf. - vs_index_partial_ref: - name: ${var.catalog}.${var.schema}.partial_ref_index - endpoint_name: ${resources.vector_search_endpoints.vs_endpoint.name} - primary_key: id - index_type: DELTA_SYNC - -variables: - catalog: - default: my_catalog - schema: - default: my_schema - -presets: - name_prefix: "$PREFIX" diff --git a/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/out.test.toml b/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/out.test.toml deleted file mode 100644 index f784a183258..00000000000 --- a/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/out.test.toml +++ /dev/null @@ -1,3 +0,0 @@ -Local = true -Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/output.txt b/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/output.txt deleted file mode 100644 index 4f9b10e9a8e..00000000000 --- a/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/output.txt +++ /dev/null @@ -1,30 +0,0 @@ - ->>> [CLI] bundle validate -o json -{ - "vector_search_endpoints": { - "vs_endpoint": { - "endpoint_type": "STANDARD", - "name": "vs_endpoint" - } - }, - "vector_search_indexes": { - "vs_index_literal": { - "endpoint_name": "vs_endpoint", - "index_type": "DELTA_SYNC", - "name": "my_catalog.my_schema.literal_index", - "primary_key": "id" - }, - "vs_index_partial_ref": { - "endpoint_name": "${resources.vector_search_endpoints.vs_endpoint.name}", - "index_type": "DELTA_SYNC", - "name": "my_catalog.my_schema.partial_ref_index", - "primary_key": "id" - }, - "vs_index_ref": { - "endpoint_name": "${resources.vector_search_endpoints.vs_endpoint.name}", - "index_type": "DELTA_SYNC", - "name": "my_catalog.my_schema.ref_index", - "primary_key": "id" - } - } -} diff --git a/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/script b/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/script deleted file mode 100644 index a5c6230a957..00000000000 --- a/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/script +++ /dev/null @@ -1,4 +0,0 @@ -PREFIX="[prefix]" envsubst < databricks.yml.tmpl > databricks.yml -trace $CLI bundle validate -o json | jq '.resources | {vector_search_endpoints, vector_search_indexes}' - -rm databricks.yml diff --git a/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/test.toml b/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/test.toml deleted file mode 100644 index bb36c8e2502..00000000000 --- a/acceptance/bundle/validate/presets_name_prefix_vs_index_endpoint/test.toml +++ /dev/null @@ -1 +0,0 @@ -Badness = "Literal vector_search_indexes.*.endpoint_name is not rewritten by the name_prefix preset, even when it points at a bundle-managed endpoint that gets prefixed. The DABs convention is to use ${resources.vector_search_endpoints.X.name} (covered by vs_index_ref below), but the literal form silently breaks. Until apply_presets.go cross-references endpoint_name against bundle endpoints, prefer the reference form." From 67b417f29adc4228df251411199c367235b2d866 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Mon, 18 May 2026 13:35:51 +0200 Subject: [PATCH 47/47] schema: regenerate annotations for VectorSearchIndex.index_subtype generate-schema picked up the missing placeholder for index_subtype after the SDK bump; previously this field wasn't in the resource and the schema_test caught the gap on rebase. Co-authored-by: Isaac --- bundle/internal/schema/annotations.yml | 3 +++ bundle/schema/jsonschema.json | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index ae01c42df21..cb2dcfb24ad 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1002,6 +1002,9 @@ github.com/databricks/cli/bundle/config/resources.VectorSearchIndex: "grants": "description": |- PLACEHOLDER + "index_subtype": + "description": |- + PLACEHOLDER "index_type": "description": |- PLACEHOLDER diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index b6a87536321..37e6469e6dc 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1988,6 +1988,9 @@ "grants": { "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/catalog.PrivilegeAssignment" }, + "index_subtype": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/vectorsearch.IndexSubtype" + }, "index_type": { "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/vectorsearch.VectorIndexType" }, @@ -11475,6 +11478,9 @@ { "type": "object", "properties": { + "columns_to_index": { + "$ref": "#/$defs/slice/string" + }, "columns_to_sync": { "$ref": "#/$defs/slice/string" }, @@ -11585,6 +11591,9 @@ } ] }, + "vectorsearch.IndexSubtype": { + "type": "string" + }, "vectorsearch.PipelineType": { "type": "string" },