From c23a67919dd8849e7fd28c2f51ca88948dcae64f Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 1 Jul 2025 19:14:12 +0200 Subject: [PATCH 1/2] Refactor deletes in direct deployment --- .../destroy/all-resources/databricks.yml | 13 ++++++++++ .../bundle/destroy/all-resources/foo.py | 0 .../bundle/destroy/all-resources/output.txt | 16 ++++++++++++ .../bundle/destroy/all-resources/script | 2 ++ bundle/terranova/apply.go | 4 +-- bundle/terranova/tnresources/app.go | 2 +- bundle/terranova/tnresources/job.go | 4 +-- bundle/terranova/tnresources/pipeline.go | 4 +-- bundle/terranova/tnresources/resource.go | 25 ++++++++++++++----- bundle/terranova/tnresources/schema.go | 2 +- 10 files changed, 58 insertions(+), 14 deletions(-) create mode 100644 acceptance/bundle/destroy/all-resources/databricks.yml create mode 100644 acceptance/bundle/destroy/all-resources/foo.py create mode 100644 acceptance/bundle/destroy/all-resources/output.txt create mode 100644 acceptance/bundle/destroy/all-resources/script diff --git a/acceptance/bundle/destroy/all-resources/databricks.yml b/acceptance/bundle/destroy/all-resources/databricks.yml new file mode 100644 index 0000000000..8020b7ec5a --- /dev/null +++ b/acceptance/bundle/destroy/all-resources/databricks.yml @@ -0,0 +1,13 @@ +resources: + pipelines: + my_pipeline: + name: test-pipeline + libraries: + - file: + path: "./foo.py" + + schemas: + my_schema: + name: test-schema + catalog_name: main + comment: COMMENT1 diff --git a/acceptance/bundle/destroy/all-resources/foo.py b/acceptance/bundle/destroy/all-resources/foo.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/acceptance/bundle/destroy/all-resources/output.txt b/acceptance/bundle/destroy/all-resources/output.txt new file mode 100644 index 0000000000..6f4688f8e9 --- /dev/null +++ b/acceptance/bundle/destroy/all-resources/output.txt @@ -0,0 +1,16 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete pipeline my_pipeline + delete schema my_schema + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/destroy/all-resources/script b/acceptance/bundle/destroy/all-resources/script new file mode 100644 index 0000000000..ceb8acee91 --- /dev/null +++ b/acceptance/bundle/destroy/all-resources/script @@ -0,0 +1,2 @@ +trace $CLI bundle deploy +trace $CLI bundle destroy --auto-approve diff --git a/bundle/terranova/apply.go b/bundle/terranova/apply.go index eca7fc80d4..ba4c953a82 100644 --- a/bundle/terranova/apply.go +++ b/bundle/terranova/apply.go @@ -207,7 +207,7 @@ func (d *Deployer) Create(ctx context.Context, resource tnresources.IResource, c } func (d *Deployer) Recreate(ctx context.Context, oldResource tnresources.IResource, oldID string, config any) error { - err := oldResource.DoDelete(ctx, oldID) + err := tnresources.DeleteResource(ctx, d.client, d.group, oldID) if err != nil { return fmt.Errorf("deleting old id=%s: %w", oldID, err) } @@ -267,7 +267,7 @@ func (d *Deployer) Update(ctx context.Context, resource tnresources.IResource, o func (d *Deployer) Delete(ctx context.Context, resource tnresources.IResource, oldID string) error { // TODO: recognize 404 and 403 as "deleted" and proceed to removing state - err := resource.DoDelete(ctx, oldID) + err := tnresources.DeleteResource(ctx, d.client, d.group, oldID) if err != nil { return fmt.Errorf("deleting id=%s: %w", oldID, err) } diff --git a/bundle/terranova/tnresources/app.go b/bundle/terranova/tnresources/app.go index 273366c578..a69a462b16 100644 --- a/bundle/terranova/tnresources/app.go +++ b/bundle/terranova/tnresources/app.go @@ -54,7 +54,7 @@ func (r *ResourceApp) DoUpdate(ctx context.Context, id string) (string, error) { return response.Name, nil } -func (r *ResourceApp) DoDelete(ctx context.Context, id string) error { +func DeleteApp(ctx context.Context, client *databricks.WorkspaceClient, id string) error { // TODO: implement app deletion return nil } diff --git a/bundle/terranova/tnresources/job.go b/bundle/terranova/tnresources/job.go index 144535ea1e..373551d025 100644 --- a/bundle/terranova/tnresources/job.go +++ b/bundle/terranova/tnresources/job.go @@ -52,12 +52,12 @@ func (r *ResourceJob) DoUpdate(ctx context.Context, id string) (string, error) { return id, nil } -func (r *ResourceJob) DoDelete(ctx context.Context, id string) error { +func DeleteJob(ctx context.Context, client *databricks.WorkspaceClient, id string) error { idInt, err := strconv.ParseInt(id, 10, 64) if err != nil { return err } - err = r.client.Jobs.DeleteByJobId(ctx, idInt) + err = client.Jobs.DeleteByJobId(ctx, idInt) if err != nil { return SDKError{Method: "Jobs.DeleteByJobId", Err: err} } diff --git a/bundle/terranova/tnresources/pipeline.go b/bundle/terranova/tnresources/pipeline.go index 481555a770..4e9351d768 100644 --- a/bundle/terranova/tnresources/pipeline.go +++ b/bundle/terranova/tnresources/pipeline.go @@ -49,8 +49,8 @@ func (r *ResourcePipeline) DoUpdate(ctx context.Context, id string) (string, err return id, nil } -func (r *ResourcePipeline) DoDelete(ctx context.Context, id string) error { - err := r.client.Pipelines.DeleteByPipelineId(ctx, id) +func DeletePipeline(ctx context.Context, client *databricks.WorkspaceClient, id string) error { + err := client.Pipelines.DeleteByPipelineId(ctx, id) if err != nil { return SDKError{Method: "Pipelines.DeleteByPipelineId", Err: err} } diff --git a/bundle/terranova/tnresources/resource.go b/bundle/terranova/tnresources/resource.go index 3595a38004..d613a21352 100644 --- a/bundle/terranova/tnresources/resource.go +++ b/bundle/terranova/tnresources/resource.go @@ -26,6 +26,15 @@ var supportedResourcesTypes = map[string]reflect.Type{ "apps": reflect.TypeOf(ResourceApp{}.config), } +type DeleteResourceFN = func(ctx context.Context, client *databricks.WorkspaceClient, oldID string) error + +var deletableResources = map[string]DeleteResourceFN{ + "jobs": DeleteJob, + "pipelines": DeletePipeline, + "schemas": DeleteSchema, + "apps": DeleteApp, +} + type IResource interface { Config() any @@ -34,9 +43,7 @@ type IResource interface { // Update the resource. Returns id of the resource. // Usually returns the same id as oldId but can also return a different one (e.g. schemas and volumes when certain fields are changed) - DoUpdate(ctx context.Context, oldId string) (string, error) - - DoDelete(ctx context.Context, oldId string) error + DoUpdate(ctx context.Context, oldID string) (string, error) WaitAfterCreate(ctx context.Context) error WaitAfterUpdate(ctx context.Context) error @@ -59,9 +66,7 @@ func invokeConstructor(ctor reflect.Value, client *databricks.WorkspaceClient, c // Prepare the config value matching the expected type. var cfgVal reflect.Value if cfg == nil { - // Treat nil as a request for the zero value of the expected config type. This - // is useful for actions (like deletion) where the config is irrelevant. - cfgVal = reflect.Zero(expectedCfgType) + return nil, errors.New("internal error, config must not be nil") } else { suppliedVal := reflect.ValueOf(cfg) if suppliedVal.Type() != expectedCfgType { @@ -116,3 +121,11 @@ func New(client *databricks.WorkspaceClient, group, name string, config any) (IR return result, cfgType, nil } + +func DeleteResource(ctx context.Context, client *databricks.WorkspaceClient, group, id string) error { + fn, ok := deletableResources[group] + if !ok { + return fmt.Errorf("cannot delete %s", group) + } + return fn(ctx, client, id) +} diff --git a/bundle/terranova/tnresources/schema.go b/bundle/terranova/tnresources/schema.go index 5d55328e42..fbd4359543 100644 --- a/bundle/terranova/tnresources/schema.go +++ b/bundle/terranova/tnresources/schema.go @@ -51,7 +51,7 @@ func (r *ResourceSchema) DoUpdate(ctx context.Context, id string) (string, error return response.FullName, nil } -func (r *ResourceSchema) DoDelete(ctx context.Context, id string) error { +func DeleteSchema(ctx context.Context, client *databricks.WorkspaceClient, id string) error { // TODO: implement schema deletion return nil } From f80eba8ed4f60c5723bee06f0df58022210ef0b9 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 2 Jul 2025 09:06:53 +0200 Subject: [PATCH 2/2] constants for key names --- bundle/terranova/tnresources/resource.go | 31 +++++++++++++++--------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/bundle/terranova/tnresources/resource.go b/bundle/terranova/tnresources/resource.go index d613a21352..ec28298bbc 100644 --- a/bundle/terranova/tnresources/resource.go +++ b/bundle/terranova/tnresources/resource.go @@ -11,28 +11,35 @@ import ( "github.com/databricks/databricks-sdk-go" ) +const ( + _jobs = "jobs" + _pipelines = "pipelines" + _schemas = "schemas" + _apps = "apps" +) + var supportedResources = map[string]reflect.Value{ - "jobs": reflect.ValueOf(NewResourceJob), - "pipelines": reflect.ValueOf(NewResourcePipeline), - "schemas": reflect.ValueOf(NewResourceSchema), - "apps": reflect.ValueOf(NewResourceApp), + _jobs: reflect.ValueOf(NewResourceJob), + _pipelines: reflect.ValueOf(NewResourcePipeline), + _schemas: reflect.ValueOf(NewResourceSchema), + _apps: reflect.ValueOf(NewResourceApp), } // This types matches what Config() returns and should match 'config' field in the resource struct var supportedResourcesTypes = map[string]reflect.Type{ - "jobs": reflect.TypeOf(ResourceJob{}.config), - "pipelines": reflect.TypeOf(ResourcePipeline{}.config), - "schemas": reflect.TypeOf(ResourceSchema{}.config), - "apps": reflect.TypeOf(ResourceApp{}.config), + _jobs: reflect.TypeOf(ResourceJob{}.config), + _pipelines: reflect.TypeOf(ResourcePipeline{}.config), + _schemas: reflect.TypeOf(ResourceSchema{}.config), + _apps: reflect.TypeOf(ResourceApp{}.config), } type DeleteResourceFN = func(ctx context.Context, client *databricks.WorkspaceClient, oldID string) error var deletableResources = map[string]DeleteResourceFN{ - "jobs": DeleteJob, - "pipelines": DeletePipeline, - "schemas": DeleteSchema, - "apps": DeleteApp, + _jobs: DeleteJob, + _pipelines: DeletePipeline, + _schemas: DeleteSchema, + _apps: DeleteApp, } type IResource interface {