Allow referencing bundle resources by name

.codegen.json

@@ -5,7 +5,8 @@
     },
     "batch": {
         ".codegen/cmds-workspace.go.tmpl": "cmd/workspace/cmd.go",
-        ".codegen/cmds-account.go.tmpl": "cmd/account/cmd.go"
+        ".codegen/cmds-account.go.tmpl": "cmd/account/cmd.go",
+        ".codegen/lookup.go.tmpl": "bundle/config/variable/lookup.go"
     },
     "toolchain": {
         "required": ["go"],

.codegen/lookup.go.tmpl

@@ -0,0 +1,122 @@
+// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT.
+
+package variable
+
+{{ $allowlist :=
+	list
+		"alerts"
+		"clusters"
+		"cluster-policies"
+		"clusters"
+		"dashboards"
+		"instance-pools"
+		"jobs"
+		"metastores"
+		"pipelines"
+		"queries"
+		"warehouses"
+}}
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/databricks/databricks-sdk-go"
+)
+
+type Lookup struct {
+	{{range .Services -}}
+	{{- if in $allowlist .KebabName -}}
+	{{.Singular.PascalName}} string `json:"{{.Singular.KebabName}},omitempty"`
+
+	{{end}}
+	{{- end}}
+}
+
+func LookupFromMap(m map[string]interface{}) *Lookup {
+	l := &Lookup{}
+	{{range .Services -}}
+	{{- if in $allowlist .KebabName -}}
+	if v, ok := m["{{.Singular.KebabName}}"]; ok {
+		l.{{.Singular.PascalName}} = v.(string)
+	}
+	{{end -}}
+	{{- end}}
+	return l
+}
+
+func (l *Lookup) Resolve(ctx context.Context, w *databricks.WorkspaceClient) (string, error) {
+	if err := l.validate(); err != nil {
+		return "", err
+	}
+
+	resolvers := resolvers()
+	{{range .Services -}}
+	{{- if in $allowlist .KebabName -}}
+	if l.{{.Singular.PascalName}} != "" {
+		return resolvers["{{.Singular.KebabName}}"](ctx, w, l.{{.Singular.PascalName}})
+	}
+	{{end -}}
+	{{- end}}
+
+	return "", fmt.Errorf("no valid lookup fields provided")
+}
+
+func (l *Lookup) String() string {
+	{{range .Services -}}
+	{{- if in $allowlist .KebabName -}}
+	if l.{{.Singular.PascalName}} != "" {
+		return fmt.Sprintf("{{.Singular.KebabName}}: %s", l.{{.Singular.PascalName}})
+	}
+	{{end -}}
+	{{- end}}
+	return ""
+}
+
+func (l *Lookup) validate() error {
+	// Validate that only one field is set
+	count := 0
+	{{range .Services -}}
+	{{- if in $allowlist .KebabName -}}
+	if l.{{.Singular.PascalName}} != "" {
+		count++
+	}
+	{{end -}}
+	{{- end}}
+
+	if count != 1 {
+		return fmt.Errorf("exactly one lookup field must be provided")
+	}
+
+	if strings.Contains(l.String(), "${var") {
+		return fmt.Errorf("lookup fields cannot contain variable references")
+	}
+
+	return nil
+}
+
+
+type resolverFunc func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error)
+
+func resolvers() map[string](resolverFunc) {
+	resolvers := make(map[string](resolverFunc), 0)
+	{{range .Services -}}
+	{{- if in $allowlist .KebabName -}}
+	resolvers["{{.Singular.KebabName}}"] = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) {
+		entity, err := w.{{.PascalName}}.GetBy{{range .List.NamedIdMap.NamePath}}{{.PascalName}}{{end}}(ctx, name)
+		if err != nil {
+			return "", err
+		}
+
+		return fmt.Sprint(entity{{ template "field-path" .List.NamedIdMap.IdPath }}), nil
+	}
+	{{end -}}
+	{{- end}}
+
+	return resolvers
+}
+
+
+{{- define "field-path" -}}
+	{{- range .}}.{{.PascalName}}{{end}}
+{{- end -}}

.gitattributes

@@ -1,3 +1,4 @@
+bundle/config/variable/lookup.go linguist-generated=true
 cmd/account/access-control/access-control.go linguist-generated=true
 cmd/account/billable-usage/billable-usage.go linguist-generated=true
 cmd/account/budgets/budgets.go linguist-generated=true

bundle/config/mutator/resolve_resource_references.go

@@ -0,0 +1,48 @@
+package mutator
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/libs/log"
+	"golang.org/x/sync/errgroup"
+)
+
+type resolveResourceReferences struct{}
+
+func ResolveResourceReferences() bundle.Mutator {
+	return &resolveResourceReferences{}
+}
+
+func (m *resolveResourceReferences) Apply(ctx context.Context, b *bundle.Bundle) error {
+	errs, errCtx := errgroup.WithContext(ctx)
+
+	for k := range b.Config.Variables {
+		v := b.Config.Variables[k]
+		if v.Lookup == nil {
+			continue
+		}
+
+		if v.HasValue() {
+			log.Debugf(ctx, "Ignoring '%s' lookup for the variable '%s' because the value is set", v.Lookup, k)
+			continue
+		}
+
+		errs.Go(func() error {
+			id, err := v.Lookup.Resolve(errCtx, b.WorkspaceClient())
+			if err != nil {
+				return fmt.Errorf("failed to resolve %s, err: %w", v.Lookup, err)
+			}
+
+			v.Set(id)
+			return nil
+		})
+	}
+
+	return errs.Wait()
+}
+
+func (*resolveResourceReferences) Name() string {
+	return "ResolveResourceReferences"
+}

bundle/config/mutator/resolve_resource_references_test.go

@@ -0,0 +1,197 @@
+package mutator
+
+import (
+	"context"
+	"testing"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/config/variable"
+	"github.com/databricks/databricks-sdk-go/service/compute"
+	"github.com/stretchr/testify/require"
+)
+
+type MockClusterService struct{}
+
+// ChangeOwner implements compute.ClustersService.
+func (MockClusterService) ChangeOwner(ctx context.Context, request compute.ChangeClusterOwner) error {
+	panic("unimplemented")
+}
+
+// Create implements compute.ClustersService.
+func (MockClusterService) Create(ctx context.Context, request compute.CreateCluster) (*compute.CreateClusterResponse, error) {
+	panic("unimplemented")
+}
+
+// Delete implements compute.ClustersService.
+func (MockClusterService) Delete(ctx context.Context, request compute.DeleteCluster) error {
+	panic("unimplemented")
+}
+
+// Edit implements compute.ClustersService.
+func (MockClusterService) Edit(ctx context.Context, request compute.EditCluster) error {
+	panic("unimplemented")
+}
+
+// Events implements compute.ClustersService.
+func (MockClusterService) Events(ctx context.Context, request compute.GetEvents) (*compute.GetEventsResponse, error) {
+	panic("unimplemented")
+}
+
+// Get implements compute.ClustersService.
+func (MockClusterService) Get(ctx context.Context, request compute.GetClusterRequest) (*compute.ClusterDetails, error) {
+	panic("unimplemented")
+}
+
+// GetPermissionLevels implements compute.ClustersService.
+func (MockClusterService) GetPermissionLevels(ctx context.Context, request compute.GetClusterPermissionLevelsRequest) (*compute.GetClusterPermissionLevelsResponse, error) {
+	panic("unimplemented")
+}
+
+// GetPermissions implements compute.ClustersService.
+func (MockClusterService) GetPermissions(ctx context.Context, request compute.GetClusterPermissionsRequest) (*compute.ClusterPermissions, error) {
+	panic("unimplemented")
+}
+
+// List implements compute.ClustersService.
+func (MockClusterService) List(ctx context.Context, request compute.ListClustersRequest) (*compute.ListClustersResponse, error) {
+	return &compute.ListClustersResponse{
+		Clusters: []compute.ClusterDetails{
+			{ClusterId: "1234-5678-abcd", ClusterName: "Some Custom Cluster"},
+			{ClusterId: "9876-5432-xywz", ClusterName: "Some Other Name"},
+		},
+	}, nil
+}
+
+// ListNodeTypes implements compute.ClustersService.
+func (MockClusterService) ListNodeTypes(ctx context.Context) (*compute.ListNodeTypesResponse, error) {
+	panic("unimplemented")
+}
+
+// ListZones implements compute.ClustersService.
+func (MockClusterService) ListZones(ctx context.Context) (*compute.ListAvailableZonesResponse, error) {
+	panic("unimplemented")
+}
+
+// PermanentDelete implements compute.ClustersService.
+func (MockClusterService) PermanentDelete(ctx context.Context, request compute.PermanentDeleteCluster) error {
+	panic("unimplemented")
+}
+
+// Pin implements compute.ClustersService.
+func (MockClusterService) Pin(ctx context.Context, request compute.PinCluster) error {
+	panic("unimplemented")
+}
+
+// Resize implements compute.ClustersService.
+func (MockClusterService) Resize(ctx context.Context, request compute.ResizeCluster) error {
+	panic("unimplemented")
+}
+
+// Restart implements compute.ClustersService.
+func (MockClusterService) Restart(ctx context.Context, request compute.RestartCluster) error {
+	panic("unimplemented")
+}
+
+// SetPermissions implements compute.ClustersService.
+func (MockClusterService) SetPermissions(ctx context.Context, request compute.ClusterPermissionsRequest) (*compute.ClusterPermissions, error) {
+	panic("unimplemented")
+}
+
+// SparkVersions implements compute.ClustersService.
+func (MockClusterService) SparkVersions(ctx context.Context) (*compute.GetSparkVersionsResponse, error) {
+	panic("unimplemented")
+}
+
+// Start implements compute.ClustersService.
+func (MockClusterService) Start(ctx context.Context, request compute.StartCluster) error {
+	panic("unimplemented")
+}
+
+// Unpin implements compute.ClustersService.
+func (MockClusterService) Unpin(ctx context.Context, request compute.UnpinCluster) error {
+	panic("unimplemented")
+}
+
+// UpdatePermissions implements compute.ClustersService.
+func (MockClusterService) UpdatePermissions(ctx context.Context, request compute.ClusterPermissionsRequest) (*compute.ClusterPermissions, error) {
+	panic("unimplemented")
+}
+
+func TestResolveClusterReference(t *testing.T) {
+	clusterRef1 := "Some Custom Cluster"
+	clusterRef2 := "Some Other Name"
+	justString := "random string"
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Variables: map[string]*variable.Variable{
+				"my-cluster-id-1": {
+					Lookup: &variable.Lookup{
+						Cluster: clusterRef1,
+					},
+				},
+				"my-cluster-id-2": {
+					Lookup: &variable.Lookup{
+						Cluster: clusterRef2,
+					},
+				},
+				"some-variable": {
+					Value: &justString,
+				},
+			},
+		},
+	}
+
+	b.WorkspaceClient().Clusters.WithImpl(MockClusterService{})
+
+	err := bundle.Apply(context.Background(), b, ResolveResourceReferences())
+	require.NoError(t, err)
+	require.Equal(t, "1234-5678-abcd", *b.Config.Variables["my-cluster-id-1"].Value)
+	require.Equal(t, "9876-5432-xywz", *b.Config.Variables["my-cluster-id-2"].Value)
+}
+
+func TestResolveNonExistentClusterReference(t *testing.T) {
+	clusterRef := "Random"
+	justString := "random string"
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Variables: map[string]*variable.Variable{
+				"my-cluster-id": {
+					Lookup: &variable.Lookup{
+						Cluster: clusterRef,
+					},
+				},
+				"some-variable": {
+					Value: &justString,
+				},
+			},
+		},
+	}
+
+	b.WorkspaceClient().Clusters.WithImpl(MockClusterService{})
+
+	err := bundle.Apply(context.Background(), b, ResolveResourceReferences())
+	require.ErrorContains(t, err, "failed to resolve cluster: Random, err: ClusterDetails named 'Random' does not exist")
+}
+
+func TestNoLookupIfVariableIsSet(t *testing.T) {
+	clusterRef := "donotexist"
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Variables: map[string]*variable.Variable{
+				"my-cluster-id": {
+					Lookup: &variable.Lookup{
+						Cluster: clusterRef,
+					},
+				},
+			},
+		},
+	}
+
+	b.WorkspaceClient().Clusters.WithImpl(MockClusterService{})
+	b.Config.Variables["my-cluster-id"].Set("random value")
+
+	err := bundle.Apply(context.Background(), b, ResolveResourceReferences())
+	require.NoError(t, err)
+	require.Equal(t, "random value", *b.Config.Variables["my-cluster-id"].Value)
+}

bundle/config/mutator/set_variables.go

@@ -46,6 +46,12 @@ func setVariable(ctx context.Context, v *variable.Variable, name string) error {
 		return nil
 	}
 
+	// case: Defined a variable for named lookup for a resource
+	// It will be resolved later in ResolveResourceReferences mutator
+	if v.Lookup != nil {
+		return nil
+	}
+
 	// We should have had a value to set for the variable at this point.
 	// TODO: use cmdio to request values for unassigned variables if current
 	// terminal is a tty. Tracked in https://github.com/databricks/cli/issues/379