Added support for sync.include and sync.exclude sections

bundle/bundle.go

@@ -186,7 +186,7 @@ func (b *Bundle) GetSyncIncludePatterns() ([]string, error) {
 	if err != nil {
 		return nil, err
 	}
-	return append(b.Config.Sync.Include, filepath.Join(internalDirRel, "*.*")), nil
+	return append(b.Config.Sync.Include, filepath.ToSlash(filepath.Join(internalDirRel, "*.*"))), nil
 }
 
 func (b *Bundle) GitRepository() (*git.Repository, error) {

libs/core/set.go

@@ -0,0 +1,75 @@
+package core
+
+import (
+	"fmt"
+
+	"golang.org/x/exp/maps"
+)
+
+type hashFunc[T any] func(a T) string
+
+// Set struct represents set data structure
+type Set[T any] struct {
+	key  hashFunc[T]
+	data map[string]T
+}
+
+// NewSetFromF initialise a new set with initial values and a hash function
+// to define uniqueness of value
+func NewSetFromF[T any](values []T, f hashFunc[T]) *Set[T] {
+	s := &Set[T]{
+		key:  f,
+		data: make(map[string]T),
+	}
+
+	for _, v := range values {
+		s.Add(v)
+	}
+
+	return s
+}
+
+// NewSetF initialise a new empty and a hash function
+// to define uniqueness of value
+func NewSetF[T any](f hashFunc[T]) *Set[T] {
+	return NewSetFromF([]T{}, f)
+}
+
+// NewSetFrom initialise a new set with initial values which are comparable
+func NewSetFrom[T comparable](values []T) *Set[T] {
+	return NewSetFromF(values, func(item T) string {
+		return fmt.Sprintf("%v", item)
+	})
+}
+
+// NewSetFrom initialise a new empty set for comparable values
+func NewSet[T comparable]() *Set[T] {
+	return NewSetFrom([]T{})
+}
+
+func (s *Set[T]) addOne(item T) {
+	s.data[s.key(item)] = item
+}
+
+// Add one or multiple items to set
+func (s *Set[T]) Add(items ...T) {
+	for _, i := range items {
+		s.addOne(i)
+	}
+}
+
+// Remove an item from set. No-op if the item does not exist
+func (s *Set[T]) Remove(item T) {
+	delete(s.data, s.key(item))
+}
+
+// Indicates if the item exists in the set
+func (s *Set[T]) Has(item T) bool {
+	_, ok := s.data[s.key(item)]
+	return ok
+}
+
+// Returns an iterable slice of values from set
+func (s *Set[T]) Iter() []T {
+	return maps.Values(s.data)
+}

libs/core/set_test.go

@@ -0,0 +1,111 @@
+package core
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestSet(t *testing.T) {
+	s := NewSetFrom([]string{})
+	require.ElementsMatch(t, []string{}, s.Iter())
+
+	s = NewSetFrom([]string{"a", "a", "a", "b", "b", "c", "d", "e"})
+	require.ElementsMatch(t, []string{"a", "b", "c", "d", "e"}, s.Iter())
+
+	i := NewSetFrom([]int{1, 1, 2, 3, 4, 5, 7, 7, 7, 10, 11})
+	require.ElementsMatch(t, []int{1, 2, 3, 4, 5, 7, 10, 11}, i.Iter())
+
+	f := NewSetFrom([]float32{1.1, 1.1, 2.0, 3.1, 4.5, 5.1, 7.1, 7.2, 7.1, 10.1, 11.0})
+	require.ElementsMatch(t, []float32{1.1, 2.0, 3.1, 4.5, 5.1, 7.1, 7.2, 10.1, 11.0}, f.Iter())
+}
+
+type testStruct struct {
+	key   string
+	value int
+}
+
+func TestSetCustomKey(t *testing.T) {
+	s := NewSetF(func(item *testStruct) string {
+		return fmt.Sprintf("%s:%d", item.key, item.value)
+	})
+	s.Add(&testStruct{"a", 1})
+	s.Add(&testStruct{"b", 2})
+	s.Add(&testStruct{"c", 1})
+	s.Add(&testStruct{"a", 1})
+	s.Add(&testStruct{"a", 1})
+	s.Add(&testStruct{"a", 1})
+	s.Add(&testStruct{"c", 1})
+	s.Add(&testStruct{"c", 3})
+
+	require.ElementsMatch(t, []*testStruct{
+		{"a", 1},
+		{"b", 2},
+		{"c", 1},
+		{"c", 3},
+	}, s.Iter())
+}
+
+func TestSetAdd(t *testing.T) {
+	s := NewSet[string]()
+	s.Add("a")
+	s.Add("a")
+	s.Add("a")
+	s.Add("b")
+	s.Add("c")
+	s.Add("c")
+	s.Add("d")
+	s.Add("d")
+
+	require.ElementsMatch(t, []string{"a", "b", "c", "d"}, s.Iter())
+}
+
+func TestSetRemove(t *testing.T) {
+	s := NewSet[string]()
+	s.Add("a")
+	s.Add("a")
+	s.Add("a")
+	s.Add("b")
+	s.Add("c")
+	s.Add("c")
+	s.Add("d")
+	s.Add("d")
+
+	s.Remove("d")
+	s.Remove("d")
+	s.Remove("a")
+
+	require.ElementsMatch(t, []string{"b", "c"}, s.Iter())
+}
+
+func TestSetHas(t *testing.T) {
+	s := NewSet[string]()
+	require.False(t, s.Has("a"))
+
+	s.Add("a")
+	require.True(t, s.Has("a"))
+
+	s.Add("a")
+	s.Add("a")
+	require.True(t, s.Has("a"))
+
+	s.Add("b")
+	s.Add("c")
+	s.Add("c")
+	s.Add("d")
+	s.Add("d")
+
+	require.True(t, s.Has("a"))
+	require.True(t, s.Has("b"))
+	require.True(t, s.Has("c"))
+	require.True(t, s.Has("d"))
+
+	s.Remove("d")
+	s.Remove("a")
+
+	require.False(t, s.Has("a"))
+	require.True(t, s.Has("b"))
+	require.True(t, s.Has("c"))
+	require.False(t, s.Has("d"))
+}

libs/sync/sync.go

@@ -3,9 +3,9 @@ package sync
 import (
 	"context"
 	"fmt"
-	"slices"
 	"time"
 
+	"github.com/databricks/cli/libs/core"
 	"github.com/databricks/cli/libs/filer"
 	"github.com/databricks/cli/libs/fileset"
 	"github.com/databricks/cli/libs/git"
@@ -144,51 +144,10 @@ func (s *Sync) notifyComplete(ctx context.Context, d diff) {
 }
 
 func (s *Sync) RunOnce(ctx context.Context) error {
-	// tradeoff: doing portable monitoring only due to macOS max descriptor manual ulimit setting requirement
-	// https://github.com/gorakhargosh/watchdog/blob/master/src/watchdog/observers/kqueue.py#L394-L418
-	all := make([]fileset.File, 0)
-	gitFiles, err := s.fileSet.All()
+	files, err := getFileList(ctx, s)
 	if err != nil {
-		log.Errorf(ctx, "cannot list files: %s", err)
 		return err
 	}
-	all = append(all, gitFiles...)
-
-	include, err := s.includeFileSet.All()
-	if err != nil {
-		log.Errorf(ctx, "cannot list include files: %s", err)
-		return err
-	}
-
-	// Avoiding duplicates with Git tracked and include files
-	for _, i := range include {
-		if slices.ContainsFunc(all, func(a fileset.File) bool {
-			return a.Absolute == i.Absolute
-		}) {
-			continue
-		}
-
-		all = append(all, i)
-	}
-
-	all = append(all, include...)
-
-	exclude, err := s.excludeFileSet.All()
-	if err != nil {
-		log.Errorf(ctx, "cannot list exclude files: %s", err)
-		return err
-	}
-
-	files := make([]fileset.File, 0)
-	for _, f := range all {
-		if slices.ContainsFunc(exclude, func(a fileset.File) bool {
-			return a.Absolute == f.Absolute
-		}) {
-			continue
-		}
-
-		files = append(files, f)
-	}
 
 	change, err := s.snapshot.diff(ctx, files)
 	if err != nil {
@@ -216,6 +175,40 @@ func (s *Sync) RunOnce(ctx context.Context) error {
 	return nil
 }
 
+func getFileList(ctx context.Context, s *Sync) ([]fileset.File, error) {
+	// tradeoff: doing portable monitoring only due to macOS max descriptor manual ulimit setting requirement
+	// https://github.com/gorakhargosh/watchdog/blob/master/src/watchdog/observers/kqueue.py#L394-L418
+	all := core.NewSetF(func(f fileset.File) string {
+		return f.Absolute
+	})
+	gitFiles, err := s.fileSet.All()
+	if err != nil {
+		log.Errorf(ctx, "cannot list files: %s", err)
+		return nil, err
+	}
+	all.Add(gitFiles...)
+
+	include, err := s.includeFileSet.All()
+	if err != nil {
+		log.Errorf(ctx, "cannot list include files: %s", err)
+		return nil, err
+	}
+
+	all.Add(include...)
+
+	exclude, err := s.excludeFileSet.All()
+	if err != nil {
+		log.Errorf(ctx, "cannot list exclude files: %s", err)
+		return nil, err
+	}
+
+	for _, f := range exclude {
+		all.Remove(f)
+	}
+
+	return all.Iter(), nil
+}
+
 func (s *Sync) DestroySnapshot(ctx context.Context) error {
 	return s.snapshot.Destroy(ctx)
 }