Faster globbing (#335)

cli/internal/context/context.go

@@ -11,7 +11,7 @@ import (
 	"turbo/internal/backends"
 	"turbo/internal/config"
 	"turbo/internal/fs"
-	"turbo/internal/fs/globby"
+	"turbo/internal/globby"
 	"turbo/internal/util"
 
 	mapset "github.com/deckarep/golang-set"
@@ -180,7 +180,7 @@ func WithGraph(rootpath string, config *config.Config) Option {
 		globalDeps := make(util.Set)
 
 		if len(pkg.Turbo.GlobalDependencies) > 0 {
-			f := globby.GlobFiles(rootpath, &pkg.Turbo.GlobalDependencies, nil)
+			f := globby.GlobFiles(rootpath, pkg.Turbo.GlobalDependencies, []string{})
 			for _, val := range f {
 				globalDeps.Add(val)
 			}
@@ -243,7 +243,7 @@ func WithGraph(rootpath string, config *config.Config) Option {
 			"**/tests/**/*",
 		}
 
-		f := globby.GlobFiles(rootpath, &justJsons, &ignore)
+		f := globby.GlobFiles(rootpath, justJsons, ignore)
 
 		for i, val := range f {
 			_, val := i, val // https://golang.org/doc/faq#closures_and_goroutines

cli/internal/fs/copy_file.go

@@ -85,9 +85,12 @@ func WalkMode(rootPath string, callback func(name string, isDir bool, mode os.Fi
 	} else if !info.IsDir() {
 		return callback(rootPath, false, info.Mode())
 	}
-	return godirwalk.Walk(rootPath, &godirwalk.Options{Callback: func(name string, info *godirwalk.Dirent) error {
-		return callback(name, info.IsDir(), info.ModeType())
-	}})
+	return godirwalk.Walk(rootPath, &godirwalk.Options{
+		Callback: func(name string, info *godirwalk.Dirent) error {
+			return callback(name, info.IsDir(), info.ModeType())
+		},
+		Unsorted: true,
+	})
 }
 
 // SameFile returns true if the two given paths refer to the same physical

cli/internal/globby/globby.go

@@ -0,0 +1,47 @@
+package globby
+
+import (
+	"turbo/internal/fs"
+
+	"path/filepath"
+	"strings"
+
+	"github.com/bmatcuk/doublestar/v4"
+)
+
+func GlobFiles(basePath string, includePatterns []string, excludePatterns []string) []string {
+	var include []string
+	var exclude []string
+	var result []string
+
+	for _, p := range includePatterns {
+		include = append(include, filepath.Join(basePath, p))
+	}
+
+	for _, p := range excludePatterns {
+		exclude = append(exclude, filepath.Join(basePath, p))
+	}
+
+	includePattern := "{" + strings.Join(include, ",") + "}"
+	excludePattern := "{" + strings.Join(exclude, ",") + "}"
+	_ = fs.Walk(basePath, func(p string, isDir bool) error {
+		if val, _ := doublestar.PathMatch(excludePattern, p); val {
+			if isDir {
+				return filepath.SkipDir
+			}
+			return nil
+		}
+
+		if isDir {
+			return nil
+		}
+
+		if val, _ := doublestar.PathMatch(includePattern, p); val || len(includePatterns) == 0 {
+			result = append(result, p)
+		}
+
+		return nil
+	})
+
+	return result
+}

cli/internal/run/run.go

@@ -20,7 +20,7 @@ import (
 	"turbo/internal/context"
 	"turbo/internal/core"
 	"turbo/internal/fs"
-	"turbo/internal/fs/globby"
+	"turbo/internal/globby"
 	"turbo/internal/scm"
 	"turbo/internal/ui"
 	"turbo/internal/util"
@@ -583,7 +583,7 @@ func (c *RunCommand) Run(args []string) int {
 				if runOptions.cache && (pipeline.Cache == nil || *pipeline.Cache) {
 					targetLogger.Debug("caching output", "outputs", outputs)
 					ignore := []string{}
-					filesToBeCached := globby.GlobFiles(pack.Dir, &outputs, &ignore)
+					filesToBeCached := globby.GlobFiles(pack.Dir, outputs, ignore)
 					if err := turboCache.Put(pack.Dir, hash, int(time.Since(cmdTime).Milliseconds()), filesToBeCached); err != nil {
 						c.logError(targetLogger, "", fmt.Errorf("Error caching output: %w", err))
 					}