File filtering improvements (#332)

* Ignore .git and .idea directories
* Use more efficient algorithm for filtering than
  the one provided by OPA

Fixes #330

Signed-off-by: Anders Eknert <anders@styra.com>

e2e/cli_test.go

@@ -133,7 +133,7 @@ func TestLintNonExistentDir(t *testing.T) {
 	}
 
 	if exp, act := "error(s) encountered while linting: errors encountered when reading files to lint: "+
-		"failed to filter paths: 1 error occurred during loading: stat "+td+"/what/ever: no such file or directory\n",
+		"failed to filter paths:\nstat "+td+"/what/ever: no such file or directory\n",
 		stdout.String(); exp != act {
 		t.Errorf("expected stdout %q, got %q", exp, act)
 	}

pkg/config/filter.go

@@ -1,35 +1,63 @@
 package config
 
 import (
+	"errors"
 	"fmt"
 	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/gobwas/glob"
 
 	"github.com/open-policy-agent/opa/bundle"
-	"github.com/open-policy-agent/opa/loader"
 )
 
 func FilterIgnoredPaths(paths, ignore []string, checkFileExists bool) ([]string, error) {
-	policyPaths := paths
-
 	if checkFileExists {
-		filteredPaths, err := loader.FilteredPaths(paths, func(_ string, info os.FileInfo, depth int) bool {
-			return !info.IsDir() && !strings.HasSuffix(info.Name(), bundle.RegoExt)
-		})
-		if err != nil {
-			return nil, fmt.Errorf("failed to filter paths: %w", err)
+		filtered := make([]string, 0, len(paths))
+
+		if err := walkPaths(paths, func(path string, info os.DirEntry, err error) error {
+			if info.IsDir() && (info.Name() == ".git" || info.Name() == ".idea") {
+				return filepath.SkipDir
+			}
+			if !info.IsDir() && strings.HasSuffix(path, bundle.RegoExt) {
+				filtered = append(filtered, path)
+			}
+
+			return err
+		}); err != nil {
+			return nil, fmt.Errorf("failed to filter paths:\n%w", err)
 		}
 
-		policyPaths = filteredPaths
+		return filterPaths(filtered, ignore)
 	}
 
 	if len(ignore) == 0 {
-		return policyPaths, nil
+		return paths, nil
+	}
+
+	return filterPaths(paths, ignore)
+}
+
+func walkPaths(paths []string, filter func(path string, info os.DirEntry, err error) error) error {
+	var errs error
+
+	for _, path := range paths {
+		// We need to stat the initial set of paths, as filepath.WalkDir
+		// will panic on non-existent paths.
+		_, err := os.Stat(path)
+		if err != nil {
+			errs = errors.Join(errs, err)
+
+			continue
+		}
+
+		if err := filepath.WalkDir(path, filter); err != nil {
+			errs = errors.Join(errs, err)
+		}
 	}
 
-	return filterPaths(policyPaths, ignore)
+	return errs
 }
 
 func filterPaths(policyPaths []string, ignore []string) ([]string, error) {