diff --git a/pkg/config/filter.go b/pkg/config/filter.go
index 9a908e8e..07ddac4e 100644
--- a/pkg/config/filter.go
+++ b/pkg/config/filter.go
@@ -3,26 +3,35 @@ package config
 import (
 	"fmt"
 	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/gobwas/glob"
 
 	"github.com/open-policy-agent/opa/bundle"
-	"github.com/open-policy-agent/opa/loader"
 )
 
+type FileFilter = func(abspath string) bool
+
+//nolint:gochecknoglobals
+var ignoreDirs = map[string]struct{}{
+	".git":  {},
+	".idea": {},
+}
+
 func FilterIgnoredPaths(paths, ignore []string, checkFileExists bool) ([]string, error) {
 	policyPaths := paths
 
 	if checkFileExists {
-		filteredPaths, err := loader.FilteredPaths(paths, func(_ string, info os.FileInfo, depth int) bool {
-			return !info.IsDir() && !strings.HasSuffix(info.Name(), bundle.RegoExt)
+		fp, errs := filteredPaths(paths, func(path string) bool {
+			return !strings.HasSuffix(path, bundle.RegoExt)
 		})
-		if err != nil {
-			return nil, fmt.Errorf("failed to filter paths: %w", err)
+
+		if len(errs) > 0 {
+			return nil, fmt.Errorf("failed to filter paths: %v", errorsToString(errs))
 		}
 
-		policyPaths = filteredPaths
+		policyPaths = fp
 	}
 
 	if len(ignore) == 0 {
@@ -32,6 +41,73 @@ func FilterIgnoredPaths(paths, ignore []string, checkFileExists bool) ([]string,
 	return filterPaths(policyPaths, ignore)
 }
 
+// filteredPaths is a simplified version of the same function in OPA, but one that
+// avoids stat-ing files in favor of DirEntry introspection. This is faster due to
+// the reduced number of system calls. Since we avoid stat-ing files, we don't get
+// to work with FileInfo objects, but will rather work directly with the files as
+// string paths. Finally, this function differs from OPA in that we never will
+// traverse down into .git or .idea directories, as that's always a waste of time.
+func filteredPaths(paths []string, filter FileFilter) ([]string, []error) {
+	result := make([]string, 0, len(paths))
+	errs := make([]error, 0, len(paths))
+
+	all(paths, filter, func(path *string, err error) {
+		if err != nil {
+			errs = append(errs, err)
+		} else {
+			result = append(result, *path)
+		}
+	})
+
+	if len(errs) > 0 {
+		return nil, errs
+	}
+
+	return result, nil
+}
+
+func all(paths []string, filter FileFilter, f func(*string, error)) {
+	for _, path := range paths {
+		info, err := os.Stat(filepath.Clean(path))
+		if err != nil {
+			f(nil, err)
+
+			continue
+		}
+
+		allRec(path, info.IsDir(), filter, 0, f)
+	}
+}
+
+func allRec(path string, isDir bool, filter FileFilter, depth int, f func(*string, error)) {
+	if !isDir && filter != nil && filter(path) {
+		return
+	}
+
+	if !isDir {
+		f(&path, nil)
+
+		return
+	}
+
+	files, err := os.ReadDir(path)
+	if err != nil {
+		f(nil, err)
+
+		return
+	}
+
+	for _, file := range files {
+		if file.IsDir() {
+			if _, ignored := ignoreDirs[file.Name()]; ignored {
+				continue
+			}
+		}
+
+		allRec(filepath.Join(path, file.Name()), file.IsDir(), filter, depth+1, f)
+	}
+}
+
 func filterPaths(policyPaths []string, ignore []string) ([]string, error) {
 	filtered := make([]string, 0, len(policyPaths))
 
@@ -108,3 +184,22 @@ func excludeFile(pattern string, filename string) (bool, error) {
 
 	return false, nil
 }
+
+// Taken from OPA, but while we use a different error type there, we'll settle
+// for a plain array here until we need something more fancy elsewhere.
+func errorsToString(e []error) string {
+	if len(e) == 0 {
+		return "no error(s)"
+	}
+
+	if len(e) == 1 {
+		return "1 error occurred during loading: " + e[0].Error()
+	}
+
+	buf := make([]string, len(e))
+	for i := range buf {
+		buf[i] = e[i].Error()
+	}
+
+	return fmt.Sprintf("%v errors occurred during loading:\n", len(e)) + strings.Join(buf, "\n")
+}