always set the filter to skip navigating baseDir (#12984)

baseDir is empty if the top level prefix does not
end with `/` this causes large recursive listings
without any filtering, to fix this filtering make
sure to set the filter prefix appropriately.

also do not navigate folders at top level that do
not match the filter prefix, entries don't need
to match prefix since they are never prefixed
with the prefix anyways.

cmd/metacache-server-pool.go

@@ -97,6 +97,7 @@ func (z *erasureServerPools) listPath(ctx context.Context, o *listPathOptions) (
 	o.parseMarker()
 	o.BaseDir = baseDirFromPrefix(o.Prefix)
 	o.Transient = o.Transient || isReservedOrInvalidBucket(o.Bucket, false)
+	o.SetFilter()
 	if o.Transient {
 		o.Create = false
 	}

cmd/metacache-walk.go

@@ -110,6 +110,11 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 	var scanDir func(path string) error
 
 	scanDir = func(current string) error {
+		// always skip the directory that doesn't match the prefix
+		if len(current) > 0 && !strings.HasPrefix(current, prefix) {
+			return nil
+		}
+
 		// Skip forward, if requested...
 		forward := ""
 		if len(opts.ForwardTo) > 0 && strings.HasPrefix(opts.ForwardTo, current) {
@@ -138,9 +143,6 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 		}
 		dirObjects := make(map[string]struct{})
 		for i, entry := range entries {
-			if len(prefix) > 0 && !strings.HasPrefix(entry, prefix) {
-				continue
-			}
 			if len(forward) > 0 && entry < forward {
 				continue
 			}
@@ -197,7 +199,6 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 		// Process in sort order.
 		sort.Strings(entries)
 		dirStack := make([]string, 0, 5)
-		prefix = "" // Remove prefix after first level.
 		if len(forward) > 0 {
 			idx := sort.SearchStrings(entries, forward)
 			if idx > 0 {