diff --git a/pkg/blobfs_prefetch.go b/pkg/blobfs_prefetch.go
index ddcf0f2..2b3ad92 100644
--- a/pkg/blobfs_prefetch.go
+++ b/pkg/blobfs_prefetch.go
@@ -3,6 +3,7 @@ package blobcache
 import (
 	"context"
 	"fmt"
+	"runtime"
 	"sync"
 	"time"
 )
@@ -200,13 +201,11 @@ func (pb *PrefetchBuffer) fetch(windowIndex uint64) {
 func (pb *PrefetchBuffer) IsIdle() bool {
 	idle := true
 
-	pb.mu.Lock()
-	defer pb.mu.Unlock()
-
+	windowsToDelete := make([]int64, 0)
 	pb.windows.Range(func(key, value any) bool {
 		w := value.(*window)
 		if w != nil && time.Since(w.lastRead) > pb.manager.windowIdleTTL && !w.fetching {
-			return true
+			windowsToDelete = append(windowsToDelete, w.index)
 		} else {
 			idle = false
 		}
@@ -214,6 +213,15 @@ func (pb *PrefetchBuffer) IsIdle() bool {
 		return true
 	})
 
+	for _, index := range windowsToDelete {
+		w, exists := pb.windows.Load(index)
+		if exists {
+			w.(*window).data = nil
+		}
+
+		pb.windows.Delete(index)
+	}
+
 	return idle
 }
 
@@ -231,8 +239,11 @@ func (pb *PrefetchBuffer) Clear() {
 		if w != nil {
 			w.data = nil
 		}
+
 		return true
 	})
+
+	runtime.GC()
 }
 
 func (pb *PrefetchBuffer) GetRange(offset, length uint64) ([]byte, error) {
@@ -261,9 +272,6 @@ func (pb *PrefetchBuffer) GetRange(offset, length uint64) ([]byte, error) {
 }
 
 func (pb *PrefetchBuffer) tryGetRange(offset, length uint64) ([]byte, bool, bool) {
-	pb.mu.Lock()
-	defer pb.mu.Unlock()
-
 	windowIndex := offset / pb.windowSize
 
 	var w *window
@@ -282,8 +290,10 @@ func (pb *PrefetchBuffer) tryGetRange(offset, length uint64) ([]byte, bool, bool
 		return nil, false, false
 	}
 
-	w.mu.Lock()
-	defer w.mu.Unlock()
+	if w.fetching {
+		w.mu.Lock()
+		defer w.mu.Unlock()
+	}
 
 	windowOffset := offset - (windowIndex * pb.windowSize)
 	windowHead := (windowIndex * pb.windowSize) + w.readLength