Bugfix: S3 accessor was unable to handle 416 byte range error.

accessors/ext4/utils.go

@@ -34,7 +34,7 @@ func GetExt4Cache(scope vfilter.Scope,
 		lru_size := vql_subsystem.GetIntFromRow(
 			scope, scope, constants.NTFS_CACHE_SIZE)
 
-		paged_reader, err := readers.NewPagedReader(
+		paged_reader, err := readers.NewAccessorReader(
 			scope, accessor, device, int(lru_size))
 
 		cache_ctx, err = ext4.GetEXT4Context(paged_reader)

accessors/fat/utils.go

@@ -34,7 +34,7 @@ func GetFATCache(scope vfilter.Scope,
 		lru_size := vql_subsystem.GetIntFromRow(
 			scope, scope, constants.NTFS_CACHE_SIZE)
 
-		paged_reader, err := readers.NewPagedReader(
+		paged_reader, err := readers.NewAccessorReader(
 			scope, accessor, device, int(lru_size))
 
 		cache_ctx, err = fat.GetFATContext(paged_reader)

accessors/ntfs/readers/ntfs_reader.go

@@ -77,7 +77,7 @@ func (self *NTFSCachedContext) Start(
 
 	lru_size := vql_subsystem.GetIntFromRow(
 		self.scope, self.scope, constants.NTFS_CACHE_SIZE)
-	self.paged_reader, err = readers.NewPagedReader(
+	self.paged_reader, err = readers.NewAccessorReader(
 		self.scope, self.accessor, self.device, int(lru_size))
 
 	if err != nil {

accessors/raw_registry/raw_registry.go

@@ -307,7 +307,7 @@ func getRegHive(scope vfilter.Scope,
 		return nil, err
 	}
 
-	paged_reader, err := readers.NewPagedReader(
+	paged_reader, err := readers.NewAccessorReader(
 		scope, pathspec.DelegateAccessor, delegate, int(lru_size))
 	if err != nil {
 		scope.Log("%v: did you provide a URL or Pathspec?", err)

accessors/s3/reader.go

@@ -2,9 +2,12 @@ package s3
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"io"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
+	awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
 	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 )
@@ -30,6 +33,13 @@ func (self *S3Reader) Read(buff []byte) (int, error) {
 		manager.NewWriteAtBuffer(buff), req)
 
 	if err != nil {
+		var re *awshttp.ResponseError
+		if errors.As(err, &re) {
+			if re.HTTPStatusCode() == 416 {
+				return 0, io.EOF
+			}
+		}
+
 		return 0, err
 	}
 	self.offset += n

accessors/s3/s3.go

@@ -14,7 +14,6 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
-	ntfs "www.velocidex.com/golang/go-ntfs/parser"
 	"www.velocidex.com/golang/velociraptor/accessors"
 	"www.velocidex.com/golang/velociraptor/acls"
 	"www.velocidex.com/golang/velociraptor/utils"
@@ -201,7 +200,7 @@ func (self RawS3SystemAccessor) OpenWithOSPath(
 
 	// Wrap the reader in an in memory cache so we do not have many
 	// small reads from the network.
-	paged_reader, err := ntfs.NewPagedReader(
+	paged_reader, err := utils.NewPagedReader(
 		utils.MakeReaderAtter(reader), 1024*1024, 20)
 	return utils.NewReadSeekReaderAdapter(paged_reader), err
 }

go.mod

@@ -99,7 +99,7 @@ require (
 	www.velocidex.com/golang/go-prefetch v0.0.0-20220801101854-338dbe61982a
 	www.velocidex.com/golang/oleparse v0.0.0-20230217092320-383a0121aafe
 	www.velocidex.com/golang/regparser v0.0.0-20240404115756-2169ac0e3c09
-	www.velocidex.com/golang/vfilter v0.0.0-20240605081519-e681962fca60
+	www.velocidex.com/golang/vfilter v0.0.0-20240608150317-307fc598a311
 )
 
 require (
@@ -109,6 +109,7 @@ require (
 	github.com/Masterminds/sprig/v3 v3.2.2
 	github.com/Velocidex/file-rotatelogs v0.0.0-20211221020724-d12e4dae4e11
 	github.com/Velocidex/go-ewf v0.0.0-20240210123447-97dc81b7d8c3
+	github.com/Velocidex/go-ext4 v0.0.0-20240608083317-8dd00855b069
 	github.com/Velocidex/go-fat v0.0.0-20230923165230-3e6c4265297a
 	github.com/Velocidex/go-vhdx v0.0.0-20240601014259-b204818c95fd
 	github.com/Velocidex/grok v0.0.1
@@ -163,11 +164,11 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.1 // indirect
 	github.com/PuerkitoBio/goquery v1.8.1 // indirect
-	github.com/Velocidex/go-ext4 v0.0.0-20240608083317-8dd00855b069 // indirect
 	github.com/alecthomas/colour v0.1.0 // indirect
 	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
 	github.com/andybalholm/cascadia v1.3.2 // indirect
+	github.com/aws/aws-sdk-go v1.53.19 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.2 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.2 // indirect

go.sum

@@ -197,6 +197,8 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd
 github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA=
 github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw=
 github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
+github.com/aws/aws-sdk-go v1.53.19 h1:WEuWc918RXlIaPCyU11F7hH9H1ItK+8m2c/uoQNRUok=
+github.com/aws/aws-sdk-go v1.53.19/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
 github.com/aws/aws-sdk-go-v2 v1.25.2 h1:/uiG1avJRgLGiQM9X3qJM8+Qa6KRGK5rRPuXE0HUM+w=
 github.com/aws/aws-sdk-go-v2 v1.25.2/go.mod h1:Evoc5AsmtveRt1komDwIsjHFyrP5tDuF1D1U+6z6pNo=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 h1:gTK2uhtAPtFcdRRJilZPx8uJLL2J85xK11nKtWL0wfU=
@@ -1362,7 +1364,7 @@ www.velocidex.com/golang/oleparse v0.0.0-20230217092320-383a0121aafe h1:o9jQWSwK
 www.velocidex.com/golang/oleparse v0.0.0-20230217092320-383a0121aafe/go.mod h1:R7IisRzDO7q5LVRJsCQf1xA50LrIavsPWzAjVE4THyY=
 www.velocidex.com/golang/regparser v0.0.0-20240404115756-2169ac0e3c09 h1:G1RWYBXP2lSzxKcrAU1YhiUlBetZ7hGIzIiWuuazvfo=
 www.velocidex.com/golang/regparser v0.0.0-20240404115756-2169ac0e3c09/go.mod h1:pxSECT5mWM3goJ4sxB4HCJNKnKqiAlpyT8XnvBwkLGU=
-www.velocidex.com/golang/vfilter v0.0.0-20240605081519-e681962fca60 h1:r+PPvW2n4AgLEO9Awo19Q4dmqDMKTQgNXbWezKZ+5fA=
-www.velocidex.com/golang/vfilter v0.0.0-20240605081519-e681962fca60/go.mod h1:P50KPQr2LpWVAu7ilGH8CBLBASGtOJ2971yA9YhR8rY=
+www.velocidex.com/golang/vfilter v0.0.0-20240608150317-307fc598a311 h1:IujLDZmLQ/cMraqK2gXcvvbwe6GJqpmCABjMwFXNqI4=
+www.velocidex.com/golang/vfilter v0.0.0-20240608150317-307fc598a311/go.mod h1:P50KPQr2LpWVAu7ilGH8CBLBASGtOJ2971yA9YhR8rY=
 www.velocidex.com/golang/vtypes v0.0.0-20240123105603-069d4a7f435c h1:rL/It+Ig+mvIhmy9vl5gg5b6CX2J12x0v2SXIT2RoWE=
 www.velocidex.com/golang/vtypes v0.0.0-20240123105603-069d4a7f435c/go.mod h1:tjaJNlBWbvH4cEMrEu678CFR2hrtcdyPINIpRxrOh4U=

utils/lru.go

@@ -0,0 +1,230 @@
+// Based on
+// https://github.com/hashicorp/golang-lru/blob/master/simplelru/lru.go
+// but more optimized for speed: by changing keys to int mapaccess is
+// much faster. We also added locking to the LRU to make it thread
+// safe.
+
+package utils
+
+import (
+	"container/list"
+	"errors"
+	"fmt"
+	"sync"
+)
+
+// EvictCallback is used to get a callback when a cache entry is evicted
+type EvictCallback func(key int, value interface{})
+
+// LRU implements a thread safe fixed size LRU cache
+type LRU struct {
+	size      int
+	evictList *list.List
+	items     map[int]*list.Element
+	onEvict   EvictCallback
+
+	mu sync.Mutex
+
+	name  string
+	hits  int64
+	miss  int64
+	total int64
+}
+
+// entry is used to hold a value in the evictList
+type entry struct {
+	key   int
+	value interface{}
+}
+
+// NewLRU constructs an LRU of the given size
+func NewLRU(size int, onEvict EvictCallback, name string) (*LRU, error) {
+	if size <= 0 {
+		return nil, errors.New("Must provide a positive size")
+	}
+	c := &LRU{
+		size:      size,
+		evictList: list.New(),
+		items:     make(map[int]*list.Element),
+		onEvict:   onEvict,
+		name:      name,
+	}
+	return c, nil
+}
+
+// Purge is used to completely clear the cache.
+func (self *LRU) Purge() {
+	self.mu.Lock()
+	defer self.mu.Unlock()
+
+	for k, v := range self.items {
+		if self.onEvict != nil {
+			self.onEvict(k, v.Value.(*entry).value)
+		}
+		delete(self.items, k)
+	}
+	self.evictList.Init()
+}
+
+// Add adds a value to the cache.  Returns true if an eviction occurred.
+func (self *LRU) Add(key int, value interface{}) (evicted bool) {
+	self.mu.Lock()
+	defer self.mu.Unlock()
+
+	self.total++
+
+	// Check for existing item
+	if ent, ok := self.items[key]; ok {
+		self.evictList.MoveToFront(ent)
+		ent.Value.(*entry).value = value
+		return false
+	}
+
+	// Add new item
+	ent := &entry{key, value}
+	entry := self.evictList.PushFront(ent)
+	self.items[key] = entry
+
+	evict := self.evictList.Len() > self.size
+	// Verify size not exceeded
+	if evict {
+		self.removeOldest()
+	}
+	return evict
+}
+
+func (self *LRU) Touch(key int) {
+	self.mu.Lock()
+	defer self.mu.Unlock()
+
+	if ent, ok := self.items[key]; ok {
+		self.evictList.MoveToFront(ent)
+	}
+}
+
+// Get looks up a key's value from the cache.
+func (self *LRU) Get(key int) (value interface{}, ok bool) {
+	self.mu.Lock()
+	defer self.mu.Unlock()
+
+	if ent, ok := self.items[key]; ok {
+		self.evictList.MoveToFront(ent)
+		self.hits++
+		return ent.Value.(*entry).value, true
+	}
+	self.miss++
+	return
+}
+
+// Contains checks if a key is in the cache, without updating the recent-ness
+// or deleting it for being stale.
+func (self *LRU) Contains(key int) (ok bool) {
+	self.mu.Lock()
+	defer self.mu.Unlock()
+
+	_, ok = self.items[key]
+	return ok
+}
+
+// Peek returns the key value (or undefined if not found) without updating
+// the "recently used"-ness of the key.
+func (self *LRU) Peek(key int) (value interface{}, ok bool) {
+	self.mu.Lock()
+	defer self.mu.Unlock()
+
+	var ent *list.Element
+	if ent, ok = self.items[key]; ok {
+		return ent.Value.(*entry).value, true
+	}
+	return nil, ok
+}
+
+// Remove removes the provided key from the cache, returning if the
+// key was contained.
+func (self *LRU) Remove(key int) (present bool) {
+	self.mu.Lock()
+	defer self.mu.Unlock()
+
+	if ent, ok := self.items[key]; ok {
+		self.removeElement(ent)
+		return true
+	}
+	return false
+}
+
+// RemoveOldest removes the oldest item from the cache.
+func (self *LRU) RemoveOldest() (key int, value interface{}, ok bool) {
+	self.mu.Lock()
+	defer self.mu.Unlock()
+
+	ent := self.evictList.Back()
+	if ent != nil {
+		self.removeElement(ent)
+		kv := ent.Value.(*entry)
+		return kv.key, kv.value, true
+	}
+	return 0, nil, false
+}
+
+// GetOldest returns the oldest entry
+func (self *LRU) GetOldest() (key int, value interface{}, ok bool) {
+	self.mu.Lock()
+	defer self.mu.Unlock()
+
+	ent := self.evictList.Back()
+	if ent != nil {
+		kv := ent.Value.(*entry)
+		return kv.key, kv.value, true
+	}
+	return 0, nil, false
+}
+
+// Keys returns a slice of the keys in the cache, from oldest to newest.
+func (self *LRU) Keys() []int {
+	self.mu.Lock()
+	defer self.mu.Unlock()
+
+	keys := make([]int, len(self.items))
+	i := 0
+	for ent := self.evictList.Back(); ent != nil; ent = ent.Prev() {
+		keys[i] = ent.Value.(*entry).key
+		i++
+	}
+	return keys
+}
+
+// Len returns the number of items in the cache.
+func (self *LRU) Len() int {
+	self.mu.Lock()
+	defer self.mu.Unlock()
+
+	return self.evictList.Len()
+}
+
+func (self *LRU) DebugString() string {
+	self.mu.Lock()
+	defer self.mu.Unlock()
+
+	return fmt.Sprintf("%s LRU %p hit %d miss %d - total %v (%f)\n",
+		self.name, self,
+		self.hits, self.miss, self.total,
+		float64(self.hits)/float64(self.miss))
+}
+
+// removeOldest removes the oldest item from the cache.
+func (self *LRU) removeOldest() {
+	ent := self.evictList.Back()
+	if ent != nil {
+		self.removeElement(ent)
+	}
+}
+
+// removeElement is used to remove a given list element from the cache
+func (self *LRU) removeElement(e *list.Element) {
+	self.evictList.Remove(e)
+	kv := e.Value.(*entry)
+	delete(self.items, kv.key)
+	if self.onEvict != nil {
+		self.onEvict(kv.key, kv.value)
+	}
+}