Materialize eval result to avoid memory leaks

accessors/offset/offset.go

@@ -126,7 +126,8 @@ func init() {
 For Example
 
 FileName = pathspec(
-      DelegateAccessor="data", DelegatePath=MyData,
-      Path=[dict(Offset=0,Length=5), dict(Offset=10,Length=5)])
+      DelegateAccessor="data",
+      DelegatePath=MyData,
+      Path="/5")
 `)
 }

accessors/raw_registry/raw_registry.go

@@ -352,6 +352,10 @@ func (self *RawRegFileSystemAccessor) New(scope vfilter.Scope) (
 		my_readdir_lru.SetCacheSizeLimit(1000)
 		my_readdir_lru.SetTTL(time.Minute)
 	}
+	scope.AddDestructor(func() {
+		my_lru.Close()
+		my_readdir_lru.Close()
+	})
 
 	return &RawRegFileSystemAccessor{
 		scope: scope,

accessors/registry/registry_windows.go

@@ -392,6 +392,10 @@ func (self *RegFileSystemAccessor) New(scope vfilter.Scope) (
 		my_readdir_lru.SetCacheSizeLimit(1000)
 		my_readdir_lru.SetTTL(time.Minute)
 	}
+	scope.AddDestructor(func() {
+		my_lru.Close()
+		my_readdir_lru.Close()
+	})
 
 	return &RegFileSystemAccessor{
 		lru:         my_lru,

accessors/smb/cache.go

@@ -172,6 +172,7 @@ func NewSMBMountCache(scope vfilter.Scope) *SMBMountCache {
 
 	vql_subsystem.GetRootScope(scope).AddDestructor(func() {
 		result.lru.Flush()
+		result.lru.Close()
 		cancel()
 	})
 	return result

crypto/server/manager.go

@@ -208,5 +208,14 @@ func NewServerPublicKeyResolver(
 
 	result.negative_lru.SetTTL(timeout)
 
+	// Close the LRU when we are done here.
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		<-ctx.Done()
+
+		result.negative_lru.Close()
+	}()
+
 	return result, nil
 }

docs/references/vql.yaml

@@ -1385,6 +1385,10 @@
   - name: action
     type: string
     description: Either index or create. For data streams this must be create.
+  - name: secret
+    type: string
+    description: Alternatively use a secret from the secrets service. Secret must
+      be of type 'AWS S3 Creds'
   category: server
   metadata:
     permissions: COLLECT_SERVER
@@ -1421,7 +1425,7 @@
   args:
   - name: items
     type: Any
-    description: Not used anymore
+    description: The items to enumerate
   category: basic
 - name: enumerate_flow
   description: |
@@ -1476,6 +1480,15 @@
     description: 'The count of sessions to retrieve (default 64) '
   metadata:
     permissions: MACHINE_STATE
+- name: eval
+  description: Evaluate a vql lambda function on the current scope.
+  type: Function
+  args:
+  - name: func
+    type: vfilter.Lambda
+    description: Lambda function to evaluate e.g. x=>1+1 where x will be the current
+      scope.
+    required: true
 - name: execve
   description: |
     This plugin launches an external command and captures its STDERR,
@@ -2800,6 +2813,14 @@
   - name: else
     type: StoredQuery
   category: plugin
+- name: import
+  description: Imports an artifact into the current scope. This only works in notebooks!
+  type: Function
+  args:
+  - name: artifact
+    type: string
+    description: The Artifact to import
+    required: true
 - name: import_collection
   description: |
     Imports an offline collection zip file (experimental).
@@ -3560,6 +3581,9 @@
   - name: period
     type: int64
     description: The latest age of the cache.
+  - name: name
+    type: string
+    description: The name of this cache.
   category: basic
 - name: min
   description: |
@@ -5818,6 +5842,7 @@
   - name: url
     type: string
     description: The Splunk Event Collector URL.
+    required: true
   - name: token
     type: string
     description: Splunk HEC Token.
@@ -5853,6 +5878,10 @@
   - name: hostname_field
     type: string
     description: Field to use as event hostname. Overrides hostname parameter.
+  - name: secret
+    type: string
+    description: Alternatively use a secret from the secrets service. Secret must
+      be of type 'AWS S3 Creds'
   category: server
   metadata:
     permissions: COLLECT_SERVER
@@ -5951,6 +5980,19 @@
     type: Any
     description: Dictionary of values to feed into Starlark environment
   category: parsers
+- name: stat
+  description: Get file information. Unlike glob() this does not support wildcards.
+  type: Function
+  args:
+  - name: filename
+    type: accessors.OSPath
+    description: One or more files to open.
+    required: true
+  - name: accessor
+    type: string
+    description: An accessor to use.
+  metadata:
+    permissions: FILESYSTEM_READ
 - name: stat
   description: Get file information. Unlike glob() this does not support wildcards.
   type: Plugin
@@ -6641,6 +6683,9 @@
   - name: hunt_id
     type: string
     description: A hunt ID
+  - name: notebook_id
+    type: string
+    description: A notebook ID
   category: server
   metadata:
     permissions: READ_RESULTS
@@ -7307,3 +7352,4 @@
   category: plugin
   metadata:
     permissions: FILESYSTEM_READ
+

vql/common/lru.go

@@ -88,6 +88,9 @@ func (self LRUFunction) Call(ctx context.Context, scope vfilter.Scope,
 	result := &LRUCache{
 		lru: ttlcache.NewCache(),
 	}
+	scope.AddDestructor(func() {
+		result.lru.Close()
+	})
 
 	if arg.Size <= 0 {
 		arg.Size = 1000

vql/functions/eval.go

@@ -33,7 +33,10 @@ func (self EvalFunction) Call(ctx context.Context, scope vfilter.Scope,
 	}
 
 	// Evaluate the lambda on the current scope.
-	return arg.Func.Reduce(ctx, scope, []vfilter.Any{scope})
+	res := arg.Func.Reduce(ctx, scope, []vfilter.Any{scope})
+
+	// Materialize the lambda
+	return vql_subsystem.Materialize(ctx, scope, res)
 }
 
 func init() {

vql/functions/log.go

@@ -1,19 +1,19 @@
 /*
-   Velociraptor - Dig Deeper
-   Copyright (C) 2019-2024 Rapid7 Inc.
+Velociraptor - Dig Deeper
+Copyright (C) 2019-2024 Rapid7 Inc.
 
-   This program is free software: you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation, either version 3 of the License, or
-   (at your option) any later version.
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published
+by the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
 
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU Affero General Public License for more details.
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
 
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see <https://www.gnu.org/licenses/>.
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
 package functions
 
@@ -78,6 +78,10 @@ func (self *LogFunction) Call(ctx context.Context,
 		}
 		log_cache.lru.SetCacheSizeLimit(100)
 
+		vql_subsystem.GetRootScope(scope).AddDestructor(func() {
+			log_cache.lru.Close()
+		})
+
 	} else {
 		log_cache, _ = log_cache_any.(*logCache)
 		if log_cache == nil {

vql/readers/paged.go

@@ -25,7 +25,7 @@ know when is it safe to remove the file reference - at the end of
 the row? at the end the root scope?
 
 Having an LRU allows us to be flexible and not worry about the
-scope lifetime so much. Files will eventually get closed and cached
+scope lifetime so much. Files will eventually get closed and caches
 will be evicted.
 */
 package readers
@@ -74,6 +74,7 @@ func (self *ReaderPool) Close() {
 	for _, k := range self.lru.GetKeys() {
 		self.lru.Remove(k)
 	}
+	self.lru.Close()
 }
 
 type AccessorReader struct {

vql/tools/process/tracker.go

@@ -393,13 +393,16 @@ func (self *ProcessTracker) CallChain(
 	}
 }
 
-func NewProcessTracker(max_size int) *ProcessTracker {
+func NewProcessTracker(scope vfilter.Scope, max_size int) *ProcessTracker {
 	result := &ProcessTracker{
 		lookup:      ttlcache.NewCache(),
 		enrichments: ordereddict.NewDict(),
 	}
 
 	result.lookup.SetCacheSizeLimit(max_size)
+	scope.AddDestructor(func() {
+		result.lookup.Close()
+	})
 
 	return result
 }
@@ -445,7 +448,7 @@ func (self _InstallProcessTracker) Call(ctx context.Context,
 		max_size = 10000
 	}
 
-	tracker := NewProcessTracker(int(max_size))
+	tracker := NewProcessTracker(scope, int(max_size))
 
 	// Any any enrichments to the tracker.
 	for _, enrichment := range arg.Enrichments {

vql_plugins/server.go

@@ -1,21 +1,22 @@
+//go:build server_vql
 // +build server_vql
 
 /*
-   Velociraptor - Dig Deeper
-   Copyright (C) 2019-2024 Rapid7 Inc.
+Velociraptor - Dig Deeper
+Copyright (C) 2019-2024 Rapid7 Inc.
 
-   This program is free software: you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation, either version 3 of the License, or
-   (at your option) any later version.
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published
+by the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
 
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU Affero General Public License for more details.
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
 
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see <https://www.gnu.org/licenses/>.
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
 package plugins