Sigma plugin now supports VQL transformation rules. (#3474)

constants/constants.go

@@ -23,7 +23,7 @@ import (
 )
 
 const (
-	VERSION = "0.72.0"
+	VERSION = "0.72.1"
 
 	ENROLLMENT_WELL_KNOWN_FLOW = "E:Enrol"
 	MONITORING_WELL_KNOWN_FLOW = FLOW_PREFIX + "Monitoring"

file_store/directory/directory.go

@@ -174,9 +174,8 @@ func (self *DirectoryFileStore) StatFile(
 		return nil, err
 	}
 
-	return &file_store_file_info.FileStoreFileInfo{
-		FileInfo: file,
-	}, nil
+	return file_store_file_info.NewFileStoreFileInfo(
+		self.config_obj, filename, file), nil
 }
 
 func (self *DirectoryFileStore) WriteFile(

go.mod

@@ -278,4 +278,6 @@ replace github.com/alecthomas/chroma => github.com/Velocidex/chroma v0.6.8-0.202
 
 replace github.com/go-errors/errors => github.com/Velocidex/errors v0.0.0-20221019164655-9ace6bf61e26
 
-replace github.com/bradleyjkemp/sigma-go => github.com/Velocidex/sigma-go v0.0.0-20231015053605-117f1827960e
+replace github.com/bradleyjkemp/sigma-go => github.com/Velocidex/sigma-go v0.0.0-20240505024531-e8ce54ec3aed
+
+//replace github.com/bradleyjkemp/sigma-go => ../sigma-go

go.sum

@@ -139,8 +139,8 @@ github.com/Velocidex/pkcs7 v0.0.0-20230220112103-d4ed02e1862a h1:H7dVazNcaE80V8c
 github.com/Velocidex/pkcs7 v0.0.0-20230220112103-d4ed02e1862a/go.mod h1:/fy/Eg4TQz9KkJduvZfGCnbWTQ/LKaknS2wYB52cU6c=
 github.com/Velocidex/sflags v0.3.1-0.20231011011525-620ab7ca8617 h1:pxAOaYTYwbWhoSwRoJOT3TJmUAjD2D011A1c8M2yyEE=
 github.com/Velocidex/sflags v0.3.1-0.20231011011525-620ab7ca8617/go.mod h1:TTYBEgQFkjJvyBOC2s7G1+mNPZ3IHuqLZmVFRzyPfq4=
-github.com/Velocidex/sigma-go v0.0.0-20231015053605-117f1827960e h1:OyrCQ2wjJ0Y/pgClrOE+oIYlJCnzQyR0uz5bbwcdO3U=
-github.com/Velocidex/sigma-go v0.0.0-20231015053605-117f1827960e/go.mod h1:fHCN8y8cC1l5CYY7oOhPIznHmj/yeGxUvU+vAV7alr4=
+github.com/Velocidex/sigma-go v0.0.0-20240505024531-e8ce54ec3aed h1:zqhuWeg6oqO3jNabjKJaGO7DreiGhbVfeyqleICMAZk=
+github.com/Velocidex/sigma-go v0.0.0-20240505024531-e8ce54ec3aed/go.mod h1:fHCN8y8cC1l5CYY7oOhPIznHmj/yeGxUvU+vAV7alr4=
 github.com/Velocidex/ttlcache/v2 v2.9.1-0.20230724083715-1eb048b1f6d6 h1:3HSrLwAt64gM7FNTPRXYMszpS5bRijQ6xaPVq2vsbuI=
 github.com/Velocidex/ttlcache/v2 v2.9.1-0.20230724083715-1eb048b1f6d6/go.mod h1:3/pI9BBAF7gydBWvMVtV7W1qRwshEG9lBwed/d8xfFg=
 github.com/Velocidex/yaml/v2 v2.2.8 h1:GUrSy4SBJ6RjGt43k6MeBKtw2z/27gh4A3hfFmFY3No=

vql/sigma/evaluator/checks.go

@@ -0,0 +1,65 @@
+package evaluator
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/Velocidex/ordereddict"
+	"github.com/bradleyjkemp/sigma-go"
+	"www.velocidex.com/golang/vfilter"
+)
+
+func (self *VQLRuleEvaluator) CheckRule() error {
+	// Rule has no condition - just and all the selections
+	if len(self.Detection.Conditions) == 0 {
+		fields := []string{}
+		for k := range self.Detection.Searches {
+			fields = append(fields, k)
+		}
+
+		self.Detection.Conditions = append(self.Detection.Conditions,
+			sigma.Condition{
+				Search: sigma.SearchIdentifier{
+					Name: strings.Join(fields, " and "),
+				},
+			})
+	}
+
+	if self.Detection.Timeframe != "" {
+		return fmt.Errorf("In rule %v: Timeframe detections not supported",
+			self.Title)
+	}
+
+	// Make sure if the rule has a VQL lambda it is valid.
+	if self.AdditionalFields != nil {
+		lambda_any, pres := self.AdditionalFields["vql"]
+		if pres {
+			lambda_str, ok := lambda_any.(string)
+			if ok {
+				lambda, err := vfilter.ParseLambda(lambda_str)
+				if err != nil {
+					return fmt.Errorf(
+						"Rule provides invalid lambda: %v, Error: %v",
+						lambda_str, err)
+				}
+				self.lambda = lambda
+			}
+		}
+
+		self.lambda_args = ordereddict.NewDict()
+		lambda_args_any, pres := self.AdditionalFields["vql_args"]
+		if pres {
+			lambda_args_dict, ok := lambda_args_any.(map[string]interface{})
+			if ok {
+				for k, v := range lambda_args_dict {
+					self.lambda_args.Set(k, v)
+				}
+			} else {
+				return fmt.Errorf("Rule %v: vql_args should be a dict",
+					self.Title)
+			}
+		}
+	}
+
+	return nil
+}

vql/sigma/evaluator/evaluate.go

@@ -20,6 +20,11 @@ type VQLRuleEvaluator struct {
 	sigma.Rule
 	scope types.Scope
 
+	// If the rule specifies a VQL transformer we use that to
+	// transform the event.
+	lambda      *vfilter.Lambda
+	lambda_args *ordereddict.Dict
+
 	fieldmappings []FieldMappingRecord
 }
 
@@ -47,6 +52,27 @@ func (self *VQLRuleEvaluator) evaluateAggregationExpression(
 	return false, nil
 }
 
+func (self *VQLRuleEvaluator) MaybeEnrichWithVQL(
+	ctx context.Context, scope types.Scope, event *Event) *Event {
+	if self.lambda != nil {
+		new_event := NewEvent(event.Copy())
+		subscope := scope.Copy().AppendVars(self.lambda_args)
+		defer subscope.Close()
+
+		row := self.lambda.Reduce(ctx, subscope, []vfilter.Any{event})
+
+		// Merge the row into the event. This allows the VQL lambda to
+		// set any field.
+		for _, k := range scope.GetMembers(row) {
+			v, _ := scope.Associative(row, k)
+			new_event.Set(k, v)
+		}
+		return new_event
+	}
+
+	return event
+}
+
 func (self *VQLRuleEvaluator) Match(ctx context.Context,
 	scope types.Scope, event *Event) (Result, error) {
 	subscope := scope.Copy().AppendVars(
@@ -70,7 +96,6 @@ func (self *VQLRuleEvaluator) Match(ctx context.Context,
 		if err != nil {
 			return Result{}, fmt.Errorf("error evaluating search %s: %w", identifier, err)
 		}
-
 		result.SearchResults[identifier] = eval_result
 	}
 

vql/sigma/evaluator/event.go

@@ -25,8 +25,13 @@ import (
 // lambda for each rule and instead call it once for the first rule to
 // use this field.
 type Event struct {
+	// This is the original event from the log source.
 	*ordereddict.Dict
 
+	// This caches the sigma fields which are reduced by the sigma
+	// field mapping lambdas. The same event is passed through the
+	// entire rule chain so this caching avoids calculating the sigma
+	// fields multiple times.
 	mu         sync.Mutex
 	cache      map[string]types.Any
 	cache_json string

vql/sigma/evaluator/modifiers/vql.go

@@ -72,6 +72,7 @@ func (self vql) Matches(
 		}
 		lambda_cache.Set(expected_str, lambda)
 	}
+
 	return scope.Bool(
 		lambda.Reduce(ctx, scope, []types.Any{actual})), nil
 }

vql/sigma/fixtures/TestSigma.golden

@@ -1694,5 +1694,161 @@
     }
    }
   }
+ ],
+ "Test VQL Events": [
+  {
+   "Foo": 1,
+   "Bar": "Baz",
+   "Details": null,
+   "_Match": {
+    "Match": true,
+    "SearchResults": {
+     "selection1": true,
+     "selection2": true
+    },
+    "ConditionResults": [
+     true
+    ]
+   },
+   "_Rule": {
+    "Title": "VQL Events",
+    "Logsource": {
+     "Product": "windows",
+     "Service": "application"
+    },
+    "Detection": {
+     "Searches": {
+      "selection1": {
+       "EventMatchers": [
+        [
+         {
+          "Field": "Foo",
+          "Values": [
+           1
+          ]
+         }
+        ]
+       ]
+      },
+      "selection2": {
+       "EventMatchers": [
+        [
+         {
+          "Field": "Bar",
+          "Modifiers": [
+           "contains"
+          ],
+          "Values": [
+           "B"
+          ]
+         }
+        ]
+       ]
+      }
+     },
+     "condition": [
+      {
+       "Search": [
+        {
+         "Name": "selection1"
+        },
+        {
+         "Name": "selection2"
+        }
+       ]
+      }
+     ]
+    },
+    "AdditionalFields": {
+     "vql": "x=\u003edict(Foo=1, Bar=\"Baz\")"
+    }
+   }
+  }
+ ],
+ "Test Conditions": [
+  {
+   "Foo": 1,
+   "Bar": "Baz",
+   "Proc": 1,
+   "Details": null,
+   "_Match": {
+    "Match": true,
+    "SearchResults": {
+     "process_creation": true,
+     "selection_1_1": true,
+     "selection_1_2": true
+    },
+    "ConditionResults": [
+     true
+    ]
+   },
+   "_Rule": {
+    "Title": "VQL Events",
+    "Logsource": {
+     "Product": "windows",
+     "Service": "application"
+    },
+    "Detection": {
+     "Searches": {
+      "process_creation": {
+       "EventMatchers": [
+        [
+         {
+          "Field": "Proc",
+          "Values": [
+           1
+          ]
+         }
+        ]
+       ]
+      },
+      "selection_1_1": {
+       "EventMatchers": [
+        [
+         {
+          "Field": "Foo",
+          "Values": [
+           1
+          ]
+         }
+        ]
+       ]
+      },
+      "selection_1_2": {
+       "EventMatchers": [
+        [
+         {
+          "Field": "Bar",
+          "Modifiers": [
+           "contains"
+          ],
+          "Values": [
+           "B"
+          ]
+         }
+        ]
+       ]
+      }
+     },
+     "condition": [
+      {
+       "Search": [
+        {
+         "Name": "process_creation"
+        },
+        [
+         {
+          "Pattern": "selection_1_*"
+         },
+         {
+          "Pattern": "selection_1_*"
+         }
+        ]
+       ]
+      }
+     ]
+    }
+   }
+  }
  ]
 }

vql/sigma/pool.go

@@ -30,7 +30,8 @@ func (self *workerJob) Run() {
 	defer self.wg.Done()
 
 	for _, rule := range self.rules {
-		match, err := rule.Match(self.ctx, self.scope, self.event)
+		event := rule.MaybeEnrichWithVQL(self.ctx, self.scope, self.event)
+		match, err := rule.Match(self.ctx, self.scope, event)
 		if err != nil {
 			functions.DeduplicatedLog(self.ctx, self.scope,
 				"While evaluating rule %v: %v", rule.Title, err)
@@ -44,16 +45,17 @@ func (self *workerJob) Run() {
 		// Make a copy here because another thread might match at the same
 		// time.
 		event_copy := self.sigma_context.AddDetail(
-			self.ctx, self.scope, self.event, rule)
+			self.ctx, self.scope, event, rule)
 		event_copy.Set("_Match", match).
 			Set("_Rule", rule)
 
+		self.sigma_context.IncHitCount()
+
 		select {
 		case <-self.ctx.Done():
 			return
 
 		case self.output_chan <- event_copy:
-			self.sigma_context.IncHitCount()
 		}
 	}
 }