diff --git a/constants/constants.go b/constants/constants.go index 6fd670ecd8..a193944d22 100644 --- a/constants/constants.go +++ b/constants/constants.go @@ -23,7 +23,7 @@ import ( ) const ( - VERSION = "0.72.0" + VERSION = "0.72.1" ENROLLMENT_WELL_KNOWN_FLOW = "E:Enrol" MONITORING_WELL_KNOWN_FLOW = FLOW_PREFIX + "Monitoring" diff --git a/file_store/directory/directory.go b/file_store/directory/directory.go index 9dba1d00cf..3adcc010e3 100644 --- a/file_store/directory/directory.go +++ b/file_store/directory/directory.go @@ -174,9 +174,8 @@ func (self *DirectoryFileStore) StatFile( return nil, err } - return &file_store_file_info.FileStoreFileInfo{ - FileInfo: file, - }, nil + return file_store_file_info.NewFileStoreFileInfo( + self.config_obj, filename, file), nil } func (self *DirectoryFileStore) WriteFile( diff --git a/go.mod b/go.mod index 65331d71c3..7f5740cf62 100644 --- a/go.mod +++ b/go.mod @@ -278,4 +278,6 @@ replace github.com/alecthomas/chroma => github.com/Velocidex/chroma v0.6.8-0.202 replace github.com/go-errors/errors => github.com/Velocidex/errors v0.0.0-20221019164655-9ace6bf61e26 -replace github.com/bradleyjkemp/sigma-go => github.com/Velocidex/sigma-go v0.0.0-20231015053605-117f1827960e +replace github.com/bradleyjkemp/sigma-go => github.com/Velocidex/sigma-go v0.0.0-20240505024531-e8ce54ec3aed + +//replace github.com/bradleyjkemp/sigma-go => ../sigma-go diff --git a/go.sum b/go.sum index 6b74b9791a..64f2c2a2fe 100644 --- a/go.sum +++ b/go.sum @@ -139,8 +139,8 @@ github.com/Velocidex/pkcs7 v0.0.0-20230220112103-d4ed02e1862a h1:H7dVazNcaE80V8c github.com/Velocidex/pkcs7 v0.0.0-20230220112103-d4ed02e1862a/go.mod h1:/fy/Eg4TQz9KkJduvZfGCnbWTQ/LKaknS2wYB52cU6c= github.com/Velocidex/sflags v0.3.1-0.20231011011525-620ab7ca8617 h1:pxAOaYTYwbWhoSwRoJOT3TJmUAjD2D011A1c8M2yyEE= github.com/Velocidex/sflags v0.3.1-0.20231011011525-620ab7ca8617/go.mod h1:TTYBEgQFkjJvyBOC2s7G1+mNPZ3IHuqLZmVFRzyPfq4= -github.com/Velocidex/sigma-go v0.0.0-20231015053605-117f1827960e h1:OyrCQ2wjJ0Y/pgClrOE+oIYlJCnzQyR0uz5bbwcdO3U= -github.com/Velocidex/sigma-go v0.0.0-20231015053605-117f1827960e/go.mod h1:fHCN8y8cC1l5CYY7oOhPIznHmj/yeGxUvU+vAV7alr4= +github.com/Velocidex/sigma-go v0.0.0-20240505024531-e8ce54ec3aed h1:zqhuWeg6oqO3jNabjKJaGO7DreiGhbVfeyqleICMAZk= +github.com/Velocidex/sigma-go v0.0.0-20240505024531-e8ce54ec3aed/go.mod h1:fHCN8y8cC1l5CYY7oOhPIznHmj/yeGxUvU+vAV7alr4= github.com/Velocidex/ttlcache/v2 v2.9.1-0.20230724083715-1eb048b1f6d6 h1:3HSrLwAt64gM7FNTPRXYMszpS5bRijQ6xaPVq2vsbuI= github.com/Velocidex/ttlcache/v2 v2.9.1-0.20230724083715-1eb048b1f6d6/go.mod h1:3/pI9BBAF7gydBWvMVtV7W1qRwshEG9lBwed/d8xfFg= github.com/Velocidex/yaml/v2 v2.2.8 h1:GUrSy4SBJ6RjGt43k6MeBKtw2z/27gh4A3hfFmFY3No= diff --git a/vql/sigma/checks.go b/vql/sigma/checks.go deleted file mode 100644 index 475eb3f1e5..0000000000 --- a/vql/sigma/checks.go +++ /dev/null @@ -1,26 +0,0 @@ -package sigma - -import ( - "strings" - - "github.com/bradleyjkemp/sigma-go" -) - -func CheckRule(rule *sigma.Rule) error { - // Rule has no condition - just and all the selections - if len(rule.Detection.Conditions) == 0 { - fields := []string{} - for k := range rule.Detection.Searches { - fields = append(fields, k) - } - - rule.Detection.Conditions = append(rule.Detection.Conditions, - sigma.Condition{ - Search: sigma.SearchIdentifier{ - Name: strings.Join(fields, " and "), - }, - }) - } - - return nil -} diff --git a/vql/sigma/evaluator/checks.go b/vql/sigma/evaluator/checks.go new file mode 100644 index 0000000000..aa0960e9e8 --- /dev/null +++ b/vql/sigma/evaluator/checks.go @@ -0,0 +1,65 @@ +package evaluator + +import ( + "fmt" + "strings" + + "github.com/Velocidex/ordereddict" + "github.com/bradleyjkemp/sigma-go" + "www.velocidex.com/golang/vfilter" +) + +func (self *VQLRuleEvaluator) CheckRule() error { + // Rule has no condition - just and all the selections + if len(self.Detection.Conditions) == 0 { + fields := []string{} + for k := range self.Detection.Searches { + fields = append(fields, k) + } + + self.Detection.Conditions = append(self.Detection.Conditions, + sigma.Condition{ + Search: sigma.SearchIdentifier{ + Name: strings.Join(fields, " and "), + }, + }) + } + + if self.Detection.Timeframe != "" { + return fmt.Errorf("In rule %v: Timeframe detections not supported", + self.Title) + } + + // Make sure if the rule has a VQL lambda it is valid. + if self.AdditionalFields != nil { + lambda_any, pres := self.AdditionalFields["vql"] + if pres { + lambda_str, ok := lambda_any.(string) + if ok { + lambda, err := vfilter.ParseLambda(lambda_str) + if err != nil { + return fmt.Errorf( + "Rule provides invalid lambda: %v, Error: %v", + lambda_str, err) + } + self.lambda = lambda + } + } + + self.lambda_args = ordereddict.NewDict() + lambda_args_any, pres := self.AdditionalFields["vql_args"] + if pres { + lambda_args_dict, ok := lambda_args_any.(map[string]interface{}) + if ok { + for k, v := range lambda_args_dict { + self.lambda_args.Set(k, v) + } + } else { + return fmt.Errorf("Rule %v: vql_args should be a dict", + self.Title) + } + } + } + + return nil +} diff --git a/vql/sigma/evaluator/evaluate.go b/vql/sigma/evaluator/evaluate.go index 4d50c51cdf..603ebf15a9 100644 --- a/vql/sigma/evaluator/evaluate.go +++ b/vql/sigma/evaluator/evaluate.go @@ -20,6 +20,11 @@ type VQLRuleEvaluator struct { sigma.Rule scope types.Scope + // If the rule specifies a VQL transformer we use that to + // transform the event. + lambda *vfilter.Lambda + lambda_args *ordereddict.Dict + fieldmappings []FieldMappingRecord } @@ -47,6 +52,27 @@ func (self *VQLRuleEvaluator) evaluateAggregationExpression( return false, nil } +func (self *VQLRuleEvaluator) MaybeEnrichWithVQL( + ctx context.Context, scope types.Scope, event *Event) *Event { + if self.lambda != nil { + new_event := NewEvent(event.Copy()) + subscope := scope.Copy().AppendVars(self.lambda_args) + defer subscope.Close() + + row := self.lambda.Reduce(ctx, subscope, []vfilter.Any{event}) + + // Merge the row into the event. This allows the VQL lambda to + // set any field. + for _, k := range scope.GetMembers(row) { + v, _ := scope.Associative(row, k) + new_event.Set(k, v) + } + return new_event + } + + return event +} + func (self *VQLRuleEvaluator) Match(ctx context.Context, scope types.Scope, event *Event) (Result, error) { subscope := scope.Copy().AppendVars( @@ -70,7 +96,6 @@ func (self *VQLRuleEvaluator) Match(ctx context.Context, if err != nil { return Result{}, fmt.Errorf("error evaluating search %s: %w", identifier, err) } - result.SearchResults[identifier] = eval_result } diff --git a/vql/sigma/evaluator/event.go b/vql/sigma/evaluator/event.go index e53d4f9148..090fe14a38 100644 --- a/vql/sigma/evaluator/event.go +++ b/vql/sigma/evaluator/event.go @@ -25,8 +25,13 @@ import ( // lambda for each rule and instead call it once for the first rule to // use this field. type Event struct { + // This is the original event from the log source. *ordereddict.Dict + // This caches the sigma fields which are reduced by the sigma + // field mapping lambdas. The same event is passed through the + // entire rule chain so this caching avoids calculating the sigma + // fields multiple times. mu sync.Mutex cache map[string]types.Any cache_json string diff --git a/vql/sigma/evaluator/modifiers/vql.go b/vql/sigma/evaluator/modifiers/vql.go index 07a261befd..b5fa90aedd 100644 --- a/vql/sigma/evaluator/modifiers/vql.go +++ b/vql/sigma/evaluator/modifiers/vql.go @@ -72,6 +72,7 @@ func (self vql) Matches( } lambda_cache.Set(expected_str, lambda) } + return scope.Bool( lambda.Reduce(ctx, scope, []types.Any{actual})), nil } diff --git a/vql/sigma/fixtures/TestSigma.golden b/vql/sigma/fixtures/TestSigma.golden index ca17b63645..2cfdc6cc44 100644 --- a/vql/sigma/fixtures/TestSigma.golden +++ b/vql/sigma/fixtures/TestSigma.golden @@ -1694,5 +1694,161 @@ } } } + ], + "Test VQL Events": [ + { + "Foo": 1, + "Bar": "Baz", + "Details": null, + "_Match": { + "Match": true, + "SearchResults": { + "selection1": true, + "selection2": true + }, + "ConditionResults": [ + true + ] + }, + "_Rule": { + "Title": "VQL Events", + "Logsource": { + "Product": "windows", + "Service": "application" + }, + "Detection": { + "Searches": { + "selection1": { + "EventMatchers": [ + [ + { + "Field": "Foo", + "Values": [ + 1 + ] + } + ] + ] + }, + "selection2": { + "EventMatchers": [ + [ + { + "Field": "Bar", + "Modifiers": [ + "contains" + ], + "Values": [ + "B" + ] + } + ] + ] + } + }, + "condition": [ + { + "Search": [ + { + "Name": "selection1" + }, + { + "Name": "selection2" + } + ] + } + ] + }, + "AdditionalFields": { + "vql": "x=\u003edict(Foo=1, Bar=\"Baz\")" + } + } + } + ], + "Test Conditions": [ + { + "Foo": 1, + "Bar": "Baz", + "Proc": 1, + "Details": null, + "_Match": { + "Match": true, + "SearchResults": { + "process_creation": true, + "selection_1_1": true, + "selection_1_2": true + }, + "ConditionResults": [ + true + ] + }, + "_Rule": { + "Title": "VQL Events", + "Logsource": { + "Product": "windows", + "Service": "application" + }, + "Detection": { + "Searches": { + "process_creation": { + "EventMatchers": [ + [ + { + "Field": "Proc", + "Values": [ + 1 + ] + } + ] + ] + }, + "selection_1_1": { + "EventMatchers": [ + [ + { + "Field": "Foo", + "Values": [ + 1 + ] + } + ] + ] + }, + "selection_1_2": { + "EventMatchers": [ + [ + { + "Field": "Bar", + "Modifiers": [ + "contains" + ], + "Values": [ + "B" + ] + } + ] + ] + } + }, + "condition": [ + { + "Search": [ + { + "Name": "process_creation" + }, + [ + { + "Pattern": "selection_1_*" + }, + { + "Pattern": "selection_1_*" + } + ] + ] + } + ] + } + } + } ] } \ No newline at end of file diff --git a/vql/sigma/pool.go b/vql/sigma/pool.go index d59fc512ba..8c2ecede05 100644 --- a/vql/sigma/pool.go +++ b/vql/sigma/pool.go @@ -30,7 +30,8 @@ func (self *workerJob) Run() { defer self.wg.Done() for _, rule := range self.rules { - match, err := rule.Match(self.ctx, self.scope, self.event) + event := rule.MaybeEnrichWithVQL(self.ctx, self.scope, self.event) + match, err := rule.Match(self.ctx, self.scope, event) if err != nil { functions.DeduplicatedLog(self.ctx, self.scope, "While evaluating rule %v: %v", rule.Title, err) @@ -44,16 +45,17 @@ func (self *workerJob) Run() { // Make a copy here because another thread might match at the same // time. event_copy := self.sigma_context.AddDetail( - self.ctx, self.scope, self.event, rule) + self.ctx, self.scope, event, rule) event_copy.Set("_Match", match). Set("_Rule", rule) + self.sigma_context.IncHitCount() + select { case <-self.ctx.Done(): return case self.output_chan <- event_copy: - self.sigma_context.IncHitCount() } } } diff --git a/vql/sigma/runner.go b/vql/sigma/runner.go index 14c5d7c192..9b3e286973 100644 --- a/vql/sigma/runner.go +++ b/vql/sigma/runner.go @@ -42,6 +42,13 @@ type SigmaContext struct { default_details *vfilter.Lambda } +func (self *SigmaContext) GetHitCount() int { + self.mu.Lock() + defer self.mu.Unlock() + + return self.hit_count +} + func (self *SigmaContext) IncHitCount() { self.mu.Lock() defer self.mu.Unlock() @@ -61,12 +68,11 @@ func (self *SigmaContext) Rows( defer subscope.Close() count := 0 - hit_count := 0 start := utils.GetTime().Now() defer func() { - scope.Log("INFO:sigma: Consumed %v messages from log source %v with %v hits on %v rules (%v)", - count, runner.Name, hit_count, len(runner.rules), + scope.Log("INFO:sigma: Consumed %v messages from log source %v on %v rules (%v)", + count, runner.Name, len(runner.rules), utils.GetTime().Now().Sub(start)) }() @@ -140,9 +146,18 @@ func NewSigmaContext( for _, r := range rules { if matchLogSource(log_target, r) { - runner.rules = append(runner.rules, - evaluator.NewVQLRuleEvaluator( - scope, r, compiled_fieldmappings)) + evaluator_rule := evaluator.NewVQLRuleEvaluator( + scope, r, compiled_fieldmappings) + + // Check rule for sanity + err := evaluator_rule.CheckRule() + if err != nil { + scope.Log("sigma: Error parsing: %v in rule '%v'", + err, evaluator_rule.Rule.Title) + continue + } + + runner.rules = append(runner.rules, evaluator_rule) total_rules++ } } diff --git a/vql/sigma/sigma.go b/vql/sigma/sigma.go index c36ecd36d7..704c0cc1c7 100644 --- a/vql/sigma/sigma.go +++ b/vql/sigma/sigma.go @@ -60,16 +60,13 @@ func (self SigmaPlugin) Call( continue } - if arg.RuleFilter != nil && - !scope.Bool(arg.RuleFilter.Reduce(ctx, scope, []vfilter.Any{rule})) { + // A rule must have a title + if rule.Title == "" { continue } - // Check rule for sanity - err = CheckRule(&rule) - if err != nil { - scope.Log("sigma: Error parsing: %v in rule '%v'", - err, utils.Elide(r, 20)) + if arg.RuleFilter != nil && + !scope.Bool(arg.RuleFilter.Reduce(ctx, scope, []vfilter.Any{rule})) { continue } @@ -96,6 +93,7 @@ func (self SigmaPlugin) Call( for row := range sigma_context.Rows(ctx, scope) { output_chan <- row } + scope.Log("INFO:sigma: Completed with %v hits", sigma_context.GetHitCount()) }() return output_chan diff --git a/vql/sigma/sigma_test.go b/vql/sigma/sigma_test.go index 0e5b2328f9..8cdbb03096 100644 --- a/vql/sigma/sigma_test.go +++ b/vql/sigma/sigma_test.go @@ -498,6 +498,60 @@ detection: Set("Foo", base64.StdEncoding.EncodeToString([]byte("kgkrpepsigrgspriteefjefe"))), }, }, + { + description: "Test VQL Events", + rule: ` +title: VQL Events +logsource: + product: windows + service: application + +detection: + selection1: + Foo: 1 + selection2: + Bar|contains: B + + condition: selection1 and selection2 + +vql: x=>dict(Foo=1, Bar="Baz") +`, + fieldmappings: ordereddict.NewDict(). + Set("Foo", "x=>x.Foo"). + Set("Bar", "x=>x.Bar"), + rows: []*ordereddict.Dict{ + ordereddict.NewDict(), + }, + }, + { + description: "Test Conditions", + rule: ` +title: VQL Events +logsource: + product: windows + service: application + +detection: + process_creation: + Proc: 1 + selection_1_1: + Foo: 1 + selection_1_2: + Bar|contains: B + + condition: "process_creation and (all of selection_1_* or all of selection_1_*)" +`, + fieldmappings: ordereddict.NewDict(). + Set("Foo", "x=>x.Foo"). + Set("Bar", "x=>x.Bar"). + Set("Proc", "x=>x.Proc"), + rows: []*ordereddict.Dict{ + ordereddict.NewDict(). + Set("Foo", 1). + Set("Bar", "Baz"). + Set("Proc", 1), + }, + }, } ) @@ -517,7 +571,7 @@ func (self *SigmaTestSuite) TestSigmaModifiers() { plugin := SigmaPlugin{} for _, test_case := range sigmaTestCases { - if false && test_case.description != "Match single base64offset field" { + if false && test_case.description != "Test Conditions" { continue }