Bugfix: Handle NULL in stacking (#3570)

Since NULL does not compare to anything sorting on columns that have
NULL in them is not stable. This change considers those values as empty
strings so stacking is stable.

accessors/protocols.go

@@ -191,8 +191,12 @@ func (self _AssociativeOSPath) Associative(
 		second_item := length
 		if t[1] != nil {
 			second_item = *t[1]
+			if second_item > length {
+				second_item = length
+			}
 		}
 
+		// Wrap around behavior for negative index.
 		if first_item < 0 {
 			first_item += length
 		}

artifacts/definitions/Generic/Detection/Yara/Zip.yaml

@@ -105,11 +105,13 @@ sources:
                                 Rule, Tags, Meta,
                                 String.Name as YaraString,
                                 String.Offset as HitOffset,
-                                upload( accessor='scope',
+                                if(condition=String.Data,
+                                   then=upload(
+                                    accessor='scope',
                                     file='String.Data',
                                     name=format(format="%v_%v",
                                     args=[ OSPath.HumanString, String.Offset ]
-                                        )) as HitContext
+                                   ))) as HitContext
                               FROM yara(accessor='zip',files=OSPath,rules=YaraRule,
                                 context=ContextBytes, number=NumberOfHits)
                             })

artifacts/definitions/Linux/Detection/Yara/Process.yaml

@@ -95,11 +95,13 @@ sources:
                 Meta,
                 String.Name as YaraString,
                 String.Offset as HitOffset,
-                upload( accessor='scope',
-                    file='String.Data',
-                    name=format(format="%v-%v_%v_%v",
-                    args=[ ProcessName, Pid, String.Offset, ContextBytes ]
-                        )) as HitContext
+                if(condition=String.Data,
+                   then=upload(
+                     accessor='scope',
+                     file='String.Data',
+                     name=format(format="%v-%v_%v_%v",
+                     args=[ ProcessName, Pid, String.Offset, ContextBytes ]
+                    ))) as HitContext
              FROM proc_yara(
                         pid=Pid,
                         rules=yara_rules,

artifacts/definitions/Windows/Carving/CobaltStrike.yaml

@@ -462,7 +462,7 @@ sources:
             FROM switch( -- switchcase will find beacon as priority, then search for shellcode
                 beacon = {
                     SELECT *,
-                        substr(start=0,end=1,str=String.Data) as Xor,
+                        substr(start=0, end=1, str=String.Data) as Xor,
                         read_file(accessor='data',
                                   filename=TargetBytes,
                                   offset= String.Offset,

artifacts/definitions/Windows/Detection/Yara/NTFS.yaml

@@ -25,7 +25,7 @@ description: |
 
   Note: no drive and forward slashes - these expressions are for paths
   relative to the root of the filesystem.
-  If upload is selected NumberOfHits is redundant and not advised as hits are 
+  If upload is selected NumberOfHits is redundant and not advised as hits are
   grouped by path to ensure files only downloaded once.
 
 type: CLIENT
@@ -80,8 +80,8 @@ parameters:
     description: Include this amount of bytes around hit as context.
     default: 0
     type: int
-    
-    
+
+
 sources:
   - precondition:
       SELECT OS From info() where OS = 'windows'
@@ -95,7 +95,7 @@ sources:
             OSPath, IsDir
         FROM Artifact.Windows.NTFS.MFT(
             MFTDrive=DriveLetter, AllDrives=AllDrives,
-            FileRegex=FileNameRegex,PathRegex=PathRegex, 
+            FileRegex=FileNameRegex,PathRegex=PathRegex,
             SizeMax=SizeMax, SizeMin=SizeMin)
         WHERE NOT IsDir
             AND NOT OSPath =~ '''\\\\.\\.:\\<Err>\\'''
@@ -122,9 +122,11 @@ sources:
                     Rule, Tags, Meta,
                     String.Name as YaraString,
                     String.Offset as HitOffset,
-                    upload( accessor='scope', 
-                            file='String.Data', 
-                            name=format(format="%v-%v-%v", 
+                    if(condition=String.Data,
+                       then=upload(
+                            accessor='scope',
+                            file='String.Data',
+                            name=format(format="%v-%v-%v",
                             args=[
                                 OSPath,
                                 if(condition= String.Offset - ContextBytes < 0,
@@ -133,8 +135,9 @@ sources:
                                 if(condition= String.Offset + ContextBytes > File.Size,
                                     then= File.Size,
                                     else= String.Offset + ContextBytes) ]
-                            )) as HitContext
-                FROM yara(rules=yara_rules, files=OSPath, context=ContextBytes,number=NumberOfHits)
+                            ))) as HitContext
+                FROM yara(rules=yara_rules,
+                   files=OSPath, context=ContextBytes, number=NumberOfHits)
             })
 
       -- upload files that have hit

artifacts/testdata/server/testcases/yara_detection.in.yaml

@@ -88,8 +88,9 @@ Queries:
        DriveLetter=srcDir,
        PathRegex='wkscli.dll', ContextBytes=10, YaraRule=MZRule)
 
+  # This rule has no strings so HitContext should be NULL
   - SELECT relpath(path=OSPath, base=srcDir, sep="/") as TestPath,
-           Size, Rule, CleanContext(HitContext=HitContext), HitOffset
+           Size, Rule, HitContext, HitOffset
     FROM Artifact.Windows.Detection.Yara.NTFS(
          DriveLetter=srcDir,
          PathRegex='wkscli.dll', YaraRule=IsPE)

artifacts/testdata/server/testcases/yara_detection.out.yaml

@@ -80,18 +80,13 @@ FROM scope()
   },
   "HitOffset": 0
  }
-]SELECT relpath(path=OSPath, base=srcDir, sep="/") as TestPath, Size, Rule, CleanContext(HitContext=HitContext), HitOffset FROM Artifact.Windows.Detection.Yara.NTFS( DriveLetter=srcDir, PathRegex='wkscli.dll', YaraRule=IsPE)[
+]SELECT relpath(path=OSPath, base=srcDir, sep="/") as TestPath, Size, Rule, HitContext, HitOffset FROM Artifact.Windows.Detection.Yara.NTFS( DriveLetter=srcDir, PathRegex='wkscli.dll', YaraRule=IsPE)[
  {
   "TestPath": "artifacts/testdata/files/wkscli.dll",
   "Size": 9728,
   "Rule": "IsPE",
-  "CleanContext(HitContext=HitContext)": {
-   "StoredSize": 4,
-   "Path": "String.Data",
-   "Size": 4,
-   "sha256": "fa1d2db62d4d952e2031452e1bc1ddcad0b192c2e29a706f11ce426ae5acddea"
-  },
-  "HitOffset": null
+  "HitContext": null,
+  "HitOffset": 0
  }
 ]SELECT relpath(path=OSPath, base=srcDir, sep="/") as TestPath,Size, Rule,Mtime,Atime,Ctime,Btime FROM Artifact.Generic.Detection.Yara.Glob(YaraRule=IsPE)[
  {

go.mod

@@ -99,7 +99,7 @@ require (
 	www.velocidex.com/golang/go-prefetch v0.0.0-20220801101854-338dbe61982a
 	www.velocidex.com/golang/oleparse v0.0.0-20230217092320-383a0121aafe
 	www.velocidex.com/golang/regparser v0.0.0-20240404115756-2169ac0e3c09
-	www.velocidex.com/golang/vfilter v0.0.0-20240608150317-307fc598a311
+	www.velocidex.com/golang/vfilter v0.0.0-20240618023104-cd2ef63ee978
 )
 
 require (
@@ -168,7 +168,6 @@ require (
 	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
 	github.com/andybalholm/cascadia v1.3.2 // indirect
-	github.com/aws/aws-sdk-go v1.53.19 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.2 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.2 // indirect

go.sum

@@ -197,8 +197,6 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd
 github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA=
 github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw=
 github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
-github.com/aws/aws-sdk-go v1.53.19 h1:WEuWc918RXlIaPCyU11F7hH9H1ItK+8m2c/uoQNRUok=
-github.com/aws/aws-sdk-go v1.53.19/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
 github.com/aws/aws-sdk-go-v2 v1.25.2 h1:/uiG1avJRgLGiQM9X3qJM8+Qa6KRGK5rRPuXE0HUM+w=
 github.com/aws/aws-sdk-go-v2 v1.25.2/go.mod h1:Evoc5AsmtveRt1komDwIsjHFyrP5tDuF1D1U+6z6pNo=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 h1:gTK2uhtAPtFcdRRJilZPx8uJLL2J85xK11nKtWL0wfU=
@@ -1364,7 +1362,7 @@ www.velocidex.com/golang/oleparse v0.0.0-20230217092320-383a0121aafe h1:o9jQWSwK
 www.velocidex.com/golang/oleparse v0.0.0-20230217092320-383a0121aafe/go.mod h1:R7IisRzDO7q5LVRJsCQf1xA50LrIavsPWzAjVE4THyY=
 www.velocidex.com/golang/regparser v0.0.0-20240404115756-2169ac0e3c09 h1:G1RWYBXP2lSzxKcrAU1YhiUlBetZ7hGIzIiWuuazvfo=
 www.velocidex.com/golang/regparser v0.0.0-20240404115756-2169ac0e3c09/go.mod h1:pxSECT5mWM3goJ4sxB4HCJNKnKqiAlpyT8XnvBwkLGU=
-www.velocidex.com/golang/vfilter v0.0.0-20240608150317-307fc598a311 h1:IujLDZmLQ/cMraqK2gXcvvbwe6GJqpmCABjMwFXNqI4=
-www.velocidex.com/golang/vfilter v0.0.0-20240608150317-307fc598a311/go.mod h1:P50KPQr2LpWVAu7ilGH8CBLBASGtOJ2971yA9YhR8rY=
+www.velocidex.com/golang/vfilter v0.0.0-20240618023104-cd2ef63ee978 h1:4OsjvVgF4L4B9lzLjHFaxBLk4c6V7IK4uohb4z1qsGI=
+www.velocidex.com/golang/vfilter v0.0.0-20240618023104-cd2ef63ee978/go.mod h1:P50KPQr2LpWVAu7ilGH8CBLBASGtOJ2971yA9YhR8rY=
 www.velocidex.com/golang/vtypes v0.0.0-20240123105603-069d4a7f435c h1:rL/It+Ig+mvIhmy9vl5gg5b6CX2J12x0v2SXIT2RoWE=
 www.velocidex.com/golang/vtypes v0.0.0-20240123105603-069d4a7f435c/go.mod h1:tjaJNlBWbvH4cEMrEu678CFR2hrtcdyPINIpRxrOh4U=

gui/velociraptor/src/components/clients/search.css

@@ -35,6 +35,7 @@
 }
 
 .react-autosuggest__suggestions-container--open {
+  margin-top: 2px;
   display: block;
   max-height: 400px;
   overflow-y: auto;

gui/velociraptor/src/themes/coolgray-dark.css

@@ -241,7 +241,6 @@ body.coolgray-dark {
 }
 
 .coolgray-dark .react-autosuggest__suggestions-container--open {
-  margin-left: 5px;
   background: var(--color-form-control-background);
   font-family: var(--font-family-sans-serif);
   font-size: var(--font-size-base);

gui/velociraptor/src/themes/github-dimmed-dark.css

@@ -219,7 +219,6 @@ body.github-dimmed-dark {
 }
 
 .github-dimmed-dark .react-autosuggest__suggestions-container--open {
-  margin-left: 5px;
   background: var(--color-form-control-background);
   font-family: var(--font-family-sans-serif);
   font-size: var(--font-size-base);

gui/velociraptor/src/themes/midnight.css

@@ -252,7 +252,6 @@ body.midnight {
 }
 
 .midnight .react-autosuggest__suggestions-container--open {
-  margin-left: 5px;
   background: var(--color-form-control-background);
   font-family: var(--font-family-sans-serif);
   font-size: var(--font-size-base);

gui/velociraptor/src/themes/ncurses-dark.css

@@ -346,7 +346,6 @@ body.ncurses-dark {
 }
 
 .ncurses-dark .react-autosuggest__suggestions-container--open {
-  margin-left: 5px;
   background: grey;
   font-family: var(--font-family-sans-serif);
   font-size: var(--font-size-base);

gui/velociraptor/src/themes/ncurses-light.css

@@ -337,7 +337,6 @@ body.ncurses-light {
 }
 
 .ncurses-light .react-autosuggest__suggestions-container--open {
-  margin-left: 5px;
   background: whitesmoke;
   font-family: var(--font-family-sans-serif);
   font-size: var(--font-size-base);

gui/velociraptor/src/themes/pink-light.css

@@ -230,7 +230,6 @@ body.pink-light {
 }
 
 .pink-light .react-autosuggest__suggestions-container--open {
-  margin-left: 5px;
   background: var(--color-form-control-background);
   font-family: var(--font-family-sans-serif);
   font-size: var(--font-size-base);

gui/velociraptor/src/themes/veloci-dark.css

@@ -231,7 +231,6 @@ body.veloci-dark {
 }
 
 .veloci-dark .react-autosuggest__suggestions-container--open {
-  margin-left: 5px;
   background: var(--color-form-control-background);
   font-family: var(--font-family-sans-serif);
   font-size: var(--font-size-base);

gui/velociraptor/src/themes/veloci-light.css

@@ -216,7 +216,6 @@ body.veloci-light {
 }
 
 .veloci-light .react-autosuggest__suggestions-container--open {
-    margin-left: 5px;
     background: var(--color-canvas-background);
     font-family: var(--font-family-sans-serif);
     font-size: var(--font-size-base);

result_sets/simple/transformed.go

@@ -294,20 +294,25 @@ func (self *Stacker) Start(ctx context.Context) {
 	for row := range self.sorted_chan {
 		// Get the value for the sorted column
 		value, pres := self.scope.Associative(row, self.sort_column)
-		if pres {
-			// Flush the current value
-			if !self.scope.Eq(value, self.value) {
-				if self.count > 0 {
-					self.writer.WriteJSONL(
-						[]byte(json.Format(`{"value":%q,"idx":%q,"c":%q}
+
+		// Empty values are treated as an empty string so they can be
+		// grouped into a single group.
+		if !pres || utils.IsNil(value) {
+			value = ""
+		}
+
+		// Flush the current value
+		if !self.scope.Eq(value, self.value) {
+			if self.count > 0 {
+				self.writer.WriteJSONL(
+					[]byte(json.Format(`{"value":%q,"idx":%q,"c":%q}
 `, self.value, self.index, self.count)), 1)
-				}
-				self.count = 0
-				self.value = value
-				self.index = index
 			}
-			self.count++
+			self.count = 0
+			self.value = value
+			self.index = index
 		}
+		self.count++
 
 		select {
 		case <-ctx.Done():

services/hunt_dispatcher/index.go

@@ -55,7 +55,7 @@ func (self *HuntStorageManagerImpl) FlushIndex(
 		now := utils.GetTime().Now()
 
 		logger := logging.GetLogger(self.config_obj, &logging.FrontendComponent)
-		logger.Info("HuntDispatcher: <green>Rebuilt Hunt Index in %v for %v (%v hunts)</>",
+		logger.Debug("HuntDispatcher: <green>Rebuilt Hunt Index in %v for %v (%v hunts)</>",
 			now.Sub(start), services.GetOrgName(self.config_obj), len(hunt_ids))
 	}()
 

services/indexing/search.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/Velocidex/ordereddict"
 	api_proto "www.velocidex.com/golang/velociraptor/api/proto"
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
 	"www.velocidex.com/golang/velociraptor/datastore"
@@ -450,8 +451,11 @@ func (self *Indexer) searchVerbs(ctx context.Context,
 	in *api_proto.SearchClientsRequest,
 	limit uint64) (*api_proto.SearchClientsResponse, error) {
 
+	total := uint64(0)
 	terms := []string{}
-	items := []*api_proto.ApiClient{}
+
+	// Dedup by client id
+	items := ordereddict.NewDict()
 
 	term := strings.ToLower(in.Query)
 	for _, verb := range verbs {
@@ -472,7 +476,10 @@ func (self *Indexer) searchVerbs(ctx context.Context,
 			}, limit)
 		if err == nil {
 			terms = append(terms, res.Names...)
-			items = append(items, res.Items...)
+			for _, i := range res.Items {
+				items.Update(i.ClientId, i)
+			}
+			total += res.Total
 		}
 	}
 
@@ -487,12 +494,23 @@ func (self *Indexer) searchVerbs(ctx context.Context,
 			}, limit)
 		if err == nil {
 			terms = append(terms, res.Names...)
-			items = append(items, res.Items...)
+			for _, i := range res.Items {
+				items.Update(i.ClientId, i)
+			}
+			total += res.Total
 		}
 	}
 
-	return &api_proto.SearchClientsResponse{
-		Names: terms,
-		Items: items,
-	}, nil
+	res := &api_proto.SearchClientsResponse{
+		Names:      terms,
+		Total:      total,
+		SearchTerm: in,
+	}
+
+	for _, k := range items.Keys() {
+		v, _ := items.Get(k)
+		res.Items = append(res.Items, v.(*api_proto.ApiClient))
+	}
+
+	return res, nil
 }

vql/common/yara.go

@@ -468,6 +468,9 @@ func (self *scanReporter) RuleMatching(
 			Meta:     metas,
 			File:     self.file_info,
 			FileName: self.filename,
+
+			// There are no strings so produce an empty String member.
+			String: &YaraHit{},
 		}
 
 		select {