diff --git a/accessors/protocols.go b/accessors/protocols.go index 934e9868fe..b44d20acc3 100644 --- a/accessors/protocols.go +++ b/accessors/protocols.go @@ -191,8 +191,12 @@ func (self _AssociativeOSPath) Associative( second_item := length if t[1] != nil { second_item = *t[1] + if second_item > length { + second_item = length + } } + // Wrap around behavior for negative index. if first_item < 0 { first_item += length } diff --git a/artifacts/definitions/Generic/Detection/Yara/Zip.yaml b/artifacts/definitions/Generic/Detection/Yara/Zip.yaml index e1f0841e3a..40370c3a71 100644 --- a/artifacts/definitions/Generic/Detection/Yara/Zip.yaml +++ b/artifacts/definitions/Generic/Detection/Yara/Zip.yaml @@ -105,11 +105,13 @@ sources: Rule, Tags, Meta, String.Name as YaraString, String.Offset as HitOffset, - upload( accessor='scope', + if(condition=String.Data, + then=upload( + accessor='scope', file='String.Data', name=format(format="%v_%v", args=[ OSPath.HumanString, String.Offset ] - )) as HitContext + ))) as HitContext FROM yara(accessor='zip',files=OSPath,rules=YaraRule, context=ContextBytes, number=NumberOfHits) }) diff --git a/artifacts/definitions/Linux/Detection/Yara/Process.yaml b/artifacts/definitions/Linux/Detection/Yara/Process.yaml index 521df95809..ddedfd43a4 100644 --- a/artifacts/definitions/Linux/Detection/Yara/Process.yaml +++ b/artifacts/definitions/Linux/Detection/Yara/Process.yaml @@ -95,11 +95,13 @@ sources: Meta, String.Name as YaraString, String.Offset as HitOffset, - upload( accessor='scope', - file='String.Data', - name=format(format="%v-%v_%v_%v", - args=[ ProcessName, Pid, String.Offset, ContextBytes ] - )) as HitContext + if(condition=String.Data, + then=upload( + accessor='scope', + file='String.Data', + name=format(format="%v-%v_%v_%v", + args=[ ProcessName, Pid, String.Offset, ContextBytes ] + ))) as HitContext FROM proc_yara( pid=Pid, rules=yara_rules, diff --git a/artifacts/definitions/Windows/Carving/CobaltStrike.yaml b/artifacts/definitions/Windows/Carving/CobaltStrike.yaml index 1868cfa3bb..70ce7d6504 100644 --- a/artifacts/definitions/Windows/Carving/CobaltStrike.yaml +++ b/artifacts/definitions/Windows/Carving/CobaltStrike.yaml @@ -462,7 +462,7 @@ sources: FROM switch( -- switchcase will find beacon as priority, then search for shellcode beacon = { SELECT *, - substr(start=0,end=1,str=String.Data) as Xor, + substr(start=0, end=1, str=String.Data) as Xor, read_file(accessor='data', filename=TargetBytes, offset= String.Offset, diff --git a/artifacts/definitions/Windows/Detection/Yara/NTFS.yaml b/artifacts/definitions/Windows/Detection/Yara/NTFS.yaml index fe6fe5421e..01279712d4 100644 --- a/artifacts/definitions/Windows/Detection/Yara/NTFS.yaml +++ b/artifacts/definitions/Windows/Detection/Yara/NTFS.yaml @@ -25,7 +25,7 @@ description: | Note: no drive and forward slashes - these expressions are for paths relative to the root of the filesystem. - If upload is selected NumberOfHits is redundant and not advised as hits are + If upload is selected NumberOfHits is redundant and not advised as hits are grouped by path to ensure files only downloaded once. type: CLIENT @@ -80,8 +80,8 @@ parameters: description: Include this amount of bytes around hit as context. default: 0 type: int - - + + sources: - precondition: SELECT OS From info() where OS = 'windows' @@ -95,7 +95,7 @@ sources: OSPath, IsDir FROM Artifact.Windows.NTFS.MFT( MFTDrive=DriveLetter, AllDrives=AllDrives, - FileRegex=FileNameRegex,PathRegex=PathRegex, + FileRegex=FileNameRegex,PathRegex=PathRegex, SizeMax=SizeMax, SizeMin=SizeMin) WHERE NOT IsDir AND NOT OSPath =~ '''\\\\.\\.:\\\\''' @@ -122,9 +122,11 @@ sources: Rule, Tags, Meta, String.Name as YaraString, String.Offset as HitOffset, - upload( accessor='scope', - file='String.Data', - name=format(format="%v-%v-%v", + if(condition=String.Data, + then=upload( + accessor='scope', + file='String.Data', + name=format(format="%v-%v-%v", args=[ OSPath, if(condition= String.Offset - ContextBytes < 0, @@ -133,8 +135,9 @@ sources: if(condition= String.Offset + ContextBytes > File.Size, then= File.Size, else= String.Offset + ContextBytes) ] - )) as HitContext - FROM yara(rules=yara_rules, files=OSPath, context=ContextBytes,number=NumberOfHits) + ))) as HitContext + FROM yara(rules=yara_rules, + files=OSPath, context=ContextBytes, number=NumberOfHits) }) -- upload files that have hit diff --git a/artifacts/testdata/server/testcases/yara_detection.in.yaml b/artifacts/testdata/server/testcases/yara_detection.in.yaml index 924fa33c5d..fd6294c7ac 100644 --- a/artifacts/testdata/server/testcases/yara_detection.in.yaml +++ b/artifacts/testdata/server/testcases/yara_detection.in.yaml @@ -88,8 +88,9 @@ Queries: DriveLetter=srcDir, PathRegex='wkscli.dll', ContextBytes=10, YaraRule=MZRule) + # This rule has no strings so HitContext should be NULL - SELECT relpath(path=OSPath, base=srcDir, sep="/") as TestPath, - Size, Rule, CleanContext(HitContext=HitContext), HitOffset + Size, Rule, HitContext, HitOffset FROM Artifact.Windows.Detection.Yara.NTFS( DriveLetter=srcDir, PathRegex='wkscli.dll', YaraRule=IsPE) diff --git a/artifacts/testdata/server/testcases/yara_detection.out.yaml b/artifacts/testdata/server/testcases/yara_detection.out.yaml index c18f64b945..d7473eb2d9 100644 --- a/artifacts/testdata/server/testcases/yara_detection.out.yaml +++ b/artifacts/testdata/server/testcases/yara_detection.out.yaml @@ -80,18 +80,13 @@ FROM scope() }, "HitOffset": 0 } -]SELECT relpath(path=OSPath, base=srcDir, sep="/") as TestPath, Size, Rule, CleanContext(HitContext=HitContext), HitOffset FROM Artifact.Windows.Detection.Yara.NTFS( DriveLetter=srcDir, PathRegex='wkscli.dll', YaraRule=IsPE)[ +]SELECT relpath(path=OSPath, base=srcDir, sep="/") as TestPath, Size, Rule, HitContext, HitOffset FROM Artifact.Windows.Detection.Yara.NTFS( DriveLetter=srcDir, PathRegex='wkscli.dll', YaraRule=IsPE)[ { "TestPath": "artifacts/testdata/files/wkscli.dll", "Size": 9728, "Rule": "IsPE", - "CleanContext(HitContext=HitContext)": { - "StoredSize": 4, - "Path": "String.Data", - "Size": 4, - "sha256": "fa1d2db62d4d952e2031452e1bc1ddcad0b192c2e29a706f11ce426ae5acddea" - }, - "HitOffset": null + "HitContext": null, + "HitOffset": 0 } ]SELECT relpath(path=OSPath, base=srcDir, sep="/") as TestPath,Size, Rule,Mtime,Atime,Ctime,Btime FROM Artifact.Generic.Detection.Yara.Glob(YaraRule=IsPE)[ { diff --git a/go.mod b/go.mod index 6c4a4a1e21..69cc116184 100644 --- a/go.mod +++ b/go.mod @@ -99,7 +99,7 @@ require ( www.velocidex.com/golang/go-prefetch v0.0.0-20220801101854-338dbe61982a www.velocidex.com/golang/oleparse v0.0.0-20230217092320-383a0121aafe www.velocidex.com/golang/regparser v0.0.0-20240404115756-2169ac0e3c09 - www.velocidex.com/golang/vfilter v0.0.0-20240608150317-307fc598a311 + www.velocidex.com/golang/vfilter v0.0.0-20240618023104-cd2ef63ee978 ) require ( @@ -168,7 +168,6 @@ require ( github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect - github.com/aws/aws-sdk-go v1.53.19 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.2 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.2 // indirect diff --git a/go.sum b/go.sum index 6a7d925a48..8fe4779eda 100644 --- a/go.sum +++ b/go.sum @@ -197,8 +197,6 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA= github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw= github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= -github.com/aws/aws-sdk-go v1.53.19 h1:WEuWc918RXlIaPCyU11F7hH9H1ItK+8m2c/uoQNRUok= -github.com/aws/aws-sdk-go v1.53.19/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/aws/aws-sdk-go-v2 v1.25.2 h1:/uiG1avJRgLGiQM9X3qJM8+Qa6KRGK5rRPuXE0HUM+w= github.com/aws/aws-sdk-go-v2 v1.25.2/go.mod h1:Evoc5AsmtveRt1komDwIsjHFyrP5tDuF1D1U+6z6pNo= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 h1:gTK2uhtAPtFcdRRJilZPx8uJLL2J85xK11nKtWL0wfU= @@ -1364,7 +1362,7 @@ www.velocidex.com/golang/oleparse v0.0.0-20230217092320-383a0121aafe h1:o9jQWSwK www.velocidex.com/golang/oleparse v0.0.0-20230217092320-383a0121aafe/go.mod h1:R7IisRzDO7q5LVRJsCQf1xA50LrIavsPWzAjVE4THyY= www.velocidex.com/golang/regparser v0.0.0-20240404115756-2169ac0e3c09 h1:G1RWYBXP2lSzxKcrAU1YhiUlBetZ7hGIzIiWuuazvfo= www.velocidex.com/golang/regparser v0.0.0-20240404115756-2169ac0e3c09/go.mod h1:pxSECT5mWM3goJ4sxB4HCJNKnKqiAlpyT8XnvBwkLGU= -www.velocidex.com/golang/vfilter v0.0.0-20240608150317-307fc598a311 h1:IujLDZmLQ/cMraqK2gXcvvbwe6GJqpmCABjMwFXNqI4= -www.velocidex.com/golang/vfilter v0.0.0-20240608150317-307fc598a311/go.mod h1:P50KPQr2LpWVAu7ilGH8CBLBASGtOJ2971yA9YhR8rY= +www.velocidex.com/golang/vfilter v0.0.0-20240618023104-cd2ef63ee978 h1:4OsjvVgF4L4B9lzLjHFaxBLk4c6V7IK4uohb4z1qsGI= +www.velocidex.com/golang/vfilter v0.0.0-20240618023104-cd2ef63ee978/go.mod h1:P50KPQr2LpWVAu7ilGH8CBLBASGtOJ2971yA9YhR8rY= www.velocidex.com/golang/vtypes v0.0.0-20240123105603-069d4a7f435c h1:rL/It+Ig+mvIhmy9vl5gg5b6CX2J12x0v2SXIT2RoWE= www.velocidex.com/golang/vtypes v0.0.0-20240123105603-069d4a7f435c/go.mod h1:tjaJNlBWbvH4cEMrEu678CFR2hrtcdyPINIpRxrOh4U= diff --git a/gui/velociraptor/src/components/clients/search.css b/gui/velociraptor/src/components/clients/search.css index b35e2c140b..d044bdfd53 100644 --- a/gui/velociraptor/src/components/clients/search.css +++ b/gui/velociraptor/src/components/clients/search.css @@ -35,6 +35,7 @@ } .react-autosuggest__suggestions-container--open { + margin-top: 2px; display: block; max-height: 400px; overflow-y: auto; diff --git a/gui/velociraptor/src/themes/coolgray-dark.css b/gui/velociraptor/src/themes/coolgray-dark.css index 5ee01de76d..1538e28a3e 100644 --- a/gui/velociraptor/src/themes/coolgray-dark.css +++ b/gui/velociraptor/src/themes/coolgray-dark.css @@ -241,7 +241,6 @@ body.coolgray-dark { } .coolgray-dark .react-autosuggest__suggestions-container--open { - margin-left: 5px; background: var(--color-form-control-background); font-family: var(--font-family-sans-serif); font-size: var(--font-size-base); diff --git a/gui/velociraptor/src/themes/github-dimmed-dark.css b/gui/velociraptor/src/themes/github-dimmed-dark.css index 26d1b0bca6..4d5123abc3 100644 --- a/gui/velociraptor/src/themes/github-dimmed-dark.css +++ b/gui/velociraptor/src/themes/github-dimmed-dark.css @@ -219,7 +219,6 @@ body.github-dimmed-dark { } .github-dimmed-dark .react-autosuggest__suggestions-container--open { - margin-left: 5px; background: var(--color-form-control-background); font-family: var(--font-family-sans-serif); font-size: var(--font-size-base); diff --git a/gui/velociraptor/src/themes/midnight.css b/gui/velociraptor/src/themes/midnight.css index 167e0a2159..a24b5c4521 100644 --- a/gui/velociraptor/src/themes/midnight.css +++ b/gui/velociraptor/src/themes/midnight.css @@ -252,7 +252,6 @@ body.midnight { } .midnight .react-autosuggest__suggestions-container--open { - margin-left: 5px; background: var(--color-form-control-background); font-family: var(--font-family-sans-serif); font-size: var(--font-size-base); diff --git a/gui/velociraptor/src/themes/ncurses-dark.css b/gui/velociraptor/src/themes/ncurses-dark.css index c7a279c658..82c367d3d9 100644 --- a/gui/velociraptor/src/themes/ncurses-dark.css +++ b/gui/velociraptor/src/themes/ncurses-dark.css @@ -346,7 +346,6 @@ body.ncurses-dark { } .ncurses-dark .react-autosuggest__suggestions-container--open { - margin-left: 5px; background: grey; font-family: var(--font-family-sans-serif); font-size: var(--font-size-base); diff --git a/gui/velociraptor/src/themes/ncurses-light.css b/gui/velociraptor/src/themes/ncurses-light.css index 970b60a08d..995cbf72b6 100644 --- a/gui/velociraptor/src/themes/ncurses-light.css +++ b/gui/velociraptor/src/themes/ncurses-light.css @@ -337,7 +337,6 @@ body.ncurses-light { } .ncurses-light .react-autosuggest__suggestions-container--open { - margin-left: 5px; background: whitesmoke; font-family: var(--font-family-sans-serif); font-size: var(--font-size-base); diff --git a/gui/velociraptor/src/themes/pink-light.css b/gui/velociraptor/src/themes/pink-light.css index 8041a7b21b..b7fb46164d 100644 --- a/gui/velociraptor/src/themes/pink-light.css +++ b/gui/velociraptor/src/themes/pink-light.css @@ -230,7 +230,6 @@ body.pink-light { } .pink-light .react-autosuggest__suggestions-container--open { - margin-left: 5px; background: var(--color-form-control-background); font-family: var(--font-family-sans-serif); font-size: var(--font-size-base); diff --git a/gui/velociraptor/src/themes/veloci-dark.css b/gui/velociraptor/src/themes/veloci-dark.css index a67e4e1b4f..92c46e5a8e 100644 --- a/gui/velociraptor/src/themes/veloci-dark.css +++ b/gui/velociraptor/src/themes/veloci-dark.css @@ -231,7 +231,6 @@ body.veloci-dark { } .veloci-dark .react-autosuggest__suggestions-container--open { - margin-left: 5px; background: var(--color-form-control-background); font-family: var(--font-family-sans-serif); font-size: var(--font-size-base); diff --git a/gui/velociraptor/src/themes/veloci-light.css b/gui/velociraptor/src/themes/veloci-light.css index cbd4c82350..eda71cdcdd 100644 --- a/gui/velociraptor/src/themes/veloci-light.css +++ b/gui/velociraptor/src/themes/veloci-light.css @@ -216,7 +216,6 @@ body.veloci-light { } .veloci-light .react-autosuggest__suggestions-container--open { - margin-left: 5px; background: var(--color-canvas-background); font-family: var(--font-family-sans-serif); font-size: var(--font-size-base); diff --git a/result_sets/simple/transformed.go b/result_sets/simple/transformed.go index 83bb4d13d5..b1250f77c1 100644 --- a/result_sets/simple/transformed.go +++ b/result_sets/simple/transformed.go @@ -294,20 +294,25 @@ func (self *Stacker) Start(ctx context.Context) { for row := range self.sorted_chan { // Get the value for the sorted column value, pres := self.scope.Associative(row, self.sort_column) - if pres { - // Flush the current value - if !self.scope.Eq(value, self.value) { - if self.count > 0 { - self.writer.WriteJSONL( - []byte(json.Format(`{"value":%q,"idx":%q,"c":%q} + + // Empty values are treated as an empty string so they can be + // grouped into a single group. + if !pres || utils.IsNil(value) { + value = "" + } + + // Flush the current value + if !self.scope.Eq(value, self.value) { + if self.count > 0 { + self.writer.WriteJSONL( + []byte(json.Format(`{"value":%q,"idx":%q,"c":%q} `, self.value, self.index, self.count)), 1) - } - self.count = 0 - self.value = value - self.index = index } - self.count++ + self.count = 0 + self.value = value + self.index = index } + self.count++ select { case <-ctx.Done(): diff --git a/services/hunt_dispatcher/index.go b/services/hunt_dispatcher/index.go index 322f365db4..423c337363 100644 --- a/services/hunt_dispatcher/index.go +++ b/services/hunt_dispatcher/index.go @@ -55,7 +55,7 @@ func (self *HuntStorageManagerImpl) FlushIndex( now := utils.GetTime().Now() logger := logging.GetLogger(self.config_obj, &logging.FrontendComponent) - logger.Info("HuntDispatcher: Rebuilt Hunt Index in %v for %v (%v hunts)", + logger.Debug("HuntDispatcher: Rebuilt Hunt Index in %v for %v (%v hunts)", now.Sub(start), services.GetOrgName(self.config_obj), len(hunt_ids)) }() diff --git a/services/indexing/search.go b/services/indexing/search.go index cc786fde30..0fb6416d50 100644 --- a/services/indexing/search.go +++ b/services/indexing/search.go @@ -9,6 +9,7 @@ import ( "strings" "time" + "github.com/Velocidex/ordereddict" api_proto "www.velocidex.com/golang/velociraptor/api/proto" config_proto "www.velocidex.com/golang/velociraptor/config/proto" "www.velocidex.com/golang/velociraptor/datastore" @@ -450,8 +451,11 @@ func (self *Indexer) searchVerbs(ctx context.Context, in *api_proto.SearchClientsRequest, limit uint64) (*api_proto.SearchClientsResponse, error) { + total := uint64(0) terms := []string{} - items := []*api_proto.ApiClient{} + + // Dedup by client id + items := ordereddict.NewDict() term := strings.ToLower(in.Query) for _, verb := range verbs { @@ -472,7 +476,10 @@ func (self *Indexer) searchVerbs(ctx context.Context, }, limit) if err == nil { terms = append(terms, res.Names...) - items = append(items, res.Items...) + for _, i := range res.Items { + items.Update(i.ClientId, i) + } + total += res.Total } } @@ -487,12 +494,23 @@ func (self *Indexer) searchVerbs(ctx context.Context, }, limit) if err == nil { terms = append(terms, res.Names...) - items = append(items, res.Items...) + for _, i := range res.Items { + items.Update(i.ClientId, i) + } + total += res.Total } } - return &api_proto.SearchClientsResponse{ - Names: terms, - Items: items, - }, nil + res := &api_proto.SearchClientsResponse{ + Names: terms, + Total: total, + SearchTerm: in, + } + + for _, k := range items.Keys() { + v, _ := items.Get(k) + res.Items = append(res.Items, v.(*api_proto.ApiClient)) + } + + return res, nil } diff --git a/vql/common/yara.go b/vql/common/yara.go index aa007e18b0..b682ebabd4 100644 --- a/vql/common/yara.go +++ b/vql/common/yara.go @@ -468,6 +468,9 @@ func (self *scanReporter) RuleMatching( Meta: metas, File: self.file_info, FileName: self.filename, + + // There are no strings so produce an empty String member. + String: &YaraHit{}, } select { diff --git a/vql/sorter/mergesort.go b/vql/sorter/mergesort.go index 995f6b7598..f5294e020e 100644 --- a/vql/sorter/mergesort.go +++ b/vql/sorter/mergesort.go @@ -111,21 +111,12 @@ func (self *MergeSorterCtx) Feed(row types.Row) { Desc: memory_sorter.Desc, } - // Do this in parallel. - self.wg.Add(1) - go func() { - defer self.wg.Done() + new_data_file := newDataFile( + memory_sorter.Scope, + memory_sorter.Items, + memory_sorter.OrderBy) - self.mu.Lock() - defer self.mu.Unlock() - - new_data_file := newDataFile( - memory_sorter.Scope, - memory_sorter.Items, - memory_sorter.OrderBy) - - self.addProvider(new_data_file) - }() + self.addProvider(new_data_file) } } @@ -193,6 +184,11 @@ func (self *MergeSorterCtx) Merge(ctx context.Context, output_chan chan types.Ro self.memory_sorter.OrderBy) } + // Treat NULL as a string so they sort properly. + if utils.IsNil(value) { + value = "" + } + // smallest_value is not set yet. if utils.IsNil(smallest_value) { smallest_value = value @@ -256,6 +252,13 @@ type dataFile struct { } func (self *dataFile) Close() { + self.mu.Lock() + defer self.mu.Unlock() + + self._Close() +} + +func (self *dataFile) _Close() { if self.fd != nil { self.fd.Close() os.Remove(self.fd.Name()) @@ -265,12 +268,22 @@ func (self *dataFile) Close() { } func (self *dataFile) Last() types.Row { + self.mu.Lock() + defer self.mu.Unlock() + return self.lastValue } // Called when the current value is consumed - read the next row from // the file and sets the current value. func (self *dataFile) Consume() { + self.mu.Lock() + defer self.mu.Unlock() + + self._Consume() +} + +func (self *dataFile) _Consume() { if self.reader == nil { return } @@ -279,7 +292,7 @@ func (self *dataFile) Consume() { if err != nil { // File is exhausted, close it and reset. self.lastValue = nil - self.Close() + self._Close() return } @@ -287,51 +300,63 @@ func (self *dataFile) Consume() { err = item.UnmarshalJSON(row_data) if err != nil { self.lastValue = nil - self.Close() + self._Close() return } self.lastValue = item } -func newDataFile(scope types.Scope, items []types.Row, key string) *dataFile { - result := &dataFile{ - scope: scope, - key: key, - } +// Initialize the file by writing it to storage. Writing is done in the background. +func (self *dataFile) prepareFile(scope vfilter.Scope, items []vfilter.Row) { - tmpfile, err := ioutil.TempFile("", "vql") - if err != nil { - scope.Log("Unable to create tempfile: %v", err) - return result - } + // We hold the lock for the duration of writing the file until we + // are ready. + self.mu.Lock() + go func() { + defer self.mu.Unlock() - // Serialize all the rows into the file. - serialized, err := json.MarshalJsonl(items) - if err != nil { - scope.Log("Unable to serialize: %v", err) - return result - } - _, err = tmpfile.Write(serialized) - if err != nil { - scope.Log("Unable to serialize: %v", err) - return result - } - tmpfile.Close() + tmpfile, err := ioutil.TempFile("", "vql") + if err != nil { + scope.Log("Unable to create tempfile: %v", err) + return + } - // Reopen the file for reading. - fd, err := os.Open(tmpfile.Name()) - if err != nil { - scope.Log("Unable to open file: %v", err) - return result - } + // Serialize all the rows into the file. + serialized, err := json.MarshalJsonl(items) + if err != nil { + scope.Log("Unable to serialize: %v", err) + return + } + _, err = tmpfile.Write(serialized) + if err != nil { + scope.Log("Unable to serialize: %v", err) + return + } + tmpfile.Close() + + // Reopen the file for reading. + fd, err := os.Open(tmpfile.Name()) + if err != nil { + scope.Log("Unable to open file: %v", err) + return + } + + self.fd = fd + self.reader = bufio.NewReader(fd) - result.fd = fd - result.reader = bufio.NewReader(fd) + self._Consume() + }() +} - result.Consume() +func newDataFile(scope types.Scope, items []types.Row, key string) *dataFile { + self := &dataFile{ + scope: scope, + key: key, + } - return result + self.prepareFile(scope, items) + return self } // A provider for in memory rows diff --git a/vql/sorter/mergesort_test.go b/vql/sorter/mergesort_test.go index 3cfb8334dc..00f7a87fa4 100644 --- a/vql/sorter/mergesort_test.go +++ b/vql/sorter/mergesort_test.go @@ -27,6 +27,9 @@ func TestDataFile(t *testing.T) { data_file := newDataFile(scope, rows, "X") defer data_file.Close() + // Wait here until its ready + data_file.Last() + // Check the content of the backing file. fd, err := os.Open(data_file.fd.Name()) defer fd.Close()