diff --git a/api/api.go b/api/api.go index 35a2729fe6..ad175a09af 100644 --- a/api/api.go +++ b/api/api.go @@ -36,10 +36,9 @@ type VersionResponse struct { } type PoolPostRequest struct { - Name string `json:"name"` - SortKeys SortKeys `json:"layout"` - SeekStride int `json:"seek_stride"` - Thresh int64 `json:"thresh"` + Name string `json:"name"` + SortKeys SortKeys `json:"layout"` + Thresh int64 `json:"thresh"` } type SortKeys struct { diff --git a/book/src/command/db.md b/book/src/command/db.md index fdea3778be..e958ef5dc2 100644 --- a/book/src/command/db.md +++ b/book/src/command/db.md @@ -151,9 +151,6 @@ We can narrow the span of the query by specifying a filter on the database ``` super db -c 'from logs | ts >= 2018-03-24T17:36:30.090766Z and ts <= 2018-03-24T17:36:30.090758Z' ``` -Filters on sort keys are efficiently implemented as the data is laid out -according to the sort key and seek indexes keyed by the sort key -are computed for each data object. When querying data to the [BSUP](../formats/bsup.md) output format, output from a pool can be easily piped to other commands like `super`, e.g., diff --git a/book/src/database/api.md b/book/src/database/api.md index 765d3ccf90..e9f0166155 100644 --- a/book/src/database/api.md +++ b/book/src/database/api.md @@ -25,7 +25,6 @@ POST /pool | name | string | body | **Required.** Name of the pool. Must be unique to lake. | | layout.order | string | body | Order of storage by primary key(s) in pool. Possible values: desc, asc. Default: asc. | | layout.keys | [[string]] | body | Primary key(s) of pool. The element of each inner string array should reflect the hierarchical ordering of named fields within indexed records. Default: [[ts]]. | -| thresh | int | body | The size in bytes of each seek index. | | Content-Type | string | header | [MIME type](#mime-types) of the request payload. | | Accept | string | header | Preferred [MIME type](#mime-types) of the response. | diff --git a/book/src/database/format.md b/book/src/database/format.md index 53963495aa..ee28d4998d 100644 --- a/book/src/database/format.md +++ b/book/src/database/format.md @@ -77,7 +77,7 @@ resulting object is immutable, there is no possible write concurrency to manage with respect to a given object. A data object is composed of the primary data object stored as one or two objects -(for sequence and/or vector layout) and an optional seek index. +(for sequence and/or vector layout). Data objects may be either in sequence form (i.e., BSUP) or vector form (i.e., CSUP), or both forms may be present as a query optimizer may choose to use whatever @@ -91,19 +91,9 @@ Immutable objects are named as follows: |-----------|----| |vector data|`/data/.csup`| |sequence data|`/data/.bsup`| -|sequence seek index|`/data/-seek.bsup`| `` is the KSUID of the data object. -The seek index maps pool key values to seek offsets in the BSUP file thereby -allowing a scan to do a byte-range retrieval of the BSUP object when -processing only a subset of data. - ->[!NOTE] -> The CSUP format allows individual vector segments to be read in isolation -> and the in-memory CSUP representation supports random access so there is -> no need to have a seek index for the vector object. - #### Commit History A branch's commit history is the definitive record of the evolution of data in diff --git a/cmd/super/db/create/command.go b/cmd/super/db/create/command.go index fced217cee..d51d0f9281 100644 --- a/cmd/super/db/create/command.go +++ b/cmd/super/db/create/command.go @@ -20,16 +20,14 @@ var spec = &charm.Spec{ Long: ` See https://superdb.org/command/db.html#super-db-create `, - HiddenFlags: "seekstride", - New: New, + New: New, } type Command struct { *db.Command - sortKey string - thresh units.Bytes - seekStride units.Bytes - use bool + sortKey string + thresh units.Bytes + use bool } func init() { @@ -37,11 +35,7 @@ func init() { } func New(parent charm.Command, f *flag.FlagSet) (charm.Command, error) { - c := &Command{ - Command: parent.(*db.Command), - seekStride: units.Bytes(data.DefaultSeekStride), - } - f.Var(&c.seekStride, "seekstride", "size of seek-index unit for BSUP data, as '32KB', '1MB', etc.") + c := &Command{Command: parent.(*db.Command)} c.thresh = data.DefaultThreshold f.Var(&c.thresh, "S", "target size of pool data objects, as '10MB' or '4GiB', etc.") f.BoolVar(&c.use, "use", false, "set created pool as the current pool") @@ -67,7 +61,7 @@ func (c *Command) Run(args []string) error { return err } poolName := args[0] - id, err := db.CreatePool(ctx, poolName, sortKey, int(c.seekStride), int64(c.thresh)) + id, err := db.CreatePool(ctx, poolName, sortKey, int64(c.thresh)) if err != nil { return err } diff --git a/db/api/api.go b/db/api/api.go index f7d2025064..952f91a91c 100644 --- a/db/api/api.go +++ b/db/api/api.go @@ -30,7 +30,7 @@ type Interface interface { Query(ctx context.Context, query []srcfiles.Input) (vio.Scanner, error) PoolID(ctx context.Context, poolName string) (ksuid.KSUID, error) CommitObject(ctx context.Context, poolID ksuid.KSUID, branchName string) (ksuid.KSUID, error) - CreatePool(context.Context, string, order.SortKeys, int, int64) (ksuid.KSUID, error) + CreatePool(context.Context, string, order.SortKeys, int64) (ksuid.KSUID, error) RemovePool(context.Context, ksuid.KSUID) error RenamePool(context.Context, ksuid.KSUID, string) error CreateBranch(ctx context.Context, pool ksuid.KSUID, name string, parent ksuid.KSUID) error diff --git a/db/api/local.go b/db/api/local.go index bc193cd9c8..3d60ec0b68 100644 --- a/db/api/local.go +++ b/db/api/local.go @@ -63,11 +63,11 @@ func (l *local) Root() *db.Root { return l.db } -func (l *local) CreatePool(ctx context.Context, name string, sortKeys order.SortKeys, seekStride int, thresh int64) (ksuid.KSUID, error) { +func (l *local) CreatePool(ctx context.Context, name string, sortKeys order.SortKeys, thresh int64) (ksuid.KSUID, error) { if name == "" { return ksuid.Nil, errors.New("no pool name provided") } - pool, err := l.db.CreatePool(ctx, name, sortKeys, seekStride, thresh) + pool, err := l.db.CreatePool(ctx, name, sortKeys, thresh) if err != nil { return ksuid.Nil, err } diff --git a/db/api/remote.go b/db/api/remote.go index 8b0bb4dead..af4fe9f0e9 100644 --- a/db/api/remote.go +++ b/db/api/remote.go @@ -53,15 +53,14 @@ func (r *remote) CommitObject(ctx context.Context, poolID ksuid.KSUID, branchNam return res.Commit, err } -func (r *remote) CreatePool(ctx context.Context, name string, sortKeys order.SortKeys, seekStride int, thresh int64) (ksuid.KSUID, error) { +func (r *remote) CreatePool(ctx context.Context, name string, sortKeys order.SortKeys, thresh int64) (ksuid.KSUID, error) { res, err := r.conn.CreatePool(ctx, api.PoolPostRequest{ Name: name, SortKeys: api.SortKeys{ Order: sortKeys.Primary().Order, Keys: field.List{sortKeys.Primary().Key}, }, - SeekStride: seekStride, - Thresh: thresh, + Thresh: thresh, }) if err != nil { return ksuid.Nil, err diff --git a/db/data/object.go b/db/data/object.go index f7d783e9cd..b7db78ca07 100644 --- a/db/data/object.go +++ b/db/data/object.go @@ -15,8 +15,7 @@ import ( ) const ( - DefaultSeekStride = 64 * 1024 - DefaultThreshold = 500 * 1024 * 1024 + DefaultThreshold = 500 * 1024 * 1024 ) // A FileKind is the first part of a file name, used to differentiate files @@ -27,7 +26,6 @@ const ( FileKindUnknown FileKind = "" FileKindData FileKind = "data" FileKindMetadata FileKind = "meta" - FileKindSeek FileKind = "seek" ) func (k FileKind) Description() string { @@ -36,18 +34,11 @@ func (k FileKind) Description() string { return "data" case FileKindMetadata: return "metadata" - case "FileKindSeek": - return "seekindex" default: return "unknown" } } -//XXX all file types are cacheable but seek index etc is not matched here. -//and seekindexes really do want to be cached as they are small and -// eliminate round-trips, especially when you are ready sub-ranges of -// cached data files! - var fileRegex = regexp.MustCompile(`([0-9A-Za-z]{27}-(data|meta)).bsup$`) // XXX this won't work right until we integrate segID @@ -125,14 +116,6 @@ func SequenceURI(path *storage.URI, id ksuid.KSUID) *storage.URI { return path.JoinPath(fmt.Sprintf("%s.bsup", id)) } -func (o Object) SeekIndexURI(path *storage.URI) *storage.URI { - return SeekIndexURI(path, o.ID) -} - -func SeekIndexURI(path *storage.URI, id ksuid.KSUID) *storage.URI { - return path.JoinPath(fmt.Sprintf("%s-seek.bsup", id)) -} - func (o Object) VectorURI(path *storage.URI) *storage.URI { return VectorURI(path, o.ID) } @@ -141,7 +124,7 @@ func VectorURI(path *storage.URI, id ksuid.KSUID) *storage.URI { return path.JoinPath(fmt.Sprintf("%s.csup", id)) } -// Remove deletes the row object and its seek index. +// Remove deletes the object. // Any 'not found' errors are ignored. func (o Object) Remove(ctx context.Context, engine storage.Engine, path *storage.URI) error { if err := engine.DeleteByPrefix(ctx, o.ObjectPrefix(path)); err != nil && !errors.Is(err, fs.ErrNotExist) { diff --git a/db/data/reader.go b/db/data/reader.go index beaf258dd6..e490daa590 100644 --- a/db/data/reader.go +++ b/db/data/reader.go @@ -5,7 +5,6 @@ import ( "fmt" "io" - "github.com/brimdata/super/db/seekindex" "github.com/brimdata/super/pkg/storage" ) @@ -19,29 +18,16 @@ type Reader struct { // NewReader returns a Reader for this data object. If the object has a seek index // and if the provided span skips part of the object, the seek index will be used to // limit the reading window of the returned reader. -func (o *Object) NewReader(ctx context.Context, engine storage.Engine, path *storage.URI, ranges []seekindex.Range) (*Reader, error) { +func (o *Object) NewReader(ctx context.Context, engine storage.Engine, path *storage.URI) (*Reader, error) { objectPath := o.SequenceURI(path) reader, err := engine.Get(ctx, objectPath) if err != nil { return nil, fmt.Errorf("%s: %w", objectPath, err) } - var r io.Reader - var readBytes int64 - if len(ranges) == 0 { - r = reader - readBytes = o.Size - } else { - readers := make([]io.Reader, 0, len(ranges)) - for _, rg := range ranges { - readers = append(readers, io.NewSectionReader(reader, rg.Offset, rg.Length)) - readBytes += rg.Length - } - r = io.MultiReader(readers...) - } return &Reader{ - Reader: r, + Reader: reader, Closer: reader, TotalBytes: o.Size, - ReadBytes: readBytes, + ReadBytes: o.Size, }, nil } diff --git a/db/data/seekindex.go b/db/data/seekindex.go deleted file mode 100644 index 57f254ab1b..0000000000 --- a/db/data/seekindex.go +++ /dev/null @@ -1,77 +0,0 @@ -package data - -import ( - "context" - "fmt" - - "github.com/brimdata/super" - "github.com/brimdata/super/db/seekindex" - "github.com/brimdata/super/pkg/storage" - "github.com/brimdata/super/runtime/sam/expr" - "github.com/brimdata/super/sio/bsupio" - "github.com/brimdata/super/sup" - "github.com/brimdata/super/vector" -) - -func LookupSeekRange(ctx context.Context, engine storage.Engine, path *storage.URI, - obj *Object, pruner expr.Evaluator) ([]seekindex.Range, error) { - if pruner == nil { - // scan whole object - return nil, nil - } - r, err := engine.Get(ctx, obj.SeekIndexURI(path)) - if err != nil { - return nil, err - } - defer r.Close() - var ranges seekindex.Ranges - unmarshaler := sup.NewBSUPUnmarshaler() - reader := bsupio.NewReader(super.NewContext(), r) - defer reader.Close() - for { - val, err := reader.Read() - if val == nil || err != nil { - return ranges, err - } - result := pruner.Eval(*val) - if result.Type() == super.TypeBool && result.Bool() { - continue - } - var entry seekindex.Entry - if err := unmarshaler.Unmarshal(*val, &entry); err != nil { - return nil, fmt.Errorf("corrupt seek index entry for %q at value: %q (%w)", obj.ID.String(), sup.String(val), err) - } - ranges.Append(entry) - } -} - -func RangeFromBitVector(ctx context.Context, engine storage.Engine, path *storage.URI, - o *Object, b *vector.Bool) ([]seekindex.Range, error) { - index, err := readSeekIndex(ctx, engine, path, o) - if err != nil { - return nil, err - } - return index.Filter(b), nil -} - -func readSeekIndex(ctx context.Context, engine storage.Engine, path *storage.URI, o *Object) (seekindex.Index, error) { - r, err := engine.Get(ctx, o.SeekIndexURI(path)) - if err != nil { - return nil, err - } - defer r.Close() - zr := bsupio.NewReader(super.NewContext(), r) - u := sup.NewBSUPUnmarshaler() - var index seekindex.Index - for { - val, err := zr.Read() - if val == nil { - return index, err - } - var entry seekindex.Entry - if err := u.Unmarshal(*val, &entry); err != nil { - return nil, err - } - index = append(index, entry) - } -} diff --git a/db/data/writer.go b/db/data/writer.go index 5b4aa5f87d..d8c40db85e 100644 --- a/db/data/writer.go +++ b/db/data/writer.go @@ -5,7 +5,6 @@ import ( "io" "github.com/brimdata/super" - "github.com/brimdata/super/db/seekindex" "github.com/brimdata/super/order" "github.com/brimdata/super/pkg/bufwriter" "github.com/brimdata/super/pkg/storage" @@ -15,46 +14,30 @@ import ( // Writer is a sio.Writer that writes a stream of sorted records into a // data object. type Writer struct { - object *Object - byteCounter *writeCounter - count uint64 - writer *bsupio.Writer - sortKey order.SortKey - seekIndex *seekindex.Writer - seekIndexStride int - seekIndexTrigger int - first bool - seekMin *super.Value + object *Object + byteCounter *writeCounter + count uint64 + writer *bsupio.Writer + sortKey order.SortKey + first bool } -// NewWriter returns a writer for writing the data of a BSUP object as -// well as optionally creating a seek index for the row object when the -// seekIndexStride is non-zero. We assume all records are non-volatile until -// Close as super.Values from the various record bodies are referenced across +// NewWriter returns a writer for writing the data of a BSUP object. We assume all records are +// non-volatile until Close as super.Values from the various record bodies are referenced across // calls to Write. -func (o *Object) NewWriter(ctx context.Context, engine storage.Engine, path *storage.URI, sortKey order.SortKey, seekIndexStride int) (*Writer, error) { +func (o *Object) NewWriter(ctx context.Context, engine storage.Engine, path *storage.URI, sortKey order.SortKey) (*Writer, error) { out, err := engine.Put(ctx, o.SequenceURI(path)) if err != nil { return nil, err } counter := &writeCounter{bufwriter.New(out), 0} - w := &Writer{ + return &Writer{ object: o, byteCounter: counter, writer: bsupio.NewWriter(counter), sortKey: sortKey, first: true, - } - if seekIndexStride == 0 { - seekIndexStride = DefaultSeekStride - } - w.seekIndexStride = seekIndexStride - seekOut, err := engine.Put(ctx, o.SeekIndexURI(path)) - if err != nil { - return nil, err - } - w.seekIndex = seekindex.NewWriter(bsupio.NewWriter(bufwriter.New(seekOut))) - return w, nil + }, nil } func (w *Writer) Write(val super.Value) error { @@ -67,39 +50,11 @@ func (w *Writer) WriteWithKey(key, val super.Value) error { if err := w.writer.Write(val); err != nil { return err } - w.object.Max.CopyFrom(key) - return w.writeIndex(key) -} - -func (w *Writer) writeIndex(key super.Value) error { - w.seekIndexTrigger += len(key.Bytes()) if w.first { w.first = false w.object.Min.CopyFrom(key) } - if w.seekMin == nil { - w.seekMin = key.Copy().Ptr() - } - if w.seekIndexTrigger < w.seekIndexStride { - return nil - } - if err := w.writer.EndStream(); err != nil { - return err - } - return w.flushSeekIndex() -} - -func (w *Writer) flushSeekIndex() error { - if w.seekMin != nil { - w.seekIndexTrigger = 0 - min := *w.seekMin - max := w.object.Max - if w.sortKey.Order == order.Desc { - min, max = max, min - } - w.seekMin = nil - return w.seekIndex.Write(min, max, w.count, uint64(w.writer.Position())) - } + w.object.Max.CopyFrom(key) return nil } @@ -107,7 +62,6 @@ func (w *Writer) flushSeekIndex() error { // because the write error will be more informative and should be returned. func (w *Writer) Abort() { w.writer.Close() - w.seekIndex.Close() } func (w *Writer) Close(ctx context.Context) error { @@ -115,14 +69,6 @@ func (w *Writer) Close(ctx context.Context) error { w.Abort() return err } - if err := w.flushSeekIndex(); err != nil { - w.Abort() - return err - } - if err := w.seekIndex.Close(); err != nil { - w.Abort() - return err - } w.object.Count = w.count w.object.Size = w.writer.Position() if w.sortKey.Order == order.Desc { diff --git a/db/data/writer_test.go b/db/data/writer_test.go index e915ed4bef..ad79c1445c 100644 --- a/db/data/writer_test.go +++ b/db/data/writer_test.go @@ -20,7 +20,7 @@ func TestDataReaderWriterVector(t *testing.T) { tmp := storage.MustParseURI(t.TempDir()) object := data.NewObject() ctx := t.Context() - w, err := object.NewWriter(ctx, engine, tmp, order.NewSortKey(order.Asc, field.Path{"a"}), 1000) + w, err := object.NewWriter(ctx, engine, tmp, order.NewSortKey(order.Asc, field.Path{"a"})) require.NoError(t, err) sctx := super.NewContext() require.NoError(t, w.Write(sup.MustParseValue(sctx, "{a:1,b:4}"))) diff --git a/db/pool.go b/db/pool.go index bf7afda05f..288ffda9f5 100644 --- a/db/pool.go +++ b/db/pool.go @@ -319,14 +319,6 @@ func (p *Pool) Vacuum(ctx context.Context, commit ksuid.KSUID, dryrun bool) ([]k } return err }) - // Delete the seek index as well. - group.Go(func() error { - err := p.engine.Delete(ctx, data.SeekIndexURI(p.DataPath, o.ID)) - if errors.Is(err, fs.ErrNotExist) { - err = nil - } - return err - }) } if err := group.Wait(); err != nil { return nil, err diff --git a/db/pools/config.go b/db/pools/config.go index 56a8a714f9..9592d080d8 100644 --- a/db/pools/config.go +++ b/db/pools/config.go @@ -13,33 +13,28 @@ import ( ) type Config struct { - Ts nano.Ts `super:"ts"` - Name string `super:"name"` - ID ksuid.KSUID `super:"id"` - SortKeys order.SortKeys `super:"layout"` - SeekStride int `super:"seek_stride"` - Threshold int64 `super:"threshold"` + Ts nano.Ts `super:"ts"` + Name string `super:"name"` + ID ksuid.KSUID `super:"id"` + SortKeys order.SortKeys `super:"layout"` + Threshold int64 `super:"threshold"` } var _ journal.Entry = (*Config)(nil) -func NewConfig(name string, sortKeys order.SortKeys, thresh int64, seekStride int) *Config { +func NewConfig(name string, sortKeys order.SortKeys, thresh int64) *Config { if sortKeys.IsNil() { sortKeys = order.SortKeys{order.NewSortKey(order.Desc, field.Dotted("ts"))} } if thresh == 0 { thresh = data.DefaultThreshold } - if seekStride == 0 { - seekStride = data.DefaultSeekStride - } return &Config{ - Ts: nano.Now(), - Name: name, - ID: ksuid.New(), - SortKeys: sortKeys, - SeekStride: seekStride, - Threshold: thresh, + Ts: nano.Now(), + Name: name, + ID: ksuid.New(), + SortKeys: sortKeys, + Threshold: thresh, } } @@ -55,12 +50,11 @@ func (p *Config) Path(root *storage.URI) *storage.URI { // previous versions. At some point we'll do a migration so we don't have to do // this. type marshalConfig struct { - Ts nano.Ts `super:"ts"` - Name string `super:"name"` - ID ksuid.KSUID `super:"id"` - SortKey oldSortKey `super:"layout"` - SeekStride int `super:"seek_stride"` - Threshold int64 `super:"threshold"` + Ts nano.Ts `super:"ts"` + Name string `super:"name"` + ID ksuid.KSUID `super:"id"` + SortKey oldSortKey `super:"layout"` + Threshold int64 `super:"threshold"` } type oldSortKey struct { @@ -76,11 +70,10 @@ var hackedBindings = []sup.Binding{ func (p Config) MarshalBSUP(ctx *sup.MarshalBSUPContext) (super.Type, error) { ctx.NamedBindings(hackedBindings) m := marshalConfig{ - Ts: p.Ts, - Name: p.Name, - ID: p.ID, - SeekStride: p.SeekStride, - Threshold: p.Threshold, + Ts: p.Ts, + Name: p.Name, + ID: p.ID, + Threshold: p.Threshold, } if !p.SortKeys.IsNil() { m.SortKey.Order = p.SortKeys[0].Order @@ -101,7 +94,6 @@ func (p *Config) UnmarshalBSUP(ctx *sup.UnmarshalBSUPContext, val super.Value) e p.Ts = m.Ts p.Name = m.Name p.ID = m.ID - p.SeekStride = m.SeekStride p.Threshold = m.Threshold for _, k := range m.SortKey.Keys { p.SortKeys = append(p.SortKeys, order.NewSortKey(m.SortKey.Order, k)) diff --git a/db/root.go b/db/root.go index c5075fa2a5..3eff534745 100644 --- a/db/root.go +++ b/db/root.go @@ -313,7 +313,7 @@ func (r *Root) RenamePool(ctx context.Context, id ksuid.KSUID, newName string) e return r.pools.Rename(ctx, id, newName) } -func (r *Root) CreatePool(ctx context.Context, name string, sortKeys order.SortKeys, seekStride int, thresh int64) (*Pool, error) { +func (r *Root) CreatePool(ctx context.Context, name string, sortKeys order.SortKeys, thresh int64) (*Pool, error) { if name == "HEAD" { return nil, fmt.Errorf("pool cannot be named %q", name) } @@ -326,7 +326,7 @@ func (r *Root) CreatePool(ctx context.Context, name string, sortKeys order.SortK if len(sortKeys) > 1 { return nil, errors.New("multiple pool keys not supported") } - config := pools.NewConfig(name, sortKeys, thresh, seekStride) + config := pools.NewConfig(name, sortKeys, thresh) if err := CreatePool(ctx, r.engine, r.logger, r.path, config); err != nil { return nil, err } diff --git a/db/seekindex/entry.go b/db/seekindex/entry.go deleted file mode 100644 index 89cf6c2c24..0000000000 --- a/db/seekindex/entry.go +++ /dev/null @@ -1,37 +0,0 @@ -package seekindex - -import ( - "github.com/brimdata/super" - "github.com/brimdata/super/vector" -) - -type Entry struct { - Min super.Value `super:"min"` - Max super.Value `super:"max"` - ValOff uint64 `super:"val_off"` - ValCnt uint64 `super:"val_cnt"` - Offset uint64 `super:"offset"` - Length uint64 `super:"length"` -} - -func (e Entry) Range() Range { - return Range{ - Offset: int64(e.Offset), - Length: int64(e.Length), - } -} - -type Index []Entry - -func (i Index) Filter(b *vector.Bool) Ranges { - var ranges Ranges - for _, e := range i { - for off := e.ValOff; off < uint64(b.Len()) && off < e.ValOff+e.ValCnt; off++ { - if b.IsSet(uint32(off)) { - ranges.Append(e) - break - } - } - } - return ranges -} diff --git a/db/seekindex/range.go b/db/seekindex/range.go deleted file mode 100644 index 8e44cf0ff9..0000000000 --- a/db/seekindex/range.go +++ /dev/null @@ -1,34 +0,0 @@ -package seekindex - -import ( - "fmt" - "io" -) - -type Range struct { - Offset int64 `super:"offset"` - Length int64 `super:"length"` -} - -func (r Range) IsZero() bool { - return r.Length == 0 -} - -func (r Range) String() string { - return fmt.Sprintf("offset %d length %d", r.Offset, r.Length) -} - -func (r Range) Reader(reader io.ReaderAt) (io.Reader, error) { - return io.NewSectionReader(reader, r.Offset, r.Length), nil -} - -type Ranges []Range - -func (r *Ranges) Append(entry Entry) { - n := len(*r) - if n == 0 || (*r)[n-1].Offset+(*r)[n-1].Length < int64(entry.Offset) { - *r = append(*r, entry.Range()) - return - } - (*r)[n-1].Length += int64(entry.Length) -} diff --git a/db/seekindex/writer.go b/db/seekindex/writer.go deleted file mode 100644 index bbf5493e16..0000000000 --- a/db/seekindex/writer.go +++ /dev/null @@ -1,40 +0,0 @@ -package seekindex - -import ( - "github.com/brimdata/super" - "github.com/brimdata/super/sio" - "github.com/brimdata/super/sup" -) - -type Writer struct { - marshal *sup.MarshalBSUPContext - writer sio.WriteCloser - offset uint64 - valoff uint64 -} - -func NewWriter(w sio.WriteCloser) *Writer { - return &Writer{ - marshal: sup.NewBSUPMarshaler(), - writer: w, - } -} - -func (w *Writer) Write(min, max super.Value, valoff uint64, offset uint64) error { - val, err := w.marshal.Marshal(&Entry{ - Min: min, - Max: max, - ValOff: w.valoff, - ValCnt: valoff - w.valoff, - Offset: w.offset, - Length: offset - w.offset, - }) - w.valoff = valoff - w.offset = offset - if err != nil { - return err - } - return w.writer.Write(val) -} - -func (w *Writer) Close() error { return w.writer.Close() } diff --git a/db/writer.go b/db/writer.go index d264d7f27c..0df0702707 100644 --- a/db/writer.go +++ b/db/writer.go @@ -143,7 +143,7 @@ func (w *Writer) writeObject(object *data.Object, recs []super.Value) error { return w.ctx.Err() } } - writer, err := object.NewWriter(w.ctx, w.pool.engine, w.pool.DataPath, w.pool.SortKeys.Primary(), w.pool.SeekStride) + writer, err := object.NewWriter(w.ctx, w.pool.engine, w.pool.DataPath, w.pool.SortKeys.Primary()) if err != nil { return err } @@ -246,7 +246,7 @@ func (w *SortedWriter) Abort() { func (w *SortedWriter) newWriter() error { o := data.NewObject() var err error - w.writer, err = o.NewWriter(w.ctx, w.pool.engine, w.pool.DataPath, w.sortKey, w.pool.SeekStride) + w.writer, err = o.NewWriter(w.ctx, w.pool.engine, w.pool.DataPath, w.sortKey) if err != nil { return err } diff --git a/db/ztests/consecutive-ts.yaml b/db/ztests/consecutive-ts.yaml deleted file mode 100644 index 55fc5305e5..0000000000 --- a/db/ztests/consecutive-ts.yaml +++ /dev/null @@ -1,36 +0,0 @@ -script: | - export SUPER_DB=test - super db init -q - super db create -q -seekstride 11B -orderby ts:desc logs - super db load -use logs -q in.sup - super -s -c "min:=defuse(min),max:=defuse(max)" test/*/data/*-seek.bsup - -inputs: - - name: in.sup - data: | - {ts:1970-01-01T00:00:00Z} - {ts:1970-01-01T00:00:02Z} - {ts:1970-01-01T00:00:02Z} - {ts:1970-01-01T00:00:02Z} - {ts:1970-01-01T00:00:02Z} - {ts:1970-01-01T00:00:02Z} - {ts:1970-01-01T00:00:03Z} - {ts:1970-01-01T00:00:03Z} - {ts:1970-01-01T00:00:03Z} - {ts:1970-01-01T00:00:03Z} - {ts:1970-01-01T00:00:03Z} - {ts:1970-01-01T00:00:03Z} - {ts:1970-01-01T00:00:03Z} - {ts:1970-01-01T00:00:06Z} - {ts:1970-01-01T00:00:07Z} - {ts:1970-01-01T00:00:08Z} - -outputs: - - name: stdout - data: | - {min:1970-01-01T00:00:06Z,max:1970-01-01T00:00:08Z,val_off:0::uint64,val_cnt:3::uint64,offset:0::uint64,length:37::uint64} - {min:1970-01-01T00:00:03Z,max:1970-01-01T00:00:03Z,val_off:3::uint64,val_cnt:3::uint64,offset:37::uint64,length:37::uint64} - {min:1970-01-01T00:00:03Z,max:1970-01-01T00:00:03Z,val_off:6::uint64,val_cnt:3::uint64,offset:74::uint64,length:37::uint64} - {min:1970-01-01T00:00:02Z,max:1970-01-01T00:00:03Z,val_off:9::uint64,val_cnt:3::uint64,offset:111::uint64,length:35::uint64} - {min:1970-01-01T00:00:02Z,max:1970-01-01T00:00:02Z,val_off:12::uint64,val_cnt:3::uint64,offset:146::uint64,length:34::uint64} - {min:1970-01-01T00:00:00Z,max:1970-01-01T00:00:00Z,val_off:15::uint64,val_cnt:1::uint64,offset:180::uint64,length:16::uint64} diff --git a/db/ztests/ls.yaml b/db/ztests/ls.yaml index f904a39fcb..a2368e2c5b 100644 --- a/db/ztests/ls.yaml +++ b/db/ztests/ls.yaml @@ -34,7 +34,6 @@ outputs: ] ] }::order.SortKey, - seek_stride: 65536, threshold: 524288000 } === diff --git a/db/ztests/meta.yaml b/db/ztests/meta.yaml index 5db49a60c2..48c60bc537 100644 --- a/db/ztests/meta.yaml +++ b/db/ztests/meta.yaml @@ -41,7 +41,6 @@ outputs: ] ] }::order.SortKey, - seek_stride: 65536, threshold: 524288000 } { @@ -54,7 +53,6 @@ outputs: ] ] }::order.SortKey, - seek_stride: 65536, threshold: 524288000 } === diff --git a/db/ztests/seek-index-boundaries.yaml b/db/ztests/seek-index-boundaries.yaml deleted file mode 100644 index e8285f33f0..0000000000 --- a/db/ztests/seek-index-boundaries.yaml +++ /dev/null @@ -1,67 +0,0 @@ -script: | - export SUPER_DB=test - super db init -q - for o in asc desc; do - super db create -use -q -seekstride 1B -orderby ts:$o $o - seq 20 | super -c 'values {ts:this}' - | super db load -q - - source query.sh "from $o | ts == 5" - source query.sh "from $o | ts < 2" - source query.sh "from $o | ts <= 2" - source query.sh "from $o | ts > 19" - source query.sh "from $o | ts >= 19" - done - -inputs: - - name: query.sh - data: | - echo // $1 | tee /dev/stderr - super db -s -stats -c "$1" -outputs: - - name: stdout - data: | - // from asc | ts == 5 - {ts:5} - // from asc | ts < 2 - {ts:1} - // from asc | ts <= 2 - {ts:1} - {ts:2} - // from asc | ts > 19 - {ts:20} - // from asc | ts >= 19 - {ts:19} - {ts:20} - // from desc | ts == 5 - {ts:5} - // from desc | ts < 2 - {ts:1} - // from desc | ts <= 2 - {ts:2} - {ts:1} - // from desc | ts > 19 - {ts:20} - // from desc | ts >= 19 - {ts:20} - {ts:19} - - name: stderr - data: | - // from asc | ts == 5 - {bytes_read:2,bytes_matched:2,records_read:1,records_matched:1} - // from asc | ts < 2 - {bytes_read:2,bytes_matched:2,records_read:1,records_matched:1} - // from asc | ts <= 2 - {bytes_read:4,bytes_matched:4,records_read:2,records_matched:2} - // from asc | ts > 19 - {bytes_read:2,bytes_matched:2,records_read:1,records_matched:1} - // from asc | ts >= 19 - {bytes_read:4,bytes_matched:4,records_read:2,records_matched:2} - // from desc | ts == 5 - {bytes_read:2,bytes_matched:2,records_read:1,records_matched:1} - // from desc | ts < 2 - {bytes_read:2,bytes_matched:2,records_read:1,records_matched:1} - // from desc | ts <= 2 - {bytes_read:4,bytes_matched:4,records_read:2,records_matched:2} - // from desc | ts > 19 - {bytes_read:2,bytes_matched:2,records_read:1,records_matched:1} - // from desc | ts >= 19 - {bytes_read:4,bytes_matched:4,records_read:2,records_matched:2} diff --git a/db/ztests/seek-index-null.yaml b/db/ztests/seek-index-null.yaml deleted file mode 100644 index b426b3d621..0000000000 --- a/db/ztests/seek-index-null.yaml +++ /dev/null @@ -1,34 +0,0 @@ -script: | - export SUPER_DB=test - super db init -q - for o in asc desc; do - echo // $o | tee /dev/stderr - super db create -q -seekstride 2KB -orderby ts:$o $o - super db load -q -use $o babble.sup null.sup - super db -s -stats -c "from $o | ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z" - done - -inputs: - - name: babble.sup - source: ../../testdata/babble.sup - - name: null.sup - data: | - {ts:null,s:"foo-bar",v:1} - -outputs: - - name: stdout - data: | - // asc - {ts:2020-04-21T23:59:26.06326664Z,s:"potbellied-Dedanim",v:230} - {ts:2020-04-21T23:59:29.06985813Z,s:"areek-ashless",v:266} - {ts:2020-04-21T23:59:38.0687693Z,s:"topcoating-rhexis",v:415} - // desc - {ts:2020-04-21T23:59:38.0687693Z,s:"topcoating-rhexis",v:415} - {ts:2020-04-21T23:59:29.06985813Z,s:"areek-ashless",v:266} - {ts:2020-04-21T23:59:26.06326664Z,s:"potbellied-Dedanim",v:230} - - name: stderr - data: | - // asc - {bytes_read:16403,bytes_matched:87,records_read:500,records_matched:3} - // desc - {bytes_read:16403,bytes_matched:87,records_read:500,records_matched:3} diff --git a/db/ztests/seek-index-overlap.yaml b/db/ztests/seek-index-overlap.yaml index 53bfdda071..461657df89 100644 --- a/db/ztests/seek-index-overlap.yaml +++ b/db/ztests/seek-index-overlap.yaml @@ -1,8 +1,8 @@ script: | export SUPER_DB=test super db init -q - super db create -seekstride 2KB -orderby ts:asc -q asc - super db create -seekstride 2KB -orderby ts:desc -q desc + super db create -orderby ts:asc -q asc + super db create -orderby ts:desc -q desc super db use -q asc super -c "tail 900" babble.sup | super db load -q - super -c "head 250" babble.sup | super db load -q - diff --git a/db/ztests/seek-index-simple.yaml b/db/ztests/seek-index-simple.yaml deleted file mode 100644 index 4eded9e98b..0000000000 --- a/db/ztests/seek-index-simple.yaml +++ /dev/null @@ -1,41 +0,0 @@ -script: | - export SUPER_DB=test - super db init -q - super db create -q -seekstride 1KB -orderby k:asc asc - super db create -q -seekstride 1KB -orderby k:desc desc - seq -f "{k:%g,foo:123}" 1 15000 | super db load -q -use asc - - seq -f "{k:%g,foo:123}" 1 15000 | super db load -q -use desc - - super db -s -stats -c 'from asc | k >= 1000 and k <= 1002' - super db -s -stats -c 'from asc | k == 12321' - super db -s -stats -c 'from asc | k == 12322 or foo != 123' - super db -s -stats -c 'from asc | k == 12323 and foo == 123' - super db -s -stats -c 'from desc | k >= 1000 and k <= 1002' - super db -s -stats -c 'from desc | k == 12321' - super db -s -stats -c 'from desc | k == 12322 or foo != 123' - super db -s -stats -c 'from desc | k == 12323 and foo == 123' - -outputs: - - name: stdout - data: | - {k:1000,foo:123} - {k:1001,foo:123} - {k:1002,foo:123} - {k:12321,foo:123} - {k:12322,foo:123} - {k:12323,foo:123} - {k:1002,foo:123} - {k:1001,foo:123} - {k:1000,foo:123} - {k:12321,foo:123} - {k:12322,foo:123} - {k:12323,foo:123} - - name: stderr - data: | - {bytes_read:2500,bytes_matched:15,records_read:500,records_matched:3} - {bytes_read:2500,bytes_matched:5,records_read:500,records_matched:1} - {bytes_read:74873,bytes_matched:5,records_read:15000,records_matched:1} - {bytes_read:2500,bytes_matched:5,records_read:500,records_matched:1} - {bytes_read:5000,bytes_matched:15,records_read:1000,records_matched:3} - {bytes_read:2500,bytes_matched:5,records_read:500,records_matched:1} - {bytes_read:74873,bytes_matched:5,records_read:15000,records_matched:1} - {bytes_read:2500,bytes_matched:5,records_read:500,records_matched:1} diff --git a/db/ztests/seek-index-ts.yaml b/db/ztests/seek-index-ts.yaml deleted file mode 100644 index fc6fa40216..0000000000 --- a/db/ztests/seek-index-ts.yaml +++ /dev/null @@ -1,53 +0,0 @@ -script: | - export SUPER_DB=test - super db init -q - for o in asc desc; do - super db create -use -seekstride 2KB -orderby ts:$o -q $o - super db load -q babble.sup - source query.sh "from $o | ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z" - source query.sh "from $o | ts == 2020-04-21T23:59:26.06326664Z" - source query.sh "from $o | ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar'" - - done - -inputs: - - name: babble.sup - source: ../../testdata/babble.sup - - name: query.sh - data: | - echo // $1 | tee /dev/stderr - super db -s -stats -c "$1" - -outputs: - - name: stdout - data: | - // from asc | ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z - {ts:2020-04-21T23:59:26.06326664Z,s:"potbellied-Dedanim",v:230} - {ts:2020-04-21T23:59:29.06985813Z,s:"areek-ashless",v:266} - {ts:2020-04-21T23:59:38.0687693Z,s:"topcoating-rhexis",v:415} - // from asc | ts == 2020-04-21T23:59:26.06326664Z - {ts:2020-04-21T23:59:26.06326664Z,s:"potbellied-Dedanim",v:230} - // from asc | ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar' - {ts:2020-04-21T23:59:26.06326664Z,s:"potbellied-Dedanim",v:230} - // from desc | ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z - {ts:2020-04-21T23:59:38.0687693Z,s:"topcoating-rhexis",v:415} - {ts:2020-04-21T23:59:29.06985813Z,s:"areek-ashless",v:266} - {ts:2020-04-21T23:59:26.06326664Z,s:"potbellied-Dedanim",v:230} - // from desc | ts == 2020-04-21T23:59:26.06326664Z - {ts:2020-04-21T23:59:26.06326664Z,s:"potbellied-Dedanim",v:230} - // from desc | ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar' - {ts:2020-04-21T23:59:26.06326664Z,s:"potbellied-Dedanim",v:230} - - name: stderr - data: | - // from asc | ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z - {bytes_read:16403,bytes_matched:87,records_read:500,records_matched:3} - // from asc | ts == 2020-04-21T23:59:26.06326664Z - {bytes_read:8141,bytes_matched:31,records_read:250,records_matched:1} - // from asc | ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar' - {bytes_read:32889,bytes_matched:31,records_read:1000,records_matched:1} - // from desc | ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z - {bytes_read:16403,bytes_matched:87,records_read:500,records_matched:3} - // from desc | ts == 2020-04-21T23:59:26.06326664Z - {bytes_read:8141,bytes_matched:31,records_read:250,records_matched:1} - // from desc | ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar' - {bytes_read:32889,bytes_matched:31,records_read:1000,records_matched:1} diff --git a/runtime/sam/op/meta/sequence.go b/runtime/sam/op/meta/sequence.go index 49be40cde5..9f1d52de9b 100644 --- a/runtime/sam/op/meta/sequence.go +++ b/runtime/sam/op/meta/sequence.go @@ -8,7 +8,6 @@ import ( "github.com/brimdata/super" "github.com/brimdata/super/db" "github.com/brimdata/super/db/data" - "github.com/brimdata/super/db/seekindex" "github.com/brimdata/super/runtime" "github.com/brimdata/super/runtime/sam/expr" "github.com/brimdata/super/runtime/sam/op/merge" @@ -133,14 +132,7 @@ func (s *SearchScanner) Pull(done bool) (sbuf.Batch, error) { if b == nil || err != nil { return nil, err } - ranges, err := data.RangeFromBitVector(s.rctx.Context, s.pool.Storage(), s.pool.DataPath, o, b) - if err != nil { - return nil, err - } - if len(ranges) == 0 { - continue - } - s.scanner, err = newObjectScanner(s.rctx.Context, s.rctx.Sctx, s.pool, o, ranges, s.pushdown, s.progress) + s.scanner, err = newObjectScanner(s.rctx.Context, s.rctx.Sctx, s.pool, o, s.pushdown, s.progress) if err != nil { return nil, err } @@ -187,11 +179,7 @@ func newObjectsScanner(ctx context.Context, sctx *super.Context, pool *db.Pool, } } for _, object := range objects { - ranges, err := data.LookupSeekRange(ctx, pool.Storage(), pool.DataPath, object, pruner) - if err != nil { - return nil, err - } - s, err := newObjectScanner(ctx, sctx, pool, object, ranges, pushdown, progress) + s, err := newObjectScanner(ctx, sctx, pool, object, pushdown, progress) if err != nil { pullersDone() return nil, err @@ -204,8 +192,8 @@ func newObjectsScanner(ctx context.Context, sctx *super.Context, pool *db.Pool, return merge.New(ctx, pullers, db.ImportComparator(sctx, pool).Compare), nil } -func newObjectScanner(ctx context.Context, sctx *super.Context, pool *db.Pool, object *data.Object, ranges []seekindex.Range, pushdown sbuf.Pushdown, progress *vio.Progress) (sbuf.Puller, error) { - rc, err := object.NewReader(ctx, pool.Storage(), pool.DataPath, ranges) +func newObjectScanner(ctx context.Context, sctx *super.Context, pool *db.Pool, object *data.Object, pushdown sbuf.Pushdown, progress *vio.Progress) (sbuf.Puller, error) { + rc, err := object.NewReader(ctx, pool.Storage(), pool.DataPath) if err != nil { return nil, err } diff --git a/service/handlers.go b/service/handlers.go index 655f899306..b074d4c5fc 100644 --- a/service/handlers.go +++ b/service/handlers.go @@ -246,7 +246,7 @@ func handlePoolPost(c *Core, w *ResponseWriter, r *Request) { if len(req.SortKeys.Keys) > 0 { sortKeys = append(sortKeys, order.NewSortKey(req.SortKeys.Order, req.SortKeys.Keys[0])) } - pool, err := c.root.CreatePool(r.Context(), req.Name, sortKeys, req.SeekStride, req.Thresh) + pool, err := c.root.CreatePool(r.Context(), req.Name, sortKeys, req.Thresh) if err != nil { w.Error(err) return diff --git a/service/ztests/curl-create.yaml b/service/ztests/curl-create.yaml index e02602453c..c96bce2f9d 100644 --- a/service/ztests/curl-create.yaml +++ b/service/ztests/curl-create.yaml @@ -25,7 +25,6 @@ outputs: ] ] }, - seek_stride: 65536, threshold: 524288000 }, branch: { diff --git a/service/ztests/curl-pool-rename.yaml b/service/ztests/curl-pool-rename.yaml index 1166494c3c..2458e9f832 100644 --- a/service/ztests/curl-pool-rename.yaml +++ b/service/ztests/curl-pool-rename.yaml @@ -31,6 +31,5 @@ outputs: ] ] }, - seek_stride: 65536, threshold: 524288000 } diff --git a/service/ztests/seek-index-null.yaml b/service/ztests/seek-index-null.yaml deleted file mode 100644 index c3a0c97495..0000000000 --- a/service/ztests/seek-index-null.yaml +++ /dev/null @@ -1,34 +0,0 @@ -script: | - source service.sh - for o in asc desc; do - echo // $o | tee /dev/stderr - super db create -q -seekstride=2KB -orderby ts:$o $o - super db load -q -use $o babble.sup null.sup - super db -s -stats -c "from $o | ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z" - done - -inputs: - - name: service.sh - - name: babble.sup - source: ../../testdata/babble.sup - - name: null.sup - data: | - {ts:null,s:"foo-bar",v:1} - -outputs: - - name: stdout - data: | - // asc - {ts:2020-04-21T23:59:26.06326664Z,s:"potbellied-Dedanim",v:230} - {ts:2020-04-21T23:59:29.06985813Z,s:"areek-ashless",v:266} - {ts:2020-04-21T23:59:38.0687693Z,s:"topcoating-rhexis",v:415} - // desc - {ts:2020-04-21T23:59:38.0687693Z,s:"topcoating-rhexis",v:415} - {ts:2020-04-21T23:59:29.06985813Z,s:"areek-ashless",v:266} - {ts:2020-04-21T23:59:26.06326664Z,s:"potbellied-Dedanim",v:230} - - name: stderr - data: | - // asc - {bytes_read:16403,bytes_matched:87,records_read:500,records_matched:3} - // desc - {bytes_read:16403,bytes_matched:87,records_read:500,records_matched:3} diff --git a/service/ztests/seek-index-overlap.yaml b/service/ztests/seek-index-overlap.yaml deleted file mode 100644 index 80747b0722..0000000000 --- a/service/ztests/seek-index-overlap.yaml +++ /dev/null @@ -1,30 +0,0 @@ -script: | - source service.sh - super db create -seekstride 2KB -orderby ts:asc -q asc - super db create -seekstride 2KB -orderby ts:desc -q desc - super db use -q asc - super -c "tail 900" babble.sup | super db load -q - - super -c "head 250" babble.sup | super db load -q - - super db -s -stats -c "from asc | count()" - echo === | tee /dev/stderr - super db use -q desc - super -c "tail 900" babble.sup | super db load -q - - super -c "head 250" babble.sup | super db load -q - - super db -s -stats -c "from desc | count()" - -inputs: - - name: service.sh - - name: babble.sup - source: ../../testdata/babble.sup - -outputs: - - name: stdout - data: | - 1150 - === - 1150 - - name: stderr - data: | - {bytes_read:37833,bytes_matched:37833,records_read:1150,records_matched:1150} - === - {bytes_read:37833,bytes_matched:37833,records_read:1150,records_matched:1150} diff --git a/service/ztests/seek-index.yaml b/service/ztests/seek-index.yaml deleted file mode 100644 index 094efe14c6..0000000000 --- a/service/ztests/seek-index.yaml +++ /dev/null @@ -1,57 +0,0 @@ -script: | - source service.sh - for o in asc desc; do - echo // $o | tee /dev/stderr - super db create -use -seekstride 2KB -orderby ts:$o -q $o - super db load -q babble.sup - source query.sh "from $o | ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z" - source query.sh "from $o | ts == 2020-04-21T23:59:26.06326664Z" - source query.sh "from $o | ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar'" - done - -inputs: - - name: service.sh - - name: babble.sup - source: ../../testdata/babble.sup - - name: query.sh - data: | - echo // $1 | tee /dev/stderr - super db -s -stats -c "$1" - -outputs: - - name: stdout - data: | - // asc - // from asc | ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z - {ts:2020-04-21T23:59:26.06326664Z,s:"potbellied-Dedanim",v:230} - {ts:2020-04-21T23:59:29.06985813Z,s:"areek-ashless",v:266} - {ts:2020-04-21T23:59:38.0687693Z,s:"topcoating-rhexis",v:415} - // from asc | ts == 2020-04-21T23:59:26.06326664Z - {ts:2020-04-21T23:59:26.06326664Z,s:"potbellied-Dedanim",v:230} - // from asc | ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar' - {ts:2020-04-21T23:59:26.06326664Z,s:"potbellied-Dedanim",v:230} - // desc - // from desc | ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z - {ts:2020-04-21T23:59:38.0687693Z,s:"topcoating-rhexis",v:415} - {ts:2020-04-21T23:59:29.06985813Z,s:"areek-ashless",v:266} - {ts:2020-04-21T23:59:26.06326664Z,s:"potbellied-Dedanim",v:230} - // from desc | ts == 2020-04-21T23:59:26.06326664Z - {ts:2020-04-21T23:59:26.06326664Z,s:"potbellied-Dedanim",v:230} - // from desc | ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar' - {ts:2020-04-21T23:59:26.06326664Z,s:"potbellied-Dedanim",v:230} - - name: stderr - data: | - // asc - // from asc | ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z - {bytes_read:16403,bytes_matched:87,records_read:500,records_matched:3} - // from asc | ts == 2020-04-21T23:59:26.06326664Z - {bytes_read:8141,bytes_matched:31,records_read:250,records_matched:1} - // from asc | ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar' - {bytes_read:32889,bytes_matched:31,records_read:1000,records_matched:1} - // desc - // from desc | ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z - {bytes_read:16403,bytes_matched:87,records_read:500,records_matched:3} - // from desc | ts == 2020-04-21T23:59:26.06326664Z - {bytes_read:8141,bytes_matched:31,records_read:250,records_matched:1} - // from desc | ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar' - {bytes_read:32889,bytes_matched:31,records_read:1000,records_matched:1}