remove database seek index

api/api.go

@@ -36,10 +36,9 @@ type VersionResponse struct {
 }
 
 type PoolPostRequest struct {
-	Name       string   `json:"name"`
-	SortKeys   SortKeys `json:"layout"`
-	SeekStride int      `json:"seek_stride"`
-	Thresh     int64    `json:"thresh"`
+	Name     string   `json:"name"`
+	SortKeys SortKeys `json:"layout"`
+	Thresh   int64    `json:"thresh"`
 }
 
 type SortKeys struct {

book/src/command/db.md

@@ -151,9 +151,6 @@ We can narrow the span of the query by specifying a filter on the database
 ```
 super db -c 'from logs | ts >= 2018-03-24T17:36:30.090766Z and ts <= 2018-03-24T17:36:30.090758Z'
 ```
-Filters on sort keys are efficiently implemented as the data is laid out
-according to the sort key and seek indexes keyed by the sort key
-are computed for each data object.
 
 When querying data to the [BSUP](../formats/bsup.md) output format,
 output from a pool can be easily piped to other commands like `super`, e.g.,

book/src/database/api.md

@@ -25,7 +25,6 @@ POST /pool
 | name | string | body | **Required.** Name of the pool. Must be unique to lake. |
 | layout.order | string | body | Order of storage by primary key(s) in pool. Possible values: desc, asc. Default: asc. |
 | layout.keys | [[string]] | body | Primary key(s) of pool. The element of each inner string array should reflect the hierarchical ordering of named fields within indexed records. Default: [[ts]]. |
-| thresh | int | body | The size in bytes of each seek index. |
 | Content-Type | string | header | [MIME type](#mime-types) of the request payload. |
 | Accept | string | header | Preferred [MIME type](#mime-types) of the response. |
 

book/src/database/format.md

@@ -77,7 +77,7 @@ resulting object is immutable, there is no possible write concurrency to manage
 with respect to a given object.
 
 A data object is composed of the primary data object stored as one or two objects
-(for sequence and/or vector layout) and an optional seek index.
+(for sequence and/or vector layout).
 
 Data objects may be either in sequence form (i.e., BSUP) or vector form (i.e., CSUP),
 or both forms may be present as a query optimizer may choose to use whatever
@@ -91,19 +91,9 @@ Immutable objects are named as follows:
 |-----------|----|
 |vector data|`<pool-id>/data/<id>.csup`|
 |sequence data|`<pool-id>/data/<id>.bsup`|
-|sequence seek index|`<pool-id>/data/<id>-seek.bsup`|
 
 `<id>` is the KSUID of the data object.
 
-The seek index maps pool key values to seek offsets in the BSUP file thereby
-allowing a scan to do a byte-range retrieval of the BSUP object when
-processing only a subset of data.
-
->[!NOTE]
-> The CSUP format allows individual vector segments to be read in isolation
-> and the in-memory CSUP representation supports random access so there is
-> no need to have a seek index for the vector object.
-
 #### Commit History
 
 A branch's commit history is the definitive record of the evolution of data in

cmd/super/db/create/command.go

@@ -20,28 +20,22 @@ var spec = &charm.Spec{
 	Long: `
 See https://superdb.org/command/db.html#super-db-create
 `,
-	HiddenFlags: "seekstride",
-	New:         New,
+	New: New,
 }
 
 type Command struct {
 	*db.Command
-	sortKey    string
-	thresh     units.Bytes
-	seekStride units.Bytes
-	use        bool
+	sortKey string
+	thresh  units.Bytes
+	use     bool
 }
 
 func init() {
 	db.Spec.Add(spec)
 }
 
 func New(parent charm.Command, f *flag.FlagSet) (charm.Command, error) {
-	c := &Command{
-		Command:    parent.(*db.Command),
-		seekStride: units.Bytes(data.DefaultSeekStride),
-	}
-	f.Var(&c.seekStride, "seekstride", "size of seek-index unit for BSUP data, as '32KB', '1MB', etc.")
+	c := &Command{Command: parent.(*db.Command)}
 	c.thresh = data.DefaultThreshold
 	f.Var(&c.thresh, "S", "target size of pool data objects, as '10MB' or '4GiB', etc.")
 	f.BoolVar(&c.use, "use", false, "set created pool as the current pool")
@@ -67,7 +61,7 @@ func (c *Command) Run(args []string) error {
 		return err
 	}
 	poolName := args[0]
-	id, err := db.CreatePool(ctx, poolName, sortKey, int(c.seekStride), int64(c.thresh))
+	id, err := db.CreatePool(ctx, poolName, sortKey, int64(c.thresh))
 	if err != nil {
 		return err
 	}

db/api/api.go

@@ -30,7 +30,7 @@ type Interface interface {
 	Query(ctx context.Context, query []srcfiles.Input) (vio.Scanner, error)
 	PoolID(ctx context.Context, poolName string) (ksuid.KSUID, error)
 	CommitObject(ctx context.Context, poolID ksuid.KSUID, branchName string) (ksuid.KSUID, error)
-	CreatePool(context.Context, string, order.SortKeys, int, int64) (ksuid.KSUID, error)
+	CreatePool(context.Context, string, order.SortKeys, int64) (ksuid.KSUID, error)
 	RemovePool(context.Context, ksuid.KSUID) error
 	RenamePool(context.Context, ksuid.KSUID, string) error
 	CreateBranch(ctx context.Context, pool ksuid.KSUID, name string, parent ksuid.KSUID) error

db/api/local.go

@@ -63,11 +63,11 @@ func (l *local) Root() *db.Root {
 	return l.db
 }
 
-func (l *local) CreatePool(ctx context.Context, name string, sortKeys order.SortKeys, seekStride int, thresh int64) (ksuid.KSUID, error) {
+func (l *local) CreatePool(ctx context.Context, name string, sortKeys order.SortKeys, thresh int64) (ksuid.KSUID, error) {
 	if name == "" {
 		return ksuid.Nil, errors.New("no pool name provided")
 	}
-	pool, err := l.db.CreatePool(ctx, name, sortKeys, seekStride, thresh)
+	pool, err := l.db.CreatePool(ctx, name, sortKeys, thresh)
 	if err != nil {
 		return ksuid.Nil, err
 	}

db/api/remote.go

@@ -53,15 +53,14 @@ func (r *remote) CommitObject(ctx context.Context, poolID ksuid.KSUID, branchNam
 	return res.Commit, err
 }
 
-func (r *remote) CreatePool(ctx context.Context, name string, sortKeys order.SortKeys, seekStride int, thresh int64) (ksuid.KSUID, error) {
+func (r *remote) CreatePool(ctx context.Context, name string, sortKeys order.SortKeys, thresh int64) (ksuid.KSUID, error) {
 	res, err := r.conn.CreatePool(ctx, api.PoolPostRequest{
 		Name: name,
 		SortKeys: api.SortKeys{
 			Order: sortKeys.Primary().Order,
 			Keys:  field.List{sortKeys.Primary().Key},
 		},
-		SeekStride: seekStride,
-		Thresh:     thresh,
+		Thresh: thresh,
 	})
 	if err != nil {
 		return ksuid.Nil, err

db/data/object.go

@@ -15,8 +15,7 @@ import (
 )
 
 const (
-	DefaultSeekStride = 64 * 1024
-	DefaultThreshold  = 500 * 1024 * 1024
+	DefaultThreshold = 500 * 1024 * 1024
 )
 
 // A FileKind is the first part of a file name, used to differentiate files
@@ -27,7 +26,6 @@ const (
 	FileKindUnknown  FileKind = ""
 	FileKindData     FileKind = "data"
 	FileKindMetadata FileKind = "meta"
-	FileKindSeek     FileKind = "seek"
 )
 
 func (k FileKind) Description() string {
@@ -36,18 +34,11 @@ func (k FileKind) Description() string {
 		return "data"
 	case FileKindMetadata:
 		return "metadata"
-	case "FileKindSeek":
-		return "seekindex"
 	default:
 		return "unknown"
 	}
 }
 
-//XXX all file types are cacheable but seek index etc is not matched here.
-//and seekindexes really do want to be cached as they are small and
-// eliminate round-trips, especially when you are ready sub-ranges of
-// cached data files!
-
 var fileRegex = regexp.MustCompile(`([0-9A-Za-z]{27}-(data|meta)).bsup$`)
 
 // XXX this won't work right until we integrate segID
@@ -125,14 +116,6 @@ func SequenceURI(path *storage.URI, id ksuid.KSUID) *storage.URI {
 	return path.JoinPath(fmt.Sprintf("%s.bsup", id))
 }
 
-func (o Object) SeekIndexURI(path *storage.URI) *storage.URI {
-	return SeekIndexURI(path, o.ID)
-}
-
-func SeekIndexURI(path *storage.URI, id ksuid.KSUID) *storage.URI {
-	return path.JoinPath(fmt.Sprintf("%s-seek.bsup", id))
-}
-
 func (o Object) VectorURI(path *storage.URI) *storage.URI {
 	return VectorURI(path, o.ID)
 }
@@ -141,7 +124,7 @@ func VectorURI(path *storage.URI, id ksuid.KSUID) *storage.URI {
 	return path.JoinPath(fmt.Sprintf("%s.csup", id))
 }
 
-// Remove deletes the row object and its seek index.
+// Remove deletes the object.
 // Any 'not found' errors are ignored.
 func (o Object) Remove(ctx context.Context, engine storage.Engine, path *storage.URI) error {
 	if err := engine.DeleteByPrefix(ctx, o.ObjectPrefix(path)); err != nil && !errors.Is(err, fs.ErrNotExist) {

db/data/reader.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"io"
 
-	"github.com/brimdata/super/db/seekindex"
 	"github.com/brimdata/super/pkg/storage"
 )
 
@@ -19,29 +18,16 @@ type Reader struct {
 // NewReader returns a Reader for this data object. If the object has a seek index
 // and if the provided span skips part of the object, the seek index will be used to
 // limit the reading window of the returned reader.
-func (o *Object) NewReader(ctx context.Context, engine storage.Engine, path *storage.URI, ranges []seekindex.Range) (*Reader, error) {
+func (o *Object) NewReader(ctx context.Context, engine storage.Engine, path *storage.URI) (*Reader, error) {
 	objectPath := o.SequenceURI(path)
 	reader, err := engine.Get(ctx, objectPath)
 	if err != nil {
 		return nil, fmt.Errorf("%s: %w", objectPath, err)
 	}
-	var r io.Reader
-	var readBytes int64
-	if len(ranges) == 0 {
-		r = reader
-		readBytes = o.Size
-	} else {
-		readers := make([]io.Reader, 0, len(ranges))
-		for _, rg := range ranges {
-			readers = append(readers, io.NewSectionReader(reader, rg.Offset, rg.Length))
-			readBytes += rg.Length
-		}
-		r = io.MultiReader(readers...)
-	}
 	return &Reader{
-		Reader:     r,
+		Reader:     reader,
 		Closer:     reader,
 		TotalBytes: o.Size,
-		ReadBytes:  readBytes,
+		ReadBytes:  o.Size,
 	}, nil
 }