Re-work segment.Fields/Terms to use iterators

.excludemetalint

@@ -2,4 +2,4 @@ vendor/
 generated/
 _mock.go
 .pb.go
-index/segment/mem/ids_map_gen.go
+_gen.go

generated-source-files.mk

@@ -17,7 +17,7 @@ install-m3x-repo: install-glide install-generics-bin
 
 # Generation rule for all generated types
 .PHONY: genny-all
-genny-all: genny-map-all
+genny-all: genny-map-all genny-arraypool-all
 
 # Tests that all currently generated types match their contents if they were regenerated
 .PHONY: test-genny-all
@@ -115,3 +115,21 @@ genny-map-segment-fs-fst-terms-offset: install-m3x-repo
 		rename_type_prefix=fstTermsOffsets
 	# Rename generated map file
 	mv -f $(m3ninx_package_path)/index/segment/fs/map_gen.go $(m3ninx_package_path)/index/segment/fs/fst_terms_offsets_map_gen.go
+
+
+# generation rule for all generated arraypools
+.PHONY: genny-arraypool-all
+genny-arraypool-all:                     \
+	genny-arraypool-bytes-slice-array-pool
+
+# arraypool generation rule for ./x/bytes.SliceArrayPool
+.PHONY: genny-arraypool-bytes-slice-array-pool
+genny-arraypool-bytes-slice-array-pool: install-m3x-repo
+	cd $(m3x_package_path) && make genny-arraypool \
+	pkg=bytes                                      \
+	elem_type=[]byte                               \
+	target_package=$(m3ninx_package)/x/bytes       \
+	out_file=slice_arraypool_gen.go                \
+	rename_type_prefix=Slice                       \
+	rename_type_middle=Slice                       \
+	rename_constructor=NewSliceArrayPool

index/segment/fs/fst_terms_iterator.go

@@ -0,0 +1,120 @@
+// Copyright (c) 2018 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package fs
+
+import (
+	sgmt "github.com/m3db/m3ninx/index/segment"
+	"github.com/m3db/m3x/context"
+	xerrors "github.com/m3db/m3x/errors"
+
+	"github.com/couchbase/vellum"
+)
+
+type newFstTermsIterOpts struct {
+	ctx         context.Context
+	opts        Options
+	fst         *vellum.FST
+	finalizeFST bool
+}
+
+func (o *newFstTermsIterOpts) Close() error {
+	o.ctx.Close()
+	if o.finalizeFST {
+		return o.fst.Close()
+	}
+	return nil
+}
+
+func newFSTTermsIter(opts newFstTermsIterOpts) *fstTermsIter {
+	return &fstTermsIter{iterOpts: opts}
+}
+
+// nolint: maligned
+type fstTermsIter struct {
+	iterOpts newFstTermsIterOpts
+
+	initialized bool
+	iter        *vellum.FSTIterator
+	err         error
+	done        bool
+
+	current      []byte
+	currentValue uint64
+}
+
+var _ sgmt.OrderedBytesSliceIterator = &fstTermsIter{}
+
+func (f *fstTermsIter) Next() bool {
+	if f.done || f.err != nil {
+		return false
+	}
+
+	var err error
+
+	if !f.initialized {
+		f.initialized = true
+		f.iter = &vellum.FSTIterator{}
+		err = f.iter.Reset(f.iterOpts.fst, minByteKey, maxByteKey, nil)
+	} else {
+		err = f.iter.Next()
+	}
+
+	if err != nil {
+		f.done = true
+		if err != vellum.ErrIteratorDone {
+			f.err = err
+		}
+		return false
+	}
+
+	nextBytes, nextValue := f.iter.Current()
+	// taking a copy of the bytes to avoid referring to mmap'd memory
+	// TODO(prateek): back this by a bytes pool.
+	f.current = append([]byte(nil), nextBytes...)
+	f.currentValue = nextValue
+	return true
+}
+
+func (f *fstTermsIter) CurrentExtended() ([]byte, uint64) {
+	return f.current, f.currentValue
+}
+
+func (f *fstTermsIter) Current() []byte {
+	return f.current
+}
+
+func (f *fstTermsIter) Err() error {
+	return f.err
+}
+
+func (f *fstTermsIter) Close() error {
+	f.current = nil
+
+	multiErr := xerrors.MultiError{}
+
+	if f.iter != nil {
+		multiErr = multiErr.Add(f.iter.Close())
+	}
+	f.iter = nil
+
+	multiErr = multiErr.Add(f.iterOpts.Close())
+	return multiErr.FinalError()
+}

index/segment/fs/options.go

@@ -0,0 +1,106 @@
+// Copyright (c) 2018 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package fs
+
+import (
+	"github.com/m3db/m3ninx/postings"
+	"github.com/m3db/m3ninx/postings/roaring"
+	"github.com/m3db/m3ninx/x/bytes"
+
+	"github.com/m3db/m3x/instrument"
+	"github.com/m3db/m3x/pool"
+)
+
+const (
+	defaultBytesArrayPoolCapacity = 1024
+)
+
+// Options is a collection of knobs for a fs segment.
+type Options interface {
+	// SetInstrumentOptions sets the instrument options.
+	SetInstrumentOptions(value instrument.Options) Options
+
+	// InstrumentOptions returns the instrument options.
+	InstrumentOptions() instrument.Options
+
+	// SetBytesSliceArrayPool sets the bytes slice array pool.
+	SetBytesSliceArrayPool(value bytes.SliceArrayPool) Options
+
+	// BytesSliceArrayPool returns the bytes slice array pool.
+	BytesSliceArrayPool() bytes.SliceArrayPool
+
+	// SetPostingsListPool sets the postings list pool.
+	SetPostingsListPool(value postings.Pool) Options
+
+	// PostingsListPool returns the postings list pool.
+	PostingsListPool() postings.Pool
+}
+
+type opts struct {
+	iopts             instrument.Options
+	bytesSliceArrPool bytes.SliceArrayPool
+	postingsPool      postings.Pool
+}
+
+// NewOptions returns new options.
+func NewOptions() Options {
+	arrPool := bytes.NewSliceArrayPool(bytes.SliceArrayPoolOpts{
+		Capacity: defaultBytesArrayPoolCapacity,
+		Options:  pool.NewObjectPoolOptions(),
+	})
+	arrPool.Init()
+
+	return &opts{
+		iopts:             instrument.NewOptions(),
+		bytesSliceArrPool: arrPool,
+		postingsPool:      postings.NewPool(nil, roaring.NewPostingsList),
+	}
+}
+
+func (o *opts) SetInstrumentOptions(v instrument.Options) Options {
+	opts := *o
+	opts.iopts = v
+	return &opts
+}
+
+func (o *opts) InstrumentOptions() instrument.Options {
+	return o.iopts
+}
+
+func (o *opts) SetBytesSliceArrayPool(value bytes.SliceArrayPool) Options {
+	opts := *o
+	opts.bytesSliceArrPool = value
+	return &opts
+}
+
+func (o *opts) BytesSliceArrayPool() bytes.SliceArrayPool {
+	return o.bytesSliceArrPool
+}
+
+func (o *opts) SetPostingsListPool(v postings.Pool) Options {
+	opts := *o
+	opts.postingsPool = v
+	return &opts
+}
+
+func (o *opts) PostingsListPool() postings.Pool {
+	return o.postingsPool
+}