diff --git a/.excludemetalint b/.excludemetalint index 8c44d2f..b6dbc14 100644 --- a/.excludemetalint +++ b/.excludemetalint @@ -1,5 +1,6 @@ vendor/ generated/ +index/template/ values/template/ _mock.go _gen.go diff --git a/digest/writer_mock.go b/digest/digest_mock.go similarity index 100% rename from digest/writer_mock.go rename to digest/digest_mock.go diff --git a/generated/generics/generate.go b/generated/generics/generate.go index b2de588..1ec6369 100644 --- a/generated/generics/generate.go +++ b/generated/generics/generate.go @@ -43,6 +43,12 @@ // Iterator related template instantiations. +//go:generate sh -c "cat $GOPATH/src/$PACKAGE/index/template/at_position_value_field_iterator.go | awk '/^package/{i++}i' | genny -out=$GOPATH/src/$PACKAGE/index/field/at_position_bool_field_iterator.gen.go -pkg=field -imp \"github.com/xichen2020/eventdb/values/iterator\" gen \"GenericValue=bool ForwardValueIterator=BoolIterator:iterator.ForwardBoolIterator SeekableValueIterator=SeekableBoolIterator:iterator.SeekableBoolIterator atPositionValueFieldIterator=atPositionBoolFieldIterator newAtPositionValueFieldIterator=newAtPositionBoolFieldIterator errPositionIterValueIterCountMismatch=errPositionIterBoolIterCountMismatch\"" +//go:generate sh -c "cat $GOPATH/src/$PACKAGE/index/template/at_position_value_field_iterator.go | awk '/^package/{i++}i' | genny -out=$GOPATH/src/$PACKAGE/index/field/at_position_int_field_iterator.gen.go -pkg=field -imp \"github.com/xichen2020/eventdb/values/iterator\" gen \"GenericValue=int ForwardValueIterator=IntIterator:iterator.ForwardIntIterator SeekableValueIterator=SeekableIntIterator:iterator.SeekableIntIterator atPositionValueFieldIterator=atPositionIntFieldIterator newAtPositionValueFieldIterator=newAtPositionIntFieldIterator errPositionIterValueIterCountMismatch=errPositionIterIntIterCountMismatch\"" +//go:generate sh -c "cat $GOPATH/src/$PACKAGE/index/template/at_position_value_field_iterator.go | awk '/^package/{i++}i' | genny -out=$GOPATH/src/$PACKAGE/index/field/at_position_double_field_iterator.gen.go -pkg=field -imp \"github.com/xichen2020/eventdb/values/iterator\" gen \"GenericValue=float64 ForwardValueIterator=DoubleIterator:iterator.ForwardDoubleIterator SeekableValueIterator=SeekableDoubleIterator:iterator.SeekableDoubleIterator atPositionValueFieldIterator=atPositionDoubleFieldIterator newAtPositionValueFieldIterator=newAtPositionDoubleFieldIterator errPositionIterValueIterCountMismatch=errPositionIterDoubleIterCountMismatch\"" +//go:generate sh -c "cat $GOPATH/src/$PACKAGE/index/template/at_position_value_field_iterator.go | awk '/^package/{i++}i' | genny -out=$GOPATH/src/$PACKAGE/index/field/at_position_string_field_iterator.gen.go -pkg=field -imp \"github.com/xichen2020/eventdb/values/iterator\" gen \"GenericValue=string ForwardValueIterator=StringIterator:iterator.ForwardStringIterator SeekableValueIterator=SeekableStringIterator:iterator.SeekableStringIterator atPositionValueFieldIterator=atPositionStringFieldIterator newAtPositionValueFieldIterator=newAtPositionStringFieldIterator errPositionIterValueIterCountMismatch=errPositionIterStringIterCountMismatch\"" +//go:generate sh -c "cat $GOPATH/src/$PACKAGE/index/template/at_position_value_field_iterator.go | awk '/^package/{i++}i' | genny -out=$GOPATH/src/$PACKAGE/index/field/at_position_time_field_iterator.gen.go -pkg=field -imp \"github.com/xichen2020/eventdb/values/iterator\" gen \"GenericValue=int64 ForwardValueIterator=TimeIterator:iterator.ForwardTimeIterator SeekableValueIterator=SeekableTimeIterator:iterator.SeekableTimeIterator atPositionValueFieldIterator=atPositionTimeFieldIterator newAtPositionValueFieldIterator=newAtPositionTimeFieldIterator errPositionIterValueIterCountMismatch=errPositionIterTimeIterCountMismatch\"" + //go:generate sh -c "cat $GOPATH/src/$PACKAGE/values/template/filtered_value_iterator.go | awk '/^package/{i++}i' | genny -out=$GOPATH/src/$PACKAGE/values/iterator/impl/filtered_bool_iterator.gen.go -pkg=impl -imp \"github.com/xichen2020/eventdb/values/iterator\" -imp \"github.com/xichen2020/eventdb/filter\" gen \"GenericValue=bool ForwardValueIterator=BoolIterator:iterator.ForwardBoolIterator FilteredValueIterator=FilteredBoolIterator ValueFilter=BoolFilter:filter.BoolFilter\"" //go:generate sh -c "cat $GOPATH/src/$PACKAGE/values/template/filtered_value_iterator.go | awk '/^package/{i++}i' | genny -out=$GOPATH/src/$PACKAGE/values/iterator/impl/filtered_int_iterator.gen.go -pkg=impl -imp \"github.com/xichen2020/eventdb/values/iterator\" -imp \"github.com/xichen2020/eventdb/filter\" gen \"GenericValue=int ForwardValueIterator=IntIterator:iterator.ForwardIntIterator FilteredValueIterator=FilteredIntIterator ValueFilter=IntFilter:filter.IntFilter\"" //go:generate sh -c "cat $GOPATH/src/$PACKAGE/values/template/filtered_value_iterator.go | awk '/^package/{i++}i' | genny -out=$GOPATH/src/$PACKAGE/values/iterator/impl/filtered_double_iterator.gen.go -pkg=impl -imp \"github.com/xichen2020/eventdb/values/iterator\" -imp \"github.com/xichen2020/eventdb/filter\" gen \"GenericValue=float64 ForwardValueIterator=DoubleIterator:iterator.ForwardDoubleIterator FilteredValueIterator=FilteredDoubleIterator ValueFilter=DoubleFilter:filter.DoubleFilter\"" diff --git a/generated/mocks/generate.go b/generated/mocks/generate.go index 9c02fab..ea18cac 100644 --- a/generated/mocks/generate.go +++ b/generated/mocks/generate.go @@ -1,9 +1,9 @@ package mocks // mockgen rules for generating mocks for exported interfaces (reflection mode). -//go:generate sh -c "mockgen -package=digest $PACKAGE/digest FdWithDigestWriter | genclean -pkg $PACKAGE/digest -out $GOPATH/src/$PACKAGE/digest/writer_mock.go" -//go:generate sh -c "mockgen -package=index $PACKAGE/index DocIDSetIterator | genclean -pkg $PACKAGE/index -out $GOPATH/src/$PACKAGE/index/document_mock.go" -//go:generate sh -c "mockgen -package=iterator $PACKAGE/values/iterator ForwardBoolIterator,ForwardIntIterator,ForwardDoubleIterator,ForwardStringIterator,ForwardTimeIterator,PositionIterator | genclean -pkg $PACKAGE/values/iterator -out $GOPATH/src/$PACKAGE/values/iterator/iterator_mock.go" +//go:generate sh -c "mockgen -package=digest $PACKAGE/digest FdWithDigestWriter | genclean -pkg $PACKAGE/digest -out $GOPATH/src/$PACKAGE/digest/digest_mock.go" +//go:generate sh -c "mockgen -package=index $PACKAGE/index DocIDSetIterator,SeekableDocIDSetIterator,DocIDPositionIterator | genclean -pkg $PACKAGE/index -out $GOPATH/src/$PACKAGE/index/index_mock.go" +//go:generate sh -c "mockgen -package=iterator $PACKAGE/values/iterator ForwardBoolIterator,ForwardIntIterator,ForwardDoubleIterator,ForwardStringIterator,ForwardTimeIterator,SeekableBoolIterator,SeekableIntIterator,SeekableDoubleIterator,SeekableStringIterator,SeekableTimeIterator,PositionIterator | genclean -pkg $PACKAGE/values/iterator -out $GOPATH/src/$PACKAGE/values/iterator/iterator_mock.go" //go:generate sh -c "mockgen -package=values $PACKAGE/values BoolValues,IntValues,DoubleValues,StringValues,TimeValues | genclean -pkg $PACKAGE/values -out $GOPATH/src/$PACKAGE/values/values_mock.go" // mockgen rules for generating mocks for unexported interfaces (file mode). diff --git a/index/at_position_doc_id_set_iterator.go b/index/at_position_doc_id_set_iterator.go index e2c51ab..ab990d3 100644 --- a/index/at_position_doc_id_set_iterator.go +++ b/index/at_position_doc_id_set_iterator.go @@ -1,18 +1,27 @@ package index import ( + "errors" + "github.com/xichen2020/eventdb/values/iterator" ) +var ( + errPositionIterDocIDIterCountMismatch = errors.New("doc ID iterator and the position iterator iterator count mismatch") +) + // AtPositionDocIDSetIterator outputs the doc IDs from the doc ID set iterator at the // given positions from the position iterator. type AtPositionDocIDSetIterator struct { - docIt DocIDSetIterator - positionIt iterator.PositionIterator + docIt DocIDSetIterator + seekableDocIt SeekableDocIDSetIterator + positionIt iterator.PositionIterator + done bool + err error + firstTime bool currDocID int32 currPos int - done bool } // NewAtPositionDocIDSetIterator creates a new at position iterator. @@ -20,31 +29,55 @@ func NewAtPositionDocIDSetIterator( docIt DocIDSetIterator, positionIt iterator.PositionIterator, ) *AtPositionDocIDSetIterator { + seekableDocIt, _ := docIt.(SeekableDocIDSetIterator) + if seekableDocIt != nil { + docIt = nil + } return &AtPositionDocIDSetIterator{ - docIt: docIt, - positionIt: positionIt, + docIt: docIt, + seekableDocIt: seekableDocIt, + positionIt: positionIt, + firstTime: true, } } // Next returns true if there are more doc IDs to be iterated over. func (it *AtPositionDocIDSetIterator) Next() bool { - if it.done { + if it.done || it.err != nil { return false } if !it.positionIt.Next() { it.done = true return false } - nextPos := it.positionIt.Current() + nextPos := it.positionIt.Position() distance := nextPos - it.currPos - // TODO(xichen): Look into optimizations to speed this up if the doc ID set iterator - // supports a `Seek` or `Advance` API. - for i := 0; i < distance; i++ { - if !it.docIt.Next() { - panic("doc ID iterator and the position iterator iterator count mismatch") + + // We have a next position, now advance the doc ID set iterator for the first time. + if it.firstTime { + it.firstTime = false + if hasNoValues := + (it.seekableDocIt != nil && !it.seekableDocIt.Next()) || + (it.docIt != nil && !it.docIt.Next()); hasNoValues { + it.err = errPositionIterDocIDIterCountMismatch + return false } } - it.currDocID = it.docIt.DocID() + + if it.seekableDocIt != nil { + if it.err = it.seekableDocIt.SeekForward(distance); it.err != nil { + return false + } + it.currDocID = it.seekableDocIt.DocID() + } else { + for i := 0; i < distance; i++ { + if !it.docIt.Next() { + it.err = errPositionIterDocIDIterCountMismatch + return false + } + } + it.currDocID = it.docIt.DocID() + } it.currPos = nextPos return true } @@ -52,9 +85,18 @@ func (it *AtPositionDocIDSetIterator) Next() bool { // DocID returns the current doc ID. func (it *AtPositionDocIDSetIterator) DocID() int32 { return it.currDocID } +// Err returns any error encountered. +func (it *AtPositionDocIDSetIterator) Err() error { return it.err } + // Close closes the iterator. func (it *AtPositionDocIDSetIterator) Close() { - it.docIt.Close() - it.docIt = nil + if it.docIt != nil { + it.docIt.Close() + it.docIt = nil + } else { + it.seekableDocIt.Close() + it.seekableDocIt = nil + } it.positionIt = nil + it.err = nil } diff --git a/index/at_position_doc_id_set_iterator_test.go b/index/at_position_doc_id_set_iterator_test.go index 1582348..942d744 100644 --- a/index/at_position_doc_id_set_iterator_test.go +++ b/index/at_position_doc_id_set_iterator_test.go @@ -10,7 +10,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestAtPositionDocIDSetIterator(t *testing.T) { +func TestAtPositionDocIDSetIteratorForwardOnly(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -28,19 +28,58 @@ func TestAtPositionDocIDSetIterator(t *testing.T) { mockPositionIt := iterator.NewMockPositionIterator(ctrl) gomock.InOrder( mockPositionIt.EXPECT().Next().Return(true), - mockPositionIt.EXPECT().Current().Return(2), + mockPositionIt.EXPECT().Position().Return(2), mockPositionIt.EXPECT().Next().Return(true), - mockPositionIt.EXPECT().Current().Return(4), + mockPositionIt.EXPECT().Position().Return(4), mockPositionIt.EXPECT().Next().Return(true), - mockPositionIt.EXPECT().Current().Return(7), + mockPositionIt.EXPECT().Position().Return(7), mockPositionIt.EXPECT().Next().Return(false), ) atPositionIt := NewAtPositionDocIDSetIterator(docIDSetIter, mockPositionIt) + defer atPositionIt.Close() - expected := []int32{5, 20, 90} + expected := []int32{7, 54, 107} var actual []int32 for atPositionIt.Next() { actual = append(actual, atPositionIt.DocID()) } require.Equal(t, expected, actual) } + +func TestAtPositionDocIDSetIteratorSeekable(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + docIDSetIter := NewMockSeekableDocIDSetIterator(ctrl) + gomock.InOrder( + docIDSetIter.EXPECT().Next().Return(true), + docIDSetIter.EXPECT().SeekForward(2).Return(nil), + docIDSetIter.EXPECT().DocID().Return(int32(7)), + docIDSetIter.EXPECT().SeekForward(2).Return(nil), + docIDSetIter.EXPECT().DocID().Return(int32(54)), + docIDSetIter.EXPECT().SeekForward(3).Return(nil), + docIDSetIter.EXPECT().DocID().Return(int32(107)), + docIDSetIter.EXPECT().Close(), + ) + + mockPositionIt := iterator.NewMockPositionIterator(ctrl) + gomock.InOrder( + mockPositionIt.EXPECT().Next().Return(true), + mockPositionIt.EXPECT().Position().Return(2), + mockPositionIt.EXPECT().Next().Return(true), + mockPositionIt.EXPECT().Position().Return(4), + mockPositionIt.EXPECT().Next().Return(true), + mockPositionIt.EXPECT().Position().Return(7), + mockPositionIt.EXPECT().Next().Return(false), + ) + atPositionIt := NewAtPositionDocIDSetIterator(docIDSetIter, mockPositionIt) + defer atPositionIt.Close() + + expected := []int32{7, 54, 107} + var actual []int32 + for atPositionIt.Next() { + actual = append(actual, atPositionIt.DocID()) + } + require.NoError(t, atPositionIt.Err()) + require.Equal(t, expected, actual) +} diff --git a/index/bitmap_based_doc_id_position_iterator.go b/index/bitmap_based_doc_id_position_iterator.go new file mode 100644 index 0000000..c61e137 --- /dev/null +++ b/index/bitmap_based_doc_id_position_iterator.go @@ -0,0 +1,55 @@ +package index + +import "github.com/pilosa/pilosa/roaring" + +type bitmapBasedDocIDPositionIterator struct { + bm *roaring.Bitmap + maskingIt DocIDSetIterator + + done bool + currDocID int32 + currPosition int +} + +// nolint: deadcode +func newBitmapBasedDocIDPositionIterator( + bm *roaring.Bitmap, + maskingIt DocIDSetIterator, +) *bitmapBasedDocIDPositionIterator { + return &bitmapBasedDocIDPositionIterator{ + bm: bm, + maskingIt: maskingIt, + currDocID: -1, + currPosition: -1, + } +} + +func (it *bitmapBasedDocIDPositionIterator) Next() bool { + if it.done { + return false + } + if !it.maskingIt.Next() { + it.done = true + return false + } + currDocID := it.maskingIt.DocID() + if !it.bm.Contains(uint64(currDocID)) { + return it.Next() + } + // Find the number of bits set between [prevDocID+1, it.currDocID+1). + prevDocID := it.currDocID + it.currDocID = currDocID + numBitsSet := it.bm.CountRange(uint64(prevDocID+1), uint64(it.currDocID+1)) + it.currPosition += int(numBitsSet) + return true +} + +func (it *bitmapBasedDocIDPositionIterator) DocID() int32 { return it.currDocID } + +func (it *bitmapBasedDocIDPositionIterator) Position() int { return it.currPosition } + +func (it *bitmapBasedDocIDPositionIterator) Close() { + it.bm = nil + it.maskingIt.Close() + it.maskingIt = nil +} diff --git a/index/bitmap_based_doc_id_position_iterator_bench_test.go b/index/bitmap_based_doc_id_position_iterator_bench_test.go new file mode 100644 index 0000000..1768b75 --- /dev/null +++ b/index/bitmap_based_doc_id_position_iterator_bench_test.go @@ -0,0 +1,57 @@ +package index + +import ( + "testing" + + "github.com/pilosa/pilosa/roaring" +) + +// Summary: The default doc ID position iterator is as fast as the custom one, with +// fewer memory allocations. + +func BenchmarkDefaultBitmapDocIDPositionIterator(b *testing.B) { + bm := initBenchBitmap(benchNumTotalDocs, 5) + ds := initBenchDocIDSet(benchNumTotalDocs, 8) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + bmIt := newBitmapBasedDocIDSet(bm).Iter() + dsIt := newArrayBasedDocIDSetIterator(ds) + defaultIt := NewDocIDPositionIterator(bmIt, dsIt) + count := 0 + for defaultIt.Next() { + benchDocID = defaultIt.DocID() + benchPos = defaultIt.Position() + count++ + } + } +} + +func BenchmarkCustomBitmapDocIDPositionIterator(b *testing.B) { + bm := initBenchBitmap(benchNumTotalDocs, 5) + ds := initBenchDocIDSet(benchNumTotalDocs, 8) + + bm.Optimize() + b.ResetTimer() + for i := 0; i < b.N; i++ { + dsIt := newArrayBasedDocIDSetIterator(ds) + it := newBitmapBasedDocIDPositionIterator(bm, dsIt) + count := 0 + for it.Next() { + benchDocID = it.DocID() + benchPos = it.Position() + count++ + } + } +} + +// nolint: unparam +func initBenchBitmap(n int, everyN int) *roaring.Bitmap { + bm := roaring.NewBitmap() + for j := 0; j < n; j++ { + if j%everyN == 0 { + bm.DirectAdd(uint64(j)) + } + } + return bm +} diff --git a/index/bitmap_based_doc_id_position_iterator_test.go b/index/bitmap_based_doc_id_position_iterator_test.go new file mode 100644 index 0000000..e8c7fa1 --- /dev/null +++ b/index/bitmap_based_doc_id_position_iterator_test.go @@ -0,0 +1,41 @@ +package index + +import ( + "testing" + + "github.com/golang/mock/gomock" + "github.com/pilosa/pilosa/roaring" + "github.com/stretchr/testify/require" +) + +func TestBitmapBasedDocIDPositionIterator(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + bm := roaring.NewBitmap(0, 3, 5, 9, 12, 19, 23) + + maskingIt := NewMockDocIDSetIterator(ctrl) + gomock.InOrder( + maskingIt.EXPECT().Next().Return(true), + maskingIt.EXPECT().DocID().Return(int32(0)).MinTimes(1), + maskingIt.EXPECT().Next().Return(true), + maskingIt.EXPECT().DocID().Return(int32(7)).MinTimes(1), + maskingIt.EXPECT().Next().Return(true), + maskingIt.EXPECT().DocID().Return(int32(12)).MinTimes(1), + maskingIt.EXPECT().Next().Return(false).AnyTimes(), + ) + + var ( + docIDs []int32 + positions []int + expectedDocIDs = []int32{0, 12} + expectedPositions = []int{0, 4} + ) + it := newBitmapBasedDocIDPositionIterator(bm, maskingIt) + for it.Next() { + docIDs = append(docIDs, it.DocID()) + positions = append(positions, it.Position()) + } + require.Equal(t, expectedDocIDs, docIDs) + require.Equal(t, expectedPositions, positions) +} diff --git a/index/bitmap_based_doc_id_set_iterator.go b/index/bitmap_based_doc_id_set_iterator.go new file mode 100644 index 0000000..13e75d8 --- /dev/null +++ b/index/bitmap_based_doc_id_set_iterator.go @@ -0,0 +1,38 @@ +package index + +import "github.com/pilosa/pilosa/roaring" + +type bitmapBasedDocIDIterator struct { + rit *roaring.Iterator + + closed bool + done bool + curr int32 +} + +func newbitmapBasedDocIDIterator(rit *roaring.Iterator) *bitmapBasedDocIDIterator { + return &bitmapBasedDocIDIterator{rit: rit} +} + +func (it *bitmapBasedDocIDIterator) Next() bool { + if it.done || it.closed { + return false + } + curr, eof := it.rit.Next() + if eof { + it.done = true + return false + } + it.curr = int32(curr) + return true +} + +func (it *bitmapBasedDocIDIterator) DocID() int32 { return it.curr } + +func (it *bitmapBasedDocIDIterator) Close() { + if it.closed { + return + } + it.closed = true + it.rit = nil +} diff --git a/index/doc_id_position_iterator.go b/index/doc_id_position_iterator.go new file mode 100644 index 0000000..b624cd8 --- /dev/null +++ b/index/doc_id_position_iterator.go @@ -0,0 +1,91 @@ +package index + +// DocIDPositionIterator is an iterator that contains a backing doc ID set iterator +// and a "masking" doc ID set iterator. The iterator outputs the position of doc IDs +// in the backing doc ID set iterator, as well as the doc IDs at the corresponding positions. +// It implements both `DocIDSetIterator` and `PositionIterator` interface. +type DocIDPositionIterator interface { + DocIDSetIterator + + Position() int +} + +type docIDPositionIterator struct { + backingIt DocIDSetIterator + maskingIt DocIDSetIterator + + backingDone bool + maskingDone bool + backingDocID int32 + maskingDocID int32 + currPosition int +} + +// NewDocIDPositionIterator creates a new doc ID position iterator. +func NewDocIDPositionIterator( + backingIt DocIDSetIterator, + maskingIt DocIDSetIterator, +) DocIDPositionIterator { + return &docIDPositionIterator{ + backingIt: backingIt, + maskingIt: maskingIt, + backingDocID: invalidDocID, + maskingDocID: invalidDocID, + currPosition: -1, + } +} + +// Next returns true if there are more items to be iterated over. +func (it *docIDPositionIterator) Next() bool { + if it.backingDone || it.maskingDone { + return false + } + it.advanceBackingIter() + if it.backingDone { + return false + } + if it.maskingDocID == invalidDocID { + it.advanceMaskingIter() + } + for { + if it.maskingDone { + return false + } + if it.backingDocID == it.maskingDocID { + return true + } + if it.backingDocID < it.maskingDocID { + return it.Next() + } + it.advanceMaskingIter() + } +} + +// DocID returns the current doc ID. +func (it *docIDPositionIterator) DocID() int32 { return it.backingDocID } + +// Position returns the current doc ID position. +func (it *docIDPositionIterator) Position() int { return it.currPosition } + +// Close closes the iterator. +func (it *docIDPositionIterator) Close() { + it.backingIt.Close() + it.maskingIt.Close() +} + +func (it *docIDPositionIterator) advanceBackingIter() { + if it.backingIt.Next() { + it.backingDocID = it.backingIt.DocID() + it.currPosition++ + } else { + it.backingDone = true + } +} + +func (it *docIDPositionIterator) advanceMaskingIter() { + if it.maskingIt.Next() { + it.maskingDocID = it.maskingIt.DocID() + } else { + it.maskingDone = true + } +} diff --git a/index/doc_id_position_iterator_test.go b/index/doc_id_position_iterator_test.go new file mode 100644 index 0000000..c032fc5 --- /dev/null +++ b/index/doc_id_position_iterator_test.go @@ -0,0 +1,148 @@ +package index + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/golang/mock/gomock" +) + +func TestDocIDPositionIteratorNoOverlap(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + backingIt := NewMockDocIDSetIterator(ctrl) + gomock.InOrder( + backingIt.EXPECT().Next().Return(true), + backingIt.EXPECT().DocID().Return(int32(3)).MinTimes(1), + backingIt.EXPECT().Next().Return(true), + backingIt.EXPECT().DocID().Return(int32(4)).MinTimes(1), + backingIt.EXPECT().Next().Return(true), + backingIt.EXPECT().DocID().Return(int32(7)).MinTimes(1), + backingIt.EXPECT().Next().Return(true), + backingIt.EXPECT().DocID().Return(int32(10)).MinTimes(1), + backingIt.EXPECT().Next().Return(true), + backingIt.EXPECT().DocID().Return(int32(15)).MinTimes(1), + backingIt.EXPECT().Next().Return(false).AnyTimes(), + ) + + maskingIt := NewMockDocIDSetIterator(ctrl) + gomock.InOrder( + maskingIt.EXPECT().Next().Return(true), + maskingIt.EXPECT().DocID().Return(int32(2)).MinTimes(1), + maskingIt.EXPECT().Next().Return(true), + maskingIt.EXPECT().DocID().Return(int32(6)).MinTimes(1), + maskingIt.EXPECT().Next().Return(true), + maskingIt.EXPECT().DocID().Return(int32(13)).MinTimes(1), + maskingIt.EXPECT().Next().Return(false).AnyTimes(), + ) + + var ( + docIDs []int32 + positions []int + ) + it := NewDocIDPositionIterator(backingIt, maskingIt) + for it.Next() { + docIDs = append(docIDs, it.DocID()) + positions = append(positions, it.Position()) + } + require.Equal(t, 0, len(docIDs)) + require.Equal(t, 0, len(positions)) +} + +func TestDocIDPositionIteratorAllOverlap(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + backingIt := NewMockDocIDSetIterator(ctrl) + gomock.InOrder( + backingIt.EXPECT().Next().Return(true), + backingIt.EXPECT().DocID().Return(int32(3)).MinTimes(1), + backingIt.EXPECT().Next().Return(true), + backingIt.EXPECT().DocID().Return(int32(4)).MinTimes(1), + backingIt.EXPECT().Next().Return(true), + backingIt.EXPECT().DocID().Return(int32(7)).MinTimes(1), + backingIt.EXPECT().Next().Return(true), + backingIt.EXPECT().DocID().Return(int32(10)).MinTimes(1), + backingIt.EXPECT().Next().Return(true), + backingIt.EXPECT().DocID().Return(int32(15)).MinTimes(1), + backingIt.EXPECT().Next().Return(false).AnyTimes(), + ) + + maskingIt := NewMockDocIDSetIterator(ctrl) + gomock.InOrder( + maskingIt.EXPECT().Next().Return(true), + maskingIt.EXPECT().DocID().Return(int32(3)).MinTimes(1), + maskingIt.EXPECT().Next().Return(true), + maskingIt.EXPECT().DocID().Return(int32(4)).MinTimes(1), + maskingIt.EXPECT().Next().Return(true), + maskingIt.EXPECT().DocID().Return(int32(7)).MinTimes(1), + maskingIt.EXPECT().Next().Return(true), + maskingIt.EXPECT().DocID().Return(int32(10)).MinTimes(1), + maskingIt.EXPECT().Next().Return(true), + maskingIt.EXPECT().DocID().Return(int32(15)).MinTimes(1), + maskingIt.EXPECT().Next().Return(false).AnyTimes(), + ) + + var ( + docIDs []int32 + positions []int + expectedDocIDs = []int32{3, 4, 7, 10, 15} + expectedPositions = []int{0, 1, 2, 3, 4} + ) + it := NewDocIDPositionIterator(backingIt, maskingIt) + for it.Next() { + docIDs = append(docIDs, it.DocID()) + positions = append(positions, it.Position()) + } + require.Equal(t, expectedDocIDs, docIDs) + require.Equal(t, expectedPositions, positions) +} + +func TestDocIDPositionIteratorPartialOverlap(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + backingIt := NewMockDocIDSetIterator(ctrl) + gomock.InOrder( + backingIt.EXPECT().Next().Return(true), + backingIt.EXPECT().DocID().Return(int32(3)).MinTimes(1), + backingIt.EXPECT().Next().Return(true), + backingIt.EXPECT().DocID().Return(int32(4)).MinTimes(1), + backingIt.EXPECT().Next().Return(true), + backingIt.EXPECT().DocID().Return(int32(7)).MinTimes(1), + backingIt.EXPECT().Next().Return(true), + backingIt.EXPECT().DocID().Return(int32(10)).MinTimes(1), + backingIt.EXPECT().Next().Return(true), + backingIt.EXPECT().DocID().Return(int32(15)).MinTimes(1), + backingIt.EXPECT().Next().Return(false).AnyTimes(), + ) + + maskingIt := NewMockDocIDSetIterator(ctrl) + gomock.InOrder( + maskingIt.EXPECT().Next().Return(true), + maskingIt.EXPECT().DocID().Return(int32(3)).MinTimes(1), + maskingIt.EXPECT().Next().Return(true), + maskingIt.EXPECT().DocID().Return(int32(8)).MinTimes(1), + maskingIt.EXPECT().Next().Return(true), + maskingIt.EXPECT().DocID().Return(int32(10)).MinTimes(1), + maskingIt.EXPECT().Next().Return(true), + maskingIt.EXPECT().DocID().Return(int32(12)).MinTimes(1), + maskingIt.EXPECT().Next().Return(false).AnyTimes(), + ) + + var ( + docIDs []int32 + positions []int + expectedDocIDs = []int32{3, 10} + expectedPositions = []int{0, 3} + ) + it := NewDocIDPositionIterator(backingIt, maskingIt) + for it.Next() { + docIDs = append(docIDs, it.DocID()) + positions = append(positions, it.Position()) + } + require.Equal(t, expectedDocIDs, docIDs) + require.Equal(t, expectedPositions, positions) +} diff --git a/index/doc_id.go b/index/doc_id_set.go similarity index 75% rename from index/doc_id.go rename to index/doc_id_set.go index 85633da..9b33f29 100644 --- a/index/doc_id.go +++ b/index/doc_id_set.go @@ -15,6 +15,10 @@ type DocIDSet interface { // Iter returns the document ID set iterator. Iter() DocIDSetIterator + // Fetch returns the set of positions in the current doc ID set that + // are also in the doc ID set given by the iterator passed in. + Fetch(it DocIDSetIterator) DocIDPositionIterator + // WriteTo writes the document ID set to an io.Writer. // NB: extBuf is an external buffer for reuse. WriteTo(writer io.Writer, extBuf *bytes.Buffer) error @@ -29,21 +33,6 @@ type unmarshallableDocIDSet interface { readFrom(buf []byte) (int, error) } -// DocIDSetIterator is the document ID set iterator. -type DocIDSetIterator interface { - // Next returns true if there are more document IDs to be iterated over. - Next() bool - - // DocID returns the current document ID. - // NB: This is not called `Current` because it needs to - // be embedded with other iterators so the method name is - // more specific w.r.t. what value this is referring to. - DocID() int32 - - // Close closes the iterator. - Close() -} - // DocIDSetBuilder builds a document ID set. type DocIDSetBuilder interface { // Add adds a single document ID. @@ -102,6 +91,10 @@ func newFullDocIDSet(numTotalDocs int32) *fullDocIDSet { func (s *fullDocIDSet) Iter() DocIDSetIterator { return NewFullDocIDSetIterator(s.numTotalDocs) } +func (s *fullDocIDSet) Fetch(it DocIDSetIterator) DocIDPositionIterator { + return newFullDocIDPositionIterator(s.numTotalDocs, it) +} + func (s *fullDocIDSet) WriteTo(writer io.Writer, _ *bytes.Buffer) error { // Write Doc ID set type. if err := xio.WriteVarint(writer, int64(fullDocIDSetType)); err != nil { @@ -164,6 +157,10 @@ func (s *bitmapBasedDocIDSet) Iter() DocIDSetIterator { return newbitmapBasedDocIDIterator(s.bm.Iterator()) } +func (s *bitmapBasedDocIDSet) Fetch(it DocIDSetIterator) DocIDPositionIterator { + return NewDocIDPositionIterator(s.Iter(), it) +} + func (s *bitmapBasedDocIDSet) WriteTo(writer io.Writer, extBuf *bytes.Buffer) error { // Write Doc ID set type. if err := xio.WriteVarint(writer, int64(bitmapBasedDocIDSetType)); err != nil { @@ -201,52 +198,3 @@ func (s *bitmapBasedDocIDSet) readFrom(buf []byte) (int, error) { bytesRead = encodeEnd return bytesRead, nil } - -type bitmapBasedDocIDIterator struct { - rit *roaring.Iterator - - closed bool - done bool - curr int32 -} - -func newbitmapBasedDocIDIterator(rit *roaring.Iterator) *bitmapBasedDocIDIterator { - return &bitmapBasedDocIDIterator{rit: rit} -} - -func (it *bitmapBasedDocIDIterator) Next() bool { - if it.done || it.closed { - return false - } - curr, eof := it.rit.Next() - if eof { - it.done = true - return false - } - it.curr = int32(curr) - return true -} - -func (it *bitmapBasedDocIDIterator) DocID() int32 { return it.curr } - -func (it *bitmapBasedDocIDIterator) Close() { - if it.closed { - return - } - it.closed = true - it.rit = nil -} - -// DocIDSetIteratorFn transforms an input doc ID set iterator into a new doc ID set iterator. -type DocIDSetIteratorFn func(it DocIDSetIterator) DocIDSetIterator - -// NoOpDocIDSetIteratorFn is a no op transformation function that returns the input iterator as is. -func NoOpDocIDSetIteratorFn(it DocIDSetIterator) DocIDSetIterator { return it } - -// ExcludeDocIDSetIteratorFn returns a transformation function that excludes the doc ID set -// associated with the input iterator from the full doc ID set. -func ExcludeDocIDSetIteratorFn(numTotalDocs int32) DocIDSetIteratorFn { - return func(it DocIDSetIterator) DocIDSetIterator { - return NewExcludeDocIDSetIterator(numTotalDocs, it) - } -} diff --git a/index/doc_id_set_iterator.go b/index/doc_id_set_iterator.go new file mode 100644 index 0000000..9df00c1 --- /dev/null +++ b/index/doc_id_set_iterator.go @@ -0,0 +1,41 @@ +package index + +// DocIDSetIterator is the document ID set iterator. +// TODO(xichen): Add `Err` API. +type DocIDSetIterator interface { + // Next returns true if there are more document IDs to be iterated over. + Next() bool + + // DocID returns the current document ID. + // NB: This is not called `Current` because it needs to + // be embedded with other iterators so the method name is + // more specific w.r.t. what value this is referring to. + DocID() int32 + + // Close closes the iterator. + Close() +} + +// SeekableDocIDSetIterator is a doc ID set iterator that can seek to positions. +// TODO(xichen): DocIDSetIterator implementations should implement this interface +// where possible to speed things up. +type SeekableDocIDSetIterator interface { + DocIDSetIterator + + // SeekForward moves the iterator forward n positions. + SeekForward(n int) error +} + +// DocIDSetIteratorFn transforms an input doc ID set iterator into a new doc ID set iterator. +type DocIDSetIteratorFn func(it DocIDSetIterator) DocIDSetIterator + +// NoOpDocIDSetIteratorFn is a no op transformation function that returns the input iterator as is. +func NoOpDocIDSetIteratorFn(it DocIDSetIterator) DocIDSetIterator { return it } + +// ExcludeDocIDSetIteratorFn returns a transformation function that excludes the doc ID set +// associated with the input iterator from the full doc ID set. +func ExcludeDocIDSetIteratorFn(numTotalDocs int32) DocIDSetIteratorFn { + return func(it DocIDSetIterator) DocIDSetIterator { + return NewExcludeDocIDSetIterator(numTotalDocs, it) + } +} diff --git a/index/doc_id_test.go b/index/doc_id_set_test.go similarity index 100% rename from index/doc_id_test.go rename to index/doc_id_set_test.go diff --git a/index/document_mock.go b/index/document_mock.go deleted file mode 100644 index 9e1995f..0000000 --- a/index/document_mock.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright (c) 2019 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -// Automatically generated by MockGen. DO NOT EDIT! -// Source: github.com/xichen2020/eventdb/index (interfaces: DocIDSetIterator) - -package index - -import ( - "github.com/golang/mock/gomock" -) - -// Mock of DocIDSetIterator interface -type MockDocIDSetIterator struct { - ctrl *gomock.Controller - recorder *_MockDocIDSetIteratorRecorder -} - -// Recorder for MockDocIDSetIterator (not exported) -type _MockDocIDSetIteratorRecorder struct { - mock *MockDocIDSetIterator -} - -func NewMockDocIDSetIterator(ctrl *gomock.Controller) *MockDocIDSetIterator { - mock := &MockDocIDSetIterator{ctrl: ctrl} - mock.recorder = &_MockDocIDSetIteratorRecorder{mock} - return mock -} - -func (_m *MockDocIDSetIterator) EXPECT() *_MockDocIDSetIteratorRecorder { - return _m.recorder -} - -func (_m *MockDocIDSetIterator) Close() { - _m.ctrl.Call(_m, "Close") -} - -func (_mr *_MockDocIDSetIteratorRecorder) Close() *gomock.Call { - return _mr.mock.ctrl.RecordCall(_mr.mock, "Close") -} - -func (_m *MockDocIDSetIterator) DocID() int32 { - ret := _m.ctrl.Call(_m, "DocID") - ret0, _ := ret[0].(int32) - return ret0 -} - -func (_mr *_MockDocIDSetIteratorRecorder) DocID() *gomock.Call { - return _mr.mock.ctrl.RecordCall(_mr.mock, "DocID") -} - -func (_m *MockDocIDSetIterator) Next() bool { - ret := _m.ctrl.Call(_m, "Next") - ret0, _ := ret[0].(bool) - return ret0 -} - -func (_mr *_MockDocIDSetIteratorRecorder) Next() *gomock.Call { - return _mr.mock.ctrl.RecordCall(_mr.mock, "Next") -} diff --git a/index/field/at_position_bool_field_iterator.gen.go b/index/field/at_position_bool_field_iterator.gen.go new file mode 100644 index 0000000..7c2b485 --- /dev/null +++ b/index/field/at_position_bool_field_iterator.gen.go @@ -0,0 +1,126 @@ +// Copyright (c) 2019 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// This file was automatically generated by genny. +// Any changes will be lost if this file is regenerated. +// see https://github.com/mauricelam/genny + +package field + +import "github.com/xichen2020/eventdb/values/iterator" +import ( + "errors" + + "github.com/xichen2020/eventdb/index" +) + +var ( + errPositionIterBoolIterCountMismatch = errors.New("value iterator and the doc ID position iterator count mismatch") +) + +type atPositionBoolFieldIterator struct { + docIDPosIt index.DocIDPositionIterator + valsIt iterator.ForwardBoolIterator + seekableValsIt iterator.SeekableBoolIterator + + done bool + err error + firstTime bool + currPos int + currDocID int32 + currVal bool +} + +func newAtPositionBoolFieldIterator( + docIDPosIt index.DocIDPositionIterator, + valsIt iterator.ForwardBoolIterator, +) *atPositionBoolFieldIterator { + seekableValsIt, _ := valsIt.(iterator.SeekableBoolIterator) + if seekableValsIt != nil { + valsIt = nil + } + return &atPositionBoolFieldIterator{ + docIDPosIt: docIDPosIt, + valsIt: valsIt, + seekableValsIt: seekableValsIt, + firstTime: true, + } +} + +func (it *atPositionBoolFieldIterator) Next() bool { + if it.done || it.err != nil { + return false + } + if !it.docIDPosIt.Next() { + it.done = true + return false + } + nextPos := it.docIDPosIt.Position() + distance := nextPos - it.currPos + + // We have a next position, now advance the values iterator for the first time. + if it.firstTime { + it.firstTime = false + if hasNoValues := + (it.seekableValsIt != nil && !it.seekableValsIt.Next()) || + (it.valsIt != nil && !it.valsIt.Next()); hasNoValues { + it.err = errPositionIterBoolIterCountMismatch + return false + } + } + + if it.seekableValsIt != nil { + if it.err = it.seekableValsIt.SeekForward(distance); it.err != nil { + return false + } + it.currVal = it.seekableValsIt.Current() + } else { + for i := 0; i < distance; i++ { + if !it.valsIt.Next() { + it.err = errPositionIterBoolIterCountMismatch + return false + } + } + it.currVal = it.valsIt.Current() + } + + it.currPos = nextPos + it.currDocID = it.docIDPosIt.DocID() + return true +} + +func (it *atPositionBoolFieldIterator) DocID() int32 { return it.currDocID } + +func (it *atPositionBoolFieldIterator) Value() bool { return it.currVal } + +func (it *atPositionBoolFieldIterator) Err() error { return it.err } + +func (it *atPositionBoolFieldIterator) Close() { + it.docIDPosIt.Close() + it.docIDPosIt = nil + if it.valsIt != nil { + it.valsIt.Close() + it.valsIt = nil + } else { + it.seekableValsIt.Close() + it.seekableValsIt = nil + } + it.err = nil +} diff --git a/index/field/at_position_double_field_iterator.gen.go b/index/field/at_position_double_field_iterator.gen.go new file mode 100644 index 0000000..ad83f4b --- /dev/null +++ b/index/field/at_position_double_field_iterator.gen.go @@ -0,0 +1,126 @@ +// Copyright (c) 2019 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// This file was automatically generated by genny. +// Any changes will be lost if this file is regenerated. +// see https://github.com/mauricelam/genny + +package field + +import "github.com/xichen2020/eventdb/values/iterator" +import ( + "errors" + + "github.com/xichen2020/eventdb/index" +) + +var ( + errPositionIterDoubleIterCountMismatch = errors.New("value iterator and the doc ID position iterator count mismatch") +) + +type atPositionDoubleFieldIterator struct { + docIDPosIt index.DocIDPositionIterator + valsIt iterator.ForwardDoubleIterator + seekableValsIt iterator.SeekableDoubleIterator + + done bool + err error + firstTime bool + currPos int + currDocID int32 + currVal float64 +} + +func newAtPositionDoubleFieldIterator( + docIDPosIt index.DocIDPositionIterator, + valsIt iterator.ForwardDoubleIterator, +) *atPositionDoubleFieldIterator { + seekableValsIt, _ := valsIt.(iterator.SeekableDoubleIterator) + if seekableValsIt != nil { + valsIt = nil + } + return &atPositionDoubleFieldIterator{ + docIDPosIt: docIDPosIt, + valsIt: valsIt, + seekableValsIt: seekableValsIt, + firstTime: true, + } +} + +func (it *atPositionDoubleFieldIterator) Next() bool { + if it.done || it.err != nil { + return false + } + if !it.docIDPosIt.Next() { + it.done = true + return false + } + nextPos := it.docIDPosIt.Position() + distance := nextPos - it.currPos + + // We have a next position, now advance the values iterator for the first time. + if it.firstTime { + it.firstTime = false + if hasNoValues := + (it.seekableValsIt != nil && !it.seekableValsIt.Next()) || + (it.valsIt != nil && !it.valsIt.Next()); hasNoValues { + it.err = errPositionIterDoubleIterCountMismatch + return false + } + } + + if it.seekableValsIt != nil { + if it.err = it.seekableValsIt.SeekForward(distance); it.err != nil { + return false + } + it.currVal = it.seekableValsIt.Current() + } else { + for i := 0; i < distance; i++ { + if !it.valsIt.Next() { + it.err = errPositionIterDoubleIterCountMismatch + return false + } + } + it.currVal = it.valsIt.Current() + } + + it.currPos = nextPos + it.currDocID = it.docIDPosIt.DocID() + return true +} + +func (it *atPositionDoubleFieldIterator) DocID() int32 { return it.currDocID } + +func (it *atPositionDoubleFieldIterator) Value() float64 { return it.currVal } + +func (it *atPositionDoubleFieldIterator) Err() error { return it.err } + +func (it *atPositionDoubleFieldIterator) Close() { + it.docIDPosIt.Close() + it.docIDPosIt = nil + if it.valsIt != nil { + it.valsIt.Close() + it.valsIt = nil + } else { + it.seekableValsIt.Close() + it.seekableValsIt = nil + } + it.err = nil +} diff --git a/index/field/at_position_int_field_iterator.gen.go b/index/field/at_position_int_field_iterator.gen.go new file mode 100644 index 0000000..3fb636b --- /dev/null +++ b/index/field/at_position_int_field_iterator.gen.go @@ -0,0 +1,126 @@ +// Copyright (c) 2019 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// This file was automatically generated by genny. +// Any changes will be lost if this file is regenerated. +// see https://github.com/mauricelam/genny + +package field + +import "github.com/xichen2020/eventdb/values/iterator" +import ( + "errors" + + "github.com/xichen2020/eventdb/index" +) + +var ( + errPositionIterIntIterCountMismatch = errors.New("value iterator and the doc ID position iterator count mismatch") +) + +type atPositionIntFieldIterator struct { + docIDPosIt index.DocIDPositionIterator + valsIt iterator.ForwardIntIterator + seekableValsIt iterator.SeekableIntIterator + + done bool + err error + firstTime bool + currPos int + currDocID int32 + currVal int +} + +func newAtPositionIntFieldIterator( + docIDPosIt index.DocIDPositionIterator, + valsIt iterator.ForwardIntIterator, +) *atPositionIntFieldIterator { + seekableValsIt, _ := valsIt.(iterator.SeekableIntIterator) + if seekableValsIt != nil { + valsIt = nil + } + return &atPositionIntFieldIterator{ + docIDPosIt: docIDPosIt, + valsIt: valsIt, + seekableValsIt: seekableValsIt, + firstTime: true, + } +} + +func (it *atPositionIntFieldIterator) Next() bool { + if it.done || it.err != nil { + return false + } + if !it.docIDPosIt.Next() { + it.done = true + return false + } + nextPos := it.docIDPosIt.Position() + distance := nextPos - it.currPos + + // We have a next position, now advance the values iterator for the first time. + if it.firstTime { + it.firstTime = false + if hasNoValues := + (it.seekableValsIt != nil && !it.seekableValsIt.Next()) || + (it.valsIt != nil && !it.valsIt.Next()); hasNoValues { + it.err = errPositionIterIntIterCountMismatch + return false + } + } + + if it.seekableValsIt != nil { + if it.err = it.seekableValsIt.SeekForward(distance); it.err != nil { + return false + } + it.currVal = it.seekableValsIt.Current() + } else { + for i := 0; i < distance; i++ { + if !it.valsIt.Next() { + it.err = errPositionIterIntIterCountMismatch + return false + } + } + it.currVal = it.valsIt.Current() + } + + it.currPos = nextPos + it.currDocID = it.docIDPosIt.DocID() + return true +} + +func (it *atPositionIntFieldIterator) DocID() int32 { return it.currDocID } + +func (it *atPositionIntFieldIterator) Value() int { return it.currVal } + +func (it *atPositionIntFieldIterator) Err() error { return it.err } + +func (it *atPositionIntFieldIterator) Close() { + it.docIDPosIt.Close() + it.docIDPosIt = nil + if it.valsIt != nil { + it.valsIt.Close() + it.valsIt = nil + } else { + it.seekableValsIt.Close() + it.seekableValsIt = nil + } + it.err = nil +} diff --git a/index/field/at_position_string_field_iterator.gen.go b/index/field/at_position_string_field_iterator.gen.go new file mode 100644 index 0000000..81d9e84 --- /dev/null +++ b/index/field/at_position_string_field_iterator.gen.go @@ -0,0 +1,126 @@ +// Copyright (c) 2019 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// This file was automatically generated by genny. +// Any changes will be lost if this file is regenerated. +// see https://github.com/mauricelam/genny + +package field + +import "github.com/xichen2020/eventdb/values/iterator" +import ( + "errors" + + "github.com/xichen2020/eventdb/index" +) + +var ( + errPositionIterStringIterCountMismatch = errors.New("value iterator and the doc ID position iterator count mismatch") +) + +type atPositionStringFieldIterator struct { + docIDPosIt index.DocIDPositionIterator + valsIt iterator.ForwardStringIterator + seekableValsIt iterator.SeekableStringIterator + + done bool + err error + firstTime bool + currPos int + currDocID int32 + currVal string +} + +func newAtPositionStringFieldIterator( + docIDPosIt index.DocIDPositionIterator, + valsIt iterator.ForwardStringIterator, +) *atPositionStringFieldIterator { + seekableValsIt, _ := valsIt.(iterator.SeekableStringIterator) + if seekableValsIt != nil { + valsIt = nil + } + return &atPositionStringFieldIterator{ + docIDPosIt: docIDPosIt, + valsIt: valsIt, + seekableValsIt: seekableValsIt, + firstTime: true, + } +} + +func (it *atPositionStringFieldIterator) Next() bool { + if it.done || it.err != nil { + return false + } + if !it.docIDPosIt.Next() { + it.done = true + return false + } + nextPos := it.docIDPosIt.Position() + distance := nextPos - it.currPos + + // We have a next position, now advance the values iterator for the first time. + if it.firstTime { + it.firstTime = false + if hasNoValues := + (it.seekableValsIt != nil && !it.seekableValsIt.Next()) || + (it.valsIt != nil && !it.valsIt.Next()); hasNoValues { + it.err = errPositionIterStringIterCountMismatch + return false + } + } + + if it.seekableValsIt != nil { + if it.err = it.seekableValsIt.SeekForward(distance); it.err != nil { + return false + } + it.currVal = it.seekableValsIt.Current() + } else { + for i := 0; i < distance; i++ { + if !it.valsIt.Next() { + it.err = errPositionIterStringIterCountMismatch + return false + } + } + it.currVal = it.valsIt.Current() + } + + it.currPos = nextPos + it.currDocID = it.docIDPosIt.DocID() + return true +} + +func (it *atPositionStringFieldIterator) DocID() int32 { return it.currDocID } + +func (it *atPositionStringFieldIterator) Value() string { return it.currVal } + +func (it *atPositionStringFieldIterator) Err() error { return it.err } + +func (it *atPositionStringFieldIterator) Close() { + it.docIDPosIt.Close() + it.docIDPosIt = nil + if it.valsIt != nil { + it.valsIt.Close() + it.valsIt = nil + } else { + it.seekableValsIt.Close() + it.seekableValsIt = nil + } + it.err = nil +} diff --git a/index/field/at_position_string_field_iterator_test.go b/index/field/at_position_string_field_iterator_test.go new file mode 100644 index 0000000..94f470a --- /dev/null +++ b/index/field/at_position_string_field_iterator_test.go @@ -0,0 +1,111 @@ +package field + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/xichen2020/eventdb/index" + "github.com/xichen2020/eventdb/values/iterator" + + "github.com/golang/mock/gomock" +) + +func TestNewAtPositionStringFieldIteratorForwardOnly(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + docPosIt := index.NewMockDocIDPositionIterator(ctrl) + gomock.InOrder( + docPosIt.EXPECT().Next().Return(true), + docPosIt.EXPECT().Position().Return(0), + docPosIt.EXPECT().DocID().Return(int32(12)), + docPosIt.EXPECT().Next().Return(true), + docPosIt.EXPECT().Position().Return(2), + docPosIt.EXPECT().DocID().Return(int32(23)), + docPosIt.EXPECT().Next().Return(true), + docPosIt.EXPECT().Position().Return(4), + docPosIt.EXPECT().DocID().Return(int32(45)), + docPosIt.EXPECT().Next().Return(false), + docPosIt.EXPECT().Close(), + ) + + valsIt := iterator.NewMockForwardStringIterator(ctrl) + gomock.InOrder( + valsIt.EXPECT().Next().Return(true), + valsIt.EXPECT().Current().Return("a"), + valsIt.EXPECT().Next().Return(true), + valsIt.EXPECT().Next().Return(true), + valsIt.EXPECT().Current().Return("c"), + valsIt.EXPECT().Next().Return(true), + valsIt.EXPECT().Next().Return(true), + valsIt.EXPECT().Current().Return("e"), + valsIt.EXPECT().Close(), + ) + + var ( + expectedDocIDs = []int32{12, 23, 45} + expectedValues = []string{"a", "c", "e"} + actualDocIDs []int32 + actualValues []string + ) + it := newAtPositionStringFieldIterator(docPosIt, valsIt) + defer it.Close() + + for it.Next() { + actualDocIDs = append(actualDocIDs, it.DocID()) + actualValues = append(actualValues, it.Value()) + } + require.NoError(t, it.Err()) + require.Equal(t, expectedDocIDs, actualDocIDs) + require.Equal(t, expectedValues, actualValues) +} + +func TestNewAtPositionStringFieldIteratorSeekable(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + docPosIt := index.NewMockDocIDPositionIterator(ctrl) + gomock.InOrder( + docPosIt.EXPECT().Next().Return(true), + docPosIt.EXPECT().Position().Return(0), + docPosIt.EXPECT().DocID().Return(int32(12)), + docPosIt.EXPECT().Next().Return(true), + docPosIt.EXPECT().Position().Return(2), + docPosIt.EXPECT().DocID().Return(int32(23)), + docPosIt.EXPECT().Next().Return(true), + docPosIt.EXPECT().Position().Return(4), + docPosIt.EXPECT().DocID().Return(int32(45)), + docPosIt.EXPECT().Next().Return(false), + docPosIt.EXPECT().Close(), + ) + + valsIt := iterator.NewMockSeekableStringIterator(ctrl) + gomock.InOrder( + valsIt.EXPECT().Next().Return(true), + valsIt.EXPECT().SeekForward(0).Return(nil), + valsIt.EXPECT().Current().Return("a"), + valsIt.EXPECT().SeekForward(2).Return(nil), + valsIt.EXPECT().Current().Return("c"), + valsIt.EXPECT().SeekForward(2).Return(nil), + valsIt.EXPECT().Current().Return("e"), + valsIt.EXPECT().Close(), + ) + + var ( + expectedDocIDs = []int32{12, 23, 45} + expectedValues = []string{"a", "c", "e"} + actualDocIDs []int32 + actualValues []string + ) + it := newAtPositionStringFieldIterator(docPosIt, valsIt) + defer it.Close() + + for it.Next() { + actualDocIDs = append(actualDocIDs, it.DocID()) + actualValues = append(actualValues, it.Value()) + } + require.NoError(t, it.Err()) + require.Equal(t, expectedDocIDs, actualDocIDs) + require.Equal(t, expectedValues, actualValues) +} diff --git a/index/field/at_position_time_field_iterator.gen.go b/index/field/at_position_time_field_iterator.gen.go new file mode 100644 index 0000000..398e20a --- /dev/null +++ b/index/field/at_position_time_field_iterator.gen.go @@ -0,0 +1,126 @@ +// Copyright (c) 2019 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// This file was automatically generated by genny. +// Any changes will be lost if this file is regenerated. +// see https://github.com/mauricelam/genny + +package field + +import "github.com/xichen2020/eventdb/values/iterator" +import ( + "errors" + + "github.com/xichen2020/eventdb/index" +) + +var ( + errPositionIterTimeIterCountMismatch = errors.New("value iterator and the doc ID position iterator count mismatch") +) + +type atPositionTimeFieldIterator struct { + docIDPosIt index.DocIDPositionIterator + valsIt iterator.ForwardTimeIterator + seekableValsIt iterator.SeekableTimeIterator + + done bool + err error + firstTime bool + currPos int + currDocID int32 + currVal int64 +} + +func newAtPositionTimeFieldIterator( + docIDPosIt index.DocIDPositionIterator, + valsIt iterator.ForwardTimeIterator, +) *atPositionTimeFieldIterator { + seekableValsIt, _ := valsIt.(iterator.SeekableTimeIterator) + if seekableValsIt != nil { + valsIt = nil + } + return &atPositionTimeFieldIterator{ + docIDPosIt: docIDPosIt, + valsIt: valsIt, + seekableValsIt: seekableValsIt, + firstTime: true, + } +} + +func (it *atPositionTimeFieldIterator) Next() bool { + if it.done || it.err != nil { + return false + } + if !it.docIDPosIt.Next() { + it.done = true + return false + } + nextPos := it.docIDPosIt.Position() + distance := nextPos - it.currPos + + // We have a next position, now advance the values iterator for the first time. + if it.firstTime { + it.firstTime = false + if hasNoValues := + (it.seekableValsIt != nil && !it.seekableValsIt.Next()) || + (it.valsIt != nil && !it.valsIt.Next()); hasNoValues { + it.err = errPositionIterTimeIterCountMismatch + return false + } + } + + if it.seekableValsIt != nil { + if it.err = it.seekableValsIt.SeekForward(distance); it.err != nil { + return false + } + it.currVal = it.seekableValsIt.Current() + } else { + for i := 0; i < distance; i++ { + if !it.valsIt.Next() { + it.err = errPositionIterTimeIterCountMismatch + return false + } + } + it.currVal = it.valsIt.Current() + } + + it.currPos = nextPos + it.currDocID = it.docIDPosIt.DocID() + return true +} + +func (it *atPositionTimeFieldIterator) DocID() int32 { return it.currDocID } + +func (it *atPositionTimeFieldIterator) Value() int64 { return it.currVal } + +func (it *atPositionTimeFieldIterator) Err() error { return it.err } + +func (it *atPositionTimeFieldIterator) Close() { + it.docIDPosIt.Close() + it.docIDPosIt = nil + if it.valsIt != nil { + it.valsIt.Close() + it.valsIt = nil + } else { + it.seekableValsIt.Close() + it.seekableValsIt = nil + } + it.err = nil +} diff --git a/index/field/bool_field.go b/index/field/bool_field.go index 2df6e63..20b058a 100644 --- a/index/field/bool_field.go +++ b/index/field/bool_field.go @@ -12,7 +12,6 @@ import ( ) // BoolField contains data in documents for which such field are bool values. -// TODO(xichen): Potentially support query APIs. type BoolField interface { // DocIDSet returns the doc ID set for which the documents have bool values. DocIDSet() index.DocIDSet @@ -28,6 +27,11 @@ type BoolField interface { filterValue *field.ValueUnion, numTotalDocs int32, ) (index.DocIDSetIterator, error) + + // Fetch fetches the field values from the set of documents given by + // the doc ID set iterator passed in. If the field doesn't exist in + // a document from the doc ID set iterator output, it is ignored. + Fetch(it index.DocIDSetIterator) (BoolFieldIterator, error) } // CloseableBoolField is a bool field that can be closed. @@ -120,6 +124,15 @@ func (f *boolField) Filter( return index.NewAtPositionDocIDSetIterator(docIDSetIter, positionIt), nil } +func (f *boolField) Fetch(it index.DocIDSetIterator) (BoolFieldIterator, error) { + valsIt, err := f.values.Iter() + if err != nil { + return nil, err + } + docIDPosIt := f.docIDSet.Fetch(it) + return newAtPositionBoolFieldIterator(docIDPosIt, valsIt), nil +} + func (f *boolField) ShallowCopy() CloseableBoolField { f.IncRef() shallowCopy := *f diff --git a/index/field/double_field.go b/index/field/double_field.go index d6891b4..b24da96 100644 --- a/index/field/double_field.go +++ b/index/field/double_field.go @@ -12,7 +12,6 @@ import ( ) // DoubleField contains data in documents for which such field are double values. -// TODO(xichen): Potentially support query APIs. type DoubleField interface { // DocIDSet returns the doc ID set for which the documents have double values. DocIDSet() index.DocIDSet @@ -28,6 +27,11 @@ type DoubleField interface { filterValue *field.ValueUnion, numTotalDocs int32, ) (index.DocIDSetIterator, error) + + // Fetch fetches the field values from the set of documents given by + // the doc ID set iterator passed in. If the field doesn't exist in + // a document from the doc ID set iterator output, it is ignored. + Fetch(it index.DocIDSetIterator) (DoubleFieldIterator, error) } // CloseableDoubleField is a double field that can be closed. @@ -120,6 +124,15 @@ func (f *doubleField) Filter( return index.NewAtPositionDocIDSetIterator(docIDSetIter, positionIt), nil } +func (f *doubleField) Fetch(it index.DocIDSetIterator) (DoubleFieldIterator, error) { + valsIt, err := f.values.Iter() + if err != nil { + return nil, err + } + docIDPosIt := f.docIDSet.Fetch(it) + return newAtPositionDoubleFieldIterator(docIDPosIt, valsIt), nil +} + func (f *doubleField) ShallowCopy() CloseableDoubleField { f.IncRef() shallowCopy := *f diff --git a/index/field/field_iterator.go b/index/field/field_iterator.go new file mode 100644 index 0000000..9060772 --- /dev/null +++ b/index/field/field_iterator.go @@ -0,0 +1,63 @@ +package field + +import ( + "github.com/xichen2020/eventdb/index" +) + +// NB(xichen): Hand-written typed field iterator interfaces for (doc ID, value) pairs +// because genny doesn't support generating typed interfaces. + +// BoolFieldIterator iterates over (doc ID, bool value) pairs in a bool field. +type BoolFieldIterator interface { + index.DocIDSetIterator + + // Value returns the current bool value. + Value() bool + + // Err returns the error encountered if any. + Err() error +} + +// IntFieldIterator iterates over (doc ID, int value) pairs in an int field. +type IntFieldIterator interface { + index.DocIDSetIterator + + // Value returns the current int value. + Value() int + + // Err returns the error encountered if any. + Err() error +} + +// DoubleFieldIterator iterates over (doc ID, double value) pairs in a double field. +type DoubleFieldIterator interface { + index.DocIDSetIterator + + // Value returns the current double value. + Value() float64 + + // Err returns the error encountered if any. + Err() error +} + +// StringFieldIterator iterates over (doc ID, string value) pairs in a string field. +type StringFieldIterator interface { + index.DocIDSetIterator + + // Value returns the current string value. + Value() string + + // Err returns the error encountered if any. + Err() error +} + +// TimeFieldIterator iterates over (doc ID, time value) pairs in a time field. +type TimeFieldIterator interface { + index.DocIDSetIterator + + // Value returns the current time value in nanoseconds. + Value() int64 + + // Err returns the error encountered if any. + Err() error +} diff --git a/index/field/int_field.go b/index/field/int_field.go index 860ebdf..0c4bdf4 100644 --- a/index/field/int_field.go +++ b/index/field/int_field.go @@ -12,7 +12,6 @@ import ( ) // IntField contains data in documents for which such field are int values. -// TODO(xichen): Potentially support query APIs. type IntField interface { // DocIDSet returns the doc ID set for which the documents have int values. DocIDSet() index.DocIDSet @@ -28,6 +27,11 @@ type IntField interface { filterValue *field.ValueUnion, numTotalDocs int32, ) (index.DocIDSetIterator, error) + + // Fetch fetches the field values from the set of documents given by + // the doc ID set iterator passed in. If the field doesn't exist in + // a document from the doc ID set iterator output, it is ignored. + Fetch(it index.DocIDSetIterator) (IntFieldIterator, error) } // CloseableIntField is a int field that can be closed. @@ -120,6 +124,15 @@ func (f *intField) Filter( return index.NewAtPositionDocIDSetIterator(docIDSetIter, positionIt), nil } +func (f *intField) Fetch(it index.DocIDSetIterator) (IntFieldIterator, error) { + valsIt, err := f.values.Iter() + if err != nil { + return nil, err + } + docIDPosIt := f.docIDSet.Fetch(it) + return newAtPositionIntFieldIterator(docIDPosIt, valsIt), nil +} + func (f *intField) ShallowCopy() CloseableIntField { f.IncRef() shallowCopy := *f diff --git a/index/field/null_field.go b/index/field/null_field.go index 1666c3b..c786d88 100644 --- a/index/field/null_field.go +++ b/index/field/null_field.go @@ -11,7 +11,6 @@ import ( ) // NullField contains data in documents for which such field are null values. -// TODO(xichen): Potentially support query APIs. type NullField interface { // DocIDSet returns the doc ID set for which the documents have null values. DocIDSet() index.DocIDSet @@ -23,6 +22,11 @@ type NullField interface { filterValue *field.ValueUnion, numTotalDocs int32, ) (index.DocIDSetIterator, error) + + // Fetch fetches the field doc IDs from the set of documents given by + // the doc ID set iterator passed in. If the field doesn't exist in + // a document from the doc ID set iterator output, it is ignored. + Fetch(it index.DocIDSetIterator) (index.DocIDSetIterator, error) } // CloseableNullField is a null field that can be closed. @@ -106,6 +110,10 @@ func (f *nullField) Filter( return docIDSetIter, nil } +func (f *nullField) Fetch(it index.DocIDSetIterator) (index.DocIDSetIterator, error) { + return f.docIDSet.Fetch(it), nil +} + func (f *nullField) ShallowCopy() CloseableNullField { f.IncRef() shallowCopy := *f diff --git a/index/field/string_field.go b/index/field/string_field.go index b37b3d5..d6354d2 100644 --- a/index/field/string_field.go +++ b/index/field/string_field.go @@ -12,7 +12,6 @@ import ( ) // StringField contains data in documents for which such field are string values. -// TODO(xichen): Potentially support query APIs. type StringField interface { // DocIDSet returns the doc ID set for which the documents have string values. DocIDSet() index.DocIDSet @@ -28,6 +27,11 @@ type StringField interface { filterValue *field.ValueUnion, numTotalDocs int32, ) (index.DocIDSetIterator, error) + + // Fetch fetches the field values from the set of documents given by + // the doc ID set iterator passed in. If the field doesn't exist in + // a document from the doc ID set iterator output, it is ignored. + Fetch(it index.DocIDSetIterator) (StringFieldIterator, error) } // CloseableStringField is a string field that can be closed. @@ -120,6 +124,15 @@ func (f *stringField) Filter( return index.NewAtPositionDocIDSetIterator(docIDSetIter, positionIt), nil } +func (f *stringField) Fetch(it index.DocIDSetIterator) (StringFieldIterator, error) { + valsIt, err := f.values.Iter() + if err != nil { + return nil, err + } + docIDPosIt := f.docIDSet.Fetch(it) + return newAtPositionStringFieldIterator(docIDPosIt, valsIt), nil +} + func (f *stringField) ShallowCopy() CloseableStringField { f.IncRef() shallowCopy := *f diff --git a/index/field/time_field.go b/index/field/time_field.go index 3ff8898..4a9e845 100644 --- a/index/field/time_field.go +++ b/index/field/time_field.go @@ -12,7 +12,6 @@ import ( ) // TimeField contains data in documents for which such field are time values. -// TODO(xichen): Potentially support query APIs. type TimeField interface { // DocIDSet returns the doc ID set for which the documents have time values. DocIDSet() index.DocIDSet @@ -28,6 +27,11 @@ type TimeField interface { filterValue *field.ValueUnion, numTotalDocs int32, ) (index.DocIDSetIterator, error) + + // Fetch fetches the field values from the set of documents given by + // the doc ID set iterator passed in. If the field doesn't exist in + // a document from the doc ID set iterator output, it is ignored. + Fetch(it index.DocIDSetIterator) (TimeFieldIterator, error) } // CloseableTimeField is a time field that can be closed. @@ -120,6 +124,15 @@ func (f *timeField) Filter( return index.NewAtPositionDocIDSetIterator(docIDSetIter, positionIt), nil } +func (f *timeField) Fetch(it index.DocIDSetIterator) (TimeFieldIterator, error) { + valsIt, err := f.values.Iter() + if err != nil { + return nil, err + } + docIDPosIt := f.docIDSet.Fetch(it) + return newAtPositionTimeFieldIterator(docIDPosIt, valsIt), nil +} + func (f *timeField) ShallowCopy() CloseableTimeField { f.IncRef() shallowCopy := *f diff --git a/index/field/typed_field_iterator.go b/index/field/typed_field_iterator.go deleted file mode 100644 index eafe01a..0000000 --- a/index/field/typed_field_iterator.go +++ /dev/null @@ -1,46 +0,0 @@ -package field - -import "github.com/xichen2020/eventdb/index" - -// NB(xichen): Value-type-specific iterator interfacees for (doc ID, value) pairs -// because genny doesn't support generating typed interfaces. - -// DocIDBoolPairIterator iterates over (doc ID, bool value) pairs. -type DocIDBoolPairIterator interface { - index.DocIDSetIterator - - // Value returns the current bool value. - Value() bool -} - -// DocIDIntPairIterator iterates over (doc ID, int value) pairs. -type DocIDIntPairIterator interface { - index.DocIDSetIterator - - // Value returns the current int value. - Value() int -} - -// DocIDDoublePairIterator iterates over (doc ID, double value) pairs. -type DocIDDoublePairIterator interface { - index.DocIDSetIterator - - // Value returns the current double value. - Value() float64 -} - -// DocIDStringPairIterator iterates over (doc ID, string value) pairs. -type DocIDStringPairIterator interface { - index.DocIDSetIterator - - // Value returns the current string value. - Value() string -} - -// DocIDTimePairIterator iterates over (doc ID, time value) pairs. -type DocIDTimePairIterator interface { - index.DocIDSetIterator - - // Value returns the current time value in nanoseconds. - Value() int64 -} diff --git a/index/full_doc_id_position_iterator.go b/index/full_doc_id_position_iterator.go new file mode 100644 index 0000000..c50ec54 --- /dev/null +++ b/index/full_doc_id_position_iterator.go @@ -0,0 +1,51 @@ +package index + +// fullDocIDPositionIterator is a (doc ID, position) iterator for a full +// doc ID set representing [0, numTotalDocs) masked by the given doc ID +// set iterator `maskingIt`. +type fullDocIDPositionIterator struct { + numTotalDocs int32 + maskingIt DocIDSetIterator + + done bool + currDocID int32 + currPosition int +} + +func newFullDocIDPositionIterator( + numTotalDocs int32, + maskingIt DocIDSetIterator, +) *fullDocIDPositionIterator { + return &fullDocIDPositionIterator{ + numTotalDocs: numTotalDocs, + maskingIt: maskingIt, + currDocID: invalidDocID, + currPosition: -1, + } +} + +func (it *fullDocIDPositionIterator) Next() bool { + if it.done { + return false + } + if !it.maskingIt.Next() { + it.done = true + return false + } + it.currPosition++ + it.currDocID = it.maskingIt.DocID() + if it.currDocID < it.numTotalDocs { + return true + } + it.done = true + return false +} + +func (it *fullDocIDPositionIterator) DocID() int32 { return it.currDocID } + +func (it *fullDocIDPositionIterator) Position() int { return it.currPosition } + +func (it *fullDocIDPositionIterator) Close() { + it.maskingIt.Close() + it.maskingIt = nil +} diff --git a/index/full_doc_id_position_iterator_bench_test.go b/index/full_doc_id_position_iterator_bench_test.go new file mode 100644 index 0000000..ebded3d --- /dev/null +++ b/index/full_doc_id_position_iterator_bench_test.go @@ -0,0 +1,88 @@ +package index + +import ( + "testing" +) + +// Summary: The custom full doc ID position iterator is 13x faster than the default +// doc ID position iterator implementation with fewer memory allocations. +var ( + benchNumTotalDocs = 2 * 1024 * 1024 + benchDocID int32 + benchPos int +) + +func BenchmarkDefaultFullSetDocIDPositionIterator(b *testing.B) { + fullDocIDSet := newFullDocIDSet(int32(benchNumTotalDocs)) + ds := initBenchDocIDSet(benchNumTotalDocs, 10) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + fullDocSetIt := fullDocIDSet.Iter() + maskingIt := newArrayBasedDocIDSetIterator(ds) + it := NewDocIDPositionIterator(fullDocSetIt, maskingIt) + var ( + docID int32 + pos int + ) + for it.Next() { + docID = it.DocID() + pos = it.Position() + } + benchDocID = docID + benchPos = pos + } +} + +func BenchmarkCustomFullSetDocIDPositionIterator(b *testing.B) { + ds := initBenchDocIDSet(benchNumTotalDocs, 10) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + maskingIt := newArrayBasedDocIDSetIterator(ds) + it := newFullDocIDPositionIterator(int32(benchNumTotalDocs), maskingIt) + var ( + docID int32 + pos int + ) + for it.Next() { + docID += it.DocID() + pos += it.Position() + } + benchDocID = docID + benchPos = pos + } +} + +type arrayBasedDocIDSetIterator struct { + docIDs []int32 + + currIdx int +} + +func newArrayBasedDocIDSetIterator(docIDs []int32) *arrayBasedDocIDSetIterator { + return &arrayBasedDocIDSetIterator{ + docIDs: docIDs, + currIdx: -1, + } +} + +func (it *arrayBasedDocIDSetIterator) Next() bool { + if it.currIdx >= len(it.docIDs) { + return false + } + it.currIdx++ + return it.currIdx < len(it.docIDs) +} + +func (it *arrayBasedDocIDSetIterator) DocID() int32 { return it.docIDs[it.currIdx] } + +func (it *arrayBasedDocIDSetIterator) Close() { it.docIDs = nil } + +func initBenchDocIDSet(n int, everyN int) []int32 { + var arr []int32 + for i := 0; i < n; i += everyN { + arr = append(arr, int32(i)) + } + return arr +} diff --git a/index/full_doc_id_position_iterator_test.go b/index/full_doc_id_position_iterator_test.go new file mode 100644 index 0000000..6ff8c4a --- /dev/null +++ b/index/full_doc_id_position_iterator_test.go @@ -0,0 +1,70 @@ +package index + +import ( + "testing" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" +) + +func TestFullDocIDPositionIteratorWithinRange(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + maskingIt := NewMockDocIDSetIterator(ctrl) + gomock.InOrder( + maskingIt.EXPECT().Next().Return(true), + maskingIt.EXPECT().DocID().Return(int32(2)).MinTimes(1), + maskingIt.EXPECT().Next().Return(true), + maskingIt.EXPECT().DocID().Return(int32(6)).MinTimes(1), + maskingIt.EXPECT().Next().Return(true), + maskingIt.EXPECT().DocID().Return(int32(13)).MinTimes(1), + maskingIt.EXPECT().Next().Return(false).AnyTimes(), + ) + + var ( + numTotalDocs = 14 + docIDs []int32 + positions []int + expectedDocIDs = []int32{2, 6, 13} + expectedPositions = []int{0, 1, 2} + ) + it := newFullDocIDPositionIterator(int32(numTotalDocs), maskingIt) + for it.Next() { + docIDs = append(docIDs, it.DocID()) + positions = append(positions, it.Position()) + } + require.Equal(t, expectedDocIDs, docIDs) + require.Equal(t, expectedPositions, positions) +} + +func TestFullDocIDPositionIteratorOutsideRange(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + maskingIt := NewMockDocIDSetIterator(ctrl) + gomock.InOrder( + maskingIt.EXPECT().Next().Return(true), + maskingIt.EXPECT().DocID().Return(int32(2)).MinTimes(1), + maskingIt.EXPECT().Next().Return(true), + maskingIt.EXPECT().DocID().Return(int32(6)).MinTimes(1), + maskingIt.EXPECT().Next().Return(true), + maskingIt.EXPECT().DocID().Return(int32(13)).MinTimes(1), + maskingIt.EXPECT().Next().Return(false).AnyTimes(), + ) + + var ( + numTotalDocs = 13 + docIDs []int32 + positions []int + expectedDocIDs = []int32{2, 6} + expectedPositions = []int{0, 1} + ) + it := newFullDocIDPositionIterator(int32(numTotalDocs), maskingIt) + for it.Next() { + docIDs = append(docIDs, it.DocID()) + positions = append(positions, it.Position()) + } + require.Equal(t, expectedDocIDs, docIDs) + require.Equal(t, expectedPositions, positions) +} diff --git a/index/index_mock.go b/index/index_mock.go new file mode 100644 index 0000000..c1d6d50 --- /dev/null +++ b/index/index_mock.go @@ -0,0 +1,195 @@ +// Copyright (c) 2019 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// Automatically generated by MockGen. DO NOT EDIT! +// Source: github.com/xichen2020/eventdb/index (interfaces: DocIDSetIterator,SeekableDocIDSetIterator,DocIDPositionIterator) + +package index + +import ( + "github.com/golang/mock/gomock" +) + +// Mock of DocIDSetIterator interface +type MockDocIDSetIterator struct { + ctrl *gomock.Controller + recorder *_MockDocIDSetIteratorRecorder +} + +// Recorder for MockDocIDSetIterator (not exported) +type _MockDocIDSetIteratorRecorder struct { + mock *MockDocIDSetIterator +} + +func NewMockDocIDSetIterator(ctrl *gomock.Controller) *MockDocIDSetIterator { + mock := &MockDocIDSetIterator{ctrl: ctrl} + mock.recorder = &_MockDocIDSetIteratorRecorder{mock} + return mock +} + +func (_m *MockDocIDSetIterator) EXPECT() *_MockDocIDSetIteratorRecorder { + return _m.recorder +} + +func (_m *MockDocIDSetIterator) Close() { + _m.ctrl.Call(_m, "Close") +} + +func (_mr *_MockDocIDSetIteratorRecorder) Close() *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "Close") +} + +func (_m *MockDocIDSetIterator) DocID() int32 { + ret := _m.ctrl.Call(_m, "DocID") + ret0, _ := ret[0].(int32) + return ret0 +} + +func (_mr *_MockDocIDSetIteratorRecorder) DocID() *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "DocID") +} + +func (_m *MockDocIDSetIterator) Next() bool { + ret := _m.ctrl.Call(_m, "Next") + ret0, _ := ret[0].(bool) + return ret0 +} + +func (_mr *_MockDocIDSetIteratorRecorder) Next() *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "Next") +} + +// Mock of SeekableDocIDSetIterator interface +type MockSeekableDocIDSetIterator struct { + ctrl *gomock.Controller + recorder *_MockSeekableDocIDSetIteratorRecorder +} + +// Recorder for MockSeekableDocIDSetIterator (not exported) +type _MockSeekableDocIDSetIteratorRecorder struct { + mock *MockSeekableDocIDSetIterator +} + +func NewMockSeekableDocIDSetIterator(ctrl *gomock.Controller) *MockSeekableDocIDSetIterator { + mock := &MockSeekableDocIDSetIterator{ctrl: ctrl} + mock.recorder = &_MockSeekableDocIDSetIteratorRecorder{mock} + return mock +} + +func (_m *MockSeekableDocIDSetIterator) EXPECT() *_MockSeekableDocIDSetIteratorRecorder { + return _m.recorder +} + +func (_m *MockSeekableDocIDSetIterator) Close() { + _m.ctrl.Call(_m, "Close") +} + +func (_mr *_MockSeekableDocIDSetIteratorRecorder) Close() *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "Close") +} + +func (_m *MockSeekableDocIDSetIterator) DocID() int32 { + ret := _m.ctrl.Call(_m, "DocID") + ret0, _ := ret[0].(int32) + return ret0 +} + +func (_mr *_MockSeekableDocIDSetIteratorRecorder) DocID() *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "DocID") +} + +func (_m *MockSeekableDocIDSetIterator) Next() bool { + ret := _m.ctrl.Call(_m, "Next") + ret0, _ := ret[0].(bool) + return ret0 +} + +func (_mr *_MockSeekableDocIDSetIteratorRecorder) Next() *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "Next") +} + +func (_m *MockSeekableDocIDSetIterator) SeekForward(_param0 int) error { + ret := _m.ctrl.Call(_m, "SeekForward", _param0) + ret0, _ := ret[0].(error) + return ret0 +} + +func (_mr *_MockSeekableDocIDSetIteratorRecorder) SeekForward(arg0 interface{}) *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "SeekForward", arg0) +} + +// Mock of DocIDPositionIterator interface +type MockDocIDPositionIterator struct { + ctrl *gomock.Controller + recorder *_MockDocIDPositionIteratorRecorder +} + +// Recorder for MockDocIDPositionIterator (not exported) +type _MockDocIDPositionIteratorRecorder struct { + mock *MockDocIDPositionIterator +} + +func NewMockDocIDPositionIterator(ctrl *gomock.Controller) *MockDocIDPositionIterator { + mock := &MockDocIDPositionIterator{ctrl: ctrl} + mock.recorder = &_MockDocIDPositionIteratorRecorder{mock} + return mock +} + +func (_m *MockDocIDPositionIterator) EXPECT() *_MockDocIDPositionIteratorRecorder { + return _m.recorder +} + +func (_m *MockDocIDPositionIterator) Close() { + _m.ctrl.Call(_m, "Close") +} + +func (_mr *_MockDocIDPositionIteratorRecorder) Close() *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "Close") +} + +func (_m *MockDocIDPositionIterator) DocID() int32 { + ret := _m.ctrl.Call(_m, "DocID") + ret0, _ := ret[0].(int32) + return ret0 +} + +func (_mr *_MockDocIDPositionIteratorRecorder) DocID() *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "DocID") +} + +func (_m *MockDocIDPositionIterator) Next() bool { + ret := _m.ctrl.Call(_m, "Next") + ret0, _ := ret[0].(bool) + return ret0 +} + +func (_mr *_MockDocIDPositionIteratorRecorder) Next() *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "Next") +} + +func (_m *MockDocIDPositionIterator) Position() int { + ret := _m.ctrl.Call(_m, "Position") + ret0, _ := ret[0].(int) + return ret0 +} + +func (_mr *_MockDocIDPositionIteratorRecorder) Position() *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "Position") +} diff --git a/index/template/at_position_value_field_iterator.go b/index/template/at_position_value_field_iterator.go new file mode 100644 index 0000000..c18f145 --- /dev/null +++ b/index/template/at_position_value_field_iterator.go @@ -0,0 +1,101 @@ +package template + +import ( + "errors" + + "github.com/xichen2020/eventdb/index" +) + +var ( + errPositionIterValueIterCountMismatch = errors.New("value iterator and the doc ID position iterator count mismatch") +) + +type atPositionValueFieldIterator struct { + docIDPosIt index.DocIDPositionIterator + valsIt ForwardValueIterator + seekableValsIt SeekableValueIterator + + done bool + err error + firstTime bool + currPos int + currDocID int32 + currVal GenericValue +} + +func newAtPositionValueFieldIterator( + docIDPosIt index.DocIDPositionIterator, + valsIt ForwardValueIterator, +) *atPositionValueFieldIterator { + seekableValsIt, _ := valsIt.(SeekableValueIterator) + if seekableValsIt != nil { + valsIt = nil + } + return &atPositionValueFieldIterator{ + docIDPosIt: docIDPosIt, + valsIt: valsIt, + seekableValsIt: seekableValsIt, + firstTime: true, + } +} + +func (it *atPositionValueFieldIterator) Next() bool { + if it.done || it.err != nil { + return false + } + if !it.docIDPosIt.Next() { + it.done = true + return false + } + nextPos := it.docIDPosIt.Position() + distance := nextPos - it.currPos + + // We have a next position, now advance the values iterator for the first time. + if it.firstTime { + it.firstTime = false + if hasNoValues := + (it.seekableValsIt != nil && !it.seekableValsIt.Next()) || + (it.valsIt != nil && !it.valsIt.Next()); hasNoValues { + it.err = errPositionIterValueIterCountMismatch + return false + } + } + + if it.seekableValsIt != nil { + if it.err = it.seekableValsIt.SeekForward(distance); it.err != nil { + return false + } + it.currVal = it.seekableValsIt.Current() + } else { + for i := 0; i < distance; i++ { + if !it.valsIt.Next() { + it.err = errPositionIterValueIterCountMismatch + return false + } + } + it.currVal = it.valsIt.Current() + } + + it.currPos = nextPos + it.currDocID = it.docIDPosIt.DocID() + return true +} + +func (it *atPositionValueFieldIterator) DocID() int32 { return it.currDocID } + +func (it *atPositionValueFieldIterator) Value() GenericValue { return it.currVal } + +func (it *atPositionValueFieldIterator) Err() error { return it.err } + +func (it *atPositionValueFieldIterator) Close() { + it.docIDPosIt.Close() + it.docIDPosIt = nil + if it.valsIt != nil { + it.valsIt.Close() + it.valsIt = nil + } else { + it.seekableValsIt.Close() + it.seekableValsIt = nil + } + it.err = nil +} diff --git a/index/template/generic.go b/index/template/generic.go new file mode 100644 index 0000000..74cc983 --- /dev/null +++ b/index/template/generic.go @@ -0,0 +1,25 @@ +package template + +import ( + "github.com/mauricelam/genny/generic" +) + +// GenericValue is a generic type. +type GenericValue generic.Type + +// ForwardValueIterator allows iterating over a stream of GenericValue. +type ForwardValueIterator interface { + generic.Type + + Next() bool + Err() error + Current() GenericValue + Close() +} + +// SeekableValueIterator allows iterating and seeking over a stream of GenericValue. +type SeekableValueIterator interface { + ForwardValueIterator + + SeekForward(n int) error +} diff --git a/storage/immutable_segment.go b/storage/immutable_segment.go index fc0d1f8..9219dc7 100644 --- a/storage/immutable_segment.go +++ b/storage/immutable_segment.go @@ -669,6 +669,7 @@ func (s *immutableSeg) insertFields( // nolint: unparam // TODO(xichen): Collapse filters against the same field. // TODO(xichen): Remove the nolint directive once the implementation is finished. +// TODO(xichen): Check for iteration error at the end. func applyFilters( startNanosInclusive, endNanosExclusive int64, filters []query.FilterList, diff --git a/values/decoding/delta_int_iterator.gen.go b/values/decoding/delta_int_iterator.gen.go index b5329bb..cf4fd87 100644 --- a/values/decoding/delta_int_iterator.gen.go +++ b/values/decoding/delta_int_iterator.gen.go @@ -41,7 +41,6 @@ type deltaIntIterator struct { addFn applyOpToIntIntFn negativeBit uint64 - closed bool curr int err error isDeltaValue bool @@ -62,7 +61,7 @@ func newDeltaIntIterator( // Next returns true if there are more values to be iterated over. func (it *deltaIntIterator) Next() bool { - if it.closed || it.err != nil { + if it.err != nil { return false } @@ -111,13 +110,8 @@ func (it *deltaIntIterator) Err() error { } // Close closes the iterator. -func (it *deltaIntIterator) Close() error { - if it.closed { - return nil - } - it.closed = true +func (it *deltaIntIterator) Close() { it.bitReader = nil it.err = nil it.addFn = nil - return nil } diff --git a/values/decoding/delta_time_iterator.gen.go b/values/decoding/delta_time_iterator.gen.go index 24f9a51..c4ba776 100644 --- a/values/decoding/delta_time_iterator.gen.go +++ b/values/decoding/delta_time_iterator.gen.go @@ -41,7 +41,6 @@ type deltaTimeIterator struct { addFn applyOpToTimeIntFn negativeBit uint64 - closed bool curr int64 err error isDeltaValue bool @@ -62,7 +61,7 @@ func newDeltaTimeIterator( // Next returns true if there are more values to be iterated over. func (it *deltaTimeIterator) Next() bool { - if it.closed || it.err != nil { + if it.err != nil { return false } @@ -111,13 +110,8 @@ func (it *deltaTimeIterator) Err() error { } // Close closes the iterator. -func (it *deltaTimeIterator) Close() error { - if it.closed { - return nil - } - it.closed = true +func (it *deltaTimeIterator) Close() { it.bitReader = nil it.err = nil it.addFn = nil - return nil } diff --git a/values/decoding/dictionary_based_int_iterator.go b/values/decoding/dictionary_based_int_iterator.go index 254f16f..f96373a 100644 --- a/values/decoding/dictionary_based_int_iterator.go +++ b/values/decoding/dictionary_based_int_iterator.go @@ -18,9 +18,8 @@ type dictionaryBasedIntIterator struct { bytesPerDictValue int bitsPerEncodedValue int - curr int - err error - closed bool + curr int + err error } func newDictionaryBasedIntIterator( @@ -41,7 +40,7 @@ func newDictionaryBasedIntIterator( // Next iteration. func (it *dictionaryBasedIntIterator) Next() bool { - if it.closed || it.err != nil { + if it.err != nil { return false } @@ -75,13 +74,8 @@ func (it *dictionaryBasedIntIterator) Err() error { } // Close closes the iterator. -func (it *dictionaryBasedIntIterator) Close() error { - if it.closed { - return nil - } - it.closed = true +func (it *dictionaryBasedIntIterator) Close() { it.bitReader = nil it.encodedDict = nil it.err = nil - return nil } diff --git a/values/decoding/dictionary_based_string_iterator.go b/values/decoding/dictionary_based_string_iterator.go index 5d96ca4..78d07a3 100644 --- a/values/decoding/dictionary_based_string_iterator.go +++ b/values/decoding/dictionary_based_string_iterator.go @@ -16,9 +16,8 @@ type dictionaryBasedStringIterator struct { // and should not be mutated during iteration. extDict []string - curr string - err error - closed bool + curr string + err error } func newDictionaryBasedStringIterator( @@ -33,7 +32,7 @@ func newDictionaryBasedStringIterator( // Next iteration. func (it *dictionaryBasedStringIterator) Next() bool { - if it.closed || it.err != nil { + if it.err != nil { return false } @@ -64,13 +63,8 @@ func (it *dictionaryBasedStringIterator) Err() error { } // Close the iterator. -func (it *dictionaryBasedStringIterator) Close() error { - if it.closed { - return nil - } - it.closed = true +func (it *dictionaryBasedStringIterator) Close() { it.extDict = nil it.err = nil it.reader = nil - return nil } diff --git a/values/decoding/double_decode.go b/values/decoding/double_decode.go index 486ed78..3cd6a73 100644 --- a/values/decoding/double_decode.go +++ b/values/decoding/double_decode.go @@ -49,9 +49,8 @@ func newDoubleIteratorFromMeta( type bitPatternDoubleIterator struct { encodedBytes []byte - closed bool currIdx int - curr float64 + curr uint64 err error } @@ -63,7 +62,7 @@ func newBitPatternDoubleIterator(encodedBytes []byte) *bitPatternDoubleIterator // Next iteration. func (it *bitPatternDoubleIterator) Next() bool { - if it.closed || it.err != nil { + if it.err != nil { return false } if it.currIdx == len(it.encodedBytes) { @@ -74,25 +73,19 @@ func (it *bitPatternDoubleIterator) Next() bool { it.err = fmt.Errorf("double iterator index %d out of range %d", it.currIdx+uint64SizeBytes, len(it.encodedBytes)) return false } - v := xio.ReadInt(uint64SizeBytes, it.encodedBytes[it.currIdx:]) - it.curr = math.Float64frombits(v) + it.curr = xio.ReadInt(uint64SizeBytes, it.encodedBytes[it.currIdx:]) it.currIdx += uint64SizeBytes return true } // Current returns the current double. -func (it *bitPatternDoubleIterator) Current() float64 { return it.curr } +func (it *bitPatternDoubleIterator) Current() float64 { return math.Float64frombits(it.curr) } // Err returns any error recorded while iterating. func (it *bitPatternDoubleIterator) Err() error { return it.err } // Close the iterator. -func (it *bitPatternDoubleIterator) Close() error { - if it.closed { - return nil - } - it.closed = true +func (it *bitPatternDoubleIterator) Close() { it.encodedBytes = nil it.err = nil - return nil } diff --git a/values/decoding/raw_size_string_iterator.go b/values/decoding/raw_size_string_iterator.go index b64592e..423f1af 100644 --- a/values/decoding/raw_size_string_iterator.go +++ b/values/decoding/raw_size_string_iterator.go @@ -19,10 +19,9 @@ const ( type rawSizeStringIterator struct { reader xio.Reader - closed bool - curr string - err error - buf []byte + curr string + err error + buf []byte } func newRawSizeStringIterator( @@ -36,7 +35,7 @@ func newRawSizeStringIterator( // Next iteration. func (it *rawSizeStringIterator) Next() bool { - if it.closed || it.err != nil { + if it.err != nil { return false } @@ -70,10 +69,8 @@ func (it *rawSizeStringIterator) Err() error { } // Close closes the iterator. -func (it *rawSizeStringIterator) Close() error { - it.closed = true +func (it *rawSizeStringIterator) Close() { it.buf = nil it.err = nil it.reader = nil - return nil } diff --git a/values/decoding/run_length_decode_bool.gen.go b/values/decoding/run_length_decode_bool.gen.go index f9c659a..4eaaee8 100644 --- a/values/decoding/run_length_decode_bool.gen.go +++ b/values/decoding/run_length_decode_bool.gen.go @@ -56,13 +56,12 @@ type runLengthBoolIterator struct { curr bool repetitions int64 - closed bool err error } // Next returns true if there are more values to be iterated over, and false otherwise. func (it *runLengthBoolIterator) Next() bool { - if it.closed || it.err != nil { + if it.err != nil { return false } @@ -101,15 +100,10 @@ func (it *runLengthBoolIterator) Err() error { } // Close the iterator. -func (it *runLengthBoolIterator) Close() error { - if it.closed { - return nil - } - it.closed = true +func (it *runLengthBoolIterator) Close() { it.err = nil it.reader = nil it.readValueFn = nil - return nil } func newRunLengthBoolIterator( diff --git a/values/iterator/bool_iterator.go b/values/iterator/bool_iterator.go index 247c12a..ee332ca 100644 --- a/values/iterator/bool_iterator.go +++ b/values/iterator/bool_iterator.go @@ -7,3 +7,13 @@ type ForwardBoolIterator interface { // Current returns the current value in the iteration. Current() bool } + +// SeekableBoolIterator is a bool iterator that seek to positions. +// TODO(xichen): SeekableBoolIterator implementations should implement this +// interface where possible to speed things up. +type SeekableBoolIterator interface { + ForwardBoolIterator + + // SeekForward moves the iterator forward n positions. + SeekForward(n int) error +} diff --git a/values/iterator/double_iterator.go b/values/iterator/double_iterator.go index 13d4994..fee2293 100644 --- a/values/iterator/double_iterator.go +++ b/values/iterator/double_iterator.go @@ -7,3 +7,13 @@ type ForwardDoubleIterator interface { // Current returns the current value in the iteration. Current() float64 } + +// SeekableDoubleIterator is a double iterator that seek to positions. +// TODO(xichen): SeekableDoubleIterator implementations should implement this +// interface where possible to speed things up. +type SeekableDoubleIterator interface { + ForwardDoubleIterator + + // SeekForward moves the iterator forward n positions. + SeekForward(n int) error +} diff --git a/values/iterator/impl/array_based_bool_iterator.go b/values/iterator/impl/array_based_bool_iterator.go index e3cc2fd..f3ab442 100644 --- a/values/iterator/impl/array_based_bool_iterator.go +++ b/values/iterator/impl/array_based_bool_iterator.go @@ -30,4 +30,4 @@ func (it *ArrayBasedBoolIterator) Current() bool { return it.values[it.idx] } func (it *ArrayBasedBoolIterator) Err() error { return nil } // Close closes the iterator. -func (it *ArrayBasedBoolIterator) Close() error { return nil } +func (it *ArrayBasedBoolIterator) Close() {} diff --git a/values/iterator/impl/array_based_double_iterator.go b/values/iterator/impl/array_based_double_iterator.go index f1bd5ea..e785424 100644 --- a/values/iterator/impl/array_based_double_iterator.go +++ b/values/iterator/impl/array_based_double_iterator.go @@ -30,4 +30,4 @@ func (it *ArrayBasedDoubleIterator) Current() float64 { return it.values[it.idx] func (it *ArrayBasedDoubleIterator) Err() error { return nil } // Close closes the iterator. -func (it *ArrayBasedDoubleIterator) Close() error { return nil } +func (it *ArrayBasedDoubleIterator) Close() {} diff --git a/values/iterator/impl/array_based_int_iterator.go b/values/iterator/impl/array_based_int_iterator.go index 5791e69..16ee37f 100644 --- a/values/iterator/impl/array_based_int_iterator.go +++ b/values/iterator/impl/array_based_int_iterator.go @@ -30,4 +30,4 @@ func (it *ArrayBasedIntIterator) Current() int { return it.values[it.idx] } func (it *ArrayBasedIntIterator) Err() error { return nil } // Close closes the iterator. -func (it *ArrayBasedIntIterator) Close() error { return nil } +func (it *ArrayBasedIntIterator) Close() {} diff --git a/values/iterator/impl/array_based_string_iterator.go b/values/iterator/impl/array_based_string_iterator.go index b0f6fb6..ff74c13 100644 --- a/values/iterator/impl/array_based_string_iterator.go +++ b/values/iterator/impl/array_based_string_iterator.go @@ -30,4 +30,4 @@ func (it *ArrayBasedStringIterator) Current() string { return it.values[it.idx] func (it *ArrayBasedStringIterator) Err() error { return nil } // Close closes the iterator. -func (it *ArrayBasedStringIterator) Close() error { return nil } +func (it *ArrayBasedStringIterator) Close() {} diff --git a/values/iterator/impl/array_based_time_iterator.go b/values/iterator/impl/array_based_time_iterator.go index cfa5086..461315e 100644 --- a/values/iterator/impl/array_based_time_iterator.go +++ b/values/iterator/impl/array_based_time_iterator.go @@ -31,4 +31,4 @@ func (it *ArrayBasedTimeIterator) Current() int64 { return it.timeNanos[it.idx] func (it *ArrayBasedTimeIterator) Err() error { return nil } // Close closes the iterator. -func (it *ArrayBasedTimeIterator) Close() error { return nil } +func (it *ArrayBasedTimeIterator) Close() {} diff --git a/values/iterator/impl/empty_position_iterator.go b/values/iterator/impl/empty_position_iterator.go index 12736e7..e33936c 100644 --- a/values/iterator/impl/empty_position_iterator.go +++ b/values/iterator/impl/empty_position_iterator.go @@ -12,4 +12,4 @@ type emptyPositionIterator struct{} // NewEmptyPositionIterator creates an empty selection iterator. func NewEmptyPositionIterator() iterator.PositionIterator { return emptyPositionIter } func (it emptyPositionIterator) Next() bool { return false } -func (it emptyPositionIterator) Current() int { return -1 } +func (it emptyPositionIterator) Position() int { return -1 } diff --git a/values/iterator/impl/filtered_bool_iterator.gen.go b/values/iterator/impl/filtered_bool_iterator.gen.go index 262d235..cce6c15 100644 --- a/values/iterator/impl/filtered_bool_iterator.gen.go +++ b/values/iterator/impl/filtered_bool_iterator.gen.go @@ -64,8 +64,8 @@ func (it *FilteredBoolIterator) Next() bool { return false } -// Current returns the current position. -func (it *FilteredBoolIterator) Current() int { return it.currPos } +// Position returns the current position. +func (it *FilteredBoolIterator) Position() int { return it.currPos } // Close closes the iterator. func (it *FilteredBoolIterator) Close() { diff --git a/values/iterator/impl/filtered_double_iterator.gen.go b/values/iterator/impl/filtered_double_iterator.gen.go index 18c8657..5841722 100644 --- a/values/iterator/impl/filtered_double_iterator.gen.go +++ b/values/iterator/impl/filtered_double_iterator.gen.go @@ -64,8 +64,8 @@ func (it *FilteredDoubleIterator) Next() bool { return false } -// Current returns the current position. -func (it *FilteredDoubleIterator) Current() int { return it.currPos } +// Position returns the current position. +func (it *FilteredDoubleIterator) Position() int { return it.currPos } // Close closes the iterator. func (it *FilteredDoubleIterator) Close() { diff --git a/values/iterator/impl/filtered_int_iterator.gen.go b/values/iterator/impl/filtered_int_iterator.gen.go index 6f1b346..98a761b 100644 --- a/values/iterator/impl/filtered_int_iterator.gen.go +++ b/values/iterator/impl/filtered_int_iterator.gen.go @@ -64,8 +64,8 @@ func (it *FilteredIntIterator) Next() bool { return false } -// Current returns the current position. -func (it *FilteredIntIterator) Current() int { return it.currPos } +// Position returns the current position. +func (it *FilteredIntIterator) Position() int { return it.currPos } // Close closes the iterator. func (it *FilteredIntIterator) Close() { diff --git a/values/iterator/impl/filtered_string_iterator.gen.go b/values/iterator/impl/filtered_string_iterator.gen.go index 4be0cf6..cec2544 100644 --- a/values/iterator/impl/filtered_string_iterator.gen.go +++ b/values/iterator/impl/filtered_string_iterator.gen.go @@ -64,8 +64,8 @@ func (it *FilteredStringIterator) Next() bool { return false } -// Current returns the current position. -func (it *FilteredStringIterator) Current() int { return it.currPos } +// Position returns the current position. +func (it *FilteredStringIterator) Position() int { return it.currPos } // Close closes the iterator. func (it *FilteredStringIterator) Close() { diff --git a/values/iterator/impl/filtered_time_iterator.gen.go b/values/iterator/impl/filtered_time_iterator.gen.go index 8568a85..6f4d3f3 100644 --- a/values/iterator/impl/filtered_time_iterator.gen.go +++ b/values/iterator/impl/filtered_time_iterator.gen.go @@ -64,8 +64,8 @@ func (it *FilteredTimeIterator) Next() bool { return false } -// Current returns the current position. -func (it *FilteredTimeIterator) Current() int { return it.currPos } +// Position returns the current position. +func (it *FilteredTimeIterator) Position() int { return it.currPos } // Close closes the iterator. func (it *FilteredTimeIterator) Close() { diff --git a/values/iterator/impl/scaled_time_iterator.go b/values/iterator/impl/scaled_time_iterator.go index 6ef253c..529a4fc 100644 --- a/values/iterator/impl/scaled_time_iterator.go +++ b/values/iterator/impl/scaled_time_iterator.go @@ -14,9 +14,9 @@ type scaledTimeIterator struct { valuesIt iterator.ForwardTimeIterator scaleTimeFn scaleTimeFn - closed bool - curr int64 - err error + done bool + curr int64 + err error } // NewScaledTimeIterator creates a new scaled time iterator. @@ -34,10 +34,11 @@ func NewScaledTimeIterator( // Next iteration. func (it *scaledTimeIterator) Next() bool { - if it.closed || it.err != nil { + if it.done || it.err != nil { return false } if !it.valuesIt.Next() { + it.done = true it.err = it.valuesIt.Err() return false } @@ -53,14 +54,9 @@ func (it *scaledTimeIterator) Current() int64 { return it.scaleTimeFn(it.curr, i func (it *scaledTimeIterator) Err() error { return it.err } // Close the iterator. -func (it *scaledTimeIterator) Close() error { - if it.closed { - return nil - } - it.closed = true +func (it *scaledTimeIterator) Close() { it.valuesIt.Close() it.valuesIt = nil it.scaleTimeFn = nil it.err = nil - return nil } diff --git a/values/iterator/int_iterator.go b/values/iterator/int_iterator.go index feacc79..7d79438 100644 --- a/values/iterator/int_iterator.go +++ b/values/iterator/int_iterator.go @@ -7,3 +7,13 @@ type ForwardIntIterator interface { // Current returns the current value in the iteration. Current() int } + +// SeekableIntIterator is a int iterator that seek to positions. +// TODO(xichen): SeekableIntIterator implementations should implement this +// interface where possible to speed things up. +type SeekableIntIterator interface { + ForwardIntIterator + + // SeekForward moves the iterator forward n positions. + SeekForward(n int) error +} diff --git a/values/iterator/base_iterator.go b/values/iterator/iterator.go similarity index 87% rename from values/iterator/base_iterator.go rename to values/iterator/iterator.go index f84987e..46d27fb 100644 --- a/values/iterator/base_iterator.go +++ b/values/iterator/iterator.go @@ -1,15 +1,14 @@ package iterator -import "io" - // nolint: megacheck type baseIterator interface { - io.Closer - // Next returns true if there is another value in the data stream. // If it returns false, err should be checked for errors. Next() bool // Err returns any error encountered during iteration. Err() error + + // Close closes the iterator. + Close() } diff --git a/values/iterator/iterator_mock.go b/values/iterator/iterator_mock.go index de00b1a..7ea0729 100644 --- a/values/iterator/iterator_mock.go +++ b/values/iterator/iterator_mock.go @@ -19,7 +19,7 @@ // THE SOFTWARE. // Automatically generated by MockGen. DO NOT EDIT! -// Source: github.com/xichen2020/eventdb/values/iterator (interfaces: ForwardBoolIterator,ForwardIntIterator,ForwardDoubleIterator,ForwardStringIterator,ForwardTimeIterator,PositionIterator) +// Source: github.com/xichen2020/eventdb/values/iterator (interfaces: ForwardBoolIterator,ForwardIntIterator,ForwardDoubleIterator,ForwardStringIterator,ForwardTimeIterator,SeekableBoolIterator,SeekableIntIterator,SeekableDoubleIterator,SeekableStringIterator,SeekableTimeIterator,PositionIterator) package iterator @@ -48,10 +48,8 @@ func (_m *MockForwardBoolIterator) EXPECT() *_MockForwardBoolIteratorRecorder { return _m.recorder } -func (_m *MockForwardBoolIterator) Close() error { - ret := _m.ctrl.Call(_m, "Close") - ret0, _ := ret[0].(error) - return ret0 +func (_m *MockForwardBoolIterator) Close() { + _m.ctrl.Call(_m, "Close") } func (_mr *_MockForwardBoolIteratorRecorder) Close() *gomock.Call { @@ -109,10 +107,8 @@ func (_m *MockForwardIntIterator) EXPECT() *_MockForwardIntIteratorRecorder { return _m.recorder } -func (_m *MockForwardIntIterator) Close() error { - ret := _m.ctrl.Call(_m, "Close") - ret0, _ := ret[0].(error) - return ret0 +func (_m *MockForwardIntIterator) Close() { + _m.ctrl.Call(_m, "Close") } func (_mr *_MockForwardIntIteratorRecorder) Close() *gomock.Call { @@ -170,10 +166,8 @@ func (_m *MockForwardDoubleIterator) EXPECT() *_MockForwardDoubleIteratorRecorde return _m.recorder } -func (_m *MockForwardDoubleIterator) Close() error { - ret := _m.ctrl.Call(_m, "Close") - ret0, _ := ret[0].(error) - return ret0 +func (_m *MockForwardDoubleIterator) Close() { + _m.ctrl.Call(_m, "Close") } func (_mr *_MockForwardDoubleIteratorRecorder) Close() *gomock.Call { @@ -231,10 +225,8 @@ func (_m *MockForwardStringIterator) EXPECT() *_MockForwardStringIteratorRecorde return _m.recorder } -func (_m *MockForwardStringIterator) Close() error { - ret := _m.ctrl.Call(_m, "Close") - ret0, _ := ret[0].(error) - return ret0 +func (_m *MockForwardStringIterator) Close() { + _m.ctrl.Call(_m, "Close") } func (_mr *_MockForwardStringIteratorRecorder) Close() *gomock.Call { @@ -292,10 +284,8 @@ func (_m *MockForwardTimeIterator) EXPECT() *_MockForwardTimeIteratorRecorder { return _m.recorder } -func (_m *MockForwardTimeIterator) Close() error { - ret := _m.ctrl.Call(_m, "Close") - ret0, _ := ret[0].(error) - return ret0 +func (_m *MockForwardTimeIterator) Close() { + _m.ctrl.Call(_m, "Close") } func (_mr *_MockForwardTimeIteratorRecorder) Close() *gomock.Call { @@ -332,6 +322,351 @@ func (_mr *_MockForwardTimeIteratorRecorder) Next() *gomock.Call { return _mr.mock.ctrl.RecordCall(_mr.mock, "Next") } +// Mock of SeekableBoolIterator interface +type MockSeekableBoolIterator struct { + ctrl *gomock.Controller + recorder *_MockSeekableBoolIteratorRecorder +} + +// Recorder for MockSeekableBoolIterator (not exported) +type _MockSeekableBoolIteratorRecorder struct { + mock *MockSeekableBoolIterator +} + +func NewMockSeekableBoolIterator(ctrl *gomock.Controller) *MockSeekableBoolIterator { + mock := &MockSeekableBoolIterator{ctrl: ctrl} + mock.recorder = &_MockSeekableBoolIteratorRecorder{mock} + return mock +} + +func (_m *MockSeekableBoolIterator) EXPECT() *_MockSeekableBoolIteratorRecorder { + return _m.recorder +} + +func (_m *MockSeekableBoolIterator) Close() { + _m.ctrl.Call(_m, "Close") +} + +func (_mr *_MockSeekableBoolIteratorRecorder) Close() *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "Close") +} + +func (_m *MockSeekableBoolIterator) Current() bool { + ret := _m.ctrl.Call(_m, "Current") + ret0, _ := ret[0].(bool) + return ret0 +} + +func (_mr *_MockSeekableBoolIteratorRecorder) Current() *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "Current") +} + +func (_m *MockSeekableBoolIterator) Err() error { + ret := _m.ctrl.Call(_m, "Err") + ret0, _ := ret[0].(error) + return ret0 +} + +func (_mr *_MockSeekableBoolIteratorRecorder) Err() *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "Err") +} + +func (_m *MockSeekableBoolIterator) Next() bool { + ret := _m.ctrl.Call(_m, "Next") + ret0, _ := ret[0].(bool) + return ret0 +} + +func (_mr *_MockSeekableBoolIteratorRecorder) Next() *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "Next") +} + +func (_m *MockSeekableBoolIterator) SeekForward(_param0 int) error { + ret := _m.ctrl.Call(_m, "SeekForward", _param0) + ret0, _ := ret[0].(error) + return ret0 +} + +func (_mr *_MockSeekableBoolIteratorRecorder) SeekForward(arg0 interface{}) *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "SeekForward", arg0) +} + +// Mock of SeekableIntIterator interface +type MockSeekableIntIterator struct { + ctrl *gomock.Controller + recorder *_MockSeekableIntIteratorRecorder +} + +// Recorder for MockSeekableIntIterator (not exported) +type _MockSeekableIntIteratorRecorder struct { + mock *MockSeekableIntIterator +} + +func NewMockSeekableIntIterator(ctrl *gomock.Controller) *MockSeekableIntIterator { + mock := &MockSeekableIntIterator{ctrl: ctrl} + mock.recorder = &_MockSeekableIntIteratorRecorder{mock} + return mock +} + +func (_m *MockSeekableIntIterator) EXPECT() *_MockSeekableIntIteratorRecorder { + return _m.recorder +} + +func (_m *MockSeekableIntIterator) Close() { + _m.ctrl.Call(_m, "Close") +} + +func (_mr *_MockSeekableIntIteratorRecorder) Close() *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "Close") +} + +func (_m *MockSeekableIntIterator) Current() int { + ret := _m.ctrl.Call(_m, "Current") + ret0, _ := ret[0].(int) + return ret0 +} + +func (_mr *_MockSeekableIntIteratorRecorder) Current() *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "Current") +} + +func (_m *MockSeekableIntIterator) Err() error { + ret := _m.ctrl.Call(_m, "Err") + ret0, _ := ret[0].(error) + return ret0 +} + +func (_mr *_MockSeekableIntIteratorRecorder) Err() *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "Err") +} + +func (_m *MockSeekableIntIterator) Next() bool { + ret := _m.ctrl.Call(_m, "Next") + ret0, _ := ret[0].(bool) + return ret0 +} + +func (_mr *_MockSeekableIntIteratorRecorder) Next() *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "Next") +} + +func (_m *MockSeekableIntIterator) SeekForward(_param0 int) error { + ret := _m.ctrl.Call(_m, "SeekForward", _param0) + ret0, _ := ret[0].(error) + return ret0 +} + +func (_mr *_MockSeekableIntIteratorRecorder) SeekForward(arg0 interface{}) *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "SeekForward", arg0) +} + +// Mock of SeekableDoubleIterator interface +type MockSeekableDoubleIterator struct { + ctrl *gomock.Controller + recorder *_MockSeekableDoubleIteratorRecorder +} + +// Recorder for MockSeekableDoubleIterator (not exported) +type _MockSeekableDoubleIteratorRecorder struct { + mock *MockSeekableDoubleIterator +} + +func NewMockSeekableDoubleIterator(ctrl *gomock.Controller) *MockSeekableDoubleIterator { + mock := &MockSeekableDoubleIterator{ctrl: ctrl} + mock.recorder = &_MockSeekableDoubleIteratorRecorder{mock} + return mock +} + +func (_m *MockSeekableDoubleIterator) EXPECT() *_MockSeekableDoubleIteratorRecorder { + return _m.recorder +} + +func (_m *MockSeekableDoubleIterator) Close() { + _m.ctrl.Call(_m, "Close") +} + +func (_mr *_MockSeekableDoubleIteratorRecorder) Close() *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "Close") +} + +func (_m *MockSeekableDoubleIterator) Current() float64 { + ret := _m.ctrl.Call(_m, "Current") + ret0, _ := ret[0].(float64) + return ret0 +} + +func (_mr *_MockSeekableDoubleIteratorRecorder) Current() *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "Current") +} + +func (_m *MockSeekableDoubleIterator) Err() error { + ret := _m.ctrl.Call(_m, "Err") + ret0, _ := ret[0].(error) + return ret0 +} + +func (_mr *_MockSeekableDoubleIteratorRecorder) Err() *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "Err") +} + +func (_m *MockSeekableDoubleIterator) Next() bool { + ret := _m.ctrl.Call(_m, "Next") + ret0, _ := ret[0].(bool) + return ret0 +} + +func (_mr *_MockSeekableDoubleIteratorRecorder) Next() *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "Next") +} + +func (_m *MockSeekableDoubleIterator) SeekForward(_param0 int) error { + ret := _m.ctrl.Call(_m, "SeekForward", _param0) + ret0, _ := ret[0].(error) + return ret0 +} + +func (_mr *_MockSeekableDoubleIteratorRecorder) SeekForward(arg0 interface{}) *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "SeekForward", arg0) +} + +// Mock of SeekableStringIterator interface +type MockSeekableStringIterator struct { + ctrl *gomock.Controller + recorder *_MockSeekableStringIteratorRecorder +} + +// Recorder for MockSeekableStringIterator (not exported) +type _MockSeekableStringIteratorRecorder struct { + mock *MockSeekableStringIterator +} + +func NewMockSeekableStringIterator(ctrl *gomock.Controller) *MockSeekableStringIterator { + mock := &MockSeekableStringIterator{ctrl: ctrl} + mock.recorder = &_MockSeekableStringIteratorRecorder{mock} + return mock +} + +func (_m *MockSeekableStringIterator) EXPECT() *_MockSeekableStringIteratorRecorder { + return _m.recorder +} + +func (_m *MockSeekableStringIterator) Close() { + _m.ctrl.Call(_m, "Close") +} + +func (_mr *_MockSeekableStringIteratorRecorder) Close() *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "Close") +} + +func (_m *MockSeekableStringIterator) Current() string { + ret := _m.ctrl.Call(_m, "Current") + ret0, _ := ret[0].(string) + return ret0 +} + +func (_mr *_MockSeekableStringIteratorRecorder) Current() *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "Current") +} + +func (_m *MockSeekableStringIterator) Err() error { + ret := _m.ctrl.Call(_m, "Err") + ret0, _ := ret[0].(error) + return ret0 +} + +func (_mr *_MockSeekableStringIteratorRecorder) Err() *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "Err") +} + +func (_m *MockSeekableStringIterator) Next() bool { + ret := _m.ctrl.Call(_m, "Next") + ret0, _ := ret[0].(bool) + return ret0 +} + +func (_mr *_MockSeekableStringIteratorRecorder) Next() *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "Next") +} + +func (_m *MockSeekableStringIterator) SeekForward(_param0 int) error { + ret := _m.ctrl.Call(_m, "SeekForward", _param0) + ret0, _ := ret[0].(error) + return ret0 +} + +func (_mr *_MockSeekableStringIteratorRecorder) SeekForward(arg0 interface{}) *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "SeekForward", arg0) +} + +// Mock of SeekableTimeIterator interface +type MockSeekableTimeIterator struct { + ctrl *gomock.Controller + recorder *_MockSeekableTimeIteratorRecorder +} + +// Recorder for MockSeekableTimeIterator (not exported) +type _MockSeekableTimeIteratorRecorder struct { + mock *MockSeekableTimeIterator +} + +func NewMockSeekableTimeIterator(ctrl *gomock.Controller) *MockSeekableTimeIterator { + mock := &MockSeekableTimeIterator{ctrl: ctrl} + mock.recorder = &_MockSeekableTimeIteratorRecorder{mock} + return mock +} + +func (_m *MockSeekableTimeIterator) EXPECT() *_MockSeekableTimeIteratorRecorder { + return _m.recorder +} + +func (_m *MockSeekableTimeIterator) Close() { + _m.ctrl.Call(_m, "Close") +} + +func (_mr *_MockSeekableTimeIteratorRecorder) Close() *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "Close") +} + +func (_m *MockSeekableTimeIterator) Current() int64 { + ret := _m.ctrl.Call(_m, "Current") + ret0, _ := ret[0].(int64) + return ret0 +} + +func (_mr *_MockSeekableTimeIteratorRecorder) Current() *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "Current") +} + +func (_m *MockSeekableTimeIterator) Err() error { + ret := _m.ctrl.Call(_m, "Err") + ret0, _ := ret[0].(error) + return ret0 +} + +func (_mr *_MockSeekableTimeIteratorRecorder) Err() *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "Err") +} + +func (_m *MockSeekableTimeIterator) Next() bool { + ret := _m.ctrl.Call(_m, "Next") + ret0, _ := ret[0].(bool) + return ret0 +} + +func (_mr *_MockSeekableTimeIteratorRecorder) Next() *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "Next") +} + +func (_m *MockSeekableTimeIterator) SeekForward(_param0 int) error { + ret := _m.ctrl.Call(_m, "SeekForward", _param0) + ret0, _ := ret[0].(error) + return ret0 +} + +func (_mr *_MockSeekableTimeIteratorRecorder) SeekForward(arg0 interface{}) *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "SeekForward", arg0) +} + // Mock of PositionIterator interface type MockPositionIterator struct { ctrl *gomock.Controller @@ -353,16 +688,6 @@ func (_m *MockPositionIterator) EXPECT() *_MockPositionIteratorRecorder { return _m.recorder } -func (_m *MockPositionIterator) Current() int { - ret := _m.ctrl.Call(_m, "Current") - ret0, _ := ret[0].(int) - return ret0 -} - -func (_mr *_MockPositionIteratorRecorder) Current() *gomock.Call { - return _mr.mock.ctrl.RecordCall(_mr.mock, "Current") -} - func (_m *MockPositionIterator) Next() bool { ret := _m.ctrl.Call(_m, "Next") ret0, _ := ret[0].(bool) @@ -372,3 +697,13 @@ func (_m *MockPositionIterator) Next() bool { func (_mr *_MockPositionIteratorRecorder) Next() *gomock.Call { return _mr.mock.ctrl.RecordCall(_mr.mock, "Next") } + +func (_m *MockPositionIterator) Position() int { + ret := _m.ctrl.Call(_m, "Position") + ret0, _ := ret[0].(int) + return ret0 +} + +func (_mr *_MockPositionIteratorRecorder) Position() *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "Position") +} diff --git a/values/iterator/position_iterator.go b/values/iterator/position_iterator.go index 45639d5..7bbd1f6 100644 --- a/values/iterator/position_iterator.go +++ b/values/iterator/position_iterator.go @@ -1,10 +1,11 @@ package iterator // PositionIterator iterates over positions in a sequence. +// TODO(xichen): Add `Err` to position iterator. type PositionIterator interface { // Next returns true if there are more positions to iterate over. Next() bool - // Current returns the current position in the sequence. - Current() int + // Position returns the current position in the sequence. + Position() int } diff --git a/values/iterator/string_iterator.go b/values/iterator/string_iterator.go index 0820e74..ab9521a 100644 --- a/values/iterator/string_iterator.go +++ b/values/iterator/string_iterator.go @@ -7,3 +7,13 @@ type ForwardStringIterator interface { // Current returns the current value in the iteration. Current() string } + +// SeekableStringIterator is a string iterator that seek to positions. +// TODO(xichen): SeekableStringIterator implementations should implement this +// interface where possible to speed things up. +type SeekableStringIterator interface { + ForwardStringIterator + + // SeekForward moves the iterator forward n positions. + SeekForward(n int) error +} diff --git a/values/iterator/time_iterator.go b/values/iterator/time_iterator.go index fdaa44a..3cb4559 100644 --- a/values/iterator/time_iterator.go +++ b/values/iterator/time_iterator.go @@ -7,3 +7,13 @@ type ForwardTimeIterator interface { // Current returns the current value in nanoseconds in the iteration. Current() int64 } + +// SeekableTimeIterator is a time iterator that seek to positions. +// TODO(xichen): SeekableTimeIterator implementations should implement this +// interface where possible to speed things up. +type SeekableTimeIterator interface { + ForwardTimeIterator + + // SeekForward moves the iterator forward n positions. + SeekForward(n int) error +} diff --git a/values/roundtriptest/bool_test.go b/values/roundtriptest/bool_test.go index 493d699..0d20409 100644 --- a/values/roundtriptest/bool_test.go +++ b/values/roundtriptest/bool_test.go @@ -74,5 +74,5 @@ func produceMockBoolData( } iter.EXPECT().Next().Return(false).Times(1) iter.EXPECT().Err().Return(nil).Times(1) - iter.EXPECT().Close().Return(nil) + iter.EXPECT().Close() } diff --git a/values/roundtriptest/double_test.go b/values/roundtriptest/double_test.go index b4c525b..7a281bc 100644 --- a/values/roundtriptest/double_test.go +++ b/values/roundtriptest/double_test.go @@ -46,7 +46,7 @@ func produceMockDoubleData(data []float64, iter *iterator.MockForwardDoubleItera } iter.EXPECT().Next().Return(false).Times(1) iter.EXPECT().Err().Return(nil).Times(1) - iter.EXPECT().Close().Return(nil) + iter.EXPECT().Close() } func ensureEncodeAndDecodeDouble(t *testing.T, data []float64) { diff --git a/values/roundtriptest/int_test.go b/values/roundtriptest/int_test.go index 80d124d..4cc4fc8 100644 --- a/values/roundtriptest/int_test.go +++ b/values/roundtriptest/int_test.go @@ -91,7 +91,7 @@ func TestPositiveIntDeltaEncodeAndDecode(t *testing.T) { iter1.EXPECT().Next().Return(true), iter1.EXPECT().Current().Return(3), iter1.EXPECT().Err().Return(nil), - iter1.EXPECT().Close().Return(nil), + iter1.EXPECT().Close(), ) iter2 := iterator.NewMockForwardIntIterator(ctrl) @@ -129,7 +129,7 @@ func TestNegativeIntDeltaEncodeAndDecode(t *testing.T) { iter1.EXPECT().Next().Return(true), iter1.EXPECT().Current().Return(-3), iter1.EXPECT().Err().Return(nil), - iter1.EXPECT().Close().Return(nil), + iter1.EXPECT().Close(), ) iter2 := iterator.NewMockForwardIntIterator(ctrl) @@ -199,7 +199,7 @@ func TestMixedIntDeltaEncodeAndDecode(t *testing.T) { iter1.EXPECT().Next().Return(true), iter1.EXPECT().Current().Return(-3), iter1.EXPECT().Err().Return(nil), - iter1.EXPECT().Close().Return(nil), + iter1.EXPECT().Close(), ) iter2 := iterator.NewMockForwardIntIterator(ctrl) @@ -222,7 +222,7 @@ func produceMockIntData(data []int, iter *iterator.MockForwardIntIterator) { } iter.EXPECT().Next().Return(false).Times(1) iter.EXPECT().Err().Return(nil).Times(1) - iter.EXPECT().Close().Return(nil) + iter.EXPECT().Close() } // Ensure that encoding/decoding test data gives the same result. diff --git a/values/roundtriptest/string_test.go b/values/roundtriptest/string_test.go index 69b89e0..9f53763 100644 --- a/values/roundtriptest/string_test.go +++ b/values/roundtriptest/string_test.go @@ -86,7 +86,7 @@ func TestRawSizeEncodeAndDecode(t *testing.T) { iter1.EXPECT().Next().Return(true), iter1.EXPECT().Current().Return("unique string 2"), iter1.EXPECT().Err().Return(nil), - iter1.EXPECT().Close().Return(nil), + iter1.EXPECT().Close(), ) iter2 := iterator.NewMockForwardStringIterator(ctrl) @@ -127,5 +127,5 @@ func produceMockStringData(data []string, iter *iterator.MockForwardStringIterat } iter.EXPECT().Next().Return(false).Times(1) iter.EXPECT().Err().Return(nil).Times(1) - iter.EXPECT().Close().Return(nil).Times(1) + iter.EXPECT().Close().Times(1) } diff --git a/values/roundtriptest/time_test.go b/values/roundtriptest/time_test.go index 4920c78..af62fe2 100644 --- a/values/roundtriptest/time_test.go +++ b/values/roundtriptest/time_test.go @@ -60,7 +60,7 @@ func produceMockTimeData(data []int64, iter *iterator.MockForwardTimeIterator) { } iter.EXPECT().Next().Return(false).Times(1) iter.EXPECT().Err().Return(nil).Times(1) - iter.EXPECT().Close().Return(nil).Times(1) + iter.EXPECT().Close().Times(1) } func testEncodeAndDecodeTime( diff --git a/values/template/delta_iterator.go b/values/template/delta_iterator.go index c1ef9d9..053316e 100644 --- a/values/template/delta_iterator.go +++ b/values/template/delta_iterator.go @@ -17,7 +17,6 @@ type deltaValueIterator struct { addFn applyOpToValueIntFn negativeBit uint64 - closed bool curr GenericValue err error isDeltaValue bool @@ -38,7 +37,7 @@ func newDeltaValueIterator( // Next returns true if there are more values to be iterated over. func (it *deltaValueIterator) Next() bool { - if it.closed || it.err != nil { + if it.err != nil { return false } @@ -87,13 +86,8 @@ func (it *deltaValueIterator) Err() error { } // Close closes the iterator. -func (it *deltaValueIterator) Close() error { - if it.closed { - return nil - } - it.closed = true +func (it *deltaValueIterator) Close() { it.bitReader = nil it.err = nil it.addFn = nil - return nil } diff --git a/values/template/filtered_value_iterator.go b/values/template/filtered_value_iterator.go index 1087901..37dfa6d 100644 --- a/values/template/filtered_value_iterator.go +++ b/values/template/filtered_value_iterator.go @@ -37,8 +37,8 @@ func (it *FilteredValueIterator) Next() bool { return false } -// Current returns the current position. -func (it *FilteredValueIterator) Current() int { return it.currPos } +// Position returns the current position. +func (it *FilteredValueIterator) Position() int { return it.currPos } // Close closes the iterator. func (it *FilteredValueIterator) Close() { diff --git a/values/template/generic.go b/values/template/generic.go index 781fd47..1155e84 100644 --- a/values/template/generic.go +++ b/values/template/generic.go @@ -1,8 +1,6 @@ package template import ( - "io" - "github.com/mauricelam/genny/generic" "github.com/xichen2020/eventdb/values/iterator" ) @@ -13,11 +11,11 @@ type GenericValue generic.Type // ForwardValueIterator allows iterating over a stream of GenericValue. type ForwardValueIterator interface { generic.Type - io.Closer Next() bool Err() error Current() GenericValue + Close() } // ValueFilter performs filtering against values. diff --git a/values/template/run_length_decode.go b/values/template/run_length_decode.go index 9ee0f9c..8998654 100644 --- a/values/template/run_length_decode.go +++ b/values/template/run_length_decode.go @@ -30,13 +30,12 @@ type runLengthValueIterator struct { curr GenericValue repetitions int64 - closed bool err error } // Next returns true if there are more values to be iterated over, and false otherwise. func (it *runLengthValueIterator) Next() bool { - if it.closed || it.err != nil { + if it.err != nil { return false } @@ -75,15 +74,10 @@ func (it *runLengthValueIterator) Err() error { } // Close the iterator. -func (it *runLengthValueIterator) Close() error { - if it.closed { - return nil - } - it.closed = true +func (it *runLengthValueIterator) Close() { it.err = nil it.reader = nil it.readValueFn = nil - return nil } func newRunLengthValueIterator(