-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
1991c4e
commit 138c65f
Showing
73 changed files
with
2,492 additions
and
373 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,6 @@ | ||
| vendor/ | ||
| generated/ | ||
| index/template/ | ||
| values/template/ | ||
| _mock.go | ||
| _gen.go |
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,60 +1,102 @@ | ||
| package index | ||
|
|
||
| import ( | ||
| "errors" | ||
|
|
||
| "github.com/xichen2020/eventdb/values/iterator" | ||
| ) | ||
|
|
||
| var ( | ||
| errPositionIterDocIDIterCountMismatch = errors.New("doc ID iterator and the position iterator iterator count mismatch") | ||
| ) | ||
|
|
||
| // AtPositionDocIDSetIterator outputs the doc IDs from the doc ID set iterator at the | ||
| // given positions from the position iterator. | ||
| type AtPositionDocIDSetIterator struct { | ||
| docIt DocIDSetIterator | ||
| positionIt iterator.PositionIterator | ||
| docIt DocIDSetIterator | ||
| seekableDocIt SeekableDocIDSetIterator | ||
| positionIt iterator.PositionIterator | ||
|
|
||
| done bool | ||
| err error | ||
| firstTime bool | ||
| currDocID int32 | ||
| currPos int | ||
| done bool | ||
| } | ||
|
|
||
| // NewAtPositionDocIDSetIterator creates a new at position iterator. | ||
| func NewAtPositionDocIDSetIterator( | ||
| docIt DocIDSetIterator, | ||
| positionIt iterator.PositionIterator, | ||
| ) *AtPositionDocIDSetIterator { | ||
| seekableDocIt, _ := docIt.(SeekableDocIDSetIterator) | ||
| if seekableDocIt != nil { | ||
| docIt = nil | ||
| } | ||
| return &AtPositionDocIDSetIterator{ | ||
| docIt: docIt, | ||
| positionIt: positionIt, | ||
| docIt: docIt, | ||
| seekableDocIt: seekableDocIt, | ||
| positionIt: positionIt, | ||
| firstTime: true, | ||
| } | ||
| } | ||
|
|
||
| // Next returns true if there are more doc IDs to be iterated over. | ||
| func (it *AtPositionDocIDSetIterator) Next() bool { | ||
| if it.done { | ||
| if it.done || it.err != nil { | ||
| return false | ||
| } | ||
| if !it.positionIt.Next() { | ||
| it.done = true | ||
| return false | ||
| } | ||
| nextPos := it.positionIt.Current() | ||
| nextPos := it.positionIt.Position() | ||
| distance := nextPos - it.currPos | ||
| // TODO(xichen): Look into optimizations to speed this up if the doc ID set iterator | ||
| // supports a `Seek` or `Advance` API. | ||
| for i := 0; i < distance; i++ { | ||
| if !it.docIt.Next() { | ||
| panic("doc ID iterator and the position iterator iterator count mismatch") | ||
|
|
||
| // We have a next position, now advance the doc ID set iterator for the first time. | ||
| if it.firstTime { | ||
| it.firstTime = false | ||
| if hasNoValues := | ||
| (it.seekableDocIt != nil && !it.seekableDocIt.Next()) || | ||
| (it.docIt != nil && !it.docIt.Next()); hasNoValues { | ||
| it.err = errPositionIterDocIDIterCountMismatch | ||
| return false | ||
| } | ||
| } | ||
| it.currDocID = it.docIt.DocID() | ||
|
|
||
| if it.seekableDocIt != nil { | ||
| if it.err = it.seekableDocIt.SeekForward(distance); it.err != nil { | ||
| return false | ||
| } | ||
| it.currDocID = it.seekableDocIt.DocID() | ||
| } else { | ||
| for i := 0; i < distance; i++ { | ||
| if !it.docIt.Next() { | ||
| it.err = errPositionIterDocIDIterCountMismatch | ||
| return false | ||
| } | ||
| } | ||
| it.currDocID = it.docIt.DocID() | ||
| } | ||
| it.currPos = nextPos | ||
| return true | ||
| } | ||
|
|
||
| // DocID returns the current doc ID. | ||
| func (it *AtPositionDocIDSetIterator) DocID() int32 { return it.currDocID } | ||
|
|
||
| // Err returns any error encountered. | ||
| func (it *AtPositionDocIDSetIterator) Err() error { return it.err } | ||
|
|
||
| // Close closes the iterator. | ||
| func (it *AtPositionDocIDSetIterator) Close() { | ||
| it.docIt.Close() | ||
| it.docIt = nil | ||
| if it.docIt != nil { | ||
| it.docIt.Close() | ||
| it.docIt = nil | ||
| } else { | ||
| it.seekableDocIt.Close() | ||
| it.seekableDocIt = nil | ||
| } | ||
| it.positionIt = nil | ||
| it.err = nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,55 @@ | ||
| package index | ||
|
|
||
| import "github.com/pilosa/pilosa/roaring" | ||
|
|
||
| type bitmapBasedDocIDPositionIterator struct { | ||
| bm *roaring.Bitmap | ||
| maskingIt DocIDSetIterator | ||
|
|
||
| done bool | ||
| currDocID int32 | ||
| currPosition int | ||
| } | ||
|
|
||
| // nolint: deadcode | ||
| func newBitmapBasedDocIDPositionIterator( | ||
| bm *roaring.Bitmap, | ||
| maskingIt DocIDSetIterator, | ||
| ) *bitmapBasedDocIDPositionIterator { | ||
| return &bitmapBasedDocIDPositionIterator{ | ||
| bm: bm, | ||
| maskingIt: maskingIt, | ||
| currDocID: -1, | ||
| currPosition: -1, | ||
| } | ||
| } | ||
|
|
||
| func (it *bitmapBasedDocIDPositionIterator) Next() bool { | ||
| if it.done { | ||
| return false | ||
| } | ||
| if !it.maskingIt.Next() { | ||
| it.done = true | ||
| return false | ||
| } | ||
| currDocID := it.maskingIt.DocID() | ||
| if !it.bm.Contains(uint64(currDocID)) { | ||
| return it.Next() | ||
| } | ||
| // Find the number of bits set between [prevDocID+1, it.currDocID+1). | ||
| prevDocID := it.currDocID | ||
| it.currDocID = currDocID | ||
| numBitsSet := it.bm.CountRange(uint64(prevDocID+1), uint64(it.currDocID+1)) | ||
| it.currPosition += int(numBitsSet) | ||
| return true | ||
| } | ||
|
|
||
| func (it *bitmapBasedDocIDPositionIterator) DocID() int32 { return it.currDocID } | ||
|
|
||
| func (it *bitmapBasedDocIDPositionIterator) Position() int { return it.currPosition } | ||
|
|
||
| func (it *bitmapBasedDocIDPositionIterator) Close() { | ||
| it.bm = nil | ||
| it.maskingIt.Close() | ||
| it.maskingIt = nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,57 @@ | ||
| package index | ||
|
|
||
| import ( | ||
| "testing" | ||
|
|
||
| "github.com/pilosa/pilosa/roaring" | ||
| ) | ||
|
|
||
| // Summary: The default doc ID position iterator is as fast as the custom one, with | ||
| // fewer memory allocations. | ||
|
|
||
| func BenchmarkDefaultBitmapDocIDPositionIterator(b *testing.B) { | ||
| bm := initBenchBitmap(benchNumTotalDocs, 5) | ||
| ds := initBenchDocIDSet(benchNumTotalDocs, 8) | ||
|
|
||
| b.ResetTimer() | ||
| for i := 0; i < b.N; i++ { | ||
| bmIt := newBitmapBasedDocIDSet(bm).Iter() | ||
| dsIt := newArrayBasedDocIDSetIterator(ds) | ||
| defaultIt := NewDocIDPositionIterator(bmIt, dsIt) | ||
| count := 0 | ||
| for defaultIt.Next() { | ||
| benchDocID = defaultIt.DocID() | ||
| benchPos = defaultIt.Position() | ||
| count++ | ||
| } | ||
| } | ||
| } | ||
|
|
||
| func BenchmarkCustomBitmapDocIDPositionIterator(b *testing.B) { | ||
| bm := initBenchBitmap(benchNumTotalDocs, 5) | ||
| ds := initBenchDocIDSet(benchNumTotalDocs, 8) | ||
|
|
||
| bm.Optimize() | ||
| b.ResetTimer() | ||
| for i := 0; i < b.N; i++ { | ||
| dsIt := newArrayBasedDocIDSetIterator(ds) | ||
| it := newBitmapBasedDocIDPositionIterator(bm, dsIt) | ||
| count := 0 | ||
| for it.Next() { | ||
| benchDocID = it.DocID() | ||
| benchPos = it.Position() | ||
| count++ | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // nolint: unparam | ||
| func initBenchBitmap(n int, everyN int) *roaring.Bitmap { | ||
| bm := roaring.NewBitmap() | ||
| for j := 0; j < n; j++ { | ||
| if j%everyN == 0 { | ||
| bm.DirectAdd(uint64(j)) | ||
| } | ||
| } | ||
| return bm | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,41 @@ | ||
| package index | ||
|
|
||
| import ( | ||
| "testing" | ||
|
|
||
| "github.com/golang/mock/gomock" | ||
| "github.com/pilosa/pilosa/roaring" | ||
| "github.com/stretchr/testify/require" | ||
| ) | ||
|
|
||
| func TestBitmapBasedDocIDPositionIterator(t *testing.T) { | ||
| ctrl := gomock.NewController(t) | ||
| defer ctrl.Finish() | ||
|
|
||
| bm := roaring.NewBitmap(0, 3, 5, 9, 12, 19, 23) | ||
|
|
||
| maskingIt := NewMockDocIDSetIterator(ctrl) | ||
| gomock.InOrder( | ||
| maskingIt.EXPECT().Next().Return(true), | ||
| maskingIt.EXPECT().DocID().Return(int32(0)).MinTimes(1), | ||
| maskingIt.EXPECT().Next().Return(true), | ||
| maskingIt.EXPECT().DocID().Return(int32(7)).MinTimes(1), | ||
| maskingIt.EXPECT().Next().Return(true), | ||
| maskingIt.EXPECT().DocID().Return(int32(12)).MinTimes(1), | ||
| maskingIt.EXPECT().Next().Return(false).AnyTimes(), | ||
| ) | ||
|
|
||
| var ( | ||
| docIDs []int32 | ||
| positions []int | ||
| expectedDocIDs = []int32{0, 12} | ||
| expectedPositions = []int{0, 4} | ||
| ) | ||
| it := newBitmapBasedDocIDPositionIterator(bm, maskingIt) | ||
| for it.Next() { | ||
| docIDs = append(docIDs, it.DocID()) | ||
| positions = append(positions, it.Position()) | ||
| } | ||
| require.Equal(t, expectedDocIDs, docIDs) | ||
| require.Equal(t, expectedPositions, positions) | ||
| } |
Oops, something went wrong.