Support querying ordered results in immutable segments during query e…

…xecution

document/field/field.go

@@ -1,5 +1,9 @@
 package field
 
+import (
+	"fmt"
+)
+
 // ValueType is the type of a field value.
 type ValueType int
 
@@ -146,6 +150,101 @@ func NewTimeUnion(v int64) ValueUnion {
 	}
 }
 
+// ValueCompareFn compares two value unions.
+type ValueCompareFn func(v1, v2 ValueUnion) int
+
+// CompareUnion is a convience method that compares two unions.
+// If the two unions have different field types, the result is undefined and the method always returns -1.
+// Otherwise, the corresponding values are compared, and the method returns
+// * -1 if v1 < v2
+// * 0 if v1 == v2
+// * 1 if v1 > v2
+func CompareUnion(v1, v2 ValueUnion) (int, error) {
+	if v1.Type != v2.Type {
+		return 0, fmt.Errorf("cannot compare unions of different types %v and %v", v1.Type, v2.Type)
+	}
+	switch v1.Type {
+	case NullType:
+		return 0, nil
+	case BoolType:
+		return compareBool(v1.BoolVal, v2.BoolVal), nil
+	case IntType:
+		return compareInt(v1.IntVal, v2.IntVal), nil
+	case DoubleType:
+		return compareDouble(v1.DoubleVal, v2.DoubleVal), nil
+	case StringType:
+		return compareString(v1.StringVal, v2.StringVal), nil
+	case TimeType:
+		return compareTimeNanos(v1.TimeNanosVal, v2.TimeNanosVal), nil
+	default:
+		return 0, fmt.Errorf("invalid value type %v", v1.Type)
+	}
+}
+
+// MustCompareUnion compares two value unions, and panics if it encounters an error.
+func MustCompareUnion(v1, v2 ValueUnion) int {
+	res, err := CompareUnion(v1, v2)
+	if err != nil {
+		panic(err)
+	}
+	return res
+}
+
+// MustReverseCompareUnion reverse compares two value unions, and panics if it encounters an error.
+func MustReverseCompareUnion(v1, v2 ValueUnion) int {
+	return MustCompareUnion(v2, v1)
+}
+
+func compareBool(v1, v2 bool) int {
+	if v1 == v2 {
+		return 0
+	}
+	if !v1 {
+		return -1
+	}
+	return 1
+}
+
+func compareInt(v1, v2 int) int {
+	if v1 < v2 {
+		return -1
+	}
+	if v1 > v2 {
+		return 1
+	}
+	return 0
+}
+
+func compareDouble(v1, v2 float64) int {
+	if v1 < v2 {
+		return -1
+	}
+	if v1 > v2 {
+		return 1
+	}
+	return 0
+}
+
+func compareString(v1, v2 string) int {
+	if v1 < v2 {
+		return -1
+	}
+	if v1 > v2 {
+		return 1
+	}
+	return 0
+}
+
+func compareTimeNanos(v1, v2 int64) int {
+	if v1 < v2 {
+		return -1
+	}
+	if v1 > v2 {
+		return 1
+	}
+	return 0
+}
+
 // Field is an event field.
 type Field struct {
 	Path  []string

index/bitmap_based_doc_id_position_iterator.go

@@ -6,9 +6,10 @@ type bitmapBasedDocIDPositionIterator struct {
 	bm        *roaring.Bitmap
 	maskingIt DocIDSetIterator
 
-	done         bool
-	currDocID    int32
-	currPosition int
+	done            bool
+	currDocID       int32
+	backingPosition int
+	maskingPosition int
 }
 
 // nolint: deadcode
@@ -17,10 +18,11 @@ func newBitmapBasedDocIDPositionIterator(
 	maskingIt DocIDSetIterator,
 ) *bitmapBasedDocIDPositionIterator {
 	return &bitmapBasedDocIDPositionIterator{
-		bm:           bm,
-		maskingIt:    maskingIt,
-		currDocID:    -1,
-		currPosition: -1,
+		bm:              bm,
+		maskingIt:       maskingIt,
+		currDocID:       -1,
+		backingPosition: -1,
+		maskingPosition: -1,
 	}
 }
 
@@ -32,6 +34,7 @@ func (it *bitmapBasedDocIDPositionIterator) Next() bool {
 		it.done = true
 		return false
 	}
+	it.maskingPosition++
 	currDocID := it.maskingIt.DocID()
 	if !it.bm.Contains(uint64(currDocID)) {
 		return it.Next()
@@ -40,13 +43,15 @@ func (it *bitmapBasedDocIDPositionIterator) Next() bool {
 	prevDocID := it.currDocID
 	it.currDocID = currDocID
 	numBitsSet := it.bm.CountRange(uint64(prevDocID+1), uint64(it.currDocID+1))
-	it.currPosition += int(numBitsSet)
+	it.backingPosition += int(numBitsSet)
 	return true
 }
 
 func (it *bitmapBasedDocIDPositionIterator) DocID() int32 { return it.currDocID }
 
-func (it *bitmapBasedDocIDPositionIterator) Position() int { return it.currPosition }
+func (it *bitmapBasedDocIDPositionIterator) Position() int { return it.backingPosition }
+
+func (it *bitmapBasedDocIDPositionIterator) MaskingPosition() int { return it.maskingPosition }
 
 func (it *bitmapBasedDocIDPositionIterator) Close() {
 	it.bm = nil

index/bitmap_based_doc_id_position_iterator_test.go

@@ -26,16 +26,20 @@ func TestBitmapBasedDocIDPositionIterator(t *testing.T) {
 	)
 
 	var (
-		docIDs            []int32
-		positions         []int
-		expectedDocIDs    = []int32{0, 12}
-		expectedPositions = []int{0, 4}
+		docIDs                   []int32
+		backingPositions         []int
+		maskingPositions         []int
+		expectedDocIDs           = []int32{0, 12}
+		expectedBackingPositions = []int{0, 4}
+		expectedMaskingPositions = []int{0, 2}
 	)
 	it := newBitmapBasedDocIDPositionIterator(bm, maskingIt)
 	for it.Next() {
 		docIDs = append(docIDs, it.DocID())
-		positions = append(positions, it.Position())
+		backingPositions = append(backingPositions, it.Position())
+		maskingPositions = append(maskingPositions, it.MaskingPosition())
 	}
 	require.Equal(t, expectedDocIDs, docIDs)
-	require.Equal(t, expectedPositions, positions)
+	require.Equal(t, expectedBackingPositions, backingPositions)
+	require.Equal(t, expectedMaskingPositions, maskingPositions)
 }

index/doc_id_position_iterator.go

@@ -7,18 +7,23 @@ package index
 type DocIDPositionIterator interface {
 	DocIDSetIterator
 
+	// Position is the position in the backing doc ID set.
 	Position() int
+
+	// MaskingPosition is the position in the masking doc ID set.
+	MaskingPosition() int
 }
 
 type docIDPositionIterator struct {
 	backingIt DocIDSetIterator
 	maskingIt DocIDSetIterator
 
-	backingDone  bool
-	maskingDone  bool
-	backingDocID int32
-	maskingDocID int32
-	currPosition int
+	backingDone     bool
+	maskingDone     bool
+	backingDocID    int32
+	maskingDocID    int32
+	backingPosition int
+	maskingPosition int
 }
 
 // NewDocIDPositionIterator creates a new doc ID position iterator.
@@ -27,11 +32,12 @@ func NewDocIDPositionIterator(
 	maskingIt DocIDSetIterator,
 ) DocIDPositionIterator {
 	it := &docIDPositionIterator{
-		backingIt:    backingIt,
-		maskingIt:    maskingIt,
-		backingDocID: invalidDocID,
-		maskingDocID: invalidDocID,
-		currPosition: -1,
+		backingIt:       backingIt,
+		maskingIt:       maskingIt,
+		backingDocID:    invalidDocID,
+		maskingDocID:    invalidDocID,
+		backingPosition: -1,
+		maskingPosition: -1,
 	}
 	it.advanceMaskingIter()
 	return it
@@ -63,8 +69,11 @@ func (it *docIDPositionIterator) Next() bool {
 // DocID returns the current doc ID.
 func (it *docIDPositionIterator) DocID() int32 { return it.backingDocID }
 
-// Position returns the current doc ID position.
-func (it *docIDPositionIterator) Position() int { return it.currPosition }
+// Position returns the current doc ID position in the backing doc ID set.
+func (it *docIDPositionIterator) Position() int { return it.backingPosition }
+
+// MaskingPosition returns the current doc ID position in the masking doc ID set.
+func (it *docIDPositionIterator) MaskingPosition() int { return it.maskingPosition }
 
 // Close closes the iterator.
 func (it *docIDPositionIterator) Close() {
@@ -75,7 +84,7 @@ func (it *docIDPositionIterator) Close() {
 func (it *docIDPositionIterator) advanceBackingIter() {
 	if it.backingIt.Next() {
 		it.backingDocID = it.backingIt.DocID()
-		it.currPosition++
+		it.backingPosition++
 	} else {
 		it.backingDone = true
 	}
@@ -84,6 +93,7 @@ func (it *docIDPositionIterator) advanceBackingIter() {
 func (it *docIDPositionIterator) advanceMaskingIter() {
 	if it.maskingIt.Next() {
 		it.maskingDocID = it.maskingIt.DocID()
+		it.maskingPosition++
 	} else {
 		it.maskingDone = true
 	}

index/doc_id_position_iterator_test.go

@@ -39,16 +39,19 @@ func TestDocIDPositionIteratorNoOverlap(t *testing.T) {
 	)
 
 	var (
-		docIDs    []int32
-		positions []int
+		docIDs           []int32
+		backingPositions []int
+		maskingPositions []int
 	)
 	it := NewDocIDPositionIterator(backingIt, maskingIt)
 	for it.Next() {
 		docIDs = append(docIDs, it.DocID())
-		positions = append(positions, it.Position())
+		backingPositions = append(backingPositions, it.Position())
+		maskingPositions = append(maskingPositions, it.MaskingPosition())
 	}
 	require.Equal(t, 0, len(docIDs))
-	require.Equal(t, 0, len(positions))
+	require.Equal(t, 0, len(backingPositions))
+	require.Equal(t, 0, len(maskingPositions))
 }
 
 func TestDocIDPositionIteratorAllOverlap(t *testing.T) {
@@ -86,18 +89,22 @@ func TestDocIDPositionIteratorAllOverlap(t *testing.T) {
 	)
 
 	var (
-		docIDs            []int32
-		positions         []int
-		expectedDocIDs    = []int32{3, 4, 7, 10, 15}
-		expectedPositions = []int{0, 1, 2, 3, 4}
+		docIDs                   []int32
+		backingPositions         []int
+		maskingPositions         []int
+		expectedDocIDs           = []int32{3, 4, 7, 10, 15}
+		expectedBackingPositions = []int{0, 1, 2, 3, 4}
+		expectedMaskingPositions = []int{0, 1, 2, 3, 4}
 	)
 	it := NewDocIDPositionIterator(backingIt, maskingIt)
 	for it.Next() {
 		docIDs = append(docIDs, it.DocID())
-		positions = append(positions, it.Position())
+		backingPositions = append(backingPositions, it.Position())
+		maskingPositions = append(maskingPositions, it.MaskingPosition())
 	}
 	require.Equal(t, expectedDocIDs, docIDs)
-	require.Equal(t, expectedPositions, positions)
+	require.Equal(t, expectedBackingPositions, backingPositions)
+	require.Equal(t, expectedMaskingPositions, maskingPositions)
 }
 
 func TestDocIDPositionIteratorPartialOverlap(t *testing.T) {
@@ -133,16 +140,20 @@ func TestDocIDPositionIteratorPartialOverlap(t *testing.T) {
 	)
 
 	var (
-		docIDs            []int32
-		positions         []int
-		expectedDocIDs    = []int32{3, 10}
-		expectedPositions = []int{0, 3}
+		docIDs                   []int32
+		backingPositions         []int
+		maskingPositions         []int
+		expectedDocIDs           = []int32{3, 10}
+		expectedBackingPositions = []int{0, 3}
+		expectedMaskingPositions = []int{0, 2}
 	)
 	it := NewDocIDPositionIterator(backingIt, maskingIt)
 	for it.Next() {
 		docIDs = append(docIDs, it.DocID())
-		positions = append(positions, it.Position())
+		backingPositions = append(backingPositions, it.Position())
+		maskingPositions = append(maskingPositions, it.MaskingPosition())
 	}
 	require.Equal(t, expectedDocIDs, docIDs)
-	require.Equal(t, expectedPositions, positions)
+	require.Equal(t, expectedBackingPositions, backingPositions)
+	require.Equal(t, expectedMaskingPositions, maskingPositions)
 }

index/field/at_position_bool_field_iterator.gen.go

@@ -23,12 +23,13 @@ type atPositionBoolFieldIterator struct {
 	seekableValsIt iterator.SeekableBoolIterator
 	valAsUnionFn   field.BoolAsUnionFn
 
-	done      bool
-	err       error
-	firstTime bool
-	currPos   int
-	currDocID int32
-	currVal   bool
+	done           bool
+	err            error
+	firstTime      bool
+	currPos        int
+	currMaskingPos int
+	currDocID      int32
+	currVal        bool
 }
 
 func newAtPositionBoolFieldIterator(
@@ -87,6 +88,7 @@ func (it *atPositionBoolFieldIterator) Next() bool {
 	}
 
 	it.currPos = nextPos
+	it.currMaskingPos = it.docIDPosIt.MaskingPosition()
 	it.currDocID = it.docIDPosIt.DocID()
 	return true
 }
@@ -99,6 +101,8 @@ func (it *atPositionBoolFieldIterator) ValueUnion() field.ValueUnion {
 	return it.valAsUnionFn(it.currVal)
 }
 
+func (it *atPositionBoolFieldIterator) MaskingPosition() int { return it.currMaskingPos }
+
 func (it *atPositionBoolFieldIterator) Err() error { return it.err }
 
 func (it *atPositionBoolFieldIterator) Close() {