Skip to content

Commit

Permalink
Reduced index range complexity, removed range overlapping
Browse files Browse the repository at this point in the history
  • Loading branch information
@Hydrocharged
Hydrocharged committed Nov 5, 2021
1 parent 92de2b5 commit 4a443cd
Show file tree
Hide file tree
Showing 8 changed files with 724 additions and 164 deletions.
135 changes: 61 additions & 74 deletions memory/index.go
View file
Expand Up @@ -85,89 +85,76 @@ func (i *Index) NewLookup(ctx *sql.Context, ranges ...sql.Range) (sql.IndexLooku
exprs = mergeableIndex.ColumnExpressions()
}

var completeExpr sql.Expression
var rangeCollectionExpr sql.Expression
for _, rang := range ranges {
var filterExpr sql.Expression
for i, rangeColumn := range rang {
var rangeExpr sql.Expression
for _, rangeColumnExpr := range rangeColumn {
switch rangeColumnExpr.Type() {
// Both Empty and All may seem like strange inclusions, but if only one range is given we need some
// expression to evaluate, otherwise our expression would be a nil expression which would panic.
case sql.RangeType_Empty:
rangeExpr = or(rangeExpr, expression.NewEquals(expression.NewLiteral(1, sql.Int8), expression.NewLiteral(2, sql.Int8)))
case sql.RangeType_All:
rangeExpr = or(rangeExpr, expression.NewEquals(expression.NewLiteral(1, sql.Int8), expression.NewLiteral(1, sql.Int8)))
case sql.RangeType_GreaterThan:
lit, typ := getType(sql.GetRangeCutKey(rangeColumnExpr.LowerBound))
rangeExpr = or(rangeExpr, expression.NewNullSafeGreaterThan(exprs[i], expression.NewLiteral(lit, typ)))
case sql.RangeType_GreaterOrEqual:
lit, typ := getType(sql.GetRangeCutKey(rangeColumnExpr.LowerBound))
rangeExpr = or(rangeExpr, expression.NewNullSafeGreaterThanOrEqual(exprs[i], expression.NewLiteral(lit, typ)))
case sql.RangeType_LessThan:
lit, typ := getType(sql.GetRangeCutKey(rangeColumnExpr.UpperBound))
rangeExpr = or(rangeExpr, expression.NewNullSafeLessThan(exprs[i], expression.NewLiteral(lit, typ)))
case sql.RangeType_LessOrEqual:
lit, typ := getType(sql.GetRangeCutKey(rangeColumnExpr.UpperBound))
rangeExpr = or(rangeExpr, expression.NewNullSafeLessThanOrEqual(exprs[i], expression.NewLiteral(lit, typ)))
case sql.RangeType_ClosedClosed:
if ok, err := rangeColumnExpr.RepresentsEquals(); err != nil {
return nil, err
} else if ok {
lit, typ := getType(sql.GetRangeCutKey(rangeColumnExpr.LowerBound))
if typ == sql.Null {
rangeExpr = or(rangeExpr, expression.NewIsNull(exprs[i]))
} else {
rangeExpr = or(rangeExpr, expression.NewNullSafeEquals(exprs[i], expression.NewLiteral(lit, typ)))
}
var rangeExpr sql.Expression
for i, rce := range rang {
var rangeColumnExpr sql.Expression
switch rce.Type() {
// Both Empty and All may seem like strange inclusions, but if only one range is given we need some
// expression to evaluate, otherwise our expression would be a nil expression which would panic.
case sql.RangeType_Empty:
rangeColumnExpr = expression.NewEquals(expression.NewLiteral(1, sql.Int8), expression.NewLiteral(2, sql.Int8))
case sql.RangeType_All:
rangeColumnExpr = expression.NewEquals(expression.NewLiteral(1, sql.Int8), expression.NewLiteral(1, sql.Int8))
case sql.RangeType_GreaterThan:
lit, typ := getType(sql.GetRangeCutKey(rce.LowerBound))
rangeColumnExpr = expression.NewNullSafeGreaterThan(exprs[i], expression.NewLiteral(lit, typ))
case sql.RangeType_GreaterOrEqual:
lit, typ := getType(sql.GetRangeCutKey(rce.LowerBound))
rangeColumnExpr = expression.NewNullSafeGreaterThanOrEqual(exprs[i], expression.NewLiteral(lit, typ))
case sql.RangeType_LessThan:
lit, typ := getType(sql.GetRangeCutKey(rce.UpperBound))
rangeColumnExpr = expression.NewNullSafeLessThan(exprs[i], expression.NewLiteral(lit, typ))
case sql.RangeType_LessOrEqual:
lit, typ := getType(sql.GetRangeCutKey(rce.UpperBound))
rangeColumnExpr = expression.NewNullSafeLessThanOrEqual(exprs[i], expression.NewLiteral(lit, typ))
case sql.RangeType_ClosedClosed:
if ok, err := rce.RepresentsEquals(); err != nil {
return nil, err
} else if ok {
lit, typ := getType(sql.GetRangeCutKey(rce.LowerBound))
if typ == sql.Null {
rangeColumnExpr = expression.NewIsNull(exprs[i])
} else {
lowLit, lowTyp := getType(sql.GetRangeCutKey(rangeColumnExpr.LowerBound))
upLit, upTyp := getType(sql.GetRangeCutKey(rangeColumnExpr.UpperBound))
rangeExpr = or(rangeExpr,
and(
expression.NewNullSafeGreaterThanOrEqual(exprs[i], expression.NewLiteral(lowLit, lowTyp)),
expression.NewNullSafeLessThanOrEqual(exprs[i], expression.NewLiteral(upLit, upTyp)),
),
)
rangeColumnExpr = expression.NewNullSafeEquals(exprs[i], expression.NewLiteral(lit, typ))
}
case sql.RangeType_OpenOpen:
lowLit, lowTyp := getType(sql.GetRangeCutKey(rangeColumnExpr.LowerBound))
upLit, upTyp := getType(sql.GetRangeCutKey(rangeColumnExpr.UpperBound))
rangeExpr = or(rangeExpr,
and(
expression.NewNullSafeGreaterThan(exprs[i], expression.NewLiteral(lowLit, lowTyp)),
expression.NewNullSafeLessThan(exprs[i], expression.NewLiteral(upLit, upTyp)),
),
)
case sql.RangeType_OpenClosed:
lowLit, lowTyp := getType(sql.GetRangeCutKey(rangeColumnExpr.LowerBound))
upLit, upTyp := getType(sql.GetRangeCutKey(rangeColumnExpr.UpperBound))
rangeExpr = or(rangeExpr,
and(
expression.NewNullSafeGreaterThan(exprs[i], expression.NewLiteral(lowLit, lowTyp)),
expression.NewNullSafeLessThanOrEqual(exprs[i], expression.NewLiteral(upLit, upTyp)),
),
)
case sql.RangeType_ClosedOpen:
lowLit, lowTyp := getType(sql.GetRangeCutKey(rangeColumnExpr.LowerBound))
upLit, upTyp := getType(sql.GetRangeCutKey(rangeColumnExpr.UpperBound))
rangeExpr = or(rangeExpr,
and(
expression.NewNullSafeGreaterThanOrEqual(exprs[i], expression.NewLiteral(lowLit, lowTyp)),
expression.NewNullSafeLessThan(exprs[i], expression.NewLiteral(upLit, upTyp)),
),
} else {
lowLit, lowTyp := getType(sql.GetRangeCutKey(rce.LowerBound))
upLit, upTyp := getType(sql.GetRangeCutKey(rce.UpperBound))
rangeColumnExpr = and(
expression.NewNullSafeGreaterThanOrEqual(exprs[i], expression.NewLiteral(lowLit, lowTyp)),
expression.NewNullSafeLessThanOrEqual(exprs[i], expression.NewLiteral(upLit, upTyp)),
)
}
case sql.RangeType_OpenOpen:
lowLit, lowTyp := getType(sql.GetRangeCutKey(rce.LowerBound))
upLit, upTyp := getType(sql.GetRangeCutKey(rce.UpperBound))
rangeColumnExpr = and(
expression.NewNullSafeGreaterThan(exprs[i], expression.NewLiteral(lowLit, lowTyp)),
expression.NewNullSafeLessThan(exprs[i], expression.NewLiteral(upLit, upTyp)),
)
case sql.RangeType_OpenClosed:
lowLit, lowTyp := getType(sql.GetRangeCutKey(rce.LowerBound))
upLit, upTyp := getType(sql.GetRangeCutKey(rce.UpperBound))
rangeColumnExpr = and(
expression.NewNullSafeGreaterThan(exprs[i], expression.NewLiteral(lowLit, lowTyp)),
expression.NewNullSafeLessThanOrEqual(exprs[i], expression.NewLiteral(upLit, upTyp)),
)
case sql.RangeType_ClosedOpen:
lowLit, lowTyp := getType(sql.GetRangeCutKey(rce.LowerBound))
upLit, upTyp := getType(sql.GetRangeCutKey(rce.UpperBound))
rangeColumnExpr = and(
expression.NewNullSafeGreaterThanOrEqual(exprs[i], expression.NewLiteral(lowLit, lowTyp)),
expression.NewNullSafeLessThan(exprs[i], expression.NewLiteral(upLit, upTyp)),
)
}
if rangeExpr == nil {
continue
}
filterExpr = and(filterExpr, rangeExpr)
rangeExpr = and(rangeExpr, rangeColumnExpr)
}
completeExpr = or(completeExpr, filterExpr)
rangeCollectionExpr = or(rangeCollectionExpr, rangeExpr)
}

return NewIndexLookup(ctx, mergeableIndex, completeExpr, ranges...), nil
return NewIndexLookup(ctx, mergeableIndex, rangeCollectionExpr, ranges...), nil
}

// ColumnExpressionTypes implements the interface sql.Index.
Expand Down
24 changes: 12 additions & 12 deletions sql/analyzer/indexes.go
View file
Expand Up @@ -128,10 +128,10 @@ func getIndexes(
foundRightIdx := false
if rightIdx, ok := rightIndexes[table]; ok {
if canMergeIndexes(leftIdx.lookup, rightIdx.lookup) {
var allRanges []sql.Range
allRanges = append([]sql.Range{}, leftIdx.lookup.Ranges()...)
var allRanges sql.RangeCollection
allRanges = append(sql.RangeCollection{}, leftIdx.lookup.Ranges()...)
allRanges = append(allRanges, rightIdx.lookup.Ranges()...)
newRanges, err := sql.SimplifyRanges(allRanges...)
newRanges, err := sql.RemoveOverlappingRanges(allRanges...)
if err != nil {
return nil, nil
}
Expand Down Expand Up @@ -184,15 +184,15 @@ func getIndexes(
return nil, errInvalidInRightEvaluation.New(value)
}

var toUnion []sql.Range
var toUnion sql.RangeCollection
for _, val := range values {
ranges := sql.NewIndexBuilder(ctx, idx).Equals(ctx, colExprs[0].String(), val).Range()
if ranges == nil {
ranges := sql.NewIndexBuilder(ctx, idx).Equals(ctx, colExprs[0].String(), val).Ranges()
if len(ranges) == 0 {
return nil, nil
}
toUnion = append(toUnion, ranges)
toUnion = append(toUnion, ranges...)
}
allRanges, err := sql.SimplifyRanges(toUnion...)
allRanges, err := sql.RemoveOverlappingRanges(toUnion...)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -503,13 +503,13 @@ func getNegatedIndexes(
return nil, errInvalidInRightEvaluation.New(value)
}

var toIntersect []sql.Range
var toIntersect sql.RangeCollection
for _, val := range values {
ranges := sql.NewIndexBuilder(ctx, idx).NotEquals(ctx, normalizedExpressions[0].String(), val).Range()
if ranges == nil {
ranges := sql.NewIndexBuilder(ctx, idx).NotEquals(ctx, normalizedExpressions[0].String(), val).Ranges()
if len(ranges) == 0 {
return nil, nil
}
toIntersect = append(toIntersect, ranges)
toIntersect = append(toIntersect, ranges...)
}
allRanges := sql.IntersectRanges(toIntersect...)
if allRanges == nil {
Expand Down
13 changes: 6 additions & 7 deletions sql/index.go
View file
Expand Up @@ -39,13 +39,12 @@ type Index interface {
// IsGenerated returns whether this index was generated. Generated indexes
// are used for index access, but are not displayed (such as with SHOW INDEXES).
IsGenerated() bool
// NewLookup returns a new IndexLookup for the ranges given. Ranges represent filters over columns (RangeColumns),
// and it is possible for ranges to overlap (however they will not be subsets of one another). Each Range
// is ordered by the column expressions (as returned by Expressions) with the RangeColumn representing the
// searchable area for each column expression. Multiple RangeColumnExprs per RangeColumn are equivalent to the union
// of those RangeColumnExprs, e.g. (x < 1 OR x > 10) will return two RangeColumnExprs, where matching either is
// valid for the column "x". If an integrator is unable to process the given ranges, then a nil may be returned. An
// error should be returned only in the event that an error occurred.
// NewLookup returns a new IndexLookup for the ranges given. Ranges represent filters over columns. Each Range
// is ordered by the column expressions (as returned by Expressions) with the RangeColumnExpr representing the
// searchable area for each column expression. Each Range given will not overlap with any other ranges. Additionally,
// all ranges will have the same length, and may represent a partial index (matching a prefix rather than the entire
// index). If an integrator is unable to process the given ranges, then a nil may be returned. An error should be
// returned only in the event that an error occurred.
NewLookup(ctx *Context, ranges ...Range) (IndexLookup, error)
// ColumnExpressionTypes returns each expression and its associated Type. Each expression string should exactly
// match the string returned from Index.Expressions().
Expand Down
41 changes: 33 additions & 8 deletions sql/index_builder.go
View file
Expand Up @@ -31,7 +31,7 @@ type IndexBuilder struct {
isInvalid bool
err error
colExprTypes map[string]Type
ranges map[string]RangeColumn
ranges map[string][]RangeColumnExpr
}

// NewIndexBuilder returns a new IndexBuilder. Used internally to construct a range that will later be passed to
Expand All @@ -46,7 +46,7 @@ func NewIndexBuilder(ctx *Context, idx Index) *IndexBuilder {
isInvalid: false,
err: nil,
colExprTypes: colExprTypes,
ranges: make(map[string]RangeColumn),
ranges: make(map[string][]RangeColumnExpr),
}
}

Expand Down Expand Up @@ -154,22 +154,47 @@ func (b *IndexBuilder) LessOrEqual(ctx *Context, colExpr string, key interface{}
return b
}

// Range returns the range for this index builder. If the builder is invalid for any reason then this returns nil.
func (b *IndexBuilder) Range() Range {
// Ranges returns all ranges for this index builder. If the builder is invalid for any reason then this returns nil.
func (b *IndexBuilder) Ranges() RangeCollection {
if b.err != nil || b.isInvalid {
return nil
}
var rangeCollection Range
var allColumns [][]RangeColumnExpr
for _, colExpr := range b.idx.Expressions() {
ranges, ok := b.ranges[colExpr]
if !ok {
// An index builder is guaranteed to cover the first n expressions, so if we hit an expression that we do
// not have an entry for then we've hit all the ranges.
break
}
rangeCollection = append(rangeCollection, ranges)
allColumns = append(allColumns, ranges)
}

// In the builder ranges map we store multiple column expressions per column, however we want all permutations to
// be their own range, so here we're creating a new range for every permutation.
colCounts := make([]int, len(allColumns))
permutation := make([]int, len(allColumns))
for i, rangeColumn := range allColumns {
colCounts[i] = len(rangeColumn)
}
var ranges []Range
exit := false
for !exit {
exit = true
currentRange := make(Range, len(allColumns))
for colIdx, exprCount := range colCounts {
permutation[colIdx] = (permutation[colIdx] + 1) % exprCount
if permutation[colIdx] != 0 {
exit = false
break
}
}
for colIdx, exprIdx := range permutation {
currentRange[colIdx] = allColumns[colIdx][exprIdx]
}
ranges = append(ranges, currentRange)
}
return rangeCollection
return ranges
}

// Build constructs a new IndexLookup based on the ranges that have been built internally by this builder.
Expand All @@ -179,7 +204,7 @@ func (b *IndexBuilder) Build(ctx *Context) (IndexLookup, error) {
} else if b.isInvalid {
return nil, nil
} else {
return b.idx.NewLookup(ctx, b.Range())
return b.idx.NewLookup(ctx, b.Ranges()...)
}
}

Expand Down

0 comments on commit 4a443cd

Please sign in to comment.