diff --git a/pkg/container/types/decimal.go b/pkg/container/types/decimal.go index b1c36628b2172..e30fa01faf1c9 100644 --- a/pkg/container/types/decimal.go +++ b/pkg/container/types/decimal.go @@ -199,6 +199,9 @@ func CompareDecimal128(x Decimal128, y Decimal128) int { } func CompareDecimal64WithScale(x, y Decimal64, scale1, scale2 int32) int { + if scale1 == scale2 { + return x.Compare(y) + } if x.Sign() != y.Sign() { if x.Sign() { return -1 @@ -208,25 +211,29 @@ func CompareDecimal64WithScale(x, y Decimal64, scale1, scale2 int32) int { } var err error if scale1 < scale2 { - x, err = x.Scale(scale2 - scale1) - if err != nil { - if x.Sign() { - return -1 - } else { - return 1 - } - } - return x.Compare(y) + scaled := x + scaled, err = scaled.Scale(scale2 - scale1) + if err != nil || scaled.Sign() != x.Sign() { + return CompareDecimal128WithScale( + Decimal128FromDecimal64(x, scale1), + Decimal128FromDecimal64(y, scale2), + scale1, + scale2, + ) + } + return scaled.Compare(y) } else { - y, err = y.Scale(scale1 - scale2) - if err != nil { - if x.Sign() { - return 1 - } else { - return -1 - } - } - return x.Compare(y) + scaled := y + scaled, err = scaled.Scale(scale1 - scale2) + if err != nil || scaled.Sign() != y.Sign() { + return CompareDecimal128WithScale( + Decimal128FromDecimal64(x, scale1), + Decimal128FromDecimal64(y, scale2), + scale1, + scale2, + ) + } + return x.Compare(scaled) } } diff --git a/pkg/container/types/decimal_test.go b/pkg/container/types/decimal_test.go index 724af1afff4ec..c74c49f285795 100644 --- a/pkg/container/types/decimal_test.go +++ b/pkg/container/types/decimal_test.go @@ -200,6 +200,32 @@ func TestCompare64(t *testing.T) { } } +func TestCompareDecimal64WithScaleFallsBackOnSignedOverflow(t *testing.T) { + positive, err := ParseDecimal64("9.99999999999999998", 18, 17) + require.NoError(t, err) + negative, err := ParseDecimal64("-9.99999999999999998", 18, 17) + require.NoError(t, err) + positiveBound, err := ParseDecimal64("100", 18, 0) + require.NoError(t, err) + negativeBound, err := ParseDecimal64("-100", 18, 0) + require.NoError(t, err) + + require.Less(t, CompareDecimal64WithScale(positive, positiveBound, 17, 0), 0) + require.Greater(t, CompareDecimal64WithScale(negative, negativeBound, 17, 0), 0) + require.Greater(t, CompareDecimal64WithScale(positiveBound, positive, 0, 17), 0) + require.Less(t, CompareDecimal64WithScale(negativeBound, negative, 0, 17), 0) +} + +func TestCompareDecimal64WithScaleFastPaths(t *testing.T) { + negative := Decimal64(1).Minus() + + require.Less(t, CompareDecimal64WithScale(Decimal64(1), Decimal64(2), 2, 2), 0) + require.Less(t, CompareDecimal64WithScale(negative, Decimal64(1), 1, 2), 0) + require.Greater(t, CompareDecimal64WithScale(Decimal64(1), negative, 2, 1), 0) + require.Zero(t, CompareDecimal64WithScale(Decimal64(12), Decimal64(120), 1, 2)) + require.Zero(t, CompareDecimal64WithScale(Decimal64(120), Decimal64(12), 2, 1)) +} + func TestCompare128(t *testing.T) { x := Decimal128{0, 0} y := Decimal128{^x.B0_63, ^x.B64_127} diff --git a/pkg/vm/engine/readutil/expr_filter.go b/pkg/vm/engine/readutil/expr_filter.go index 85fa28986b9ac..cb9e4454fc082 100644 --- a/pkg/vm/engine/readutil/expr_filter.go +++ b/pkg/vm/engine/readutil/expr_filter.go @@ -100,6 +100,300 @@ func isSortedKey(colDef *plan.ColDef) (isPK, isSorted bool) { return } +// makeDecimalZoneMapBound preserves the bound's own scale for pruning. Folded +// constants carry correctly encoded bytes and type metadata, but raw-byte ZM +// helpers assume that the bytes already use the persisted ZM's scale. That +// assumption is not valid for comparisons such as DECIMAL(20,4) < DECIMAL(38,0). +func makeDecimalZoneMapBound(colDef *plan.ColDef, value []byte, valueExpr *plan.Expr) (objectio.ZoneMap, bool) { + columnType := types.T(colDef.Typ.Id) + if !columnType.IsDecimal() { + return nil, true + } + if valueExpr == nil || types.T(valueExpr.Typ.Id) != columnType || + columnType == types.T_decimal256 || len(value) != columnType.FixedLength() { + return nil, false + } + bound := index.NewZM(columnType, valueExpr.Typ.Scale) + index.UpdateZM(bound, value) + return bound, true +} + +type zoneMapMatch struct { + matches bool + comparable bool +} + +func (m zoneMapMatch) mayMatch() bool { + return !m.comparable || m.matches +} + +func (m zoneMapMatch) excludes() bool { + return m.comparable && !m.matches +} + +func (m zoneMapMatch) and(other zoneMapMatch) zoneMapMatch { + if m.excludes() || other.excludes() { + return zoneMapMatch{comparable: true} + } + if m.comparable && other.comparable { + return zoneMapMatch{matches: true, comparable: true} + } + return zoneMapMatch{} +} + +func rawZoneMapComparable(zm objectio.ZoneMap, columnType types.T) bool { + return zm.IsInited() && zm.GetType() == columnType +} + +func anyLTByBound( + zm objectio.ZoneMap, value []byte, bound objectio.ZoneMap, columnType types.T, +) zoneMapMatch { + if bound == nil { + if !rawZoneMapComparable(zm, columnType) { + return zoneMapMatch{} + } + return zoneMapMatch{matches: zm.AnyLTByValue(value), comparable: true} + } + result, ok := zm.AnyLT(bound) + return zoneMapMatch{matches: result, comparable: ok} +} + +func anyLEByBound( + zm objectio.ZoneMap, value []byte, bound objectio.ZoneMap, columnType types.T, +) zoneMapMatch { + if bound == nil { + if !rawZoneMapComparable(zm, columnType) { + return zoneMapMatch{} + } + return zoneMapMatch{matches: zm.AnyLEByValue(value), comparable: true} + } + result, ok := zm.AnyLE(bound) + return zoneMapMatch{matches: result, comparable: ok} +} + +func anyGTByBound( + zm objectio.ZoneMap, value []byte, bound objectio.ZoneMap, columnType types.T, +) zoneMapMatch { + if bound == nil { + if !rawZoneMapComparable(zm, columnType) { + return zoneMapMatch{} + } + return zoneMapMatch{matches: zm.AnyGTByValue(value), comparable: true} + } + result, ok := zm.AnyGT(bound) + return zoneMapMatch{matches: result, comparable: ok} +} + +func anyGEByBound( + zm objectio.ZoneMap, value []byte, bound objectio.ZoneMap, columnType types.T, +) zoneMapMatch { + if bound == nil { + if !rawZoneMapComparable(zm, columnType) { + return zoneMapMatch{} + } + return zoneMapMatch{matches: zm.AnyGEByValue(value), comparable: true} + } + result, ok := zm.AnyGE(bound) + return zoneMapMatch{matches: result, comparable: ok} +} + +func intersectsBound( + zm objectio.ZoneMap, value []byte, bound objectio.ZoneMap, columnType types.T, +) zoneMapMatch { + if bound == nil { + if !rawZoneMapComparable(zm, columnType) { + return zoneMapMatch{} + } + return zoneMapMatch{matches: zm.ContainsKey(value), comparable: true} + } + result, ok := zm.Intersect(bound) + return zoneMapMatch{matches: result, comparable: ok} +} + +func anyBetweenBounds( + zm objectio.ZoneMap, + lowerValue, upperValue []byte, + lowerBound, upperBound objectio.ZoneMap, + columnType types.T, +) zoneMapMatch { + if lowerBound == nil { + if !rawZoneMapComparable(zm, columnType) { + return zoneMapMatch{} + } + return zoneMapMatch{matches: zm.Between(lowerValue, upperValue), comparable: true} + } + result, ok := zm.AnyBetween(lowerBound, upperBound) + return zoneMapMatch{matches: result, comparable: ok} +} + +func inRangeBounds( + zm objectio.ZoneMap, + lowerValue, upperValue []byte, + lowerBound, upperBound objectio.ZoneMap, + hint uint8, + columnType types.T, +) zoneMapMatch { + switch hint { + case 1: // (lb, ub] + return anyGTByBound(zm, lowerValue, lowerBound, columnType). + and(anyLEByBound(zm, upperValue, upperBound, columnType)) + case 2: // [lb, ub) + return anyGEByBound(zm, lowerValue, lowerBound, columnType). + and(anyLTByBound(zm, upperValue, upperBound, columnType)) + case 3: // (lb, ub) + return anyGTByBound(zm, lowerValue, lowerBound, columnType). + and(anyLTByBound(zm, upperValue, upperBound, columnType)) + default: // [lb, ub] + return anyGEByBound(zm, lowerValue, lowerBound, columnType). + and(anyLEByBound(zm, upperValue, upperBound, columnType)) + } +} + +func zoneMapVectorComparable( + zm objectio.ZoneMap, vec *vector.Vector, columnType types.T, +) bool { + if !rawZoneMapComparable(zm, columnType) || vec == nil || vec.GetType().Oid != columnType { + return false + } + return !columnType.IsDecimal() || zm.GetScale() == vec.GetType().Scale +} + +func anyInVector( + zm objectio.ZoneMap, vec *vector.Vector, columnType types.T, +) zoneMapMatch { + if !zoneMapVectorComparable(zm, vec, columnType) { + return zoneMapMatch{} + } + return zoneMapMatch{matches: zm.AnyIn(vec), comparable: true} +} + +func prefixEqByValue(zm objectio.ZoneMap, value []byte, columnType types.T) zoneMapMatch { + if !rawZoneMapComparable(zm, columnType) { + return zoneMapMatch{} + } + return zoneMapMatch{matches: zm.PrefixEq(value), comparable: true} +} + +func prefixBetweenByValue( + zm objectio.ZoneMap, lower, upper []byte, columnType types.T, +) zoneMapMatch { + if !rawZoneMapComparable(zm, columnType) { + return zoneMapMatch{} + } + return zoneMapMatch{matches: zm.PrefixBetween(lower, upper), comparable: true} +} + +func prefixInRangeByValue( + zm objectio.ZoneMap, lower, upper []byte, hint uint8, columnType types.T, +) zoneMapMatch { + if !rawZoneMapComparable(zm, columnType) { + return zoneMapMatch{} + } + return zoneMapMatch{matches: zm.PrefixInRange(lower, upper, hint), comparable: true} +} + +func prefixInVector( + zm objectio.ZoneMap, vec *vector.Vector, columnType types.T, +) zoneMapMatch { + if !zoneMapVectorComparable(zm, vec, columnType) { + return zoneMapMatch{} + } + if vec.IsConstNull() || vec.GetNulls().Any() { + return zoneMapMatch{} + } + return zoneMapMatch{matches: zm.PrefixIn(vec), comparable: true} +} + +func anyPrefixLTByValue(zm objectio.ZoneMap, value []byte, columnType types.T) zoneMapMatch { + if !rawZoneMapComparable(zm, columnType) { + return zoneMapMatch{} + } + return zoneMapMatch{ + matches: types.PrefixCompare(zm.GetMinBuf(), value) < 0, + comparable: true, + } +} + +func anyPrefixLEByValue(zm objectio.ZoneMap, value []byte, columnType types.T) zoneMapMatch { + if !rawZoneMapComparable(zm, columnType) { + return zoneMapMatch{} + } + return zoneMapMatch{ + matches: types.PrefixCompare(zm.GetMinBuf(), value) <= 0, + comparable: true, + } +} + +func anyPrefixGTByValue(zm objectio.ZoneMap, value []byte, columnType types.T) zoneMapMatch { + if !rawZoneMapComparable(zm, columnType) { + return zoneMapMatch{} + } + return zoneMapMatch{ + matches: types.PrefixCompare(zm.GetMaxBuf(), value) > 0, + comparable: true, + } +} + +func anyPrefixGEByValue(zm objectio.ZoneMap, value []byte, columnType types.T) zoneMapMatch { + if !rawZoneMapComparable(zm, columnType) { + return zoneMapMatch{} + } + return zoneMapMatch{ + matches: types.PrefixCompare(zm.GetMaxBuf(), value) >= 0, + comparable: true, + } +} + +func makeVectorValueZoneMapBound( + columnType types.T, vec *vector.Vector, value []byte, +) (objectio.ZoneMap, bool) { + if vec == nil || vec.GetType().Oid != columnType { + return nil, false + } + if !columnType.IsDecimal() { + return nil, true + } + if columnType == types.T_decimal256 || len(value) != columnType.FixedLength() { + return nil, false + } + bound := index.NewZM(columnType, vec.GetType().Scale) + index.UpdateZM(bound, value) + return bound, true +} + +func seekFirstBlockByZoneMap( + meta objectio.ObjectDataMeta, + seqNum uint16, + bound objectio.ZoneMap, + columnType types.T, + compare func(objectio.ZoneMap) zoneMapMatch, +) int { + blockCnt := int(meta.BlockCount()) + if blockCnt == 0 || !zoneMapMetadataComparable(meta.MustGetColumn(seqNum).ZoneMap(), bound, columnType) { + return 0 + } + for j := range blockCnt { + if !zoneMapMetadataComparable( + meta.GetBlockMeta(uint32(j)).MustGetColumn(seqNum).ZoneMap(), bound, columnType, + ) { + return 0 + } + } + return sort.Search(blockCnt, func(j int) bool { + result := compare(meta.GetBlockMeta(uint32(j)).MustGetColumn(seqNum).ZoneMap()) + return result.matches + }) +} + +func zoneMapMetadataComparable( + zm objectio.ZoneMap, bound objectio.ZoneMap, columnType types.T, +) bool { + if !zm.IsInited() || zm.GetType() != columnType { + return false + } + return bound == nil || (bound.IsInited() && bound.GetType() == columnType) +} + func CompileFilterExprs( exprs []*plan.Expr, tableDef *plan.TableDef, @@ -433,19 +727,24 @@ func CompileFilterExpr( } case "<=": - colExpr, vals, ok := mustColConstValueFromBinaryFuncExpr(exprImpl) + colExpr, vals, valExprs, ok := mustColConstValueWithTypeFromBinaryFuncExpr(exprImpl) if !ok { canCompile = false return } colDef := getColDefByName(expr, colExpr.Col.Name, colExpr.Col.ColPos, tableDef) + bound, ok := makeDecimalZoneMapBound(colDef, vals[0], valExprs[0]) + if !ok { + canCompile = false + return + } _, isSorted := isSortedKey(colDef) if isSorted { fastFilterOp = func(obj *objectio.ObjectStats) (bool, error) { if obj.ZMIsEmpty() { return true, nil } - return obj.SortKeyZoneMap().AnyLEByValue(vals[0]), nil + return anyLEByBound(obj.SortKeyZoneMap(), vals[0], bound, types.T(colDef.Typ.Id)).mayMatch(), nil } } loadOp = loadMetadataOnlyOpFactory(fs) @@ -455,31 +754,36 @@ func CompileFilterExpr( return true, nil } dataMeta := meta.MustDataMeta() - return dataMeta.MustGetColumn(uint16(seqNum)).ZoneMap().AnyLEByValue(vals[0]), nil + return anyLEByBound(dataMeta.MustGetColumn(uint16(seqNum)).ZoneMap(), vals[0], bound, types.T(colDef.Typ.Id)).mayMatch(), nil } blockFilterOp = func( blkIdx int, blkMeta objectio.BlockObject, bf objectio.BloomFilter, ) (bool, bool, error) { - ok := blkMeta.MustGetColumn(uint16(seqNum)).ZoneMap().AnyLEByValue(vals[0]) + result := anyLEByBound(blkMeta.MustGetColumn(uint16(seqNum)).ZoneMap(), vals[0], bound, types.T(colDef.Typ.Id)) if isSorted { - return !ok, ok, nil + return result.excludes(), result.mayMatch(), nil } - return false, ok, nil + return false, result.mayMatch(), nil } case ">=": - colExpr, vals, ok := mustColConstValueFromBinaryFuncExpr(exprImpl) + colExpr, vals, valExprs, ok := mustColConstValueWithTypeFromBinaryFuncExpr(exprImpl) if !ok { canCompile = false return } colDef := getColDefByName(expr, colExpr.Col.Name, colExpr.Col.ColPos, tableDef) + bound, ok := makeDecimalZoneMapBound(colDef, vals[0], valExprs[0]) + if !ok { + canCompile = false + return + } _, isSorted := isSortedKey(colDef) if isSorted { fastFilterOp = func(obj *objectio.ObjectStats) (bool, error) { if obj.ZMIsEmpty() { return true, nil } - return obj.SortKeyZoneMap().AnyGEByValue(vals[0]), nil + return anyGEByBound(obj.SortKeyZoneMap(), vals[0], bound, types.T(colDef.Typ.Id)).mayMatch(), nil } } loadOp = loadMetadataOnlyOpFactory(fs) @@ -489,36 +793,39 @@ func CompileFilterExpr( return true, nil } dataMeta := meta.MustDataMeta() - return dataMeta.MustGetColumn(uint16(seqNum)).ZoneMap().AnyGEByValue(vals[0]), nil + return anyGEByBound(dataMeta.MustGetColumn(uint16(seqNum)).ZoneMap(), vals[0], bound, types.T(colDef.Typ.Id)).mayMatch(), nil } blockFilterOp = func( _ int, blkMeta objectio.BlockObject, bf objectio.BloomFilter, ) (bool, bool, error) { - return false, blkMeta.MustGetColumn(uint16(seqNum)).ZoneMap().AnyGEByValue(vals[0]), nil + return false, anyGEByBound(blkMeta.MustGetColumn(uint16(seqNum)).ZoneMap(), vals[0], bound, types.T(colDef.Typ.Id)).mayMatch(), nil } if isSorted { seekOp = func(meta objectio.ObjectDataMeta) int { - blockCnt := int(meta.BlockCount()) - blkIdx := sort.Search(blockCnt, func(j int) bool { - return meta.GetBlockMeta(uint32(j)).MustGetColumn(uint16(seqNum)).ZoneMap().AnyGEByValue(vals[0]) + return seekFirstBlockByZoneMap(meta, uint16(seqNum), bound, types.T(colDef.Typ.Id), func(zm objectio.ZoneMap) zoneMapMatch { + return anyGEByBound(zm, vals[0], bound, types.T(colDef.Typ.Id)) }) - return blkIdx } } case ">": - colExpr, vals, ok := mustColConstValueFromBinaryFuncExpr(exprImpl) + colExpr, vals, valExprs, ok := mustColConstValueWithTypeFromBinaryFuncExpr(exprImpl) if !ok { canCompile = false return } colDef := getColDefByName(expr, colExpr.Col.Name, colExpr.Col.ColPos, tableDef) + bound, ok := makeDecimalZoneMapBound(colDef, vals[0], valExprs[0]) + if !ok { + canCompile = false + return + } _, isSorted := isSortedKey(colDef) if isSorted { fastFilterOp = func(obj *objectio.ObjectStats) (bool, error) { if obj.ZMIsEmpty() { return true, nil } - return obj.SortKeyZoneMap().AnyGTByValue(vals[0]), nil + return anyGTByBound(obj.SortKeyZoneMap(), vals[0], bound, types.T(colDef.Typ.Id)).mayMatch(), nil } } loadOp = loadMetadataOnlyOpFactory(fs) @@ -528,36 +835,39 @@ func CompileFilterExpr( return true, nil } dataMeta := meta.MustDataMeta() - return dataMeta.MustGetColumn(uint16(seqNum)).ZoneMap().AnyGTByValue(vals[0]), nil + return anyGTByBound(dataMeta.MustGetColumn(uint16(seqNum)).ZoneMap(), vals[0], bound, types.T(colDef.Typ.Id)).mayMatch(), nil } blockFilterOp = func( _ int, blkMeta objectio.BlockObject, bf objectio.BloomFilter, ) (bool, bool, error) { - return false, blkMeta.MustGetColumn(uint16(seqNum)).ZoneMap().AnyGTByValue(vals[0]), nil + return false, anyGTByBound(blkMeta.MustGetColumn(uint16(seqNum)).ZoneMap(), vals[0], bound, types.T(colDef.Typ.Id)).mayMatch(), nil } if isSorted { seekOp = func(meta objectio.ObjectDataMeta) int { - blockCnt := int(meta.BlockCount()) - blkIdx := sort.Search(blockCnt, func(j int) bool { - return meta.GetBlockMeta(uint32(j)).MustGetColumn(uint16(seqNum)).ZoneMap().AnyGTByValue(vals[0]) + return seekFirstBlockByZoneMap(meta, uint16(seqNum), bound, types.T(colDef.Typ.Id), func(zm objectio.ZoneMap) zoneMapMatch { + return anyGTByBound(zm, vals[0], bound, types.T(colDef.Typ.Id)) }) - return blkIdx } } case "<": - colExpr, vals, ok := mustColConstValueFromBinaryFuncExpr(exprImpl) + colExpr, vals, valExprs, ok := mustColConstValueWithTypeFromBinaryFuncExpr(exprImpl) if !ok { canCompile = false return } colDef := getColDefByName(expr, colExpr.Col.Name, colExpr.Col.ColPos, tableDef) + bound, ok := makeDecimalZoneMapBound(colDef, vals[0], valExprs[0]) + if !ok { + canCompile = false + return + } _, isSorted := isSortedKey(colDef) if isSorted { fastFilterOp = func(obj *objectio.ObjectStats) (bool, error) { if obj.ZMIsEmpty() { return true, nil } - return obj.SortKeyZoneMap().AnyLTByValue(vals[0]), nil + return anyLTByBound(obj.SortKeyZoneMap(), vals[0], bound, types.T(colDef.Typ.Id)).mayMatch(), nil } } loadOp = loadMetadataOnlyOpFactory(fs) @@ -567,16 +877,16 @@ func CompileFilterExpr( return true, nil } dataMeta := meta.MustDataMeta() - return dataMeta.MustGetColumn(uint16(seqNum)).ZoneMap().AnyLTByValue(vals[0]), nil + return anyLTByBound(dataMeta.MustGetColumn(uint16(seqNum)).ZoneMap(), vals[0], bound, types.T(colDef.Typ.Id)).mayMatch(), nil } blockFilterOp = func( _ int, blkMeta objectio.BlockObject, bf objectio.BloomFilter, ) (bool, bool, error) { - ok := blkMeta.MustGetColumn(uint16(seqNum)).ZoneMap().AnyLTByValue(vals[0]) + result := anyLTByBound(blkMeta.MustGetColumn(uint16(seqNum)).ZoneMap(), vals[0], bound, types.T(colDef.Typ.Id)) if isSorted { - return !ok, ok, nil + return result.excludes(), result.mayMatch(), nil } - return false, ok, nil + return false, result.mayMatch(), nil } case "prefix_eq": colExpr, vals, ok := mustColConstValueFromBinaryFuncExpr(exprImpl) @@ -585,13 +895,14 @@ func CompileFilterExpr( return } colDef := getColDefByName(expr, colExpr.Col.Name, colExpr.Col.ColPos, tableDef) + columnType := types.T(colDef.Typ.Id) isPK, isSorted := isSortedKey(colDef) if isSorted { fastFilterOp = func(obj *objectio.ObjectStats) (bool, error) { if obj.ZMIsEmpty() { return true, nil } - return obj.SortKeyZoneMap().PrefixEq(vals[0]), nil + return prefixEqByValue(obj.SortKeyZoneMap(), vals[0], columnType).mayMatch(), nil } } highSelectivityHint = isPK @@ -603,25 +914,23 @@ func CompileFilterExpr( return true, nil } dataMeta := meta.MustDataMeta() - return dataMeta.MustGetColumn(uint16(seqNum)).ZoneMap().PrefixEq(vals[0]), nil + return prefixEqByValue( + dataMeta.MustGetColumn(uint16(seqNum)).ZoneMap(), vals[0], columnType, + ).mayMatch(), nil } blockFilterOp = func( _ int, blkMeta objectio.BlockObject, bf objectio.BloomFilter, ) (bool, bool, error) { zm := blkMeta.MustGetColumn(uint16(seqNum)).ZoneMap() - if isSorted && types.PrefixCompare(zm.GetMinBuf(), vals[0]) > 0 { + if isSorted && anyPrefixLEByValue(zm, vals[0], columnType).excludes() { return true, false, nil } - return false, zm.PrefixEq(vals[0]), nil + return false, prefixEqByValue(zm, vals[0], columnType).mayMatch(), nil } if isSorted { seekOp = func(meta objectio.ObjectDataMeta) int { - blockCnt := int(meta.BlockCount()) - return sort.Search(blockCnt, func(j int) bool { - return types.PrefixCompare( - meta.GetBlockMeta(uint32(j)).MustGetColumn(uint16(seqNum)).ZoneMap().GetMaxBuf(), - vals[0], - ) >= 0 + return seekFirstBlockByZoneMap(meta, uint16(seqNum), nil, columnType, func(zm objectio.ZoneMap) zoneMapMatch { + return anyPrefixGEByValue(zm, vals[0], columnType) }) } } @@ -632,13 +941,14 @@ func CompileFilterExpr( return } colDef := getColDefByName(expr, colExpr.Col.Name, colExpr.Col.ColPos, tableDef) + columnType := types.T(colDef.Typ.Id) _, isSorted := isSortedKey(colDef) if isSorted { fastFilterOp = func(obj *objectio.ObjectStats) (bool, error) { if obj.ZMIsEmpty() { return true, nil } - return obj.SortKeyZoneMap().PrefixBetween(vals[0], vals[1]), nil + return prefixBetweenByValue(obj.SortKeyZoneMap(), vals[0], vals[1], columnType).mayMatch(), nil } } loadOp = loadMetadataOnlyOpFactory(fs) @@ -648,25 +958,23 @@ func CompileFilterExpr( return true, nil } dataMeta := meta.MustDataMeta() - return dataMeta.MustGetColumn(uint16(seqNum)).ZoneMap().PrefixBetween(vals[0], vals[1]), nil + return prefixBetweenByValue( + dataMeta.MustGetColumn(uint16(seqNum)).ZoneMap(), vals[0], vals[1], columnType, + ).mayMatch(), nil } blockFilterOp = func( _ int, blkMeta objectio.BlockObject, bf objectio.BloomFilter, ) (bool, bool, error) { zm := blkMeta.MustGetColumn(uint16(seqNum)).ZoneMap() - if isSorted && types.PrefixCompare(zm.GetMinBuf(), vals[1]) > 0 { + if isSorted && anyPrefixLEByValue(zm, vals[1], columnType).excludes() { return true, false, nil } - return false, zm.PrefixBetween(vals[0], vals[1]), nil + return false, prefixBetweenByValue(zm, vals[0], vals[1], columnType).mayMatch(), nil } if isSorted { seekOp = func(meta objectio.ObjectDataMeta) int { - blockCnt := int(meta.BlockCount()) - return sort.Search(blockCnt, func(j int) bool { - return types.PrefixCompare( - meta.GetBlockMeta(uint32(j)).MustGetColumn(uint16(seqNum)).ZoneMap().GetMaxBuf(), - vals[0], - ) >= 0 + return seekFirstBlockByZoneMap(meta, uint16(seqNum), nil, columnType, func(zm objectio.ZoneMap) zoneMapMatch { + return anyPrefixGEByValue(zm, vals[0], columnType) }) } } @@ -678,13 +986,16 @@ func CompileFilterExpr( } hint := vals[2][0] colDef := getColDefByName(expr, colExpr.Col.Name, colExpr.Col.ColPos, tableDef) + columnType := types.T(colDef.Typ.Id) _, isSorted := isSortedKey(colDef) if isSorted { fastFilterOp = func(obj *objectio.ObjectStats) (bool, error) { if obj.ZMIsEmpty() { return true, nil } - return obj.SortKeyZoneMap().PrefixInRange(vals[0], vals[1], hint), nil + return prefixInRangeByValue( + obj.SortKeyZoneMap(), vals[0], vals[1], hint, columnType, + ).mayMatch(), nil } } loadOp = loadMetadataOnlyOpFactory(fs) @@ -694,46 +1005,61 @@ func CompileFilterExpr( return true, nil } dataMeta := meta.MustDataMeta() - return dataMeta.MustGetColumn(uint16(seqNum)).ZoneMap().PrefixInRange(vals[0], vals[1], hint), nil + return prefixInRangeByValue( + dataMeta.MustGetColumn(uint16(seqNum)).ZoneMap(), vals[0], vals[1], hint, columnType, + ).mayMatch(), nil } blockFilterOp = func( _ int, blkMeta objectio.BlockObject, bf objectio.BloomFilter, ) (bool, bool, error) { zm := blkMeta.MustGetColumn(uint16(seqNum)).ZoneMap() if isSorted { - cmp := types.PrefixCompare(zm.GetMinBuf(), vals[1]) - if cmp > 0 || (cmp == 0 && (hint == 2 || hint == 3)) { + upperResult := anyPrefixLEByValue(zm, vals[1], columnType) + if hint == 2 || hint == 3 { + upperResult = anyPrefixLTByValue(zm, vals[1], columnType) + } + if upperResult.excludes() { return true, false, nil } } - return false, zm.PrefixInRange(vals[0], vals[1], hint), nil + return false, prefixInRangeByValue(zm, vals[0], vals[1], hint, columnType).mayMatch(), nil } if isSorted { seekOp = func(meta objectio.ObjectDataMeta) int { - blockCnt := int(meta.BlockCount()) - return sort.Search(blockCnt, func(j int) bool { - zm := meta.GetBlockMeta(uint32(j)).MustGetColumn(uint16(seqNum)).ZoneMap() + return seekFirstBlockByZoneMap(meta, uint16(seqNum), nil, columnType, func(zm objectio.ZoneMap) zoneMapMatch { if hint == 1 || hint == 3 { - return types.PrefixCompare(zm.GetMaxBuf(), vals[0]) > 0 + return anyPrefixGTByValue(zm, vals[0], columnType) } - return types.PrefixCompare(zm.GetMaxBuf(), vals[0]) >= 0 + return anyPrefixGEByValue(zm, vals[0], columnType) }) } } case "between": - colExpr, vals, ok := mustColConstValueFromBinaryFuncExpr(exprImpl) + colExpr, vals, valExprs, ok := mustColConstValueWithTypeFromBinaryFuncExpr(exprImpl) if !ok { canCompile = false return } colDef := getColDefByName(expr, colExpr.Col.Name, colExpr.Col.ColPos, tableDef) + lowerBound, ok := makeDecimalZoneMapBound(colDef, vals[0], valExprs[0]) + if !ok { + canCompile = false + return + } + upperBound, ok := makeDecimalZoneMapBound(colDef, vals[1], valExprs[1]) + if !ok { + canCompile = false + return + } _, isSorted := isSortedKey(colDef) if isSorted { fastFilterOp = func(obj *objectio.ObjectStats) (bool, error) { if obj.ZMIsEmpty() { return true, nil } - return obj.SortKeyZoneMap().Between(vals[0], vals[1]), nil + return anyBetweenBounds( + obj.SortKeyZoneMap(), vals[0], vals[1], lowerBound, upperBound, types.T(colDef.Typ.Id), + ).mayMatch(), nil } } loadOp = loadMetadataOnlyOpFactory(fs) @@ -743,40 +1069,56 @@ func CompileFilterExpr( return true, nil } dataMeta := meta.MustDataMeta() - return dataMeta.MustGetColumn(uint16(seqNum)).ZoneMap().Between(vals[0], vals[1]), nil + return anyBetweenBounds( + dataMeta.MustGetColumn(uint16(seqNum)).ZoneMap(), vals[0], vals[1], lowerBound, upperBound, types.T(colDef.Typ.Id), + ).mayMatch(), nil } blockFilterOp = func( _ int, blkMeta objectio.BlockObject, bf objectio.BloomFilter, ) (bool, bool, error) { zm := blkMeta.MustGetColumn(uint16(seqNum)).ZoneMap() - if isSorted && !zm.AnyLEByValue(vals[1]) { + upperResult := anyLEByBound(zm, vals[1], upperBound, types.T(colDef.Typ.Id)) + if isSorted && upperResult.excludes() { return true, false, nil } - return false, zm.Between(vals[0], vals[1]), nil + return false, anyBetweenBounds( + zm, vals[0], vals[1], lowerBound, upperBound, types.T(colDef.Typ.Id), + ).mayMatch(), nil } if isSorted { seekOp = func(meta objectio.ObjectDataMeta) int { - blockCnt := int(meta.BlockCount()) - return sort.Search(blockCnt, func(j int) bool { - return meta.GetBlockMeta(uint32(j)).MustGetColumn(uint16(seqNum)).ZoneMap().AnyGEByValue(vals[0]) + return seekFirstBlockByZoneMap(meta, uint16(seqNum), lowerBound, types.T(colDef.Typ.Id), func(zm objectio.ZoneMap) zoneMapMatch { + return anyGEByBound(zm, vals[0], lowerBound, types.T(colDef.Typ.Id)) }) } } case "in_range": - colExpr, vals, ok := mustColConstValueFromBinaryFuncExpr(exprImpl) + colExpr, vals, valExprs, ok := mustColConstValueWithTypeFromBinaryFuncExpr(exprImpl) if !ok || len(vals) < 3 || len(vals[2]) == 0 { canCompile = false return } hint := vals[2][0] colDef := getColDefByName(expr, colExpr.Col.Name, colExpr.Col.ColPos, tableDef) + lowerBound, ok := makeDecimalZoneMapBound(colDef, vals[0], valExprs[0]) + if !ok { + canCompile = false + return + } + upperBound, ok := makeDecimalZoneMapBound(colDef, vals[1], valExprs[1]) + if !ok { + canCompile = false + return + } _, isSorted := isSortedKey(colDef) if isSorted { fastFilterOp = func(obj *objectio.ObjectStats) (bool, error) { if obj.ZMIsEmpty() { return true, nil } - return obj.SortKeyZoneMap().InRange(vals[0], vals[1], hint), nil + return inRangeBounds( + obj.SortKeyZoneMap(), vals[0], vals[1], lowerBound, upperBound, hint, types.T(colDef.Typ.Id), + ).mayMatch(), nil } } loadOp = loadMetadataOnlyOpFactory(fs) @@ -786,7 +1128,9 @@ func CompileFilterExpr( return true, nil } dataMeta := meta.MustDataMeta() - return dataMeta.MustGetColumn(uint16(seqNum)).ZoneMap().InRange(vals[0], vals[1], hint), nil + return inRangeBounds( + dataMeta.MustGetColumn(uint16(seqNum)).ZoneMap(), vals[0], vals[1], lowerBound, upperBound, hint, types.T(colDef.Typ.Id), + ).mayMatch(), nil } blockFilterOp = func( _ int, blkMeta objectio.BlockObject, bf objectio.BloomFilter, @@ -795,27 +1139,27 @@ func CompileFilterExpr( if isSorted { if hint == 2 || hint == 3 { // open UB: break when min >= ub - if !zm.AnyLTByValue(vals[1]) { + if anyLTByBound(zm, vals[1], upperBound, types.T(colDef.Typ.Id)).excludes() { return true, false, nil } } else { // closed UB: break when min > ub - if !zm.AnyLEByValue(vals[1]) { + if anyLEByBound(zm, vals[1], upperBound, types.T(colDef.Typ.Id)).excludes() { return true, false, nil } } } - return false, zm.InRange(vals[0], vals[1], hint), nil + return false, inRangeBounds( + zm, vals[0], vals[1], lowerBound, upperBound, hint, types.T(colDef.Typ.Id), + ).mayMatch(), nil } if isSorted { seekOp = func(meta objectio.ObjectDataMeta) int { - blockCnt := int(meta.BlockCount()) - return sort.Search(blockCnt, func(j int) bool { - zm := meta.GetBlockMeta(uint32(j)).MustGetColumn(uint16(seqNum)).ZoneMap() + return seekFirstBlockByZoneMap(meta, uint16(seqNum), lowerBound, types.T(colDef.Typ.Id), func(zm objectio.ZoneMap) zoneMapMatch { if hint == 1 || hint == 3 { - return zm.AnyGTByValue(vals[0]) + return anyGTByBound(zm, vals[0], lowerBound, types.T(colDef.Typ.Id)) } - return zm.AnyGEByValue(vals[0]) + return anyGEByBound(zm, vals[0], lowerBound, types.T(colDef.Typ.Id)) }) } } @@ -826,15 +1170,23 @@ func CompileFilterExpr( return } vec := vector.NewVec(types.T_any.ToType()) - _ = vec.UnmarshalBinary(val) + if err := vec.UnmarshalBinary(val); err != nil { + canCompile = false + return + } colDef := getColDefByName(expr, colExpr.Col.Name, colExpr.Col.ColPos, tableDef) + columnType := types.T(colDef.Typ.Id) + if columnType != types.T_varchar || vec.GetType().Oid != types.T_varchar { + canCompile = false + return + } isPK, isSorted := isSortedKey(colDef) if isSorted { fastFilterOp = func(obj *objectio.ObjectStats) (bool, error) { if obj.ZMIsEmpty() { return true, nil } - return obj.SortKeyZoneMap().PrefixIn(vec), nil + return prefixInVector(obj.SortKeyZoneMap(), vec, columnType).mayMatch(), nil } } highSelectivityHint = isPK && vec.Length() <= 10 @@ -845,10 +1197,12 @@ func CompileFilterExpr( return true, nil } dataMeta := meta.MustDataMeta() - return dataMeta.MustGetColumn(uint16(seqNum)).ZoneMap().PrefixIn(vec), nil + return prefixInVector( + dataMeta.MustGetColumn(uint16(seqNum)).ZoneMap(), vec, columnType, + ).mayMatch(), nil } var minPrefix, maxPrefix []byte - if vec.Length() > 0 { + if vec.Length() > 0 && !vec.IsConstNull() && !vec.GetNulls().Any() { col, area := vector.MustVarlenaRawData(vec) minPrefix = col[0].GetByteSlice(area) maxPrefix = col[len(col)-1].GetByteSlice(area) @@ -860,22 +1214,18 @@ func CompileFilterExpr( return false, true, nil } zm := blkMeta.MustGetColumn(uint16(seqNum)).ZoneMap() - if isSorted && maxPrefix != nil && types.PrefixCompare(zm.GetMinBuf(), maxPrefix) > 0 { + if isSorted && maxPrefix != nil && anyPrefixLEByValue(zm, maxPrefix, columnType).excludes() { return true, false, nil } - if !zm.PrefixIn(vec) { + if prefixInVector(zm, vec, columnType).excludes() { return false, false, nil } return false, true, nil } if isSorted && minPrefix != nil { seekOp = func(meta objectio.ObjectDataMeta) int { - blockCnt := int(meta.BlockCount()) - return sort.Search(blockCnt, func(j int) bool { - return types.PrefixCompare( - meta.GetBlockMeta(uint32(j)).MustGetColumn(uint16(seqNum)).ZoneMap().GetMaxBuf(), - minPrefix, - ) >= 0 + return seekFirstBlockByZoneMap(meta, uint16(seqNum), nil, columnType, func(zm objectio.ZoneMap) zoneMapMatch { + return anyPrefixGEByValue(zm, minPrefix, columnType) }) } } @@ -928,15 +1278,19 @@ func CompileFilterExpr( return } vec := vector.NewVec(types.T_any.ToType()) - _ = vec.UnmarshalBinary(val) + if err := vec.UnmarshalBinary(val); err != nil { + canCompile = false + return + } colDef := getColDefByName(expr, colExpr.Col.Name, colExpr.Col.ColPos, tableDef) + columnType := types.T(colDef.Typ.Id) isPK, isSorted := isSortedKey(colDef) if isSorted { fastFilterOp = func(obj *objectio.ObjectStats) (bool, error) { if obj.ZMIsEmpty() { return true, nil } - return obj.SortKeyZoneMap().AnyIn(vec), nil + return anyInVector(obj.SortKeyZoneMap(), vec, columnType).mayMatch(), nil } } if isPK { @@ -953,24 +1307,41 @@ func CompileFilterExpr( return true, nil } dataMeta := meta.MustDataMeta() - return dataMeta.MustGetColumn(uint16(seqNum)).ZoneMap().AnyIn(vec), nil + return anyInVector( + dataMeta.MustGetColumn(uint16(seqNum)).ZoneMap(), vec, columnType, + ).mayMatch(), nil } vecHasNull := vec.IsConstNull() || vec.GetNulls().Any() - var maxVal []byte + var minVal, maxVal []byte if vec.Length() > 0 && !vecHasNull { + minVal = vec.GetRawBytesAt(0) maxVal = vec.GetRawBytesAt(vec.Length() - 1) } + var minBound, maxBound objectio.ZoneMap + if minVal != nil { + minBound, ok = makeVectorValueZoneMapBound(columnType, vec, minVal) + if !ok { + canCompile = false + return + } + maxBound, ok = makeVectorValueZoneMapBound(columnType, vec, maxVal) + if !ok { + canCompile = false + return + } + } blockFilterOp = func( blkIdx int, blkMeta objectio.BlockObject, bf objectio.BloomFilter, ) (bool, bool, error) { zm := blkMeta.MustGetColumn(uint16(seqNum)).ZoneMap() - if isSorted && maxVal != nil && !zm.AnyLEByValue(maxVal) { + if isSorted && maxVal != nil && anyLEByBound(zm, maxVal, maxBound, columnType).excludes() { return true, false, nil } - if !zm.AnyIn(vec) { + membership := anyInVector(zm, vec, columnType) + if membership.excludes() { return false, false, nil } - if isPK { + if isPK && membership.comparable { blkBf := bf.GetBloomFilter(uint32(blkIdx)) blkBfIdx := index.NewEmptyBloomFilter() if err := index.DecodeBloomFilter(blkBfIdx, blkBf); err != nil { @@ -983,29 +1354,32 @@ func CompileFilterExpr( } return false, true, nil } - if isSorted && vec.Length() > 0 && !vecHasNull { - minVal := vec.GetRawBytesAt(0) + if isSorted && minVal != nil { seekOp = func(meta objectio.ObjectDataMeta) int { - blockCnt := int(meta.BlockCount()) - return sort.Search(blockCnt, func(j int) bool { - return meta.GetBlockMeta(uint32(j)).MustGetColumn(uint16(seqNum)).ZoneMap().AnyGEByValue(minVal) + return seekFirstBlockByZoneMap(meta, uint16(seqNum), minBound, columnType, func(zm objectio.ZoneMap) zoneMapMatch { + return anyGEByBound(zm, minVal, minBound, columnType) }) } } case "=": - colExpr, vals, ok := mustColConstValueFromBinaryFuncExpr(exprImpl) + colExpr, vals, valExprs, ok := mustColConstValueWithTypeFromBinaryFuncExpr(exprImpl) if !ok { canCompile = false return } colDef := getColDefByName(expr, colExpr.Col.Name, colExpr.Col.ColPos, tableDef) + bound, ok := makeDecimalZoneMapBound(colDef, vals[0], valExprs[0]) + if !ok { + canCompile = false + return + } isPK, isSorted := isSortedKey(colDef) if isSorted { fastFilterOp = func(obj *objectio.ObjectStats) (bool, error) { if obj.ZMIsEmpty() { return true, nil } - return obj.SortKeyZoneMap().ContainsKey(vals[0]), nil + return intersectsBound(obj.SortKeyZoneMap(), vals[0], bound, types.T(colDef.Typ.Id)).mayMatch(), nil } } if isPK { @@ -1022,7 +1396,9 @@ func CompileFilterExpr( return true, nil } dataMeta := meta.MustDataMeta() - return dataMeta.MustGetColumn(uint16(seqNum)).ZoneMap().ContainsKey(vals[0]), nil + return intersectsBound( + dataMeta.MustGetColumn(uint16(seqNum)).ZoneMap(), vals[0], bound, types.T(colDef.Typ.Id), + ).mayMatch(), nil } blockFilterOp = func( blkIdx int, blkMeta objectio.BlockObject, bf objectio.BloomFilter, @@ -1031,21 +1407,25 @@ func CompileFilterExpr( can, ok bool ) zm := blkMeta.MustGetColumn(uint16(seqNum)).ZoneMap() + intersection := intersectsBound(zm, vals[0], bound, types.T(colDef.Typ.Id)) if isSorted { - can = !zm.AnyLEByValue(vals[0]) + can = anyLEByBound(zm, vals[0], bound, types.T(colDef.Typ.Id)).excludes() if can { ok = false } else { - ok = zm.ContainsKey(vals[0]) + ok = intersection.mayMatch() } } else { can = false - ok = zm.ContainsKey(vals[0]) + ok = intersection.mayMatch() } if !ok { return can, ok, nil } - if isPK { + // Bloom keys are raw encoded values and carry no scale. A decimal + // bound with a different persisted scale cannot be queried safely. + if isPK && intersection.comparable && (bound == nil || + (bound.GetType() == zm.GetType() && bound.GetScale() == zm.GetScale())) { var blkBF index.BloomFilter buf := bf.GetBloomFilter(uint32(blkIdx)) if err := blkBF.Unmarshal(buf); err != nil { @@ -1060,11 +1440,9 @@ func CompileFilterExpr( } if isSorted { seekOp = func(meta objectio.ObjectDataMeta) int { - blockCnt := int(meta.BlockCount()) - blkIdx := sort.Search(blockCnt, func(j int) bool { - return meta.GetBlockMeta(uint32(j)).MustGetColumn(uint16(seqNum)).ZoneMap().AnyGEByValue(vals[0]) + return seekFirstBlockByZoneMap(meta, uint16(seqNum), bound, types.T(colDef.Typ.Id), func(zm objectio.ZoneMap) zoneMapMatch { + return anyGEByBound(zm, vals[0], bound, types.T(colDef.Typ.Id)) }) - return blkIdx } } default: diff --git a/pkg/vm/engine/readutil/expr_filter_decimal_test.go b/pkg/vm/engine/readutil/expr_filter_decimal_test.go new file mode 100644 index 0000000000000..68a4f7134c2a9 --- /dev/null +++ b/pkg/vm/engine/readutil/expr_filter_decimal_test.go @@ -0,0 +1,1279 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package readutil + +import ( + "context" + "strconv" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/objectio" + "github.com/matrixorigin/matrixone/pkg/pb/plan" + "github.com/matrixorigin/matrixone/pkg/sql/colexec" + "github.com/matrixorigin/matrixone/pkg/sql/parsers/dialect/mysql" + plan2 "github.com/matrixorigin/matrixone/pkg/sql/plan" + "github.com/matrixorigin/matrixone/pkg/testutil" + "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/index" + "github.com/stretchr/testify/require" +) + +type decimalZoneMapCompilerContext struct { + *plan2.MockCompilerContext +} + +func (c *decimalZoneMapCompilerContext) Resolve( + dbName string, + tableName string, + _ *plan2.Snapshot, +) (*plan2.ObjectRef, *plan2.TableDef, error) { + typ := plan.Type{Id: int32(types.T_decimal128), Width: 20, Scale: 4} + switch tableName { + case "decimal_scan64": + typ = plan.Type{Id: int32(types.T_decimal64), Width: 12, Scale: 4} + case "decimal_scan128": + default: + return c.MockCompilerContext.Resolve(dbName, tableName, nil) + } + return &plan.ObjectRef{SchemaName: dbName, ObjName: tableName}, &plan.TableDef{ + Name: tableName, + DbName: dbName, + Name2ColIndex: map[string]int32{"amount": 0}, + Cols: []*plan.ColDef{{ + Name: "amount", + ColId: 1, + Seqnum: 0, + Typ: typ, + }}, + }, nil +} + +func TestCompileFilterExprDecimalScaleMatchesPublicPlan(t *testing.T) { + tests := []struct { + name string + table string + literalScale int32 + literalRaw any + blockMin string + blockMax string + selectedExpected bool + }{ + { + name: "decimal64 planner rescales literal", + table: "decimal_scan64", + literalScale: 4, + literalRaw: types.Decimal64(20000000), + blockMin: "1000.0000", + blockMax: "2500.0000", + selectedExpected: true, + }, + { + name: "decimal128 planner preserves low scale literal", + table: "decimal_scan128", + literalScale: 0, + literalRaw: types.Decimal128{B0_63: 2000}, + blockMin: "1000.0000", + blockMax: "2500.0000", + selectedExpected: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ctx := &decimalZoneMapCompilerContext{MockCompilerContext: plan2.NewMockCompilerContext(true)} + ctx.SetContext(context.Background()) + stmt, err := mysql.ParseOne(ctx.GetContext(), "select count(*) from "+test.table+" where amount < 2000", 1) + require.NoError(t, err) + queryPlan, err := plan2.BuildPlan(ctx, stmt, false) + require.NoError(t, err) + + var scan *plan.Node + for _, node := range queryPlan.GetQuery().Nodes { + if node.NodeType == plan.Node_TABLE_SCAN { + scan = node + break + } + } + require.NotNil(t, scan) + require.Len(t, scan.FilterList, 1) + filter := scan.FilterList[0] + require.Equal(t, test.literalScale, filter.GetF().Args[1].Typ.Scale) + require.Equal(t, test.literalRaw, decimalLiteralValue(filter.GetF().Args[1])) + + meta := makeDecimalBlockMeta(t, scan.TableDef.Cols[0].Typ, test.blockMin, test.blockMax) + proc := testutil.NewProcess(t) + need := plan2.AssignAuxIdForExpr(filter, 0) + selectedByGeneralPath := colexec.EvaluateFilterByZoneMap( + proc.Ctx, + proc, + filter, + meta, + map[int]int{0: 0}, + make([]objectio.ZoneMap, need), + make([]*vector.Vector, need), + ) + require.Equal(t, test.selectedExpected, selectedByGeneralPath) + + compiledFilter := plan2.DeepCopyExpr(filter) + var executors []colexec.ExpressionExecutor + _, err = plan2.ReplaceFoldExpr(proc, compiledFilter, &executors) + require.NoError(t, err) + require.NoError(t, plan2.EvalFoldExpr(proc, compiledFilter, &executors)) + for _, executor := range executors { + executor.Free() + } + + _, _, _, blockFilter, _, canCompile, _ := CompileFilterExpr(compiledFilter, scan.TableDef, nil) + require.True(t, canCompile) + require.NotNil(t, blockFilter) + _, selectedByFastPath, err := blockFilter(0, meta, nil) + require.NoError(t, err) + require.Equal(t, selectedByGeneralPath, selectedByFastPath) + }) + } +} + +type decimalBound struct { + text string + scale int32 +} + +type decimalScalePruningCase struct { + name string + colType plan.Type + min string + max string + op string + bounds []decimalBound + primary bool + uninitialized bool + want bool +} + +func TestCompileFilterExprDecimalScaleMatrix(t *testing.T) { + decimalTypes := []struct { + name string + typ plan.Type + }{ + {name: "decimal64", typ: plan.Type{Id: int32(types.T_decimal64), Width: 12, Scale: 4}}, + {name: "decimal128", typ: plan.Type{Id: int32(types.T_decimal128), Width: 20, Scale: 4}}, + } + + tests := make([]decimalScalePruningCase, 0, len(decimalTypes)*25+6) + for _, decimalType := range decimalTypes { + add := func(name, min, max, op string, want bool, bounds ...decimalBound) { + tests = append(tests, decimalScalePruningCase{ + name: decimalType.name + "/" + name, colType: decimalType.typ, + min: min, max: max, op: op, bounds: bounds, want: want, + }) + } + add("lt_scale0_keeps", "1000.0000", "2500.0000", "<", true, decimalBound{"2000", 0}) + add("lt_scale0_prunes", "2000.0000", "2500.0000", "<", false, decimalBound{"2000", 0}) + add("lt_scale1_keeps", "1000.0000", "2500.0000", "<", true, decimalBound{"2000.0", 1}) + add("lt_scale4_prunes", "2000.0000", "2500.0000", "<", false, decimalBound{"2000.0000", 4}) + add("le_scale0_boundary", "2000.0000", "2500.0000", "<=", true, decimalBound{"2000", 0}) + add("gt_scale0_keeps", "1000.0000", "2500.0000", ">", true, decimalBound{"2000", 0}) + add("gt_scale0_prunes", "1000.0000", "2000.0000", ">", false, decimalBound{"2000", 0}) + add("ge_scale0_boundary", "1000.0000", "2000.0000", ">=", true, decimalBound{"2000", 0}) + add("eq_scale0_keeps", "1999.9000", "2000.1000", "=", true, decimalBound{"2000", 0}) + add("eq_scale0_prunes", "2000.0001", "2500.0000", "=", false, decimalBound{"2000", 0}) + add("between_scale0_keeps", "500.0000", "1500.0000", "between", true, + decimalBound{"1000", 0}, decimalBound{"2000", 0}) + add("between_scale0_prunes_upper", "2000.0001", "2500.0000", "between", false, + decimalBound{"1000", 0}, decimalBound{"2000", 0}) + add("between_scale0_prunes_lower", "500.0000", "999.9999", "between", false, + decimalBound{"1000", 0}, decimalBound{"2000", 0}) + add("between_mixed_scale_keeps", "500.0000", "1500.0000", "between", true, + decimalBound{"1000.0", 1}, decimalBound{"2000.0000", 4}) + add("negative_lt_keeps", "-2500.0000", "-1000.0000", "<", true, decimalBound{"-2000", 0}) + add("negative_lt_prunes", "-1999.9999", "-1000.0000", "<", false, decimalBound{"-2000", 0}) + add("zero_eq_keeps", "-0.0001", "0.0001", "=", true, decimalBound{"0", 0}) + } + + for _, oid := range []types.T{types.T_decimal64, types.T_decimal128} { + width := int32(12) + name := "decimal64" + if oid == types.T_decimal128 { + width = 20 + name = "decimal128" + } + colType := plan.Type{Id: int32(oid), Width: width, Scale: 2} + tests = append(tests, + decimalScalePruningCase{ + name: name + "/lossy_upper_lt_keeps", colType: colType, + min: "2.00", max: "2.00", op: "<", bounds: []decimalBound{{"2.0010", 4}}, want: true, + }, + decimalScalePruningCase{ + name: name + "/lossy_lower_gt_keeps", colType: colType, + min: "2.00", max: "2.00", op: ">", bounds: []decimalBound{{"1.9990", 4}}, want: true, + }, + decimalScalePruningCase{ + name: name + "/lossy_between_lower_keeps", colType: colType, + min: "1.01", max: "1.01", op: "between", + bounds: []decimalBound{{"1.0050", 4}, {"2", 0}}, want: true, + }, + decimalScalePruningCase{ + name: name + "/lossy_between_lower_prunes", colType: colType, + min: "1.00", max: "1.00", op: "between", + bounds: []decimalBound{{"1.0050", 4}, {"2", 0}}, want: false, + }, + decimalScalePruningCase{ + name: name + "/lossy_between_upper_keeps", colType: colType, + min: "2.00", max: "2.00", op: "between", + bounds: []decimalBound{{"1", 0}, {"2.0050", 4}}, want: true, + }, + decimalScalePruningCase{ + name: name + "/lossy_between_upper_prunes", colType: colType, + min: "2.01", max: "2.01", op: "between", + bounds: []decimalBound{{"1", 0}, {"2.0050", 4}}, want: false, + }, + decimalScalePruningCase{ + name: name + "/unknown_zm_fails_open", colType: colType, op: "<", + bounds: []decimalBound{{"2", 0}}, uninitialized: true, want: true, + }, + decimalScalePruningCase{ + name: name + "/primary_eq_mismatched_scale_skips_bloom", colType: colType, + min: "1.00", max: "3.00", op: "=", bounds: []decimalBound{{"2.0000", 4}}, primary: true, want: true, + }, + ) + } + + tests = append(tests, + decimalScalePruningCase{ + name: "decimal64/positive_extreme_scale", colType: plan.Type{Id: int32(types.T_decimal64), Width: 18, Scale: 17}, + min: "9.99999999999999998", max: "9.99999999999999999", op: "<", + bounds: []decimalBound{{"10", 0}}, want: true, + }, + decimalScalePruningCase{ + name: "decimal64/negative_extreme_scale", colType: plan.Type{Id: int32(types.T_decimal64), Width: 18, Scale: 17}, + min: "-9.99999999999999999", max: "-9.99999999999999998", op: ">", + bounds: []decimalBound{{"-10", 0}}, want: true, + }, + decimalScalePruningCase{ + name: "decimal64/positive_scale_overflow", colType: plan.Type{Id: int32(types.T_decimal64), Width: 18, Scale: 17}, + min: "9.99999999999999998", max: "9.99999999999999999", op: "<", + bounds: []decimalBound{{"100", 0}}, want: true, + }, + decimalScalePruningCase{ + name: "decimal64/negative_scale_overflow", colType: plan.Type{Id: int32(types.T_decimal64), Width: 18, Scale: 17}, + min: "-9.99999999999999999", max: "-9.99999999999999998", op: ">", + bounds: []decimalBound{{"-100", 0}}, want: true, + }, + decimalScalePruningCase{ + name: "decimal128/positive_scale_overflow", colType: plan.Type{Id: int32(types.T_decimal128), Width: 38, Scale: 37}, + min: "9.9999999999999999999999999999999999998", max: "9.9999999999999999999999999999999999999", op: "<", + bounds: []decimalBound{{"100", 0}}, want: true, + }, + decimalScalePruningCase{ + name: "decimal128/negative_scale_overflow", colType: plan.Type{Id: int32(types.T_decimal128), Width: 38, Scale: 37}, + min: "-9.9999999999999999999999999999999999999", max: "-9.9999999999999999999999999999999999998", op: ">", + bounds: []decimalBound{{"-100", 0}}, want: true, + }, + ) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + tableDef := decimalTableDef(test.colType, test.primary) + expr := decimalFoldedFilter(t, test.colType, test.op, test.bounds...) + _, _, _, blockFilter, _, canCompile, _ := CompileFilterExpr(expr, tableDef, nil) + require.True(t, canCompile) + require.NotNil(t, blockFilter) + + var meta objectio.BlockObject + if test.uninitialized { + meta = makeDecimalBlockMeta(t, test.colType) + } else { + meta = makeDecimalBlockMeta(t, test.colType, test.min, test.max) + } + _, selected, err := blockFilter(0, meta, nil) + require.NoError(t, err) + require.Equal(t, test.want, selected) + }) + } +} + +func TestDecimalZoneMapScaleAwareComparisons(t *testing.T) { + decimalTypes := []struct { + name string + typ plan.Type + }{ + {name: "decimal64", typ: plan.Type{Id: int32(types.T_decimal64), Width: 12, Scale: 4}}, + {name: "decimal128", typ: plan.Type{Id: int32(types.T_decimal128), Width: 20, Scale: 4}}, + } + for _, decimalType := range decimalTypes { + t.Run(decimalType.name, func(t *testing.T) { + zm := decimalZoneMap(t, decimalType.typ, "-2.0000", "2.0000") + oneValue, one := decimalBoundZoneMap(t, decimalType.typ, decimalBound{"1", 0}) + minusTwoValue, minusTwo := decimalBoundZoneMap(t, decimalType.typ, decimalBound{"-2", 0}) + twoValue, two := decimalBoundZoneMap(t, decimalType.typ, decimalBound{"2", 0}) + threeValue, three := decimalBoundZoneMap(t, decimalType.typ, decimalBound{"3", 0}) + minusOneValue, minusOne := decimalBoundZoneMap(t, decimalType.typ, decimalBound{"-1", 0}) + oneScale1Value, oneScale1 := decimalBoundZoneMap(t, decimalType.typ, decimalBound{"1.0", 1}) + fourValue, four := decimalBoundZoneMap(t, decimalType.typ, decimalBound{"4.00", 2}) + + columnType := types.T(decimalType.typ.Id) + require.True(t, anyLTByBound(zm, oneValue, one, columnType).mayMatch()) + require.True(t, anyLTByBound(zm, minusTwoValue, minusTwo, columnType).excludes()) + require.True(t, anyLEByBound(zm, minusTwoValue, minusTwo, columnType).mayMatch()) + require.True(t, anyGTByBound(zm, oneValue, one, columnType).mayMatch()) + require.True(t, anyGTByBound(zm, twoValue, two, columnType).excludes()) + require.True(t, anyGEByBound(zm, twoValue, two, columnType).mayMatch()) + require.True(t, intersectsBound(zm, oneValue, one, columnType).mayMatch()) + require.True(t, intersectsBound(zm, threeValue, three, columnType).excludes()) + require.True(t, anyBetweenBounds( + zm, minusOneValue, oneScale1Value, minusOne, oneScale1, columnType, + ).mayMatch()) + require.True(t, anyBetweenBounds(zm, threeValue, fourValue, three, four, columnType).excludes()) + + mismatchType := plan.Type{Id: int32(types.T_decimal128), Width: 20, Scale: 0} + if types.T(decimalType.typ.Id) == types.T_decimal128 { + mismatchType = plan.Type{Id: int32(types.T_decimal64), Width: 12, Scale: 0} + } + mismatch := decimalZoneMap(t, mismatchType, "1") + // A persisted type mismatch makes index.ZM comparison return ok=false. + // Every decimal pruning helper must fail open in that case. + require.True(t, anyLTByBound(zm, nil, mismatch, columnType).mayMatch()) + require.True(t, anyLEByBound(zm, nil, mismatch, columnType).mayMatch()) + require.True(t, anyGTByBound(zm, nil, mismatch, columnType).mayMatch()) + require.True(t, anyGEByBound(zm, nil, mismatch, columnType).mayMatch()) + require.True(t, intersectsBound(zm, nil, mismatch, columnType).mayMatch()) + require.True(t, anyBetweenBounds(zm, nil, nil, mismatch, mismatch, columnType).mayMatch()) + for hint := uint8(0); hint < 4; hint++ { + require.True(t, inRangeBounds(zm, nil, nil, mismatch, mismatch, hint, columnType).mayMatch()) + } + }) + } +} + +var zoneMapMatchSink zoneMapMatch +var zoneMapSeekSink int + +func TestZoneMapMatchHelpersDoNotAllocate(t *testing.T) { + intType := plan.Type{Id: int32(types.T_int64)} + intZM := sortedUnknownZoneMap(t, intType, "10", "20") + intBound := encodeSortedUnknownValue(t, intType, "15") + unknownIntZM := index.NewZM(types.T_int64, 0) + + decimalType := plan.Type{Id: int32(types.T_decimal128), Width: 20, Scale: 4} + decimalZM := decimalZoneMap(t, decimalType, "10.0000", "20.0000") + decimalValue, decimalBoundZM := decimalBoundZoneMap(t, decimalType, decimalBound{"15", 0}) + unknownDecimalZM := index.NewZM(types.T_decimal128, decimalType.Scale) + intInExpr := sortedUnknownVectorFilter(t, intType, []string{"15", "20"}, false) + intInVec := vector.NewVec(types.T_any.ToType()) + require.NoError(t, intInVec.UnmarshalBinary(intInExpr.GetF().Args[1].GetVec().Data)) + prefixType := plan.Type{Id: int32(types.T_varchar), Width: 8} + prefixZM := sortedUnknownZoneMap(t, prefixType, "10", "20") + unknownPrefixZM := index.NewZM(types.T_varchar, 0) + prefixInExpr := sortedUnknownVectorFilter(t, prefixType, []string{"15", "20"}, true) + prefixInVec := vector.NewVec(types.T_any.ToType()) + require.NoError(t, prefixInVec.UnmarshalBinary(prefixInExpr.GetF().Args[1].GetVec().Data)) + prefixValue := encodeSortedUnknownValue(t, prefixType, "15") + + tests := []struct { + name string + fn func() zoneMapMatch + }{ + { + name: "raw initialized", + fn: func() zoneMapMatch { + return anyLTByBound(intZM, intBound, nil, types.T_int64) + }, + }, + { + name: "raw unknown", + fn: func() zoneMapMatch { + return anyLTByBound(unknownIntZM, intBound, nil, types.T_int64) + }, + }, + { + name: "decimal initialized", + fn: func() zoneMapMatch { + return anyLTByBound(decimalZM, decimalValue, decimalBoundZM, types.T_decimal128) + }, + }, + { + name: "decimal unknown", + fn: func() zoneMapMatch { + return anyLTByBound(unknownDecimalZM, decimalValue, decimalBoundZM, types.T_decimal128) + }, + }, + { + name: "in initialized", + fn: func() zoneMapMatch { + return anyInVector(intZM, intInVec, types.T_int64) + }, + }, + { + name: "in unknown", + fn: func() zoneMapMatch { + return anyInVector(unknownIntZM, intInVec, types.T_int64) + }, + }, + { + name: "prefix initialized", + fn: func() zoneMapMatch { + return prefixEqByValue(prefixZM, prefixValue, types.T_varchar) + }, + }, + { + name: "prefix unknown", + fn: func() zoneMapMatch { + return prefixInVector(unknownPrefixZM, prefixInVec, types.T_varchar) + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + allocations := testing.AllocsPerRun(1000, func() { + zoneMapMatchSink = test.fn() + }) + require.Zero(t, allocations) + }) + } + + tableDef := decimalTableDef(decimalType, false) + tableDef.Cols[0].ClusterBy = true + expr := decimalFoldedFilter(t, decimalType, ">=", decimalBound{"15", 0}) + _, _, _, _, seek, canCompile, _ := CompileFilterExpr(expr, tableDef, nil) + require.True(t, canCompile) + require.NotNil(t, seek) + dataMeta := decimalObjectDataMeta(t, decimalType, "10.0000", "20.0000", "30.0000") + allocations := testing.AllocsPerRun(1000, func() { + zoneMapSeekSink = seek(dataMeta) + }) + require.Zero(t, allocations) +} + +func BenchmarkSeekFirstBlockByZoneMap(b *testing.B) { + for _, blockCount := range []int{1, 16, 256, 4096} { + b.Run(strconv.Itoa(blockCount)+"_blocks", func(b *testing.B) { + dataMeta := objectio.BuildMetaData(uint16(blockCount), 1) + objectZM := index.NewZM(types.T_int64, 0) + for i := range blockCount { + value := int64(i) + valueBytes := types.EncodeInt64(&value) + index.UpdateZM(objectZM, valueBytes) + blockZM := index.NewZM(types.T_int64, 0) + index.UpdateZM(blockZM, valueBytes) + dataMeta.GetBlockMeta(uint32(i)).MustGetColumn(0).SetZoneMap(blockZM) + } + dataMeta.MustGetColumn(0).SetZoneMap(objectZM) + boundValue := int64(blockCount / 2) + boundBytes := types.EncodeInt64(&boundValue) + + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + zoneMapSeekSink = seekFirstBlockByZoneMap( + dataMeta, 0, nil, types.T_int64, + func(zm objectio.ZoneMap) zoneMapMatch { + return anyGEByBound(zm, boundBytes, nil, types.T_int64) + }, + ) + } + }) + } +} + +func TestCompileFilterExprDecimalComparisonFailureSkipsBloom(t *testing.T) { + decimalTypes := []struct { + name string + colType plan.Type + metadataType plan.Type + }{ + { + name: "decimal64_column_decimal128_metadata", + colType: plan.Type{Id: int32(types.T_decimal64), Width: 12, Scale: 0}, + metadataType: plan.Type{Id: int32(types.T_decimal128), Width: 20, Scale: 0}, + }, + { + name: "decimal128_column_decimal64_metadata", + colType: plan.Type{Id: int32(types.T_decimal128), Width: 20, Scale: 0}, + metadataType: plan.Type{Id: int32(types.T_decimal64), Width: 12, Scale: 0}, + }, + } + for _, decimalType := range decimalTypes { + t.Run(decimalType.name, func(t *testing.T) { + tableDef := decimalTableDef(decimalType.colType, true) + expr := decimalFoldedFilter(t, decimalType.colType, "=", decimalBound{"1", 0}) + _, _, _, blockFilter, _, canCompile, _ := CompileFilterExpr(expr, tableDef, nil) + require.True(t, canCompile) + + dataMeta := objectio.BuildMetaData(1, 1) + meta := dataMeta.GetBlockMeta(0) + meta.MustGetColumn(0).SetZoneMap(decimalZoneMap(t, decimalType.metadataType, "1")) + quickBreak, selected, err := blockFilter(0, meta, nil) + require.NoError(t, err) + require.False(t, quickBreak) + require.True(t, selected) + }) + } +} + +func TestCompileFilterExprDecimalSortedBetweenPaths(t *testing.T) { + decimalTypes := []struct { + name string + typ plan.Type + }{ + {name: "decimal64", typ: plan.Type{Id: int32(types.T_decimal64), Width: 12, Scale: 4}}, + {name: "decimal128", typ: plan.Type{Id: int32(types.T_decimal128), Width: 20, Scale: 4}}, + } + for _, decimalType := range decimalTypes { + t.Run(decimalType.name, func(t *testing.T) { + tableDef := decimalTableDef(decimalType.typ, false) + tableDef.Cols[0].ClusterBy = true + expr := decimalFoldedFilter(t, decimalType.typ, "between", + decimalBound{"1", 0}, decimalBound{"2.0", 1}) + fastFilter, _, objectFilter, blockFilter, seek, canCompile, _ := CompileFilterExpr(expr, tableDef, nil) + require.True(t, canCompile) + require.NotNil(t, fastFilter) + require.NotNil(t, objectFilter) + require.NotNil(t, blockFilter) + require.NotNil(t, seek) + + dataMeta := decimalObjectDataMeta(t, decimalType.typ, "0.0000", "1.0000", "2.0000", "3.0000") + stats := decimalObjectStats(t, decimalType.typ, "0.0000", "3.0000") + selected, err := fastFilter(stats) + require.NoError(t, err) + require.True(t, selected) + selected, err = objectFilter(nil, nil) + require.NoError(t, err) + require.True(t, selected) + require.Equal(t, 1, seek(dataMeta)) + + quickBreak, selected, err := blockFilter(1, dataMeta.GetBlockMeta(1), nil) + require.NoError(t, err) + require.False(t, quickBreak) + require.True(t, selected) + quickBreak, selected, err = blockFilter(3, dataMeta.GetBlockMeta(3), nil) + require.NoError(t, err) + require.True(t, quickBreak) + require.False(t, selected) + }) + } +} + +func TestCompileFilterExprDecimalSortedInRangeHints(t *testing.T) { + decimalTypes := []struct { + name string + typ plan.Type + }{ + {name: "decimal64", typ: plan.Type{Id: int32(types.T_decimal64), Width: 12, Scale: 2}}, + {name: "decimal128", typ: plan.Type{Id: int32(types.T_decimal128), Width: 20, Scale: 2}}, + } + for _, decimalType := range decimalTypes { + for hint := uint8(0); hint < 4; hint++ { + t.Run(decimalType.name+"/hint_"+string(rune('0'+hint)), func(t *testing.T) { + tableDef := decimalTableDef(decimalType.typ, false) + tableDef.Cols[0].ClusterBy = true + expr := decimalInRangeFoldedFilter(t, decimalType.typ, + decimalBound{"1", 0}, decimalBound{"3.0", 1}, hint) + fastFilter, _, objectFilter, blockFilter, seek, canCompile, _ := CompileFilterExpr(expr, tableDef, nil) + require.True(t, canCompile) + require.NotNil(t, fastFilter) + require.NotNil(t, objectFilter) + require.NotNil(t, blockFilter) + require.NotNil(t, seek) + + dataMeta := decimalObjectDataMeta(t, decimalType.typ, "1.00", "2.00", "3.00", "4.00") + stats := decimalObjectStats(t, decimalType.typ, "1.00", "4.00") + selected, err := fastFilter(stats) + require.NoError(t, err) + require.True(t, selected) + selected, err = objectFilter(nil, nil) + require.NoError(t, err) + require.True(t, selected) + + wantSeek := 0 + if hint == 1 || hint == 3 { + wantSeek = 1 + } + require.Equal(t, wantSeek, seek(dataMeta)) + + quickBreak, selected, err := blockFilter(2, dataMeta.GetBlockMeta(2), nil) + require.NoError(t, err) + if hint == 2 || hint == 3 { + require.True(t, quickBreak) + require.False(t, selected) + } else { + require.False(t, quickBreak) + require.True(t, selected) + } + quickBreak, selected, err = blockFilter(3, dataMeta.GetBlockMeta(3), nil) + require.NoError(t, err) + require.True(t, quickBreak) + require.False(t, selected) + }) + } + } +} + +func TestCompileFilterExprSortedUnknownZoneMapDoesNotExcludeLaterBlocks(t *testing.T) { + typesUnderTest := []struct { + name string + typ plan.Type + }{ + {name: "decimal64", typ: plan.Type{Id: int32(types.T_decimal64), Width: 12, Scale: 2}}, + {name: "decimal128", typ: plan.Type{Id: int32(types.T_decimal128), Width: 20, Scale: 2}}, + {name: "int64", typ: plan.Type{Id: int32(types.T_int64)}}, + {name: "varchar", typ: plan.Type{Id: int32(types.T_varchar), Width: 8}}, + } + operations := []struct { + name string + op string + bounds []string + quickBreak bool + hints []uint8 + prefixOnly bool + }{ + {name: "lt", op: "<", bounds: []string{"25"}, quickBreak: true}, + {name: "le", op: "<=", bounds: []string{"20"}, quickBreak: true}, + {name: "gt", op: ">", bounds: []string{"15"}}, + {name: "ge", op: ">=", bounds: []string{"20"}}, + {name: "eq", op: "=", bounds: []string{"20"}, quickBreak: true}, + {name: "between", op: "between", bounds: []string{"15", "25"}, quickBreak: true}, + {name: "in_range", op: "in_range", bounds: []string{"15", "25"}, quickBreak: true, + hints: []uint8{0, 1, 2, 3}}, + {name: "in", op: "in", bounds: []string{"20", "25"}, quickBreak: true}, + {name: "prefix_eq", op: "prefix_eq", bounds: []string{"20"}, quickBreak: true, prefixOnly: true}, + {name: "prefix_between", op: "prefix_between", bounds: []string{"15", "25"}, quickBreak: true, prefixOnly: true}, + {name: "prefix_in_range", op: "prefix_in_range", bounds: []string{"15", "25"}, quickBreak: true, + hints: []uint8{0, 1, 2, 3}, prefixOnly: true}, + {name: "prefix_in", op: "prefix_in", bounds: []string{"20", "25"}, quickBreak: true, prefixOnly: true}, + } + + for _, typ := range typesUnderTest { + for _, operation := range operations { + if operation.prefixOnly && types.T(typ.typ.Id) != types.T_varchar { + continue + } + hints := operation.hints + if len(hints) == 0 { + hints = []uint8{0} + } + for _, hint := range hints { + name := typ.name + "/" + operation.name + if operation.op == "in_range" || operation.op == "prefix_in_range" { + name += "/hint_" + strconv.Itoa(int(hint)) + } + t.Run(name, func(t *testing.T) { + tableDef := decimalTableDef(typ.typ, false) + tableDef.Cols[0].ClusterBy = true + expr := sortedUnknownFilter(t, typ.typ, operation.op, operation.bounds, hint) + fastFilter, _, _, blockFilter, seek, canCompile, _ := CompileFilterExpr(expr, tableDef, nil) + require.True(t, canCompile) + require.NotNil(t, fastFilter) + require.NotNil(t, blockFilter) + + dataMeta := sortedUnknownDataMeta(t, typ.typ) + stats := objectio.NewObjectStats() + require.NoError(t, objectio.SetObjectStatsSortKeyZoneMap( + stats, index.NewZM(types.T(typ.typ.Id), typ.typ.Scale))) + selected, err := fastFilter(stats) + require.NoError(t, err) + require.True(t, selected, "unknown object ZM must fail open") + + quickBreak, selected, err := blockFilter(0, dataMeta.GetBlockMeta(0), nil) + require.NoError(t, err) + require.False(t, quickBreak, "unknown first block must not stop the scan") + require.True(t, selected, "unknown first block must fail open") + + quickBreak, selected, err = blockFilter(1, dataMeta.GetBlockMeta(1), nil) + require.NoError(t, err) + require.False(t, quickBreak) + require.True(t, selected, "later matching block must remain reachable") + + quickBreak, _, err = blockFilter(2, dataMeta.GetBlockMeta(2), nil) + require.NoError(t, err) + require.Equal(t, operation.quickBreak, quickBreak) + + if seek != nil { + require.Equal(t, 0, seek(dataMeta), "unknown leading block must make seek fail open") + } + }) + } + } + } +} + +func TestCompileFilterExprSortedIncompatibleZoneMapFailsOpen(t *testing.T) { + tests := []struct { + name string + columnType plan.Type + metadataType plan.Type + }{ + { + name: "int64 column with varchar metadata", + columnType: plan.Type{Id: int32(types.T_int64)}, + metadataType: plan.Type{Id: int32(types.T_varchar), Width: 8}, + }, + { + name: "varchar column with int64 metadata", + columnType: plan.Type{Id: int32(types.T_varchar), Width: 8}, + metadataType: plan.Type{Id: int32(types.T_int64)}, + }, + { + name: "varchar column with text metadata", + columnType: plan.Type{Id: int32(types.T_varchar), Width: 8}, + metadataType: plan.Type{Id: int32(types.T_text)}, + }, + { + name: "varchar column with blob metadata", + columnType: plan.Type{Id: int32(types.T_varchar), Width: 8}, + metadataType: plan.Type{Id: int32(types.T_blob)}, + }, + { + name: "varchar column with json metadata", + columnType: plan.Type{Id: int32(types.T_varchar), Width: 8}, + metadataType: plan.Type{Id: int32(types.T_json)}, + }, + } + operations := []struct { + op string + bounds []string + hint uint8 + prefixOnly bool + }{ + {op: "=", bounds: []string{"20"}}, + {op: "in", bounds: []string{"20", "25"}}, + {op: "prefix_eq", bounds: []string{"20"}, prefixOnly: true}, + {op: "prefix_between", bounds: []string{"15", "25"}, prefixOnly: true}, + {op: "prefix_in_range", bounds: []string{"15", "25"}, hint: 3, prefixOnly: true}, + {op: "prefix_in", bounds: []string{"20", "25"}, prefixOnly: true}, + } + for _, test := range tests { + for _, operation := range operations { + if operation.prefixOnly && types.T(test.columnType.Id) != types.T_varchar { + continue + } + t.Run(test.name+"/"+operation.op, func(t *testing.T) { + tableDef := decimalTableDef(test.columnType, false) + tableDef.Cols[0].ClusterBy = true + expr := sortedUnknownFilter(t, test.columnType, operation.op, operation.bounds, operation.hint) + fastFilter, _, _, blockFilter, seek, canCompile, _ := CompileFilterExpr(expr, tableDef, nil) + require.True(t, canCompile) + + mismatch := sortedUnknownZoneMap(t, test.metadataType, "20") + stats := objectio.NewObjectStats() + require.NoError(t, objectio.SetObjectStatsSortKeyZoneMap(stats, mismatch)) + selected, err := fastFilter(stats) + require.NoError(t, err) + require.True(t, selected) + + dataMeta := objectio.BuildMetaData(2, 1) + dataMeta.MustGetColumn(0).SetZoneMap(mismatch) + dataMeta.GetBlockMeta(0).MustGetColumn(0).SetZoneMap(mismatch.Clone()) + dataMeta.GetBlockMeta(1).MustGetColumn(0).SetZoneMap(mismatch.Clone()) + quickBreak, selected, err := blockFilter(0, dataMeta.GetBlockMeta(0), nil) + require.NoError(t, err) + require.False(t, quickBreak) + require.True(t, selected) + require.Equal(t, 0, seek(dataMeta)) + }) + } + } +} + +func TestCompileFilterExprInVectorMetadataMismatchFailsOpen(t *testing.T) { + columnType := plan.Type{Id: int32(types.T_decimal128), Width: 20, Scale: 2} + vectorType := plan.Type{Id: int32(types.T_decimal128), Width: 20, Scale: 4} + tableDef := decimalTableDef(columnType, true) + expr := sortedUnknownVectorFilterWithType( + t, columnType, vectorType, []string{"20.0000", "25.0000"}, false, + ) + fastFilter, _, _, blockFilter, _, canCompile, _ := CompileFilterExpr(expr, tableDef, nil) + require.True(t, canCompile) + + stats := decimalObjectStats(t, columnType, "20.00", "25.00") + selected, err := fastFilter(stats) + require.NoError(t, err) + require.True(t, selected, "scale-mismatched IN vector must fail open") + + dataMeta := decimalObjectDataMeta(t, columnType, "20.00", "25.00") + quickBreak, selected, err := blockFilter(0, dataMeta.GetBlockMeta(0), nil) + require.NoError(t, err) + require.False(t, quickBreak) + require.True(t, selected, "unsafe raw-byte Bloom lookup must be skipped") +} + +func TestCompileFilterExprMalformedVectorDoesNotCompile(t *testing.T) { + typ := plan.Type{Id: int32(types.T_varchar), Width: 8} + tableDef := decimalTableDef(typ, false) + for _, prefix := range []bool{false, true} { + name := "in" + if prefix { + name = "prefix_in" + } + t.Run(name, func(t *testing.T) { + expr := sortedUnknownVectorFilter(t, typ, []string{"20"}, prefix) + expr.GetF().Args[1].GetVec().Data = []byte{1, 2, 3} + _, _, _, _, _, canCompile, _ := CompileFilterExpr(expr, tableDef, nil) + require.False(t, canCompile) + }) + } + + t.Run("prefix_in physical type mismatch", func(t *testing.T) { + expr := sortedUnknownVectorFilterWithType( + t, typ, plan.Type{Id: int32(types.T_int64)}, []string{"20"}, true, + ) + _, _, _, _, _, canCompile, _ := CompileFilterExpr(expr, tableDef, nil) + require.False(t, canCompile) + }) +} + +func TestCompileFilterExprPrefixInNullableVectorFailsOpen(t *testing.T) { + typ := plan.Type{Id: int32(types.T_varchar), Width: 8} + proc := testutil.NewProcess(t) + vec := vector.NewVec(plan2.MakeTypeByPlan2Type(typ)) + defer vec.Free(proc.Mp()) + require.NoError(t, vector.AppendBytes(vec, nil, true, proc.Mp())) + require.NoError(t, vector.AppendBytes(vec, []byte("00000020"), false, proc.Mp())) + data, err := vec.MarshalBinary() + require.NoError(t, err) + expr := plan2.MakeInExpr( + context.Background(), + &plan.Expr{ + Typ: typ, + Expr: &plan.Expr_Col{Col: &plan.ColRef{ + RelPos: 0, ColPos: 0, Name: "amount", + }}, + }, + int32(vec.Length()), + data, + true, + ) + tableDef := decimalTableDef(typ, false) + tableDef.Cols[0].ClusterBy = true + fastFilter, _, _, blockFilter, seek, canCompile, _ := CompileFilterExpr(expr, tableDef, nil) + require.True(t, canCompile) + require.Nil(t, seek) + + stats := objectio.NewObjectStats() + require.NoError(t, objectio.SetObjectStatsSortKeyZoneMap( + stats, sortedUnknownZoneMap(t, typ, "40"), + )) + selected, err := fastFilter(stats) + require.NoError(t, err) + require.True(t, selected) + + dataMeta := objectio.BuildMetaData(1, 1) + dataMeta.GetBlockMeta(0).MustGetColumn(0).SetZoneMap(sortedUnknownZoneMap(t, typ, "40")) + quickBreak, selected, err := blockFilter(0, dataMeta.GetBlockMeta(0), nil) + require.NoError(t, err) + require.False(t, quickBreak) + require.True(t, selected) +} + +func TestSeekFirstBlockFailsOpenForUnsampledUnknownZoneMap(t *testing.T) { + typesUnderTest := []struct { + name string + columnType plan.Type + mismatchType plan.Type + }{ + { + name: "decimal64", + columnType: plan.Type{Id: int32(types.T_decimal64), Width: 12, Scale: 2}, + mismatchType: plan.Type{Id: int32(types.T_decimal128), Width: 20, Scale: 2}, + }, + { + name: "decimal128", + columnType: plan.Type{Id: int32(types.T_decimal128), Width: 20, Scale: 2}, + mismatchType: plan.Type{Id: int32(types.T_decimal64), Width: 12, Scale: 2}, + }, + { + name: "int64", + columnType: plan.Type{Id: int32(types.T_int64)}, + mismatchType: plan.Type{Id: int32(types.T_varchar), Width: 8}, + }, + { + name: "varchar", + columnType: plan.Type{Id: int32(types.T_varchar), Width: 8}, + mismatchType: plan.Type{Id: int32(types.T_int64)}, + }, + } + for _, typ := range typesUnderTest { + for _, state := range []string{"uninitialized", "incompatible"} { + t.Run(typ.name+"/"+state, func(t *testing.T) { + tableDef := decimalTableDef(typ.columnType, false) + tableDef.Cols[0].ClusterBy = true + expr := sortedUnknownFilter(t, typ.columnType, ">=", []string{"35"}, 0) + _, _, _, _, seek, canCompile, _ := CompileFilterExpr(expr, tableDef, nil) + require.True(t, canCompile) + require.NotNil(t, seek) + + dataMeta := objectio.BuildMetaData(5, 1) + dataMeta.MustGetColumn(0).SetZoneMap(sortedUnknownZoneMap(t, typ.columnType, "10", "50")) + dataMeta.GetBlockMeta(0).MustGetColumn(0).SetZoneMap(sortedUnknownZoneMap(t, typ.columnType, "10")) + if state == "uninitialized" { + dataMeta.GetBlockMeta(1).MustGetColumn(0).SetZoneMap( + index.NewZM(types.T(typ.columnType.Id), typ.columnType.Scale)) + } else { + dataMeta.GetBlockMeta(1).MustGetColumn(0).SetZoneMap( + sortedUnknownZoneMap(t, typ.mismatchType, "15")) + } + dataMeta.GetBlockMeta(2).MustGetColumn(0).SetZoneMap(sortedUnknownZoneMap(t, typ.columnType, "20")) + dataMeta.GetBlockMeta(3).MustGetColumn(0).SetZoneMap(sortedUnknownZoneMap(t, typ.columnType, "40")) + dataMeta.GetBlockMeta(4).MustGetColumn(0).SetZoneMap(sortedUnknownZoneMap(t, typ.columnType, "50")) + + // sort.Search for the ordinary boundary probes blocks 2, 4 and 3; + // block 1 is deliberately unsampled and must still force seek=0. + require.Equal(t, 0, seek(dataMeta)) + }) + } + } +} + +func TestCompileFilterExprDecimalScaleNonDecimalControl(t *testing.T) { + tableDef := &plan.TableDef{ + Name: "int_scan", + Name2ColIndex: map[string]int32{"amount": 0}, + Cols: []*plan.ColDef{{ + Name: "amount", ColId: 1, Seqnum: 0, + Typ: plan.Type{Id: int32(types.T_int64)}, + }}, + } + dataMeta := objectio.BuildMetaData(1, 1) + meta := dataMeta.GetBlockMeta(0) + zm := index.NewZM(types.T_int64, 0) + minValue, maxValue := int64(10), int64(20) + index.UpdateZM(zm, types.EncodeInt64(&minValue)) + index.UpdateZM(zm, types.EncodeInt64(&maxValue)) + meta.MustGetColumn(0).SetZoneMap(zm) + + tests := []struct { + name string + op string + bounds []int64 + want bool + }{ + {name: "lt prunes", op: "<", bounds: []int64{5}, want: false}, + {name: "le keeps", op: "<=", bounds: []int64{10}, want: true}, + {name: "gt prunes", op: ">", bounds: []int64{20}, want: false}, + {name: "ge keeps", op: ">=", bounds: []int64{20}, want: true}, + {name: "eq keeps", op: "=", bounds: []int64{15}, want: true}, + {name: "between prunes", op: "between", bounds: []int64{21, 30}, want: false}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + expr := int64FoldedFilter(test.op, test.bounds...) + _, _, _, blockFilter, _, canCompile, _ := CompileFilterExpr(expr, tableDef, nil) + require.True(t, canCompile) + _, selected, err := blockFilter(0, meta, nil) + require.NoError(t, err) + require.Equal(t, test.want, selected) + }) + } +} + +func decimalTableDef(typ plan.Type, primary bool) *plan.TableDef { + return &plan.TableDef{ + Name: "decimal_scan", + Name2ColIndex: map[string]int32{"amount": 0}, + Cols: []*plan.ColDef{{ + Name: "amount", ColId: 1, Seqnum: 0, Primary: primary, Typ: typ, + }}, + } +} + +func decimalFoldedFilter(t *testing.T, colType plan.Type, op string, bounds ...decimalBound) *plan.Expr { + t.Helper() + args := []*plan.Expr{{ + Typ: colType, + Expr: &plan.Expr_Col{Col: &plan.ColRef{ + RelPos: 0, ColPos: 0, Name: "amount", + }}, + }} + for _, bound := range bounds { + boundType := colType + boundType.Scale = bound.scale + if types.T(boundType.Id) == types.T_decimal64 { + boundType.Width = 18 + } else { + boundType.Width = 38 + } + args = append(args, &plan.Expr{ + Typ: boundType, + Expr: &plan.Expr_Fold{Fold: &plan.FoldVal{IsConst: true, Data: encodeDecimal(t, boundType, bound.text)}}, + }) + } + return foldedFunction(op, args) +} + +func decimalInRangeFoldedFilter( + t *testing.T, + colType plan.Type, + lower, upper decimalBound, + hint uint8, +) *plan.Expr { + expr := decimalFoldedFilter(t, colType, "in_range", lower, upper) + hintValue := hint + expr.GetF().Args = append(expr.GetF().Args, &plan.Expr{ + Typ: plan.Type{Id: int32(types.T_uint8)}, + Expr: &plan.Expr_Fold{Fold: &plan.FoldVal{IsConst: true, Data: types.EncodeUint8(&hintValue)}}, + }) + return expr +} + +func int64FoldedFilter(op string, bounds ...int64) *plan.Expr { + args := []*plan.Expr{{ + Typ: plan.Type{Id: int32(types.T_int64)}, + Expr: &plan.Expr_Col{Col: &plan.ColRef{ + RelPos: 0, ColPos: 0, Name: "amount", + }}, + }} + for _, value := range bounds { + args = append(args, &plan.Expr{ + Typ: plan.Type{Id: int32(types.T_int64)}, + Expr: &plan.Expr_Fold{Fold: &plan.FoldVal{IsConst: true, Data: types.EncodeInt64(&value)}}, + }) + } + return foldedFunction(op, args) +} + +func sortedUnknownFilter( + t *testing.T, + typ plan.Type, + op string, + bounds []string, + hint uint8, +) *plan.Expr { + t.Helper() + if op == "in" || op == "prefix_in" { + return sortedUnknownVectorFilter(t, typ, bounds, op == "prefix_in") + } + if types.T(typ.Id).IsDecimal() { + decimalBounds := make([]decimalBound, len(bounds)) + for i, bound := range bounds { + decimalBounds[i] = decimalBound{text: bound, scale: 0} + } + if op == "in_range" { + return decimalInRangeFoldedFilter(t, typ, decimalBounds[0], decimalBounds[1], hint) + } + return decimalFoldedFilter(t, typ, op, decimalBounds...) + } + + args := []*plan.Expr{{ + Typ: typ, + Expr: &plan.Expr_Col{Col: &plan.ColRef{ + RelPos: 0, ColPos: 0, Name: "amount", + }}, + }} + for _, bound := range bounds { + args = append(args, &plan.Expr{ + Typ: typ, + Expr: &plan.Expr_Fold{Fold: &plan.FoldVal{ + IsConst: true, + Data: encodeSortedUnknownValue(t, typ, bound), + }}, + }) + } + if op == "in_range" || op == "prefix_in_range" { + hintValue := hint + args = append(args, &plan.Expr{ + Typ: plan.Type{Id: int32(types.T_uint8)}, + Expr: &plan.Expr_Fold{Fold: &plan.FoldVal{IsConst: true, Data: types.EncodeUint8(&hintValue)}}, + }) + } + return foldedFunction(op, args) +} + +func sortedUnknownVectorFilter( + t *testing.T, + typ plan.Type, + values []string, + prefix bool, +) *plan.Expr { + return sortedUnknownVectorFilterWithType(t, typ, typ, values, prefix) +} + +func sortedUnknownVectorFilterWithType( + t *testing.T, + columnType plan.Type, + vectorType plan.Type, + values []string, + prefix bool, +) *plan.Expr { + t.Helper() + proc := testutil.NewProcess(t) + vec := vector.NewVec(plan2.MakeTypeByPlan2Type(vectorType)) + defer vec.Free(proc.Mp()) + for _, value := range values { + encoded := encodeSortedUnknownValue(t, vectorType, value) + switch types.T(vectorType.Id) { + case types.T_decimal64: + require.NoError(t, vector.AppendFixed(vec, types.DecodeDecimal64(encoded), false, proc.Mp())) + case types.T_decimal128: + require.NoError(t, vector.AppendFixed(vec, types.DecodeDecimal128(encoded), false, proc.Mp())) + case types.T_int64: + require.NoError(t, vector.AppendFixed(vec, types.DecodeInt64(encoded), false, proc.Mp())) + case types.T_char, types.T_varchar, types.T_text, types.T_blob, types.T_json: + require.NoError(t, vector.AppendBytes(vec, encoded, false, proc.Mp())) + default: + t.Fatalf("unsupported vector zonemap test type %v", types.T(vectorType.Id)) + } + } + data, err := vec.MarshalBinary() + require.NoError(t, err) + return plan2.MakeInExpr( + context.Background(), + &plan.Expr{ + Typ: columnType, + Expr: &plan.Expr_Col{Col: &plan.ColRef{ + RelPos: 0, ColPos: 0, Name: "amount", + }}, + }, + int32(len(values)), + data, + prefix, + ) +} + +func sortedUnknownDataMeta(t *testing.T, typ plan.Type) objectio.ObjectDataMeta { + t.Helper() + dataMeta := objectio.BuildMetaData(3, 1) + unknown := index.NewZM(types.T(typ.Id), typ.Scale) + dataMeta.MustGetColumn(0).SetZoneMap(unknown) + dataMeta.GetBlockMeta(0).MustGetColumn(0).SetZoneMap(unknown.Clone()) + dataMeta.GetBlockMeta(1).MustGetColumn(0).SetZoneMap(sortedUnknownZoneMap(t, typ, "20")) + dataMeta.GetBlockMeta(2).MustGetColumn(0).SetZoneMap(sortedUnknownZoneMap(t, typ, "40")) + return dataMeta +} + +func sortedUnknownZoneMap(t *testing.T, typ plan.Type, values ...string) objectio.ZoneMap { + t.Helper() + zm := index.NewZM(types.T(typ.Id), typ.Scale) + for _, value := range values { + index.UpdateZM(zm, encodeSortedUnknownValue(t, typ, value)) + } + return zm +} + +func encodeSortedUnknownValue(t *testing.T, typ plan.Type, value string) []byte { + t.Helper() + switch types.T(typ.Id) { + case types.T_decimal64, types.T_decimal128: + return encodeDecimal(t, typ, value) + case types.T_int64: + parsed, err := strconv.ParseInt(value, 10, 64) + require.NoError(t, err) + return types.EncodeInt64(&parsed) + case types.T_char, types.T_varchar, types.T_text, types.T_blob, types.T_json: + return []byte("000000" + value) + default: + t.Fatalf("unsupported sorted zonemap test type %v", types.T(typ.Id)) + return nil + } +} + +func foldedFunction(name string, args []*plan.Expr) *plan.Expr { + return &plan.Expr{ + Typ: plan.Type{Id: int32(types.T_bool)}, + Expr: &plan.Expr_F{F: &plan.Function{ + Func: &plan.ObjectRef{ObjName: name}, Args: args, + }}, + } +} + +func decimalLiteralValue(expr *plan.Expr) any { + switch types.T(expr.Typ.Id) { + case types.T_decimal64: + return types.Decimal64(expr.GetLit().GetDecimal64Val().A) + case types.T_decimal128: + value := expr.GetLit().GetDecimal128Val() + return types.Decimal128{B0_63: uint64(value.A), B64_127: uint64(value.B)} + default: + return nil + } +} + +func encodeDecimal(t *testing.T, typ plan.Type, text string) []byte { + t.Helper() + switch types.T(typ.Id) { + case types.T_decimal64: + value, err := types.ParseDecimal64(text, typ.Width, typ.Scale) + require.NoError(t, err) + return types.EncodeDecimal64(&value) + case types.T_decimal128: + value, err := types.ParseDecimal128(text, typ.Width, typ.Scale) + require.NoError(t, err) + return types.EncodeDecimal128(&value) + default: + t.Fatalf("unsupported decimal type %v", types.T(typ.Id)) + return nil + } +} + +func decimalBoundZoneMap( + t *testing.T, + colType plan.Type, + bound decimalBound, +) ([]byte, objectio.ZoneMap) { + t.Helper() + boundType := colType + boundType.Scale = bound.scale + if types.T(boundType.Id) == types.T_decimal64 { + boundType.Width = 18 + } else { + boundType.Width = 38 + } + value := encodeDecimal(t, boundType, bound.text) + zoneMap, ok := makeDecimalZoneMapBound( + &plan.ColDef{Typ: colType}, + value, + &plan.Expr{Typ: boundType}, + ) + require.True(t, ok) + return value, zoneMap +} + +func decimalZoneMap(t *testing.T, typ plan.Type, values ...string) objectio.ZoneMap { + t.Helper() + zm := index.NewZM(types.T(typ.Id), typ.Scale) + for _, text := range values { + index.UpdateZM(zm, encodeDecimal(t, typ, text)) + } + return zm +} + +func decimalObjectDataMeta(t *testing.T, typ plan.Type, values ...string) objectio.ObjectDataMeta { + t.Helper() + dataMeta := objectio.BuildMetaData(uint16(len(values)), 1) + objectZM := decimalZoneMap(t, typ, values...) + dataMeta.MustGetColumn(0).SetZoneMap(objectZM) + for i, text := range values { + dataMeta.GetBlockMeta(uint32(i)).MustGetColumn(0).SetZoneMap(decimalZoneMap(t, typ, text)) + } + return dataMeta +} + +func decimalObjectStats(t *testing.T, typ plan.Type, values ...string) *objectio.ObjectStats { + t.Helper() + stats := objectio.NewObjectStats() + require.NoError(t, objectio.SetObjectStatsSortKeyZoneMap(stats, decimalZoneMap(t, typ, values...))) + return stats +} + +func makeDecimalBlockMeta(t *testing.T, typ plan.Type, values ...string) objectio.BlockObject { + t.Helper() + dataMeta := objectio.BuildMetaData(1, 1) + meta := dataMeta.GetBlockMeta(0) + zm := index.NewZM(types.T(typ.Id), typ.Scale) + for _, text := range values { + index.UpdateZM(zm, encodeDecimal(t, typ, text)) + } + meta.MustGetColumn(0).SetZoneMap(zm) + return meta +} diff --git a/pkg/vm/engine/readutil/expr_util.go b/pkg/vm/engine/readutil/expr_util.go index 7dead64b48738..32f677784a447 100644 --- a/pkg/vm/engine/readutil/expr_util.go +++ b/pkg/vm/engine/readutil/expr_util.go @@ -160,6 +160,13 @@ func evalValue( func mustColConstValueFromBinaryFuncExpr( expr *plan.Expr_F, ) (*plan.Expr_Col, [][]byte, bool) { + colExpr, vals, _, ok := mustColConstValueWithTypeFromBinaryFuncExpr(expr) + return colExpr, vals, ok +} + +func mustColConstValueWithTypeFromBinaryFuncExpr( + expr *plan.Expr_F, +) (*plan.Expr_Col, [][]byte, []*plan.Expr, bool) { var ( colExpr *plan.Expr_Col tmpExpr *plan.Expr_Col @@ -176,14 +183,14 @@ func mustColConstValueFromBinaryFuncExpr( } if len(valExprs) == 0 || colExpr == nil { - return nil, nil, false + return nil, nil, nil, false } vals, ok := getConstBytesFromExpr(valExprs) if !ok { - return nil, nil, false + return nil, nil, nil, false } - return colExpr, vals, true + return colExpr, vals, valExprs, true } func getConstBytesFromExpr(exprs []*plan.Expr) ([][]byte, bool) { diff --git a/test/distributed/cases/dtype/decimal.result b/test/distributed/cases/dtype/decimal.result index 456e0ecca0f15..bf70b8c79432e 100644 --- a/test/distributed/cases/dtype/decimal.result +++ b/test/distributed/cases/dtype/decimal.result @@ -1759,3 +1759,218 @@ ORDER BY r.id, l.id; 2 ¦ 12 DROP TABLE decimal_join_left; DROP TABLE decimal_join_right; +DROP TABLE IF EXISTS decimal_zm_scale_scan; +CREATE TABLE decimal_zm_scale_scan ( +id BIGINT NOT NULL PRIMARY KEY, +d64 DECIMAL(12,4), +d128 DECIMAL(20,4) +); +INSERT INTO decimal_zm_scale_scan +SELECT result, +CAST((result - 20001) / 10.0 AS DECIMAL(12,4)), +CAST((result - 20001) / 10.0 AS DECIMAL(20,4)) +FROM generate_series(1, 40001) g; +INSERT INTO decimal_zm_scale_scan VALUES (40002, NULL, NULL); +SELECT mo_ctl('dn','flush', concat(database(), '.decimal_zm_scale_scan')); +➤ mo_ctl(dn, flush, concat(database(), .decimal_zm_scale_scan))[12,-1,0] 𝄀 +{ + "method": "Flush", + "result": [ + { + "returnStr": "OK" + } + ] +} + +SELECT COUNT(*) AS d64_lt_s0 FROM decimal_zm_scale_scan WHERE d64 < 1000; +➤ d64_lt_s0[-5,64,0] 𝄀 +30000 +SELECT COUNT(*) AS d64_le_s0 FROM decimal_zm_scale_scan WHERE d64 <= 1000; +➤ d64_le_s0[-5,64,0] 𝄀 +30001 +SELECT COUNT(*) AS d64_gt_s0 FROM decimal_zm_scale_scan WHERE d64 > 1000; +➤ d64_gt_s0[-5,64,0] 𝄀 +10000 +SELECT COUNT(*) AS d64_ge_s0 FROM decimal_zm_scale_scan WHERE d64 >= 1000; +➤ d64_ge_s0[-5,64,0] 𝄀 +10001 +SELECT COUNT(*) AS d64_eq_s0 FROM decimal_zm_scale_scan WHERE d64 = 0; +➤ d64_eq_s0[-5,64,0] 𝄀 +1 +SELECT COUNT(*) AS d64_between_s0 FROM decimal_zm_scale_scan WHERE d64 BETWEEN -1000 AND 1000; +➤ d64_between_s0[-5,64,0] 𝄀 +20001 +SELECT COUNT(*) AS d64_lt_lossy_upper FROM decimal_zm_scale_scan WHERE d64 < 1000.00001; +➤ d64_lt_lossy_upper[-5,64,0] 𝄀 +30001 +SELECT COUNT(*) AS d64_between_lossy_lower FROM decimal_zm_scale_scan WHERE d64 BETWEEN -999.99999 AND 1000; +➤ d64_between_lossy_lower[-5,64,0] 𝄀 +20000 +SELECT COUNT(*) AS d128_lt_s0 FROM decimal_zm_scale_scan WHERE d128 < 1000; +➤ d128_lt_s0[-5,64,0] 𝄀 +30000 +SELECT COUNT(*) AS d128_lt_s1 FROM decimal_zm_scale_scan WHERE d128 < 1000.0; +➤ d128_lt_s1[-5,64,0] 𝄀 +30000 +SELECT COUNT(*) AS d128_lt_s4 FROM decimal_zm_scale_scan WHERE d128 < 1000.0000; +➤ d128_lt_s4[-5,64,0] 𝄀 +30000 +SELECT COUNT(*) AS d128_le_s0 FROM decimal_zm_scale_scan WHERE d128 <= 1000; +➤ d128_le_s0[-5,64,0] 𝄀 +30001 +SELECT COUNT(*) AS d128_gt_s0 FROM decimal_zm_scale_scan WHERE d128 > 1000; +➤ d128_gt_s0[-5,64,0] 𝄀 +10000 +SELECT COUNT(*) AS d128_ge_s0 FROM decimal_zm_scale_scan WHERE d128 >= 1000; +➤ d128_ge_s0[-5,64,0] 𝄀 +10001 +SELECT COUNT(*) AS d128_eq_s0 FROM decimal_zm_scale_scan WHERE d128 = 0; +➤ d128_eq_s0[-5,64,0] 𝄀 +1 +SELECT COUNT(*) AS d128_between_s0 FROM decimal_zm_scale_scan WHERE d128 BETWEEN -1000 AND 1000; +➤ d128_between_s0[-5,64,0] 𝄀 +20001 +SELECT COUNT(*) AS d128_between_mixed FROM decimal_zm_scale_scan WHERE d128 BETWEEN -1000.0 AND 1000.0000; +➤ d128_between_mixed[-5,64,0] 𝄀 +20001 +SELECT COUNT(*) AS d128_lt_lossy_upper FROM decimal_zm_scale_scan WHERE d128 < 1000.00001; +➤ d128_lt_lossy_upper[-5,64,0] 𝄀 +30001 +SELECT COUNT(*) AS d128_between_lossy_lower FROM decimal_zm_scale_scan WHERE d128 BETWEEN -999.99999 AND 1000; +➤ d128_between_lossy_lower[-5,64,0] 𝄀 +20000 +SELECT COUNT(*) AS d128_lt_oracle FROM decimal_zm_scale_scan WHERE d128 + 0 < 1000; +➤ d128_lt_oracle[-5,64,0] 𝄀 +30000 +SELECT COUNT(*) AS decimal_nulls FROM decimal_zm_scale_scan WHERE d64 IS NULL AND d128 IS NULL; +➤ decimal_nulls[-5,64,0] 𝄀 +1 +SELECT COUNT(*) AS int_control FROM decimal_zm_scale_scan WHERE id < 1000; +➤ int_control[-5,64,0] 𝄀 +999 +DROP TABLE decimal_zm_scale_scan; +DROP TABLE IF EXISTS nullable_decimal_cluster_zm; +CREATE TABLE nullable_decimal_cluster_zm ( +id BIGINT NOT NULL, +amount DECIMAL(20,4) +) CLUSTER BY(amount); +INSERT INTO nullable_decimal_cluster_zm +SELECT result, +CASE WHEN result <= 20000 THEN NULL +ELSE CAST((result - 20000) / 10.0 AS DECIMAL(20,4)) END +FROM generate_series(1, 50000) g; +SELECT mo_ctl('dn','flush', concat(database(), '.nullable_decimal_cluster_zm')); +➤ mo_ctl(dn, flush, concat(database(), .nullable_decimal_cluster_zm))[12,-1,0] 𝄀 +{ + "method": "Flush", + "result": [ + { + "returnStr": "OK" + } + ] +} + +SELECT COUNT(*), SUM(id), SUM(amount) FROM nullable_decimal_cluster_zm WHERE amount < 2000.0000; +➤ count(*)[-5,64,0] ¦ sum(id)[-5,64,0] ¦ sum(amount)[3,38,4] 𝄀 +19999 ¦ 599970000 ¦ 19999000.0000 +SELECT COUNT(*), SUM(id), SUM(amount) FROM nullable_decimal_cluster_zm WHERE amount + 0 < 2000.0000; +➤ count(*)[-5,64,0] ¦ sum(id)[-5,64,0] ¦ sum(amount)[3,38,4] 𝄀 +19999 ¦ 599970000 ¦ 19999000.0000 +SELECT COUNT(*) FROM nullable_decimal_cluster_zm WHERE amount <= 2000.0000; +➤ count(*)[-5,64,0] 𝄀 +20000 +SELECT COUNT(*) FROM nullable_decimal_cluster_zm WHERE amount > 2000.0000; +➤ count(*)[-5,64,0] 𝄀 +10000 +SELECT COUNT(*) FROM nullable_decimal_cluster_zm WHERE amount IN (1000.0000, 2000.0000); +➤ count(*)[-5,64,0] 𝄀 +2 +SELECT COUNT(*) FROM nullable_decimal_cluster_zm WHERE amount + 0 IN (1000.0000, 2000.0000); +➤ count(*)[-5,64,0] 𝄀 +2 +DROP TABLE nullable_decimal_cluster_zm; +DROP TABLE IF EXISTS nullable_bigint_cluster_zm; +CREATE TABLE nullable_bigint_cluster_zm ( +id BIGINT NOT NULL, +amount BIGINT +) CLUSTER BY(amount); +INSERT INTO nullable_bigint_cluster_zm +SELECT result, +CASE WHEN result <= 20000 THEN NULL ELSE result - 20000 END +FROM generate_series(1, 50000) g; +SELECT mo_ctl('dn','flush', concat(database(), '.nullable_bigint_cluster_zm')); +➤ mo_ctl(dn, flush, concat(database(), .nullable_bigint_cluster_zm))[12,-1,0] 𝄀 +{ + "method": "Flush", + "result": [ + { + "returnStr": "OK" + } + ] +} + +SELECT COUNT(*) FROM nullable_bigint_cluster_zm WHERE amount < 20000; +➤ count(*)[-5,64,0] 𝄀 +19999 +SELECT COUNT(*) FROM nullable_bigint_cluster_zm WHERE amount + 0 < 20000; +➤ count(*)[-5,64,0] 𝄀 +19999 +SELECT COUNT(*) FROM nullable_bigint_cluster_zm WHERE amount <= 20000; +➤ count(*)[-5,64,0] 𝄀 +20000 +SELECT COUNT(*) FROM nullable_bigint_cluster_zm WHERE amount > 20000; +➤ count(*)[-5,64,0] 𝄀 +10000 +SELECT COUNT(*) FROM nullable_bigint_cluster_zm WHERE amount IN (10000, 20000); +➤ count(*)[-5,64,0] 𝄀 +2 +SELECT COUNT(*) FROM nullable_bigint_cluster_zm WHERE amount + 0 IN (10000, 20000); +➤ count(*)[-5,64,0] 𝄀 +2 +DROP TABLE nullable_bigint_cluster_zm; +DROP TABLE IF EXISTS nullable_varchar_cluster_zm; +CREATE TABLE nullable_varchar_cluster_zm ( +id BIGINT NOT NULL, +amount VARCHAR(8) +) CLUSTER BY(amount); +INSERT INTO nullable_varchar_cluster_zm +SELECT result, +CASE WHEN result <= 20000 THEN NULL +ELSE LPAD(CAST(result - 20000 AS VARCHAR), 8, '0') END +FROM generate_series(1, 50000) g; +SELECT mo_ctl('dn','flush', concat(database(), '.nullable_varchar_cluster_zm')); +➤ mo_ctl(dn, flush, concat(database(), .nullable_varchar_cluster_zm))[12,-1,0] 𝄀 +{ + "method": "Flush", + "result": [ + { + "returnStr": "OK" + } + ] +} + +SELECT COUNT(*) FROM nullable_varchar_cluster_zm WHERE amount < '00020000'; +➤ count(*)[-5,64,0] 𝄀 +19999 +SELECT COUNT(*) FROM nullable_varchar_cluster_zm WHERE CONCAT(amount, '') < '00020000'; +➤ count(*)[-5,64,0] 𝄀 +19999 +SELECT COUNT(*) FROM nullable_varchar_cluster_zm WHERE amount <= '00020000'; +➤ count(*)[-5,64,0] 𝄀 +20000 +SELECT COUNT(*) FROM nullable_varchar_cluster_zm WHERE amount > '00020000'; +➤ count(*)[-5,64,0] 𝄀 +10000 +SELECT COUNT(*) FROM nullable_varchar_cluster_zm WHERE amount IN ('00010000', '00020000'); +➤ count(*)[-5,64,0] 𝄀 +2 +SELECT COUNT(*) FROM nullable_varchar_cluster_zm WHERE CONCAT(amount, '') IN ('00010000', '00020000'); +➤ count(*)[-5,64,0] 𝄀 +2 +SELECT COUNT(*) FROM nullable_varchar_cluster_zm WHERE amount LIKE '0001%'; +➤ count(*)[-5,64,0] 𝄀 +10000 +SELECT COUNT(*) FROM nullable_varchar_cluster_zm WHERE CONCAT(amount, '') LIKE '0001%'; +➤ count(*)[-5,64,0] 𝄀 +10000 +DROP TABLE nullable_varchar_cluster_zm; diff --git a/test/distributed/cases/dtype/decimal.test b/test/distributed/cases/dtype/decimal.test index a97f40e4df77c..268c025cb3d15 100644 --- a/test/distributed/cases/dtype/decimal.test +++ b/test/distributed/cases/dtype/decimal.test @@ -971,3 +971,108 @@ ORDER BY r.id, l.id; DROP TABLE decimal_join_left; DROP TABLE decimal_join_right; + +-- issue #26808: normal-table zonemap pruning must preserve the literal scale. +DROP TABLE IF EXISTS decimal_zm_scale_scan; +CREATE TABLE decimal_zm_scale_scan ( + id BIGINT NOT NULL PRIMARY KEY, + d64 DECIMAL(12,4), + d128 DECIMAL(20,4) +); +INSERT INTO decimal_zm_scale_scan +SELECT result, + CAST((result - 20001) / 10.0 AS DECIMAL(12,4)), + CAST((result - 20001) / 10.0 AS DECIMAL(20,4)) +FROM generate_series(1, 40001) g; +INSERT INTO decimal_zm_scale_scan VALUES (40002, NULL, NULL); +-- @separator:table +SELECT mo_ctl('dn','flush', concat(database(), '.decimal_zm_scale_scan')); + +SELECT COUNT(*) AS d64_lt_s0 FROM decimal_zm_scale_scan WHERE d64 < 1000; +SELECT COUNT(*) AS d64_le_s0 FROM decimal_zm_scale_scan WHERE d64 <= 1000; +SELECT COUNT(*) AS d64_gt_s0 FROM decimal_zm_scale_scan WHERE d64 > 1000; +SELECT COUNT(*) AS d64_ge_s0 FROM decimal_zm_scale_scan WHERE d64 >= 1000; +SELECT COUNT(*) AS d64_eq_s0 FROM decimal_zm_scale_scan WHERE d64 = 0; +SELECT COUNT(*) AS d64_between_s0 FROM decimal_zm_scale_scan WHERE d64 BETWEEN -1000 AND 1000; +SELECT COUNT(*) AS d64_lt_lossy_upper FROM decimal_zm_scale_scan WHERE d64 < 1000.00001; +SELECT COUNT(*) AS d64_between_lossy_lower FROM decimal_zm_scale_scan WHERE d64 BETWEEN -999.99999 AND 1000; + +SELECT COUNT(*) AS d128_lt_s0 FROM decimal_zm_scale_scan WHERE d128 < 1000; +SELECT COUNT(*) AS d128_lt_s1 FROM decimal_zm_scale_scan WHERE d128 < 1000.0; +SELECT COUNT(*) AS d128_lt_s4 FROM decimal_zm_scale_scan WHERE d128 < 1000.0000; +SELECT COUNT(*) AS d128_le_s0 FROM decimal_zm_scale_scan WHERE d128 <= 1000; +SELECT COUNT(*) AS d128_gt_s0 FROM decimal_zm_scale_scan WHERE d128 > 1000; +SELECT COUNT(*) AS d128_ge_s0 FROM decimal_zm_scale_scan WHERE d128 >= 1000; +SELECT COUNT(*) AS d128_eq_s0 FROM decimal_zm_scale_scan WHERE d128 = 0; +SELECT COUNT(*) AS d128_between_s0 FROM decimal_zm_scale_scan WHERE d128 BETWEEN -1000 AND 1000; +SELECT COUNT(*) AS d128_between_mixed FROM decimal_zm_scale_scan WHERE d128 BETWEEN -1000.0 AND 1000.0000; +SELECT COUNT(*) AS d128_lt_lossy_upper FROM decimal_zm_scale_scan WHERE d128 < 1000.00001; +SELECT COUNT(*) AS d128_between_lossy_lower FROM decimal_zm_scale_scan WHERE d128 BETWEEN -999.99999 AND 1000; +SELECT COUNT(*) AS d128_lt_oracle FROM decimal_zm_scale_scan WHERE d128 + 0 < 1000; +SELECT COUNT(*) AS decimal_nulls FROM decimal_zm_scale_scan WHERE d64 IS NULL AND d128 IS NULL; +SELECT COUNT(*) AS int_control FROM decimal_zm_scale_scan WHERE id < 1000; + +DROP TABLE decimal_zm_scale_scan; + +-- issues #26808 and #26817: an all-NULL leading block on a nullable sorted key +-- leaves its zonemap uninitialized and must not exclude later matching blocks. +DROP TABLE IF EXISTS nullable_decimal_cluster_zm; +CREATE TABLE nullable_decimal_cluster_zm ( + id BIGINT NOT NULL, + amount DECIMAL(20,4) +) CLUSTER BY(amount); +INSERT INTO nullable_decimal_cluster_zm +SELECT result, + CASE WHEN result <= 20000 THEN NULL + ELSE CAST((result - 20000) / 10.0 AS DECIMAL(20,4)) END +FROM generate_series(1, 50000) g; +-- @separator:table +SELECT mo_ctl('dn','flush', concat(database(), '.nullable_decimal_cluster_zm')); +SELECT COUNT(*), SUM(id), SUM(amount) FROM nullable_decimal_cluster_zm WHERE amount < 2000.0000; +SELECT COUNT(*), SUM(id), SUM(amount) FROM nullable_decimal_cluster_zm WHERE amount + 0 < 2000.0000; +SELECT COUNT(*) FROM nullable_decimal_cluster_zm WHERE amount <= 2000.0000; +SELECT COUNT(*) FROM nullable_decimal_cluster_zm WHERE amount > 2000.0000; +SELECT COUNT(*) FROM nullable_decimal_cluster_zm WHERE amount IN (1000.0000, 2000.0000); +SELECT COUNT(*) FROM nullable_decimal_cluster_zm WHERE amount + 0 IN (1000.0000, 2000.0000); +DROP TABLE nullable_decimal_cluster_zm; + +DROP TABLE IF EXISTS nullable_bigint_cluster_zm; +CREATE TABLE nullable_bigint_cluster_zm ( + id BIGINT NOT NULL, + amount BIGINT +) CLUSTER BY(amount); +INSERT INTO nullable_bigint_cluster_zm +SELECT result, + CASE WHEN result <= 20000 THEN NULL ELSE result - 20000 END +FROM generate_series(1, 50000) g; +-- @separator:table +SELECT mo_ctl('dn','flush', concat(database(), '.nullable_bigint_cluster_zm')); +SELECT COUNT(*) FROM nullable_bigint_cluster_zm WHERE amount < 20000; +SELECT COUNT(*) FROM nullable_bigint_cluster_zm WHERE amount + 0 < 20000; +SELECT COUNT(*) FROM nullable_bigint_cluster_zm WHERE amount <= 20000; +SELECT COUNT(*) FROM nullable_bigint_cluster_zm WHERE amount > 20000; +SELECT COUNT(*) FROM nullable_bigint_cluster_zm WHERE amount IN (10000, 20000); +SELECT COUNT(*) FROM nullable_bigint_cluster_zm WHERE amount + 0 IN (10000, 20000); +DROP TABLE nullable_bigint_cluster_zm; + +DROP TABLE IF EXISTS nullable_varchar_cluster_zm; +CREATE TABLE nullable_varchar_cluster_zm ( + id BIGINT NOT NULL, + amount VARCHAR(8) +) CLUSTER BY(amount); +INSERT INTO nullable_varchar_cluster_zm +SELECT result, + CASE WHEN result <= 20000 THEN NULL + ELSE LPAD(CAST(result - 20000 AS VARCHAR), 8, '0') END +FROM generate_series(1, 50000) g; +-- @separator:table +SELECT mo_ctl('dn','flush', concat(database(), '.nullable_varchar_cluster_zm')); +SELECT COUNT(*) FROM nullable_varchar_cluster_zm WHERE amount < '00020000'; +SELECT COUNT(*) FROM nullable_varchar_cluster_zm WHERE CONCAT(amount, '') < '00020000'; +SELECT COUNT(*) FROM nullable_varchar_cluster_zm WHERE amount <= '00020000'; +SELECT COUNT(*) FROM nullable_varchar_cluster_zm WHERE amount > '00020000'; +SELECT COUNT(*) FROM nullable_varchar_cluster_zm WHERE amount IN ('00010000', '00020000'); +SELECT COUNT(*) FROM nullable_varchar_cluster_zm WHERE CONCAT(amount, '') IN ('00010000', '00020000'); +SELECT COUNT(*) FROM nullable_varchar_cluster_zm WHERE amount LIKE '0001%'; +SELECT COUNT(*) FROM nullable_varchar_cluster_zm WHERE CONCAT(amount, '') LIKE '0001%'; +DROP TABLE nullable_varchar_cluster_zm;