diff --git a/be/src/common/config.h b/be/src/common/config.h index 9620cd54e0e58d..a9c9357905024a 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -784,8 +784,6 @@ CONF_mInt32(orc_natural_read_size_mb, "8"); // if it is lower than a specific threshold, the predicate will be disabled. CONF_mInt32(bloom_filter_predicate_check_row_num, "204800"); -CONF_Bool(enable_decimalv3, "false"); - //whether turn on quick compaction feature CONF_Bool(enable_quick_compaction, "false"); // For continuous versions that rows less than quick_compaction_max_rows will trigger compaction quickly diff --git a/be/src/exec/csv_scan_node.cpp b/be/src/exec/csv_scan_node.cpp index f84c25cbba8ed9..4750e54b36ed66 100644 --- a/be/src/exec/csv_scan_node.cpp +++ b/be/src/exec/csv_scan_node.cpp @@ -484,7 +484,7 @@ bool CsvScanNode::check_and_write_text_slot(const std::string& column_name, char_len); value_to_convert_length = char_len; } - } else if (slot->type().is_decimal_type()) { + } else if (slot->type().is_decimal_v2_type()) { int precision = column_type.precision; int scale = column_type.scale; bool is_success = check_decimal_input(value, value_length, precision, scale, error_msg); diff --git a/be/src/exec/olap_common.h b/be/src/exec/olap_common.h index aa200ec5a80794..92487bbf5fed6f 100644 --- a/be/src/exec/olap_common.h +++ b/be/src/exec/olap_common.h @@ -43,7 +43,7 @@ std::string cast_to_string(T value, int scale) { std::stringstream ss; vectorized::write_text((int64_t)value, scale, ss); return ss.str(); - } else if constexpr (primitive_type == TYPE_DECIMAL128) { + } else if constexpr (primitive_type == TYPE_DECIMAL128I) { std::stringstream ss; vectorized::write_text((int128_t)value, scale, ss); return ss.str(); @@ -389,7 +389,7 @@ using ColumnValueRangeType = ColumnValueRange, ColumnValueRange, ColumnValueRange, ColumnValueRange, ColumnValueRange, ColumnValueRange, - ColumnValueRange, ColumnValueRange>; + ColumnValueRange, ColumnValueRange>; template const typename ColumnValueRange::CppType diff --git a/be/src/exec/table_connector.cpp b/be/src/exec/table_connector.cpp index 77c3a13449cd40..fa725832d3681e 100644 --- a/be/src/exec/table_connector.cpp +++ b/be/src/exec/table_connector.cpp @@ -285,7 +285,7 @@ Status TableConnector::append(const std::string& table_name, vectorized::Block* } case TYPE_DECIMAL32: case TYPE_DECIMAL64: - case TYPE_DECIMAL128: { + case TYPE_DECIMAL128I: { auto val = type_ptr->to_string(*column, i); fmt::format_to(_insert_stmt_buffer, "{}", val); break; diff --git a/be/src/exprs/anyval_util.cpp b/be/src/exprs/anyval_util.cpp index 346da8731720db..b83e04e9ce789f 100644 --- a/be/src/exprs/anyval_util.cpp +++ b/be/src/exprs/anyval_util.cpp @@ -100,7 +100,7 @@ AnyVal* create_any_val(ObjectPool* pool, const TypeDescriptor& type) { case TYPE_DECIMAL64: return pool->add(new BigIntVal); - case TYPE_DECIMAL128: + case TYPE_DECIMAL128I: return pool->add(new LargeIntVal); case TYPE_DATE: @@ -175,8 +175,8 @@ FunctionContext::TypeDesc AnyValUtil::column_type_to_type_desc(const TypeDescrip out.precision = type.precision; out.scale = type.scale; break; - case TYPE_DECIMAL128: - out.type = FunctionContext::TYPE_DECIMAL128; + case TYPE_DECIMAL128I: + out.type = FunctionContext::TYPE_DECIMAL128I; out.precision = type.precision; out.scale = type.scale; break; diff --git a/be/src/exprs/anyval_util.h b/be/src/exprs/anyval_util.h index 742d44214e69ab..007f95f0eb2ece 100644 --- a/be/src/exprs/anyval_util.h +++ b/be/src/exprs/anyval_util.h @@ -431,7 +431,7 @@ class AnyValUtil { reinterpret_cast(dst)->val = *reinterpret_cast(slot); return; - case TYPE_DECIMAL128: + case TYPE_DECIMAL128I: memcpy(&reinterpret_cast(dst)->val, slot, sizeof(__int128)); return; diff --git a/be/src/exprs/binary_predicate.cpp b/be/src/exprs/binary_predicate.cpp index f06757716631d1..6fa54942debcdc 100644 --- a/be/src/exprs/binary_predicate.cpp +++ b/be/src/exprs/binary_predicate.cpp @@ -44,7 +44,7 @@ Expr* BinaryPredicate::from_thrift(const TExprNode& node) { case TPrimitiveType::DATETIMEV2: return new EqBigIntValPred(node); case TPrimitiveType::LARGEINT: - case TPrimitiveType::DECIMAL128: + case TPrimitiveType::DECIMAL128I: return new EqLargeIntValPred(node); case TPrimitiveType::FLOAT: return new EqFloatValPred(node); @@ -81,7 +81,7 @@ Expr* BinaryPredicate::from_thrift(const TExprNode& node) { case TPrimitiveType::DATETIMEV2: return new NeBigIntValPred(node); case TPrimitiveType::LARGEINT: - case TPrimitiveType::DECIMAL128: + case TPrimitiveType::DECIMAL128I: return new NeLargeIntValPred(node); case TPrimitiveType::FLOAT: return new NeFloatValPred(node); @@ -118,7 +118,7 @@ Expr* BinaryPredicate::from_thrift(const TExprNode& node) { case TPrimitiveType::DATETIMEV2: return new LtBigIntValPred(node); case TPrimitiveType::LARGEINT: - case TPrimitiveType::DECIMAL128: + case TPrimitiveType::DECIMAL128I: return new LtLargeIntValPred(node); case TPrimitiveType::FLOAT: return new LtFloatValPred(node); @@ -155,7 +155,7 @@ Expr* BinaryPredicate::from_thrift(const TExprNode& node) { case TPrimitiveType::DATETIMEV2: return new LeBigIntValPred(node); case TPrimitiveType::LARGEINT: - case TPrimitiveType::DECIMAL128: + case TPrimitiveType::DECIMAL128I: return new LeLargeIntValPred(node); case TPrimitiveType::FLOAT: return new LeFloatValPred(node); @@ -192,7 +192,7 @@ Expr* BinaryPredicate::from_thrift(const TExprNode& node) { case TPrimitiveType::DATETIMEV2: return new GtBigIntValPred(node); case TPrimitiveType::LARGEINT: - case TPrimitiveType::DECIMAL128: + case TPrimitiveType::DECIMAL128I: return new GtLargeIntValPred(node); case TPrimitiveType::FLOAT: return new GtFloatValPred(node); @@ -229,7 +229,7 @@ Expr* BinaryPredicate::from_thrift(const TExprNode& node) { case TPrimitiveType::DATETIMEV2: return new GeBigIntValPred(node); case TPrimitiveType::LARGEINT: - case TPrimitiveType::DECIMAL128: + case TPrimitiveType::DECIMAL128I: return new GeLargeIntValPred(node); case TPrimitiveType::FLOAT: return new GeFloatValPred(node); @@ -266,7 +266,7 @@ Expr* BinaryPredicate::from_thrift(const TExprNode& node) { case TPrimitiveType::DATETIMEV2: return new EqForNullBigIntValPred(node); case TPrimitiveType::LARGEINT: - case TPrimitiveType::DECIMAL128: + case TPrimitiveType::DECIMAL128I: return new EqForNullLargeIntValPred(node); case TPrimitiveType::FLOAT: return new EqForNullFloatValPred(node); diff --git a/be/src/exprs/cast_functions.cpp b/be/src/exprs/cast_functions.cpp index d2d887ac615bc6..e5079216f1ee4f 100644 --- a/be/src/exprs/cast_functions.cpp +++ b/be/src/exprs/cast_functions.cpp @@ -31,7 +31,6 @@ #include "util/array_parser.h" #include "util/mysql_global.h" #include "util/string_parser.hpp" -#include "vec/data_types/data_type_decimal.h" #include "vec/runtime/vdatetime_value.h" namespace doris { @@ -362,254 +361,6 @@ DateTimeVal CastFunctions::cast_to_date_val(FunctionContext* ctx, const StringVa return result; } -#define CAST_TYPE_DECIMAL32(from_type) \ - Decimal32Val CastFunctions::cast_to_decimal32_val(FunctionContext* ctx, \ - const from_type& val) { \ - if (val.is_null) { \ - return Decimal32Val::null(); \ - } \ - auto scale_to = ctx->get_return_type().scale; \ - return Decimal32Val( \ - val.val * \ - vectorized::DataTypeDecimal::get_scale_multiplier( \ - scale_to)); \ - } - -#define CAST_TYPE_DECIMAL64(from_type) \ - Decimal64Val CastFunctions::cast_to_decimal64_val(FunctionContext* ctx, \ - const from_type& val) { \ - if (val.is_null) { \ - return Decimal64Val::null(); \ - } \ - auto scale_to = ctx->get_return_type().scale; \ - return Decimal64Val( \ - val.val * \ - vectorized::DataTypeDecimal::get_scale_multiplier( \ - scale_to)); \ - } - -#define CAST_TYPE_DECIMAL128(from_type) \ - Decimal128Val CastFunctions::cast_to_decimal128_val(FunctionContext* ctx, \ - const from_type& val) { \ - if (val.is_null) { \ - return Decimal128Val::null(); \ - } \ - auto scale_to = ctx->get_return_type().scale; \ - return Decimal128Val( \ - val.val * \ - vectorized::DataTypeDecimal::get_scale_multiplier( \ - scale_to)); \ - } - -#define CAST_TYPE_DECIMAL(to_type) \ - CAST_TYPE_##to_type(TinyIntVal) CAST_TYPE_##to_type(SmallIntVal) CAST_TYPE_##to_type(IntVal) \ - CAST_TYPE_##to_type(BigIntVal) CAST_TYPE_##to_type(LargeIntVal) \ - CAST_TYPE_##to_type(FloatVal) CAST_TYPE_##to_type(DoubleVal) - -CAST_TYPE_DECIMAL(DECIMAL32) -CAST_TYPE_DECIMAL(DECIMAL64) -CAST_TYPE_DECIMAL(DECIMAL128) - -Decimal32Val CastFunctions::cast_to_decimal32_val(FunctionContext* context, - const DateTimeVal& val) { - if (val.is_null) { - return Decimal32Val::null(); - } - auto scale_to = context->get_return_type().scale; - DateTimeValue dt_value = DateTimeValue::from_datetime_val(val); - return Decimal32Val( - dt_value.to_int64() * - vectorized::DataTypeDecimal::get_scale_multiplier(scale_to)); -} - -Decimal32Val CastFunctions::cast_to_decimal32_val(FunctionContext* context, const StringVal& val) { - if (val.is_null) { - return Decimal32Val::null(); - } - std::stringstream ss; - StringParser::ParseResult result; - int32_t v = StringParser::string_to_decimal((const char*)val.ptr, val.len, - context->get_return_type().precision, - context->get_return_type().scale, &result); - return Decimal32Val(v); -} - -Decimal32Val CastFunctions::cast_to_decimal32_val(FunctionContext* ctx, const Decimal32Val& val) { - if (ctx->get_arg_type(0)->scale == ctx->get_return_type().scale && - ctx->get_arg_type(0)->precision == ctx->get_return_type().precision) { - return val; - } - if (val.is_null) { - return Decimal32Val::null(); - } - auto scale_from = ctx->get_arg_type(0)->scale; - auto scale_to = ctx->get_return_type().scale; - if (scale_to > scale_from) { - return Decimal32Val( - val.val * vectorized::DataTypeDecimal::get_scale_multiplier( - scale_to - scale_from)); - } else { - return Decimal32Val( - val.val / vectorized::DataTypeDecimal::get_scale_multiplier( - scale_from - scale_to)); - } -} - -Decimal64Val CastFunctions::cast_to_decimal64_val(FunctionContext* context, - const DateTimeVal& val) { - if (val.is_null) { - return Decimal64Val::null(); - } - auto scale_to = context->get_return_type().scale; - DateTimeValue dt_value = DateTimeValue::from_datetime_val(val); - return Decimal64Val( - dt_value.to_int64() * - vectorized::DataTypeDecimal::get_scale_multiplier(scale_to)); -} - -Decimal64Val CastFunctions::cast_to_decimal64_val(FunctionContext* context, const StringVal& val) { - if (val.is_null) { - return Decimal64Val::null(); - } - std::stringstream ss; - StringParser::ParseResult result; - int64_t v = StringParser::string_to_decimal((const char*)val.ptr, val.len, - context->get_return_type().precision, - context->get_return_type().scale, &result); - return Decimal64Val(v); -} - -Decimal64Val CastFunctions::cast_to_decimal64_val(FunctionContext* ctx, const Decimal64Val& val) { - if (ctx->get_arg_type(0)->scale == ctx->get_return_type().scale && - ctx->get_arg_type(0)->precision == ctx->get_return_type().precision) { - return val; - } - if (val.is_null) { - return Decimal64Val::null(); - } - auto scale_from = ctx->get_arg_type(0)->scale; - auto scale_to = ctx->get_return_type().scale; - if (scale_to > scale_from) { - return Decimal64Val( - val.val * vectorized::DataTypeDecimal::get_scale_multiplier( - scale_to - scale_from)); - } else { - return Decimal64Val( - val.val / vectorized::DataTypeDecimal::get_scale_multiplier( - scale_from - scale_to)); - } -} - -Decimal128Val CastFunctions::cast_to_decimal128_val(FunctionContext* context, - const DateTimeVal& val) { - if (val.is_null) { - return Decimal128Val::null(); - } - auto scale_to = context->get_return_type().scale; - DateTimeValue dt_value = DateTimeValue::from_datetime_val(val); - return Decimal128Val( - dt_value.to_int64() * - vectorized::DataTypeDecimal::get_scale_multiplier(scale_to)); -} - -Decimal128Val CastFunctions::cast_to_decimal128_val(FunctionContext* context, - const StringVal& val) { - if (val.is_null) { - return Decimal128Val::null(); - } - std::stringstream ss; - StringParser::ParseResult result; - int128_t v = StringParser::string_to_decimal( - (const char*)val.ptr, val.len, context->get_return_type().precision, - context->get_return_type().scale, &result); - return Decimal128Val(v); -} - -Decimal128Val CastFunctions::cast_to_decimal128_val(FunctionContext* ctx, - const Decimal128Val& val) { - if (ctx->get_arg_type(0)->scale == ctx->get_return_type().scale && - ctx->get_arg_type(0)->precision == ctx->get_return_type().precision) { - return val; - } - if (val.is_null) { - return Decimal128Val::null(); - } - auto scale_from = ctx->get_arg_type(0)->scale; - auto scale_to = ctx->get_return_type().scale; - if (scale_to > scale_from) { - return Decimal128Val( - val.val * vectorized::DataTypeDecimal::get_scale_multiplier( - scale_to - scale_from)); - } else { - return Decimal128Val( - val.val / vectorized::DataTypeDecimal::get_scale_multiplier( - scale_from - scale_to)); - } -} - -Decimal128Val CastFunctions::cast_to_decimal128_val(FunctionContext* ctx, const Decimal32Val& val) { - if (ctx->get_arg_type(0)->scale == ctx->get_return_type().scale && - ctx->get_arg_type(0)->precision == ctx->get_return_type().precision) { - return Decimal128Val(val.val); - } - if (val.is_null) { - return Decimal128Val::null(); - } - auto scale_from = ctx->get_arg_type(0)->scale; - auto scale_to = ctx->get_return_type().scale; - if (scale_to > scale_from) { - return Decimal128Val( - val.val * vectorized::DataTypeDecimal::get_scale_multiplier( - scale_to - scale_from)); - } else { - return Decimal128Val( - val.val / vectorized::DataTypeDecimal::get_scale_multiplier( - scale_from - scale_to)); - } -} - -Decimal128Val CastFunctions::cast_to_decimal128_val(FunctionContext* ctx, const Decimal64Val& val) { - if (ctx->get_arg_type(0)->scale == ctx->get_return_type().scale && - ctx->get_arg_type(0)->precision == ctx->get_return_type().precision) { - return Decimal128Val(val.val); - } - if (val.is_null) { - return Decimal128Val::null(); - } - auto scale_from = ctx->get_arg_type(0)->scale; - auto scale_to = ctx->get_return_type().scale; - if (scale_to > scale_from) { - return Decimal128Val( - val.val * vectorized::DataTypeDecimal::get_scale_multiplier( - scale_to - scale_from)); - } else { - return Decimal128Val( - val.val / vectorized::DataTypeDecimal::get_scale_multiplier( - scale_from - scale_to)); - } -} - -Decimal64Val CastFunctions::cast_to_decimal64_val(FunctionContext* ctx, const Decimal32Val& val) { - if (ctx->get_arg_type(0)->scale == ctx->get_return_type().scale && - ctx->get_arg_type(0)->precision == ctx->get_return_type().precision) { - return Decimal64Val(val.val); - } - if (val.is_null) { - return Decimal64Val::null(); - } - auto scale_from = ctx->get_arg_type(0)->scale; - auto scale_to = ctx->get_return_type().scale; - if (scale_to > scale_from) { - return Decimal64Val( - val.val * vectorized::DataTypeDecimal::get_scale_multiplier( - scale_to - scale_from)); - } else { - return Decimal64Val( - val.val / vectorized::DataTypeDecimal::get_scale_multiplier( - scale_from - scale_to)); - } -} - DateTimeVal CastFunctions::cast_to_date_val(FunctionContext* ctx, const DateV2Val& val) { if (val.is_null) { return DateTimeVal::null(); diff --git a/be/src/exprs/cast_functions.h b/be/src/exprs/cast_functions.h index 9281b132931469..df370fdba23562 100644 --- a/be/src/exprs/cast_functions.h +++ b/be/src/exprs/cast_functions.h @@ -185,28 +185,6 @@ class CastFunctions { const DateTimeVal& val); static doris_udf::DateTimeV2Val cast_to_datetimev2_val(FunctionContext* context, const doris_udf::DateTimeV2Val& val); - -#define DECLARE_CAST_TO_DECIMAL(width) \ - static Decimal##width##Val cast_to_decimal##width##_val(FunctionContext*, const TinyIntVal&); \ - static Decimal##width##Val cast_to_decimal##width##_val(FunctionContext*, const SmallIntVal&); \ - static Decimal##width##Val cast_to_decimal##width##_val(FunctionContext*, const IntVal&); \ - static Decimal##width##Val cast_to_decimal##width##_val(FunctionContext*, const BigIntVal&); \ - static Decimal##width##Val cast_to_decimal##width##_val(FunctionContext*, const LargeIntVal&); \ - static Decimal##width##Val cast_to_decimal##width##_val(FunctionContext*, const FloatVal&); \ - static Decimal##width##Val cast_to_decimal##width##_val(FunctionContext*, const DoubleVal&); \ - static Decimal##width##Val cast_to_decimal##width##_val(FunctionContext*, const DateTimeVal&); \ - static Decimal##width##Val cast_to_decimal##width##_val(FunctionContext*, const StringVal&); \ - static Decimal##width##Val cast_to_decimal##width##_val(FunctionContext*, \ - const Decimal##width##Val&); - - DECLARE_CAST_TO_DECIMAL(32) - DECLARE_CAST_TO_DECIMAL(64) - DECLARE_CAST_TO_DECIMAL(128) - - static Decimal64Val cast_to_decimal64_val(FunctionContext*, const Decimal32Val&); - static Decimal128Val cast_to_decimal128_val(FunctionContext*, const Decimal32Val&); - static Decimal128Val cast_to_decimal128_val(FunctionContext*, const Decimal64Val&); - static CollectionVal cast_to_array_val(FunctionContext* context, const StringVal& val); }; diff --git a/be/src/exprs/create_predicate_function.h b/be/src/exprs/create_predicate_function.h index 445df3a7e30c92..1f249044fb0b6f 100644 --- a/be/src/exprs/create_predicate_function.h +++ b/be/src/exprs/create_predicate_function.h @@ -108,8 +108,8 @@ typename Traits::BasePtr create_predicate_function(PrimitiveType type) { return Creator::template create(); case TYPE_DECIMAL64: return Creator::template create(); - case TYPE_DECIMAL128: - return Creator::template create(); + case TYPE_DECIMAL128I: + return Creator::template create(); default: DCHECK(false) << "Invalid type."; diff --git a/be/src/exprs/expr.cpp b/be/src/exprs/expr.cpp index c618f1bf29bc80..8605a12371509d 100644 --- a/be/src/exprs/expr.cpp +++ b/be/src/exprs/expr.cpp @@ -726,7 +726,7 @@ doris_udf::AnyVal* Expr::get_const_val(ExprContext* context) { _constant_val.reset(new Decimal64Val(get_decimal64_val(context, nullptr))); break; } - case TYPE_DECIMAL128: { + case TYPE_DECIMAL128I: { _constant_val.reset(new Decimal128Val(get_decimal128_val(context, nullptr))); break; } diff --git a/be/src/exprs/expr.h b/be/src/exprs/expr.h index f0c802b4a00d3e..448784d0f2a53c 100644 --- a/be/src/exprs/expr.h +++ b/be/src/exprs/expr.h @@ -562,7 +562,7 @@ Status create_texpr_literal_node(const void* data, TExprNode* node, int precisio decimal_literal.__set_value(ss.str()); (*node).__set_decimal_literal(decimal_literal); (*node).__set_type(create_type_desc(PrimitiveType::TYPE_DECIMAL64, precision, scale)); - } else if constexpr (T == TYPE_DECIMAL128) { + } else if constexpr (T == TYPE_DECIMAL128I) { auto origin_value = reinterpret_cast(data); (*node).__set_node_type(TExprNodeType::DECIMAL_LITERAL); TDecimalLiteral decimal_literal; @@ -570,7 +570,7 @@ Status create_texpr_literal_node(const void* data, TExprNode* node, int precisio vectorized::write_text(*origin_value, scale, ss); decimal_literal.__set_value(ss.str()); (*node).__set_decimal_literal(decimal_literal); - (*node).__set_type(create_type_desc(PrimitiveType::TYPE_DECIMAL128, precision, scale)); + (*node).__set_type(create_type_desc(PrimitiveType::TYPE_DECIMAL128I, precision, scale)); } else if constexpr (T == TYPE_FLOAT) { auto origin_value = reinterpret_cast(data); (*node).__set_node_type(TExprNodeType::FLOAT_LITERAL); diff --git a/be/src/exprs/expr_context.cpp b/be/src/exprs/expr_context.cpp index c37195a6bbb626..b019de99c01884 100644 --- a/be/src/exprs/expr_context.cpp +++ b/be/src/exprs/expr_context.cpp @@ -298,7 +298,7 @@ void* ExprContext::get_value(Expr* e, TupleRow* row, int precision, int scale) { _result.bigint_val = v.val; return &_result.bigint_val; } - case TYPE_DECIMAL128: { + case TYPE_DECIMAL128I: { doris_udf::Decimal128Val v = e->get_decimal128_val(this, row); if (v.is_null) { return nullptr; diff --git a/be/src/exprs/literal.cpp b/be/src/exprs/literal.cpp index 6d2d381c045146..28211abc2ac946 100644 --- a/be/src/exprs/literal.cpp +++ b/be/src/exprs/literal.cpp @@ -109,7 +109,7 @@ Literal::Literal(const TExprNode& node) : Expr(node) { } case TYPE_DECIMAL32: case TYPE_DECIMAL64: - case TYPE_DECIMAL128: { + case TYPE_DECIMAL128I: { DCHECK_EQ(node.node_type, TExprNodeType::DECIMAL_LITERAL); DCHECK(node.__isset.decimal_literal); _value.set_string_val(node.decimal_literal.value); @@ -183,7 +183,7 @@ Decimal64Val Literal::get_decimal64_val(ExprContext* context, TupleRow* row) { } Decimal128Val Literal::get_decimal128_val(ExprContext* context, TupleRow* row) { - DCHECK(_type.type == TYPE_DECIMAL128) << _type; + DCHECK(_type.type == TYPE_DECIMAL128I) << _type; StringParser::ParseResult result; auto decimal_value = StringParser::string_to_decimal( _value.string_val.ptr, _value.string_val.len, _type.precision, _type.scale, &result); diff --git a/be/src/exprs/runtime_filter.cpp b/be/src/exprs/runtime_filter.cpp index 9657d921465d63..f92869b5f4eeb4 100644 --- a/be/src/exprs/runtime_filter.cpp +++ b/be/src/exprs/runtime_filter.cpp @@ -73,7 +73,7 @@ TExprNodeType::type get_expr_node_type(PrimitiveType type) { case TYPE_DECIMAL32: case TYPE_DECIMAL64: - case TYPE_DECIMAL128: + case TYPE_DECIMAL128I: case TYPE_DECIMALV2: return TExprNodeType::DECIMAL_LITERAL; @@ -129,8 +129,8 @@ PColumnType to_proto(PrimitiveType type) { return PColumnType::COLUMN_TYPE_DECIMAL32; case TYPE_DECIMAL64: return PColumnType::COLUMN_TYPE_DECIMAL64; - case TYPE_DECIMAL128: - return PColumnType::COLUMN_TYPE_DECIMAL128; + case TYPE_DECIMAL128I: + return PColumnType::COLUMN_TYPE_DECIMAL128I; case TYPE_CHAR: return PColumnType::COLUMN_TYPE_CHAR; case TYPE_VARCHAR: @@ -178,8 +178,8 @@ PrimitiveType to_primitive_type(PColumnType type) { return TYPE_DECIMAL32; case PColumnType::COLUMN_TYPE_DECIMAL64: return TYPE_DECIMAL64; - case PColumnType::COLUMN_TYPE_DECIMAL128: - return TYPE_DECIMAL128; + case PColumnType::COLUMN_TYPE_DECIMAL128I: + return TYPE_DECIMAL128I; case PColumnType::COLUMN_TYPE_VARCHAR: return TYPE_VARCHAR; case PColumnType::COLUMN_TYPE_CHAR: @@ -289,8 +289,8 @@ Status create_literal(ObjectPool* pool, const TypeDescriptor& type, const void* create_texpr_literal_node(data, &node, type.precision, type.scale); break; } - case TYPE_DECIMAL128: { - create_texpr_literal_node(data, &node, type.precision, type.scale); + case TYPE_DECIMAL128I: { + create_texpr_literal_node(data, &node, type.precision, type.scale); break; } case TYPE_CHAR: { @@ -841,7 +841,7 @@ class RuntimePredicateWrapper { }); break; } - case TYPE_DECIMAL128: { + case TYPE_DECIMAL128I: { batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column, ObjectPool* pool) { auto string_val = column.stringval(); @@ -974,7 +974,7 @@ class RuntimePredicateWrapper { int64_t max_val = minmax_filter->max_val().longval(); return _minmax_func->assign(&min_val, &max_val); } - case TYPE_DECIMAL128: { + case TYPE_DECIMAL128I: { auto min_string_val = minmax_filter->min_val().stringval(); auto max_string_val = minmax_filter->max_val().stringval(); StringParser::ParseResult result; @@ -1553,7 +1553,7 @@ void IRuntimeFilter::to_protobuf(PInFilter* filter) { }); return; } - case TYPE_DECIMAL128: { + case TYPE_DECIMAL128I: { batch_copy(filter, it, [](PColumnValue* column, const int128_t* value) { column->set_stringval(LargeIntValue::to_string(*value)); }); @@ -1661,7 +1661,7 @@ void IRuntimeFilter::to_protobuf(PMinMaxFilter* filter) { filter->mutable_max_val()->set_longval(*reinterpret_cast(max_data)); return; } - case TYPE_DECIMAL128: { + case TYPE_DECIMAL128I: { filter->mutable_min_val()->set_stringval( LargeIntValue::to_string(*reinterpret_cast(min_data))); filter->mutable_max_val()->set_stringval( diff --git a/be/src/exprs/scalar_fn_call.cpp b/be/src/exprs/scalar_fn_call.cpp index 3eae4e6f4149e3..7212b619bb9562 100644 --- a/be/src/exprs/scalar_fn_call.cpp +++ b/be/src/exprs/scalar_fn_call.cpp @@ -516,7 +516,7 @@ Decimal64Val ScalarFnCall::get_decimal64_val(ExprContext* context, TupleRow* row } Decimal128Val ScalarFnCall::get_decimal128_val(ExprContext* context, TupleRow* row) { - DCHECK_EQ(_type.type, TYPE_DECIMAL128); + DCHECK_EQ(_type.type, TYPE_DECIMAL128I); DCHECK(context != nullptr); if (_scalar_fn_wrapper == nullptr) { return interpret_eval(context, row); diff --git a/be/src/exprs/slot_ref.cpp b/be/src/exprs/slot_ref.cpp index ecc749404d2c5b..ce4c3319e7abe3 100644 --- a/be/src/exprs/slot_ref.cpp +++ b/be/src/exprs/slot_ref.cpp @@ -284,7 +284,7 @@ Decimal64Val SlotRef::get_decimal64_val(ExprContext* context, TupleRow* row) { } Decimal128Val SlotRef::get_decimal128_val(ExprContext* context, TupleRow* row) { - DCHECK_EQ(_type.type, TYPE_DECIMAL128); + DCHECK_EQ(_type.type, TYPE_DECIMAL128I); Tuple* t = row->get_tuple(_tuple_idx); if (t == nullptr || t->is_null(_null_indicator_offset)) { return Decimal128Val::null(); diff --git a/be/src/olap/aggregate_func.cpp b/be/src/olap/aggregate_func.cpp index d7e7b7e82c2daf..027105952c1cbc 100644 --- a/be/src/olap/aggregate_func.cpp +++ b/be/src/olap/aggregate_func.cpp @@ -100,7 +100,7 @@ AggregateFuncResolver::AggregateFuncResolver() { add_aggregate_mapping(); add_aggregate_mapping(); add_aggregate_mapping(); - add_aggregate_mapping(); + add_aggregate_mapping(); add_aggregate_mapping(); add_aggregate_mapping(); add_aggregate_mapping(); @@ -144,7 +144,7 @@ AggregateFuncResolver::AggregateFuncResolver() { add_aggregate_mapping(); add_aggregate_mapping(); + OLAP_FIELD_TYPE_DECIMAL128I>(); add_aggregate_mapping(); @@ -159,7 +159,7 @@ AggregateFuncResolver::AggregateFuncResolver() { add_aggregate_mapping(); add_aggregate_mapping(); add_aggregate_mapping(); - add_aggregate_mapping(); + add_aggregate_mapping(); add_aggregate_mapping(); add_aggregate_mapping(); add_aggregate_mapping(); @@ -180,7 +180,7 @@ AggregateFuncResolver::AggregateFuncResolver() { add_aggregate_mapping(); add_aggregate_mapping(); add_aggregate_mapping(); - add_aggregate_mapping(); + add_aggregate_mapping(); add_aggregate_mapping(); add_aggregate_mapping(); add_aggregate_mapping(); @@ -201,7 +201,7 @@ AggregateFuncResolver::AggregateFuncResolver() { add_aggregate_mapping(); add_aggregate_mapping(); add_aggregate_mapping(); - add_aggregate_mapping(); + add_aggregate_mapping(); // Replace Aggregate Function add_aggregate_mapping(); @@ -215,7 +215,7 @@ AggregateFuncResolver::AggregateFuncResolver() { add_aggregate_mapping(); add_aggregate_mapping(); add_aggregate_mapping(); - add_aggregate_mapping(); + add_aggregate_mapping(); add_aggregate_mapping(); add_aggregate_mapping(); add_aggregate_mapping(); @@ -237,7 +237,8 @@ AggregateFuncResolver::AggregateFuncResolver() { add_aggregate_mapping(); add_aggregate_mapping(); add_aggregate_mapping(); - add_aggregate_mapping(); + add_aggregate_mapping(); add_aggregate_mapping(); add_aggregate_mapping(); add_aggregate_mapping(); diff --git a/be/src/olap/column_vector.cpp b/be/src/olap/column_vector.cpp index cee5d69b4b5b48..8211bd9d1a592f 100644 --- a/be/src/olap/column_vector.cpp +++ b/be/src/olap/column_vector.cpp @@ -97,10 +97,10 @@ Status ColumnVectorBatch::create(size_t init_capacity, bool is_nullable, const T new ScalarColumnVectorBatch::CppType>( type_info, is_nullable)); break; - case OLAP_FIELD_TYPE_DECIMAL128: - local.reset( - new ScalarColumnVectorBatch::CppType>( - type_info, is_nullable)); + case OLAP_FIELD_TYPE_DECIMAL128I: + local.reset(new ScalarColumnVectorBatch< + CppTypeTraits::CppType>(type_info, + is_nullable)); break; case OLAP_FIELD_TYPE_DATE: local.reset(new ScalarColumnVectorBatch::CppType>( diff --git a/be/src/olap/delete_handler.cpp b/be/src/olap/delete_handler.cpp index bbb07353564264..017cef5ecc500d 100644 --- a/be/src/olap/delete_handler.cpp +++ b/be/src/olap/delete_handler.cpp @@ -143,7 +143,7 @@ bool DeleteHandler::is_condition_value_valid(const TabletColumn& column, return valid_decimal(value_str, column.precision(), column.frac()); case OLAP_FIELD_TYPE_DECIMAL64: return valid_decimal(value_str, column.precision(), column.frac()); - case OLAP_FIELD_TYPE_DECIMAL128: + case OLAP_FIELD_TYPE_DECIMAL128I: return valid_decimal(value_str, column.precision(), column.frac()); case OLAP_FIELD_TYPE_CHAR: case OLAP_FIELD_TYPE_VARCHAR: diff --git a/be/src/olap/field.h b/be/src/olap/field.h index ea5a33d91a731f..391f7f45be68c9 100644 --- a/be/src/olap/field.h +++ b/be/src/olap/field.h @@ -758,7 +758,7 @@ class FieldFactory { [[fallthrough]]; case OLAP_FIELD_TYPE_DECIMAL64: [[fallthrough]]; - case OLAP_FIELD_TYPE_DECIMAL128: + case OLAP_FIELD_TYPE_DECIMAL128I: [[fallthrough]]; case OLAP_FIELD_TYPE_DATETIMEV2: { Field* field = new Field(column); @@ -798,7 +798,7 @@ class FieldFactory { [[fallthrough]]; case OLAP_FIELD_TYPE_DECIMAL64: [[fallthrough]]; - case OLAP_FIELD_TYPE_DECIMAL128: + case OLAP_FIELD_TYPE_DECIMAL128I: [[fallthrough]]; case OLAP_FIELD_TYPE_DATETIMEV2: { Field* field = new Field(column); diff --git a/be/src/olap/in_list_predicate.h b/be/src/olap/in_list_predicate.h index a4b608f16041c6..cac644040fb087 100644 --- a/be/src/olap/in_list_predicate.h +++ b/be/src/olap/in_list_predicate.h @@ -99,7 +99,7 @@ class InListPredicateBase : public ColumnPredicate { if constexpr (Type == TYPE_STRING || Type == TYPE_CHAR) { tmp = convert(*col, condition, pool); } else if constexpr (Type == TYPE_DECIMAL32 || Type == TYPE_DECIMAL64 || - Type == TYPE_DECIMAL128) { + Type == TYPE_DECIMAL128I) { tmp = convert(*col, condition); } else { tmp = convert(condition); diff --git a/be/src/olap/key_coder.cpp b/be/src/olap/key_coder.cpp index d4fa6bd6c7f239..9f1c2a2788e2b3 100644 --- a/be/src/olap/key_coder.cpp +++ b/be/src/olap/key_coder.cpp @@ -77,7 +77,7 @@ class KeyCoderResolver { add_mapping(); add_mapping(); add_mapping(); - add_mapping(); + add_mapping(); } template diff --git a/be/src/olap/olap_common.h b/be/src/olap/olap_common.h index e16aa8e5b4f344..7a396a188edefc 100644 --- a/be/src/olap/olap_common.h +++ b/be/src/olap/olap_common.h @@ -151,7 +151,7 @@ enum FieldType { OLAP_FIELD_TYPE_TIMEV2 = 30, OLAP_FIELD_TYPE_DECIMAL32 = 31, OLAP_FIELD_TYPE_DECIMAL64 = 32, - OLAP_FIELD_TYPE_DECIMAL128 = 33, + OLAP_FIELD_TYPE_DECIMAL128I = 33, OLAP_FIELD_TYPE_JSONB = 34, }; diff --git a/be/src/olap/predicate_creator.h b/be/src/olap/predicate_creator.h index 3e39b7cbc1ed38..2d3916529c4c79 100644 --- a/be/src/olap/predicate_creator.h +++ b/be/src/olap/predicate_creator.h @@ -174,8 +174,8 @@ inline std::unique_ptr> get_creator(const FieldT case OLAP_FIELD_TYPE_DECIMAL64: { return std::make_unique>(); } - case OLAP_FIELD_TYPE_DECIMAL128: { - return std::make_unique>(); + case OLAP_FIELD_TYPE_DECIMAL128I: { + return std::make_unique>(); } case OLAP_FIELD_TYPE_CHAR: { return std::make_unique>(); @@ -308,7 +308,7 @@ inline ColumnPredicate* parse_to_predicate(TabletSchemaSPtr tablet_schema, M(TYPE_STRING) \ M(TYPE_DECIMAL32) \ M(TYPE_DECIMAL64) \ - M(TYPE_DECIMAL128) + M(TYPE_DECIMAL128I) template inline ColumnPredicate* create_olap_column_predicate( diff --git a/be/src/olap/row_block2.cpp b/be/src/olap/row_block2.cpp index 0e7f5cd4277312..6d68090a402c44 100644 --- a/be/src/olap/row_block2.cpp +++ b/be/src/olap/row_block2.cpp @@ -321,9 +321,9 @@ Status RowBlockV2::_copy_data_to_column(int cid, insert_data_directly(cid, column_decimal); break; } - case OLAP_FIELD_TYPE_DECIMAL128: { + case OLAP_FIELD_TYPE_DECIMAL128I: { auto column_decimal = - assert_cast*>(column); + assert_cast*>(column); insert_data_directly(cid, column_decimal); break; } @@ -601,7 +601,7 @@ Status RowBlockV2::_append_data_to_column(const ColumnVectorBatch* batch, size_t } case OLAP_FIELD_TYPE_DECIMAL: { auto column_decimal = - assert_cast*>(column); + assert_cast*>(column); for (uint32_t j = 0; j < selected_size; ++j) { if (!nullable_mark_array[j]) { @@ -628,9 +628,9 @@ Status RowBlockV2::_append_data_to_column(const ColumnVectorBatch* batch, size_t assert_cast*>(column); insert_data_directly(batch, column_decimal, start, len); } - case OLAP_FIELD_TYPE_DECIMAL128: { + case OLAP_FIELD_TYPE_DECIMAL128I: { auto column_decimal = - assert_cast*>(column); + assert_cast*>(column); insert_data_directly(batch, column_decimal, start, len); } case OLAP_FIELD_TYPE_ARRAY: { diff --git a/be/src/olap/rowset/segment_v2/bitmap_index_writer.cpp b/be/src/olap/rowset/segment_v2/bitmap_index_writer.cpp index f8bc37a1278c8a..7e37593b606fbd 100644 --- a/be/src/olap/rowset/segment_v2/bitmap_index_writer.cpp +++ b/be/src/olap/rowset/segment_v2/bitmap_index_writer.cpp @@ -238,8 +238,8 @@ Status BitmapIndexWriter::create(const TypeInfo* type_info, case OLAP_FIELD_TYPE_DECIMAL64: res->reset(new BitmapIndexWriterImpl(type_info)); break; - case OLAP_FIELD_TYPE_DECIMAL128: - res->reset(new BitmapIndexWriterImpl(type_info)); + case OLAP_FIELD_TYPE_DECIMAL128I: + res->reset(new BitmapIndexWriterImpl(type_info)); break; case OLAP_FIELD_TYPE_BOOL: res->reset(new BitmapIndexWriterImpl(type_info)); diff --git a/be/src/olap/rowset/segment_v2/encoding_info.cpp b/be/src/olap/rowset/segment_v2/encoding_info.cpp index d74cec5bb0ed39..8f5ec0f8181ecf 100644 --- a/be/src/olap/rowset/segment_v2/encoding_info.cpp +++ b/be/src/olap/rowset/segment_v2/encoding_info.cpp @@ -324,9 +324,9 @@ EncodingInfoResolver::EncodingInfoResolver() { _add_map(); _add_map(); - _add_map(); - _add_map(); - _add_map(); + _add_map(); + _add_map(); + _add_map(); _add_map(); diff --git a/be/src/olap/rowset/segment_v2/zone_map_index.cpp b/be/src/olap/rowset/segment_v2/zone_map_index.cpp index 7b6d287836414a..c6ed824b4d4c25 100644 --- a/be/src/olap/rowset/segment_v2/zone_map_index.cpp +++ b/be/src/olap/rowset/segment_v2/zone_map_index.cpp @@ -181,7 +181,7 @@ Status ZoneMapIndexReader::load(bool use_page_cache, bool kept_in_memory) { M(TYPE_STRING) \ M(TYPE_DECIMAL32) \ M(TYPE_DECIMAL64) \ - M(TYPE_DECIMAL128) + M(TYPE_DECIMAL128I) Status ZoneMapIndexWriter::create(Field* field, std::unique_ptr& res) { switch (field->type()) { diff --git a/be/src/olap/schema.cpp b/be/src/olap/schema.cpp index 01f03433179fc8..c8c27c42697053 100644 --- a/be/src/olap/schema.cpp +++ b/be/src/olap/schema.cpp @@ -179,8 +179,8 @@ vectorized::IColumn::MutablePtr Schema::get_predicate_column_ptr(FieldType type) return doris::vectorized::PredicateColumnType::create(); case OLAP_FIELD_TYPE_DECIMAL64: return doris::vectorized::PredicateColumnType::create(); - case OLAP_FIELD_TYPE_DECIMAL128: - return doris::vectorized::PredicateColumnType::create(); + case OLAP_FIELD_TYPE_DECIMAL128I: + return doris::vectorized::PredicateColumnType::create(); default: LOG(FATAL) << "Unexpected type when choosing predicate column, type=" << type; diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp index fe4b353e5ad922..d9c46888c133fe 100644 --- a/be/src/olap/tablet_schema.cpp +++ b/be/src/olap/tablet_schema.cpp @@ -74,8 +74,8 @@ FieldType TabletColumn::get_field_type_by_string(const std::string& type_str) { type = OLAP_FIELD_TYPE_DECIMAL32; } else if (0 == upper_type_str.compare("DECIMAL64")) { type = OLAP_FIELD_TYPE_DECIMAL64; - } else if (0 == upper_type_str.compare("DECIMAL128")) { - type = OLAP_FIELD_TYPE_DECIMAL128; + } else if (0 == upper_type_str.compare("DECIMAL128I")) { + type = OLAP_FIELD_TYPE_DECIMAL128I; } else if (0 == upper_type_str.compare(0, 7, "DECIMAL")) { type = OLAP_FIELD_TYPE_DECIMAL; } else if (0 == upper_type_str.compare(0, 7, "VARCHAR")) { @@ -202,8 +202,8 @@ std::string TabletColumn::get_string_by_field_type(FieldType type) { case OLAP_FIELD_TYPE_DECIMAL64: return "DECIMAL64"; - case OLAP_FIELD_TYPE_DECIMAL128: - return "DECIMAL128"; + case OLAP_FIELD_TYPE_DECIMAL128I: + return "DECIMAL128I"; case OLAP_FIELD_TYPE_VARCHAR: return "VARCHAR"; @@ -316,7 +316,7 @@ uint32_t TabletColumn::get_field_length_by_type(TPrimitiveType::type type, uint3 return 4; case TPrimitiveType::DECIMAL64: return 8; - case TPrimitiveType::DECIMAL128: + case TPrimitiveType::DECIMAL128I: return 16; case TPrimitiveType::DECIMALV2: return 12; // use 12 bytes in olap engine. diff --git a/be/src/olap/types.cpp b/be/src/olap/types.cpp index e92510a1d72e2a..aa2226a84ac0db 100644 --- a/be/src/olap/types.cpp +++ b/be/src/olap/types.cpp @@ -89,7 +89,7 @@ const TypeInfo* get_scalar_type_info(FieldType field_type) { get_scalar_type_info(), get_scalar_type_info(), get_scalar_type_info(), - get_scalar_type_info(), + get_scalar_type_info(), get_scalar_type_info(), }; return field_type_array[field_type]; @@ -160,7 +160,7 @@ const TypeInfo* get_array_type_info(FieldType leaf_type, int32_t iterations) { INIT_ARRAY_TYPE_INFO_LIST(OLAP_FIELD_TYPE_TIMEV2), INIT_ARRAY_TYPE_INFO_LIST(OLAP_FIELD_TYPE_DECIMAL32), INIT_ARRAY_TYPE_INFO_LIST(OLAP_FIELD_TYPE_DECIMAL64), - INIT_ARRAY_TYPE_INFO_LIST(OLAP_FIELD_TYPE_DECIMAL128), + INIT_ARRAY_TYPE_INFO_LIST(OLAP_FIELD_TYPE_DECIMAL128I), INIT_ARRAY_TYPE_INFO_LIST(OLAP_FIELD_TYPE_JSONB), }; return array_type_Info_arr[leaf_type][iterations]; diff --git a/be/src/olap/types.h b/be/src/olap/types.h index 8754e610c6a46d..3c784d8804fda1 100644 --- a/be/src/olap/types.h +++ b/be/src/olap/types.h @@ -513,7 +513,7 @@ struct CppTypeTraits { using UnsignedCppType = uint64_t; }; template <> -struct CppTypeTraits { +struct CppTypeTraits { using CppType = int128_t; using UnsignedCppType = uint128_t; }; @@ -991,8 +991,8 @@ struct FieldTypeTraits }; template <> -struct FieldTypeTraits - : public BaseFieldtypeTraits { +struct FieldTypeTraits + : public BaseFieldtypeTraits { static Status from_string(void* buf, const std::string& scan_key, const int precision, const int scale) { StringParser::ParseResult result = StringParser::PARSE_SUCCESS; diff --git a/be/src/runtime/define_primitive_type.h b/be/src/runtime/define_primitive_type.h index 83cb0b81a52c43..5a87112398543d 100644 --- a/be/src/runtime/define_primitive_type.h +++ b/be/src/runtime/define_primitive_type.h @@ -52,7 +52,7 @@ enum PrimitiveType { TYPE_TIMEV2, /* 27 */ TYPE_DECIMAL32, /* 28 */ TYPE_DECIMAL64, /* 29 */ - TYPE_DECIMAL128, /* 30 */ + TYPE_DECIMAL128I, /* 30 */ TYPE_JSONB, /* 31 */ }; diff --git a/be/src/runtime/mysql_result_writer.cpp b/be/src/runtime/mysql_result_writer.cpp index 2d4e3541c9593c..823f05d2d4954f 100644 --- a/be/src/runtime/mysql_result_writer.cpp +++ b/be/src/runtime/mysql_result_writer.cpp @@ -187,7 +187,7 @@ int MysqlResultWriter::_add_row_value(int index, const TypeDescriptor& type, voi iter.get(&data); auto datetime_value = DateTimeValue::from_datetime_val(data); buf_ret = _add_row_value(index, child_type, &datetime_value); - } else if (child_type.is_decimal_type()) { + } else if (child_type.is_decimal_v2_type()) { DecimalV2Val data; iter.get(&data); auto decimal_value = DecimalV2Value::from_decimal_val(data); diff --git a/be/src/runtime/primitive_type.cpp b/be/src/runtime/primitive_type.cpp index 9c17ba8fec7dfe..779676a45d9780 100644 --- a/be/src/runtime/primitive_type.cpp +++ b/be/src/runtime/primitive_type.cpp @@ -47,8 +47,8 @@ PrimitiveType convert_type_to_primitive(FunctionContext::Type type) { return PrimitiveType::TYPE_DECIMAL32; case FunctionContext::Type::TYPE_DECIMAL64: return PrimitiveType::TYPE_DECIMAL64; - case FunctionContext::Type::TYPE_DECIMAL128: - return PrimitiveType::TYPE_DECIMAL128; + case FunctionContext::Type::TYPE_DECIMAL128I: + return PrimitiveType::TYPE_DECIMAL128I; case FunctionContext::Type::TYPE_BOOLEAN: return PrimitiveType::TYPE_BOOLEAN; case FunctionContext::Type::TYPE_ARRAY: @@ -121,7 +121,7 @@ int get_byte_size(PrimitiveType type) { case TYPE_DATE: case TYPE_LARGEINT: case TYPE_DECIMALV2: - case TYPE_DECIMAL128: + case TYPE_DECIMAL128I: return 16; case INVALID_TYPE: @@ -244,8 +244,8 @@ PrimitiveType thrift_to_type(TPrimitiveType::type ttype) { case TPrimitiveType::DECIMAL64: return TYPE_DECIMAL64; - case TPrimitiveType::DECIMAL128: - return TYPE_DECIMAL128; + case TPrimitiveType::DECIMAL128I: + return TYPE_DECIMAL128I; case TPrimitiveType::CHAR: return TYPE_CHAR; @@ -338,8 +338,8 @@ TPrimitiveType::type to_thrift(PrimitiveType ptype) { case TYPE_DECIMAL64: return TPrimitiveType::DECIMAL64; - case TYPE_DECIMAL128: - return TPrimitiveType::DECIMAL128; + case TYPE_DECIMAL128I: + return TPrimitiveType::DECIMAL128I; case TYPE_CHAR: return TPrimitiveType::CHAR; @@ -432,8 +432,8 @@ std::string type_to_string(PrimitiveType t) { case TYPE_DECIMAL64: return "DECIMAL64"; - case TYPE_DECIMAL128: - return "DECIMAL128"; + case TYPE_DECIMAL128I: + return "DECIMAL128I"; case TYPE_CHAR: return "CHAR"; @@ -527,7 +527,7 @@ std::string type_to_odbc_string(PrimitiveType t) { case TYPE_DECIMAL64: return "decimal64"; - case TYPE_DECIMAL128: + case TYPE_DECIMAL128I: return "decimal128"; case TYPE_CHAR: @@ -621,7 +621,7 @@ int get_slot_size(PrimitiveType type) { return sizeof(DateTimeValue); case TYPE_DECIMALV2: - case TYPE_DECIMAL128: + case TYPE_DECIMAL128I: return 16; case INVALID_TYPE: diff --git a/be/src/runtime/primitive_type.h b/be/src/runtime/primitive_type.h index 104c738bac2eee..f785514696b312 100644 --- a/be/src/runtime/primitive_type.h +++ b/be/src/runtime/primitive_type.h @@ -52,7 +52,7 @@ constexpr bool is_enumeration_type(PrimitiveType type) { case TYPE_DECIMALV2: case TYPE_DECIMAL32: case TYPE_DECIMAL64: - case TYPE_DECIMAL128: + case TYPE_DECIMAL128I: case TYPE_BOOLEAN: case TYPE_ARRAY: case TYPE_HLL: @@ -197,9 +197,9 @@ struct PrimitiveTypeTraits { using ColumnType = vectorized::ColumnDecimal; }; template <> -struct PrimitiveTypeTraits { +struct PrimitiveTypeTraits { using CppType = __int128_t; - using ColumnType = vectorized::ColumnDecimal; + using ColumnType = vectorized::ColumnDecimal; }; template <> struct PrimitiveTypeTraits { diff --git a/be/src/runtime/raw_value.cpp b/be/src/runtime/raw_value.cpp index fe918611cc239e..67e70d4be2f9ad 100644 --- a/be/src/runtime/raw_value.cpp +++ b/be/src/runtime/raw_value.cpp @@ -109,7 +109,7 @@ void RawValue::print_value_as_bytes(const void* value, const TypeDescriptor& typ stream->write(chars, 8); break; - case TYPE_DECIMAL128: + case TYPE_DECIMAL128I: stream->write(chars, 16); break; @@ -214,8 +214,8 @@ void RawValue::print_value(const void* value, const TypeDescriptor& type, int sc break; } - case TYPE_DECIMAL128: { - auto decimal_val = reinterpret_cast(value); + case TYPE_DECIMAL128I: { + auto decimal_val = reinterpret_cast(value); write_text(*decimal_val, type.scale, *stream); break; } @@ -248,7 +248,7 @@ void RawValue::print_value(const void* value, const TypeDescriptor& type, int sc iter.get(&data); auto datetime_value = DateTimeValue::from_datetime_val(data); print_value(&datetime_value, child_type, scale, stream); - } else if (child_type.is_decimal_type()) { + } else if (child_type.is_decimal_v2_type()) { DecimalV2Val data; iter.get(&data); auto decimal_value = DecimalV2Value::from_decimal_val(data); @@ -405,9 +405,9 @@ void RawValue::write(const void* value, void* dst, const TypeDescriptor& type, M *reinterpret_cast(dst) = *reinterpret_cast(value); break; - case TYPE_DECIMAL128: - *reinterpret_cast(dst) = - *reinterpret_cast(value); + case TYPE_DECIMAL128I: + *reinterpret_cast(dst) = + *reinterpret_cast(value); break; case TYPE_OBJECT: @@ -529,9 +529,9 @@ void RawValue::write(const void* value, const TypeDescriptor& type, void* dst, u *reinterpret_cast(dst) = *reinterpret_cast(value); break; - case TYPE_DECIMAL128: - *reinterpret_cast(dst) = - *reinterpret_cast(value); + case TYPE_DECIMAL128I: + *reinterpret_cast(dst) = + *reinterpret_cast(value); break; default: @@ -654,7 +654,7 @@ int RawValue::compare(const void* v1, const void* v2, const TypeDescriptor& type return b1 > b2 ? 1 : (b1 < b2 ? -1 : 0); } - case TYPE_DECIMAL128: { + case TYPE_DECIMAL128I: { __int128 large_int_value1 = reinterpret_cast(v1)->value; __int128 large_int_value2 = reinterpret_cast(v2)->value; return large_int_value1 > large_int_value2 ? 1 diff --git a/be/src/runtime/raw_value.h b/be/src/runtime/raw_value.h index 990e8b90155702..70d12c60d53add 100644 --- a/be/src/runtime/raw_value.h +++ b/be/src/runtime/raw_value.h @@ -181,7 +181,7 @@ inline bool RawValue::lt(const void* v1, const void* v2, const TypeDescriptor& t return *reinterpret_cast(v1) < *reinterpret_cast(v2); case TYPE_DECIMAL64: return *reinterpret_cast(v1) < *reinterpret_cast(v2); - case TYPE_DECIMAL128: + case TYPE_DECIMAL128I: return reinterpret_cast(v1)->value < reinterpret_cast(v2)->value; @@ -254,7 +254,7 @@ inline bool RawValue::eq(const void* v1, const void* v2, const TypeDescriptor& t return *reinterpret_cast(v1) == *reinterpret_cast(v2); case TYPE_DECIMAL64: return *reinterpret_cast(v1) == *reinterpret_cast(v2); - case TYPE_DECIMAL128: + case TYPE_DECIMAL128I: return reinterpret_cast(v1)->value == reinterpret_cast(v2)->value; @@ -327,7 +327,7 @@ inline uint32_t RawValue::get_hash_value(const void* v, const PrimitiveType& typ return HashUtil::hash(v, 4, seed); case TYPE_DECIMAL64: return HashUtil::hash(v, 8, seed); - case TYPE_DECIMAL128: + case TYPE_DECIMAL128I: return HashUtil::hash(v, 16, seed); case TYPE_LARGEINT: @@ -396,7 +396,7 @@ inline uint32_t RawValue::get_hash_value_fvn(const void* v, const PrimitiveType& return HashUtil::fnv_hash(v, 4, seed); case TYPE_DECIMAL64: return HashUtil::fnv_hash(v, 8, seed); - case TYPE_DECIMAL128: + case TYPE_DECIMAL128I: return HashUtil::fnv_hash(v, 16, seed); case TYPE_LARGEINT: @@ -450,7 +450,7 @@ inline uint32_t RawValue::zlib_crc32(const void* v, const TypeDescriptor& type, case TYPE_DECIMAL64: return HashUtil::zlib_crc_hash(v, 8, seed); case TYPE_LARGEINT: - case TYPE_DECIMAL128: + case TYPE_DECIMAL128I: return HashUtil::zlib_crc_hash(v, 16, seed); case TYPE_FLOAT: return HashUtil::zlib_crc_hash(v, 4, seed); @@ -544,7 +544,7 @@ inline uint32_t RawValue::zlib_crc32(const void* v, size_t len, const TypeDescri return HashUtil::zlib_crc_hash(v, 4, seed); case TYPE_DECIMAL64: return HashUtil::zlib_crc_hash(v, 8, seed); - case TYPE_DECIMAL128: + case TYPE_DECIMAL128I: return HashUtil::zlib_crc_hash(v, 16, seed); default: DCHECK(false) << "invalid type: " << type; diff --git a/be/src/runtime/types.cpp b/be/src/runtime/types.cpp index aa38f35542f996..37f49335698a6b 100644 --- a/be/src/runtime/types.cpp +++ b/be/src/runtime/types.cpp @@ -38,7 +38,7 @@ TypeDescriptor::TypeDescriptor(const std::vector& types, int* idx) DCHECK(scalar_type.__isset.len); len = scalar_type.len; } else if (type == TYPE_DECIMALV2 || type == TYPE_DECIMAL32 || type == TYPE_DECIMAL64 || - type == TYPE_DECIMAL128 || type == TYPE_DATETIMEV2) { + type == TYPE_DECIMAL128I || type == TYPE_DATETIMEV2) { DCHECK(scalar_type.__isset.precision); DCHECK(scalar_type.__isset.scale); precision = scalar_type.precision; @@ -121,7 +121,7 @@ void TypeDescriptor::to_thrift(TTypeDesc* thrift_type) const { // DCHECK_NE(len, -1); scalar_type.__set_len(len); } else if (type == TYPE_DECIMALV2 || type == TYPE_DECIMAL32 || type == TYPE_DECIMAL64 || - type == TYPE_DECIMAL128 || type == TYPE_DATETIMEV2) { + type == TYPE_DECIMAL128I || type == TYPE_DATETIMEV2) { DCHECK_NE(precision, -1); DCHECK_NE(scale, -1); scalar_type.__set_precision(precision); @@ -140,7 +140,7 @@ void TypeDescriptor::to_protobuf(PTypeDesc* ptype) const { if (type == TYPE_CHAR || type == TYPE_VARCHAR || type == TYPE_HLL || type == TYPE_STRING) { scalar_type->set_len(len); } else if (type == TYPE_DECIMALV2 || type == TYPE_DECIMAL32 || type == TYPE_DECIMAL64 || - type == TYPE_DECIMAL128 || type == TYPE_DATETIMEV2) { + type == TYPE_DECIMAL128I || type == TYPE_DATETIMEV2) { DCHECK_NE(precision, -1); DCHECK_NE(scale, -1); scalar_type->set_precision(precision); @@ -168,7 +168,7 @@ TypeDescriptor::TypeDescriptor(const google::protobuf::RepeatedPtrField(type_scale - scale); - return is_negative ? -value : value; + if constexpr (std::is_same_v) { + value *= get_scale_multiplier<__int128>(type_scale - scale); + } else { + value *= get_scale_multiplier(type_scale - scale); + } + + return is_negative ? T(-value) : T(value); } } @@ -714,7 +719,12 @@ inline T StringParser::string_to_decimal(const char* s, int len, int type_precis // Ex: 0.1e3 (which at this point would have precision == 1 and scale == 1), the // scale must be set to 0 and the value set to 100 which means a precision of 3. precision += exponent - scale; - value *= get_scale_multiplier(exponent - scale); + + if constexpr (std::is_same_v) { + value *= get_scale_multiplier<__int128>(exponent - scale); + } else { + value *= get_scale_multiplier(exponent - scale); + } scale = 0; } else { // Ex: 100e-4, the scale must be set to 4 but no adjustment to the value is needed, @@ -740,11 +750,16 @@ inline T StringParser::string_to_decimal(const char* s, int len, int type_precis shift -= truncated_digit_count; } if (shift > 0) { - T divisor = get_scale_multiplier(shift); + T divisor; + if constexpr (std::is_same_v) { + divisor = get_scale_multiplier<__int128>(shift); + } else { + divisor = get_scale_multiplier(shift); + } if (LIKELY(divisor >= 0)) { value /= divisor; T remainder = value % divisor; - if ((remainder > 0 ? remainder : -remainder) >= (divisor >> 1)) { + if ((remainder > 0 ? T(remainder) : T(-remainder)) >= (divisor >> 1)) { value += 1; } } else { @@ -758,10 +773,14 @@ inline T StringParser::string_to_decimal(const char* s, int len, int type_precis } if (type_scale > scale) { - value *= get_scale_multiplier(type_scale - scale); + if constexpr (std::is_same_v) { + value *= get_scale_multiplier<__int128>(type_scale - scale); + } else { + value *= get_scale_multiplier(type_scale - scale); + } } - return is_negative ? -value : value; + return is_negative ? T(-value) : T(value); } } // end namespace doris diff --git a/be/src/util/symbols_util.cpp b/be/src/util/symbols_util.cpp index 06df655082e9a0..7ccd0793b6b8b2 100644 --- a/be/src/util/symbols_util.cpp +++ b/be/src/util/symbols_util.cpp @@ -176,7 +176,7 @@ static void append_any_val_type(int namespace_id, const TypeDescriptor& type, case TYPE_DECIMAL64: append_mangled_token("Decimal64Val", s); break; - case TYPE_DECIMAL128: + case TYPE_DECIMAL128I: append_mangled_token("Decimal128Val", s); break; default: diff --git a/be/src/vec/aggregate_functions/aggregate_function_avg.cpp b/be/src/vec/aggregate_functions/aggregate_function_avg.cpp index e4838279877827..5875b831f38746 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_avg.cpp +++ b/be/src/vec/aggregate_functions/aggregate_function_avg.cpp @@ -29,9 +29,7 @@ namespace doris::vectorized { template struct Avg { - using FieldType = std::conditional_t< - IsDecimalNumber, Decimal128, - std::conditional_t, double, NearestFieldType>>; + using FieldType = typename AvgNearestFieldTypeTrait::Type; using Function = AggregateFunctionAvg>; }; diff --git a/be/src/vec/aggregate_functions/aggregate_function_avg.h b/be/src/vec/aggregate_functions/aggregate_function_avg.h index c80c46b8f9497f..955dd5644f86c7 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_avg.h +++ b/be/src/vec/aggregate_functions/aggregate_function_avg.h @@ -53,17 +53,18 @@ struct AggregateFunctionAvgData { // null is handled in AggregationNode::_get_without_key_result return static_cast(sum); } - if (!config::enable_decimalv3) { - // to keep the same result with row vesion; see AggregateFunctions::decimalv2_avg_get_value - if constexpr (std::is_same_v && std::is_same_v) { - DecimalV2Value decimal_val_count(count, 0); - DecimalV2Value decimal_val_sum(static_cast(sum)); - DecimalV2Value cal_ret = decimal_val_sum / decimal_val_count; - Decimal128 ret(cal_ret.value()); - return ret; - } + // to keep the same result with row vesion; see AggregateFunctions::decimalv2_avg_get_value + if constexpr (IsDecimalV2 && IsDecimalV2) { + DecimalV2Value decimal_val_count(count, 0); + DecimalV2Value decimal_val_sum(static_cast(sum)); + DecimalV2Value cal_ret = decimal_val_sum / decimal_val_count; + Decimal128 ret(cal_ret.value()); + return ret; + } else if constexpr (IsDecimal128I && IsDecimal128I) { + return static_cast(sum).value.val / count; + } else { + return static_cast(sum) / count; } - return static_cast(sum) / count; } void write(BufferWritable& buf) const { @@ -82,12 +83,18 @@ template class AggregateFunctionAvg final : public IAggregateFunctionDataHelper> { public: - using ResultType = std::conditional_t, Decimal128, Float64>; - using ResultDataType = std::conditional_t, DataTypeDecimal, - DataTypeNumber>; + using ResultType = + std::conditional_t, Decimal128, + std::conditional_t, Decimal128I, Float64>>; + using ResultDataType = + std::conditional_t, DataTypeDecimal, + std::conditional_t, DataTypeDecimal, + DataTypeNumber>>; using ColVecType = std::conditional_t, ColumnDecimal, ColumnVector>; - using ColVecResult = std::conditional_t, ColumnDecimal, - ColumnVector>; + using ColVecResult = + std::conditional_t, ColumnDecimal, + std::conditional_t, ColumnDecimal, + ColumnVector>>; /// ctor for native types AggregateFunctionAvg(const DataTypes& argument_types_) @@ -114,7 +121,13 @@ class AggregateFunctionAvg final void add(AggregateDataPtr __restrict place, const IColumn** columns, size_t row_num, Arena*) const override { const auto& column = static_cast(*columns[0]); - this->data(place).sum += column.get_data()[row_num]; + if constexpr (IsDecimal128I && IsDecimal128I) { + this->data(place).sum.value.val += column.get_data()[row_num].value.val; + } else if constexpr (IsDecimal128I && IsDecimalNumber) { + this->data(place).sum.value.val += column.get_data()[row_num]; + } else { + this->data(place).sum += column.get_data()[row_num]; + } ++this->data(place).count; } diff --git a/be/src/vec/aggregate_functions/aggregate_function_avg_weighted.cpp b/be/src/vec/aggregate_functions/aggregate_function_avg_weighted.cpp index 310082431d0b85..d5d85cf11e3b82 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_avg_weighted.cpp +++ b/be/src/vec/aggregate_functions/aggregate_function_avg_weighted.cpp @@ -38,9 +38,12 @@ AggregateFunctionPtr create_aggregate_function_avg_weight(const std::string& nam return AggregateFunctionPtr(new AggregateFunctionAvgWeight(argument_types)); FOR_NUMERIC_TYPES(DISPATCH) #undef DISPATCH - if (which.is_decimal()) { + if (which.is_decimal128()) { return AggregateFunctionPtr(new AggregateFunctionAvgWeight(argument_types)); } + if (which.is_decimal()) { + return AggregateFunctionPtr(new AggregateFunctionAvgWeight(argument_types)); + } LOG(WARNING) << fmt::format("Illegal argument type for aggregate function topn_array is: {}", type->get_name()); diff --git a/be/src/vec/aggregate_functions/aggregate_function_avg_weighted.h b/be/src/vec/aggregate_functions/aggregate_function_avg_weighted.h index 0f652c01dba3b2..0d86055468a6df 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_avg_weighted.h +++ b/be/src/vec/aggregate_functions/aggregate_function_avg_weighted.h @@ -29,9 +29,11 @@ namespace doris::vectorized { template struct AggregateFunctionAvgWeightedData { void add(const T& data_val, double weight_val) { - if constexpr (IsDecimalNumber) { + if constexpr (IsDecimalV2) { DecimalV2Value value = binary_cast(data_val); data_sum = data_sum + (static_cast(value) * weight_val); + } else if constexpr (IsDecimal128I) { + data_sum = data_sum + (data_val.value.val * weight_val); } else { data_sum = data_sum + (data_val * weight_val); } diff --git a/be/src/vec/aggregate_functions/aggregate_function_collect.cpp b/be/src/vec/aggregate_functions/aggregate_function_collect.cpp index 34b7c7e7778936..b0e713c87f29bd 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_collect.cpp +++ b/be/src/vec/aggregate_functions/aggregate_function_collect.cpp @@ -70,6 +70,8 @@ AggregateFunctionPtr create_aggregate_function_collect(const std::string& name, return create_agg_function_collect(distinct, argument_types); } else if (type.is_decimal128()) { return create_agg_function_collect(distinct, argument_types); + } else if (type.is_decimal128i()) { + return create_agg_function_collect(distinct, argument_types); } else if (type.is_date()) { return create_agg_function_collect(distinct, argument_types); } else if (type.is_date_time()) { diff --git a/be/src/vec/aggregate_functions/aggregate_function_collect.h b/be/src/vec/aggregate_functions/aggregate_function_collect.h index e78fdfe08299be..f385aa30008245 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_collect.h +++ b/be/src/vec/aggregate_functions/aggregate_function_collect.h @@ -41,7 +41,11 @@ struct AggregateFunctionCollectSetData { void add(const IColumn& column, size_t row_num) { const auto& vec = assert_cast(column).get_data(); - set.insert(vec[row_num]); + if constexpr (IsDecimal128I) { + set.insert(vec[row_num].value); + } else { + set.insert(vec[row_num]); + } } void merge(const AggregateFunctionCollectSetData& rhs) { set.merge(rhs.set); } void write(BufferWritable& buf) const { set.write(buf); } diff --git a/be/src/vec/aggregate_functions/aggregate_function_min_max.cpp b/be/src/vec/aggregate_functions/aggregate_function_min_max.cpp index d4357367a435c5..83045dbd001d81 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_min_max.cpp +++ b/be/src/vec/aggregate_functions/aggregate_function_min_max.cpp @@ -70,6 +70,10 @@ static IAggregateFunction* create_aggregate_function_single_value(const String& return new AggregateFunctionTemplate>, false>( argument_type); } + if (which.idx == TypeIndex::Decimal128I) { + return new AggregateFunctionTemplate>, false>( + argument_type); + } return nullptr; } diff --git a/be/src/vec/aggregate_functions/aggregate_function_min_max.h b/be/src/vec/aggregate_functions/aggregate_function_min_max.h index 5bb86dd2712a90..08108526d492d8 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_min_max.h +++ b/be/src/vec/aggregate_functions/aggregate_function_min_max.h @@ -206,11 +206,23 @@ struct SingleValueDataDecimal { } bool change_if_less(const IColumn& column, size_t row_num, Arena* arena) { - if (!has() || assert_cast&>(column).get_data()[row_num] < value) { - change(column, row_num, arena); - return true; + if constexpr (IsDecimal128I) { + if (!has() || + (assert_cast&>(column).get_data()[row_num]).value.val < + value.val) { + change(column, row_num, arena); + return true; + } else { + return false; + } } else { - return false; + if (!has() || + assert_cast&>(column).get_data()[row_num] < value) { + change(column, row_num, arena); + return true; + } else { + return false; + } } } @@ -224,11 +236,23 @@ struct SingleValueDataDecimal { } bool change_if_greater(const IColumn& column, size_t row_num, Arena* arena) { - if (!has() || assert_cast&>(column).get_data()[row_num] > value) { - change(column, row_num, arena); - return true; + if constexpr (IsDecimal128I) { + if (!has() || + (assert_cast&>(column).get_data()[row_num]).value.val > + value.val) { + change(column, row_num, arena); + return true; + } else { + return false; + } } else { - return false; + if (!has() || + assert_cast&>(column).get_data()[row_num] > value) { + change(column, row_num, arena); + return true; + } else { + return false; + } } } diff --git a/be/src/vec/aggregate_functions/aggregate_function_min_max_by.cpp b/be/src/vec/aggregate_functions/aggregate_function_min_max_by.cpp index 506c0371300d54..ea513a26bbccfc 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_min_max_by.cpp +++ b/be/src/vec/aggregate_functions/aggregate_function_min_max_by.cpp @@ -63,6 +63,10 @@ static IAggregateFunction* create_aggregate_function_min_max_by_impl( return new AggregateFunctionTemplate>, false>( value_arg_type, key_arg_type); } + if (which.idx == TypeIndex::Decimal128I) { + return new AggregateFunctionTemplate>, false>( + value_arg_type, key_arg_type); + } return nullptr; } @@ -99,7 +103,7 @@ static IAggregateFunction* create_aggregate_function_min_max_by(const String& na SingleValueDataFixed>( argument_types); } - if (which.idx == TypeIndex::Decimal128 && !config::enable_decimalv3) { + if (which.idx == TypeIndex::Decimal128) { return create_aggregate_function_min_max_by_impl>( argument_types); @@ -114,9 +118,9 @@ static IAggregateFunction* create_aggregate_function_min_max_by(const String& na SingleValueDataFixed>( argument_types); } - if (which.idx == TypeIndex::Decimal128) { + if (which.idx == TypeIndex::Decimal128I) { return create_aggregate_function_min_max_by_impl>( + SingleValueDataFixed>( argument_types); } return nullptr; diff --git a/be/src/vec/aggregate_functions/aggregate_function_product.h b/be/src/vec/aggregate_functions/aggregate_function_product.h index 28af3ead15febb..d9260554beb738 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_product.h +++ b/be/src/vec/aggregate_functions/aggregate_function_product.h @@ -114,7 +114,9 @@ class AggregateFunctionProduct final } void reset(AggregateDataPtr place) const override { - if constexpr (IsDecimalNumber) { + if constexpr (IsDecimal128I) { + this->data(place).reset(T(1 * ResultDataType::get_scale_multiplier(scale).value.val)); + } else if constexpr (IsDecimalNumber) { this->data(place).reset(T(1 * ResultDataType::get_scale_multiplier(scale))); } else { this->data(place).reset(1); diff --git a/be/src/vec/aggregate_functions/aggregate_function_stddev.cpp b/be/src/vec/aggregate_functions/aggregate_function_stddev.cpp index 357b09a7a21742..f1510dd568adc5 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_stddev.cpp +++ b/be/src/vec/aggregate_functions/aggregate_function_stddev.cpp @@ -58,6 +58,11 @@ static IAggregateFunction* create_function_single_value(const String& name, NameData>>, is_nullable>( argument_types); } + if (which.is_decimal128i()) { + return new AggregateFunctionTemplate< + NameData>>, is_nullable>( + argument_types); + } DCHECK(false) << "with unknowed type, failed in create_aggregate_function_stddev_variance"; return nullptr; } diff --git a/be/src/vec/aggregate_functions/aggregate_function_sum.cpp b/be/src/vec/aggregate_functions/aggregate_function_sum.cpp index 60313f654798e7..22bf6fd80f494a 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_sum.cpp +++ b/be/src/vec/aggregate_functions/aggregate_function_sum.cpp @@ -31,7 +31,9 @@ namespace doris::vectorized { template struct SumSimple { /// @note It uses slow Decimal128 (cause we need such a variant). sumWithOverflow is faster for Decimal32/64 - using ResultType = std::conditional_t, Decimal128, NearestFieldType>; + using ResultType = std::conditional_t< + IsDecimalV2, Decimal128, + std::conditional_t, Decimal128I, NearestFieldType>>; // using ResultType = NearestFieldType; using AggregateDataType = AggregateFunctionSumData; using Function = AggregateFunctionSum; diff --git a/be/src/vec/aggregate_functions/aggregate_function_uniq.cpp b/be/src/vec/aggregate_functions/aggregate_function_uniq.cpp index e9a66901b6cb55..450327ed651347 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_uniq.cpp +++ b/be/src/vec/aggregate_functions/aggregate_function_uniq.cpp @@ -57,6 +57,9 @@ AggregateFunctionPtr create_aggregate_function_uniq(const std::string& name, } else if (which.is_decimal128()) { return std::make_shared>>( argument_types); + } else if (which.is_decimal128i()) { + return std::make_shared>>( + argument_types); } else if (which.is_string_or_fixed_string()) { return std::make_shared>>(argument_types); } diff --git a/be/src/vec/aggregate_functions/helpers.h b/be/src/vec/aggregate_functions/helpers.h index fce6e82cb77bda..36e11f7011f24f 100644 --- a/be/src/vec/aggregate_functions/helpers.h +++ b/be/src/vec/aggregate_functions/helpers.h @@ -186,6 +186,9 @@ static IAggregateFunction* create_with_decimal_type(const IDataType& argument_ty if (which.idx == TypeIndex::Decimal128) { return new AggregateFunctionTemplate(std::forward(args)...); } + if (which.idx == TypeIndex::Decimal128I) { + return new AggregateFunctionTemplate(std::forward(args)...); + } return nullptr; } @@ -203,6 +206,9 @@ static IAggregateFunction* create_with_decimal_type(const IDataType& argument_ty if (which.idx == TypeIndex::Decimal128) { return new AggregateFunctionTemplate(std::forward(args)...); } + if (which.idx == TypeIndex::Decimal128I) { + return new AggregateFunctionTemplate(std::forward(args)...); + } return nullptr; } diff --git a/be/src/vec/columns/column_decimal.cpp b/be/src/vec/columns/column_decimal.cpp index 8194b1ff7ebfc5..a6bd40ab95f8bc 100644 --- a/be/src/vec/columns/column_decimal.cpp +++ b/be/src/vec/columns/column_decimal.cpp @@ -40,7 +40,12 @@ int ColumnDecimal::compare_at(size_t n, size_t m, const IColumn& rhs_, int) c const T& a = data[n]; const T& b = other.data[m]; - if (scale == other.scale) return a > b ? 1 : (a < b ? -1 : 0); + if constexpr (doris::vectorized::IsDecimal128I) { + if (scale == other.scale) + return a.value.val > b.value.val ? 1 : (a.value.val < b.value.val ? -1 : 0); + } else { + if (scale == other.scale) return a > b ? 1 : (a < b ? -1 : 0); + } return decimal_less(b, a, other.scale, scale) ? 1 : (decimal_less(a, b, scale, other.scale) ? -1 : 0); @@ -133,29 +138,26 @@ void ColumnDecimal::update_crcs_with_value(std::vector& hashes, Pri auto s = hashes.size(); DCHECK(s == size()); - if constexpr (!std::is_same_v) { + if constexpr (!IsDecimalV2) { DO_CRC_HASHES_FUNCTION_COLUMN_IMPL() } else { - if (type == TYPE_DECIMALV2) { - auto decimalv2_do_crc = [&](size_t i) { - const DecimalV2Value& dec_val = (const DecimalV2Value&)data[i]; - int64_t int_val = dec_val.int_value(); - int32_t frac_val = dec_val.frac_value(); - hashes[i] = HashUtil::zlib_crc_hash(&int_val, sizeof(int_val), hashes[i]); - hashes[i] = HashUtil::zlib_crc_hash(&frac_val, sizeof(frac_val), hashes[i]); - }; - - if (null_data == nullptr) { - for (size_t i = 0; i < s; i++) { - decimalv2_do_crc(i); - } - } else { - for (size_t i = 0; i < s; i++) { - if (null_data[i] == 0) decimalv2_do_crc(i); - } + DCHECK(type == TYPE_DECIMALV2); + auto decimalv2_do_crc = [&](size_t i) { + const DecimalV2Value& dec_val = (const DecimalV2Value&)data[i]; + int64_t int_val = dec_val.int_value(); + int32_t frac_val = dec_val.frac_value(); + hashes[i] = HashUtil::zlib_crc_hash(&int_val, sizeof(int_val), hashes[i]); + hashes[i] = HashUtil::zlib_crc_hash(&frac_val, sizeof(frac_val), hashes[i]); + }; + + if (null_data == nullptr) { + for (size_t i = 0; i < s; i++) { + decimalv2_do_crc(i); } } else { - DO_CRC_HASHES_FUNCTION_COLUMN_IMPL() + for (size_t i = 0; i < s; i++) { + if (null_data[i] == 0) decimalv2_do_crc(i); + } } } } @@ -377,11 +379,20 @@ void ColumnDecimal::get_extremes(Field& min, Field& max) const { T cur_min = data[0]; T cur_max = data[0]; - for (const T& x : data) { - if (x < cur_min) - cur_min = x; - else if (x > cur_max) - cur_max = x; + if constexpr (doris::vectorized::IsDecimal128I) { + for (const T& x : data) { + if (x.value.val < cur_min.value.val) + cur_min.value.val = x.value.val; + else if (x.value.val > cur_max.value.val) + cur_max.value.val = x.value.val; + } + } else { + for (const T& x : data) { + if (x < cur_min) + cur_min = x; + else if (x > cur_max) + cur_max = x; + } } min = NearestFieldType(cur_min, scale); @@ -409,7 +420,14 @@ void ColumnDecimal::compare_internal(size_t rhs_row_id, const IColumn& rhs, size_t end = simd::find_one(cmp_res, begin + 1); for (size_t row_id = begin; row_id < end; row_id++) { auto value_a = get_data()[row_id]; - int res = value_a > cmp_base ? 1 : (value_a < cmp_base ? -1 : 0); + int res = 0; + if constexpr (doris::vectorized::IsDecimal128I) { + res = value_a.value.val > cmp_base.value.val + ? 1 + : (value_a.value.val < cmp_base.value.val ? -1 : 0); + } else { + res = value_a > cmp_base ? 1 : (value_a < cmp_base ? -1 : 0); + } if (res * direction < 0) { filter[row_id] = 1; cmp_res[row_id] = 1; @@ -436,7 +454,13 @@ Decimal128 ColumnDecimal::get_scale_multiplier() const { return common::exp10_i128(scale); } +template <> +Decimal128I ColumnDecimal::get_scale_multiplier() const { + return common::exp10_i128(scale); +} + template class ColumnDecimal; template class ColumnDecimal; template class ColumnDecimal; +template class ColumnDecimal; } // namespace doris::vectorized diff --git a/be/src/vec/columns/column_decimal.h b/be/src/vec/columns/column_decimal.h index a3960067cb2b86..b19e5ff31e4ab6 100644 --- a/be/src/vec/columns/column_decimal.h +++ b/be/src/vec/columns/column_decimal.h @@ -70,6 +70,7 @@ class ColumnDecimal final : public COWHelper; + static constexpr bool IsDecimal128I = std::is_same_v; public: using value_type = T; @@ -175,10 +176,28 @@ class ColumnDecimal final : public COWHelper(&data[n]), sizeof(data[n])); } void get(size_t n, Field& res) const override { res = (*this)[n]; } - bool get_bool(size_t n) const override { return bool(data[n]); } - Int64 get_int(size_t n) const override { return Int64(data[n] * scale); } + bool get_bool(size_t n) const override { + if constexpr (IsDecimal128I) { + return bool(data[n].value.val); + } else { + return bool(data[n]); + } + } + Int64 get_int(size_t n) const override { + if constexpr (IsDecimal128I) { + return Int64(data[n].value.val * scale); + } else { + return Int64(data[n] * scale); + } + } UInt64 get64(size_t n) const override; - bool is_default_at(size_t n) const override { return data[n] == 0; } + bool is_default_at(size_t n) const override { + if constexpr (IsDecimal128I) { + return data[n].value.val == 0; + } else { + return data[n] == 0; + } + } void clear() override { data.clear(); } @@ -240,8 +259,20 @@ class ColumnDecimal final : public COWHelper data[b]; }); - else - std::partial_sort(res.begin(), sort_end, res.end(), - [this](size_t a, size_t b) { return data[a] < data[b]; }); + if constexpr (doris::vectorized::IsDecimal128I) { + if (reverse) + std::partial_sort(res.begin(), sort_end, res.end(), [this](size_t a, size_t b) { + return data[a].value.val > data[b].value.val; + }); + else + std::partial_sort(res.begin(), sort_end, res.end(), [this](size_t a, size_t b) { + return data[a].value.val < data[b].value.val; + }); + } else { + if (reverse) + std::partial_sort(res.begin(), sort_end, res.end(), + [this](size_t a, size_t b) { return data[a] > data[b]; }); + else + std::partial_sort(res.begin(), sort_end, res.end(), + [this](size_t a, size_t b) { return data[a] < data[b]; }); + } } }; diff --git a/be/src/vec/columns/columns_number.h b/be/src/vec/columns/columns_number.h index df21f7e567b407..1ce930e4cd1a66 100644 --- a/be/src/vec/columns/columns_number.h +++ b/be/src/vec/columns/columns_number.h @@ -52,6 +52,7 @@ using ColumnFloat64 = ColumnVector; using ColumnDecimal32 = ColumnDecimal; using ColumnDecimal64 = ColumnDecimal; using ColumnDecimal128 = ColumnDecimal; +using ColumnDecimal128I = ColumnDecimal; template struct IsFixLenColumnType { diff --git a/be/src/vec/common/field_visitors.h b/be/src/vec/common/field_visitors.h index 9b19f1946b1397..42ba1bff0bd1a1 100644 --- a/be/src/vec/common/field_visitors.h +++ b/be/src/vec/common/field_visitors.h @@ -65,6 +65,8 @@ typename std::decay_t::ResultType apply_visitor(Visitor&& visitor, F&& return visitor(field.template get>()); case Field::Types::Decimal128: return visitor(field.template get>()); + case Field::Types::Decimal128I: + return visitor(field.template get>()); case Field::Types::AggregateFunctionState: return visitor(field.template get()); @@ -101,6 +103,8 @@ static typename std::decay_t::ResultType apply_binary_visitor_impl(Visi return visitor(field1, field2.template get>()); case Field::Types::Decimal128: return visitor(field1, field2.template get>()); + case Field::Types::Decimal128I: + return visitor(field1, field2.template get>()); case Field::Types::AggregateFunctionState: return visitor(field1, field2.template get()); @@ -150,6 +154,10 @@ typename std::decay_t::ResultType apply_visitor(Visitor&& visitor, F1&& return apply_binary_visitor_impl(std::forward(visitor), field1.template get>(), std::forward(field2)); + case Field::Types::Decimal128I: + return apply_binary_visitor_impl(std::forward(visitor), + field1.template get>(), + std::forward(field2)); case Field::Types::AggregateFunctionState: return apply_binary_visitor_impl(std::forward(visitor), field1.template get(), @@ -175,6 +183,7 @@ class FieldVisitorToString : public StaticVisitor { String operator()(const DecimalField& x) const; String operator()(const DecimalField& x) const; String operator()(const DecimalField& x) const; + String operator()(const DecimalField& x) const; String operator()(const AggregateFunctionStateData& x) const; }; @@ -192,6 +201,7 @@ class FieldVisitorDump : public StaticVisitor { String operator()(const DecimalField& x) const; String operator()(const DecimalField& x) const; String operator()(const DecimalField& x) const; + String operator()(const DecimalField& x) const; String operator()(const AggregateFunctionStateData& x) const; }; @@ -232,8 +242,13 @@ class FieldVisitorConvertToNumber : public StaticVisitor { T operator()(const DecimalField& x) const { if constexpr (std::is_floating_point_v) return static_cast(x.get_value()) / x.get_scale_multiplier(); - else - return x.get_value() / x.get_scale_multiplier(); + else { + if constexpr (std::is_same_v) { + return x.get_value().value.val / x.get_scale_multiplier().value.val; + } else { + return x.get_value() / x.get_scale_multiplier(); + } + } } T operator()(const AggregateFunctionStateData&) const { @@ -261,6 +276,7 @@ class FieldVisitorHash : public StaticVisitor<> { void operator()(const DecimalField& x) const; void operator()(const DecimalField& x) const; void operator()(const DecimalField& x) const; + void operator()(const DecimalField& x) const; void operator()(const AggregateFunctionStateData& x) const; }; @@ -280,6 +296,10 @@ template <> constexpr bool is_decimalField>() { return true; } +template <> +constexpr bool is_decimalField>() { + return true; +} /** More precise comparison, used for index. * Differs from Field::operator< and Field::operator== in that it also compares values of different types. diff --git a/be/src/vec/common/hash_table/hash.h b/be/src/vec/common/hash_table/hash.h index ab995636252d8e..6d63a167359a4b 100644 --- a/be/src/vec/common/hash_table/hash.h +++ b/be/src/vec/common/hash_table/hash.h @@ -92,6 +92,13 @@ struct DefaultHash>> { size_t operator()(T key) const { return default_hash64(key); } }; +template <> +struct DefaultHash { + size_t operator()(doris::vectorized::Int128I key) const { + return default_hash64(key); + } +}; + template <> struct DefaultHash : public StringRefHash {}; diff --git a/be/src/vec/core/block.cpp b/be/src/vec/core/block.cpp index 6d6abe49fc32cb..3cdadd5bdcb4ed 100644 --- a/be/src/vec/core/block.cpp +++ b/be/src/vec/core/block.cpp @@ -876,7 +876,7 @@ void Block::deep_copy_slot(void* dst, MemPool* pool, const doris::TypeDescriptor DateTimeVal datetime_val; datetime_value.to_datetime_val(&datetime_val); iterator.set(&datetime_val); - } else if (item_type_desc.is_decimal_type()) { + } else if (item_type_desc.is_decimal_v2_type()) { // In CollectionValue, decimal type data is stored as decimal12_t. DecimalV2Value decimal_value; deep_copy_slot(&decimal_value, pool, item_type_desc, data_ref, item_column, diff --git a/be/src/vec/core/call_on_type_index.h b/be/src/vec/core/call_on_type_index.h index 0e9dd5468c4ecb..ff4bb9f6933ac5 100644 --- a/be/src/vec/core/call_on_type_index.h +++ b/be/src/vec/core/call_on_type_index.h @@ -69,6 +69,8 @@ bool call_on_basic_type(TypeIndex number, F&& f) { return f(TypePair()); case TypeIndex::Decimal128: return f(TypePair()); + case TypeIndex::Decimal128I: + return f(TypePair()); default: break; } @@ -137,6 +139,9 @@ inline bool call_on_basic_types(TypeIndex type_num1, TypeIndex type_num2, F&& f) case TypeIndex::Decimal128: return call_on_basic_type( type_num2, std::forward(f)); + case TypeIndex::Decimal128I: + return call_on_basic_type( + type_num2, std::forward(f)); default: break; } @@ -204,6 +209,8 @@ bool call_on_index_and_data_type(TypeIndex number, F&& f) { return f(TypePair, T>()); case TypeIndex::Decimal128: return f(TypePair, T>()); + case TypeIndex::Decimal128I: + return f(TypePair, T>()); case TypeIndex::Date: return f(TypePair()); @@ -257,6 +264,8 @@ bool call_on_index_and_number_data_type(TypeIndex number, F&& f) { return f(TypePair, T>()); case TypeIndex::Decimal128: return f(TypePair, T>()); + case TypeIndex::Decimal128I: + return f(TypePair, T>()); default: break; } diff --git a/be/src/vec/core/decimal_comparison.h b/be/src/vec/core/decimal_comparison.h index 6ce3e9bb569a09..4a4491d304ab8e 100644 --- a/be/src/vec/core/decimal_comparison.h +++ b/be/src/vec/core/decimal_comparison.h @@ -143,12 +143,33 @@ class DecimalComparison { Shift shift; if (decimal0 && decimal1) { auto result_type = decimal_result_type(*decimal0, *decimal1, false, false); - shift.a = result_type.scale_factor_for(*decimal0, false); - shift.b = result_type.scale_factor_for(*decimal1, false); - } else if (decimal0) - shift.b = decimal0->get_scale_multiplier(); - else if (decimal1) - shift.a = decimal1->get_scale_multiplier(); + if constexpr (sizeof(T) >= sizeof(U) && IsDecimal128I) { + shift.a = result_type.scale_factor_for(*decimal0, false).value.val; + } else if constexpr (sizeof(T) < sizeof(U) && IsDecimal128I) { + shift.a = result_type.scale_factor_for(*decimal0, false).value.val; + } else { + shift.a = result_type.scale_factor_for(*decimal0, false); + } + if constexpr (sizeof(T) >= sizeof(U) && IsDecimal128I) { + shift.b = result_type.scale_factor_for(*decimal1, false).value.val; + } else if constexpr (sizeof(T) < sizeof(U) && IsDecimal128I) { + shift.b = result_type.scale_factor_for(*decimal1, false).value.val; + } else { + shift.b = result_type.scale_factor_for(*decimal1, false); + } + } else if (decimal0) { + if constexpr (IsDecimal128I) { + shift.b = decimal0->get_scale_multiplier().value.val; + } else { + shift.b = decimal0->get_scale_multiplier(); + } + } else if (decimal1) { + if constexpr (IsDecimal128I) { + shift.a = decimal1->get_scale_multiplier().value.val; + } else { + shift.a = decimal1->get_scale_multiplier(); + } + } return shift; } @@ -158,7 +179,11 @@ class DecimalComparison { const DataTypePtr& left_type, const DataTypePtr&) { Shift shift; const DataTypeDecimal* decimal0 = check_decimal(*left_type); - if (decimal0) shift.b = decimal0->get_scale_multiplier(); + if constexpr (IsDecimal128I) { + if (decimal0) shift.b = decimal0->get_scale_multiplier().value.val; + } else { + if (decimal0) shift.b = decimal0->get_scale_multiplier(); + } return shift; } @@ -167,7 +192,11 @@ class DecimalComparison { const DataTypePtr&, const DataTypePtr& right_type) { Shift shift; const DataTypeDecimal* decimal1 = check_decimal(*right_type); - if (decimal1) shift.a = decimal1->get_scale_multiplier(); + if constexpr (IsDecimal128I) { + if (decimal1) shift.a = decimal1->get_scale_multiplier().value.val; + } else { + if (decimal1) shift.a = decimal1->get_scale_multiplier(); + } return shift; } @@ -227,8 +256,19 @@ class DecimalComparison { template static NO_INLINE UInt8 apply(A a, B b, CompareInt scale [[maybe_unused]]) { - CompareInt x = a; - CompareInt y = b; + CompareInt x {}; + if constexpr (IsDecimal128I) { + x = a.value.val; + } else { + x = a; + } + + CompareInt y {}; + if constexpr (IsDecimal128I) { + y = b.value.val; + } else { + y = b; + } if constexpr (_check_overflow) { bool overflow = false; diff --git a/be/src/vec/core/field.cpp b/be/src/vec/core/field.cpp index 722b9663aa3101..5d11aca10b1878 100644 --- a/be/src/vec/core/field.cpp +++ b/be/src/vec/core/field.cpp @@ -129,18 +129,8 @@ void write_binary(const Array& x, BufferWritable& buf) { } template <> -Decimal32 DecimalField::get_scale_multiplier() const { - return DataTypeDecimal::get_scale_multiplier(scale); -} - -template <> -Decimal64 DecimalField::get_scale_multiplier() const { - return DataTypeDecimal::get_scale_multiplier(scale); -} - -template <> -Decimal128 DecimalField::get_scale_multiplier() const { - return DataTypeDecimal::get_scale_multiplier(scale); +Decimal128I DecimalField::get_scale_multiplier() const { + return DataTypeDecimal::get_scale_multiplier(scale); } template @@ -161,42 +151,38 @@ static bool dec_less_or_equal(T x, T y, UInt32 x_scale, UInt32 y_scale) { return Comparator::compare(x, y, x_scale, y_scale); } -template <> -bool decimal_equal(Decimal32 x, Decimal32 y, UInt32 xs, UInt32 ys) { - return dec_equal(x, y, xs, ys); -} -template <> -bool decimal_less(Decimal32 x, Decimal32 y, UInt32 xs, UInt32 ys) { - return dec_less(x, y, xs, ys); -} -template <> -bool decimal_less_or_equal(Decimal32 x, Decimal32 y, UInt32 xs, UInt32 ys) { - return dec_less_or_equal(x, y, xs, ys); -} +#define DECLARE_DECIMAL_COMPARISON(TYPE) \ + template <> \ + bool decimal_equal(TYPE x, TYPE y, UInt32 xs, UInt32 ys) { \ + return dec_equal(x, y, xs, ys); \ + } \ + template <> \ + bool decimal_less(TYPE x, TYPE y, UInt32 xs, UInt32 ys) { \ + return dec_less(x, y, xs, ys); \ + } \ + template <> \ + bool decimal_less_or_equal(TYPE x, TYPE y, UInt32 xs, UInt32 ys) { \ + return dec_less_or_equal(x, y, xs, ys); \ + } \ + template <> \ + TYPE DecimalField::get_scale_multiplier() const { \ + return DataTypeDecimal::get_scale_multiplier(scale); \ + } -template <> -bool decimal_equal(Decimal64 x, Decimal64 y, UInt32 xs, UInt32 ys) { - return dec_equal(x, y, xs, ys); -} -template <> -bool decimal_less(Decimal64 x, Decimal64 y, UInt32 xs, UInt32 ys) { - return dec_less(x, y, xs, ys); -} -template <> -bool decimal_less_or_equal(Decimal64 x, Decimal64 y, UInt32 xs, UInt32 ys) { - return dec_less_or_equal(x, y, xs, ys); -} +DECLARE_DECIMAL_COMPARISON(Decimal32) +DECLARE_DECIMAL_COMPARISON(Decimal64) +DECLARE_DECIMAL_COMPARISON(Decimal128) template <> -bool decimal_equal(Decimal128 x, Decimal128 y, UInt32 xs, UInt32 ys) { - return dec_equal(x, y, xs, ys); +bool decimal_equal(Decimal128I x, Decimal128I y, UInt32 xs, UInt32 ys) { + return dec_equal(Decimal128(x.value.val), Decimal128(y.value.val), xs, ys); } template <> -bool decimal_less(Decimal128 x, Decimal128 y, UInt32 xs, UInt32 ys) { - return dec_less(x, y, xs, ys); +bool decimal_less(Decimal128I x, Decimal128I y, UInt32 xs, UInt32 ys) { + return dec_less(Decimal128(x.value.val), Decimal128(y.value.val), xs, ys); } template <> -bool decimal_less_or_equal(Decimal128 x, Decimal128 y, UInt32 xs, UInt32 ys) { - return dec_less_or_equal(x, y, xs, ys); +bool decimal_less_or_equal(Decimal128I x, Decimal128I y, UInt32 xs, UInt32 ys) { + return dec_less_or_equal(Decimal128(x.value.val), Decimal128(y.value.val), xs, ys); } } // namespace doris::vectorized diff --git a/be/src/vec/core/field.h b/be/src/vec/core/field.h index cf3586f8dff6b5..89fe0de7d7ed01 100644 --- a/be/src/vec/core/field.h +++ b/be/src/vec/core/field.h @@ -40,6 +40,31 @@ struct NearestFieldTypeImpl; template using NearestFieldType = typename NearestFieldTypeImpl::Type; +template +struct AvgNearestFieldTypeTrait { + using Type = typename NearestFieldTypeImpl::Type; +}; + +template <> +struct AvgNearestFieldTypeTrait { + using Type = Decimal128I; +}; + +template <> +struct AvgNearestFieldTypeTrait { + using Type = Decimal128I; +}; + +template <> +struct AvgNearestFieldTypeTrait { + using Type = Decimal128; +}; + +template <> +struct AvgNearestFieldTypeTrait { + using Type = Decimal128I; +}; + class Field; using FieldVector = std::vector; @@ -277,6 +302,7 @@ class Field { Decimal128 = 21, AggregateFunctionState = 22, JSONB = 23, + Decimal128I = 24, }; static const int MIN_NON_POD = 16; @@ -309,6 +335,8 @@ class Field { return "Decimal64"; case Decimal128: return "Decimal128"; + case Decimal128I: + return "Decimal128I"; case AggregateFunctionState: return "AggregateFunctionState"; case FixedLengthObject: @@ -327,7 +355,8 @@ class Field { struct EnumToType; static bool is_decimal(Types::Which which) { - return which >= Types::Decimal32 && which <= Types::Decimal128; + return (which >= Types::Decimal32 && which <= Types::Decimal128) || + which == Types::Decimal128I; } Field() : which(Types::Null) {} @@ -477,6 +506,8 @@ class Field { return get>() < rhs.get>(); case Types::Decimal128: return get>() < rhs.get>(); + case Types::Decimal128I: + return get>() < rhs.get>(); case Types::AggregateFunctionState: return get() < rhs.get(); case Types::FixedLengthObject: @@ -520,6 +551,8 @@ class Field { return get>() <= rhs.get>(); case Types::Decimal128: return get>() <= rhs.get>(); + case Types::Decimal128I: + return get>() <= rhs.get>(); case Types::AggregateFunctionState: return get() <= rhs.get(); case Types::FixedLengthObject: @@ -559,6 +592,8 @@ class Field { return get>() == rhs.get>(); case Types::Decimal128: return get>() == rhs.get>(); + case Types::Decimal128I: + return get>() == rhs.get>(); case Types::AggregateFunctionState: return get() == rhs.get(); case Types::FixedLengthObject: @@ -574,7 +609,7 @@ class Field { std::aligned_union_t, DecimalField, DecimalField, - AggregateFunctionStateData> + DecimalField, AggregateFunctionStateData> storage; Types::Which which; @@ -646,6 +681,9 @@ class Field { case Types::Decimal128: f(field.template get>()); return; + case Types::Decimal128I: + f(field.template get>()); + return; case Types::AggregateFunctionState: f(field.template get()); return; @@ -779,6 +817,10 @@ struct Field::TypeToEnum> { static const Types::Which value = Types::Decimal128; }; template <> +struct Field::TypeToEnum> { + static const Types::Which value = Types::Decimal128I; +}; +template <> struct Field::TypeToEnum { static const Types::Which value = Types::AggregateFunctionState; }; @@ -836,6 +878,10 @@ struct Field::EnumToType { using Type = DecimalField; }; template <> +struct Field::EnumToType { + using Type = DecimalField; +}; +template <> struct Field::EnumToType { using Type = DecimalField; }; @@ -948,6 +994,10 @@ struct NearestFieldTypeImpl { using Type = DecimalField; }; template <> +struct NearestFieldTypeImpl { + using Type = DecimalField; +}; +template <> struct NearestFieldTypeImpl> { using Type = DecimalField; }; @@ -960,6 +1010,10 @@ struct NearestFieldTypeImpl> { using Type = DecimalField; }; template <> +struct NearestFieldTypeImpl> { + using Type = DecimalField; +}; +template <> struct NearestFieldTypeImpl { using Type = Float64; }; diff --git a/be/src/vec/core/sort_block.h b/be/src/vec/core/sort_block.h index 9d4b9ca2b75de1..015d3f3235b662 100644 --- a/be/src/vec/core/sort_block.h +++ b/be/src/vec/core/sort_block.h @@ -332,7 +332,13 @@ class ColumnSorter { EqualRange& range, bool last_column) const { int new_limit = _limit; auto comparator = [&](const size_t a, const size_t b) { - if constexpr (!std::is_same_v) { + if constexpr (std::is_same_v) { + auto value_a = column.get_data()[a]; + auto value_b = column.get_data()[b]; + return value_a.value.val > value_b.value.val + ? 1 + : (value_a.value.val < value_b.value.val ? -1 : 0); + } else if constexpr (!std::is_same_v) { auto value_a = column.get_data()[a]; auto value_b = column.get_data()[b]; return value_a > value_b ? 1 : (value_a < value_b ? -1 : 0); @@ -397,7 +403,11 @@ class ColumnSorter { _create_permutation(column, permutation_for_column.data(), perms); auto comparator = [&](const PermutationWithInlineValue& a, const PermutationWithInlineValue& b) { - if constexpr (!std::is_same_v) { + if constexpr (std::is_same_v) { + return a.inline_value.value.val > b.inline_value.value.val + ? 1 + : (a.inline_value.value.val < b.inline_value.value.val ? -1 : 0); + } else if constexpr (!std::is_same_v) { return a.inline_value > b.inline_value ? 1 : (a.inline_value < b.inline_value ? -1 : 0); } else { diff --git a/be/src/vec/core/types.h b/be/src/vec/core/types.h index 3a694e6fa8b7be..08bd4695408d3a 100644 --- a/be/src/vec/core/types.h +++ b/be/src/vec/core/types.h @@ -79,6 +79,7 @@ enum class TypeIndex { TimeV2, FixedLengthObject, JSONB, + Decimal128I, }; struct Consted { @@ -263,6 +264,62 @@ using DateTime = Int64; using DateV2 = UInt32; using DateTimeV2 = UInt64; +struct Int128I { + Int128I() = default; + Int128I(Int128I&&) = default; + Int128I(const Int128I&) = default; + +#define DECLARE_NUMERIC_CTOR(TYPE) \ + constexpr Int128I(const TYPE& value_) : val(value_) {} \ + constexpr Int128I(const TYPE&& value_) : val(value_) {} + + DECLARE_NUMERIC_CTOR(Int128) + DECLARE_NUMERIC_CTOR(Int32) + DECLARE_NUMERIC_CTOR(Int64) + DECLARE_NUMERIC_CTOR(UInt32) + DECLARE_NUMERIC_CTOR(UInt64) + DECLARE_NUMERIC_CTOR(Float32) + DECLARE_NUMERIC_CTOR(Float64) +#undef DECLARE_NUMERIC_CTOR + constexpr Int128I& operator=(Int128I&&) = default; + constexpr Int128I& operator=(const Int128I&) = default; + + constexpr Int128I& operator=(Int128&& v) { + val = v; + return *this; + }; + + constexpr Int128I& operator=(const Int128& v) { + val = v; + return *this; + }; + + operator Int128() const { return val; } + + const Int128I& operator+=(const Int128& x) { + val += x; + return *this; + } + const Int128I& operator-=(const Int128& x) { + val -= x; + return *this; + } + const Int128I& operator*=(const Int128& x) { + val *= x; + return *this; + } + const Int128I& operator/=(const Int128& x) { + val /= x; + return *this; + } + const Int128I& operator%=(const Int128& x) { + val %= x; + return *this; + } + + Int128 val; +}; + /// Own FieldType for Decimal. /// It is only a "storage" for decimal. To perform operations, you also have to provide a scale (number of digits after point). template @@ -273,7 +330,17 @@ struct Decimal { Decimal(Decimal&&) = default; Decimal(const Decimal&) = default; - Decimal(const T& value_) : value(value_) {} +#define DECLARE_NUMERIC_CTOR(TYPE) \ + Decimal(const TYPE& value_) : value(value_) {} + + DECLARE_NUMERIC_CTOR(Int128) + DECLARE_NUMERIC_CTOR(Int32) + DECLARE_NUMERIC_CTOR(Int64) + DECLARE_NUMERIC_CTOR(UInt32) + DECLARE_NUMERIC_CTOR(UInt64) + DECLARE_NUMERIC_CTOR(Float32) + DECLARE_NUMERIC_CTOR(Float64) +#undef DECLARE_NUMERIC_CTOR static Decimal double_to_decimal(double value_) { DecimalV2Value decimal_value; @@ -282,7 +349,13 @@ struct Decimal { } template - Decimal(const Decimal& x) : value(x) {} + Decimal(const Decimal& x) { + if constexpr (std::is_same_v) { + value = x.value.val; + } else { + value = x; + } + } constexpr Decimal& operator=(Decimal&&) = default; constexpr Decimal& operator=(const Decimal&) = default; @@ -316,6 +389,7 @@ struct Decimal { using Decimal32 = Decimal; using Decimal64 = Decimal; using Decimal128 = Decimal; +using Decimal128I = Decimal; template <> struct TypeName { @@ -329,6 +403,10 @@ template <> struct TypeName { static const char* get() { return "Decimal128"; } }; +template <> +struct TypeName { + static const char* get() { return "Decimal128I"; } +}; template <> struct TypeId { @@ -342,6 +420,10 @@ template <> struct TypeId { static constexpr const TypeIndex value = TypeIndex::Decimal128; }; +template <> +struct TypeId { + static constexpr const TypeIndex value = TypeIndex::Decimal128I; +}; template constexpr bool IsDecimalNumber = false; @@ -351,6 +433,18 @@ template <> inline constexpr bool IsDecimalNumber = true; template <> inline constexpr bool IsDecimalNumber = true; +template <> +inline constexpr bool IsDecimalNumber = true; + +template +constexpr bool IsDecimalV2 = false; +template <> +inline constexpr bool IsDecimalV2 = true; + +template +constexpr bool IsDecimal128I = false; +template <> +inline constexpr bool IsDecimal128I = true; template constexpr bool IsFloatNumber = false; @@ -375,6 +469,10 @@ template <> struct NativeType { using Type = Int128; }; +template <> +struct NativeType { + using Type = Int128I; +}; inline const char* getTypeName(TypeIndex idx) { switch (idx) { @@ -428,6 +526,8 @@ inline const char* getTypeName(TypeIndex idx) { return TypeName::get(); case TypeIndex::Decimal128: return TypeName::get(); + case TypeIndex::Decimal128I: + return TypeName::get(); case TypeIndex::UUID: return "UUID"; case TypeIndex::Array: @@ -477,6 +577,15 @@ struct hash { } }; +template <> +struct hash { + size_t operator()(const doris::vectorized::Decimal128I& x) const { + return std::hash()(x.value.val >> 64) ^ + std::hash()( + x.value.val & std::numeric_limits::max()); + } +}; + constexpr bool is_integer(doris::vectorized::TypeIndex index) { using TypeIndex = doris::vectorized::TypeIndex; switch (index) { diff --git a/be/src/vec/data_types/data_type.cpp b/be/src/vec/data_types/data_type.cpp index 09b4c954e9e241..7a0d67d4a41c68 100644 --- a/be/src/vec/data_types/data_type.cpp +++ b/be/src/vec/data_types/data_type.cpp @@ -127,6 +127,8 @@ PGenericType_TypeId IDataType::get_pdata_type(const IDataType* data_type) { return PGenericType::DECIMAL64; case TypeIndex::Decimal128: return PGenericType::DECIMAL128; + case TypeIndex::Decimal128I: + return PGenericType::DECIMAL128I; case TypeIndex::String: return PGenericType::STRING; case TypeIndex::Date: diff --git a/be/src/vec/data_types/data_type.h b/be/src/vec/data_types/data_type.h index 945472b75423c6..e5f0e6aa4cd639 100644 --- a/be/src/vec/data_types/data_type.h +++ b/be/src/vec/data_types/data_type.h @@ -285,7 +285,10 @@ struct WhichDataType { bool is_decimal32() const { return idx == TypeIndex::Decimal32; } bool is_decimal64() const { return idx == TypeIndex::Decimal64; } bool is_decimal128() const { return idx == TypeIndex::Decimal128; } - bool is_decimal() const { return is_decimal32() || is_decimal64() || is_decimal128(); } + bool is_decimal128i() const { return idx == TypeIndex::Decimal128I; } + bool is_decimal() const { + return is_decimal32() || is_decimal64() || is_decimal128() || is_decimal128i(); + } bool is_float32() const { return idx == TypeIndex::Float32; } bool is_float64() const { return idx == TypeIndex::Float64; } @@ -343,6 +346,9 @@ inline bool is_enum(const DataTypePtr& data_type) { inline bool is_decimal(const DataTypePtr& data_type) { return WhichDataType(data_type).is_decimal(); } +inline bool is_decimal_v2(const DataTypePtr& data_type) { + return WhichDataType(data_type).is_decimal128(); +} inline bool is_tuple(const DataTypePtr& data_type) { return WhichDataType(data_type).is_tuple(); } diff --git a/be/src/vec/data_types/data_type_decimal.cpp b/be/src/vec/data_types/data_type_decimal.cpp index 0a1d9f5b5b2e7e..6cd9f59fbeb9f3 100644 --- a/be/src/vec/data_types/data_type_decimal.cpp +++ b/be/src/vec/data_types/data_type_decimal.cpp @@ -56,21 +56,20 @@ template void DataTypeDecimal::to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const { // TODO: Reduce the copy in std::string mem to ostr, like DataTypeNumber - if (config::enable_decimalv3) { + if constexpr (!IsDecimalV2) { T value = assert_cast(*column.convert_to_full_column_if_const().get()) .get_data()[row_num]; std::ostringstream buf; write_text(value, scale, buf); std::string str = buf.str(); ostr.write(str.data(), str.size()); - return; + } else { + DecimalV2Value value = (DecimalV2Value)assert_cast( + *column.convert_to_full_column_if_const().get()) + .get_data()[row_num]; + auto str = value.to_string(); + ostr.write(str.data(), str.size()); } - - DecimalV2Value value = (DecimalV2Value)assert_cast( - *column.convert_to_full_column_if_const().get()) - .get_data()[row_num]; - auto str = value.to_string(); - ostr.write(str.data(), str.size()); } template @@ -134,18 +133,19 @@ Field DataTypeDecimal::get_default() const { template DataTypePtr DataTypeDecimal::promote_numeric_type() const { - using PromotedType = DataTypeDecimal; + using PromotedType = std::conditional_t, DataTypeDecimal, + DataTypeDecimal>; return std::make_shared(PromotedType::max_precision(), scale); } template MutableColumnPtr DataTypeDecimal::create_column() const { - if (config::enable_decimalv3) { - return ColumnType::create(0, scale); - } else { + if constexpr (IsDecimalV2) { auto col = ColumnDecimal128::create(0, scale); col->set_decimalv2_type(); return col; + } else { + return ColumnType::create(0, scale); } } @@ -160,7 +160,7 @@ T DataTypeDecimal::parse_from_string(const std::string& str) const { return value; } -DataTypePtr create_decimal(UInt64 precision_value, UInt64 scale_value) { +DataTypePtr create_decimal(UInt64 precision_value, UInt64 scale_value, bool use_v2) { if (precision_value < min_decimal_precision() || precision_value > max_decimal_precision()) { LOG(FATAL) << "Wrong precision"; @@ -170,12 +170,16 @@ DataTypePtr create_decimal(UInt64 precision_value, UInt64 scale_value) { LOG(FATAL) << "Negative scales and scales larger than precision are not supported"; } + if (use_v2) { + return std::make_shared>(precision_value, scale_value); + } + if (precision_value <= max_decimal_precision()) { return std::make_shared>(precision_value, scale_value); } else if (precision_value <= max_decimal_precision()) { return std::make_shared>(precision_value, scale_value); } - return std::make_shared>(precision_value, scale_value); + return std::make_shared>(precision_value, scale_value); } template <> @@ -193,6 +197,11 @@ Decimal128 DataTypeDecimal::get_scale_multiplier(UInt32 scale) { return common::exp10_i128(scale); } +template <> +Decimal128I DataTypeDecimal::get_scale_multiplier(UInt32 scale) { + return common::exp10_i128(scale); +} + template void convert_to_decimal(T* from_value, T* to_value, int32_t from_scale, int32_t to_scale, bool* loss_accuracy) { @@ -238,6 +247,14 @@ Int128 max_decimal_value(UInt32 precision) { DataTypeDecimal::get_scale_multiplier( (UInt64)max_decimal_precision() - precision); } +template <> +Int128I max_decimal_value(UInt32 precision) { + return (static_cast(999999999999999999ll) * 100000000000000000ll * 1000ll + + static_cast(99999999999999999ll) * 1000ll + 999ll) / + DataTypeDecimal::get_scale_multiplier( + (UInt64)max_decimal_precision() - precision) + .value.val; +} template typename T::NativeType min_decimal_value(UInt32 precision) { @@ -260,10 +277,19 @@ Int128 min_decimal_value(UInt32 precision) { DataTypeDecimal::get_scale_multiplier( (UInt64)max_decimal_precision() - precision); } +template <> +Int128I min_decimal_value(UInt32 precision) { + return -(static_cast(999999999999999999ll) * 100000000000000000ll * 1000ll + + static_cast(99999999999999999ll) * 1000ll + 999ll) / + DataTypeDecimal::get_scale_multiplier( + (UInt64)max_decimal_precision() - precision) + .value.val; +} /// Explicit template instantiations. template class DataTypeDecimal; template class DataTypeDecimal; template class DataTypeDecimal; +template class DataTypeDecimal; } // namespace doris::vectorized diff --git a/be/src/vec/data_types/data_type_decimal.h b/be/src/vec/data_types/data_type_decimal.h index 1f2f2fa2b2eb11..e972f79fa8eeec 100644 --- a/be/src/vec/data_types/data_type_decimal.h +++ b/be/src/vec/data_types/data_type_decimal.h @@ -49,6 +49,10 @@ template <> constexpr size_t max_decimal_precision() { return 38; } +template <> +constexpr size_t max_decimal_precision() { + return 38; +} template static constexpr typename T::NativeType max_decimal_value() { @@ -67,8 +71,13 @@ constexpr Int128 max_decimal_value() { return static_cast(999999999999999999ll) * 100000000000000000ll * 1000ll + static_cast(99999999999999999ll) * 1000ll + 999ll; } +template <> +constexpr Int128I max_decimal_value() { + return static_cast(999999999999999999ll) * 100000000000000000ll * 1000ll + + static_cast(99999999999999999ll) * 1000ll + 999ll; +} -DataTypePtr create_decimal(UInt64 precision, UInt64 scale); +DataTypePtr create_decimal(UInt64 precision, UInt64 scale, bool use_v2); inline UInt32 least_decimal_precision_for(TypeIndex int_type) { switch (int_type) { @@ -171,20 +180,39 @@ class DataTypeDecimal final : public IDataType { T whole_part(T x) const { if (scale == 0) return x; - return x / get_scale_multiplier(); + if constexpr (IsDecimal128I) { + return x.value.val / get_scale_multiplier().value.val; + } else { + return x / get_scale_multiplier(); + } } T fractional_part(T x) const { if (scale == 0) return 0; - if (x < T(0)) x *= T(-1); - return x % get_scale_multiplier(); + if constexpr (IsDecimal128I) { + if (x.value.val < 0) x.value.val *= -1; + return x.value.val % get_scale_multiplier().value.val; + } else { + if (x < T(0)) x *= T(-1); + return x % get_scale_multiplier(); + } } - T max_whole_value() const { return get_scale_multiplier(max_precision() - scale) - T(1); } + T max_whole_value() const { + if constexpr (IsDecimal128I) { + return get_scale_multiplier(max_precision() - scale).value.val - 1; + } else { + return get_scale_multiplier(max_precision() - scale) - T(1); + } + } bool can_store_whole(T x) const { T max = max_whole_value(); - if (x > max || x < -max) return false; + if constexpr (IsDecimal128I) { + if (x.value.val > max.value.val || x.value.val < -max.value.val) return false; + } else { + if (x > max || x < -max) return false; + } return true; } @@ -218,7 +246,9 @@ template typename std::enable_if_t<(sizeof(T) >= sizeof(U)), const DataTypeDecimal> decimal_result_type( const DataTypeDecimal& tx, const DataTypeDecimal& ty, bool is_multiply, bool is_divide) { - if (config::enable_decimalv3) { + if constexpr (IsDecimalV2 && IsDecimalV2) { + return DataTypeDecimal(max_decimal_precision(), 9); + } else { UInt32 scale = (tx.get_scale() > ty.get_scale() ? tx.get_scale() : ty.get_scale()); if (is_multiply) { scale = tx.get_scale() + ty.get_scale(); @@ -226,8 +256,6 @@ typename std::enable_if_t<(sizeof(T) >= sizeof(U)), const DataTypeDecimal> de scale = tx.get_scale(); } return DataTypeDecimal(max_decimal_precision(), scale); - } else { - return DataTypeDecimal(max_decimal_precision(), 9); } } @@ -235,7 +263,9 @@ template typename std::enable_if_t<(sizeof(T) < sizeof(U)), const DataTypeDecimal> decimal_result_type( const DataTypeDecimal& tx, const DataTypeDecimal& ty, bool is_multiply, bool is_divide) { - if (config::enable_decimalv3) { + if constexpr (IsDecimalV2 && IsDecimalV2) { + return DataTypeDecimal(max_decimal_precision(), 9); + } else { UInt32 scale = (tx.get_scale() > ty.get_scale() ? tx.get_scale() : ty.get_scale()); if (is_multiply) { scale = tx.get_scale() + ty.get_scale(); @@ -243,28 +273,26 @@ typename std::enable_if_t<(sizeof(T) < sizeof(U)), const DataTypeDecimal> dec scale = tx.get_scale(); } return DataTypeDecimal(max_decimal_precision(), scale); - } else { - return DataTypeDecimal(max_decimal_precision(), 9); } } template const DataTypeDecimal decimal_result_type(const DataTypeDecimal& tx, const DataTypeNumber&, bool, bool) { - if (config::enable_decimalv3) { - return DataTypeDecimal(max_decimal_precision(), tx.get_scale()); - } else { + if constexpr (IsDecimalV2 && IsDecimalV2) { return DataTypeDecimal(max_decimal_precision(), 9); + } else { + return DataTypeDecimal(max_decimal_precision(), tx.get_scale()); } } template const DataTypeDecimal decimal_result_type(const DataTypeNumber&, const DataTypeDecimal& ty, bool, bool) { - if (config::enable_decimalv3) { - return DataTypeDecimal(max_decimal_precision(), ty.get_scale()); - } else { + if constexpr (IsDecimalV2 && IsDecimalV2) { return DataTypeDecimal(max_decimal_precision(), 9); + } else { + return DataTypeDecimal(max_decimal_precision(), ty.get_scale()); } } @@ -278,6 +306,8 @@ inline UInt32 get_decimal_scale(const IDataType& data_type, if (auto* decimal_type = check_decimal(data_type)) return decimal_type->get_scale(); if (auto* decimal_type = check_decimal(data_type)) return decimal_type->get_scale(); if (auto* decimal_type = check_decimal(data_type)) return decimal_type->get_scale(); + if (auto* decimal_type = check_decimal(data_type)) + return decimal_type->get_scale(); return default_value; } @@ -291,6 +321,18 @@ template <> inline constexpr bool IsDataTypeDecimal> = true; template <> inline constexpr bool IsDataTypeDecimal> = true; +template <> +inline constexpr bool IsDataTypeDecimal> = true; + +template +constexpr bool IsDataTypeDecimalV2 = false; +template <> +inline constexpr bool IsDataTypeDecimalV2> = true; + +template +constexpr bool IsDataTypeDecimal128I = false; +template <> +inline constexpr bool IsDataTypeDecimal128I> = true; template constexpr bool IsDataTypeDecimalOrNumber = @@ -303,21 +345,52 @@ convert_decimals(const typename FromDataType::FieldType& value, UInt32 scale_fro UInt32 scale_to) { using FromFieldType = typename FromDataType::FieldType; using ToFieldType = typename ToDataType::FieldType; - using MaxFieldType = std::conditional_t<(sizeof(FromFieldType) > sizeof(ToFieldType)), - FromFieldType, ToFieldType>; + using MaxFieldType = + std::conditional_t<(sizeof(FromFieldType) == sizeof(ToFieldType)) && + (std::is_same_v || + std::is_same_v), + Decimal128I, + std::conditional_t<(sizeof(FromFieldType) > sizeof(ToFieldType)), + FromFieldType, ToFieldType>>; using MaxNativeType = typename MaxFieldType::NativeType; MaxNativeType converted_value; if (scale_to > scale_from) { converted_value = DataTypeDecimal::get_scale_multiplier(scale_to - scale_from); - if (common::mul_overflow(static_cast(value), converted_value, - converted_value)) { - LOG(WARNING) << "Decimal convert overflow"; + if constexpr (IsDecimal128I) { + if (common::mul_overflow(static_cast(value).val, converted_value.val, + converted_value.val)) { + LOG(WARNING) << "Decimal convert overflow"; + } + } else if constexpr (IsDecimal128I) { + if (common::mul_overflow(static_cast(value.value.val), converted_value, + converted_value)) { + LOG(WARNING) << "Decimal convert overflow"; + } + } else { + if (common::mul_overflow(static_cast(value), converted_value, + converted_value)) { + LOG(WARNING) << "Decimal convert overflow"; + } } - } else - converted_value = - value / DataTypeDecimal::get_scale_multiplier(scale_from - scale_to); + } else { + if constexpr (IsDecimal128I) { + if constexpr (IsDecimal128I) { + converted_value = + value.value.val / + DataTypeDecimal::get_scale_multiplier(scale_from - scale_to) + .value.val; + } else { + converted_value = value / DataTypeDecimal::get_scale_multiplier( + scale_from - scale_to) + .value.val; + } + } else { + converted_value = value / DataTypeDecimal::get_scale_multiplier( + scale_from - scale_to); + } + } if constexpr (sizeof(FromFieldType) > sizeof(ToFieldType)) { if (converted_value < std::numeric_limits::min() || @@ -326,7 +399,11 @@ convert_decimals(const typename FromDataType::FieldType& value, UInt32 scale_fro } } - return converted_value; + if constexpr (IsDecimal128I) { + return converted_value.val; + } else { + return converted_value; + } } template @@ -337,10 +414,14 @@ convert_from_decimal(const typename FromDataType::FieldType& value, UInt32 scale using ToFieldType = typename ToDataType::FieldType; if constexpr (std::is_floating_point_v) { - if (config::enable_decimalv3) { + if constexpr (IsDecimalV2) { + return binary_cast(value); + } else if constexpr (IsDecimal128I) { + return *reinterpret_cast(&value) / + FromDataType::get_scale_multiplier(scale).value.val; + } else { return static_cast(value) / FromDataType::get_scale_multiplier(scale); } - return binary_cast(value); } else { FromFieldType converted_value = convert_decimals(value, scale, 0); @@ -348,22 +429,41 @@ convert_from_decimal(const typename FromDataType::FieldType& value, UInt32 scale if constexpr (sizeof(FromFieldType) > sizeof(ToFieldType) || !std::numeric_limits::is_signed) { if constexpr (std::numeric_limits::is_signed) { - if (converted_value < std::numeric_limits::min() || - converted_value > std::numeric_limits::max()) { - LOG(WARNING) << "Decimal convert overflow"; + if constexpr (IsDecimal128I) { + if (converted_value.value.val < std::numeric_limits::min() || + converted_value.value.val > std::numeric_limits::max()) { + LOG(WARNING) << "Decimal convert overflow"; + } + } else { + if (converted_value < std::numeric_limits::min() || + converted_value > std::numeric_limits::max()) { + LOG(WARNING) << "Decimal convert overflow"; + } } } else { using CastIntType = std::conditional_t, Int128, Int64>; - if (converted_value < 0 || - converted_value > - static_cast(std::numeric_limits::max())) { - LOG(WARNING) << "Decimal convert overflow"; + if constexpr (IsDecimal128I) { + if (converted_value.value.val < 0 || + converted_value.value.val > + static_cast(std::numeric_limits::max())) { + LOG(WARNING) << "Decimal convert overflow"; + } + } else { + if (converted_value < 0 || + converted_value > + static_cast(std::numeric_limits::max())) { + LOG(WARNING) << "Decimal convert overflow"; + } } } } - return converted_value; + if constexpr (IsDecimal128I) { + return converted_value.value.val; + } else { + return converted_value; + } } } @@ -379,7 +479,12 @@ convert_to_decimal(const typename FromDataType::FieldType& value, UInt32 scale) LOG(WARNING) << "Decimal convert overflow. Cannot convert infinity or NaN to decimal"; } - auto out = value * ToDataType::get_scale_multiplier(scale); + FromFieldType out; + if constexpr (IsDataTypeDecimal128I) { + out = value * ToDataType::get_scale_multiplier(scale).value.val; + } else { + out = value * ToDataType::get_scale_multiplier(scale); + } if (out <= static_cast(std::numeric_limits::min()) || out >= static_cast(std::numeric_limits::max())) { LOG(WARNING) << "Decimal convert overflow. Float is out of Decimal range"; diff --git a/be/src/vec/data_types/data_type_factory.cpp b/be/src/vec/data_types/data_type_factory.cpp index 2696b742664e14..b40c03cb03622a 100644 --- a/be/src/vec/data_types/data_type_factory.cpp +++ b/be/src/vec/data_types/data_type_factory.cpp @@ -117,8 +117,8 @@ DataTypePtr DataTypeFactory::create_data_type(const TypeDescriptor& col_desc, bo break; case TYPE_DECIMAL32: case TYPE_DECIMAL64: - case TYPE_DECIMAL128: - nested = vectorized::create_decimal(col_desc.precision, col_desc.scale); + case TYPE_DECIMAL128I: + nested = vectorized::create_decimal(col_desc.precision, col_desc.scale, false); break; // Just Mock A NULL Type in Vec Exec Engine case TYPE_NULL: @@ -200,8 +200,8 @@ DataTypePtr DataTypeFactory::_create_primitive_data_type(const FieldType& type, break; case OLAP_FIELD_TYPE_DECIMAL32: case OLAP_FIELD_TYPE_DECIMAL64: - case OLAP_FIELD_TYPE_DECIMAL128: - result = vectorized::create_decimal(precision, scale); + case OLAP_FIELD_TYPE_DECIMAL128I: + result = vectorized::create_decimal(precision, scale, false); break; default: DCHECK(false) << "Invalid FieldType:" << (int)type; @@ -280,6 +280,10 @@ DataTypePtr DataTypeFactory::create_data_type(const PColumnMeta& pcolumn) { nested = std::make_shared>(pcolumn.decimal_param().precision(), pcolumn.decimal_param().scale()); break; + case PGenericType::DECIMAL128I: + nested = std::make_shared>(pcolumn.decimal_param().precision(), + pcolumn.decimal_param().scale()); + break; case PGenericType::BITMAP: nested = std::make_shared(); break; diff --git a/be/src/vec/data_types/data_type_factory.hpp b/be/src/vec/data_types/data_type_factory.hpp index 52e034f31363d1..8d24f28cc0faea 100644 --- a/be/src/vec/data_types/data_type_factory.hpp +++ b/be/src/vec/data_types/data_type_factory.hpp @@ -76,6 +76,8 @@ class DataTypeFactory { BeConsts::MAX_DECIMAL64_PRECISION, 0)}, {"Decimal128", std::make_shared>( BeConsts::MAX_DECIMAL128_PRECISION, 0)}, + {"Decimal128I", std::make_shared>( + BeConsts::MAX_DECIMAL128_PRECISION, 0)}, {"Json", std::make_shared()}, }; for (auto const& [key, val] : base_type_map) { diff --git a/be/src/vec/data_types/get_least_supertype.cpp b/be/src/vec/data_types/get_least_supertype.cpp index 4d22ab48702716..8c8b9d7e7ba7d6 100644 --- a/be/src/vec/data_types/get_least_supertype.cpp +++ b/be/src/vec/data_types/get_least_supertype.cpp @@ -168,9 +168,11 @@ DataTypePtr get_least_supertype(const DataTypes& types) { UInt32 have_decimal32 = type_ids.count(TypeIndex::Decimal32); UInt32 have_decimal64 = type_ids.count(TypeIndex::Decimal64); UInt32 have_decimal128 = type_ids.count(TypeIndex::Decimal128); + UInt32 have_decimal128i = type_ids.count(TypeIndex::Decimal128I); - if (have_decimal32 || have_decimal64 || have_decimal128) { - UInt32 num_supported = have_decimal32 + have_decimal64 + have_decimal128; + if (have_decimal32 || have_decimal64 || have_decimal128 || have_decimal128i) { + UInt32 num_supported = + have_decimal32 + have_decimal64 + have_decimal128 + have_decimal128i; std::vector int_ids = { TypeIndex::Int8, TypeIndex::UInt8, TypeIndex::Int16, TypeIndex::UInt16, @@ -215,6 +217,9 @@ DataTypePtr get_least_supertype(const DataTypes& types) { if (have_decimal128 || min_precision > DataTypeDecimal::max_precision()) return std::make_shared>( DataTypeDecimal::max_precision(), max_scale); + if (have_decimal128i || min_precision > DataTypeDecimal::max_precision()) + return std::make_shared>( + DataTypeDecimal::max_precision(), max_scale); if (have_decimal64 || min_precision > DataTypeDecimal::max_precision()) return std::make_shared>( DataTypeDecimal::max_precision(), max_scale); diff --git a/be/src/vec/exec/format/format_common.h b/be/src/vec/exec/format/format_common.h index 33e218aca23a00..c5570f967dc806 100644 --- a/be/src/vec/exec/format/format_common.h +++ b/be/src/vec/exec/format/format_common.h @@ -38,7 +38,8 @@ struct DecimalScaleParams { return common::exp10_i32(n); } else if constexpr (std::is_same_v) { return common::exp10_i64(n); - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v || + std::is_same_v) { return common::exp10_i128(n); } else { return DecimalPrimitiveType(1); diff --git a/be/src/vec/exec/format/orc/vorc_reader.cpp b/be/src/vec/exec/format/orc/vorc_reader.cpp index 17f4bab7abdb05..6b8c23c760102c 100644 --- a/be/src/vec/exec/format/orc/vorc_reader.cpp +++ b/be/src/vec/exec/format/orc/vorc_reader.cpp @@ -691,6 +691,9 @@ Status OrcReader::_orc_column_to_doris_column(const std::string& col_name, case TypeIndex::Decimal128: return _decode_decimal_column(col_name, data_column, data_type, _decimal_scale_params, cvb, num_values); + case TypeIndex::Decimal128I: + return _decode_decimal_column(col_name, data_column, data_type, + _decimal_scale_params, cvb, num_values); case TypeIndex::Date: return _decode_time_column( col_name, data_column, cvb, num_values); diff --git a/be/src/vec/exec/format/parquet/parquet_common.cpp b/be/src/vec/exec/format/parquet/parquet_common.cpp index 5b8ff2f80129ce..fe514d76e2084a 100644 --- a/be/src/vec/exec/format/parquet/parquet_common.cpp +++ b/be/src/vec/exec/format/parquet/parquet_common.cpp @@ -388,6 +388,17 @@ Status FixLengthDecoder::decode_values(MutableColumnPtr& doris_column, DataTypeP return _decode_primitive_decimal(doris_column, data_type, select_vector); } break; + case TypeIndex::Decimal128I: + if (_physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY) { + return _decode_binary_decimal(doris_column, data_type, select_vector); + } else if (_physical_type == tparquet::Type::INT32) { + return _decode_primitive_decimal(doris_column, data_type, + select_vector); + } else if (_physical_type == tparquet::Type::INT64) { + return _decode_primitive_decimal(doris_column, data_type, + select_vector); + } + break; case TypeIndex::String: case TypeIndex::FixedString: if (_physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY) { @@ -572,6 +583,8 @@ Status ByteArrayDecoder::decode_values(MutableColumnPtr& doris_column, DataTypeP return _decode_binary_decimal(doris_column, data_type, select_vector); case TypeIndex::Decimal128: return _decode_binary_decimal(doris_column, data_type, select_vector); + case TypeIndex::Decimal128I: + return _decode_binary_decimal(doris_column, data_type, select_vector); default: break; } diff --git a/be/src/vec/exec/join/vhash_join_node.cpp b/be/src/vec/exec/join/vhash_join_node.cpp index f6da393576cd18..79beb01a83bce4 100644 --- a/be/src/vec/exec/join/vhash_join_node.cpp +++ b/be/src/vec/exec/join/vhash_join_node.cpp @@ -870,7 +870,7 @@ void HashJoinNode::_hash_table_init() { case TYPE_DECIMALV2: case TYPE_DECIMAL32: case TYPE_DECIMAL64: - case TYPE_DECIMAL128: { + case TYPE_DECIMAL128I: { DataTypePtr& type_ptr = _build_expr_ctxs[0]->root()->data_type(); TypeIndex idx = _build_expr_ctxs[0]->root()->is_nullable() ? assert_cast(*type_ptr) diff --git a/be/src/vec/exec/scan/vscan_node.cpp b/be/src/vec/exec/scan/vscan_node.cpp index 156a8c4def329c..74a3c9f80112a7 100644 --- a/be/src/vec/exec/scan/vscan_node.cpp +++ b/be/src/vec/exec/scan/vscan_node.cpp @@ -343,7 +343,7 @@ Status VScanNode::_normalize_conjuncts() { M(HLL) \ M(DECIMAL32) \ M(DECIMAL64) \ - M(DECIMAL128) \ + M(DECIMAL128I) \ M(DECIMALV2) \ M(BOOLEAN) APPLY_FOR_PRIMITIVE_TYPE(M) @@ -863,7 +863,7 @@ Status VScanNode::_change_value_range(ColumnValueRange& temp_rang (PrimitiveType == TYPE_SMALLINT) || (PrimitiveType == TYPE_INT) || (PrimitiveType == TYPE_BIGINT) || (PrimitiveType == TYPE_LARGEINT) || (PrimitiveType == TYPE_DECIMAL32) || (PrimitiveType == TYPE_DECIMAL64) || - (PrimitiveType == TYPE_DECIMAL128) || (PrimitiveType == TYPE_STRING) || + (PrimitiveType == TYPE_DECIMAL128I) || (PrimitiveType == TYPE_STRING) || (PrimitiveType == TYPE_BOOLEAN) || (PrimitiveType == TYPE_DATEV2)) { if constexpr (IsFixed) { func(temp_range, diff --git a/be/src/vec/exec/vaggregation_node.cpp b/be/src/vec/exec/vaggregation_node.cpp index 83057e1b845bdf..86891efaad6564 100644 --- a/be/src/vec/exec/vaggregation_node.cpp +++ b/be/src/vec/exec/vaggregation_node.cpp @@ -187,7 +187,7 @@ void AggregationNode::_init_hash_method(std::vector& probe_exprs) case TYPE_DECIMALV2: case TYPE_DECIMAL32: case TYPE_DECIMAL64: - case TYPE_DECIMAL128: { + case TYPE_DECIMAL128I: { DataTypePtr& type_ptr = probe_exprs[0]->root()->data_type(); TypeIndex idx = is_nullable ? assert_cast(*type_ptr) .get_nested_type() diff --git a/be/src/vec/exec/vjdbc_connector.cpp b/be/src/vec/exec/vjdbc_connector.cpp index b2f730ed140aa5..ad2733873495f3 100644 --- a/be/src/vec/exec/vjdbc_connector.cpp +++ b/be/src/vec/exec/vjdbc_connector.cpp @@ -312,6 +312,36 @@ Status JdbcConnector::_convert_column_data(JNIEnv* env, jobject jobj, decimal_slot.value()); break; } + case TYPE_DECIMAL32: { + std::string data = _jobject_to_string(env, jobj); + StringParser::ParseResult result = StringParser::PARSE_SUCCESS; + const Int32 decimal_slot = StringParser::string_to_decimal( + data.c_str(), data.length(), slot_desc->type().precision, slot_desc->type().scale, + &result); + reinterpret_cast(col_ptr)->insert_data( + reinterpret_cast(&decimal_slot), 0); + break; + } + case TYPE_DECIMAL64: { + std::string data = _jobject_to_string(env, jobj); + StringParser::ParseResult result = StringParser::PARSE_SUCCESS; + const Int64 decimal_slot = StringParser::string_to_decimal( + data.c_str(), data.length(), slot_desc->type().precision, slot_desc->type().scale, + &result); + reinterpret_cast(col_ptr)->insert_data( + reinterpret_cast(&decimal_slot), 0); + break; + } + case TYPE_DECIMAL128I: { + std::string data = _jobject_to_string(env, jobj); + StringParser::ParseResult result = StringParser::PARSE_SUCCESS; + const Int128I decimal_slot = StringParser::string_to_decimal( + data.c_str(), data.length(), slot_desc->type().precision, slot_desc->type().scale, + &result); + reinterpret_cast(col_ptr)->insert_data( + reinterpret_cast(&decimal_slot), 0); + break; + } default: { std::string error_msg = fmt::format("Fail to convert jdbc value to {} on column: {}.", diff --git a/be/src/vec/exec/vschema_scan_node.cpp b/be/src/vec/exec/vschema_scan_node.cpp index 0f2710b6655b84..f6e36800a0391e 100644 --- a/be/src/vec/exec/vschema_scan_node.cpp +++ b/be/src/vec/exec/vschema_scan_node.cpp @@ -433,7 +433,7 @@ Status VSchemaScanNode::write_slot_to_vectorized_column(void* slot, SlotDescript } case TYPE_DECIMALV2: - case TYPE_DECIMAL128: { + case TYPE_DECIMAL128I: { Int128 num = (reinterpret_cast(slot))->value; reinterpret_cast*>(col_ptr)->insert_value(num); break; diff --git a/be/src/vec/exec/vset_operation_node.cpp b/be/src/vec/exec/vset_operation_node.cpp index d4be9f4eb4ce84..65d7de572492cd 100644 --- a/be/src/vec/exec/vset_operation_node.cpp +++ b/be/src/vec/exec/vset_operation_node.cpp @@ -175,7 +175,7 @@ void VSetOperationNode::hash_table_init() { break; case TYPE_LARGEINT: case TYPE_DECIMALV2: - case TYPE_DECIMAL128: + case TYPE_DECIMAL128I: _hash_table_variants->emplace>(); break; default: diff --git a/be/src/vec/exprs/vliteral.cpp b/be/src/vec/exprs/vliteral.cpp index 8851b53331e268..c6f3aa422469f8 100644 --- a/be/src/vec/exprs/vliteral.cpp +++ b/be/src/vec/exprs/vliteral.cpp @@ -143,7 +143,7 @@ void VLiteral::init(const TExprNode& node) { DCHECK_EQ(node.node_type, TExprNodeType::DECIMAL_LITERAL); DCHECK(node.__isset.decimal_literal); DataTypePtr type_ptr = create_decimal(node.type.types[0].scalar_type.precision, - node.type.types[0].scalar_type.scale); + node.type.types[0].scalar_type.scale, false); auto val = typeid_cast*>(type_ptr.get()) ->parse_from_string(node.decimal_literal.value); auto scale = @@ -155,7 +155,7 @@ void VLiteral::init(const TExprNode& node) { DCHECK_EQ(node.node_type, TExprNodeType::DECIMAL_LITERAL); DCHECK(node.__isset.decimal_literal); DataTypePtr type_ptr = create_decimal(node.type.types[0].scalar_type.precision, - node.type.types[0].scalar_type.scale); + node.type.types[0].scalar_type.scale, false); auto val = typeid_cast*>(type_ptr.get()) ->parse_from_string(node.decimal_literal.value); auto scale = @@ -163,16 +163,16 @@ void VLiteral::init(const TExprNode& node) { field = DecimalField(val, scale); break; } - case TYPE_DECIMAL128: { + case TYPE_DECIMAL128I: { DCHECK_EQ(node.node_type, TExprNodeType::DECIMAL_LITERAL); DCHECK(node.__isset.decimal_literal); DataTypePtr type_ptr = create_decimal(node.type.types[0].scalar_type.precision, - node.type.types[0].scalar_type.scale); + node.type.types[0].scalar_type.scale, false); auto val = typeid_cast*>(type_ptr.get()) ->parse_from_string(node.decimal_literal.value); auto scale = typeid_cast*>(type_ptr.get())->get_scale(); - field = DecimalField(val, scale); + field = DecimalField(val, scale); break; } default: { @@ -271,7 +271,7 @@ std::string VLiteral::debug_string() const { out); break; } - case TYPE_DECIMAL128: { + case TYPE_DECIMAL128I: { write_text(*(reinterpret_cast(ref.data)), _type.scale, out); break; diff --git a/be/src/vec/functions/array/function_array_aggregation.cpp b/be/src/vec/functions/array/function_array_aggregation.cpp index d1b8623151a992..9d513191f668fe 100644 --- a/be/src/vec/functions/array/function_array_aggregation.cpp +++ b/be/src/vec/functions/array/function_array_aggregation.cpp @@ -55,20 +55,27 @@ struct ArrayAggregateResultImpl { template struct ArrayAggregateResultImpl { - using Result = std::conditional_t, Decimal128, Float64>; + using Result = + std::conditional_t, Decimal128, + std::conditional_t, Decimal128I, Float64>>; }; template struct ArrayAggregateResultImpl { - using Result = std::conditional_t, Decimal128, Float64>; + using Result = + std::conditional_t, Decimal128, + std::conditional_t, Decimal128I, Float64>>; }; template struct ArrayAggregateResultImpl { using Result = std::conditional_t< - IsDecimalNumber, Decimal128, - std::conditional_t, Float64, - std::conditional_t, Int128, Int64>>>; + IsDecimalV2, Decimal128, + std::conditional_t< + IsDecimalNumber, Decimal128I, + std::conditional_t< + IsFloatNumber, Float64, + std::conditional_t, Int128, Int64>>>>; }; template @@ -168,6 +175,7 @@ struct ArrayAggregateImpl { execute_type(res, type, data, offsets) || execute_type(res, type, data, offsets) || execute_type(res, type, data, offsets) || + execute_type(res, type, data, offsets) || execute_type(res, type, data, offsets) || execute_type(res, type, data, offsets) || execute_type(res, type, data, offsets) || diff --git a/be/src/vec/functions/function_binary_arithmetic.h b/be/src/vec/functions/function_binary_arithmetic.h index 60d329cae083e8..50b1ab84df442a 100644 --- a/be/src/vec/functions/function_binary_arithmetic.h +++ b/be/src/vec/functions/function_binary_arithmetic.h @@ -259,7 +259,12 @@ struct DecimalBinaryOperation { ArrayC& c, ResultType scale_a [[maybe_unused]], ResultType scale_b [[maybe_unused]], NullMap& null_map) { size_t size = a.size(); - if (config::enable_decimalv3) { + if constexpr (IsDecimalV2 || IsDecimalV2) { + /// default: use it if no return before + for (size_t i = 0; i < size; ++i) { + c[i] = apply(a[i], b[i], null_map[i]); + } + } else { if constexpr (OpTraits::is_division && IsDecimalNumber) { for (size_t i = 0; i < size; ++i) { c[i] = apply_scaled_div(a[i], b[i], scale_a, null_map[i]); @@ -279,10 +284,6 @@ struct DecimalBinaryOperation { } } } - /// default: use it if no return before - for (size_t i = 0; i < size; ++i) { - c[i] = apply(a[i], b[i], null_map[i]); - } } static void vector_constant(const typename Traits::ArrayA& a, B b, ArrayC& c, @@ -508,7 +509,16 @@ struct DecimalBinaryOperation { private: /// there's implicit type conversion here static NativeResultType apply(NativeResultType a, NativeResultType b) { - if (config::enable_decimalv3) { + if constexpr (IsDecimalV2 || IsDecimalV2) { + // Now, Doris only support decimal +-*/ decimal. + // overflow in consider in operator + DecimalV2Value l(a); + DecimalV2Value r(b); + auto ans = Op::template apply(l, r); + NativeResultType result; + memcpy(&result, &ans, sizeof(NativeResultType)); + return result; + } else { if constexpr (OpTraits::can_overflow && check_overflow) { NativeResultType res; // TODO handle overflow gracefully @@ -521,29 +531,20 @@ struct DecimalBinaryOperation { return Op::template apply(a, b); } } - - // Now, Doris only support decimal +-*/ decimal. - // overflow in consider in operator - DecimalV2Value l(a); - DecimalV2Value r(b); - auto ans = Op::template apply(l, r); - NativeResultType result; - memcpy(&result, &ans, sizeof(NativeResultType)); - return result; } /// null_map for divide and mod static NativeResultType apply(NativeResultType a, NativeResultType b, UInt8& is_null) { - if (config::enable_decimalv3) { + if constexpr (IsDecimalV2 || IsDecimalV2) { + DecimalV2Value l(a); + DecimalV2Value r(b); + auto ans = Op::template apply(l, r, is_null); + NativeResultType result; + memcpy(&result, &ans, std::min(sizeof(result), sizeof(ans))); + return result; + } else { return Op::template apply(a, b, is_null); } - - DecimalV2Value l(a); - DecimalV2Value r(b); - auto ans = Op::template apply(l, r, is_null); - NativeResultType result; - memcpy(&result, &ans, std::min(sizeof(result), sizeof(ans))); - return result; } template @@ -753,12 +754,11 @@ struct ConstOrVectorAdapter { typename ResultDataType::FieldType scale_a; typename ResultDataType::FieldType scale_b; - if constexpr (OpTraits::is_division && IsDataTypeDecimal) { - if (config::enable_decimalv3) { - scale_a = type_right.get_scale_multiplier(); - scale_b = 1; - return std::make_tuple(type, scale_a, scale_b); - } + if constexpr (OpTraits::is_division && IsDataTypeDecimal && + !IsDecimalV2 && !IsDecimalV2) { + scale_a = type_right.get_scale_multiplier(); + scale_b = 1; + return std::make_tuple(type, scale_a, scale_b); } scale_a = type.scale_factor_for(type_left, OpTraits::is_multiply); scale_b = type.scale_factor_for(type_right, OpTraits::is_multiply || OpTraits::is_division); diff --git a/be/src/vec/functions/function_case.h b/be/src/vec/functions/function_case.h index f3961c4eef0182..c9e001ca405ec1 100644 --- a/be/src/vec/functions/function_case.h +++ b/be/src/vec/functions/function_case.h @@ -292,8 +292,15 @@ class FunctionCase : public IFunction { ->get_data() .data(); - for (int row_idx = 0; row_idx < rows_count; row_idx++) { - result_raw_data[row_idx] += (then_idx[row_idx] == i) * column_raw_data[row_idx]; + if constexpr (std::is_same_v) { + for (int row_idx = 0; row_idx < rows_count; row_idx++) { + result_raw_data[row_idx] += + (then_idx[row_idx] == i) * column_raw_data[row_idx].value.val; + } + } else { + for (int row_idx = 0; row_idx < rows_count; row_idx++) { + result_raw_data[row_idx] += (then_idx[row_idx] == i) * column_raw_data[row_idx]; + } } } } diff --git a/be/src/vec/functions/function_cast.h b/be/src/vec/functions/function_cast.h index befb22532ae8ea..0880a723f1f22a 100644 --- a/be/src/vec/functions/function_cast.h +++ b/be/src/vec/functions/function_cast.h @@ -564,6 +564,9 @@ struct NameToDecimal64 { struct NameToDecimal128 { static constexpr auto name = "toDecimal128"; }; +struct NameToDecimal128I { + static constexpr auto name = "toDecimal128I"; +}; struct NameToUInt8 { static constexpr auto name = "toUInt8"; }; @@ -805,9 +808,9 @@ class FunctionConvert : public IFunction { using Monotonic = MonotonicityImpl; static constexpr auto name = Name::name; - static constexpr bool to_decimal = std::is_same_v || - std::is_same_v || - std::is_same_v; + static constexpr bool to_decimal = + std::is_same_v || std::is_same_v || + std::is_same_v || std::is_same_v; static FunctionPtr create() { return std::make_shared(); } @@ -918,6 +921,8 @@ using FunctionToDecimal64 = FunctionConvert, NameToDecimal64, UnknownMonotonicity>; using FunctionToDecimal128 = FunctionConvert, NameToDecimal128, UnknownMonotonicity>; +using FunctionToDecimal128I = + FunctionConvert, NameToDecimal128I, UnknownMonotonicity>; using FunctionToDate = FunctionConvert; using FunctionToDateTime = FunctionConvert; using FunctionToDateV2 = FunctionConvert; @@ -983,6 +988,10 @@ struct FunctionTo> { using Type = FunctionToDecimal128; }; template <> +struct FunctionTo> { + using Type = FunctionToDecimal128I; +}; +template <> struct FunctionTo { using Type = FunctionToDate; }; @@ -1131,6 +1140,9 @@ struct ConvertImpl, Name> template struct ConvertImpl, Name> : ConvertThroughParsing, Name> {}; +template +struct ConvertImpl, Name> + : ConvertThroughParsing, Name> {}; template class FunctionConvertFromString : public IFunction { @@ -1649,7 +1661,8 @@ class FunctionCast final : public IFunctionBase { if constexpr (std::is_same_v> || std::is_same_v> || - std::is_same_v>) { + std::is_same_v> || + std::is_same_v>) { ret = create_decimal_wrapper(from_type, check_and_get_data_type(to_type.get())); return true; diff --git a/be/src/vec/functions/function_coalesce.cpp b/be/src/vec/functions/function_coalesce.cpp index 1d1ae5d59388b0..768f1e77b992a5 100644 --- a/be/src/vec/functions/function_coalesce.cpp +++ b/be/src/vec/functions/function_coalesce.cpp @@ -205,8 +205,13 @@ class FunctionCoalesce : public IFunction { // true: null_map_data[row]==0 && filled_idx[row]==0 // if true, could filled current row data into result column for (size_t row = 0; row < input_rows_count; ++row) { - result_raw_data[row] += - (!(null_map_data[row] | filled_flag[row])) * column_raw_data[row]; + if constexpr (std::is_same_v) { + result_raw_data[row] += + (!(null_map_data[row] | filled_flag[row])) * column_raw_data[row].value.val; + } else { + result_raw_data[row] += + (!(null_map_data[row] | filled_flag[row])) * column_raw_data[row]; + } filled_flag[row] += (!(null_map_data[row] | filled_flag[row])); } return Status::OK(); diff --git a/be/src/vec/functions/function_variadic_arguments.h b/be/src/vec/functions/function_variadic_arguments.h index e6a0212e75be69..2d0294297c32bb 100644 --- a/be/src/vec/functions/function_variadic_arguments.h +++ b/be/src/vec/functions/function_variadic_arguments.h @@ -41,8 +41,8 @@ class FunctionVariadicArgumentsBase : public IFunction { DataTypePtr get_return_type_impl(const ColumnsWithTypeAndName& arguments) const override { DataTypePtr res; - if constexpr (IsDataTypeDecimal) { - res = create_decimal(27, 9); + if constexpr (IsDataTypeDecimalV2) { + res = create_decimal(27, 9, true); if (!res) { LOG(FATAL) << "Someting wrong with toDecimalNNOrZero() or toDecimalNNOrNull()"; } diff --git a/be/src/vec/functions/functions_comparison.h b/be/src/vec/functions/functions_comparison.h index c5f5c8888875fc..db616348649166 100644 --- a/be/src/vec/functions/functions_comparison.h +++ b/be/src/vec/functions/functions_comparison.h @@ -403,6 +403,13 @@ class FunctionComparison : public IFunction { return Status::RuntimeError("Illegal column {} of first argument of function {}", col_left_untyped->get_name(), get_name()); + } else if (is_decimal_v2(left_type) || is_decimal_v2(right_type)) { + if (!allow_decimal_comparison(left_type, right_type)) { + return Status::RuntimeError("No operation {} between {} and {}", get_name(), + left_type->get_name(), right_type->get_name()); + } + return execute_decimal(block, result, col_with_type_and_name_left, + col_with_type_and_name_right); } else if (is_decimal(left_type) || is_decimal(right_type)) { if (!allow_decimal_comparison(left_type, right_type)) { return Status::RuntimeError("No operation {} between {} and {}", get_name(), diff --git a/be/src/vec/io/io_helper.h b/be/src/vec/io/io_helper.h index d8b7edfd2e07f7..0c6a2b5f3a029a 100644 --- a/be/src/vec/io/io_helper.h +++ b/be/src/vec/io/io_helper.h @@ -56,6 +56,10 @@ template <> inline Int128 decimal_scale_multiplier(UInt32 scale) { return common::exp10_i128(scale); } +template <> +inline Int128I decimal_scale_multiplier(UInt32 scale) { + return common::exp10_i128(scale); +} inline std::string int128_to_string(__int128_t value) { fmt::memory_buffer buffer; @@ -69,25 +73,44 @@ inline std::string int128_to_string(UInt128 value) { template void write_text(Decimal value, UInt32 scale, std::ostream& ostr) { - if (value < Decimal(0)) { - value *= Decimal(-1); - ostr << '-'; + if constexpr (std::is_same_v) { + if (value.value.val < 0) { + value.value.val *= -1; + ostr << '-'; + } + } else { + if (value < Decimal(0)) { + value *= Decimal(-1); + ostr << '-'; + } } T whole_part = value; + if (scale) { - whole_part = value / decimal_scale_multiplier(scale); + if constexpr (std::is_same_v) { + whole_part = value.value.val / decimal_scale_multiplier(scale); + } else { + whole_part = value / decimal_scale_multiplier(scale); + } } - if constexpr (std::is_same::value) { + if constexpr (std::is_same_v) { ostr << int128_to_string(whole_part); + } else if constexpr (std::is_same_v) { + ostr << int128_to_string(whole_part.val); } else { ostr << whole_part; } if (scale) { ostr << '.'; String str_fractional(scale, '0'); - for (Int32 pos = scale - 1; pos >= 0; --pos, value /= Decimal(10)) - str_fractional[pos] += value % Decimal(10); + for (Int32 pos = scale - 1; pos >= 0; --pos, value /= Decimal(10)) { + if constexpr (std::is_same_v) { + str_fractional[pos] += value.value.val % 10; + } else { + str_fractional[pos] += value % Decimal(10); + } + } ostr.write(str_fractional.data(), scale); } } @@ -318,7 +341,7 @@ bool read_datetime_v2_text_impl(T& x, ReadBuffer& buf, UInt32 scale = -1) { template bool read_decimal_text_impl(T& x, ReadBuffer& buf, UInt32 precision, UInt32 scale) { static_assert(IsDecimalNumber); - if (config::enable_decimalv3) { + if constexpr (!std::is_same_v) { StringParser::ParseResult result = StringParser::PARSE_SUCCESS; x.value = StringParser::string_to_decimal( @@ -326,15 +349,16 @@ bool read_decimal_text_impl(T& x, ReadBuffer& buf, UInt32 precision, UInt32 scal // only to match the is_all_read() check to prevent return null buf.position() = buf.end(); return result != StringParser::PARSE_FAILURE; - } - auto dv = binary_cast(x.value); - auto ans = dv.parse_from_str((const char*)buf.position(), buf.count()) == 0; + } else { + auto dv = binary_cast(x.value); + auto ans = dv.parse_from_str((const char*)buf.position(), buf.count()) == 0; - // only to match the is_all_read() check to prevent return null - buf.position() = buf.end(); + // only to match the is_all_read() check to prevent return null + buf.position() = buf.end(); - x.value = dv.value(); - return ans; + x.value = dv.value(); + return ans; + } } template diff --git a/be/src/vec/olap/olap_data_convertor.cpp b/be/src/vec/olap/olap_data_convertor.cpp index 33598c4e982e5e..48fa0c6313d54a 100644 --- a/be/src/vec/olap/olap_data_convertor.cpp +++ b/be/src/vec/olap/olap_data_convertor.cpp @@ -75,8 +75,8 @@ OlapBlockDataConvertor::create_olap_column_data_convertor(const TabletColumn& co case FieldType::OLAP_FIELD_TYPE_DECIMAL64: { return std::make_unique>(); } - case FieldType::OLAP_FIELD_TYPE_DECIMAL128: { - return std::make_unique>(); + case FieldType::OLAP_FIELD_TYPE_DECIMAL128I: { + return std::make_unique>(); } case FieldType::OLAP_FIELD_TYPE_JSONB: { return std::make_unique(); diff --git a/be/src/vec/runtime/vfile_result_writer.cpp b/be/src/vec/runtime/vfile_result_writer.cpp index bae84e53d3875b..fe4994256adf0c 100644 --- a/be/src/vec/runtime/vfile_result_writer.cpp +++ b/be/src/vec/runtime/vfile_result_writer.cpp @@ -335,7 +335,7 @@ Status VFileResultWriter::_write_csv_file(const Block& block) { _plain_text_outstream << col.type->to_string(*col.column, i); break; } - case TYPE_DECIMAL128: { + case TYPE_DECIMAL128I: { _plain_text_outstream << col.type->to_string(*col.column, i); break; } diff --git a/be/src/vec/runtime/vorc_writer.cpp b/be/src/vec/runtime/vorc_writer.cpp index 64a0d60e7e8574..c2c8057f25679d 100644 --- a/be/src/vec/runtime/vorc_writer.cpp +++ b/be/src/vec/runtime/vorc_writer.cpp @@ -431,8 +431,7 @@ Status VOrcWriterWrapper::write(const Block& block) { SET_NUM_ELEMENTS break; } - case TYPE_DECIMALV2: - case TYPE_DECIMAL128: { + case TYPE_DECIMALV2: { orc::Decimal128VectorBatch* cur_batch = dynamic_cast(root->fields[i]); if (null_map != nullptr) { @@ -450,9 +449,40 @@ Status VOrcWriterWrapper::write(const Block& block) { } } else if (const auto& not_null_column = check_and_get_column(col)) { - auto ptr = not_null_column->get_data().data(); + auto col_ptr = not_null_column->get_data().data(); for (size_t row_id = 0; row_id < sz; row_id++) { - auto v = ptr[row_id]; + auto v = col_ptr[row_id]; + orc::Int128 value(v >> 64, (uint64_t)v); + cur_batch->values[row_id] = value; + } + } else { + RETURN_WRONG_TYPE + } + SET_NUM_ELEMENTS + break; + } + case TYPE_DECIMAL128I: { + orc::Decimal128VectorBatch* cur_batch = + dynamic_cast(root->fields[i]); + if (null_map != nullptr) { + cur_batch->hasNulls = true; + auto& null_data = assert_cast(*null_map).get_data(); + for (size_t row_id = 0; row_id < sz; row_id++) { + if (null_data[row_id] != 0) { + cur_batch->notNull[row_id] = 0; + } else { + cur_batch->notNull[row_id] = 1; + auto& v = + assert_cast(*col).get_data()[row_id]; + orc::Int128 value(v.value >> 64, (uint64_t)v.value); + cur_batch->values[row_id] = value; + } + } + } else if (const auto& not_null_column = + check_and_get_column(col)) { + auto col_ptr = not_null_column->get_data().data(); + for (size_t row_id = 0; row_id < sz; row_id++) { + auto v = col_ptr[row_id].value.val; orc::Int128 value(v >> 64, (uint64_t)v); cur_batch->values[row_id] = value; } diff --git a/be/src/vec/runtime/vparquet_writer.cpp b/be/src/vec/runtime/vparquet_writer.cpp index 67d94d8f14c904..4a80d65c7b0e71 100644 --- a/be/src/vec/runtime/vparquet_writer.cpp +++ b/be/src/vec/runtime/vparquet_writer.cpp @@ -458,8 +458,8 @@ Status VParquetWriterWrapper::write(const Block& block) { DISPATCH_PARQUET_DECIMAL_WRITER(Decimal64) break; } - case TYPE_DECIMAL128: { - DISPATCH_PARQUET_DECIMAL_WRITER(Decimal128) + case TYPE_DECIMAL128I: { + DISPATCH_PARQUET_DECIMAL_WRITER(Decimal128I) break; } default: { diff --git a/be/src/vec/sink/vmysql_result_writer.cpp b/be/src/vec/sink/vmysql_result_writer.cpp index 9483f13c004449..b592206752a553 100644 --- a/be/src/vec/sink/vmysql_result_writer.cpp +++ b/be/src/vec/sink/vmysql_result_writer.cpp @@ -170,7 +170,7 @@ Status VMysqlResultWriter::_add_one_column(const ColumnPtr& column_ptr, result->result_batch.rows[i].append(_buffer.buf(), _buffer.length()); } } else if constexpr (type == TYPE_DECIMAL32 || type == TYPE_DECIMAL64 || - type == TYPE_DECIMAL128) { + type == TYPE_DECIMAL128I) { for (int i = 0; i < row_size; ++i) { if (0 != buf_ret) { return Status::InternalError("pack mysql buffer failed."); @@ -367,21 +367,19 @@ int VMysqlResultWriter::_add_one_cell(const ColumnPtr& column_ptr, size_t row_id ->to_string(*column, row_idx); return buffer.push_string(decimal_str.c_str(), decimal_str.length()); } else if (which.is_decimal128()) { - if (config::enable_decimalv3) { - DataTypePtr nested_type = type; - if (type->is_nullable()) { - nested_type = assert_cast(*type).get_nested_type(); - } - auto decimal_str = assert_cast*>(nested_type.get()) - ->to_string(*column, row_idx); - return buffer.push_string(decimal_str.c_str(), decimal_str.length()); - } else { - auto& column_data = - static_cast&>(*column).get_data(); - DecimalV2Value decimal_val(column_data[row_idx]); - auto decimal_str = decimal_val.to_string(); - return buffer.push_string(decimal_str.c_str(), decimal_str.length()); + auto& column_data = + static_cast&>(*column).get_data(); + DecimalV2Value decimal_val(column_data[row_idx]); + auto decimal_str = decimal_val.to_string(); + return buffer.push_string(decimal_str.c_str(), decimal_str.length()); + } else if (which.is_decimal128i()) { + DataTypePtr nested_type = type; + if (type->is_nullable()) { + nested_type = assert_cast(*type).get_nested_type(); } + auto decimal_str = assert_cast*>(nested_type.get()) + ->to_string(*column, row_idx); + return buffer.push_string(decimal_str.c_str(), decimal_str.length()); } else if (which.is_array()) { auto& column_array = assert_cast(*column); auto& offsets = column_array.get_offsets(); @@ -574,15 +572,15 @@ Status VMysqlResultWriter::append_block(Block& input_block) { } break; } - case TYPE_DECIMAL128: { + case TYPE_DECIMAL128I: { if (type_ptr->is_nullable()) { auto& nested_type = assert_cast(*type_ptr).get_nested_type(); - status = _add_one_column(column_ptr, result, - nested_type, scale); + status = _add_one_column(column_ptr, result, + nested_type, scale); } else { - status = _add_one_column(column_ptr, result, - type_ptr, scale); + status = _add_one_column(column_ptr, result, + type_ptr, scale); } break; } diff --git a/be/src/vec/sink/vmysql_table_writer.cpp b/be/src/vec/sink/vmysql_table_writer.cpp index 251844e9ef1f8a..f302513c7a8c59 100644 --- a/be/src/vec/sink/vmysql_table_writer.cpp +++ b/be/src/vec/sink/vmysql_table_writer.cpp @@ -176,7 +176,7 @@ Status VMysqlTableWriter::insert_row(vectorized::Block& block, size_t row) { } case TYPE_DECIMAL32: case TYPE_DECIMAL64: - case TYPE_DECIMAL128: { + case TYPE_DECIMAL128I: { auto val = type_ptr->to_string(*column, row); fmt::format_to(_insert_stmt_buffer, "{}", val); break; diff --git a/be/src/vec/utils/template_helpers.hpp b/be/src/vec/utils/template_helpers.hpp index 187ec7accc5666..b67f28f3ddd190 100644 --- a/be/src/vec/utils/template_helpers.hpp +++ b/be/src/vec/utils/template_helpers.hpp @@ -37,10 +37,11 @@ M(Float32, ColumnFloat32) \ M(Float64, ColumnFloat64) -#define DECIMAL_TYPE_TO_COLUMN_TYPE(M) \ - M(Decimal32, ColumnDecimal) \ - M(Decimal64, ColumnDecimal) \ - M(Decimal128, ColumnDecimal) +#define DECIMAL_TYPE_TO_COLUMN_TYPE(M) \ + M(Decimal32, ColumnDecimal) \ + M(Decimal64, ColumnDecimal) \ + M(Decimal128, ColumnDecimal) \ + M(Decimal128I, ColumnDecimal) #define STRING_TYPE_TO_COLUMN_TYPE(M) M(String, ColumnString) diff --git a/be/test/vec/core/block_test.cpp b/be/test/vec/core/block_test.cpp index 26fe4dfcf18471..c0bbd1faf4adea 100644 --- a/be/test/vec/core/block_test.cpp +++ b/be/test/vec/core/block_test.cpp @@ -287,7 +287,7 @@ void serialize_and_deserialize_test(segment_v2::CompressionTypePB compression_ty } // decimal { - vectorized::DataTypePtr decimal_data_type(doris::vectorized::create_decimal(27, 9)); + vectorized::DataTypePtr decimal_data_type(doris::vectorized::create_decimal(27, 9, true)); auto decimal_column = decimal_data_type->create_column(); auto& data = ((vectorized::ColumnDecimal>*) decimal_column.get()) @@ -357,7 +357,7 @@ void serialize_and_deserialize_test(segment_v2::CompressionTypePB compression_ty } // nullable decimal { - vectorized::DataTypePtr decimal_data_type(doris::vectorized::create_decimal(27, 9)); + vectorized::DataTypePtr decimal_data_type(doris::vectorized::create_decimal(27, 9, true)); vectorized::DataTypePtr nullable_data_type( std::make_shared(decimal_data_type)); auto nullable_column = nullable_data_type->create_column(); @@ -439,7 +439,7 @@ TEST(BlockTest, dump_data) { vectorized::DataTypePtr string_type(std::make_shared()); vectorized::ColumnWithTypeAndName test_string(strcol->get_ptr(), string_type, "test_string"); - vectorized::DataTypePtr decimal_data_type(doris::vectorized::create_decimal(27, 9)); + vectorized::DataTypePtr decimal_data_type(doris::vectorized::create_decimal(27, 9, true)); auto decimal_column = decimal_data_type->create_column(); auto& decimal_data = ((vectorized::ColumnDecimal>*) decimal_column.get()) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PrimitiveType.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PrimitiveType.java index cd9e6d6881f0df..8f03e8d8bccbb0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PrimitiveType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PrimitiveType.java @@ -54,7 +54,7 @@ public enum PrimitiveType { DECIMALV2("DECIMALV2", 16, TPrimitiveType.DECIMALV2), DECIMAL32("DECIMAL32", 4, TPrimitiveType.DECIMAL32), DECIMAL64("DECIMAL64", 8, TPrimitiveType.DECIMAL64), - DECIMAL128("DECIMAL128", 16, TPrimitiveType.DECIMAL128), + DECIMAL128("DECIMAL128", 16, TPrimitiveType.DECIMAL128I), TIME("TIME", 8, TPrimitiveType.TIME), // these following types are stored as object binary in BE. HLL("HLL", 16, TPrimitiveType.HLL), @@ -969,7 +969,7 @@ public static PrimitiveType fromThrift(TPrimitiveType tPrimitiveType) { return DECIMAL32; case DECIMAL64: return DECIMAL64; - case DECIMAL128: + case DECIMAL128I: return DECIMAL128; case TIME: return TIME; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Type.java index 74cab468088cd6..8aec90c4527784 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Type.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Type.java @@ -762,7 +762,7 @@ protected static Pair fromThrift(TTypeDesc col, int nodeIdx) { } else if (scalarType.getType() == TPrimitiveType.DECIMALV2 || scalarType.getType() == TPrimitiveType.DECIMAL32 || scalarType.getType() == TPrimitiveType.DECIMAL64 - || scalarType.getType() == TPrimitiveType.DECIMAL128) { + || scalarType.getType() == TPrimitiveType.DECIMAL128I) { Preconditions.checkState(scalarType.isSetPrecision() && scalarType.isSetPrecision()); type = ScalarType.createDecimalType(scalarType.getPrecision(), diff --git a/gensrc/proto/internal_service.proto b/gensrc/proto/internal_service.proto index 7d427db5823d63..bf61b2270721d8 100644 --- a/gensrc/proto/internal_service.proto +++ b/gensrc/proto/internal_service.proto @@ -407,7 +407,7 @@ enum PColumnType { COLUMN_TYPE_DATETIMEV2 = 16; COLUMN_TYPE_DECIMAL32 = 17; COLUMN_TYPE_DECIMAL64 = 18; - COLUMN_TYPE_DECIMAL128 = 19; + COLUMN_TYPE_DECIMAL128I = 19; } message PMinMaxFilter { diff --git a/gensrc/proto/types.proto b/gensrc/proto/types.proto index 5385961562154c..5686a6ac9d8edf 100644 --- a/gensrc/proto/types.proto +++ b/gensrc/proto/types.proto @@ -103,6 +103,7 @@ message PGenericType { DATETIMEV2 = 29; FIXEDLENGTHOBJECT = 30; JSONB = 31; + DECIMAL128I = 32; UNKNOWN = 999; } required TypeId id = 2; diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift index 5104045f237744..994880e69ce13f 100644 --- a/gensrc/thrift/Types.thrift +++ b/gensrc/thrift/Types.thrift @@ -89,7 +89,7 @@ enum TPrimitiveType { TIMEV2, DECIMAL32, DECIMAL64, - DECIMAL128, + DECIMAL128I, JSONB, }