From 5517dc7e670a3aa2d815be47cacf43a8fa167164 Mon Sep 17 00:00:00 2001 From: jacktengg Date: Fri, 31 Jul 2026 11:02:57 +0800 Subject: [PATCH 1/2] [feature](timestamp_ns) Add basic TIMESTAMP_NS type support ### What problem does this PR solve? Issue Number: None Related PR: #66333 Problem Summary: Doris DATETIME and DATETIMEV2 support at most six fractional-second digits. Extending their existing packed civil-time representation to nanoseconds would mix incompatible storage representations and semantics. Introduce TIMESTAMP_NS as a separate, fixed-precision timestamp type: - TIMESTAMP_NS always has nine fractional-second digits and does not accept a precision parameter. - Values are stored as signed Int64 nanoseconds relative to the Unix epoch. - The supported range is [1677-09-21 00:12:43.145224192, 2262-04-11 23:47:16.854775807]. - TIMESTAMP_NS has no time-zone attribute. UTC is used as the stable mapping between its civil-time fields and physical epoch-nanosecond representation. - DATETIME(p) and DATETIMEV2(p) retain their existing behavior and continue to support only precision 0 through 6. Add the new primitive type to Thrift, protobuf, FE type metadata, Nereids, BE data types, and cloud schema metadata. Add dedicated TIMESTAMP_NS literal, value, column, and SerDe implementations. Support the basic end-to-end SQL and storage paths, including: - DDL, literals, default values, partitions, buckets, and schema metadata - Insert, stream load, partial update, and decoded timestamp input - Segment storage, key coding, zone map, bloom filter, and inverted index - Comparison, predicates, hashing, grouping, ordering, joins, runtime filters, and basic aggregate functions - FE literal validation, constant folding, type coercion, and predicate simplification - MySQL text and prepared-statement binary results, using string-compatible output to preserve all nine fractional digits Arrow Flight SQL output, ORC OUTFILE, and Java UDF signatures explicitly reject TIMESTAMP_NS because those integrations are not supported yet. The broader explicit cast matrix and scalar date/time function signatures will be implemented in follow-up changes. ### Release note Add the fixed-precision TIMESTAMP_NS SQL type for nanosecond timestamps. TIMESTAMP_NS stores signed Int64 Unix-epoch nanoseconds, supports the range from 1677-09-21 00:12:43.145224192 through 2262-04-11 23:47:16.854775807, and always displays nine fractional digits. Existing DATETIME and DATETIMEV2 types remain limited to precision 0 through 6. ### Check List (For Author) - Test: Unit Test and Regression Test - Added BE unit tests for TIMESTAMP_NS values, SerDe, storage indexes, hashing, predicates, aggregates, and partitioning - Added FE unit tests for types, literals, boundaries, partitions, coercion, folding, MySQL output, and unsupported integrations - Added regression tests for DDL, predicates, storage, stream load, partial/default values, and MySQL prepared-statement output - Behavior changed: Yes; adds the TIMESTAMP_NS SQL and storage type without changing DATETIME or DATETIMEV2 precision semantics - Does this need documentation: Yes; TIMESTAMP_NS syntax, range, storage semantics, and current integration limitations require documentation --- be/src/core/call_on_type_index.h | 5 + be/src/core/column/column_array.cpp | 6 + be/src/core/column/column_vector.cpp | 1 + be/src/core/column/column_vector.h | 3 +- be/src/core/column/columns_common.cpp | 2 + .../core/data_type/convert_field_to_type.cpp | 8 +- be/src/core/data_type/data_type.cpp | 2 + be/src/core/data_type/data_type_array.cpp | 3 +- .../data_type_date_or_datetime_v2.cpp | 20 + .../data_type/data_type_date_or_datetime_v2.h | 8 + be/src/core/data_type/data_type_factory.cpp | 10 + .../core/data_type/data_type_number_base.cpp | 1 + be/src/core/data_type/data_type_time.h | 5 +- .../core/data_type/data_type_timestamp_ns.cpp | 66 ++ .../core/data_type/data_type_timestamp_ns.h | 58 ++ be/src/core/data_type/define_primitive_type.h | 3 +- be/src/core/data_type/primitive_type.cpp | 9 + be/src/core/data_type/primitive_type.h | 19 +- be/src/core/data_type/storage_field_type.cpp | 4 + .../data_type_number_serde.cpp | 40 +- .../data_type_serde/data_type_number_serde.h | 12 + .../core/data_type_serde/data_type_serde.cpp | 2 + .../data_type_timestamp_ns_serde.cpp | 458 +++++++++++ .../data_type_timestamp_ns_serde.h | 83 ++ be/src/core/field.cpp | 23 + be/src/core/type_limit.h | 7 + be/src/core/value/timestamp_ns_value.cpp | 67 ++ be/src/core/value/timestamp_ns_value.h | 264 ++++++ be/src/core/value/vdatetime_value.cpp | 1 + be/src/exec/common/field_visitors.h | 3 + be/src/exec/common/hash_table/hash.h | 12 + be/src/exec/operator/scan_operator.cpp | 13 +- .../runtime_filter/runtime_filter_wrapper.cpp | 4 + ...gregate_function_approx_count_distinct.cpp | 2 +- .../aggregate_function_histogram.cpp | 12 +- .../aggregate/aggregate_function_map_v2.cpp | 3 +- .../aggregate_function_min_max_impl.h | 4 + .../aggregate_function_sequence_match.cpp | 3 + .../aggregate_function_topn_array.cpp | 9 +- .../aggregate_function_topn_weighted.cpp | 9 +- .../aggregate/aggregate_function_uniq.cpp | 13 +- be/src/exprs/create_predicate_function.h | 1 + .../array/function_array_aggregation.cpp | 1 + be/src/exprs/function/cast/cast_to_date.h | 4 +- be/src/exprs/function/cast/function_cast.cpp | 2 + .../function/cast/function_cast_date.cpp | 15 +- be/src/exprs/function/date_time_transforms.h | 4 + be/src/exprs/function/function.h | 3 +- be/src/exprs/function/function_convert_tz.cpp | 152 +++- .../function_date_or_datetime_computation.cpp | 76 +- .../function_date_or_datetime_to_string.cpp | 8 + .../function/function_datetime_floor_ceil.cpp | 1 + .../function_datetime_string_to_string.cpp | 6 + .../function_datetime_string_to_string.h | 1 + .../function/function_other_types_to_date.cpp | 60 +- be/src/exprs/function/functions_comparison.h | 2 + be/src/exprs/function/least_greast.cpp | 9 +- be/src/exprs/function/time_of_function.cpp | 18 + be/src/exprs/function/to_time_function.cpp | 31 + .../lambda_function/varray_sort_function.cpp | 3 +- be/src/exprs/vcase_expr.h | 3 + be/src/exprs/vcondition_expr.cpp | 6 +- be/src/exprs/vdirect_in_predicate.h | 1 + be/src/exprs/vectorized_fn_call.cpp | 2 + be/src/exprs/vexpr.cpp | 10 + be/src/exprs/vexpr.h | 9 + be/src/exprs/vin_predicate.cpp | 1 + be/src/exprs/vtopn_pred.cpp | 2 + .../schema_columns_scanner.cpp | 2 + be/src/runtime/fold_constant_executor.cpp | 7 +- be/src/storage/delete/delete_handler.cpp | 6 + .../bloom_filter_index_writer.cpp | 1 + be/src/storage/index/index_writer.cpp | 1 + .../index/inverted/inverted_index_writer.cpp | 3 +- .../storage/index/zone_map/zone_map_index.cpp | 1 + .../storage/iterator/olap_data_convertor.cpp | 3 + be/src/storage/key_coder.cpp | 1 + be/src/storage/key_coder.h | 1 + be/src/storage/olap_common.h | 2 + be/src/storage/olap_scan_common.h | 9 +- be/src/storage/partial_update_info.cpp | 15 +- .../storage/predicate/predicate_creator.cpp | 4 + .../predicate_creator_comparison.cpp | 4 + .../predicate_creator_in_list_in.cpp | 4 + .../predicate_creator_in_list_not_in.cpp | 4 + be/src/storage/segment/encoding_info.cpp | 5 + be/src/storage/tablet/tablet_schema.cpp | 8 + be/src/storage/tablet_info.cpp | 41 +- be/src/storage/types.h | 9 + be/src/tools/meta_tool.cpp | 10 + be/src/util/raw_value.h | 3 +- .../data_type/data_type_datetime_v2_test.cpp | 5 +- .../data_type/data_type_timestamp_ns_test.cpp | 778 ++++++++++++++++++ .../data_type/storage_field_type_test.cpp | 4 +- .../data_type_serde_decoded_values_test.cpp | 2 +- .../tablet_sink_hash_partitioner_test.cpp | 109 +++ .../agg_approx_count_distinct_test.cpp | 64 ++ be/test/exprs/aggregate/agg_min_max_test.cpp | 37 + be/test/exprs/function/function_test_util.cpp | 19 +- be/test/exprs/function/function_test_util.h | 7 + be/test/exprs/function/function_time_test.cpp | 45 + be/test/exprs/vcondition_expr_test.cpp | 56 +- be/test/exprs/vexpr_unaligned_int128_test.cpp | 11 + .../format/orc/orc_reader_fill_data_test.cpp | 20 + be/test/storage/key_coder_test.cpp | 1 + .../bloom_filter_index_reader_writer_test.cpp | 27 + .../segment/inverted_index_writer_test.cpp | 40 +- .../storage/segment/zone_map_index_test.cpp | 38 + be/test/testutil/datetime_ut_util.h | 11 +- cloud/src/meta-service/meta_service_schema.h | 3 + cloud/test/schema_kv_test.cpp | 14 + .../apache/doris/analysis/DateLiteral.java | 3 + .../apache/doris/analysis/ExprVisitor.java | 6 +- .../doris/analysis/TimeStampNsLiteral.java | 376 +++++++++ .../apache/doris/catalog/AggregateType.java | 2 + .../org/apache/doris/catalog/ColumnType.java | 9 + .../doris/persist/gson/GsonUtilsCatalog.java | 2 + .../org/apache/doris/analysis/ColumnDef.java | 10 +- .../doris/analysis/DateLiteralUtils.java | 113 ++- .../doris/analysis/ExprToSqlVisitor.java | 5 + .../analysis/ExprToStringValueVisitor.java | 8 + .../doris/analysis/ExprToThriftVisitor.java | 12 + .../doris/analysis/LiteralExprUtils.java | 5 +- .../doris/analysis/PartitionExprUtil.java | 39 +- .../doris/catalog/ColumnToProtobuf.java | 1 + .../apache/doris/catalog/PartitionInfo.java | 3 +- .../apache/doris/catalog/PartitionKey.java | 34 +- .../doris/common/proc/BuildIndexProcDir.java | 9 +- .../doris/common/proc/PartitionsProcDir.java | 8 +- .../doris/common/proc/RollupProcDir.java | 9 +- .../common/proc/SchemaChangeProcDir.java | 9 +- .../common/util/DynamicPartitionUtil.java | 1 + .../util/FetchRemoteTabletSchemaUtil.java | 1 + .../doris/common/util/PropertyAnalyzer.java | 4 +- .../org/apache/doris/common/util/Util.java | 1 + .../converter/ConnectorColumnConverter.java | 2 + .../jdbc/client/JdbcMySQLClient.java | 6 +- .../doris/httpv2/util/StatementSubmitter.java | 3 +- .../streamresponse/JsonStreamResponse.java | 3 +- .../mtmv/MTMVPartitionExprDateTrunc.java | 3 +- .../apache/doris/mysql/MysqlSerializer.java | 26 +- .../rules/expression/check/CheckCast.java | 7 + .../rules/FoldConstantRuleOnBE.java | 51 +- .../rules/FoldConstantRuleOnFE.java | 6 +- .../rules/SimplifyComparisonPredicate.java | 128 ++- .../expression/rules/SimplifyInPredicate.java | 10 +- .../doris/nereids/stats/FilterEstimation.java | 35 +- .../functions/ComputeSignatureHelper.java | 19 +- .../functions/DateTimeWithPrecision.java | 10 +- .../functions/agg/SequenceCount.java | 5 +- .../functions/agg/SequenceMatch.java | 5 +- .../functions/agg/TopNWeighted.java | 8 + .../functions/agg/WindowFunnel.java | 4 + .../functions/agg/WindowFunnelV2.java | 4 + .../executable/DateTimeArithmetic.java | 2 +- .../DateTimeExtractAndTransform.java | 18 +- .../functions/executable/TimeRoundSeries.java | 118 +-- .../functions/scalar/ConvertTz.java | 4 +- .../functions/scalar/DayMicrosecondAdd.java | 12 +- .../functions/scalar/DayMicrosecondSub.java | 12 +- .../functions/scalar/HourMicrosecondAdd.java | 12 +- .../functions/scalar/HourMicrosecondSub.java | 12 +- .../functions/scalar/MicroSecondsAdd.java | 10 +- .../functions/scalar/MicroSecondsSub.java | 10 +- .../functions/scalar/MilliSecondsAdd.java | 10 +- .../functions/scalar/MilliSecondsSub.java | 10 +- .../scalar/MinuteMicrosecondAdd.java | 12 +- .../scalar/MinuteMicrosecondSub.java | 12 +- .../scalar/SecondMicrosecondAdd.java | 7 +- .../scalar/SecondMicrosecondSub.java | 7 +- .../functions/window/RequireTrivialTypes.java | 2 + .../expressions/literal/DateLiteral.java | 37 +- .../expressions/literal/DateTimeLiteral.java | 174 ++-- .../literal/DateTimeV2Literal.java | 228 +++-- .../expressions/literal/DateV2Literal.java | 12 +- .../trees/expressions/literal/Literal.java | 18 +- .../expressions/literal/NumericLiteral.java | 14 +- .../literal/StringLikeLiteral.java | 12 +- .../literal/TimeStampNsLiteral.java | 332 ++++++++ .../expressions/literal/TimeV2Literal.java | 2 +- .../literal/TimestampTzLiteral.java | 10 +- .../visitor/ExpressionVisitor.java | 5 + .../plans/commands/CreateFunctionCommand.java | 7 + .../plans/commands/info/DefaultValue.java | 20 +- .../apache/doris/nereids/types/DataType.java | 36 +- .../doris/nereids/types/DateTimeV2Type.java | 28 +- .../doris/nereids/types/TimeStampNsType.java | 53 ++ .../doris/nereids/types/TimeStampTzType.java | 4 +- .../doris/nereids/types/TimeV2Type.java | 2 +- .../nereids/types/coercion/DateLikeType.java | 6 +- .../nereids/util/DateTimeFormatterUtils.java | 8 +- .../apache/doris/nereids/util/DateUtils.java | 97 +++ .../doris/nereids/util/TypeCoercionUtils.java | 110 ++- .../apache/doris/persist/gson/GsonUtils.java | 2 + .../org/apache/doris/qe/ConnectProcessor.java | 2 +- .../org/apache/doris/qe/StmtExecutor.java | 39 + .../apache/doris/qe/cache/PartitionRange.java | 1 + .../arrowflight/FlightSqlSchemaHelper.java | 2 + .../doris/statistics/util/StatisticsUtil.java | 4 +- .../tablefunction/MetadataGenerator.java | 6 + .../CurrentTimestampDefaultValueTest.java | 49 ++ .../doris/analysis/DateLiteralUtilsTest.java | 228 +++++ .../analysis/ExprGsonSerializationTest.java | 2 + .../ExprToStringValueVisitorTest.java | 11 + .../doris/catalog/CreateFunctionTest.java | 12 + .../doris/catalog/PartitionKeyTest.java | 56 ++ .../org/apache/doris/catalog/TypeTest.java | 14 + ...idsToConnectorExpressionConverterTest.java | 5 + .../mysql/MysqlSerializerDateTimeTest.java | 72 ++ .../rules/expression/FoldConstantTest.java | 4 +- .../rules/expression/check/CheckCastTest.java | 15 + .../rules/SimplifyCastRuleTest.java | 5 + .../SimplifyComparisonPredicateTest.java | 67 ++ .../nereids/stats/FilterEstimationTest.java | 28 + .../trees/expressions/LiteralTest.java | 50 ++ .../functions/ComputeSignatureHelperTest.java | 46 ++ .../DateTimeMicrosecondFunctionTest.java | 120 +++ .../functions/scalar/StrToDateTest.java | 10 + .../TimestampTzLeadLagSignatureTest.java | 14 + .../expressions/literal/DateLiteralTest.java | 65 +- .../literal/DateTimeLiteralTest.java | 152 +++- .../literal/IntegerLiteralTest.java | 4 + .../literal/TimeV2LiteralTest.java | 14 + .../literal/TimestampTzLiteralTest.java | 47 ++ .../nereids/trees/plans/OutFileTest.java | 12 +- .../doris/nereids/types/DataTypeTest.java | 36 + .../nereids/util/TypeCoercionMatrixTest.java | 4 + .../nereids/util/TypeCoercionUtilsTest.java | 50 ++ .../org/apache/doris/qe/StmtExecutorTest.java | 52 ++ .../FlightSqlSchemaHelperArrowTypeTest.java | 7 + .../org/apache/doris/nereids/DorisLexer.g4 | 1 + .../org/apache/doris/nereids/DorisParser.g4 | 2 + .../apache/doris/catalog/PrimitiveType.java | 11 +- .../org/apache/doris/catalog/ScalarType.java | 21 +- .../java/org/apache/doris/catalog/Type.java | 24 +- gensrc/proto/types.proto | 1 + gensrc/thrift/Types.thrift | 3 +- .../datetimev2/test_timestamp_ns.out | 100 +++ .../test_timestamp_ns_binary_output.out | 17 + .../test_timestamp_ns_ddl_predicate.out | 61 ++ .../test_timestamp_ns_partial_default.csv | 1 + .../test_timestamp_ns_stream_load.csv | 9 + .../test_timestamp_ns_stream_load.out | 10 + .../datetimev2/test_timestamp_ns.groovy | 416 ++++++++++ .../test_timestamp_ns_binary_output.groovy | 63 ++ .../test_timestamp_ns_ddl_predicate.groovy | 399 +++++++++ .../test_timestamp_ns_stream_load.groovy | 56 ++ 247 files changed, 7363 insertions(+), 772 deletions(-) create mode 100644 be/src/core/data_type/data_type_timestamp_ns.cpp create mode 100644 be/src/core/data_type/data_type_timestamp_ns.h create mode 100644 be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp create mode 100644 be/src/core/data_type_serde/data_type_timestamp_ns_serde.h create mode 100644 be/src/core/value/timestamp_ns_value.cpp create mode 100644 be/src/core/value/timestamp_ns_value.h create mode 100644 be/test/core/data_type/data_type_timestamp_ns_test.cpp create mode 100644 be/test/exprs/aggregate/agg_approx_count_distinct_test.cpp create mode 100644 fe/fe-catalog/src/main/java/org/apache/doris/analysis/TimeStampNsLiteral.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimeStampNsLiteral.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/types/TimeStampNsType.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/analysis/CurrentTimestampDefaultValueTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/mysql/MysqlSerializerDateTimeTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateTimeMicrosecondFunctionTest.java create mode 100644 regression-test/data/datatype_p0/datetimev2/test_timestamp_ns.out create mode 100644 regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_binary_output.out create mode 100644 regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_ddl_predicate.out create mode 100644 regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_partial_default.csv create mode 100644 regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_stream_load.csv create mode 100644 regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_stream_load.out create mode 100644 regression-test/suites/datatype_p0/datetimev2/test_timestamp_ns.groovy create mode 100644 regression-test/suites/datatype_p0/datetimev2/test_timestamp_ns_binary_output.groovy create mode 100644 regression-test/suites/datatype_p0/datetimev2/test_timestamp_ns_ddl_predicate.groovy create mode 100644 regression-test/suites/datatype_p0/datetimev2/test_timestamp_ns_stream_load.groovy diff --git a/be/src/core/call_on_type_index.h b/be/src/core/call_on_type_index.h index 98aad077413a56..7056abb6d721ce 100644 --- a/be/src/core/call_on_type_index.h +++ b/be/src/core/call_on_type_index.h @@ -37,6 +37,7 @@ struct TypePair { class DataTypeDate; class DataTypeDateV2; class DataTypeDateTimeV2; +class DataTypeTimeStampNs; class DataTypeDateTime; class DataTypeIPv4; class DataTypeIPv6; @@ -83,6 +84,8 @@ bool call_on_index_and_data_type(PrimitiveType number, F&& f) { return f(TypePair()); case PrimitiveType::TYPE_DATETIMEV2: return f(TypePair()); + case PrimitiveType::TYPE_TIMESTAMP_NS: + return f(TypePair()); case PrimitiveType::TYPE_DATETIME: return f(TypePair()); case PrimitiveType::TYPE_TIMEV2: @@ -236,6 +239,8 @@ bool dispatch_type_base(PrimitiveType number, F&& f) { return f(DispatchDataType()); case PrimitiveType::TYPE_DATETIMEV2: return f(DispatchDataType()); + case PrimitiveType::TYPE_TIMESTAMP_NS: + return f(DispatchDataType()); case PrimitiveType::TYPE_DATETIME: return f(DispatchDataType()); case PrimitiveType::TYPE_TIMEV2: diff --git a/be/src/core/column/column_array.cpp b/be/src/core/column/column_array.cpp index 604c46948539b1..dad91af31c01cf 100644 --- a/be/src/core/column/column_array.cpp +++ b/be/src/core/column/column_array.cpp @@ -874,6 +874,9 @@ ColumnArrayDataOffsets filter_return_new_dispatch(const Filter& filt, ssize_t re return filter_number_return_new(filt, result_size_hint, data, offsets); if (check_and_get_column(data.get())) return filter_number_return_new(filt, result_size_hint, data, offsets); + if (check_and_get_column(data.get())) { + return filter_number_return_new(filt, result_size_hint, data, offsets); + } if (check_and_get_column(data.get())) return filter_number_return_new(filt, result_size_hint, data, offsets); if (check_and_get_column(data.get())) @@ -940,6 +943,9 @@ size_t filter_inplace_dispatch(const Filter& filter, IColumn& src_data, return filter_number_inplace(filter, src_data, src_offsets); if (check_and_get_column(&src_data)) return filter_number_inplace(filter, src_data, src_offsets); + if (check_and_get_column(&src_data)) { + return filter_number_inplace(filter, src_data, src_offsets); + } if (check_and_get_column(&src_data)) return filter_number_inplace(filter, src_data, src_offsets); if (check_and_get_column(&src_data)) diff --git a/be/src/core/column/column_vector.cpp b/be/src/core/column/column_vector.cpp index 31e4862187fb79..8d43364e5a898d 100644 --- a/be/src/core/column/column_vector.cpp +++ b/be/src/core/column/column_vector.cpp @@ -559,6 +559,7 @@ template class ColumnVector; template class ColumnVector; template class ColumnVector; template class ColumnVector; +template class ColumnVector; template class ColumnVector; template class ColumnVector; template class ColumnVector; diff --git a/be/src/core/column/column_vector.h b/be/src/core/column/column_vector.h index 4f3a7a6c46f2b4..8105ade3d02916 100644 --- a/be/src/core/column/column_vector.h +++ b/be/src/core/column/column_vector.h @@ -412,7 +412,8 @@ class ColumnVector final : public COWHelper> { // when run function which need_replace_null_data_to_default, use the value far from 0 to avoid // raise errors for null cell. static value_type default_value() { - if constexpr (is_date_type(T) || T == PrimitiveType::TYPE_TIMESTAMPTZ) { + if constexpr ((is_date_type(T) && T != PrimitiveType::TYPE_TIMESTAMP_NS) || + T == PrimitiveType::TYPE_TIMESTAMPTZ) { return PrimitiveTypeTraits::CppType::DEFAULT_VALUE; } else { return value_type(); diff --git a/be/src/core/column/columns_common.cpp b/be/src/core/column/columns_common.cpp index bfaee36a8e0058..1129032dc07c11 100644 --- a/be/src/core/column/columns_common.cpp +++ b/be/src/core/column/columns_common.cpp @@ -359,6 +359,8 @@ INSTANTIATE(DateV2Value, IColumn::Offset) INSTANTIATE(DateV2Value, ColumnArray::Offset64) INSTANTIATE(DateV2Value, IColumn::Offset) INSTANTIATE(DateV2Value, ColumnArray::Offset64) +INSTANTIATE(TimeStampNsValue, IColumn::Offset) +INSTANTIATE(TimeStampNsValue, ColumnArray::Offset64) INSTANTIATE(TimestampTzValue, IColumn::Offset) INSTANTIATE(TimestampTzValue, ColumnArray::Offset64) INSTANTIATE(UInt32, IColumn::Offset) diff --git a/be/src/core/data_type/convert_field_to_type.cpp b/be/src/core/data_type/convert_field_to_type.cpp index 8873c09346b2c0..c5582d8bca1d0d 100644 --- a/be/src/core/data_type/convert_field_to_type.cpp +++ b/be/src/core/data_type/convert_field_to_type.cpp @@ -58,6 +58,9 @@ void dispatch(F&& f, const Field& field) { case PrimitiveType::TYPE_DATETIMEV2: f(field.template get()); return; + case PrimitiveType::TYPE_TIMESTAMP_NS: + f(field.template get()); + return; case PrimitiveType::TYPE_TIMESTAMPTZ: f(field.template get()); return; @@ -180,6 +183,9 @@ class FieldVisitorToJsonb : public StaticVisitor { void operator()(const DateV2Value& x, JsonbWriter* writer) const { writer->writeInt64(*(UInt64*)&x); } + void operator()(const TimeStampNsValue& x, JsonbWriter* writer) const { + writer->writeInt64(x.epoch_nanos()); + } void operator()(const TimestampTzValue& x, JsonbWriter* writer) const { writer->writeInt64(*(UInt64*)&x); } @@ -828,4 +834,4 @@ void convert_field_to_type(const Field& from_value, const IDataType& to_type, Fi return convert_field_to_typeImpl(from_value, to_type, from_type_hint, to); } } -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/core/data_type/data_type.cpp b/be/src/core/data_type/data_type.cpp index 941a0377b05e66..d5980d4a8ecd80 100644 --- a/be/src/core/data_type/data_type.cpp +++ b/be/src/core/data_type/data_type.cpp @@ -126,6 +126,8 @@ PGenericType_TypeId IDataType::get_pdata_type(const IDataType* data_type) { return PGenericType::VARIANT; case PrimitiveType::TYPE_DATETIMEV2: return PGenericType::DATETIMEV2; + case PrimitiveType::TYPE_TIMESTAMP_NS: + return PGenericType::TIMESTAMP_NS; case PrimitiveType::TYPE_TIMESTAMPTZ: return PGenericType::TIMESTAMPTZ; case PrimitiveType::TYPE_BITMAP: diff --git a/be/src/core/data_type/data_type_array.cpp b/be/src/core/data_type/data_type_array.cpp index 1cd003e42893aa..369689a90b40b0 100644 --- a/be/src/core/data_type/data_type_array.cpp +++ b/be/src/core/data_type/data_type_array.cpp @@ -162,7 +162,8 @@ FieldWithDataType DataTypeArray::get_field_with_data_type(const IColumn& column, if (is_decimal(nested_type_id)) { precision = nested_type->get_precision(); scale = nested_type->get_scale(); - } else if (nested_type_id == TYPE_DATETIMEV2 || nested_type_id == TYPE_TIMESTAMPTZ) { + } else if (nested_type_id == TYPE_DATETIMEV2 || nested_type_id == TYPE_TIMESTAMP_NS || + nested_type_id == TYPE_TIMESTAMPTZ) { scale = nested_type->get_scale(); } else if (nested_type_id == TYPE_JSONB) { // Array should return JsonbField as element diff --git a/be/src/core/data_type/data_type_date_or_datetime_v2.cpp b/be/src/core/data_type/data_type_date_or_datetime_v2.cpp index 35b3ba7c56a2bf..87a1d667f374e2 100644 --- a/be/src/core/data_type/data_type_date_or_datetime_v2.cpp +++ b/be/src/core/data_type/data_type_date_or_datetime_v2.cpp @@ -106,6 +106,13 @@ void DataTypeDateV2::cast_to_date_time_v2(const DateV2Value fro << TIME_PART_LENGTH); } +void DataTypeDateV2::cast_to_date_time_v2(const DateV2Value from, + TimeStampNsValue& to) { + DateV2Value value; + cast_to_date_time_v2(from, value); + DORIS_CHECK(to.from_datetime(value)); +} + void DataTypeDateV2::cast_from_date(const VecDateTimeValue from, DateV2Value& to) { auto& to_value = (DateV2Value&)(to); auto from_value = binary_cast(from); @@ -183,6 +190,19 @@ void DataTypeDateTimeV2::cast_to_date_v2(const DateV2Value UInt32(from.to_date_int_val() >> TIME_PART_LENGTH)); } +void DataTypeDateTimeV2::cast_to_date(const TimeStampNsValue from, VecDateTimeValue& to) { + cast_to_date(from.to_datetime(), to); +} + +void DataTypeDateTimeV2::cast_to_date_time(const TimeStampNsValue from, VecDateTimeValue& to) { + cast_to_date_time(from.to_datetime(), to); +} + +void DataTypeDateTimeV2::cast_to_date_v2(const TimeStampNsValue from, + DateV2Value& to) { + cast_to_date_v2(from.to_datetime(), to); +} + DataTypePtr create_datetimev2(UInt64 scale_value) { if (scale_value > 6) { throw doris::Exception(doris::ErrorCode::NOT_IMPLEMENTED_ERROR, "scale_value {} > 6", diff --git a/be/src/core/data_type/data_type_date_or_datetime_v2.h b/be/src/core/data_type/data_type_date_or_datetime_v2.h index 0e16644863ec9c..34c85bd69dd643 100644 --- a/be/src/core/data_type/data_type_date_or_datetime_v2.h +++ b/be/src/core/data_type/data_type_date_or_datetime_v2.h @@ -81,6 +81,7 @@ class DataTypeDateV2 final : public DataTypeNumberBase from, VecDateTimeValue& to); static void cast_to_date_time_v2(const DateV2Value from, DateV2Value& to); + static void cast_to_date_time_v2(const DateV2Value from, TimeStampNsValue& to); static void cast_from_date(const VecDateTimeValue from, DateV2Value& to); static void cast_from_date_time(const VecDateTimeValue from, DateV2Value& to); }; @@ -146,6 +147,9 @@ class DataTypeDateTimeV2 final : public DataTypeNumberBase from, DateV2Value& to); + static void cast_to_date(const TimeStampNsValue from, VecDateTimeValue& to); + static void cast_to_date_time(const TimeStampNsValue from, VecDateTimeValue& to); + static void cast_to_date_v2(const TimeStampNsValue from, DateV2Value& to); static void cast_from_date(const VecDateTimeValue from, DateV2Value& to); static void cast_from_date_time(const VecDateTimeValue from, DateV2Value& to); @@ -156,8 +160,12 @@ class DataTypeDateTimeV2 final : public DataTypeNumberBase constexpr bool IsDataTypeDateTimeV2 = false; template <> inline constexpr bool IsDataTypeDateTimeV2 = true; +template <> +inline constexpr bool IsDataTypeDateTimeV2 = true; } // namespace doris diff --git a/be/src/core/data_type/data_type_factory.cpp b/be/src/core/data_type/data_type_factory.cpp index 463f79f352a29b..9e2ef01633a19d 100644 --- a/be/src/core/data_type/data_type_factory.cpp +++ b/be/src/core/data_type/data_type_factory.cpp @@ -59,6 +59,7 @@ #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" #include "core/data_type/data_type_time.h" +#include "core/data_type/data_type_timestamp_ns.h" #include "core/data_type/data_type_timestamptz.h" #include "core/data_type/data_type_varbinary.h" #include "core/data_type/data_type_variant.h" @@ -152,6 +153,9 @@ DataTypePtr DataTypeFactory::_create_primitive_data_type(const FieldType& type, case FieldType::OLAP_FIELD_TYPE_DATETIMEV2: result = create_datetimev2(scale); break; + case FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS: + result = std::make_shared(); + break; case FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ: result = std::make_shared(scale); break; @@ -258,6 +262,9 @@ DataTypePtr DataTypeFactory::create_data_type(const PColumnMeta& pcolumn) { case PGenericType::DATETIMEV2: nested = std::make_shared(pcolumn.decimal_param().scale()); break; + case PGenericType::TIMESTAMP_NS: + nested = std::make_shared(); + break; case PGenericType::DATETIME: nested = std::make_shared(); break; @@ -436,6 +443,9 @@ DataTypePtr DataTypeFactory::create_data_type(const PrimitiveType primitive_type case TYPE_DATETIMEV2: nested = create_datetimev2(scale); break; + case TYPE_TIMESTAMP_NS: + nested = std::make_shared(); + break; case TYPE_DATETIME: nested = std::make_shared(); break; diff --git a/be/src/core/data_type/data_type_number_base.cpp b/be/src/core/data_type/data_type_number_base.cpp index dcd167bce1df0d..15bc934c2594f0 100644 --- a/be/src/core/data_type/data_type_number_base.cpp +++ b/be/src/core/data_type/data_type_number_base.cpp @@ -203,6 +203,7 @@ template class DataTypeNumberBase; template class DataTypeNumberBase; template class DataTypeNumberBase; template class DataTypeNumberBase; +template class DataTypeNumberBase; template class DataTypeNumberBase; template class DataTypeNumberBase; template class DataTypeNumberBase; diff --git a/be/src/core/data_type/data_type_time.h b/be/src/core/data_type/data_type_time.h index afcc97fe8e1853..7ff188c9b2457b 100644 --- a/be/src/core/data_type/data_type_time.h +++ b/be/src/core/data_type/data_type_time.h @@ -26,6 +26,7 @@ #include #include #include +#include #include "core/data_type/data_type.h" #include "core/data_type/data_type_number_base.h" @@ -40,8 +41,10 @@ class IColumn; class DataTypeTimeV2 final : public DataTypeNumberBase { public: + static constexpr UInt32 MAX_SCALE = 6; + DataTypeTimeV2(int scale = 0) : _scale(scale) { - if (UNLIKELY(scale > 6)) { + if (UNLIKELY(std::cmp_greater(scale, MAX_SCALE))) { throw doris::Exception(ErrorCode::INTERNAL_ERROR, "Scale {} is out of bounds", scale); } if (scale == -1) { diff --git a/be/src/core/data_type/data_type_timestamp_ns.cpp b/be/src/core/data_type/data_type_timestamp_ns.cpp new file mode 100644 index 00000000000000..80d5651fc0c7ba --- /dev/null +++ b/be/src/core/data_type/data_type_timestamp_ns.cpp @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "core/data_type/data_type_timestamp_ns.h" + +#include + +#include +#include +#include + +#include "common/exception.h" +#include "core/assert_cast.h" +#include "core/column/column_vector.h" +#include "core/string_ref.h" +#include "core/value/timestamp_ns_value.h" + +namespace doris { + +Field DataTypeTimeStampNs::get_field(const TExprNode& node) const { + int64_t value = 0; + const StringRef string_value(node.date_literal.value.data(), node.date_literal.value.size()); + const auto status = parse_timestamp_ns(string_value, &value); + if (!status.ok()) { + throw doris::Exception(doris::ErrorCode::INVALID_ARGUMENT, + "Invalid value: {} for type TimeStampNs: {}", + node.date_literal.value, status.to_string()); + } + return Field::create_field(TimeStampNsValue(value)); +} + +bool DataTypeTimeStampNs::equals(const IDataType& rhs) const { + return typeid(rhs) == typeid(*this); +} + +void DataTypeTimeStampNs::to_pb_column_meta(PColumnMeta* col_meta) const { + IDataType::to_pb_column_meta(col_meta); +} + +FieldWithDataType DataTypeTimeStampNs::get_field_with_data_type(const IColumn& column, + size_t row_num) const { + const auto& column_data = + assert_cast(column); + Field field; + column_data.get(row_num, field); + return FieldWithDataType {.field = std::move(field), + .base_scalar_type_id = get_primitive_type(), + .precision = -1, + .scale = static_cast(get_scale())}; +} + +} // namespace doris diff --git a/be/src/core/data_type/data_type_timestamp_ns.h b/be/src/core/data_type/data_type_timestamp_ns.h new file mode 100644 index 00000000000000..38c23a60d3ee09 --- /dev/null +++ b/be/src/core/data_type/data_type_timestamp_ns.h @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "core/data_type/data_type_number_base.h" +#include "core/data_type_serde/data_type_timestamp_ns_serde.h" + +namespace doris { + +class DataTypeTimeStampNs final : public DataTypeNumberBase { +public: + static constexpr PrimitiveType PType = TYPE_TIMESTAMP_NS; + static constexpr bool is_parametric = false; + static constexpr UInt32 SCALE = 9; + + PrimitiveType get_primitive_type() const override { return TYPE_TIMESTAMP_NS; } + doris::FieldType get_storage_field_type() const override { + return doris::FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS; + } + const std::string get_family_name() const override { return "TimeStampNs"; } + std::string do_get_name() const override { return "TimeStampNs"; } + + bool equals(const IDataType& rhs) const override; + bool equals_ignore_precision(const IDataType& rhs) const override { + return rhs.get_primitive_type() == TYPE_TIMESTAMP_NS; + } + + using SerDeType = DataTypeTimeStampNsSerDe; + DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override { + return std::make_shared(nesting_level); + } + + Field get_field(const TExprNode& node) const override; + UInt32 get_scale() const override { return SCALE; } + void to_pb_column_meta(PColumnMeta* col_meta) const override; + FieldWithDataType get_field_with_data_type(const IColumn& column, + size_t row_num) const override; +}; + +} // namespace doris diff --git a/be/src/core/data_type/define_primitive_type.h b/be/src/core/data_type/define_primitive_type.h index 852b68a96b5358..6606242cf1d257 100644 --- a/be/src/core/data_type/define_primitive_type.h +++ b/be/src/core/data_type/define_primitive_type.h @@ -73,7 +73,8 @@ enum PrimitiveType : PrimitiveNative { TYPE_UINT64, /* 39, used as offset */ TYPE_FIXED_LENGTH_OBJECT, /* 40, represent fixed-length object on BE */ TYPE_VARBINARY, /* 41, varbinary */ - TYPE_TIMESTAMPTZ /* 42, timestamptz */ + TYPE_TIMESTAMPTZ, /* 42, timestamptz */ + TYPE_TIMESTAMP_NS /* 43, signed Int64 epoch nanoseconds */ }; } // namespace doris diff --git a/be/src/core/data_type/primitive_type.cpp b/be/src/core/data_type/primitive_type.cpp index 746fc655323a22..208e9e9f411caa 100644 --- a/be/src/core/data_type/primitive_type.cpp +++ b/be/src/core/data_type/primitive_type.cpp @@ -69,6 +69,9 @@ PrimitiveType thrift_to_type(TPrimitiveType::type ttype) { case TPrimitiveType::DATETIMEV2: return TYPE_DATETIMEV2; + case TPrimitiveType::TIMESTAMP_NS: + return TYPE_TIMESTAMP_NS; + case TPrimitiveType::TIMEV2: return TYPE_TIMEV2; @@ -191,6 +194,9 @@ TPrimitiveType::type to_thrift(PrimitiveType ptype) { case TYPE_DATETIMEV2: return TPrimitiveType::DATETIMEV2; + case TYPE_TIMESTAMP_NS: + return TPrimitiveType::TIMESTAMP_NS; + case TYPE_TIMEV2: return TPrimitiveType::TIMEV2; @@ -305,6 +311,9 @@ std::string type_to_string(PrimitiveType t) { case TYPE_DATETIMEV2: return "DATETIMEV2"; + case TYPE_TIMESTAMP_NS: + return "TIMESTAMP_NS"; + case TYPE_TIMEV2: return "TIMEV2"; diff --git a/be/src/core/data_type/primitive_type.h b/be/src/core/data_type/primitive_type.h index cd4c55b19b121f..73e78c5213c6d6 100644 --- a/be/src/core/data_type/primitive_type.h +++ b/be/src/core/data_type/primitive_type.h @@ -31,6 +31,7 @@ #include "core/string_view.h" #include "core/types.h" #include "core/uint24.h" +#include "core/value/timestamp_ns_value.h" #include "core/value/timestamptz_value.h" #include "core/value/vdatetime_value.h" #include "exec/common/template_helpers.hpp" @@ -75,6 +76,7 @@ class DataTypeTimeV2; class DataTypeDateTime; class DataTypeDate; class DataTypeDateTimeV2; +class DataTypeTimeStampNs; class DataTypeDateV2; class DataTypeTimeStampTz; template @@ -112,6 +114,7 @@ using ColumnDateTime = ColumnVector; using ColumnDateV2 = ColumnVector; using ColumnTimeStampTz = ColumnVector; using ColumnDateTimeV2 = ColumnVector; +using ColumnTimeStampNs = ColumnVector; using ColumnFloat32 = ColumnVector; using ColumnFloat64 = ColumnVector; using ColumnIPv4 = ColumnVector; @@ -140,6 +143,7 @@ constexpr bool is_enumeration_type(PrimitiveType type) { case TYPE_STRING: case TYPE_DATETIME: case TYPE_DATETIMEV2: + case TYPE_TIMESTAMP_NS: case TYPE_TIMESTAMPTZ: case TYPE_TIMEV2: case TYPE_DECIMALV2: @@ -175,7 +179,7 @@ constexpr bool is_enumeration_type(PrimitiveType type) { constexpr bool is_date_type(PrimitiveType type) { return type == TYPE_DATETIME || type == TYPE_DATE || type == TYPE_DATETIMEV2 || - type == TYPE_DATEV2; + type == TYPE_TIMESTAMP_NS || type == TYPE_DATEV2; } constexpr bool is_time_type(PrimitiveType type) { @@ -191,7 +195,11 @@ constexpr bool is_date_or_datetime(PrimitiveType type) { } constexpr bool is_date_v2_or_datetime_v2(PrimitiveType type) { - return type == TYPE_DATETIMEV2 || type == TYPE_DATEV2; + return type == TYPE_DATETIMEV2 || type == TYPE_TIMESTAMP_NS || type == TYPE_DATEV2; +} + +constexpr bool is_datetime_v2(PrimitiveType type) { + return type == TYPE_DATETIMEV2 || type == TYPE_TIMESTAMP_NS; } constexpr bool is_ip(PrimitiveType type) { @@ -366,6 +374,13 @@ struct PrimitiveTypeTraits { using ColumnType = ColumnDateTimeV2; }; template <> +struct PrimitiveTypeTraits { + using CppType = TimeStampNsValue; + using StorageFieldType = int64_t; + using DataType = DataTypeTimeStampNs; + using ColumnType = ColumnTimeStampNs; +}; +template <> struct PrimitiveTypeTraits { using CppType = DateV2Value; using StorageFieldType = uint32_t; diff --git a/be/src/core/data_type/storage_field_type.cpp b/be/src/core/data_type/storage_field_type.cpp index 269015401dcab7..ec10e25ee68e8a 100644 --- a/be/src/core/data_type/storage_field_type.cpp +++ b/be/src/core/data_type/storage_field_type.cpp @@ -73,6 +73,8 @@ FieldType primitive_type_to_storage_field_type(PrimitiveType type) { return FieldType::OLAP_FIELD_TYPE_DATEV2; case PrimitiveType::TYPE_DATETIMEV2: return FieldType::OLAP_FIELD_TYPE_DATETIMEV2; + case PrimitiveType::TYPE_TIMESTAMP_NS: + return FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS; case PrimitiveType::TYPE_TIMEV2: return FieldType::OLAP_FIELD_TYPE_TIMEV2; case PrimitiveType::TYPE_DECIMAL32: @@ -167,6 +169,8 @@ PrimitiveType storage_field_type_to_primitive_type(FieldType type) { return PrimitiveType::TYPE_DATEV2; case FieldType::OLAP_FIELD_TYPE_DATETIMEV2: return PrimitiveType::TYPE_DATETIMEV2; + case FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS: + return PrimitiveType::TYPE_TIMESTAMP_NS; case FieldType::OLAP_FIELD_TYPE_TIMEV2: return PrimitiveType::TYPE_TIMEV2; case FieldType::OLAP_FIELD_TYPE_DECIMAL32: diff --git a/be/src/core/data_type_serde/data_type_number_serde.cpp b/be/src/core/data_type_serde/data_type_number_serde.cpp index a1a4a3df65df19..7cec876900cd72 100644 --- a/be/src/core/data_type_serde/data_type_number_serde.cpp +++ b/be/src/core/data_type_serde/data_type_number_serde.cpp @@ -655,9 +655,10 @@ using DORIS_NUMERIC_ARROW_BUILDER = TypeMap; @@ -727,6 +728,13 @@ Status DataTypeNumberSerDe::write_column_to_arrow(const IColumn& column, cons builder.AppendValues((uint64_t*)col_data.data() + start, end - start, reinterpret_cast(arrow_null_map_data)), column, *array_builder)); + } else if constexpr (T == TYPE_TIMESTAMP_NS) { + auto& builder = assert_cast(*array_builder); + RETURN_IF_ERROR(checkArrowStatus( + builder.AppendValues(reinterpret_cast(col_data.data()) + start, + end - start, + reinterpret_cast(arrow_null_map_data)), + column, *array_builder)); } else { auto& builder = assert_cast(*array_builder); RETURN_IF_ERROR(checkArrowStatus( @@ -1117,8 +1125,8 @@ template constexpr bool can_write_to_jsonb_from_number() { return T == TYPE_BOOLEAN || T == TYPE_TINYINT || T == TYPE_SMALLINT || T == TYPE_INT || T == TYPE_BIGINT || T == TYPE_LARGEINT || T == TYPE_FLOAT || T == TYPE_DOUBLE || - T == TYPE_DATEV2 || T == TYPE_DATETIMEV2 || T == TYPE_TIMESTAMPTZ || T == TYPE_IPV4 || - T == TYPE_IPV6 || T == TYPE_TIMEV2; + T == TYPE_DATEV2 || T == TYPE_DATETIMEV2 || T == TYPE_TIMESTAMP_NS || + T == TYPE_TIMESTAMPTZ || T == TYPE_IPV4 || T == TYPE_IPV6 || T == TYPE_TIMEV2; } template @@ -1156,6 +1164,8 @@ bool write_to_jsonb_from_number(auto& data, JsonbWriter& writer, int scale) { return jsonb_writer_string(writer, CastToString::from_datev2(data)); } else if constexpr (T == TYPE_DATETIMEV2) { return jsonb_writer_string(writer, CastToString::from_datetimev2(data, scale)); + } else if constexpr (T == TYPE_TIMESTAMP_NS) { + return jsonb_writer_string(writer, TimeStampNsValue(data).to_string(scale)); } else if constexpr (T == TYPE_TIMESTAMPTZ) { return jsonb_writer_string(writer, CastToString::from_timestamptz(data, scale)); } else if constexpr (T == TYPE_IPV4) { @@ -1445,6 +1455,8 @@ void DataTypeNumberSerDe::read_one_cell_from_jsonb(IColumn& column, col.insert_value(binary_cast(static_cast(read_int()))); } else if constexpr (T == TYPE_BIGINT) { col.insert_value(static_cast(read_int())); + } else if constexpr (T == TYPE_TIMESTAMP_NS) { + col.insert_value(TimeStampNsValue(static_cast(read_int()))); } else if constexpr (T == TYPE_LARGEINT) { col.insert_value(static_cast<__int128_t>(read_int())); } else if constexpr (T == TYPE_FLOAT) { @@ -1489,7 +1501,7 @@ void DataTypeNumberSerDe::write_one_cell_to_jsonb(const IColumn& column, int32_t val = *reinterpret_cast(data_ref.data); result.writeInt32(val); } else if constexpr (T == TYPE_BIGINT || T == TYPE_DATE || T == TYPE_DATETIME || - T == TYPE_DATETIMEV2 || T == TYPE_TIMESTAMPTZ) { + T == TYPE_DATETIMEV2 || T == TYPE_TIMESTAMP_NS || T == TYPE_TIMESTAMPTZ) { int64_t val = *reinterpret_cast(data_ref.data); if (options.enable_row_store_compact_jsonb) { result.writeInt(val); @@ -1748,6 +1760,10 @@ const uint8_t* DataTypeNumberSerDe::deserialize_binary_to_column(const uint8_ col.insert_value(binary_cast>( unaligned_load(data))); data += sizeof(UInt64); + } else if constexpr (T == TYPE_TIMESTAMP_NS) { + data += sizeof(uint8_t); + col.insert_value(TimeStampNsValue(unaligned_load(data))); + data += sizeof(Int64); } else { throw doris::Exception(ErrorCode::NOT_IMPLEMENTED_ERROR, "deserialize_binary_to_column with type '{}'", type_to_string(T)); @@ -1810,6 +1826,14 @@ const uint8_t* DataTypeNumberSerDe::deserialize_binary_to_field(const uint8_t info.scale = static_cast(scale); field = Field::create_field(*(typename PrimitiveTypeTraits::CppType*)&v); data += sizeof(UInt64); + } else if constexpr (T == TYPE_TIMESTAMP_NS) { + const uint8_t scale = *data; + data += sizeof(uint8_t); + info.precision = -1; + info.scale = static_cast(scale); + field = Field::create_field( + TimeStampNsValue(unaligned_load(data))); + data += sizeof(Int64); } else { throw doris::Exception(ErrorCode::NOT_IMPLEMENTED_ERROR, "deserialize_binary_to_column with type '{}'", type_to_string(T)); @@ -1828,6 +1852,9 @@ void value_to_string(const typename PrimitiveTypeTraits::CppType value, Buffe CastToString::push_datev2(value, bw); } else if constexpr (T == TYPE_DATETIMEV2) { CastToString::push_datetimev2(value, scale, bw); + } else if constexpr (T == TYPE_TIMESTAMP_NS) { + const auto string_value = TimeStampNsValue(value).to_string(scale); + bw.write(string_value.data(), string_value.size()); } else if constexpr (T == TYPE_TIMESTAMPTZ) { CastToString::push_timestamptz(value, scale, bw, options); } else if constexpr (T == TYPE_TIMEV2) { @@ -1954,6 +1981,7 @@ template class DataTypeNumberSerDe; template class DataTypeNumberSerDe; template class DataTypeNumberSerDe; template class DataTypeNumberSerDe; +template class DataTypeNumberSerDe; template class DataTypeNumberSerDe; template class DataTypeNumberSerDe; template class DataTypeNumberSerDe; diff --git a/be/src/core/data_type_serde/data_type_number_serde.h b/be/src/core/data_type_serde/data_type_number_serde.h index 9c0de8cb0c0e8a..c69540ff462df4 100644 --- a/be/src/core/data_type_serde/data_type_number_serde.h +++ b/be/src/core/data_type_serde/data_type_number_serde.h @@ -247,6 +247,12 @@ Status DataTypeNumberSerDe::read_column_from_pb(IColumn& column, const PValue for (int i = 0; i < arg.int64_value_size(); ++i) { data[old_column_size + i] = arg.int64_value(i); } + } else if constexpr (T == TYPE_TIMESTAMP_NS) { + column.resize(old_column_size + arg.int64_value_size()); + auto& data = reinterpret_cast(column).get_data(); + for (int i = 0; i < arg.int64_value_size(); ++i) { + data[old_column_size + i] = TimeStampNsValue(arg.int64_value(i)); + } } else if constexpr (T == TYPE_FLOAT) { column.resize(old_column_size + arg.float_value_size()); auto& data = reinterpret_cast(column).get_data(); @@ -332,6 +338,12 @@ Status DataTypeNumberSerDe::write_column_to_pb(const IColumn& column, PValues auto* values = result.mutable_int64_value(); values->Reserve(row_count); values->Add(data.begin() + start, data.begin() + end); + } else if constexpr (T == TYPE_TIMESTAMP_NS) { + ptype->set_id(PGenericType::INT64); + auto* values = result.mutable_int64_value(); + values->Reserve(row_count); + values->Add(reinterpret_cast(data.begin()) + start, + reinterpret_cast(data.begin()) + end); } else if constexpr (T == TYPE_FLOAT) { ptype->set_id(PGenericType::FLOAT); auto* values = result.mutable_float_value(); diff --git a/be/src/core/data_type_serde/data_type_serde.cpp b/be/src/core/data_type_serde/data_type_serde.cpp index ca5f2c206ada16..b8dc7ccb09e5da 100644 --- a/be/src/core/data_type_serde/data_type_serde.cpp +++ b/be/src/core/data_type_serde/data_type_serde.cpp @@ -288,6 +288,7 @@ const uint8_t* DataTypeSerDe::deserialize_binary_to_column(const uint8_t* data, HANDLE_T_NUM_SERDE(OLAP_FIELD_TYPE_IPV6, TYPE_IPV6) HANDLE_T_NUM_SERDE(OLAP_FIELD_TYPE_DATEV2, TYPE_DATEV2) HANDLE_T_NUM_SERDE(OLAP_FIELD_TYPE_DATETIMEV2, TYPE_DATETIMEV2) + HANDLE_T_NUM_SERDE(OLAP_FIELD_TYPE_TIMESTAMP_NS, TYPE_TIMESTAMP_NS) HANDLE_T_DEC_SERDE(OLAP_FIELD_TYPE_DECIMAL32, TYPE_DECIMAL32) HANDLE_T_DEC_SERDE(OLAP_FIELD_TYPE_DECIMAL64, TYPE_DECIMAL64) HANDLE_T_DEC_SERDE(OLAP_FIELD_TYPE_DECIMAL128I, TYPE_DECIMAL128I) @@ -350,6 +351,7 @@ const uint8_t* DataTypeSerDe::deserialize_binary_to_field(const uint8_t* data, F HANDLE_T_NUM_SERDE(OLAP_FIELD_TYPE_IPV6, TYPE_IPV6) HANDLE_T_NUM_SERDE(OLAP_FIELD_TYPE_DATEV2, TYPE_DATEV2) HANDLE_T_NUM_SERDE(OLAP_FIELD_TYPE_DATETIMEV2, TYPE_DATETIMEV2) + HANDLE_T_NUM_SERDE(OLAP_FIELD_TYPE_TIMESTAMP_NS, TYPE_TIMESTAMP_NS) HANDLE_T_DEC_SERDE(OLAP_FIELD_TYPE_DECIMAL32, TYPE_DECIMAL32) HANDLE_T_DEC_SERDE(OLAP_FIELD_TYPE_DECIMAL64, TYPE_DECIMAL64) HANDLE_T_DEC_SERDE(OLAP_FIELD_TYPE_DECIMAL128I, TYPE_DECIMAL128I) diff --git a/be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp b/be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp new file mode 100644 index 00000000000000..5224162671db8a --- /dev/null +++ b/be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp @@ -0,0 +1,458 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "core/data_type_serde/data_type_timestamp_ns_serde.h" + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "common/config.h" +#include "common/exception.h" +#include "core/assert_cast.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/data_type_serde/arrow_validation.h" +#include "core/data_type_serde/decoded_column_view.h" +#include "core/value/vdatetime_value.h" +#include "exprs/function/cast/cast_to_datetimev2_impl.hpp" +#include "util/mysql_row_buffer.h" +#include "util/unaligned.h" + +namespace doris { +namespace { + +constexpr int64_t NANOS_PER_MILLISECOND = 1000000; +constexpr int64_t NANOS_PER_MICROSECOND = 1000; +constexpr size_t NANOSECOND_SCALE = 9; + +bool checked_scale_to_nanos(int64_t value, int64_t multiplier, int64_t* result) { + return !__builtin_mul_overflow(value, multiplier, result); +} + +Status utc_epoch_nanos_to_local_epoch_nanos(int64_t source, const cctz::time_zone& timezone, + int64_t* result) { + const TimeStampNsValue source_value(source); + DateV2Value local_value; + local_value.from_unixtime(source_value.epoch_seconds(), timezone); + local_value.set_microsecond(source_value.microsecond()); + TimeStampNsValue target_value; + if (!target_value.from_datetime(local_value, source_value.nanosecond_remainder())) { + return Status::DataQualityError("Timestamp {} is outside TIMESTAMP_NS range", source); + } + *result = target_value.epoch_nanos(); + return Status::OK(); +} + +/* +Status local_epoch_nanos_to_utc_epoch_nanos(int64_t source, const cctz::time_zone& timezone, + int64_t* result) { + const TimeStampNsValue source_value(source); + auto local_value = source_value.to_datetime(); + int64_t seconds = 0; + local_value.unix_timestamp(&seconds, timezone); + const __int128 epoch_nanos = + static_cast<__int128>(seconds) * TimeStampNsValue::NANOS_PER_SECOND + + source_value.nanosecond(); + if (epoch_nanos < std::numeric_limits::min() || + epoch_nanos > std::numeric_limits::max()) { + return Status::DataQualityError("TIMESTAMP_NS value {} is outside epoch nanosecond range", + source_value.to_string(9)); + } + *result = static_cast(epoch_nanos); + return Status::OK(); +} +*/ + +} // namespace + +Status parse_timestamp_ns(StringRef str, int64_t* epoch_nanos, + const cctz::time_zone* local_time_zone) { + std::string input(str.data, str.size); + const size_t dot = input.rfind('.'); + size_t fraction_begin = std::string::npos; + size_t fraction_end = std::string::npos; + if (dot != std::string::npos && dot + 1 < input.size() && + std::isdigit(static_cast(input[dot + 1]))) { + fraction_begin = dot + 1; + fraction_end = fraction_begin; + while (fraction_end < input.size() && + std::isdigit(static_cast(input[fraction_end]))) { + ++fraction_end; + } + } + + uint32_t nanos = 0; + size_t fraction_length = 0; + if (fraction_begin != std::string::npos) { + fraction_length = fraction_end - fraction_begin; + const size_t copied_digits = std::min(fraction_length, 9); + for (size_t i = 0; i < copied_digits; ++i) { + nanos = nanos * 10 + static_cast(input[fraction_begin + i] - '0'); + } + for (size_t i = copied_digits; i < 9; ++i) { + nanos *= 10; + } + } + + if (fraction_length > NANOSECOND_SCALE && input[fraction_begin + NANOSECOND_SCALE] >= '5') { + ++nanos; + } + + // Keep the fractional token in place so that the legacy parser validates its position and all + // trailing syntax. Zeroing the digits prevents its scale-0 rounding from changing the second; + // the nanosecond rounding above remains the only source of fractional rounding. + if (fraction_begin != std::string::npos) { + std::fill(input.begin() + fraction_begin, input.begin() + fraction_end, '0'); + } + const StringRef input_ref(input.data(), input.size()); + DateV2Value datetime; + CastParameters params {.status = Status::OK(), .is_strict = true}; + CastToDatetimeV2::from_string_strict_mode( + input_ref, datetime, local_time_zone, 0, params); + if (!params.status.ok()) { + if (dot != std::string::npos) { + return Status::InvalidArgument("Invalid TIMESTAMP_NS value '{}'", + std::string(str.data, str.size)); + } + return params.status; + } + + if (nanos == TimeStampNsValue::NANOS_PER_SECOND) { + if (!datetime.date_add_interval( + TimeInterval {TimeUnit::SECOND, 1, false})) { + return Status::InvalidArgument("TIMESTAMP_NS value overflows while rounding '{}'", + std::string(str.data, str.size)); + } + nanos = 0; + } + datetime.set_microsecond(nanos / NANOS_PER_MICROSECOND); + TimeStampNsValue value; + if (!value.from_datetime(datetime, static_cast(nanos % NANOS_PER_MICROSECOND))) { + return Status::InvalidArgument( + "TIMESTAMP_NS value '{}' is outside [{}, {}]", std::string(str.data, str.size), + TimeStampNsValue(std::numeric_limits::min()).to_string(), + TimeStampNsValue(std::numeric_limits::max()).to_string()); + } + *epoch_nanos = value.epoch_nanos(); + return Status::OK(); +} + +Status DataTypeTimeStampNsSerDe::from_string_batch(const ColumnString& strings, + ColumnNullable& result, + const FormatOptions& options) const { + auto& data = assert_cast(result.get_nested_column()).get_data(); + auto& null_map = result.get_null_map_column().get_data(); + result.resize(strings.size()); + for (size_t i = 0; i < strings.size(); ++i) { + int64_t value = 0; + const auto status = parse_timestamp_ns(strings.get_data_at(i), &value, options.timezone); + null_map[i] = !status.ok(); + data[i] = TimeStampNsValue(value); + } + return Status::OK(); +} + +Status DataTypeTimeStampNsSerDe::from_string_strict_mode_batch( + const ColumnString& strings, IColumn& result, const FormatOptions& options, + const NullMap::value_type* null_map) const { + auto& data = assert_cast(result).get_data(); + result.resize(strings.size()); + for (size_t i = 0; i < strings.size(); ++i) { + if (null_map != nullptr && null_map[i]) { + continue; + } + int64_t value = 0; + RETURN_IF_ERROR(parse_timestamp_ns(strings.get_data_at(i), &value, options.timezone)); + data[i] = TimeStampNsValue(value); + } + return Status::OK(); +} + +Status DataTypeTimeStampNsSerDe::from_string(StringRef& str, IColumn& column, + const FormatOptions& options) const { + int64_t value = 0; + RETURN_IF_ERROR(parse_timestamp_ns(str, &value, options.timezone)); + assert_cast(column).insert_value(TimeStampNsValue(value)); + return Status::OK(); +} + +Status DataTypeTimeStampNsSerDe::from_string_strict_mode(StringRef& str, IColumn& column, + const FormatOptions& options) const { + return from_string(str, column, options); +} + +Status DataTypeTimeStampNsSerDe::serialize_column_to_json(const IColumn& column, int64_t start_idx, + int64_t end_idx, BufferWritable& bw, + FormatOptions& options) const { + SERIALIZE_COLUMN_TO_JSON(); +} + +Status DataTypeTimeStampNsSerDe::serialize_one_cell_to_json(const IColumn& column, int64_t row_num, + BufferWritable& bw, + FormatOptions& options) const { + auto [column_ptr, index] = check_column_const_set_readability(column, row_num); + if (_nesting_level > 1) { + bw.write('"'); + } + const auto value = + assert_cast(*column_ptr) + .get_element(index); + const std::string result = value.to_string(9); + bw.write(result.data(), result.size()); + if (_nesting_level > 1) { + bw.write('"'); + } + return Status::OK(); +} + +Status DataTypeTimeStampNsSerDe::deserialize_column_from_json_vector( + IColumn& column, std::vector& slices, uint64_t* num_deserialized, + const FormatOptions& options) const { + DESERIALIZE_COLUMN_FROM_JSON_VECTOR(); + return Status::OK(); +} + +Status DataTypeTimeStampNsSerDe::deserialize_one_cell_from_json( + IColumn& column, Slice& slice, const FormatOptions& options) const { + if (_nesting_level > 1) { + slice.trim_quote(); + } + StringRef str(slice.data, slice.size); + return from_string(str, column, options); +} + +Status DataTypeTimeStampNsSerDe::deserialize_column_from_jsonb(IColumn& column, + const JsonbValue* jsonb_value, + CastParameters& cast_params) const { + DORIS_CHECK(jsonb_value->isString()); + return parse_column_from_jsonb_string(column, jsonb_value, cast_params); +} + +Status DataTypeTimeStampNsSerDe::deserialize_column_from_jsonb_vector( + ColumnNullable& column_to, const ColumnString& column_from, + CastParameters& cast_params) const { + return DataTypeSerDe::deserialize_column_from_jsonb_vector(column_to, column_from, cast_params); +} + +Status DataTypeTimeStampNsSerDe::write_column_to_arrow(const IColumn& column, + const NullMap* null_map, + arrow::ArrayBuilder* array_builder, + int64_t start, int64_t end, + const cctz::time_zone& ctz) const { + return Status::NotSupported("DataTypeTimeStampNsSerDe::write_column_to_arrow"); + // TODO: support write_column_to_arrow + /* + const auto& data = assert_cast(column).get_data(); + auto& builder = assert_cast(*array_builder); + const auto timestamp_type = std::static_pointer_cast(builder.type()); + const auto& timezone = timestamp_type->timezone(); + for (int64_t i = start; i < end; ++i) { + if (null_map != nullptr && (*null_map)[i]) { + RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(), column, builder)); + continue; + } + int64_t value = data[i].epoch_nanos(); + if (!timezone.empty()) { + RETURN_IF_ERROR(local_epoch_nanos_to_utc_epoch_nanos(value, ctz, &value)); + } + switch (timestamp_type->unit()) { + case arrow::TimeUnit::SECOND: + value /= TimeStampNsValue::NANOS_PER_SECOND; + break; + case arrow::TimeUnit::MILLI: + value /= NANOS_PER_MILLISECOND; + break; + case arrow::TimeUnit::MICRO: + value /= NANOS_PER_MICROSECOND; + break; + case arrow::TimeUnit::NANO: + break; + } + RETURN_IF_ERROR(checkArrowStatus(builder.Append(value), column, builder)); + } + return Status::OK(); + */ +} + +Status DataTypeTimeStampNsSerDe::read_column_from_arrow(IColumn& column, + const arrow::Array* arrow_array, + int64_t start, int64_t end, + const cctz::time_zone& ctz) const { + return Status::NotSupported("DataTypeTimeStampNsSerDe::read_column_from_arrow"); + // TODO: + /* + if (arrow_array->type_id() != arrow::Type::TIMESTAMP) { + return Status::InvalidArgument("Expected Arrow timestamp, got {}", arrow_array->type_id()); + } + if (config::enable_arrow_input_validation) { + check_arrow_no_offset(*arrow_array); + } + const auto& array = assert_cast(*arrow_array); + const auto type = std::static_pointer_cast(array.type()); + auto& data = assert_cast(column).get_data(); + for (int64_t i = start; i < end; ++i) { + int64_t value = array.Value(i); + int64_t nanos = 0; + switch (type->unit()) { + case arrow::TimeUnit::SECOND: + if (!checked_scale_to_nanos(value, TimeStampNsValue::NANOS_PER_SECOND, &nanos)) { + return Status::DataQualityError("Arrow timestamp {} overflows nanoseconds", value); + } + break; + case arrow::TimeUnit::MILLI: + if (!checked_scale_to_nanos(value, NANOS_PER_MILLISECOND, &nanos)) { + return Status::DataQualityError("Arrow timestamp {} overflows nanoseconds", value); + } + break; + case arrow::TimeUnit::MICRO: + if (!checked_scale_to_nanos(value, NANOS_PER_MICROSECOND, &nanos)) { + return Status::DataQualityError("Arrow timestamp {} overflows nanoseconds", value); + } + break; + case arrow::TimeUnit::NANO: + nanos = value; + break; + } + if (!type->timezone().empty()) { + RETURN_IF_ERROR(utc_epoch_nanos_to_local_epoch_nanos(nanos, ctz, &nanos)); + } + data.push_back(TimeStampNsValue(nanos)); + } + return Status::OK(); + */ +} + +Status DataTypeTimeStampNsSerDe::read_column_from_decoded_values( + IColumn& column, const DecodedColumnView& view) const { + if (view.value_kind != DecodedValueKind::INT64) { + return decoded_column_view_handle_conversion_failure( + column, view, + Status::NotSupported("TIMESTAMP_NS decoded reader expects INT64 source")); + } + auto& data = assert_cast(column).get_data(); + const auto* values = reinterpret_cast(view.values); + static const auto utc = cctz::utc_time_zone(); + const auto& timezone = view.timezone == nullptr ? utc : *view.timezone; + for (int64_t row = 0; row < view.row_count; ++row) { + if (decoded_column_view_row_is_null(view, row)) { + data.push_back(TimeStampNsValue(0)); + continue; + } + int64_t nanos = 0; + int64_t multiplier = 1; + switch (view.time_unit) { + case DecodedTimeUnit::MILLIS: + multiplier = NANOS_PER_MILLISECOND; + break; + case DecodedTimeUnit::MICROS: + multiplier = NANOS_PER_MICROSECOND; + break; + case DecodedTimeUnit::NANOS: + break; + case DecodedTimeUnit::UNKNOWN: + return decoded_column_view_handle_conversion_failure( + column, view, + Status::NotSupported("TIMESTAMP_NS decoded reader requires a time unit")); + } + if (!checked_scale_to_nanos(values[row], multiplier, &nanos)) { + return decoded_column_view_handle_conversion_failure( + column, view, + Status::DataQualityError("Timestamp {} overflows nanoseconds", values[row])); + } + if (view.timestamp_is_adjusted_to_utc) { + RETURN_IF_ERROR(utc_epoch_nanos_to_local_epoch_nanos(nanos, timezone, &nanos)); + } + data.push_back(TimeStampNsValue(nanos)); + } + return Status::OK(); +} + +Status DataTypeTimeStampNsSerDe::write_column_to_mysql_binary(const IColumn& column, + MysqlRowBinaryBuffer& row_buffer, + int64_t row_idx, bool col_const, + const FormatOptions& options) const { + const auto& data = assert_cast(column).get_data(); + const auto index = index_check_const(row_idx, col_const); + const auto value = TimeStampNsValue(data[index]).to_string(9); + if (row_buffer.push_string(value.data(), value.size()) != 0) { + return Status::InternalError("pack MySQL TIMESTAMP_NS buffer failed"); + } + return Status::OK(); +} + +Status DataTypeTimeStampNsSerDe::write_column_to_orc(const std::string& timezone, + const IColumn& column, const NullMap* null_map, + orc::ColumnVectorBatch* orc_col_batch, + int64_t start, int64_t end, Arena& arena, + const FormatOptions& options) const { + return Status::NotSupported("DataTypeTimeStampNsSerDe::write_column_to_orc"); + // TODO: support write_column_to_orc + /* + const auto& data = assert_cast(column).get_data(); + auto& batch = assert_cast(*orc_col_batch); + cctz::time_zone parsed_timezone; + if (!cctz::load_time_zone(timezone, &parsed_timezone)) { + return Status::InvalidArgument("Invalid timezone '{}'", timezone); + } + for (int64_t row = start; row < end; ++row) { + if (batch.notNull[row] == 0) { + continue; + } + int64_t value = data[row].epoch_nanos(); + RETURN_IF_ERROR(local_epoch_nanos_to_utc_epoch_nanos(value, parsed_timezone, &value)); + const TimeStampNsValue timestamp(value); + batch.data[row] = timestamp.epoch_seconds(); + batch.nanoseconds[row] = timestamp.nanosecond(); + } + batch.numElements = end - start; + return Status::OK(); + */ +} + +void DataTypeTimeStampNsSerDe::write_one_cell_to_binary(const IColumn& src_column, + ColumnString::Chars& chars, + int64_t row_num) const { + const auto type = static_cast(FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS); + constexpr auto scale = static_cast(9); + const auto value = assert_cast(src_column).get_element(row_num); + const size_t old_size = chars.size(); + chars.resize(old_size + sizeof(type) + sizeof(scale) + sizeof(value)); + memcpy(chars.data() + old_size, &type, sizeof(type)); + memcpy(chars.data() + old_size + sizeof(type), &scale, sizeof(scale)); + memcpy(chars.data() + old_size + sizeof(type) + sizeof(scale), &value, sizeof(value)); +} + +std::string DataTypeTimeStampNsSerDe::to_olap_string(const Field& field) const { + return field.get().to_string(9); +} + +Status DataTypeTimeStampNsSerDe::from_olap_string(const std::string& str, Field& field, + const FormatOptions& options) const { + int64_t value = 0; + RETURN_IF_ERROR(parse_timestamp_ns(StringRef(str.data(), str.size()), &value)); + field = Field::create_field(TimeStampNsValue(value)); + return Status::OK(); +} + +} // namespace doris diff --git a/be/src/core/data_type_serde/data_type_timestamp_ns_serde.h b/be/src/core/data_type_serde/data_type_timestamp_ns_serde.h new file mode 100644 index 00000000000000..5489b34eeb8c11 --- /dev/null +++ b/be/src/core/data_type_serde/data_type_timestamp_ns_serde.h @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "core/data_type_serde/data_type_number_serde.h" + +namespace doris { + +Status parse_timestamp_ns(StringRef str, int64_t* epoch_nanos, + const cctz::time_zone* local_time_zone = nullptr); + +class DataTypeTimeStampNsSerDe final + : public DataTypeNumberSerDe { +public: + explicit DataTypeTimeStampNsSerDe(int nesting_level = 1) + : DataTypeNumberSerDe(nesting_level) {} + + Status from_string_batch(const ColumnString& str, ColumnNullable& column, + const FormatOptions& options) const override; + Status from_string_strict_mode_batch( + const ColumnString& str, IColumn& column, const FormatOptions& options, + const NullMap::value_type* null_map = nullptr) const override; + Status from_string(StringRef& str, IColumn& column, + const FormatOptions& options) const override; + Status from_string_strict_mode(StringRef& str, IColumn& column, + const FormatOptions& options) const override; + + Status serialize_one_cell_to_json(const IColumn& column, int64_t row_num, BufferWritable& bw, + FormatOptions& options) const override; + Status serialize_column_to_json(const IColumn& column, int64_t start_idx, int64_t end_idx, + BufferWritable& bw, FormatOptions& options) const override; + Status deserialize_one_cell_from_json(IColumn& column, Slice& slice, + const FormatOptions& options) const override; + Status deserialize_column_from_json_vector(IColumn& column, std::vector& slices, + uint64_t* num_deserialized, + const FormatOptions& options) const override; + Status deserialize_column_from_jsonb(IColumn& column, const JsonbValue* jsonb_value, + CastParameters& cast_params) const override; + Status deserialize_column_from_jsonb_vector(ColumnNullable& column_to, + const ColumnString& column_from, + CastParameters& cast_params) const override; + + Status write_column_to_arrow(const IColumn& column, const NullMap* null_map, + arrow::ArrayBuilder* array_builder, int64_t start, int64_t end, + const cctz::time_zone& ctz) const override; + Status read_column_from_arrow(IColumn& column, const arrow::Array* arrow_array, int64_t start, + int64_t end, const cctz::time_zone& ctz) const override; + Status read_column_from_decoded_values(IColumn& column, + const DecodedColumnView& view) const override; + Status write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& row_buffer, + int64_t row_idx, bool col_const, + const FormatOptions& options) const override; + Status write_column_to_orc(const std::string& timezone, const IColumn& column, + const NullMap* null_map, orc::ColumnVectorBatch* orc_col_batch, + int64_t start, int64_t end, Arena& arena, + const FormatOptions& options) const override; + + void write_one_cell_to_binary(const IColumn& src_column, ColumnString::Chars& chars, + int64_t row_num) const override; + int get_scale() const override { return 9; } + std::string to_olap_string(const Field& field) const override; + +protected: + Status from_olap_string(const std::string& str, Field& field, + const FormatOptions& options) const override; +}; + +} // namespace doris diff --git a/be/src/core/field.cpp b/be/src/core/field.cpp index f63aaf8227a7c6..1466a0d44d2d8a 100644 --- a/be/src/core/field.cpp +++ b/be/src/core/field.cpp @@ -124,6 +124,9 @@ void Field::create(Field&& field) { case PrimitiveType::TYPE_DATETIMEV2: create_concrete(std::move(field.template get())); return; + case PrimitiveType::TYPE_TIMESTAMP_NS: + create_concrete(std::move(field.template get())); + return; case PrimitiveType::TYPE_DATEV2: create_concrete(std::move(field.template get())); return; @@ -253,6 +256,9 @@ void Field::create(const Field& field) { case PrimitiveType::TYPE_DATETIMEV2: create_concrete(field.template get()); return; + case PrimitiveType::TYPE_TIMESTAMP_NS: + create_concrete(field.template get()); + return; case PrimitiveType::TYPE_DATEV2: create_concrete(field.template get()); return; @@ -414,6 +420,9 @@ void Field::assign(Field&& field) { case PrimitiveType::TYPE_DATETIMEV2: assign_concrete(std::move(field.template get())); return; + case PrimitiveType::TYPE_TIMESTAMP_NS: + assign_concrete(std::move(field.template get())); + return; case PrimitiveType::TYPE_DATETIME: assign_concrete(std::move(field.template get())); return; @@ -523,6 +532,9 @@ void Field::assign(const Field& field) { case PrimitiveType::TYPE_DATETIMEV2: assign_concrete(field.template get()); return; + case PrimitiveType::TYPE_TIMESTAMP_NS: + assign_concrete(field.template get()); + return; case PrimitiveType::TYPE_DATETIME: assign_concrete(field.template get()); return; @@ -708,6 +720,9 @@ std::strong_ordering Field::operator<=>(const Field& rhs) const { case PrimitiveType::TYPE_DATETIMEV2: return get().to_date_int_val() <=> rhs.get().to_date_int_val(); + case PrimitiveType::TYPE_TIMESTAMP_NS: + return get() <=> + rhs.get(); case PrimitiveType::TYPE_DATEV2: return get().to_date_int_val() <=> rhs.get().to_date_int_val(); @@ -826,6 +841,7 @@ std::string_view Field::as_string_view() const { // MATCH_PRIMITIVE_TYPE(TYPE_QUANTILE_STATE); MATCH_PRIMITIVE_TYPE(TYPE_DATEV2); MATCH_PRIMITIVE_TYPE(TYPE_DATETIMEV2); + MATCH_PRIMITIVE_TYPE(TYPE_TIMESTAMP_NS); MATCH_PRIMITIVE_TYPE(TYPE_TIMEV2); MATCH_PRIMITIVE_TYPE(TYPE_DECIMAL32); MATCH_PRIMITIVE_TYPE(TYPE_DECIMAL64); @@ -883,6 +899,8 @@ std::string Field::to_debug_string(int scale) const { return CastToString::from_datev2(get()); case PrimitiveType::TYPE_DATETIMEV2: return CastToString::from_datetimev2(get(), scale); + case PrimitiveType::TYPE_TIMESTAMP_NS: + return get().to_string(scale); case PrimitiveType::TYPE_TIMESTAMPTZ: return CastToString::from_timestamptz(get(), scale); case PrimitiveType::TYPE_DECIMALV2: @@ -927,6 +945,8 @@ std::string Field::to_debug_string(int scale) const { typename PrimitiveTypeTraits::CppType && rhs); \ template void Field::FUNC_NAME( \ typename PrimitiveTypeTraits::CppType && rhs); \ + template void Field::FUNC_NAME( \ + typename PrimitiveTypeTraits::CppType && rhs); \ template void Field::FUNC_NAME( \ typename PrimitiveTypeTraits::CppType && rhs); \ template void Field::FUNC_NAME( \ @@ -973,6 +993,8 @@ std::string Field::to_debug_string(int scale) const { const typename PrimitiveTypeTraits::CppType& rhs); \ template void Field::FUNC_NAME( \ const typename PrimitiveTypeTraits::CppType& rhs); \ + template void Field::FUNC_NAME( \ + const typename PrimitiveTypeTraits::CppType& rhs); \ template void Field::FUNC_NAME( \ const typename PrimitiveTypeTraits::CppType& rhs); \ template void Field::FUNC_NAME( \ @@ -1070,6 +1092,7 @@ DECLARE_FUNCTION(TYPE_DATE) DECLARE_FUNCTION(TYPE_DATETIME) DECLARE_FUNCTION(TYPE_DATEV2) DECLARE_FUNCTION(TYPE_DATETIMEV2) +DECLARE_FUNCTION(TYPE_TIMESTAMP_NS) DECLARE_FUNCTION(TYPE_TIMESTAMPTZ) DECLARE_FUNCTION(TYPE_DECIMAL32) DECLARE_FUNCTION(TYPE_DECIMAL64) diff --git a/be/src/core/type_limit.h b/be/src/core/type_limit.h index 824433a03191dc..7a286470f5a25d 100644 --- a/be/src/core/type_limit.h +++ b/be/src/core/type_limit.h @@ -20,6 +20,7 @@ #include "core/extended_types.h" #include "core/string_ref.h" #include "core/value/decimalv2_value.h" +#include "core/value/timestamp_ns_value.h" #include "core/value/timestamptz_value.h" namespace doris { @@ -118,6 +119,12 @@ struct type_limit> { } }; +template <> +struct type_limit { + static TimeStampNsValue min() { return TimeStampNsValue(std::numeric_limits::min()); } + static TimeStampNsValue max() { return TimeStampNsValue(std::numeric_limits::max()); } +}; + template <> struct type_limit { static TimestampTzValue min() { diff --git a/be/src/core/value/timestamp_ns_value.cpp b/be/src/core/value/timestamp_ns_value.cpp new file mode 100644 index 00000000000000..83ee9019242abd --- /dev/null +++ b/be/src/core/value/timestamp_ns_value.cpp @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "core/value/timestamp_ns_value.h" + +#include + +#include "exec/common/int_exp.h" + +namespace doris { + +DateV2Value TimeStampNsValue::to_datetime() const { + DateV2Value value; + value.from_unixtime(epoch_seconds(), cctz::utc_time_zone()); + value.set_microsecond(microsecond()); + return value; +} + +bool TimeStampNsValue::from_datetime(const DateV2Value& value, + uint16_t nanosecond_remainder) { + DORIS_CHECK_LE(nanosecond_remainder, 999); + int64_t seconds = 0; + value.unix_timestamp(&seconds, cctz::utc_time_zone()); + const __int128 epoch_nanos = static_cast<__int128>(seconds) * NANOS_PER_SECOND + + static_cast<__int128>(value.microsecond()) * 1000 + + nanosecond_remainder; + if (epoch_nanos < std::numeric_limits::min() || + epoch_nanos > std::numeric_limits::max()) { + return false; + } + _epoch_nanos = static_cast(epoch_nanos); + return true; +} + +int32_t TimeStampNsValue::to_buffer(char* buffer, int scale) const { + DORIS_CHECK_GE(scale, 0); + DORIS_CHECK_LE(scale, 9); + const auto value = to_datetime(); + const int32_t base_length = value.to_buffer(buffer, 0); + if (scale == 0) { + return base_length; + } + buffer[base_length] = '.'; + uint32_t nanos = nanosecond(); + for (int i = 0; i < scale; ++i) { + buffer[base_length + 1 + i] = + static_cast('0' + nanos / static_cast(int_exp10(8 - i))); + nanos %= static_cast(int_exp10(8 - i)); + } + return base_length + 1 + scale; +} + +} // namespace doris diff --git a/be/src/core/value/timestamp_ns_value.h b/be/src/core/value/timestamp_ns_value.h new file mode 100644 index 00000000000000..9a6b48c24c69bf --- /dev/null +++ b/be/src/core/value/timestamp_ns_value.h @@ -0,0 +1,264 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "core/value/vdatetime_value.h" + +namespace doris { + +// TIMESTAMP_NS uses a signed Int64 nanosecond offset from the Unix epoch. UTC is used only as a +// stable mapping between timezone-naive civil fields and the physical integer representation. +class TimeStampNsValue { +public: + static constexpr int64_t NANOS_PER_SECOND = 1000000000; + static constexpr int64_t NANOS_PER_MILLISECOND = 1000000; + static constexpr int64_t NANOS_PER_MICROSECOND = 1000; + + constexpr TimeStampNsValue() = default; + explicit constexpr TimeStampNsValue(int64_t epoch_nanos) : _epoch_nanos(epoch_nanos) {} + + constexpr int64_t to_date_int_val() const { return _epoch_nanos; } + constexpr int64_t epoch_nanos() const { return _epoch_nanos; } + + int64_t epoch_seconds() const { + int64_t seconds = _epoch_nanos / NANOS_PER_SECOND; + if (_epoch_nanos % NANOS_PER_SECOND < 0) { + --seconds; + } + return seconds; + } + + uint32_t nanosecond() const { + int64_t nanos = _epoch_nanos % NANOS_PER_SECOND; + if (nanos < 0) { + nanos += NANOS_PER_SECOND; + } + return static_cast(nanos); + } + + uint32_t microsecond() const { return nanosecond() / 1000; } + uint16_t nanosecond_remainder() const { return static_cast(nanosecond() % 1000); } + + DateV2Value to_datetime() const; + bool from_datetime(const DateV2Value& value, + uint16_t nanosecond_remainder = 0); + + uint16_t year() const { return to_datetime().year(); } + uint8_t month() const { return to_datetime().month(); } + uint8_t day() const { return to_datetime().day(); } + uint8_t hour() const { return to_datetime().hour(); } + uint8_t minute() const { return to_datetime().minute(); } + uint8_t second() const { return to_datetime().second(); } + int quarter() const { return to_datetime().quarter(); } + int64_t daynr() const { return to_datetime().daynr(); } + uint16_t year_of_week() const { return to_datetime().year_of_week(); } + uint8_t week(uint8_t mode) const { return to_datetime().week(mode); } + uint32_t year_week(uint8_t mode) const { return to_datetime().year_week(mode); } + int day_of_year() const { return to_datetime().day_of_year(); } + int day_of_week() const { return to_datetime().day_of_week(); } + uint8_t weekday() const { return to_datetime().weekday(); } + int64_t time_part_to_seconds() const { return to_datetime().time_part_to_seconds(); } + int64_t time_part_to_microsecond() const { + return time_part_to_seconds() * 1000000 + microsecond(); + } + + template + int64_t time_part_diff_in_ms(const RHS& rhs) const { + return time_part_to_microsecond() - rhs.time_part_to_microsecond(); + } + + template + int64_t datetime_diff_in_seconds(const RHS& rhs) const { + return (daynr() - rhs.daynr()) * SECOND_PER_HOUR * HOUR_PER_DAY + time_part_to_seconds() - + rhs.time_part_to_seconds(); + } + + template + int64_t datetime_diff_in_microseconds(const RHS& rhs) const { + if constexpr (std::is_same_v, TimeStampNsValue>) { + return static_cast((static_cast<__int128>(_epoch_nanos) - rhs._epoch_nanos) / + 1000); + } + return (daynr() - rhs.daynr()) * HOUR_PER_DAY * SECOND_PER_HOUR * MS_PER_SECOND + + time_part_diff_in_ms(rhs); + } + + template + int32_t date_diff_in_days(const RHS& rhs) const { + return static_cast(daynr() - rhs.daynr()); + } + + int32_t date_diff_in_days_round_to_zero_by_time(const auto& rhs) const { + int32_t days = date_diff_in_days(rhs); + const int64_t time_diff = time_part_diff_in_ms(rhs); + if (days > 0 && time_diff < 0) { + --days; + } else if (days < 0 && time_diff > 0) { + ++days; + } + return days; + } + + int64_t datetime_diff_in_seconds_round_to_zero_by_ms(const auto& rhs) const { + int64_t seconds = datetime_diff_in_seconds(rhs); + const int64_t fraction_diff = static_cast(nanosecond()) - rhs.nanosecond(); + if (seconds > 0 && fraction_diff < 0) { + --seconds; + } else if (seconds < 0 && fraction_diff > 0) { + ++seconds; + } + return seconds; + } + + const char* day_name_with_locale(const char* const* day_names) const { + return to_datetime().day_name_with_locale(day_names); + } + const char* month_name_with_locale(const char* const* month_names) const { + return to_datetime().month_name_with_locale(month_names); + } + bool to_format_string_conservative(const char* format, size_t len, char* to, + size_t max_valid_length) const { + return to_datetime().to_format_string_conservative(format, len, to, max_valid_length); + } + + bool is_valid_date() const { return true; } + + template + bool date_add_interval(const TimeInterval& interval) { + auto value = to_datetime(); + const uint16_t remainder = nanosecond_remainder(); + if (!value.date_add_interval(interval)) { + return false; + } + return from_datetime(value, remainder); + } + + template + bool datetime_trunc() { + auto value = to_datetime(); + if (!value.template datetime_trunc()) { + return false; + } + return from_datetime(value); + } + + int32_t to_buffer(char* buffer, int scale = 9) const; + char* to_string(char* buffer, int scale = 9) const { + const int32_t length = to_buffer(buffer, scale); + buffer[length] = '\0'; + return buffer + length + 1; + } + std::string to_string(int scale = 9) const { + char buffer[40]; + const int32_t length = to_buffer(buffer, scale); + return {buffer, static_cast(length)}; + } + + auto operator<=>(const TimeStampNsValue&) const = default; + + TimeStampNsValue& operator+=(int64_t seconds) { + int64_t delta = 0; + DORIS_CHECK(!__builtin_mul_overflow(seconds, NANOS_PER_SECOND, &delta)); + DORIS_CHECK(!__builtin_add_overflow(_epoch_nanos, delta, &_epoch_nanos)); + return *this; + } + + TimeStampNsValue& operator-=(int64_t seconds) { + DORIS_CHECK_NE(seconds, std::numeric_limits::min()); + return *this += -seconds; + } + + uint32_t hash(int seed) const { + return HashUtil::hash(&_epoch_nanos, sizeof(_epoch_nanos), seed); + } + +private: + int64_t _epoch_nanos = 0; +}; + +static_assert(sizeof(TimeStampNsValue) == sizeof(int64_t)); +static_assert(std::is_trivially_copyable_v); + +template +int64_t datetime_diff(const TimeStampNsValue& ts_value1, const TimeStampNsValue& ts_value2) { + const auto time_key = [](const TimeStampNsValue& value, bool include_month) { + int64_t result = include_month ? value.month() : 0; + result = result * 32 + value.day(); + result = result * 24 + value.hour(); + result = result * 60 + value.minute(); + result = result * 60 + value.second(); + return result * TimeStampNsValue::NANOS_PER_SECOND + value.nanosecond(); + }; + if constexpr (UNIT == YEAR) { + int year = ts_value2.year() - ts_value1.year(); + const int64_t remainder1 = time_key(ts_value1, true); + const int64_t remainder2 = time_key(ts_value2, true); + if (year > 0) { + year -= remainder2 < remainder1; + } else if (year < 0) { + year += remainder2 > remainder1; + } + return year; + } else if constexpr (UNIT == QUARTER || UNIT == MONTH) { + int month = (ts_value2.year() - ts_value1.year()) * 12 + + (ts_value2.month() - ts_value1.month()); + const int64_t remainder1 = time_key(ts_value1, false); + const int64_t remainder2 = time_key(ts_value2, false); + if (month > 0) { + month -= remainder2 < remainder1; + } else if (month < 0) { + month += remainder2 > remainder1; + } + return UNIT == QUARTER ? month / 3 : month; + } else if constexpr (UNIT == WEEK || UNIT == DAY) { + int64_t day = ts_value2.daynr() - ts_value1.daynr(); + const int64_t time1 = + ts_value1.time_part_to_seconds() * TimeStampNsValue::NANOS_PER_SECOND + + ts_value1.nanosecond(); + const int64_t time2 = + ts_value2.time_part_to_seconds() * TimeStampNsValue::NANOS_PER_SECOND + + ts_value2.nanosecond(); + if (day > 0) { + day -= time2 < time1; + } else if (day < 0) { + day += time2 > time1; + } + return UNIT == WEEK ? day / 7 : day; + } else { + constexpr int64_t divisor = UNIT == HOUR ? 3600 * TimeStampNsValue::NANOS_PER_SECOND + : UNIT == MINUTE ? 60 * TimeStampNsValue::NANOS_PER_SECOND + : UNIT == SECOND ? TimeStampNsValue::NANOS_PER_SECOND + : UNIT == MILLISECOND ? 1000000 + : 1000; + static_assert(UNIT == HOUR || UNIT == MINUTE || UNIT == SECOND || UNIT == MILLISECOND || + UNIT == MICROSECOND, + "Unsupported TimeUnit for datetime_diff"); + return static_cast( + (static_cast<__int128>(ts_value2.epoch_nanos()) - ts_value1.epoch_nanos()) / + divisor); + } +} + +} // namespace doris + +template <> +struct std::hash { + size_t operator()(const doris::TimeStampNsValue& value) const { + return std::hash {}(value.epoch_nanos()); + } +}; diff --git a/be/src/core/value/vdatetime_value.cpp b/be/src/core/value/vdatetime_value.cpp index 2bceb41b809537..6dfd7062f75d78 100644 --- a/be/src/core/value/vdatetime_value.cpp +++ b/be/src/core/value/vdatetime_value.cpp @@ -47,6 +47,7 @@ #include "util/timezone_utils.h" namespace doris { + #include "common/compile_check_avoid_begin.h" static const char* s_ab_month_name[] = {"", "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec", nullptr}; diff --git a/be/src/exec/common/field_visitors.h b/be/src/exec/common/field_visitors.h index 8b14b6bbd17626..25ca811dd889ae 100644 --- a/be/src/exec/common/field_visitors.h +++ b/be/src/exec/common/field_visitors.h @@ -47,6 +47,9 @@ typename std::decay_t::ResultType apply_visitor(Visitor&& visitor, F&& case PrimitiveType::TYPE_DATETIMEV2: return visitor.template apply( field.template get()); + case PrimitiveType::TYPE_TIMESTAMP_NS: + return visitor.template apply( + field.template get()); case PrimitiveType::TYPE_TIMESTAMPTZ: return visitor.template apply( field.template get()); diff --git a/be/src/exec/common/hash_table/hash.h b/be/src/exec/common/hash_table/hash.h index e8487bf4c8db96..ea598fd7fc383a 100644 --- a/be/src/exec/common/hash_table/hash.h +++ b/be/src/exec/common/hash_table/hash.h @@ -26,6 +26,7 @@ #include "core/string_ref.h" #include "core/types.h" #include "core/uint128.h" +#include "core/value/timestamp_ns_value.h" #include "parallel_hashmap/phmap_utils.h" // Here is an empirical value. @@ -111,6 +112,11 @@ struct DefaultHash> { } }; +template <> +struct DefaultHash { + size_t operator()(doris::TimeStampNsValue key) const { return int_hash64(key.epoch_nanos()); } +}; + template <> struct DefaultHash { size_t operator()(doris::TimestampTzValue key) const { @@ -168,6 +174,11 @@ inline size_t hash_crc32(doris::DateV2Value u) { return hash_crc32(u.to_date_int_val()); } +template <> +inline size_t hash_crc32(doris::TimeStampNsValue u) { + return hash_crc32(u.epoch_nanos()); +} + template <> inline size_t hash_crc32(doris::TimestampTzValue u) { return hash_crc32(u.to_date_int_val()); @@ -196,6 +207,7 @@ DEFINE_HASH(doris::Float64) DEFINE_HASH(doris::VecDateTimeValue) DEFINE_HASH(doris::DateV2Value) DEFINE_HASH(doris::DateV2Value) +DEFINE_HASH(doris::TimeStampNsValue) DEFINE_HASH(doris::TimestampTzValue) DEFINE_HASH(unsigned __int128) diff --git a/be/src/exec/operator/scan_operator.cpp b/be/src/exec/operator/scan_operator.cpp index e1aff774e7a39d..1f31ad0a630b2e 100644 --- a/be/src/exec/operator/scan_operator.cpp +++ b/be/src/exec/operator/scan_operator.cpp @@ -933,12 +933,13 @@ Status ScanLocalStateBase::_change_value_range(bool is_equal_op, func(temp_range, to_olap_filter_type(fn_name), tmp_value); } } else if constexpr ((PrimitiveType == TYPE_DECIMALV2) || (PrimitiveType == TYPE_DATETIMEV2) || - (PrimitiveType == TYPE_TINYINT) || (PrimitiveType == TYPE_SMALLINT) || - (PrimitiveType == TYPE_INT) || (PrimitiveType == TYPE_BIGINT) || - (PrimitiveType == TYPE_LARGEINT) || (PrimitiveType == TYPE_FLOAT) || - (PrimitiveType == TYPE_DOUBLE) || (PrimitiveType == TYPE_IPV4) || - (PrimitiveType == TYPE_IPV6) || (PrimitiveType == TYPE_DECIMAL32) || - (PrimitiveType == TYPE_DECIMAL64) || (PrimitiveType == TYPE_DECIMAL128I) || + (PrimitiveType == TYPE_TIMESTAMP_NS) || (PrimitiveType == TYPE_TINYINT) || + (PrimitiveType == TYPE_SMALLINT) || (PrimitiveType == TYPE_INT) || + (PrimitiveType == TYPE_BIGINT) || (PrimitiveType == TYPE_LARGEINT) || + (PrimitiveType == TYPE_FLOAT) || (PrimitiveType == TYPE_DOUBLE) || + (PrimitiveType == TYPE_IPV4) || (PrimitiveType == TYPE_IPV6) || + (PrimitiveType == TYPE_DECIMAL32) || (PrimitiveType == TYPE_DECIMAL64) || + (PrimitiveType == TYPE_DECIMAL128I) || (PrimitiveType == TYPE_DECIMAL256) || (PrimitiveType == TYPE_BOOLEAN) || (PrimitiveType == TYPE_DATEV2) || (PrimitiveType == TYPE_TIMESTAMPTZ) || (PrimitiveType == TYPE_DATETIME) || is_string_type(PrimitiveType)) { diff --git a/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp b/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp index 109fa6f0171878..d8f108c3ec07b1 100644 --- a/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp +++ b/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp @@ -298,6 +298,8 @@ Status RuntimeFilterWrapper::_assign(const PInFilter& in_filter, bool contain_nu } case TYPE_TIMESTAMPTZ: [[fallthrough]]; + case TYPE_TIMESTAMP_NS: + [[fallthrough]]; case TYPE_DATETIMEV2: { batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) { auto date_v2_val = column.longval(); @@ -491,6 +493,8 @@ Status RuntimeFilterWrapper::_assign(const PMinMaxFilter& minmax_filter, bool co } case TYPE_TIMESTAMPTZ: [[fallthrough]]; + case TYPE_TIMESTAMP_NS: + [[fallthrough]]; case TYPE_DATETIMEV2: { int64_t min_val = minmax_filter.min_val().longval(); int64_t max_val = minmax_filter.max_val().longval(); diff --git a/be/src/exprs/aggregate/aggregate_function_approx_count_distinct.cpp b/be/src/exprs/aggregate/aggregate_function_approx_count_distinct.cpp index 28aa90fc4420d2..2821382ebeb4e2 100644 --- a/be/src/exprs/aggregate/aggregate_function_approx_count_distinct.cpp +++ b/be/src/exprs/aggregate/aggregate_function_approx_count_distinct.cpp @@ -30,7 +30,7 @@ AggregateFunctionPtr create_aggregate_function_approx_count_distinct( TYPE_BOOLEAN, TYPE_TINYINT, TYPE_SMALLINT, TYPE_INT, TYPE_BIGINT, TYPE_LARGEINT, TYPE_FLOAT, TYPE_DOUBLE, TYPE_DECIMALV2, TYPE_DECIMAL32, TYPE_DECIMAL64, TYPE_DECIMAL128I, TYPE_DECIMAL256, TYPE_VARCHAR, TYPE_DATEV2, TYPE_DATETIMEV2, - TYPE_IPV4, TYPE_IPV6, + TYPE_TIMESTAMP_NS, TYPE_IPV4, TYPE_IPV6, TYPE_TIMESTAMPTZ>::create(argument_types, result_is_nullable, attr); diff --git a/be/src/exprs/aggregate/aggregate_function_histogram.cpp b/be/src/exprs/aggregate/aggregate_function_histogram.cpp index 622359b3da2199..17cc1cc96ca77d 100644 --- a/be/src/exprs/aggregate/aggregate_function_histogram.cpp +++ b/be/src/exprs/aggregate/aggregate_function_histogram.cpp @@ -39,11 +39,11 @@ AggregateFunctionPtr create_aggregate_function_histogram(const std::string& name const bool result_is_nullable, const AggregateFunctionAttr& attr) { assert_arity_range(name, argument_types, 1, 2); - using creator = - creator_with_type_list; + using creator = creator_with_type_list; if (argument_types.size() == 2) { return creator::create( argument_types, result_is_nullable, attr); @@ -58,4 +58,4 @@ void register_aggregate_function_histogram(AggregateFunctionSimpleFactory& facto factory.register_alias("histogram", "hist"); } -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/exprs/aggregate/aggregate_function_map_v2.cpp b/be/src/exprs/aggregate/aggregate_function_map_v2.cpp index eaaec268914488..28f6f020ab0afd 100644 --- a/be/src/exprs/aggregate/aggregate_function_map_v2.cpp +++ b/be/src/exprs/aggregate/aggregate_function_map_v2.cpp @@ -53,6 +53,7 @@ AggregateFunctionPtr create_aggregate_function_map_agg_v2(const std::string& nam case PrimitiveType::TYPE_VARCHAR: case PrimitiveType::TYPE_DATEV2: case PrimitiveType::TYPE_DATETIMEV2: + case PrimitiveType::TYPE_TIMESTAMP_NS: case PrimitiveType::TYPE_TIMEV2: case PrimitiveType::TYPE_TIMESTAMPTZ: return create_agg_function_map_agg_v2(argument_types, result_is_nullable, attr); @@ -67,4 +68,4 @@ void register_aggregate_function_map_agg_v2(AggregateFunctionSimpleFactory& fact factory.register_function_both("map_agg_v2", create_aggregate_function_map_agg_v2); } -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/exprs/aggregate/aggregate_function_min_max_impl.h b/be/src/exprs/aggregate/aggregate_function_min_max_impl.h index 75a03f8893b53a..9717cc0461c7c1 100644 --- a/be/src/exprs/aggregate/aggregate_function_min_max_impl.h +++ b/be/src/exprs/aggregate/aggregate_function_min_max_impl.h @@ -59,6 +59,10 @@ AggregateFunctionPtr create_aggregate_function_single_value(const String& name, return creator_without_type::create_unary_arguments< AggregateFunctionsSingleValue>>>( argument_types, result_is_nullable, attr); + case PrimitiveType::TYPE_TIMESTAMP_NS: + return creator_without_type::create_unary_arguments< + AggregateFunctionsSingleValue>>>( + argument_types, result_is_nullable, attr); case PrimitiveType::TYPE_TIMESTAMPTZ: return creator_without_type::create_unary_arguments< AggregateFunctionsSingleValue>>>( diff --git a/be/src/exprs/aggregate/aggregate_function_sequence_match.cpp b/be/src/exprs/aggregate/aggregate_function_sequence_match.cpp index 7e55c11e2219c3..3de5d44fcc091b 100644 --- a/be/src/exprs/aggregate/aggregate_function_sequence_match.cpp +++ b/be/src/exprs/aggregate/aggregate_function_sequence_match.cpp @@ -48,6 +48,9 @@ AggregateFunctionPtr create_aggregate_function_sequence_base(const std::string& case TYPE_DATETIMEV2: return creator_without_type::create>( argument_types, result_is_nullable, attr); + case TYPE_TIMESTAMP_NS: + return creator_without_type::create>( + argument_types, result_is_nullable, attr); case TYPE_DATEV2: return creator_without_type::create>( argument_types, result_is_nullable, attr); diff --git a/be/src/exprs/aggregate/aggregate_function_topn_array.cpp b/be/src/exprs/aggregate/aggregate_function_topn_array.cpp index 8c8fee3f4fde9d..7f1ebf7b4c2e5d 100644 --- a/be/src/exprs/aggregate/aggregate_function_topn_array.cpp +++ b/be/src/exprs/aggregate/aggregate_function_topn_array.cpp @@ -27,11 +27,10 @@ using ImplArray = AggregateFunctionTopNImplArray; template using ImplArrayWithDefault = AggregateFunctionTopNImplArray; -using topn_array_creator = - creator_with_type_list; +using topn_array_creator = creator_with_type_list< + TYPE_TINYINT, TYPE_SMALLINT, TYPE_INT, TYPE_BIGINT, TYPE_LARGEINT, TYPE_FLOAT, TYPE_DOUBLE, + TYPE_DECIMAL32, TYPE_DECIMAL64, TYPE_DECIMAL128I, TYPE_DECIMAL256, TYPE_VARCHAR, + TYPE_DATEV2, TYPE_DATETIMEV2, TYPE_TIMESTAMP_NS, TYPE_TIMESTAMPTZ, TYPE_IPV4, TYPE_IPV6>; AggregateFunctionPtr create_aggregate_function_topn_array(const std::string& name, const DataTypes& argument_types, diff --git a/be/src/exprs/aggregate/aggregate_function_topn_weighted.cpp b/be/src/exprs/aggregate/aggregate_function_topn_weighted.cpp index d1c8102458443c..dc955320d32ff5 100644 --- a/be/src/exprs/aggregate/aggregate_function_topn_weighted.cpp +++ b/be/src/exprs/aggregate/aggregate_function_topn_weighted.cpp @@ -27,11 +27,10 @@ using ImplWeight = AggregateFunctionTopNImplWeight; template using ImplWeightWithDefault = AggregateFunctionTopNImplWeight; -using topn_weighted_creator = - creator_with_type_list; +using topn_weighted_creator = creator_with_type_list< + TYPE_TINYINT, TYPE_SMALLINT, TYPE_INT, TYPE_BIGINT, TYPE_LARGEINT, TYPE_FLOAT, TYPE_DOUBLE, + TYPE_DECIMAL32, TYPE_DECIMAL64, TYPE_DECIMAL128I, TYPE_DECIMAL256, TYPE_VARCHAR, + TYPE_DATEV2, TYPE_DATETIMEV2, TYPE_TIMESTAMP_NS, TYPE_TIMESTAMPTZ, TYPE_IPV4, TYPE_IPV6>; AggregateFunctionPtr create_aggregate_function_topn_weighted(const std::string& name, const DataTypes& argument_types, diff --git a/be/src/exprs/aggregate/aggregate_function_uniq.cpp b/be/src/exprs/aggregate/aggregate_function_uniq.cpp index 9bfcc38855c46e..f9f5d75de774bc 100644 --- a/be/src/exprs/aggregate/aggregate_function_uniq.cpp +++ b/be/src/exprs/aggregate/aggregate_function_uniq.cpp @@ -52,12 +52,13 @@ AggregateFunctionPtr create_aggregate_function_uniq(const std::string& name, argument_types, result_is_nullable, attr); } } - return creator_with_type_list< - TYPE_BOOLEAN, TYPE_TINYINT, TYPE_SMALLINT, TYPE_INT, TYPE_BIGINT, TYPE_LARGEINT, - TYPE_DECIMAL32, TYPE_DECIMAL64, TYPE_DECIMAL128I, TYPE_DECIMAL256, TYPE_VARCHAR, - TYPE_ARRAY, TYPE_FLOAT, TYPE_DOUBLE, TYPE_DATEV2, TYPE_DATETIMEV2, TYPE_TIMESTAMPTZ, - TYPE_VARBINARY>::create(argument_types, result_is_nullable, - attr); + return creator_with_type_list::create( + argument_types, result_is_nullable, attr); } void register_aggregate_function_uniq(AggregateFunctionSimpleFactory& factory) { diff --git a/be/src/exprs/create_predicate_function.h b/be/src/exprs/create_predicate_function.h index b20c6613199d52..49b87750046032 100644 --- a/be/src/exprs/create_predicate_function.h +++ b/be/src/exprs/create_predicate_function.h @@ -92,6 +92,7 @@ class PredicateFunctionCreator { M(TYPE_DATETIME) \ M(TYPE_DATEV2) \ M(TYPE_DATETIMEV2) \ + M(TYPE_TIMESTAMP_NS) \ M(TYPE_TIMESTAMPTZ) \ M(TYPE_TIMEV2) \ M(TYPE_CHAR) \ diff --git a/be/src/exprs/function/array/function_array_aggregation.cpp b/be/src/exprs/function/array/function_array_aggregation.cpp index 671f39ea67114d..39c58ee6a2f2ad 100644 --- a/be/src/exprs/function/array/function_array_aggregation.cpp +++ b/be/src/exprs/function/array/function_array_aggregation.cpp @@ -189,6 +189,7 @@ struct ArrayAggregateImpl { execute_type(res, type, data, offsets) || execute_type(res, type, data, offsets) || execute_type(res, type, data, offsets) || + execute_type(res, type, data, offsets) || execute_type(res, type, data, offsets) || execute_type(res, type, data, offsets)) { block.replace_by_position(result, std::move(res)); diff --git a/be/src/exprs/function/cast/cast_to_date.h b/be/src/exprs/function/cast/cast_to_date.h index 009739b0b0bbbf..a4eb08df07699b 100644 --- a/be/src/exprs/function/cast/cast_to_date.h +++ b/be/src/exprs/function/cast/cast_to_date.h @@ -32,6 +32,7 @@ #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_time.h" +#include "core/data_type/data_type_timestamp_ns.h" #include "core/data_type/primitive_type.h" #include "core/data_type_serde/data_type_serde.h" #include "core/types.h" @@ -43,7 +44,8 @@ namespace doris { template - requires(IsStringType && IsDatelikeTypes) + requires(IsStringType && + (IsDatelikeTypes || std::is_same_v)) class CastToImpl : public CastToBase { public: Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, diff --git a/be/src/exprs/function/cast/function_cast.cpp b/be/src/exprs/function/cast/function_cast.cpp index 5561a3a76de4a6..d6c46a4de5e662 100644 --- a/be/src/exprs/function/cast/function_cast.cpp +++ b/be/src/exprs/function/cast/function_cast.cpp @@ -21,6 +21,7 @@ #include "core/data_type/data_type_decimal.h" #include "core/data_type/data_type_number.h" // IWYU pragma: keep #include "core/data_type/data_type_quantilestate.h" +#include "core/data_type/data_type_timestamp_ns.h" #include "core/data_type/data_type_variant.h" #include "core/data_type/data_type_variant_v2.h" #include "core/data_type/primitive_type.h" @@ -312,6 +313,7 @@ WrapperType prepare_impl(FunctionContext* context, const DataTypePtr& origin_fro case PrimitiveType::TYPE_DATETIME: case PrimitiveType::TYPE_DATEV2: case PrimitiveType::TYPE_DATETIMEV2: + case PrimitiveType::TYPE_TIMESTAMP_NS: case PrimitiveType::TYPE_TIMEV2: return create_datelike_wrapper(context, from_type, to_type->get_primitive_type()); case PrimitiveType::TYPE_TIMESTAMPTZ: diff --git a/be/src/exprs/function/cast/function_cast_date.cpp b/be/src/exprs/function/cast/function_cast_date.cpp index 88013ab5e3a04b..5005e329f8082e 100644 --- a/be/src/exprs/function/cast/function_cast_date.cpp +++ b/be/src/exprs/function/cast/function_cast_date.cpp @@ -19,6 +19,7 @@ #include "core/data_type/data_type_date_or_datetime_v2.h" #include "core/data_type/data_type_date_time.h" #include "core/data_type/data_type_time.h" +#include "core/data_type/data_type_timestamp_ns.h" #include "exprs/function/cast/cast_to_date.h" namespace doris::CastWrapper { @@ -30,9 +31,15 @@ WrapperType create_datelike_wrapper(FunctionContext* context, const DataTypePtr& auto make_datelike_wrapper = [&](const auto& types) -> bool { using Types = std::decay_t; using FromDataType = typename Types::LeftType; - if constexpr (CastUtil::IsPureDigitType || IsDatelikeTypes || - IsStringType || - std::is_same_v) { + constexpr bool is_timestamp_ns_source = std::is_same_v; + constexpr bool is_timestamp_ns_target = std::is_same_v; + constexpr bool is_supported_timestamp_ns_cast = + is_timestamp_ns_target && IsStringType; + constexpr bool is_supported_legacy_cast = + !is_timestamp_ns_source && !is_timestamp_ns_target && + (CastUtil::IsPureDigitType || IsDatelikeTypes || + IsStringType || std::is_same_v); + if constexpr (is_supported_timestamp_ns_cast || is_supported_legacy_cast) { if (context->enable_strict_mode()) { cast_to_datelike = std::make_shared< CastToImpl>(); @@ -72,6 +79,8 @@ WrapperType create_datelike_wrapper(FunctionContext* context, const DataTypePtr& return create_datelike_wrapper(context, from_type); case TYPE_DATETIMEV2: return create_datelike_wrapper(context, from_type); + case TYPE_TIMESTAMP_NS: + return create_datelike_wrapper(context, from_type); case TYPE_TIMEV2: return create_datelike_wrapper(context, from_type); default: diff --git a/be/src/exprs/function/date_time_transforms.h b/be/src/exprs/function/date_time_transforms.h index 6bc2d6ed92f01b..d48f35e96c8719 100644 --- a/be/src/exprs/function/date_time_transforms.h +++ b/be/src/exprs/function/date_time_transforms.h @@ -145,6 +145,10 @@ struct ToDateImpl { } else if constexpr (std::is_same_v) { t.cast_to_date(); return t; + } else if constexpr (std::is_same_v) { + auto datetime = t.to_datetime(); + return binary_cast>( + static_cast(datetime.to_date_int_val() >> TIME_PART_LENGTH)); } else { return binary_cast>( (UInt32)(t.to_date_int_val() >> TIME_PART_LENGTH)); diff --git a/be/src/exprs/function/function.h b/be/src/exprs/function/function.h index 8307a6a1c7c1b0..5387b363a42c31 100644 --- a/be/src/exprs/function/function.h +++ b/be/src/exprs/function/function.h @@ -66,7 +66,8 @@ struct FunctionAttr { for (auto it : arguments) { \ is_nullable = is_nullable || it.type->is_nullable(); \ is_datev2 = is_datev2 || it.type->get_primitive_type() == TYPE_DATEV2 || \ - it.type->get_primitive_type() == TYPE_DATETIMEV2; \ + it.type->get_primitive_type() == TYPE_DATETIMEV2 || \ + it.type->get_primitive_type() == TYPE_TIMESTAMP_NS; \ } \ return is_nullable || !is_datev2 \ ? make_nullable( \ diff --git a/be/src/exprs/function/function_convert_tz.cpp b/be/src/exprs/function/function_convert_tz.cpp index aa2966cbc3fd44..b5e2c395fde04c 100644 --- a/be/src/exprs/function/function_convert_tz.cpp +++ b/be/src/exprs/function/function_convert_tz.cpp @@ -40,10 +40,12 @@ #include "core/data_type/data_type_date_time.h" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_timestamp_ns.h" #include "core/data_type/define_primitive_type.h" #include "core/data_type/primitive_type.h" #include "core/string_ref.h" #include "core/types.h" +#include "core/value/timestamp_ns_value.h" #include "core/value/vdatetime_value.h" #include "exec/common/util.hpp" #include "exprs/aggregate/aggregate_function.h" @@ -64,10 +66,6 @@ struct ConvertTzState { }; class FunctionConvertTZ : public IFunction { - constexpr static PrimitiveType PType = PrimitiveType::TYPE_DATETIMEV2; - using DateValueType = PrimitiveTypeTraits::CppType; - using ColumnType = PrimitiveTypeTraits::ColumnType; - public: static constexpr auto name = "convert_tz"; @@ -78,8 +76,11 @@ class FunctionConvertTZ : public IFunction { size_t get_number_of_arguments() const override { return 3; } DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { - return have_nullable(arguments) ? make_nullable(std::make_shared()) - : std::make_shared(); + DataTypePtr result_type = std::make_shared(); + if (remove_nullable(arguments[0])->get_primitive_type() == TYPE_TIMESTAMP_NS) { + result_type = std::make_shared(); + } + return have_nullable(arguments) ? make_nullable(result_type) : result_type; } // default value of timezone is invalid, should skip to avoid wrong exception @@ -168,23 +169,46 @@ class FunctionConvertTZ : public IFunction { argument_columns[1] = remove_nullable(argument_columns[1]); argument_columns[2] = remove_nullable(argument_columns[2]); + const auto argument_type = + remove_nullable(block.get_by_position(arguments[0]).type)->get_primitive_type(); + if (argument_type == TYPE_TIMESTAMP_NS) { + return execute_impl_typed( + context, block, result, input_rows_count, argument_columns, col_const, + convert_tz_state, std::move(result_null_map_column), result_null_map); + } + DORIS_CHECK_EQ(argument_type, TYPE_DATETIMEV2); + return execute_impl_typed( + context, block, result, input_rows_count, argument_columns, col_const, + convert_tz_state, std::move(result_null_map_column), result_null_map); + } + +private: + template + static Status execute_impl_typed(FunctionContext* context, Block& block, uint32_t result, + size_t input_rows_count, + const ColumnPtr (&argument_columns)[3], + const bool (&col_const)[3], ConvertTzState* convert_tz_state, + MutableColumnPtr result_null_map_column, + NullMap& result_null_map) { + using ColumnType = PrimitiveTypeTraits::ColumnType; auto result_column = ColumnType::create(); if (convert_tz_state->use_state) { // ignore argument columns, use cached timezone input in state - execute_tz_const_with_state(convert_tz_state, - assert_cast(argument_columns[0].get()), - result_column.get(), result_null_map, input_rows_count); + execute_tz_const_with_state( + convert_tz_state, assert_cast(argument_columns[0].get()), + result_column.get(), result_null_map, input_rows_count); } else if (col_const[1] && col_const[2]) { // arguments are const - execute_tz_const(context, assert_cast(argument_columns[0].get()), - assert_cast(argument_columns[1].get()), - assert_cast(argument_columns[2].get()), - result_column.get(), result_null_map, input_rows_count); + execute_tz_const(context, + assert_cast(argument_columns[0].get()), + assert_cast(argument_columns[1].get()), + assert_cast(argument_columns[2].get()), + result_column.get(), result_null_map, input_rows_count); } else { - _execute(context, assert_cast(argument_columns[0].get()), - assert_cast(argument_columns[1].get()), - assert_cast(argument_columns[2].get()), - result_column.get(), result_null_map, input_rows_count); + execute(context, assert_cast(argument_columns[0].get()), + assert_cast(argument_columns[1].get()), + assert_cast(argument_columns[2].get()), + result_column.get(), result_null_map, input_rows_count); } //if const if (block.get_data_type(result)->is_nullable()) { @@ -196,11 +220,12 @@ class FunctionConvertTZ : public IFunction { return Status::OK(); } -private: - static void _execute(FunctionContext* context, const ColumnType* date_column, - const ColumnString* from_tz_column, const ColumnString* to_tz_column, - ColumnType* result_column, NullMap& result_null_map, - size_t input_rows_count) { + template + static void execute(FunctionContext* context, + const typename PrimitiveTypeTraits::ColumnType* date_column, + const ColumnString* from_tz_column, const ColumnString* to_tz_column, + typename PrimitiveTypeTraits::ColumnType* result_column, + NullMap& result_null_map, size_t input_rows_count) { for (size_t i = 0; i < input_rows_count; i++) { if (result_null_map[i]) { result_column->insert_default(); @@ -208,27 +233,60 @@ class FunctionConvertTZ : public IFunction { } auto from_tz = from_tz_column->get_data_at(i).to_string(); auto to_tz = to_tz_column->get_data_at(i).to_string(); - execute_inner_loop(date_column, from_tz, to_tz, result_column, result_null_map, i); + execute_inner_loop(date_column, from_tz, to_tz, result_column, result_null_map, + i); } } - static std::pair unix_timestamp_for_convert_tz( + template + static std::pair unix_timestamp_for_convert_tz( const DateValueType& ts_value, const cctz::time_zone& from_tz) { - cctz::civil_second civil_time(ts_value.year(), ts_value.month(), ts_value.day(), - ts_value.hour(), ts_value.minute(), ts_value.second()); + const auto civil_value = [&]() { + if constexpr (std::is_same_v) { + return ts_value.to_datetime(); + } else { + return ts_value; + } + }(); + cctz::civil_second civil_time(civil_value.year(), civil_value.month(), civil_value.day(), + civil_value.hour(), civil_value.minute(), + civil_value.second()); const auto lookup = from_tz.lookup(civil_time); const bool skipped = lookup.kind == cctz::time_zone::civil_lookup::SKIPPED; const auto tp = skipped ? lookup.trans : lookup.pre; // Skipped civil times map to the transition instant. Do not keep the // input fractional part inside a local time interval that never existed. - return {tp.time_since_epoch().count(), skipped ? 0 : ts_value.microsecond()}; + const uint32_t fraction = [&]() { + if constexpr (std::is_same_v) { + return ts_value.nanosecond(); + } + return ts_value.microsecond() * 1000; + }(); + return {tp.time_since_epoch().count(), skipped ? 0 : fraction}; } - static void execute_tz_const_with_state(ConvertTzState* convert_tz_state, - const ColumnType* date_column, - ColumnType* result_column, NullMap& result_null_map, - size_t input_rows_count) { + template + static bool convert_tz_value(const DateValueType& source, const cctz::time_zone& from_tz, + const cctz::time_zone& to_tz, DateValueType& result) { + const auto [seconds, nanoseconds] = unix_timestamp_for_convert_tz(source, from_tz); + if constexpr (std::is_same_v) { + DateV2Value datetime; + datetime.from_unixtime({seconds, nanoseconds / 1000}, to_tz); + return result.from_datetime(datetime, nanoseconds % 1000); + } else { + result.from_unixtime({seconds, nanoseconds / 1000}, to_tz); + return result.is_valid_date(); + } + } + + template + static void execute_tz_const_with_state( + ConvertTzState* convert_tz_state, + const typename PrimitiveTypeTraits::ColumnType* date_column, + typename PrimitiveTypeTraits::ColumnType* result_column, + NullMap& result_null_map, size_t input_rows_count) { + using DateValueType = PrimitiveTypeTraits::CppType; cctz::time_zone& from_tz = convert_tz_state->from_tz; cctz::time_zone& to_tz = convert_tz_state->to_tz; auto push_null = [&](size_t row) { @@ -252,20 +310,21 @@ class FunctionConvertTZ : public IFunction { DateValueType ts_value = date_column->get_element(i); DateValueType ts_value2; - ts_value2.from_unixtime(unix_timestamp_for_convert_tz(ts_value, from_tz), to_tz); - - if (!ts_value2.is_valid_date()) [[unlikely]] { + if (!convert_tz_value(ts_value, from_tz, to_tz, ts_value2)) [[unlikely]] { throw_out_of_bound_convert_tz(date_column->get_element(i), from_tz.name(), to_tz.name()); } - result_column->insert(Field::create_field(ts_value2)); + result_column->insert(Field::create_field(ts_value2)); } } - static void execute_tz_const(FunctionContext* context, const ColumnType* date_column, + template + static void execute_tz_const(FunctionContext* context, + const typename PrimitiveTypeTraits::ColumnType* date_column, const ColumnString* from_tz_column, - const ColumnString* to_tz_column, ColumnType* result_column, + const ColumnString* to_tz_column, + typename PrimitiveTypeTraits::ColumnType* result_column, NullMap& result_null_map, size_t input_rows_count) { auto from_tz = from_tz_column->get_data_at(0).to_string(); auto to_tz = to_tz_column->get_data_at(0).to_string(); @@ -283,13 +342,18 @@ class FunctionConvertTZ : public IFunction { result_column->insert_default(); continue; } - execute_inner_loop(date_column, from_tz, to_tz, result_column, result_null_map, i); + execute_inner_loop(date_column, from_tz, to_tz, result_column, result_null_map, + i); } } - static void execute_inner_loop(const ColumnType* date_column, const std::string& from_tz_name, - const std::string& to_tz_name, ColumnType* result_column, - NullMap& result_null_map, const size_t index_now) { + template + static void execute_inner_loop( + const typename PrimitiveTypeTraits::ColumnType* date_column, + const std::string& from_tz_name, const std::string& to_tz_name, + typename PrimitiveTypeTraits::ColumnType* result_column, + NullMap& result_null_map, const size_t index_now) { + using DateValueType = PrimitiveTypeTraits::CppType; DateValueType ts_value = date_column->get_element(index_now); cctz::time_zone from_tz {}, to_tz {}; DateValueType ts_value2; @@ -303,14 +367,12 @@ class FunctionConvertTZ : public IFunction { to_tz_name); } - ts_value2.from_unixtime(unix_timestamp_for_convert_tz(ts_value, from_tz), to_tz); - - if (!ts_value2.is_valid_date()) [[unlikely]] { + if (!convert_tz_value(ts_value, from_tz, to_tz, ts_value2)) [[unlikely]] { throw_out_of_bound_convert_tz(date_column->get_element(index_now), from_tz.name(), to_tz.name()); } - result_column->insert(Field::create_field(ts_value2)); + result_column->insert(Field::create_field(ts_value2)); } }; diff --git a/be/src/exprs/function/function_date_or_datetime_computation.cpp b/be/src/exprs/function/function_date_or_datetime_computation.cpp index 8973cab77e36a1..d2dc3bf67c72f4 100644 --- a/be/src/exprs/function/function_date_or_datetime_computation.cpp +++ b/be/src/exprs/function/function_date_or_datetime_computation.cpp @@ -18,6 +18,7 @@ #include "exprs/function/function_date_or_datetime_computation.h" #include "core/data_type/data_type_date_or_datetime_v2.h" +#include "core/data_type/data_type_timestamp_ns.h" #include "core/data_type/define_primitive_type.h" #include "exprs/function/simple_function_factory.h" @@ -88,6 +89,23 @@ using FunctionDatetimeAddWeeks = FunctionDateOrDateTimeComputation>; using FunctionDatetimeAddYears = FunctionDateOrDateTimeComputation>; +#define TIMESTAMP_NS_CALENDAR_FUNCTION(TYPE) \ + using FunctionTimestampNsAdd##TYPE = \ + FunctionDateOrDateTimeComputation>; \ + using FunctionTimestampNsSub##TYPE = \ + FunctionDateOrDateTimeComputation> + +TIMESTAMP_NS_CALENDAR_FUNCTION(Microseconds); +TIMESTAMP_NS_CALENDAR_FUNCTION(Milliseconds); +TIMESTAMP_NS_CALENDAR_FUNCTION(Seconds); +TIMESTAMP_NS_CALENDAR_FUNCTION(Minutes); +TIMESTAMP_NS_CALENDAR_FUNCTION(Hours); +TIMESTAMP_NS_CALENDAR_FUNCTION(Days); +TIMESTAMP_NS_CALENDAR_FUNCTION(Weeks); +TIMESTAMP_NS_CALENDAR_FUNCTION(Months); +TIMESTAMP_NS_CALENDAR_FUNCTION(Quarters); +TIMESTAMP_NS_CALENDAR_FUNCTION(Years); + using FunctionTimestamptzAddMicroseconds = FunctionDateOrDateTimeComputation>; using FunctionTimestamptzAddMilliseconds = @@ -105,14 +123,18 @@ using FunctionTimestamptzAddMonths = FunctionDateOrDateTimeComputation>; using FunctionTimestamptzAddYears = FunctionDateOrDateTimeComputation>; -#define FUNCTION_TIME_UNION_CAL(TYPE) \ - using FunctionDatetimeAdd##TYPE = \ - FunctionDateOrDateTimeComputation>; \ - using FunctionDatetimeSub##TYPE = \ - FunctionDateOrDateTimeComputation>; \ - using FunctionTimestamptzAdd##TYPE = \ - FunctionDateOrDateTimeComputation>; \ - using FunctionTimestamptzSub##TYPE = \ +#define FUNCTION_TIME_UNION_CAL(TYPE) \ + using FunctionDatetimeAdd##TYPE = \ + FunctionDateOrDateTimeComputation>; \ + using FunctionDatetimeSub##TYPE = \ + FunctionDateOrDateTimeComputation>; \ + using FunctionTimestampNsAdd##TYPE = \ + FunctionDateOrDateTimeComputation>; \ + using FunctionTimestampNsSub##TYPE = \ + FunctionDateOrDateTimeComputation>; \ + using FunctionTimestamptzAdd##TYPE = \ + FunctionDateOrDateTimeComputation>; \ + using FunctionTimestamptzSub##TYPE = \ FunctionDateOrDateTimeComputation>; FUNCTION_TIME_UNION_CAL(SecondMicrosecond); @@ -184,9 +206,10 @@ using FunctionSubTimeTimestampTz = #define FUNCTION_TIME_DIFF(NAME, IMPL, TYPE) using NAME##_##TYPE = FunctionTimeDiff>; -#define ALL_FUNCTION_TIME_DIFF(NAME, IMPL) \ - FUNCTION_TIME_DIFF(NAME, IMPL, TYPE_DATETIMEV2) \ - FUNCTION_TIME_DIFF(NAME, IMPL, TYPE_DATEV2) \ +#define ALL_FUNCTION_TIME_DIFF(NAME, IMPL) \ + FUNCTION_TIME_DIFF(NAME, IMPL, TYPE_DATETIMEV2) \ + FUNCTION_TIME_DIFF(NAME, IMPL, TYPE_TIMESTAMP_NS) \ + FUNCTION_TIME_DIFF(NAME, IMPL, TYPE_DATEV2) \ FUNCTION_TIME_DIFF(NAME, IMPL, TYPE_TIMESTAMPTZ) // these diff functions accept all v2 types. but for v1 only datetime. ALL_FUNCTION_TIME_DIFF(FunctionDatetimeDateDiff, DateDiffImpl) @@ -257,6 +280,16 @@ void register_function_date_time_computation(SimpleFunctionFactory& factory) { factory.register_function(); factory.register_function(); factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); factory.register_function(); factory.register_function(); factory.register_function(); @@ -271,6 +304,8 @@ void register_function_date_time_computation(SimpleFunctionFactory& factory) { #define REGISTER_TIME_UNION_CAL(TYPE) \ factory.register_function(); \ factory.register_function(); \ + factory.register_function(); \ + factory.register_function(); \ factory.register_function(); \ factory.register_function(); @@ -302,6 +337,16 @@ void register_function_date_time_computation(SimpleFunctionFactory& factory) { factory.register_function(); factory.register_function(); factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); factory.register_function(); factory.register_function(); @@ -323,9 +368,10 @@ void register_function_date_time_computation(SimpleFunctionFactory& factory) { #define REGISTER_DATEV2_FUNCTIONS_DIFF(NAME, TYPE) factory.register_function(); -#define REGISTER_ALL_DATEV2_FUNCTIONS_DIFF(NAME) \ - REGISTER_DATEV2_FUNCTIONS_DIFF(NAME, TYPE_DATETIMEV2) \ - REGISTER_DATEV2_FUNCTIONS_DIFF(NAME, TYPE_DATEV2) \ +#define REGISTER_ALL_DATEV2_FUNCTIONS_DIFF(NAME) \ + REGISTER_DATEV2_FUNCTIONS_DIFF(NAME, TYPE_DATETIMEV2) \ + REGISTER_DATEV2_FUNCTIONS_DIFF(NAME, TYPE_TIMESTAMP_NS) \ + REGISTER_DATEV2_FUNCTIONS_DIFF(NAME, TYPE_DATEV2) \ REGISTER_DATEV2_FUNCTIONS_DIFF(NAME, TYPE_TIMESTAMPTZ) REGISTER_ALL_DATEV2_FUNCTIONS_DIFF(FunctionDatetimeDateDiff) @@ -347,4 +393,4 @@ void register_function_date_time_computation(SimpleFunctionFactory& factory) { factory.register_function(); } -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/exprs/function/function_date_or_datetime_to_string.cpp b/be/src/exprs/function/function_date_or_datetime_to_string.cpp index c7b240df961901..791c6d229b29a8 100644 --- a/be/src/exprs/function/function_date_or_datetime_to_string.cpp +++ b/be/src/exprs/function/function_date_or_datetime_to_string.cpp @@ -37,6 +37,7 @@ #include "core/column/column_vector.h" #include "core/data_type/data_type.h" #include "core/data_type/data_type_date_or_datetime_v2.h" // IWYU pragma: keep +#include "core/data_type/data_type_timestamp_ns.h" #include "core/data_type/define_primitive_type.h" #include "core/types.h" #include "exprs/aggregate/aggregate_function.h" @@ -212,6 +213,9 @@ using FunctionMonthNameV2 = FunctionDateOrDateTimeToString>; using FunctionDateTimeV2MonthName = FunctionDateOrDateTimeToString>; +using FunctionTimeStampNsDayName = FunctionDateOrDateTimeToString>; +using FunctionTimeStampNsMonthName = + FunctionDateOrDateTimeToString>; using FunctionYearMonth = FunctionDateOrDateTimeToString; using FunctionDayHour = FunctionDateOrDateTimeToString; using FunctionDayMinute = FunctionDateOrDateTimeToString; @@ -226,6 +230,7 @@ using FunctionSecondMicrosecond = FunctionDateOrDateTimeToString>; using FunctionDateTimeIso8601 = FunctionDateOrDateTimeToString>; +using FunctionTimeStampNsIso8601 = FunctionDateOrDateTimeToString>; using FunctionTimestampTzIso8601 = FunctionDateOrDateTimeToString>; void register_function_date_time_to_string(SimpleFunctionFactory& factory) { @@ -233,6 +238,8 @@ void register_function_date_time_to_string(SimpleFunctionFactory& factory) { factory.register_function(); factory.register_function(); factory.register_function(); + factory.register_function(); + factory.register_function(); factory.register_function(); factory.register_function(); factory.register_function(); @@ -246,6 +253,7 @@ void register_function_date_time_to_string(SimpleFunctionFactory& factory) { factory.register_function(); factory.register_function(); factory.register_function(); + factory.register_function(); factory.register_function(); } diff --git a/be/src/exprs/function/function_datetime_floor_ceil.cpp b/be/src/exprs/function/function_datetime_floor_ceil.cpp index 6369dd9592cda3..8cd62f87ea50c8 100644 --- a/be/src/exprs/function/function_datetime_floor_ceil.cpp +++ b/be/src/exprs/function/function_datetime_floor_ceil.cpp @@ -46,6 +46,7 @@ #include "core/data_type/data_type_date_time.h" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_timestamp_ns.h" #include "core/data_type/define_primitive_type.h" #include "core/field.h" #include "core/pod_array_fwd.h" diff --git a/be/src/exprs/function/function_datetime_string_to_string.cpp b/be/src/exprs/function/function_datetime_string_to_string.cpp index 9fca37931ecc84..0559d3a4dabb0e 100644 --- a/be/src/exprs/function/function_datetime_string_to_string.cpp +++ b/be/src/exprs/function/function_datetime_string_to_string.cpp @@ -18,6 +18,7 @@ #include "exprs/function/function_datetime_string_to_string.h" #include "core/data_type/data_type_date_or_datetime_v2.h" // IWYU pragma: keep +#include "core/data_type/data_type_timestamp_ns.h" #include "core/data_type/define_primitive_type.h" #include "exprs/function/date_time_transforms.h" #include "exprs/function/simple_function_factory.h" @@ -27,6 +28,8 @@ namespace doris { using FunctionDateFormatV2 = FunctionDateTimeStringToString>; using FunctionDateTimeV2DateFormat = FunctionDateTimeStringToString>; +using FunctionTimeStampNsDateFormat = + FunctionDateTimeStringToString>; // old version using FunctionFromUnixTimeOneArg = FunctionDateTimeStringToString>; using FunctionFromUnixTimeTwoArg = FunctionDateTimeStringToString>; @@ -39,6 +42,7 @@ using FunctionFromUnixTimeNewDecimalTwoArg = FunctionDateTimeStringToString>; using FunctionTimeFormatDate = FunctionTimeFormat; using FunctionTimeFormatDateTime = FunctionTimeFormat; +using FunctionTimeFormatTimeStampNs = FunctionTimeFormat; using FunctionTimeFormatTime = FunctionTimeFormat; void register_function_date_time_string_to_string(SimpleFunctionFactory& factory) { @@ -50,8 +54,10 @@ void register_function_date_time_string_to_string(SimpleFunctionFactory& factory factory.register_function(); factory.register_function(); factory.register_function(); + factory.register_function(); factory.register_function(); factory.register_function(); + factory.register_function(); factory.register_function(); } diff --git a/be/src/exprs/function/function_datetime_string_to_string.h b/be/src/exprs/function/function_datetime_string_to_string.h index f927389ac261d5..c1d7071fdb938d 100644 --- a/be/src/exprs/function/function_datetime_string_to_string.h +++ b/be/src/exprs/function/function_datetime_string_to_string.h @@ -229,6 +229,7 @@ class FunctionDateTimeStringToString : public IFunction { std::is_same_v || std::is_same_v || std::is_same_v || + std::is_same_v || std::is_same_v) { char buf[64]; pod_array[i].to_string(buf); diff --git a/be/src/exprs/function/function_other_types_to_date.cpp b/be/src/exprs/function/function_other_types_to_date.cpp index 122fdb9338ad57..5798eb85a1eaf3 100644 --- a/be/src/exprs/function/function_other_types_to_date.cpp +++ b/be/src/exprs/function/function_other_types_to_date.cpp @@ -50,6 +50,7 @@ #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_timestamp_ns.h" #include "core/data_type/define_primitive_type.h" #include "core/data_type/primitive_type.h" #include "core/pod_array.h" @@ -695,15 +696,20 @@ template struct UnixTimeStampDateImpl { static DataTypes get_variadic_argument_types() { return {std::make_shared()}; } - using ResultDataType = - std::conditional_t, DataTypeDecimal64, - std::conditional_t>; - using ResultColumnType = - std::conditional_t, ColumnDecimal64, - std::conditional_t>; + static constexpr bool is_datetime_v2 = std::is_same_v || + std::is_same_v; + static constexpr bool is_timestamp_ns = std::is_same_v; + using ResultDataType = std::conditional_t< + is_timestamp_ns, DataTypeDecimal128, + std::conditional_t>>; + using ResultColumnType = std::conditional_t< + is_timestamp_ns, ColumnDecimal128V3, + std::conditional_t>>; static DataTypePtr get_return_type_impl(const ColumnsWithTypeAndName& arguments) { - if constexpr (std::is_same_v) { + if constexpr (is_datetime_v2) { UInt32 scale = arguments[0].type->get_scale(); return std::make_shared(12 + scale, scale); } else { @@ -747,7 +753,7 @@ struct UnixTimeStampDateImpl { col_result_data[i] = trim_timestamp(timestamp, NewVersion); } block.replace_by_position(result, std::move(col_result)); - } else { // DatetimeV2 + } else if constexpr (std::is_same_v) { const auto* col_source = assert_cast(col.get()); UInt32 scale = block.get_by_position(arguments[0]).type->get_scale(); auto col_result = ColumnDecimal64::create(input_rows_count, scale); @@ -768,6 +774,28 @@ struct UnixTimeStampDateImpl { .value; } block.replace_by_position(result, std::move(col_result)); + } else { + const auto* col_source = assert_cast(col.get()); + UInt32 scale = block.get_by_position(arguments[0]).type->get_scale(); + auto col_result = ColumnDecimal128V3::create(input_rows_count, scale); + auto& col_result_data = col_result->get_data(); + col_result->resize(input_rows_count); + + for (int i = 0; i < input_rows_count; i++) { + const auto& nano_value = col_source->get_data()[i]; + const auto datetime_value = nano_value.to_datetime(); + std::pair timestamp {}; + datetime_value.unix_timestamp(×tamp, context->state()->timezone_obj()); + + const auto trimmed_timestamp = trim_timestamp(timestamp, NewVersion); + const int64_t fraction = + trimmed_timestamp.first == timestamp.first + ? nano_value.nanosecond() / common::exp10_i64(9 - scale) + : 0; + col_result_data[i] = + Decimal128V3::from_int_frac(trimmed_timestamp.first, fraction, scale).value; + } + block.replace_by_position(result, std::move(col_result)); } return Status::OK(); @@ -1222,12 +1250,7 @@ class FunctionOtherTypesToDateType : public IFunction { } Status open(FunctionContext* context, FunctionContext::FunctionStateScope scope) override { - if constexpr (std::is_same_v> || - std::is_same_v> || - std::is_same_v> || - std::is_same_v> || - std::is_same_v> || - std::is_same_v>) { + if constexpr (requires { Impl::open(context, scope); }) { return Impl::open(context, scope); } else { return Status::OK(); @@ -1474,12 +1497,16 @@ using FunctionMakeDate = FunctionOtherTypesToDateType; using FunctionDateTruncDateV2 = FunctionOtherTypesToDateType>; using FunctionDateTruncDatetimeV2 = FunctionOtherTypesToDateType>; +using FunctionDateTruncTimestampNs = + FunctionOtherTypesToDateType>; using FunctionDateTruncTimestamptz = FunctionOtherTypesToDateType>; using FunctionDateTruncDateV2WithCommonOrder = FunctionOtherTypesToDateType>; using FunctionDateTruncDatetimeV2WithCommonOrder = FunctionOtherTypesToDateType>; +using FunctionDateTruncTimestampNsWithCommonOrder = + FunctionOtherTypesToDateType>; using FunctionDateTruncTimestamptzWithCommonOrder = FunctionOtherTypesToDateType>; using FunctionFromIso8601DateV2 = FunctionOtherTypesToDateType; @@ -1492,9 +1519,11 @@ void register_function_timestamp(SimpleFunctionFactory& factory) { factory.register_function(); factory.register_function(); factory.register_function(); + factory.register_function(); factory.register_function(); factory.register_function(); factory.register_function(); + factory.register_function(); factory.register_function(); factory.register_function(); @@ -1503,6 +1532,7 @@ void register_function_timestamp(SimpleFunctionFactory& factory) { factory.register_function>>(); factory.register_function>>(); factory.register_function>>(); + factory.register_function>>(); factory.register_function>>(); factory.register_function>>(); factory.register_function< @@ -1513,6 +1543,8 @@ void register_function_timestamp(SimpleFunctionFactory& factory) { FunctionUnixTimestampNew>>(); factory.register_function< FunctionUnixTimestampNew>>(); + factory.register_function< + FunctionUnixTimestampNew>>(); factory.register_function>>(); factory.register_function>(); diff --git a/be/src/exprs/function/functions_comparison.h b/be/src/exprs/function/functions_comparison.h index a0789c259c790b..5bd7f9fc8c2444 100644 --- a/be/src/exprs/function/functions_comparison.h +++ b/be/src/exprs/function/functions_comparison.h @@ -798,6 +798,8 @@ class FunctionComparison : public IFunction { return execute_num_type(block, result, col_left_ptr, col_right_ptr); case TYPE_DATETIMEV2: return execute_num_type(block, result, col_left_ptr, col_right_ptr); + case TYPE_TIMESTAMP_NS: + return execute_num_type(block, result, col_left_ptr, col_right_ptr); case TYPE_TIMESTAMPTZ: return execute_num_type(block, result, col_left_ptr, col_right_ptr); case TYPE_TINYINT: diff --git a/be/src/exprs/function/least_greast.cpp b/be/src/exprs/function/least_greast.cpp index 13558035636154..9cbbd4ccdc0427 100644 --- a/be/src/exprs/function/least_greast.cpp +++ b/be/src/exprs/function/least_greast.cpp @@ -304,6 +304,13 @@ struct FunctionFieldImpl { } break; } + case PrimitiveType::TYPE_TIMESTAMP_NS: { + for (int col = 1; col < arguments.size(); ++col) { + insert_result_data(res_data, argument_columns[0], + argument_columns[col], input_rows_count, col); + } + break; + } case PrimitiveType::TYPE_TIMESTAMPTZ: { for (int col = 1; col < arguments.size(); ++col) { insert_result_data(res_data, argument_columns[0], @@ -367,4 +374,4 @@ void register_function_least_greast(SimpleFunctionFactory& factory) { factory.register_function(); factory.register_function(); } -}; // namespace doris \ No newline at end of file +}; // namespace doris diff --git a/be/src/exprs/function/time_of_function.cpp b/be/src/exprs/function/time_of_function.cpp index 6162477f38517f..d9133c94242cf5 100644 --- a/be/src/exprs/function/time_of_function.cpp +++ b/be/src/exprs/function/time_of_function.cpp @@ -20,6 +20,7 @@ #include "core/data_type/data_type_date_or_datetime_v2.h" // IWYU pragma: keep #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_timestamp_ns.h" #include "exprs/function/date_time_transforms.h" #include "exprs/function/function_date_or_datetime_to_something.h" #include "exprs/function/simple_function_factory.h" @@ -51,6 +52,17 @@ using FunctionDateTimeV2YearWeek = using FunctionDateTimeV2WeekDay = FunctionDateOrDateTimeToSomething>; +#define TIMESTAMP_NS_TIME_OF_FUNCTION(NAME, RESULT_TYPE, IMPL) \ + using FunctionTimeStampNs##NAME = \ + FunctionDateOrDateTimeToSomething> + +TIMESTAMP_NS_TIME_OF_FUNCTION(WeekOfYear, DataTypeInt8, WeekOfYearImpl); +TIMESTAMP_NS_TIME_OF_FUNCTION(DayOfYear, DataTypeInt16, DayOfYearImpl); +TIMESTAMP_NS_TIME_OF_FUNCTION(DayOfWeek, DataTypeInt8, DayOfWeekImpl); +TIMESTAMP_NS_TIME_OF_FUNCTION(DayOfMonth, DataTypeInt8, DayOfMonthImpl); +TIMESTAMP_NS_TIME_OF_FUNCTION(YearWeek, DataTypeInt32, ToYearWeekOneArgImpl); +TIMESTAMP_NS_TIME_OF_FUNCTION(WeekDay, DataTypeInt8, WeekDayImpl); + void register_function_time_of_function(SimpleFunctionFactory& factory) { factory.register_function(); factory.register_function(); @@ -64,5 +76,11 @@ void register_function_time_of_function(SimpleFunctionFactory& factory) { factory.register_function(); factory.register_function(); factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); } } // namespace doris diff --git a/be/src/exprs/function/to_time_function.cpp b/be/src/exprs/function/to_time_function.cpp index 2f4b6491ceb4d2..d4e68f07f43212 100644 --- a/be/src/exprs/function/to_time_function.cpp +++ b/be/src/exprs/function/to_time_function.cpp @@ -19,6 +19,7 @@ #include "core/data_type/data_type_date_or_datetime_v2.h" #include "core/data_type/data_type_date_time.h" #include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_timestamp_ns.h" #include "core/types.h" #include "exprs/function/date_time_transforms.h" #include "exprs/function/function_date_or_datetime_to_something.h" @@ -72,6 +73,24 @@ using FunctionCenturyV2 = FunctionDateOrDateTimeToSomething>; using FunctionDateTimeV2Century = FunctionDateOrDateTimeToSomething>; + +#define TIMESTAMP_NS_TO_TIME_FUNCTION(NAME, RESULT_TYPE, IMPL) \ + using FunctionTimeStampNs##NAME = \ + FunctionDateOrDateTimeToSomething> + +TIMESTAMP_NS_TO_TIME_FUNCTION(Year, DataTypeInt16, ToYearImpl); +TIMESTAMP_NS_TO_TIME_FUNCTION(Quarter, DataTypeInt8, ToQuarterImpl); +TIMESTAMP_NS_TO_TIME_FUNCTION(Month, DataTypeInt8, ToMonthImpl); +TIMESTAMP_NS_TO_TIME_FUNCTION(Day, DataTypeInt8, ToDayImpl); +TIMESTAMP_NS_TO_TIME_FUNCTION(Week, DataTypeInt8, ToWeekOneArgImpl); +TIMESTAMP_NS_TO_TIME_FUNCTION(Hour, DataTypeInt8, ToHourImpl); +TIMESTAMP_NS_TO_TIME_FUNCTION(Minute, DataTypeInt8, ToMinuteImpl); +TIMESTAMP_NS_TO_TIME_FUNCTION(Second, DataTypeInt8, ToSecondImpl); +TIMESTAMP_NS_TO_TIME_FUNCTION(MicroSecond, DataTypeInt32, ToMicroSecondImpl); +TIMESTAMP_NS_TO_TIME_FUNCTION(ToDate, DataTypeDateV2, ToDateImpl); +TIMESTAMP_NS_TO_TIME_FUNCTION(Date, DataTypeDateV2, DateImpl); +TIMESTAMP_NS_TO_TIME_FUNCTION(Century, DataTypeInt16, ToCenturyImpl); + void register_function_to_time_function(SimpleFunctionFactory& factory) { factory.register_function(); factory.register_function(); @@ -100,6 +119,18 @@ void register_function_to_time_function(SimpleFunctionFactory& factory) { factory.register_function(); factory.register_function(); factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); factory.register_alias("date", "datev2"); factory.register_alias("to_date", "to_datev2"); } diff --git a/be/src/exprs/lambda_function/varray_sort_function.cpp b/be/src/exprs/lambda_function/varray_sort_function.cpp index dfd2ed1a8ae58e..2d0484c64f07a7 100644 --- a/be/src/exprs/lambda_function/varray_sort_function.cpp +++ b/be/src/exprs/lambda_function/varray_sort_function.cpp @@ -52,7 +52,7 @@ using ConstColumnVariant = const ColumnDecimal32*, const ColumnDecimal64*, const ColumnDecimal128V2*, const ColumnDecimal128V3*, const ColumnDecimal256*, const ColumnDate*, const ColumnDateTime*, const ColumnDateV2*, const ColumnDateTimeV2*, - const ColumnTimeV2*>; + const ColumnTimeStampNs*, const ColumnTimeV2*>; template struct is_column_vector : std::false_type {}; @@ -422,6 +422,7 @@ class ArraySortFunction : public LambdaFunction { DISPATCH_PRIMITIVE_TYPE(TYPE_DATETIME, ColumnDateTime) DISPATCH_PRIMITIVE_TYPE(TYPE_DATEV2, ColumnDateV2) DISPATCH_PRIMITIVE_TYPE(TYPE_DATETIMEV2, ColumnDateTimeV2) + DISPATCH_PRIMITIVE_TYPE(TYPE_TIMESTAMP_NS, ColumnTimeStampNs) DISPATCH_PRIMITIVE_TYPE(TYPE_TIMEV2, ColumnTimeV2) default: return Status::InternalError("Unsupported type in array_sort"); diff --git a/be/src/exprs/vcase_expr.h b/be/src/exprs/vcase_expr.h index 07793b969e019f..d1312cb4983c3a 100644 --- a/be/src/exprs/vcase_expr.h +++ b/be/src/exprs/vcase_expr.h @@ -143,6 +143,7 @@ class VCaseExpr final : public VExpr { CASE_TYPE(TYPE_DATETIME, ColumnDateTime) CASE_TYPE(TYPE_DATEV2, ColumnDateV2) CASE_TYPE(TYPE_DATETIMEV2, ColumnDateTimeV2) + CASE_TYPE(TYPE_TIMESTAMP_NS, ColumnTimeStampNs) CASE_TYPE(TYPE_TIMESTAMPTZ, ColumnTimeStampTz) CASE_TYPE(TYPE_IPV6, ColumnIPv6) CASE_TYPE(TYPE_IPV4, ColumnIPv4) @@ -231,6 +232,7 @@ class VCaseExpr final : public VExpr { std::is_same_v || std::is_same_v || std::is_same_v || + std::is_same_v || std::is_same_v) { result_raw_data[i] = ColumnType::default_value(); } else { @@ -251,6 +253,7 @@ class VCaseExpr final : public VExpr { std::is_same_v || std::is_same_v || std::is_same_v || + std::is_same_v || std::is_same_v) { for (int row_idx = 0; row_idx < rows_count; row_idx++) { result_raw_data[row_idx] = (then_idx[row_idx] == i) ? column_raw_data[row_idx] diff --git a/be/src/exprs/vcondition_expr.cpp b/be/src/exprs/vcondition_expr.cpp index 0f9d8e6a7bd427..e2293faf3dcd93 100644 --- a/be/src/exprs/vcondition_expr.cpp +++ b/be/src/exprs/vcondition_expr.cpp @@ -577,6 +577,11 @@ void insert_result_data(MutableColumnPtr& result_column, ColumnPtr& argument_col result_raw_data[row].to_date_int_val() + column_raw_data[row].to_date_int_val() * uint64_t(!(null_map_data[row] | filled_flag[row]))); + } else if constexpr (std::is_same_v) { + result_raw_data[row] = + TimeStampNsValue(result_raw_data[row].epoch_nanos() + + column_raw_data[row].epoch_nanos() * + int64_t(!(null_map_data[row] | filled_flag[row]))); } else if constexpr (std::is_same_v) { result_raw_data[row] = binary_cast( result_raw_data[row].to_date_int_val() + @@ -635,7 +640,6 @@ Status filled_result_column(const DataTypePtr& data_type, MutableColumnPtr& resu return insert_result_data_bitmap(result_column, argument_column, null_map_data, filled_flag, input_rows_count); } - auto call = [&](const auto& type) -> bool { using DispatchType = std::decay_t; insert_result_data( diff --git a/be/src/exprs/vdirect_in_predicate.h b/be/src/exprs/vdirect_in_predicate.h index 693571d25bf777..5a71294ddb42a2 100644 --- a/be/src/exprs/vdirect_in_predicate.h +++ b/be/src/exprs/vdirect_in_predicate.h @@ -264,6 +264,7 @@ class VDirectInPredicate final : public VExpr { RETURN_RAW_FIXED_SIZE(TYPE_DATETIME); RETURN_RAW_FIXED_SIZE(TYPE_DATEV2); RETURN_RAW_FIXED_SIZE(TYPE_DATETIMEV2); + RETURN_RAW_FIXED_SIZE(TYPE_TIMESTAMP_NS); RETURN_RAW_FIXED_SIZE(TYPE_TIMESTAMPTZ); RETURN_RAW_FIXED_SIZE(TYPE_TIMEV2); RETURN_RAW_FIXED_SIZE(TYPE_DECIMAL32); diff --git a/be/src/exprs/vectorized_fn_call.cpp b/be/src/exprs/vectorized_fn_call.cpp index 6e9f25be48d864..d88ac5a587a3c3 100644 --- a/be/src/exprs/vectorized_fn_call.cpp +++ b/be/src/exprs/vectorized_fn_call.cpp @@ -197,6 +197,7 @@ size_t raw_comparison_value_size(PrimitiveType primitive_type) { RETURN_RAW_COMPARISON_SIZE(TYPE_DATETIME); RETURN_RAW_COMPARISON_SIZE(TYPE_DATEV2); RETURN_RAW_COMPARISON_SIZE(TYPE_DATETIMEV2); + RETURN_RAW_COMPARISON_SIZE(TYPE_TIMESTAMP_NS); RETURN_RAW_COMPARISON_SIZE(TYPE_TIMESTAMPTZ); RETURN_RAW_COMPARISON_SIZE(TYPE_TIMEV2); RETURN_RAW_COMPARISON_SIZE(TYPE_DECIMAL32); @@ -542,6 +543,7 @@ Status VectorizedFnCall::execute_on_raw_fixed_values(const uint8_t* values, size EXECUTE_RAW_SCALAR_COMPARISON(TYPE_DATETIME); EXECUTE_RAW_SCALAR_COMPARISON(TYPE_DATEV2); EXECUTE_RAW_SCALAR_COMPARISON(TYPE_DATETIMEV2); + EXECUTE_RAW_SCALAR_COMPARISON(TYPE_TIMESTAMP_NS); EXECUTE_RAW_SCALAR_COMPARISON(TYPE_TIMESTAMPTZ); EXECUTE_RAW_SCALAR_COMPARISON(TYPE_TIMEV2); EXECUTE_RAW_SCALAR_COMPARISON(TYPE_DECIMAL32); diff --git a/be/src/exprs/vexpr.cpp b/be/src/exprs/vexpr.cpp index 5b1f835ace5a26..7ee3c772829863 100644 --- a/be/src/exprs/vexpr.cpp +++ b/be/src/exprs/vexpr.cpp @@ -138,6 +138,10 @@ TExprNode create_texpr_node_from(const void* data, const PrimitiveType& type, in THROW_IF_ERROR(create_texpr_literal_node(data, &node, precision, scale)); break; } + case TYPE_TIMESTAMP_NS: { + THROW_IF_ERROR(create_texpr_literal_node(data, &node, precision, scale)); + break; + } case TYPE_DATE: { THROW_IF_ERROR(create_texpr_literal_node(data, &node)); break; @@ -261,6 +265,12 @@ TExprNode create_texpr_node_from(const Field& field, const PrimitiveType& type, create_texpr_literal_node(&storage, &node, precision, scale)); break; } + case TYPE_TIMESTAMP_NS: { + const auto& storage = field.get(); + THROW_IF_ERROR( + create_texpr_literal_node(&storage, &node, precision, scale)); + break; + } case TYPE_TIMESTAMPTZ: { const auto& storage = field.get(); diff --git a/be/src/exprs/vexpr.h b/be/src/exprs/vexpr.h index c0b99ab7a8f4dc..52eb9987050e66 100644 --- a/be/src/exprs/vexpr.h +++ b/be/src/exprs/vexpr.h @@ -612,6 +612,15 @@ Status create_texpr_literal_node(const void* data, TExprNode* node, int precisio (*node).__set_date_literal(date_literal); (*node).__set_node_type(TExprNodeType::DATE_LITERAL); (*node).__set_type(create_type_desc(PrimitiveType::TYPE_DATETIMEV2, precision, scale)); + } else if constexpr (T == TYPE_TIMESTAMP_NS) { + const auto* origin_value = reinterpret_cast(data); + TDateLiteral date_literal; + char convert_buffer[30]; + origin_value->to_string(convert_buffer, scale); + date_literal.__set_value(convert_buffer); + (*node).__set_date_literal(date_literal); + (*node).__set_node_type(TExprNodeType::DATE_LITERAL); + (*node).__set_type(create_type_desc(PrimitiveType::TYPE_TIMESTAMP_NS)); } else if constexpr (T == TYPE_TIMESTAMPTZ) { const auto* origin_value = reinterpret_cast(data); TDateLiteral date_literal; diff --git a/be/src/exprs/vin_predicate.cpp b/be/src/exprs/vin_predicate.cpp index 0dbfb3f7e9a289..61bd18f2f8b1c8 100644 --- a/be/src/exprs/vin_predicate.cpp +++ b/be/src/exprs/vin_predicate.cpp @@ -64,6 +64,7 @@ size_t raw_in_value_size(PrimitiveType primitive_type) { RETURN_RAW_IN_SIZE(TYPE_DATETIME); RETURN_RAW_IN_SIZE(TYPE_DATEV2); RETURN_RAW_IN_SIZE(TYPE_DATETIMEV2); + RETURN_RAW_IN_SIZE(TYPE_TIMESTAMP_NS); RETURN_RAW_IN_SIZE(TYPE_TIMESTAMPTZ); RETURN_RAW_IN_SIZE(TYPE_TIMEV2); RETURN_RAW_IN_SIZE(TYPE_DECIMAL32); diff --git a/be/src/exprs/vtopn_pred.cpp b/be/src/exprs/vtopn_pred.cpp index cc26b6679a1342..07211dc1916a89 100644 --- a/be/src/exprs/vtopn_pred.cpp +++ b/be/src/exprs/vtopn_pred.cpp @@ -45,6 +45,7 @@ size_t topn_raw_value_size(PrimitiveType type) { RETURN_TOPN_RAW_SIZE(TYPE_DATETIME); RETURN_TOPN_RAW_SIZE(TYPE_DATEV2); RETURN_TOPN_RAW_SIZE(TYPE_DATETIMEV2); + RETURN_TOPN_RAW_SIZE(TYPE_TIMESTAMP_NS); RETURN_TOPN_RAW_SIZE(TYPE_TIMESTAMPTZ); // Master no longer defines a C++ carrier for legacy TYPE_TIME; Parquet time values use // TYPE_TIMEV2, so advertising the deprecated tag would make direct filtering unusable. @@ -160,6 +161,7 @@ Status VTopNPred::execute_on_raw_fixed_values(const uint8_t* values, size_t num_ EXECUTE_TOPN_RAW_SCALAR(TYPE_DATETIME); EXECUTE_TOPN_RAW_SCALAR(TYPE_DATEV2); EXECUTE_TOPN_RAW_SCALAR(TYPE_DATETIMEV2); + EXECUTE_TOPN_RAW_SCALAR(TYPE_TIMESTAMP_NS); EXECUTE_TOPN_RAW_SCALAR(TYPE_TIMESTAMPTZ); // Keep dispatch aligned with topn_raw_value_size(): legacy TYPE_TIME has no master carrier. EXECUTE_TOPN_RAW_SCALAR(TYPE_TIMEV2); diff --git a/be/src/information_schema/schema_columns_scanner.cpp b/be/src/information_schema/schema_columns_scanner.cpp index e84acc790e0b2b..158e13db97ec2b 100644 --- a/be/src/information_schema/schema_columns_scanner.cpp +++ b/be/src/information_schema/schema_columns_scanner.cpp @@ -134,6 +134,8 @@ std::string SchemaColumnsScanner::_to_mysql_data_type_string(TColumnDesc& desc) case TPrimitiveType::DATETIME: case TPrimitiveType::DATETIMEV2: return "datetime"; + case TPrimitiveType::TIMESTAMP_NS: + return "timestamp_ns"; case TPrimitiveType::TIMESTAMPTZ: return "timestamp"; case TPrimitiveType::DECIMAL32: diff --git a/be/src/runtime/fold_constant_executor.cpp b/be/src/runtime/fold_constant_executor.cpp index 97efd2d72deb5b..dcfcf656cc0e38 100644 --- a/be/src/runtime/fold_constant_executor.cpp +++ b/be/src/runtime/fold_constant_executor.cpp @@ -265,6 +265,11 @@ Status FoldConstantExecutor::_get_result(void* src, size_t size, const DataTypeP result = CastToString::from_datetimev2(value, type->get_scale()); break; } + case TYPE_TIMESTAMP_NS: { + const auto value = TimeStampNsValue(*reinterpret_cast(src)); + result = value.to_string(type->get_scale()); + break; + } case TYPE_TIMESTAMPTZ: { auto value = binary_cast(*(int64_t*)src); result = CastToString::from_timestamptz(value, type->get_scale(), options.timezone); @@ -298,4 +303,4 @@ Status FoldConstantExecutor::_get_result(void* src, size_t size, const DataTypeP return Status::OK(); } -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/storage/delete/delete_handler.cpp b/be/src/storage/delete/delete_handler.cpp index 1781c7734de7d0..132a0cb12eb3e0 100644 --- a/be/src/storage/delete/delete_handler.cpp +++ b/be/src/storage/delete/delete_handler.cpp @@ -28,6 +28,7 @@ #include "common/logging.h" #include "common/status.h" #include "core/data_type_serde/data_type_serde.h" +#include "core/data_type_serde/data_type_timestamp_ns_serde.h" #include "storage/olap_common.h" #include "storage/predicate/block_column_predicate.h" #include "storage/predicate/null_predicate.h" @@ -88,6 +89,7 @@ Status convert(const DataTypePtr& data_type, const std::list& str, FROM_FE_STRING_CASE(TYPE_DATETIME); FROM_FE_STRING_CASE(TYPE_DATEV2); FROM_FE_STRING_CASE(TYPE_DATETIMEV2); + FROM_FE_STRING_CASE(TYPE_TIMESTAMP_NS); FROM_FE_STRING_CASE(TYPE_TIMESTAMPTZ); FROM_FE_STRING_CASE(TYPE_BOOLEAN); FROM_FE_STRING_CASE(TYPE_IPV4); @@ -359,6 +361,10 @@ bool DeleteHandler::is_condition_value_valid(const TabletColumn& column, case FieldType::OLAP_FIELD_TYPE_DATETIMEV2: case FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ: return valid_datetime(value_str, column.frac()); + case FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS: { + int64_t epoch_nanos = 0; + return parse_timestamp_ns(StringRef(value_str), &epoch_nanos).ok(); + } case FieldType::OLAP_FIELD_TYPE_BOOL: return valid_bool(value_str); case FieldType::OLAP_FIELD_TYPE_IPV4: diff --git a/be/src/storage/index/bloom_filter/bloom_filter_index_writer.cpp b/be/src/storage/index/bloom_filter/bloom_filter_index_writer.cpp index 785e889a831eee..8d6a7aec6015fb 100644 --- a/be/src/storage/index/bloom_filter/bloom_filter_index_writer.cpp +++ b/be/src/storage/index/bloom_filter/bloom_filter_index_writer.cpp @@ -330,6 +330,7 @@ Status BloomFilterIndexWriter::create(const BloomFilterOptions& bf_options, Fiel M(FieldType::OLAP_FIELD_TYPE_DECIMAL) M(FieldType::OLAP_FIELD_TYPE_DATEV2) M(FieldType::OLAP_FIELD_TYPE_DATETIMEV2) + M(FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS) M(FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ) M(FieldType::OLAP_FIELD_TYPE_DECIMAL32) M(FieldType::OLAP_FIELD_TYPE_DECIMAL64) diff --git a/be/src/storage/index/index_writer.cpp b/be/src/storage/index/index_writer.cpp index 2325d280471337..785c37be49e293 100644 --- a/be/src/storage/index/index_writer.cpp +++ b/be/src/storage/index/index_writer.cpp @@ -102,6 +102,7 @@ Status IndexColumnWriter::create(const TabletColumn* column, M(FieldType::OLAP_FIELD_TYPE_DECIMAL) M(FieldType::OLAP_FIELD_TYPE_DATEV2) M(FieldType::OLAP_FIELD_TYPE_DATETIMEV2) + M(FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS) M(FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ) M(FieldType::OLAP_FIELD_TYPE_DECIMAL32) M(FieldType::OLAP_FIELD_TYPE_DECIMAL64) diff --git a/be/src/storage/index/inverted/inverted_index_writer.cpp b/be/src/storage/index/inverted/inverted_index_writer.cpp index 0f82b5225e666e..519568c28eb0bf 100644 --- a/be/src/storage/index/inverted/inverted_index_writer.cpp +++ b/be/src/storage/index/inverted/inverted_index_writer.cpp @@ -672,6 +672,7 @@ template class InvertedIndexColumnWriter; template class InvertedIndexColumnWriter; template class InvertedIndexColumnWriter; template class InvertedIndexColumnWriter; +template class InvertedIndexColumnWriter; template class InvertedIndexColumnWriter; template class InvertedIndexColumnWriter; template class InvertedIndexColumnWriter; @@ -683,4 +684,4 @@ template class InvertedIndexColumnWriter; template class InvertedIndexColumnWriter; template class InvertedIndexColumnWriter; -} // namespace doris::segment_v2 \ No newline at end of file +} // namespace doris::segment_v2 diff --git a/be/src/storage/index/zone_map/zone_map_index.cpp b/be/src/storage/index/zone_map/zone_map_index.cpp index 23e2e5a4a0feec..cd687b4a623735 100644 --- a/be/src/storage/index/zone_map/zone_map_index.cpp +++ b/be/src/storage/index/zone_map/zone_map_index.cpp @@ -358,6 +358,7 @@ ZoneMapIndexReader::~ZoneMapIndexReader() = default; M(TYPE_DATETIME) \ M(TYPE_DATEV2) \ M(TYPE_DATETIMEV2) \ + M(TYPE_TIMESTAMP_NS) \ M(TYPE_TIMESTAMPTZ) \ M(TYPE_IPV4) \ M(TYPE_IPV6) \ diff --git a/be/src/storage/iterator/olap_data_convertor.cpp b/be/src/storage/iterator/olap_data_convertor.cpp index 47da182b68e05b..294bebd8e42ec1 100644 --- a/be/src/storage/iterator/olap_data_convertor.cpp +++ b/be/src/storage/iterator/olap_data_convertor.cpp @@ -167,6 +167,9 @@ OlapBlockDataConvertor::create_olap_column_data_convertor(const TabletColumn& co case FieldType::OLAP_FIELD_TYPE_DATETIMEV2: { return std::make_unique(); } + case FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS: { + return std::make_unique>(); + } case FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ: { return std::make_unique>(); } diff --git a/be/src/storage/key_coder.cpp b/be/src/storage/key_coder.cpp index d7a4d7b1457724..e683b299e9bb1d 100644 --- a/be/src/storage/key_coder.cpp +++ b/be/src/storage/key_coder.cpp @@ -77,6 +77,7 @@ class KeyCoderResolver { add_mapping(); add_mapping(); add_mapping(); + add_mapping(); add_mapping(); add_mapping(); add_mapping(); diff --git a/be/src/storage/key_coder.h b/be/src/storage/key_coder.h index 0c4bcf08d171e5..803e5c9c7326d2 100644 --- a/be/src/storage/key_coder.h +++ b/be/src/storage/key_coder.h @@ -469,6 +469,7 @@ class KeyCoderTraits M(OLAP_FIELD_TYPE_DATETIME, TYPE_DATETIME) \ M(OLAP_FIELD_TYPE_DATEV2, TYPE_DATEV2) \ M(OLAP_FIELD_TYPE_DATETIMEV2, TYPE_DATETIMEV2) \ + M(OLAP_FIELD_TYPE_TIMESTAMP_NS, TYPE_TIMESTAMP_NS) \ M(OLAP_FIELD_TYPE_TIMESTAMPTZ, TYPE_TIMESTAMPTZ) \ M(OLAP_FIELD_TYPE_IPV4, TYPE_IPV4) \ M(OLAP_FIELD_TYPE_IPV6, TYPE_IPV6) diff --git a/be/src/storage/olap_common.h b/be/src/storage/olap_common.h index 12ca1d2e6a458f..af14528bb7937a 100644 --- a/be/src/storage/olap_common.h +++ b/be/src/storage/olap_common.h @@ -168,6 +168,7 @@ enum class FieldType { OLAP_FIELD_TYPE_IPV4 = 38, OLAP_FIELD_TYPE_IPV6 = 39, OLAP_FIELD_TYPE_TIMESTAMPTZ = 40, + OLAP_FIELD_TYPE_TIMESTAMP_NS = 41, }; // Define all aggregation methods supported by TabletColumn @@ -224,6 +225,7 @@ constexpr bool field_is_numeric_type(const FieldType& field_type) { field_type == FieldType::OLAP_FIELD_TYPE_DATEV2 || field_type == FieldType::OLAP_FIELD_TYPE_DATETIME || field_type == FieldType::OLAP_FIELD_TYPE_DATETIMEV2 || + field_type == FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS || field_type == FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ || field_type == FieldType::OLAP_FIELD_TYPE_LARGEINT || field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL || diff --git a/be/src/storage/olap_scan_common.h b/be/src/storage/olap_scan_common.h index f7162f3b1f12b4..c14596f86ad4f8 100644 --- a/be/src/storage/olap_scan_common.h +++ b/be/src/storage/olap_scan_common.h @@ -233,6 +233,7 @@ class ColumnValueRange { primitive_type == PrimitiveType::TYPE_BOOLEAN || primitive_type == PrimitiveType::TYPE_DATETIME || primitive_type == PrimitiveType::TYPE_DATETIMEV2 || + primitive_type == PrimitiveType::TYPE_TIMESTAMP_NS || primitive_type == PrimitiveType::TYPE_TIMESTAMPTZ || primitive_type == PrimitiveType::TYPE_DECIMAL256; }; @@ -337,10 +338,10 @@ using ColumnValueRangeType = std::variant< ColumnValueRange, ColumnValueRange, ColumnValueRange, ColumnValueRange, ColumnValueRange, ColumnValueRange, ColumnValueRange, ColumnValueRange, - ColumnValueRange, ColumnValueRange, - ColumnValueRange, ColumnValueRange, - ColumnValueRange, ColumnValueRange, - ColumnValueRange>; + ColumnValueRange, ColumnValueRange, + ColumnValueRange, ColumnValueRange, + ColumnValueRange, ColumnValueRange, + ColumnValueRange, ColumnValueRange>; template const typename ColumnValueRange::CppType diff --git a/be/src/storage/partial_update_info.cpp b/be/src/storage/partial_update_info.cpp index 1bea6e05cb75f5..205400a2b674f8 100644 --- a/be/src/storage/partial_update_info.cpp +++ b/be/src/storage/partial_update_info.cpp @@ -273,6 +273,7 @@ void PartialUpdateInfo::_generate_default_values_for_missing_cids( if (column.has_default_value()) { std::string default_value; if (UNLIKELY((column.type() == FieldType::OLAP_FIELD_TYPE_DATETIMEV2 || + column.type() == FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS || column.type() == FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ) && to_lower(column.default_value()).find(to_lower("CURRENT_TIMESTAMP")) != std::string::npos)) { @@ -288,7 +289,19 @@ void PartialUpdateInfo::_generate_default_values_for_missing_cids( int precision = std::stoi(column.default_value().substr(pos + 1)); DateV2Value dtv; dtv.from_unixtime(timestamp_ms / 1000, nano_seconds, timezone, precision); - default_value = dtv.to_string(); + if (column.type() == FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS) { + const int32_t scale_factor = static_cast(int_exp10(9 - precision)); + const int32_t truncated_nano_seconds = + nano_seconds / scale_factor * scale_factor; + TimeStampNsValue value; + DORIS_CHECK(value.from_datetime( + dtv, static_cast( + truncated_nano_seconds % + (TimeStampNsValue::NANOS_PER_SECOND / 1000000)))); + default_value = value.to_string(precision); + } else { + default_value = dtv.to_string(); + } if (column.type() == FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ) { default_value += timezone; } diff --git a/be/src/storage/predicate/predicate_creator.cpp b/be/src/storage/predicate/predicate_creator.cpp index 485ca89eea6f49..bcfdf93ef15386 100644 --- a/be/src/storage/predicate/predicate_creator.cpp +++ b/be/src/storage/predicate/predicate_creator.cpp @@ -94,6 +94,10 @@ std::shared_ptr create_bloom_filter_predicate( return BloomFilterColumnPredicate::create_shared(cid, col_name, filter_olap); } + case TYPE_TIMESTAMP_NS: { + return BloomFilterColumnPredicate::create_shared(cid, col_name, + filter_olap); + } case TYPE_TIMESTAMPTZ: { return BloomFilterColumnPredicate::create_shared(cid, col_name, filter_olap); diff --git a/be/src/storage/predicate/predicate_creator_comparison.cpp b/be/src/storage/predicate/predicate_creator_comparison.cpp index b10a175b016592..d444a35a0845b8 100644 --- a/be/src/storage/predicate/predicate_creator_comparison.cpp +++ b/be/src/storage/predicate/predicate_creator_comparison.cpp @@ -102,6 +102,10 @@ std::shared_ptr create_comparison_predicate(const uint32_t cid, return ComparisonPredicateBase::create_shared(cid, col_name, value, opposite); } + case TYPE_TIMESTAMP_NS: { + return ComparisonPredicateBase::create_shared(cid, col_name, value, + opposite); + } case TYPE_TIMESTAMPTZ: { return ComparisonPredicateBase::create_shared(cid, col_name, value, opposite); diff --git a/be/src/storage/predicate/predicate_creator_in_list_in.cpp b/be/src/storage/predicate/predicate_creator_in_list_in.cpp index f1ad365b4f523e..9162d02506f735 100644 --- a/be/src/storage/predicate/predicate_creator_in_list_in.cpp +++ b/be/src/storage/predicate/predicate_creator_in_list_in.cpp @@ -139,6 +139,10 @@ std::shared_ptr create_in_list_predicate( cid, col_name, set, is_opposite); } + case TYPE_TIMESTAMP_NS: { + return create_in_list_predicate_impl( + cid, col_name, set, is_opposite); + } case TYPE_TIMESTAMPTZ: { return create_in_list_predicate_impl( cid, col_name, set, is_opposite); diff --git a/be/src/storage/predicate/predicate_creator_in_list_not_in.cpp b/be/src/storage/predicate/predicate_creator_in_list_not_in.cpp index e4cb4731a57095..d90d8538dffca9 100644 --- a/be/src/storage/predicate/predicate_creator_in_list_not_in.cpp +++ b/be/src/storage/predicate/predicate_creator_in_list_not_in.cpp @@ -139,6 +139,10 @@ std::shared_ptr create_in_list_predicate( cid, col_name, set, is_opposite); } + case TYPE_TIMESTAMP_NS: { + return create_in_list_predicate_impl( + cid, col_name, set, is_opposite); + } case TYPE_TIMESTAMPTZ: { return create_in_list_predicate_impl( cid, col_name, set, is_opposite); diff --git a/be/src/storage/segment/encoding_info.cpp b/be/src/storage/segment/encoding_info.cpp index d2b9d99f114e85..5c15d34de3e062 100644 --- a/be/src/storage/segment/encoding_info.cpp +++ b/be/src/storage/segment/encoding_info.cpp @@ -306,6 +306,9 @@ EncodingInfoResolver::EncodingInfoResolver() { _register_supported_encoding(); _register_supported_encoding(); _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); + _register_supported_encoding(); _register_supported_encoding(); _register_supported_encoding(); _register_supported_encoding(); @@ -364,6 +367,7 @@ EncodingInfoResolver::EncodingInfoResolver() { _set_v2_default(); _set_v2_default(); _set_v2_default(); + _set_v2_default(); _set_v2_default(); _set_v2_default(); _set_v2_default(); @@ -397,6 +401,7 @@ EncodingInfoResolver::EncodingInfoResolver() { _set_v3_default(); _set_v3_default(); _set_v3_default(); + _set_v3_default(); _set_v3_default(); _set_v3_default(); _set_v3_default(); diff --git a/be/src/storage/tablet/tablet_schema.cpp b/be/src/storage/tablet/tablet_schema.cpp index 13ea91d032dadd..6773226d2ba9cd 100644 --- a/be/src/storage/tablet/tablet_schema.cpp +++ b/be/src/storage/tablet/tablet_schema.cpp @@ -98,6 +98,8 @@ FieldType TabletColumn::get_field_type_by_string(const std::string& type_str) { type = FieldType::OLAP_FIELD_TYPE_DATEV2; } else if (0 == upper_type_str.compare("DATETIMEV2")) { type = FieldType::OLAP_FIELD_TYPE_DATETIMEV2; + } else if (0 == upper_type_str.compare("TIMESTAMP_NS")) { + type = FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS; } else if (0 == upper_type_str.compare("DATETIME")) { type = FieldType::OLAP_FIELD_TYPE_DATETIME; } else if (0 == upper_type_str.compare("TIMESTAMPTZ")) { @@ -241,6 +243,8 @@ std::string TabletColumn::get_string_by_field_type(FieldType type) { case FieldType::OLAP_FIELD_TYPE_DATETIMEV2: return "DATETIMEV2"; + case FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS: + return "TIMESTAMP_NS"; case FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ: return "TIMESTAMPTZ"; @@ -356,6 +360,7 @@ uint32_t TabletColumn::get_field_length_by_type(TPrimitiveType::type type, uint3 case TPrimitiveType::DATETIME: return 8; case TPrimitiveType::DATETIMEV2: + case TPrimitiveType::TIMESTAMP_NS: case TPrimitiveType::TIMESTAMPTZ: return 8; case TPrimitiveType::FLOAT: @@ -479,6 +484,9 @@ void TabletColumn::init_from_pb(const ColumnPB& column) { if (column.has_frac()) { _frac = column.frac(); } + if (_type == FieldType::OLAP_FIELD_TYPE_DATETIMEV2 && _frac > 6) { + _type = FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS; + } _length = column.length(); _index_length = column.index_length(); if (column.has_is_bf_column()) { diff --git a/be/src/storage/tablet_info.cpp b/be/src/storage/tablet_info.cpp index 9fee41082442c2..c69ec16e5b0021 100644 --- a/be/src/storage/tablet_info.cpp +++ b/be/src/storage/tablet_info.cpp @@ -42,6 +42,8 @@ #include "core/data_type/data_type_factory.hpp" #include "core/data_type/define_primitive_type.h" #include "core/data_type/primitive_type.h" +#include "core/data_type/storage_field_type.h" +#include "core/data_type_serde/data_type_timestamp_ns_serde.h" #include "core/value/large_int_value.h" #include "runtime/descriptors.h" #include "runtime/memory/mem_tracker.h" @@ -181,11 +183,9 @@ Status OlapTableSchemaParam::init(const POlapTableSchemaParam& pschema) { for (const auto& p_slot_desc : pschema.slot_descs()) { auto* slot_desc = _obj_pool.add(new SlotDescriptor(p_slot_desc)); _tuple_desc->add_slot(slot_desc); - std::string data_type; - EnumToString(TPrimitiveType, to_thrift(slot_desc->col_type()), data_type); std::string is_null_str = slot_desc->is_nullable() ? "true" : "false"; - std::string data_type_str = - std::to_string(int64_t(TabletColumn::get_field_type_by_string(data_type))); + std::string data_type_str = std::to_string( + int64_t(primitive_type_to_storage_field_type(slot_desc->col_type()))); slots_map.emplace(to_lower(slot_desc->col_name()) + "+" + data_type_str + is_null_str, slot_desc); } @@ -198,11 +198,12 @@ Status OlapTableSchemaParam::init(const POlapTableSchemaParam& pschema) { index->row_binlog_id = p_index.row_binlog_id(); } for (const auto& pcolumn_desc : p_index.columns_desc()) { + TabletColumn* tc = _obj_pool.add(new TabletColumn()); + tc->init_from_pb(pcolumn_desc); if (_unique_key_update_mode != UniqueKeyUpdateModePB::UPDATE_FIXED_COLUMNS || _partial_update_input_columns.contains(pcolumn_desc.name())) { std::string is_null_str = pcolumn_desc.is_nullable() ? "true" : "false"; - std::string data_type_str = std::to_string( - int64_t(TabletColumn::get_field_type_by_string(pcolumn_desc.type()))); + std::string data_type_str = std::to_string(int64_t(tc->type())); auto it = slots_map.find(to_lower(pcolumn_desc.name()) + "+" + data_type_str + is_null_str); if (it == std::end(slots_map)) { @@ -222,8 +223,6 @@ Status OlapTableSchemaParam::init(const POlapTableSchemaParam& pschema) { } index->slots.emplace_back(it->second); } - TabletColumn* tc = _obj_pool.add(new TabletColumn()); - tc->init_from_pb(pcolumn_desc); index->columns.emplace_back(tc); } for (const auto& pindex_desc : p_index.indexes_desc()) { @@ -359,11 +358,13 @@ Status OlapTableSchemaParam::init(const TOlapTableSchemaParam& tschema) { index->row_binlog_id = t_index.row_binlog_id; } for (const auto& tcolumn_desc : t_index.columns_desc) { + TabletColumn* tc = _obj_pool.add(new TabletColumn()); + tc->init_from_thrift(tcolumn_desc); if (_unique_key_update_mode != UniqueKeyUpdateModePB::UPDATE_FIXED_COLUMNS || _partial_update_input_columns.contains(tcolumn_desc.column_name)) { std::string is_null_str = tcolumn_desc.is_allow_null ? "true" : "false"; std::string data_type_str = - std::to_string(int64_t(thrift_to_type(tcolumn_desc.column_type.type))); + std::to_string(int64_t(storage_field_type_to_primitive_type(tc->type()))); auto it = slots_map.find(to_lower(tcolumn_desc.column_name) + "+" + data_type_str + is_null_str); if (it == slots_map.end()) { @@ -386,8 +387,6 @@ Status OlapTableSchemaParam::init(const TOlapTableSchemaParam& tschema) { index->slots.emplace_back(it->second); } index_slots_map.emplace(to_lower(tcolumn_desc.column_name), tcolumn_desc.col_unique_id); - TabletColumn* tc = _obj_pool.add(new TabletColumn()); - tc->init_from_thrift(tcolumn_desc); index->columns.emplace_back(tc); } if (t_index.__isset.indexes_desc) { @@ -618,11 +617,12 @@ bool VOlapTablePartitionParam::_part_contains(VOlapTablePartition* part, // insert value into _partition_block's column // NOLINTBEGIN(readability-function-size) static Status _create_partition_key(const TExprNode& t_expr, BlockRow* part_key, uint16_t pos) { - auto column = std::move(*part_key->first->get_by_position(pos).column).mutate(); + const auto& partition_column = part_key->first->get_by_position(pos); + const auto partition_type = remove_nullable(partition_column.type); + auto column = std::move(*partition_column.column).mutate(); switch (t_expr.node_type) { case TExprNodeType::DATE_LITERAL: { - auto primitive_type = - DataTypeFactory::instance().create_data_type(t_expr.type)->get_primitive_type(); + const auto primitive_type = partition_type->get_primitive_type(); if (primitive_type == TYPE_DATEV2) { DateV2Value dt; CastParameters params; @@ -636,8 +636,7 @@ static Status _create_partition_key(const TExprNode& t_expr, BlockRow* part_key, column->insert_data(reinterpret_cast(&dt), 0); } else if (primitive_type == TYPE_DATETIMEV2) { DateV2Value dt; - const int32_t scale = - t_expr.type.types.empty() ? -1 : t_expr.type.types.front().scalar_type.scale; + const int32_t scale = cast_set(partition_type->get_scale()); CastParameters params; if (!CastToDatetimeV2::from_string_strict_mode( {t_expr.date_literal.value.c_str(), t_expr.date_literal.value.size()}, dt, @@ -647,11 +646,17 @@ static Status _create_partition_key(const TExprNode& t_expr, BlockRow* part_key, return Status::InternalError(ss.str()); } column->insert_data(reinterpret_cast(&dt), 0); + } else if (primitive_type == TYPE_TIMESTAMP_NS) { + int64_t epoch_nanos = 0; + RETURN_IF_ERROR(parse_timestamp_ns( + {t_expr.date_literal.value.data(), t_expr.date_literal.value.size()}, + &epoch_nanos)); + const TimeStampNsValue dt(epoch_nanos); + column->insert_data(reinterpret_cast(&dt), 0); } else if (primitive_type == TYPE_TIMESTAMPTZ) { TimestampTzValue res; CastParameters params {.status = Status::OK(), .is_strict = true}; - const int32_t scale = - t_expr.type.types.empty() ? -1 : t_expr.type.types.front().scalar_type.scale; + const int32_t scale = cast_set(partition_type->get_scale()); if (!CastToTimestampTz::from_string( {t_expr.date_literal.value.c_str(), t_expr.date_literal.value.size()}, res, params, nullptr, scale)) [[unlikely]] { diff --git a/be/src/storage/types.h b/be/src/storage/types.h index 7d66631ef16180..432715b56a01c0 100644 --- a/be/src/storage/types.h +++ b/be/src/storage/types.h @@ -173,6 +173,11 @@ struct CppTypeTraits { using UnsignedCppType = uint64_t; }; template <> +struct CppTypeTraits { + using CppType = int64_t; + using UnsignedCppType = uint64_t; +}; +template <> struct CppTypeTraits { using CppType = uint64_t; using UnsignedCppType = uint64_t; @@ -306,6 +311,9 @@ struct FieldTypeTraits template <> struct FieldTypeTraits : public BaseFieldTypeTraits {}; +template <> +struct FieldTypeTraits + : public BaseFieldTypeTraits {}; template <> struct FieldTypeTraits @@ -421,6 +429,7 @@ inline size_t field_type_size(FieldType field_type) { DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_DATETIME) DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_DATEV2) DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_DATETIMEV2) + DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_TIMESTAMP_NS) DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_TIMEV2) DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_TIMESTAMPTZ) DORIS_FIELD_TYPE_SIZE_CASE(OLAP_FIELD_TYPE_IPV4) diff --git a/be/src/tools/meta_tool.cpp b/be/src/tools/meta_tool.cpp index 18bc27c1d839c2..2c95ce29d9b0f7 100644 --- a/be/src/tools/meta_tool.cpp +++ b/be/src/tools/meta_tool.cpp @@ -386,6 +386,8 @@ std::string get_field_type_string(doris::FieldType type) { return "DATEV2"; case doris::FieldType::OLAP_FIELD_TYPE_DATETIMEV2: return "DATETIMEV2"; + case doris::FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS: + return "TIMESTAMP_NS"; case doris::FieldType::OLAP_FIELD_TYPE_BOOL: return "BOOLEAN"; case doris::FieldType::OLAP_FIELD_TYPE_STRUCT: @@ -513,6 +515,14 @@ std::string format_column_value(const doris::IColumn& column, size_t row, } return ""; } + case FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS: { + const StringRef& data = column.get_data_at(row); + if (data.size == sizeof(int64_t)) { + int64_t val = *reinterpret_cast(data.data); + return std::to_string(val); + } + return ""; + } case FieldType::OLAP_FIELD_TYPE_CHAR: case FieldType::OLAP_FIELD_TYPE_VARCHAR: case FieldType::OLAP_FIELD_TYPE_STRING: diff --git a/be/src/util/raw_value.h b/be/src/util/raw_value.h index 4babb5c0810037..ca9914e4064d9d 100644 --- a/be/src/util/raw_value.h +++ b/be/src/util/raw_value.h @@ -85,7 +85,8 @@ inline uint32_t RawValue::zlib_crc32(const void* v, size_t len, const PrimitiveT return HashUtil::zlib_crc_hash(v, 4, seed); } - case TYPE_DATETIMEV2: { + case TYPE_DATETIMEV2: + case TYPE_TIMESTAMP_NS: { return HashUtil::zlib_crc_hash(v, 8, seed); } diff --git a/be/test/core/data_type/data_type_datetime_v2_test.cpp b/be/test/core/data_type/data_type_datetime_v2_test.cpp index e1a58ae49a543e..b3aeec27da2e8e 100644 --- a/be/test/core/data_type/data_type_datetime_v2_test.cpp +++ b/be/test/core/data_type/data_type_datetime_v2_test.cpp @@ -120,6 +120,9 @@ TEST_F(DataTypeDateTimeV2Test, simple_func_test) { test_func(dt_time_v2_6); EXPECT_THROW(DataTypeDateTimeV2(7), Exception); EXPECT_THROW(create_datetimev2(7), Exception); + EXPECT_THROW(create_datetimev2(8), Exception); + EXPECT_THROW(create_datetimev2(9), Exception); + EXPECT_THROW(create_datetimev2(10), Exception); EXPECT_THROW(DataTypeTimeV2(7), Exception); } @@ -869,4 +872,4 @@ TEST_F(DataTypeDateTimeV2Test, GetFieldWithDataTypeTest) { field_datetime_v2); } -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/test/core/data_type/data_type_timestamp_ns_test.cpp b/be/test/core/data_type/data_type_timestamp_ns_test.cpp new file mode 100644 index 00000000000000..0583d1040c2005 --- /dev/null +++ b/be/test/core/data_type/data_type_timestamp_ns_test.cpp @@ -0,0 +1,778 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "core/data_type/data_type_timestamp_ns.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "core/assert_cast.h" +#include "core/column/column_const.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/data_type/data_type_date_or_datetime_v2.h" +#include "core/data_type_serde/data_type_serde.h" +#include "core/data_type_serde/data_type_timestamp_ns_serde.h" +#include "core/string_buffer.hpp" +#include "core/value/vdatetime_value.h" +#include "exprs/function/cast/cast_parameters.h" +#include "util/jsonb_utils.h" +#include "util/jsonb_writer.h" +#include "util/mysql_row_buffer.h" +#include "util/slice.h" +#include "util/timezone_utils.h" + +namespace doris { + +TEST(DataTypeTimeStampNsTest, Int64EpochRangeAndOrdering) { + const TimeStampNsValue epoch(0); + const TimeStampNsValue before_epoch(-1); + const TimeStampNsValue minimum(std::numeric_limits::min()); + const TimeStampNsValue maximum(std::numeric_limits::max()); + + EXPECT_EQ(epoch.to_string(9), "1970-01-01 00:00:00.000000000"); + EXPECT_EQ(before_epoch.to_string(9), "1969-12-31 23:59:59.999999999"); + EXPECT_EQ(minimum.to_string(9), "1677-09-21 00:12:43.145224192"); + EXPECT_EQ(maximum.to_string(9), "2262-04-11 23:47:16.854775807"); + EXPECT_LT(minimum, before_epoch); + EXPECT_LT(before_epoch, epoch); + EXPECT_LT(epoch, maximum); +} + +TEST(DataTypeTimeStampNsTest, NegativeEpochUsesFloorSecondAndNormalizedFraction) { + struct TestCase { + int64_t epoch_nanos; + int64_t epoch_seconds; + uint32_t nanosecond; + }; + const std::vector cases = { + {-1000000001, -2, 999999999}, + {-1000000000, -1, 0}, + {-999999999, -1, 1}, + {-1, -1, 999999999}, + {0, 0, 0}, + {1, 0, 1}, + }; + + for (const auto& test_case : cases) { + const TimeStampNsValue value(test_case.epoch_nanos); + EXPECT_EQ(value.epoch_seconds(), test_case.epoch_seconds); + EXPECT_EQ(value.nanosecond(), test_case.nanosecond); + EXPECT_EQ( + static_cast<__int128>(value.epoch_seconds()) * TimeStampNsValue::NANOS_PER_SECOND + + value.nanosecond(), + test_case.epoch_nanos); + } +} + +TEST(DataTypeTimeStampNsTest, ParseAtFixedNanosecondPrecision) { + int64_t value = 0; + + ASSERT_TRUE(parse_timestamp_ns(StringRef("1970-01-01 00:00:00.12345675"), &value).ok()); + EXPECT_EQ(TimeStampNsValue(value).to_string(), "1970-01-01 00:00:00.123456750"); + + ASSERT_TRUE(parse_timestamp_ns(StringRef("1969-12-31 23:59:59.999999999"), &value).ok()); + EXPECT_EQ(value, -1); + + ASSERT_TRUE(parse_timestamp_ns(StringRef("1970-01-01 00:00:00.999999995"), &value).ok()); + EXPECT_EQ(TimeStampNsValue(value).to_string(), "1970-01-01 00:00:00.999999995"); + + ASSERT_TRUE(parse_timestamp_ns(StringRef("1970-01-01 00:00:00.9999999995"), &value).ok()); + EXPECT_EQ(TimeStampNsValue(value).to_string(), "1970-01-01 00:00:01.000000000"); +} + +TEST(DataTypeTimeStampNsTest, ParseTimezoneSuffixInSessionTimezone) { + TimezoneUtils::load_timezones_to_cache(); + cctz::time_zone shanghai; + ASSERT_TRUE(cctz::load_time_zone("Asia/Shanghai", &shanghai)); + + int64_t value = 0; + auto status = + parse_timestamp_ns(StringRef("2023-08-17T01:41:18.123456789Z"), &value, &shanghai); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(TimeStampNsValue(value).to_string(9), "2023-08-17 09:41:18.123456789"); + + ASSERT_TRUE(parse_timestamp_ns(StringRef("2023-08-17T01:41:18.123456789America/Los_Angeles"), + &value, &shanghai) + .ok()); + EXPECT_EQ(TimeStampNsValue(value).to_string(9), "2023-08-17 16:41:18.123456789"); + + EXPECT_FALSE( + parse_timestamp_ns(StringRef("1677-09-21T00:12:43.145224192+14:00"), &value, &shanghai) + .ok()); + EXPECT_FALSE( + parse_timestamp_ns(StringRef("2262-04-11T23:47:16.854775807-01:00"), &value, &shanghai) + .ok()); +} + +TEST(DataTypeTimeStampNsTest, ParseAcceptsFractionalWidthsAndRejectsMalformedValues) { + struct ValidCase { + const char* input; + const char* expected; + }; + const std::vector valid_cases = { + {"2024-02-29 12:34:56.1234567", "2024-02-29 12:34:56.123456700"}, + {"2024-02-29 12:34:56.12345678", "2024-02-29 12:34:56.123456780"}, + {"2024-02-29 12:34:56.123456789", "2024-02-29 12:34:56.123456789"}, + {"2024-02-29 12:34:56.1234567894", "2024-02-29 12:34:56.123456789"}, + {"2024-02-29 12:34:56.1234567895", "2024-02-29 12:34:56.123456790"}, + {"2024-02-29 12:34:56", "2024-02-29 12:34:56.000000000"}, + }; + + for (const auto& test_case : valid_cases) { + int64_t value = 0; + ASSERT_TRUE(parse_timestamp_ns(StringRef(test_case.input), &value).ok()) << test_case.input; + EXPECT_EQ(TimeStampNsValue(value).to_string(), test_case.expected); + } + + const std::vector invalid_values = { + "", + "not-a-date", + "2023-02-29 00:00:00.000000000", + "2024-13-01", + "2024-01-01 24:00:00", + "2024-01-01 00:00:00.trailing", + "2024-01-01 00:00:00.123.456", + "2024-01-01.123 00:00:00", + "2024.01-01 00:00:00", + }; + for (const char* input : invalid_values) { + int64_t value = 0; + EXPECT_FALSE(parse_timestamp_ns(StringRef(input), &value).ok()) << input; + } +} + +TEST(DataTypeTimeStampNsTest, RejectValuesOutsideEpochRange) { + int64_t value = 0; + EXPECT_FALSE(parse_timestamp_ns(StringRef("0000-01-01 00:00:00.000000000"), &value).ok()); + EXPECT_FALSE(parse_timestamp_ns(StringRef("1677-09-21 00:12:43.145224191"), &value).ok()); + EXPECT_FALSE(parse_timestamp_ns(StringRef("2262-04-11 23:47:16.854775808"), &value).ok()); + EXPECT_FALSE(parse_timestamp_ns(StringRef("9999-12-31 23:59:59.999999999"), &value).ok()); +} + +TEST(DataTypeTimeStampNsTest, CivilRoundTripPreservesSubMicrosecondDigits) { + DateV2Value civil; + civil.unchecked_set_time(2024, 2, 29, 23, 59, 58, 123456); + + TimeStampNsValue value; + ASSERT_TRUE(value.from_datetime(civil, 789)); + EXPECT_EQ(value.to_string(9), "2024-02-29 23:59:58.123456789"); + EXPECT_EQ(value.year(), 2024); + EXPECT_EQ(value.month(), 2); + EXPECT_EQ(value.day(), 29); + EXPECT_EQ(value.hour(), 23); + EXPECT_EQ(value.minute(), 59); + EXPECT_EQ(value.second(), 58); + EXPECT_EQ(value.microsecond(), 123456); + EXPECT_EQ(value.nanosecond_remainder(), 789); + EXPECT_EQ(value.to_datetime().to_date_int_val(), civil.to_date_int_val()); +} + +TEST(DataTypeTimeStampNsTest, CalendarArithmeticPreservesSubMicrosecondDigits) { + int64_t raw_value = 0; + ASSERT_TRUE(parse_timestamp_ns(StringRef("2024-01-31 23:59:59.123456789"), &raw_value).ok()); + TimeStampNsValue value(raw_value); + + ASSERT_TRUE(value.date_add_interval(TimeInterval(TimeUnit::MONTH, 1, false))); + EXPECT_EQ(value.to_string(9), "2024-02-29 23:59:59.123456789"); + + ASSERT_TRUE( + value.date_add_interval(TimeInterval(TimeUnit::SECOND, 1, false))); + EXPECT_EQ(value.to_string(9), "2024-03-01 00:00:00.123456789"); +} + +TEST(DataTypeTimeStampNsTest, DiffTruncatesIncompleteUnitsTowardZero) { + int64_t lhs_raw = 0; + int64_t rhs_raw = 0; + ASSERT_TRUE(parse_timestamp_ns(StringRef("1970-01-01 00:00:00.000000001"), &lhs_raw).ok()); + ASSERT_TRUE(parse_timestamp_ns(StringRef("1970-01-01 00:00:01.999999999"), &rhs_raw).ok()); + const TimeStampNsValue lhs(lhs_raw); + const TimeStampNsValue rhs(rhs_raw); + + EXPECT_EQ(datetime_diff(lhs, rhs), 1); + EXPECT_EQ(datetime_diff(rhs, lhs), -1); + EXPECT_EQ(datetime_diff(lhs, rhs), 1999); + EXPECT_EQ(datetime_diff(rhs, lhs), -1999); + EXPECT_EQ(lhs.datetime_diff_in_microseconds(rhs), -1999999); + EXPECT_EQ(rhs.datetime_diff_in_microseconds(lhs), 1999999); +} + +TEST(DataTypeTimeStampNsTest, FactoryKeepsTimestampNsSeparateFromDateTimeV2) { + const auto microseconds = create_datetimev2(6); + const auto timestamp_ns = std::make_shared(); + + EXPECT_EQ(microseconds->get_primitive_type(), TYPE_DATETIMEV2); + EXPECT_THROW(create_datetimev2(7), Exception); + EXPECT_THROW(create_datetimev2(8), Exception); + EXPECT_THROW(create_datetimev2(9), Exception); + EXPECT_EQ(timestamp_ns->get_primitive_type(), TYPE_TIMESTAMP_NS); + EXPECT_EQ(timestamp_ns->get_storage_field_type(), FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS); + EXPECT_EQ(timestamp_ns->get_scale(), 9); + EXPECT_EQ(microseconds->get_family_name(), "DateTimeV2"); + EXPECT_EQ(timestamp_ns->get_family_name(), "TimeStampNs"); +} + +TEST(DataTypeTimeStampNsTest, SerDeRoundTripsTextProtobufAndBinary) { + const auto type = std::make_shared(); + const auto serde = type->get_serde(); + auto source = type->create_column(); + DataTypeSerDe::FormatOptions options; + const std::vector inputs = { + "1677-09-21 00:12:43.145224192", "1969-12-31 23:59:59.999999999", + "1970-01-01 00:00:00.000000000", "2024-02-29 12:34:56.123456789", + "2262-04-11 23:47:16.854775807", + }; + for (const auto& input : inputs) { + StringRef ref(input); + ASSERT_TRUE(serde->from_string(ref, *source, options).ok()) << input; + } + const auto& source_data = assert_cast(*source).get_data(); + + PValues protobuf_values; + ASSERT_TRUE(serde->write_column_to_pb(*source, protobuf_values, 0, source->size()).ok()); + auto protobuf_result = type->create_column(); + ASSERT_TRUE(serde->read_column_from_pb(*protobuf_result, protobuf_values).ok()); + const auto& protobuf_data = assert_cast(*protobuf_result).get_data(); + EXPECT_EQ(protobuf_data, source_data); + + ColumnString::Chars binary; + std::vector offsets = {0}; + for (size_t row = 0; row < source->size(); ++row) { + serde->write_one_cell_to_binary(*source, binary, row); + offsets.push_back(binary.size()); + } + constexpr size_t bytes_per_row = sizeof(uint8_t) + sizeof(uint8_t) + sizeof(int64_t); + ASSERT_EQ(binary.size(), source->size() * bytes_per_row); + auto binary_result = ColumnNullable::create(type->create_column(), ColumnUInt8::create()); + for (size_t row = 0; row < source->size(); ++row) { + const uint8_t* begin = binary.data() + offsets[row]; + const uint8_t* end = DataTypeSerDe::deserialize_binary_to_column(begin, *binary_result); + EXPECT_EQ(end - begin, bytes_per_row); + } + const auto& binary_data = + assert_cast(binary_result->get_nested_column()).get_data(); + EXPECT_EQ(binary_data, source_data); +} + +/* +TEST(DataTypeTimeStampNsTest, SerDeRoundTripsArrowNanosecondsBeforeEpoch) { + const auto type = std::make_shared(); + const auto serde = type->get_serde(); + auto source = type->create_column(); + auto& source_data = assert_cast(*source).get_data(); + source_data.push_back(TimeStampNsValue(-1)); + source_data.push_back(TimeStampNsValue(0)); + source_data.push_back(TimeStampNsValue(1234567890)); + + arrow::TimestampBuilder builder(arrow::timestamp(arrow::TimeUnit::NANO), + arrow::default_memory_pool()); + ASSERT_TRUE(serde->write_column_to_arrow(*source, nullptr, &builder, 0, source->size(), + cctz::utc_time_zone()) + .ok()); + std::shared_ptr array; + ASSERT_TRUE(builder.Finish(&array).ok()); + const auto* timestamps = assert_cast(array.get()); + EXPECT_EQ(timestamps->Value(0), -1); + EXPECT_EQ(timestamps->Value(1), 0); + EXPECT_EQ(timestamps->Value(2), 1234567890); + + auto result = type->create_column(); + ASSERT_TRUE(serde->read_column_from_arrow(*result, array.get(), 0, array->length(), + cctz::utc_time_zone()) + .ok()); + EXPECT_EQ(assert_cast(*result).get_data(), source_data); +} +*/ + +TEST(DataTypeTimeStampNsTest, DecodedTimestampUnitsAndValidation) { + const auto type = std::make_shared(); + const auto serde = type->get_serde(); + const int64_t values[] = {-1, 0, 1}; + + const auto check_unit = [&](DecodedTimeUnit unit, int64_t multiplier) { + auto column = type->create_column(); + DecodedColumnView view; + view.value_kind = DecodedValueKind::INT64; + view.time_unit = unit; + view.row_count = 3; + view.values = reinterpret_cast(values); + ASSERT_TRUE(serde->read_column_from_decoded_values(*column, view).ok()); + const auto& data = assert_cast(*column).get_data(); + ASSERT_EQ(data.size(), 3); + EXPECT_EQ(data[0].epoch_nanos(), -multiplier); + EXPECT_EQ(data[1].epoch_nanos(), 0); + EXPECT_EQ(data[2].epoch_nanos(), multiplier); + }; + + check_unit(DecodedTimeUnit::MILLIS, TimeStampNsValue::NANOS_PER_MILLISECOND); + check_unit(DecodedTimeUnit::MICROS, TimeStampNsValue::NANOS_PER_MICROSECOND); + check_unit(DecodedTimeUnit::NANOS, 1); + + { + auto column = type->create_column(); + DecodedColumnView view; + view.value_kind = DecodedValueKind::INT64; + view.time_unit = DecodedTimeUnit::UNKNOWN; + view.row_count = 3; + view.values = reinterpret_cast(values); + EXPECT_FALSE(serde->read_column_from_decoded_values(*column, view).ok()); + } + + { + const int64_t overflow = std::numeric_limits::max(); + auto column = type->create_column(); + DecodedColumnView view; + view.value_kind = DecodedValueKind::INT64; + view.time_unit = DecodedTimeUnit::MILLIS; + view.row_count = 1; + view.values = reinterpret_cast(&overflow); + EXPECT_FALSE(serde->read_column_from_decoded_values(*column, view).ok()); + } +} + +TEST(DataTypeTimeStampNsTest, DataTypeLiteralFieldAndCivilCasts) { + const DataTypeTimeStampNs type; + const DataTypeDateTimeV2 legacy6(6); + + EXPECT_TRUE(type.equals(DataTypeTimeStampNs {})); + EXPECT_FALSE(type.equals(legacy6)); + EXPECT_FALSE(type.equals_ignore_precision(legacy6)); + EXPECT_FALSE(legacy6.equals_ignore_precision(type)); + + TExprNode node; + node.date_literal.value = "2024-02-29 12:34:56.123456789"; + const Field field = type.get_field(node); + EXPECT_EQ(field.get().to_string(9), "2024-02-29 12:34:56.123456789"); + node.date_literal.value = "not-a-datetime"; + EXPECT_THROW(type.get_field(node), Exception); + + auto column = type.create_column(); + column->insert(field); + const auto field_with_type = type.get_field_with_data_type(*column, 0); + EXPECT_EQ(field_with_type.field, field); + EXPECT_EQ(field_with_type.base_scalar_type_id, TYPE_TIMESTAMP_NS); + EXPECT_EQ(field_with_type.precision, -1); + EXPECT_EQ(field_with_type.scale, 9); + + DateV2Value date; + date.unchecked_set_time(2024, 2, 29, 0, 0, 0, 0); + TimeStampNsValue nano_from_date; + DataTypeDateV2::cast_to_date_time_v2(date, nano_from_date); + EXPECT_EQ(nano_from_date.to_string(9), "2024-02-29 00:00:00.000000000"); + + VecDateTimeValue date_v1; + VecDateTimeValue datetime_v1; + DateV2Value date_v2; + DataTypeDateTimeV2::cast_to_date(field.get(), date_v1); + DataTypeDateTimeV2::cast_to_date_time(field.get(), datetime_v1); + DataTypeDateTimeV2::cast_to_date_v2(field.get(), date_v2); + char date_v1_text[64] = {}; + char datetime_v1_text[64] = {}; + date_v1.to_string(date_v1_text); + datetime_v1.to_string(datetime_v1_text); + EXPECT_STREQ(date_v1_text, "2024-02-29"); + EXPECT_STREQ(datetime_v1_text, "2024-02-29 12:34:56"); + EXPECT_EQ(date_v2.to_string(), "2024-02-29"); +} + +TEST(DataTypeTimeStampNsTest, CalendarHelpersArithmeticAndHash) { + int64_t raw = 0; + ASSERT_TRUE(parse_timestamp_ns(StringRef("2024-02-29 12:34:56.123456789"), &raw).ok()); + TimeStampNsValue value(raw); + + EXPECT_EQ(value.quarter(), 1); + EXPECT_GT(value.daynr(), 0); + EXPECT_GT(value.year_of_week(), 0); + EXPECT_GT(value.week(0), 0); + EXPECT_GT(value.year_week(0), 0); + EXPECT_EQ(value.day_of_year(), 60); + EXPECT_GT(value.day_of_week(), 0); + EXPECT_LT(value.weekday(), 7); + EXPECT_EQ(value.time_part_to_seconds(), 12 * 3600 + 34 * 60 + 56); + EXPECT_EQ(value.time_part_to_microsecond(), (12 * 3600 + 34 * 60 + 56) * 1000000LL + 123456); + EXPECT_TRUE(value.is_valid_date()); + + static const char* const day_names[] = {"Monday", "Tuesday", "Wednesday", "Thursday", + "Friday", "Saturday", "Sunday"}; + static const char* const month_names[] = { + "", "January", "February", "March", "April", "May", "June", + "July", "August", "September", "October", "November", "December"}; + EXPECT_STREQ(value.day_name_with_locale(day_names), "Thursday"); + EXPECT_STREQ(value.month_name_with_locale(month_names), "February"); + + char formatted[64] = {}; + constexpr char format[] = "%Y-%m-%d %H:%i:%s.%f"; + ASSERT_TRUE(value.to_format_string_conservative(format, std::strlen(format), formatted, + sizeof(formatted))); + EXPECT_STREQ(formatted, "2024-02-29 12:34:56.123456"); + + char text[40] = {}; + const char* end = value.to_string(text, 9); + EXPECT_STREQ(text, "2024-02-29 12:34:56.123456789"); + EXPECT_EQ(end, text + std::strlen(text) + 1); + + TimeStampNsValue adjusted = value; + adjusted += 2; + EXPECT_EQ(adjusted.epoch_nanos(), value.epoch_nanos() + 2000000000LL); + adjusted -= 3; + EXPECT_EQ(adjusted.epoch_nanos(), value.epoch_nanos() - 1000000000LL); + EXPECT_EQ(value.hash(17), value.hash(17)); + EXPECT_EQ(std::hash {}(value), std::hash {}(value.epoch_nanos())); + + int64_t earlier_raw = 0; + int64_t later_raw = 0; + ASSERT_TRUE(parse_timestamp_ns(StringRef("2020-01-31 12:00:00.900000009"), &earlier_raw).ok()); + ASSERT_TRUE(parse_timestamp_ns(StringRef("2022-03-31 11:00:00.100000001"), &later_raw).ok()); + const TimeStampNsValue earlier(earlier_raw); + const TimeStampNsValue later(later_raw); + + EXPECT_EQ(datetime_diff(earlier, later), 2); + EXPECT_EQ(datetime_diff(later, earlier), -2); + EXPECT_EQ(datetime_diff(earlier, later), 25); + EXPECT_EQ(datetime_diff(later, earlier), -25); + EXPECT_EQ(datetime_diff(earlier, later), 8); + EXPECT_EQ(datetime_diff(earlier, later), 112); + EXPECT_EQ(datetime_diff(earlier, later), 789); + EXPECT_EQ(datetime_diff(later, earlier), -789); + EXPECT_EQ(earlier.date_diff_in_days(later), -790); + EXPECT_EQ(later.date_diff_in_days_round_to_zero_by_time(earlier), 789); + EXPECT_EQ(earlier.date_diff_in_days_round_to_zero_by_time(later), -789); + EXPECT_EQ(later.datetime_diff_in_seconds_round_to_zero_by_ms(earlier), + -earlier.datetime_diff_in_seconds_round_to_zero_by_ms(later)); + + const auto legacy = later.to_datetime(); + EXPECT_EQ(earlier.time_part_diff_in_ms(legacy), 3600800000LL); + EXPECT_EQ(earlier.datetime_diff_in_microseconds(legacy), + earlier.datetime_diff_in_seconds(legacy) * 1000000LL + + earlier.time_part_diff_in_ms(legacy) % 1000000LL); + + TimeStampNsValue truncated = value; + ASSERT_TRUE(truncated.datetime_trunc()); + EXPECT_EQ(truncated.to_string(9), "2024-02-29 00:00:00.000000000"); +} + +TEST(DataTypeTimeStampNsTest, SerDeStrictBatchJsonJsonbMysqlAndBinaryField) { + const DataTypeTimeStampNs type; + const auto serde = type.get_serde(); + DataTypeSerDe::FormatOptions options; + options.field_delim = ";"; + + auto strings = ColumnString::create(); + strings->insert_data("1970-01-01 00:00:00.000000001", 29); + strings->insert_data("ignored-invalid-value", 21); + strings->insert_data("2024-02-29 12:34:56.123456789", 29); + NullMap null_map = {0, 1, 0}; + auto strict_result = type.create_column(); + ASSERT_TRUE( + serde->from_string_strict_mode_batch(*strings, *strict_result, options, null_map.data()) + .ok()); + const auto& strict_data = assert_cast(*strict_result).get_data(); + EXPECT_EQ(strict_data[0].epoch_nanos(), 1); + EXPECT_EQ(strict_data[2].to_string(9), "2024-02-29 12:34:56.123456789"); + + auto invalid_strings = ColumnString::create(); + invalid_strings->insert_data("invalid", 7); + auto invalid_result = type.create_column(); + EXPECT_FALSE(serde->from_string_strict_mode_batch(*invalid_strings, *invalid_result, options, + nullptr) + .ok()); + + auto source = type.create_column(); + for (const std::string input : + {"1970-01-01 00:00:00.000000001", "2024-02-29 12:34:56.123456789"}) { + StringRef ref(input); + ASSERT_TRUE(serde->from_string_strict_mode(ref, *source, options).ok()); + } + + auto serialized = ColumnString::create(); + VectorBufferWriter writer(*serialized); + ASSERT_TRUE(serde->serialize_column_to_json(*source, 0, source->size(), writer, options).ok()); + writer.commit(); + EXPECT_EQ(serialized->get_data_at(0).to_string(), + "1970-01-01 00:00:00.000000001;2024-02-29 12:34:56.123456789"); + + std::vector json_values = {"1970-01-01 00:00:00.000000001", + "2024-02-29 12:34:56.123456789"}; + std::vector slices; + for (auto& json_value : json_values) { + slices.emplace_back(json_value.data(), json_value.size()); + } + auto json_result = type.create_column(); + uint64_t num_deserialized = 0; + ASSERT_TRUE(serde->deserialize_column_from_json_vector(*json_result, slices, &num_deserialized, + options) + .ok()); + EXPECT_EQ(num_deserialized, json_values.size()); + EXPECT_EQ(assert_cast(*json_result).get_data(), + assert_cast(*source).get_data()); + + const auto nested_serde = type.get_serde(2); + auto nested_json = ColumnString::create(); + VectorBufferWriter nested_writer(*nested_json); + ASSERT_TRUE(nested_serde->serialize_one_cell_to_json(*source, 1, nested_writer, options).ok()); + nested_writer.commit(); + EXPECT_EQ(nested_json->get_data_at(0).to_string(), "\"2024-02-29 12:34:56.123456789\""); + auto nested_result = type.create_column(); + std::string quoted = nested_json->get_data_at(0).to_string(); + Slice quoted_slice(quoted.data(), quoted.size()); + ASSERT_TRUE(nested_serde->deserialize_one_cell_from_json(*nested_result, quoted_slice, options) + .ok()); + EXPECT_EQ(assert_cast(*nested_result).get_element(0), + assert_cast(*source).get_element(1)); + + auto one_value = source->clone_resized(1); + auto const_column = ColumnConst::create(std::move(one_value), 2); + auto const_json = ColumnString::create(); + VectorBufferWriter const_writer(*const_json); + ASSERT_TRUE(serde->serialize_one_cell_to_json(*const_column, 1, const_writer, options).ok()); + const_writer.commit(); + EXPECT_EQ(const_json->get_data_at(0).to_string(), "1970-01-01 00:00:00.000000001"); + + JsonbWriter jsonb_writer; + ASSERT_TRUE(serde->serialize_column_to_jsonb(*source, 1, jsonb_writer).ok()); + EXPECT_EQ(JsonbToJson::jsonb_to_json_string(jsonb_writer.getOutput()->getBuffer(), + jsonb_writer.getOutput()->getSize()), + "\"2024-02-29 12:34:56.123456789\""); + CastParameters cast_params {.status = Status::OK(), .is_strict = true}; + auto jsonb_result = type.create_column(); + ASSERT_TRUE(serde->deserialize_column_from_jsonb(*jsonb_result, jsonb_writer.getValue(), + cast_params) + .ok()); + EXPECT_EQ(assert_cast(*jsonb_result).get_element(0), + assert_cast(*source).get_element(1)); + + auto jsonb_values = ColumnString::create(); + ASSERT_TRUE(serde->serialize_column_to_jsonb_vector(*source, *jsonb_values).ok()); + auto jsonb_vector_result = ColumnNullable::create(type.create_column(), ColumnUInt8::create()); + ASSERT_TRUE(serde->deserialize_column_from_jsonb_vector(*jsonb_vector_result, *jsonb_values, + cast_params) + .ok()); + EXPECT_EQ(assert_cast(jsonb_vector_result->get_nested_column()) + .get_data(), + assert_cast(*source).get_data()); + + JsonbWriter row_store_writer; + Arena row_store_arena; + row_store_writer.writeStartObject(); + serde->write_one_cell_to_jsonb(*source, row_store_writer, row_store_arena, 0, 1, options); + row_store_writer.writeEndObject(); + const JsonbDocument* row_store_document = nullptr; + ASSERT_TRUE(JsonbDocument::checkAndCreateDocument(row_store_writer.getOutput()->getBuffer(), + row_store_writer.getOutput()->getSize(), + &row_store_document) + .ok()); + auto row_store_result = type.create_column(); + serde->read_one_cell_from_jsonb(*row_store_result, (*row_store_document)->begin()->value()); + EXPECT_EQ(assert_cast(*row_store_result).get_element(0), + assert_cast(*source).get_element(1)); + + MysqlRowBinaryBuffer mysql_buffer; + ASSERT_TRUE(serde->write_column_to_mysql_binary(*source, mysql_buffer, 1, false, options).ok()); + ASSERT_EQ(static_cast(mysql_buffer.buf()[0]), 29); + EXPECT_EQ(std::string(mysql_buffer.buf() + 1, 29), "2024-02-29 12:34:56.123456789"); + + ColumnString::Chars binary; + serde->write_one_cell_to_binary(*source, binary, 1); + Field binary_field; + FieldInfo info; + const uint8_t* end = + DataTypeSerDe::deserialize_binary_to_field(binary.data(), binary_field, info); + EXPECT_EQ(end, binary.data() + binary.size()); + EXPECT_EQ(info.scalar_type_id, TYPE_TIMESTAMP_NS); + EXPECT_EQ(info.scale, 9); + EXPECT_EQ(binary_field.get().to_string(9), "2024-02-29 12:34:56.123456789"); +} + +TEST(DataTypeTimeStampNsTest, SerDeBatchRejectsMultipleFractionalSeparators) { + const DataTypeTimeStampNs type; + const auto serde = type.get_serde(); + DataTypeSerDe::FormatOptions options; + + auto strings = ColumnString::create(); + strings->insert_data("2024-01-01 00:00:00.123456789", 29); + strings->insert_data("2024-01-01 00:00:00.123.456", 27); + + auto permissive_result = ColumnNullable::create(type.create_column(), ColumnUInt8::create()); + ASSERT_TRUE(serde->from_string_batch(*strings, *permissive_result, options).ok()); + const auto& permissive_null_map = permissive_result->get_null_map_data(); + ASSERT_EQ(permissive_null_map.size(), 2); + EXPECT_EQ(permissive_null_map[0], 0); + EXPECT_EQ(permissive_null_map[1], 1); + + auto invalid_string = ColumnString::create(); + invalid_string->insert_data("2024-01-01 00:00:00.123.456", 27); + auto strict_result = type.create_column(); + EXPECT_FALSE( + serde->from_string_strict_mode_batch(*invalid_string, *strict_result, options, nullptr) + .ok()); +} + +/* +TEST(DataTypeTimeStampNsTest, SerDeArrowUnitsNullTimezoneAndErrors) { + TimezoneUtils::load_timezones_to_cache(); + cctz::time_zone shanghai; + ASSERT_TRUE(TimezoneUtils::find_cctz_time_zone("Asia/Shanghai", shanghai)); + + const DataTypeTimeStampNs type; + const auto serde = type.get_serde(); + auto source = type.create_column(); + DataTypeSerDe::FormatOptions options; + for (const std::string input : + {"1970-01-01 08:00:01.000000000", "1970-01-01 08:00:02.000000000"}) { + StringRef ref(input); + ASSERT_TRUE(serde->from_string(ref, *source, options).ok()); + } + const NullMap null_map = {0, 1}; + + const std::array, 4> units = { + std::pair {arrow::TimeUnit::SECOND, 1LL}, + std::pair {arrow::TimeUnit::MILLI, 1000LL}, + std::pair {arrow::TimeUnit::MICRO, 1000000LL}, + std::pair {arrow::TimeUnit::NANO, 1000000000LL}, + }; + for (const auto& [unit, multiplier] : units) { + arrow::TimestampBuilder builder(arrow::timestamp(unit, "Asia/Shanghai"), + arrow::default_memory_pool()); + ASSERT_TRUE(serde->write_column_to_arrow(*source, &null_map, &builder, 0, source->size(), + shanghai) + .ok()); + std::shared_ptr array; + ASSERT_TRUE(builder.Finish(&array).ok()); + const auto& timestamps = assert_cast(*array); + EXPECT_EQ(timestamps.Value(0), multiplier); + EXPECT_TRUE(timestamps.IsNull(1)); + + auto result = type.create_column(); + ASSERT_TRUE(serde->read_column_from_arrow(*result, array.get(), 0, 1, shanghai).ok()); + EXPECT_EQ(assert_cast(*result).get_element(0), + assert_cast(*source).get_element(0)); + } + + arrow::Int64Builder wrong_builder; + ASSERT_TRUE(wrong_builder.Append(1).ok()); + std::shared_ptr wrong_array; + ASSERT_TRUE(wrong_builder.Finish(&wrong_array).ok()); + auto wrong_result = type.create_column(); + EXPECT_FALSE( + serde->read_column_from_arrow(*wrong_result, wrong_array.get(), 0, 1, shanghai).ok()); + + for (const auto unit : + {arrow::TimeUnit::SECOND, arrow::TimeUnit::MILLI, arrow::TimeUnit::MICRO}) { + arrow::TimestampBuilder overflow_builder(arrow::timestamp(unit), + arrow::default_memory_pool()); + ASSERT_TRUE(overflow_builder.Append(std::numeric_limits::max()).ok()); + std::shared_ptr overflow_array; + ASSERT_TRUE(overflow_builder.Finish(&overflow_array).ok()); + auto overflow_result = type.create_column(); + EXPECT_FALSE(serde->read_column_from_arrow(*overflow_result, overflow_array.get(), 0, 1, + cctz::utc_time_zone()) + .ok()); + } + + arrow::TimestampBuilder boundary_builder( + arrow::timestamp(arrow::TimeUnit::NANO, "Asia/Shanghai"), arrow::default_memory_pool()); + ASSERT_TRUE(boundary_builder.Append(std::numeric_limits::max()).ok()); + std::shared_ptr boundary_array; + ASSERT_TRUE(boundary_builder.Finish(&boundary_array).ok()); + auto boundary_result = type.create_column(); + EXPECT_FALSE( + serde->read_column_from_arrow(*boundary_result, boundary_array.get(), 0, 1, shanghai) + .ok()); + + auto minimum = type.create_column(); + assert_cast(*minimum).insert_value( + TimeStampNsValue(std::numeric_limits::min())); + arrow::TimestampBuilder minimum_builder( + arrow::timestamp(arrow::TimeUnit::NANO, "Asia/Shanghai"), arrow::default_memory_pool()); + EXPECT_FALSE( + serde->write_column_to_arrow(*minimum, nullptr, &minimum_builder, 0, 1, shanghai).ok()); +} + +TEST(DataTypeTimeStampNsTest, SerDeDecodedNullTimezoneAndOrc) { + TimezoneUtils::load_timezones_to_cache(); + cctz::time_zone shanghai; + ASSERT_TRUE(TimezoneUtils::find_cctz_time_zone("Asia/Shanghai", shanghai)); + + const DataTypeTimeStampNs type; + const auto serde = type.get_serde(); + const int64_t values[] = {0, 1}; + const NullMap null_map = {0, 1}; + DecodedColumnView view; + view.value_kind = DecodedValueKind::INT64; + view.time_unit = DecodedTimeUnit::NANOS; + view.row_count = 2; + view.values = reinterpret_cast(values); + view.null_map = null_map.data(); + view.timestamp_is_adjusted_to_utc = true; + view.timezone = &shanghai; + auto decoded = type.create_column(); + ASSERT_TRUE(serde->read_column_from_decoded_values(*decoded, view).ok()); + const auto& decoded_data = assert_cast(*decoded).get_data(); + EXPECT_EQ(decoded_data[0].to_string(9), "1970-01-01 08:00:00.000000000"); + EXPECT_EQ(decoded_data[1].epoch_nanos(), 0); + + view.value_kind = DecodedValueKind::INT32; + auto invalid_kind = type.create_column(); + EXPECT_FALSE(serde->read_column_from_decoded_values(*invalid_kind, view).ok()); + + auto source = type.create_column(); + int64_t source_raw = 0; + ASSERT_TRUE( + parse_timestamp_ns(StringRef("1970-01-01 08:00:01.123456789"), &source_raw).ok()); + assert_cast(*source).insert_value(TimeStampNsValue(source_raw)); + assert_cast(*source).insert_value(TimeStampNsValue(0)); + + Arena arena; + DataTypeSerDe::FormatOptions options; + orc::TimestampVectorBatch batch(2, *orc::getDefaultPool()); + batch.resize(2); + batch.notNull[0] = 1; + batch.notNull[1] = 0; + batch.hasNulls = true; + ASSERT_TRUE(serde->write_column_to_orc("Asia/Shanghai", *source, nullptr, &batch, 0, 2, arena, + options) + .ok()); + EXPECT_EQ(batch.numElements, 2); + EXPECT_EQ(batch.data[0], 1); + EXPECT_EQ(batch.nanoseconds[0], 123456789); + EXPECT_EQ(batch.data[1], 0); + EXPECT_FALSE(serde->write_column_to_orc("invalid/timezone", *source, nullptr, &batch, 0, 2, + arena, options) + .ok()); + + auto minimum = type.create_column(); + assert_cast(*minimum).insert_value( + TimeStampNsValue(std::numeric_limits::min())); + orc::TimestampVectorBatch minimum_batch(1, *orc::getDefaultPool()); + minimum_batch.resize(1); + minimum_batch.notNull[0] = 1; + EXPECT_FALSE(serde->write_column_to_orc("Asia/Shanghai", *minimum, nullptr, &minimum_batch, 0, + 1, arena, options) + .ok()); +} +*/ + +} // namespace doris diff --git a/be/test/core/data_type/storage_field_type_test.cpp b/be/test/core/data_type/storage_field_type_test.cpp index af621ceb9c6abe..f2f40f2a265429 100644 --- a/be/test/core/data_type/storage_field_type_test.cpp +++ b/be/test/core/data_type/storage_field_type_test.cpp @@ -72,6 +72,7 @@ constexpr std::array supported_mappings { TypePair {PrimitiveType::TYPE_UINT32, FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT}, TypePair {PrimitiveType::TYPE_UINT64, FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT}, TypePair {PrimitiveType::TYPE_TIMESTAMPTZ, FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ}, + TypePair {PrimitiveType::TYPE_TIMESTAMP_NS, FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS}, }; TEST(StorageFieldTypeTest, SupportedMappingsRoundTrip) { @@ -90,7 +91,6 @@ TEST(StorageFieldTypeTest, UnsupportedPrimitiveTypesThrow) { static_cast(33), // TYPE_LAMBDA_FUNCTION (deprecated) PrimitiveType::TYPE_FIXED_LENGTH_OBJECT, PrimitiveType::TYPE_VARBINARY, - static_cast(43), static_cast(255), }; @@ -107,7 +107,6 @@ TEST(StorageFieldTypeTest, UnsupportedOrInvalidFieldTypesThrow) { FieldType::OLAP_FIELD_TYPE_DISCRETE_DOUBLE, static_cast(-1), static_cast(0), - static_cast(41), static_cast(255), }; @@ -159,6 +158,7 @@ TEST(StorageFieldTypeTest, PersistedFieldTypeValuesStayStable) { FieldType::OLAP_FIELD_TYPE_IPV4, FieldType::OLAP_FIELD_TYPE_IPV6, FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ, + FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS, }; for (size_t i = 0; i < persisted_types.size(); ++i) { diff --git a/be/test/core/data_type_serde/data_type_serde_decoded_values_test.cpp b/be/test/core/data_type_serde/data_type_serde_decoded_values_test.cpp index 18e4aa8a1cf848..1303fa1e29a032 100644 --- a/be/test/core/data_type_serde/data_type_serde_decoded_values_test.cpp +++ b/be/test/core/data_type_serde/data_type_serde_decoded_values_test.cpp @@ -830,7 +830,7 @@ TEST(DataTypeSerDeDecodedValuesTest, ReadDateTimeV2Millis) { "1970-01-01 00:00:00.001000", "1970-01-01 00:00:01.234000"}); } -TEST(DataTypeSerDeDecodedValuesTest, ReadDateTimeV2Nanos) { +TEST(DataTypeSerDeDecodedValuesTest, ReadTimeStampNss) { auto type = std::make_shared(6); std::vector values = {-1000, 0, 1000, 1234567890}; auto view = make_fixed_view(DecodedValueKind::INT64, values); diff --git a/be/test/exec/sink/tablet_sink_hash_partitioner_test.cpp b/be/test/exec/sink/tablet_sink_hash_partitioner_test.cpp index ba587b3c3ba89b..f280b202fc8c50 100644 --- a/be/test/exec/sink/tablet_sink_hash_partitioner_test.cpp +++ b/be/test/exec/sink/tablet_sink_hash_partitioner_test.cpp @@ -36,6 +36,7 @@ #include "common/config.h" #include "core/assert_cast.h" #include "core/column/column_vector.h" +#include "core/data_type/data_type_date_or_datetime_v2.h" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" #include "exec/operator/exchange_sink_operator.h" @@ -59,6 +60,69 @@ using doris::ExchangeSinkLocalState; using doris::ExchangeSinkOperatorX; using doris::OperatorContext; +TExprNode _make_timestamp_ns_literal(std::string value) { + TExprNode node; + node.__set_node_type(TExprNodeType::DATE_LITERAL); + node.__set_num_children(0); + + TDateLiteral literal; + literal.__set_value(std::move(value)); + node.__set_date_literal(literal); + + TTypeDesc type_desc = create_type_desc(PrimitiveType::TYPE_TIMESTAMP_NS); + type_desc.__set_is_nullable(false); + node.__set_type(type_desc); + node.__set_is_nullable(false); + return node; +} + +void _build_timestamp_ns_schema(OperatorContext& ctx, TOlapTableSchemaParam& tschema, + int64_t& schema_index_id) { + TTypeDesc type_desc = create_type_desc(PrimitiveType::TYPE_TIMESTAMP_NS); + + TDescriptorTableBuilder dtb; + TTupleDescriptorBuilder tuple_builder; + tuple_builder.add_slot(TSlotDescriptorBuilder() + .set_slotType(type_desc) + .nullable(true) + .column_name("dt") + .column_pos(0) + .build()); + tuple_builder.build(&dtb); + + auto thrift_desc_tbl = dtb.desc_tbl(); + DescriptorTbl* desc_tbl = nullptr; + auto st = DescriptorTbl::create(ctx.state.obj_pool(), thrift_desc_tbl, &desc_tbl); + ASSERT_TRUE(st.ok()) << st.to_string(); + ctx.state.set_desc_tbl(desc_tbl); + + tschema.db_id = 1; + tschema.table_id = 2; + tschema.version = 0; + tschema.slot_descs = thrift_desc_tbl.slotDescriptors; + tschema.tuple_desc = thrift_desc_tbl.tupleDescriptors.front(); + + TOlapTableIndexSchema index_schema; + index_schema.id = 10; + index_schema.columns = {"dt"}; + index_schema.schema_hash = 123; + tschema.indexes = {index_schema}; + schema_index_id = index_schema.id; +} + +TOlapTablePartition _make_timestamp_ns_partition(int64_t id, int64_t schema_index_id) { + TOlapTablePartition partition; + partition.id = id; + partition.num_buckets = 1; + partition.__set_is_mutable(true); + + TOlapTableIndexTablets index_tablets; + index_tablets.index_id = schema_index_id; + index_tablets.tablets = {id * 100}; + partition.indexes = {index_tablets}; + return partition; +} + std::shared_ptr _create_parent_operator( OperatorContext& ctx, const std::shared_ptr& row_desc_holder) { TDataStreamSink sink; @@ -275,5 +339,50 @@ TEST(TabletSinkHashPartitionerTest, OlapTabletFinderRoundRobinEveryBatch) { EXPECT_EQ(tablet_index[0], 0); } } + +TEST(TabletSinkHashPartitionerTest, TimeStampNsRangeUsesPartitionColumnScale) { + OperatorContext ctx; + + TOlapTableSchemaParam tschema; + int64_t schema_index_id = 0; + _build_timestamp_ns_schema(ctx, tschema, schema_index_id); + + auto schema = std::make_shared(); + auto st = schema->init(tschema); + ASSERT_TRUE(st.ok()) << st.to_string(); + + TOlapTablePartitionParam partition_param; + partition_param.db_id = 1; + partition_param.table_id = 2; + partition_param.version = 0; + partition_param.__set_partition_type(TPartitionType::RANGE_PARTITIONED); + partition_param.__set_partition_columns({"dt"}); + + auto before_epoch = _make_timestamp_ns_partition(1, schema_index_id); + before_epoch.__set_end_keys({_make_timestamp_ns_literal("1970-01-01 00:00:00")}); + + auto epoch = _make_timestamp_ns_partition(2, schema_index_id); + epoch.__set_start_keys({_make_timestamp_ns_literal("1970-01-01 00:00:00")}); + epoch.__set_end_keys({_make_timestamp_ns_literal("1970-01-01 00:00:00.000000002")}); + + auto after_epoch = _make_timestamp_ns_partition(3, schema_index_id); + after_epoch.__set_start_keys({_make_timestamp_ns_literal("1970-01-01 00:00:00.000000002")}); + partition_param.partitions = {epoch, before_epoch, after_epoch}; + + VOlapTablePartitionParam partitions(schema, partition_param); + st = partitions.init(); + ASSERT_TRUE(st.ok()) << st.to_string(); + + auto* slot = schema->tuple_desc()->slots().front(); + auto column = slot->get_empty_mutable_column(); + const TimeStampNsValue min_value(std::numeric_limits::min()); + column->insert_data(reinterpret_cast(&min_value), 0); + Block block({{std::move(column), slot->get_data_type_ptr(), slot->col_name()}}); + + VOlapTablePartition* result = nullptr; + EXPECT_TRUE(partitions.find_partition(&block, 0, result)); + ASSERT_NE(result, nullptr); + EXPECT_EQ(result->id, before_epoch.id); +} } // anonymous namespace } // namespace doris diff --git a/be/test/exprs/aggregate/agg_approx_count_distinct_test.cpp b/be/test/exprs/aggregate/agg_approx_count_distinct_test.cpp new file mode 100644 index 00000000000000..1f8c3f4310fbc9 --- /dev/null +++ b/be/test/exprs/aggregate/agg_approx_count_distinct_test.cpp @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include + +#include "core/column/column_vector.h" +#include "core/data_type/data_type_timestamp_ns.h" +#include "exprs/aggregate/aggregate_function.h" +#include "exprs/aggregate/aggregate_function_simple_factory.h" + +namespace doris { + +void register_aggregate_function_approx_count_distinct(AggregateFunctionSimpleFactory& factory); + +TEST(AggApproxCountDistinctTest, TimeStampNs) { + AggregateFunctionSimpleFactory factory; + register_aggregate_function_approx_count_distinct(factory); + + auto data_type = std::make_shared(); + const DataTypes argument_types = {data_type}; + for (const std::string function_name : {"approx_count_distinct", "ndv"}) { + auto function = factory.get(function_name, argument_types, nullptr, false, -1); + ASSERT_NE(function, nullptr); + + auto column = ColumnTimeStampNs::create(); + for (const int64_t epoch_nanos : {-1, 0, 1, 1, 1'000'000'001}) { + column->insert_value(TimeStampNsValue(epoch_nanos)); + } + + std::unique_ptr memory(new char[function->size_of_data()]); + AggregateDataPtr place = memory.get(); + function->create(place); + Arena arena; + const IColumn* columns[] = {column.get()}; + for (size_t row = 0; row < column->size(); ++row) { + function->add(place, columns, row, arena); + } + + auto result = ColumnInt64::create(); + function->insert_result_into(place, *result); + EXPECT_EQ(result->get_element(0), 4); + function->destroy(place); + } +} + +} // namespace doris diff --git a/be/test/exprs/aggregate/agg_min_max_test.cpp b/be/test/exprs/aggregate/agg_min_max_test.cpp index 3f2080fc8304e6..86b19d7462bad8 100644 --- a/be/test/exprs/aggregate/agg_min_max_test.cpp +++ b/be/test/exprs/aggregate/agg_min_max_test.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -34,6 +35,7 @@ #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_timestamp_ns.h" #include "core/field.h" #include "core/string_ref.h" #include "core/types.h" @@ -135,6 +137,41 @@ TEST_P(AggMinMaxTest, min_max_decimal_test) { } } +TEST_P(AggMinMaxTest, min_max_timestamp_ns_test) { + Arena arena; + const std::string min_max_type = GetParam(); + const std::vector epoch_nanos = {0, -1, std::numeric_limits::max(), + std::numeric_limits::min(), 1}; + + auto data_type = std::make_shared(); + auto column = ColumnTimeStampNs::create(); + for (const int64_t value : epoch_nanos) { + column->insert_value(TimeStampNsValue(value)); + } + + AggregateFunctionSimpleFactory factory; + register_aggregate_function_minmax(factory); + DataTypes data_types = {data_type}; + auto agg_function = factory.get(min_max_type, data_types, data_type, false, -1); + ASSERT_NE(agg_function, nullptr); + + std::unique_ptr memory(new char[agg_function->size_of_data()]); + AggregateDataPtr place = memory.get(); + agg_function->create(place); + + const IColumn* columns[1] = {column.get()}; + for (size_t row = 0; row < epoch_nanos.size(); ++row) { + agg_function->add(place, columns, row, arena); + } + + auto result = ColumnTimeStampNs::create(); + agg_function->insert_result_into(place, *result); + const int64_t expected = min_max_type == "min" ? std::numeric_limits::min() + : std::numeric_limits::max(); + EXPECT_EQ(result->get_element(0).epoch_nanos(), expected); + agg_function->destroy(place); +} + TEST_P(AggMinMaxTest, min_max_string_test) { Arena arena; std::string min_max_type = GetParam(); diff --git a/be/test/exprs/function/function_test_util.cpp b/be/test/exprs/function/function_test_util.cpp index b37e452b69a52a..d3662c1ec627aa 100644 --- a/be/test/exprs/function/function_test_util.cpp +++ b/be/test/exprs/function/function_test_util.cpp @@ -44,6 +44,7 @@ #include "core/data_type/data_type_struct.h" #include "core/data_type/data_type_time.h" #include "core/data_type/data_type_varbinary.h" +#include "core/data_type_serde/data_type_timestamp_ns_serde.h" #include "core/field.h" #include "core/types.h" #include "core/value/bitmap_value.h" @@ -194,7 +195,11 @@ static size_t type_index_to_data_type(const std::vector& input_types, s desc = type; return 1; case PrimitiveType::TYPE_DATETIMEV2: - type = std::make_shared(input_types[index].scale_or(0)); + type = create_datetimev2(input_types[index].scale_or(0)); + desc = type; + return 1; + case PrimitiveType::TYPE_TIMESTAMP_NS: + type = std::make_shared(); desc = type; return 1; case PrimitiveType::TYPE_TIMEV2: @@ -318,7 +323,12 @@ bool insert_datetime_cell(MutableColumnPtr& column, DataTypePtr date_type_ptr, c auto datetime_str = any_cast(cell); CastParameters params; - if constexpr (PType == PrimitiveType::TYPE_DATETIMEV2) { + if constexpr (PType == PrimitiveType::TYPE_TIMESTAMP_NS) { + int64_t epoch_nanos = 0; + result = parse_timestamp_ns({datetime_str.c_str(), datetime_str.size()}, &epoch_nanos) + .ok(); + date_value = TimeStampNsValue(epoch_nanos); + } else if constexpr (PType == PrimitiveType::TYPE_DATETIMEV2) { result = CastToDatetimeV2::from_string_non_strict_mode( {datetime_str.c_str(), datetime_str.size()}, date_value, nullptr, date_type_ptr->get_scale(), params); @@ -529,6 +539,11 @@ bool insert_cell(MutableColumnPtr& column, DataTypePtr type_ptr, const AnyType& column, type_ptr, cell, datetime_is_string_format))); break; } + case PrimitiveType::TYPE_TIMESTAMP_NS: { + RETURN_IF_FALSE((insert_datetime_cell( + column, type_ptr, cell, datetime_is_string_format))); + break; + } case PrimitiveType::TYPE_TIMEV2: { TimeValue::TimeType time_value = 0; if (datetime_is_string_format) { diff --git a/be/test/exprs/function/function_test_util.h b/be/test/exprs/function/function_test_util.h index ebbb9c2f77c486..b486037a5d44d9 100644 --- a/be/test/exprs/function/function_test_util.h +++ b/be/test/exprs/function/function_test_util.h @@ -50,6 +50,7 @@ #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" #include "core/data_type/data_type_time.h" +#include "core/data_type/data_type_timestamp_ns.h" #include "core/data_type/data_type_varbinary.h" #include "core/extended_types.h" #include "core/types.h" @@ -376,6 +377,9 @@ Status check_function(const std::string& func_name, const InputTypeSet& input_ty return ResultNullable ? make_nullable(std::make_shared(result_precision, result_scale)) : std::make_shared(result_precision, result_scale); + } else if constexpr (std::is_same_v) { + return ResultNullable ? make_nullable(std::make_shared()) + : std::make_shared(); } else if constexpr (IsDataTypeDateTimeV2 || IsTimeV2Type) { // datetimev2 or timev2 UInt32 real_scale = 0; @@ -428,6 +432,9 @@ Status check_function(const std::string& func_name, const InputTypeSet& input_ty ? make_nullable(std::make_shared(result_precision, result_scale)) : std::make_shared(result_precision, result_scale); + } else if constexpr (std::is_same_v) { + result_type_ptr = ResultNullable ? make_nullable(std::make_shared()) + : std::make_shared(); } else if constexpr (IsDataTypeDateTimeV2 || IsTimeV2Type) { // datetimev2 UInt32 real_scale = 0; diff --git a/be/test/exprs/function/function_time_test.cpp b/be/test/exprs/function/function_time_test.cpp index e012c7784e97ea..bc7e2d75dff1dd 100644 --- a/be/test/exprs/function/function_time_test.cpp +++ b/be/test/exprs/function/function_time_test.cpp @@ -26,6 +26,7 @@ #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_time.h" +#include "core/data_type/data_type_timestamp_ns.h" #include "core/types.h" #include "core/value/time_value.h" #include "core/value/vdatetime_value.h" @@ -267,6 +268,17 @@ TEST(VTimestampFunctionsTest, unix_timestamp_test) { check_function(func_name, input_types, data_set, 6, 18)); } + { + InputTypeSet input_types = {PrimitiveType::TYPE_TIMESTAMP_NS}; + DataSet data_set = { + {{std::string("1970-01-01 08:00:00.123456789")}, DECIMAL128V3(0, 123456789, 9)}, + {{std::string("2022-05-24 12:34:56.789123456")}, + DECIMAL128V3(1653366896, 789123456, 9)}, + }; + static_cast( + check_function(func_name, input_types, data_set, 9, 21)); + } + // test out of range { InputTypeSet input_types = {{PrimitiveType::TYPE_DATETIMEV2, 6}}; @@ -343,6 +355,23 @@ TEST(VTimestampFunctionsTest, convert_tz_test) { data_set, 6)); } + { + InputTypeSet input_types_timestamp_ns = {PrimitiveType::TYPE_TIMESTAMP_NS, + PrimitiveType::TYPE_VARCHAR, + PrimitiveType::TYPE_VARCHAR}; + DataSet data_set = {{{std::string {"1970-01-01 08:00:00.123456789"}, + std::string {"Asia/Shanghai"}, std::string {"UTC"}}, + std::string("1970-01-01 00:00:00.123456789")}, + {{std::string {"2021-03-28 02:15:30.123456789"}, + std::string {"Europe/Paris"}, std::string {"UTC"}}, + std::string("2021-03-28 01:00:00.000000000")}, + {{std::string {"2021-03-28 03:00:30.123456789"}, + std::string {"Europe/Paris"}, std::string {"UTC"}}, + std::string("2021-03-28 01:00:30.123456789")}}; + static_cast(check_function( + func_name, input_types_timestamp_ns, data_set)); + } + { DataSet data_set = {{{std::string {"2019-08-01 02:18:27"}, std::string {"Asia/Shanghai"}, std::string {"UTC"}}, @@ -1449,6 +1478,22 @@ TEST(VTimestampFunctionsTest, dayname_test) { TEST(VTimestampFunctionsTest, datetrunc_test) { std::string func_name = "date_trunc"; + { + InputTypeSet input_types = {AnyType {PrimitiveType::TYPE_TIMESTAMP_NS, 9}, + Consted {PrimitiveType::TYPE_VARCHAR}}; + DataSet data_set = {{{std::string("2023-10-17 12:34:56.123456789"), std::string("day")}, + std::string("2023-10-17 00:00:00.000000000")}}; + static_cast( + check_function(func_name, input_types, data_set, 9)); + } + { + InputTypeSet input_types = {Consted {PrimitiveType::TYPE_VARCHAR}, + AnyType {PrimitiveType::TYPE_TIMESTAMP_NS, 9}}; + DataSet data_set = {{{std::string("day"), std::string("2023-10-17 12:34:56.123456789")}, + std::string("2023-10-17 00:00:00.000000000")}}; + static_cast( + check_function(func_name, input_types, data_set, 9)); + } { InputTypeSet input_types = {PrimitiveType::TYPE_DATETIMEV2, Consted {PrimitiveType::TYPE_VARCHAR}}; diff --git a/be/test/exprs/vcondition_expr_test.cpp b/be/test/exprs/vcondition_expr_test.cpp index b26f01cc2664c5..83b5ea26aa8fd9 100644 --- a/be/test/exprs/vcondition_expr_test.cpp +++ b/be/test/exprs/vcondition_expr_test.cpp @@ -29,8 +29,10 @@ #include "core/column/column_nullable.h" #include "core/column/column_vector.h" +#include "core/data_type/data_type_date_or_datetime_v2.h" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_timestamp_ns.h" #include "exprs/vexpr.h" #include "exprs/vexpr_context.h" @@ -38,7 +40,7 @@ namespace doris { // Build a minimal TExprNode as the input of VectorizedCoalesceExpr. // Only fields required by the VExpr base ctor (so that create_data_type works) are set. -static TExprNode make_coalesce_node(TPrimitiveType::type ptype, bool is_nullable) { +static TExprNode make_coalesce_node(TPrimitiveType::type ptype, bool is_nullable, int scale = -1) { TExprNode node; node.node_type = TExprNodeType::FUNCTION_CALL; node.num_children = 0; @@ -49,6 +51,9 @@ static TExprNode make_coalesce_node(TPrimitiveType::type ptype, bool is_nullable type_node.type = TTypeNodeType::SCALAR; TScalarType scalar_type; scalar_type.__set_type(ptype); + if (scale >= 0) { + scalar_type.__set_scale(scale); + } type_node.__set_scalar_type(scalar_type); type_desc.types.push_back(type_node); node.__set_type(type_desc); @@ -355,4 +360,53 @@ TEST_F(VConditionExprCoalesceTest, Int32_NormalPathStillWorks) { EXPECT_FALSE(is_null); } +TEST_F(VConditionExprCoalesceTest, TimeStampNs) { + auto coalesce_node = + make_coalesce_node(TPrimitiveType::TIMESTAMP_NS, /*is_nullable=*/true, /*scale=*/9); + auto coalesce_expr = VectorizedCoalesceExpr::create_shared(coalesce_node); + auto nano_type = std::make_shared(); + coalesce_expr->data_type() = std::make_shared(nano_type); + + auto nested0 = ColumnTimeStampNs::create(); + nested0->insert_value(TimeStampNsValue(11)); + nested0->insert_value(TimeStampNsValue(0)); + nested0->insert_value(TimeStampNsValue(-17)); + nested0->insert_value(TimeStampNsValue(0)); + auto null_map0 = ColumnUInt8::create(); + null_map0->insert_value(0); + null_map0->insert_value(1); + null_map0->insert_value(0); + null_map0->insert_value(1); + ColumnPtr col0 = ColumnNullable::create(std::move(nested0), std::move(null_map0)); + + auto col1 = ColumnTimeStampNs::create(); + col1->insert_value(TimeStampNsValue(99)); + col1->insert_value(TimeStampNsValue(22)); + col1->insert_value(TimeStampNsValue(std::numeric_limits::max())); + col1->insert_value(TimeStampNsValue(std::numeric_limits::min())); + + coalesce_expr->add_child( + std::make_shared(col0, std::make_shared(nano_type))); + coalesce_expr->add_child(std::make_shared(std::move(col1), nano_type)); + + VExprContext context(coalesce_expr); + ColumnPtr result; + auto st = coalesce_expr->execute_column_impl(&context, /*block=*/nullptr, + /*selector=*/nullptr, /*count=*/4, result); + ASSERT_TRUE(st.ok()) << st.to_string(); + ASSERT_EQ(result->size(), 4); + + const auto& nullable = assert_cast(*result); + const auto& values = + assert_cast(nullable.get_nested_column()).get_data(); + EXPECT_FALSE(nullable.is_null_at(0)); + EXPECT_EQ(values[0].epoch_nanos(), 11); + EXPECT_FALSE(nullable.is_null_at(1)); + EXPECT_EQ(values[1].epoch_nanos(), 22); + EXPECT_FALSE(nullable.is_null_at(2)); + EXPECT_EQ(values[2].epoch_nanos(), -17); + EXPECT_FALSE(nullable.is_null_at(3)); + EXPECT_EQ(values[3].epoch_nanos(), std::numeric_limits::min()); +} + } // namespace doris diff --git a/be/test/exprs/vexpr_unaligned_int128_test.cpp b/be/test/exprs/vexpr_unaligned_int128_test.cpp index d34a6da7f9430a..844c5fc633e463 100644 --- a/be/test/exprs/vexpr_unaligned_int128_test.cpp +++ b/be/test/exprs/vexpr_unaligned_int128_test.cpp @@ -34,6 +34,7 @@ #include "core/value/decimalv2_value.h" #include "core/value/large_int_value.h" +#include "core/value/timestamp_ns_value.h" #include "exprs/vexpr.h" namespace doris { @@ -104,4 +105,14 @@ TEST(UnalignedInt128Test, DecimalV2LiteralFromUnalignedBuffer) { EXPECT_EQ(node.decimal_literal.value, src.to_string()); } +TEST(UnalignedInt128Test, TimeStampNsLiteralKeepsItsPhysicalType) { + const TimeStampNsValue value(1); + TExprNode node; + Status st = create_texpr_literal_node(&value, &node); + ASSERT_TRUE(st.ok()) << st; + ASSERT_TRUE(node.__isset.date_literal); + ASSERT_EQ(node.type.types.size(), 1); + EXPECT_EQ(node.type.types.front().scalar_type.type, TPrimitiveType::TIMESTAMP_NS); +} + } // namespace doris diff --git a/be/test/format/orc/orc_reader_fill_data_test.cpp b/be/test/format/orc/orc_reader_fill_data_test.cpp index 44177ddb29df46..ae0f035432d1ad 100644 --- a/be/test/format/orc/orc_reader_fill_data_test.cpp +++ b/be/test/format/orc/orc_reader_fill_data_test.cpp @@ -161,6 +161,26 @@ TEST_F(OrcReaderFillDataTest, TimestampDecodeNormalizesCstTimezone) { EXPECT_EQ(data_type->to_string(timestamp_column.get_data()[0]), "2020-01-02 03:04:05.321000"); } +TEST_F(OrcReaderFillDataTest, TimestampDecodeKeepsWideDateTimeV2Range) { + auto batch = create_timestamp_batch(1, {253402300799}, {999999000}); + auto data_type = std::make_shared(6); + auto column = data_type->create_column(); + auto orc_type_ptr = createPrimitiveType(orc::TypeKind::TIMESTAMP); + + TFileScanRangeParams params; + TFileRangeDesc range; + auto reader = OrcReader::create_unique(params, range, 4064, "UTC", nullptr, nullptr, true); + + MutableColumnPtr mutable_column = column->assert_mutable(); + Status status = reader->_fill_doris_data_column( + "test_ts", mutable_column, data_type, const_node, orc_type_ptr.get(), batch.get(), 1); + + ASSERT_TRUE(status.ok()) << status.to_string(); + const auto& timestamp_column = assert_cast(*mutable_column); + ASSERT_EQ(timestamp_column.size(), 1); + EXPECT_EQ(data_type->to_string(timestamp_column.get_data()[0]), "9999-12-31 23:59:59.999999"); +} + TEST_F(OrcReaderFillDataTest, SchemaChangeNullableNullMapUsesAppendedSlice) { std::vector values = {10, 20, 30}; std::vector nulls = {true, false, true}; diff --git a/be/test/storage/key_coder_test.cpp b/be/test/storage/key_coder_test.cpp index 5362645078677c..ee521f3a86df54 100644 --- a/be/test/storage/key_coder_test.cpp +++ b/be/test/storage/key_coder_test.cpp @@ -174,6 +174,7 @@ TEST_F(KeyCoderTest, test_int) { test_integer_encode(); test_integer_encode(); + test_integer_encode(); } TEST_F(KeyCoderTest, test_date) { diff --git a/be/test/storage/segment/bloom_filter_index_reader_writer_test.cpp b/be/test/storage/segment/bloom_filter_index_reader_writer_test.cpp index 17eeac11a7db15..2d798a1498b21b 100644 --- a/be/test/storage/segment/bloom_filter_index_reader_writer_test.cpp +++ b/be/test/storage/segment/bloom_filter_index_reader_writer_test.cpp @@ -476,6 +476,33 @@ TEST_F(BloomFilterIndexReaderWriterTest, test_datetimev2) { delete[] val; } +TEST_F(BloomFilterIndexReaderWriterTest, test_timestamp_ns) { + size_t num = 1024 * 3; + std::vector values(num); + for (size_t i = 0; i < num; ++i) { + values[i] = -1'000'000'000 + static_cast(i); + } + + { + const std::string file_name = "bloom_filter_timestamp_ns"; + int64_t not_exist_value = -1'000'000'001; + auto st = test_bloom_filter_index_reader_writer_template< + FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS>(file_name, values.data(), num, 1, + ¬_exist_value); + EXPECT_TRUE(st.ok()); + } + // Primary key bloom filters consume encoded Slice keys rather than raw column values. + { + const std::string file_name = "bloom_filter_timestamp_ns_pk"; + int64_t not_exist_value = -1'000'000'001; + auto st = test_bloom_filter_index_reader_writer_template< + FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS>(file_name, values.data(), num, 1, + ¬_exist_value, false, true); + EXPECT_FALSE(st.ok()); + EXPECT_EQ(st.code(), TStatusCode::NOT_IMPLEMENTED_ERROR); + } +} + TEST_F(BloomFilterIndexReaderWriterTest, test_timestamptz) { size_t num = 1024 * 3; auto base_dt = make_timestamptz(2025, 11, 14, 14, 37, 30, 999998); diff --git a/be/test/storage/segment/inverted_index_writer_test.cpp b/be/test/storage/segment/inverted_index_writer_test.cpp index 8b7f56221cf8cf..563e9cbdd79c67 100644 --- a/be/test/storage/segment/inverted_index_writer_test.cpp +++ b/be/test/storage/segment/inverted_index_writer_test.cpp @@ -790,6 +790,44 @@ TEST_F(InvertedIndexWriterTest, NumericWrite) { test_numeric_write("test_rowset_3", 0); } +TEST_F(InvertedIndexWriterTest, TimeStampNsWrite) { + TabletColumn field; + field.set_name("dt"); + field.set_unique_id(0); + field.set_type(FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS); + field.set_is_nullable(false); + + TabletIndexPB index_meta_pb; + index_meta_pb.set_index_type(IndexType::INVERTED); + index_meta_pb.set_index_id(1); + index_meta_pb.set_index_name("test_timestamp_ns"); + index_meta_pb.add_col_unique_id(0); + TabletIndex index_meta; + index_meta.init_from_pb(index_meta_pb); + + const std::string rowset_id = "test_timestamp_ns"; + const std::string index_path_prefix {InvertedIndexDescriptor::get_index_file_path_prefix( + local_segment_path(kTestDir, rowset_id, 0))}; + const std::string index_path = + InvertedIndexDescriptor::get_index_file_path_v2(index_path_prefix); + io::FileWriterPtr file_writer; + ASSERT_TRUE(io::global_local_filesystem()->create_file(index_path, &file_writer).ok()); + auto index_file_writer = std::make_unique( + io::global_local_filesystem(), index_path_prefix, rowset_id, 0, + InvertedIndexStorageFormatPB::V2, std::move(file_writer)); + + std::unique_ptr column_writer; + ASSERT_TRUE( + IndexColumnWriter::create(&field, &column_writer, index_file_writer.get(), &index_meta) + .ok()); + const std::vector values = {std::numeric_limits::min(), -1, 0, 1, + std::numeric_limits::max()}; + ASSERT_TRUE(column_writer->add_values("dt", values.data(), values.size()).ok()); + ASSERT_TRUE(column_writer->finish().ok()); + ASSERT_TRUE(index_file_writer->begin_close().ok()); + ASSERT_TRUE(index_file_writer->finish_close().ok()); +} + // Test case for Unicode string values with enable_correct_term_write=true TEST_F(InvertedIndexWriterTest, UnicodeStringWriteEnabled) { test_unicode_string_write("test_rowset_4", 0, true); @@ -1544,4 +1582,4 @@ TEST_F(InvertedIndexWriterTest, NormsFileCreationWithTokenization) { << "inverted_index_writer.cpp where .nrm file creation depends on _should_analyzer."; } -} // namespace doris::segment_v2 \ No newline at end of file +} // namespace doris::segment_v2 diff --git a/be/test/storage/segment/zone_map_index_test.cpp b/be/test/storage/segment/zone_map_index_test.cpp index db4b692b1f1477..079a2d237dc553 100644 --- a/be/test/storage/segment/zone_map_index_test.cpp +++ b/be/test/storage/segment/zone_map_index_test.cpp @@ -1119,6 +1119,44 @@ TEST_F(ColumnZoneMapTest, TimestamptzPage) { EXPECT_EQ(false, zone_maps[4].has_not_null()); } +TEST_F(ColumnZoneMapTest, TimeStampNsPage) { + auto column = std::make_shared(); + column->_unique_id = 0; + column->_col_name = "timestamp_ns"; + column->_type = FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS; + column->_is_key = true; + column->_is_nullable = true; + column->_length = 8; + column->_index_length = 8; + + auto data_type = DataTypeFactory::instance().create_data_type(TYPE_TIMESTAMP_NS, false, 0, 9); + std::unique_ptr writer; + ASSERT_TRUE(ZoneMapIndexWriter::create(data_type, column.get(), writer).ok()); + + TimeStampNsValue values[] = { + TimeStampNsValue(std::numeric_limits::min()), + TimeStampNsValue(-1), + TimeStampNsValue(0), + TimeStampNsValue(std::numeric_limits::max()), + }; + writer->add_values(values, std::size(values)); + ASSERT_TRUE(writer->flush().ok()); + + const std::string file_path = kTestDir + "/timestamp_ns_zonemap"; + io::FileWriterPtr file_writer; + ASSERT_TRUE(_fs->create_file(file_path, &file_writer).ok()); + ColumnIndexMetaPB index_meta; + ASSERT_TRUE(writer->finish(file_writer.get(), &index_meta).ok()); + ASSERT_TRUE(file_writer->close().ok()); + + ZoneMap segment_zone_map; + ASSERT_TRUE(ZoneMap::from_proto(index_meta.zone_map_index().segment_zone_map(), data_type, + segment_zone_map) + .ok()); + EXPECT_EQ(segment_zone_map.min_value.get(), values[0]); + EXPECT_EQ(segment_zone_map.max_value.get(), values[3]); +} + // Regression test for "all-null page after a value page" — int variant. // // Page 1 has integers, page 2 is all nulls. The fix in flush() guards the diff --git a/be/test/testutil/datetime_ut_util.h b/be/test/testutil/datetime_ut_util.h index 9f0967ce1d76dc..9ced7af876c0a0 100644 --- a/be/test/testutil/datetime_ut_util.h +++ b/be/test/testutil/datetime_ut_util.h @@ -27,6 +27,15 @@ inline auto make_datetime(int year, int month, int day, int hour, int minute, in return dt; } +inline auto make_datetime_nano(int year, int month, int day, int hour, int minute, int second, + int nanosecond) { + DateV2Value datetime; + datetime.unchecked_set_time(year, month, day, hour, minute, second, nanosecond / 1000); + TimeStampNsValue result; + DORIS_CHECK(result.from_datetime(datetime, nanosecond % 1000)); + return result; +} + inline auto make_timestamptz(int year, int month, int day, int hour, int minute, int second, int microsecond) { DateV2Value dt; @@ -34,4 +43,4 @@ inline auto make_timestamptz(int year, int month, int day, int hour, int minute, return TimestampTzValue(dt.to_date_int_val()); } -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/cloud/src/meta-service/meta_service_schema.h b/cloud/src/meta-service/meta_service_schema.h index ffade3a150fbcf..4293de93d02508 100644 --- a/cloud/src/meta-service/meta_service_schema.h +++ b/cloud/src/meta-service/meta_service_schema.h @@ -24,6 +24,9 @@ namespace doris::cloud { class Transaction; struct ValueBuf; +bool check_tablet_schema(const doris::TabletSchemaCloudPB& schema, + doris::TabletSchemaCloudPB& saved_schema); + void put_schema_kv(MetaServiceCode& code, std::string& msg, Transaction* txn, std::string_view schema_key, const doris::TabletSchemaCloudPB& schema); diff --git a/cloud/test/schema_kv_test.cpp b/cloud/test/schema_kv_test.cpp index 32a9302a0150e4..146dbd7e2e1e64 100644 --- a/cloud/test/schema_kv_test.cpp +++ b/cloud/test/schema_kv_test.cpp @@ -335,6 +335,20 @@ TEST(DetachSchemaKVTest, PutSchemaKvTest) { ASSERT_EQ(document_get(txn.get(), versioned_key, &saved_schema), TxnErrorCode::TXN_OK); EXPECT_EQ(saved_schema.schema_version(), schema_version); } + + { + doris::TabletSchemaCloudPB timestamp_ns_schema; + auto* timestamp_ns_column = timestamp_ns_schema.add_column(); + timestamp_ns_column->set_unique_id(1); + timestamp_ns_column->set_type("TIMESTAMP_NS"); + + doris::TabletSchemaCloudPB saved_schema(timestamp_ns_schema); + EXPECT_TRUE(check_tablet_schema(timestamp_ns_schema, saved_schema)); + + doris::TabletSchemaCloudPB datetimev2_schema(timestamp_ns_schema); + datetimev2_schema.mutable_column(0)->set_type("DATETIMEV2"); + EXPECT_FALSE(check_tablet_schema(datetimev2_schema, saved_schema)); + } } static void begin_txn(MetaServiceProxy* meta_service, int64_t db_id, const std::string& label, diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/DateLiteral.java b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/DateLiteral.java index edb4c7b3c533c1..88ec6851995cba 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/DateLiteral.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/DateLiteral.java @@ -422,6 +422,9 @@ public int compareLiteral(LiteralExpr expr) { long diff = getMicroPartWithinScale() - other.getMicroPartWithinScale(); return diff < 0 ? -1 : (diff == 0 ? 0 : 1); } + if (expr instanceof TimeStampNsLiteral) { + return -expr.compareLiteral(this); + } // date time will not overflow when doing addition and subtraction return Integer.signum(getStringValue().compareTo(expr.getStringValue())); } diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/ExprVisitor.java b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/ExprVisitor.java index fbed2844989f71..2fb6d46f2bdae9 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/ExprVisitor.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/ExprVisitor.java @@ -18,7 +18,7 @@ package org.apache.doris.analysis; /** - * Abstract visitor base class for {@link Expr} and all 39 concrete subclasses. + * Abstract visitor base class for {@link Expr} and all concrete subclasses. * Follows the same {@code } pattern as * {@code org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor}. * @@ -69,6 +69,10 @@ public R visitDateLiteral(DateLiteral expr, C context) { return visit(expr, context); } + public R visitTimeStampNsLiteral(TimeStampNsLiteral expr, C context) { + return visit(expr, context); + } + public R visitTimeV2Literal(TimeV2Literal expr, C context) { return visit(expr, context); } diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/TimeStampNsLiteral.java b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/TimeStampNsLiteral.java new file mode 100644 index 00000000000000..42fc483c69259f --- /dev/null +++ b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/TimeStampNsLiteral.java @@ -0,0 +1,376 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.Type; +import org.apache.doris.common.AnalysisException; + +import com.google.common.base.Preconditions; +import com.google.gson.annotations.SerializedName; + +import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.time.DateTimeException; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.Year; +import java.time.ZoneOffset; +import java.util.TimeZone; + +/** + * Legacy literal for the fixed nanosecond-precision TIMESTAMP_NS type. + * + *

TIMESTAMP_NS owns its value and range logic instead of inheriting the unrelated calendar + * encodings and microsecond rules in {@link DateLiteral}.

+ */ +public final class TimeStampNsLiteral extends LiteralExpr { + private static final long NANOSECONDS_PER_SECOND = 1_000_000_000L; + private static final long MAX_NANOSECOND = NANOSECONDS_PER_SECOND - 1; + private static final LocalDateTime MIN_VALUE + = LocalDateTime.of(1677, 9, 21, 0, 12, 43, 145224192); + private static final LocalDateTime MAX_VALUE + = LocalDateTime.of(2262, 4, 11, 23, 47, 16, 854775807); + + @SerializedName("y") + private long year; + @SerializedName("m") + private long month; + @SerializedName("d") + private long day; + @SerializedName("h") + private long hour; + @SerializedName("M") + private long minute; + @SerializedName("s") + private long second; + @SerializedName("ns") + private long nanosecond; + + public TimeStampNsLiteral() { + type = Type.TIMESTAMP_NS; + nullable = false; + } + + public TimeStampNsLiteral(boolean isMax) { + this(isMax ? MAX_VALUE : MIN_VALUE); + } + + public TimeStampNsLiteral(long year, long month, long day, long hour, long minute, long second, + long nanosecond) { + this(); + this.year = year; + this.month = month; + this.day = day; + this.hour = hour; + this.minute = minute; + this.second = second; + this.nanosecond = nanosecond; + } + + public TimeStampNsLiteral(LocalDateTime value) { + this(value.getYear(), value.getMonthValue(), value.getDayOfMonth(), value.getHour(), + value.getMinute(), value.getSecond(), value.getNano()); + } + + private TimeStampNsLiteral(TimeStampNsLiteral other) { + super(other); + year = other.year; + month = other.month; + day = other.day; + hour = other.hour; + minute = other.minute; + second = other.second; + nanosecond = other.nanosecond; + type = Type.TIMESTAMP_NS; + } + + public static TimeStampNsLiteral createMinValue() { + return new TimeStampNsLiteral(false); + } + + @Override + public Expr clone() { + return new TimeStampNsLiteral(this); + } + + @Override + public boolean isMinValue() { + return compareBoundary(MIN_VALUE) == 0; + } + + @Override + public Object getRealValue() { + LocalDateTime value = toLocalDateTime(); + return BigInteger.valueOf(value.toEpochSecond(ZoneOffset.UTC)) + .multiply(BigInteger.valueOf(NANOSECONDS_PER_SECOND)) + .add(BigInteger.valueOf(nanosecond)) + .longValueExact(); + } + + @Override + public ByteBuffer getHashValue(PrimitiveType primitiveType) { + Preconditions.checkArgument(primitiveType == PrimitiveType.TIMESTAMP_NS, + "Expected TIMESTAMP_NS hash type, but got %s", primitiveType); + ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES).order(ByteOrder.LITTLE_ENDIAN); + buffer.putLong((long) getRealValue()); + return buffer; + } + + @Override + public int compareLiteral(LiteralExpr expr) { + if (expr instanceof PlaceHolderExpr) { + return compareLiteral(((PlaceHolderExpr) expr).getLiteral()); + } + if (expr instanceof NullLiteral) { + return 1; + } + if (expr == MaxLiteral.MAX_VALUE) { + return -1; + } + if (expr instanceof TimeStampNsLiteral) { + return compareDateTime((TimeStampNsLiteral) expr); + } + if (expr instanceof DateLiteral) { + DateLiteral other = (DateLiteral) expr; + int result = compareDateTime(other.getYear(), other.getMonth(), other.getDay(), + other.getHour(), other.getMinute(), other.getSecond(), other.getMicrosecond() * 1000); + if (result == 0 && other.isDateType()) { + return 1; + } + return result; + } + return Integer.signum(getStringValue().compareTo(expr.getStringValue())); + } + + private int compareDateTime(TimeStampNsLiteral other) { + return compareDateTime(other.year, other.month, other.day, other.hour, other.minute, + other.second, other.nanosecond); + } + + private int compareDateTime(long otherYear, long otherMonth, long otherDay, long otherHour, + long otherMinute, long otherSecond, long otherNanosecond) { + int result = Long.compare(year, otherYear); + if (result == 0) { + result = Long.compare(month, otherMonth); + } + if (result == 0) { + result = Long.compare(day, otherDay); + } + if (result == 0) { + result = Long.compare(hour, otherHour); + } + if (result == 0) { + result = Long.compare(minute, otherMinute); + } + if (result == 0) { + result = Long.compare(second, otherSecond); + } + if (result == 0) { + result = Long.compare(nanosecond, otherNanosecond); + } + return result; + } + + @Override + public String getStringValue() { + return String.format("%04d-%02d-%02d %02d:%02d:%02d.%09d", + year, month, day, hour, minute, second, nanosecond); + } + + public void roundFloor(int newScale) { + Preconditions.checkArgument(newScale == ScalarType.TIMESTAMP_NS_SCALE, + "TIMESTAMP_NS has fixed scale %s, but got %s", + ScalarType.TIMESTAMP_NS_SCALE, newScale); + } + + @Override + public void checkValueValid() throws AnalysisException { + if (year < 0 || year > 9999) { + throw new AnalysisException("TimeStampNsLiteral has invalid year value: " + year); + } + if (month < 1 || month > 12) { + throw new AnalysisException("TimeStampNsLiteral has invalid month value: " + month); + } + if (day < 1 || day > daysInMonth(year, month)) { + throw new AnalysisException("TimeStampNsLiteral has invalid day value: " + day); + } + if (hour < 0 || hour > 23) { + throw new AnalysisException("TimeStampNsLiteral has invalid hour value: " + hour); + } + if (minute < 0 || minute > 59) { + throw new AnalysisException("TimeStampNsLiteral has invalid minute value: " + minute); + } + if (second < 0 || second > 59) { + throw new AnalysisException("TimeStampNsLiteral has invalid second value: " + second); + } + if (nanosecond < 0 || nanosecond > MAX_NANOSECOND) { + throw new AnalysisException("TimeStampNsLiteral has invalid nanosecond value: " + nanosecond); + } + if (checkRange()) { + throw new AnalysisException("TimeStampNsLiteral is outside Int64 epoch nanosecond range: " + + getStringValue()); + } + } + + boolean checkRange() { + if (year < 0 || year > 9999 || month < 1 || month > 12 + || hour < 0 || hour > 23 || minute < 0 || minute > 59 + || second < 0 || second > 59) { + return true; + } + if (day < 1 || day > daysInMonth(year, month)) { + return true; + } + if (nanosecond < 0 || nanosecond > MAX_NANOSECOND) { + return true; + } + return compareBoundary(MIN_VALUE) < 0 || compareBoundary(MAX_VALUE) > 0; + } + + private int compareBoundary(LocalDateTime boundary) { + return compareDateTime(boundary.getYear(), boundary.getMonthValue(), boundary.getDayOfMonth(), + boundary.getHour(), boundary.getMinute(), boundary.getSecond(), boundary.getNano()); + } + + private static int daysInMonth(long year, long month) { + switch ((int) month) { + case 2: + return Year.isLeap(year) ? 29 : 28; + case 4: + case 6: + case 9: + case 11: + return 30; + default: + return 31; + } + } + + public LocalDateTime toLocalDateTime() { + try { + return LocalDateTime.of((int) year, (int) month, (int) day, (int) hour, + (int) minute, (int) second, (int) nanosecond); + } catch (DateTimeException e) { + throw new IllegalStateException("Invalid TIMESTAMP_NS literal: " + getStringValue(), e); + } + } + + public TimeStampNsLiteral plusYears(long years) { + return new TimeStampNsLiteral(toLocalDateTime().plusYears(years)); + } + + public TimeStampNsLiteral plusMonths(long months) { + return new TimeStampNsLiteral(toLocalDateTime().plusMonths(months)); + } + + public TimeStampNsLiteral plusDays(long days) { + return new TimeStampNsLiteral(toLocalDateTime().plusDays(days)); + } + + public TimeStampNsLiteral plusHours(long hours) { + return new TimeStampNsLiteral(toLocalDateTime().plusHours(hours)); + } + + public TimeStampNsLiteral plusMinutes(long minutes) { + return new TimeStampNsLiteral(toLocalDateTime().plusMinutes(minutes)); + } + + public TimeStampNsLiteral plusSeconds(long seconds) { + return new TimeStampNsLiteral(toLocalDateTime().plusSeconds(seconds)); + } + + public void setMinValue() { + TimeStampNsLiteral minimum = createMinValue(); + year = minimum.year; + month = minimum.month; + day = minimum.day; + hour = minimum.hour; + minute = minimum.minute; + second = minimum.second; + nanosecond = minimum.nanosecond; + } + + public long getUnixTimestampWithNanoseconds(TimeZone timeZone) { + Instant instant = toLocalDateTime().atZone(timeZone.toZoneId()).toInstant(); + return Math.addExact(Math.multiplyExact(instant.getEpochSecond(), NANOSECONDS_PER_SECOND), + instant.getNano()); + } + + public long getYear() { + return year; + } + + public long getMonth() { + return month; + } + + public long getDay() { + return day; + } + + public long getHour() { + return hour; + } + + public long getMinute() { + return minute; + } + + public long getSecond() { + return second; + } + + public long getMicrosecond() { + return nanosecond / 1000; + } + + public long getNanosecond() { + return nanosecond; + } + + @Override + public long getLongValue() { + return (year * 10000 + month * 100 + day) * 1000000L + hour * 10000 + minute * 100 + second; + } + + @Override + public double getDoubleValue() { + return getLongValue(); + } + + public double getDoubleValueAsDateTime() { + return getDoubleValue(); + } + + @Override + public int hashCode() { + int legacyHash = Long.hashCode(getLongValue()); + if (nanosecond % 1000 == 0) { + return legacyHash; + } + return 31 * legacyHash + Long.hashCode(nanosecond); + } + + @Override + public R accept(ExprVisitor visitor, C context) { + return visitor.visitTimeStampNsLiteral(this, context); + } +} diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/AggregateType.java b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/AggregateType.java index 2bcdc2165f68db..a49644ce505b89 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/AggregateType.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/AggregateType.java @@ -86,6 +86,7 @@ public enum AggregateType { primitiveTypeList.add(PrimitiveType.DATETIME); primitiveTypeList.add(PrimitiveType.DATEV2); primitiveTypeList.add(PrimitiveType.DATETIMEV2); + primitiveTypeList.add(PrimitiveType.TIMESTAMP_NS); primitiveTypeList.add(PrimitiveType.TIMESTAMPTZ); primitiveTypeList.add(PrimitiveType.CHAR); primitiveTypeList.add(PrimitiveType.VARCHAR); @@ -108,6 +109,7 @@ public enum AggregateType { primitiveTypeList.add(PrimitiveType.DATETIME); primitiveTypeList.add(PrimitiveType.DATEV2); primitiveTypeList.add(PrimitiveType.DATETIMEV2); + primitiveTypeList.add(PrimitiveType.TIMESTAMP_NS); primitiveTypeList.add(PrimitiveType.TIMESTAMPTZ); primitiveTypeList.add(PrimitiveType.CHAR); primitiveTypeList.add(PrimitiveType.VARCHAR); diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/ColumnType.java b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/ColumnType.java index 483f22e2425f16..d3e6ef3fbbcf64 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/catalog/ColumnType.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/catalog/ColumnType.java @@ -149,17 +149,26 @@ public abstract class ColumnType { schemaChangeMatrix[PrimitiveType.DATETIME.ordinal()][PrimitiveType.DATEV2.ordinal()] = true; schemaChangeMatrix[PrimitiveType.DATE.ordinal()][PrimitiveType.DATETIMEV2.ordinal()] = true; schemaChangeMatrix[PrimitiveType.DATETIME.ordinal()][PrimitiveType.DATETIMEV2.ordinal()] = true; + schemaChangeMatrix[PrimitiveType.DATE.ordinal()][PrimitiveType.TIMESTAMP_NS.ordinal()] = true; + schemaChangeMatrix[PrimitiveType.DATETIME.ordinal()][PrimitiveType.TIMESTAMP_NS.ordinal()] = true; schemaChangeMatrix[PrimitiveType.DATE.ordinal()][PrimitiveType.DATEV2.ordinal()] = true; schemaChangeMatrix[PrimitiveType.DATETIMEV2.ordinal()][PrimitiveType.DATE.ordinal()] = true; + schemaChangeMatrix[PrimitiveType.TIMESTAMP_NS.ordinal()][PrimitiveType.DATE.ordinal()] = true; schemaChangeMatrix[PrimitiveType.DATEV2.ordinal()][PrimitiveType.DATETIME.ordinal()] = true; schemaChangeMatrix[PrimitiveType.DATETIMEV2.ordinal()][PrimitiveType.DATEV2.ordinal()] = true; + schemaChangeMatrix[PrimitiveType.TIMESTAMP_NS.ordinal()][PrimitiveType.DATEV2.ordinal()] = true; schemaChangeMatrix[PrimitiveType.DATEV2.ordinal()][PrimitiveType.DATETIMEV2.ordinal()] = true; + schemaChangeMatrix[PrimitiveType.DATEV2.ordinal()][PrimitiveType.TIMESTAMP_NS.ordinal()] = true; schemaChangeMatrix[PrimitiveType.DATETIMEV2.ordinal()][PrimitiveType.DATETIME.ordinal()] = true; + schemaChangeMatrix[PrimitiveType.TIMESTAMP_NS.ordinal()][PrimitiveType.DATETIME.ordinal()] = true; schemaChangeMatrix[PrimitiveType.DATEV2.ordinal()][PrimitiveType.DATE.ordinal()] = true; // we should support schema change between different precision schemaChangeMatrix[PrimitiveType.DATETIMEV2.ordinal()][PrimitiveType.DATETIMEV2.ordinal()] = true; + schemaChangeMatrix[PrimitiveType.DATETIMEV2.ordinal()][PrimitiveType.TIMESTAMP_NS.ordinal()] = true; + schemaChangeMatrix[PrimitiveType.TIMESTAMP_NS.ordinal()][PrimitiveType.DATETIMEV2.ordinal()] = true; + schemaChangeMatrix[PrimitiveType.TIMESTAMP_NS.ordinal()][PrimitiveType.TIMESTAMP_NS.ordinal()] = true; // Currently, we do not support schema change between complex types with subtypes. schemaChangeMatrix[PrimitiveType.ARRAY.ordinal()][PrimitiveType.ARRAY.ordinal()] = true; diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/persist/gson/GsonUtilsCatalog.java b/fe/fe-catalog/src/main/java/org/apache/doris/persist/gson/GsonUtilsCatalog.java index ec65e8db39df99..13bee42ee1e91d 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/persist/gson/GsonUtilsCatalog.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/persist/gson/GsonUtilsCatalog.java @@ -54,6 +54,7 @@ import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.StringLiteral; import org.apache.doris.analysis.StructLiteral; +import org.apache.doris.analysis.TimeStampNsLiteral; import org.apache.doris.analysis.TimeV2Literal; import org.apache.doris.analysis.TimestampArithmeticExpr; import org.apache.doris.analysis.TryCastExpr; @@ -131,6 +132,7 @@ public class GsonUtilsCatalog { .registerSubtype(ArrayLiteral.class, ArrayLiteral.class.getSimpleName()) .registerSubtype(BoolLiteral.class, BoolLiteral.class.getSimpleName()) .registerSubtype(DateLiteral.class, DateLiteral.class.getSimpleName()) + .registerSubtype(TimeStampNsLiteral.class, TimeStampNsLiteral.class.getSimpleName()) .registerSubtype(IPv4Literal.class, IPv4Literal.class.getSimpleName()) .registerSubtype(IPv6Literal.class, IPv6Literal.class.getSimpleName()) .registerSubtype(JsonLiteral.class, JsonLiteral.class.getSimpleName()) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java index 597a006707ed2c..f9a475fe78fa63 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java @@ -323,6 +323,7 @@ public static void validateDefaultValue(Type type, String defaultValue, DefaultV switch (primitiveType) { case DATETIME: case DATETIMEV2: + case TIMESTAMP_NS: case TIMESTAMPTZ: break; default: @@ -400,16 +401,19 @@ public static void validateDefaultValue(Type type, String defaultValue, DefaultV break; case DATETIME: case DATETIMEV2: + case TIMESTAMP_NS: case TIMESTAMPTZ: if (defaultValueExprDef == null) { - DateLiteralUtils.createDateLiteral(defaultValue, scalarType); + DateLiteralUtils.createLiteral(defaultValue, scalarType); } else { if (defaultValueExprDef.getExprName().equals(DefaultValue.NOW)) { if (defaultValueExprDef.getPrecision() != null) { Long defaultValuePrecision = defaultValueExprDef.getPrecision(); String typeStr = scalarType.toString(); - int typePrecision = - Integer.parseInt(typeStr.substring(typeStr.indexOf("(") + 1, typeStr.indexOf(")"))); + int typePrecision = scalarType.isTimeStampNs() + ? ScalarType.TIMESTAMP_NS_SCALE + : Integer.parseInt(typeStr.substring( + typeStr.indexOf("(") + 1, typeStr.indexOf(")"))); if (defaultValuePrecision > typePrecision) { typeStr = typeStr.replace("V2", ""); throw new AnalysisException("default value precision: " + defaultValue diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteralUtils.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteralUtils.java index 84d8f2bb0ea22b..709ac3f2912820 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteralUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteralUtils.java @@ -21,6 +21,7 @@ import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.util.TimeUtils; +import org.apache.doris.nereids.util.DateUtils; import com.google.common.base.Preconditions; @@ -37,11 +38,7 @@ import java.util.regex.Pattern; import javax.annotation.Nullable; -/** - * Utility class providing a static factory method for creating {@link DateLiteral} instances - * from string representations. This replicates the parsing logic of the - * {@code DateLiteral(String, Type)} constructor and its {@code init()} method. - */ +/** Utility methods for parsing legacy date-like literals from strings. */ public class DateLiteralUtils { private static final Pattern HAS_OFFSET_PART = Pattern.compile("[\\+\\-]\\d{2}:\\d{2}"); @@ -49,16 +46,22 @@ public class DateLiteralUtils { private DateLiteralUtils() { } - /** - * Creates a {@link DateLiteral} by parsing the given string, replicating the exact behavior - * of the {@code DateLiteral(String, Type)} constructor and its internal {@code init()} method. - * - * @param s the date/datetime string to parse - * @param type the target date type, or {@code null} for auto-detection - * @return a new {@link DateLiteral} instance - * @throws AnalysisException if the string cannot be parsed or the resulting value is out of range - */ + /** Parse a DATE, DATETIME, DATETIMEV2, TIMESTAMPTZ, or TIMESTAMP_NS literal. */ + public static LiteralExpr createLiteral(String s, @Nullable Type type) throws AnalysisException { + return createDateTimeLiteral(s, type); + } + + /** Parse a literal that is represented by {@link DateLiteral}. */ public static DateLiteral createDateLiteral(String s, @Nullable Type type) throws AnalysisException { + LiteralExpr literal = createDateTimeLiteral(s, type); + if (!(literal instanceof DateLiteral)) { + throw new AnalysisException("date literal [" + s + + "] resolves to TIMESTAMP_NS; use DateLiteralUtils.createLiteral instead"); + } + return (DateLiteral) literal; + } + + private static LiteralExpr createDateTimeLiteral(String s, @Nullable Type type) throws AnalysisException { try { if (type != null) { Preconditions.checkArgument(type.isDateType()); @@ -84,6 +87,12 @@ public static DateLiteral createDateLiteral(String s, @Nullable Type type) throw sourceZone = ZoneId.of(tzString); } + int nanosecondGuardDigit = -1; + if (type != null && (type.isDatetimeV2() || type.isTimeStampNs())) { + nanosecondGuardDigit = DateUtils.getNanosecondGuardDigit(s); + s = DateUtils.truncateFractionalSecondForJavaParser(s); + } + if (!s.contains("-")) { // handle format like 20210106, but should not handle 2021-1-6 for (DateTimeFormatter formatter : DateLiteral.formatterList) { @@ -140,7 +149,8 @@ public static DateLiteral createDateLiteral(String s, @Nullable Type type) throw throw new AnalysisException("Invalid date value: " + s); } if (timePart.length == 0 && type != null - && (type.equals(Type.DATETIME) || type.equals(Type.DATETIMEV2))) { + && (type.equals(Type.DATETIME) || type.equals(Type.DATETIMEV2) + || type.equals(Type.TIMESTAMP_NS))) { throw new AnalysisException("Invalid datetime value: " + s); } for (int i = 0; i < timePart.length; i++) { @@ -159,7 +169,7 @@ public static DateLiteral createDateLiteral(String s, @Nullable Type type) throw ? timePart[i].split("\\.")[0].length() : timePart[i].length(), "s"))); if (timePart[i].contains(".")) { - builder.appendFraction(ChronoField.MICRO_OF_SECOND, 0, 6, true); + builder.appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true); } break; default: @@ -183,29 +193,32 @@ public static DateLiteral createDateLiteral(String s, @Nullable Type type) throw long hour = getOrDefault(dateTime, ChronoField.HOUR_OF_DAY, 0); long minute = getOrDefault(dateTime, ChronoField.MINUTE_OF_HOUR, 0); long second = getOrDefault(dateTime, ChronoField.SECOND_OF_MINUTE, 0); - long microsecond = getOrDefault(dateTime, ChronoField.MICRO_OF_SECOND, 0); + long nanosecond = getOrDefault(dateTime, ChronoField.NANO_OF_SECOND, 0); + long microsecond = nanosecond / 1000; if (type != null) { - if (microsecond != 0 && type.isDatetime()) { + if (nanosecond != 0 && type.isDatetime()) { int dotIndex = s.lastIndexOf("."); int scale = s.length() - dotIndex - 1; - type = ScalarType.createDatetimeV2Type(scale); + type = scale > ScalarType.MAX_DATETIMEV2_SCALE + ? ScalarType.createTimeStampNsType() + : ScalarType.createDatetimeV2Type(scale); } } else { - if (hour == 0 && minute == 0 && second == 0 && microsecond == 0) { + if (hour == 0 && minute == 0 && second == 0 && nanosecond == 0) { type = ScalarType.getDefaultDateType(Type.DATE); } else { type = ScalarType.getDefaultDateType(Type.DATETIME); - if (type.isDatetimeV2() && microsecond != 0) { - int scale = 6; - for (int i = 0; i < 6; i++) { - if (microsecond % Math.pow(10.0, i + 1) > 0) { - break; - } else { - scale -= 1; - } + if (type.isDatetimeV2() && nanosecond != 0) { + int scale = ScalarType.TIMESTAMP_NS_SCALE; + long fractionalSecond = nanosecond; + while (fractionalSecond % 10 == 0) { + fractionalSecond /= 10; + scale--; } - type = ScalarType.createDatetimeV2Type(scale); + type = scale > ScalarType.MAX_DATETIMEV2_SCALE + ? ScalarType.createTimeStampNsType() + : ScalarType.createDatetimeV2Type(scale); } } } @@ -230,7 +243,8 @@ public static DateLiteral createDateLiteral(String s, @Nullable Type type) throw } LocalDateTime parsedLdt = LocalDateTime.of( (int) year, (int) month, (int) day, - (int) hour, (int) minute, (int) second); + (int) hour, (int) minute, (int) second, + (int) nanosecond); Instant targetInstant = parsedLdt.atZone(sourceZone).toInstant(); LocalDateTime destLdt = LocalDateTime.ofInstant(targetInstant, dorisZone); year = destLdt.getYear(); @@ -243,11 +257,31 @@ public static DateLiteral createDateLiteral(String s, @Nullable Type type) throw // the original init() behavior (not affected by zone conversion) } - // Construct DateLiteral using the appropriate constructor based on the determined type + if (type.isDatetimeV2() || type.isTimeStampNs()) { + LocalDateTime roundedDateTime = roundFractionalSecond( + year, month, day, hour, minute, second, nanosecond, + type.isTimeStampNs() ? 9 : ((ScalarType) type).getScalarScale(), nanosecondGuardDigit); + year = roundedDateTime.getYear(); + month = roundedDateTime.getMonthValue(); + day = roundedDateTime.getDayOfMonth(); + hour = roundedDateTime.getHour(); + minute = roundedDateTime.getMinute(); + second = roundedDateTime.getSecond(); + nanosecond = roundedDateTime.getNano(); + microsecond = nanosecond / 1000; + } + + if (type.isTimeStampNs()) { + TimeStampNsLiteral result = new TimeStampNsLiteral( + year, month, day, hour, minute, second, nanosecond); + result.checkValueValid(); + return result; + } + DateLiteral result; if (type.isDate() || type.isDateV2()) { result = new DateLiteral(year, month, day, type); - } else if (microsecond != 0 && (type.isDatetimeV2() || type.isTimeStampTz())) { + } else if (nanosecond != 0 && (type.isDatetimeV2() || type.isTimeStampTz())) { result = new DateLiteral(year, month, day, hour, minute, second, microsecond, type); } else { result = new DateLiteral(year, month, day, hour, minute, second, type); @@ -256,6 +290,7 @@ public static DateLiteral createDateLiteral(String s, @Nullable Type type) throw if (result.checkRange() || result.checkDate()) { throw new AnalysisException("Datetime value is out of range"); } + result.checkValueValid(); return result; } catch (Exception ex) { throw new AnalysisException("date literal [" + s + "] is invalid: " + ex.getMessage()); @@ -266,6 +301,20 @@ private static int getOrDefault(TemporalAccessor accessor, ChronoField field, in return accessor.isSupported(field) ? accessor.get(field) : defaultValue; } + private static LocalDateTime roundFractionalSecond(long year, long month, long day, long hour, + long minute, long second, long nanosecond, int scale, int nanosecondGuardDigit) { + long factor = (long) Math.pow(10, ScalarType.TIMESTAMP_NS_SCALE - scale); + long roundingOffset = scale == ScalarType.TIMESTAMP_NS_SCALE && nanosecondGuardDigit >= 5 + ? 1 : factor / 2; + long roundedNanosecond = (nanosecond + roundingOffset) / factor * factor; + LocalDateTime result = LocalDateTime.of( + (int) year, (int) month, (int) day, (int) hour, (int) minute, (int) second); + if (roundedNanosecond == 1000000000L) { + return result.plusSeconds(1); + } + return result.withNano((int) roundedNanosecond); + } + private static boolean haveTimeZoneOffset(String arg) { Preconditions.checkArgument(arg.length() > 6); return HAS_OFFSET_PART.matcher(arg.substring(arg.length() - 6)).matches(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToSqlVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToSqlVisitor.java index 3bbbbc902cc9c0..003fca86bfd959 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToSqlVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToSqlVisitor.java @@ -86,6 +86,11 @@ public String visitDateLiteral(DateLiteral expr, ToSqlParams context) { return "'" + expr.getStringValue() + "'"; } + @Override + public String visitTimeStampNsLiteral(TimeStampNsLiteral expr, ToSqlParams context) { + return "'" + expr.getStringValue() + "'"; + } + @Override public String visitTimeV2Literal(TimeV2Literal expr, ToSqlParams context) { return "\"" + expr.getStringValue() + "\""; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToStringValueVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToStringValueVisitor.java index 0c59cf99f5b068..46fe332ccbcff9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToStringValueVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToStringValueVisitor.java @@ -105,6 +105,14 @@ public String visitDateLiteral(DateLiteral expr, StringValueContext ctx) { return value; } + @Override + public String visitTimeStampNsLiteral(TimeStampNsLiteral expr, StringValueContext ctx) { + if (ctx.isInComplexType()) { + return wrapWithQuotes(expr.getStringValue(), ctx); + } + return expr.getStringValue(); + } + @Override public String visitFloatLiteral(FloatLiteral expr, StringValueContext ctx) { String value; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToThriftVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToThriftVisitor.java index 3df252e8fed56b..3f1521ff376459 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToThriftVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprToThriftVisitor.java @@ -202,6 +202,18 @@ public Void visitDateLiteral(DateLiteral expr, TExprNode msg) { return null; } + @Override + public Void visitTimeStampNsLiteral(TimeStampNsLiteral expr, TExprNode msg) { + msg.node_type = TExprNodeType.DATE_LITERAL; + msg.date_literal = new TDateLiteral(expr.getStringValue()); + try { + expr.checkValueValid(); + } catch (AnalysisException e) { + LOG.warn("meet invalid value when plan to translate " + expr + " to thrift node"); + } + return null; + } + @Override public Void visitTimeV2Literal(TimeV2Literal expr, TExprNode msg) { msg.node_type = TExprNodeType.TIMEV2_LITERAL; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExprUtils.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExprUtils.java index b9a5d392a349ac..2e0d833f652179 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExprUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExprUtils.java @@ -68,8 +68,9 @@ public static LiteralExpr createLiteral(String value, Type type) throws Analysis case DATETIME: case DATEV2: case DATETIMEV2: + case TIMESTAMP_NS: case TIMESTAMPTZ: - literalExpr = DateLiteralUtils.createDateLiteral(value, type); + literalExpr = DateLiteralUtils.createLiteral(value, type); break; case IPV4: literalExpr = new IPv4Literal(value); @@ -104,6 +105,8 @@ public static LiteralExpr createInfinity(Type type, boolean isMax) throws Analys case DATETIMEV2: case TIMESTAMPTZ: return DateLiteral.createMinValue(type); + case TIMESTAMP_NS: + return TimeStampNsLiteral.createMinValue(); default: throw new AnalysisException("Invalid data type for creating infinity: " + type); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionExprUtil.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionExprUtil.java index 43e1b53badc91c..96421206125a1a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionExprUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionExprUtil.java @@ -36,6 +36,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; @@ -100,6 +101,11 @@ public static FunctionIntervalInfo getFunctionIntervalInfo(ArrayList parti public static DateLiteral getRangeEnd(DateLiteral beginTime, FunctionIntervalInfo intervalInfo) throws AnalysisException { + LocalDateTime endTime = getRangeEnd(beginTime.getTimeFormatter(), intervalInfo); + return endTime == null ? null : new DateLiteral(endTime, beginTime.getType()); + } + + private static LocalDateTime getRangeEnd(LocalDateTime beginTime, FunctionIntervalInfo intervalInfo) { String timeUnit = intervalInfo.timeUnit; long interval = intervalInfo.interval; switch (timeUnit) { @@ -161,11 +167,18 @@ public static Map getAddPartitionClauseFromPartitionValu if (partitionType == PartitionType.RANGE) { String beginTime = curPartitionValues.get(0); // have check range type size must be 1 Type partitionColumnType = partitionColumn.get(0).getType(); - DateLiteral beginDateTime = DateLiteralUtils.createDateLiteral(beginTime, partitionColumnType); + LiteralExpr beginDateTime = DateLiteralUtils.createLiteral(beginTime, partitionColumnType); + LocalDateTime beginLocalDateTime = beginDateTime instanceof DateLiteral + ? ((DateLiteral) beginDateTime).getTimeFormatter() + : ((TimeStampNsLiteral) beginDateTime).toLocalDateTime(); partitionName += String.format(DATETIME_NAME_FORMATTER, - beginDateTime.getYear(), beginDateTime.getMonth(), beginDateTime.getDay(), - beginDateTime.getHour(), beginDateTime.getMinute(), beginDateTime.getSecond()); - DateLiteral endDateTime = getRangeEnd(beginDateTime, intervalInfo); + beginLocalDateTime.getYear(), beginLocalDateTime.getMonthValue(), + beginLocalDateTime.getDayOfMonth(), beginLocalDateTime.getHour(), + beginLocalDateTime.getMinute(), beginLocalDateTime.getSecond()); + LocalDateTime endLocalDateTime = getRangeEnd(beginLocalDateTime, intervalInfo); + LiteralExpr endDateTime = beginDateTime instanceof DateLiteral + ? new DateLiteral(endLocalDateTime, beginDateTime.getType()) + : new TimeStampNsLiteral(endLocalDateTime); partitionKeyDesc = createPartitionKeyDescWithRange(beginDateTime, endDateTime, partitionColumnType); } else if (partitionType == PartitionType.LIST) { List> listValues = new ArrayList<>(); @@ -232,8 +245,8 @@ public static Map getAddPartitionClauseFromPartitionValu return result; } - private static PartitionKeyDesc createPartitionKeyDescWithRange(DateLiteral beginDateTime, - DateLiteral endDateTime, Type partitionColumnType) throws AnalysisException { + private static PartitionKeyDesc createPartitionKeyDescWithRange(LiteralExpr beginDateTime, + LiteralExpr endDateTime, Type partitionColumnType) throws AnalysisException { PartitionValue lowerValue = getPartitionFromDate(partitionColumnType, beginDateTime); PartitionValue upperValue = getPartitionFromDate(partitionColumnType, endDateTime); return PartitionKeyDesc.createFixed( @@ -241,7 +254,7 @@ private static PartitionKeyDesc createPartitionKeyDescWithRange(DateLiteral begi Collections.singletonList(upperValue)); } - private static PartitionValue getPartitionFromDate(Type partitionColumnType, DateLiteral dateLiteral) + private static PartitionValue getPartitionFromDate(Type partitionColumnType, LiteralExpr dateLiteral) throws AnalysisException { // check out of range. try { @@ -252,15 +265,19 @@ private static PartitionValue getPartitionFromDate(Type partitionColumnType, Dat return PartitionValue.MAX_VALUE; } + LocalDateTime dateTime = dateLiteral instanceof DateLiteral + ? ((DateLiteral) dateLiteral).getTimeFormatter() + : ((TimeStampNsLiteral) dateLiteral).toLocalDateTime(); String timeString; if (partitionColumnType.isDate() || partitionColumnType.isDateV2()) { - timeString = String.format(DATE_FORMATTER, dateLiteral.getYear(), dateLiteral.getMonth(), - dateLiteral.getDay()); + timeString = String.format(DATE_FORMATTER, dateTime.getYear(), dateTime.getMonthValue(), + dateTime.getDayOfMonth()); } else if (partitionColumnType.isDatetime() || partitionColumnType.isDatetimeV2() + || partitionColumnType.isTimeStampNs() || partitionColumnType.isTimeStampTz()) { timeString = String.format(DATETIME_FORMATTER, - dateLiteral.getYear(), dateLiteral.getMonth(), dateLiteral.getDay(), - dateLiteral.getHour(), dateLiteral.getMinute(), dateLiteral.getSecond()); + dateTime.getYear(), dateTime.getMonthValue(), dateTime.getDayOfMonth(), + dateTime.getHour(), dateTime.getMinute(), dateTime.getSecond()); } else { throw new AnalysisException( "not support range partition with column type : " + partitionColumnType.toString()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnToProtobuf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnToProtobuf.java index d9c17012a64d28..aa0c89a85bd25c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnToProtobuf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnToProtobuf.java @@ -153,6 +153,7 @@ static int getFieldLengthByType(PrimitiveType type, int stringLength) throws Ddl case DATETIME: return 8; case DATETIMEV2: + case TIMESTAMP_NS: case TIMESTAMPTZ: return 8; case FLOAT: diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java index f23954280f3a65..ab6f610003a74f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java @@ -24,6 +24,7 @@ import org.apache.doris.analysis.PartitionDesc; import org.apache.doris.analysis.PartitionValue; import org.apache.doris.analysis.SinglePartitionDesc; +import org.apache.doris.analysis.TimeStampNsLiteral; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; @@ -398,7 +399,7 @@ public static List toPartitionValue(PartitionKey partitionKey) { return partitionKey.getKeys().stream().map(expr -> { if (expr == MaxLiteral.MAX_VALUE) { return PartitionValue.MAX_VALUE; - } else if (expr instanceof DateLiteral) { + } else if (expr instanceof DateLiteral || expr instanceof TimeStampNsLiteral) { return new PartitionValue(expr.getStringValue()); } else if (expr instanceof NullLiteral) { return new PartitionValue("NULL", true); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java index 3832876cd84401..db6ccea6564773 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java @@ -34,8 +34,10 @@ import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral; import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal; import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.expressions.literal.TimeStampNsLiteral; import org.apache.doris.nereids.trees.expressions.literal.TimestampTzLiteral; import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.DateTimeV2Type; import org.apache.doris.nereids.types.TimeStampTzType; import org.apache.doris.persist.gson.GsonUtils; @@ -118,7 +120,8 @@ public static PartitionKey createPartitionKey(List keys, List keys) { value = expr.accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITH_TABLE); } else { value = expr.getRealValue(); - if (expr instanceof DateLiteral) { - DateLiteral dateLiteral = (DateLiteral) expr; - value = dateLiteral.getStringValue(); + if (expr instanceof DateLiteral || expr instanceof org.apache.doris.analysis.TimeStampNsLiteral) { + value = expr.getStringValue(); } } if (keys.size() - 1 == i) { @@ -571,7 +581,11 @@ public PartitionKey deserialize(JsonElement json, java.lang.reflect.Type typeOfT LOG.warn("Value {} for partition key [type = {}] is invalid! This is a bug exists " + "in Doris 1.2.0 and fixed since Doris 1.2.1. You should create this table " + "again using Doris 1.2.1+ .", key.getStringValue(), type); - ((DateLiteral) key).setMinValue(); + if (key instanceof org.apache.doris.analysis.TimeStampNsLiteral) { + ((org.apache.doris.analysis.TimeStampNsLiteral) key).setMinValue(); + } else { + ((DateLiteral) key).setMinValue(); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/BuildIndexProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/BuildIndexProcDir.java index a0552af614d94e..2f2ca45c2f562d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/BuildIndexProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/BuildIndexProcDir.java @@ -23,6 +23,7 @@ import org.apache.doris.analysis.DateLiteralUtils; import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.LimitElement; +import org.apache.doris.analysis.LiteralExpr; import org.apache.doris.analysis.StringLiteral; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Type; @@ -88,7 +89,8 @@ boolean filterResult(String columnName, Comparable element, HashMap tablets) { typeMap.put("DATE", Type.DATE); typeMap.put("DATEV2", Type.DATEV2); typeMap.put("DATETIMEV2", Type.DATETIMEV2); + typeMap.put("TIMESTAMP_NS", Type.TIMESTAMP_NS); typeMap.put("DATETIME", Type.DATETIME); typeMap.put("DECIMAL32", Type.DECIMAL32); typeMap.put("DECIMAL64", Type.DECIMAL64); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index 0b56c1a244dda5..308c0b0bb9fdbb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -410,8 +410,10 @@ public static DataProperty analyzeDataProperty(Map properties, f } } else if (key.equalsIgnoreCase(PROPERTIES_STORAGE_COOLDOWN_TIME)) { try { + // Cooldown timestamps are stored in milliseconds, so DATETIMEV2(6) preserves their + // fractional part without imposing the narrower TIMESTAMP_NS epoch range. DateLiteral dateLiteral = DateLiteralUtils.createDateLiteral(value, - ScalarType.getDefaultDateType(Type.DATETIME)); + ScalarType.createDatetimeV2Type(6)); cooldownTimestamp = dateLiteral.unixTimestamp(TimeUtils.getTimeZone()); } catch (AnalysisException e) { LOG.warn("dateLiteral failed, use max cool down time", e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java index 0a4e8f967e310e..2f706a4c660492 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java @@ -78,6 +78,7 @@ public class Util { TYPE_STRING_MAP.put(PrimitiveType.DATETIME, "datetime"); TYPE_STRING_MAP.put(PrimitiveType.DATEV2, "datev2"); TYPE_STRING_MAP.put(PrimitiveType.DATETIMEV2, "datetimev2"); + TYPE_STRING_MAP.put(PrimitiveType.TIMESTAMP_NS, "timestamp_ns"); TYPE_STRING_MAP.put(PrimitiveType.TIMESTAMPTZ, "timestamptz"); TYPE_STRING_MAP.put(PrimitiveType.CHAR, "char(%d)"); TYPE_STRING_MAP.put(PrimitiveType.VARCHAR, "varchar(%d)"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/connector/converter/ConnectorColumnConverter.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/connector/converter/ConnectorColumnConverter.java index 62598eec708bef..e90209ecb06031 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/connector/converter/ConnectorColumnConverter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/connector/converter/ConnectorColumnConverter.java @@ -359,6 +359,8 @@ private static Type convertScalarType(String typeName, int precision, int scale) return ScalarType.createDatetimeV2Type(precision); } return ScalarType.DATETIMEV2; + case "TIMESTAMP_NS": + return ScalarType.createTimeStampNsType(); case "TIMESTAMPTZ": if (precision >= 0) { return ScalarType.createTimeStampTzType(precision); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java index bcdc9ccc03bd69..e6b8b56f91bf9d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java @@ -450,11 +450,13 @@ private Type dorisTypeToDoris(JdbcFieldSchema fieldSchema) { case "DATETIMEV2": { int scale = (openParen == -1) ? 0 : Integer.parseInt(upperType.substring(openParen + 1, upperType.length() - 1)); - if (scale > 6) { - scale = 6; + if (scale > ScalarType.MAX_DATETIMEV2_SCALE) { + scale = ScalarType.MAX_DATETIMEV2_SCALE; } return ScalarType.createDatetimeV2Type(scale); } + case "TIMESTAMP_NS": + return ScalarType.createTimeStampNsType(); case "CHAR": case "CHARACTER": return ScalarType.createCharType(fieldSchema.requiredColumnSize()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/StatementSubmitter.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/StatementSubmitter.java index 60b44ec9384bfb..03c85ba771e2fd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/StatementSubmitter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/StatementSubmitter.java @@ -225,7 +225,8 @@ private ExecutionResultSet generateResultSet(ResultSet rs, long startTime, boole for (int i = 1; i <= colNum && (!isCopyStmt || i <= copyResult.length); ++i) { String type = rs.getMetaData().getColumnTypeName(i); if ("DATE".equalsIgnoreCase(type) || "DATETIME".equalsIgnoreCase(type) - || "DATEV2".equalsIgnoreCase(type) || "DATETIMEV2".equalsIgnoreCase(type)) { + || "DATEV2".equalsIgnoreCase(type) || "DATETIMEV2".equalsIgnoreCase(type) + || "TIMESTAMP_NS".equalsIgnoreCase(type)) { row.add(rs.getString(i)); } else { row.add(rs.getObject(i)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/streamresponse/JsonStreamResponse.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/streamresponse/JsonStreamResponse.java index ba77589b710d27..61afcffdcffde6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/streamresponse/JsonStreamResponse.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/util/streamresponse/JsonStreamResponse.java @@ -154,7 +154,8 @@ private void writeResultSetData(ResultSet rs, JsonWriter jsonWriter, long startT for (int i = 1; i <= colNum; ++i) { String type = rs.getMetaData().getColumnTypeName(i); if ("DATE".equalsIgnoreCase(type) || "DATETIME".equalsIgnoreCase(type) - || "DATEV2".equalsIgnoreCase(type) || "DATETIMEV2".equalsIgnoreCase(type)) { + || "DATEV2".equalsIgnoreCase(type) || "DATETIMEV2".equalsIgnoreCase(type) + || "TIMESTAMP_NS".equalsIgnoreCase(type)) { row.add(rs.getString(i)); } else { row.add(rs.getObject(i)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java index 6a23cabdb93e0a..f563c34f25659b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java @@ -247,7 +247,8 @@ private String dateTimeToStr(DateTimeV2Literal literal, if (partitionColumnType.isDate() || partitionColumnType.isDateV2()) { return String.format(PartitionExprUtil.DATE_FORMATTER, literal.getYear(), literal.getMonth(), literal.getDay()); - } else if (partitionColumnType.isDatetime() || partitionColumnType.isDatetimeV2()) { + } else if (partitionColumnType.isDatetime() || partitionColumnType.isDatetimeV2() + || partitionColumnType.isTimeStampNs()) { return String.format(PartitionExprUtil.DATETIME_FORMATTER, literal.getYear(), literal.getMonth(), literal.getDay(), literal.getHour(), literal.getMinute(), literal.getSecond()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlSerializer.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlSerializer.java index 4469830f320eb3..e67f843acefcdc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlSerializer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlSerializer.java @@ -18,6 +18,7 @@ package org.apache.doris.mysql; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.MysqlColType; import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; @@ -187,7 +188,7 @@ public void writeField(FieldInfo fieldInfo, Type type) { // Column length: four byte integer writeInt4(getMysqlTypeLength(type)); // Column type: one byte integer - writeInt1(type.getPrimitiveType().toMysqlType().getCode()); + writeInt1(getMysqlResultType(type).getCode()); // Flags: two byte integer writeInt2(getMysqlFlags(type)); // Decimals: one byte integer @@ -216,7 +217,7 @@ public void writeField(String db, String table, Column column, boolean sendDefau // TODO(zhaochun): fix Column length: four byte integer writeInt4(getMysqlTypeLength(column.getType())); // Column type: one byte integer - writeInt1(column.getDataType().toMysqlType().getCode()); + writeInt1(getMysqlResultType(column.getType()).getCode()); // Flags: two byte integer writeInt2(getMysqlFlags(column.getType())); // Decimals: one byte integer @@ -251,7 +252,7 @@ public void writeField(String colName, Type type) { // Column length: four byte integer writeInt4(getMysqlTypeLength(type)); // Column type: one byte integer - writeInt1(type.getPrimitiveType().toMysqlType().getCode()); + writeInt1(getMysqlResultType(type).getCode()); // Flags: two byte integer writeInt2(getMysqlFlags(type)); // Decimals: one byte integer @@ -268,7 +269,7 @@ public void writeField(String colName, Type type) { * @param type * @return */ - private int getMysqlTypeLength(Type type) { + int getMysqlTypeLength(Type type) { switch (type.getPrimitiveType()) { // MySQL use Tinyint(1) to represent boolean case BOOLEAN: @@ -291,6 +292,9 @@ private int getMysqlTypeLength(Type type) { case TIMESTAMPTZ: // yyyy-MM-dd HH:mm:ss[.ffffff]+HH:mm return 32; + case TIMESTAMP_NS: + // yyyy-MM-dd HH:mm:ss.nnnnnnnnn + return 29; case DATETIME: case DATETIMEV2: { if (type.getPrimitiveType().isTimeType()) { @@ -332,6 +336,9 @@ private int getMysqlTypeLength(Type type) { // this is used for decimal scale public int getMysqlDecimals(Type type) { + if (isTimestampNs(type)) { + return 0; + } switch (type.getPrimitiveType()) { case DECIMALV2: case DECIMAL32: @@ -349,6 +356,17 @@ public int getMysqlDecimals(Type type) { } } + private MysqlColType getMysqlResultType(Type type) { + // MySQL's binary temporal value contains at most microseconds. TIMESTAMP_NS is + // serialized by BE as a length-encoded string so the metadata must advertise a string too. + return isTimestampNs(type) ? MysqlColType.MYSQL_TYPE_STRING + : type.getPrimitiveType().toMysqlType(); + } + + private boolean isTimestampNs(Type type) { + return type.isTimeStampNs(); + } + // see https://github.com/mysql/mysql-server/blob/trunk/include/mysql_com.h#L161 private int getMysqlFlags(Type type) { int flags = 0; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/check/CheckCast.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/check/CheckCast.java index b6b3127994682d..3f92c072afbb87 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/check/CheckCast.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/check/CheckCast.java @@ -50,6 +50,7 @@ import org.apache.doris.nereids.types.StringType; import org.apache.doris.nereids.types.StructField; import org.apache.doris.nereids.types.StructType; +import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TimeStampTzType; import org.apache.doris.nereids.types.TimeV2Type; import org.apache.doris.nereids.types.TinyIntType; @@ -150,6 +151,11 @@ public class CheckCast implements ExpressionPatternRuleFactory { allowedTypes.add(TimeStampTzType.class); strictCastWhiteList.put(DateTimeV2Type.class, allowedTypes); + // TimestampNs + allowedTypes = Sets.newHashSet(); + allowToStringLikeType(allowedTypes); + strictCastWhiteList.put(TimeStampNsType.class, allowedTypes); + // timestamp tz allowedTypes = Sets.newHashSet(); allowedTypes.add(DateTimeV2Type.class); @@ -180,6 +186,7 @@ public class CheckCast implements ExpressionPatternRuleFactory { allowedTypes.add(IPv6Type.class); allowedTypes.add(VarBinaryType.class); allowedTypes.add(TimeStampTzType.class); + allowedTypes.add(TimeStampNsType.class); allowToComplexType(allowedTypes); allowedTypes.remove(HllType.class); allowedTypes.remove(BitmapType.class); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java index 3f6812405e864b..96e0e09f987afe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java @@ -68,6 +68,7 @@ import org.apache.doris.nereids.trees.expressions.literal.SmallIntLiteral; import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; import org.apache.doris.nereids.trees.expressions.literal.StructLiteral; +import org.apache.doris.nereids.trees.expressions.literal.TimeStampNsLiteral; import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.DataType; @@ -76,6 +77,7 @@ import org.apache.doris.nereids.types.MapType; import org.apache.doris.nereids.types.StructField; import org.apache.doris.nereids.types.StructType; +import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.proto.InternalService; import org.apache.doris.proto.InternalService.PConstantExprResult; import org.apache.doris.proto.Types.PScalarType; @@ -107,6 +109,7 @@ import java.time.DateTimeException; import java.time.LocalDate; import java.time.LocalDateTime; +import java.time.ZoneOffset; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -125,6 +128,7 @@ public class FoldConstantRuleOnBE implements ExpressionPatternRuleFactory { public static final FoldConstantRuleOnBE INSTANCE = new FoldConstantRuleOnBE(); private static final Logger LOG = LogManager.getLogger(FoldConstantRuleOnBE.class); + private static final long NANOS_PER_SECOND = 1_000_000_000L; @Override public List> buildRules() { @@ -472,17 +476,26 @@ public static List getResultExpression(DataType type, PValues resultCon res.add(literal); } } else if (type.isDateTimeV2Type()) { - int num = resultContent.getUint64ValueCount(); - for (int i = 0; i < num; ++i) { - long uint64Value = resultContent.getUint64Value(i); - LocalDateTime dateTimeV2 = convertToJavaDateTimeV2(uint64Value); - if (dateTimeV2 == null && resultContent.hasHasNull()) { - res.add(new NullLiteral(type)); - } else { - Literal literal = new DateTimeV2Literal((DateTimeV2Type) type, dateTimeV2.getYear(), - dateTimeV2.getMonthValue(), dateTimeV2.getDayOfMonth(), dateTimeV2.getHour(), - dateTimeV2.getMinute(), dateTimeV2.getSecond(), dateTimeV2.getNano() / 1000); - res.add(literal); + DateTimeV2Type dateTimeV2Type = (DateTimeV2Type) type; + if (dateTimeV2Type instanceof TimeStampNsType) { + // TIMESTAMP_NS is sent as signed epoch nanoseconds, unlike the packed uint64 civil + // layout of DATETIMEV2. + int num = resultContent.getInt64ValueCount(); + for (int i = 0; i < num; ++i) { + LocalDateTime dateTimeV2 = convertEpochNanosToJavaDateTimeV2( + resultContent.getInt64Value(i)); + res.add(createDateTimeLikeLiteral(dateTimeV2Type, dateTimeV2)); + } + } else { + int num = resultContent.getUint64ValueCount(); + for (int i = 0; i < num; ++i) { + long uint64Value = resultContent.getUint64Value(i); + LocalDateTime dateTimeV2 = convertToJavaDateTimeV2(uint64Value); + if (dateTimeV2 == null && resultContent.hasHasNull()) { + res.add(new NullLiteral(type)); + } else { + res.add(createDateTimeLikeLiteral(dateTimeV2Type, dateTimeV2)); + } } } } else if (type.isDateV2Type()) { @@ -688,6 +701,22 @@ private static LocalDateTime convertToJavaDateTimeV2(long time) { } } + private static LocalDateTime convertEpochNanosToJavaDateTimeV2(long epochNanos) { + long epochSecond = Math.floorDiv(epochNanos, NANOS_PER_SECOND); + int nanoOfSecond = (int) Math.floorMod(epochNanos, NANOS_PER_SECOND); + return LocalDateTime.ofEpochSecond(epochSecond, nanoOfSecond, ZoneOffset.UTC); + } + + private static Literal createDateTimeLikeLiteral( + DateTimeV2Type type, LocalDateTime dateTime) { + if (type instanceof TimeStampNsType) { + return TimeStampNsLiteral.fromJavaDateType(dateTime); + } + return DateTimeV2Literal.create(type, dateTime.getYear(), dateTime.getMonthValue(), + dateTime.getDayOfMonth(), dateTime.getHour(), dateTime.getMinute(), + dateTime.getSecond(), dateTime.getNano() / 1000); + } + private static LocalDate convertToJavaDateV2(int date) { int year = date >> 9; int month = (date >> 5) & 0XF; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java index 5338bf7df9b2ef..97911650afb0a2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java @@ -82,8 +82,6 @@ import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; import org.apache.doris.nereids.trees.expressions.literal.ComparableLiteral; import org.apache.doris.nereids.trees.expressions.literal.DateLiteral; -import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral; -import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal; import org.apache.doris.nereids.trees.expressions.literal.DateV2Literal; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; @@ -702,10 +700,10 @@ public Expression visitDate(Date date, ExpressionRewriteContext context) { } DataType dataType = child.getDataType(); if (dataType.isDateTimeType()) { - DateTimeLiteral dateTimeLiteral = (DateTimeLiteral) child; + DateLiteral dateTimeLiteral = (DateLiteral) child; return new DateLiteral(dateTimeLiteral.getYear(), dateTimeLiteral.getMonth(), dateTimeLiteral.getDay()); } else if (dataType.isDateTimeV2Type()) { - DateTimeV2Literal dateTimeLiteral = (DateTimeV2Literal) child; + DateLiteral dateTimeLiteral = (DateLiteral) child; return new DateV2Literal(dateTimeLiteral.getYear(), dateTimeLiteral.getMonth(), dateTimeLiteral.getDay()); } return date; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicate.java index 335fe78307180f..03c61517e86f4c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicate.java @@ -48,6 +48,7 @@ import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.literal.NumericLiteral; import org.apache.doris.nereids.trees.expressions.literal.SmallIntLiteral; +import org.apache.doris.nereids.trees.expressions.literal.TimeStampNsLiteral; import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.DataType; @@ -58,6 +59,7 @@ import org.apache.doris.nereids.types.DecimalV3Type; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.SmallIntType; +import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TinyIntType; import org.apache.doris.nereids.types.coercion.DateLikeType; import org.apache.doris.nereids.types.coercion.IntegralType; @@ -183,6 +185,10 @@ private static Expression processDateLikeTypeCoercion(ComparisonPredicate cp, Ex if (cast.child().getDataType() instanceof DateTimeType || cast.child().getDataType() instanceof DateTimeV2Type) { // right is datetime + if (right instanceof TimeStampNsLiteral) { + return processDateTimeLikeComparisonPredicateTimeStampNsLiteral( + cp, cast.child(), (TimeStampNsLiteral) right); + } if (right instanceof DateTimeV2Literal) { return processDateTimeLikeComparisonPredicateDateTimeV2Literal( cp, cast.child(), (DateTimeV2Literal) right); @@ -203,6 +209,37 @@ private static Expression processDateLikeTypeCoercion(ComparisonPredicate cp, Ex return cp; } + private static Expression processDateTimeLikeComparisonPredicateTimeStampNsLiteral( + ComparisonPredicate comparisonPredicate, Expression left, TimeStampNsLiteral right) { + DataType leftType = left.getDataType(); + if (!(leftType instanceof DateTimeType) && !(leftType instanceof DateTimeV2Type)) { + return comparisonPredicate; + } + int toScale = leftType instanceof DateTimeV2Type ? ((DateTimeV2Type) leftType).getScale() : 0; + if (toScale >= TimeStampNsType.SCALE) { + return comparisonPredicate; + } + + DateTimeV2Literal rounded; + if (comparisonPredicate instanceof EqualTo || comparisonPredicate instanceof NullSafeEqual) { + rounded = right.roundFloorToDateTimeV2(toScale); + if (rounded.getNanoSecond() != right.getNanoSecond()) { + return comparisonPredicate instanceof NullSafeEqual + ? BooleanLiteral.FALSE : ExpressionUtils.falseOrNull(left); + } + } else if (comparisonPredicate instanceof GreaterThan + || comparisonPredicate instanceof LessThanEqual) { + rounded = right.roundFloorToDateTimeV2(toScale); + } else if (comparisonPredicate instanceof LessThan + || comparisonPredicate instanceof GreaterThanEqual) { + rounded = right.roundCeilingToDateTimeV2(toScale); + } else { + return comparisonPredicate; + } + Expression newRight = leftType instanceof DateTimeType ? migrateToDateTime(rounded) : rounded; + return comparisonPredicate.withChildren(left, newRight); + } + // process cast(datetime as datetime) cmp datetime private static Expression processDateTimeLikeComparisonPredicateDateTimeV2Literal( ComparisonPredicate comparisonPredicate, Expression left, DateTimeV2Literal right) { @@ -214,9 +251,9 @@ private static Expression processDateTimeLikeComparisonPredicateDateTimeV2Litera DateTimeV2Type rightType = right.getDataType(); if (toScale < rightType.getScale()) { if (comparisonPredicate instanceof EqualTo) { - long originValue = right.getMicroSecond(); + long originValue = right.getNanoSecond(); right = right.roundFloor(toScale); - if (right.getMicroSecond() != originValue) { + if (right.getNanoSecond() != originValue) { // TODO: the ideal way is to return an If expr like: // return new If(new IsNull(left), new NullLiteral(BooleanType.INSTANCE), // BooleanLiteral.of(false)); @@ -226,9 +263,9 @@ private static Expression processDateTimeLikeComparisonPredicateDateTimeV2Litera return ExpressionUtils.falseOrNull(left); } } else if (comparisonPredicate instanceof NullSafeEqual) { - long originValue = right.getMicroSecond(); + long originValue = right.getNanoSecond(); right = right.roundFloor(toScale); - if (right.getMicroSecond() != originValue) { + if (right.getNanoSecond() != originValue) { return BooleanLiteral.of(false); } } else if (comparisonPredicate instanceof GreaterThan @@ -284,21 +321,50 @@ private static Expression processDateTimeLikeComparisonPredicateDateLiteral( return comparisonPredicate; } - DateTimeLiteral lowBound = null; - DateTimeLiteral upBound = null; + DateLiteral lowBound; + DateLiteral upBound; if (leftType instanceof DateTimeType) { lowBound = new DateTimeLiteral(right.getYear(), right.getMonth(), right.getDay(), 0, 0, 0); upBound = new DateTimeLiteral(right.getYear(), right.getMonth(), right.getDay(), 23, 59, 59); } else { - long upMicroSecond = 0; + long upperFraction = 0; for (int i = 0; i < ((DateTimeV2Type) leftType).getScale(); i++) { - upMicroSecond = 10 * upMicroSecond + 9; + upperFraction = 10 * upperFraction + 9; + } + DateTimeV2Type dateTimeV2Type = (DateTimeV2Type) leftType; + int scale = dateTimeV2Type.getScale(); + // DateTimeV2Literal accepts a 6-digit microsecond value for DATETIMEV2, but a + // 9-digit nanosecond value for TIMESTAMP_NS. + boolean isTimestampNs = dateTimeV2Type instanceof TimeStampNsType; + int fractionalWidth = isTimestampNs ? TimeStampNsType.SCALE : DateTimeV2Type.MAX_SCALE; + upperFraction *= (long) Math.pow(10, fractionalWidth - scale); + if (isTimestampNs) { + // The signed Int64 epoch-nanosecond range starts and ends in the middle of its + // boundary dates. Clamp the rewritten whole-day interval to the exact min/max on + // those dates, and simplify dates completely outside the range without constructing + // an invalid midnight or end-of-day literal. + TimeStampNsLiteral minValue = TimeStampNsLiteral.getMinValue(); + TimeStampNsLiteral maxValue = TimeStampNsLiteral.getMaxValue(); + DateV2Literal minDate = new DateV2Literal( + minValue.getYear(), minValue.getMonth(), minValue.getDay()); + DateV2Literal maxDate = new DateV2Literal( + maxValue.getYear(), maxValue.getMonth(), maxValue.getDay()); + int compareToMin = right.compareTo(minDate); + int compareToMax = right.compareTo(maxDate); + if (compareToMin < 0 || compareToMax > 0) { + return simplifyDateComparisonOutsideRange( + comparisonPredicate, left, compareToMin < 0); + } + lowBound = compareToMin == 0 ? minValue : new TimeStampNsLiteral( + right.getYear(), right.getMonth(), right.getDay(), 0, 0, 0, 0); + upBound = compareToMax == 0 ? maxValue : new TimeStampNsLiteral( + right.getYear(), right.getMonth(), right.getDay(), 23, 59, 59, upperFraction); + } else { + lowBound = new DateTimeV2Literal(dateTimeV2Type, + right.getYear(), right.getMonth(), right.getDay(), 0, 0, 0, 0); + upBound = new DateTimeV2Literal(dateTimeV2Type, + right.getYear(), right.getMonth(), right.getDay(), 23, 59, 59, upperFraction); } - upMicroSecond *= (int) Math.pow(10, 6 - ((DateTimeV2Type) leftType).getScale()); - lowBound = new DateTimeV2Literal((DateTimeV2Type) leftType, - right.getYear(), right.getMonth(), right.getDay(), 0, 0, 0, 0); - upBound = new DateTimeV2Literal((DateTimeV2Type) leftType, - right.getYear(), right.getMonth(), right.getDay(), 23, 59, 59, upMicroSecond); } if (comparisonPredicate instanceof GreaterThanEqual || comparisonPredicate instanceof LessThan) { @@ -321,17 +387,43 @@ private static Expression processDateTimeLikeComparisonPredicateDateLiteral( return comparisonPredicate; } + /** + * Simplify a date/datetime comparison whose right literal is outside the range representable by + * the left expression. For every non-null left value, the result is constant and depends only on + * the comparison direction and whether the literal is below the minimum or above the maximum. + * Null-safe equality is always false; other predicates return true-or-null or false-or-null to + * preserve SQL three-valued logic when the left expression is nullable. + * + * @param comparisonPredicate comparison to simplify + * @param left left expression whose type defines the representable range + * @param rightBeforeMin true if the right literal is below that range; false if it is above + * @return the constant-equivalent expression with the original NULL semantics + */ + private static Expression simplifyDateComparisonOutsideRange( + ComparisonPredicate comparisonPredicate, Expression left, boolean rightBeforeMin) { + if (comparisonPredicate instanceof NullSafeEqual) { + return BooleanLiteral.FALSE; + } + if (comparisonPredicate instanceof EqualTo) { + return ExpressionUtils.falseOrNull(left); + } + boolean alwaysTrue = rightBeforeMin + ? comparisonPredicate instanceof GreaterThan || comparisonPredicate instanceof GreaterThanEqual + : comparisonPredicate instanceof LessThan || comparisonPredicate instanceof LessThanEqual; + return alwaysTrue ? ExpressionUtils.trueOrNull(left) : ExpressionUtils.falseOrNull(left); + } + // process cast(date as datetime/date) cmp datetime/date private static Expression processDateLikeComparisonPredicateDateLiteral( ComparisonPredicate comparisonPredicate, Expression left, DateLiteral right) { if (!(left.getDataType() instanceof DateType) && !(left.getDataType() instanceof DateV2Type)) { return comparisonPredicate; } - if (right instanceof DateTimeLiteral) { - DateTimeLiteral dateTimeLiteral = (DateTimeLiteral) right; - right = migrateToDateV2(dateTimeLiteral); - if (dateTimeLiteral.getHour() != 0 || dateTimeLiteral.getMinute() != 0 - || dateTimeLiteral.getSecond() != 0 || dateTimeLiteral.getMicroSecond() != 0) { + if (right.getDataType().isDateTimeType() || right.getDataType().isDateTimeV2Type() + || right.getDataType().isTimeStampTzType()) { + boolean isMidnight = right.isMidnight(); + right = new DateV2Literal(right.getYear(), right.getMonth(), right.getDay()); + if (!isMidnight) { if (comparisonPredicate instanceof EqualTo) { return ExpressionUtils.falseOrNull(left); } else if (comparisonPredicate instanceof NullSafeEqual) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyInPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyInPredicate.java index b66db8e66f0d03..003b0ed7299c76 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyInPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyInPredicate.java @@ -26,6 +26,7 @@ import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal; import org.apache.doris.nereids.trees.expressions.literal.DateV2Literal; import org.apache.doris.nereids.types.DateTimeV2Type; +import org.apache.doris.nereids.types.TimeStampNsType; import com.google.common.collect.ImmutableList; @@ -61,6 +62,7 @@ && canLosslessConvertToDateV2Literal((DateTimeV2Literal) literal))) { return expr.withChildren(children.build()); } } else if (cast.child().getDataType().isDateTimeV2Type() + && !(cast.child().getDataType() instanceof TimeStampNsType) && expr.child(1) instanceof DateTimeV2Literal) { List literals = expr.children().subList(1, expr.children().size()); DateTimeV2Type castType = (DateTimeV2Type) cast.getDataType(); @@ -71,7 +73,7 @@ && canLosslessConvertToLowScaleLiteral( (DateTimeV2Literal) literal, compareType.getScale()))) { ImmutableList.Builder children = ImmutableList.builder(); children.add(cast.child()); - literals.forEach(l -> children.add(new DateTimeV2Literal(compareType, + literals.forEach(l -> children.add(DateTimeV2Literal.create(compareType, ((DateTimeV2Literal) l).getStringValue()))); return expr.withChildren(children.build()); } @@ -93,7 +95,7 @@ && canLosslessConvertToLowScaleLiteral( */ private static boolean canLosslessConvertToDateV2Literal(DateTimeV2Literal literal) { return (literal.getHour() | literal.getMinute() | literal.getSecond() - | literal.getMicroSecond()) == 0L; + | literal.getNanoSecond()) == 0L; } private static DateV2Literal convertToDateV2Literal(DateTimeV2Literal literal) { @@ -101,7 +103,7 @@ private static DateV2Literal convertToDateV2Literal(DateTimeV2Literal literal) { } private static boolean canLosslessConvertToLowScaleLiteral(DateTimeV2Literal literal, int targetScale) { - long scaleFactor = (long) Math.pow(10, DateTimeV2Type.MAX_SCALE - targetScale); - return literal.getMicroSecond() % scaleFactor == 0; + long scaleFactor = (long) Math.pow(10, TimeStampNsType.SCALE - targetScale); + return literal.getNanoSecond() % scaleFactor == 0; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java index 482f7872be6015..c14628ca3b0b14 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java @@ -21,6 +21,7 @@ import org.apache.doris.analysis.DateLiteralUtils; import org.apache.doris.analysis.LiteralExpr; import org.apache.doris.analysis.StringLiteral; +import org.apache.doris.analysis.TimeStampNsLiteral; import org.apache.doris.nereids.stats.FilterEstimation.EstimationContext; import org.apache.doris.nereids.trees.expressions.And; import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; @@ -475,7 +476,7 @@ private Statistics estimateColumnToConstant(ComparisonPredicate cp, EstimationCo return estimateColumnEqualToConstant(cp, statsForLeft, statsForRight, context); } else { // literal Map used to covert dateLiteral back to stringLiteral - Map literalMap = new HashMap<>(); + Map literalMap = new HashMap<>(); DataType compareType = cp.left().getDataType(); Optional statsForLeftMayConvertedOpt = tryConvertStringColStatsToDateColStats(statsForLeft, literalMap); @@ -523,7 +524,7 @@ private Statistics estimateColumnToConstant(ComparisonPredicate cp, EstimationCo } private ColumnStatistic convertDateColStatsToStringColStats(ColumnStatistic colStats, - Map literalMap) { + Map literalMap) { ColumnStatisticBuilder builder = new ColumnStatisticBuilder(colStats); if (colStats.minExpr != null) { builder.setMinExpr(literalMap.get(colStats.minExpr)) @@ -553,18 +554,18 @@ private ColumnStatistic convertDateColStatsToStringColStats(ColumnStatistic colS } private Optional tryConvertStringColStatsToDateColStats(ColumnStatistic colStats, - Map literalMap) { + Map literalMap) { if (colStats.minExpr == null || colStats.maxExpr == null) { return Optional.empty(); } if (!(colStats.minExpr instanceof StringLiteral) || !(colStats.maxExpr instanceof StringLiteral)) { return Optional.empty(); } - Optional newMinExpr = tryConvertStrLiteralToDateLiteral(colStats.minExpr); + Optional newMinExpr = tryConvertStrLiteralToDateLiteral(colStats.minExpr); if (!newMinExpr.isPresent()) { return Optional.empty(); } - Optional newMaxExpr = tryConvertStrLiteralToDateLiteral(colStats.maxExpr); + Optional newMaxExpr = tryConvertStrLiteralToDateLiteral(colStats.maxExpr); if (!newMaxExpr.isPresent()) { return Optional.empty(); } @@ -578,7 +579,10 @@ private Optional tryConvertStringColStatsToDateColStats(ColumnS if (colStats.hotValues != null) { for (Literal oneHot : colStats.hotValues.keySet()) { try { - DateTimeLiteral oneHotDate = new DateTimeLiteral(oneHot.getStringValue()); + Literal oneHotDate = newMinExpr.get().getType().isTimeStampNs() + ? new org.apache.doris.nereids.trees.expressions.literal.TimeStampNsLiteral( + oneHot.getStringValue()) + : new DateTimeLiteral(oneHot.getStringValue()); newHotValues.put(oneHotDate, colStats.hotValues.get(oneHot)); } catch (Exception e) { @@ -588,15 +592,15 @@ private Optional tryConvertStringColStatsToDateColStats(ColumnS } ColumnStatisticBuilder builder = new ColumnStatisticBuilder(colStats); - return Optional.of(builder.setMinValue(newMinExpr.get().getDoubleValueAsDateTime()) + return Optional.of(builder.setMinValue(getDoubleValueAsDateTime(newMinExpr.get())) .setMinExpr(newMinExpr.get()) - .setMaxValue(newMaxExpr.get().getDoubleValueAsDateTime()) + .setMaxValue(getDoubleValueAsDateTime(newMaxExpr.get())) .setMaxExpr(newMaxExpr.get()) .setHotValues(newHotValues.isEmpty() ? null : newHotValues) .build()); } - private Optional tryConvertStrLiteralToDateLiteral(LiteralExpr literal) { + private Optional tryConvertStrLiteralToDateLiteral(LiteralExpr literal) { if (literal == null) { return Optional.empty(); } @@ -604,9 +608,9 @@ private Optional tryConvertStrLiteralToDateLiteral(LiteralExpr lite return Optional.empty(); } - DateLiteral dt = null; + LiteralExpr dt = null; try { - dt = DateLiteralUtils.createDateLiteral(literal.getStringValue(), null); + dt = DateLiteralUtils.createLiteral(literal.getStringValue(), null); dt.checkValueValid(); } catch (Exception e) { // ignore @@ -614,6 +618,15 @@ private Optional tryConvertStrLiteralToDateLiteral(LiteralExpr lite return dt == null ? Optional.empty() : Optional.of(dt); } + private double getDoubleValueAsDateTime(LiteralExpr literal) { + if (literal instanceof DateLiteral) { + return ((DateLiteral) literal).getDoubleValueAsDateTime(); + } + Preconditions.checkState(literal instanceof TimeStampNsLiteral, + "Expected a date-like literal, but got %s", literal.getClass()); + return ((TimeStampNsLiteral) literal).getDoubleValueAsDateTime(); + } + private Statistics estimateColumnEqualToConstant(ComparisonPredicate cp, ColumnStatistic statsForLeft, ColumnStatistic statsForRight, EstimationContext context) { double selectivity = DEFAULT_ISNULL_SELECTIVITY; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ComputeSignatureHelper.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ComputeSignatureHelper.java index 0d0abe92789834..0d69200677d551 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ComputeSignatureHelper.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ComputeSignatureHelper.java @@ -528,19 +528,24 @@ private static FunctionSignature defaultTimePrecisionPromotion(FunctionSignature DataType inputType = nestedInputTypes.get(j); // corresponding target slot type for inputType DataType nestedTargetType = nestedTargetTypes.get(j); - int targetScale = 0; + ScaleTimeType promotedType; // for string input, try to get the most suitable scale if (arg instanceof StringLikeLiteral) { ScaleTimeType timelikeType = (ScaleTimeType) nestedTargetType; - targetScale = timelikeType.forTypeFromString((StringLikeLiteral) arg).getScale(); + promotedType = timelikeType.forTypeFromString((StringLikeLiteral) arg); } else { // for all other input types, get the target scale when cast it to targetType ScaleTimeType targetScaleType = (ScaleTimeType) nestedTargetType; - ScaleTimeType promotedType = targetScaleType.scaleTypeForType(inputType); - targetScale = promotedType.getScale(); + promotedType = targetScaleType.scaleTypeForType(inputType); } + int targetScale = promotedType.getScale(); + // TimeStampNsType inherits DateTimeV2Type only to reuse civil-time behavior. A + // DATETIMEV2 signature must stay within its own precision and physical layout. + if (nestedTargetType.getClass() == DateTimeV2Type.class) { + targetScale = Math.min(targetScale, DateTimeV2Type.MAX_SCALE); + } finalTypeScale = Math.max(finalTypeScale, targetScale); // init value -1 always promotes } } @@ -552,7 +557,8 @@ private static FunctionSignature defaultTimePrecisionPromotion(FunctionSignature // promote the precision of return type ImmutableList.Builder newArgTypesBuilder = ImmutableList.builderWithExpectedSize(signature.arity); for (DataType signatureArgType : signature.argumentsTypes) { - newArgTypesBuilder.add(TypeCoercionUtils.replaceTimesWithTargetPrecision(signatureArgType, finalTypeScale)); + newArgTypesBuilder.add(TypeCoercionUtils.replaceTimesWithTargetPrecision( + signatureArgType, finalTypeScale)); } List newArgTypes = newArgTypesBuilder.build(); signature = signature.withArgumentTypes(signature.hasVarArgs, newArgTypes); @@ -560,7 +566,8 @@ private static FunctionSignature defaultTimePrecisionPromotion(FunctionSignature || signature.returnType instanceof TimeStampTzType || signature.returnType instanceof ComplexDataType) { signature = signature.withReturnType( - TypeCoercionUtils.replaceTimesWithTargetPrecision(signature.returnType, finalTypeScale)); + TypeCoercionUtils.replaceTimesWithTargetPrecision( + signature.returnType, finalTypeScale)); } return signature; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/DateTimeWithPrecision.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/DateTimeWithPrecision.java index ab67c9e365a2e0..87cf6e8e08662c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/DateTimeWithPrecision.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/DateTimeWithPrecision.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.trees.expressions.functions; import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.scalar.ScalarFunction; import org.apache.doris.nereids.trees.expressions.functions.scalar.ScalarFunctionParams; @@ -46,9 +47,14 @@ public FunctionSignature computeSignature(FunctionSignature signature) { // searching in FunctionSet. So we adjust the return type by hand here. if (getArgument(0) instanceof IntegerLikeLiteral) { IntegerLikeLiteral integerLikeLiteral = (IntegerLikeLiteral) getArgument(0); - signature = signature.withReturnType(DateTimeV2Type.of(integerLikeLiteral.getIntValue())); + int scale = integerLikeLiteral.getIntValue(); + if (scale < 0 || scale > DateTimeV2Type.MAX_SCALE) { + throw new AnalysisException("Scale of Datetime/Time must between 0 and 6. Scale was set to: " + + scale); + } + signature = signature.withReturnType(DateTimeV2Type.of(scale)); } else { - signature = signature.withReturnType(DateTimeV2Type.of(6)); + signature = signature.withReturnType(DateTimeV2Type.MAX); } } return super.computeSignature(signature); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/SequenceCount.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/SequenceCount.java index e4ad5956c1c290..33370f39ffabef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/SequenceCount.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/SequenceCount.java @@ -27,6 +27,7 @@ import org.apache.doris.nereids.types.DateTimeV2Type; import org.apache.doris.nereids.types.DateV2Type; import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TimeStampTzType; import org.apache.doris.nereids.util.ExpressionUtils; @@ -47,7 +48,9 @@ public class SequenceCount extends NotNullableAggregateFunction FunctionSignature.ret(BigIntType.INSTANCE) .varArgs(StringType.INSTANCE, TimeStampTzType.WILDCARD, BooleanType.INSTANCE), FunctionSignature.ret(BigIntType.INSTANCE) - .varArgs(StringType.INSTANCE, DateTimeV2Type.WILDCARD, BooleanType.INSTANCE) + .varArgs(StringType.INSTANCE, DateTimeV2Type.WILDCARD, BooleanType.INSTANCE), + FunctionSignature.ret(BigIntType.INSTANCE) + .varArgs(StringType.INSTANCE, TimeStampNsType.INSTANCE, BooleanType.INSTANCE) ); /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/SequenceMatch.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/SequenceMatch.java index c9f15731a196ee..cae83665551003 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/SequenceMatch.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/SequenceMatch.java @@ -25,6 +25,7 @@ import org.apache.doris.nereids.types.DateTimeV2Type; import org.apache.doris.nereids.types.DateV2Type; import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TimeStampTzType; import org.apache.doris.nereids.util.ExpressionUtils; @@ -45,7 +46,9 @@ public class SequenceMatch extends NullableAggregateFunction FunctionSignature.ret(BooleanType.INSTANCE) .varArgs(StringType.INSTANCE, TimeStampTzType.WILDCARD, BooleanType.INSTANCE), FunctionSignature.ret(BooleanType.INSTANCE) - .varArgs(StringType.INSTANCE, DateTimeV2Type.WILDCARD, BooleanType.INSTANCE) + .varArgs(StringType.INSTANCE, DateTimeV2Type.WILDCARD, BooleanType.INSTANCE), + FunctionSignature.ret(BooleanType.INSTANCE) + .varArgs(StringType.INSTANCE, TimeStampNsType.INSTANCE, BooleanType.INSTANCE) ); /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/TopNWeighted.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/TopNWeighted.java index 9d3152685a4606..332d36ded2b64a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/TopNWeighted.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/TopNWeighted.java @@ -36,6 +36,7 @@ import org.apache.doris.nereids.types.LargeIntType; import org.apache.doris.nereids.types.SmallIntType; import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TimeStampTzType; import org.apache.doris.nereids.types.TinyIntType; import org.apache.doris.nereids.types.VarcharType; @@ -77,6 +78,8 @@ public class TopNWeighted extends NullableAggregateFunction .args(TimeStampTzType.WILDCARD, BigIntType.INSTANCE, IntegerType.INSTANCE), FunctionSignature.ret(ArrayType.of(DateTimeV2Type.WILDCARD)) .args(DateTimeV2Type.WILDCARD, BigIntType.INSTANCE, IntegerType.INSTANCE), + FunctionSignature.ret(ArrayType.of(TimeStampNsType.INSTANCE)) + .args(TimeStampNsType.INSTANCE, BigIntType.INSTANCE, IntegerType.INSTANCE), FunctionSignature.ret(ArrayType.of(StringType.INSTANCE)) .args(StringType.INSTANCE, BigIntType.INSTANCE, IntegerType.INSTANCE), FunctionSignature.ret(ArrayType.of(VarcharType.SYSTEM_DEFAULT)) @@ -119,6 +122,11 @@ public class TopNWeighted extends NullableAggregateFunction BigIntType.INSTANCE, IntegerType.INSTANCE, IntegerType.INSTANCE), + FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT) + .args(TimeStampNsType.INSTANCE, + BigIntType.INSTANCE, + IntegerType.INSTANCE, + IntegerType.INSTANCE), FunctionSignature.ret(ArrayType.of(StringType.INSTANCE)) .args(StringType.INSTANCE, BigIntType.INSTANCE, IntegerType.INSTANCE, IntegerType.INSTANCE), FunctionSignature.ret(ArrayType.of(VarcharType.SYSTEM_DEFAULT)) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/WindowFunnel.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/WindowFunnel.java index ef60c0a8c07d31..7ae384621d1dde 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/WindowFunnel.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/WindowFunnel.java @@ -29,6 +29,7 @@ import org.apache.doris.nereids.types.DateV2Type; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TimeStampTzType; import org.apache.doris.nereids.util.ExpressionUtils; @@ -50,6 +51,9 @@ public class WindowFunnel extends NullableAggregateFunction BooleanType.INSTANCE), FunctionSignature.ret(IntegerType.INSTANCE) .varArgs(BigIntType.INSTANCE, StringType.INSTANCE, DateTimeV2Type.WILDCARD, + BooleanType.INSTANCE), + FunctionSignature.ret(IntegerType.INSTANCE) + .varArgs(BigIntType.INSTANCE, StringType.INSTANCE, TimeStampNsType.INSTANCE, BooleanType.INSTANCE) ); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/WindowFunnelV2.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/WindowFunnelV2.java index 697774eb47f4b6..1c51948a483900 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/WindowFunnelV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/WindowFunnelV2.java @@ -29,6 +29,7 @@ import org.apache.doris.nereids.types.DateV2Type; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TimeStampTzType; import org.apache.doris.nereids.util.ExpressionUtils; @@ -53,6 +54,9 @@ public class WindowFunnelV2 extends NullableAggregateFunction BooleanType.INSTANCE), FunctionSignature.ret(IntegerType.INSTANCE) .varArgs(BigIntType.INSTANCE, StringType.INSTANCE, DateTimeV2Type.WILDCARD, + BooleanType.INSTANCE), + FunctionSignature.ret(IntegerType.INSTANCE) + .varArgs(BigIntType.INSTANCE, StringType.INSTANCE, TimeStampNsType.INSTANCE, BooleanType.INSTANCE) ); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeArithmetic.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeArithmetic.java index cd12c00df210cf..ecf91b595f10db 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeArithmetic.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeArithmetic.java @@ -268,7 +268,7 @@ private static Expression applyInterval(DateTimeV2Literal date, VarcharLiteral d DateTimeV2Literal dt = (DateTimeV2Literal) result; if (dt.getScale() != date.getScale()) { // Keep original scale to match existing folding behavior. - return DateTimeV2Literal.fromJavaDateType(dt.toJavaDateType(), date.getScale()); + return DateTimeV2Literal.fromJavaDateType(dt.toJavaDateType(), date.getDataType()); } } return result; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java index ca55370c0d780d..7e2071baab1495 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java @@ -46,6 +46,7 @@ import org.apache.doris.nereids.types.DateV2Type; import org.apache.doris.nereids.types.DecimalV3Type; import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TimeV2Type; import org.apache.doris.nereids.util.DateTimeFormatterUtils; import org.apache.doris.nereids.util.DateUtils; @@ -413,7 +414,7 @@ public static Expression date(DateTimeV2Literal dateTime) throws AnalysisExcepti @ExecFunction(name = "date_trunc") public static Expression dateTrunc(DateTimeV2Literal date, StringLikeLiteral trunc) { return DateTimeV2Literal.fromJavaDateType( - dateTruncHelper(date.toJavaDateType(), trunc.getValue()), date.getScale()); + dateTruncHelper(date.toJavaDateType(), trunc.getValue()), date.getDataType()); } @ExecFunction(name = "date_trunc") @@ -424,7 +425,7 @@ public static Expression dateTrunc(DateV2Literal date, StringLikeLiteral trunc) @ExecFunction(name = "date_trunc") public static Expression dateTrunc(StringLikeLiteral trunc, DateTimeV2Literal date) { return DateTimeV2Literal.fromJavaDateType( - dateTruncHelper(date.toJavaDateType(), trunc.getValue()), date.getScale()); + dateTruncHelper(date.toJavaDateType(), trunc.getValue()), date.getDataType()); } @ExecFunction(name = "date_trunc") @@ -615,7 +616,7 @@ public static Expression unixTimestamp(DateV2Literal date) { public static Expression unixTimestamp(DateTimeV2Literal date) { int scale = date.getDataType().getScale(); return new DecimalV3Literal(DecimalV3Type.createDecimalV3TypeLooseCheck(12 + scale, scale), - new BigDecimal(getTimestamp(date.toJavaDateType()))); + new BigDecimal(getTimestamp(date.toJavaDateType(), scale))); } /** @@ -637,6 +638,10 @@ public static Expression unixTimestamp(StringLikeLiteral date, StringLikeLiteral } private static String getTimestamp(LocalDateTime dateTime) { + return getTimestamp(dateTime, 6); + } + + private static String getTimestamp(LocalDateTime dateTime, int scale) { LocalDateTime specialLowerBound = LocalDateTime.of(1970, 1, 1, 0, 0, 0); dateTime = dateTime.atZone(DateUtils.getTimeZone()) .toOffsetDateTime().atZoneSameInstant(ZoneId.of("UTC+0")) @@ -651,7 +656,8 @@ private static String getTimestamp(LocalDateTime dateTime) { if (duration.getNano() == 0) { return String.valueOf(duration.getSeconds()); } else { - return duration.getSeconds() + "." + String.format("%06d", duration.getNano() / 1000); + long fraction = duration.getNano() / (long) Math.pow(10, TimeStampNsType.SCALE - scale); + return duration.getSeconds() + "." + String.format("%0" + scale + "d", fraction); } } @@ -825,9 +831,9 @@ public static Expression convertTz(DateTimeV2Literal datetime, StringLikeLiteral ZoneId toZone = ZoneId.from(zoneFormatter.parse(toTz.getStringValue())); LocalDateTime localDateTime = datetime.toJavaDateType(); - Instant instant = DateTimeLiteral.convertLocalToInstant(localDateTime, fromZone); + Instant instant = DateUtils.convertLocalToInstant(localDateTime, fromZone); return DateTimeV2Literal.fromJavaDateType(LocalDateTime.ofInstant(instant, toZone), - datetime.getDataType().getScale()); + datetime.getDataType()); } private static void validateTimezoneOffset(String timezone) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/TimeRoundSeries.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/TimeRoundSeries.java index c32f733bedbae5..d102ab2839dd59 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/TimeRoundSeries.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/TimeRoundSeries.java @@ -104,14 +104,16 @@ private static LocalDateTime getDateCeilOrFloor(DATE tag, LocalDateTime date, in + (dt.getHour() - start.getHour()) * 60 * 60 + (dt.getMinute() - start.getMinute()) * 60 + (dt.getSecond() - start.getSecond()); - trivialPart = dt.getMicroSecond() - start.getMicroSecond(); + trivialPart = date.getNano() - origin.getNano(); break; } default: { return null; } } - trivialPart = (trivialPart == 0 ? dt.getMicroSecond() - start.getMicroSecond() : trivialPart); + // The coarser fields can be equal while the values still differ within the second. Use the full + // fractional precision so such values do not look aligned to a rounding boundary. + trivialPart = (trivialPart == 0 ? date.getNano() - origin.getNano() : trivialPart); if (getCeil) { diff = diff + (trivialPart > 0 ? 1 : 0); } else { @@ -237,25 +239,25 @@ public static Expression yearCeil(DateV2Literal date, IntegerLiteral period, Dat @ExecFunction(name = "year_ceil") public static Expression yearCeil(DateTimeV2Literal date) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.YEAR, date.toJavaDateType(), - 1, START_ORIGINAL_DAY, true), date.getScale()); + 1, START_ORIGINAL_DAY, true), date.getDataType()); } @ExecFunction(name = "year_ceil") public static Expression yearCeil(DateTimeV2Literal date, IntegerLiteral period) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.YEAR, date.toJavaDateType(), - period.getValue(), START_ORIGINAL_DAY, true), date.getScale()); + period.getValue(), START_ORIGINAL_DAY, true), date.getDataType()); } @ExecFunction(name = "year_ceil") public static Expression yearCeil(DateTimeV2Literal date, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.YEAR, date.toJavaDateType(), - 1, origin.toJavaDateType(), true), date.commonScale(origin)); + 1, origin.toJavaDateType(), true), date.commonType(origin)); } @ExecFunction(name = "year_ceil") public static Expression yearCeil(DateTimeV2Literal date, IntegerLiteral period, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.YEAR, date.toJavaDateType(), - period.getValue(), origin.toJavaDateType(), true), date.commonScale(origin)); + period.getValue(), origin.toJavaDateType(), true), date.commonType(origin)); } @ExecFunction(name = "year_ceil") @@ -312,25 +314,25 @@ public static Expression quarterCeil(DateV2Literal date, IntegerLiteral period, @ExecFunction(name = "quarter_ceil") public static Expression quarterCeil(DateTimeV2Literal date) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.QUARTER, date.toJavaDateType(), - 1, START_ORIGINAL_DAY, true), date.getScale()); + 1, START_ORIGINAL_DAY, true), date.getDataType()); } @ExecFunction(name = "quarter_ceil") public static Expression quarterCeil(DateTimeV2Literal date, IntegerLiteral period) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.QUARTER, date.toJavaDateType(), - period.getValue(), START_ORIGINAL_DAY, true), date.getScale()); + period.getValue(), START_ORIGINAL_DAY, true), date.getDataType()); } @ExecFunction(name = "quarter_ceil") public static Expression quarterCeil(DateTimeV2Literal date, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.QUARTER, date.toJavaDateType(), - 1, origin.toJavaDateType(), true), date.commonScale(origin)); + 1, origin.toJavaDateType(), true), date.commonType(origin)); } @ExecFunction(name = "quarter_ceil") public static Expression quarterCeil(DateTimeV2Literal date, IntegerLiteral period, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.QUARTER, date.toJavaDateType(), - period.getValue(), origin.toJavaDateType(), true), date.commonScale(origin)); + period.getValue(), origin.toJavaDateType(), true), date.commonType(origin)); } @ExecFunction(name = "quarter_ceil") @@ -387,25 +389,25 @@ public static Expression monthCeil(DateV2Literal date, IntegerLiteral period, Da @ExecFunction(name = "month_ceil") public static Expression monthCeil(DateTimeV2Literal date) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.MONTH, date.toJavaDateType(), - 1, START_ORIGINAL_DAY, true), date.getScale()); + 1, START_ORIGINAL_DAY, true), date.getDataType()); } @ExecFunction(name = "month_ceil") public static Expression monthCeil(DateTimeV2Literal date, IntegerLiteral period) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.MONTH, date.toJavaDateType(), - period.getValue(), START_ORIGINAL_DAY, true), date.getScale()); + period.getValue(), START_ORIGINAL_DAY, true), date.getDataType()); } @ExecFunction(name = "month_ceil") public static Expression monthCeil(DateTimeV2Literal date, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.MONTH, date.toJavaDateType(), - 1, origin.toJavaDateType(), true), date.commonScale(origin)); + 1, origin.toJavaDateType(), true), date.commonType(origin)); } @ExecFunction(name = "month_ceil") public static Expression monthCeil(DateTimeV2Literal date, IntegerLiteral period, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.MONTH, date.toJavaDateType(), - period.getValue(), origin.toJavaDateType(), true), date.commonScale(origin)); + period.getValue(), origin.toJavaDateType(), true), date.commonType(origin)); } @ExecFunction(name = "month_ceil") @@ -462,25 +464,25 @@ public static Expression dayCeil(DateV2Literal date, IntegerLiteral period, Date @ExecFunction(name = "day_ceil") public static Expression dayCeil(DateTimeV2Literal date) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.DAY, date.toJavaDateType(), - 1, START_ORIGINAL_DAY, true), date.getScale()); + 1, START_ORIGINAL_DAY, true), date.getDataType()); } @ExecFunction(name = "day_ceil") public static Expression dayCeil(DateTimeV2Literal date, IntegerLiteral period) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.DAY, date.toJavaDateType(), - period.getValue(), START_ORIGINAL_DAY, true), date.getScale()); + period.getValue(), START_ORIGINAL_DAY, true), date.getDataType()); } @ExecFunction(name = "day_ceil") public static Expression dayCeil(DateTimeV2Literal date, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.DAY, date.toJavaDateType(), - 1, origin.toJavaDateType(), true), date.commonScale(origin)); + 1, origin.toJavaDateType(), true), date.commonType(origin)); } @ExecFunction(name = "day_ceil") public static Expression dayCeil(DateTimeV2Literal date, IntegerLiteral period, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.DAY, date.toJavaDateType(), - period.getValue(), origin.toJavaDateType(), true), date.commonScale(origin)); + period.getValue(), origin.toJavaDateType(), true), date.commonType(origin)); } @ExecFunction(name = "day_ceil") @@ -537,25 +539,25 @@ public static Expression hourCeil(DateV2Literal date, IntegerLiteral period, Dat @ExecFunction(name = "hour_ceil") public static Expression hourCeil(DateTimeV2Literal date) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.HOUR, date.toJavaDateType(), - 1, START_ORIGINAL_DAY, true), date.getScale()); + 1, START_ORIGINAL_DAY, true), date.getDataType()); } @ExecFunction(name = "hour_ceil") public static Expression hourCeil(DateTimeV2Literal date, IntegerLiteral period) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.HOUR, date.toJavaDateType(), - period.getValue(), START_ORIGINAL_DAY, true), date.getScale()); + period.getValue(), START_ORIGINAL_DAY, true), date.getDataType()); } @ExecFunction(name = "hour_ceil") public static Expression hourCeil(DateTimeV2Literal date, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.HOUR, date.toJavaDateType(), - 1, origin.toJavaDateType(), true), date.commonScale(origin)); + 1, origin.toJavaDateType(), true), date.commonType(origin)); } @ExecFunction(name = "hour_ceil") public static Expression hourCeil(DateTimeV2Literal date, IntegerLiteral period, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.HOUR, date.toJavaDateType(), - period.getValue(), origin.toJavaDateType(), true), date.commonScale(origin)); + period.getValue(), origin.toJavaDateType(), true), date.commonType(origin)); } @ExecFunction(name = "hour_ceil") @@ -612,26 +614,26 @@ public static Expression minuteCeil(DateV2Literal date, IntegerLiteral period, D @ExecFunction(name = "minute_ceil") public static Expression minuteCeil(DateTimeV2Literal date) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.MINUTE, date.toJavaDateType(), - 1, START_ORIGINAL_DAY, true), date.getScale()); + 1, START_ORIGINAL_DAY, true), date.getDataType()); } @ExecFunction(name = "minute_ceil") public static Expression minuteCeil(DateTimeV2Literal date, IntegerLiteral period) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.MINUTE, date.toJavaDateType(), - period.getValue(), START_ORIGINAL_DAY, true), date.getScale()); + period.getValue(), START_ORIGINAL_DAY, true), date.getDataType()); } @ExecFunction(name = "minute_ceil") public static Expression minuteCeil(DateTimeV2Literal date, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.MINUTE, date.toJavaDateType(), - 1, origin.toJavaDateType(), true), date.commonScale(origin)); + 1, origin.toJavaDateType(), true), date.commonType(origin)); } @ExecFunction(name = "minute_ceil") public static Expression minuteCeil(DateTimeV2Literal date, IntegerLiteral period, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.MINUTE, date.toJavaDateType(), - period.getValue(), origin.toJavaDateType(), true), date.commonScale(origin)); + period.getValue(), origin.toJavaDateType(), true), date.commonType(origin)); } @ExecFunction(name = "minute_ceil") @@ -689,26 +691,26 @@ public static Expression secondCeil(DateV2Literal date, IntegerLiteral period, D @ExecFunction(name = "second_ceil") public static Expression secondCeil(DateTimeV2Literal date) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.SECOND, date.toJavaDateType(), - 1, START_ORIGINAL_DAY, true), date.getScale()); + 1, START_ORIGINAL_DAY, true), date.getDataType()); } @ExecFunction(name = "second_ceil") public static Expression secondCeil(DateTimeV2Literal date, IntegerLiteral period) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.SECOND, date.toJavaDateType(), - period.getValue(), START_ORIGINAL_DAY, true), date.getScale()); + period.getValue(), START_ORIGINAL_DAY, true), date.getDataType()); } @ExecFunction(name = "second_ceil") public static Expression secondCeil(DateTimeV2Literal date, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.SECOND, date.toJavaDateType(), - 1, origin.toJavaDateType(), true), date.commonScale(origin)); + 1, origin.toJavaDateType(), true), date.commonType(origin)); } @ExecFunction(name = "second_ceil") public static Expression secondCeil(DateTimeV2Literal date, IntegerLiteral period, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.SECOND, date.toJavaDateType(), - period.getValue(), origin.toJavaDateType(), true), date.commonScale(origin)); + period.getValue(), origin.toJavaDateType(), true), date.commonType(origin)); } @ExecFunction(name = "second_ceil") @@ -766,25 +768,25 @@ public static Expression yearFloor(DateV2Literal date, IntegerLiteral period, Da @ExecFunction(name = "year_floor") public static Expression yearFloor(DateTimeV2Literal date) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.YEAR, date.toJavaDateType(), - 1, START_ORIGINAL_DAY, false), date.getScale()); + 1, START_ORIGINAL_DAY, false), date.getDataType()); } @ExecFunction(name = "year_floor") public static Expression yearFloor(DateTimeV2Literal date, IntegerLiteral period) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.YEAR, date.toJavaDateType(), - period.getValue(), START_ORIGINAL_DAY, false), date.getScale()); + period.getValue(), START_ORIGINAL_DAY, false), date.getDataType()); } @ExecFunction(name = "year_floor") public static Expression yearFloor(DateTimeV2Literal date, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.YEAR, date.toJavaDateType(), - 1, origin.toJavaDateType(), false), date.commonScale(origin)); + 1, origin.toJavaDateType(), false), date.commonType(origin)); } @ExecFunction(name = "year_floor") public static Expression yearFloor(DateTimeV2Literal date, IntegerLiteral period, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.YEAR, date.toJavaDateType(), - period.getValue(), origin.toJavaDateType(), false), date.commonScale(origin)); + period.getValue(), origin.toJavaDateType(), false), date.commonType(origin)); } @ExecFunction(name = "year_floor") @@ -841,26 +843,26 @@ public static Expression quarterFloor(DateV2Literal date, IntegerLiteral period, @ExecFunction(name = "quarter_floor") public static Expression quarterFloor(DateTimeV2Literal date) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.QUARTER, date.toJavaDateType(), - 1, START_ORIGINAL_DAY, false), date.getScale()); + 1, START_ORIGINAL_DAY, false), date.getDataType()); } @ExecFunction(name = "quarter_floor") public static Expression quarterFloor(DateTimeV2Literal date, IntegerLiteral period) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.QUARTER, date.toJavaDateType(), - period.getValue(), START_ORIGINAL_DAY, false), date.getScale()); + period.getValue(), START_ORIGINAL_DAY, false), date.getDataType()); } @ExecFunction(name = "quarter_floor") public static Expression quarterFloor(DateTimeV2Literal date, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.QUARTER, date.toJavaDateType(), - 1, origin.toJavaDateType(), false), date.commonScale(origin)); + 1, origin.toJavaDateType(), false), date.commonType(origin)); } @ExecFunction(name = "quarter_floor") public static Expression quarterFloor(DateTimeV2Literal date, IntegerLiteral period, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.QUARTER, date.toJavaDateType(), - period.getValue(), origin.toJavaDateType(), false), date.commonScale(origin)); + period.getValue(), origin.toJavaDateType(), false), date.commonType(origin)); } @ExecFunction(name = "quarter_floor") @@ -918,26 +920,26 @@ public static Expression monthFloor(DateV2Literal date, IntegerLiteral period, D @ExecFunction(name = "month_floor") public static Expression monthFloor(DateTimeV2Literal date) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.MONTH, date.toJavaDateType(), - 1, START_ORIGINAL_DAY, false), date.getScale()); + 1, START_ORIGINAL_DAY, false), date.getDataType()); } @ExecFunction(name = "month_floor") public static Expression monthFloor(DateTimeV2Literal date, IntegerLiteral period) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.MONTH, date.toJavaDateType(), - period.getValue(), START_ORIGINAL_DAY, false), date.getScale()); + period.getValue(), START_ORIGINAL_DAY, false), date.getDataType()); } @ExecFunction(name = "month_floor") public static Expression monthFloor(DateTimeV2Literal date, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.MONTH, date.toJavaDateType(), - 1, origin.toJavaDateType(), false), date.commonScale(origin)); + 1, origin.toJavaDateType(), false), date.commonType(origin)); } @ExecFunction(name = "month_floor") public static Expression monthFloor(DateTimeV2Literal date, IntegerLiteral period, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.MONTH, date.toJavaDateType(), - period.getValue(), origin.toJavaDateType(), false), date.commonScale(origin)); + period.getValue(), origin.toJavaDateType(), false), date.commonType(origin)); } @ExecFunction(name = "month_floor") @@ -995,25 +997,25 @@ public static Expression dayFloor(DateV2Literal date, IntegerLiteral period, Dat @ExecFunction(name = "day_floor") public static Expression dayFloor(DateTimeV2Literal date) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.DAY, date.toJavaDateType(), - 1, START_ORIGINAL_DAY, false), date.getScale()); + 1, START_ORIGINAL_DAY, false), date.getDataType()); } @ExecFunction(name = "day_floor") public static Expression dayFloor(DateTimeV2Literal date, IntegerLiteral period) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.DAY, date.toJavaDateType(), - period.getValue(), START_ORIGINAL_DAY, false), date.getScale()); + period.getValue(), START_ORIGINAL_DAY, false), date.getDataType()); } @ExecFunction(name = "day_floor") public static Expression dayFloor(DateTimeV2Literal date, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.DAY, date.toJavaDateType(), - 1, origin.toJavaDateType(), false), date.commonScale(origin)); + 1, origin.toJavaDateType(), false), date.commonType(origin)); } @ExecFunction(name = "day_floor") public static Expression dayFloor(DateTimeV2Literal date, IntegerLiteral period, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.DAY, date.toJavaDateType(), - period.getValue(), origin.toJavaDateType(), false), date.commonScale(origin)); + period.getValue(), origin.toJavaDateType(), false), date.commonType(origin)); } @ExecFunction(name = "day_floor") @@ -1070,25 +1072,25 @@ public static Expression hourFloor(DateV2Literal date, IntegerLiteral period, Da @ExecFunction(name = "hour_floor") public static Expression hourFloor(DateTimeV2Literal date) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.HOUR, date.toJavaDateType(), - 1, START_ORIGINAL_DAY, false), date.getScale()); + 1, START_ORIGINAL_DAY, false), date.getDataType()); } @ExecFunction(name = "hour_floor") public static Expression hourFloor(DateTimeV2Literal date, IntegerLiteral period) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.HOUR, date.toJavaDateType(), - period.getValue(), START_ORIGINAL_DAY, false), date.getScale()); + period.getValue(), START_ORIGINAL_DAY, false), date.getDataType()); } @ExecFunction(name = "hour_floor") public static Expression hourFloor(DateTimeV2Literal date, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.HOUR, date.toJavaDateType(), - 1, origin.toJavaDateType(), false), date.commonScale(origin)); + 1, origin.toJavaDateType(), false), date.commonType(origin)); } @ExecFunction(name = "hour_floor") public static Expression hourFloor(DateTimeV2Literal date, IntegerLiteral period, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.HOUR, date.toJavaDateType(), - period.getValue(), origin.toJavaDateType(), false), date.commonScale(origin)); + period.getValue(), origin.toJavaDateType(), false), date.commonType(origin)); } @ExecFunction(name = "hour_floor") @@ -1145,26 +1147,26 @@ public static Expression minuteFloor(DateV2Literal date, IntegerLiteral period, @ExecFunction(name = "minute_floor") public static Expression minuteFloor(DateTimeV2Literal date) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.MINUTE, date.toJavaDateType(), - 1, START_ORIGINAL_DAY, false), date.getScale()); + 1, START_ORIGINAL_DAY, false), date.getDataType()); } @ExecFunction(name = "minute_floor") public static Expression minuteFloor(DateTimeV2Literal date, IntegerLiteral period) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.MINUTE, date.toJavaDateType(), - period.getValue(), START_ORIGINAL_DAY, false), date.getScale()); + period.getValue(), START_ORIGINAL_DAY, false), date.getDataType()); } @ExecFunction(name = "minute_floor") public static Expression minuteFloor(DateTimeV2Literal date, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.MINUTE, date.toJavaDateType(), - 1, origin.toJavaDateType(), false), date.commonScale(origin)); + 1, origin.toJavaDateType(), false), date.commonType(origin)); } @ExecFunction(name = "minute_floor") public static Expression minuteFloor(DateTimeV2Literal date, IntegerLiteral period, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.MINUTE, date.toJavaDateType(), - period.getValue(), origin.toJavaDateType(), false), date.commonScale(origin)); + period.getValue(), origin.toJavaDateType(), false), date.commonType(origin)); } @ExecFunction(name = "minute_floor") @@ -1222,26 +1224,26 @@ public static Expression secondFloor(DateV2Literal date, IntegerLiteral period, @ExecFunction(name = "second_floor") public static Expression secondFloor(DateTimeV2Literal date) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.SECOND, date.toJavaDateType(), - 1, START_ORIGINAL_DAY, false), date.getScale()); + 1, START_ORIGINAL_DAY, false), date.getDataType()); } @ExecFunction(name = "second_floor") public static Expression secondFloor(DateTimeV2Literal date, IntegerLiteral period) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.SECOND, date.toJavaDateType(), - period.getValue(), START_ORIGINAL_DAY, false), date.getScale()); + period.getValue(), START_ORIGINAL_DAY, false), date.getDataType()); } @ExecFunction(name = "second_floor") public static Expression secondFloor(DateTimeV2Literal date, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.SECOND, date.toJavaDateType(), - 1, origin.toJavaDateType(), false), date.commonScale(origin)); + 1, origin.toJavaDateType(), false), date.commonType(origin)); } @ExecFunction(name = "second_floor") public static Expression secondFloor(DateTimeV2Literal date, IntegerLiteral period, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.SECOND, date.toJavaDateType(), - period.getValue(), origin.toJavaDateType(), false), date.commonScale(origin)); + period.getValue(), origin.toJavaDateType(), false), date.commonType(origin)); } @ExecFunction(name = "second_floor") diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConvertTz.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConvertTz.java index 195ea273af90b8..afc4890da0a85a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConvertTz.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConvertTz.java @@ -142,8 +142,8 @@ public boolean isMonotonic(Literal lower, Literal upper) { * Partition pruning folds both endpoints before deriving the function range, so a fall-back * instant inside (fromInstant(lower), fromInstant(upper)] disables the monotonic shortcut. */ - Instant lowerInstant = DateTimeLiteral.convertLocalToInstant(lowerDateTime, fromZone); - Instant upperInstant = DateTimeLiteral.convertLocalToInstant(upperDateTime, fromZone); + Instant lowerInstant = DateUtils.convertLocalToInstant(lowerDateTime, fromZone); + Instant upperInstant = DateUtils.convertLocalToInstant(upperDateTime, fromZone); if (upperInstant.isBefore(lowerInstant)) { return false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DayMicrosecondAdd.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DayMicrosecondAdd.java index 22b319464bc08a..a3a236a34baff0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DayMicrosecondAdd.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DayMicrosecondAdd.java @@ -41,8 +41,8 @@ public class DayMicrosecondAdd extends ScalarFunction implements BinaryExpression, ExplicitlyCastableSignature, ComputeSignatureForDateArithmetic, PropagateNullable, DateAddSubMonotonic { public static final List SIGNATURES = ImmutableList.of( - FunctionSignature.ret(DateTimeV2Type.MAX) - .args(DateTimeV2Type.MAX, VarcharType.SYSTEM_DEFAULT), + FunctionSignature.ret(DateTimeV2Type.WILDCARD) + .args(DateTimeV2Type.WILDCARD, VarcharType.SYSTEM_DEFAULT), FunctionSignature.ret(TimeStampTzType.MAX) .args(TimeStampTzType.MAX, VarcharType.SYSTEM_DEFAULT) ); @@ -80,6 +80,12 @@ public Expression withConstantArgs(Expression literal) { @Override public FunctionSignature computeSignature(FunctionSignature signature) { signature = super.computeSignature(signature); - return signature.withArgumentType(0, DateTimeV2Type.MAX).withReturnType(DateTimeV2Type.MAX); + if (getArgument(0).getDataType() instanceof TimeStampTzType + || signature.argumentsTypes.get(0) instanceof TimeStampTzType) { + return signature.withArgumentType(0, TimeStampTzType.MAX).withReturnType(TimeStampTzType.MAX); + } + DateTimeV2Type returnType = DateTimeV2Type.forTypeWithMinimumScale( + getArgument(0).getDataType(), DateTimeV2Type.MAX_SCALE); + return signature.withArgumentType(0, returnType).withReturnType(returnType); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DayMicrosecondSub.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DayMicrosecondSub.java index efb61c5f72ec17..1fd6aaeb56c601 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DayMicrosecondSub.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DayMicrosecondSub.java @@ -41,8 +41,8 @@ public class DayMicrosecondSub extends ScalarFunction implements BinaryExpression, ExplicitlyCastableSignature, ComputeSignatureForDateArithmetic, PropagateNullable, DateAddSubMonotonic { public static final List SIGNATURES = ImmutableList.of( - FunctionSignature.ret(DateTimeV2Type.MAX) - .args(DateTimeV2Type.MAX, VarcharType.SYSTEM_DEFAULT), + FunctionSignature.ret(DateTimeV2Type.WILDCARD) + .args(DateTimeV2Type.WILDCARD, VarcharType.SYSTEM_DEFAULT), FunctionSignature.ret(TimeStampTzType.MAX) .args(TimeStampTzType.MAX, VarcharType.SYSTEM_DEFAULT) ); @@ -80,6 +80,12 @@ public Expression withConstantArgs(Expression literal) { @Override public FunctionSignature computeSignature(FunctionSignature signature) { signature = super.computeSignature(signature); - return signature.withArgumentType(0, DateTimeV2Type.MAX).withReturnType(DateTimeV2Type.MAX); + if (getArgument(0).getDataType() instanceof TimeStampTzType + || signature.argumentsTypes.get(0) instanceof TimeStampTzType) { + return signature.withArgumentType(0, TimeStampTzType.MAX).withReturnType(TimeStampTzType.MAX); + } + DateTimeV2Type returnType = DateTimeV2Type.forTypeWithMinimumScale( + getArgument(0).getDataType(), DateTimeV2Type.MAX_SCALE); + return signature.withArgumentType(0, returnType).withReturnType(returnType); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HourMicrosecondAdd.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HourMicrosecondAdd.java index d7f7c9e2a31faa..7d6193b11a1c42 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HourMicrosecondAdd.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HourMicrosecondAdd.java @@ -41,8 +41,8 @@ public class HourMicrosecondAdd extends ScalarFunction implements BinaryExpression, ExplicitlyCastableSignature, ComputeSignatureForDateArithmetic, PropagateNullable, DateAddSubMonotonic { public static final List SIGNATURES = ImmutableList.of( - FunctionSignature.ret(DateTimeV2Type.MAX) - .args(DateTimeV2Type.MAX, VarcharType.SYSTEM_DEFAULT), + FunctionSignature.ret(DateTimeV2Type.WILDCARD) + .args(DateTimeV2Type.WILDCARD, VarcharType.SYSTEM_DEFAULT), FunctionSignature.ret(TimeStampTzType.MAX) .args(TimeStampTzType.MAX, VarcharType.SYSTEM_DEFAULT) ); @@ -80,6 +80,12 @@ public Expression withConstantArgs(Expression literal) { @Override public FunctionSignature computeSignature(FunctionSignature signature) { signature = super.computeSignature(signature); - return signature.withArgumentType(0, DateTimeV2Type.MAX).withReturnType(DateTimeV2Type.MAX); + if (getArgument(0).getDataType() instanceof TimeStampTzType + || signature.argumentsTypes.get(0) instanceof TimeStampTzType) { + return signature.withArgumentType(0, TimeStampTzType.MAX).withReturnType(TimeStampTzType.MAX); + } + DateTimeV2Type returnType = DateTimeV2Type.forTypeWithMinimumScale( + getArgument(0).getDataType(), DateTimeV2Type.MAX_SCALE); + return signature.withArgumentType(0, returnType).withReturnType(returnType); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HourMicrosecondSub.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HourMicrosecondSub.java index 1d43cb8e94cc5b..2b7983543f2f51 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HourMicrosecondSub.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HourMicrosecondSub.java @@ -41,8 +41,8 @@ public class HourMicrosecondSub extends ScalarFunction implements BinaryExpression, ExplicitlyCastableSignature, ComputeSignatureForDateArithmetic, PropagateNullable, DateAddSubMonotonic { public static final List SIGNATURES = ImmutableList.of( - FunctionSignature.ret(DateTimeV2Type.MAX) - .args(DateTimeV2Type.MAX, VarcharType.SYSTEM_DEFAULT), + FunctionSignature.ret(DateTimeV2Type.WILDCARD) + .args(DateTimeV2Type.WILDCARD, VarcharType.SYSTEM_DEFAULT), FunctionSignature.ret(TimeStampTzType.MAX) .args(TimeStampTzType.MAX, VarcharType.SYSTEM_DEFAULT) ); @@ -80,6 +80,12 @@ public Expression withConstantArgs(Expression literal) { @Override public FunctionSignature computeSignature(FunctionSignature signature) { signature = super.computeSignature(signature); - return signature.withArgumentType(0, DateTimeV2Type.MAX).withReturnType(DateTimeV2Type.MAX); + if (getArgument(0).getDataType() instanceof TimeStampTzType + || signature.argumentsTypes.get(0) instanceof TimeStampTzType) { + return signature.withArgumentType(0, TimeStampTzType.MAX).withReturnType(TimeStampTzType.MAX); + } + DateTimeV2Type returnType = DateTimeV2Type.forTypeWithMinimumScale( + getArgument(0).getDataType(), DateTimeV2Type.MAX_SCALE); + return signature.withArgumentType(0, returnType).withReturnType(returnType); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MicroSecondsAdd.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MicroSecondsAdd.java index c83c2f8f5edc3b..125e8d16575bc2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MicroSecondsAdd.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MicroSecondsAdd.java @@ -40,7 +40,8 @@ public class MicroSecondsAdd extends ScalarFunction implements BinaryExpression, PropagateNullable, DateAddSubMonotonic { private static final List SIGNATURES = ImmutableList.of( - FunctionSignature.ret(DateTimeV2Type.MAX).args(DateTimeV2Type.MAX, BigIntType.INSTANCE), + FunctionSignature.ret(DateTimeV2Type.WILDCARD) + .args(DateTimeV2Type.WILDCARD, BigIntType.INSTANCE), FunctionSignature.ret(TimeStampTzType.MAX).args(TimeStampTzType.MAX, BigIntType.INSTANCE) ); @@ -67,10 +68,13 @@ public List getSignatures() { @Override public FunctionSignature computeSignature(FunctionSignature signature) { signature = super.computeSignature(signature); - if (signature.argumentsTypes.get(0) instanceof TimeStampTzType) { + if (getArgument(0).getDataType() instanceof TimeStampTzType + || signature.argumentsTypes.get(0) instanceof TimeStampTzType) { return signature.withArgumentType(0, TimeStampTzType.MAX).withReturnType(TimeStampTzType.MAX); } - return signature.withArgumentType(0, DateTimeV2Type.MAX).withReturnType(DateTimeV2Type.MAX); + DateTimeV2Type returnType = DateTimeV2Type.forTypeWithMinimumScale( + getArgument(0).getDataType(), DateTimeV2Type.MAX_SCALE); + return signature.withArgumentType(0, returnType).withReturnType(returnType); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MicroSecondsSub.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MicroSecondsSub.java index a407a945481918..5f6f54bc7b675a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MicroSecondsSub.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MicroSecondsSub.java @@ -40,7 +40,8 @@ public class MicroSecondsSub extends ScalarFunction implements BinaryExpression, PropagateNullable, DateAddSubMonotonic { private static final List SIGNATURES = ImmutableList - .of(FunctionSignature.ret(DateTimeV2Type.MAX).args(DateTimeV2Type.MAX, BigIntType.INSTANCE), + .of(FunctionSignature.ret(DateTimeV2Type.WILDCARD) + .args(DateTimeV2Type.WILDCARD, BigIntType.INSTANCE), FunctionSignature.ret(TimeStampTzType.MAX).args(TimeStampTzType.MAX, BigIntType.INSTANCE) ); @@ -67,10 +68,13 @@ public List getSignatures() { @Override public FunctionSignature computeSignature(FunctionSignature signature) { signature = super.computeSignature(signature); - if (signature.argumentsTypes.get(0) instanceof TimeStampTzType) { + if (getArgument(0).getDataType() instanceof TimeStampTzType + || signature.argumentsTypes.get(0) instanceof TimeStampTzType) { return signature.withArgumentType(0, TimeStampTzType.MAX).withReturnType(TimeStampTzType.MAX); } - return signature.withArgumentType(0, DateTimeV2Type.MAX).withReturnType(DateTimeV2Type.MAX); + DateTimeV2Type returnType = DateTimeV2Type.forTypeWithMinimumScale( + getArgument(0).getDataType(), DateTimeV2Type.MAX_SCALE); + return signature.withArgumentType(0, returnType).withReturnType(returnType); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MilliSecondsAdd.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MilliSecondsAdd.java index 4880e260eb2064..e4b0f95e044d73 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MilliSecondsAdd.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MilliSecondsAdd.java @@ -40,7 +40,8 @@ public class MilliSecondsAdd extends ScalarFunction implements BinaryExpression, PropagateNullable, DateAddSubMonotonic { private static final List SIGNATURES = ImmutableList - .of(FunctionSignature.ret(DateTimeV2Type.MAX).args(DateTimeV2Type.MAX, BigIntType.INSTANCE), + .of(FunctionSignature.ret(DateTimeV2Type.WILDCARD) + .args(DateTimeV2Type.WILDCARD, BigIntType.INSTANCE), FunctionSignature.ret(TimeStampTzType.MAX).args(TimeStampTzType.MAX, BigIntType.INSTANCE) ); @@ -67,10 +68,13 @@ public List getSignatures() { @Override public FunctionSignature computeSignature(FunctionSignature signature) { signature = super.computeSignature(signature); - if (signature.argumentsTypes.get(0) instanceof TimeStampTzType) { + if (getArgument(0).getDataType() instanceof TimeStampTzType + || signature.argumentsTypes.get(0) instanceof TimeStampTzType) { return signature.withArgumentType(0, TimeStampTzType.MAX).withReturnType(TimeStampTzType.MAX); } - return signature.withArgumentType(0, DateTimeV2Type.MAX).withReturnType(DateTimeV2Type.MAX); + DateTimeV2Type returnType = DateTimeV2Type.forTypeWithMinimumScale( + getArgument(0).getDataType(), DateTimeV2Type.MAX_SCALE); + return signature.withArgumentType(0, returnType).withReturnType(returnType); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MilliSecondsSub.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MilliSecondsSub.java index 2bf37e4b81fcc8..a844cabe446709 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MilliSecondsSub.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MilliSecondsSub.java @@ -40,7 +40,8 @@ public class MilliSecondsSub extends ScalarFunction implements BinaryExpression, PropagateNullable, DateAddSubMonotonic { private static final List SIGNATURES = ImmutableList - .of(FunctionSignature.ret(DateTimeV2Type.MAX).args(DateTimeV2Type.MAX, BigIntType.INSTANCE), + .of(FunctionSignature.ret(DateTimeV2Type.WILDCARD) + .args(DateTimeV2Type.WILDCARD, BigIntType.INSTANCE), FunctionSignature.ret(TimeStampTzType.MAX).args(TimeStampTzType.MAX, BigIntType.INSTANCE) ); @@ -67,10 +68,13 @@ public List getSignatures() { @Override public FunctionSignature computeSignature(FunctionSignature signature) { signature = super.computeSignature(signature); - if (signature.argumentsTypes.get(0) instanceof TimeStampTzType) { + if (getArgument(0).getDataType() instanceof TimeStampTzType + || signature.argumentsTypes.get(0) instanceof TimeStampTzType) { return signature.withArgumentType(0, TimeStampTzType.MAX).withReturnType(TimeStampTzType.MAX); } - return signature.withArgumentType(0, DateTimeV2Type.MAX).withReturnType(DateTimeV2Type.MAX); + DateTimeV2Type returnType = DateTimeV2Type.forTypeWithMinimumScale( + getArgument(0).getDataType(), DateTimeV2Type.MAX_SCALE); + return signature.withArgumentType(0, returnType).withReturnType(returnType); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MinuteMicrosecondAdd.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MinuteMicrosecondAdd.java index 135666d6d52cf4..41379116417ec3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MinuteMicrosecondAdd.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MinuteMicrosecondAdd.java @@ -41,8 +41,8 @@ public class MinuteMicrosecondAdd extends ScalarFunction implements BinaryExpression, ExplicitlyCastableSignature, ComputeSignatureForDateArithmetic, PropagateNullable, DateAddSubMonotonic { public static final List SIGNATURES = ImmutableList.of( - FunctionSignature.ret(DateTimeV2Type.MAX) - .args(DateTimeV2Type.MAX, VarcharType.SYSTEM_DEFAULT), + FunctionSignature.ret(DateTimeV2Type.WILDCARD) + .args(DateTimeV2Type.WILDCARD, VarcharType.SYSTEM_DEFAULT), FunctionSignature.ret(TimeStampTzType.MAX) .args(TimeStampTzType.MAX, VarcharType.SYSTEM_DEFAULT) ); @@ -80,6 +80,12 @@ public Expression withConstantArgs(Expression literal) { @Override public FunctionSignature computeSignature(FunctionSignature signature) { signature = super.computeSignature(signature); - return signature.withArgumentType(0, DateTimeV2Type.MAX).withReturnType(DateTimeV2Type.MAX); + if (getArgument(0).getDataType() instanceof TimeStampTzType + || signature.argumentsTypes.get(0) instanceof TimeStampTzType) { + return signature.withArgumentType(0, TimeStampTzType.MAX).withReturnType(TimeStampTzType.MAX); + } + DateTimeV2Type returnType = DateTimeV2Type.forTypeWithMinimumScale( + getArgument(0).getDataType(), DateTimeV2Type.MAX_SCALE); + return signature.withArgumentType(0, returnType).withReturnType(returnType); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MinuteMicrosecondSub.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MinuteMicrosecondSub.java index 45b7272d505f46..f3e53b460a1e28 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MinuteMicrosecondSub.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MinuteMicrosecondSub.java @@ -41,8 +41,8 @@ public class MinuteMicrosecondSub extends ScalarFunction implements BinaryExpression, ExplicitlyCastableSignature, ComputeSignatureForDateArithmetic, PropagateNullable, DateAddSubMonotonic { public static final List SIGNATURES = ImmutableList.of( - FunctionSignature.ret(DateTimeV2Type.MAX) - .args(DateTimeV2Type.MAX, VarcharType.SYSTEM_DEFAULT), + FunctionSignature.ret(DateTimeV2Type.WILDCARD) + .args(DateTimeV2Type.WILDCARD, VarcharType.SYSTEM_DEFAULT), FunctionSignature.ret(TimeStampTzType.MAX) .args(TimeStampTzType.MAX, VarcharType.SYSTEM_DEFAULT) ); @@ -80,6 +80,12 @@ public Expression withConstantArgs(Expression literal) { @Override public FunctionSignature computeSignature(FunctionSignature signature) { signature = super.computeSignature(signature); - return signature.withArgumentType(0, DateTimeV2Type.MAX).withReturnType(DateTimeV2Type.MAX); + if (getArgument(0).getDataType() instanceof TimeStampTzType + || signature.argumentsTypes.get(0) instanceof TimeStampTzType) { + return signature.withArgumentType(0, TimeStampTzType.MAX).withReturnType(TimeStampTzType.MAX); + } + DateTimeV2Type returnType = DateTimeV2Type.forTypeWithMinimumScale( + getArgument(0).getDataType(), DateTimeV2Type.MAX_SCALE); + return signature.withArgumentType(0, returnType).withReturnType(returnType); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SecondMicrosecondAdd.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SecondMicrosecondAdd.java index 8411699aedf2f9..6542fa142a026a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SecondMicrosecondAdd.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SecondMicrosecondAdd.java @@ -80,9 +80,12 @@ public Expression withConstantArgs(Expression literal) { @Override public FunctionSignature computeSignature(FunctionSignature signature) { signature = super.computeSignature(signature); - if (signature.argumentsTypes.get(0) instanceof TimeStampTzType) { + if (getArgument(0).getDataType() instanceof TimeStampTzType + || signature.argumentsTypes.get(0) instanceof TimeStampTzType) { return signature.withArgumentType(0, TimeStampTzType.MAX).withReturnType(TimeStampTzType.MAX); } - return signature.withArgumentType(0, DateTimeV2Type.MAX).withReturnType(DateTimeV2Type.MAX); + DateTimeV2Type returnType = DateTimeV2Type.forTypeWithMinimumScale( + getArgument(0).getDataType(), DateTimeV2Type.MAX_SCALE); + return signature.withArgumentType(0, returnType).withReturnType(returnType); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SecondMicrosecondSub.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SecondMicrosecondSub.java index af4350f28f26ba..2ec3d2d83256fc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SecondMicrosecondSub.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SecondMicrosecondSub.java @@ -80,9 +80,12 @@ public Expression withConstantArgs(Expression literal) { @Override public FunctionSignature computeSignature(FunctionSignature signature) { signature = super.computeSignature(signature); - if (signature.argumentsTypes.get(0) instanceof TimeStampTzType) { + if (getArgument(0).getDataType() instanceof TimeStampTzType + || signature.argumentsTypes.get(0) instanceof TimeStampTzType) { return signature.withArgumentType(0, TimeStampTzType.MAX).withReturnType(TimeStampTzType.MAX); } - return signature.withArgumentType(0, DateTimeV2Type.MAX).withReturnType(DateTimeV2Type.MAX); + DateTimeV2Type returnType = DateTimeV2Type.forTypeWithMinimumScale( + getArgument(0).getDataType(), DateTimeV2Type.MAX_SCALE); + return signature.withArgumentType(0, returnType).withReturnType(returnType); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/window/RequireTrivialTypes.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/window/RequireTrivialTypes.java index c868f3501b5bff..074b13dab7dca6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/window/RequireTrivialTypes.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/window/RequireTrivialTypes.java @@ -32,6 +32,7 @@ import org.apache.doris.nereids.types.LargeIntType; import org.apache.doris.nereids.types.SmallIntType; import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TimeStampTzType; import org.apache.doris.nereids.types.TimeV2Type; import org.apache.doris.nereids.types.TinyIntType; @@ -61,6 +62,7 @@ public interface RequireTrivialTypes { DateTimeType.INSTANCE, DateV2Type.INSTANCE, DateTimeV2Type.WILDCARD, + TimeStampNsType.INSTANCE, TimeV2Type.WILDCARD, TimeStampTzType.WILDCARD, VarcharType.SYSTEM_DEFAULT, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java index 3d1f5017be3b14..9af785bd94fee5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java @@ -26,6 +26,7 @@ import org.apache.doris.nereids.types.DateTimeType; import org.apache.doris.nereids.types.DateTimeV2Type; import org.apache.doris.nereids.types.DateType; +import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.coercion.DateLikeType; import org.apache.doris.nereids.util.DateTimeFormatterUtils; import org.apache.doris.nereids.util.DateUtils; @@ -254,14 +255,14 @@ static Result normalize(String s) { sb.append(":00"); } - // parse MicroSecond - // Keep up to 7 digits at most, 7th digit is use for overflow. + // Keep one more digit than Java's NANO_OF_SECOND supports. The tenth digit is the guard + // digit used when a DATETIMEV2(9) literal is rounded. int j = i; if (partNumber == 6 && i < s.length() && s.charAt(i) == '.') { sb.append(s.charAt(i)); i += 1; while (i < s.length() && Character.isDigit(s.charAt(i))) { - if (i - j <= 7) { + if (i - j <= DateUtils.NANOSECOND_SCALE + 1) { sb.append(s.charAt(i)); } i += 1; @@ -329,6 +330,7 @@ public static Result parseDateTime(String s } if (!containsPunctuation) { s = normalizeBasic(s); + s = DateUtils.truncateFractionalSecondForJavaParser(s); // mysql reject "20200219 010101" "200219 010101", can't use ' ' spilt basic date time. if (!s.contains("T")) { @@ -344,6 +346,7 @@ public static Result parseDateTime(String s return normalizeResult.cast(); } s = normalizeResult.get(); + s = DateUtils.truncateFractionalSecondForJavaParser(s); if (!s.contains(" ")) { dateTime = DateTimeFormatterUtils.ZONE_DATE_FORMATTER.parse(s); @@ -397,7 +400,7 @@ private static boolean checkDatetime(TemporalAccessor dateTime) { return DateUtils.getOrDefault(dateTime, ChronoField.HOUR_OF_DAY) != 0 || DateUtils.getOrDefault(dateTime, ChronoField.MINUTE_OF_HOUR) != 0 || DateUtils.getOrDefault(dateTime, ChronoField.SECOND_OF_MINUTE) != 0 - || DateUtils.getOrDefault(dateTime, ChronoField.MICRO_OF_SECOND) != 0; + || DateUtils.getOrDefault(dateTime, ChronoField.NANO_OF_SECOND) != 0; } @Override @@ -472,10 +475,8 @@ public int compareTo(ComparableLiteral other) { return cmp; } - long thisMicrosecond = this instanceof DateTimeV2Literal ? ((DateTimeV2Literal) this).getMicroSecond() : 0L; - long otherMicrosecond = other instanceof DateTimeV2Literal - ? ((DateTimeV2Literal) other).getMicroSecond() : 0L; - return Long.compare(thisMicrosecond, otherMicrosecond); + return Long.compare(getFractionalSecondInNanoseconds(), + ((DateLiteral) other).getFractionalSecondInNanoseconds()); } if (other instanceof NullLiteral) { return 1; @@ -500,6 +501,21 @@ public long getDay() { return day; } + /** Return the time of day in nanoseconds; DATE literals have no time part. */ + public long getTimePartInNanoseconds() { + return 0; + } + + /** Return the fractional second in nanoseconds; DATE literals have no fractional part. */ + public long getFractionalSecondInNanoseconds() { + return 0; + } + + /** DATE values represent the beginning of a day. Datetime subclasses override this as needed. */ + public boolean isMidnight() { + return true; + } + public int getDayOfYear() { if (year == 0 && month == 3 && (day == 1 || day == 2)) { return toJavaDateType().getDayOfYear() - 1; @@ -623,7 +639,10 @@ protected Expression uncheckedCastTo(DataType targetType) throws AnalysisExcepti } else if (targetType.isDoubleType()) { return new DoubleLiteral(value); } else if (targetType.isDateTimeV2Type()) { - return new DateTimeV2Literal((DateTimeV2Type) targetType, year, month, day, 0, 0, 0, 0); + if (targetType instanceof TimeStampNsType) { + return new TimeStampNsLiteral(year, month, day, 0, 0, 0, 0); + } + return DateTimeV2Literal.create((DateTimeV2Type) targetType, year, month, day, 0, 0, 0, 0); } else if (targetType.isDateTimeType()) { return new DateTimeLiteral((DateTimeType) targetType, year, month, day, 0, 0, 0, 0); } else if (targetType.isDateV2Type()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java index c50f98b45a1119..e9e7bea5d1006b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java @@ -26,27 +26,23 @@ import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.DateTimeType; import org.apache.doris.nereids.types.DateTimeV2Type; +import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TimeStampTzType; import org.apache.doris.nereids.types.TimeV2Type; import org.apache.doris.nereids.types.coercion.DateLikeType; import org.apache.doris.nereids.util.DateUtils; -import org.apache.doris.nereids.util.StandardDateFormat; import com.google.common.base.Preconditions; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.math.BigInteger; -import java.time.Instant; +import java.time.DateTimeException; import java.time.LocalDateTime; import java.time.ZoneId; -import java.time.ZoneOffset; import java.time.temporal.ChronoField; import java.time.temporal.TemporalAccessor; import java.time.temporal.TemporalQueries; -import java.time.zone.ZoneOffsetTransition; -import java.time.zone.ZoneRules; -import java.util.List; import java.util.Objects; /** @@ -56,6 +52,7 @@ public class DateTimeLiteral extends DateLiteral { public static final DateTimeLiteral MIN_DATETIME = new DateTimeLiteral(0000, 1, 1, 0, 0, 0); public static final DateTimeLiteral MAX_DATETIME = new DateTimeLiteral(9999, 12, 31, 23, 59, 59); protected static final int MAX_MICROSECOND = 999999; + protected static final int MAX_NANOSECOND = 999999999; private static final Logger LOG = LogManager.getLogger(DateTimeLiteral.class); @@ -63,6 +60,7 @@ public class DateTimeLiteral extends DateLiteral { protected long minute; protected long second; protected long microSecond; + protected long nanoSecond; public DateTimeLiteral(String s) { this(DateTimeType.INSTANCE, s); @@ -98,13 +96,15 @@ public DateTimeLiteral(DateLikeType dataType, long year, long month, long day, this.minute = minute; this.second = second; this.microSecond = microSecond; + this.nanoSecond = microSecond * 1000; this.year = year; this.month = month; this.day = day; } + @Override public boolean isMidnight() { - return hour == 0 && minute == 0 && second == 0 && microSecond == 0; + return hour == 0 && minute == 0 && second == 0 && nanoSecond == 0; } /** @@ -157,7 +157,7 @@ public static Result parseDateTimeLiteral(St ZoneId zoneId = temporal.query(TemporalQueries.zone()); if (zoneId != null) { - LocalDateTime localDateTime = convertTimeZone(year, month, day, hour, minute, second, + LocalDateTime localDateTime = DateUtils.convertTimeZone(year, month, day, hour, minute, second, zoneId, DateUtils.getTimeZone()); year = localDateTime.getYear(); month = localDateTime.getMonthValue(); @@ -167,7 +167,9 @@ public static Result parseDateTimeLiteral(St second = localDateTime.getSecond(); } - long microSecond = DateUtils.getOrDefault(temporal, ChronoField.NANO_OF_SECOND) / 100L; + long nanoSecond = DateUtils.getOrDefault(temporal, ChronoField.NANO_OF_SECOND); + DateTimeV2Type dateTimeV2Type = isV2 ? DateTimeV2Type.forTypeFromString(s) : null; + long microSecond = nanoSecond / 100L; // Microseconds have 7 digits. long sevenDigit = microSecond % 10; microSecond = microSecond / 10; @@ -188,8 +190,8 @@ public static Result parseDateTimeLiteral(St } if (isV2) { - DateTimeV2Type type = DateTimeV2Type.forTypeFromString(s); - return Result.ok(new DateTimeV2Literal(type, year, month, day, hour, minute, second, microSecond)); + return Result.ok(DateTimeV2Literal.create(dateTimeV2Type, year, month, day, hour, minute, second, + microSecond)); } else { return Result.ok(new DateTimeLiteral(DateTimeType.INSTANCE, year, month, day, hour, minute, second)); } @@ -198,21 +200,20 @@ public static Result parseDateTimeLiteral(St protected void roundMicroSecond(int scale) { Preconditions.checkArgument(scale >= 0 && scale <= DateTimeV2Type.MAX_SCALE, "invalid datetime v2 scale: %s", scale); - double factor = Math.pow(10, 6 - scale); - - this.microSecond = Math.round(this.microSecond / factor) * (int) factor; - - if (this.microSecond >= 1000000) { - LocalDateTime localDateTime = DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER_TO_MICRO_SECOND, - getStringValue()).plusSeconds(1); + long factor = (long) Math.pow(10, DateUtils.NANOSECOND_SCALE - scale); + this.nanoSecond = (this.nanoSecond + factor / 2) / factor * factor; + if (this.nanoSecond >= 1000000000) { + LocalDateTime localDateTime = LocalDateTime.of((int) year, (int) month, (int) day, + (int) hour, (int) minute, (int) second).plusSeconds(1); this.year = localDateTime.getYear(); this.month = localDateTime.getMonthValue(); this.day = localDateTime.getDayOfMonth(); this.hour = localDateTime.getHour(); this.minute = localDateTime.getMinute(); this.second = localDateTime.getSecond(); - this.microSecond -= 1000000; + this.nanoSecond -= 1000000000; } + this.microSecond = this.nanoSecond / 1000; if (checkRange() || checkDate(year, month, day)) { // may fallback to legacy planner. make sure the behaviour of rounding is same. throw new AnalysisException("datetime literal [" + toString() + "] is out of range"); @@ -232,7 +233,7 @@ protected void init(String s) throws AnalysisException { ZoneId zoneId = temporal.query(TemporalQueries.zone()); if (zoneId != null) { ZoneId targetZone = this.dataType instanceof TimeStampTzType ? ZoneId.of("UTC") : DateUtils.getTimeZone(); - LocalDateTime localDateTime = convertTimeZone(year, month, day, hour, minute, second, + LocalDateTime localDateTime = DateUtils.convertTimeZone(year, month, day, hour, minute, second, zoneId, targetZone); this.year = localDateTime.getYear(); this.month = localDateTime.getMonthValue(); @@ -242,13 +243,14 @@ protected void init(String s) throws AnalysisException { this.second = localDateTime.getSecond(); } - microSecond = DateUtils.getOrDefault(temporal, ChronoField.NANO_OF_SECOND) / 100L; + nanoSecond = DateUtils.getOrDefault(temporal, ChronoField.NANO_OF_SECOND); + microSecond = nanoSecond / 100L; // Microseconds have 7 digits. long sevenDigit = microSecond % 10; microSecond = microSecond / 10; + nanoSecond = microSecond * 1000; if (sevenDigit >= 5) { - DateTimeLiteral result; - result = this.plusMicroSeconds(1); + DateTimeLiteral result = this.plusMicroSeconds(1); this.second = result.second; this.minute = result.minute; this.hour = result.hour; @@ -256,6 +258,7 @@ protected void init(String s) throws AnalysisException { this.month = result.month; this.year = result.year; this.microSecond = result.microSecond; + this.nanoSecond = result.nanoSecond; } if (checkRange(year, month, day) || checkDate(year, month, day)) { @@ -269,40 +272,9 @@ public DateTimeLiteral plusMicroSeconds(long microSeconds) { return fromJavaDateType(toJavaDateType().plusNanos(Math.multiplyExact(microSeconds, 1000L)), 6); } - private static LocalDateTime convertTimeZone(long year, long month, long day, long hour, long minute, - long second, ZoneId fromZone, ZoneId toZone) { - LocalDateTime localDateTime = LocalDateTime.of((int) year, (int) month, (int) day, - (int) hour, (int) minute, (int) second); - Instant instant = convertLocalToInstant(localDateTime, fromZone); - return LocalDateTime.ofInstant(instant, toZone); - } - - /** - * Convert a local civil datetime in {@code fromZone} to an instant with the same DST transition - * policy as BE cctz::convert(civil_second, zone). - * - *

For normal local times, there is one valid offset. For fall-back overlap times, two offsets - * are valid and the first one is the pre-transition offset. For spring-forward gap times, the - * local time does not exist, so any value inside the skipped interval maps to the transition - * instant. - */ - public static Instant convertLocalToInstant(LocalDateTime localDateTime, ZoneId fromZone) { - ZoneRules rules = fromZone.getRules(); - List validOffsets = rules.getValidOffsets(localDateTime); - int size = validOffsets.size(); - // Match BE cctz::convert(civil_second, zone) semantics for constant folding. - // Normal local time has one offset; repeated local time uses the pre-transition offset. - if (size == 1 || size == 2) { - return localDateTime.atOffset(validOffsets.get(0)).toInstant(); - } - // Skipped local time maps to the transition instant, e.g. 2021-03-28 02:15 Europe/Paris. - ZoneOffsetTransition transition = rules.getTransition(localDateTime); - return transition.getInstant(); - } - public boolean checkRange() { return checkRange(year, month, day) || hour > MAX_DATETIME.getHour() || minute > MAX_DATETIME.getMinute() - || second > MAX_DATETIME.getSecond() || microSecond > MAX_MICROSECOND; + || second > MAX_DATETIME.getSecond() || nanoSecond > MAX_NANOSECOND; } @Override @@ -319,6 +291,16 @@ public long timePartToMicroSecond() { return ((hour * 60L + minute) * 60L + second) * 1000L * 1000L + microSecond; } + @Override + public long getTimePartInNanoseconds() { + return ((hour * 60L + minute) * 60L + second) * 1_000_000_000L + nanoSecond; + } + + @Override + public long getFractionalSecondInNanoseconds() { + return nanoSecond; + } + @Override public double getDouble() { return (double) getValue(); @@ -336,54 +318,7 @@ public String toString() { @Override public String getStringValue() { - if (0 <= year && year <= 9999 && 0 <= month && month <= 99 && 0 <= day && day <= 99 - && 0 <= hour && hour <= 99 && 0 <= minute && minute <= 99 && 0 <= second && second <= 99) { - char[] format = new char[] { - '0', '0', '0', '0', '-', '0', '0', '-', '0', '0', ' ', '0', '0', ':', '0', '0', ':', '0', '0'}; - int offset = 3; - long year = this.year; - while (year > 0) { - format[offset--] = (char) ('0' + (year % 10)); - year /= 10; - } - - offset = 6; - long month = this.month; - while (month > 0) { - format[offset--] = (char) ('0' + (month % 10)); - month /= 10; - } - - offset = 9; - long day = this.day; - while (day > 0) { - format[offset--] = (char) ('0' + (day % 10)); - day /= 10; - } - - offset = 12; - long hour = this.hour; - while (hour > 0) { - format[offset--] = (char) ('0' + (hour % 10)); - hour /= 10; - } - - offset = 15; - long minute = this.minute; - while (minute > 0) { - format[offset--] = (char) ('0' + (minute % 10)); - minute /= 10; - } - - offset = 18; - long second = this.second; - while (second > 0) { - format[offset--] = (char) ('0' + (second % 10)); - second /= 10; - } - return String.valueOf(format); - } - return String.format("%04d-%02d-%02d %02d:%02d:%02d", year, month, day, hour, minute, second); + return DateUtils.formatDateTime(year, month, day, hour, minute, second, nanoSecond, 0); } @Override @@ -411,14 +346,31 @@ protected Expression uncheckedCastTo(DataType targetType) throws AnalysisExcepti } else if (targetType.isDateTimeV2Type()) { // High scale datetime to low scale datetime may overflow. try { - return new DateTimeV2Literal((DateTimeV2Type) targetType, - year, month, day, hour, minute, second, microSecond); - } catch (AnalysisException e) { + DateTimeV2Type dateTimeV2Type = (DateTimeV2Type) targetType; + int scale = dateTimeV2Type.getScale(); + long factor = (long) Math.pow(10, DateUtils.NANOSECOND_SCALE - scale); + LocalDateTime roundedDateTime = LocalDateTime.of( + (int) year, (int) month, (int) day, + (int) hour, (int) minute, (int) second, (int) nanoSecond) + .plusNanos(factor / 2); + if (dateTimeV2Type instanceof TimeStampNsType) { + return TimeStampNsLiteral.fromJavaDateType(roundedDateTime); + } + return DateTimeV2Literal.fromJavaDateType(roundedDateTime, dateTimeV2Type); + } catch (AnalysisException | DateTimeException e) { throw new CastException(e.getMessage(), e); } } else if (targetType.isTimeType()) { - return new TimeV2Literal((int) hour, (int) minute, (int) second, (int) microSecond, - ((TimeV2Type) targetType).getScale(), false); + int scale = ((TimeV2Type) targetType).getScale(); + long factor = (long) Math.pow(10, DateUtils.NANOSECOND_SCALE - scale); + long roundedNanoSecond = (nanoSecond + factor / 2) / factor * factor; + long totalNanoSecond = ((hour * 60 + minute) * 60 + second) * 1000000000L + + roundedNanoSecond; + int resultHour = (int) (totalNanoSecond / 1000000000L / 60 / 60); + int resultMinute = (int) (totalNanoSecond / 1000000000L / 60 % 60); + int resultSecond = (int) (totalNanoSecond / 1000000000L % 60); + int resultMicroSecond = (int) (totalNanoSecond % 1000000000L / 1000); + return new TimeV2Literal(resultHour, resultMinute, resultSecond, resultMicroSecond, scale, false); } else if (targetType.isFloatType()) { return new FloatLiteral(getValue()); } else if (targetType.isDoubleType()) { @@ -476,6 +428,10 @@ public long getMicroSecond() { return microSecond; } + public long getNanoSecond() { + return nanoSecond; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -490,7 +446,7 @@ public boolean equals(Object o) { public LocalDateTime toJavaDateType() { return LocalDateTime.of(((int) getYear()), ((int) getMonth()), ((int) getDay()), - ((int) getHour()), ((int) getMinute()), ((int) getSecond()), (int) getMicroSecond() * 1000); + ((int) getHour()), ((int) getMinute()), ((int) getSecond()), (int) getNanoSecond()); } public static DateTimeLiteral fromJavaDateType(LocalDateTime dateTime, int precision) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeV2Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeV2Literal.java index bfdf1e3be6eba5..c7e7720cb64597 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeV2Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeV2Literal.java @@ -22,13 +22,15 @@ import org.apache.doris.nereids.exceptions.NotSupportedException; import org.apache.doris.nereids.exceptions.UnboundException; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeExtractAndTransform; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.DateTimeType; import org.apache.doris.nereids.types.DateTimeV2Type; +import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TimeStampTzType; -import org.apache.doris.qe.ConnectContext; +import org.apache.doris.nereids.util.DateUtils; + +import com.google.common.base.Preconditions; import java.time.LocalDateTime; import java.util.Objects; @@ -46,7 +48,7 @@ public DateTimeV2Literal(String s) { } public DateTimeV2Literal(DateTimeV2Type dateType, String s) { - super(dateType, s); + super(requireDateTimeV2Type(dateType), s); roundMicroSecond(dateType.getScale()); } @@ -60,17 +62,43 @@ public DateTimeV2Literal(long year, long month, long day, long hour, long minute public DateTimeV2Literal(DateTimeV2Type dateType, long year, long month, long day, long hour, long minute, long second, long microSecond) { - super(dateType, year, month, day, hour, minute, second, microSecond); + super(requireDateTimeV2Type(dateType), year, month, day, hour, minute, second, microSecond); roundMicroSecond(dateType.getScale()); } + private static DateTimeV2Type requireDateTimeV2Type(DateTimeV2Type dateType) { + Preconditions.checkArgument(!(dateType instanceof TimeStampNsType), + "Use TimeStampNsLiteral for TIMESTAMP_NS values"); + return dateType; + } + + /** Create a DATETIMEV2 literal. TIMESTAMP_NS has its own literal factory. */ + public static DateTimeV2Literal create(DateTimeV2Type dateType, String value) { + return new DateTimeV2Literal(dateType, value); + } + + /** Create a DATETIMEV2 literal. TIMESTAMP_NS has its own literal factory. */ + public static DateTimeV2Literal create(DateTimeV2Type dateType, + long year, long month, long day, long hour, long minute, long second, long fractionalSecond) { + return new DateTimeV2Literal(dateType, year, month, day, hour, minute, second, fractionalSecond); + } + + /** Create the last value of a date for a DATETIMEV2 type. */ + public static DateTimeV2Literal createEndOfDay(DateTimeV2Type dateType, long year, long month, long day) { + long microSecond = 0; + for (int i = 0; i < DateTimeV2Type.MAX_SCALE; i++) { + microSecond = microSecond * 10 + (i < dateType.getScale() ? 9 : 0); + } + return new DateTimeV2Literal(dateType, year, month, day, 23, 59, 59, microSecond); + } + /** Date difference rounded toward zero by time part. */ public static long dateDiffInDaysRoundToZeroByTime(DateLiteral lhs, DateLiteral rhs) { long days = DateV2Literal.dateDiffInDays(lhs, rhs); - long microSecondDiff = timePartToMicroSecond(lhs) - timePartToMicroSecond(rhs); - if (days > 0 && microSecondDiff < 0) { + long nanoSecondDiff = timePartToNanoSecond(lhs) - timePartToNanoSecond(rhs); + if (days > 0 && nanoSecondDiff < 0) { days--; - } else if (days < 0 && microSecondDiff > 0) { + } else if (days < 0 && nanoSecondDiff > 0) { days++; } return days; @@ -83,15 +111,28 @@ public static long datetimeDiffInSecondsRoundToZeroByMicroSecond(DateLiteral lhs /** Datetime difference in microseconds. */ public static long datetimeDiffInMicroSeconds(DateLiteral lhs, DateLiteral rhs) { - return DateV2Literal.dateDiffInDays(lhs, rhs) * 24L * 60L * 60L * 1000L * 1000L - + timePartToMicroSecond(lhs) - timePartToMicroSecond(rhs); + long secondDiff = DateV2Literal.dateDiffInDays(lhs, rhs) * 24L * 60L * 60L + + timePartToSecond(lhs) - timePartToSecond(rhs); + long nanoDiff = fractionalNanoSecond(lhs) - fractionalNanoSecond(rhs); + long result = secondDiff * 1000000L + nanoDiff / 1000L; + if (secondDiff > 0 && nanoDiff < 0 && nanoDiff % 1000 != 0) { + result--; + } else if (secondDiff < 0 && nanoDiff > 0 && nanoDiff % 1000 != 0) { + result++; + } + return result; } - private static long timePartToMicroSecond(DateLiteral date) { - if (date instanceof DateTimeLiteral) { - return ((DateTimeLiteral) date).timePartToMicroSecond(); - } - return 0; + private static long timePartToNanoSecond(DateLiteral date) { + return date.getTimePartInNanoseconds(); + } + + private static long timePartToSecond(DateLiteral date) { + return date.getTimePartInNanoseconds() / 1_000_000_000L; + } + + private static long fractionalNanoSecond(DateLiteral date) { + return date.getFractionalSecondInNanoseconds(); } @Override @@ -112,7 +153,7 @@ public LiteralExpr toLegacyLiteral() { @Override public double getDouble() { - return super.getDouble() + microSecond / 1000000.0; + return super.getDouble() + nanoSecond / 1000000000.0; } @Override @@ -123,71 +164,7 @@ public String toString() { @Override public String getStringValue() { int scale = getDataType().getScale(); - if (scale <= 0) { - return super.getStringValue(); - } - - if (0 <= year && year <= 9999 && 0 <= month && month <= 99 && 0 <= day && day <= 99 - && 0 <= hour && hour <= 99 && 0 <= minute && minute <= 99 && 0 <= second && second <= 99 - && 0 <= microSecond && microSecond <= MAX_MICROSECOND) { - char[] format = new char[] { - '0', '0', '0', '0', '-', '0', '0', '-', '0', '0', ' ', '0', '0', ':', '0', '0', ':', '0', '0', - '.', '0', '0', '0', '0', '0', '0'}; - int offset = 3; - long year = this.year; - while (year > 0) { - format[offset--] = (char) ('0' + (year % 10)); - year /= 10; - } - - offset = 6; - long month = this.month; - while (month > 0) { - format[offset--] = (char) ('0' + (month % 10)); - month /= 10; - } - - offset = 9; - long day = this.day; - while (day > 0) { - format[offset--] = (char) ('0' + (day % 10)); - day /= 10; - } - - offset = 12; - long hour = this.hour; - while (hour > 0) { - format[offset--] = (char) ('0' + (hour % 10)); - hour /= 10; - } - - offset = 15; - long minute = this.minute; - while (minute > 0) { - format[offset--] = (char) ('0' + (minute % 10)); - minute /= 10; - } - - offset = 18; - long second = this.second; - while (second > 0) { - format[offset--] = (char) ('0' + (second % 10)); - second /= 10; - } - - offset = 19 + scale; - long microSecond = (int) (this.microSecond / Math.pow(10, DateTimeV2Type.MAX_SCALE - scale)); - while (microSecond > 0) { - format[offset--] = (char) ('0' + (microSecond % 10)); - microSecond /= 10; - } - return String.valueOf(format, 0, 20 + scale); - } - - return String.format("%04d-%02d-%02d %02d:%02d:%02d" - + (scale > 0 ? ".%0" + scale + "d" : ""), - year, month, day, hour, minute, second, - (int) (microSecond / Math.pow(10, DateTimeV2Type.MAX_SCALE - scale))); + return DateUtils.formatDateTime(year, month, day, hour, minute, second, nanoSecond, scale); } @Override @@ -200,19 +177,13 @@ protected Expression uncheckedCastTo(DataType targetType) throws AnalysisExcepti year, month, day, hour, minute, second, microSecond); } if (targetType.isTimeStampTzType()) { - DateTimeV2Literal dtV2Lit = (DateTimeV2Literal) (DateTimeExtractAndTransform.convertTz( - this, - new StringLiteral(ConnectContext.get().getSessionVariable().timeZone), - new StringLiteral("UTC"))); - return new TimestampTzLiteral((TimeStampTzType) targetType, - dtV2Lit.getYear(), dtV2Lit.getMonth(), dtV2Lit.getDay(), - dtV2Lit.getHour(), dtV2Lit.getMinute(), dtV2Lit.getSecond(), dtV2Lit.getMicroSecond()); + return TimestampTzLiteral.fromSessionTimeZone((TimeStampTzType) targetType, this); } return super.uncheckedCastTo(targetType); } public Expression plusDays(long days) { - return fromJavaDateType(toJavaDateType().plusDays(days), getDataType().getScale()); + return fromJavaDateType(toJavaDateType().plusDays(days), getDataType()); } /** @@ -259,34 +230,34 @@ public Expression plusDaySecond(VarcharLiteral daySecond) { .plusDays(days) .plusHours(hours) .plusMinutes(minutes) - .plusSeconds(seconds), getDataType().getScale()); + .plusSeconds(seconds), getDataType()); } catch (NumberFormatException e) { throw new NotSupportedException("Invalid time format"); } } public Expression plusMonths(long months) { - return fromJavaDateType(toJavaDateType().plusMonths(months), getDataType().getScale()); + return fromJavaDateType(toJavaDateType().plusMonths(months), getDataType()); } public Expression plusWeeks(long weeks) { - return fromJavaDateType(toJavaDateType().plusWeeks(weeks), getDataType().getScale()); + return fromJavaDateType(toJavaDateType().plusWeeks(weeks), getDataType()); } public Expression plusYears(long years) { - return fromJavaDateType(toJavaDateType().plusYears(years), getDataType().getScale()); + return fromJavaDateType(toJavaDateType().plusYears(years), getDataType()); } public Expression plusHours(long hours) { - return fromJavaDateType(toJavaDateType().plusHours(hours), getDataType().getScale()); + return fromJavaDateType(toJavaDateType().plusHours(hours), getDataType()); } public Expression plusMinutes(long minutes) { - return fromJavaDateType(toJavaDateType().plusMinutes(minutes), getDataType().getScale()); + return fromJavaDateType(toJavaDateType().plusMinutes(minutes), getDataType()); } public Expression plusSeconds(long seconds) { - return fromJavaDateType(toJavaDateType().plusSeconds(seconds), getDataType().getScale()); + return fromJavaDateType(toJavaDateType().plusSeconds(seconds), getDataType()); } /** @@ -321,7 +292,7 @@ public Expression plusDayHour(VarcharLiteral dayHour) { return fromJavaDateType(toJavaDateType() .plusDays(days) - .plusHours(hours), getDataType().getScale()); + .plusHours(hours), getDataType()); } catch (NumberFormatException e) { throw new NotSupportedException("Invalid time format"); } @@ -359,7 +330,7 @@ public Expression plusMinuteSecond(VarcharLiteral minuteSecond) { return fromJavaDateType(toJavaDateType() .plusMinutes(minutes) - .plusSeconds(seconds), getDataType().getScale()); + .plusSeconds(seconds), getDataType()); } catch (NumberFormatException e) { throw new NotSupportedException("Invalid time format"); } @@ -401,7 +372,7 @@ public Expression plusSecondMicrosecond(VarcharLiteral secondMicrosecond) { return fromJavaDateType(toJavaDateType() .plusSeconds(seconds) - .plusNanos(Math.multiplyExact(microseconds, 1000L)), getDataType().getScale()); + .plusNanos(Math.multiplyExact(microseconds, 1000L)), getDataType()); } catch (NumberFormatException e) { throw new NotSupportedException("Invalid time format"); } @@ -410,7 +381,9 @@ public Expression plusSecondMicrosecond(VarcharLiteral secondMicrosecond) { // When performing addition or subtraction with MicroSeconds, the precision must be set to 6 to display it // completely. use multiplyExact to be aware of multiplication overflow possibility. public DateTimeV2Literal plusMicroSeconds(long microSeconds) { - return fromJavaDateType(toJavaDateType().plusNanos(Math.multiplyExact(microSeconds, 1000L)), 6); + return fromJavaDateType( + toJavaDateType().plusNanos(Math.multiplyExact(microSeconds, 1000L)), + DateTimeV2Type.MAX); } public Expression plusMilliSeconds(long microSeconds) { @@ -421,16 +394,19 @@ public int getScale() { return ((DateTimeV2Type) dataType).getScale(); } - public int commonScale(DateTimeV2Literal other) { - return (int) Math.max(getScale(), other.getScale()); + public DateTimeV2Type commonType(DateTimeV2Literal other) { + return DateTimeV2Type.getWiderDatetimeV2Type(getDataType(), other.getDataType()); } /** * roundCeiling */ public DateTimeV2Literal roundCeiling(int newScale) { - long remain = Double.valueOf(microSecond % (Math.pow(10, 6 - newScale))).longValue(); - long newMicroSecond = microSecond; + DateTimeV2Type targetType = DateTimeV2Type.of(newScale); + int targetScale = targetType.getScale(); + long factor = (long) Math.pow(10, DateUtils.NANOSECOND_SCALE - targetScale); + long remain = nanoSecond % factor; + long newNanoSecond = nanoSecond; long newSecond = second; long newMinute = minute; long newHour = hour; @@ -438,13 +414,10 @@ public DateTimeV2Literal roundCeiling(int newScale) { long newMonth = month; long newYear = year; if (remain != 0) { - newMicroSecond = Double - .valueOf((microSecond + (int) (Math.pow(10, 6 - newScale))) - / (int) (Math.pow(10, 6 - newScale)) * (Math.pow(10, 6 - newScale))) - .longValue(); + newNanoSecond = (nanoSecond + factor) / factor * factor; } - if (newMicroSecond > MAX_MICROSECOND) { - newMicroSecond %= newMicroSecond; + if (newNanoSecond > MAX_NANOSECOND) { + newNanoSecond = 0; Expression plus1Second = this.plusSeconds(1); if (plus1Second.isNullLiteral()) { throw new AnalysisException("round ceil datetime literal (" + toString() + ", " @@ -458,13 +431,18 @@ public DateTimeV2Literal roundCeiling(int newScale) { newMonth = result.month; newYear = result.year; } - return new DateTimeV2Literal(DateTimeV2Type.of(newScale), newYear, newMonth, newDay, - newHour, newMinute, newSecond, newMicroSecond); + return create(targetType, newYear, newMonth, newDay, + newHour, newMinute, newSecond, newNanoSecond / 1000); } + /** Round down the fractional second to the requested normalized datetime scale. */ public DateTimeV2Literal roundFloor(int newScale) { - return new DateTimeV2Literal(DateTimeV2Type.of(newScale), year, month, day, hour, minute, second, - microSecond / (int) Math.pow(10, 6 - newScale) * (int) Math.pow(10, 6 - newScale)); + DateTimeV2Type targetType = DateTimeV2Type.of(newScale); + int targetScale = targetType.getScale(); + long factor = (long) Math.pow(10, DateUtils.NANOSECOND_SCALE - targetScale); + long newNanoSecond = nanoSecond / factor * factor; + return create(targetType, year, month, day, hour, minute, second, + newNanoSecond / 1000); } public static Expression fromJavaDateType(LocalDateTime dateTime) { @@ -475,14 +453,21 @@ public static Expression fromJavaDateType(LocalDateTime dateTime) { * convert java LocalDateTime object to DateTimeV2Literal object. */ public static DateTimeV2Literal fromJavaDateType(LocalDateTime dateTime, int precision) { - long value = (long) Math.pow(10, DateTimeV2Type.MAX_SCALE - precision); + return fromJavaDateType(dateTime, DateTimeV2Type.of(precision)); + } + + /** Convert a Java datetime to the explicit DATETIMEV2 target type. */ + public static DateTimeV2Literal fromJavaDateType(LocalDateTime dateTime, DateTimeV2Type targetType) { + requireDateTimeV2Type(targetType); + int targetScale = targetType.getScale(); + long factor = (long) Math.pow(10, DateUtils.NANOSECOND_SCALE - targetScale); if (isDateOutOfRange(dateTime)) { throw new AnalysisException("datetime out of range" + dateTime.toString()); } - return new DateTimeV2Literal(DateTimeV2Type.of(precision), dateTime.getYear(), + long nanoSecond = dateTime.getNano() / factor * factor; + return create(targetType, dateTime.getYear(), dateTime.getMonthValue(), dateTime.getDayOfMonth(), dateTime.getHour(), - dateTime.getMinute(), dateTime.getSecond(), - (dateTime.getNano() / 1000) / value * value); + dateTime.getMinute(), dateTime.getSecond(), nanoSecond / 1000); } @Override @@ -490,13 +475,12 @@ public boolean equals(Object o) { if (this == o) { return true; } - if (o == null || getClass() != o.getClass()) { - return false; - } - if (!super.equals(o)) { + if (!(o instanceof DateTimeV2Literal)) { return false; } DateTimeV2Literal literal = (DateTimeV2Literal) o; - return Objects.equals(dataType, literal.dataType) && Objects.equals(microSecond, literal.microSecond); + return Objects.equals(getValue(), literal.getValue()) + && Objects.equals(dataType, literal.dataType) + && Objects.equals(nanoSecond, literal.nanoSecond); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateV2Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateV2Literal.java index 1462397bb08092..d7a108cce4f493 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateV2Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateV2Literal.java @@ -95,7 +95,7 @@ public DateTimeV2Literal toBeginOfTheDay() { * @return 2020-01-01 00:00:00 */ public DateTimeV2Literal toBeginOfTheDay(DateTimeV2Type dateType) { - return new DateTimeV2Literal(dateType, year, month, day, 0, 0, 0, 000000); + return DateTimeV2Literal.create(dateType, year, month, day, 0, 0, 0, 000000); } /** @@ -111,15 +111,7 @@ public DateTimeV2Literal toEndOfTheDay() { * @return 2020-01-01 23:59:59.9[scale] */ public DateTimeV2Literal toEndOfTheDay(DateTimeV2Type dateType) { - long microSecond = 0; - // eg. scale == 4 -> 999900 - for (int i = 0; i < 6; ++i) { - microSecond *= 10; - if (i < dateType.getScale()) { - microSecond += 9; - } - } - return new DateTimeV2Literal(dateType, year, month, day, 23, 59, 59, microSecond); + return DateTimeV2Literal.createEndOfDay(dateType, year, month, day); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java index 1c368179005be3..b252f30a86a60c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java @@ -335,11 +335,22 @@ public static Literal fromLegacyLiteral(LiteralExpr literalExpr, Type type) { } case DATETIMEV2: { org.apache.doris.analysis.DateLiteral dateLiteral = (org.apache.doris.analysis.DateLiteral) literalExpr; + DateTimeV2Type dateTimeV2Type = (DateTimeV2Type) DateType.fromCatalogType(type); + long fractionalSecond = dateLiteral.getMicrosecond(); return new DateTimeV2Literal( - (DateTimeV2Type) DateType.fromCatalogType(type), + dateTimeV2Type, dateLiteral.getYear(), dateLiteral.getMonth(), dateLiteral.getDay(), dateLiteral.getHour(), dateLiteral.getMinute(), dateLiteral.getSecond(), - dateLiteral.getMicrosecond() + fractionalSecond + ); + } + case TIMESTAMP_NS: { + org.apache.doris.analysis.TimeStampNsLiteral timestampNsLiteral + = (org.apache.doris.analysis.TimeStampNsLiteral) literalExpr; + return new TimeStampNsLiteral( + timestampNsLiteral.getYear(), timestampNsLiteral.getMonth(), timestampNsLiteral.getDay(), + timestampNsLiteral.getHour(), timestampNsLiteral.getMinute(), timestampNsLiteral.getSecond(), + timestampNsLiteral.getNanosecond() ); } case TIMESTAMPTZ: { @@ -628,7 +639,8 @@ private static Literal handleDateTimeLiteral(ByteBuffer data) { microsecond = data.getInt(); } if (Config.enable_date_conversion) { - return new DateTimeV2Literal(DateTimeV2Type.MAX, year, month, day, hour, minute, second, microsecond); + return new DateTimeV2Literal(DateTimeV2Type.MAX, + year, month, day, hour, minute, second, microsecond); } return new DateTimeLiteral(DateTimeType.INSTANCE, year, month, day, hour, minute, second, microsecond); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/NumericLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/NumericLiteral.java index c871a7dc03cc7a..c46aa019e98be4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/NumericLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/NumericLiteral.java @@ -25,6 +25,7 @@ import org.apache.doris.nereids.types.DateTimeV2Type; import org.apache.doris.nereids.types.DateType; import org.apache.doris.nereids.types.DateV2Type; +import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TimeStampTzType; import java.math.BigDecimal; @@ -154,23 +155,26 @@ protected String getDateTimeString(long value) { } protected Expression getDateLikeLiteral(String s, DataType targetType) { - DateTimeV2Literal l; + DateTimeV2Literal dateTime; try { - l = new DateTimeV2Literal(DateTimeV2Type.MAX, s); + dateTime = new DateTimeV2Literal(DateTimeV2Type.MAX, s); } catch (AnalysisException e) { throw new CastException(e.getMessage(), e); } if (targetType instanceof DateType) { - return new DateLiteral(l.getYear(), l.getMonth(), l.getDay()); + return new DateLiteral(dateTime.getYear(), dateTime.getMonth(), dateTime.getDay()); } if (targetType instanceof DateV2Type) { - return new DateV2Literal(l.getYear(), l.getMonth(), l.getDay()); + return new DateV2Literal(dateTime.getYear(), dateTime.getMonth(), dateTime.getDay()); } if (targetType instanceof DateTimeType) { return new DateTimeLiteral(s); } + if (targetType instanceof TimeStampNsType) { + return new TimeStampNsLiteral(s); + } if (targetType instanceof DateTimeV2Type) { - return new DateTimeV2Literal((DateTimeV2Type) targetType, s); + return DateTimeV2Literal.create((DateTimeV2Type) targetType, s); } if (targetType instanceof TimeStampTzType) { return new TimestampTzLiteral((TimeStampTzType) targetType, s); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteral.java index afe473ccf75ca7..f6ad99ad91b924 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteral.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.trees.expressions.literal.format.DateTimeChecker; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.DateTimeV2Type; +import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TimeStampTzType; import org.apache.doris.nereids.types.TimeV2Type; import org.apache.doris.qe.SessionVariable; @@ -144,7 +145,8 @@ protected Expression uncheckedCastTo(DataType targetType) throws AnalysisExcepti if (DateTimeChecker.hasTimeZone(value)) { return new TimestampTzLiteral(timeStampTzType, value); } - DateTimeV2Literal datetime = (DateTimeV2Literal) castToDateTime(DateTimeV2Type.MAX, strictCast); + DateTimeV2Literal datetime = (DateTimeV2Literal) castToDateTime( + DateTimeV2Type.MAX, strictCast); return TimestampTzLiteral.fromSessionTimeZone(timeStampTzType, datetime); } else if (targetType.isDateTimeV2Type()) { return castToDateTime(targetType, strictCast); @@ -379,9 +381,15 @@ protected DateLiteral getDateTimeLiteral(String year, String month, String date, } catch (AnalysisException e) { throw new CastException(e.getMessage(), e); } + } else if (targetType instanceof TimeStampNsType) { + try { + return new TimeStampNsLiteral(format); + } catch (AnalysisException e) { + throw new CastException(e.getMessage(), e); + } } else { try { - return new DateTimeV2Literal((DateTimeV2Type) targetType, format); + return DateTimeV2Literal.create((DateTimeV2Type) targetType, format); } catch (AnalysisException e) { throw new CastException(e.getMessage(), e); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimeStampNsLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimeStampNsLiteral.java new file mode 100644 index 00000000000000..805e4bca6cd3e4 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimeStampNsLiteral.java @@ -0,0 +1,332 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.literal; + +import org.apache.doris.analysis.LiteralExpr; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.exceptions.CastException; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.DateTimeType; +import org.apache.doris.nereids.types.DateTimeV2Type; +import org.apache.doris.nereids.types.TimeStampNsType; +import org.apache.doris.nereids.types.TimeStampTzType; +import org.apache.doris.nereids.types.TimeV2Type; +import org.apache.doris.nereids.util.DateUtils; + +import java.math.BigInteger; +import java.time.DateTimeException; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.temporal.ChronoField; +import java.time.temporal.TemporalAccessor; +import java.time.temporal.TemporalQueries; +import java.util.Objects; + +/** Literal for the fixed nanosecond-precision TIMESTAMP_NS type. */ +public final class TimeStampNsLiteral extends DateLiteral { + private static final long NANOS_PER_SECOND = 1_000_000_000L; + private static final long MAX_NANOSECOND = NANOS_PER_SECOND - 1; + private static final LocalDateTime MIN_VALUE + = LocalDateTime.of(1677, 9, 21, 0, 12, 43, 145224192); + private static final LocalDateTime MAX_VALUE + = LocalDateTime.of(2262, 4, 11, 23, 47, 16, 854775807); + + private final long hour; + private final long minute; + private final long second; + private final long nanosecond; + + public TimeStampNsLiteral(String value) { + this(parse(value)); + } + + /** Construct a TIMESTAMP_NS literal from civil datetime fields. */ + public TimeStampNsLiteral(long year, long month, long day, long hour, long minute, long second, + long nanosecond) { + super(TimeStampNsType.INSTANCE, year, month, day); + this.hour = hour; + this.minute = minute; + this.second = second; + this.nanosecond = nanosecond; + if (checkRange()) { + throw new AnalysisException("timestamp_ns literal [" + toString() + + "] is outside Int64 epoch nanosecond range"); + } + } + + private TimeStampNsLiteral(LocalDateTime value) { + this(value.getYear(), value.getMonthValue(), value.getDayOfMonth(), + value.getHour(), value.getMinute(), value.getSecond(), value.getNano()); + } + + private static LocalDateTime parse(String value) { + TemporalAccessor temporal = parseDateTime(value).get(); + long year = DateUtils.getOrDefault(temporal, ChronoField.YEAR); + long month = DateUtils.getOrDefault(temporal, ChronoField.MONTH_OF_YEAR); + long day = DateUtils.getOrDefault(temporal, ChronoField.DAY_OF_MONTH); + long hour = DateUtils.getOrDefault(temporal, ChronoField.HOUR_OF_DAY); + long minute = DateUtils.getOrDefault(temporal, ChronoField.MINUTE_OF_HOUR); + long second = DateUtils.getOrDefault(temporal, ChronoField.SECOND_OF_MINUTE); + + ZoneId zoneId = temporal.query(TemporalQueries.zone()); + if (zoneId != null) { + LocalDateTime converted = DateUtils.convertTimeZone( + year, month, day, hour, minute, second, zoneId, DateUtils.getTimeZone()); + year = converted.getYear(); + month = converted.getMonthValue(); + day = converted.getDayOfMonth(); + hour = converted.getHour(); + minute = converted.getMinute(); + second = converted.getSecond(); + } + + LocalDateTime result = LocalDateTime.of((int) year, (int) month, (int) day, + (int) hour, (int) minute, (int) second, + DateUtils.getOrDefault(temporal, ChronoField.NANO_OF_SECOND)); + if (DateUtils.getNanosecondGuardDigit(value) >= 5) { + result = result.plusNanos(1); + } + return result; + } + + public static TimeStampNsLiteral getMinValue() { + return new TimeStampNsLiteral(MIN_VALUE); + } + + public static TimeStampNsLiteral getMaxValue() { + return new TimeStampNsLiteral(MAX_VALUE); + } + + public static TimeStampNsLiteral createEndOfDay(long year, long month, long day) { + return new TimeStampNsLiteral(year, month, day, 23, 59, 59, MAX_NANOSECOND); + } + + public static TimeStampNsLiteral fromJavaDateType(LocalDateTime dateTime) { + return new TimeStampNsLiteral(dateTime); + } + + public boolean checkRange() { + if (checkRange(year, month, day) || month < 1 || day < 1 || checkDate(year, month, day) + || hour < 0 || hour > 23 || minute < 0 || minute > 59 + || second < 0 || second > 59 || nanosecond < 0 || nanosecond > MAX_NANOSECOND) { + return true; + } + LocalDateTime value = toJavaDateType(); + return value.isBefore(MIN_VALUE) || value.isAfter(MAX_VALUE); + } + + @Override + public boolean isMidnight() { + return hour == 0 && minute == 0 && second == 0 && nanosecond == 0; + } + + @Override + public TimeStampNsType getDataType() { + return TimeStampNsType.INSTANCE; + } + + @Override + public Long getValue() { + return (year * 10000 + month * 100 + day) * 1000000L + hour * 10000 + minute * 100 + second; + } + + @Override + public long getTimePartInNanoseconds() { + return ((hour * 60L + minute) * 60L + second) * NANOS_PER_SECOND + nanosecond; + } + + @Override + public long getFractionalSecondInNanoseconds() { + return nanosecond; + } + + @Override + public double getDouble() { + return getValue() + nanosecond / (double) NANOS_PER_SECOND; + } + + @Override + public String getStringValue() { + return DateUtils.formatDateTime(year, month, day, hour, minute, second, + nanosecond, TimeStampNsType.SCALE); + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitTimeStampNsLiteral(this, context); + } + + @Override + public LiteralExpr toLegacyLiteral() { + return new org.apache.doris.analysis.TimeStampNsLiteral( + year, month, day, hour, minute, second, nanosecond); + } + + @Override + protected Expression uncheckedCastTo(DataType targetType) throws AnalysisException { + if (dataType.equals(targetType)) { + return this; + } + if (targetType.isIntegralType()) { + if (targetType.isBigIntType()) { + return new BigIntLiteral(getValue()); + } else if (targetType.isLargeIntType()) { + return new LargeIntLiteral(new BigInteger(String.valueOf(getValue()))); + } + throw new AnalysisException("TIMESTAMP_NS can not cast to " + targetType); + } else if (targetType.isDateV2Type()) { + return new DateV2Literal(year, month, day); + } else if (targetType.isDateType()) { + return new DateLiteral(year, month, day); + } else if (targetType.isDateTimeType()) { + return new DateTimeLiteral((DateTimeType) targetType, + year, month, day, hour, minute, second, nanosecond / 1000); + } else if (targetType.isDateTimeV2Type()) { + try { + DateTimeV2Type dateTimeV2Type = (DateTimeV2Type) targetType; + int scale = dateTimeV2Type.getScale(); + long factor = (long) Math.pow(10, DateUtils.NANOSECOND_SCALE - scale); + LocalDateTime roundedDateTime = toJavaDateType().plusNanos(factor / 2); + return DateTimeV2Literal.fromJavaDateType(roundedDateTime, dateTimeV2Type); + } catch (AnalysisException | DateTimeException e) { + throw new CastException(e.getMessage(), e); + } + } else if (targetType.isTimeStampTzType()) { + TimeStampTzType timeStampTzType = (TimeStampTzType) targetType; + DateTimeV2Literal rounded = (DateTimeV2Literal) uncheckedCastTo( + DateTimeV2Type.of(timeStampTzType.getScale())); + return TimestampTzLiteral.fromSessionTimeZone(timeStampTzType, rounded); + } else if (targetType.isTimeType()) { + int scale = ((TimeV2Type) targetType).getScale(); + long factor = (long) Math.pow(10, DateUtils.NANOSECOND_SCALE - scale); + long roundedNanosecond = (nanosecond + factor / 2) / factor * factor; + long totalNanosecond = ((hour * 60 + minute) * 60 + second) * NANOS_PER_SECOND + + roundedNanosecond; + int resultHour = (int) (totalNanosecond / NANOS_PER_SECOND / 60 / 60); + int resultMinute = (int) (totalNanosecond / NANOS_PER_SECOND / 60 % 60); + int resultSecond = (int) (totalNanosecond / NANOS_PER_SECOND % 60); + int resultMicrosecond = (int) (totalNanosecond % NANOS_PER_SECOND / 1000); + return new TimeV2Literal(resultHour, resultMinute, resultSecond, + resultMicrosecond, scale, false); + } else if (targetType.isFloatType()) { + return new FloatLiteral(getValue()); + } else if (targetType.isDoubleType()) { + return new DoubleLiteral(getValue()); + } + return super.uncheckedCastTo(targetType); + } + + public TimeStampNsLiteral plusDays(long days) { + return fromJavaDateType(toJavaDateType().plusDays(days)); + } + + public TimeStampNsLiteral plusMonths(long months) { + return fromJavaDateType(toJavaDateType().plusMonths(months)); + } + + public TimeStampNsLiteral plusWeeks(long weeks) { + return fromJavaDateType(toJavaDateType().plusWeeks(weeks)); + } + + public TimeStampNsLiteral plusYears(long years) { + return fromJavaDateType(toJavaDateType().plusYears(years)); + } + + public TimeStampNsLiteral plusHours(long hours) { + return fromJavaDateType(toJavaDateType().plusHours(hours)); + } + + public TimeStampNsLiteral plusMinutes(long minutes) { + return fromJavaDateType(toJavaDateType().plusMinutes(minutes)); + } + + public TimeStampNsLiteral plusSeconds(long seconds) { + return fromJavaDateType(toJavaDateType().plusSeconds(seconds)); + } + + public TimeStampNsLiteral plusMicroSeconds(long microseconds) { + return fromJavaDateType(toJavaDateType().plusNanos(Math.multiplyExact(microseconds, 1000L))); + } + + public TimeStampNsLiteral plusMilliSeconds(long milliseconds) { + return plusMicroSeconds(Math.multiplyExact(milliseconds, 1000L)); + } + + public DateTimeV2Literal roundFloorToDateTimeV2(int scale) { + long factor = (long) Math.pow(10, DateUtils.NANOSECOND_SCALE - scale); + LocalDateTime rounded = toJavaDateType().withNano((int) (nanosecond / factor * factor)); + return DateTimeV2Literal.fromJavaDateType(rounded, scale); + } + + public DateTimeV2Literal roundCeilingToDateTimeV2(int scale) { + long factor = (long) Math.pow(10, DateUtils.NANOSECOND_SCALE - scale); + long remainder = nanosecond % factor; + LocalDateTime rounded = remainder == 0 ? toJavaDateType() + : toJavaDateType().plusNanos(factor - remainder); + return DateTimeV2Literal.fromJavaDateType(rounded, scale); + } + + @Override + public LocalDateTime toJavaDateType() { + return LocalDateTime.of((int) year, (int) month, (int) day, + (int) hour, (int) minute, (int) second, (int) nanosecond); + } + + public long getHour() { + return hour; + } + + public long getMinute() { + return minute; + } + + public long getSecond() { + return second; + } + + public long getMicroSecond() { + return nanosecond / 1000; + } + + public long getNanoSecond() { + return nanosecond; + } + + public int getScale() { + return TimeStampNsType.SCALE; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof TimeStampNsLiteral)) { + return false; + } + TimeStampNsLiteral literal = (TimeStampNsLiteral) o; + return Objects.equals(getValue(), literal.getValue()) && nanosecond == literal.nanosecond; + } + + @Override + protected int computeHashCode() { + return Objects.hash(getValue(), nanosecond); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimeV2Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimeV2Literal.java index 8d04a006baf51a..bab104370b1ad7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimeV2Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimeV2Literal.java @@ -315,7 +315,7 @@ protected Expression uncheckedCastTo(DataType targetType) throws AnalysisExcepti return new DateTimeLiteral(time.getYear(), time.getMonth(), time.getDay(), time.getHour(), time.getMinute(), time.getSecond()); } else if (targetType.isDateTimeV2Type()) { - return time; + return time.uncheckedCastTo(targetType); } return super.uncheckedCastTo(targetType); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteral.java index 810168d4985a51..359ff0e0b90f4f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteral.java @@ -109,8 +109,10 @@ public static TimestampTzLiteral fromTimeZone(TimeStampTzType dateType, String s */ public static TimestampTzLiteral fromTimeZone(TimeStampTzType dateType, DateTimeV2Literal literal, String timeZone) { + DateTimeV2Literal roundedLiteral = (DateTimeV2Literal) literal.uncheckedCastTo( + DateTimeV2Type.of(dateType.getScale())); DateTimeV2Literal utcLiteral = (DateTimeV2Literal) DateTimeExtractAndTransform.convertTz( - literal, + roundedLiteral, new StringLiteral(timeZone), new StringLiteral("UTC")); return new TimestampTzLiteral(dateType, @@ -171,13 +173,13 @@ protected Expression uncheckedCastTo(DataType targetType) throws AnalysisExcepti return new TimestampTzLiteral((TimeStampTzType) targetType, year, month, day, hour, minute, second, microSecond); } else if (targetType.isDateTimeV2Type()) { - DateTimeV2Literal dtV2Lit = new DateTimeV2Literal((DateTimeV2Type) targetType, + DateTimeV2Literal dtV2Lit = DateTimeV2Literal.create(DateTimeV2Type.of(getDataType().getScale()), year, month, day, hour, minute, second, microSecond); dtV2Lit = (DateTimeV2Literal) (DateTimeExtractAndTransform.convertTz( dtV2Lit, new StringLiteral("UTC"), new StringLiteral(ConnectContext.get().getSessionVariable().timeZone))); - return dtV2Lit; + return dtV2Lit.uncheckedCastTo(targetType); } throw new AnalysisException(String.format("Cast from %s to %s not supported", this, targetType)); } @@ -440,7 +442,7 @@ public TimestampTzLiteral roundFloor(int newScale) { @Override protected void roundMicroSecond(int scale) { - Preconditions.checkArgument(scale >= 0 && scale <= DateTimeV2Type.MAX_SCALE, + Preconditions.checkArgument(scale >= 0 && scale <= TimeStampTzType.MAX_SCALE, "invalid datetime v2 scale: %s", scale); double factor = Math.pow(10, 6 - scale); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java index a25b110dfb438a..acf4d0a35711d9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java @@ -120,6 +120,7 @@ import org.apache.doris.nereids.trees.expressions.literal.SmallIntLiteral; import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; import org.apache.doris.nereids.trees.expressions.literal.StructLiteral; +import org.apache.doris.nereids.trees.expressions.literal.TimeStampNsLiteral; import org.apache.doris.nereids.trees.expressions.literal.TimeV2Literal; import org.apache.doris.nereids.trees.expressions.literal.TimestampTzLiteral; import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; @@ -337,6 +338,10 @@ public R visitDateTimeV2Literal(DateTimeV2Literal dateTimeV2Literal, C context) return visitLiteral(dateTimeV2Literal, context); } + public R visitTimeStampNsLiteral(TimeStampNsLiteral timeStampNsLiteral, C context) { + return visitLiteral(timeStampNsLiteral, context); + } + public R visitTimestampTzLiteral(TimestampTzLiteral timestampTzLiteral, C context) { return visitLiteral(timestampTzLiteral, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateFunctionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateFunctionCommand.java index 66a4c1f428c82f..555b22e715bcf5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateFunctionCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateFunctionCommand.java @@ -425,6 +425,10 @@ private void checkUdfSupportedTypes() throws AnalysisException { } private void checkUdfSupportedType(Type type, String typePosition) throws AnalysisException { + if (binaryType == Function.BinaryType.JAVA_UDF && type.isTimeStampNs()) { + throw new AnalysisException(String.format( + "%s does not support %s type %s", binaryType, typePosition, type.toSql())); + } // Reject bitmap/hll/quantile_state type if (type.isObjectStored()) { throw new AnalysisException(String.format( @@ -1063,6 +1067,9 @@ private Types.PGenericType convertToPParameterType(Type arg) throws AnalysisExce case TIMEV2: typeBuilder.setId(Types.PGenericType.TypeId.DATETIMEV2); break; + case TIMESTAMP_NS: + typeBuilder.setId(Types.PGenericType.TypeId.TIMESTAMP_NS); + break; case TIMESTAMPTZ: typeBuilder.setId(Types.PGenericType.TypeId.TIMESTAMPTZ); break; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DefaultValue.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DefaultValue.java index ad5f41ec2d1816..b6a0bb014cb480 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DefaultValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DefaultValue.java @@ -18,9 +18,9 @@ package org.apache.doris.nereids.trees.plans.commands.info; import org.apache.doris.analysis.DefaultValueExprDef; -import org.apache.doris.catalog.ScalarType; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.types.DateTimeV2Type; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; @@ -81,9 +81,9 @@ public DefaultValue(String value, String exprName, Long precision) { * default value current_timestamp(precision) */ public static DefaultValue currentTimeStampDefaultValueWithPrecision(Long precision) { - if (precision > ScalarType.MAX_DATETIMEV2_SCALE || precision < 0) { + if (precision > DateTimeV2Type.MAX_SCALE || precision < 0) { throw new AnalysisException("column's default value current_timestamp" - + " precision must be between 0 and 6"); + + " precision must be between 0 and " + DateTimeV2Type.MAX_SCALE); } if (precision == 0) { return new DefaultValue(CURRENT_TIMESTAMP, NOW); @@ -118,22 +118,10 @@ public String getRawValue() { return LocalDateTime.now(TimeUtils.getTimeZone().toZoneId()).toString().replace('T', ' '); } else if (isCurrentTimeStampWithPrecision()) { long precision = getCurrentTimeStampPrecision(); - String format = "yyyy-MM-dd HH:mm:ss"; if (precision == 0) { return LocalDateTime.now(TimeUtils.getTimeZone().toZoneId()).toString().replace('T', ' '); - } else if (precision == 1) { - format = "yyyy-MM-dd HH:mm:ss.S"; - } else if (precision == 2) { - format = "yyyy-MM-dd HH:mm:ss.SS"; - } else if (precision == 3) { - format = "yyyy-MM-dd HH:mm:ss.SSS"; - } else if (precision == 4) { - format = "yyyy-MM-dd HH:mm:ss.SSSS"; - } else if (precision == 5) { - format = "yyyy-MM-dd HH:mm:ss.SSSSS"; - } else if (precision == 6) { - format = "yyyy-MM-dd HH:mm:ss.SSSSSS"; } + String format = "yyyy-MM-dd HH:mm:ss." + "S".repeat((int) precision); return LocalDateTime.now(TimeUtils.getTimeZone().toZoneId()) .format(DateTimeFormatter.ofPattern(format)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java index c7ef8f21de9cfc..d8655e5fe2d01c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java @@ -77,6 +77,7 @@ private static Map initMap() { .put(Type.DATEV2.getPrimitiveType(), DateType.INSTANCE) .put(Type.DATETIME.getPrimitiveType(), DateTimeType.INSTANCE) .put(Type.DATETIMEV2.getPrimitiveType(), DateTimeV2Type.SYSTEM_DEFAULT) + .put(Type.TIMESTAMP_NS.getPrimitiveType(), TimeStampNsType.INSTANCE) .put(Type.TIMESTAMPTZ.getPrimitiveType(), TimeStampTzType.SYSTEM_DEFAULT) .put(Type.DECIMALV2.getPrimitiveType(), DecimalV2Type.SYSTEM_DEFAULT) .put(Type.DECIMAL32.getPrimitiveType(), DecimalV3Type.SYSTEM_DEFAULT) @@ -118,7 +119,8 @@ private static Map initMap() { .add(DateType.class, () -> ImmutableList.of( DateTimeType.INSTANCE, DateV2Type.INSTANCE, StringType.INSTANCE)) .add(DateV2Type.class, () -> ImmutableList.of(DateTimeV2Type.SYSTEM_DEFAULT, StringType.INSTANCE)) - .add(TimeV2Type.class, () -> ImmutableList.of(DateTimeV2Type.MAX, StringType.INSTANCE)) + .add(TimeV2Type.class, () -> ImmutableList.of( + DateTimeV2Type.MAX, StringType.INSTANCE)) .build(); public static Map legacyTypeToNereidsType() { @@ -351,6 +353,12 @@ public static DataType convertPrimitiveFromStrings(List types) { throw new AnalysisException("Nereids do not support type: " + type); } break; + case "timestamp_ns": + if (types.size() != 1) { + throw new AnalysisException("timestamp_ns does not support precision"); + } + dataType = TimeStampNsType.INSTANCE; + break; case "timestamptz": switch (types.size()) { case 1: @@ -424,6 +432,7 @@ public static DataType fromCatalogType(Type type) { case DOUBLE: return DoubleType.INSTANCE; case NULL_TYPE: return NullType.INSTANCE; case DATETIMEV2: return DateTimeV2Type.of(((ScalarType) type).getScalarScale()); + case TIMESTAMP_NS: return TimeStampNsType.INSTANCE; case DATETIME: return DateTimeType.INSTANCE; case DATEV2: return DateV2Type.INSTANCE; case DATE: return DateType.INSTANCE; @@ -1098,20 +1107,35 @@ private static void validateScalarType(ScalarType scalarType) { + precision + " in not supported."); } } + case TIMESTAMP_NS: + // TIMESTAMP_NS has fixed nanosecond precision and carries no variable precision/scale metadata. + break; + case DATETIMEV2: { + int precision = scalarType.decimalPrecision(); + int scale = scalarType.decimalScale(); + if (precision != ScalarType.DATETIME_PRECISION) { + throw new AnalysisException( + "Precision of Datetime must be " + ScalarType.DATETIME_PRECISION + + "." + " Precision was set to: " + precision + "."); + } + if (scale < 0 || scale > ScalarType.MAX_DATETIMEV2_SCALE) { + throw new AnalysisException("Scale of Datetime must between 0 and " + + ScalarType.MAX_DATETIMEV2_SCALE + "." + + " Scale was set to: " + scale + "."); + } + break; + } case TIMEV2: - case DATETIMEV2: case TIMESTAMPTZ: { int precision = scalarType.decimalPrecision(); int scale = scalarType.decimalScale(); - // precision: 18 if (precision != ScalarType.DATETIME_PRECISION) { throw new AnalysisException( - "Precision of Datetime/Time must be " + ScalarType.DATETIME_PRECISION + "Precision of Time must be " + ScalarType.DATETIME_PRECISION + "." + " Precision was set to: " + precision + "."); } - // scale: [0, 6] if (scale < 0 || scale > 6) { - throw new AnalysisException("Scale of Datetime/Time must between 0 and 6." + throw new AnalysisException("Scale of Time must between 0 and 6." + " Scale was set to: " + scale + "."); } break; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeV2Type.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeV2Type.java index 13097339554e3e..4406a346695420 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeV2Type.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeV2Type.java @@ -37,7 +37,7 @@ * Datetime type in Nereids. */ public class DateTimeV2Type extends DateLikeType implements ScaleTimeType { - public static final int MAX_SCALE = 6; + public static final int MAX_SCALE = ScalarType.MAX_DATETIMEV2_SCALE; public static final DateTimeV2Type SYSTEM_DEFAULT = new DateTimeV2Type(0); public static final DateTimeV2Type MAX = new DateTimeV2Type(MAX_SCALE); public static final DateTimeV2Type WILDCARD = new DateTimeV2Type(-1); @@ -63,7 +63,11 @@ public static DateTimeV2Type of(int scale) { } } + /** Return the wider type without losing the distinct TIMESTAMP_NS physical type. */ public static DateTimeV2Type getWiderDatetimeV2Type(DateTimeV2Type t1, DateTimeV2Type t2) { + if (t1 instanceof TimeStampNsType || t2 instanceof TimeStampNsType) { + return TimeStampNsType.INSTANCE; + } if (t1.scale > t2.scale) { return t1; } @@ -85,10 +89,10 @@ public static DateTimeV2Type forType(DataType dataType) { return SYSTEM_DEFAULT; } if (dataType instanceof DecimalV3Type) { - return DateTimeV2Type.of(Math.min(((DecimalV3Type) dataType).getScale(), 6)); + return DateTimeV2Type.of(Math.min(((DecimalV3Type) dataType).getScale(), MAX_SCALE)); } if (dataType instanceof DecimalV2Type) { - return DateTimeV2Type.of(Math.min(((DecimalV2Type) dataType).getScale(), 6)); + return DateTimeV2Type.of(Math.min(((DecimalV2Type) dataType).getScale(), MAX_SCALE)); } if (dataType instanceof TimeV2Type) { return DateTimeV2Type.of(((TimeV2Type) dataType).getScale()); @@ -96,6 +100,19 @@ public static DateTimeV2Type forType(DataType dataType) { return MAX; } + /** + * Return the DATETIMEV2 type inferred from {@code dataType}, widened only when the operation + * itself requires more fractional digits. + */ + public static DateTimeV2Type forTypeWithMinimumScale(DataType dataType, int minimumScale) { + if (dataType instanceof CharacterType) { + return DateTimeV2Type.of(minimumScale); + } + DateTimeV2Type dateTimeV2Type = forType(dataType); + return dateTimeV2Type instanceof TimeStampNsType + ? dateTimeV2Type : DateTimeV2Type.of(Math.max(dateTimeV2Type.getScale(), minimumScale)); + } + public ScaleTimeType scaleTypeForType(DataType dataType) { return forType(dataType); } @@ -118,10 +135,7 @@ public static DateTimeV2Type forTypeFromString(String s) { // let be to process it } } - if (scale > MAX_SCALE) { - scale = MAX_SCALE; - } - return DateTimeV2Type.of(scale); + return DateTimeV2Type.of(Math.min(scale, MAX_SCALE)); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/TimeStampNsType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/TimeStampNsType.java new file mode 100644 index 00000000000000..e16b647b1d023c --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/TimeStampNsType.java @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.types; + +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.Type; +import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral; +import org.apache.doris.nereids.types.coercion.ScaleTimeType; + +/** Timestamp represented as signed nanoseconds since the Unix epoch. */ +public final class TimeStampNsType extends DateTimeV2Type { + public static final int SCALE = ScalarType.TIMESTAMP_NS_SCALE; + public static final TimeStampNsType INSTANCE = new TimeStampNsType(); + + private TimeStampNsType() { + super(SCALE); + } + + @Override + public Type toCatalogDataType() { + return ScalarType.createTimeStampNsType(); + } + + @Override + public ScaleTimeType scaleTypeForType(DataType dataType) { + return INSTANCE; + } + + @Override + public ScaleTimeType forTypeFromString(StringLikeLiteral str) { + return INSTANCE; + } + + @Override + public String toSql() { + return "timestamp_ns"; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/TimeStampTzType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/TimeStampTzType.java index 4f9c09b5f7f805..b5cd1ce2a3e738 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/TimeStampTzType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/TimeStampTzType.java @@ -85,13 +85,13 @@ public ScaleTimeType forTypeFromString(StringLikeLiteral s) { public static TimeStampTzType forTypeFromString(String s) { DateTimeV2Type dateTimeV2Type = DateTimeV2Type.forTypeFromString(s); - return TimeStampTzType.of(dateTimeV2Type.getScale()); + return TimeStampTzType.of(Math.min(dateTimeV2Type.getScale(), MAX_SCALE)); } @Override public ScaleTimeType scaleTypeForType(DataType dataType) { DateTimeV2Type dateTimeV2Type = DateTimeV2Type.forType(dataType); - return TimeStampTzType.of(dateTimeV2Type.getScale()); + return TimeStampTzType.of(Math.min(dateTimeV2Type.getScale(), MAX_SCALE)); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/TimeV2Type.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/TimeV2Type.java index af758c02cb522b..e3d7748c59ce36 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/TimeV2Type.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/TimeV2Type.java @@ -117,7 +117,7 @@ public static TimeV2Type forType(DataType dataType) { return TimeV2Type.of(Math.min(((DecimalV2Type) dataType).getScale(), 6)); } if (dataType instanceof DateTimeV2Type) { - return TimeV2Type.of(((DateTimeV2Type) dataType).getScale()); + return TimeV2Type.of(Math.min(((DateTimeV2Type) dataType).getScale(), MAX_SCALE)); } return MAX; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/coercion/DateLikeType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/coercion/DateLikeType.java index 1f8130215b04e2..4c5eb100e3ee90 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/coercion/DateLikeType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/coercion/DateLikeType.java @@ -22,10 +22,12 @@ import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral; import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal; import org.apache.doris.nereids.trees.expressions.literal.DateV2Literal; +import org.apache.doris.nereids.trees.expressions.literal.TimeStampNsLiteral; import org.apache.doris.nereids.types.DateTimeType; import org.apache.doris.nereids.types.DateTimeV2Type; import org.apache.doris.nereids.types.DateType; import org.apache.doris.nereids.types.DateV2Type; +import org.apache.doris.nereids.types.TimeStampNsType; import java.time.LocalDate; import java.time.LocalDateTime; @@ -70,8 +72,10 @@ public DateLiteral fromString(String s) { return new DateV2Literal(l.getYear(), l.getMonth(), l.getDay()); } else if (this instanceof DateTimeType) { return new DateTimeLiteral(s); + } else if (this instanceof TimeStampNsType) { + return new TimeStampNsLiteral(s); } else if (this instanceof DateTimeV2Type) { - return new DateTimeV2Literal((DateTimeV2Type) this, s); + return DateTimeV2Literal.create((DateTimeV2Type) this, s); } else { throw new AnalysisException("unknown date like type"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateTimeFormatterUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateTimeFormatterUtils.java index e8ff854945a0a2..e86e7ebdbbea3e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateTimeFormatterUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateTimeFormatterUtils.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal; import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral; import org.apache.doris.nereids.trees.expressions.literal.TimeV2Literal; +import org.apache.doris.nereids.types.TimeStampNsType; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatterBuilder; @@ -56,14 +57,13 @@ public class DateTimeFormatterUtils { .appendLiteral('-').appendValue(ChronoField.MONTH_OF_YEAR, 2) .appendLiteral('-').appendValue(ChronoField.DAY_OF_MONTH, 2) .toFormatter().withResolverStyle(ResolverStyle.STRICT); - // HH[:mm][:ss][.microsecond] + // HH[:mm][:ss][.fraction] public static final DateTimeFormatter TIME_FORMATTER = new DateTimeFormatterBuilder() .appendValue(ChronoField.HOUR_OF_DAY, 2) .appendLiteral(':').appendValue(ChronoField.MINUTE_OF_HOUR, 2) .appendLiteral(':').appendValue(ChronoField.SECOND_OF_MINUTE, 2) - // microsecond maxWidth is 7, we may need 7th digit to judge overflow .appendOptional(new DateTimeFormatterBuilder() - .appendFraction(ChronoField.NANO_OF_SECOND, 1, 7, true).toFormatter()) + .appendFraction(ChronoField.NANO_OF_SECOND, 1, TimeStampNsType.SCALE, true).toFormatter()) .toFormatter().withResolverStyle(ResolverStyle.STRICT); // Time without delimiter: HHmmss[.microsecond] private static final DateTimeFormatter BASIC_TIME_FORMATTER = new DateTimeFormatterBuilder() @@ -71,7 +71,7 @@ public class DateTimeFormatterUtils { .appendValue(ChronoField.MINUTE_OF_HOUR, 2) .appendValue(ChronoField.SECOND_OF_MINUTE, 2) .appendOptional(new DateTimeFormatterBuilder() - .appendFraction(ChronoField.NANO_OF_SECOND, 1, 7, true).toFormatter()) + .appendFraction(ChronoField.NANO_OF_SECOND, 1, TimeStampNsType.SCALE, true).toFormatter()) .toFormatter().withResolverStyle(ResolverStyle.STRICT); // yyyymmdd private static final DateTimeFormatter BASIC_DATE_FORMATTER = new DateTimeFormatterBuilder() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateUtils.java index e339471e447fb9..685819c481d579 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateUtils.java @@ -27,6 +27,7 @@ import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; +import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatterBuilder; import java.time.format.ResolverStyle; @@ -37,6 +38,9 @@ import java.time.temporal.TemporalAccessor; import java.time.temporal.WeekFields; import java.time.zone.ZoneOffsetTransition; +import java.time.zone.ZoneRules; +import java.util.Arrays; +import java.util.List; import java.util.Locale; import java.util.Set; @@ -44,6 +48,7 @@ * date util tools. */ public class DateUtils { + public static final int NANOSECOND_SCALE = 9; public static final Set monoFormat = ImmutableSet.of("yyyyMMdd", "yyyy-MM-dd", "yyyy-MM-dd HH:mm:ss", "%Y", "%Y-%m", "%Y-%m-%d", "%Y-%m-%d %H", "%Y-%m-%d %H:%i", "%Y-%m-%d %H:%i:%s", "%Y-%m-%d %H:%i:%S", "%Y-%m-%d %T", "%Y%m%d", "%Y%m"); @@ -389,6 +394,98 @@ public static int getOrDefault(final TemporalAccessor accessor, final ChronoFiel return accessor.isSupported(field) ? accessor.get(field) : /* default value */ 0; } + /** + * Remove the TIMESTAMP_NS rounding guard digit before parsing because Java accepts at most + * nine fractional-second digits. The guard remains available in the original string and can + * be retrieved by {@link #getNanosecondGuardDigit(String)} during target-scale rounding. + */ + public static String truncateFractionalSecondForJavaParser(String s) { + int dot = s.lastIndexOf('.'); + if (dot < 0) { + return s; + } + int fractionEnd = dot + 1; + while (fractionEnd < s.length() && Character.isDigit(s.charAt(fractionEnd))) { + fractionEnd++; + } + int retainedEnd = Math.min(dot + NANOSECOND_SCALE + 1, fractionEnd); + return retainedEnd == fractionEnd ? s : s.substring(0, retainedEnd) + s.substring(fractionEnd); + } + + /** Return the TIMESTAMP_NS rounding guard digit, or -1 when the input has none. */ + public static int getNanosecondGuardDigit(String s) { + int dot = s.lastIndexOf('.'); + int guardIndex = dot + NANOSECOND_SCALE + 1; + if (dot < 0 || guardIndex >= s.length() || !Character.isDigit(s.charAt(guardIndex))) { + return -1; + } + return s.charAt(guardIndex) - '0'; + } + + /** Format a datetime with the requested number of fractional-second digits. */ + public static String formatDateTime(long year, long month, long day, long hour, long minute, + long second, long nanosecond, int scale) { + if (0 <= year && year <= 9999 && 0 <= month && month <= 99 && 0 <= day && day <= 99 + && 0 <= hour && hour <= 99 && 0 <= minute && minute <= 99 + && 0 <= second && second <= 99 && 0 <= nanosecond && nanosecond < 1_000_000_000L) { + int length = scale == 0 ? 19 : 20 + scale; + char[] value = new char[length]; + Arrays.fill(value, '0'); + value[4] = '-'; + value[7] = '-'; + value[10] = ' '; + value[13] = ':'; + value[16] = ':'; + fillPaddedValue(value, 0, year, 4); + fillPaddedValue(value, 5, month, 2); + fillPaddedValue(value, 8, day, 2); + fillPaddedValue(value, 11, hour, 2); + fillPaddedValue(value, 14, minute, 2); + fillPaddedValue(value, 17, second, 2); + if (scale > 0) { + value[19] = '.'; + long divisor = (long) Math.pow(10, NANOSECOND_SCALE - scale); + fillPaddedValue(value, 20, nanosecond / divisor, scale); + } + return String.valueOf(value); + } + String fractionFormat = scale > 0 ? ".%0" + scale + "d" : ""; + long divisor = (long) Math.pow(10, NANOSECOND_SCALE - scale); + return String.format("%04d-%02d-%02d %02d:%02d:%02d" + fractionFormat, + year, month, day, hour, minute, second, nanosecond / divisor); + } + + private static void fillPaddedValue(char[] buffer, int start, long value, int length) { + for (int i = start + length - 1; i >= start; i--) { + buffer[i] = (char) ('0' + value % 10); + value /= 10; + } + } + + /** Convert a civil datetime between time zones using the same DST policy as BE cctz. */ + public static LocalDateTime convertTimeZone(long year, long month, long day, long hour, long minute, + long second, ZoneId fromZone, ZoneId toZone) { + LocalDateTime localDateTime = LocalDateTime.of((int) year, (int) month, (int) day, + (int) hour, (int) minute, (int) second); + return LocalDateTime.ofInstant(convertLocalToInstant(localDateTime, fromZone), toZone); + } + + /** + * Convert a local civil datetime to an instant with the same overlap/gap policy as BE cctz. + * Repeated local times use the pre-transition offset; skipped local times map to the transition + * instant. + */ + public static Instant convertLocalToInstant(LocalDateTime localDateTime, ZoneId fromZone) { + ZoneRules rules = fromZone.getRules(); + List validOffsets = rules.getValidOffsets(localDateTime); + int size = validOffsets.size(); + if (size == 1 || size == 2) { + return localDateTime.atOffset(validOffsets.get(0)).toInstant(); + } + ZoneOffsetTransition transition = rules.getTransition(localDateTime); + return transition.getInstant(); + } + /** * get hour from accessor, if not support hour field, return 0 */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java index 4a5fa860a90703..64dc3316ef4aed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java @@ -64,6 +64,7 @@ import org.apache.doris.nereids.trees.expressions.literal.SmallIntLiteral; import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral; import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; +import org.apache.doris.nereids.trees.expressions.literal.TimeStampNsLiteral; import org.apache.doris.nereids.trees.expressions.literal.TimeV2Literal; import org.apache.doris.nereids.trees.expressions.literal.TimestampTzLiteral; import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; @@ -96,6 +97,7 @@ import org.apache.doris.nereids.types.StringType; import org.apache.doris.nereids.types.StructField; import org.apache.doris.nereids.types.StructType; +import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TimeStampTzType; import org.apache.doris.nereids.types.TimeV2Type; import org.apache.doris.nereids.types.TinyIntType; @@ -131,6 +133,7 @@ import java.util.Map; import java.util.Optional; import java.util.function.Function; +import java.util.function.Predicate; import java.util.stream.Collectors; /** @@ -336,6 +339,10 @@ public static boolean hasDateTimeV2Type(DataType dataType) { return hasSpecifiedType(dataType, DateTimeV2Type.class); } + public static boolean hasTimeStampNsType(DataType dataType) { + return hasSpecifiedType(dataType, TimeStampNsType.class); + } + public static boolean hasTimeV2Type(DataType dataType) { return hasSpecifiedType(dataType, TimeV2Type.class); } @@ -377,7 +384,14 @@ public static DataType replaceDecimalV3WithWildcard(DataType dataType) { } public static DataType replaceDateTimeV2WithMax(DataType dataType) { - return replaceSpecifiedType(dataType, DateTimeV2Type.class, DateTimeV2Type.MAX); + return replaceSpecifiedType(dataType, + type -> type.getClass() == DateTimeV2Type.class, DateTimeV2Type.MAX); + } + + private static DataType replaceDateLikeWithMaxPrecision(DataType dataType) { + DataType targetType = hasTimeStampNsType(dataType) + ? TimeStampNsType.INSTANCE : DateTimeV2Type.MAX; + return replaceSpecifiedType(dataType, DateLikeType.class, targetType); } /** @@ -386,10 +400,14 @@ public static DataType replaceDateTimeV2WithMax(DataType dataType) { public static DataType replaceTimesWithTargetPrecision(DataType dataType, int targetScale) { return replaceSpecifiedType( replaceSpecifiedType( - replaceSpecifiedType(dataType, DateTimeV2Type.class, - DateTimeV2Type.of(targetScale)), - TimeV2Type.class, TimeV2Type.of(targetScale)), - TimeStampTzType.class, TimeStampTzType.of(targetScale)); + // Preserve TimeStampNsType: its DateTimeV2Type inheritance is for code reuse, + // not type-family precision promotion. + replaceSpecifiedType(dataType, + type -> type.getClass() == DateTimeV2Type.class, + DateTimeV2Type.of(Math.min(targetScale, DateTimeV2Type.MAX_SCALE))), + TimeV2Type.class, TimeV2Type.of(Math.min(targetScale, TimeV2Type.MAX_SCALE))), + TimeStampTzType.class, + TimeStampTzType.of(Math.min(targetScale, TimeStampTzType.MAX_SCALE))); } /** @@ -397,17 +415,27 @@ public static DataType replaceTimesWithTargetPrecision(DataType dataType, int ta */ public static DataType replaceSpecifiedType(DataType dataType, Class specifiedType, DataType newType) { + return replaceSpecifiedType(dataType, + type -> specifiedType.isAssignableFrom(type.getClass()), newType); + } + + private static DataType replaceSpecifiedType(DataType dataType, + Predicate shouldReplace, DataType newType) { if (dataType instanceof ArrayType) { - return ArrayType.of(replaceSpecifiedType(((ArrayType) dataType).getItemType(), specifiedType, newType)); + return ArrayType.of(replaceSpecifiedType( + ((ArrayType) dataType).getItemType(), shouldReplace, newType)); } else if (dataType instanceof MapType) { - return MapType.of(replaceSpecifiedType(((MapType) dataType).getKeyType(), specifiedType, newType), - replaceSpecifiedType(((MapType) dataType).getValueType(), specifiedType, newType)); + return MapType.of(replaceSpecifiedType( + ((MapType) dataType).getKeyType(), shouldReplace, newType), + replaceSpecifiedType( + ((MapType) dataType).getValueType(), shouldReplace, newType)); } else if (dataType instanceof StructType) { List newFields = ((StructType) dataType).getFields().stream() - .map(f -> f.withDataType(replaceSpecifiedType(f.getDataType(), specifiedType, newType))) + .map(f -> f.withDataType(replaceSpecifiedType( + f.getDataType(), shouldReplace, newType))) .collect(ImmutableList.toImmutableList()); return new StructType(newFields); - } else if (specifiedType.isAssignableFrom(dataType.getClass())) { + } else if (shouldReplace.test(dataType)) { return newType; } else { return dataType; @@ -665,6 +693,8 @@ public static Optional characterLiteralTypeCoercion(String value, Da ret = new VarcharLiteral(value, ((VarcharType) dataType).getLen()); } else if (dataType instanceof StringType) { ret = new StringLiteral(value); + } else if (dataType instanceof TimeStampNsType && DateTimeChecker.isValidDateTime(value)) { + ret = new TimeStampNsLiteral(value); } else if ((dataType.isDateTimeV2Type() || dataType.isDateTimeType()) && DateTimeChecker.isValidDateTime(value)) { ret = DateTimeLiteral.parseDateTimeLiteral(value, true).orElse(null); @@ -681,10 +711,14 @@ public static Optional characterLiteralTypeCoercion(String value, Da if (parseResult.isOk()) { ret = parseResult.get(); } else { - Result parseResult2 - = DateTimeV2Literal.parseDateTimeLiteral(value, true); - if (parseResult2.isOk()) { - ret = parseResult2.get(); + if (DateTimeLiteral.determineScale(value) > DateTimeV2Type.MAX_SCALE) { + ret = new TimeStampNsLiteral(value); + } else { + Result parseResult2 + = DateTimeV2Literal.parseDateTimeLiteral(value, true); + if (parseResult2.isOk()) { + ret = parseResult2.get(); + } } } } else if (dataType instanceof TimeV2Type && TimeChecker.isValidTime(value)) { @@ -953,10 +987,12 @@ private static Optional getCommonDataTypeWithDateType(DataType leftTyp return Optional.of(DateTimeV2Type.SYSTEM_DEFAULT); } else if (rightType instanceof DecimalV2Type) { DecimalV2Type decimalV2Type = (DecimalV2Type) rightType; - return Optional.of(DateTimeV2Type.of(Math.min(DateTimeV2Type.MAX_SCALE, decimalV2Type.getScale()))); + return Optional.of(DateTimeV2Type.of(Math.min(DateTimeV2Type.MAX_SCALE, + decimalV2Type.getScale()))); } else if (rightType instanceof DecimalV3Type) { DecimalV3Type decimalV3Type = (DecimalV3Type) rightType; - return Optional.of(DateTimeV2Type.of(Math.min(DateTimeV2Type.MAX_SCALE, decimalV3Type.getScale()))); + return Optional.of(DateTimeV2Type.of(Math.min(DateTimeV2Type.MAX_SCALE, + decimalV3Type.getScale()))); } else { return Optional.of(DateTimeV2Type.MAX); } @@ -972,7 +1008,7 @@ private static Optional getCommonDataTypeWithDateType(DataType leftTyp } } else if (rightType instanceof TimeV2Type) { TimeV2Type timeV2Type = (TimeV2Type) rightType; - return Optional.of(DateTimeV2Type.of(Math.min(DateTimeV2Type.MAX_SCALE, timeV2Type.getScale()))); + return Optional.of(DateTimeV2Type.of(timeV2Type.getScale())); } else if (rightType.isStringLikeType()) { return Optional.of(DateTimeV2Type.MAX); } @@ -989,30 +1025,34 @@ private static Optional getCommonDataTypeWithDateTimeV2Type( return Optional.of(leftType); } else if (rightType instanceof DecimalV2Type) { DecimalV2Type decimalV2Type = (DecimalV2Type) rightType; - return Optional.of(DateTimeV2Type.of(Math.min(DateTimeV2Type.MAX_SCALE, - Math.max(leftType.getScale(), decimalV2Type.getScale())))); + return Optional.of(DateTimeV2Type.forTypeWithMinimumScale(leftType, + Math.min(DateTimeV2Type.MAX_SCALE, decimalV2Type.getScale()))); } else if (rightType instanceof DecimalV3Type) { DecimalV3Type decimalV3Type = (DecimalV3Type) rightType; - return Optional.of(DateTimeV2Type.of(Math.min(DateTimeV2Type.MAX_SCALE, - Math.max(leftType.getScale(), decimalV3Type.getScale())))); + return Optional.of(DateTimeV2Type.forTypeWithMinimumScale(leftType, + Math.min(DateTimeV2Type.MAX_SCALE, decimalV3Type.getScale()))); } else { - return Optional.of(DateTimeV2Type.MAX); + return Optional.of(DateTimeV2Type.forTypeWithMinimumScale( + leftType, DateTimeV2Type.MAX_SCALE)); } } else if (rightType.isDateLikeType()) { if (rightType instanceof DateTimeV2Type) { DateTimeV2Type dateTimeV2Type = (DateTimeV2Type) rightType; - return Optional.of(DateTimeV2Type.of(Math.max(leftType.getScale(), dateTimeV2Type.getScale()))); + return Optional.of(DateTimeV2Type.getWiderDatetimeV2Type(leftType, dateTimeV2Type)); } else if (rightType instanceof TimeStampTzType) { TimeStampTzType timeStampTzType = (TimeStampTzType) rightType; - return Optional.of(DateTimeV2Type.of(Math.max(leftType.getScale(), timeStampTzType.getScale()))); + return leftType instanceof TimeStampNsType ? Optional.of(leftType) + : Optional.of(DateTimeV2Type.of(Math.max(leftType.getScale(), timeStampTzType.getScale()))); } else { return Optional.of(leftType); } } else if (rightType instanceof TimeV2Type) { TimeV2Type timeV2Type = (TimeV2Type) rightType; - return Optional.of(DateTimeV2Type.of(Math.max(leftType.getScale(), timeV2Type.getScale()))); + return leftType instanceof TimeStampNsType ? Optional.of(leftType) + : Optional.of(DateTimeV2Type.of(Math.max(leftType.getScale(), timeV2Type.getScale()))); } else if (rightType.isStringLikeType()) { - return Optional.of(DateTimeV2Type.MAX); + return Optional.of(DateTimeV2Type.forTypeWithMinimumScale( + leftType, DateTimeV2Type.MAX_SCALE)); } return Optional.empty(); } @@ -1124,16 +1164,16 @@ public static Optional findWiderTypeForTwo(DataType left, DataType rig return findCommonVariantType((VariantType) left, (VariantType) right); } else if (left instanceof VariantType) { return Optional.of(replaceSpecifiedType(replaceDecimalV3WithTarget(replaceSpecifiedType( - replaceSpecifiedType(replaceSpecifiedType(replaceCharacterToString(right), - IntegralType.class, DecimalV3Type.SYSTEM_DEFAULT), - DateLikeType.class, DateTimeV2Type.MAX), + replaceDateLikeWithMaxPrecision(replaceSpecifiedType( + replaceCharacterToString(right), + IntegralType.class, DecimalV3Type.SYSTEM_DEFAULT)), DecimalV2Type.class, DecimalV3Type.SYSTEM_DEFAULT), DecimalV3Type.SYSTEM_DEFAULT), FloatType.class, DoubleType.INSTANCE)); } else if (right instanceof VariantType) { return Optional.of(replaceSpecifiedType(replaceDecimalV3WithTarget(replaceSpecifiedType( - replaceSpecifiedType(replaceSpecifiedType(replaceCharacterToString(left), - IntegralType.class, DecimalV3Type.SYSTEM_DEFAULT), - DateLikeType.class, DateTimeV2Type.MAX), + replaceDateLikeWithMaxPrecision(replaceSpecifiedType( + replaceCharacterToString(left), + IntegralType.class, DecimalV3Type.SYSTEM_DEFAULT)), DecimalV2Type.class, DecimalV3Type.SYSTEM_DEFAULT), DecimalV3Type.SYSTEM_DEFAULT), FloatType.class, DoubleType.INSTANCE)); } else if (left instanceof ComplexDataType || right instanceof ComplexDataType) { @@ -1760,14 +1800,16 @@ private static Optional findCommonPrimitiveTypeForComparison( || rightType.isDateV2Type() || rightType.isDateTimeType()) { return Optional.of(leftType); } else { - return Optional.of(DateTimeV2Type.MAX); + return Optional.of(DateTimeV2Type.forTypeWithMinimumScale( + leftType, DateTimeV2Type.MAX_SCALE)); } } else if (rightType.isDateTimeV2Type()) { if (leftType instanceof IntegralType || leftType.isDateType() || leftType.isDateV2Type() || leftType.isDateTimeType()) { return Optional.of(rightType); } else { - return Optional.of(DateTimeV2Type.MAX); + return Optional.of(DateTimeV2Type.forTypeWithMinimumScale( + rightType, DateTimeV2Type.MAX_SCALE)); } } else if (leftType.isDateV2Type()) { if (rightType instanceof IntegralType || rightType.isDateType() || rightType.isDateV2Type()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java index a02f5bbf204b1a..3e61102801bfe7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java @@ -59,6 +59,7 @@ import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.StringLiteral; import org.apache.doris.analysis.StructLiteral; +import org.apache.doris.analysis.TimeStampNsLiteral; import org.apache.doris.analysis.TimeV2Literal; import org.apache.doris.analysis.TimestampArithmeticExpr; import org.apache.doris.analysis.TryCastExpr; @@ -266,6 +267,7 @@ public class GsonUtils { .registerSubtype(ArrayLiteral.class, ArrayLiteral.class.getSimpleName()) .registerSubtype(BoolLiteral.class, BoolLiteral.class.getSimpleName()) .registerSubtype(DateLiteral.class, DateLiteral.class.getSimpleName()) + .registerSubtype(TimeStampNsLiteral.class, TimeStampNsLiteral.class.getSimpleName()) .registerSubtype(IPv4Literal.class, IPv4Literal.class.getSimpleName()) .registerSubtype(IPv6Literal.class, IPv6Literal.class.getSimpleName()) .registerSubtype(JsonLiteral.class, JsonLiteral.class.getSimpleName()) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java index c83397f58dc0e7..9dd256b70c3bea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java @@ -888,7 +888,7 @@ private static LiteralExpr getLiteralExprFromThrift(TExprNode node) throws Analy case DECIMAL_LITERAL: return DecimalLiteralUtils.create(node.decimal_literal.value); case STRING_LITERAL: return new StringLiteral(node.string_literal.value); case JSON_LITERAL: return new JsonLiteral(node.json_literal.value); - case DATE_LITERAL: return DateLiteralUtils.createDateLiteral(node.date_literal.value, null); + case DATE_LITERAL: return DateLiteralUtils.createLiteral(node.date_literal.value, null); case IPV4_LITERAL: return new IPv4Literal(node.ipv4_literal.value); case IPV6_LITERAL: return new IPv6Literal(node.ipv6_literal.value); default: throw new AnalysisException("Wrong type from thrift;"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index b117b334f611f7..21ff1318ef92b0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -29,13 +29,17 @@ import org.apache.doris.analysis.StorageBackend.StorageType; import org.apache.doris.analysis.StringValueContext; import org.apache.doris.analysis.ToSqlParams; +import org.apache.doris.catalog.ArrayType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.EnvFactory; import org.apache.doris.catalog.FsBroker; +import org.apache.doris.catalog.MapType; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.StructField; +import org.apache.doris.catalog.StructType; import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.Type; import org.apache.doris.cloud.catalog.CloudEnv; @@ -1394,6 +1398,10 @@ private void handleQueryStmt() throws Exception { return; } + if (context.getConnectType() == ConnectType.ARROW_FLIGHT_SQL) { + checkArrowFlightSqlOutput(queryStmt.getResultExprs()); + } + if (parsedStmt instanceof LogicalPlanAdapter) { LogicalPlanAdapter logicalPlanAdapter = (LogicalPlanAdapter) parsedStmt; LogicalPlan logicalPlan = logicalPlanAdapter.getLogicalPlan(); @@ -1450,6 +1458,32 @@ private void handleQueryStmt() throws Exception { executeAndSendResult(isOutfileQuery, false, queryStmt, channel, null, null); } + static void checkArrowFlightSqlOutput(List resultExprs) throws AnalysisException { + for (Expr resultExpr : resultExprs) { + if (containsTimeStampNs(resultExpr.getType())) { + throw new AnalysisException("TIMESTAMP_NS is not supported by Arrow Flight SQL"); + } + } + } + + private static boolean containsTimeStampNs(Type type) { + if (type.isTimeStampNs()) { + return true; + } else if (type.isArrayType()) { + return containsTimeStampNs(((ArrayType) type).getItemType()); + } else if (type.isMapType()) { + MapType mapType = (MapType) type; + return containsTimeStampNs(mapType.getKeyType()) || containsTimeStampNs(mapType.getValueType()); + } else if (type.isStructType()) { + for (StructField field : ((StructType) type).getFields()) { + if (containsTimeStampNs(field.getType())) { + return true; + } + } + } + return false; + } + public void executeAndSendResult(boolean isOutfileQuery, boolean isSendFields, Queriable queryStmt, MysqlChannel channel, CacheAnalyzer cacheAnalyzer, InternalService.PFetchCacheResult cacheResult) throws Exception { @@ -2025,6 +2059,11 @@ protected void sendBinaryResultRow(ResultSet resultSet) throws IOException { serializer.writeInt4((int) microSecond); } break; + case TIMESTAMP_NS: + // MySQL temporal binary values cannot carry nanoseconds. The metadata advertises + // MYSQL_TYPE_STRING, so encode the result as length-encoded text. + serializer.writeLenEncodedString(item); + break; default: serializer.writeLenEncodedString(item); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionRange.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionRange.java index acdb244e482939..bbf0984a765f32 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionRange.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionRange.java @@ -129,6 +129,7 @@ public boolean init(Type type, LiteralExpr expr) { case TIMEV2: case DATETIME: case DATETIMEV2: + case TIMESTAMP_NS: case TIMESTAMPTZ: case FLOAT: case DOUBLE: diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlSchemaHelper.java b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlSchemaHelper.java index b89abf16e83395..03859b923720d8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlSchemaHelper.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlSchemaHelper.java @@ -134,6 +134,8 @@ private static ArrowType getArrowType(PrimitiveType primitiveType, Integer preci } else { return new ArrowType.Timestamp(TimeUnit.SECOND, timeZone); } + case TIMESTAMP_NS: + throw new IllegalArgumentException("TIMESTAMP_NS is not supported by Arrow Flight SQL"); case TIMESTAMPTZ: if (scale > 3) { return new ArrowType.Timestamp(TimeUnit.MICROSECOND, "UTC"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java index e3b67a4dd51baf..e22db8c7d7a18c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java @@ -261,8 +261,9 @@ public static LiteralExpr readableValue(Type type, String columnValue) throws An case DATETIME: case DATEV2: case DATETIMEV2: + case TIMESTAMP_NS: case TIMESTAMPTZ: - return DateLiteralUtils.createDateLiteral(columnValue, type); + return DateLiteralUtils.createLiteral(columnValue, type); case CHAR: case VARCHAR: case STRING: @@ -313,6 +314,7 @@ public static double convertToDouble(Type type, String columnValue) throws Analy return literal.getDouble(); case DATETIMEV2: + case TIMESTAMP_NS: case DATETIME: DateTimeLiteral dateTimeLiteral = new DateTimeLiteral(columnValue); return dateTimeLiteral.getDouble(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java index 19ac5631c93482..6ac88ab8a56df3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java @@ -17,6 +17,7 @@ package org.apache.doris.tablefunction; +import org.apache.doris.analysis.DateLiteralUtils; import org.apache.doris.analysis.UserIdentity; import org.apache.doris.authentication.AuthenticationIntegrationMeta; import org.apache.doris.authentication.RoleMappingMeta; @@ -2157,6 +2158,11 @@ private static List partitionValuesRows(List partitionCols, List org.apache.doris.nereids.trees.plans.commands.info.DefaultValue + .currentTimeStampDefaultValueWithPrecision(invalidPrecision)); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/DateLiteralUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/DateLiteralUtilsTest.java index 08d472d54865ff..a2d515835c2528 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/DateLiteralUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/DateLiteralUtilsTest.java @@ -23,15 +23,23 @@ import org.apache.doris.common.util.TimeUtils; import org.apache.doris.foundation.format.FormatOptions; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.thrift.TExpr; +import org.apache.doris.thrift.TExprNodeType; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.time.Instant; +import java.time.LocalDateTime; import java.util.TimeZone; public class DateLiteralUtilsTest { + private static TimeStampNsLiteral createTimeStampNsLiteral(String value) throws AnalysisException { + return (TimeStampNsLiteral) DateLiteralUtils.createLiteral( + value, ScalarType.createTimeStampNsType()); + } + @Test public void testTimestampTzInit() throws AnalysisException { String value; @@ -205,6 +213,226 @@ public void testDatetimeV2WithMicroseconds() throws AnalysisException { Assertions.assertTrue(dl0.getType().isDatetimeV2()); } + @Test + public void testTimestampNsWithNanoseconds() throws AnalysisException { + TimeStampNsLiteral dl7 = createTimeStampNsLiteral("2023-06-15 14:30:45.1234567"); + Assertions.assertTrue(dl7.getType().isTimeStampNs()); + Assertions.assertEquals(123456700, dl7.getNanosecond()); + Assertions.assertEquals(123456, dl7.getMicrosecond()); + Assertions.assertEquals("2023-06-15 14:30:45.123456700", dl7.getStringValue()); + + TimeStampNsLiteral dl8 = createTimeStampNsLiteral("2023-06-15 14:30:45.12345678"); + Assertions.assertTrue(dl8.getType().isTimeStampNs()); + Assertions.assertEquals(123456780, dl8.getNanosecond()); + Assertions.assertEquals("2023-06-15 14:30:45.123456780", dl8.getStringValue()); + + TimeStampNsLiteral dl9 = createTimeStampNsLiteral("2023-06-15 14:30:45.123456789"); + Assertions.assertTrue(dl9.getType().isTimeStampNs()); + Assertions.assertEquals(123456789, dl9.getNanosecond()); + Assertions.assertEquals("2023-06-15 14:30:45.123456789", dl9.getStringValue()); + } + + @Test + public void testDateTimeV2MaximumSentinelUsesMicroseconds() throws AnalysisException { + DateLiteral maximum = new DateLiteral(Type.DATETIMEV2_WITH_MAX_SCALAR, true); + Assertions.assertEquals("9999-12-31 23:59:59.999999", maximum.getStringValue()); + Assertions.assertEquals(999999, maximum.getMicrosecond()); + } + + @Test + public void testTimestampNsUsesFixedNanosecondPrecision() throws AnalysisException { + TimeStampNsLiteral rounded7 = createTimeStampNsLiteral("2023-06-15 14:30:45.12345675"); + TimeStampNsLiteral canonical7 = createTimeStampNsLiteral("2023-06-15 14:30:45.123456750"); + Assertions.assertEquals("2023-06-15 14:30:45.123456750", rounded7.getStringValue()); + Assertions.assertEquals(123456750, rounded7.getNanosecond()); + Assertions.assertEquals(canonical7, rounded7); + Assertions.assertEquals(canonical7.hashCode(), rounded7.hashCode()); + + TimeStampNsLiteral rounded8 = createTimeStampNsLiteral("2023-06-15 14:30:45.123456785"); + Assertions.assertEquals("2023-06-15 14:30:45.123456785", rounded8.getStringValue()); + Assertions.assertEquals(123456785, rounded8.getNanosecond()); + + TimeStampNsLiteral rounded9 = createTimeStampNsLiteral("2023-06-15 14:30:45.1234567895"); + Assertions.assertEquals("2023-06-15 14:30:45.123456790", rounded9.getStringValue()); + Assertions.assertEquals(123456790, rounded9.getNanosecond()); + + TimeStampNsLiteral carry7 = createTimeStampNsLiteral("2023-06-15 14:30:45.99999995"); + TimeStampNsLiteral carry8 = createTimeStampNsLiteral("2023-06-15 14:30:45.999999995"); + TimeStampNsLiteral carry9 = createTimeStampNsLiteral("2023-06-15 14:30:45.9999999995"); + Assertions.assertEquals("2023-06-15 14:30:45.999999950", carry7.getStringValue()); + Assertions.assertEquals("2023-06-15 14:30:45.999999995", carry8.getStringValue()); + Assertions.assertEquals("2023-06-15 14:30:46.000000000", carry9.getStringValue()); + } + + @Test + public void testTimestampNsRoundFloorRequiresFixedScale() throws AnalysisException { + TimeStampNsLiteral timestampNs = createTimeStampNsLiteral("2023-06-15 14:30:45.123456789"); + + timestampNs.roundFloor(ScalarType.TIMESTAMP_NS_SCALE); + Assertions.assertEquals(123456789, timestampNs.getNanosecond()); + + int[] invalidScales = {-1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 10}; + for (int scale : invalidScales) { + IllegalArgumentException exception = Assertions.assertThrows( + IllegalArgumentException.class, () -> timestampNs.roundFloor(scale)); + Assertions.assertEquals("TIMESTAMP_NS has fixed scale " + ScalarType.TIMESTAMP_NS_SCALE + + ", but got " + scale, exception.getMessage()); + Assertions.assertEquals(123456789, timestampNs.getNanosecond()); + } + } + + @Test + public void testTimestampNsTypeInference() throws AnalysisException { + TimeStampNsLiteral upgraded = (TimeStampNsLiteral) DateLiteralUtils.createLiteral( + "2023-06-15 14:30:45.123456789", Type.DATETIME); + Assertions.assertTrue(upgraded.getType().isTimeStampNs()); + Assertions.assertEquals(123456789, upgraded.getNanosecond()); + + TimeStampNsLiteral inferredScale7 = (TimeStampNsLiteral) DateLiteralUtils.createLiteral( + "2023-06-15 14:30:45.123456700", null); + Assertions.assertTrue(inferredScale7.getType().isTimeStampNs()); + Assertions.assertEquals(123456700, inferredScale7.getNanosecond()); + Assertions.assertEquals("2023-06-15 14:30:45.123456700", inferredScale7.getStringValue()); + + TimeStampNsLiteral inferredScale9 = (TimeStampNsLiteral) DateLiteralUtils.createLiteral( + "1970-01-01 00:00:00.000000001", null); + Assertions.assertTrue(inferredScale9.getType().isTimeStampNs()); + Assertions.assertEquals(1, inferredScale9.getNanosecond()); + Assertions.assertEquals(1L, inferredScale9.getRealValue()); + } + + @Test + public void testTimestampNsEpochAndBoundaries() throws AnalysisException { + TimeStampNsLiteral beforeEpoch = createTimeStampNsLiteral("1969-12-31 23:59:59.999999999"); + Assertions.assertEquals(-1L, beforeEpoch.getRealValue()); + + TimeStampNsLiteral epoch = createTimeStampNsLiteral("1970-01-01 00:00:00.000000000"); + Assertions.assertEquals(0L, epoch.getRealValue()); + + TimeStampNsLiteral lowerBoundary = createTimeStampNsLiteral("1677-09-21 00:12:43.145224192"); + Assertions.assertEquals(Long.MIN_VALUE, lowerBoundary.getRealValue()); + + TimeStampNsLiteral upperBoundary = createTimeStampNsLiteral("2262-04-11 23:47:16.854775807"); + Assertions.assertEquals(Long.MAX_VALUE, upperBoundary.getRealValue()); + + TimeStampNsLiteral minimum = new TimeStampNsLiteral(false); + Assertions.assertEquals(Long.MIN_VALUE, minimum.getRealValue()); + Assertions.assertTrue(minimum.isMinValue()); + + TimeStampNsLiteral maximum = new TimeStampNsLiteral(true); + Assertions.assertEquals(Long.MAX_VALUE, maximum.getRealValue()); + + epoch.setMinValue(); + Assertions.assertEquals(Long.MIN_VALUE, epoch.getRealValue()); + Assertions.assertTrue(epoch.isMinValue()); + } + + @Test + public void testTimestampNsDateArithmeticPreservesLiteralType() throws AnalysisException { + TimeStampNsLiteral literal = new TimeStampNsLiteral( + 1970, 1, 1, 0, 0, 0, 123456789); + + TimeStampNsLiteral result = literal.plusSeconds(1).plusDays(1); + + Assertions.assertInstanceOf(TimeStampNsLiteral.class, result); + Assertions.assertEquals("1970-01-02 00:00:01.123456789", result.getStringValue()); + Assertions.assertEquals(86401123456789L, result.getRealValue()); + } + + @Test + public void testTimestampNsIsIndependentFromDateLiteral() throws AnalysisException { + TimeStampNsLiteral timestampNs = createTimeStampNsLiteral( + "2024-01-02 03:04:05.123456000"); + DateLiteral datetimeV2 = DateLiteralUtils.createDateLiteral( + "2024-01-02 03:04:05.123456", ScalarType.createDatetimeV2Type(6)); + + Assertions.assertInstanceOf(LiteralExpr.class, timestampNs); + Assertions.assertFalse(DateLiteral.class.isInstance(timestampNs)); + Assertions.assertEquals(0, timestampNs.compareLiteral(datetimeV2)); + Assertions.assertEquals(0, datetimeV2.compareLiteral(timestampNs)); + Assertions.assertEquals(timestampNs, datetimeV2); + Assertions.assertEquals(datetimeV2, timestampNs); + Assertions.assertEquals(datetimeV2.hashCode(), timestampNs.hashCode()); + Assertions.assertInstanceOf(TimeStampNsLiteral.class, timestampNs.clone()); + Assertions.assertEquals("'2024-01-02 03:04:05.123456000'", + timestampNs.accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITH_TABLE)); + TExpr thrift = ExprToThriftVisitor.treeToThrift(timestampNs); + Assertions.assertEquals(TExprNodeType.DATE_LITERAL, thrift.nodes.get(0).node_type); + Assertions.assertEquals(timestampNs.getStringValue(), thrift.nodes.get(0).date_literal.value); + + Assertions.assertThrows(AnalysisException.class, () -> DateLiteralUtils.createDateLiteral( + timestampNs.getStringValue(), ScalarType.createTimeStampNsType())); + } + + @Test + public void testTimestampNsCheckRangeUsesInt64EpochNanosecondBounds() { + Assertions.assertFalse(new TimeStampNsLiteral( + 1677, 9, 21, 0, 12, 43, 145224192).checkRange()); + Assertions.assertTrue(new TimeStampNsLiteral( + 1677, 9, 21, 0, 12, 43, 145224191).checkRange()); + Assertions.assertFalse(new TimeStampNsLiteral( + 2262, 4, 11, 23, 47, 16, 854775807).checkRange()); + Assertions.assertTrue(new TimeStampNsLiteral( + 2262, 4, 11, 23, 47, 16, 854775808).checkRange()); + Assertions.assertTrue(new TimeStampNsLiteral( + 2024, 13, 1, 0, 0, 0, 0).checkRange()); + + DateLiteral maxDatetimeV2 = new DateLiteral( + 9999, 12, 31, 23, 59, 59, 999999, ScalarType.createDatetimeV2Type(6)); + Assertions.assertFalse(maxDatetimeV2.checkRange()); + } + + @Test + public void testFractionalSecondRangeIsTypeSpecific() throws AnalysisException { + LocalDateTime lastMicrosecond = LocalDateTime.of(2024, 1, 1, 0, 0, 0, 999999000); + LocalDateTime afterLastMicrosecond = LocalDateTime.of(2024, 1, 1, 0, 0, 0, 999999001); + + for (Type type : new Type[] { + ScalarType.createDatetimeV2Type(6), ScalarType.createTimeStampTzType(6)}) { + DateLiteral valid = new DateLiteral(lastMicrosecond, type); + Assertions.assertFalse(valid.checkRange()); + Assertions.assertDoesNotThrow(valid::checkValueValid); + + DateLiteral canonicalized = new DateLiteral(afterLastMicrosecond, type); + Assertions.assertEquals(999999, canonicalized.getMicrosecond()); + Assertions.assertFalse(canonicalized.checkRange()); + Assertions.assertDoesNotThrow(canonicalized::checkValueValid); + } + + TimeStampNsLiteral timestampNs = new TimeStampNsLiteral( + LocalDateTime.of(2024, 1, 1, 0, 0, 0, 999999999)); + Assertions.assertFalse(timestampNs.checkRange()); + Assertions.assertDoesNotThrow(timestampNs::checkValueValid); + + TimeStampNsLiteral invalidTimestampNs = new TimeStampNsLiteral( + 2024, 1, 1, 0, 0, 0, 1000000000L); + Assertions.assertTrue(invalidTimestampNs.checkRange()); + Assertions.assertThrows(AnalysisException.class, invalidTimestampNs::checkValueValid); + } + + @Test + public void testTimestampNsTimezoneOffset() throws AnalysisException { + TimeStampNsLiteral utc = createTimeStampNsLiteral("2023-06-15 12:00:00.123456789+00:00"); + TimeStampNsLiteral plus8 = createTimeStampNsLiteral("2023-06-15 20:00:00.123456789+08:00"); + TimeStampNsLiteral minus5 = createTimeStampNsLiteral("2023-06-15 07:00:00.123456789-05:00"); + + Assertions.assertEquals(utc.getStringValue(), plus8.getStringValue()); + Assertions.assertEquals(utc.getStringValue(), minus5.getStringValue()); + Assertions.assertEquals(123456789, utc.getNanosecond()); + Assertions.assertEquals(123456789, plus8.getNanosecond()); + Assertions.assertEquals(123456789, minus5.getNanosecond()); + } + + @Test + public void testInvalidTimestampNsseconds() { + Assertions.assertThrows(AnalysisException.class, () -> DateLiteralUtils.createLiteral( + "2023-02-29 14:30:45.123456789", ScalarType.createTimeStampNsType())); + Assertions.assertThrows(AnalysisException.class, () -> DateLiteralUtils.createLiteral( + "1677-09-21 00:12:43.145224191", ScalarType.createTimeStampNsType())); + Assertions.assertThrows(AnalysisException.class, () -> DateLiteralUtils.createLiteral( + "2262-04-11 23:47:16.854775808", ScalarType.createTimeStampNsType())); + } + @Test public void testCompactDateFormats() throws AnalysisException { // YYYYMMDD compact date diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprGsonSerializationTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprGsonSerializationTest.java index 83b057759ed5a5..5eda48b0d0251b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprGsonSerializationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprGsonSerializationTest.java @@ -146,6 +146,8 @@ private Map, Expr> createExprSamples() throws Exception { samples.put(StringLiteral.class, new StringLiteral("expr-gson")); samples.put(StructLiteral.class, new StructLiteral(new StructType(), new IntLiteral(7L), new StringLiteral("field"))); + samples.put(TimeStampNsLiteral.class, + new TimeStampNsLiteral(2024, 4, 27, 12, 34, 56, 123456789)); samples.put(TimeV2Literal.class, new TimeV2Literal(12, 34, 56, 123456, 6, false)); samples.put(VarBinaryLiteral.class, new VarBinaryLiteral("bin".getBytes(StandardCharsets.UTF_8))); samples.put(BetweenPredicate.class, createBetweenPredicate()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprToStringValueVisitorTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprToStringValueVisitorTest.java index 9391d622cc6226..7b3d741b60b089 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprToStringValueVisitorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprToStringValueVisitorTest.java @@ -319,6 +319,17 @@ public void testDateLiteralInComplexType() throws Exception { V.visitDateLiteral(d, StringValueContext.forQuery(FormatOptions.getDefault()).asComplexType())); } + @Test + public void testTimeStampNsLiteral() { + TimeStampNsLiteral timestampNs = new TimeStampNsLiteral( + 2024, 1, 15, 10, 30, 0, 123456789); + Assertions.assertEquals("2024-01-15 10:30:00.123456789", + timestampNs.accept(V, StringValueContext.forQuery(FormatOptions.getDefault()))); + Assertions.assertEquals("\"2024-01-15 10:30:00.123456789\"", + timestampNs.accept(V, + StringValueContext.forQuery(FormatOptions.getDefault()).asComplexType())); + } + // ======================== TimeStampTz ======================== @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateFunctionTest.java index 977d11c24a9141..90421e81053c6a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateFunctionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateFunctionTest.java @@ -155,6 +155,18 @@ public void testCreatePythonFunctionRejectsObjectTypes() throws Exception { assertCreateFunctionAnalysisException(ctx, "create function py_obj_type_db.j_bitmap_arg(bitmap) returns int " + "properties('type'='JAVA_UDF', 'symbol'='evaluate');", "JAVA_UDF does not support argument 1 type bitmap"); + assertCreateFunctionAnalysisException(ctx, + "create function py_obj_type_db.j_timestamp_ns_arg(timestamp_ns) returns int " + + "properties('type'='JAVA_UDF', 'symbol'='evaluate');", + "JAVA_UDF does not support argument 1 type timestamp_ns"); + assertCreateFunctionAnalysisException(ctx, + "create function py_obj_type_db.j_timestamp_ns_ret(int) returns timestamp_ns " + + "properties('type'='JAVA_UDF', 'symbol'='evaluate');", + "JAVA_UDF does not support return type timestamp_ns"); + assertCreateFunctionAnalysisException(ctx, + "create function py_obj_type_db.j_timestamp_ns_array(array) returns int " + + "properties('type'='JAVA_UDF', 'symbol'='evaluate');", + "JAVA_UDF does not support argument 1 element type timestamp_ns"); assertCreateFunctionAnalysisException(ctx, "create function py_obj_type_db.py_hll_ret(int) returns hll " + "properties('type'='PYTHON_UDF', 'symbol'='evaluate', 'runtime_version'='3.10.2');", "PYTHON_UDF does not support return type hll"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/PartitionKeyTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/PartitionKeyTest.java index b5148f982eacf7..65d56ec8a95946 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/PartitionKeyTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/PartitionKeyTest.java @@ -27,6 +27,8 @@ import org.junit.BeforeClass; import org.junit.Test; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.DataOutputStream; import java.nio.file.Files; @@ -292,6 +294,60 @@ public void testMaxValueToSql() throws Exception { Assert.assertEquals("(MAXVALUE, MAXVALUE, MAXVALUE, MAXVALUE, MAXVALUE, MAXVALUE, MAXVALUE)", key.toSql()); } + @Test + public void testTimeStampNsMinValue() throws Exception { + assertTimeStampNsMinValue( + "1677-09-21 00:12:43.145224192", "1677-09-21 00:12:43.145224193"); + } + + @Test + public void testTimeStampNsSuccessor() throws Exception { + Column column = new Column("timestamp_ns", ScalarType.createTimeStampNsType()); + PartitionKey key = PartitionKey.createPartitionKey( + Arrays.asList(new PartitionValue("1970-01-01 00:00:00.123456789")), + Arrays.asList(column)); + + PartitionKey successor = key.successor(); + + Assert.assertTrue(successor.getKeys().get(0) + instanceof org.apache.doris.analysis.TimeStampNsLiteral); + Assert.assertEquals("1970-01-01 00:00:00.123456790", + successor.getKeys().get(0).getStringValue()); + } + + @Test + public void testTimeStampNsSerialization() throws Exception { + Column column = new Column("timestamp_ns", ScalarType.createTimeStampNsType()); + PartitionKey key = PartitionKey.createPartitionKey( + Arrays.asList(new PartitionValue("1970-01-01 00:00:00.123456789")), + Arrays.asList(column)); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + key.write(new DataOutputStream(bytes)); + PartitionKey restored = PartitionKey.read( + new DataInputStream(new ByteArrayInputStream(bytes.toByteArray()))); + + Assert.assertEquals(key, restored); + Assert.assertTrue(restored.getKeys().get(0) + instanceof org.apache.doris.analysis.TimeStampNsLiteral); + Assert.assertEquals("1970-01-01 00:00:00.123456789", + restored.getKeys().get(0).getStringValue()); + } + + private void assertTimeStampNsMinValue(String minValue, String nextValue) throws Exception { + Column column = new Column("timestamp_ns", ScalarType.createTimeStampNsType()); + PartitionKey infinityMin = PartitionKey.createInfinityPartitionKey(Arrays.asList(column), false); + PartitionKey literalMin = PartitionKey.createPartitionKey( + Arrays.asList(new PartitionValue(minValue)), Arrays.asList(column)); + PartitionKey literalNext = PartitionKey.createPartitionKey( + Arrays.asList(new PartitionValue(nextValue)), Arrays.asList(column)); + + Assert.assertTrue(infinityMin.isMinValue()); + Assert.assertTrue(literalMin.isMinValue()); + Assert.assertEquals(infinityMin, literalMin); + Assert.assertFalse(literalNext.isMinValue()); + } + @Test public void testTimestampTzPartitionKeyKeepsExplicitOffset() throws Exception { boolean originalRunningUnitTest = FeConstants.runningUnitTest; diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/TypeTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/TypeTest.java index 37a66edfc1f51d..f36268d61fe0f5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/TypeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/TypeTest.java @@ -246,5 +246,19 @@ public void testDatetimeV2ScaleMatching() { Assert.assertFalse(Type.matchExactType(dtv2s3, dtv2s6, true)); // Same scale -> match Assert.assertTrue(Type.matchExactType(dtv2s6, ScalarType.createDatetimeV2Type(6), false)); + + ScalarType timestampNs = ScalarType.createTimeStampNsType(); + Assert.assertNotEquals(timestampNs, dtv2s6); + for (int invalidScale = 7; invalidScale <= 9; invalidScale++) { + ScalarType invalidDatetimeV2 = ScalarType.createDatetimeV2Type(invalidScale); + Assert.assertTrue(invalidDatetimeV2.isDatetimeV2()); + Assert.assertFalse(invalidDatetimeV2.isTimeStampNs()); + Assert.assertNotEquals(timestampNs, invalidDatetimeV2); + } + ScalarType wildcardDatetimeV2 = ScalarType.createDatetimeV2Type(-1); + Assert.assertFalse(timestampNs.matchesType(wildcardDatetimeV2)); + Assert.assertFalse(timestampNs.matchesType(dtv2s6)); + Assert.assertFalse(dtv2s6.matchesType(timestampNs)); + Assert.assertFalse(Type.matchExactType(timestampNs, wildcardDatetimeV2, false)); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/connector/converter/NereidsToConnectorExpressionConverterTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/connector/converter/NereidsToConnectorExpressionConverterTest.java index 3d2e9e96eead7d..717ff586bad95d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/connector/converter/NereidsToConnectorExpressionConverterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/connector/converter/NereidsToConnectorExpressionConverterTest.java @@ -145,6 +145,11 @@ public void dateAndDatetimeLiteralsBecomeJavaTimeValues() { new EqualTo(slot("ts", ScalarType.createDatetimeV2Type(0)), new DateTimeV2Literal(2024, 1, 2, 12, 34, 56)))); Assertions.assertEquals(LocalDateTime.of(2024, 1, 2, 12, 34, 56), ts.getValue()); + + ConnectorLiteral nanoTs = rightLiteralOf(NereidsToConnectorExpressionConverter.convert( + new EqualTo(slot("timestamp_ns", ScalarType.createTimeStampNsType()), + new DateTimeV2Literal("2024-01-02 12:34:56.123456789")))); + Assertions.assertEquals(LocalDateTime.of(2024, 1, 2, 12, 34, 56, 123456789), nanoTs.getValue()); } // ---- node shape mapping ---- diff --git a/fe/fe-core/src/test/java/org/apache/doris/mysql/MysqlSerializerDateTimeTest.java b/fe/fe-core/src/test/java/org/apache/doris/mysql/MysqlSerializerDateTimeTest.java new file mode 100644 index 00000000000000..924e07755f66de --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/mysql/MysqlSerializerDateTimeTest.java @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mysql; + +import org.apache.doris.catalog.MysqlColType; +import org.apache.doris.catalog.ScalarType; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class MysqlSerializerDateTimeTest { + private static int skipLenEncodedString(byte[] buffer, int offset) { + return offset + 1 + (buffer[offset] & 0xFF); + } + + private static int fieldMetadataOffset(byte[] buffer) { + int offset = 0; + for (int i = 0; i < 6; i++) { + offset = skipLenEncodedString(buffer, offset); + } + return offset + 1; + } + + @Test + public void testTimestampNsMetadata() { + MysqlSerializer serializer = MysqlSerializer.newInstance(); + ScalarType type = ScalarType.createTimeStampNsType(); + Assertions.assertTrue(type.isTimeStampNs()); + Assertions.assertEquals(29, serializer.getMysqlTypeLength(type)); + Assertions.assertEquals(0, serializer.getMysqlDecimals(type)); + + serializer.writeField("dt", type); + byte[] field = serializer.toArray(); + int metadataOffset = fieldMetadataOffset(field); + int mysqlTypeOffset = metadataOffset + 2 + 4; + Assertions.assertEquals(MysqlColType.MYSQL_TYPE_STRING.getCode(), + field[mysqlTypeOffset] & 0xFF); + int decimalsOffset = mysqlTypeOffset + 1 + 2; + Assertions.assertEquals(0, field[decimalsOffset] & 0xFF); + } + + @Test + public void testDatetimeV2MicrosecondMetadata() { + for (int scale = 0; scale <= 6; scale++) { + MysqlSerializer serializer = MysqlSerializer.newInstance(); + ScalarType type = ScalarType.createDatetimeV2Type(scale); + Assertions.assertEquals(scale, serializer.getMysqlDecimals(type)); + + serializer.writeField("dt", type); + byte[] field = serializer.toArray(); + int metadataOffset = fieldMetadataOffset(field); + int mysqlTypeOffset = metadataOffset + 2 + 4; + Assertions.assertEquals(MysqlColType.MYSQL_TYPE_DATETIME.getCode(), + field[mysqlTypeOffset] & 0xFF); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantTest.java index 9f46a7970a2c72..6dbc84ae24cf7b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantTest.java @@ -331,7 +331,8 @@ void testFoldDate() { MilliSecondsAdd millisecondsAdd = new MilliSecondsAdd( DateTimeV2Literal.fromJavaDateType(LocalDateTime.of(1, 1, 1, 1, 1, 1)), new BigIntLiteral(1)); rewritten = executor.rewrite(millisecondsAdd, context); - Assertions.assertEquals(new DateTimeV2Literal(DateTimeV2Type.MAX, "0001-01-01 01:01:01.001000"), rewritten); + Assertions.assertEquals( + new DateTimeV2Literal(DateTimeV2Type.of(6), "0001-01-01 01:01:01.001000"), rewritten); // fail to fold, because the result is out of range millisecondsAdd = new MilliSecondsAdd( DateTimeV2Literal.fromJavaDateType(LocalDateTime.of(9999, 12, 31, 23, 59, 59)), @@ -1313,6 +1314,7 @@ void testDateTimeV2TypeDateTimeArithmeticFunctions() { .secondCeil(new DateTimeV2Literal("2021-01-01 12:12:12.123"), new IntegerLiteral(2))) == 0); Assertions.assertTrue(new DateTimeV2Literal("2021-01-01 12:12:12.000000").compareTo((ComparableLiteral) TimeRoundSeries .secondFloor(new DateTimeV2Literal("2021-01-01 12:12:12.123"), new IntegerLiteral(2))) == 0); + } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/check/CheckCastTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/check/CheckCastTest.java index b946bd5749a6c3..7969727e14ea28 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/check/CheckCastTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/check/CheckCastTest.java @@ -42,6 +42,7 @@ import org.apache.doris.nereids.types.StringType; import org.apache.doris.nereids.types.StructField; import org.apache.doris.nereids.types.StructType; +import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TimeV2Type; import org.apache.doris.nereids.types.TinyIntType; import org.apache.doris.nereids.types.VarcharType; @@ -56,6 +57,20 @@ import java.util.List; public class CheckCastTest { + @Test + public void testTimeStampNsOnlySupportsStringCasts() { + Assertions.assertTrue(CheckCast.check(StringType.INSTANCE, TimeStampNsType.INSTANCE, true)); + Assertions.assertTrue(CheckCast.check(TimeStampNsType.INSTANCE, StringType.INSTANCE, true)); + Assertions.assertTrue(CheckCast.check(StringType.INSTANCE, TimeStampNsType.INSTANCE, false)); + Assertions.assertTrue(CheckCast.check(TimeStampNsType.INSTANCE, StringType.INSTANCE, false)); + Assertions.assertFalse(CheckCast.check(DateTimeV2Type.MAX, TimeStampNsType.INSTANCE, true)); + Assertions.assertFalse(CheckCast.check(TimeStampNsType.INSTANCE, DateTimeV2Type.MAX, true)); + Assertions.assertFalse(CheckCast.check(DateV2Type.INSTANCE, TimeStampNsType.INSTANCE, true)); + Assertions.assertFalse(CheckCast.check(TimeStampNsType.INSTANCE, DateV2Type.INSTANCE, true)); + Assertions.assertFalse(CheckCast.check(DateTimeV2Type.MAX, TimeStampNsType.INSTANCE, false)); + Assertions.assertFalse(CheckCast.check(TimeStampNsType.INSTANCE, DateTimeV2Type.MAX, false)); + } + @Test public void testCastBetweenVariantTypes() { VariantType v1Source = new VariantType(100); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyCastRuleTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyCastRuleTest.java index 0daf90ca5abf65..36281d7b75a5af 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyCastRuleTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyCastRuleTest.java @@ -22,6 +22,7 @@ import org.apache.doris.nereids.rules.expression.ExpressionRuleExecutor; import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral; import org.apache.doris.nereids.trees.expressions.literal.CharLiteral; import org.apache.doris.nereids.trees.expressions.literal.DecimalLiteral; @@ -37,6 +38,7 @@ import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.SmallIntType; import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TinyIntType; import org.apache.doris.nereids.types.VarcharType; @@ -67,6 +69,9 @@ public void testSimplify() { Expression tinyIntLiteral = new TinyIntLiteral((byte) 12); // cast tinyint as tinyint assertRewrite(new Cast(tinyIntLiteral, TinyIntType.INSTANCE), tinyIntLiteral); + + Expression timestampNs = new SlotReference("timestamp_ns", TimeStampNsType.INSTANCE, true); + assertRewrite(new Cast(timestampNs, TimeStampNsType.INSTANCE), timestampNs); // cast tinyint as decimalv2(3,0) assertRewrite(new Cast(tinyIntLiteral, DecimalV2Type.forType(TinyIntType.INSTANCE)), new DecimalLiteral(DecimalV2Type.forType(TinyIntType.INSTANCE), new BigDecimal(12))); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicateTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicateTest.java index 7d6967e6b3bd10..3e6a5882ff1ef9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicateTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicateTest.java @@ -46,6 +46,7 @@ import org.apache.doris.nereids.trees.expressions.literal.LargeIntLiteral; import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.trees.expressions.literal.SmallIntLiteral; +import org.apache.doris.nereids.trees.expressions.literal.TimeStampNsLiteral; import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.BooleanType; @@ -60,6 +61,7 @@ import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.LargeIntType; import org.apache.doris.nereids.types.SmallIntType; +import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TinyIntType; import org.apache.doris.nereids.util.ExpressionUtils; @@ -460,6 +462,71 @@ void testDateTimeV2CmpDateTimeV2() { ExpressionUtils.falseOrNull(datetime2)); } + @Test + void testTimeStampNsBoundaryDateComparison() { + executor = new ExpressionRuleExecutor(ImmutableList.of( + bottomUp(SimplifyComparisonPredicate.INSTANCE) + )); + + Expression datetime9 = new SlotReference("timestampNs", TimeStampNsType.INSTANCE, true); + Expression castToDate = new Cast(datetime9, DateV2Type.INSTANCE); + DateV2Literal minDate = new DateV2Literal("1677-09-21"); + DateV2Literal maxDate = new DateV2Literal("2262-04-11"); + TimeStampNsLiteral minValue = TimeStampNsLiteral.getMinValue(); + TimeStampNsLiteral maxValue = TimeStampNsLiteral.getMaxValue(); + TimeStampNsLiteral minDayEnd = new TimeStampNsLiteral("1677-09-21 23:59:59.999999999"); + TimeStampNsLiteral maxDayStart = new TimeStampNsLiteral("2262-04-11 00:00:00.000000000"); + + assertRewrite(new EqualTo(castToDate, minDate), new And( + new GreaterThanEqual(datetime9, minValue), new LessThanEqual(datetime9, minDayEnd))); + assertRewrite(new EqualTo(castToDate, maxDate), new And( + new GreaterThanEqual(datetime9, maxDayStart), new LessThanEqual(datetime9, maxValue))); + assertRewrite(new GreaterThanEqual(castToDate, minDate), new GreaterThanEqual(datetime9, minValue)); + assertRewrite(new LessThan(castToDate, minDate), new LessThan(datetime9, minValue)); + assertRewrite(new GreaterThan(castToDate, maxDate), new GreaterThan(datetime9, maxValue)); + assertRewrite(new LessThanEqual(castToDate, maxDate), new LessThanEqual(datetime9, maxValue)); + + DateV2Literal beforeMinDate = new DateV2Literal("1677-09-20"); + DateV2Literal afterMaxDate = new DateV2Literal("2262-04-12"); + assertRewrite(new EqualTo(castToDate, beforeMinDate), ExpressionUtils.falseOrNull(datetime9)); + assertRewrite(new GreaterThan(castToDate, beforeMinDate), ExpressionUtils.trueOrNull(datetime9)); + assertRewrite(new EqualTo(castToDate, afterMaxDate), ExpressionUtils.falseOrNull(datetime9)); + assertRewrite(new LessThan(castToDate, afterMaxDate), ExpressionUtils.trueOrNull(datetime9)); + } + + @Test + void testDateToTimeStampNsSubMicrosecondComparison() { + executor = new ExpressionRuleExecutor(ImmutableList.of( + bottomUp(SimplifyComparisonPredicate.INSTANCE) + )); + + Expression date = new SlotReference("date", DateV2Type.INSTANCE, true); + Expression castToNano = new Cast(date, TimeStampNsType.INSTANCE); + TimeStampNsLiteral nanoAfterMidnight = new TimeStampNsLiteral( + "2024-01-01 00:00:00.000000001"); + DateV2Literal currentDate = new DateV2Literal("2024-01-01"); + DateV2Literal nextDate = new DateV2Literal("2024-01-02"); + + assertRewrite(new EqualTo(castToNano, nanoAfterMidnight), ExpressionUtils.falseOrNull(date)); + assertRewrite(new EqualTo(nanoAfterMidnight, castToNano), ExpressionUtils.falseOrNull(date)); + assertRewrite(new NullSafeEqual(castToNano, nanoAfterMidnight), BooleanLiteral.FALSE); + assertRewrite(new NullSafeEqual(nanoAfterMidnight, castToNano), BooleanLiteral.FALSE); + + assertRewrite(new LessThan(castToNano, nanoAfterMidnight), new LessThan(date, nextDate)); + assertRewrite(new LessThan(nanoAfterMidnight, castToNano), new GreaterThan(date, currentDate)); + assertRewrite(new LessThanEqual(castToNano, nanoAfterMidnight), new LessThanEqual(date, currentDate)); + assertRewrite(new LessThanEqual(nanoAfterMidnight, castToNano), new GreaterThanEqual(date, nextDate)); + assertRewrite(new GreaterThan(castToNano, nanoAfterMidnight), new GreaterThan(date, currentDate)); + assertRewrite(new GreaterThan(nanoAfterMidnight, castToNano), new LessThan(date, nextDate)); + assertRewrite(new GreaterThanEqual(castToNano, nanoAfterMidnight), new GreaterThanEqual(date, nextDate)); + assertRewrite(new GreaterThanEqual(nanoAfterMidnight, castToNano), new LessThanEqual(date, currentDate)); + + assertRewrite(new Not(new EqualTo(castToNano, nanoAfterMidnight)), + new Not(ExpressionUtils.falseOrNull(date))); + assertRewrite(new Not(new EqualTo(nanoAfterMidnight, castToNano)), + new Not(ExpressionUtils.falseOrNull(date))); + } + @Test void testRound() { executor = new ExpressionRuleExecutor(ImmutableList.of( diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/FilterEstimationTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/FilterEstimationTest.java index 8533adb3061ad3..033ba1c8fc4938 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/FilterEstimationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/FilterEstimationTest.java @@ -1406,6 +1406,34 @@ public void testStringRangeColToDateLiteral() { Assertions.assertEquals(4.24, filter2021.getRowCount(), 0.01); } + @Test + public void testStringRangeColToTimeStampNsLiteral() { + SlotReference timestamp = new SlotReference("timestamp", new VarcharType(30)); + ColumnStatistic timestampStats = new ColumnStatisticBuilder(100) + .setNdv(100) + .setAvgSizeByte(30) + .setNumNulls(0) + .setMaxExpr(new StringLiteral( + "2020-2-01 00:00:00.000000002").toLegacyLiteral()) + .setMaxValue(new VarcharLiteral( + "2020-2-01 00:00:00.000000002").getDouble()) + .setMinExpr(new StringLiteral( + "2020-1-01 00:00:00.000000001").toLegacyLiteral()) + .setMinValue(new VarcharLiteral( + "2020-1-01 00:00:00.000000001").getDouble()) + .build(); + Statistics baseStats = new StatisticsBuilder() + .setRowCount(100) + .putColumnStatistics(timestamp, timestampStats) + .build(); + + VarcharLiteral october = new VarcharLiteral("2020-10-01 00:00:00.000000003"); + Statistics result = new FilterEstimation().estimate( + new LessThan(timestamp, october), baseStats); + + Assertions.assertEquals(100, result.getRowCount()); + } + @Test public void testStringRangeColToCol() { SlotReference a = new SlotReference("a", new VarcharType(25)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/LiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/LiteralTest.java index 4964414e6b3db5..104db5c21118f6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/LiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/LiteralTest.java @@ -23,12 +23,14 @@ import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; +import org.apache.doris.nereids.trees.expressions.literal.TimeStampNsLiteral; import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.StringType; import org.apache.doris.nereids.types.StructField; import org.apache.doris.nereids.types.StructType; +import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.proto.Types.PGenericType; import org.apache.doris.proto.Types.PGenericType.TypeId; import org.apache.doris.proto.Types.PValues; @@ -57,6 +59,54 @@ public void testEqual() { Assertions.assertTrue(Literal.of(false) instanceof BooleanLiteral); } + @Test + public void testGetResultExpressionTimeStampNs() { + assertTimeStampNsResult(123456800L, "1970-01-01 00:00:00.123456800"); + assertTimeStampNsResult(-10L, "1969-12-31 23:59:59.999999990"); + assertTimeStampNsResult(-1L, "1969-12-31 23:59:59.999999999"); + } + + @Test + public void testGetResultExpressionNestedTimeStampNs() { + TimeStampNsType nanoType = TimeStampNsType.INSTANCE; + PGenericType int64Type = PGenericType.newBuilder().setId(TypeId.INT64).build(); + PValues nanoValues = PValues.newBuilder() + .setType(int64Type) + .addInt64Value(-1L) + .addInt64Value(123456789L) + .build(); + PValues arrayValues = PValues.newBuilder() + .setType(PGenericType.newBuilder().setId(TypeId.LIST).build()) + .addChildElement(nanoValues) + .addChildOffset(2) + .build(); + + List result = org.apache.doris.nereids.rules.expression.rules.FoldConstantRuleOnBE + .getResultExpression(ArrayType.of(nanoType), arrayValues); + Assertions.assertEquals(1, result.size()); + Assertions.assertTrue(result.get(0) instanceof ArrayLiteral); + List items = ((ArrayLiteral) result.get(0)).getValue(); + Assertions.assertEquals("1969-12-31 23:59:59.999999999", + ((TimeStampNsLiteral) items.get(0)).getStringValue()); + Assertions.assertEquals("1970-01-01 00:00:00.123456789", + ((TimeStampNsLiteral) items.get(1)).getStringValue()); + } + + private void assertTimeStampNsResult(long epochNanos, String expected) { + PValues values = PValues.newBuilder() + .setType(PGenericType.newBuilder().setId(TypeId.INT64).build()) + .addInt64Value(epochNanos) + .build(); + List result = org.apache.doris.nereids.rules.expression.rules.FoldConstantRuleOnBE + .getResultExpression(TimeStampNsType.INSTANCE, values); + + Assertions.assertEquals(1, result.size()); + Assertions.assertInstanceOf(TimeStampNsLiteral.class, result.get(0)); + TimeStampNsLiteral literal = (TimeStampNsLiteral) result.get(0); + Assertions.assertSame(TimeStampNsType.INSTANCE, literal.getDataType()); + Assertions.assertEquals(expected, literal.getStringValue()); + } + @Test public void testGetResultExpressionArrayInt() { int num = 10; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/ComputeSignatureHelperTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/ComputeSignatureHelperTest.java index e0e1f0f30b6be9..b410391b9b1e45 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/ComputeSignatureHelperTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/ComputeSignatureHelperTest.java @@ -30,6 +30,7 @@ import org.apache.doris.nereids.trees.expressions.literal.MapLiteral; import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.trees.expressions.literal.SmallIntLiteral; +import org.apache.doris.nereids.trees.expressions.literal.TimeStampNsLiteral; import org.apache.doris.nereids.trees.expressions.literal.TimeV2Literal; import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.BigIntType; @@ -45,6 +46,7 @@ import org.apache.doris.nereids.types.MapType; import org.apache.doris.nereids.types.NullType; import org.apache.doris.nereids.types.SmallIntType; +import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TimeV2Type; import org.apache.doris.nereids.types.coercion.AnyDataType; import org.apache.doris.nereids.types.coercion.FollowToAnyDataType; @@ -449,6 +451,50 @@ void testMixedDateTimeV2AndTimeV2PrecisionPromotion() { Assertions.assertEquals(DateTimeV2Type.of(3), signature.returnType); } + @Test + void testDateTimeV2SignatureDoesNotPromoteToTimestampNs() { + FunctionSignature signature = FunctionSignature.ret(DateV2Type.INSTANCE) + .args(DateTimeV2Type.SYSTEM_DEFAULT); + List arguments = Lists.newArrayList( + new TimeStampNsLiteral("2020-02-02 00:00:00.123456789")); + + signature = ComputeSignatureHelper.computePrecision( + new FakeComputeSignature(), signature, arguments); + + Assertions.assertEquals(DateTimeV2Type.MAX, signature.getArgType(0)); + Assertions.assertEquals(DateV2Type.INSTANCE, signature.returnType); + } + + @Test + void testTimestampNsDoesNotPromoteDateTimeV2OrTimeV2Signatures() { + FunctionSignature signature = FunctionSignature.ret(DateTimeV2Type.SYSTEM_DEFAULT) + .args(DateTimeV2Type.SYSTEM_DEFAULT, TimeV2Type.SYSTEM_DEFAULT); + List arguments = Lists.newArrayList( + new TimeStampNsLiteral("2020-02-02 00:00:00.123456789"), + new TimeV2Literal("12:34:56.123456")); + + signature = ComputeSignatureHelper.computePrecision( + new FakeComputeSignature(), signature, arguments); + + Assertions.assertEquals(DateTimeV2Type.MAX, signature.getArgType(0)); + Assertions.assertEquals(TimeV2Type.of(6), signature.getArgType(1)); + Assertions.assertEquals(DateTimeV2Type.MAX, signature.returnType); + } + + @Test + void testExplicitTimestampNsSignatureRemainsTimestampNs() { + FunctionSignature signature = FunctionSignature.ret(TimeStampNsType.INSTANCE) + .args(TimeStampNsType.INSTANCE); + List arguments = Lists.newArrayList( + new TimeStampNsLiteral("2020-02-02 00:00:00.123456789")); + + signature = ComputeSignatureHelper.computePrecision( + new FakeComputeSignature(), signature, arguments); + + Assertions.assertEquals(TimeStampNsType.INSTANCE, signature.getArgType(0)); + Assertions.assertEquals(TimeStampNsType.INSTANCE, signature.returnType); + } + @Test void testNestedTimeV2PrecisionPromotion() { FunctionSignature signature = FunctionSignature.ret(ArrayType.of(TimeV2Type.SYSTEM_DEFAULT)).args( diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateTimeMicrosecondFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateTimeMicrosecondFunctionTest.java new file mode 100644 index 00000000000000..ad249d092dcf99 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateTimeMicrosecondFunctionTest.java @@ -0,0 +1,120 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.DateTimeV2Type; +import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.TimeStampTzType; +import org.apache.doris.nereids.types.TimeV2Type; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; + +class DateTimeMicrosecondFunctionTest { + + @Test + void testDateTimeV2MicrosecondArithmeticMinimumScale() { + assertDateTimeV2Scale(0, DateTimeV2Type.MAX_SCALE); + assertDateTimeV2Scale(6, DateTimeV2Type.MAX_SCALE); + } + + @Test + void testTimestampTzMicrosecondArithmeticScale() { + Expression timestamp = new SlotReference("timestamp", TimeStampTzType.of(3)); + for (ScalarFunction function : createFunctions(timestamp)) { + FunctionSignature signature = function.getSignature(); + Assertions.assertEquals(TimeStampTzType.MAX, signature.getArgType(0), function.getName()); + Assertions.assertEquals(TimeStampTzType.MAX, signature.returnType, function.getName()); + } + } + + @Test + void testTimestampTzStringLiteralKeepsTimezoneType() { + SecondMicrosecondAdd function = new SecondMicrosecondAdd( + new VarcharLiteral("2004-11-30 01:00:00+08:00"), new VarcharLiteral("10.123456")); + FunctionSignature signature = function.getSignature(); + Assertions.assertEquals(TimeStampTzType.MAX, signature.getArgType(0)); + Assertions.assertEquals(TimeStampTzType.MAX, signature.returnType); + } + + @Test + void testImplicitStringDateTimeFunctionsKeepMicrosecondPrecision() { + Expression stringSlot = new SlotReference("string", StringType.INSTANCE); + Expression nullableString = new Nullable(new VarcharLiteral("9999-01-01 00:00:00")); + + assertDateTimeV2MicrosecondSignature(new AddTime(stringSlot, + new SlotReference("time", TimeV2Type.MAX)), 0); + assertDateTimeV2MicrosecondSignature(new DateFormat(stringSlot, + new VarcharLiteral("%Y-%m-%d")), 0); + assertDateTimeV2MicrosecondSignature(new ToSeconds(stringSlot), 0); + assertDateTimeV2MicrosecondSignature(new YearsAdd(nullableString, new IntegerLiteral(-1)), 0); + assertDateTimeV2MicrosecondSignature(new Timestamp( + new VarcharLiteral("1st Jun 2007 09:45:30")), 0); + + FunctionSignature timeSignature = new Time(nullableString).getSignature(); + Assertions.assertEquals(DateTimeV2Type.MAX, timeSignature.getArgType(0)); + Assertions.assertEquals(TimeV2Type.MAX, timeSignature.returnType); + } + + private void assertDateTimeV2MicrosecondSignature(ScalarFunction function, int argumentIndex) { + FunctionSignature signature = function.getSignature(); + Assertions.assertEquals(DateTimeV2Type.MAX, + signature.getArgType(argumentIndex), function.getName()); + if (signature.returnType instanceof DateTimeV2Type) { + Assertions.assertEquals(DateTimeV2Type.MAX, + signature.returnType, function.getName()); + } + } + + private void assertDateTimeV2Scale(int inputScale, int expectedScale) { + Expression datetime = new SlotReference("datetime", DateTimeV2Type.of(inputScale)); + DataType expectedType = DateTimeV2Type.of(expectedScale); + for (ScalarFunction function : createFunctions(datetime)) { + FunctionSignature signature = function.getSignature(); + Assertions.assertEquals(expectedType, signature.getArgType(0), function.getName()); + Assertions.assertEquals(expectedType, signature.returnType, function.getName()); + } + } + + private List createFunctions(Expression datetime) { + BigIntLiteral numericInterval = new BigIntLiteral(1); + VarcharLiteral compositeInterval = new VarcharLiteral("1"); + return List.of( + new MicroSecondsAdd(datetime, numericInterval), + new MicroSecondsSub(datetime, numericInterval), + new MilliSecondsAdd(datetime, numericInterval), + new MilliSecondsSub(datetime, numericInterval), + new DayMicrosecondAdd(datetime, compositeInterval), + new DayMicrosecondSub(datetime, compositeInterval), + new HourMicrosecondAdd(datetime, compositeInterval), + new HourMicrosecondSub(datetime, compositeInterval), + new MinuteMicrosecondAdd(datetime, compositeInterval), + new MinuteMicrosecondSub(datetime, compositeInterval), + new SecondMicrosecondAdd(datetime, compositeInterval), + new SecondMicrosecondSub(datetime, compositeInterval)); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrToDateTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrToDateTest.java index abb37cfc9198a6..54104a48e7dbca 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrToDateTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrToDateTest.java @@ -18,9 +18,11 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar; import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; import org.apache.doris.nereids.types.DateTimeV2Type; import org.apache.doris.nereids.types.DateV2Type; +import org.apache.doris.nereids.types.StringType; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -33,11 +35,19 @@ void testComputeSignatureWithFoldableFormat() { new Concat(new StringLiteral("%Y-%m"), new StringLiteral("-%d"))); StrToDate dateTimeFormat = new StrToDate(new StringLiteral("2024-01-01 12:34:56"), new Concat(new StringLiteral("%Y-%m-%d"), new StringLiteral(" %H:%i:%s"))); + StrToDate microsecondFormat = new StrToDate(new StringLiteral("2024-01-01 12:34:56.123456"), + new Concat(new StringLiteral("%Y-%m-%d %H:%i:%s"), new StringLiteral(".%f"))); + StrToDate nonConstantFormat = new StrToDate(new StringLiteral("2024-01-01 12:34:56.123456"), + new SlotReference("format", StringType.INSTANCE)); FunctionSignature dateSignature = dateFormat.computeSignature(StrToDate.SIGNATURES.get(0)); FunctionSignature dateTimeSignature = dateTimeFormat.computeSignature(StrToDate.SIGNATURES.get(0)); + FunctionSignature microsecondSignature = microsecondFormat.computeSignature(StrToDate.SIGNATURES.get(0)); + FunctionSignature nonConstantSignature = nonConstantFormat.computeSignature(StrToDate.SIGNATURES.get(1)); Assertions.assertEquals(DateV2Type.INSTANCE, dateSignature.returnType); Assertions.assertEquals(DateTimeV2Type.SYSTEM_DEFAULT, dateTimeSignature.returnType); + Assertions.assertEquals(DateTimeV2Type.MAX, microsecondSignature.returnType); + Assertions.assertEquals(DateTimeV2Type.MAX, nonConstantSignature.returnType); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/window/TimestampTzLeadLagSignatureTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/window/TimestampTzLeadLagSignatureTest.java index 238e863a745d2e..1d7a0920aaf7ee 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/window/TimestampTzLeadLagSignatureTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/window/TimestampTzLeadLagSignatureTest.java @@ -19,6 +19,7 @@ import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TimeStampTzType; import org.junit.jupiter.api.Assertions; @@ -29,6 +30,19 @@ */ public class TimestampTzLeadLagSignatureTest { + @Test + public void testLeadAndLagSignaturesMatchTimeStampNs() { + SlotReference slot = SlotReference.of("ts", TimeStampNsType.INSTANCE); + + FunctionSignature leadSignature = new Lead(slot).getSignature(); + Assertions.assertEquals(TimeStampNsType.INSTANCE, leadSignature.returnType); + Assertions.assertEquals(TimeStampNsType.INSTANCE, leadSignature.getArgType(0)); + + FunctionSignature lagSignature = new Lag(slot).getSignature(); + Assertions.assertEquals(TimeStampNsType.INSTANCE, lagSignature.returnType); + Assertions.assertEquals(TimeStampNsType.INSTANCE, lagSignature.getArgType(0)); + } + @Test public void testLeadSignatureMatchesTimeStampTz() { SlotReference slot = SlotReference.of("ts", TimeStampTzType.of(6)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteralTest.java index 8b2db5b3511e8b..9bdaf95d59b51c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteralTest.java @@ -30,6 +30,7 @@ import org.apache.doris.nereids.types.LargeIntType; import org.apache.doris.nereids.types.SmallIntType; import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.TimeStampNsType; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Disabled; @@ -59,11 +60,13 @@ void testNormalize() { s = DateLiteral.normalize("2021-5-01 0:0:0.001").get(); Assertions.assertEquals("2021-05-01 00:00:00.001", s); s = DateLiteral.normalize("2021-5-01 0:0:0.12345678").get(); - Assertions.assertEquals("2021-05-01 00:00:00.1234567", s); + Assertions.assertEquals("2021-05-01 00:00:00.12345678", s); + s = DateLiteral.normalize("2021-5-01 0:0:0.12345678951").get(); + Assertions.assertEquals("2021-05-01 00:00:00.1234567895", s); s = DateLiteral.normalize("2021-5-1 Asia/Shanghai").get(); Assertions.assertEquals("2021-05-01Asia/Shanghai", s); s = DateLiteral.normalize("2021-5-1 0:0:0.12345678 Asia/Shanghai").get(); - Assertions.assertEquals("2021-05-01 00:00:00.1234567Asia/Shanghai", s); + Assertions.assertEquals("2021-05-01 00:00:00.12345678Asia/Shanghai", s); } @Test @@ -224,6 +227,64 @@ void testPoint() { new DateTimeV2Literal("2020.02.01 00.00.00.0000001"); } + @Test + void testTimeStampNsseconds() { + Assertions.assertFalse(DateTimeLiteral.class.isAssignableFrom(TimeStampNsLiteral.class)); + TimeStampNsLiteral nanoseconds = + new TimeStampNsLiteral("1970-01-01 00:00:00.123456789"); + Assertions.assertEquals(TimeStampNsType.INSTANCE, nanoseconds.getDataType()); + Assertions.assertEquals(123456789, nanoseconds.getNanoSecond()); + Assertions.assertEquals("1970-01-01 00:00:00.123456789", + nanoseconds.getStringValue()); + Assertions.assertEquals("1970-01-01 00:00:00.123456789", + nanoseconds.toLegacyLiteral().getStringValue()); + Assertions.assertInstanceOf(org.apache.doris.analysis.TimeStampNsLiteral.class, + nanoseconds.toLegacyLiteral()); + Assertions.assertThrows(IllegalArgumentException.class, + () -> DateTimeV2Literal.create(TimeStampNsType.INSTANCE, + "1970-01-01 00:00:00.123456789")); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new DateTimeV2Literal( + TimeStampNsType.INSTANCE, "1970-01-01 00:00:00.123456789")); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new DateTimeV2Literal( + TimeStampNsType.INSTANCE, 1970, 1, 1, 0, 0, 0, 123456789)); + + TimeStampNsLiteral rounded = new TimeStampNsLiteral("1970-01-01 00:00:00.12345675"); + Assertions.assertEquals("1970-01-01 00:00:00.123456750", + rounded.getStringValue()); + + TimeStampNsLiteral scale7 = new TimeStampNsLiteral("1969-12-31 23:59:59.99999994"); + Assertions.assertEquals("1969-12-31 23:59:59.999999940", + scale7.getStringValue()); + TimeStampNsLiteral scale8 = new TimeStampNsLiteral("1969-12-31 23:59:59.999999995"); + Assertions.assertEquals("1969-12-31 23:59:59.999999995", + scale8.getStringValue()); + for (String value : new String[] { + "2024-02-29 12:34:56.1234567", + "2024-02-29 12:34:56.12345678", + "2024-02-29 12:34:56.123456789"}) { + DateTimeV2Type inferredType = DateTimeV2Type.forTypeFromString(value); + Assertions.assertEquals(DateTimeV2Type.MAX_SCALE, inferredType.getScale()); + Assertions.assertFalse(inferredType instanceof TimeStampNsType); + } + + Assertions.assertDoesNotThrow(() -> + new TimeStampNsLiteral("1677-09-21 00:12:43.145224192")); + Assertions.assertDoesNotThrow(() -> + new TimeStampNsLiteral("2262-04-11 23:47:16.854775807")); + Assertions.assertEquals(Long.MIN_VALUE, + new TimeStampNsLiteral("1677-09-21 00:12:43.145224192") + .toLegacyLiteral().getRealValue()); + Assertions.assertEquals(Long.MAX_VALUE, + new TimeStampNsLiteral("2262-04-11 23:47:16.854775807") + .toLegacyLiteral().getRealValue()); + Assertions.assertThrows(AnalysisException.class, () -> + new TimeStampNsLiteral("1677-09-21 00:12:43.145224191")); + Assertions.assertThrows(AnalysisException.class, () -> + new TimeStampNsLiteral("2262-04-11 23:47:16.854775808")); + } + @Test void testSuffixSpace() { new DateLiteral("2016-07-02 "); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteralTest.java index c3556f1bef4bcb..9e1c73c4a0ee88 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteralTest.java @@ -17,6 +17,8 @@ package org.apache.doris.nereids.trees.expressions.literal; +import org.apache.doris.catalog.MysqlColType; +import org.apache.doris.common.Config; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.exceptions.CastException; import org.apache.doris.nereids.trees.expressions.Expression; @@ -31,15 +33,46 @@ import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.LargeIntType; import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.TimeStampNsType; +import org.apache.doris.nereids.types.TimeV2Type; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.util.function.Consumer; import java.util.function.Function; class DateTimeLiteralTest { + @Test + void testMysqlTimestampKeepsMicrosecondRange() { + boolean previousEnableDateConversion = Config.enable_date_conversion; + try { + Config.enable_date_conversion = true; + ByteBuffer data = ByteBuffer.allocate(12).order(ByteOrder.LITTLE_ENDIAN); + data.put((byte) 11) + .putChar((char) 3022) + .put((byte) 1) + .put((byte) 2) + .put((byte) 3) + .put((byte) 4) + .put((byte) 5) + .putInt(123456) + .flip(); + + Literal literal = Literal.getLiteralByMysqlType( + MysqlColType.MYSQL_TYPE_DATETIME, false, data); + + DateTimeV2Literal dateTime = Assertions.assertInstanceOf(DateTimeV2Literal.class, literal); + Assertions.assertEquals(DateTimeV2Type.MAX, dateTime.getDataType()); + Assertions.assertEquals("3022-01-02 03:04:05.123456", dateTime.getStringValue()); + } finally { + Config.enable_date_conversion = previousEnableDateConversion; + } + } + @Test void reject() { // Assertions.assertThrows(IllegalArgumentException.class, () -> { @@ -402,28 +435,34 @@ void testIrregularDateTimeHourMinuteSecondMicrosecond() { check("2016-07-02 01:01:01.123456", DateTimeV2Literal::new); check("2016-7-02 01:01:01.123456", DateTimeV2Literal::new); - // Testing with microsecond of length 7 - DateTimeV2Literal literal = check("2016-07-02 01:01:01.12345678", DateTimeV2Literal::new); - Assertions.assertEquals(123457, literal.microSecond); - - literal = check("2016-07-02 01:01:01.44444444", DateTimeV2Literal::new); - Assertions.assertEquals(444444, literal.microSecond); - - literal = check("2016-07-02 01:01:01.44444445", DateTimeV2Literal::new); - Assertions.assertEquals(444444, literal.microSecond); - - literal = check("2016-07-02 01:01:01.4444445", DateTimeV2Literal::new); - Assertions.assertEquals(444445, literal.microSecond); - - literal = check("2016-07-02 01:01:01.9999995", DateTimeV2Literal::new); - Assertions.assertEquals(0, literal.microSecond); - Assertions.assertEquals(2, literal.second); - - literal = check("2021-01-01 23:59:59.9999995", DateTimeV2Literal::new); - Assertions.assertEquals(0, literal.microSecond); - Assertions.assertEquals(0, literal.second); - Assertions.assertEquals(0, literal.minute); - Assertions.assertEquals(0, literal.hour); + // Fractional digits above microsecond precision are preserved by TIMESTAMP_NS literals. + TimeStampNsLiteral literal = check("2016-07-02 01:01:01.12345678", TimeStampNsLiteral::new); + Assertions.assertEquals(123456, literal.getMicroSecond()); + Assertions.assertEquals(123456780, literal.getNanoSecond()); + + literal = check("2016-07-02 01:01:01.44444444", TimeStampNsLiteral::new); + Assertions.assertEquals(444444, literal.getMicroSecond()); + Assertions.assertEquals(444444440, literal.getNanoSecond()); + + literal = check("2016-07-02 01:01:01.44444445", TimeStampNsLiteral::new); + Assertions.assertEquals(444444, literal.getMicroSecond()); + Assertions.assertEquals(444444450, literal.getNanoSecond()); + + literal = check("2016-07-02 01:01:01.4444445", TimeStampNsLiteral::new); + Assertions.assertEquals(444444, literal.getMicroSecond()); + Assertions.assertEquals(444444500, literal.getNanoSecond()); + + literal = check("2016-07-02 01:01:01.9999995", TimeStampNsLiteral::new); + Assertions.assertEquals(999999, literal.getMicroSecond()); + Assertions.assertEquals(999999500, literal.getNanoSecond()); + Assertions.assertEquals(1, literal.getSecond()); + + literal = check("2021-01-01 23:59:59.9999995", TimeStampNsLiteral::new); + Assertions.assertEquals(999999, literal.getMicroSecond()); + Assertions.assertEquals(999999500, literal.getNanoSecond()); + Assertions.assertEquals(59, literal.getSecond()); + Assertions.assertEquals(59, literal.getMinute()); + Assertions.assertEquals(23, literal.getHour()); } @Test @@ -458,6 +497,25 @@ void testDateTimeV2Scale() { check("2017-01-01 00:00:00.00000", s -> new DateTimeV2Literal(DateTimeV2Type.of(5), s))); } + @Test + void testTimeStampNsGuardDigitRounding() { + TimeStampNsLiteral literal = new TimeStampNsLiteral("1970-01-01 00:00:00.1234567894"); + Assertions.assertEquals("1970-01-01 00:00:00.123456789", literal.getStringValue()); + + literal = new TimeStampNsLiteral("1970-01-01 00:00:00.1234567895"); + Assertions.assertEquals("1970-01-01 00:00:00.123456790", literal.getStringValue()); + + literal = new TimeStampNsLiteral("1970-01-01 00:00:00.9999999995"); + Assertions.assertEquals("1970-01-01 00:00:01.000000000", literal.getStringValue()); + + literal = new TimeStampNsLiteral("1970-01-01 00:00:00.1234567845"); + Assertions.assertEquals("1970-01-01 00:00:00.123456785", literal.getStringValue()); + + DateTimeLiteral parsed = DateTimeLiteral.parseDateTimeLiteral( + "1970-01-01 00:00:00.9999999995", true).get(); + Assertions.assertEquals("1970-01-01 00:00:01.000000", parsed.getStringValue()); + } + @Test void testRoundFloor() { DateTimeV2Literal literal; @@ -625,7 +683,8 @@ void testDateTimeV1UncheckedCastTo() { @Test void testDateTimeV2UncheckedCastTo() { - DateTimeV2Literal v2 = new DateTimeV2Literal(DateTimeV2Type.MAX, 2025, 7, 23, 13, 25, 59, 999999); + DateTimeV2Literal v2 = new DateTimeV2Literal( + DateTimeV2Type.of(6), 2025, 7, 23, 13, 25, 59, 999999); Expression expression = v2.uncheckedCastTo(BigIntType.INSTANCE); Assertions.assertInstanceOf(BigIntLiteral.class, expression); @@ -689,9 +748,54 @@ void testDateTimeV2UncheckedCastTo() { Assertions.assertInstanceOf(StringLiteral.class, expression); Assertions.assertEquals("2025-07-23 13:25:59", ((StringLiteral) expression).value); - v2 = new DateTimeV2Literal(DateTimeV2Type.MAX, 2025, 7, 23, 13, 25, 59, 0); + v2 = new DateTimeV2Literal(DateTimeV2Type.of(6), 2025, 7, 23, 13, 25, 59, 0); expression = v2.uncheckedCastTo(StringType.INSTANCE); Assertions.assertInstanceOf(StringLiteral.class, expression); Assertions.assertEquals("2025-07-23 13:25:59.000000", ((StringLiteral) expression).value); } + + @Test + void testTimeStampNsUncheckedCastRoundingAndRange() { + TimeStampNsLiteral nano = new TimeStampNsLiteral("2024-02-29 12:34:56.123456789"); + DateTimeV2Literal micro = (DateTimeV2Literal) nano.uncheckedCastTo(DateTimeV2Type.of(6)); + Assertions.assertEquals("2024-02-29 12:34:56.123457", micro.getStringValue()); + + nano = new TimeStampNsLiteral("1969-12-31 23:59:59.999999999"); + micro = (DateTimeV2Literal) nano.uncheckedCastTo(DateTimeV2Type.of(6)); + Assertions.assertEquals("1970-01-01 00:00:00.000000", micro.getStringValue()); + TimeV2Literal time = (TimeV2Literal) nano.uncheckedCastTo(TimeV2Type.of(6)); + Assertions.assertEquals("24:00:00.000000", time.getStringValue()); + + nano = new TimeStampNsLiteral("2024-02-29 12:34:56.123456789"); + time = (TimeV2Literal) nano.uncheckedCastTo(TimeV2Type.of(6)); + Assertions.assertEquals("12:34:56.123457", time.getStringValue()); + + TimeStampNsLiteral lower = new TimeStampNsLiteral("1677-09-21 00:12:43.145224192"); + Assertions.assertSame(lower, lower.uncheckedCastTo(TimeStampNsType.INSTANCE)); + + TimeStampNsLiteral upper = new TimeStampNsLiteral("2262-04-11 23:47:16.854775807"); + Assertions.assertSame(upper, upper.uncheckedCastTo(TimeStampNsType.INSTANCE)); + + DateTimeV2Literal lowerMicro = new DateTimeV2Literal( + DateTimeV2Type.of(6), "1677-09-21 00:12:43.145225"); + Assertions.assertEquals("1677-09-21 00:12:43.145225000", + ((TimeStampNsLiteral) lowerMicro.uncheckedCastTo(TimeStampNsType.INSTANCE)).getStringValue()); + + DateTimeV2Literal yearZero = new DateTimeV2Literal( + DateTimeV2Type.of(6), "0000-01-01 00:00:00.000000"); + DateTimeV2Literal yearNineNineNineNine = new DateTimeV2Literal( + DateTimeV2Type.of(6), "9999-12-31 23:59:59.999999"); + Assertions.assertThrows(CastException.class, + () -> yearZero.uncheckedCastTo(TimeStampNsType.INSTANCE)); + Assertions.assertThrows(CastException.class, + () -> yearNineNineNineNine.uncheckedCastTo(TimeStampNsType.INSTANCE)); + } + + @Test + void testTimestampNsEpochNanoBoundaries() { + Assertions.assertEquals("1677-09-21 00:12:43.145224192", + TimeStampNsLiteral.getMinValue().getStringValue()); + Assertions.assertEquals("2262-04-11 23:47:16.854775807", + TimeStampNsLiteral.getMaxValue().getStringValue()); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/IntegerLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/IntegerLiteralTest.java index 0a9a34f669418b..a1af7a9647b9bc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/IntegerLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/IntegerLiteralTest.java @@ -187,6 +187,10 @@ void testUncheckedCastTo() { Assertions.assertEquals(0, ((DateTimeV2Literal) expression).second); Assertions.assertEquals(0, ((DateTimeV2Literal) expression).microSecond); + IntegerLiteral invalidDate = new IntegerLiteral(1000); + Assertions.assertThrows(CastException.class, + () -> invalidDate.uncheckedCastTo(DateTimeV2Type.SYSTEM_DEFAULT)); + // to string d1 = new IntegerLiteral(701231); expression = d1.uncheckedCastTo(StringType.INSTANCE); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimeV2LiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimeV2LiteralTest.java index 96775d44bfc628..2b740edd37d3e6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimeV2LiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimeV2LiteralTest.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.types.DateTimeV2Type; import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TimeV2Type; import org.junit.jupiter.api.Assertions; @@ -186,4 +187,17 @@ public void testCastNegativeTimeToDateTimeV2KeepsSign() { Assertions.assertEquals(59, dateTime.getSecond()); } + @Test + public void testCastToTimeStampNsKeepsTargetScale() { + TimeV2Literal literal = new TimeV2Literal(TimeV2Type.of(6), "12:34:56.123456"); + + TimeStampNsLiteral dateTime = (TimeStampNsLiteral) literal.uncheckedCastTo(TimeStampNsType.INSTANCE); + + Assertions.assertEquals(TimeStampNsType.INSTANCE, dateTime.getDataType()); + Assertions.assertEquals(12, dateTime.getHour()); + Assertions.assertEquals(34, dateTime.getMinute()); + Assertions.assertEquals(56, dateTime.getSecond()); + Assertions.assertEquals(123456000, dateTime.getNanoSecond()); + } + } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteralTest.java index ac5235fdf34500..f5bb13f96f0901 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteralTest.java @@ -17,7 +17,9 @@ package org.apache.doris.nereids.trees.expressions.literal; +import org.apache.doris.nereids.exceptions.CastException; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TimeStampTzType; import org.apache.doris.qe.ConnectContext; @@ -57,6 +59,8 @@ void testConstructorsAndParsing() { Assertions.assertEquals(14, literal.second); Assertions.assertEquals(123457, literal.microSecond); Assertions.assertEquals(6, literal.getDataType().getScale()); + Assertions.assertEquals(TimeStampTzType.of(6), + TimeStampTzType.MAX.scaleTypeForType(TimeStampNsType.INSTANCE)); literal = new TimestampTzLiteral("2022-12-31 21:00:14.1234567 -06:45"); Assertions.assertEquals(2023, literal.year); @@ -126,6 +130,49 @@ void testGetStringValueKeepsUtcSuffix() { Assertions.assertEquals("2024-01-15 12:00:00.000000+00:00", literal.getStringValue()); } + @Test + void testCastBetweenTimestampTzAndTimeStampNs() { + ConnectContext context = new ConnectContext(); + context.getSessionVariable().setTimeZone("+08:00"); + context.setThreadLocalInfo(); + try { + TimeStampNsLiteral nano = new TimeStampNsLiteral("2024-02-29 12:34:56.123456789"); + TimestampTzLiteral timestampTz = (TimestampTzLiteral) nano.uncheckedCastTo(TimeStampTzType.of(6)); + Assertions.assertEquals("2024-02-29 04:34:56.123457+00:00", timestampTz.getStringValue()); + + timestampTz = new TimestampTzLiteral( + TimeStampTzType.of(6), 2024, 2, 29, 4, 34, 56, 123456); + nano = (TimeStampNsLiteral) timestampTz.uncheckedCastTo(TimeStampNsType.INSTANCE); + Assertions.assertEquals("2024-02-29 12:34:56.123456000", nano.getStringValue()); + + context.getSessionVariable().setTimeZone("+14:00"); + timestampTz = new TimestampTzLiteral( + TimeStampTzType.of(6), 1677, 9, 20, 10, 12, 43, 145225); + nano = (TimeStampNsLiteral) timestampTz.uncheckedCastTo(TimeStampNsType.INSTANCE); + Assertions.assertEquals("1677-09-21 00:12:43.145225000", nano.getStringValue()); + + context.getSessionVariable().setTimeZone("UTC"); + TimestampTzLiteral yearZero = new TimestampTzLiteral( + TimeStampTzType.of(6), 0, 1, 1, 0, 0, 0, 0); + TimestampTzLiteral yearNineNineNineNine = new TimestampTzLiteral( + TimeStampTzType.of(6), 9999, 12, 31, 23, 59, 59, 999999); + TimestampTzLiteral belowLowerBound = new TimestampTzLiteral( + TimeStampTzType.of(6), 1677, 9, 21, 0, 12, 43, 145224); + TimestampTzLiteral aboveUpperBound = new TimestampTzLiteral( + TimeStampTzType.of(6), 2262, 4, 11, 23, 47, 16, 854776); + Assertions.assertThrows(CastException.class, + () -> yearZero.uncheckedCastTo(TimeStampNsType.INSTANCE)); + Assertions.assertThrows(CastException.class, + () -> yearNineNineNineNine.uncheckedCastTo(TimeStampNsType.INSTANCE)); + Assertions.assertThrows(CastException.class, + () -> belowLowerBound.uncheckedCastTo(TimeStampNsType.INSTANCE)); + Assertions.assertThrows(CastException.class, + () -> aboveUpperBound.uncheckedCastTo(TimeStampNsType.INSTANCE)); + } finally { + ConnectContext.remove(); + } + } + @Test void testBasicArithmetic() { TimestampTzLiteral base1 = new TimestampTzLiteral("2025-12-31 23:59:59.999999"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/OutFileTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/OutFileTest.java index 6a10110cf90a92..e6a13b9db4e91a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/OutFileTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/OutFileTest.java @@ -45,7 +45,8 @@ protected void runBeforeAll() throws Exception { createTables( "CREATE TABLE IF NOT EXISTS T1 (\n" + " id bigint,\n" - + " score bigint\n" + + " score bigint,\n" + + " ts timestamp_ns\n" + ")\n" + "DUPLICATE KEY(id)\n" + "DISTRIBUTED BY HASH(id) BUCKETS 1\n" @@ -78,6 +79,15 @@ public void testWriteOutFile() throws Exception { .contains("FILE SINK")); } + @Test + public void testTimestampNsOrcOutfileIsRejected() { + String sql = "select ts from T1 into outfile 'file://~/timestamp_ns.orc' format as orc " + + "properties ('broker.name' = 'my_broker')"; + Exception exception = Assertions.assertThrows(Exception.class, () -> getOutputFragment(sql)); + Assertions.assertTrue(exception.getMessage().contains( + "currently orc do not support column type: TIMESTAMP_NS"), exception.getMessage()); + } + private PlanFragment getOutputFragment(String sql) throws Exception { StatementScopeIdGenerator.clear(); StatementContext statementContext = MemoTestUtils.createStatementContext(connectContext, sql); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/types/DataTypeTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/types/DataTypeTest.java index e8e1eff4e4f170..8af1d320585cc2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/types/DataTypeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/types/DataTypeTest.java @@ -74,6 +74,32 @@ void testFromPrimitiveType() { Assertions.assertEquals(DataType.fromCatalogType(Type.STRING), StringType.INSTANCE); } + @Test + void testTimeStampNsValidation() { + Assertions.assertDoesNotThrow(TimeStampNsType.INSTANCE::validateDataType); + } + + @Test + void testTimeV2PromotionKeepsMicrosecondPrecision() { + Assertions.assertEquals( + ImmutableList.of(DateTimeV2Type.MAX, StringType.INSTANCE), + TimeV2Type.of(6).getAllPromotions()); + } + + @Test + void testImplicitDateTimeV2PrecisionKeepsMicrosecondCompatibility() { + Assertions.assertEquals(DateTimeV2Type.MAX, + DateTimeV2Type.forType(StringType.INSTANCE)); + Assertions.assertEquals(DateTimeV2Type.MAX, + DateTimeV2Type.forType(DecimalV3Type.createDecimalV3Type(18, 9))); + Assertions.assertEquals(DateTimeV2Type.MAX, + DateTimeV2Type.forTypeFromString("1st Jun 2007 09:45:30")); + Assertions.assertEquals(DateTimeV2Type.MAX, + DateTimeV2Type.forType(DateTimeV2Type.MAX)); + Assertions.assertEquals(DateTimeV2Type.MAX, + DateTimeV2Type.forTypeFromString("2024-01-01 00:00:00.123456789")); + } + @Test void testConvertFromString() { // boolean @@ -121,6 +147,16 @@ void testConvertFromString() { // datetimev2 Assertions.assertEquals(DateTimeV2Type.of(3), DataType.convertFromString("datetimev2(3)")); + Assertions.assertSame(TimeStampNsType.INSTANCE, DataType.convertFromString("timestamp_ns")); + for (String datetimeType : ImmutableList.of("datetime", "datetimev2")) { + for (int invalidScale = 7; invalidScale <= 9; invalidScale++) { + int scale = invalidScale; + Assertions.assertThrows(AnalysisException.class, + () -> DataType.convertFromString(datetimeType + "(" + scale + ")")); + } + } + Assertions.assertThrows(AnalysisException.class, + () -> DataType.convertFromString("timestamp_ns(9)")); // hll Assertions.assertEquals(HllType.INSTANCE, DataType.convertFromString("hll")); // bitmap diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionMatrixTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionMatrixTest.java index e654a2794bd237..a8741482501405 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionMatrixTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionMatrixTest.java @@ -45,6 +45,7 @@ import org.apache.doris.nereids.types.StringType; import org.apache.doris.nereids.types.StructField; import org.apache.doris.nereids.types.StructType; +import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TimeV2Type; import org.apache.doris.nereids.types.TinyIntType; import org.apache.doris.nereids.types.VarcharType; @@ -695,6 +696,9 @@ public void testProcessComparisonPredicateForDateTimeV2Type() { testProcessComparisonPredicate(DateTimeV2Type.of(4), CharType.createCharType(5), DateTimeV2Type.of(6)); testProcessComparisonPredicate(DateTimeV2Type.of(4), VarcharType.createVarcharType(5), DateTimeV2Type.of(6)); testProcessComparisonPredicate(DateTimeV2Type.of(4), StringType.INSTANCE, DateTimeV2Type.of(6)); + testProcessComparisonPredicate(TimeStampNsType.INSTANCE, DecimalV2Type.SYSTEM_DEFAULT, + TimeStampNsType.INSTANCE); + testProcessComparisonPredicate(TimeStampNsType.INSTANCE, StringType.INSTANCE, TimeStampNsType.INSTANCE); testProcessComparisonPredicate(DateTimeV2Type.of(4), ArrayType.of(StringType.INSTANCE), null); testProcessComparisonPredicate(DateTimeV2Type.of(4), MapType.of(StringType.INSTANCE, StringType.INSTANCE), null); testProcessComparisonPredicate(DateTimeV2Type.of(4), new StructType(ImmutableList.of(new StructField("c1", StringType.INSTANCE, true, ""))), null); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionUtilsTest.java index 7255e6530b2747..cc3baf2c7800d6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionUtilsTest.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.util; import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.rules.analysis.ExpressionAnalyzer; import org.apache.doris.nereids.rules.expression.check.CheckCast; import org.apache.doris.nereids.trees.expressions.Add; import org.apache.doris.nereids.trees.expressions.Cast; @@ -27,6 +28,7 @@ import org.apache.doris.nereids.trees.expressions.GreaterThan; import org.apache.doris.nereids.trees.expressions.InPredicate; import org.apache.doris.nereids.trees.expressions.Multiply; +import org.apache.doris.nereids.trees.expressions.Not; import org.apache.doris.nereids.trees.expressions.NullSafeEqual; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.Subtract; @@ -73,6 +75,7 @@ import org.apache.doris.nereids.types.StringType; import org.apache.doris.nereids.types.StructField; import org.apache.doris.nereids.types.StructType; +import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TimeStampTzType; import org.apache.doris.nereids.types.TimeV2Type; import org.apache.doris.nereids.types.TinyIntType; @@ -87,8 +90,29 @@ import org.junit.jupiter.api.Test; import java.math.BigDecimal; +import java.util.Optional; public class TypeCoercionUtilsTest { + @Test + public void testReplaceDateTimeV2WithMaxKeepsTimestampNsIndependent() { + Assertions.assertEquals(DateTimeV2Type.MAX, + TypeCoercionUtils.replaceDateTimeV2WithMax(DateTimeV2Type.of(3))); + Assertions.assertSame(TimeStampNsType.INSTANCE, + TypeCoercionUtils.replaceDateTimeV2WithMax(TimeStampNsType.INSTANCE)); + Assertions.assertEquals(ArrayType.of(TimeStampNsType.INSTANCE), + TypeCoercionUtils.replaceDateTimeV2WithMax(ArrayType.of(TimeStampNsType.INSTANCE))); + } + + @Test + public void testVariantWiderTypeKeepsTimestampNsPrecision() { + Assertions.assertEquals(Optional.of(TimeStampNsType.INSTANCE), + TypeCoercionUtils.findWiderTypeForTwo( + VariantType.INSTANCE, TimeStampNsType.INSTANCE, false, false)); + Assertions.assertEquals(Optional.of(ArrayType.of(TimeStampNsType.INSTANCE)), + TypeCoercionUtils.findWiderTypeForTwo( + VariantType.INSTANCE, ArrayType.of(TimeStampNsType.INSTANCE), false, false)); + } + @Test public void testImplicitCastAccept() { IntegerType integerType = IntegerType.INSTANCE; @@ -488,6 +512,32 @@ public void testProcessInStringCoercion() { Assertions.assertEquals(DecimalV3Type.createDecimalV3Type(23, 3), smallIntString.getOptions().get(0).getDataType()); } + @Test + public void testDateStringSubMicrosecondComparisonCoercion() { + Expression date = new SlotReference("date", DateV2Type.INSTANCE, true); + Expression nanoString = new StringLiteral("2024-01-01 00:00:00.000000001"); + + // Preserve the nanosecond literal instead of silently truncating it to DATETIMEV2(6). + // DATEV2/TIMESTAMP_NS casts are intentionally unsupported until cross-type casts land. + Assertions.assertThrows(AnalysisException.class, + () -> TypeCoercionUtils.processComparisonPredicate(new EqualTo(date, nanoString))); + Assertions.assertThrows(AnalysisException.class, + () -> TypeCoercionUtils.processComparisonPredicate(new EqualTo(nanoString, date))); + } + + @Test + public void testDateStringSubMicrosecondInCoercion() { + Expression date = new SlotReference("date", DateV2Type.INSTANCE, true); + Expression nanoString = new StringLiteral("2024-01-01 00:00:00.000000001"); + Expression dateString = new StringLiteral("2024-01-02"); + + Assertions.assertThrows(AnalysisException.class, () -> TypeCoercionUtils.processInPredicate( + new InPredicate(date, ImmutableList.of(nanoString, dateString)))); + Assertions.assertThrows(AnalysisException.class, + () -> ExpressionAnalyzer.FUNCTION_ANALYZER_RULE.rewrite( + new Not(new InPredicate(date, ImmutableList.of(dateString, nanoString))), null)); + } + @Test public void testCharacterLiteralTypeCoercion() { // datev2 diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java index 607cea3b40a302..67999830275f31 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java @@ -17,9 +17,14 @@ package org.apache.doris.qe; +import org.apache.doris.analysis.Expr; +import org.apache.doris.catalog.ArrayType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.InternalSchemaInitializer; import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.Type; +import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.common.FeConstants; import org.apache.doris.mysql.MysqlChannel; @@ -45,6 +50,7 @@ import java.lang.reflect.Field; import java.lang.reflect.Method; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; @@ -279,6 +285,21 @@ public Void answer(InvocationOnMock invocation) { executor.sendTextResultRow(resultSet); } + @Test + public void testArrowFlightSqlRejectsTimestampNsOutput() { + Expr timestampNs = Mockito.mock(Expr.class); + Mockito.when(timestampNs.getType()).thenReturn(Type.TIMESTAMP_NS); + AnalysisException scalarException = Assertions.assertThrows(AnalysisException.class, + () -> StmtExecutor.checkArrowFlightSqlOutput(Lists.newArrayList(timestampNs))); + Assertions.assertTrue(scalarException.getMessage().contains( + "TIMESTAMP_NS is not supported by Arrow Flight SQL"), scalarException.getMessage()); + + Expr nestedTimestampNs = Mockito.mock(Expr.class); + Mockito.when(nestedTimestampNs.getType()).thenReturn(ArrayType.create(Type.TIMESTAMP_NS, true)); + Assertions.assertThrows(AnalysisException.class, + () -> StmtExecutor.checkArrowFlightSqlOutput(Lists.newArrayList(nestedTimestampNs))); + } + @Test public void testSendBinaryResultRow() throws IOException { ConnectContext mockCtx = Mockito.mock(ConnectContext.class); @@ -327,6 +348,37 @@ public Void answer(InvocationOnMock invocation) { executor.sendBinaryResultRow(resultSet); } + @Test + public void testSendBinaryTimestampNsResultRow() throws IOException { + ConnectContext mockCtx = Mockito.mock(ConnectContext.class); + MysqlChannel channel = Mockito.mock(MysqlChannel.class); + Mockito.when(mockCtx.getConnectType()).thenReturn(ConnectType.MYSQL); + Mockito.when(mockCtx.getMysqlChannel()).thenReturn(channel); + MysqlSerializer mysqlSerializer = MysqlSerializer.newInstance(); + Mockito.when(channel.getSerializer()).thenReturn(mysqlSerializer); + Mockito.when(mockCtx.getSessionVariable()).thenReturn(VariableMgr.newSessionVariable()); + + String value = "2025-01-01 01:02:03.123456789"; + List> rows = Lists.newArrayList(); + rows.add(Lists.newArrayList(value)); + ResultSet resultSet = new CommonResultSet( + new CommonResultSetMetaData(Lists.newArrayList( + new Column("timestamp_ns", ScalarType.createTimeStampNsType()))), + rows); + Mockito.doAnswer(invocation -> { + byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8); + byte[] expected = new byte[valueBytes.length + 3]; + expected[2] = (byte) valueBytes.length; + System.arraycopy(valueBytes, 0, expected, 3, valueBytes.length); + ByteBuffer buffer = invocation.getArgument(0); + Assertions.assertArrayEquals(expected, buffer.array()); + return null; + }).when(channel).sendOnePacket(Mockito.any(ByteBuffer.class)); + + StmtExecutor executor = new StmtExecutor(mockCtx, new OriginStatement("", 1), false); + executor.sendBinaryResultRow(resultSet); + } + @Test public void testSendBinaryBooleanResultRow() throws IOException { ConnectContext mockCtx = Mockito.mock(ConnectContext.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlSchemaHelperArrowTypeTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlSchemaHelperArrowTypeTest.java index 147a52aa55ed98..78db26687affdc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlSchemaHelperArrowTypeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlSchemaHelperArrowTypeTest.java @@ -89,6 +89,13 @@ public void dateV2IsDescribedAsDate32() { buildField(desc("d", TPrimitiveType.DATEV2)).getType()); } + @Test + public void timestampNsIsRejectedUntilArrowOutputIsImplemented() { + IllegalArgumentException exception = Assertions.assertThrows(IllegalArgumentException.class, + () -> buildField(desc("ts", TPrimitiveType.TIMESTAMP_NS))); + Assertions.assertEquals("TIMESTAMP_NS is not supported by Arrow Flight SQL", exception.getMessage()); + } + /** * An Arrow list carries its element type in its child and nowhere else, so the placeholder child this * replaced ({@code ZeroVector}'s Null type) described every array in the catalog as an array OF diff --git a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 index 446685892295e4..d2ba7a1203b786 100644 --- a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 +++ b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 @@ -573,6 +573,7 @@ THEN: 'THEN'; TIME: 'TIME'; TIMESTAMP: 'TIMESTAMP'; TIMESTAMPTZ: 'TIMESTAMPTZ'; +TIMESTAMP_NS: 'TIMESTAMP_NS'; TINYINT: 'TINYINT'; TO: 'TO'; TOKENIZER: 'TOKENIZER'; diff --git a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 9dfd4630ed8890..961a107585291e 100644 --- a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -1900,6 +1900,7 @@ primitiveColType | type=DATEV1 | type=DATETIMEV1 | type=TIMESTAMPTZ + | type=TIMESTAMP_NS | type=BITMAP | type=QUANTILE_STATE | type=HLL @@ -2364,6 +2365,7 @@ nonReserved | TIME | TIMESTAMP | TIMESTAMPTZ + | TIMESTAMP_NS | TRANSACTION | TREE | TRIGGERS diff --git a/fe/fe-type/src/main/java/org/apache/doris/catalog/PrimitiveType.java b/fe/fe-type/src/main/java/org/apache/doris/catalog/PrimitiveType.java index 216db92e21698a..36b640831772f1 100644 --- a/fe/fe-type/src/main/java/org/apache/doris/catalog/PrimitiveType.java +++ b/fe/fe-type/src/main/java/org/apache/doris/catalog/PrimitiveType.java @@ -66,6 +66,7 @@ public enum PrimitiveType { AGG_STATE("AGG_STATE", 16, TPrimitiveType.AGG_STATE, true), DATEV2("DATEV2", 4, TPrimitiveType.DATEV2, true), DATETIMEV2("DATETIMEV2", 8, TPrimitiveType.DATETIMEV2, true), + TIMESTAMP_NS("TIMESTAMP_NS", 8, TPrimitiveType.TIMESTAMP_NS, true), TIMEV2("TIMEV2", 8, TPrimitiveType.TIMEV2, false), TIMESTAMPTZ("TIMESTAMPTZ", 8, TPrimitiveType.TIMESTAMPTZ, false), LAMBDA_FUNCTION("LAMBDA_FUNCTION", 16, TPrimitiveType.LAMBDA_FUNCTION, false), @@ -135,6 +136,7 @@ public enum PrimitiveType { supportedTypes.add(DATETIME); supportedTypes.add(DATEV2); supportedTypes.add(DATETIMEV2); + supportedTypes.add(TIMESTAMP_NS); supportedTypes.add(TIMEV2); supportedTypes.add(TIMESTAMPTZ); supportedTypes.add(IPV4); @@ -213,6 +215,8 @@ public static PrimitiveType fromThrift(TPrimitiveType tPrimitiveType) { return DATEV2; case DATETIMEV2: return DATETIMEV2; + case TIMESTAMP_NS: + return TIMESTAMP_NS; case IPV4: return IPV4; case IPV6: @@ -328,11 +332,12 @@ public boolean isNull() { } public boolean isDateLikeType() { - return (this == DATE || this == DATETIME || this == DATEV2 || this == DATETIMEV2 || this == TIMESTAMPTZ); + return this == DATE || this == DATETIME || this == DATEV2 || this == DATETIMEV2 + || this == TIMESTAMP_NS || this == TIMESTAMPTZ; } public boolean isDateV2LikeType() { - return (this == DATEV2 || this == DATETIMEV2 || this == TIMESTAMPTZ); + return this == DATEV2 || this == DATETIMEV2 || this == TIMESTAMP_NS || this == TIMESTAMPTZ; } public boolean isTimeStampTzType() { @@ -428,6 +433,7 @@ public MysqlColType toMysqlType() { case DATEV2: return MysqlColType.MYSQL_TYPE_DATE; case TIMESTAMPTZ: + case TIMESTAMP_NS: return MysqlColType.MYSQL_TYPE_STRING; case DATETIME: case DATETIMEV2: { @@ -463,6 +469,7 @@ public int getOlapColumnIndexSize() { return DATEV2_INDEX_LEN; case DATETIME: case DATETIMEV2: + case TIMESTAMP_NS: case TIMESTAMPTZ: return DATETIME_INDEX_LEN; case VARCHAR: diff --git a/fe/fe-type/src/main/java/org/apache/doris/catalog/ScalarType.java b/fe/fe-type/src/main/java/org/apache/doris/catalog/ScalarType.java index 219a7c2384fc6c..b39bfbbd69f14c 100644 --- a/fe/fe-type/src/main/java/org/apache/doris/catalog/ScalarType.java +++ b/fe/fe-type/src/main/java/org/apache/doris/catalog/ScalarType.java @@ -81,6 +81,7 @@ public class ScalarType extends Type { public static final int MAX_DECIMAL256_PRECISION = 76; public static final int DEFAULT_MIN_AVG_DECIMAL128_SCALE = 4; public static final int MAX_DATETIMEV2_SCALE = 6; + public static final int TIMESTAMP_NS_SCALE = 9; public static final int MAX_PRECISION = MAX_DECIMAL256_PRECISION; private long byteSize = -1; @@ -146,6 +147,8 @@ public static ScalarType createType(PrimitiveType type, int len, int precision, return createDecimalType(precision, scale); case DATETIMEV2: return createDatetimeV2Type(scale); + case TIMESTAMP_NS: + return createTimeStampNsType(); case TIMEV2: return createTimeV2Type(scale); case TIMESTAMPTZ: @@ -205,6 +208,8 @@ public static ScalarType createType(PrimitiveType type) { return DATEV2; case DATETIMEV2: return DEFAULT_DATETIMEV2; + case TIMESTAMP_NS: + return Type.TIMESTAMP_NS; case TIMEV2: return TIMEV2; case TIMESTAMPTZ: @@ -281,6 +286,8 @@ public static ScalarType createType(String type) { return DATEV2; case "DATETIMEV2": return DATETIMEV2; + case "TIMESTAMP_NS": + return TIMESTAMP_NS; case "TIME": case "TIMEV2": return TIMEV2; @@ -451,6 +458,11 @@ public static ScalarType createDatetimeV2Type(int scale) { return type; } + @SuppressWarnings("checkstyle:MissingJavadocMethod") + public static ScalarType createTimeStampNsType() { + return new ScalarType(PrimitiveType.TIMESTAMP_NS); + } + @SuppressWarnings("checkstyle:MissingJavadocMethod") public static ScalarType createTimeV2Type(int scale) { ScalarType type = new ScalarType(PrimitiveType.TIMEV2); @@ -605,13 +617,15 @@ public String toString() { return "decimal(*,*)"; } return "decimal(" + precision + "," + scale + ")"; - } else if (type.isDecimalV3Type()) { + } else if (type.isDecimalV3Type()) { if (isWildcardDecimal()) { return "decimalv3(*,*)"; } return "decimalv3(" + precision + "," + scale + ")"; - } else if (type == PrimitiveType.DATETIMEV2) { + } else if (type == PrimitiveType.DATETIMEV2) { return "datetimev2(" + scale + ")"; + } else if (type == PrimitiveType.TIMESTAMP_NS) { + return "timestamp_ns"; } else if (type == PrimitiveType.TIMEV2) { return "timev2(" + scale + ")"; } else if (type == PrimitiveType.TIMESTAMPTZ) { @@ -680,6 +694,9 @@ public String toSql(int depth) { case DATETIMEV2: stringBuilder.append("datetimev2").append("(").append(scale).append(")"); break; + case TIMESTAMP_NS: + stringBuilder.append("timestamp_ns"); + break; case TIMEV2: stringBuilder.append("time").append("(").append(scale).append(")"); break; diff --git a/fe/fe-type/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-type/src/main/java/org/apache/doris/catalog/Type.java index 3c8632e1a7be1d..6f9383ee0d6b31 100644 --- a/fe/fe-type/src/main/java/org/apache/doris/catalog/Type.java +++ b/fe/fe-type/src/main/java/org/apache/doris/catalog/Type.java @@ -71,6 +71,7 @@ public abstract class Type { public static final ScalarType DATEV2 = new ScalarType(PrimitiveType.DATEV2); public static final ScalarType TIMEV2 = new ScalarType(PrimitiveType.TIMEV2); public static final ScalarType TIMESTAMPTZ = new ScalarType(PrimitiveType.TIMESTAMPTZ); + public static final ScalarType TIMESTAMP_NS = ScalarType.createTimeStampNsType(); public static final ScalarType STRING = ScalarType.createStringType(); public static final ScalarType VARBINARY = ScalarType.createVarbinaryType(-1); public static final ScalarType DEFAULT_DECIMALV2 = ScalarType.createDecimalType(PrimitiveType.DECIMALV2, @@ -97,7 +98,8 @@ public abstract class Type { public static final ScalarType DEFAULT_DECIMALV3 = DEFAULT_DECIMAL32; public static final ScalarType DEFAULT_DATETIMEV2 = ScalarType.createDatetimeV2Type(0); public static final ScalarType DATETIMEV2 = DEFAULT_DATETIMEV2; - public static final ScalarType DATETIMEV2_WITH_MAX_SCALAR = ScalarType.createDatetimeV2Type(6); + public static final ScalarType DATETIMEV2_WITH_MAX_SCALAR + = ScalarType.createDatetimeV2Type(ScalarType.MAX_DATETIMEV2_SCALE); public static final ScalarType DEFAULT_TIMESTAMP_TZ = ScalarType.createTimeStampTzType(0); public static final ScalarType TIMESTAMP_TZ = DEFAULT_TIMESTAMP_TZ; public static final ScalarType TIMESTAMP_TZ_WITH_MAX_SCALAR = ScalarType.createTimeStampTzType(6); @@ -165,6 +167,7 @@ public abstract class Type { numericDateTimeTypes.add(DATETIME); numericDateTimeTypes.add(DATEV2); numericDateTimeTypes.add(DATETIMEV2); + numericDateTimeTypes.add(TIMESTAMP_NS); numericDateTimeTypes.add(TIMEV2); numericDateTimeTypes.addAll(numericTypes); @@ -178,6 +181,7 @@ public abstract class Type { trivialTypes.add(DATETIME); trivialTypes.add(DATEV2); trivialTypes.add(DATETIMEV2); + trivialTypes.add(TIMESTAMP_NS); trivialTypes.add(IPV4); trivialTypes.add(IPV6); trivialTypes.add(TIMEV2); @@ -202,6 +206,7 @@ public abstract class Type { arraySubTypes.add(DATETIME); arraySubTypes.add(DATEV2); arraySubTypes.add(DATETIMEV2); + arraySubTypes.add(TIMESTAMP_NS); arraySubTypes.add(TIMESTAMP_TZ); arraySubTypes.add(IPV4); arraySubTypes.add(IPV6); @@ -232,6 +237,7 @@ public abstract class Type { mapSubTypes.add(DATETIME); mapSubTypes.add(DATEV2); mapSubTypes.add(DATETIMEV2); + mapSubTypes.add(TIMESTAMP_NS); mapSubTypes.add(TIMESTAMP_TZ); mapSubTypes.add(IPV4); mapSubTypes.add(IPV6); @@ -258,6 +264,7 @@ public abstract class Type { structSubTypes.add(DATETIME); structSubTypes.add(DATEV2); structSubTypes.add(DATETIMEV2); + structSubTypes.add(TIMESTAMP_NS); structSubTypes.add(TIMESTAMP_TZ); structSubTypes.add(IPV4); structSubTypes.add(IPV6); @@ -470,6 +477,8 @@ public String hideVersionForVersionColumn( typeStr.append("(").append(((ScalarType) this).getScalarScale()).append(")"); } return typeStr.toString(); + } else if (isTimeStampNs()) { + return "timestamp_ns"; } else if (isTimeStampTz()) { StringBuilder typeStr = new StringBuilder("timestamptz"); if (((ScalarType) this).getScalarScale() > 0) { @@ -538,6 +547,10 @@ public boolean isDatetimeV2() { return isScalarType(PrimitiveType.DATETIMEV2); } + public boolean isTimeStampNs() { + return isScalarType(PrimitiveType.TIMESTAMP_NS); + } + public boolean isTimeV2() { return isScalarType(PrimitiveType.TIMEV2); } @@ -687,6 +700,7 @@ public boolean isNativeType() { public boolean isDateType() { return isScalarType(PrimitiveType.DATE) || isScalarType(PrimitiveType.DATETIME) || isScalarType(PrimitiveType.DATEV2) || isScalarType(PrimitiveType.DATETIMEV2) + || isScalarType(PrimitiveType.TIMESTAMP_NS) || isScalarType(PrimitiveType.TIMESTAMPTZ); } @@ -870,6 +884,8 @@ public static Type fromPrimitiveType(PrimitiveType type) { return Type.DATEV2; case DATETIMEV2: return Type.DATETIMEV2; + case TIMESTAMP_NS: + return Type.TIMESTAMP_NS; case TIMEV2: return Type.TIMEV2; case TIMESTAMPTZ: @@ -978,6 +994,8 @@ protected static Pair fromThrift(TTypeDesc col, int nodeIdx) { Preconditions.checkState(scalarType.isSetPrecision() && scalarType.isSetScale()); type = ScalarType.createDatetimeV2Type(scalarType.getScale()); + } else if (scalarType.getType() == TPrimitiveType.TIMESTAMP_NS) { + type = ScalarType.createTimeStampNsType(); } else if (scalarType.getType() == TPrimitiveType.TIMEV2) { Preconditions.checkState(scalarType.isSetPrecision() && scalarType.isSetScale()); @@ -1119,6 +1137,8 @@ public Integer getPrecision() { case TIMESTAMPTZ: case TIMEV2: return t.decimalPrecision(); + case TIMESTAMP_NS: + return 29; default: return null; } @@ -1155,6 +1175,8 @@ public Integer getDecimalDigits() { case DECIMAL128: case DECIMAL256: return t.decimalScale(); + case TIMESTAMP_NS: + return 9; default: return null; } diff --git a/gensrc/proto/types.proto b/gensrc/proto/types.proto index 1cb49602e60a76..51acb7cb8ca324 100644 --- a/gensrc/proto/types.proto +++ b/gensrc/proto/types.proto @@ -122,6 +122,7 @@ message PGenericType { IPV6 = 40; VARBINARY = 41; TIMESTAMPTZ = 42; + TIMESTAMP_NS = 43; UNKNOWN = 999; } required TypeId id = 2; diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift index f911d17346f1f9..daef6845d5c871 100644 --- a/gensrc/thrift/Types.thrift +++ b/gensrc/thrift/Types.thrift @@ -102,7 +102,8 @@ enum TPrimitiveType { UINT64 = 41, // only used in BE to represent offsets FIXED_LENGTH_OBJECT = 42 // only used in BE to represent fixed-length object VARBINARY = 43, // represent varbinary type - TIMESTAMPTZ = 44 // timestamp with time zone + TIMESTAMPTZ = 44, // timestamp with time zone + TIMESTAMP_NS = 45 // signed nanoseconds since the Unix epoch } enum TTypeNodeType { diff --git a/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns.out b/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns.out new file mode 100644 index 00000000000000..e3debeb6621f95 --- /dev/null +++ b/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns.out @@ -0,0 +1,100 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !storage -- +1 1677-09-21 00:12:43.145224200 1677-09-21 00:12:43.145224200 1677-09-21 00:12:43.145224192 +2 1969-12-31 23:59:59.999999900 1969-12-31 23:59:59.999999990 1969-12-31 23:59:59.999999999 +3 1970-01-01 00:00:00.000000000 1970-01-01 00:00:00.000000000 1970-01-01 00:00:00.000000000 +4 1970-01-01 00:00:00.000000100 1970-01-01 00:00:00.000000010 1970-01-01 00:00:00.000000001 +5 2262-04-11 23:47:16.854775800 2262-04-11 23:47:16.854775800 2262-04-11 23:47:16.854775807 + +-- !row_store -- +1 1677-09-21 00:12:43.145224192 +2 1970-01-01 00:00:00.123456789 +3 2262-04-11 23:47:16.854775807 + +-- !rounding -- +1677-09-21 00:12:43.145224192 1970-01-01 00:00:00.123456750 1970-01-01 00:00:00.999999995 1970-01-01 00:00:00.123456789 2262-04-11 23:47:16.854775807 + +-- !cast_and_aggregate -- +1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 5 + +-- !filter -- +1 1677-09-21 00:12:43.145224192 +3 1970-01-01 00:00:00.000000000 +5 2262-04-11 23:47:16.854775807 + +-- !out_of_range -- +1677-09-21 00:12:43.145224192 \N 1970-01-01 00:00:00.000000000 \N 2262-04-11 23:47:16.854775807 + +-- !all_comparisons -- +true true true true true true true true true true + +-- !sort_limit -- +1 1677-09-21 00:12:43.145224192 +2 1969-12-31 23:59:59.999999999 +3 1970-01-01 00:00:00.000000000 +6 1970-01-01 00:00:00.000000000 +4 1970-01-01 00:00:00.000000001 +5 2262-04-11 23:47:16.854775807 +7 \N + +-- !group_by -- +\N 1 7 7 +1677-09-21 00:12:43.145224192 1 1 1 +1969-12-31 23:59:59.999999999 1 2 2 +1970-01-01 00:00:00.000000000 2 3 6 +1970-01-01 00:00:00.000000001 1 4 4 +2262-04-11 23:47:16.854775807 1 5 5 + +-- !hash_join -- +1 1 1677-09-21 00:12:43.145224192 +2 2 1969-12-31 23:59:59.999999999 +3 3 1970-01-01 00:00:00.000000000 +3 6 1970-01-01 00:00:00.000000000 +4 4 1970-01-01 00:00:00.000000001 +5 5 2262-04-11 23:47:16.854775807 +6 3 1970-01-01 00:00:00.000000000 +6 6 1970-01-01 00:00:00.000000000 + +-- !relational_aggregates -- +6 5 1677-09-21 00:12:43.145224192 2262-04-11 23:47:16.854775807 5 + +-- !unique_key -- +1677-09-21 00:12:43.145224192 1 +1969-12-31 23:59:59.999999999 2 +1970-01-01 00:00:00.000000000 20 +1970-01-01 00:00:00.000000001 4 +2262-04-11 23:47:16.854775807 5 + +-- !aggregate_key -- +1677-09-21 00:12:43.145224192 1 +1969-12-31 23:59:59.999999999 2 +1970-01-01 00:00:00.000000000 7 +1970-01-01 00:00:00.000000001 5 +2262-04-11 23:47:16.854775807 6 + +-- !range_partition -- +p_after_epoch [('2262-04-11 23:47:16.854775807'), (MAXVALUE)) +p_before_epoch [('1677-09-21 00:12:43.145224193'), ('1970-01-01 00:00:00.000000000')) +p_before_maximum [('1970-01-01 00:00:00.000000002'), ('2262-04-11 23:47:16.854775807')) +p_epoch [('1970-01-01 00:00:00.000000000'), ('1970-01-01 00:00:00.000000002')) +p_minimum [('1677-09-21 00:12:43.145224192'), ('1677-09-21 00:12:43.145224193')) + +-- !array_functions -- +1 ["1677-09-21 00:12:43.145224192", "1970-01-01 00:00:00.000000000", "2262-04-11 23:47:16.854775807"] +2 \N + +-- !index_predicates -- +1 1677-09-21 00:12:43.145224192 +2 1970-01-01 00:00:00.000000000 +3 2262-04-11 23:47:16.854775807 + +-- !cast_with_constant_folding -- +1677-09-21 00:12:43.145224192 1970-01-01 00:00:00.123456789 1970-01-01 00:00:00.123456789 1970-01-01 00:00:00.123456789 1970-01-01 00:00:00.123456789 1970-01-01 00:00:00.123456790 1970-01-01 00:00:01.000000000 2262-04-11 23:47:16.854775807 + +-- !cast_without_constant_folding -- +1677-09-21 00:12:43.145224192 1970-01-01 00:00:00.123456789 1970-01-01 00:00:00.123456789 1970-01-01 00:00:00.123456789 1970-01-01 00:00:00.123456789 1970-01-01 00:00:00.123456790 1970-01-01 00:00:01.000000000 2262-04-11 23:47:16.854775807 + +-- !nonconstant_permissive_cast -- +1 2024-01-01 00:00:00.123456789 +2 \N + diff --git a/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_binary_output.out b/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_binary_output.out new file mode 100644 index 00000000000000..ab82a201b58716 --- /dev/null +++ b/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_binary_output.out @@ -0,0 +1,17 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !text_protocol -- +1 1677-09-21 00:12:43.145224192 +2 1969-12-31 23:59:59.999999999 +3 1970-01-01 00:00:00.000000000 +4 1970-01-01 00:00:00.000000001 +5 2262-04-11 23:47:16.854775807 +6 \N + +-- !binary_protocol -- +1 1677-09-21 00:12:43.145224192 +2 1969-12-31 23:59:59.999999999 +3 1970-01-01 00:00:00.000000000 +4 1970-01-01 00:00:00.000000001 +5 2262-04-11 23:47:16.854775807 +6 \N + diff --git a/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_ddl_predicate.out b/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_ddl_predicate.out new file mode 100644 index 00000000000000..d02fde231c0b92 --- /dev/null +++ b/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_ddl_predicate.out @@ -0,0 +1,61 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !timestamp_ns_alias_values -- +1 1970-01-01 00:00:00.123456789 1970-01-01 00:00:00.123456789 1970-01-01 00:00:00.123456789 1970-01-01 00:00:00.123456789 true true true true true + +-- !timestamp_ns_alias_schema -- +1 id int YES +2 ts timestamp_ns YES +3 dt7 timestamp_ns YES +4 dt8 timestamp_ns YES +5 dt9 timestamp_ns YES + +-- !datetime_alias_scales -- +\N \N \N 6 +1677-09-21 00:12:43.145224200 1677-09-21 00:12:43.145224200 1677-09-21 00:12:43.145224192 1 +1969-12-31 23:59:59.999999900 1969-12-31 23:59:59.999999990 1969-12-31 23:59:59.999999999 2 +1970-01-01 00:00:00.000000000 1970-01-01 00:00:00.000000000 1970-01-01 00:00:00.000000000 3 +1970-01-01 00:00:00.123456800 1970-01-01 00:00:00.123456790 1970-01-01 00:00:00.123456789 4 +2262-04-11 23:47:16.854775800 2262-04-11 23:47:16.854775800 2262-04-11 23:47:16.854775807 5 + +-- !aggregate_key_and_values -- +1677-09-21 00:12:43.145224192 1677-09-21 00:12:43.145224192 1677-09-21 00:12:43.145224200 +1970-01-01 00:00:00.000000000 2262-04-11 23:47:16.854775807 1969-12-31 23:59:59.999999900 +2262-04-11 23:47:16.854775807 2262-04-11 23:47:16.854775807 2262-04-11 23:47:16.854775800 + +-- !unique_mow_primary_semantics -- +1677-09-21 00:12:43.145224192 1677-09-21 00:12:43.145224200 1 +1970-01-01 00:00:00.000000000 1970-01-01 00:00:00.000000010 20 +2262-04-11 23:47:16.854775807 2262-04-11 23:47:16.854775800 3 + +-- !invalid_and_overflow_values_become_null -- +100 \N \N \N +101 \N \N \N +102 \N \N \N + +-- !list_partition_scale_rounding -- +1970-01-01 00:00:00.123456750 1970-01-01 00:00:00.123456785 1970-01-01 00:00:00.123456790 1 +1970-01-01 00:00:00.999999950 1970-01-01 00:00:00.999999995 1970-01-01 00:00:01.000000000 2 + +-- !nullable_first_range -- +7 \N + +-- !all_storage_predicates -- +1 false true false false true true false true true false +2 false true false false true true false true false true +3 true false false true false true false true true false +4 false true true true false false false true false true +5 false true true true false false false true false true +6 false true true true false false false true true false +7 \N \N \N \N \N \N true false \N \N + +-- !topn -- +4 1970-01-01 00:00:00.000000001 +5 1970-01-01 00:00:00.123456789 +6 2262-04-11 23:47:16.854775807 + +-- !json_round_trip -- +1677-09-21 00:12:43.145224192 1970-01-01 00:00:00.000000000 2262-04-11 23:47:16.854775807 + +-- !aggregate_state -- +1 2262-04-11 23:47:16.854775807 + diff --git a/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_partial_default.csv b/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_partial_default.csv new file mode 100644 index 00000000000000..f599e28b8ab0d8 --- /dev/null +++ b/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_partial_default.csv @@ -0,0 +1 @@ +10 diff --git a/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_stream_load.csv b/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_stream_load.csv new file mode 100644 index 00000000000000..3664e0573775f9 --- /dev/null +++ b/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_stream_load.csv @@ -0,0 +1,9 @@ +1|1677-09-21 00:12:43.145224192 +2|1969-12-31 23:59:59.999999999 +3|1970-01-01 00:00:00.000000000 +4|1970-01-01 00:00:00.000000001 +5|2262-04-11 23:47:16.854775807 +6|2262-04-11 23:47:16.854775808 +7|1970-02-30 00:00:00.000000000 +8|2023-08-17T01:41:18.123456789Z +9|2023-08-17T01:41:18.123456789America/Los_Angeles diff --git a/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_stream_load.out b/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_stream_load.out new file mode 100644 index 00000000000000..9d52997436061f --- /dev/null +++ b/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_stream_load.out @@ -0,0 +1,10 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !stream_load -- +1 1677-09-21 00:12:43.145224192 +2 1969-12-31 23:59:59.999999999 +3 1970-01-01 00:00:00.000000000 +4 1970-01-01 00:00:00.000000001 +5 2262-04-11 23:47:16.854775807 +8 2023-08-17 09:41:18.123456789 +9 2023-08-17 16:41:18.123456789 + diff --git a/regression-test/suites/datatype_p0/datetimev2/test_timestamp_ns.groovy b/regression-test/suites/datatype_p0/datetimev2/test_timestamp_ns.groovy new file mode 100644 index 00000000000000..cb79df4e57e31d --- /dev/null +++ b/regression-test/suites/datatype_p0/datetimev2/test_timestamp_ns.groovy @@ -0,0 +1,416 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_timestamp_ns") { + sql "drop table if exists test_timestamp_ns" + // dt7/dt8/dt9 describe the inserted fractional widths; every column is TIMESTAMP_NS. + sql """ + create table test_timestamp_ns ( + id int, + dt7 timestamp_ns, + dt8 timestamp_ns, + dt9 timestamp_ns + ) + duplicate key(id) + distributed by hash(id) buckets 1 + properties("replication_num" = "1") + """ + + sql """ + insert into test_timestamp_ns values + (1, '1677-09-21 00:12:43.1452242', + '1677-09-21 00:12:43.14522420', + '1677-09-21 00:12:43.145224192'), + (2, '1969-12-31 23:59:59.9999999', + '1969-12-31 23:59:59.99999999', + '1969-12-31 23:59:59.999999999'), + (3, '1970-01-01 00:00:00.0000000', + '1970-01-01 00:00:00.00000000', + '1970-01-01 00:00:00.000000000'), + (4, '1970-01-01 00:00:00.0000001', + '1970-01-01 00:00:00.00000001', + '1970-01-01 00:00:00.000000001'), + (5, '2262-04-11 23:47:16.8547758', + '2262-04-11 23:47:16.85477580', + '2262-04-11 23:47:16.854775807') + """ + + order_qt_storage """ + select id, dt7, dt8, dt9 + from test_timestamp_ns + order by dt9 + """ + + sql "drop table if exists test_timestamp_ns_row_store" + sql """ + create table test_timestamp_ns_row_store ( + id int, + dt timestamp_ns + ) + duplicate key(id) + distributed by hash(id) buckets 1 + properties( + "replication_num" = "1", + "store_row_column" = "true" + ) + """ + sql """ + insert into test_timestamp_ns_row_store values + (1, '1677-09-21 00:12:43.145224192'), + (2, '1970-01-01 00:00:00.123456789'), + (3, '2262-04-11 23:47:16.854775807') + """ + order_qt_row_store """ + select id, dt + from test_timestamp_ns_row_store + order by id + """ + + qt_rounding """ + select + cast('1677-09-21 00:12:43.145224192' as timestamp_ns), + cast('1970-01-01 00:00:00.12345675' as timestamp_ns), + cast('1970-01-01 00:00:00.999999995' as timestamp_ns), + cast('1970-01-01 00:00:00.123456789' as timestamp_ns), + cast('2262-04-11 23:47:16.854775807' as timestamp_ns) + """ + + qt_cast_and_aggregate """ + select + cast(cast('1677-09-21 00:12:43.145224192' as timestamp_ns) as string), + cast(cast('2262-04-11 23:47:16.854775807' as timestamp_ns) as string), + min(dt9), max(dt9), count(distinct dt9) + from test_timestamp_ns + """ + + // Cross-family casts and datetime scalar functions are implemented in follow-up changes. + test { + sql "select cast(dt9 as datetimev2(6)) from test_timestamp_ns" + exception "cannot cast" + } + test { + sql "select seconds_add(dt9, 1) from test_timestamp_ns" + exception "Can not find the compatibility function signature: seconds_add" + } + + order_qt_filter """ + select id, dt9 + from test_timestamp_ns + where dt9 in ( + cast('1677-09-21 00:12:43.145224192' as timestamp_ns), + cast('1970-01-01 00:00:00.000000000' as timestamp_ns), + cast('2262-04-11 23:47:16.854775807' as timestamp_ns)) + order by id + """ + + qt_out_of_range """ + select + cast('1677-09-21 00:12:43.145224192' as timestamp_ns), + cast('1677-09-21 00:12:43.145224191' as timestamp_ns), + cast('1970-01-01 00:00:00.000000000' as timestamp_ns), + cast('2262-04-11 23:47:16.854775808' as timestamp_ns), + cast('2262-04-11 23:47:16.854775807' as timestamp_ns) + """ + + qt_all_comparisons """ + select + cast('1677-09-21 00:12:43.145224192' as timestamp_ns) + = cast('1677-09-21 00:12:43.145224192' as timestamp_ns), + cast('1677-09-21 00:12:43.145224192' as timestamp_ns) + < cast('1970-01-01 00:00:00.000000000' as timestamp_ns), + cast('1969-12-31 23:59:59.999999999' as timestamp_ns) + = cast('1969-12-31 23:59:59.999999999' as timestamp_ns), + cast('1969-12-31 23:59:59.999999999' as timestamp_ns) + <> cast('1970-01-01 00:00:00.000000000' as timestamp_ns), + cast('1969-12-31 23:59:59.999999999' as timestamp_ns) + < cast('1970-01-01 00:00:00.000000000' as timestamp_ns), + cast('1970-01-01 00:00:00.000000001' as timestamp_ns) + > cast('1970-01-01 00:00:00.000000000' as timestamp_ns), + cast('1970-01-01 00:00:00.000000001' as timestamp_ns) + >= cast('1970-01-01 00:00:00.000000001' as timestamp_ns), + cast('1970-01-01 00:00:00.000000001' as timestamp_ns) + <= cast('1970-01-01 00:00:00.000000001' as timestamp_ns), + cast('1970-01-01 00:00:00.000000000' as timestamp_ns) + < cast('2262-04-11 23:47:16.854775807' as timestamp_ns), + cast('2262-04-11 23:47:16.854775807' as timestamp_ns) + = cast('2262-04-11 23:47:16.854775807' as timestamp_ns) + """ + + sql "drop table if exists test_timestamp_ns_relational" + sql """ + create table test_timestamp_ns_relational ( + id int, + dt timestamp_ns, + payload varchar(16) + ) + duplicate key(id) + distributed by hash(id) buckets 1 + properties("replication_num" = "1") + """ + sql """ + insert into test_timestamp_ns_relational values + (1, '1677-09-21 00:12:43.145224192', 'minimum'), + (2, '1969-12-31 23:59:59.999999999', 'before'), + (3, '1970-01-01 00:00:00.000000000', 'epoch-a'), + (4, '1970-01-01 00:00:00.000000001', 'after'), + (5, '2262-04-11 23:47:16.854775807', 'maximum'), + (6, '1970-01-01 00:00:00.000000000', 'epoch-b'), + (7, null, 'null') + """ + + qt_sort_limit """ + select id, dt + from test_timestamp_ns_relational + order by dt asc nulls last, id + limit 7 + """ + + order_qt_group_by """ + select dt, count(*), min(id), max(id) + from test_timestamp_ns_relational + group by dt + order by dt nulls first + """ + + order_qt_hash_join """ + select l.id, r.id, l.dt + from test_timestamp_ns_relational l + join test_timestamp_ns_relational r on l.dt = r.dt + order by l.id, r.id + """ + + qt_relational_aggregates """ + select + count(dt), + count(distinct dt), + min(dt), + max(dt), + approx_count_distinct(dt) + from test_timestamp_ns_relational + """ + + sql "drop table if exists test_timestamp_ns_unique" + sql """ + create table test_timestamp_ns_unique ( + dt timestamp_ns, + value int + ) + unique key(dt) + distributed by hash(dt) buckets 1 + properties( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true" + ) + """ + sql """ + insert into test_timestamp_ns_unique values + ('1677-09-21 00:12:43.145224192', 1), + ('1969-12-31 23:59:59.999999999', 2), + ('1970-01-01 00:00:00.000000000', 3), + ('1970-01-01 00:00:00.000000001', 4), + ('2262-04-11 23:47:16.854775807', 5) + """ + sql """ + insert into test_timestamp_ns_unique values + ('1970-01-01 00:00:00.000000000', 20) + """ + order_qt_unique_key """ + select dt, value + from test_timestamp_ns_unique + order by dt + """ + + sql "drop table if exists test_timestamp_ns_aggregate" + sql """ + create table test_timestamp_ns_aggregate ( + dt timestamp_ns, + amount bigint sum + ) + aggregate key(dt) + distributed by hash(dt) buckets 1 + properties("replication_num" = "1") + """ + sql """ + insert into test_timestamp_ns_aggregate values + ('1677-09-21 00:12:43.145224192', 1), + ('1969-12-31 23:59:59.999999999', 2), + ('1970-01-01 00:00:00.000000000', 3), + ('1970-01-01 00:00:00.000000000', 4), + ('1970-01-01 00:00:00.000000001', 5), + ('2262-04-11 23:47:16.854775807', 6) + """ + order_qt_aggregate_key """ + select dt, amount + from test_timestamp_ns_aggregate + order by dt + """ + + sql "drop table if exists test_timestamp_ns_partition" + sql """ + create table test_timestamp_ns_partition ( + dt timestamp_ns, + value int + ) + duplicate key(dt) + partition by range(dt) ( + partition p_minimum values less than ('1677-09-21 00:12:43.145224193'), + partition p_before_epoch values less than ('1970-01-01 00:00:00.000000000'), + partition p_epoch values less than ('1970-01-01 00:00:00.000000002'), + partition p_before_maximum values less than ('2262-04-11 23:47:16.854775807'), + partition p_after_epoch values less than MAXVALUE + ) + distributed by hash(dt) buckets 1 + properties("replication_num" = "1") + """ + order_qt_range_partition """ + select partition_name, partition_description + from information_schema.partitions + where table_schema = database() + and table_name = 'test_timestamp_ns_partition' + order by partition_name + """ + sql "drop table if exists test_timestamp_ns_complex" + sql """ + create table test_timestamp_ns_complex ( + id int, + values_array array + ) + duplicate key(id) + distributed by hash(id) buckets 1 + properties("replication_num" = "1") + """ + sql """ + insert into test_timestamp_ns_complex values + (1, array( + cast('2262-04-11 23:47:16.854775807' as timestamp_ns), + cast('1970-01-01 00:00:00.000000000' as timestamp_ns), + cast('1677-09-21 00:12:43.145224192' as timestamp_ns) + )), + (2, null) + """ + order_qt_array_functions """ + select id, + array_sort(values_array) + from test_timestamp_ns_complex + order by id + """ + + sql "drop table if exists test_timestamp_ns_index" + sql """ + create table test_timestamp_ns_index ( + id int, + dt timestamp_ns, + index idx_dt(dt) using inverted + ) + duplicate key(id) + distributed by hash(id) buckets 1 + properties( + "replication_num" = "1", + "bloom_filter_columns" = "dt" + ) + """ + sql """ + insert into test_timestamp_ns_index values + (1, '1677-09-21 00:12:43.145224192'), + (2, '1970-01-01 00:00:00.000000000'), + (3, '2262-04-11 23:47:16.854775807') + """ + order_qt_index_predicates """ + select id, dt + from test_timestamp_ns_index + where dt in ( + cast('1677-09-21 00:12:43.145224192' as timestamp_ns), + cast('1970-01-01 00:00:00.000000000' as timestamp_ns), + cast('2262-04-11 23:47:16.854775807' as timestamp_ns)) + order by id + """ + + sql "set debug_skip_fold_constant = false" + qt_cast_with_constant_folding """ + select + cast('1677-09-21 00:12:43.145224192' as timestamp_ns), + cast('1970-01-01 00:00:00.123456789' as timestamp_ns), + cast('1970-01-01 00:00:00.123456789' as timestamp_ns), + cast('1970-01-01 00:00:00.123456789' as timestamp_ns), + cast('1970-01-01 00:00:00.1234567894' as timestamp_ns), + cast('1970-01-01 00:00:00.1234567895' as timestamp_ns), + cast('1970-01-01 00:00:00.9999999995' as timestamp_ns), + cast('2262-04-11 23:47:16.854775807' as timestamp_ns) + """ + sql "set debug_skip_fold_constant = true" + qt_cast_without_constant_folding """ + select + cast('1677-09-21 00:12:43.145224192' as timestamp_ns), + cast('1970-01-01 00:00:00.123456789' as timestamp_ns), + cast('1970-01-01 00:00:00.123456789' as timestamp_ns), + cast('1970-01-01 00:00:00.123456789' as timestamp_ns), + cast('1970-01-01 00:00:00.1234567894' as timestamp_ns), + cast('1970-01-01 00:00:00.1234567895' as timestamp_ns), + cast('1970-01-01 00:00:00.9999999995' as timestamp_ns), + cast('2262-04-11 23:47:16.854775807' as timestamp_ns) + """ + sql "set debug_skip_fold_constant = false" + + sql "drop table if exists test_timestamp_ns_cast_input" + sql """ + create table test_timestamp_ns_cast_input ( + id int, + value varchar(64) + ) + duplicate key(id) + distributed by hash(id) buckets 1 + properties("replication_num" = "1") + """ + sql """ + insert into test_timestamp_ns_cast_input values + (1, '2024-01-01 00:00:00.123456789'), + (2, '2024-01-01 00:00:00.123.456') + """ + def originalEnableStrictCast = sql("select @@enable_strict_cast")[0][0] + try { + sql "set enable_strict_cast = false" + order_qt_nonconstant_permissive_cast """ + select id, cast(value as timestamp_ns) + from test_timestamp_ns_cast_input + order by id + """ + + sql "set enable_strict_cast = true" + test { + sql """ + select cast(value as timestamp_ns) + from test_timestamp_ns_cast_input + where id = 2 + """ + exception "2024-01-01 00:00:00.123.456" + } + } finally { + sql "set enable_strict_cast = ${originalEnableStrictCast}" + } + + test { + sql """ + create table test_timestamp_ns_invalid_scale ( + id int, + dt datetimev2(10) + ) + distributed by hash(id) buckets 1 + properties("replication_num" = "1") + """ + exception "between 0 and 6" + } +} diff --git a/regression-test/suites/datatype_p0/datetimev2/test_timestamp_ns_binary_output.groovy b/regression-test/suites/datatype_p0/datetimev2/test_timestamp_ns_binary_output.groovy new file mode 100644 index 00000000000000..406c90d93524dc --- /dev/null +++ b/regression-test/suites/datatype_p0/datetimev2/test_timestamp_ns_binary_output.groovy @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import com.mysql.cj.jdbc.ServerPreparedStatement + +import java.sql.PreparedStatement +import java.sql.Types + +suite("test_timestamp_ns_binary_output") { + def user = context.config.jdbcUser + def password = context.config.jdbcPassword + + sql "drop table if exists test_timestamp_ns_binary_output" + sql """ + create table test_timestamp_ns_binary_output ( + id int, + dt timestamp_ns + ) + duplicate key(id) + distributed by hash(id) buckets 1 + properties("replication_num" = "1") + """ + sql """ + insert into test_timestamp_ns_binary_output values + (1, '1677-09-21 00:12:43.145224192'), + (2, '1969-12-31 23:59:59.999999999'), + (3, '1970-01-01 00:00:00.000000000'), + (4, '1970-01-01 00:00:00.000000001'), + (5, '2262-04-11 23:47:16.854775807'), + (6, null) + """ + + order_qt_text_protocol """ + select id, dt from test_timestamp_ns_binary_output order by id + """ + + String url = getServerPrepareJdbcUrl( + context.config.jdbcUrl, "regression_test_datatype_p0_datetimev2") + connect(user, password, url) { + PreparedStatement stmt = prepareStatement(""" + select id, dt from test_timestamp_ns_binary_output where id >= ? order by id + """) + assertEquals(ServerPreparedStatement, stmt.class) + assertEquals(Types.CHAR, stmt.metaData.getColumnType(2)) + stmt.setInt(1, 1) + qe_binary_protocol stmt + stmt.close() + } +} diff --git a/regression-test/suites/datatype_p0/datetimev2/test_timestamp_ns_ddl_predicate.groovy b/regression-test/suites/datatype_p0/datetimev2/test_timestamp_ns_ddl_predicate.groovy new file mode 100644 index 00000000000000..8c5ab6a73041b0 --- /dev/null +++ b/regression-test/suites/datatype_p0/datetimev2/test_timestamp_ns_ddl_predicate.groovy @@ -0,0 +1,399 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_timestamp_ns_ddl_predicate") { + for (def datetimeType : ["datetime", "datetimev2"]) { + for (def invalidScale : [7, 8, 9]) { + test { + sql "select cast('1970-01-01 00:00:00.123456789' as ${datetimeType}(${invalidScale}))" + exception "between 0 and 6" + } + } + } + + sql "drop table if exists test_datetime_scale_above_six" + test { + sql """ + create table test_datetime_scale_above_six ( + id int, + dt datetimev2(7) + ) + distributed by hash(id) buckets 1 + properties("replication_num" = "1") + """ + exception "between 0 and 6" + } + + sql "drop table if exists test_timestamp_ns_values" + // The legacy golden tags and dt7/dt8/dt9 names refer to input widths, not DATETIMEV2 scales. + sql """ + create table test_timestamp_ns_values ( + id int, + ts timestamp_ns, + dt7 timestamp_ns, + dt8 timestamp_ns, + dt9 timestamp_ns + ) + duplicate key(id) + distributed by hash(id) buckets 1 + properties("replication_num" = "1") + """ + sql """ + insert into test_timestamp_ns_values values + (1, + '1970-01-01 00:00:00.123456789', + '1970-01-01 00:00:00.123456789', + '1970-01-01 00:00:00.123456789', + '1970-01-01 00:00:00.123456789') + """ + order_qt_timestamp_ns_alias_values """ + select id, ts, dt7, dt8, dt9, + ts = dt7, dt7 = dt8, dt8 = dt9, + cast(dt7 as timestamp_ns) = dt7, + cast(dt8 as timestamp_ns) = dt8 + from test_timestamp_ns_values + order by id + """ + order_qt_timestamp_ns_alias_schema """ + select ordinal_position, column_name, data_type, is_nullable + from information_schema.columns + where table_schema = database() + and table_name = 'test_timestamp_ns_values' + order by ordinal_position + """ + + test { + sql "select cast('1970-01-01 00:00:00.000000001' as timestamp_ns(9))" + exception "timestamp_ns does not support precision" + } + + sql "drop table if exists test_timestamp_ns_ddl_duplicate" + sql """ + create table test_timestamp_ns_ddl_duplicate ( + dt7 timestamp_ns, + dt8 timestamp_ns, + dt9 timestamp_ns, + id int + ) + duplicate key(dt7, dt8) + distributed by hash(dt9) buckets 4 + properties("replication_num" = "1") + """ + sql """ + insert into test_timestamp_ns_ddl_duplicate values + ('1677-09-21 00:12:43.1452242', + '1677-09-21 00:12:43.14522420', + '1677-09-21 00:12:43.145224192', 1), + ('1969-12-31 23:59:59.9999999', + '1969-12-31 23:59:59.99999999', + '1969-12-31 23:59:59.999999999', 2), + ('1970-01-01 00:00:00.0000000', + '1970-01-01 00:00:00.00000000', + '1970-01-01 00:00:00.000000000', 3), + ('1970-01-01 00:00:00.1234568', + '1970-01-01 00:00:00.12345679', + '1970-01-01 00:00:00.123456789', 4), + ('2262-04-11 23:47:16.8547758', + '2262-04-11 23:47:16.85477580', + '2262-04-11 23:47:16.854775807', 5), + (null, null, null, 6) + """ + order_qt_datetime_alias_scales """ + select dt7, dt8, dt9, id + from test_timestamp_ns_ddl_duplicate + order by id + """ + sql "drop table if exists test_timestamp_ns_ddl_aggregate" + sql """ + create table test_timestamp_ns_ddl_aggregate ( + dt timestamp_ns, + max_dt timestamp_ns max, + min_dt timestamp_ns min + ) + aggregate key(dt) + distributed by hash(dt) buckets 1 + properties("replication_num" = "1") + """ + sql """ + insert into test_timestamp_ns_ddl_aggregate values + ('1677-09-21 00:12:43.145224192', + '1677-09-21 00:12:43.145224192', + '1677-09-21 00:12:43.1452242'), + ('1970-01-01 00:00:00.000000000', + '1970-01-01 00:00:00.000000001', + '1969-12-31 23:59:59.9999999'), + ('1970-01-01 00:00:00.000000000', + '2262-04-11 23:47:16.854775807', + '1970-01-01 00:00:00.0000000'), + ('2262-04-11 23:47:16.854775807', + '2262-04-11 23:47:16.854775807', + '2262-04-11 23:47:16.8547758') + """ + order_qt_aggregate_key_and_values """ + select dt, max_dt, min_dt + from test_timestamp_ns_ddl_aggregate + order by dt + """ + + sql "drop table if exists test_timestamp_ns_ddl_unique" + sql """ + create table test_timestamp_ns_ddl_unique ( + dt timestamp_ns, + value_dt timestamp_ns, + value int + ) + unique key(dt) + distributed by hash(dt) buckets 1 + properties( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true" + ) + """ + sql """ + insert into test_timestamp_ns_ddl_unique values + ('1677-09-21 00:12:43.145224192', '1677-09-21 00:12:43.14522420', 1), + ('1970-01-01 00:00:00.000000000', '1970-01-01 00:00:00.00000000', 2), + ('2262-04-11 23:47:16.854775807', '2262-04-11 23:47:16.85477580', 3) + """ + sql """ + insert into test_timestamp_ns_ddl_unique values + ('1970-01-01 00:00:00.000000000', '1970-01-01 00:00:00.00000001', 20) + """ + order_qt_unique_mow_primary_semantics """ + select dt, value_dt, value + from test_timestamp_ns_ddl_unique + order by dt + """ + + test { + sql """ + create table test_timestamp_ns_scale_overflow ( + id int, + dt datetime(10) + ) + distributed by hash(id) buckets 1 + properties("replication_num" = "1") + """ + exception "between 0 and 6" + } + + sql """ + insert into test_timestamp_ns_ddl_duplicate values + ('1970-02-30 00:00:00.0000000', + '1970-02-30 00:00:00.00000000', + '1970-02-30 00:00:00.000000000', 100) + """ + sql """ + insert into test_timestamp_ns_ddl_duplicate values + ('1677-09-21 00:12:43.1452241', + '1677-09-21 00:12:43.14522419', + '1677-09-21 00:12:43.145224191', 101) + """ + sql """ + insert into test_timestamp_ns_ddl_duplicate values + ('2262-04-11 23:47:16.8547759', + '2262-04-11 23:47:16.85477581', + '2262-04-11 23:47:16.854775808', 102) + """ + order_qt_invalid_and_overflow_values_become_null """ + select id, dt7, dt8, dt9 + from test_timestamp_ns_ddl_duplicate + where id between 100 and 102 + order by id + """ + + sql "drop table if exists test_timestamp_ns_partition_bucket" + sql """ + create table test_timestamp_ns_partition_bucket ( + dt timestamp_ns, + id int + ) + duplicate key(dt) + partition by range(dt) ( + partition p_before_epoch values less than + ('1970-01-01 00:00:00.000000000'), + partition p_epoch values less than + ('1970-01-01 00:00:00.000000002'), + partition p_after_epoch values less than MAXVALUE + ) + distributed by hash(dt) buckets 4 + properties("replication_num" = "1") + """ + sql """ + insert into test_timestamp_ns_partition_bucket values + ('1677-09-21 00:12:43.145224192', 1), + ('1969-12-31 23:59:59.999999999', 2), + ('1970-01-01 00:00:00.000000000', 3), + ('1970-01-01 00:00:00.000000001', 4), + ('1970-01-01 00:00:00.123456789', 5), + ('2262-04-11 23:47:16.854775807', 6), + (null, 7) + """ + explain { + sql """ + select * + from test_timestamp_ns_partition_bucket + where dt = cast('1970-01-01 00:00:00.000000000' as timestamp_ns) + """ + contains "partitions=1/3 (p_epoch)" + contains "tablets=1/4" + } + explain { + sql """ + select * + from test_timestamp_ns_partition_bucket + where dt >= cast('2262-04-11 23:47:16.854775807' as timestamp_ns) + """ + contains "partitions=1/3 (p_after_epoch)" + } + explain { + sql """ + select * + from test_timestamp_ns_partition_bucket + where dt is null + """ + contains "partitions=1/3 (p_before_epoch)" + } + + sql "drop table if exists test_timestamp_ns_list_partition_rounding" + sql """ + create table test_timestamp_ns_list_partition_rounding ( + dt7 timestamp_ns, + dt8 timestamp_ns, + dt9 timestamp_ns, + id int + ) + duplicate key(dt7, dt8, dt9) + partition by list(dt7, dt8, dt9) ( + partition p_round values in + (('1970-01-01 00:00:00.12345675', + '1970-01-01 00:00:00.123456785', + '1970-01-01 00:00:00.1234567895')), + partition p_carry values in + (('1970-01-01 00:00:00.99999995', + '1970-01-01 00:00:00.999999995', + '1970-01-01 00:00:00.9999999995')) + ) + distributed by hash(id) buckets 1 + properties("replication_num" = "1") + """ + sql """ + insert into test_timestamp_ns_list_partition_rounding values + ('1970-01-01 00:00:00.12345675', + '1970-01-01 00:00:00.123456785', + '1970-01-01 00:00:00.1234567895', 1), + ('1970-01-01 00:00:00.99999995', + '1970-01-01 00:00:00.999999995', + '1970-01-01 00:00:00.9999999995', 2) + """ + order_qt_list_partition_scale_rounding """ + select dt7, dt8, dt9, id + from test_timestamp_ns_list_partition_rounding + order by id + """ + + order_qt_nullable_first_range """ + select id, dt + from test_timestamp_ns_partition_bucket + where dt is null + order by id + """ + + order_qt_all_storage_predicates """ + select id, + dt = cast('1970-01-01 00:00:00.000000000' as timestamp_ns), + dt != cast('1970-01-01 00:00:00.000000000' as timestamp_ns), + dt > cast('1970-01-01 00:00:00.000000000' as timestamp_ns), + dt >= cast('1970-01-01 00:00:00.000000000' as timestamp_ns), + dt < cast('1970-01-01 00:00:00.000000000' as timestamp_ns), + dt <= cast('1970-01-01 00:00:00.000000000' as timestamp_ns), + dt is null, + dt is not null, + dt in ( + cast('1677-09-21 00:12:43.145224192' as timestamp_ns), + cast('1970-01-01 00:00:00.000000000' as timestamp_ns), + cast('2262-04-11 23:47:16.854775807' as timestamp_ns)), + dt not in ( + cast('1677-09-21 00:12:43.145224192' as timestamp_ns), + cast('1970-01-01 00:00:00.000000000' as timestamp_ns), + cast('2262-04-11 23:47:16.854775807' as timestamp_ns)) + from test_timestamp_ns_partition_bucket + order by id + """ + + order_qt_topn """ + select id, dt + from test_timestamp_ns_partition_bucket + order by dt desc nulls last + limit 3 + """ + + order_qt_json_round_trip """ + select + cast( + json_extract_string( + json_object( + 'dt', + cast( + cast('1677-09-21 00:12:43.145224192' as timestamp_ns) + as string)), + '\$.dt') + as timestamp_ns), + cast( + json_extract_string( + json_object( + 'dt', + cast( + cast('1970-01-01 00:00:00.000000000' as timestamp_ns) + as string)), + '\$.dt') + as timestamp_ns), + cast( + json_extract_string( + json_object( + 'dt', + cast( + cast('2262-04-11 23:47:16.854775807' as timestamp_ns) + as string)), + '\$.dt') + as timestamp_ns) + """ + + sql "set enable_agg_state = true" + sql "drop table if exists test_timestamp_ns_agg_state" + sql """ + create table test_timestamp_ns_agg_state ( + id int, + dt_state agg_state generic + ) + aggregate key(id) + distributed by hash(id) buckets 1 + properties("replication_num" = "1") + """ + sql """ + insert into test_timestamp_ns_agg_state values + (1, max_state(cast('1677-09-21 00:12:43.145224192' as timestamp_ns))), + (1, max_state(cast('1970-01-01 00:00:00.000000000' as timestamp_ns))), + (1, max_state(cast('2262-04-11 23:47:16.854775807' as timestamp_ns))) + """ + order_qt_aggregate_state """ + select id, max_merge(dt_state) + from test_timestamp_ns_agg_state + group by id + order by id + """ +} diff --git a/regression-test/suites/datatype_p0/datetimev2/test_timestamp_ns_stream_load.groovy b/regression-test/suites/datatype_p0/datetimev2/test_timestamp_ns_stream_load.groovy new file mode 100644 index 00000000000000..ff2786afd585d6 --- /dev/null +++ b/regression-test/suites/datatype_p0/datetimev2/test_timestamp_ns_stream_load.groovy @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_timestamp_ns_stream_load") { + sql "set time_zone = '+08:00'" + sql "drop table if exists test_timestamp_ns_stream_load" + sql """ + create table test_timestamp_ns_stream_load ( + id int, + dt timestamp_ns + ) + duplicate key(id) + distributed by hash(id) buckets 1 + properties("replication_num" = "1") + """ + + streamLoad { + table "test_timestamp_ns_stream_load" + file "test_timestamp_ns_stream_load.csv" + set "column_separator", "|" + set "strict_mode", "true" + set "max_filter_ratio", "0.3" + + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(9, json.NumberTotalRows) + assertEquals(7, json.NumberLoadedRows) + assertEquals(2, json.NumberFilteredRows) + } + } + + order_qt_stream_load """ + select id, dt + from test_timestamp_ns_stream_load + order by id + """ + +} From e92cc856aa6211e7a35d505b62bfa61968f82734 Mon Sep 17 00:00:00 2001 From: jacktengg Date: Mon, 10 Aug 2026 02:54:00 +0800 Subject: [PATCH 2/2] [refactor](timestamp_ns) Keep basic TIMESTAMP_NS support independent ### What problem does this PR solve? Issue Number: None Related PR: #66333 Problem Summary: The initial TIMESTAMP_NS implementation mixed the signed epoch-nanosecond type into DATETIMEV2 precision families and date/time function and cast paths. This made legacy datetime literals retain nanosecond-only state, advertised unsupported output and Java-function surfaces, and left inconsistent coercion and primitive classification. Keep TIMESTAMP_NS as an independent fixed-precision type, limit basic support to storage, serialization, literals, predicates, string conversion, and MySQL output, explicitly reject deferred Arrow/ORC/function/cross-temporal cast paths, and remove unused TimeStampNsValue APIs. ### Release note Add the basic fixed-precision TIMESTAMP_NS type backed by signed Int64 epoch nanoseconds. Cross-date/time casts and date/time functions remain unsupported in this PR. ### Check List (For Author) - Test: Unit Test and Regression test - Standard ASAN BE and FE builds passed - BE ASAN UT: 201 broad tests passed; focused reruns of 21 and 19 tests passed - FE UT: 495 related tests passed, then the 19-test connector rerun passed after the fix - Regression tests: test_timestamp_ns, test_timestamp_ns_binary_output, test_timestamp_ns_ddl_predicate, and test_timestamp_ns_stream_load passed - Behavior changed: Yes. TIMESTAMP_NS remains independent from DATETIMEV2 and unsupported follow-up surfaces fail explicitly. - Does this need documentation: No --- .../data_type_date_or_datetime_v2.cpp | 20 -- .../data_type/data_type_date_or_datetime_v2.h | 8 - be/src/core/data_type/primitive_type.h | 8 +- .../data_type_number_serde.cpp | 21 +- .../data_type_timestamp_ns_serde.cpp | 119 ---------- be/src/core/value/timestamp_ns_value.cpp | 6 + be/src/core/value/timestamp_ns_value.h | 183 ++------------ .../array/function_array_aggregation.cpp | 1 - be/src/exprs/function/cast/function_cast.cpp | 2 +- be/src/exprs/function/date_time_transforms.h | 4 - be/src/exprs/function/function.h | 3 +- be/src/exprs/function/function_convert_tz.cpp | 152 ++++-------- .../function_date_or_datetime_computation.cpp | 76 ++---- .../function_date_or_datetime_to_string.cpp | 8 - .../function/function_datetime_floor_ceil.cpp | 1 - .../function_datetime_string_to_string.cpp | 6 - .../function_datetime_string_to_string.h | 1 - .../function/function_other_types_to_date.cpp | 60 ++--- be/src/exprs/function/least_greast.cpp | 9 +- be/src/exprs/function/time_of_function.cpp | 18 -- be/src/exprs/function/to_time_function.cpp | 31 --- be/src/storage/tablet_info.cpp | 33 +-- .../data_type/data_type_timestamp_ns_test.cpp | 144 ++--------- .../tablet_sink_hash_partitioner_test.cpp | 2 +- be/test/exprs/function/function_test_util.cpp | 19 +- be/test/exprs/function/function_test_util.h | 7 - be/test/exprs/function/function_time_test.cpp | 45 ---- .../apache/doris/catalog/PartitionKey.java | 2 +- .../ExprToConnectorExpressionConverter.java | 3 + .../rules/FoldConstantRuleOnBE.java | 2 +- .../expression/rules/SimplifyInPredicate.java | 2 +- .../functions/ComputeSignatureHelper.java | 19 +- .../functions/DateTimeWithPrecision.java | 10 +- .../functions/agg/SequenceCount.java | 5 +- .../functions/agg/SequenceMatch.java | 5 +- .../functions/agg/TopNWeighted.java | 8 - .../functions/agg/WindowFunnel.java | 4 - .../functions/agg/WindowFunnelV2.java | 4 - .../executable/DateTimeArithmetic.java | 2 +- .../DateTimeExtractAndTransform.java | 18 +- .../functions/executable/TimeRoundSeries.java | 118 +++++---- .../functions/scalar/ConvertTz.java | 4 +- .../functions/scalar/DayMicrosecondAdd.java | 12 +- .../functions/scalar/DayMicrosecondSub.java | 12 +- .../functions/scalar/HourMicrosecondAdd.java | 12 +- .../functions/scalar/HourMicrosecondSub.java | 12 +- .../functions/scalar/MicroSecondsAdd.java | 10 +- .../functions/scalar/MicroSecondsSub.java | 10 +- .../functions/scalar/MilliSecondsAdd.java | 10 +- .../functions/scalar/MilliSecondsSub.java | 10 +- .../scalar/MinuteMicrosecondAdd.java | 12 +- .../scalar/MinuteMicrosecondSub.java | 12 +- .../scalar/SecondMicrosecondAdd.java | 7 +- .../scalar/SecondMicrosecondSub.java | 7 +- .../functions/window/RequireTrivialTypes.java | 2 - .../expressions/literal/DateLiteral.java | 6 +- .../expressions/literal/DateTimeLiteral.java | 167 ++++++++----- .../literal/DateTimeV2Literal.java | 223 ++++++++++-------- .../expressions/literal/DateV2Literal.java | 12 +- .../expressions/literal/NumericLiteral.java | 2 +- .../literal/StringLikeLiteral.java | 2 +- .../literal/TimeStampNsLiteral.java | 100 +------- .../expressions/literal/TimeV2Literal.java | 2 +- .../literal/TimestampTzLiteral.java | 10 +- .../doris/nereids/types/DateTimeV2Type.java | 12 +- .../nereids/types/coercion/DateLikeType.java | 2 +- .../doris/nereids/util/TypeCoercionUtils.java | 74 +++++- ...idsToConnectorExpressionConverterTest.java | 3 +- .../functions/ComputeSignatureHelperTest.java | 46 ---- .../DateTimeMicrosecondFunctionTest.java | 120 ---------- .../functions/scalar/StrToDateTest.java | 10 - .../TimestampTzLeadLagSignatureTest.java | 14 -- .../expressions/literal/DateLiteralTest.java | 2 +- .../literal/DateTimeLiteralTest.java | 39 --- .../literal/TimeV2LiteralTest.java | 14 -- .../literal/TimestampTzLiteralTest.java | 47 ---- .../nereids/util/TypeCoercionMatrixTest.java | 6 +- .../nereids/util/TypeCoercionUtilsTest.java | 79 +++++++ .../datetimev2/test_timestamp_ns.out | 1 - .../test_timestamp_ns_binary_output.out | 2 - .../test_timestamp_ns_ddl_predicate.out | 1 - .../test_timestamp_ns_stream_load.out | 1 - 82 files changed, 695 insertions(+), 1623 deletions(-) delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateTimeMicrosecondFunctionTest.java diff --git a/be/src/core/data_type/data_type_date_or_datetime_v2.cpp b/be/src/core/data_type/data_type_date_or_datetime_v2.cpp index 87a1d667f374e2..35b3ba7c56a2bf 100644 --- a/be/src/core/data_type/data_type_date_or_datetime_v2.cpp +++ b/be/src/core/data_type/data_type_date_or_datetime_v2.cpp @@ -106,13 +106,6 @@ void DataTypeDateV2::cast_to_date_time_v2(const DateV2Value fro << TIME_PART_LENGTH); } -void DataTypeDateV2::cast_to_date_time_v2(const DateV2Value from, - TimeStampNsValue& to) { - DateV2Value value; - cast_to_date_time_v2(from, value); - DORIS_CHECK(to.from_datetime(value)); -} - void DataTypeDateV2::cast_from_date(const VecDateTimeValue from, DateV2Value& to) { auto& to_value = (DateV2Value&)(to); auto from_value = binary_cast(from); @@ -190,19 +183,6 @@ void DataTypeDateTimeV2::cast_to_date_v2(const DateV2Value UInt32(from.to_date_int_val() >> TIME_PART_LENGTH)); } -void DataTypeDateTimeV2::cast_to_date(const TimeStampNsValue from, VecDateTimeValue& to) { - cast_to_date(from.to_datetime(), to); -} - -void DataTypeDateTimeV2::cast_to_date_time(const TimeStampNsValue from, VecDateTimeValue& to) { - cast_to_date_time(from.to_datetime(), to); -} - -void DataTypeDateTimeV2::cast_to_date_v2(const TimeStampNsValue from, - DateV2Value& to) { - cast_to_date_v2(from.to_datetime(), to); -} - DataTypePtr create_datetimev2(UInt64 scale_value) { if (scale_value > 6) { throw doris::Exception(doris::ErrorCode::NOT_IMPLEMENTED_ERROR, "scale_value {} > 6", diff --git a/be/src/core/data_type/data_type_date_or_datetime_v2.h b/be/src/core/data_type/data_type_date_or_datetime_v2.h index 34c85bd69dd643..0e16644863ec9c 100644 --- a/be/src/core/data_type/data_type_date_or_datetime_v2.h +++ b/be/src/core/data_type/data_type_date_or_datetime_v2.h @@ -81,7 +81,6 @@ class DataTypeDateV2 final : public DataTypeNumberBase from, VecDateTimeValue& to); static void cast_to_date_time_v2(const DateV2Value from, DateV2Value& to); - static void cast_to_date_time_v2(const DateV2Value from, TimeStampNsValue& to); static void cast_from_date(const VecDateTimeValue from, DateV2Value& to); static void cast_from_date_time(const VecDateTimeValue from, DateV2Value& to); }; @@ -147,9 +146,6 @@ class DataTypeDateTimeV2 final : public DataTypeNumberBase from, DateV2Value& to); - static void cast_to_date(const TimeStampNsValue from, VecDateTimeValue& to); - static void cast_to_date_time(const TimeStampNsValue from, VecDateTimeValue& to); - static void cast_to_date_v2(const TimeStampNsValue from, DateV2Value& to); static void cast_from_date(const VecDateTimeValue from, DateV2Value& to); static void cast_from_date_time(const VecDateTimeValue from, DateV2Value& to); @@ -160,12 +156,8 @@ class DataTypeDateTimeV2 final : public DataTypeNumberBase constexpr bool IsDataTypeDateTimeV2 = false; template <> inline constexpr bool IsDataTypeDateTimeV2 = true; -template <> -inline constexpr bool IsDataTypeDateTimeV2 = true; } // namespace doris diff --git a/be/src/core/data_type/primitive_type.h b/be/src/core/data_type/primitive_type.h index 73e78c5213c6d6..1d3ae0bdec88bc 100644 --- a/be/src/core/data_type/primitive_type.h +++ b/be/src/core/data_type/primitive_type.h @@ -182,6 +182,10 @@ constexpr bool is_date_type(PrimitiveType type) { type == TYPE_TIMESTAMP_NS || type == TYPE_DATEV2; } +constexpr bool is_timestamp_ns_type(PrimitiveType type) { + return type == TYPE_TIMESTAMP_NS; +} + constexpr bool is_time_type(PrimitiveType type) { return type == TYPE_TIMEV2; } @@ -195,11 +199,11 @@ constexpr bool is_date_or_datetime(PrimitiveType type) { } constexpr bool is_date_v2_or_datetime_v2(PrimitiveType type) { - return type == TYPE_DATETIMEV2 || type == TYPE_TIMESTAMP_NS || type == TYPE_DATEV2; + return type == TYPE_DATETIMEV2 || type == TYPE_DATEV2; } constexpr bool is_datetime_v2(PrimitiveType type) { - return type == TYPE_DATETIMEV2 || type == TYPE_TIMESTAMP_NS; + return type == TYPE_DATETIMEV2; } constexpr bool is_ip(PrimitiveType type) { diff --git a/be/src/core/data_type_serde/data_type_number_serde.cpp b/be/src/core/data_type_serde/data_type_number_serde.cpp index 7cec876900cd72..e35ffc7d495b3b 100644 --- a/be/src/core/data_type_serde/data_type_number_serde.cpp +++ b/be/src/core/data_type_serde/data_type_number_serde.cpp @@ -655,10 +655,9 @@ using DORIS_NUMERIC_ARROW_BUILDER = TypeMap; @@ -728,13 +727,6 @@ Status DataTypeNumberSerDe::write_column_to_arrow(const IColumn& column, cons builder.AppendValues((uint64_t*)col_data.data() + start, end - start, reinterpret_cast(arrow_null_map_data)), column, *array_builder)); - } else if constexpr (T == TYPE_TIMESTAMP_NS) { - auto& builder = assert_cast(*array_builder); - RETURN_IF_ERROR(checkArrowStatus( - builder.AppendValues(reinterpret_cast(col_data.data()) + start, - end - start, - reinterpret_cast(arrow_null_map_data)), - column, *array_builder)); } else { auto& builder = assert_cast(*array_builder); RETURN_IF_ERROR(checkArrowStatus( @@ -745,6 +737,13 @@ Status DataTypeNumberSerDe::write_column_to_arrow(const IColumn& column, cons return Status::OK(); } +template <> +Status DataTypeNumberSerDe::write_column_to_arrow( + const IColumn& column, const NullMap* null_map, arrow::ArrayBuilder* array_builder, + int64_t start, int64_t end, const cctz::time_zone& ctz) const { + return Status::NotSupported("DataTypeNumberSerDe::write_column_to_arrow"); +} + template Status DataTypeNumberSerDe::read_column_from_decoded_values( IColumn& column, const DecodedColumnView& view) const { diff --git a/be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp b/be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp index 5224162671db8a..b1bdb90323a704 100644 --- a/be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp +++ b/be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp @@ -17,22 +17,17 @@ #include "core/data_type_serde/data_type_timestamp_ns_serde.h" -#include -#include #include #include #include #include -#include #include -#include "common/config.h" #include "common/exception.h" #include "core/assert_cast.h" #include "core/column/column_nullable.h" #include "core/column/column_string.h" -#include "core/data_type_serde/arrow_validation.h" #include "core/data_type_serde/decoded_column_view.h" #include "core/value/vdatetime_value.h" #include "exprs/function/cast/cast_to_datetimev2_impl.hpp" @@ -64,26 +59,6 @@ Status utc_epoch_nanos_to_local_epoch_nanos(int64_t source, const cctz::time_zon return Status::OK(); } -/* -Status local_epoch_nanos_to_utc_epoch_nanos(int64_t source, const cctz::time_zone& timezone, - int64_t* result) { - const TimeStampNsValue source_value(source); - auto local_value = source_value.to_datetime(); - int64_t seconds = 0; - local_value.unix_timestamp(&seconds, timezone); - const __int128 epoch_nanos = - static_cast<__int128>(seconds) * TimeStampNsValue::NANOS_PER_SECOND + - source_value.nanosecond(); - if (epoch_nanos < std::numeric_limits::min() || - epoch_nanos > std::numeric_limits::max()) { - return Status::DataQualityError("TIMESTAMP_NS value {} is outside epoch nanosecond range", - source_value.to_string(9)); - } - *result = static_cast(epoch_nanos); - return Status::OK(); -} -*/ - } // namespace Status parse_timestamp_ns(StringRef str, int64_t* epoch_nanos, @@ -261,38 +236,6 @@ Status DataTypeTimeStampNsSerDe::write_column_to_arrow(const IColumn& column, int64_t start, int64_t end, const cctz::time_zone& ctz) const { return Status::NotSupported("DataTypeTimeStampNsSerDe::write_column_to_arrow"); - // TODO: support write_column_to_arrow - /* - const auto& data = assert_cast(column).get_data(); - auto& builder = assert_cast(*array_builder); - const auto timestamp_type = std::static_pointer_cast(builder.type()); - const auto& timezone = timestamp_type->timezone(); - for (int64_t i = start; i < end; ++i) { - if (null_map != nullptr && (*null_map)[i]) { - RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(), column, builder)); - continue; - } - int64_t value = data[i].epoch_nanos(); - if (!timezone.empty()) { - RETURN_IF_ERROR(local_epoch_nanos_to_utc_epoch_nanos(value, ctz, &value)); - } - switch (timestamp_type->unit()) { - case arrow::TimeUnit::SECOND: - value /= TimeStampNsValue::NANOS_PER_SECOND; - break; - case arrow::TimeUnit::MILLI: - value /= NANOS_PER_MILLISECOND; - break; - case arrow::TimeUnit::MICRO: - value /= NANOS_PER_MICROSECOND; - break; - case arrow::TimeUnit::NANO: - break; - } - RETURN_IF_ERROR(checkArrowStatus(builder.Append(value), column, builder)); - } - return Status::OK(); - */ } Status DataTypeTimeStampNsSerDe::read_column_from_arrow(IColumn& column, @@ -300,47 +243,6 @@ Status DataTypeTimeStampNsSerDe::read_column_from_arrow(IColumn& column, int64_t start, int64_t end, const cctz::time_zone& ctz) const { return Status::NotSupported("DataTypeTimeStampNsSerDe::read_column_from_arrow"); - // TODO: - /* - if (arrow_array->type_id() != arrow::Type::TIMESTAMP) { - return Status::InvalidArgument("Expected Arrow timestamp, got {}", arrow_array->type_id()); - } - if (config::enable_arrow_input_validation) { - check_arrow_no_offset(*arrow_array); - } - const auto& array = assert_cast(*arrow_array); - const auto type = std::static_pointer_cast(array.type()); - auto& data = assert_cast(column).get_data(); - for (int64_t i = start; i < end; ++i) { - int64_t value = array.Value(i); - int64_t nanos = 0; - switch (type->unit()) { - case arrow::TimeUnit::SECOND: - if (!checked_scale_to_nanos(value, TimeStampNsValue::NANOS_PER_SECOND, &nanos)) { - return Status::DataQualityError("Arrow timestamp {} overflows nanoseconds", value); - } - break; - case arrow::TimeUnit::MILLI: - if (!checked_scale_to_nanos(value, NANOS_PER_MILLISECOND, &nanos)) { - return Status::DataQualityError("Arrow timestamp {} overflows nanoseconds", value); - } - break; - case arrow::TimeUnit::MICRO: - if (!checked_scale_to_nanos(value, NANOS_PER_MICROSECOND, &nanos)) { - return Status::DataQualityError("Arrow timestamp {} overflows nanoseconds", value); - } - break; - case arrow::TimeUnit::NANO: - nanos = value; - break; - } - if (!type->timezone().empty()) { - RETURN_IF_ERROR(utc_epoch_nanos_to_local_epoch_nanos(nanos, ctz, &nanos)); - } - data.push_back(TimeStampNsValue(nanos)); - } - return Status::OK(); - */ } Status DataTypeTimeStampNsSerDe::read_column_from_decoded_values( @@ -407,27 +309,6 @@ Status DataTypeTimeStampNsSerDe::write_column_to_orc(const std::string& timezone int64_t start, int64_t end, Arena& arena, const FormatOptions& options) const { return Status::NotSupported("DataTypeTimeStampNsSerDe::write_column_to_orc"); - // TODO: support write_column_to_orc - /* - const auto& data = assert_cast(column).get_data(); - auto& batch = assert_cast(*orc_col_batch); - cctz::time_zone parsed_timezone; - if (!cctz::load_time_zone(timezone, &parsed_timezone)) { - return Status::InvalidArgument("Invalid timezone '{}'", timezone); - } - for (int64_t row = start; row < end; ++row) { - if (batch.notNull[row] == 0) { - continue; - } - int64_t value = data[row].epoch_nanos(); - RETURN_IF_ERROR(local_epoch_nanos_to_utc_epoch_nanos(value, parsed_timezone, &value)); - const TimeStampNsValue timestamp(value); - batch.data[row] = timestamp.epoch_seconds(); - batch.nanoseconds[row] = timestamp.nanosecond(); - } - batch.numElements = end - start; - return Status::OK(); - */ } void DataTypeTimeStampNsSerDe::write_one_cell_to_binary(const IColumn& src_column, diff --git a/be/src/core/value/timestamp_ns_value.cpp b/be/src/core/value/timestamp_ns_value.cpp index 83ee9019242abd..965e613a3d251e 100644 --- a/be/src/core/value/timestamp_ns_value.cpp +++ b/be/src/core/value/timestamp_ns_value.cpp @@ -24,6 +24,8 @@ namespace doris { DateV2Value TimeStampNsValue::to_datetime() const { + // epoch_seconds() is floor-divided, so set_microsecond() always receives the first six digits + // of a non-negative fractional second even for timestamps before the epoch. DateV2Value value; value.from_unixtime(epoch_seconds(), cctz::utc_time_zone()); value.set_microsecond(microsecond()); @@ -35,6 +37,8 @@ bool TimeStampNsValue::from_datetime(const DateV2Value& val DORIS_CHECK_LE(nanosecond_remainder, 999); int64_t seconds = 0; value.unix_timestamp(&seconds, cctz::utc_time_zone()); + // The civil adapter and remainder form one exact epoch-nanosecond value. Int128 is required: + // valid boundary dates overflow Int64 during the intermediate seconds-to-nanoseconds product. const __int128 epoch_nanos = static_cast<__int128>(seconds) * NANOS_PER_SECOND + static_cast<__int128>(value.microsecond()) * 1000 + nanosecond_remainder; @@ -56,6 +60,8 @@ int32_t TimeStampNsValue::to_buffer(char* buffer, int scale) const { } buffer[base_length] = '.'; uint32_t nanos = nanosecond(); + // Consume the fraction from most significant to least significant digit. Deliberately do not + // round here: type normalization happens while parsing/casting, before a value is stored. for (int i = 0; i < scale; ++i) { buffer[base_length + 1 + i] = static_cast('0' + nanos / static_cast(int_exp10(8 - i))); diff --git a/be/src/core/value/timestamp_ns_value.h b/be/src/core/value/timestamp_ns_value.h index 9a6b48c24c69bf..7e676fb6f3df15 100644 --- a/be/src/core/value/timestamp_ns_value.h +++ b/be/src/core/value/timestamp_ns_value.h @@ -21,8 +21,14 @@ namespace doris { -// TIMESTAMP_NS uses a signed Int64 nanosecond offset from the Unix epoch. UTC is used only as a -// stable mapping between timezone-naive civil fields and the physical integer representation. +// TIMESTAMP_NS is represented by a signed Int64 count of nanoseconds from the Unix epoch. The +// complete Int64 domain is valid, giving the exact range +// [1677-09-21 00:12:43.145224192, 2262-04-11 23:47:16.854775807]. The SQL type is timezone-naive: +// UTC below is only a deterministic bridge between the integer and civil calendar fields, and +// must not apply the session time zone. +// +// Keep this class an Int64-sized trivially-copyable value. Columns, storage encodings, hashing, +// and SIMD paths rely on being able to move its bytes exactly like an Int64. class TimeStampNsValue { public: static constexpr int64_t NANOS_PER_SECOND = 1000000000; @@ -32,9 +38,13 @@ class TimeStampNsValue { constexpr TimeStampNsValue() = default; explicit constexpr TimeStampNsValue(int64_t epoch_nanos) : _epoch_nanos(epoch_nanos) {} + // This is the raw physical key, not the packed civil layout used by DATETIMEV2. constexpr int64_t to_date_int_val() const { return _epoch_nanos; } constexpr int64_t epoch_nanos() const { return _epoch_nanos; } + // Split epoch nanoseconds using floor division so the fractional component is always + // non-negative. For example, -1ns becomes second -1 plus nanosecond 999,999,999 rather than + // second 0 plus a negative fraction. int64_t epoch_seconds() const { int64_t seconds = _epoch_nanos / NANOS_PER_SECOND; if (_epoch_nanos % NANOS_PER_SECOND < 0) { @@ -54,109 +64,30 @@ class TimeStampNsValue { uint32_t microsecond() const { return nanosecond() / 1000; } uint16_t nanosecond_remainder() const { return static_cast(nanosecond() % 1000); } + // DATETIMEV2 is used only as a civil-calendar adapter. It carries the first six fractional + // digits; callers must preserve nanosecond_remainder() separately when the operation should + // not discard the final three digits. DateV2Value to_datetime() const; bool from_datetime(const DateV2Value& value, uint16_t nanosecond_remainder = 0); - uint16_t year() const { return to_datetime().year(); } - uint8_t month() const { return to_datetime().month(); } - uint8_t day() const { return to_datetime().day(); } - uint8_t hour() const { return to_datetime().hour(); } - uint8_t minute() const { return to_datetime().minute(); } - uint8_t second() const { return to_datetime().second(); } - int quarter() const { return to_datetime().quarter(); } + // sequence_match still uses the common date-value concept in BE. These two integral helpers + // are the only civil accessors required by that execution path. int64_t daynr() const { return to_datetime().daynr(); } - uint16_t year_of_week() const { return to_datetime().year_of_week(); } - uint8_t week(uint8_t mode) const { return to_datetime().week(mode); } - uint32_t year_week(uint8_t mode) const { return to_datetime().year_week(mode); } - int day_of_year() const { return to_datetime().day_of_year(); } - int day_of_week() const { return to_datetime().day_of_week(); } - uint8_t weekday() const { return to_datetime().weekday(); } int64_t time_part_to_seconds() const { return to_datetime().time_part_to_seconds(); } - int64_t time_part_to_microsecond() const { - return time_part_to_seconds() * 1000000 + microsecond(); - } - - template - int64_t time_part_diff_in_ms(const RHS& rhs) const { - return time_part_to_microsecond() - rhs.time_part_to_microsecond(); - } + // This intentionally compares integral civil seconds and ignores fractional seconds. It is + // the contract used by sequence_match's second-based pattern conditions. template int64_t datetime_diff_in_seconds(const RHS& rhs) const { return (daynr() - rhs.daynr()) * SECOND_PER_HOUR * HOUR_PER_DAY + time_part_to_seconds() - rhs.time_part_to_seconds(); } - template - int64_t datetime_diff_in_microseconds(const RHS& rhs) const { - if constexpr (std::is_same_v, TimeStampNsValue>) { - return static_cast((static_cast<__int128>(_epoch_nanos) - rhs._epoch_nanos) / - 1000); - } - return (daynr() - rhs.daynr()) * HOUR_PER_DAY * SECOND_PER_HOUR * MS_PER_SECOND + - time_part_diff_in_ms(rhs); - } - - template - int32_t date_diff_in_days(const RHS& rhs) const { - return static_cast(daynr() - rhs.daynr()); - } - - int32_t date_diff_in_days_round_to_zero_by_time(const auto& rhs) const { - int32_t days = date_diff_in_days(rhs); - const int64_t time_diff = time_part_diff_in_ms(rhs); - if (days > 0 && time_diff < 0) { - --days; - } else if (days < 0 && time_diff > 0) { - ++days; - } - return days; - } - - int64_t datetime_diff_in_seconds_round_to_zero_by_ms(const auto& rhs) const { - int64_t seconds = datetime_diff_in_seconds(rhs); - const int64_t fraction_diff = static_cast(nanosecond()) - rhs.nanosecond(); - if (seconds > 0 && fraction_diff < 0) { - --seconds; - } else if (seconds < 0 && fraction_diff > 0) { - ++seconds; - } - return seconds; - } - - const char* day_name_with_locale(const char* const* day_names) const { - return to_datetime().day_name_with_locale(day_names); - } - const char* month_name_with_locale(const char* const* month_names) const { - return to_datetime().month_name_with_locale(month_names); - } - bool to_format_string_conservative(const char* format, size_t len, char* to, - size_t max_valid_length) const { - return to_datetime().to_format_string_conservative(format, len, to, max_valid_length); - } - bool is_valid_date() const { return true; } - template - bool date_add_interval(const TimeInterval& interval) { - auto value = to_datetime(); - const uint16_t remainder = nanosecond_remainder(); - if (!value.date_add_interval(interval)) { - return false; - } - return from_datetime(value, remainder); - } - - template - bool datetime_trunc() { - auto value = to_datetime(); - if (!value.template datetime_trunc()) { - return false; - } - return from_datetime(value); - } - + // Formatting emits the requested leading fractional digits without rounding. The SQL type + // normally requests scale 9; smaller scales are retained for generic date-format callers. int32_t to_buffer(char* buffer, int scale = 9) const; char* to_string(char* buffer, int scale = 9) const { const int32_t length = to_buffer(buffer, scale); @@ -171,18 +102,6 @@ class TimeStampNsValue { auto operator<=>(const TimeStampNsValue&) const = default; - TimeStampNsValue& operator+=(int64_t seconds) { - int64_t delta = 0; - DORIS_CHECK(!__builtin_mul_overflow(seconds, NANOS_PER_SECOND, &delta)); - DORIS_CHECK(!__builtin_add_overflow(_epoch_nanos, delta, &_epoch_nanos)); - return *this; - } - - TimeStampNsValue& operator-=(int64_t seconds) { - DORIS_CHECK_NE(seconds, std::numeric_limits::min()); - return *this += -seconds; - } - uint32_t hash(int seed) const { return HashUtil::hash(&_epoch_nanos, sizeof(_epoch_nanos), seed); } @@ -194,66 +113,6 @@ class TimeStampNsValue { static_assert(sizeof(TimeStampNsValue) == sizeof(int64_t)); static_assert(std::is_trivially_copyable_v); -template -int64_t datetime_diff(const TimeStampNsValue& ts_value1, const TimeStampNsValue& ts_value2) { - const auto time_key = [](const TimeStampNsValue& value, bool include_month) { - int64_t result = include_month ? value.month() : 0; - result = result * 32 + value.day(); - result = result * 24 + value.hour(); - result = result * 60 + value.minute(); - result = result * 60 + value.second(); - return result * TimeStampNsValue::NANOS_PER_SECOND + value.nanosecond(); - }; - if constexpr (UNIT == YEAR) { - int year = ts_value2.year() - ts_value1.year(); - const int64_t remainder1 = time_key(ts_value1, true); - const int64_t remainder2 = time_key(ts_value2, true); - if (year > 0) { - year -= remainder2 < remainder1; - } else if (year < 0) { - year += remainder2 > remainder1; - } - return year; - } else if constexpr (UNIT == QUARTER || UNIT == MONTH) { - int month = (ts_value2.year() - ts_value1.year()) * 12 + - (ts_value2.month() - ts_value1.month()); - const int64_t remainder1 = time_key(ts_value1, false); - const int64_t remainder2 = time_key(ts_value2, false); - if (month > 0) { - month -= remainder2 < remainder1; - } else if (month < 0) { - month += remainder2 > remainder1; - } - return UNIT == QUARTER ? month / 3 : month; - } else if constexpr (UNIT == WEEK || UNIT == DAY) { - int64_t day = ts_value2.daynr() - ts_value1.daynr(); - const int64_t time1 = - ts_value1.time_part_to_seconds() * TimeStampNsValue::NANOS_PER_SECOND + - ts_value1.nanosecond(); - const int64_t time2 = - ts_value2.time_part_to_seconds() * TimeStampNsValue::NANOS_PER_SECOND + - ts_value2.nanosecond(); - if (day > 0) { - day -= time2 < time1; - } else if (day < 0) { - day += time2 > time1; - } - return UNIT == WEEK ? day / 7 : day; - } else { - constexpr int64_t divisor = UNIT == HOUR ? 3600 * TimeStampNsValue::NANOS_PER_SECOND - : UNIT == MINUTE ? 60 * TimeStampNsValue::NANOS_PER_SECOND - : UNIT == SECOND ? TimeStampNsValue::NANOS_PER_SECOND - : UNIT == MILLISECOND ? 1000000 - : 1000; - static_assert(UNIT == HOUR || UNIT == MINUTE || UNIT == SECOND || UNIT == MILLISECOND || - UNIT == MICROSECOND, - "Unsupported TimeUnit for datetime_diff"); - return static_cast( - (static_cast<__int128>(ts_value2.epoch_nanos()) - ts_value1.epoch_nanos()) / - divisor); - } -} - } // namespace doris template <> diff --git a/be/src/exprs/function/array/function_array_aggregation.cpp b/be/src/exprs/function/array/function_array_aggregation.cpp index 39c58ee6a2f2ad..671f39ea67114d 100644 --- a/be/src/exprs/function/array/function_array_aggregation.cpp +++ b/be/src/exprs/function/array/function_array_aggregation.cpp @@ -189,7 +189,6 @@ struct ArrayAggregateImpl { execute_type(res, type, data, offsets) || execute_type(res, type, data, offsets) || execute_type(res, type, data, offsets) || - execute_type(res, type, data, offsets) || execute_type(res, type, data, offsets) || execute_type(res, type, data, offsets)) { block.replace_by_position(result, std::move(res)); diff --git a/be/src/exprs/function/cast/function_cast.cpp b/be/src/exprs/function/cast/function_cast.cpp index d6c46a4de5e662..cabbba3d125e03 100644 --- a/be/src/exprs/function/cast/function_cast.cpp +++ b/be/src/exprs/function/cast/function_cast.cpp @@ -21,7 +21,7 @@ #include "core/data_type/data_type_decimal.h" #include "core/data_type/data_type_number.h" // IWYU pragma: keep #include "core/data_type/data_type_quantilestate.h" -#include "core/data_type/data_type_timestamp_ns.h" +#include "core/data_type/data_type_timestamp_ns.h" // IWYU pragma: keep #include "core/data_type/data_type_variant.h" #include "core/data_type/data_type_variant_v2.h" #include "core/data_type/primitive_type.h" diff --git a/be/src/exprs/function/date_time_transforms.h b/be/src/exprs/function/date_time_transforms.h index d48f35e96c8719..6bc2d6ed92f01b 100644 --- a/be/src/exprs/function/date_time_transforms.h +++ b/be/src/exprs/function/date_time_transforms.h @@ -145,10 +145,6 @@ struct ToDateImpl { } else if constexpr (std::is_same_v) { t.cast_to_date(); return t; - } else if constexpr (std::is_same_v) { - auto datetime = t.to_datetime(); - return binary_cast>( - static_cast(datetime.to_date_int_val() >> TIME_PART_LENGTH)); } else { return binary_cast>( (UInt32)(t.to_date_int_val() >> TIME_PART_LENGTH)); diff --git a/be/src/exprs/function/function.h b/be/src/exprs/function/function.h index 5387b363a42c31..8307a6a1c7c1b0 100644 --- a/be/src/exprs/function/function.h +++ b/be/src/exprs/function/function.h @@ -66,8 +66,7 @@ struct FunctionAttr { for (auto it : arguments) { \ is_nullable = is_nullable || it.type->is_nullable(); \ is_datev2 = is_datev2 || it.type->get_primitive_type() == TYPE_DATEV2 || \ - it.type->get_primitive_type() == TYPE_DATETIMEV2 || \ - it.type->get_primitive_type() == TYPE_TIMESTAMP_NS; \ + it.type->get_primitive_type() == TYPE_DATETIMEV2; \ } \ return is_nullable || !is_datev2 \ ? make_nullable( \ diff --git a/be/src/exprs/function/function_convert_tz.cpp b/be/src/exprs/function/function_convert_tz.cpp index b5e2c395fde04c..aa2966cbc3fd44 100644 --- a/be/src/exprs/function/function_convert_tz.cpp +++ b/be/src/exprs/function/function_convert_tz.cpp @@ -40,12 +40,10 @@ #include "core/data_type/data_type_date_time.h" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_string.h" -#include "core/data_type/data_type_timestamp_ns.h" #include "core/data_type/define_primitive_type.h" #include "core/data_type/primitive_type.h" #include "core/string_ref.h" #include "core/types.h" -#include "core/value/timestamp_ns_value.h" #include "core/value/vdatetime_value.h" #include "exec/common/util.hpp" #include "exprs/aggregate/aggregate_function.h" @@ -66,6 +64,10 @@ struct ConvertTzState { }; class FunctionConvertTZ : public IFunction { + constexpr static PrimitiveType PType = PrimitiveType::TYPE_DATETIMEV2; + using DateValueType = PrimitiveTypeTraits::CppType; + using ColumnType = PrimitiveTypeTraits::ColumnType; + public: static constexpr auto name = "convert_tz"; @@ -76,11 +78,8 @@ class FunctionConvertTZ : public IFunction { size_t get_number_of_arguments() const override { return 3; } DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { - DataTypePtr result_type = std::make_shared(); - if (remove_nullable(arguments[0])->get_primitive_type() == TYPE_TIMESTAMP_NS) { - result_type = std::make_shared(); - } - return have_nullable(arguments) ? make_nullable(result_type) : result_type; + return have_nullable(arguments) ? make_nullable(std::make_shared()) + : std::make_shared(); } // default value of timezone is invalid, should skip to avoid wrong exception @@ -169,46 +168,23 @@ class FunctionConvertTZ : public IFunction { argument_columns[1] = remove_nullable(argument_columns[1]); argument_columns[2] = remove_nullable(argument_columns[2]); - const auto argument_type = - remove_nullable(block.get_by_position(arguments[0]).type)->get_primitive_type(); - if (argument_type == TYPE_TIMESTAMP_NS) { - return execute_impl_typed( - context, block, result, input_rows_count, argument_columns, col_const, - convert_tz_state, std::move(result_null_map_column), result_null_map); - } - DORIS_CHECK_EQ(argument_type, TYPE_DATETIMEV2); - return execute_impl_typed( - context, block, result, input_rows_count, argument_columns, col_const, - convert_tz_state, std::move(result_null_map_column), result_null_map); - } - -private: - template - static Status execute_impl_typed(FunctionContext* context, Block& block, uint32_t result, - size_t input_rows_count, - const ColumnPtr (&argument_columns)[3], - const bool (&col_const)[3], ConvertTzState* convert_tz_state, - MutableColumnPtr result_null_map_column, - NullMap& result_null_map) { - using ColumnType = PrimitiveTypeTraits::ColumnType; auto result_column = ColumnType::create(); if (convert_tz_state->use_state) { // ignore argument columns, use cached timezone input in state - execute_tz_const_with_state( - convert_tz_state, assert_cast(argument_columns[0].get()), - result_column.get(), result_null_map, input_rows_count); + execute_tz_const_with_state(convert_tz_state, + assert_cast(argument_columns[0].get()), + result_column.get(), result_null_map, input_rows_count); } else if (col_const[1] && col_const[2]) { // arguments are const - execute_tz_const(context, - assert_cast(argument_columns[0].get()), - assert_cast(argument_columns[1].get()), - assert_cast(argument_columns[2].get()), - result_column.get(), result_null_map, input_rows_count); + execute_tz_const(context, assert_cast(argument_columns[0].get()), + assert_cast(argument_columns[1].get()), + assert_cast(argument_columns[2].get()), + result_column.get(), result_null_map, input_rows_count); } else { - execute(context, assert_cast(argument_columns[0].get()), - assert_cast(argument_columns[1].get()), - assert_cast(argument_columns[2].get()), - result_column.get(), result_null_map, input_rows_count); + _execute(context, assert_cast(argument_columns[0].get()), + assert_cast(argument_columns[1].get()), + assert_cast(argument_columns[2].get()), + result_column.get(), result_null_map, input_rows_count); } //if const if (block.get_data_type(result)->is_nullable()) { @@ -220,12 +196,11 @@ class FunctionConvertTZ : public IFunction { return Status::OK(); } - template - static void execute(FunctionContext* context, - const typename PrimitiveTypeTraits::ColumnType* date_column, - const ColumnString* from_tz_column, const ColumnString* to_tz_column, - typename PrimitiveTypeTraits::ColumnType* result_column, - NullMap& result_null_map, size_t input_rows_count) { +private: + static void _execute(FunctionContext* context, const ColumnType* date_column, + const ColumnString* from_tz_column, const ColumnString* to_tz_column, + ColumnType* result_column, NullMap& result_null_map, + size_t input_rows_count) { for (size_t i = 0; i < input_rows_count; i++) { if (result_null_map[i]) { result_column->insert_default(); @@ -233,60 +208,27 @@ class FunctionConvertTZ : public IFunction { } auto from_tz = from_tz_column->get_data_at(i).to_string(); auto to_tz = to_tz_column->get_data_at(i).to_string(); - execute_inner_loop(date_column, from_tz, to_tz, result_column, result_null_map, - i); + execute_inner_loop(date_column, from_tz, to_tz, result_column, result_null_map, i); } } - template - static std::pair unix_timestamp_for_convert_tz( + static std::pair unix_timestamp_for_convert_tz( const DateValueType& ts_value, const cctz::time_zone& from_tz) { - const auto civil_value = [&]() { - if constexpr (std::is_same_v) { - return ts_value.to_datetime(); - } else { - return ts_value; - } - }(); - cctz::civil_second civil_time(civil_value.year(), civil_value.month(), civil_value.day(), - civil_value.hour(), civil_value.minute(), - civil_value.second()); + cctz::civil_second civil_time(ts_value.year(), ts_value.month(), ts_value.day(), + ts_value.hour(), ts_value.minute(), ts_value.second()); const auto lookup = from_tz.lookup(civil_time); const bool skipped = lookup.kind == cctz::time_zone::civil_lookup::SKIPPED; const auto tp = skipped ? lookup.trans : lookup.pre; // Skipped civil times map to the transition instant. Do not keep the // input fractional part inside a local time interval that never existed. - const uint32_t fraction = [&]() { - if constexpr (std::is_same_v) { - return ts_value.nanosecond(); - } - return ts_value.microsecond() * 1000; - }(); - return {tp.time_since_epoch().count(), skipped ? 0 : fraction}; + return {tp.time_since_epoch().count(), skipped ? 0 : ts_value.microsecond()}; } - template - static bool convert_tz_value(const DateValueType& source, const cctz::time_zone& from_tz, - const cctz::time_zone& to_tz, DateValueType& result) { - const auto [seconds, nanoseconds] = unix_timestamp_for_convert_tz(source, from_tz); - if constexpr (std::is_same_v) { - DateV2Value datetime; - datetime.from_unixtime({seconds, nanoseconds / 1000}, to_tz); - return result.from_datetime(datetime, nanoseconds % 1000); - } else { - result.from_unixtime({seconds, nanoseconds / 1000}, to_tz); - return result.is_valid_date(); - } - } - - template - static void execute_tz_const_with_state( - ConvertTzState* convert_tz_state, - const typename PrimitiveTypeTraits::ColumnType* date_column, - typename PrimitiveTypeTraits::ColumnType* result_column, - NullMap& result_null_map, size_t input_rows_count) { - using DateValueType = PrimitiveTypeTraits::CppType; + static void execute_tz_const_with_state(ConvertTzState* convert_tz_state, + const ColumnType* date_column, + ColumnType* result_column, NullMap& result_null_map, + size_t input_rows_count) { cctz::time_zone& from_tz = convert_tz_state->from_tz; cctz::time_zone& to_tz = convert_tz_state->to_tz; auto push_null = [&](size_t row) { @@ -310,21 +252,20 @@ class FunctionConvertTZ : public IFunction { DateValueType ts_value = date_column->get_element(i); DateValueType ts_value2; - if (!convert_tz_value(ts_value, from_tz, to_tz, ts_value2)) [[unlikely]] { + ts_value2.from_unixtime(unix_timestamp_for_convert_tz(ts_value, from_tz), to_tz); + + if (!ts_value2.is_valid_date()) [[unlikely]] { throw_out_of_bound_convert_tz(date_column->get_element(i), from_tz.name(), to_tz.name()); } - result_column->insert(Field::create_field(ts_value2)); + result_column->insert(Field::create_field(ts_value2)); } } - template - static void execute_tz_const(FunctionContext* context, - const typename PrimitiveTypeTraits::ColumnType* date_column, + static void execute_tz_const(FunctionContext* context, const ColumnType* date_column, const ColumnString* from_tz_column, - const ColumnString* to_tz_column, - typename PrimitiveTypeTraits::ColumnType* result_column, + const ColumnString* to_tz_column, ColumnType* result_column, NullMap& result_null_map, size_t input_rows_count) { auto from_tz = from_tz_column->get_data_at(0).to_string(); auto to_tz = to_tz_column->get_data_at(0).to_string(); @@ -342,18 +283,13 @@ class FunctionConvertTZ : public IFunction { result_column->insert_default(); continue; } - execute_inner_loop(date_column, from_tz, to_tz, result_column, result_null_map, - i); + execute_inner_loop(date_column, from_tz, to_tz, result_column, result_null_map, i); } } - template - static void execute_inner_loop( - const typename PrimitiveTypeTraits::ColumnType* date_column, - const std::string& from_tz_name, const std::string& to_tz_name, - typename PrimitiveTypeTraits::ColumnType* result_column, - NullMap& result_null_map, const size_t index_now) { - using DateValueType = PrimitiveTypeTraits::CppType; + static void execute_inner_loop(const ColumnType* date_column, const std::string& from_tz_name, + const std::string& to_tz_name, ColumnType* result_column, + NullMap& result_null_map, const size_t index_now) { DateValueType ts_value = date_column->get_element(index_now); cctz::time_zone from_tz {}, to_tz {}; DateValueType ts_value2; @@ -367,12 +303,14 @@ class FunctionConvertTZ : public IFunction { to_tz_name); } - if (!convert_tz_value(ts_value, from_tz, to_tz, ts_value2)) [[unlikely]] { + ts_value2.from_unixtime(unix_timestamp_for_convert_tz(ts_value, from_tz), to_tz); + + if (!ts_value2.is_valid_date()) [[unlikely]] { throw_out_of_bound_convert_tz(date_column->get_element(index_now), from_tz.name(), to_tz.name()); } - result_column->insert(Field::create_field(ts_value2)); + result_column->insert(Field::create_field(ts_value2)); } }; diff --git a/be/src/exprs/function/function_date_or_datetime_computation.cpp b/be/src/exprs/function/function_date_or_datetime_computation.cpp index d2dc3bf67c72f4..8973cab77e36a1 100644 --- a/be/src/exprs/function/function_date_or_datetime_computation.cpp +++ b/be/src/exprs/function/function_date_or_datetime_computation.cpp @@ -18,7 +18,6 @@ #include "exprs/function/function_date_or_datetime_computation.h" #include "core/data_type/data_type_date_or_datetime_v2.h" -#include "core/data_type/data_type_timestamp_ns.h" #include "core/data_type/define_primitive_type.h" #include "exprs/function/simple_function_factory.h" @@ -89,23 +88,6 @@ using FunctionDatetimeAddWeeks = FunctionDateOrDateTimeComputation>; using FunctionDatetimeAddYears = FunctionDateOrDateTimeComputation>; -#define TIMESTAMP_NS_CALENDAR_FUNCTION(TYPE) \ - using FunctionTimestampNsAdd##TYPE = \ - FunctionDateOrDateTimeComputation>; \ - using FunctionTimestampNsSub##TYPE = \ - FunctionDateOrDateTimeComputation> - -TIMESTAMP_NS_CALENDAR_FUNCTION(Microseconds); -TIMESTAMP_NS_CALENDAR_FUNCTION(Milliseconds); -TIMESTAMP_NS_CALENDAR_FUNCTION(Seconds); -TIMESTAMP_NS_CALENDAR_FUNCTION(Minutes); -TIMESTAMP_NS_CALENDAR_FUNCTION(Hours); -TIMESTAMP_NS_CALENDAR_FUNCTION(Days); -TIMESTAMP_NS_CALENDAR_FUNCTION(Weeks); -TIMESTAMP_NS_CALENDAR_FUNCTION(Months); -TIMESTAMP_NS_CALENDAR_FUNCTION(Quarters); -TIMESTAMP_NS_CALENDAR_FUNCTION(Years); - using FunctionTimestamptzAddMicroseconds = FunctionDateOrDateTimeComputation>; using FunctionTimestamptzAddMilliseconds = @@ -123,18 +105,14 @@ using FunctionTimestamptzAddMonths = FunctionDateOrDateTimeComputation>; using FunctionTimestamptzAddYears = FunctionDateOrDateTimeComputation>; -#define FUNCTION_TIME_UNION_CAL(TYPE) \ - using FunctionDatetimeAdd##TYPE = \ - FunctionDateOrDateTimeComputation>; \ - using FunctionDatetimeSub##TYPE = \ - FunctionDateOrDateTimeComputation>; \ - using FunctionTimestampNsAdd##TYPE = \ - FunctionDateOrDateTimeComputation>; \ - using FunctionTimestampNsSub##TYPE = \ - FunctionDateOrDateTimeComputation>; \ - using FunctionTimestamptzAdd##TYPE = \ - FunctionDateOrDateTimeComputation>; \ - using FunctionTimestamptzSub##TYPE = \ +#define FUNCTION_TIME_UNION_CAL(TYPE) \ + using FunctionDatetimeAdd##TYPE = \ + FunctionDateOrDateTimeComputation>; \ + using FunctionDatetimeSub##TYPE = \ + FunctionDateOrDateTimeComputation>; \ + using FunctionTimestamptzAdd##TYPE = \ + FunctionDateOrDateTimeComputation>; \ + using FunctionTimestamptzSub##TYPE = \ FunctionDateOrDateTimeComputation>; FUNCTION_TIME_UNION_CAL(SecondMicrosecond); @@ -206,10 +184,9 @@ using FunctionSubTimeTimestampTz = #define FUNCTION_TIME_DIFF(NAME, IMPL, TYPE) using NAME##_##TYPE = FunctionTimeDiff>; -#define ALL_FUNCTION_TIME_DIFF(NAME, IMPL) \ - FUNCTION_TIME_DIFF(NAME, IMPL, TYPE_DATETIMEV2) \ - FUNCTION_TIME_DIFF(NAME, IMPL, TYPE_TIMESTAMP_NS) \ - FUNCTION_TIME_DIFF(NAME, IMPL, TYPE_DATEV2) \ +#define ALL_FUNCTION_TIME_DIFF(NAME, IMPL) \ + FUNCTION_TIME_DIFF(NAME, IMPL, TYPE_DATETIMEV2) \ + FUNCTION_TIME_DIFF(NAME, IMPL, TYPE_DATEV2) \ FUNCTION_TIME_DIFF(NAME, IMPL, TYPE_TIMESTAMPTZ) // these diff functions accept all v2 types. but for v1 only datetime. ALL_FUNCTION_TIME_DIFF(FunctionDatetimeDateDiff, DateDiffImpl) @@ -280,16 +257,6 @@ void register_function_date_time_computation(SimpleFunctionFactory& factory) { factory.register_function(); factory.register_function(); factory.register_function(); - factory.register_function(); - factory.register_function(); - factory.register_function(); - factory.register_function(); - factory.register_function(); - factory.register_function(); - factory.register_function(); - factory.register_function(); - factory.register_function(); - factory.register_function(); factory.register_function(); factory.register_function(); factory.register_function(); @@ -304,8 +271,6 @@ void register_function_date_time_computation(SimpleFunctionFactory& factory) { #define REGISTER_TIME_UNION_CAL(TYPE) \ factory.register_function(); \ factory.register_function(); \ - factory.register_function(); \ - factory.register_function(); \ factory.register_function(); \ factory.register_function(); @@ -337,16 +302,6 @@ void register_function_date_time_computation(SimpleFunctionFactory& factory) { factory.register_function(); factory.register_function(); factory.register_function(); - factory.register_function(); - factory.register_function(); - factory.register_function(); - factory.register_function(); - factory.register_function(); - factory.register_function(); - factory.register_function(); - factory.register_function(); - factory.register_function(); - factory.register_function(); factory.register_function(); factory.register_function(); @@ -368,10 +323,9 @@ void register_function_date_time_computation(SimpleFunctionFactory& factory) { #define REGISTER_DATEV2_FUNCTIONS_DIFF(NAME, TYPE) factory.register_function(); -#define REGISTER_ALL_DATEV2_FUNCTIONS_DIFF(NAME) \ - REGISTER_DATEV2_FUNCTIONS_DIFF(NAME, TYPE_DATETIMEV2) \ - REGISTER_DATEV2_FUNCTIONS_DIFF(NAME, TYPE_TIMESTAMP_NS) \ - REGISTER_DATEV2_FUNCTIONS_DIFF(NAME, TYPE_DATEV2) \ +#define REGISTER_ALL_DATEV2_FUNCTIONS_DIFF(NAME) \ + REGISTER_DATEV2_FUNCTIONS_DIFF(NAME, TYPE_DATETIMEV2) \ + REGISTER_DATEV2_FUNCTIONS_DIFF(NAME, TYPE_DATEV2) \ REGISTER_DATEV2_FUNCTIONS_DIFF(NAME, TYPE_TIMESTAMPTZ) REGISTER_ALL_DATEV2_FUNCTIONS_DIFF(FunctionDatetimeDateDiff) @@ -393,4 +347,4 @@ void register_function_date_time_computation(SimpleFunctionFactory& factory) { factory.register_function(); } -} // namespace doris +} // namespace doris \ No newline at end of file diff --git a/be/src/exprs/function/function_date_or_datetime_to_string.cpp b/be/src/exprs/function/function_date_or_datetime_to_string.cpp index 791c6d229b29a8..c7b240df961901 100644 --- a/be/src/exprs/function/function_date_or_datetime_to_string.cpp +++ b/be/src/exprs/function/function_date_or_datetime_to_string.cpp @@ -37,7 +37,6 @@ #include "core/column/column_vector.h" #include "core/data_type/data_type.h" #include "core/data_type/data_type_date_or_datetime_v2.h" // IWYU pragma: keep -#include "core/data_type/data_type_timestamp_ns.h" #include "core/data_type/define_primitive_type.h" #include "core/types.h" #include "exprs/aggregate/aggregate_function.h" @@ -213,9 +212,6 @@ using FunctionMonthNameV2 = FunctionDateOrDateTimeToString>; using FunctionDateTimeV2MonthName = FunctionDateOrDateTimeToString>; -using FunctionTimeStampNsDayName = FunctionDateOrDateTimeToString>; -using FunctionTimeStampNsMonthName = - FunctionDateOrDateTimeToString>; using FunctionYearMonth = FunctionDateOrDateTimeToString; using FunctionDayHour = FunctionDateOrDateTimeToString; using FunctionDayMinute = FunctionDateOrDateTimeToString; @@ -230,7 +226,6 @@ using FunctionSecondMicrosecond = FunctionDateOrDateTimeToString>; using FunctionDateTimeIso8601 = FunctionDateOrDateTimeToString>; -using FunctionTimeStampNsIso8601 = FunctionDateOrDateTimeToString>; using FunctionTimestampTzIso8601 = FunctionDateOrDateTimeToString>; void register_function_date_time_to_string(SimpleFunctionFactory& factory) { @@ -238,8 +233,6 @@ void register_function_date_time_to_string(SimpleFunctionFactory& factory) { factory.register_function(); factory.register_function(); factory.register_function(); - factory.register_function(); - factory.register_function(); factory.register_function(); factory.register_function(); factory.register_function(); @@ -253,7 +246,6 @@ void register_function_date_time_to_string(SimpleFunctionFactory& factory) { factory.register_function(); factory.register_function(); factory.register_function(); - factory.register_function(); factory.register_function(); } diff --git a/be/src/exprs/function/function_datetime_floor_ceil.cpp b/be/src/exprs/function/function_datetime_floor_ceil.cpp index 8cd62f87ea50c8..6369dd9592cda3 100644 --- a/be/src/exprs/function/function_datetime_floor_ceil.cpp +++ b/be/src/exprs/function/function_datetime_floor_ceil.cpp @@ -46,7 +46,6 @@ #include "core/data_type/data_type_date_time.h" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" -#include "core/data_type/data_type_timestamp_ns.h" #include "core/data_type/define_primitive_type.h" #include "core/field.h" #include "core/pod_array_fwd.h" diff --git a/be/src/exprs/function/function_datetime_string_to_string.cpp b/be/src/exprs/function/function_datetime_string_to_string.cpp index 0559d3a4dabb0e..9fca37931ecc84 100644 --- a/be/src/exprs/function/function_datetime_string_to_string.cpp +++ b/be/src/exprs/function/function_datetime_string_to_string.cpp @@ -18,7 +18,6 @@ #include "exprs/function/function_datetime_string_to_string.h" #include "core/data_type/data_type_date_or_datetime_v2.h" // IWYU pragma: keep -#include "core/data_type/data_type_timestamp_ns.h" #include "core/data_type/define_primitive_type.h" #include "exprs/function/date_time_transforms.h" #include "exprs/function/simple_function_factory.h" @@ -28,8 +27,6 @@ namespace doris { using FunctionDateFormatV2 = FunctionDateTimeStringToString>; using FunctionDateTimeV2DateFormat = FunctionDateTimeStringToString>; -using FunctionTimeStampNsDateFormat = - FunctionDateTimeStringToString>; // old version using FunctionFromUnixTimeOneArg = FunctionDateTimeStringToString>; using FunctionFromUnixTimeTwoArg = FunctionDateTimeStringToString>; @@ -42,7 +39,6 @@ using FunctionFromUnixTimeNewDecimalTwoArg = FunctionDateTimeStringToString>; using FunctionTimeFormatDate = FunctionTimeFormat; using FunctionTimeFormatDateTime = FunctionTimeFormat; -using FunctionTimeFormatTimeStampNs = FunctionTimeFormat; using FunctionTimeFormatTime = FunctionTimeFormat; void register_function_date_time_string_to_string(SimpleFunctionFactory& factory) { @@ -54,10 +50,8 @@ void register_function_date_time_string_to_string(SimpleFunctionFactory& factory factory.register_function(); factory.register_function(); factory.register_function(); - factory.register_function(); factory.register_function(); factory.register_function(); - factory.register_function(); factory.register_function(); } diff --git a/be/src/exprs/function/function_datetime_string_to_string.h b/be/src/exprs/function/function_datetime_string_to_string.h index c1d7071fdb938d..f927389ac261d5 100644 --- a/be/src/exprs/function/function_datetime_string_to_string.h +++ b/be/src/exprs/function/function_datetime_string_to_string.h @@ -229,7 +229,6 @@ class FunctionDateTimeStringToString : public IFunction { std::is_same_v || std::is_same_v || std::is_same_v || - std::is_same_v || std::is_same_v) { char buf[64]; pod_array[i].to_string(buf); diff --git a/be/src/exprs/function/function_other_types_to_date.cpp b/be/src/exprs/function/function_other_types_to_date.cpp index 5798eb85a1eaf3..122fdb9338ad57 100644 --- a/be/src/exprs/function/function_other_types_to_date.cpp +++ b/be/src/exprs/function/function_other_types_to_date.cpp @@ -50,7 +50,6 @@ #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" -#include "core/data_type/data_type_timestamp_ns.h" #include "core/data_type/define_primitive_type.h" #include "core/data_type/primitive_type.h" #include "core/pod_array.h" @@ -696,20 +695,15 @@ template struct UnixTimeStampDateImpl { static DataTypes get_variadic_argument_types() { return {std::make_shared()}; } - static constexpr bool is_datetime_v2 = std::is_same_v || - std::is_same_v; - static constexpr bool is_timestamp_ns = std::is_same_v; - using ResultDataType = std::conditional_t< - is_timestamp_ns, DataTypeDecimal128, - std::conditional_t>>; - using ResultColumnType = std::conditional_t< - is_timestamp_ns, ColumnDecimal128V3, - std::conditional_t>>; + using ResultDataType = + std::conditional_t, DataTypeDecimal64, + std::conditional_t>; + using ResultColumnType = + std::conditional_t, ColumnDecimal64, + std::conditional_t>; static DataTypePtr get_return_type_impl(const ColumnsWithTypeAndName& arguments) { - if constexpr (is_datetime_v2) { + if constexpr (std::is_same_v) { UInt32 scale = arguments[0].type->get_scale(); return std::make_shared(12 + scale, scale); } else { @@ -753,7 +747,7 @@ struct UnixTimeStampDateImpl { col_result_data[i] = trim_timestamp(timestamp, NewVersion); } block.replace_by_position(result, std::move(col_result)); - } else if constexpr (std::is_same_v) { + } else { // DatetimeV2 const auto* col_source = assert_cast(col.get()); UInt32 scale = block.get_by_position(arguments[0]).type->get_scale(); auto col_result = ColumnDecimal64::create(input_rows_count, scale); @@ -774,28 +768,6 @@ struct UnixTimeStampDateImpl { .value; } block.replace_by_position(result, std::move(col_result)); - } else { - const auto* col_source = assert_cast(col.get()); - UInt32 scale = block.get_by_position(arguments[0]).type->get_scale(); - auto col_result = ColumnDecimal128V3::create(input_rows_count, scale); - auto& col_result_data = col_result->get_data(); - col_result->resize(input_rows_count); - - for (int i = 0; i < input_rows_count; i++) { - const auto& nano_value = col_source->get_data()[i]; - const auto datetime_value = nano_value.to_datetime(); - std::pair timestamp {}; - datetime_value.unix_timestamp(×tamp, context->state()->timezone_obj()); - - const auto trimmed_timestamp = trim_timestamp(timestamp, NewVersion); - const int64_t fraction = - trimmed_timestamp.first == timestamp.first - ? nano_value.nanosecond() / common::exp10_i64(9 - scale) - : 0; - col_result_data[i] = - Decimal128V3::from_int_frac(trimmed_timestamp.first, fraction, scale).value; - } - block.replace_by_position(result, std::move(col_result)); } return Status::OK(); @@ -1250,7 +1222,12 @@ class FunctionOtherTypesToDateType : public IFunction { } Status open(FunctionContext* context, FunctionContext::FunctionStateScope scope) override { - if constexpr (requires { Impl::open(context, scope); }) { + if constexpr (std::is_same_v> || + std::is_same_v> || + std::is_same_v> || + std::is_same_v> || + std::is_same_v> || + std::is_same_v>) { return Impl::open(context, scope); } else { return Status::OK(); @@ -1497,16 +1474,12 @@ using FunctionMakeDate = FunctionOtherTypesToDateType; using FunctionDateTruncDateV2 = FunctionOtherTypesToDateType>; using FunctionDateTruncDatetimeV2 = FunctionOtherTypesToDateType>; -using FunctionDateTruncTimestampNs = - FunctionOtherTypesToDateType>; using FunctionDateTruncTimestamptz = FunctionOtherTypesToDateType>; using FunctionDateTruncDateV2WithCommonOrder = FunctionOtherTypesToDateType>; using FunctionDateTruncDatetimeV2WithCommonOrder = FunctionOtherTypesToDateType>; -using FunctionDateTruncTimestampNsWithCommonOrder = - FunctionOtherTypesToDateType>; using FunctionDateTruncTimestamptzWithCommonOrder = FunctionOtherTypesToDateType>; using FunctionFromIso8601DateV2 = FunctionOtherTypesToDateType; @@ -1519,11 +1492,9 @@ void register_function_timestamp(SimpleFunctionFactory& factory) { factory.register_function(); factory.register_function(); factory.register_function(); - factory.register_function(); factory.register_function(); factory.register_function(); factory.register_function(); - factory.register_function(); factory.register_function(); factory.register_function(); @@ -1532,7 +1503,6 @@ void register_function_timestamp(SimpleFunctionFactory& factory) { factory.register_function>>(); factory.register_function>>(); factory.register_function>>(); - factory.register_function>>(); factory.register_function>>(); factory.register_function>>(); factory.register_function< @@ -1543,8 +1513,6 @@ void register_function_timestamp(SimpleFunctionFactory& factory) { FunctionUnixTimestampNew>>(); factory.register_function< FunctionUnixTimestampNew>>(); - factory.register_function< - FunctionUnixTimestampNew>>(); factory.register_function>>(); factory.register_function>(); diff --git a/be/src/exprs/function/least_greast.cpp b/be/src/exprs/function/least_greast.cpp index 9cbbd4ccdc0427..13558035636154 100644 --- a/be/src/exprs/function/least_greast.cpp +++ b/be/src/exprs/function/least_greast.cpp @@ -304,13 +304,6 @@ struct FunctionFieldImpl { } break; } - case PrimitiveType::TYPE_TIMESTAMP_NS: { - for (int col = 1; col < arguments.size(); ++col) { - insert_result_data(res_data, argument_columns[0], - argument_columns[col], input_rows_count, col); - } - break; - } case PrimitiveType::TYPE_TIMESTAMPTZ: { for (int col = 1; col < arguments.size(); ++col) { insert_result_data(res_data, argument_columns[0], @@ -374,4 +367,4 @@ void register_function_least_greast(SimpleFunctionFactory& factory) { factory.register_function(); factory.register_function(); } -}; // namespace doris +}; // namespace doris \ No newline at end of file diff --git a/be/src/exprs/function/time_of_function.cpp b/be/src/exprs/function/time_of_function.cpp index d9133c94242cf5..6162477f38517f 100644 --- a/be/src/exprs/function/time_of_function.cpp +++ b/be/src/exprs/function/time_of_function.cpp @@ -20,7 +20,6 @@ #include "core/data_type/data_type_date_or_datetime_v2.h" // IWYU pragma: keep #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" -#include "core/data_type/data_type_timestamp_ns.h" #include "exprs/function/date_time_transforms.h" #include "exprs/function/function_date_or_datetime_to_something.h" #include "exprs/function/simple_function_factory.h" @@ -52,17 +51,6 @@ using FunctionDateTimeV2YearWeek = using FunctionDateTimeV2WeekDay = FunctionDateOrDateTimeToSomething>; -#define TIMESTAMP_NS_TIME_OF_FUNCTION(NAME, RESULT_TYPE, IMPL) \ - using FunctionTimeStampNs##NAME = \ - FunctionDateOrDateTimeToSomething> - -TIMESTAMP_NS_TIME_OF_FUNCTION(WeekOfYear, DataTypeInt8, WeekOfYearImpl); -TIMESTAMP_NS_TIME_OF_FUNCTION(DayOfYear, DataTypeInt16, DayOfYearImpl); -TIMESTAMP_NS_TIME_OF_FUNCTION(DayOfWeek, DataTypeInt8, DayOfWeekImpl); -TIMESTAMP_NS_TIME_OF_FUNCTION(DayOfMonth, DataTypeInt8, DayOfMonthImpl); -TIMESTAMP_NS_TIME_OF_FUNCTION(YearWeek, DataTypeInt32, ToYearWeekOneArgImpl); -TIMESTAMP_NS_TIME_OF_FUNCTION(WeekDay, DataTypeInt8, WeekDayImpl); - void register_function_time_of_function(SimpleFunctionFactory& factory) { factory.register_function(); factory.register_function(); @@ -76,11 +64,5 @@ void register_function_time_of_function(SimpleFunctionFactory& factory) { factory.register_function(); factory.register_function(); factory.register_function(); - factory.register_function(); - factory.register_function(); - factory.register_function(); - factory.register_function(); - factory.register_function(); - factory.register_function(); } } // namespace doris diff --git a/be/src/exprs/function/to_time_function.cpp b/be/src/exprs/function/to_time_function.cpp index d4e68f07f43212..2f4b6491ceb4d2 100644 --- a/be/src/exprs/function/to_time_function.cpp +++ b/be/src/exprs/function/to_time_function.cpp @@ -19,7 +19,6 @@ #include "core/data_type/data_type_date_or_datetime_v2.h" #include "core/data_type/data_type_date_time.h" #include "core/data_type/data_type_number.h" -#include "core/data_type/data_type_timestamp_ns.h" #include "core/types.h" #include "exprs/function/date_time_transforms.h" #include "exprs/function/function_date_or_datetime_to_something.h" @@ -73,24 +72,6 @@ using FunctionCenturyV2 = FunctionDateOrDateTimeToSomething>; using FunctionDateTimeV2Century = FunctionDateOrDateTimeToSomething>; - -#define TIMESTAMP_NS_TO_TIME_FUNCTION(NAME, RESULT_TYPE, IMPL) \ - using FunctionTimeStampNs##NAME = \ - FunctionDateOrDateTimeToSomething> - -TIMESTAMP_NS_TO_TIME_FUNCTION(Year, DataTypeInt16, ToYearImpl); -TIMESTAMP_NS_TO_TIME_FUNCTION(Quarter, DataTypeInt8, ToQuarterImpl); -TIMESTAMP_NS_TO_TIME_FUNCTION(Month, DataTypeInt8, ToMonthImpl); -TIMESTAMP_NS_TO_TIME_FUNCTION(Day, DataTypeInt8, ToDayImpl); -TIMESTAMP_NS_TO_TIME_FUNCTION(Week, DataTypeInt8, ToWeekOneArgImpl); -TIMESTAMP_NS_TO_TIME_FUNCTION(Hour, DataTypeInt8, ToHourImpl); -TIMESTAMP_NS_TO_TIME_FUNCTION(Minute, DataTypeInt8, ToMinuteImpl); -TIMESTAMP_NS_TO_TIME_FUNCTION(Second, DataTypeInt8, ToSecondImpl); -TIMESTAMP_NS_TO_TIME_FUNCTION(MicroSecond, DataTypeInt32, ToMicroSecondImpl); -TIMESTAMP_NS_TO_TIME_FUNCTION(ToDate, DataTypeDateV2, ToDateImpl); -TIMESTAMP_NS_TO_TIME_FUNCTION(Date, DataTypeDateV2, DateImpl); -TIMESTAMP_NS_TO_TIME_FUNCTION(Century, DataTypeInt16, ToCenturyImpl); - void register_function_to_time_function(SimpleFunctionFactory& factory) { factory.register_function(); factory.register_function(); @@ -119,18 +100,6 @@ void register_function_to_time_function(SimpleFunctionFactory& factory) { factory.register_function(); factory.register_function(); factory.register_function(); - factory.register_function(); - factory.register_function(); - factory.register_function(); - factory.register_function(); - factory.register_function(); - factory.register_function(); - factory.register_function(); - factory.register_function(); - factory.register_function(); - factory.register_function(); - factory.register_function(); - factory.register_function(); factory.register_alias("date", "datev2"); factory.register_alias("to_date", "to_datev2"); } diff --git a/be/src/storage/tablet_info.cpp b/be/src/storage/tablet_info.cpp index c69ec16e5b0021..9b6a3fe6ed7699 100644 --- a/be/src/storage/tablet_info.cpp +++ b/be/src/storage/tablet_info.cpp @@ -42,7 +42,6 @@ #include "core/data_type/data_type_factory.hpp" #include "core/data_type/define_primitive_type.h" #include "core/data_type/primitive_type.h" -#include "core/data_type/storage_field_type.h" #include "core/data_type_serde/data_type_timestamp_ns_serde.h" #include "core/value/large_int_value.h" #include "runtime/descriptors.h" @@ -183,9 +182,11 @@ Status OlapTableSchemaParam::init(const POlapTableSchemaParam& pschema) { for (const auto& p_slot_desc : pschema.slot_descs()) { auto* slot_desc = _obj_pool.add(new SlotDescriptor(p_slot_desc)); _tuple_desc->add_slot(slot_desc); + std::string data_type; + EnumToString(TPrimitiveType, to_thrift(slot_desc->col_type()), data_type); std::string is_null_str = slot_desc->is_nullable() ? "true" : "false"; - std::string data_type_str = std::to_string( - int64_t(primitive_type_to_storage_field_type(slot_desc->col_type()))); + std::string data_type_str = + std::to_string(int64_t(TabletColumn::get_field_type_by_string(data_type))); slots_map.emplace(to_lower(slot_desc->col_name()) + "+" + data_type_str + is_null_str, slot_desc); } @@ -198,12 +199,11 @@ Status OlapTableSchemaParam::init(const POlapTableSchemaParam& pschema) { index->row_binlog_id = p_index.row_binlog_id(); } for (const auto& pcolumn_desc : p_index.columns_desc()) { - TabletColumn* tc = _obj_pool.add(new TabletColumn()); - tc->init_from_pb(pcolumn_desc); if (_unique_key_update_mode != UniqueKeyUpdateModePB::UPDATE_FIXED_COLUMNS || _partial_update_input_columns.contains(pcolumn_desc.name())) { std::string is_null_str = pcolumn_desc.is_nullable() ? "true" : "false"; - std::string data_type_str = std::to_string(int64_t(tc->type())); + std::string data_type_str = std::to_string( + int64_t(TabletColumn::get_field_type_by_string(pcolumn_desc.type()))); auto it = slots_map.find(to_lower(pcolumn_desc.name()) + "+" + data_type_str + is_null_str); if (it == std::end(slots_map)) { @@ -223,6 +223,8 @@ Status OlapTableSchemaParam::init(const POlapTableSchemaParam& pschema) { } index->slots.emplace_back(it->second); } + TabletColumn* tc = _obj_pool.add(new TabletColumn()); + tc->init_from_pb(pcolumn_desc); index->columns.emplace_back(tc); } for (const auto& pindex_desc : p_index.indexes_desc()) { @@ -358,13 +360,11 @@ Status OlapTableSchemaParam::init(const TOlapTableSchemaParam& tschema) { index->row_binlog_id = t_index.row_binlog_id; } for (const auto& tcolumn_desc : t_index.columns_desc) { - TabletColumn* tc = _obj_pool.add(new TabletColumn()); - tc->init_from_thrift(tcolumn_desc); if (_unique_key_update_mode != UniqueKeyUpdateModePB::UPDATE_FIXED_COLUMNS || _partial_update_input_columns.contains(tcolumn_desc.column_name)) { std::string is_null_str = tcolumn_desc.is_allow_null ? "true" : "false"; std::string data_type_str = - std::to_string(int64_t(storage_field_type_to_primitive_type(tc->type()))); + std::to_string(int64_t(thrift_to_type(tcolumn_desc.column_type.type))); auto it = slots_map.find(to_lower(tcolumn_desc.column_name) + "+" + data_type_str + is_null_str); if (it == slots_map.end()) { @@ -387,6 +387,8 @@ Status OlapTableSchemaParam::init(const TOlapTableSchemaParam& tschema) { index->slots.emplace_back(it->second); } index_slots_map.emplace(to_lower(tcolumn_desc.column_name), tcolumn_desc.col_unique_id); + TabletColumn* tc = _obj_pool.add(new TabletColumn()); + tc->init_from_thrift(tcolumn_desc); index->columns.emplace_back(tc); } if (t_index.__isset.indexes_desc) { @@ -617,12 +619,11 @@ bool VOlapTablePartitionParam::_part_contains(VOlapTablePartition* part, // insert value into _partition_block's column // NOLINTBEGIN(readability-function-size) static Status _create_partition_key(const TExprNode& t_expr, BlockRow* part_key, uint16_t pos) { - const auto& partition_column = part_key->first->get_by_position(pos); - const auto partition_type = remove_nullable(partition_column.type); - auto column = std::move(*partition_column.column).mutate(); + auto column = std::move(*part_key->first->get_by_position(pos).column).mutate(); switch (t_expr.node_type) { case TExprNodeType::DATE_LITERAL: { - const auto primitive_type = partition_type->get_primitive_type(); + auto primitive_type = + DataTypeFactory::instance().create_data_type(t_expr.type)->get_primitive_type(); if (primitive_type == TYPE_DATEV2) { DateV2Value dt; CastParameters params; @@ -636,7 +637,8 @@ static Status _create_partition_key(const TExprNode& t_expr, BlockRow* part_key, column->insert_data(reinterpret_cast(&dt), 0); } else if (primitive_type == TYPE_DATETIMEV2) { DateV2Value dt; - const int32_t scale = cast_set(partition_type->get_scale()); + const int32_t scale = + t_expr.type.types.empty() ? -1 : t_expr.type.types.front().scalar_type.scale; CastParameters params; if (!CastToDatetimeV2::from_string_strict_mode( {t_expr.date_literal.value.c_str(), t_expr.date_literal.value.size()}, dt, @@ -656,7 +658,8 @@ static Status _create_partition_key(const TExprNode& t_expr, BlockRow* part_key, } else if (primitive_type == TYPE_TIMESTAMPTZ) { TimestampTzValue res; CastParameters params {.status = Status::OK(), .is_strict = true}; - const int32_t scale = cast_set(partition_type->get_scale()); + const int32_t scale = + t_expr.type.types.empty() ? -1 : t_expr.type.types.front().scalar_type.scale; if (!CastToTimestampTz::from_string( {t_expr.date_literal.value.c_str(), t_expr.date_literal.value.size()}, res, params, nullptr, scale)) [[unlikely]] { diff --git a/be/test/core/data_type/data_type_timestamp_ns_test.cpp b/be/test/core/data_type/data_type_timestamp_ns_test.cpp index 0583d1040c2005..2cd70689272ce7 100644 --- a/be/test/core/data_type/data_type_timestamp_ns_test.cpp +++ b/be/test/core/data_type/data_type_timestamp_ns_test.cpp @@ -47,6 +47,22 @@ namespace doris { +TEST(DataTypeTimeStampNsTest, TypeFamilyClassifiersKeepTimestampNsIndependent) { + EXPECT_TRUE(is_date_type(TYPE_TIMESTAMP_NS)); + EXPECT_TRUE(is_timestamp_ns_type(TYPE_TIMESTAMP_NS)); + EXPECT_FALSE(is_timestamp_ns_type(TYPE_DATETIMEV2)); + + EXPECT_TRUE(is_date_v2_or_datetime_v2(TYPE_DATEV2)); + EXPECT_TRUE(is_date_v2_or_datetime_v2(TYPE_DATETIMEV2)); + EXPECT_FALSE(is_date_v2_or_datetime_v2(TYPE_TIMESTAMP_NS)); + + EXPECT_TRUE(is_datetime_v2(TYPE_DATETIMEV2)); + EXPECT_FALSE(is_datetime_v2(TYPE_TIMESTAMP_NS)); + + EXPECT_TRUE(IsDataTypeDateTimeV2); + EXPECT_FALSE(IsDataTypeDateTimeV2); +} + TEST(DataTypeTimeStampNsTest, Int64EpochRangeAndOrdering) { const TimeStampNsValue epoch(0); const TimeStampNsValue before_epoch(-1); @@ -180,44 +196,16 @@ TEST(DataTypeTimeStampNsTest, CivilRoundTripPreservesSubMicrosecondDigits) { TimeStampNsValue value; ASSERT_TRUE(value.from_datetime(civil, 789)); EXPECT_EQ(value.to_string(9), "2024-02-29 23:59:58.123456789"); - EXPECT_EQ(value.year(), 2024); - EXPECT_EQ(value.month(), 2); - EXPECT_EQ(value.day(), 29); - EXPECT_EQ(value.hour(), 23); - EXPECT_EQ(value.minute(), 59); - EXPECT_EQ(value.second(), 58); + const auto round_trip = value.to_datetime(); + EXPECT_EQ(round_trip.year(), 2024); + EXPECT_EQ(round_trip.month(), 2); + EXPECT_EQ(round_trip.day(), 29); + EXPECT_EQ(round_trip.hour(), 23); + EXPECT_EQ(round_trip.minute(), 59); + EXPECT_EQ(round_trip.second(), 58); EXPECT_EQ(value.microsecond(), 123456); EXPECT_EQ(value.nanosecond_remainder(), 789); - EXPECT_EQ(value.to_datetime().to_date_int_val(), civil.to_date_int_val()); -} - -TEST(DataTypeTimeStampNsTest, CalendarArithmeticPreservesSubMicrosecondDigits) { - int64_t raw_value = 0; - ASSERT_TRUE(parse_timestamp_ns(StringRef("2024-01-31 23:59:59.123456789"), &raw_value).ok()); - TimeStampNsValue value(raw_value); - - ASSERT_TRUE(value.date_add_interval(TimeInterval(TimeUnit::MONTH, 1, false))); - EXPECT_EQ(value.to_string(9), "2024-02-29 23:59:59.123456789"); - - ASSERT_TRUE( - value.date_add_interval(TimeInterval(TimeUnit::SECOND, 1, false))); - EXPECT_EQ(value.to_string(9), "2024-03-01 00:00:00.123456789"); -} - -TEST(DataTypeTimeStampNsTest, DiffTruncatesIncompleteUnitsTowardZero) { - int64_t lhs_raw = 0; - int64_t rhs_raw = 0; - ASSERT_TRUE(parse_timestamp_ns(StringRef("1970-01-01 00:00:00.000000001"), &lhs_raw).ok()); - ASSERT_TRUE(parse_timestamp_ns(StringRef("1970-01-01 00:00:01.999999999"), &rhs_raw).ok()); - const TimeStampNsValue lhs(lhs_raw); - const TimeStampNsValue rhs(rhs_raw); - - EXPECT_EQ(datetime_diff(lhs, rhs), 1); - EXPECT_EQ(datetime_diff(rhs, lhs), -1); - EXPECT_EQ(datetime_diff(lhs, rhs), 1999); - EXPECT_EQ(datetime_diff(rhs, lhs), -1999); - EXPECT_EQ(lhs.datetime_diff_in_microseconds(rhs), -1999999); - EXPECT_EQ(rhs.datetime_diff_in_microseconds(lhs), 1999999); + EXPECT_EQ(round_trip.to_date_int_val(), civil.to_date_int_val()); } TEST(DataTypeTimeStampNsTest, FactoryKeepsTimestampNsSeparateFromDateTimeV2) { @@ -353,7 +341,7 @@ TEST(DataTypeTimeStampNsTest, DecodedTimestampUnitsAndValidation) { } } -TEST(DataTypeTimeStampNsTest, DataTypeLiteralFieldAndCivilCasts) { +TEST(DataTypeTimeStampNsTest, DataTypeLiteralField) { const DataTypeTimeStampNs type; const DataTypeDateTimeV2 legacy6(6); @@ -376,102 +364,22 @@ TEST(DataTypeTimeStampNsTest, DataTypeLiteralFieldAndCivilCasts) { EXPECT_EQ(field_with_type.base_scalar_type_id, TYPE_TIMESTAMP_NS); EXPECT_EQ(field_with_type.precision, -1); EXPECT_EQ(field_with_type.scale, 9); - - DateV2Value date; - date.unchecked_set_time(2024, 2, 29, 0, 0, 0, 0); - TimeStampNsValue nano_from_date; - DataTypeDateV2::cast_to_date_time_v2(date, nano_from_date); - EXPECT_EQ(nano_from_date.to_string(9), "2024-02-29 00:00:00.000000000"); - - VecDateTimeValue date_v1; - VecDateTimeValue datetime_v1; - DateV2Value date_v2; - DataTypeDateTimeV2::cast_to_date(field.get(), date_v1); - DataTypeDateTimeV2::cast_to_date_time(field.get(), datetime_v1); - DataTypeDateTimeV2::cast_to_date_v2(field.get(), date_v2); - char date_v1_text[64] = {}; - char datetime_v1_text[64] = {}; - date_v1.to_string(date_v1_text); - datetime_v1.to_string(datetime_v1_text); - EXPECT_STREQ(date_v1_text, "2024-02-29"); - EXPECT_STREQ(datetime_v1_text, "2024-02-29 12:34:56"); - EXPECT_EQ(date_v2.to_string(), "2024-02-29"); } -TEST(DataTypeTimeStampNsTest, CalendarHelpersArithmeticAndHash) { +TEST(DataTypeTimeStampNsTest, FormattingAndHash) { int64_t raw = 0; ASSERT_TRUE(parse_timestamp_ns(StringRef("2024-02-29 12:34:56.123456789"), &raw).ok()); TimeStampNsValue value(raw); - EXPECT_EQ(value.quarter(), 1); - EXPECT_GT(value.daynr(), 0); - EXPECT_GT(value.year_of_week(), 0); - EXPECT_GT(value.week(0), 0); - EXPECT_GT(value.year_week(0), 0); - EXPECT_EQ(value.day_of_year(), 60); - EXPECT_GT(value.day_of_week(), 0); - EXPECT_LT(value.weekday(), 7); - EXPECT_EQ(value.time_part_to_seconds(), 12 * 3600 + 34 * 60 + 56); - EXPECT_EQ(value.time_part_to_microsecond(), (12 * 3600 + 34 * 60 + 56) * 1000000LL + 123456); EXPECT_TRUE(value.is_valid_date()); - static const char* const day_names[] = {"Monday", "Tuesday", "Wednesday", "Thursday", - "Friday", "Saturday", "Sunday"}; - static const char* const month_names[] = { - "", "January", "February", "March", "April", "May", "June", - "July", "August", "September", "October", "November", "December"}; - EXPECT_STREQ(value.day_name_with_locale(day_names), "Thursday"); - EXPECT_STREQ(value.month_name_with_locale(month_names), "February"); - - char formatted[64] = {}; - constexpr char format[] = "%Y-%m-%d %H:%i:%s.%f"; - ASSERT_TRUE(value.to_format_string_conservative(format, std::strlen(format), formatted, - sizeof(formatted))); - EXPECT_STREQ(formatted, "2024-02-29 12:34:56.123456"); - char text[40] = {}; const char* end = value.to_string(text, 9); EXPECT_STREQ(text, "2024-02-29 12:34:56.123456789"); EXPECT_EQ(end, text + std::strlen(text) + 1); - TimeStampNsValue adjusted = value; - adjusted += 2; - EXPECT_EQ(adjusted.epoch_nanos(), value.epoch_nanos() + 2000000000LL); - adjusted -= 3; - EXPECT_EQ(adjusted.epoch_nanos(), value.epoch_nanos() - 1000000000LL); EXPECT_EQ(value.hash(17), value.hash(17)); EXPECT_EQ(std::hash {}(value), std::hash {}(value.epoch_nanos())); - - int64_t earlier_raw = 0; - int64_t later_raw = 0; - ASSERT_TRUE(parse_timestamp_ns(StringRef("2020-01-31 12:00:00.900000009"), &earlier_raw).ok()); - ASSERT_TRUE(parse_timestamp_ns(StringRef("2022-03-31 11:00:00.100000001"), &later_raw).ok()); - const TimeStampNsValue earlier(earlier_raw); - const TimeStampNsValue later(later_raw); - - EXPECT_EQ(datetime_diff(earlier, later), 2); - EXPECT_EQ(datetime_diff(later, earlier), -2); - EXPECT_EQ(datetime_diff(earlier, later), 25); - EXPECT_EQ(datetime_diff(later, earlier), -25); - EXPECT_EQ(datetime_diff(earlier, later), 8); - EXPECT_EQ(datetime_diff(earlier, later), 112); - EXPECT_EQ(datetime_diff(earlier, later), 789); - EXPECT_EQ(datetime_diff(later, earlier), -789); - EXPECT_EQ(earlier.date_diff_in_days(later), -790); - EXPECT_EQ(later.date_diff_in_days_round_to_zero_by_time(earlier), 789); - EXPECT_EQ(earlier.date_diff_in_days_round_to_zero_by_time(later), -789); - EXPECT_EQ(later.datetime_diff_in_seconds_round_to_zero_by_ms(earlier), - -earlier.datetime_diff_in_seconds_round_to_zero_by_ms(later)); - - const auto legacy = later.to_datetime(); - EXPECT_EQ(earlier.time_part_diff_in_ms(legacy), 3600800000LL); - EXPECT_EQ(earlier.datetime_diff_in_microseconds(legacy), - earlier.datetime_diff_in_seconds(legacy) * 1000000LL + - earlier.time_part_diff_in_ms(legacy) % 1000000LL); - - TimeStampNsValue truncated = value; - ASSERT_TRUE(truncated.datetime_trunc()); - EXPECT_EQ(truncated.to_string(9), "2024-02-29 00:00:00.000000000"); } TEST(DataTypeTimeStampNsTest, SerDeStrictBatchJsonJsonbMysqlAndBinaryField) { diff --git a/be/test/exec/sink/tablet_sink_hash_partitioner_test.cpp b/be/test/exec/sink/tablet_sink_hash_partitioner_test.cpp index f280b202fc8c50..e1e4e7b7716f71 100644 --- a/be/test/exec/sink/tablet_sink_hash_partitioner_test.cpp +++ b/be/test/exec/sink/tablet_sink_hash_partitioner_test.cpp @@ -340,7 +340,7 @@ TEST(TabletSinkHashPartitionerTest, OlapTabletFinderRoundRobinEveryBatch) { } } -TEST(TabletSinkHashPartitionerTest, TimeStampNsRangeUsesPartitionColumnScale) { +TEST(TabletSinkHashPartitionerTest, TimeStampNsRangePartitionKey) { OperatorContext ctx; TOlapTableSchemaParam tschema; diff --git a/be/test/exprs/function/function_test_util.cpp b/be/test/exprs/function/function_test_util.cpp index d3662c1ec627aa..b37e452b69a52a 100644 --- a/be/test/exprs/function/function_test_util.cpp +++ b/be/test/exprs/function/function_test_util.cpp @@ -44,7 +44,6 @@ #include "core/data_type/data_type_struct.h" #include "core/data_type/data_type_time.h" #include "core/data_type/data_type_varbinary.h" -#include "core/data_type_serde/data_type_timestamp_ns_serde.h" #include "core/field.h" #include "core/types.h" #include "core/value/bitmap_value.h" @@ -195,11 +194,7 @@ static size_t type_index_to_data_type(const std::vector& input_types, s desc = type; return 1; case PrimitiveType::TYPE_DATETIMEV2: - type = create_datetimev2(input_types[index].scale_or(0)); - desc = type; - return 1; - case PrimitiveType::TYPE_TIMESTAMP_NS: - type = std::make_shared(); + type = std::make_shared(input_types[index].scale_or(0)); desc = type; return 1; case PrimitiveType::TYPE_TIMEV2: @@ -323,12 +318,7 @@ bool insert_datetime_cell(MutableColumnPtr& column, DataTypePtr date_type_ptr, c auto datetime_str = any_cast(cell); CastParameters params; - if constexpr (PType == PrimitiveType::TYPE_TIMESTAMP_NS) { - int64_t epoch_nanos = 0; - result = parse_timestamp_ns({datetime_str.c_str(), datetime_str.size()}, &epoch_nanos) - .ok(); - date_value = TimeStampNsValue(epoch_nanos); - } else if constexpr (PType == PrimitiveType::TYPE_DATETIMEV2) { + if constexpr (PType == PrimitiveType::TYPE_DATETIMEV2) { result = CastToDatetimeV2::from_string_non_strict_mode( {datetime_str.c_str(), datetime_str.size()}, date_value, nullptr, date_type_ptr->get_scale(), params); @@ -539,11 +529,6 @@ bool insert_cell(MutableColumnPtr& column, DataTypePtr type_ptr, const AnyType& column, type_ptr, cell, datetime_is_string_format))); break; } - case PrimitiveType::TYPE_TIMESTAMP_NS: { - RETURN_IF_FALSE((insert_datetime_cell( - column, type_ptr, cell, datetime_is_string_format))); - break; - } case PrimitiveType::TYPE_TIMEV2: { TimeValue::TimeType time_value = 0; if (datetime_is_string_format) { diff --git a/be/test/exprs/function/function_test_util.h b/be/test/exprs/function/function_test_util.h index b486037a5d44d9..ebbb9c2f77c486 100644 --- a/be/test/exprs/function/function_test_util.h +++ b/be/test/exprs/function/function_test_util.h @@ -50,7 +50,6 @@ #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" #include "core/data_type/data_type_time.h" -#include "core/data_type/data_type_timestamp_ns.h" #include "core/data_type/data_type_varbinary.h" #include "core/extended_types.h" #include "core/types.h" @@ -377,9 +376,6 @@ Status check_function(const std::string& func_name, const InputTypeSet& input_ty return ResultNullable ? make_nullable(std::make_shared(result_precision, result_scale)) : std::make_shared(result_precision, result_scale); - } else if constexpr (std::is_same_v) { - return ResultNullable ? make_nullable(std::make_shared()) - : std::make_shared(); } else if constexpr (IsDataTypeDateTimeV2 || IsTimeV2Type) { // datetimev2 or timev2 UInt32 real_scale = 0; @@ -432,9 +428,6 @@ Status check_function(const std::string& func_name, const InputTypeSet& input_ty ? make_nullable(std::make_shared(result_precision, result_scale)) : std::make_shared(result_precision, result_scale); - } else if constexpr (std::is_same_v) { - result_type_ptr = ResultNullable ? make_nullable(std::make_shared()) - : std::make_shared(); } else if constexpr (IsDataTypeDateTimeV2 || IsTimeV2Type) { // datetimev2 UInt32 real_scale = 0; diff --git a/be/test/exprs/function/function_time_test.cpp b/be/test/exprs/function/function_time_test.cpp index bc7e2d75dff1dd..e012c7784e97ea 100644 --- a/be/test/exprs/function/function_time_test.cpp +++ b/be/test/exprs/function/function_time_test.cpp @@ -26,7 +26,6 @@ #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_time.h" -#include "core/data_type/data_type_timestamp_ns.h" #include "core/types.h" #include "core/value/time_value.h" #include "core/value/vdatetime_value.h" @@ -268,17 +267,6 @@ TEST(VTimestampFunctionsTest, unix_timestamp_test) { check_function(func_name, input_types, data_set, 6, 18)); } - { - InputTypeSet input_types = {PrimitiveType::TYPE_TIMESTAMP_NS}; - DataSet data_set = { - {{std::string("1970-01-01 08:00:00.123456789")}, DECIMAL128V3(0, 123456789, 9)}, - {{std::string("2022-05-24 12:34:56.789123456")}, - DECIMAL128V3(1653366896, 789123456, 9)}, - }; - static_cast( - check_function(func_name, input_types, data_set, 9, 21)); - } - // test out of range { InputTypeSet input_types = {{PrimitiveType::TYPE_DATETIMEV2, 6}}; @@ -355,23 +343,6 @@ TEST(VTimestampFunctionsTest, convert_tz_test) { data_set, 6)); } - { - InputTypeSet input_types_timestamp_ns = {PrimitiveType::TYPE_TIMESTAMP_NS, - PrimitiveType::TYPE_VARCHAR, - PrimitiveType::TYPE_VARCHAR}; - DataSet data_set = {{{std::string {"1970-01-01 08:00:00.123456789"}, - std::string {"Asia/Shanghai"}, std::string {"UTC"}}, - std::string("1970-01-01 00:00:00.123456789")}, - {{std::string {"2021-03-28 02:15:30.123456789"}, - std::string {"Europe/Paris"}, std::string {"UTC"}}, - std::string("2021-03-28 01:00:00.000000000")}, - {{std::string {"2021-03-28 03:00:30.123456789"}, - std::string {"Europe/Paris"}, std::string {"UTC"}}, - std::string("2021-03-28 01:00:30.123456789")}}; - static_cast(check_function( - func_name, input_types_timestamp_ns, data_set)); - } - { DataSet data_set = {{{std::string {"2019-08-01 02:18:27"}, std::string {"Asia/Shanghai"}, std::string {"UTC"}}, @@ -1478,22 +1449,6 @@ TEST(VTimestampFunctionsTest, dayname_test) { TEST(VTimestampFunctionsTest, datetrunc_test) { std::string func_name = "date_trunc"; - { - InputTypeSet input_types = {AnyType {PrimitiveType::TYPE_TIMESTAMP_NS, 9}, - Consted {PrimitiveType::TYPE_VARCHAR}}; - DataSet data_set = {{{std::string("2023-10-17 12:34:56.123456789"), std::string("day")}, - std::string("2023-10-17 00:00:00.000000000")}}; - static_cast( - check_function(func_name, input_types, data_set, 9)); - } - { - InputTypeSet input_types = {Consted {PrimitiveType::TYPE_VARCHAR}, - AnyType {PrimitiveType::TYPE_TIMESTAMP_NS, 9}}; - DataSet data_set = {{{std::string("day"), std::string("2023-10-17 12:34:56.123456789")}, - std::string("2023-10-17 00:00:00.000000000")}}; - static_cast( - check_function(func_name, input_types, data_set, 9)); - } { InputTypeSet input_types = {PrimitiveType::TYPE_DATETIMEV2, Consted {PrimitiveType::TYPE_VARCHAR}}; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java index db6ccea6564773..5aaa0d602922eb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java @@ -145,7 +145,7 @@ private static Literal getDateTimeLiteral(String value, Type type) throws Analys if (type.isDatetime()) { return new DateTimeLiteral(value); } else if (type.isDatetimeV2()) { - return DateTimeV2Literal.create((DateTimeV2Type) DataType.fromCatalogType(type), value); + return new DateTimeV2Literal((DateTimeV2Type) DataType.fromCatalogType(type), value); } else if (type.isTimeStampNs()) { return new TimeStampNsLiteral(value); } else if (type.isTimeStampTz()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/connector/converter/ExprToConnectorExpressionConverter.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/connector/converter/ExprToConnectorExpressionConverter.java index 36cd21e3194084..b79b6f72c1b8eb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/connector/converter/ExprToConnectorExpressionConverter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/connector/converter/ExprToConnectorExpressionConverter.java @@ -37,6 +37,7 @@ import org.apache.doris.analysis.NullLiteral; import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.StringLiteral; +import org.apache.doris.analysis.TimeStampNsLiteral; import org.apache.doris.analysis.ToSqlParams; import org.apache.doris.catalog.ArrayType; import org.apache.doris.catalog.MapType; @@ -299,6 +300,8 @@ private static ConnectorExpression convertLiteral(LiteralExpr literal) { return new ConnectorLiteral(ct, ((DecimalLiteral) literal).getValue()); } else if (literal instanceof StringLiteral) { return new ConnectorLiteral(ct, ((StringLiteral) literal).getValue()); + } else if (literal instanceof TimeStampNsLiteral) { + return new ConnectorLiteral(ct, ((TimeStampNsLiteral) literal).toLocalDateTime()); } else if (literal instanceof DateLiteral) { return convertDateLiteral((DateLiteral) literal, ct); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java index 96e0e09f987afe..5db22ad2c7136b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java @@ -712,7 +712,7 @@ private static Literal createDateTimeLikeLiteral( if (type instanceof TimeStampNsType) { return TimeStampNsLiteral.fromJavaDateType(dateTime); } - return DateTimeV2Literal.create(type, dateTime.getYear(), dateTime.getMonthValue(), + return new DateTimeV2Literal(type, dateTime.getYear(), dateTime.getMonthValue(), dateTime.getDayOfMonth(), dateTime.getHour(), dateTime.getMinute(), dateTime.getSecond(), dateTime.getNano() / 1000); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyInPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyInPredicate.java index 003b0ed7299c76..1176881719b455 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyInPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyInPredicate.java @@ -73,7 +73,7 @@ && canLosslessConvertToLowScaleLiteral( (DateTimeV2Literal) literal, compareType.getScale()))) { ImmutableList.Builder children = ImmutableList.builder(); children.add(cast.child()); - literals.forEach(l -> children.add(DateTimeV2Literal.create(compareType, + literals.forEach(l -> children.add(new DateTimeV2Literal(compareType, ((DateTimeV2Literal) l).getStringValue()))); return expr.withChildren(children.build()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ComputeSignatureHelper.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ComputeSignatureHelper.java index 0d69200677d551..0d0abe92789834 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ComputeSignatureHelper.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ComputeSignatureHelper.java @@ -528,24 +528,19 @@ private static FunctionSignature defaultTimePrecisionPromotion(FunctionSignature DataType inputType = nestedInputTypes.get(j); // corresponding target slot type for inputType DataType nestedTargetType = nestedTargetTypes.get(j); - ScaleTimeType promotedType; + int targetScale = 0; // for string input, try to get the most suitable scale if (arg instanceof StringLikeLiteral) { ScaleTimeType timelikeType = (ScaleTimeType) nestedTargetType; - promotedType = timelikeType.forTypeFromString((StringLikeLiteral) arg); + targetScale = timelikeType.forTypeFromString((StringLikeLiteral) arg).getScale(); } else { // for all other input types, get the target scale when cast it to targetType ScaleTimeType targetScaleType = (ScaleTimeType) nestedTargetType; - promotedType = targetScaleType.scaleTypeForType(inputType); + ScaleTimeType promotedType = targetScaleType.scaleTypeForType(inputType); + targetScale = promotedType.getScale(); } - int targetScale = promotedType.getScale(); - // TimeStampNsType inherits DateTimeV2Type only to reuse civil-time behavior. A - // DATETIMEV2 signature must stay within its own precision and physical layout. - if (nestedTargetType.getClass() == DateTimeV2Type.class) { - targetScale = Math.min(targetScale, DateTimeV2Type.MAX_SCALE); - } finalTypeScale = Math.max(finalTypeScale, targetScale); // init value -1 always promotes } } @@ -557,8 +552,7 @@ private static FunctionSignature defaultTimePrecisionPromotion(FunctionSignature // promote the precision of return type ImmutableList.Builder newArgTypesBuilder = ImmutableList.builderWithExpectedSize(signature.arity); for (DataType signatureArgType : signature.argumentsTypes) { - newArgTypesBuilder.add(TypeCoercionUtils.replaceTimesWithTargetPrecision( - signatureArgType, finalTypeScale)); + newArgTypesBuilder.add(TypeCoercionUtils.replaceTimesWithTargetPrecision(signatureArgType, finalTypeScale)); } List newArgTypes = newArgTypesBuilder.build(); signature = signature.withArgumentTypes(signature.hasVarArgs, newArgTypes); @@ -566,8 +560,7 @@ private static FunctionSignature defaultTimePrecisionPromotion(FunctionSignature || signature.returnType instanceof TimeStampTzType || signature.returnType instanceof ComplexDataType) { signature = signature.withReturnType( - TypeCoercionUtils.replaceTimesWithTargetPrecision( - signature.returnType, finalTypeScale)); + TypeCoercionUtils.replaceTimesWithTargetPrecision(signature.returnType, finalTypeScale)); } return signature; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/DateTimeWithPrecision.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/DateTimeWithPrecision.java index 87cf6e8e08662c..ab67c9e365a2e0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/DateTimeWithPrecision.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/DateTimeWithPrecision.java @@ -18,7 +18,6 @@ package org.apache.doris.nereids.trees.expressions.functions; import org.apache.doris.catalog.FunctionSignature; -import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.scalar.ScalarFunction; import org.apache.doris.nereids.trees.expressions.functions.scalar.ScalarFunctionParams; @@ -47,14 +46,9 @@ public FunctionSignature computeSignature(FunctionSignature signature) { // searching in FunctionSet. So we adjust the return type by hand here. if (getArgument(0) instanceof IntegerLikeLiteral) { IntegerLikeLiteral integerLikeLiteral = (IntegerLikeLiteral) getArgument(0); - int scale = integerLikeLiteral.getIntValue(); - if (scale < 0 || scale > DateTimeV2Type.MAX_SCALE) { - throw new AnalysisException("Scale of Datetime/Time must between 0 and 6. Scale was set to: " - + scale); - } - signature = signature.withReturnType(DateTimeV2Type.of(scale)); + signature = signature.withReturnType(DateTimeV2Type.of(integerLikeLiteral.getIntValue())); } else { - signature = signature.withReturnType(DateTimeV2Type.MAX); + signature = signature.withReturnType(DateTimeV2Type.of(6)); } } return super.computeSignature(signature); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/SequenceCount.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/SequenceCount.java index 33370f39ffabef..e4ad5956c1c290 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/SequenceCount.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/SequenceCount.java @@ -27,7 +27,6 @@ import org.apache.doris.nereids.types.DateTimeV2Type; import org.apache.doris.nereids.types.DateV2Type; import org.apache.doris.nereids.types.StringType; -import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TimeStampTzType; import org.apache.doris.nereids.util.ExpressionUtils; @@ -48,9 +47,7 @@ public class SequenceCount extends NotNullableAggregateFunction FunctionSignature.ret(BigIntType.INSTANCE) .varArgs(StringType.INSTANCE, TimeStampTzType.WILDCARD, BooleanType.INSTANCE), FunctionSignature.ret(BigIntType.INSTANCE) - .varArgs(StringType.INSTANCE, DateTimeV2Type.WILDCARD, BooleanType.INSTANCE), - FunctionSignature.ret(BigIntType.INSTANCE) - .varArgs(StringType.INSTANCE, TimeStampNsType.INSTANCE, BooleanType.INSTANCE) + .varArgs(StringType.INSTANCE, DateTimeV2Type.WILDCARD, BooleanType.INSTANCE) ); /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/SequenceMatch.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/SequenceMatch.java index cae83665551003..c9f15731a196ee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/SequenceMatch.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/SequenceMatch.java @@ -25,7 +25,6 @@ import org.apache.doris.nereids.types.DateTimeV2Type; import org.apache.doris.nereids.types.DateV2Type; import org.apache.doris.nereids.types.StringType; -import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TimeStampTzType; import org.apache.doris.nereids.util.ExpressionUtils; @@ -46,9 +45,7 @@ public class SequenceMatch extends NullableAggregateFunction FunctionSignature.ret(BooleanType.INSTANCE) .varArgs(StringType.INSTANCE, TimeStampTzType.WILDCARD, BooleanType.INSTANCE), FunctionSignature.ret(BooleanType.INSTANCE) - .varArgs(StringType.INSTANCE, DateTimeV2Type.WILDCARD, BooleanType.INSTANCE), - FunctionSignature.ret(BooleanType.INSTANCE) - .varArgs(StringType.INSTANCE, TimeStampNsType.INSTANCE, BooleanType.INSTANCE) + .varArgs(StringType.INSTANCE, DateTimeV2Type.WILDCARD, BooleanType.INSTANCE) ); /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/TopNWeighted.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/TopNWeighted.java index 332d36ded2b64a..9d3152685a4606 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/TopNWeighted.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/TopNWeighted.java @@ -36,7 +36,6 @@ import org.apache.doris.nereids.types.LargeIntType; import org.apache.doris.nereids.types.SmallIntType; import org.apache.doris.nereids.types.StringType; -import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TimeStampTzType; import org.apache.doris.nereids.types.TinyIntType; import org.apache.doris.nereids.types.VarcharType; @@ -78,8 +77,6 @@ public class TopNWeighted extends NullableAggregateFunction .args(TimeStampTzType.WILDCARD, BigIntType.INSTANCE, IntegerType.INSTANCE), FunctionSignature.ret(ArrayType.of(DateTimeV2Type.WILDCARD)) .args(DateTimeV2Type.WILDCARD, BigIntType.INSTANCE, IntegerType.INSTANCE), - FunctionSignature.ret(ArrayType.of(TimeStampNsType.INSTANCE)) - .args(TimeStampNsType.INSTANCE, BigIntType.INSTANCE, IntegerType.INSTANCE), FunctionSignature.ret(ArrayType.of(StringType.INSTANCE)) .args(StringType.INSTANCE, BigIntType.INSTANCE, IntegerType.INSTANCE), FunctionSignature.ret(ArrayType.of(VarcharType.SYSTEM_DEFAULT)) @@ -122,11 +119,6 @@ public class TopNWeighted extends NullableAggregateFunction BigIntType.INSTANCE, IntegerType.INSTANCE, IntegerType.INSTANCE), - FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT) - .args(TimeStampNsType.INSTANCE, - BigIntType.INSTANCE, - IntegerType.INSTANCE, - IntegerType.INSTANCE), FunctionSignature.ret(ArrayType.of(StringType.INSTANCE)) .args(StringType.INSTANCE, BigIntType.INSTANCE, IntegerType.INSTANCE, IntegerType.INSTANCE), FunctionSignature.ret(ArrayType.of(VarcharType.SYSTEM_DEFAULT)) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/WindowFunnel.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/WindowFunnel.java index 7ae384621d1dde..ef60c0a8c07d31 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/WindowFunnel.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/WindowFunnel.java @@ -29,7 +29,6 @@ import org.apache.doris.nereids.types.DateV2Type; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.StringType; -import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TimeStampTzType; import org.apache.doris.nereids.util.ExpressionUtils; @@ -51,9 +50,6 @@ public class WindowFunnel extends NullableAggregateFunction BooleanType.INSTANCE), FunctionSignature.ret(IntegerType.INSTANCE) .varArgs(BigIntType.INSTANCE, StringType.INSTANCE, DateTimeV2Type.WILDCARD, - BooleanType.INSTANCE), - FunctionSignature.ret(IntegerType.INSTANCE) - .varArgs(BigIntType.INSTANCE, StringType.INSTANCE, TimeStampNsType.INSTANCE, BooleanType.INSTANCE) ); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/WindowFunnelV2.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/WindowFunnelV2.java index 1c51948a483900..697774eb47f4b6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/WindowFunnelV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/WindowFunnelV2.java @@ -29,7 +29,6 @@ import org.apache.doris.nereids.types.DateV2Type; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.StringType; -import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TimeStampTzType; import org.apache.doris.nereids.util.ExpressionUtils; @@ -54,9 +53,6 @@ public class WindowFunnelV2 extends NullableAggregateFunction BooleanType.INSTANCE), FunctionSignature.ret(IntegerType.INSTANCE) .varArgs(BigIntType.INSTANCE, StringType.INSTANCE, DateTimeV2Type.WILDCARD, - BooleanType.INSTANCE), - FunctionSignature.ret(IntegerType.INSTANCE) - .varArgs(BigIntType.INSTANCE, StringType.INSTANCE, TimeStampNsType.INSTANCE, BooleanType.INSTANCE) ); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeArithmetic.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeArithmetic.java index ecf91b595f10db..cd12c00df210cf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeArithmetic.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeArithmetic.java @@ -268,7 +268,7 @@ private static Expression applyInterval(DateTimeV2Literal date, VarcharLiteral d DateTimeV2Literal dt = (DateTimeV2Literal) result; if (dt.getScale() != date.getScale()) { // Keep original scale to match existing folding behavior. - return DateTimeV2Literal.fromJavaDateType(dt.toJavaDateType(), date.getDataType()); + return DateTimeV2Literal.fromJavaDateType(dt.toJavaDateType(), date.getScale()); } } return result; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java index 7e2071baab1495..ca55370c0d780d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java @@ -46,7 +46,6 @@ import org.apache.doris.nereids.types.DateV2Type; import org.apache.doris.nereids.types.DecimalV3Type; import org.apache.doris.nereids.types.StringType; -import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TimeV2Type; import org.apache.doris.nereids.util.DateTimeFormatterUtils; import org.apache.doris.nereids.util.DateUtils; @@ -414,7 +413,7 @@ public static Expression date(DateTimeV2Literal dateTime) throws AnalysisExcepti @ExecFunction(name = "date_trunc") public static Expression dateTrunc(DateTimeV2Literal date, StringLikeLiteral trunc) { return DateTimeV2Literal.fromJavaDateType( - dateTruncHelper(date.toJavaDateType(), trunc.getValue()), date.getDataType()); + dateTruncHelper(date.toJavaDateType(), trunc.getValue()), date.getScale()); } @ExecFunction(name = "date_trunc") @@ -425,7 +424,7 @@ public static Expression dateTrunc(DateV2Literal date, StringLikeLiteral trunc) @ExecFunction(name = "date_trunc") public static Expression dateTrunc(StringLikeLiteral trunc, DateTimeV2Literal date) { return DateTimeV2Literal.fromJavaDateType( - dateTruncHelper(date.toJavaDateType(), trunc.getValue()), date.getDataType()); + dateTruncHelper(date.toJavaDateType(), trunc.getValue()), date.getScale()); } @ExecFunction(name = "date_trunc") @@ -616,7 +615,7 @@ public static Expression unixTimestamp(DateV2Literal date) { public static Expression unixTimestamp(DateTimeV2Literal date) { int scale = date.getDataType().getScale(); return new DecimalV3Literal(DecimalV3Type.createDecimalV3TypeLooseCheck(12 + scale, scale), - new BigDecimal(getTimestamp(date.toJavaDateType(), scale))); + new BigDecimal(getTimestamp(date.toJavaDateType()))); } /** @@ -638,10 +637,6 @@ public static Expression unixTimestamp(StringLikeLiteral date, StringLikeLiteral } private static String getTimestamp(LocalDateTime dateTime) { - return getTimestamp(dateTime, 6); - } - - private static String getTimestamp(LocalDateTime dateTime, int scale) { LocalDateTime specialLowerBound = LocalDateTime.of(1970, 1, 1, 0, 0, 0); dateTime = dateTime.atZone(DateUtils.getTimeZone()) .toOffsetDateTime().atZoneSameInstant(ZoneId.of("UTC+0")) @@ -656,8 +651,7 @@ private static String getTimestamp(LocalDateTime dateTime, int scale) { if (duration.getNano() == 0) { return String.valueOf(duration.getSeconds()); } else { - long fraction = duration.getNano() / (long) Math.pow(10, TimeStampNsType.SCALE - scale); - return duration.getSeconds() + "." + String.format("%0" + scale + "d", fraction); + return duration.getSeconds() + "." + String.format("%06d", duration.getNano() / 1000); } } @@ -831,9 +825,9 @@ public static Expression convertTz(DateTimeV2Literal datetime, StringLikeLiteral ZoneId toZone = ZoneId.from(zoneFormatter.parse(toTz.getStringValue())); LocalDateTime localDateTime = datetime.toJavaDateType(); - Instant instant = DateUtils.convertLocalToInstant(localDateTime, fromZone); + Instant instant = DateTimeLiteral.convertLocalToInstant(localDateTime, fromZone); return DateTimeV2Literal.fromJavaDateType(LocalDateTime.ofInstant(instant, toZone), - datetime.getDataType()); + datetime.getDataType().getScale()); } private static void validateTimezoneOffset(String timezone) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/TimeRoundSeries.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/TimeRoundSeries.java index d102ab2839dd59..c32f733bedbae5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/TimeRoundSeries.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/TimeRoundSeries.java @@ -104,16 +104,14 @@ private static LocalDateTime getDateCeilOrFloor(DATE tag, LocalDateTime date, in + (dt.getHour() - start.getHour()) * 60 * 60 + (dt.getMinute() - start.getMinute()) * 60 + (dt.getSecond() - start.getSecond()); - trivialPart = date.getNano() - origin.getNano(); + trivialPart = dt.getMicroSecond() - start.getMicroSecond(); break; } default: { return null; } } - // The coarser fields can be equal while the values still differ within the second. Use the full - // fractional precision so such values do not look aligned to a rounding boundary. - trivialPart = (trivialPart == 0 ? date.getNano() - origin.getNano() : trivialPart); + trivialPart = (trivialPart == 0 ? dt.getMicroSecond() - start.getMicroSecond() : trivialPart); if (getCeil) { diff = diff + (trivialPart > 0 ? 1 : 0); } else { @@ -239,25 +237,25 @@ public static Expression yearCeil(DateV2Literal date, IntegerLiteral period, Dat @ExecFunction(name = "year_ceil") public static Expression yearCeil(DateTimeV2Literal date) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.YEAR, date.toJavaDateType(), - 1, START_ORIGINAL_DAY, true), date.getDataType()); + 1, START_ORIGINAL_DAY, true), date.getScale()); } @ExecFunction(name = "year_ceil") public static Expression yearCeil(DateTimeV2Literal date, IntegerLiteral period) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.YEAR, date.toJavaDateType(), - period.getValue(), START_ORIGINAL_DAY, true), date.getDataType()); + period.getValue(), START_ORIGINAL_DAY, true), date.getScale()); } @ExecFunction(name = "year_ceil") public static Expression yearCeil(DateTimeV2Literal date, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.YEAR, date.toJavaDateType(), - 1, origin.toJavaDateType(), true), date.commonType(origin)); + 1, origin.toJavaDateType(), true), date.commonScale(origin)); } @ExecFunction(name = "year_ceil") public static Expression yearCeil(DateTimeV2Literal date, IntegerLiteral period, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.YEAR, date.toJavaDateType(), - period.getValue(), origin.toJavaDateType(), true), date.commonType(origin)); + period.getValue(), origin.toJavaDateType(), true), date.commonScale(origin)); } @ExecFunction(name = "year_ceil") @@ -314,25 +312,25 @@ public static Expression quarterCeil(DateV2Literal date, IntegerLiteral period, @ExecFunction(name = "quarter_ceil") public static Expression quarterCeil(DateTimeV2Literal date) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.QUARTER, date.toJavaDateType(), - 1, START_ORIGINAL_DAY, true), date.getDataType()); + 1, START_ORIGINAL_DAY, true), date.getScale()); } @ExecFunction(name = "quarter_ceil") public static Expression quarterCeil(DateTimeV2Literal date, IntegerLiteral period) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.QUARTER, date.toJavaDateType(), - period.getValue(), START_ORIGINAL_DAY, true), date.getDataType()); + period.getValue(), START_ORIGINAL_DAY, true), date.getScale()); } @ExecFunction(name = "quarter_ceil") public static Expression quarterCeil(DateTimeV2Literal date, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.QUARTER, date.toJavaDateType(), - 1, origin.toJavaDateType(), true), date.commonType(origin)); + 1, origin.toJavaDateType(), true), date.commonScale(origin)); } @ExecFunction(name = "quarter_ceil") public static Expression quarterCeil(DateTimeV2Literal date, IntegerLiteral period, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.QUARTER, date.toJavaDateType(), - period.getValue(), origin.toJavaDateType(), true), date.commonType(origin)); + period.getValue(), origin.toJavaDateType(), true), date.commonScale(origin)); } @ExecFunction(name = "quarter_ceil") @@ -389,25 +387,25 @@ public static Expression monthCeil(DateV2Literal date, IntegerLiteral period, Da @ExecFunction(name = "month_ceil") public static Expression monthCeil(DateTimeV2Literal date) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.MONTH, date.toJavaDateType(), - 1, START_ORIGINAL_DAY, true), date.getDataType()); + 1, START_ORIGINAL_DAY, true), date.getScale()); } @ExecFunction(name = "month_ceil") public static Expression monthCeil(DateTimeV2Literal date, IntegerLiteral period) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.MONTH, date.toJavaDateType(), - period.getValue(), START_ORIGINAL_DAY, true), date.getDataType()); + period.getValue(), START_ORIGINAL_DAY, true), date.getScale()); } @ExecFunction(name = "month_ceil") public static Expression monthCeil(DateTimeV2Literal date, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.MONTH, date.toJavaDateType(), - 1, origin.toJavaDateType(), true), date.commonType(origin)); + 1, origin.toJavaDateType(), true), date.commonScale(origin)); } @ExecFunction(name = "month_ceil") public static Expression monthCeil(DateTimeV2Literal date, IntegerLiteral period, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.MONTH, date.toJavaDateType(), - period.getValue(), origin.toJavaDateType(), true), date.commonType(origin)); + period.getValue(), origin.toJavaDateType(), true), date.commonScale(origin)); } @ExecFunction(name = "month_ceil") @@ -464,25 +462,25 @@ public static Expression dayCeil(DateV2Literal date, IntegerLiteral period, Date @ExecFunction(name = "day_ceil") public static Expression dayCeil(DateTimeV2Literal date) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.DAY, date.toJavaDateType(), - 1, START_ORIGINAL_DAY, true), date.getDataType()); + 1, START_ORIGINAL_DAY, true), date.getScale()); } @ExecFunction(name = "day_ceil") public static Expression dayCeil(DateTimeV2Literal date, IntegerLiteral period) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.DAY, date.toJavaDateType(), - period.getValue(), START_ORIGINAL_DAY, true), date.getDataType()); + period.getValue(), START_ORIGINAL_DAY, true), date.getScale()); } @ExecFunction(name = "day_ceil") public static Expression dayCeil(DateTimeV2Literal date, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.DAY, date.toJavaDateType(), - 1, origin.toJavaDateType(), true), date.commonType(origin)); + 1, origin.toJavaDateType(), true), date.commonScale(origin)); } @ExecFunction(name = "day_ceil") public static Expression dayCeil(DateTimeV2Literal date, IntegerLiteral period, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.DAY, date.toJavaDateType(), - period.getValue(), origin.toJavaDateType(), true), date.commonType(origin)); + period.getValue(), origin.toJavaDateType(), true), date.commonScale(origin)); } @ExecFunction(name = "day_ceil") @@ -539,25 +537,25 @@ public static Expression hourCeil(DateV2Literal date, IntegerLiteral period, Dat @ExecFunction(name = "hour_ceil") public static Expression hourCeil(DateTimeV2Literal date) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.HOUR, date.toJavaDateType(), - 1, START_ORIGINAL_DAY, true), date.getDataType()); + 1, START_ORIGINAL_DAY, true), date.getScale()); } @ExecFunction(name = "hour_ceil") public static Expression hourCeil(DateTimeV2Literal date, IntegerLiteral period) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.HOUR, date.toJavaDateType(), - period.getValue(), START_ORIGINAL_DAY, true), date.getDataType()); + period.getValue(), START_ORIGINAL_DAY, true), date.getScale()); } @ExecFunction(name = "hour_ceil") public static Expression hourCeil(DateTimeV2Literal date, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.HOUR, date.toJavaDateType(), - 1, origin.toJavaDateType(), true), date.commonType(origin)); + 1, origin.toJavaDateType(), true), date.commonScale(origin)); } @ExecFunction(name = "hour_ceil") public static Expression hourCeil(DateTimeV2Literal date, IntegerLiteral period, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.HOUR, date.toJavaDateType(), - period.getValue(), origin.toJavaDateType(), true), date.commonType(origin)); + period.getValue(), origin.toJavaDateType(), true), date.commonScale(origin)); } @ExecFunction(name = "hour_ceil") @@ -614,26 +612,26 @@ public static Expression minuteCeil(DateV2Literal date, IntegerLiteral period, D @ExecFunction(name = "minute_ceil") public static Expression minuteCeil(DateTimeV2Literal date) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.MINUTE, date.toJavaDateType(), - 1, START_ORIGINAL_DAY, true), date.getDataType()); + 1, START_ORIGINAL_DAY, true), date.getScale()); } @ExecFunction(name = "minute_ceil") public static Expression minuteCeil(DateTimeV2Literal date, IntegerLiteral period) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.MINUTE, date.toJavaDateType(), - period.getValue(), START_ORIGINAL_DAY, true), date.getDataType()); + period.getValue(), START_ORIGINAL_DAY, true), date.getScale()); } @ExecFunction(name = "minute_ceil") public static Expression minuteCeil(DateTimeV2Literal date, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.MINUTE, date.toJavaDateType(), - 1, origin.toJavaDateType(), true), date.commonType(origin)); + 1, origin.toJavaDateType(), true), date.commonScale(origin)); } @ExecFunction(name = "minute_ceil") public static Expression minuteCeil(DateTimeV2Literal date, IntegerLiteral period, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.MINUTE, date.toJavaDateType(), - period.getValue(), origin.toJavaDateType(), true), date.commonType(origin)); + period.getValue(), origin.toJavaDateType(), true), date.commonScale(origin)); } @ExecFunction(name = "minute_ceil") @@ -691,26 +689,26 @@ public static Expression secondCeil(DateV2Literal date, IntegerLiteral period, D @ExecFunction(name = "second_ceil") public static Expression secondCeil(DateTimeV2Literal date) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.SECOND, date.toJavaDateType(), - 1, START_ORIGINAL_DAY, true), date.getDataType()); + 1, START_ORIGINAL_DAY, true), date.getScale()); } @ExecFunction(name = "second_ceil") public static Expression secondCeil(DateTimeV2Literal date, IntegerLiteral period) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.SECOND, date.toJavaDateType(), - period.getValue(), START_ORIGINAL_DAY, true), date.getDataType()); + period.getValue(), START_ORIGINAL_DAY, true), date.getScale()); } @ExecFunction(name = "second_ceil") public static Expression secondCeil(DateTimeV2Literal date, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.SECOND, date.toJavaDateType(), - 1, origin.toJavaDateType(), true), date.commonType(origin)); + 1, origin.toJavaDateType(), true), date.commonScale(origin)); } @ExecFunction(name = "second_ceil") public static Expression secondCeil(DateTimeV2Literal date, IntegerLiteral period, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.SECOND, date.toJavaDateType(), - period.getValue(), origin.toJavaDateType(), true), date.commonType(origin)); + period.getValue(), origin.toJavaDateType(), true), date.commonScale(origin)); } @ExecFunction(name = "second_ceil") @@ -768,25 +766,25 @@ public static Expression yearFloor(DateV2Literal date, IntegerLiteral period, Da @ExecFunction(name = "year_floor") public static Expression yearFloor(DateTimeV2Literal date) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.YEAR, date.toJavaDateType(), - 1, START_ORIGINAL_DAY, false), date.getDataType()); + 1, START_ORIGINAL_DAY, false), date.getScale()); } @ExecFunction(name = "year_floor") public static Expression yearFloor(DateTimeV2Literal date, IntegerLiteral period) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.YEAR, date.toJavaDateType(), - period.getValue(), START_ORIGINAL_DAY, false), date.getDataType()); + period.getValue(), START_ORIGINAL_DAY, false), date.getScale()); } @ExecFunction(name = "year_floor") public static Expression yearFloor(DateTimeV2Literal date, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.YEAR, date.toJavaDateType(), - 1, origin.toJavaDateType(), false), date.commonType(origin)); + 1, origin.toJavaDateType(), false), date.commonScale(origin)); } @ExecFunction(name = "year_floor") public static Expression yearFloor(DateTimeV2Literal date, IntegerLiteral period, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.YEAR, date.toJavaDateType(), - period.getValue(), origin.toJavaDateType(), false), date.commonType(origin)); + period.getValue(), origin.toJavaDateType(), false), date.commonScale(origin)); } @ExecFunction(name = "year_floor") @@ -843,26 +841,26 @@ public static Expression quarterFloor(DateV2Literal date, IntegerLiteral period, @ExecFunction(name = "quarter_floor") public static Expression quarterFloor(DateTimeV2Literal date) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.QUARTER, date.toJavaDateType(), - 1, START_ORIGINAL_DAY, false), date.getDataType()); + 1, START_ORIGINAL_DAY, false), date.getScale()); } @ExecFunction(name = "quarter_floor") public static Expression quarterFloor(DateTimeV2Literal date, IntegerLiteral period) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.QUARTER, date.toJavaDateType(), - period.getValue(), START_ORIGINAL_DAY, false), date.getDataType()); + period.getValue(), START_ORIGINAL_DAY, false), date.getScale()); } @ExecFunction(name = "quarter_floor") public static Expression quarterFloor(DateTimeV2Literal date, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.QUARTER, date.toJavaDateType(), - 1, origin.toJavaDateType(), false), date.commonType(origin)); + 1, origin.toJavaDateType(), false), date.commonScale(origin)); } @ExecFunction(name = "quarter_floor") public static Expression quarterFloor(DateTimeV2Literal date, IntegerLiteral period, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.QUARTER, date.toJavaDateType(), - period.getValue(), origin.toJavaDateType(), false), date.commonType(origin)); + period.getValue(), origin.toJavaDateType(), false), date.commonScale(origin)); } @ExecFunction(name = "quarter_floor") @@ -920,26 +918,26 @@ public static Expression monthFloor(DateV2Literal date, IntegerLiteral period, D @ExecFunction(name = "month_floor") public static Expression monthFloor(DateTimeV2Literal date) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.MONTH, date.toJavaDateType(), - 1, START_ORIGINAL_DAY, false), date.getDataType()); + 1, START_ORIGINAL_DAY, false), date.getScale()); } @ExecFunction(name = "month_floor") public static Expression monthFloor(DateTimeV2Literal date, IntegerLiteral period) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.MONTH, date.toJavaDateType(), - period.getValue(), START_ORIGINAL_DAY, false), date.getDataType()); + period.getValue(), START_ORIGINAL_DAY, false), date.getScale()); } @ExecFunction(name = "month_floor") public static Expression monthFloor(DateTimeV2Literal date, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.MONTH, date.toJavaDateType(), - 1, origin.toJavaDateType(), false), date.commonType(origin)); + 1, origin.toJavaDateType(), false), date.commonScale(origin)); } @ExecFunction(name = "month_floor") public static Expression monthFloor(DateTimeV2Literal date, IntegerLiteral period, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.MONTH, date.toJavaDateType(), - period.getValue(), origin.toJavaDateType(), false), date.commonType(origin)); + period.getValue(), origin.toJavaDateType(), false), date.commonScale(origin)); } @ExecFunction(name = "month_floor") @@ -997,25 +995,25 @@ public static Expression dayFloor(DateV2Literal date, IntegerLiteral period, Dat @ExecFunction(name = "day_floor") public static Expression dayFloor(DateTimeV2Literal date) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.DAY, date.toJavaDateType(), - 1, START_ORIGINAL_DAY, false), date.getDataType()); + 1, START_ORIGINAL_DAY, false), date.getScale()); } @ExecFunction(name = "day_floor") public static Expression dayFloor(DateTimeV2Literal date, IntegerLiteral period) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.DAY, date.toJavaDateType(), - period.getValue(), START_ORIGINAL_DAY, false), date.getDataType()); + period.getValue(), START_ORIGINAL_DAY, false), date.getScale()); } @ExecFunction(name = "day_floor") public static Expression dayFloor(DateTimeV2Literal date, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.DAY, date.toJavaDateType(), - 1, origin.toJavaDateType(), false), date.commonType(origin)); + 1, origin.toJavaDateType(), false), date.commonScale(origin)); } @ExecFunction(name = "day_floor") public static Expression dayFloor(DateTimeV2Literal date, IntegerLiteral period, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.DAY, date.toJavaDateType(), - period.getValue(), origin.toJavaDateType(), false), date.commonType(origin)); + period.getValue(), origin.toJavaDateType(), false), date.commonScale(origin)); } @ExecFunction(name = "day_floor") @@ -1072,25 +1070,25 @@ public static Expression hourFloor(DateV2Literal date, IntegerLiteral period, Da @ExecFunction(name = "hour_floor") public static Expression hourFloor(DateTimeV2Literal date) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.HOUR, date.toJavaDateType(), - 1, START_ORIGINAL_DAY, false), date.getDataType()); + 1, START_ORIGINAL_DAY, false), date.getScale()); } @ExecFunction(name = "hour_floor") public static Expression hourFloor(DateTimeV2Literal date, IntegerLiteral period) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.HOUR, date.toJavaDateType(), - period.getValue(), START_ORIGINAL_DAY, false), date.getDataType()); + period.getValue(), START_ORIGINAL_DAY, false), date.getScale()); } @ExecFunction(name = "hour_floor") public static Expression hourFloor(DateTimeV2Literal date, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.HOUR, date.toJavaDateType(), - 1, origin.toJavaDateType(), false), date.commonType(origin)); + 1, origin.toJavaDateType(), false), date.commonScale(origin)); } @ExecFunction(name = "hour_floor") public static Expression hourFloor(DateTimeV2Literal date, IntegerLiteral period, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.HOUR, date.toJavaDateType(), - period.getValue(), origin.toJavaDateType(), false), date.commonType(origin)); + period.getValue(), origin.toJavaDateType(), false), date.commonScale(origin)); } @ExecFunction(name = "hour_floor") @@ -1147,26 +1145,26 @@ public static Expression minuteFloor(DateV2Literal date, IntegerLiteral period, @ExecFunction(name = "minute_floor") public static Expression minuteFloor(DateTimeV2Literal date) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.MINUTE, date.toJavaDateType(), - 1, START_ORIGINAL_DAY, false), date.getDataType()); + 1, START_ORIGINAL_DAY, false), date.getScale()); } @ExecFunction(name = "minute_floor") public static Expression minuteFloor(DateTimeV2Literal date, IntegerLiteral period) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.MINUTE, date.toJavaDateType(), - period.getValue(), START_ORIGINAL_DAY, false), date.getDataType()); + period.getValue(), START_ORIGINAL_DAY, false), date.getScale()); } @ExecFunction(name = "minute_floor") public static Expression minuteFloor(DateTimeV2Literal date, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.MINUTE, date.toJavaDateType(), - 1, origin.toJavaDateType(), false), date.commonType(origin)); + 1, origin.toJavaDateType(), false), date.commonScale(origin)); } @ExecFunction(name = "minute_floor") public static Expression minuteFloor(DateTimeV2Literal date, IntegerLiteral period, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.MINUTE, date.toJavaDateType(), - period.getValue(), origin.toJavaDateType(), false), date.commonType(origin)); + period.getValue(), origin.toJavaDateType(), false), date.commonScale(origin)); } @ExecFunction(name = "minute_floor") @@ -1224,26 +1222,26 @@ public static Expression secondFloor(DateV2Literal date, IntegerLiteral period, @ExecFunction(name = "second_floor") public static Expression secondFloor(DateTimeV2Literal date) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.SECOND, date.toJavaDateType(), - 1, START_ORIGINAL_DAY, false), date.getDataType()); + 1, START_ORIGINAL_DAY, false), date.getScale()); } @ExecFunction(name = "second_floor") public static Expression secondFloor(DateTimeV2Literal date, IntegerLiteral period) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.SECOND, date.toJavaDateType(), - period.getValue(), START_ORIGINAL_DAY, false), date.getDataType()); + period.getValue(), START_ORIGINAL_DAY, false), date.getScale()); } @ExecFunction(name = "second_floor") public static Expression secondFloor(DateTimeV2Literal date, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.SECOND, date.toJavaDateType(), - 1, origin.toJavaDateType(), false), date.commonType(origin)); + 1, origin.toJavaDateType(), false), date.commonScale(origin)); } @ExecFunction(name = "second_floor") public static Expression secondFloor(DateTimeV2Literal date, IntegerLiteral period, DateTimeV2Literal origin) { return DateTimeV2Literal.fromJavaDateType(getDateCeilOrFloor(DATE.SECOND, date.toJavaDateType(), - period.getValue(), origin.toJavaDateType(), false), date.commonType(origin)); + period.getValue(), origin.toJavaDateType(), false), date.commonScale(origin)); } @ExecFunction(name = "second_floor") diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConvertTz.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConvertTz.java index afc4890da0a85a..195ea273af90b8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConvertTz.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConvertTz.java @@ -142,8 +142,8 @@ public boolean isMonotonic(Literal lower, Literal upper) { * Partition pruning folds both endpoints before deriving the function range, so a fall-back * instant inside (fromInstant(lower), fromInstant(upper)] disables the monotonic shortcut. */ - Instant lowerInstant = DateUtils.convertLocalToInstant(lowerDateTime, fromZone); - Instant upperInstant = DateUtils.convertLocalToInstant(upperDateTime, fromZone); + Instant lowerInstant = DateTimeLiteral.convertLocalToInstant(lowerDateTime, fromZone); + Instant upperInstant = DateTimeLiteral.convertLocalToInstant(upperDateTime, fromZone); if (upperInstant.isBefore(lowerInstant)) { return false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DayMicrosecondAdd.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DayMicrosecondAdd.java index a3a236a34baff0..22b319464bc08a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DayMicrosecondAdd.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DayMicrosecondAdd.java @@ -41,8 +41,8 @@ public class DayMicrosecondAdd extends ScalarFunction implements BinaryExpression, ExplicitlyCastableSignature, ComputeSignatureForDateArithmetic, PropagateNullable, DateAddSubMonotonic { public static final List SIGNATURES = ImmutableList.of( - FunctionSignature.ret(DateTimeV2Type.WILDCARD) - .args(DateTimeV2Type.WILDCARD, VarcharType.SYSTEM_DEFAULT), + FunctionSignature.ret(DateTimeV2Type.MAX) + .args(DateTimeV2Type.MAX, VarcharType.SYSTEM_DEFAULT), FunctionSignature.ret(TimeStampTzType.MAX) .args(TimeStampTzType.MAX, VarcharType.SYSTEM_DEFAULT) ); @@ -80,12 +80,6 @@ public Expression withConstantArgs(Expression literal) { @Override public FunctionSignature computeSignature(FunctionSignature signature) { signature = super.computeSignature(signature); - if (getArgument(0).getDataType() instanceof TimeStampTzType - || signature.argumentsTypes.get(0) instanceof TimeStampTzType) { - return signature.withArgumentType(0, TimeStampTzType.MAX).withReturnType(TimeStampTzType.MAX); - } - DateTimeV2Type returnType = DateTimeV2Type.forTypeWithMinimumScale( - getArgument(0).getDataType(), DateTimeV2Type.MAX_SCALE); - return signature.withArgumentType(0, returnType).withReturnType(returnType); + return signature.withArgumentType(0, DateTimeV2Type.MAX).withReturnType(DateTimeV2Type.MAX); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DayMicrosecondSub.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DayMicrosecondSub.java index 1fd6aaeb56c601..efb61c5f72ec17 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DayMicrosecondSub.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DayMicrosecondSub.java @@ -41,8 +41,8 @@ public class DayMicrosecondSub extends ScalarFunction implements BinaryExpression, ExplicitlyCastableSignature, ComputeSignatureForDateArithmetic, PropagateNullable, DateAddSubMonotonic { public static final List SIGNATURES = ImmutableList.of( - FunctionSignature.ret(DateTimeV2Type.WILDCARD) - .args(DateTimeV2Type.WILDCARD, VarcharType.SYSTEM_DEFAULT), + FunctionSignature.ret(DateTimeV2Type.MAX) + .args(DateTimeV2Type.MAX, VarcharType.SYSTEM_DEFAULT), FunctionSignature.ret(TimeStampTzType.MAX) .args(TimeStampTzType.MAX, VarcharType.SYSTEM_DEFAULT) ); @@ -80,12 +80,6 @@ public Expression withConstantArgs(Expression literal) { @Override public FunctionSignature computeSignature(FunctionSignature signature) { signature = super.computeSignature(signature); - if (getArgument(0).getDataType() instanceof TimeStampTzType - || signature.argumentsTypes.get(0) instanceof TimeStampTzType) { - return signature.withArgumentType(0, TimeStampTzType.MAX).withReturnType(TimeStampTzType.MAX); - } - DateTimeV2Type returnType = DateTimeV2Type.forTypeWithMinimumScale( - getArgument(0).getDataType(), DateTimeV2Type.MAX_SCALE); - return signature.withArgumentType(0, returnType).withReturnType(returnType); + return signature.withArgumentType(0, DateTimeV2Type.MAX).withReturnType(DateTimeV2Type.MAX); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HourMicrosecondAdd.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HourMicrosecondAdd.java index 7d6193b11a1c42..d7f7c9e2a31faa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HourMicrosecondAdd.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HourMicrosecondAdd.java @@ -41,8 +41,8 @@ public class HourMicrosecondAdd extends ScalarFunction implements BinaryExpression, ExplicitlyCastableSignature, ComputeSignatureForDateArithmetic, PropagateNullable, DateAddSubMonotonic { public static final List SIGNATURES = ImmutableList.of( - FunctionSignature.ret(DateTimeV2Type.WILDCARD) - .args(DateTimeV2Type.WILDCARD, VarcharType.SYSTEM_DEFAULT), + FunctionSignature.ret(DateTimeV2Type.MAX) + .args(DateTimeV2Type.MAX, VarcharType.SYSTEM_DEFAULT), FunctionSignature.ret(TimeStampTzType.MAX) .args(TimeStampTzType.MAX, VarcharType.SYSTEM_DEFAULT) ); @@ -80,12 +80,6 @@ public Expression withConstantArgs(Expression literal) { @Override public FunctionSignature computeSignature(FunctionSignature signature) { signature = super.computeSignature(signature); - if (getArgument(0).getDataType() instanceof TimeStampTzType - || signature.argumentsTypes.get(0) instanceof TimeStampTzType) { - return signature.withArgumentType(0, TimeStampTzType.MAX).withReturnType(TimeStampTzType.MAX); - } - DateTimeV2Type returnType = DateTimeV2Type.forTypeWithMinimumScale( - getArgument(0).getDataType(), DateTimeV2Type.MAX_SCALE); - return signature.withArgumentType(0, returnType).withReturnType(returnType); + return signature.withArgumentType(0, DateTimeV2Type.MAX).withReturnType(DateTimeV2Type.MAX); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HourMicrosecondSub.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HourMicrosecondSub.java index 2b7983543f2f51..1d43cb8e94cc5b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HourMicrosecondSub.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HourMicrosecondSub.java @@ -41,8 +41,8 @@ public class HourMicrosecondSub extends ScalarFunction implements BinaryExpression, ExplicitlyCastableSignature, ComputeSignatureForDateArithmetic, PropagateNullable, DateAddSubMonotonic { public static final List SIGNATURES = ImmutableList.of( - FunctionSignature.ret(DateTimeV2Type.WILDCARD) - .args(DateTimeV2Type.WILDCARD, VarcharType.SYSTEM_DEFAULT), + FunctionSignature.ret(DateTimeV2Type.MAX) + .args(DateTimeV2Type.MAX, VarcharType.SYSTEM_DEFAULT), FunctionSignature.ret(TimeStampTzType.MAX) .args(TimeStampTzType.MAX, VarcharType.SYSTEM_DEFAULT) ); @@ -80,12 +80,6 @@ public Expression withConstantArgs(Expression literal) { @Override public FunctionSignature computeSignature(FunctionSignature signature) { signature = super.computeSignature(signature); - if (getArgument(0).getDataType() instanceof TimeStampTzType - || signature.argumentsTypes.get(0) instanceof TimeStampTzType) { - return signature.withArgumentType(0, TimeStampTzType.MAX).withReturnType(TimeStampTzType.MAX); - } - DateTimeV2Type returnType = DateTimeV2Type.forTypeWithMinimumScale( - getArgument(0).getDataType(), DateTimeV2Type.MAX_SCALE); - return signature.withArgumentType(0, returnType).withReturnType(returnType); + return signature.withArgumentType(0, DateTimeV2Type.MAX).withReturnType(DateTimeV2Type.MAX); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MicroSecondsAdd.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MicroSecondsAdd.java index 125e8d16575bc2..c83c2f8f5edc3b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MicroSecondsAdd.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MicroSecondsAdd.java @@ -40,8 +40,7 @@ public class MicroSecondsAdd extends ScalarFunction implements BinaryExpression, PropagateNullable, DateAddSubMonotonic { private static final List SIGNATURES = ImmutableList.of( - FunctionSignature.ret(DateTimeV2Type.WILDCARD) - .args(DateTimeV2Type.WILDCARD, BigIntType.INSTANCE), + FunctionSignature.ret(DateTimeV2Type.MAX).args(DateTimeV2Type.MAX, BigIntType.INSTANCE), FunctionSignature.ret(TimeStampTzType.MAX).args(TimeStampTzType.MAX, BigIntType.INSTANCE) ); @@ -68,13 +67,10 @@ public List getSignatures() { @Override public FunctionSignature computeSignature(FunctionSignature signature) { signature = super.computeSignature(signature); - if (getArgument(0).getDataType() instanceof TimeStampTzType - || signature.argumentsTypes.get(0) instanceof TimeStampTzType) { + if (signature.argumentsTypes.get(0) instanceof TimeStampTzType) { return signature.withArgumentType(0, TimeStampTzType.MAX).withReturnType(TimeStampTzType.MAX); } - DateTimeV2Type returnType = DateTimeV2Type.forTypeWithMinimumScale( - getArgument(0).getDataType(), DateTimeV2Type.MAX_SCALE); - return signature.withArgumentType(0, returnType).withReturnType(returnType); + return signature.withArgumentType(0, DateTimeV2Type.MAX).withReturnType(DateTimeV2Type.MAX); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MicroSecondsSub.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MicroSecondsSub.java index 5f6f54bc7b675a..a407a945481918 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MicroSecondsSub.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MicroSecondsSub.java @@ -40,8 +40,7 @@ public class MicroSecondsSub extends ScalarFunction implements BinaryExpression, PropagateNullable, DateAddSubMonotonic { private static final List SIGNATURES = ImmutableList - .of(FunctionSignature.ret(DateTimeV2Type.WILDCARD) - .args(DateTimeV2Type.WILDCARD, BigIntType.INSTANCE), + .of(FunctionSignature.ret(DateTimeV2Type.MAX).args(DateTimeV2Type.MAX, BigIntType.INSTANCE), FunctionSignature.ret(TimeStampTzType.MAX).args(TimeStampTzType.MAX, BigIntType.INSTANCE) ); @@ -68,13 +67,10 @@ public List getSignatures() { @Override public FunctionSignature computeSignature(FunctionSignature signature) { signature = super.computeSignature(signature); - if (getArgument(0).getDataType() instanceof TimeStampTzType - || signature.argumentsTypes.get(0) instanceof TimeStampTzType) { + if (signature.argumentsTypes.get(0) instanceof TimeStampTzType) { return signature.withArgumentType(0, TimeStampTzType.MAX).withReturnType(TimeStampTzType.MAX); } - DateTimeV2Type returnType = DateTimeV2Type.forTypeWithMinimumScale( - getArgument(0).getDataType(), DateTimeV2Type.MAX_SCALE); - return signature.withArgumentType(0, returnType).withReturnType(returnType); + return signature.withArgumentType(0, DateTimeV2Type.MAX).withReturnType(DateTimeV2Type.MAX); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MilliSecondsAdd.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MilliSecondsAdd.java index e4b0f95e044d73..4880e260eb2064 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MilliSecondsAdd.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MilliSecondsAdd.java @@ -40,8 +40,7 @@ public class MilliSecondsAdd extends ScalarFunction implements BinaryExpression, PropagateNullable, DateAddSubMonotonic { private static final List SIGNATURES = ImmutableList - .of(FunctionSignature.ret(DateTimeV2Type.WILDCARD) - .args(DateTimeV2Type.WILDCARD, BigIntType.INSTANCE), + .of(FunctionSignature.ret(DateTimeV2Type.MAX).args(DateTimeV2Type.MAX, BigIntType.INSTANCE), FunctionSignature.ret(TimeStampTzType.MAX).args(TimeStampTzType.MAX, BigIntType.INSTANCE) ); @@ -68,13 +67,10 @@ public List getSignatures() { @Override public FunctionSignature computeSignature(FunctionSignature signature) { signature = super.computeSignature(signature); - if (getArgument(0).getDataType() instanceof TimeStampTzType - || signature.argumentsTypes.get(0) instanceof TimeStampTzType) { + if (signature.argumentsTypes.get(0) instanceof TimeStampTzType) { return signature.withArgumentType(0, TimeStampTzType.MAX).withReturnType(TimeStampTzType.MAX); } - DateTimeV2Type returnType = DateTimeV2Type.forTypeWithMinimumScale( - getArgument(0).getDataType(), DateTimeV2Type.MAX_SCALE); - return signature.withArgumentType(0, returnType).withReturnType(returnType); + return signature.withArgumentType(0, DateTimeV2Type.MAX).withReturnType(DateTimeV2Type.MAX); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MilliSecondsSub.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MilliSecondsSub.java index a844cabe446709..2bf37e4b81fcc8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MilliSecondsSub.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MilliSecondsSub.java @@ -40,8 +40,7 @@ public class MilliSecondsSub extends ScalarFunction implements BinaryExpression, PropagateNullable, DateAddSubMonotonic { private static final List SIGNATURES = ImmutableList - .of(FunctionSignature.ret(DateTimeV2Type.WILDCARD) - .args(DateTimeV2Type.WILDCARD, BigIntType.INSTANCE), + .of(FunctionSignature.ret(DateTimeV2Type.MAX).args(DateTimeV2Type.MAX, BigIntType.INSTANCE), FunctionSignature.ret(TimeStampTzType.MAX).args(TimeStampTzType.MAX, BigIntType.INSTANCE) ); @@ -68,13 +67,10 @@ public List getSignatures() { @Override public FunctionSignature computeSignature(FunctionSignature signature) { signature = super.computeSignature(signature); - if (getArgument(0).getDataType() instanceof TimeStampTzType - || signature.argumentsTypes.get(0) instanceof TimeStampTzType) { + if (signature.argumentsTypes.get(0) instanceof TimeStampTzType) { return signature.withArgumentType(0, TimeStampTzType.MAX).withReturnType(TimeStampTzType.MAX); } - DateTimeV2Type returnType = DateTimeV2Type.forTypeWithMinimumScale( - getArgument(0).getDataType(), DateTimeV2Type.MAX_SCALE); - return signature.withArgumentType(0, returnType).withReturnType(returnType); + return signature.withArgumentType(0, DateTimeV2Type.MAX).withReturnType(DateTimeV2Type.MAX); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MinuteMicrosecondAdd.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MinuteMicrosecondAdd.java index 41379116417ec3..135666d6d52cf4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MinuteMicrosecondAdd.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MinuteMicrosecondAdd.java @@ -41,8 +41,8 @@ public class MinuteMicrosecondAdd extends ScalarFunction implements BinaryExpression, ExplicitlyCastableSignature, ComputeSignatureForDateArithmetic, PropagateNullable, DateAddSubMonotonic { public static final List SIGNATURES = ImmutableList.of( - FunctionSignature.ret(DateTimeV2Type.WILDCARD) - .args(DateTimeV2Type.WILDCARD, VarcharType.SYSTEM_DEFAULT), + FunctionSignature.ret(DateTimeV2Type.MAX) + .args(DateTimeV2Type.MAX, VarcharType.SYSTEM_DEFAULT), FunctionSignature.ret(TimeStampTzType.MAX) .args(TimeStampTzType.MAX, VarcharType.SYSTEM_DEFAULT) ); @@ -80,12 +80,6 @@ public Expression withConstantArgs(Expression literal) { @Override public FunctionSignature computeSignature(FunctionSignature signature) { signature = super.computeSignature(signature); - if (getArgument(0).getDataType() instanceof TimeStampTzType - || signature.argumentsTypes.get(0) instanceof TimeStampTzType) { - return signature.withArgumentType(0, TimeStampTzType.MAX).withReturnType(TimeStampTzType.MAX); - } - DateTimeV2Type returnType = DateTimeV2Type.forTypeWithMinimumScale( - getArgument(0).getDataType(), DateTimeV2Type.MAX_SCALE); - return signature.withArgumentType(0, returnType).withReturnType(returnType); + return signature.withArgumentType(0, DateTimeV2Type.MAX).withReturnType(DateTimeV2Type.MAX); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MinuteMicrosecondSub.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MinuteMicrosecondSub.java index f3e53b460a1e28..45b7272d505f46 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MinuteMicrosecondSub.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MinuteMicrosecondSub.java @@ -41,8 +41,8 @@ public class MinuteMicrosecondSub extends ScalarFunction implements BinaryExpression, ExplicitlyCastableSignature, ComputeSignatureForDateArithmetic, PropagateNullable, DateAddSubMonotonic { public static final List SIGNATURES = ImmutableList.of( - FunctionSignature.ret(DateTimeV2Type.WILDCARD) - .args(DateTimeV2Type.WILDCARD, VarcharType.SYSTEM_DEFAULT), + FunctionSignature.ret(DateTimeV2Type.MAX) + .args(DateTimeV2Type.MAX, VarcharType.SYSTEM_DEFAULT), FunctionSignature.ret(TimeStampTzType.MAX) .args(TimeStampTzType.MAX, VarcharType.SYSTEM_DEFAULT) ); @@ -80,12 +80,6 @@ public Expression withConstantArgs(Expression literal) { @Override public FunctionSignature computeSignature(FunctionSignature signature) { signature = super.computeSignature(signature); - if (getArgument(0).getDataType() instanceof TimeStampTzType - || signature.argumentsTypes.get(0) instanceof TimeStampTzType) { - return signature.withArgumentType(0, TimeStampTzType.MAX).withReturnType(TimeStampTzType.MAX); - } - DateTimeV2Type returnType = DateTimeV2Type.forTypeWithMinimumScale( - getArgument(0).getDataType(), DateTimeV2Type.MAX_SCALE); - return signature.withArgumentType(0, returnType).withReturnType(returnType); + return signature.withArgumentType(0, DateTimeV2Type.MAX).withReturnType(DateTimeV2Type.MAX); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SecondMicrosecondAdd.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SecondMicrosecondAdd.java index 6542fa142a026a..8411699aedf2f9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SecondMicrosecondAdd.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SecondMicrosecondAdd.java @@ -80,12 +80,9 @@ public Expression withConstantArgs(Expression literal) { @Override public FunctionSignature computeSignature(FunctionSignature signature) { signature = super.computeSignature(signature); - if (getArgument(0).getDataType() instanceof TimeStampTzType - || signature.argumentsTypes.get(0) instanceof TimeStampTzType) { + if (signature.argumentsTypes.get(0) instanceof TimeStampTzType) { return signature.withArgumentType(0, TimeStampTzType.MAX).withReturnType(TimeStampTzType.MAX); } - DateTimeV2Type returnType = DateTimeV2Type.forTypeWithMinimumScale( - getArgument(0).getDataType(), DateTimeV2Type.MAX_SCALE); - return signature.withArgumentType(0, returnType).withReturnType(returnType); + return signature.withArgumentType(0, DateTimeV2Type.MAX).withReturnType(DateTimeV2Type.MAX); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SecondMicrosecondSub.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SecondMicrosecondSub.java index 2ec3d2d83256fc..af4350f28f26ba 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SecondMicrosecondSub.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SecondMicrosecondSub.java @@ -80,12 +80,9 @@ public Expression withConstantArgs(Expression literal) { @Override public FunctionSignature computeSignature(FunctionSignature signature) { signature = super.computeSignature(signature); - if (getArgument(0).getDataType() instanceof TimeStampTzType - || signature.argumentsTypes.get(0) instanceof TimeStampTzType) { + if (signature.argumentsTypes.get(0) instanceof TimeStampTzType) { return signature.withArgumentType(0, TimeStampTzType.MAX).withReturnType(TimeStampTzType.MAX); } - DateTimeV2Type returnType = DateTimeV2Type.forTypeWithMinimumScale( - getArgument(0).getDataType(), DateTimeV2Type.MAX_SCALE); - return signature.withArgumentType(0, returnType).withReturnType(returnType); + return signature.withArgumentType(0, DateTimeV2Type.MAX).withReturnType(DateTimeV2Type.MAX); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/window/RequireTrivialTypes.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/window/RequireTrivialTypes.java index 074b13dab7dca6..c868f3501b5bff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/window/RequireTrivialTypes.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/window/RequireTrivialTypes.java @@ -32,7 +32,6 @@ import org.apache.doris.nereids.types.LargeIntType; import org.apache.doris.nereids.types.SmallIntType; import org.apache.doris.nereids.types.StringType; -import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TimeStampTzType; import org.apache.doris.nereids.types.TimeV2Type; import org.apache.doris.nereids.types.TinyIntType; @@ -62,7 +61,6 @@ public interface RequireTrivialTypes { DateTimeType.INSTANCE, DateV2Type.INSTANCE, DateTimeV2Type.WILDCARD, - TimeStampNsType.INSTANCE, TimeV2Type.WILDCARD, TimeStampTzType.WILDCARD, VarcharType.SYSTEM_DEFAULT, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java index 9af785bd94fee5..6082b71447cf76 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java @@ -26,7 +26,6 @@ import org.apache.doris.nereids.types.DateTimeType; import org.apache.doris.nereids.types.DateTimeV2Type; import org.apache.doris.nereids.types.DateType; -import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.coercion.DateLikeType; import org.apache.doris.nereids.util.DateTimeFormatterUtils; import org.apache.doris.nereids.util.DateUtils; @@ -639,10 +638,7 @@ protected Expression uncheckedCastTo(DataType targetType) throws AnalysisExcepti } else if (targetType.isDoubleType()) { return new DoubleLiteral(value); } else if (targetType.isDateTimeV2Type()) { - if (targetType instanceof TimeStampNsType) { - return new TimeStampNsLiteral(year, month, day, 0, 0, 0, 0); - } - return DateTimeV2Literal.create((DateTimeV2Type) targetType, year, month, day, 0, 0, 0, 0); + return new DateTimeV2Literal((DateTimeV2Type) targetType, year, month, day, 0, 0, 0, 0); } else if (targetType.isDateTimeType()) { return new DateTimeLiteral((DateTimeType) targetType, year, month, day, 0, 0, 0, 0); } else if (targetType.isDateV2Type()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java index e9e7bea5d1006b..caf185c4f9907b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java @@ -26,23 +26,27 @@ import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.DateTimeType; import org.apache.doris.nereids.types.DateTimeV2Type; -import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TimeStampTzType; import org.apache.doris.nereids.types.TimeV2Type; import org.apache.doris.nereids.types.coercion.DateLikeType; import org.apache.doris.nereids.util.DateUtils; +import org.apache.doris.nereids.util.StandardDateFormat; import com.google.common.base.Preconditions; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.math.BigInteger; -import java.time.DateTimeException; +import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; +import java.time.ZoneOffset; import java.time.temporal.ChronoField; import java.time.temporal.TemporalAccessor; import java.time.temporal.TemporalQueries; +import java.time.zone.ZoneOffsetTransition; +import java.time.zone.ZoneRules; +import java.util.List; import java.util.Objects; /** @@ -52,7 +56,6 @@ public class DateTimeLiteral extends DateLiteral { public static final DateTimeLiteral MIN_DATETIME = new DateTimeLiteral(0000, 1, 1, 0, 0, 0); public static final DateTimeLiteral MAX_DATETIME = new DateTimeLiteral(9999, 12, 31, 23, 59, 59); protected static final int MAX_MICROSECOND = 999999; - protected static final int MAX_NANOSECOND = 999999999; private static final Logger LOG = LogManager.getLogger(DateTimeLiteral.class); @@ -60,7 +63,6 @@ public class DateTimeLiteral extends DateLiteral { protected long minute; protected long second; protected long microSecond; - protected long nanoSecond; public DateTimeLiteral(String s) { this(DateTimeType.INSTANCE, s); @@ -96,7 +98,6 @@ public DateTimeLiteral(DateLikeType dataType, long year, long month, long day, this.minute = minute; this.second = second; this.microSecond = microSecond; - this.nanoSecond = microSecond * 1000; this.year = year; this.month = month; this.day = day; @@ -104,7 +105,7 @@ public DateTimeLiteral(DateLikeType dataType, long year, long month, long day, @Override public boolean isMidnight() { - return hour == 0 && minute == 0 && second == 0 && nanoSecond == 0; + return hour == 0 && minute == 0 && second == 0 && microSecond == 0; } /** @@ -157,7 +158,7 @@ public static Result parseDateTimeLiteral(St ZoneId zoneId = temporal.query(TemporalQueries.zone()); if (zoneId != null) { - LocalDateTime localDateTime = DateUtils.convertTimeZone(year, month, day, hour, minute, second, + LocalDateTime localDateTime = convertTimeZone(year, month, day, hour, minute, second, zoneId, DateUtils.getTimeZone()); year = localDateTime.getYear(); month = localDateTime.getMonthValue(); @@ -167,9 +168,7 @@ public static Result parseDateTimeLiteral(St second = localDateTime.getSecond(); } - long nanoSecond = DateUtils.getOrDefault(temporal, ChronoField.NANO_OF_SECOND); - DateTimeV2Type dateTimeV2Type = isV2 ? DateTimeV2Type.forTypeFromString(s) : null; - long microSecond = nanoSecond / 100L; + long microSecond = DateUtils.getOrDefault(temporal, ChronoField.NANO_OF_SECOND) / 100L; // Microseconds have 7 digits. long sevenDigit = microSecond % 10; microSecond = microSecond / 10; @@ -190,8 +189,8 @@ public static Result parseDateTimeLiteral(St } if (isV2) { - return Result.ok(DateTimeV2Literal.create(dateTimeV2Type, year, month, day, hour, minute, second, - microSecond)); + DateTimeV2Type type = DateTimeV2Type.forTypeFromString(s); + return Result.ok(new DateTimeV2Literal(type, year, month, day, hour, minute, second, microSecond)); } else { return Result.ok(new DateTimeLiteral(DateTimeType.INSTANCE, year, month, day, hour, minute, second)); } @@ -200,20 +199,21 @@ public static Result parseDateTimeLiteral(St protected void roundMicroSecond(int scale) { Preconditions.checkArgument(scale >= 0 && scale <= DateTimeV2Type.MAX_SCALE, "invalid datetime v2 scale: %s", scale); - long factor = (long) Math.pow(10, DateUtils.NANOSECOND_SCALE - scale); - this.nanoSecond = (this.nanoSecond + factor / 2) / factor * factor; - if (this.nanoSecond >= 1000000000) { - LocalDateTime localDateTime = LocalDateTime.of((int) year, (int) month, (int) day, - (int) hour, (int) minute, (int) second).plusSeconds(1); + double factor = Math.pow(10, 6 - scale); + + this.microSecond = Math.round(this.microSecond / factor) * (int) factor; + + if (this.microSecond >= 1000000) { + LocalDateTime localDateTime = DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER_TO_MICRO_SECOND, + getStringValue()).plusSeconds(1); this.year = localDateTime.getYear(); this.month = localDateTime.getMonthValue(); this.day = localDateTime.getDayOfMonth(); this.hour = localDateTime.getHour(); this.minute = localDateTime.getMinute(); this.second = localDateTime.getSecond(); - this.nanoSecond -= 1000000000; + this.microSecond -= 1000000; } - this.microSecond = this.nanoSecond / 1000; if (checkRange() || checkDate(year, month, day)) { // may fallback to legacy planner. make sure the behaviour of rounding is same. throw new AnalysisException("datetime literal [" + toString() + "] is out of range"); @@ -233,7 +233,7 @@ protected void init(String s) throws AnalysisException { ZoneId zoneId = temporal.query(TemporalQueries.zone()); if (zoneId != null) { ZoneId targetZone = this.dataType instanceof TimeStampTzType ? ZoneId.of("UTC") : DateUtils.getTimeZone(); - LocalDateTime localDateTime = DateUtils.convertTimeZone(year, month, day, hour, minute, second, + LocalDateTime localDateTime = convertTimeZone(year, month, day, hour, minute, second, zoneId, targetZone); this.year = localDateTime.getYear(); this.month = localDateTime.getMonthValue(); @@ -243,14 +243,13 @@ protected void init(String s) throws AnalysisException { this.second = localDateTime.getSecond(); } - nanoSecond = DateUtils.getOrDefault(temporal, ChronoField.NANO_OF_SECOND); - microSecond = nanoSecond / 100L; + microSecond = DateUtils.getOrDefault(temporal, ChronoField.NANO_OF_SECOND) / 100L; // Microseconds have 7 digits. long sevenDigit = microSecond % 10; microSecond = microSecond / 10; - nanoSecond = microSecond * 1000; if (sevenDigit >= 5) { - DateTimeLiteral result = this.plusMicroSeconds(1); + DateTimeLiteral result; + result = this.plusMicroSeconds(1); this.second = result.second; this.minute = result.minute; this.hour = result.hour; @@ -258,7 +257,6 @@ protected void init(String s) throws AnalysisException { this.month = result.month; this.year = result.year; this.microSecond = result.microSecond; - this.nanoSecond = result.nanoSecond; } if (checkRange(year, month, day) || checkDate(year, month, day)) { @@ -272,9 +270,40 @@ public DateTimeLiteral plusMicroSeconds(long microSeconds) { return fromJavaDateType(toJavaDateType().plusNanos(Math.multiplyExact(microSeconds, 1000L)), 6); } + private static LocalDateTime convertTimeZone(long year, long month, long day, long hour, long minute, + long second, ZoneId fromZone, ZoneId toZone) { + LocalDateTime localDateTime = LocalDateTime.of((int) year, (int) month, (int) day, + (int) hour, (int) minute, (int) second); + Instant instant = convertLocalToInstant(localDateTime, fromZone); + return LocalDateTime.ofInstant(instant, toZone); + } + + /** + * Convert a local civil datetime in {@code fromZone} to an instant with the same DST transition + * policy as BE cctz::convert(civil_second, zone). + * + *

For normal local times, there is one valid offset. For fall-back overlap times, two offsets + * are valid and the first one is the pre-transition offset. For spring-forward gap times, the + * local time does not exist, so any value inside the skipped interval maps to the transition + * instant. + */ + public static Instant convertLocalToInstant(LocalDateTime localDateTime, ZoneId fromZone) { + ZoneRules rules = fromZone.getRules(); + List validOffsets = rules.getValidOffsets(localDateTime); + int size = validOffsets.size(); + // Match BE cctz::convert(civil_second, zone) semantics for constant folding. + // Normal local time has one offset; repeated local time uses the pre-transition offset. + if (size == 1 || size == 2) { + return localDateTime.atOffset(validOffsets.get(0)).toInstant(); + } + // Skipped local time maps to the transition instant, e.g. 2021-03-28 02:15 Europe/Paris. + ZoneOffsetTransition transition = rules.getTransition(localDateTime); + return transition.getInstant(); + } + public boolean checkRange() { return checkRange(year, month, day) || hour > MAX_DATETIME.getHour() || minute > MAX_DATETIME.getMinute() - || second > MAX_DATETIME.getSecond() || nanoSecond > MAX_NANOSECOND; + || second > MAX_DATETIME.getSecond() || microSecond > MAX_MICROSECOND; } @Override @@ -293,12 +322,12 @@ public long timePartToMicroSecond() { @Override public long getTimePartInNanoseconds() { - return ((hour * 60L + minute) * 60L + second) * 1_000_000_000L + nanoSecond; + return timePartToMicroSecond() * 1000L; } @Override public long getFractionalSecondInNanoseconds() { - return nanoSecond; + return microSecond * 1000L; } @Override @@ -318,7 +347,54 @@ public String toString() { @Override public String getStringValue() { - return DateUtils.formatDateTime(year, month, day, hour, minute, second, nanoSecond, 0); + if (0 <= year && year <= 9999 && 0 <= month && month <= 99 && 0 <= day && day <= 99 + && 0 <= hour && hour <= 99 && 0 <= minute && minute <= 99 && 0 <= second && second <= 99) { + char[] format = new char[] { + '0', '0', '0', '0', '-', '0', '0', '-', '0', '0', ' ', '0', '0', ':', '0', '0', ':', '0', '0'}; + int offset = 3; + long year = this.year; + while (year > 0) { + format[offset--] = (char) ('0' + (year % 10)); + year /= 10; + } + + offset = 6; + long month = this.month; + while (month > 0) { + format[offset--] = (char) ('0' + (month % 10)); + month /= 10; + } + + offset = 9; + long day = this.day; + while (day > 0) { + format[offset--] = (char) ('0' + (day % 10)); + day /= 10; + } + + offset = 12; + long hour = this.hour; + while (hour > 0) { + format[offset--] = (char) ('0' + (hour % 10)); + hour /= 10; + } + + offset = 15; + long minute = this.minute; + while (minute > 0) { + format[offset--] = (char) ('0' + (minute % 10)); + minute /= 10; + } + + offset = 18; + long second = this.second; + while (second > 0) { + format[offset--] = (char) ('0' + (second % 10)); + second /= 10; + } + return String.valueOf(format); + } + return String.format("%04d-%02d-%02d %02d:%02d:%02d", year, month, day, hour, minute, second); } @Override @@ -346,31 +422,14 @@ protected Expression uncheckedCastTo(DataType targetType) throws AnalysisExcepti } else if (targetType.isDateTimeV2Type()) { // High scale datetime to low scale datetime may overflow. try { - DateTimeV2Type dateTimeV2Type = (DateTimeV2Type) targetType; - int scale = dateTimeV2Type.getScale(); - long factor = (long) Math.pow(10, DateUtils.NANOSECOND_SCALE - scale); - LocalDateTime roundedDateTime = LocalDateTime.of( - (int) year, (int) month, (int) day, - (int) hour, (int) minute, (int) second, (int) nanoSecond) - .plusNanos(factor / 2); - if (dateTimeV2Type instanceof TimeStampNsType) { - return TimeStampNsLiteral.fromJavaDateType(roundedDateTime); - } - return DateTimeV2Literal.fromJavaDateType(roundedDateTime, dateTimeV2Type); - } catch (AnalysisException | DateTimeException e) { + return new DateTimeV2Literal((DateTimeV2Type) targetType, + year, month, day, hour, minute, second, microSecond); + } catch (AnalysisException e) { throw new CastException(e.getMessage(), e); } } else if (targetType.isTimeType()) { - int scale = ((TimeV2Type) targetType).getScale(); - long factor = (long) Math.pow(10, DateUtils.NANOSECOND_SCALE - scale); - long roundedNanoSecond = (nanoSecond + factor / 2) / factor * factor; - long totalNanoSecond = ((hour * 60 + minute) * 60 + second) * 1000000000L - + roundedNanoSecond; - int resultHour = (int) (totalNanoSecond / 1000000000L / 60 / 60); - int resultMinute = (int) (totalNanoSecond / 1000000000L / 60 % 60); - int resultSecond = (int) (totalNanoSecond / 1000000000L % 60); - int resultMicroSecond = (int) (totalNanoSecond % 1000000000L / 1000); - return new TimeV2Literal(resultHour, resultMinute, resultSecond, resultMicroSecond, scale, false); + return new TimeV2Literal((int) hour, (int) minute, (int) second, (int) microSecond, + ((TimeV2Type) targetType).getScale(), false); } else if (targetType.isFloatType()) { return new FloatLiteral(getValue()); } else if (targetType.isDoubleType()) { @@ -428,10 +487,6 @@ public long getMicroSecond() { return microSecond; } - public long getNanoSecond() { - return nanoSecond; - } - @Override public boolean equals(Object o) { if (this == o) { @@ -446,7 +501,7 @@ public boolean equals(Object o) { public LocalDateTime toJavaDateType() { return LocalDateTime.of(((int) getYear()), ((int) getMonth()), ((int) getDay()), - ((int) getHour()), ((int) getMinute()), ((int) getSecond()), (int) getNanoSecond()); + ((int) getHour()), ((int) getMinute()), ((int) getSecond()), (int) getMicroSecond() * 1000); } public static DateTimeLiteral fromJavaDateType(LocalDateTime dateTime, int precision) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeV2Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeV2Literal.java index c7e7720cb64597..82c994b65c9b7b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeV2Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeV2Literal.java @@ -22,13 +22,13 @@ import org.apache.doris.nereids.exceptions.NotSupportedException; import org.apache.doris.nereids.exceptions.UnboundException; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeExtractAndTransform; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.DateTimeType; import org.apache.doris.nereids.types.DateTimeV2Type; -import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TimeStampTzType; -import org.apache.doris.nereids.util.DateUtils; +import org.apache.doris.qe.ConnectContext; import com.google.common.base.Preconditions; @@ -67,38 +67,18 @@ public DateTimeV2Literal(DateTimeV2Type dateType, } private static DateTimeV2Type requireDateTimeV2Type(DateTimeV2Type dateType) { - Preconditions.checkArgument(!(dateType instanceof TimeStampNsType), + Preconditions.checkArgument(dateType.getClass() == DateTimeV2Type.class, "Use TimeStampNsLiteral for TIMESTAMP_NS values"); return dateType; } - /** Create a DATETIMEV2 literal. TIMESTAMP_NS has its own literal factory. */ - public static DateTimeV2Literal create(DateTimeV2Type dateType, String value) { - return new DateTimeV2Literal(dateType, value); - } - - /** Create a DATETIMEV2 literal. TIMESTAMP_NS has its own literal factory. */ - public static DateTimeV2Literal create(DateTimeV2Type dateType, - long year, long month, long day, long hour, long minute, long second, long fractionalSecond) { - return new DateTimeV2Literal(dateType, year, month, day, hour, minute, second, fractionalSecond); - } - - /** Create the last value of a date for a DATETIMEV2 type. */ - public static DateTimeV2Literal createEndOfDay(DateTimeV2Type dateType, long year, long month, long day) { - long microSecond = 0; - for (int i = 0; i < DateTimeV2Type.MAX_SCALE; i++) { - microSecond = microSecond * 10 + (i < dateType.getScale() ? 9 : 0); - } - return new DateTimeV2Literal(dateType, year, month, day, 23, 59, 59, microSecond); - } - /** Date difference rounded toward zero by time part. */ public static long dateDiffInDaysRoundToZeroByTime(DateLiteral lhs, DateLiteral rhs) { long days = DateV2Literal.dateDiffInDays(lhs, rhs); - long nanoSecondDiff = timePartToNanoSecond(lhs) - timePartToNanoSecond(rhs); - if (days > 0 && nanoSecondDiff < 0) { + long microSecondDiff = timePartToMicroSecond(lhs) - timePartToMicroSecond(rhs); + if (days > 0 && microSecondDiff < 0) { days--; - } else if (days < 0 && nanoSecondDiff > 0) { + } else if (days < 0 && microSecondDiff > 0) { days++; } return days; @@ -111,28 +91,15 @@ public static long datetimeDiffInSecondsRoundToZeroByMicroSecond(DateLiteral lhs /** Datetime difference in microseconds. */ public static long datetimeDiffInMicroSeconds(DateLiteral lhs, DateLiteral rhs) { - long secondDiff = DateV2Literal.dateDiffInDays(lhs, rhs) * 24L * 60L * 60L - + timePartToSecond(lhs) - timePartToSecond(rhs); - long nanoDiff = fractionalNanoSecond(lhs) - fractionalNanoSecond(rhs); - long result = secondDiff * 1000000L + nanoDiff / 1000L; - if (secondDiff > 0 && nanoDiff < 0 && nanoDiff % 1000 != 0) { - result--; - } else if (secondDiff < 0 && nanoDiff > 0 && nanoDiff % 1000 != 0) { - result++; - } - return result; - } - - private static long timePartToNanoSecond(DateLiteral date) { - return date.getTimePartInNanoseconds(); - } - - private static long timePartToSecond(DateLiteral date) { - return date.getTimePartInNanoseconds() / 1_000_000_000L; + return DateV2Literal.dateDiffInDays(lhs, rhs) * 24L * 60L * 60L * 1000L * 1000L + + timePartToMicroSecond(lhs) - timePartToMicroSecond(rhs); } - private static long fractionalNanoSecond(DateLiteral date) { - return date.getFractionalSecondInNanoseconds(); + private static long timePartToMicroSecond(DateLiteral date) { + if (date instanceof DateTimeLiteral) { + return ((DateTimeLiteral) date).timePartToMicroSecond(); + } + return 0; } @Override @@ -153,7 +120,7 @@ public LiteralExpr toLegacyLiteral() { @Override public double getDouble() { - return super.getDouble() + nanoSecond / 1000000000.0; + return super.getDouble() + microSecond / 1000000.0; } @Override @@ -164,7 +131,71 @@ public String toString() { @Override public String getStringValue() { int scale = getDataType().getScale(); - return DateUtils.formatDateTime(year, month, day, hour, minute, second, nanoSecond, scale); + if (scale <= 0) { + return super.getStringValue(); + } + + if (0 <= year && year <= 9999 && 0 <= month && month <= 99 && 0 <= day && day <= 99 + && 0 <= hour && hour <= 99 && 0 <= minute && minute <= 99 && 0 <= second && second <= 99 + && 0 <= microSecond && microSecond <= MAX_MICROSECOND) { + char[] format = new char[] { + '0', '0', '0', '0', '-', '0', '0', '-', '0', '0', ' ', '0', '0', ':', '0', '0', ':', '0', '0', + '.', '0', '0', '0', '0', '0', '0'}; + int offset = 3; + long year = this.year; + while (year > 0) { + format[offset--] = (char) ('0' + (year % 10)); + year /= 10; + } + + offset = 6; + long month = this.month; + while (month > 0) { + format[offset--] = (char) ('0' + (month % 10)); + month /= 10; + } + + offset = 9; + long day = this.day; + while (day > 0) { + format[offset--] = (char) ('0' + (day % 10)); + day /= 10; + } + + offset = 12; + long hour = this.hour; + while (hour > 0) { + format[offset--] = (char) ('0' + (hour % 10)); + hour /= 10; + } + + offset = 15; + long minute = this.minute; + while (minute > 0) { + format[offset--] = (char) ('0' + (minute % 10)); + minute /= 10; + } + + offset = 18; + long second = this.second; + while (second > 0) { + format[offset--] = (char) ('0' + (second % 10)); + second /= 10; + } + + offset = 19 + scale; + long microSecond = (int) (this.microSecond / Math.pow(10, DateTimeV2Type.MAX_SCALE - scale)); + while (microSecond > 0) { + format[offset--] = (char) ('0' + (microSecond % 10)); + microSecond /= 10; + } + return String.valueOf(format, 0, 20 + scale); + } + + return String.format("%04d-%02d-%02d %02d:%02d:%02d" + + (scale > 0 ? ".%0" + scale + "d" : ""), + year, month, day, hour, minute, second, + (int) (microSecond / Math.pow(10, DateTimeV2Type.MAX_SCALE - scale))); } @Override @@ -177,13 +208,19 @@ protected Expression uncheckedCastTo(DataType targetType) throws AnalysisExcepti year, month, day, hour, minute, second, microSecond); } if (targetType.isTimeStampTzType()) { - return TimestampTzLiteral.fromSessionTimeZone((TimeStampTzType) targetType, this); + DateTimeV2Literal dtV2Lit = (DateTimeV2Literal) (DateTimeExtractAndTransform.convertTz( + this, + new StringLiteral(ConnectContext.get().getSessionVariable().timeZone), + new StringLiteral("UTC"))); + return new TimestampTzLiteral((TimeStampTzType) targetType, + dtV2Lit.getYear(), dtV2Lit.getMonth(), dtV2Lit.getDay(), + dtV2Lit.getHour(), dtV2Lit.getMinute(), dtV2Lit.getSecond(), dtV2Lit.getMicroSecond()); } return super.uncheckedCastTo(targetType); } public Expression plusDays(long days) { - return fromJavaDateType(toJavaDateType().plusDays(days), getDataType()); + return fromJavaDateType(toJavaDateType().plusDays(days), getDataType().getScale()); } /** @@ -230,34 +267,34 @@ public Expression plusDaySecond(VarcharLiteral daySecond) { .plusDays(days) .plusHours(hours) .plusMinutes(minutes) - .plusSeconds(seconds), getDataType()); + .plusSeconds(seconds), getDataType().getScale()); } catch (NumberFormatException e) { throw new NotSupportedException("Invalid time format"); } } public Expression plusMonths(long months) { - return fromJavaDateType(toJavaDateType().plusMonths(months), getDataType()); + return fromJavaDateType(toJavaDateType().plusMonths(months), getDataType().getScale()); } public Expression plusWeeks(long weeks) { - return fromJavaDateType(toJavaDateType().plusWeeks(weeks), getDataType()); + return fromJavaDateType(toJavaDateType().plusWeeks(weeks), getDataType().getScale()); } public Expression plusYears(long years) { - return fromJavaDateType(toJavaDateType().plusYears(years), getDataType()); + return fromJavaDateType(toJavaDateType().plusYears(years), getDataType().getScale()); } public Expression plusHours(long hours) { - return fromJavaDateType(toJavaDateType().plusHours(hours), getDataType()); + return fromJavaDateType(toJavaDateType().plusHours(hours), getDataType().getScale()); } public Expression plusMinutes(long minutes) { - return fromJavaDateType(toJavaDateType().plusMinutes(minutes), getDataType()); + return fromJavaDateType(toJavaDateType().plusMinutes(minutes), getDataType().getScale()); } public Expression plusSeconds(long seconds) { - return fromJavaDateType(toJavaDateType().plusSeconds(seconds), getDataType()); + return fromJavaDateType(toJavaDateType().plusSeconds(seconds), getDataType().getScale()); } /** @@ -292,7 +329,7 @@ public Expression plusDayHour(VarcharLiteral dayHour) { return fromJavaDateType(toJavaDateType() .plusDays(days) - .plusHours(hours), getDataType()); + .plusHours(hours), getDataType().getScale()); } catch (NumberFormatException e) { throw new NotSupportedException("Invalid time format"); } @@ -330,7 +367,7 @@ public Expression plusMinuteSecond(VarcharLiteral minuteSecond) { return fromJavaDateType(toJavaDateType() .plusMinutes(minutes) - .plusSeconds(seconds), getDataType()); + .plusSeconds(seconds), getDataType().getScale()); } catch (NumberFormatException e) { throw new NotSupportedException("Invalid time format"); } @@ -372,7 +409,7 @@ public Expression plusSecondMicrosecond(VarcharLiteral secondMicrosecond) { return fromJavaDateType(toJavaDateType() .plusSeconds(seconds) - .plusNanos(Math.multiplyExact(microseconds, 1000L)), getDataType()); + .plusNanos(Math.multiplyExact(microseconds, 1000L)), getDataType().getScale()); } catch (NumberFormatException e) { throw new NotSupportedException("Invalid time format"); } @@ -381,9 +418,7 @@ public Expression plusSecondMicrosecond(VarcharLiteral secondMicrosecond) { // When performing addition or subtraction with MicroSeconds, the precision must be set to 6 to display it // completely. use multiplyExact to be aware of multiplication overflow possibility. public DateTimeV2Literal plusMicroSeconds(long microSeconds) { - return fromJavaDateType( - toJavaDateType().plusNanos(Math.multiplyExact(microSeconds, 1000L)), - DateTimeV2Type.MAX); + return fromJavaDateType(toJavaDateType().plusNanos(Math.multiplyExact(microSeconds, 1000L)), 6); } public Expression plusMilliSeconds(long microSeconds) { @@ -394,19 +429,21 @@ public int getScale() { return ((DateTimeV2Type) dataType).getScale(); } - public DateTimeV2Type commonType(DateTimeV2Literal other) { - return DateTimeV2Type.getWiderDatetimeV2Type(getDataType(), other.getDataType()); + /** Return DATETIMEV2's microsecond fraction in the common nanosecond comparison unit. */ + public long getNanoSecond() { + return microSecond * 1000L; + } + + public int commonScale(DateTimeV2Literal other) { + return (int) Math.max(getScale(), other.getScale()); } /** * roundCeiling */ public DateTimeV2Literal roundCeiling(int newScale) { - DateTimeV2Type targetType = DateTimeV2Type.of(newScale); - int targetScale = targetType.getScale(); - long factor = (long) Math.pow(10, DateUtils.NANOSECOND_SCALE - targetScale); - long remain = nanoSecond % factor; - long newNanoSecond = nanoSecond; + long remain = Double.valueOf(microSecond % (Math.pow(10, 6 - newScale))).longValue(); + long newMicroSecond = microSecond; long newSecond = second; long newMinute = minute; long newHour = hour; @@ -414,10 +451,13 @@ public DateTimeV2Literal roundCeiling(int newScale) { long newMonth = month; long newYear = year; if (remain != 0) { - newNanoSecond = (nanoSecond + factor) / factor * factor; + newMicroSecond = Double + .valueOf((microSecond + (int) (Math.pow(10, 6 - newScale))) + / (int) (Math.pow(10, 6 - newScale)) * (Math.pow(10, 6 - newScale))) + .longValue(); } - if (newNanoSecond > MAX_NANOSECOND) { - newNanoSecond = 0; + if (newMicroSecond > MAX_MICROSECOND) { + newMicroSecond %= newMicroSecond; Expression plus1Second = this.plusSeconds(1); if (plus1Second.isNullLiteral()) { throw new AnalysisException("round ceil datetime literal (" + toString() + ", " @@ -431,18 +471,13 @@ public DateTimeV2Literal roundCeiling(int newScale) { newMonth = result.month; newYear = result.year; } - return create(targetType, newYear, newMonth, newDay, - newHour, newMinute, newSecond, newNanoSecond / 1000); + return new DateTimeV2Literal(DateTimeV2Type.of(newScale), newYear, newMonth, newDay, + newHour, newMinute, newSecond, newMicroSecond); } - /** Round down the fractional second to the requested normalized datetime scale. */ public DateTimeV2Literal roundFloor(int newScale) { - DateTimeV2Type targetType = DateTimeV2Type.of(newScale); - int targetScale = targetType.getScale(); - long factor = (long) Math.pow(10, DateUtils.NANOSECOND_SCALE - targetScale); - long newNanoSecond = nanoSecond / factor * factor; - return create(targetType, year, month, day, hour, minute, second, - newNanoSecond / 1000); + return new DateTimeV2Literal(DateTimeV2Type.of(newScale), year, month, day, hour, minute, second, + microSecond / (int) Math.pow(10, 6 - newScale) * (int) Math.pow(10, 6 - newScale)); } public static Expression fromJavaDateType(LocalDateTime dateTime) { @@ -453,21 +488,14 @@ public static Expression fromJavaDateType(LocalDateTime dateTime) { * convert java LocalDateTime object to DateTimeV2Literal object. */ public static DateTimeV2Literal fromJavaDateType(LocalDateTime dateTime, int precision) { - return fromJavaDateType(dateTime, DateTimeV2Type.of(precision)); - } - - /** Convert a Java datetime to the explicit DATETIMEV2 target type. */ - public static DateTimeV2Literal fromJavaDateType(LocalDateTime dateTime, DateTimeV2Type targetType) { - requireDateTimeV2Type(targetType); - int targetScale = targetType.getScale(); - long factor = (long) Math.pow(10, DateUtils.NANOSECOND_SCALE - targetScale); + long value = (long) Math.pow(10, DateTimeV2Type.MAX_SCALE - precision); if (isDateOutOfRange(dateTime)) { throw new AnalysisException("datetime out of range" + dateTime.toString()); } - long nanoSecond = dateTime.getNano() / factor * factor; - return create(targetType, dateTime.getYear(), + return new DateTimeV2Literal(DateTimeV2Type.of(precision), dateTime.getYear(), dateTime.getMonthValue(), dateTime.getDayOfMonth(), dateTime.getHour(), - dateTime.getMinute(), dateTime.getSecond(), nanoSecond / 1000); + dateTime.getMinute(), dateTime.getSecond(), + (dateTime.getNano() / 1000) / value * value); } @Override @@ -475,12 +503,13 @@ public boolean equals(Object o) { if (this == o) { return true; } - if (!(o instanceof DateTimeV2Literal)) { + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { return false; } DateTimeV2Literal literal = (DateTimeV2Literal) o; - return Objects.equals(getValue(), literal.getValue()) - && Objects.equals(dataType, literal.dataType) - && Objects.equals(nanoSecond, literal.nanoSecond); + return Objects.equals(dataType, literal.dataType) && Objects.equals(microSecond, literal.microSecond); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateV2Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateV2Literal.java index d7a108cce4f493..1462397bb08092 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateV2Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateV2Literal.java @@ -95,7 +95,7 @@ public DateTimeV2Literal toBeginOfTheDay() { * @return 2020-01-01 00:00:00 */ public DateTimeV2Literal toBeginOfTheDay(DateTimeV2Type dateType) { - return DateTimeV2Literal.create(dateType, year, month, day, 0, 0, 0, 000000); + return new DateTimeV2Literal(dateType, year, month, day, 0, 0, 0, 000000); } /** @@ -111,7 +111,15 @@ public DateTimeV2Literal toEndOfTheDay() { * @return 2020-01-01 23:59:59.9[scale] */ public DateTimeV2Literal toEndOfTheDay(DateTimeV2Type dateType) { - return DateTimeV2Literal.createEndOfDay(dateType, year, month, day); + long microSecond = 0; + // eg. scale == 4 -> 999900 + for (int i = 0; i < 6; ++i) { + microSecond *= 10; + if (i < dateType.getScale()) { + microSecond += 9; + } + } + return new DateTimeV2Literal(dateType, year, month, day, 23, 59, 59, microSecond); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/NumericLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/NumericLiteral.java index c46aa019e98be4..802721b15b36db 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/NumericLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/NumericLiteral.java @@ -174,7 +174,7 @@ protected Expression getDateLikeLiteral(String s, DataType targetType) { return new TimeStampNsLiteral(s); } if (targetType instanceof DateTimeV2Type) { - return DateTimeV2Literal.create((DateTimeV2Type) targetType, s); + return new DateTimeV2Literal((DateTimeV2Type) targetType, s); } if (targetType instanceof TimeStampTzType) { return new TimestampTzLiteral((TimeStampTzType) targetType, s); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteral.java index f6ad99ad91b924..e9cbf6197ca931 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteral.java @@ -389,7 +389,7 @@ protected DateLiteral getDateTimeLiteral(String year, String month, String date, } } else { try { - return DateTimeV2Literal.create((DateTimeV2Type) targetType, format); + return new DateTimeV2Literal((DateTimeV2Type) targetType, format); } catch (AnalysisException e) { throw new CastException(e.getMessage(), e); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimeStampNsLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimeStampNsLiteral.java index 805e4bca6cd3e4..194add743c9a41 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimeStampNsLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimeStampNsLiteral.java @@ -19,19 +19,10 @@ import org.apache.doris.analysis.LiteralExpr; import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.nereids.exceptions.CastException; -import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; -import org.apache.doris.nereids.types.DataType; -import org.apache.doris.nereids.types.DateTimeType; -import org.apache.doris.nereids.types.DateTimeV2Type; import org.apache.doris.nereids.types.TimeStampNsType; -import org.apache.doris.nereids.types.TimeStampTzType; -import org.apache.doris.nereids.types.TimeV2Type; import org.apache.doris.nereids.util.DateUtils; -import java.math.BigInteger; -import java.time.DateTimeException; import java.time.LocalDateTime; import java.time.ZoneId; import java.time.temporal.ChronoField; @@ -122,6 +113,7 @@ public static TimeStampNsLiteral fromJavaDateType(LocalDateTime dateTime) { return new TimeStampNsLiteral(dateTime); } + /** Return whether the civil fields are invalid or outside the signed epoch-nanosecond range. */ public boolean checkRange() { if (checkRange(year, month, day) || month < 1 || day < 1 || checkDate(year, month, day) || hour < 0 || hour > 23 || minute < 0 || minute > 59 @@ -179,96 +171,6 @@ public LiteralExpr toLegacyLiteral() { year, month, day, hour, minute, second, nanosecond); } - @Override - protected Expression uncheckedCastTo(DataType targetType) throws AnalysisException { - if (dataType.equals(targetType)) { - return this; - } - if (targetType.isIntegralType()) { - if (targetType.isBigIntType()) { - return new BigIntLiteral(getValue()); - } else if (targetType.isLargeIntType()) { - return new LargeIntLiteral(new BigInteger(String.valueOf(getValue()))); - } - throw new AnalysisException("TIMESTAMP_NS can not cast to " + targetType); - } else if (targetType.isDateV2Type()) { - return new DateV2Literal(year, month, day); - } else if (targetType.isDateType()) { - return new DateLiteral(year, month, day); - } else if (targetType.isDateTimeType()) { - return new DateTimeLiteral((DateTimeType) targetType, - year, month, day, hour, minute, second, nanosecond / 1000); - } else if (targetType.isDateTimeV2Type()) { - try { - DateTimeV2Type dateTimeV2Type = (DateTimeV2Type) targetType; - int scale = dateTimeV2Type.getScale(); - long factor = (long) Math.pow(10, DateUtils.NANOSECOND_SCALE - scale); - LocalDateTime roundedDateTime = toJavaDateType().plusNanos(factor / 2); - return DateTimeV2Literal.fromJavaDateType(roundedDateTime, dateTimeV2Type); - } catch (AnalysisException | DateTimeException e) { - throw new CastException(e.getMessage(), e); - } - } else if (targetType.isTimeStampTzType()) { - TimeStampTzType timeStampTzType = (TimeStampTzType) targetType; - DateTimeV2Literal rounded = (DateTimeV2Literal) uncheckedCastTo( - DateTimeV2Type.of(timeStampTzType.getScale())); - return TimestampTzLiteral.fromSessionTimeZone(timeStampTzType, rounded); - } else if (targetType.isTimeType()) { - int scale = ((TimeV2Type) targetType).getScale(); - long factor = (long) Math.pow(10, DateUtils.NANOSECOND_SCALE - scale); - long roundedNanosecond = (nanosecond + factor / 2) / factor * factor; - long totalNanosecond = ((hour * 60 + minute) * 60 + second) * NANOS_PER_SECOND - + roundedNanosecond; - int resultHour = (int) (totalNanosecond / NANOS_PER_SECOND / 60 / 60); - int resultMinute = (int) (totalNanosecond / NANOS_PER_SECOND / 60 % 60); - int resultSecond = (int) (totalNanosecond / NANOS_PER_SECOND % 60); - int resultMicrosecond = (int) (totalNanosecond % NANOS_PER_SECOND / 1000); - return new TimeV2Literal(resultHour, resultMinute, resultSecond, - resultMicrosecond, scale, false); - } else if (targetType.isFloatType()) { - return new FloatLiteral(getValue()); - } else if (targetType.isDoubleType()) { - return new DoubleLiteral(getValue()); - } - return super.uncheckedCastTo(targetType); - } - - public TimeStampNsLiteral plusDays(long days) { - return fromJavaDateType(toJavaDateType().plusDays(days)); - } - - public TimeStampNsLiteral plusMonths(long months) { - return fromJavaDateType(toJavaDateType().plusMonths(months)); - } - - public TimeStampNsLiteral plusWeeks(long weeks) { - return fromJavaDateType(toJavaDateType().plusWeeks(weeks)); - } - - public TimeStampNsLiteral plusYears(long years) { - return fromJavaDateType(toJavaDateType().plusYears(years)); - } - - public TimeStampNsLiteral plusHours(long hours) { - return fromJavaDateType(toJavaDateType().plusHours(hours)); - } - - public TimeStampNsLiteral plusMinutes(long minutes) { - return fromJavaDateType(toJavaDateType().plusMinutes(minutes)); - } - - public TimeStampNsLiteral plusSeconds(long seconds) { - return fromJavaDateType(toJavaDateType().plusSeconds(seconds)); - } - - public TimeStampNsLiteral plusMicroSeconds(long microseconds) { - return fromJavaDateType(toJavaDateType().plusNanos(Math.multiplyExact(microseconds, 1000L))); - } - - public TimeStampNsLiteral plusMilliSeconds(long milliseconds) { - return plusMicroSeconds(Math.multiplyExact(milliseconds, 1000L)); - } - public DateTimeV2Literal roundFloorToDateTimeV2(int scale) { long factor = (long) Math.pow(10, DateUtils.NANOSECOND_SCALE - scale); LocalDateTime rounded = toJavaDateType().withNano((int) (nanosecond / factor * factor)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimeV2Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimeV2Literal.java index bab104370b1ad7..8d04a006baf51a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimeV2Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimeV2Literal.java @@ -315,7 +315,7 @@ protected Expression uncheckedCastTo(DataType targetType) throws AnalysisExcepti return new DateTimeLiteral(time.getYear(), time.getMonth(), time.getDay(), time.getHour(), time.getMinute(), time.getSecond()); } else if (targetType.isDateTimeV2Type()) { - return time.uncheckedCastTo(targetType); + return time; } return super.uncheckedCastTo(targetType); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteral.java index 359ff0e0b90f4f..810168d4985a51 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteral.java @@ -109,10 +109,8 @@ public static TimestampTzLiteral fromTimeZone(TimeStampTzType dateType, String s */ public static TimestampTzLiteral fromTimeZone(TimeStampTzType dateType, DateTimeV2Literal literal, String timeZone) { - DateTimeV2Literal roundedLiteral = (DateTimeV2Literal) literal.uncheckedCastTo( - DateTimeV2Type.of(dateType.getScale())); DateTimeV2Literal utcLiteral = (DateTimeV2Literal) DateTimeExtractAndTransform.convertTz( - roundedLiteral, + literal, new StringLiteral(timeZone), new StringLiteral("UTC")); return new TimestampTzLiteral(dateType, @@ -173,13 +171,13 @@ protected Expression uncheckedCastTo(DataType targetType) throws AnalysisExcepti return new TimestampTzLiteral((TimeStampTzType) targetType, year, month, day, hour, minute, second, microSecond); } else if (targetType.isDateTimeV2Type()) { - DateTimeV2Literal dtV2Lit = DateTimeV2Literal.create(DateTimeV2Type.of(getDataType().getScale()), + DateTimeV2Literal dtV2Lit = new DateTimeV2Literal((DateTimeV2Type) targetType, year, month, day, hour, minute, second, microSecond); dtV2Lit = (DateTimeV2Literal) (DateTimeExtractAndTransform.convertTz( dtV2Lit, new StringLiteral("UTC"), new StringLiteral(ConnectContext.get().getSessionVariable().timeZone))); - return dtV2Lit.uncheckedCastTo(targetType); + return dtV2Lit; } throw new AnalysisException(String.format("Cast from %s to %s not supported", this, targetType)); } @@ -442,7 +440,7 @@ public TimestampTzLiteral roundFloor(int newScale) { @Override protected void roundMicroSecond(int scale) { - Preconditions.checkArgument(scale >= 0 && scale <= TimeStampTzType.MAX_SCALE, + Preconditions.checkArgument(scale >= 0 && scale <= DateTimeV2Type.MAX_SCALE, "invalid datetime v2 scale: %s", scale); double factor = Math.pow(10, 6 - scale); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeV2Type.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeV2Type.java index 4406a346695420..5f414c9af1dfbe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeV2Type.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeV2Type.java @@ -28,6 +28,8 @@ import org.apache.doris.nereids.types.coercion.IntegralType; import org.apache.doris.nereids.types.coercion.ScaleTimeType; +import com.google.common.base.Preconditions; + import java.time.DateTimeException; import java.time.LocalDateTime; import java.time.temporal.ChronoUnit; @@ -46,7 +48,7 @@ public class DateTimeV2Type extends DateLikeType implements ScaleTimeType { private final int scale; - private DateTimeV2Type(int scale) { + protected DateTimeV2Type(int scale) { this.scale = scale; } @@ -63,11 +65,11 @@ public static DateTimeV2Type of(int scale) { } } - /** Return the wider type without losing the distinct TIMESTAMP_NS physical type. */ + /** Return the wider DATETIMEV2 type. */ public static DateTimeV2Type getWiderDatetimeV2Type(DateTimeV2Type t1, DateTimeV2Type t2) { - if (t1 instanceof TimeStampNsType || t2 instanceof TimeStampNsType) { - return TimeStampNsType.INSTANCE; - } + Preconditions.checkArgument(t1.getClass() == DateTimeV2Type.class + && t2.getClass() == DateTimeV2Type.class, + "TIMESTAMP_NS does not belong to the DATETIMEV2 precision family"); if (t1.scale > t2.scale) { return t1; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/coercion/DateLikeType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/coercion/DateLikeType.java index 4c5eb100e3ee90..f931ff2ce27c10 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/coercion/DateLikeType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/coercion/DateLikeType.java @@ -75,7 +75,7 @@ public DateLiteral fromString(String s) { } else if (this instanceof TimeStampNsType) { return new TimeStampNsLiteral(s); } else if (this instanceof DateTimeV2Type) { - return DateTimeV2Literal.create((DateTimeV2Type) this, s); + return new DateTimeV2Literal((DateTimeV2Type) this, s); } else { throw new AnalysisException("unknown date like type"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java index 64dc3316ef4aed..6596a4befb515d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java @@ -237,6 +237,21 @@ public static Optional implicitCast(DataType input, DataType expected) * Return Optional.empty() if we cannot do implicit cast. */ public static Optional implicitCastPrimitive(DataType input, DataType expected) { + // TIMESTAMP_NS has a different physical representation from DATETIMEV2. Until explicit + // cross-type casts are implemented, only its string conversions can be implicit. + if (input instanceof TimeStampNsType || expected instanceof TimeStampNsType) { + if (input.equals(expected) || expected instanceof AnyDataType) { + return Optional.of(input); + } else if (input instanceof NullType) { + return Optional.of(expected.defaultConcreteType()); + } else if (input instanceof CharacterType && expected instanceof TimeStampNsType) { + return Optional.of(expected); + } else if (input instanceof TimeStampNsType && expected instanceof CharacterType) { + return Optional.of(expected.defaultConcreteType()); + } + return Optional.empty(); + } + Optional castType = implicitCastPrimitiveInternal(input, expected); // TODO: complete the cast logic like FunctionCallExpr.analyzeImpl boolean legacyCastCompatible = false; @@ -336,7 +351,7 @@ public static boolean hasDecimalV3Type(DataType dataType) { } public static boolean hasDateTimeV2Type(DataType dataType) { - return hasSpecifiedType(dataType, DateTimeV2Type.class); + return hasSpecifiedType(dataType, type -> type.getClass() == DateTimeV2Type.class); } public static boolean hasTimeStampNsType(DataType dataType) { @@ -352,16 +367,20 @@ public static boolean hasTimestampTzType(DataType dataType) { } private static boolean hasSpecifiedType(DataType dataType, Class specifiedType) { + return hasSpecifiedType(dataType, type -> specifiedType.isAssignableFrom(type.getClass())); + } + + private static boolean hasSpecifiedType(DataType dataType, Predicate predicate) { if (dataType instanceof ArrayType) { - return hasSpecifiedType(((ArrayType) dataType).getItemType(), specifiedType); + return hasSpecifiedType(((ArrayType) dataType).getItemType(), predicate); } else if (dataType instanceof MapType) { - return hasSpecifiedType(((MapType) dataType).getKeyType(), specifiedType) - || hasSpecifiedType(((MapType) dataType).getValueType(), specifiedType); + return hasSpecifiedType(((MapType) dataType).getKeyType(), predicate) + || hasSpecifiedType(((MapType) dataType).getValueType(), predicate); } else if (dataType instanceof StructType) { return ((StructType) dataType).getFields().stream() - .anyMatch(f -> hasSpecifiedType(f.getDataType(), specifiedType)); + .anyMatch(f -> hasSpecifiedType(f.getDataType(), predicate)); } - return specifiedType.isAssignableFrom(dataType.getClass()); + return predicate.test(dataType); } /** @@ -389,9 +408,9 @@ public static DataType replaceDateTimeV2WithMax(DataType dataType) { } private static DataType replaceDateLikeWithMaxPrecision(DataType dataType) { - DataType targetType = hasTimeStampNsType(dataType) - ? TimeStampNsType.INSTANCE : DateTimeV2Type.MAX; - return replaceSpecifiedType(dataType, DateLikeType.class, targetType); + return replaceSpecifiedType(dataType, + type -> type instanceof DateLikeType && !(type instanceof TimeStampNsType), + DateTimeV2Type.MAX); } /** @@ -1020,6 +1039,8 @@ private static Optional getCommonDataTypeWithDateType(DataType leftTyp */ private static Optional getCommonDataTypeWithDateTimeV2Type( DateTimeV2Type leftType, DataType rightType) { + Preconditions.checkArgument(leftType.getClass() == DateTimeV2Type.class, + "TIMESTAMP_NS does not belong to the DATETIMEV2 precision family"); if (rightType.isNumericType()) { if (rightType instanceof IntegralType) { return Optional.of(leftType); @@ -1041,15 +1062,15 @@ private static Optional getCommonDataTypeWithDateTimeV2Type( return Optional.of(DateTimeV2Type.getWiderDatetimeV2Type(leftType, dateTimeV2Type)); } else if (rightType instanceof TimeStampTzType) { TimeStampTzType timeStampTzType = (TimeStampTzType) rightType; - return leftType instanceof TimeStampNsType ? Optional.of(leftType) - : Optional.of(DateTimeV2Type.of(Math.max(leftType.getScale(), timeStampTzType.getScale()))); + return Optional.of(DateTimeV2Type.of( + Math.max(leftType.getScale(), timeStampTzType.getScale()))); } else { return Optional.of(leftType); } } else if (rightType instanceof TimeV2Type) { TimeV2Type timeV2Type = (TimeV2Type) rightType; - return leftType instanceof TimeStampNsType ? Optional.of(leftType) - : Optional.of(DateTimeV2Type.of(Math.max(leftType.getScale(), timeV2Type.getScale()))); + return Optional.of(DateTimeV2Type.of( + Math.max(leftType.getScale(), timeV2Type.getScale()))); } else if (rightType.isStringLikeType()) { return Optional.of(DateTimeV2Type.forTypeWithMinimumScale( leftType, DateTimeV2Type.MAX_SCALE)); @@ -1057,6 +1078,17 @@ private static Optional getCommonDataTypeWithDateTimeV2Type( return Optional.empty(); } + /** + * TIMESTAMP_NS is not a high-scale DATETIMEV2. It can only share a coercion target with + * strings until casts between TIMESTAMP_NS and other temporal or numeric types are available. + */ + private static Optional getCommonDataTypeWithTimeStampNsType(DataType otherType) { + if (otherType instanceof TimeStampNsType || otherType.isStringLikeType()) { + return Optional.of(TimeStampNsType.INSTANCE); + } + return Optional.empty(); + } + /** * left type must be TimeV2Type Type */ @@ -1251,6 +1283,12 @@ private static Optional findWiderPrimitiveTypeForTwo( } } + if (leftType instanceof TimeStampNsType) { + return getCommonDataTypeWithTimeStampNsType(rightType); + } else if (rightType instanceof TimeStampNsType) { + return getCommonDataTypeWithTimeStampNsType(leftType); + } + // we process date like type first if (!leftType.isDateLikeType() && rightType.isDateLikeType()) { DataType temp = leftType; @@ -1779,6 +1817,12 @@ private static Optional findCommonPrimitiveTypeForComparison( return Optional.of(leftType); } + if (leftType instanceof TimeStampNsType) { + return getCommonDataTypeWithTimeStampNsType(rightType); + } else if (rightType instanceof TimeStampNsType) { + return getCommonDataTypeWithTimeStampNsType(leftType); + } + // decimal v3 if (leftType.isDecimalV3Type() && rightType.isDecimalV3Type()) { return Optional.of(DecimalV3Type.widerDecimalV3Type( @@ -2052,6 +2096,10 @@ public static Optional findCommonPrimitiveTypeForCaseWhen(DataType t1, return Optional.of(StringType.INSTANCE); } + if (t1 instanceof TimeStampNsType || t2 instanceof TimeStampNsType) { + return Optional.empty(); + } + // decimal with date should return double if ((t1.isDecimalV2Type() && t2.isDateType()) || (t2.isDecimalV2Type() && t1.isDateType())) { return Optional.of(DoubleType.INSTANCE); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/connector/converter/NereidsToConnectorExpressionConverterTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/connector/converter/NereidsToConnectorExpressionConverterTest.java index 717ff586bad95d..187eddf9e3d938 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/connector/converter/NereidsToConnectorExpressionConverterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/connector/converter/NereidsToConnectorExpressionConverterTest.java @@ -52,6 +52,7 @@ import org.apache.doris.nereids.trees.expressions.literal.DecimalV3Literal; import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; import org.apache.doris.nereids.trees.expressions.literal.SmallIntLiteral; +import org.apache.doris.nereids.trees.expressions.literal.TimeStampNsLiteral; import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; import org.apache.doris.nereids.types.BigIntType; @@ -148,7 +149,7 @@ public void dateAndDatetimeLiteralsBecomeJavaTimeValues() { ConnectorLiteral nanoTs = rightLiteralOf(NereidsToConnectorExpressionConverter.convert( new EqualTo(slot("timestamp_ns", ScalarType.createTimeStampNsType()), - new DateTimeV2Literal("2024-01-02 12:34:56.123456789")))); + new TimeStampNsLiteral("2024-01-02 12:34:56.123456789")))); Assertions.assertEquals(LocalDateTime.of(2024, 1, 2, 12, 34, 56, 123456789), nanoTs.getValue()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/ComputeSignatureHelperTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/ComputeSignatureHelperTest.java index b410391b9b1e45..e0e1f0f30b6be9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/ComputeSignatureHelperTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/ComputeSignatureHelperTest.java @@ -30,7 +30,6 @@ import org.apache.doris.nereids.trees.expressions.literal.MapLiteral; import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.trees.expressions.literal.SmallIntLiteral; -import org.apache.doris.nereids.trees.expressions.literal.TimeStampNsLiteral; import org.apache.doris.nereids.trees.expressions.literal.TimeV2Literal; import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.BigIntType; @@ -46,7 +45,6 @@ import org.apache.doris.nereids.types.MapType; import org.apache.doris.nereids.types.NullType; import org.apache.doris.nereids.types.SmallIntType; -import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TimeV2Type; import org.apache.doris.nereids.types.coercion.AnyDataType; import org.apache.doris.nereids.types.coercion.FollowToAnyDataType; @@ -451,50 +449,6 @@ void testMixedDateTimeV2AndTimeV2PrecisionPromotion() { Assertions.assertEquals(DateTimeV2Type.of(3), signature.returnType); } - @Test - void testDateTimeV2SignatureDoesNotPromoteToTimestampNs() { - FunctionSignature signature = FunctionSignature.ret(DateV2Type.INSTANCE) - .args(DateTimeV2Type.SYSTEM_DEFAULT); - List arguments = Lists.newArrayList( - new TimeStampNsLiteral("2020-02-02 00:00:00.123456789")); - - signature = ComputeSignatureHelper.computePrecision( - new FakeComputeSignature(), signature, arguments); - - Assertions.assertEquals(DateTimeV2Type.MAX, signature.getArgType(0)); - Assertions.assertEquals(DateV2Type.INSTANCE, signature.returnType); - } - - @Test - void testTimestampNsDoesNotPromoteDateTimeV2OrTimeV2Signatures() { - FunctionSignature signature = FunctionSignature.ret(DateTimeV2Type.SYSTEM_DEFAULT) - .args(DateTimeV2Type.SYSTEM_DEFAULT, TimeV2Type.SYSTEM_DEFAULT); - List arguments = Lists.newArrayList( - new TimeStampNsLiteral("2020-02-02 00:00:00.123456789"), - new TimeV2Literal("12:34:56.123456")); - - signature = ComputeSignatureHelper.computePrecision( - new FakeComputeSignature(), signature, arguments); - - Assertions.assertEquals(DateTimeV2Type.MAX, signature.getArgType(0)); - Assertions.assertEquals(TimeV2Type.of(6), signature.getArgType(1)); - Assertions.assertEquals(DateTimeV2Type.MAX, signature.returnType); - } - - @Test - void testExplicitTimestampNsSignatureRemainsTimestampNs() { - FunctionSignature signature = FunctionSignature.ret(TimeStampNsType.INSTANCE) - .args(TimeStampNsType.INSTANCE); - List arguments = Lists.newArrayList( - new TimeStampNsLiteral("2020-02-02 00:00:00.123456789")); - - signature = ComputeSignatureHelper.computePrecision( - new FakeComputeSignature(), signature, arguments); - - Assertions.assertEquals(TimeStampNsType.INSTANCE, signature.getArgType(0)); - Assertions.assertEquals(TimeStampNsType.INSTANCE, signature.returnType); - } - @Test void testNestedTimeV2PrecisionPromotion() { FunctionSignature signature = FunctionSignature.ret(ArrayType.of(TimeV2Type.SYSTEM_DEFAULT)).args( diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateTimeMicrosecondFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateTimeMicrosecondFunctionTest.java deleted file mode 100644 index ad249d092dcf99..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateTimeMicrosecondFunctionTest.java +++ /dev/null @@ -1,120 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.trees.expressions.functions.scalar; - -import org.apache.doris.catalog.FunctionSignature; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral; -import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; -import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; -import org.apache.doris.nereids.types.DataType; -import org.apache.doris.nereids.types.DateTimeV2Type; -import org.apache.doris.nereids.types.StringType; -import org.apache.doris.nereids.types.TimeStampTzType; -import org.apache.doris.nereids.types.TimeV2Type; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.util.List; - -class DateTimeMicrosecondFunctionTest { - - @Test - void testDateTimeV2MicrosecondArithmeticMinimumScale() { - assertDateTimeV2Scale(0, DateTimeV2Type.MAX_SCALE); - assertDateTimeV2Scale(6, DateTimeV2Type.MAX_SCALE); - } - - @Test - void testTimestampTzMicrosecondArithmeticScale() { - Expression timestamp = new SlotReference("timestamp", TimeStampTzType.of(3)); - for (ScalarFunction function : createFunctions(timestamp)) { - FunctionSignature signature = function.getSignature(); - Assertions.assertEquals(TimeStampTzType.MAX, signature.getArgType(0), function.getName()); - Assertions.assertEquals(TimeStampTzType.MAX, signature.returnType, function.getName()); - } - } - - @Test - void testTimestampTzStringLiteralKeepsTimezoneType() { - SecondMicrosecondAdd function = new SecondMicrosecondAdd( - new VarcharLiteral("2004-11-30 01:00:00+08:00"), new VarcharLiteral("10.123456")); - FunctionSignature signature = function.getSignature(); - Assertions.assertEquals(TimeStampTzType.MAX, signature.getArgType(0)); - Assertions.assertEquals(TimeStampTzType.MAX, signature.returnType); - } - - @Test - void testImplicitStringDateTimeFunctionsKeepMicrosecondPrecision() { - Expression stringSlot = new SlotReference("string", StringType.INSTANCE); - Expression nullableString = new Nullable(new VarcharLiteral("9999-01-01 00:00:00")); - - assertDateTimeV2MicrosecondSignature(new AddTime(stringSlot, - new SlotReference("time", TimeV2Type.MAX)), 0); - assertDateTimeV2MicrosecondSignature(new DateFormat(stringSlot, - new VarcharLiteral("%Y-%m-%d")), 0); - assertDateTimeV2MicrosecondSignature(new ToSeconds(stringSlot), 0); - assertDateTimeV2MicrosecondSignature(new YearsAdd(nullableString, new IntegerLiteral(-1)), 0); - assertDateTimeV2MicrosecondSignature(new Timestamp( - new VarcharLiteral("1st Jun 2007 09:45:30")), 0); - - FunctionSignature timeSignature = new Time(nullableString).getSignature(); - Assertions.assertEquals(DateTimeV2Type.MAX, timeSignature.getArgType(0)); - Assertions.assertEquals(TimeV2Type.MAX, timeSignature.returnType); - } - - private void assertDateTimeV2MicrosecondSignature(ScalarFunction function, int argumentIndex) { - FunctionSignature signature = function.getSignature(); - Assertions.assertEquals(DateTimeV2Type.MAX, - signature.getArgType(argumentIndex), function.getName()); - if (signature.returnType instanceof DateTimeV2Type) { - Assertions.assertEquals(DateTimeV2Type.MAX, - signature.returnType, function.getName()); - } - } - - private void assertDateTimeV2Scale(int inputScale, int expectedScale) { - Expression datetime = new SlotReference("datetime", DateTimeV2Type.of(inputScale)); - DataType expectedType = DateTimeV2Type.of(expectedScale); - for (ScalarFunction function : createFunctions(datetime)) { - FunctionSignature signature = function.getSignature(); - Assertions.assertEquals(expectedType, signature.getArgType(0), function.getName()); - Assertions.assertEquals(expectedType, signature.returnType, function.getName()); - } - } - - private List createFunctions(Expression datetime) { - BigIntLiteral numericInterval = new BigIntLiteral(1); - VarcharLiteral compositeInterval = new VarcharLiteral("1"); - return List.of( - new MicroSecondsAdd(datetime, numericInterval), - new MicroSecondsSub(datetime, numericInterval), - new MilliSecondsAdd(datetime, numericInterval), - new MilliSecondsSub(datetime, numericInterval), - new DayMicrosecondAdd(datetime, compositeInterval), - new DayMicrosecondSub(datetime, compositeInterval), - new HourMicrosecondAdd(datetime, compositeInterval), - new HourMicrosecondSub(datetime, compositeInterval), - new MinuteMicrosecondAdd(datetime, compositeInterval), - new MinuteMicrosecondSub(datetime, compositeInterval), - new SecondMicrosecondAdd(datetime, compositeInterval), - new SecondMicrosecondSub(datetime, compositeInterval)); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrToDateTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrToDateTest.java index 54104a48e7dbca..abb37cfc9198a6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrToDateTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrToDateTest.java @@ -18,11 +18,9 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar; import org.apache.doris.catalog.FunctionSignature; -import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; import org.apache.doris.nereids.types.DateTimeV2Type; import org.apache.doris.nereids.types.DateV2Type; -import org.apache.doris.nereids.types.StringType; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -35,19 +33,11 @@ void testComputeSignatureWithFoldableFormat() { new Concat(new StringLiteral("%Y-%m"), new StringLiteral("-%d"))); StrToDate dateTimeFormat = new StrToDate(new StringLiteral("2024-01-01 12:34:56"), new Concat(new StringLiteral("%Y-%m-%d"), new StringLiteral(" %H:%i:%s"))); - StrToDate microsecondFormat = new StrToDate(new StringLiteral("2024-01-01 12:34:56.123456"), - new Concat(new StringLiteral("%Y-%m-%d %H:%i:%s"), new StringLiteral(".%f"))); - StrToDate nonConstantFormat = new StrToDate(new StringLiteral("2024-01-01 12:34:56.123456"), - new SlotReference("format", StringType.INSTANCE)); FunctionSignature dateSignature = dateFormat.computeSignature(StrToDate.SIGNATURES.get(0)); FunctionSignature dateTimeSignature = dateTimeFormat.computeSignature(StrToDate.SIGNATURES.get(0)); - FunctionSignature microsecondSignature = microsecondFormat.computeSignature(StrToDate.SIGNATURES.get(0)); - FunctionSignature nonConstantSignature = nonConstantFormat.computeSignature(StrToDate.SIGNATURES.get(1)); Assertions.assertEquals(DateV2Type.INSTANCE, dateSignature.returnType); Assertions.assertEquals(DateTimeV2Type.SYSTEM_DEFAULT, dateTimeSignature.returnType); - Assertions.assertEquals(DateTimeV2Type.MAX, microsecondSignature.returnType); - Assertions.assertEquals(DateTimeV2Type.MAX, nonConstantSignature.returnType); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/window/TimestampTzLeadLagSignatureTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/window/TimestampTzLeadLagSignatureTest.java index 1d7a0920aaf7ee..238e863a745d2e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/window/TimestampTzLeadLagSignatureTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/window/TimestampTzLeadLagSignatureTest.java @@ -19,7 +19,6 @@ import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TimeStampTzType; import org.junit.jupiter.api.Assertions; @@ -30,19 +29,6 @@ */ public class TimestampTzLeadLagSignatureTest { - @Test - public void testLeadAndLagSignaturesMatchTimeStampNs() { - SlotReference slot = SlotReference.of("ts", TimeStampNsType.INSTANCE); - - FunctionSignature leadSignature = new Lead(slot).getSignature(); - Assertions.assertEquals(TimeStampNsType.INSTANCE, leadSignature.returnType); - Assertions.assertEquals(TimeStampNsType.INSTANCE, leadSignature.getArgType(0)); - - FunctionSignature lagSignature = new Lag(slot).getSignature(); - Assertions.assertEquals(TimeStampNsType.INSTANCE, lagSignature.returnType); - Assertions.assertEquals(TimeStampNsType.INSTANCE, lagSignature.getArgType(0)); - } - @Test public void testLeadSignatureMatchesTimeStampTz() { SlotReference slot = SlotReference.of("ts", TimeStampTzType.of(6)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteralTest.java index 9bdaf95d59b51c..115d10e59888e5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteralTest.java @@ -241,7 +241,7 @@ void testTimeStampNsseconds() { Assertions.assertInstanceOf(org.apache.doris.analysis.TimeStampNsLiteral.class, nanoseconds.toLegacyLiteral()); Assertions.assertThrows(IllegalArgumentException.class, - () -> DateTimeV2Literal.create(TimeStampNsType.INSTANCE, + () -> new DateTimeV2Literal(TimeStampNsType.INSTANCE, "1970-01-01 00:00:00.123456789")); Assertions.assertThrows(IllegalArgumentException.class, () -> new DateTimeV2Literal( diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteralTest.java index 9e1c73c4a0ee88..d7827722f858c9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteralTest.java @@ -33,8 +33,6 @@ import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.LargeIntType; import org.apache.doris.nereids.types.StringType; -import org.apache.doris.nereids.types.TimeStampNsType; -import org.apache.doris.nereids.types.TimeV2Type; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Disabled; @@ -754,43 +752,6 @@ void testDateTimeV2UncheckedCastTo() { Assertions.assertEquals("2025-07-23 13:25:59.000000", ((StringLiteral) expression).value); } - @Test - void testTimeStampNsUncheckedCastRoundingAndRange() { - TimeStampNsLiteral nano = new TimeStampNsLiteral("2024-02-29 12:34:56.123456789"); - DateTimeV2Literal micro = (DateTimeV2Literal) nano.uncheckedCastTo(DateTimeV2Type.of(6)); - Assertions.assertEquals("2024-02-29 12:34:56.123457", micro.getStringValue()); - - nano = new TimeStampNsLiteral("1969-12-31 23:59:59.999999999"); - micro = (DateTimeV2Literal) nano.uncheckedCastTo(DateTimeV2Type.of(6)); - Assertions.assertEquals("1970-01-01 00:00:00.000000", micro.getStringValue()); - TimeV2Literal time = (TimeV2Literal) nano.uncheckedCastTo(TimeV2Type.of(6)); - Assertions.assertEquals("24:00:00.000000", time.getStringValue()); - - nano = new TimeStampNsLiteral("2024-02-29 12:34:56.123456789"); - time = (TimeV2Literal) nano.uncheckedCastTo(TimeV2Type.of(6)); - Assertions.assertEquals("12:34:56.123457", time.getStringValue()); - - TimeStampNsLiteral lower = new TimeStampNsLiteral("1677-09-21 00:12:43.145224192"); - Assertions.assertSame(lower, lower.uncheckedCastTo(TimeStampNsType.INSTANCE)); - - TimeStampNsLiteral upper = new TimeStampNsLiteral("2262-04-11 23:47:16.854775807"); - Assertions.assertSame(upper, upper.uncheckedCastTo(TimeStampNsType.INSTANCE)); - - DateTimeV2Literal lowerMicro = new DateTimeV2Literal( - DateTimeV2Type.of(6), "1677-09-21 00:12:43.145225"); - Assertions.assertEquals("1677-09-21 00:12:43.145225000", - ((TimeStampNsLiteral) lowerMicro.uncheckedCastTo(TimeStampNsType.INSTANCE)).getStringValue()); - - DateTimeV2Literal yearZero = new DateTimeV2Literal( - DateTimeV2Type.of(6), "0000-01-01 00:00:00.000000"); - DateTimeV2Literal yearNineNineNineNine = new DateTimeV2Literal( - DateTimeV2Type.of(6), "9999-12-31 23:59:59.999999"); - Assertions.assertThrows(CastException.class, - () -> yearZero.uncheckedCastTo(TimeStampNsType.INSTANCE)); - Assertions.assertThrows(CastException.class, - () -> yearNineNineNineNine.uncheckedCastTo(TimeStampNsType.INSTANCE)); - } - @Test void testTimestampNsEpochNanoBoundaries() { Assertions.assertEquals("1677-09-21 00:12:43.145224192", diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimeV2LiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimeV2LiteralTest.java index 2b740edd37d3e6..96775d44bfc628 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimeV2LiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimeV2LiteralTest.java @@ -21,7 +21,6 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.types.DateTimeV2Type; import org.apache.doris.nereids.types.StringType; -import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TimeV2Type; import org.junit.jupiter.api.Assertions; @@ -187,17 +186,4 @@ public void testCastNegativeTimeToDateTimeV2KeepsSign() { Assertions.assertEquals(59, dateTime.getSecond()); } - @Test - public void testCastToTimeStampNsKeepsTargetScale() { - TimeV2Literal literal = new TimeV2Literal(TimeV2Type.of(6), "12:34:56.123456"); - - TimeStampNsLiteral dateTime = (TimeStampNsLiteral) literal.uncheckedCastTo(TimeStampNsType.INSTANCE); - - Assertions.assertEquals(TimeStampNsType.INSTANCE, dateTime.getDataType()); - Assertions.assertEquals(12, dateTime.getHour()); - Assertions.assertEquals(34, dateTime.getMinute()); - Assertions.assertEquals(56, dateTime.getSecond()); - Assertions.assertEquals(123456000, dateTime.getNanoSecond()); - } - } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteralTest.java index f5bb13f96f0901..ac5235fdf34500 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteralTest.java @@ -17,9 +17,7 @@ package org.apache.doris.nereids.trees.expressions.literal; -import org.apache.doris.nereids.exceptions.CastException; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.types.TimeStampNsType; import org.apache.doris.nereids.types.TimeStampTzType; import org.apache.doris.qe.ConnectContext; @@ -59,8 +57,6 @@ void testConstructorsAndParsing() { Assertions.assertEquals(14, literal.second); Assertions.assertEquals(123457, literal.microSecond); Assertions.assertEquals(6, literal.getDataType().getScale()); - Assertions.assertEquals(TimeStampTzType.of(6), - TimeStampTzType.MAX.scaleTypeForType(TimeStampNsType.INSTANCE)); literal = new TimestampTzLiteral("2022-12-31 21:00:14.1234567 -06:45"); Assertions.assertEquals(2023, literal.year); @@ -130,49 +126,6 @@ void testGetStringValueKeepsUtcSuffix() { Assertions.assertEquals("2024-01-15 12:00:00.000000+00:00", literal.getStringValue()); } - @Test - void testCastBetweenTimestampTzAndTimeStampNs() { - ConnectContext context = new ConnectContext(); - context.getSessionVariable().setTimeZone("+08:00"); - context.setThreadLocalInfo(); - try { - TimeStampNsLiteral nano = new TimeStampNsLiteral("2024-02-29 12:34:56.123456789"); - TimestampTzLiteral timestampTz = (TimestampTzLiteral) nano.uncheckedCastTo(TimeStampTzType.of(6)); - Assertions.assertEquals("2024-02-29 04:34:56.123457+00:00", timestampTz.getStringValue()); - - timestampTz = new TimestampTzLiteral( - TimeStampTzType.of(6), 2024, 2, 29, 4, 34, 56, 123456); - nano = (TimeStampNsLiteral) timestampTz.uncheckedCastTo(TimeStampNsType.INSTANCE); - Assertions.assertEquals("2024-02-29 12:34:56.123456000", nano.getStringValue()); - - context.getSessionVariable().setTimeZone("+14:00"); - timestampTz = new TimestampTzLiteral( - TimeStampTzType.of(6), 1677, 9, 20, 10, 12, 43, 145225); - nano = (TimeStampNsLiteral) timestampTz.uncheckedCastTo(TimeStampNsType.INSTANCE); - Assertions.assertEquals("1677-09-21 00:12:43.145225000", nano.getStringValue()); - - context.getSessionVariable().setTimeZone("UTC"); - TimestampTzLiteral yearZero = new TimestampTzLiteral( - TimeStampTzType.of(6), 0, 1, 1, 0, 0, 0, 0); - TimestampTzLiteral yearNineNineNineNine = new TimestampTzLiteral( - TimeStampTzType.of(6), 9999, 12, 31, 23, 59, 59, 999999); - TimestampTzLiteral belowLowerBound = new TimestampTzLiteral( - TimeStampTzType.of(6), 1677, 9, 21, 0, 12, 43, 145224); - TimestampTzLiteral aboveUpperBound = new TimestampTzLiteral( - TimeStampTzType.of(6), 2262, 4, 11, 23, 47, 16, 854776); - Assertions.assertThrows(CastException.class, - () -> yearZero.uncheckedCastTo(TimeStampNsType.INSTANCE)); - Assertions.assertThrows(CastException.class, - () -> yearNineNineNineNine.uncheckedCastTo(TimeStampNsType.INSTANCE)); - Assertions.assertThrows(CastException.class, - () -> belowLowerBound.uncheckedCastTo(TimeStampNsType.INSTANCE)); - Assertions.assertThrows(CastException.class, - () -> aboveUpperBound.uncheckedCastTo(TimeStampNsType.INSTANCE)); - } finally { - ConnectContext.remove(); - } - } - @Test void testBasicArithmetic() { TimestampTzLiteral base1 = new TimestampTzLiteral("2025-12-31 23:59:59.999999"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionMatrixTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionMatrixTest.java index a8741482501405..3ae70c57fee20b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionMatrixTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionMatrixTest.java @@ -696,8 +696,10 @@ public void testProcessComparisonPredicateForDateTimeV2Type() { testProcessComparisonPredicate(DateTimeV2Type.of(4), CharType.createCharType(5), DateTimeV2Type.of(6)); testProcessComparisonPredicate(DateTimeV2Type.of(4), VarcharType.createVarcharType(5), DateTimeV2Type.of(6)); testProcessComparisonPredicate(DateTimeV2Type.of(4), StringType.INSTANCE, DateTimeV2Type.of(6)); - testProcessComparisonPredicate(TimeStampNsType.INSTANCE, DecimalV2Type.SYSTEM_DEFAULT, - TimeStampNsType.INSTANCE); + testProcessComparisonPredicate(TimeStampNsType.INSTANCE, DecimalV2Type.SYSTEM_DEFAULT, null); + testProcessComparisonPredicate(TimeStampNsType.INSTANCE, DateV2Type.INSTANCE, null); + testProcessComparisonPredicate(TimeStampNsType.INSTANCE, DateTimeV2Type.MAX, null); + testProcessComparisonPredicate(TimeStampNsType.INSTANCE, TimeV2Type.MAX, null); testProcessComparisonPredicate(TimeStampNsType.INSTANCE, StringType.INSTANCE, TimeStampNsType.INSTANCE); testProcessComparisonPredicate(DateTimeV2Type.of(4), ArrayType.of(StringType.INSTANCE), null); testProcessComparisonPredicate(DateTimeV2Type.of(4), MapType.of(StringType.INSTANCE, StringType.INSTANCE), null); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionUtilsTest.java index cc3baf2c7800d6..05f4879bad5164 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionUtilsTest.java @@ -81,6 +81,7 @@ import org.apache.doris.nereids.types.TinyIntType; import org.apache.doris.nereids.types.VarcharType; import org.apache.doris.nereids.types.VariantType; +import org.apache.doris.nereids.types.coercion.AnyDataType; import org.apache.doris.nereids.types.coercion.IntegralType; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.GlobalVariable; @@ -111,6 +112,84 @@ public void testVariantWiderTypeKeepsTimestampNsPrecision() { Assertions.assertEquals(Optional.of(ArrayType.of(TimeStampNsType.INSTANCE)), TypeCoercionUtils.findWiderTypeForTwo( VariantType.INSTANCE, ArrayType.of(TimeStampNsType.INSTANCE), false, false)); + + StructType mixedDateLikeType = new StructType(ImmutableList.of( + new StructField("ts", TimeStampNsType.INSTANCE, false, ""), + new StructField("date", DateV2Type.INSTANCE, true, ""))); + StructType normalizedMixedDateLikeType = new StructType(ImmutableList.of( + new StructField("ts", TimeStampNsType.INSTANCE, false, ""), + new StructField("date", DateTimeV2Type.MAX, true, ""))); + Assertions.assertEquals(Optional.of(normalizedMixedDateLikeType), + TypeCoercionUtils.findWiderTypeForTwo( + VariantType.INSTANCE, mixedDateLikeType, false, false)); + } + + @Test + public void testTimestampNsIsNotInDateTimeV2PrecisionFamily() { + Assertions.assertTrue(TypeCoercionUtils.hasTimeStampNsType(TimeStampNsType.INSTANCE)); + Assertions.assertFalse(TypeCoercionUtils.hasDateTimeV2Type(TimeStampNsType.INSTANCE)); + Assertions.assertFalse(TypeCoercionUtils.hasDateTimeV2Type( + ArrayType.of(TimeStampNsType.INSTANCE))); + Assertions.assertTrue(TypeCoercionUtils.hasDateTimeV2Type(DateTimeV2Type.MAX)); + + Assertions.assertThrows(IllegalArgumentException.class, + () -> DateTimeV2Type.getWiderDatetimeV2Type( + DateTimeV2Type.MAX, TimeStampNsType.INSTANCE)); + } + + @Test + public void testTimestampNsOnlyCoercesWithTimestampNsAndStrings() { + Assertions.assertEquals(Optional.of(TimeStampNsType.INSTANCE), + TypeCoercionUtils.findWiderTypeForTwo( + TimeStampNsType.INSTANCE, StringType.INSTANCE, false, false)); + Assertions.assertEquals(Optional.of(StringType.INSTANCE), + TypeCoercionUtils.findWiderTypeForTwo( + TimeStampNsType.INSTANCE, StringType.INSTANCE, false, true)); + Assertions.assertEquals(Optional.empty(), TypeCoercionUtils.findWiderTypeForTwo( + TimeStampNsType.INSTANCE, DateTimeV2Type.MAX, false, false)); + Assertions.assertEquals(Optional.empty(), TypeCoercionUtils.findWiderTypeForTwo( + DateV2Type.INSTANCE, TimeStampNsType.INSTANCE, false, false)); + Assertions.assertEquals(Optional.empty(), TypeCoercionUtils.findWiderTypeForTwo( + TimeStampNsType.INSTANCE, DecimalV2Type.SYSTEM_DEFAULT, false, false)); + + Assertions.assertEquals(Optional.of(TimeStampNsType.INSTANCE), + TypeCoercionUtils.implicitCast(StringType.INSTANCE, TimeStampNsType.INSTANCE)); + Assertions.assertEquals(Optional.of(StringType.INSTANCE), + TypeCoercionUtils.implicitCast(TimeStampNsType.INSTANCE, StringType.INSTANCE)); + Assertions.assertEquals(Optional.of(TimeStampNsType.INSTANCE), + TypeCoercionUtils.implicitCast(NullType.INSTANCE, TimeStampNsType.INSTANCE)); + Assertions.assertEquals(Optional.of(TimeStampNsType.INSTANCE), + TypeCoercionUtils.implicitCast( + TimeStampNsType.INSTANCE, AnyDataType.INSTANCE_WITHOUT_INDEX)); + Assertions.assertEquals(Optional.empty(), + TypeCoercionUtils.implicitCast(TimeStampNsType.INSTANCE, DateTimeV2Type.MAX)); + Assertions.assertEquals(Optional.empty(), + TypeCoercionUtils.implicitCast(DateTimeV2Type.MAX, TimeStampNsType.INSTANCE)); + } + + @Test + public void testTimestampNsComparisonCoercionInBothModes() { + boolean oldBehavior = GlobalVariable.enableNewTypeCoercionBehavior; + try { + for (boolean newBehavior : ImmutableList.of(false, true)) { + GlobalVariable.enableNewTypeCoercionBehavior = newBehavior; + + EqualTo stringComparison = (EqualTo) TypeCoercionUtils.processComparisonPredicate( + new EqualTo(new SlotReference("ts", TimeStampNsType.INSTANCE), + new SlotReference("s", StringType.INSTANCE))); + Assertions.assertEquals(TimeStampNsType.INSTANCE, + stringComparison.left().getDataType()); + Assertions.assertEquals(TimeStampNsType.INSTANCE, + stringComparison.right().getDataType()); + + Assertions.assertThrows(AnalysisException.class, + () -> TypeCoercionUtils.processComparisonPredicate( + new EqualTo(new SlotReference("ts", TimeStampNsType.INSTANCE), + new SlotReference("dt", DateTimeV2Type.MAX)))); + } + } finally { + GlobalVariable.enableNewTypeCoercionBehavior = oldBehavior; + } } @Test diff --git a/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns.out b/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns.out index e3debeb6621f95..24fe3a3935b62b 100644 --- a/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns.out +++ b/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns.out @@ -97,4 +97,3 @@ p_minimum [('1677-09-21 00:12:43.145224192'), ('1677-09-21 00:12:43.145224193')) -- !nonconstant_permissive_cast -- 1 2024-01-01 00:00:00.123456789 2 \N - diff --git a/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_binary_output.out b/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_binary_output.out index ab82a201b58716..6a2d4e04a126cf 100644 --- a/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_binary_output.out +++ b/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_binary_output.out @@ -6,7 +6,6 @@ 4 1970-01-01 00:00:00.000000001 5 2262-04-11 23:47:16.854775807 6 \N - -- !binary_protocol -- 1 1677-09-21 00:12:43.145224192 2 1969-12-31 23:59:59.999999999 @@ -14,4 +13,3 @@ 4 1970-01-01 00:00:00.000000001 5 2262-04-11 23:47:16.854775807 6 \N - diff --git a/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_ddl_predicate.out b/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_ddl_predicate.out index d02fde231c0b92..7be5664f5f7143 100644 --- a/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_ddl_predicate.out +++ b/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_ddl_predicate.out @@ -58,4 +58,3 @@ -- !aggregate_state -- 1 2262-04-11 23:47:16.854775807 - diff --git a/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_stream_load.out b/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_stream_load.out index 9d52997436061f..10d48d32611768 100644 --- a/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_stream_load.out +++ b/regression-test/data/datatype_p0/datetimev2/test_timestamp_ns_stream_load.out @@ -7,4 +7,3 @@ 5 2262-04-11 23:47:16.854775807 8 2023-08-17 09:41:18.123456789 9 2023-08-17 16:41:18.123456789 -