[WIP] [feature](timestamp) support timestamp data type by justfortaste · Pull Request #41760 · apache/doris

justfortaste · 2024-10-12T07:58:53Z

Proposed changes

Issue Number: close #xxx

mysql> drop table if exists t1;
Query OK, 0 rows affected (0.01 sec)

mysql> create table t1(id int, t timestamp)ENGINE=OLAP
    ->             UNIQUE KEY(id)
    ->             DISTRIBUTED BY HASH(id) BUCKETS 2 
    ->             properties(
    ->                 "replication_num" = "1",
    ->                 "enable_unique_key_merge_on_write" = "true",
    ->                 "disable_auto_compaction" = "true");
Query OK, 0 rows affected (0.05 sec)

mysql> 
mysql> set time_zone="+8:00";
Query OK, 0 rows affected (0.01 sec)

mysql> insert into t1 values(1, '2020-01-01 10:00:00');

Query OK, 1 row affected (1.31 sec)
{'label':'label_153fb3e333924a87_be864385a5c9e63b', 'status':'VISIBLE', 'txnId':'62060'}

mysql> select * from t1;
+------+---------------------+
| id   | t                   |
+------+---------------------+
|    1 | 2020-01-01 10:00:00 |
+------+---------------------+
1 row in set (16.54 sec)

mysql> set time_zone="+10:00";
Query OK, 0 rows affected (0.00 sec)

mysql> insert into t1 values(2, '2020-01-01 10:00:00');
Query OK, 1 row affected (0.06 sec)
{'label':'label_debe49b6d6e345e6_a890671ace1a437e', 'status':'VISIBLE', 'txnId':'62061'}

mysql> select * from t1;
+------+---------------------+
| id   | t                   |
+------+---------------------+
|    1 | 2020-01-01 12:00:00 |
|    2 | 2020-01-01 10:00:00 |
+------+---------------------+
2 rows in set (0.04 sec)

mysql> 
mysql> set time_zone="+8:00";
Query OK, 0 rows affected (0.00 sec)

mysql> select * from t1;
+------+---------------------+
| id   | t                   |
+------+---------------------+
|    1 | 2020-01-01 10:00:00 |
|    2 | 2020-01-01 08:00:00 |
+------+---------------------+
2 rows in set (0.03 sec)

todo: create table t1(id int, t timestamp)ENGINE=OLAP UNIQUE KEY(k) DISTRIBUTED BY HASH(id) BUCKETS 2 properties( "replication_num" = "1", "enable_unique_key_merge_on_write" = "true", "disable_auto_compaction" = "true"); ERROR 1105 (HY000): errCode = 2, detailMessage = Nereids do not support type: timestamp

ERROR 1109 (42S02): errCode = 2, detailMessage = errCode = 2, detailMessage = Unknown table 't2' in test MySQL [test]> create table t2(t timestamp, id int)ENGINE=OLAP -> UNIQUE KEY(t) -> DISTRIBUTED BY HASH(t) BUCKETS 2 -> properties( -> "replication_num" = "1", -> "enable_unique_key_merge_on_write" = "true", -> "disable_auto_compaction" = "true"); Query OK, 0 rows affected (0.01 sec) MySQL [test]> insert into t2 values('2020-01-01 10:00:00',1); Query OK, 1 row affected (0.06 sec) {'label':'label_1946a1a147b84131_b1e0100d5679ef25', 'status':'VISIBLE', 'txnId':'3'} MySQL [test]> insert into t2 values('2025-01-01 11:11:11',2); Query OK, 1 row affected (0.05 sec) {'label':'label_d231330c9b1a4b06_9cf94bcc8c440c81', 'status':'VISIBLE', 'txnId':'4'} MySQL [test]> select * from t2; +---------------------+------+ | t | id | +---------------------+------+ | 2020-01-01 10:00:00 | 1 | | 2025-01-01 11:11:11 | 2 | +---------------------+------+ 2 rows in set (0.03 sec)

todo: MySQL [test]> create table t2(t timestamp, id int)ENGINE=OLAP -> UNIQUE KEY(t) -> DISTRIBUTED BY HASH(t) BUCKETS 2 -> properties( -> "replication_num" = "1", -> "enable_unique_key_merge_on_write" = "true", -> "disable_auto_compaction" = "true"); Query OK, 0 rows affected (0.01 sec) MySQL [test]> insert into t2 values('2020-01-01 10:00:00',1); ERROR 1105 (HY000): errCode = 2, detailMessage = unknown date like type

todo: ERROR 1105 (HY000): send fragments failed. io.grpc.StatusRuntimeException: UNAVAILABLE: Network closed for unknown reason, host: 127.0.0.1 7# google::LogMessageFatal::~LogMessageFatal() in /data/ayuanzhang/mydoris/output/be/lib/doris_be 8# doris::thrift_to_type(doris::TPrimitiveType::type) in /data/ayuanzhang/mydoris/output/be/lib/doris_be 9# doris::TypeDescriptor::TypeDescriptor(std::vector<doris::TTypeNode, std::allocator<doris::TTypeNode> > const&, int*) in /data/ayuanzhang/mydoris/output/be/lib/doris_be 10# doris::TypeDescriptor::from_thrift(doris::TTypeDesc const&) at /data/ayuanzhang/mydoris/be/src/runtime/types.h:149 11# doris::SlotDescriptor::SlotDescriptor(doris::TSlotDescriptor const&) at /data/ayuanzhang/mydoris/be/src/runtime/descriptors.cpp:46 12# doris::DescriptorTbl::create(doris::ObjectPool*, doris::TDescriptorTable const&, doris::DescriptorTbl**) in /data/ayuanzhang/mydoris/output/be/lib/doris_be

mysql> set time_zone="+8:00"; Query OK, 0 rows affected (0.01 sec) mysql> insert into t1 values(1, '2020-01-01 10:00:00'); Query OK, 1 row affected (0.22 sec) {'label':'label_5961c3a91f6f406e_ad316bfcc1d66eaa', 'status':'VISIBLE', 'txnId':'30028'} mysql> set time_zone="+10:00"; Query OK, 0 rows affected (0.00 sec) mysql> insert into t1 values(2, '2020-01-01 10:00:00'); Query OK, 1 row affected (0.06 sec) {'label':'label_684e9fee3de24054_a6fd582a5fce6440', 'status':'VISIBLE', 'txnId':'30029'} mysql> select * from t1; +------+---------------------+ | id | t | +------+---------------------+ | 1 | 2020-01-01 10:00:00 | | 2 | 2020-01-01 12:00:00 | +------+---------------------+ 2 rows in set (0.04 sec)

compile ok

-> UNIQUE KEY(id) -> DISTRIBUTED BY HASH(id) BUCKETS 2 -> properties( -> "replication_num" = "1", -> "enable_unique_key_merge_on_write" = "true", -> "disable_auto_compaction" = "true"); Query OK, 0 rows affected (0.01 sec) mysql> set time_zone="+8:00"; Query OK, 0 rows affected (0.00 sec) mysql> insert into t1 values(1, '2020-01-01 10:00:00'); Query OK, 1 row affected (0.07 sec) {'label':'label_57f023e840c49ef_87835e9756a210fc', 'status':'VISIBLE', 'txnId':'42043'} mysql> select * from t1; +------+---------------------+ | id | t | +------+---------------------+ | 1 | 2020-01-01 11:00:00 | +------+---------------------+ 1 row in set (0.04 sec) todo: PGenericType::DATETIMEV2 PColumnType::COLUMN_TYPE_TIMESTAMP

doris-robot · 2024-10-12T07:58:57Z

Thank you for your contribution to Apache Doris.
Don't know what should be done next? See How to process your PR

Since 2024-03-18, the Document has been moved to doris-website.
See Doris Document.

github-actions

clang-tidy made some suggestions

github-actions · 2024-10-12T08:12:52Z

be/src/olap/rowset/beta_rowset_reader.h

                                 std::vector<RowwiseIteratorUPtr>* out_iters,
                                 bool use_cache = false) override;
+
+    long tz_offset() { return _tz_offset; }


warning: method 'tz_offset' can be made const [readability-make-member-function-const]

Suggested change

long tz_offset() { return _tz_offset; }

long tz_offset() const { return _tz_offset; }

github-actions · 2024-10-12T08:12:53Z

be/src/vec/columns/column_string.h


    void insert_many_binary_data(char* data_array, uint32_t* len_array,
-                                 uint32_t* start_offset_array, size_t num) override {
+                                 uint32_t* start_offset_array, size_t num,


warning: pointer parameter 'start_offset_array' can be pointer to const [readability-non-const-parameter]

Suggested change

uint32_t* start_offset_array, size_t num,

const uint32_t* start_offset_array, size_t num,

github-actions · 2024-10-12T08:12:53Z

be/src/vec/columns/predicate_column.h


    void insert_many_binary_data(char* data_array, uint32_t* len_array,
-                                 uint32_t* start_offset_array, size_t num) override {
+                                 uint32_t* start_offset_array, size_t num,


warning: pointer parameter 'start_offset_array' can be pointer to const [readability-non-const-parameter]

Suggested change

uint32_t* start_offset_array, size_t num,

const uint32_t* start_offset_array, size_t num,

github-actions · 2024-10-12T08:12:53Z

be/src/vec/data_types/data_type_time_v2.h

    static constexpr bool is_parametric = true;

-    DataTypeDateTimeV2(UInt32 scale = 0) : _scale(scale) {
+    DataTypeDateTimeV2(UInt32 scale = 0, bool is_timestamp = 0)


warning: converting integer literal to bool, use bool literal instead [modernize-use-bool-literals]

Suggested change

DataTypeDateTimeV2(UInt32 scale = 0, bool is_timestamp = 0)

DataTypeDateTimeV2(UInt32 scale = 0, bool is_timestamp = false)

github-actions · 2024-10-12T08:12:53Z

be/src/vec/data_types/data_type_time_v2.h

 };

-DataTypePtr create_datetimev2(UInt64 scale);
+DataTypePtr create_datetimev2(UInt64 scale, bool is_timestamp = 0);


warning: converting integer literal to bool, use bool literal instead [modernize-use-bool-literals]

Suggested change

DataTypePtr create_datetimev2(UInt64 scale, bool is_timestamp = 0);

DataTypePtr create_datetimev2(UInt64 scale, bool is_timestamp = false);

github-actions · 2024-10-12T08:12:53Z

be/src/vec/olap/olap_data_convertor.cpp


 OlapBlockDataConvertor::OlapColumnDataConvertorBaseUPtr
-OlapBlockDataConvertor::create_olap_column_data_convertor(const TabletColumn& column) {
+OlapBlockDataConvertor::create_olap_column_data_convertor(const TabletColumn& column,


warning: function 'create_olap_column_data_convertor' exceeds recommended size/complexity thresholds [readability-function-size]

OlapBlockDataConvertor::create_olap_column_data_convertor(const TabletColumn& column, ^

Additional context

be/src/vec/olap/olap_data_convertor.cpp:115: 108 lines including whitespace and comments (threshold 80)

OlapBlockDataConvertor::create_olap_column_data_convertor(const TabletColumn& column, ^

github-actions · 2024-10-12T08:12:53Z

be/src/vec/runtime/vdatetime_value.h

+        union DateTimeV2UInt64Union {
+            DateV2Value<DateTimeV2ValueType> dt;
+            uint64_t ui64;
+            ~DateTimeV2UInt64Union() {}


warning: use '= default' to define a trivial destructor [modernize-use-equals-default]

Suggested change

~DateTimeV2UInt64Union() {}

~DateTimeV2UInt64Union() = default;

github-actions · 2024-10-12T08:12:54Z

be/src/vec/runtime/vdatetime_value.h

+        union DateTimeV2UInt64Union {
+            DateV2Value<DateTimeV2ValueType> dt;
+            uint64_t ui64;
+            ~DateTimeV2UInt64Union() {}


warning: use '= default' to define a trivial destructor [modernize-use-equals-default]

Suggested change

~DateTimeV2UInt64Union() {}

~DateTimeV2UInt64Union() = default;

github-actions · 2025-04-11T00:12:44Z

We're closing this PR because it hasn't been updated in a while.
This isn't a judgement on the merit of the PR in any way. It's just a way of keeping the PR queue manageable.
If you'd like to revive this PR, please reopen it and feel free a maintainer to remove the Stale tag!

justfortaste added 22 commits August 5, 2024 11:20

timestamp fe init

bb2cbdc

be convert to DateTimev2

9a33639

add OLAP_FIELD_TYPE_TIMESTAP

42b88e7

compile ok

DataTypeDateTimeV2 add var is_timestamp

fc574c6

tmp

45635ec

timezone arg

828e80c

timezone arg compile ok

1cba47a

timezone arg compile ok 1

6e14020

tmp xx

e610d03

read ok

b989b66

tmp insert

e428591

fix fe convert tmp

a7938b5

stream load fix

d0bcddd

stream load tmp

0d4b3b3

refactor timezone to tz_offset

702d35b

add testcase

94befc0

github-actions bot reviewed Oct 12, 2024

View reviewed changes

github-actions bot added the Stale label Apr 11, 2025

github-actions bot closed this Apr 12, 2025

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

[WIP] [feature](timestamp) support timestamp data type#41760

[WIP] [feature](timestamp) support timestamp data type#41760
justfortaste wants to merge 22 commits intoapache:masterfrom
justfortaste:feature_timestamp

justfortaste commented Oct 12, 2024 •

edited

Loading

Uh oh!

doris-robot commented Oct 12, 2024

Uh oh!

github-actions bot left a comment

Uh oh!

github-actions bot Oct 12, 2024

Uh oh!

github-actions bot Oct 12, 2024

Uh oh!

github-actions bot Oct 12, 2024

Uh oh!

github-actions bot Oct 12, 2024

Uh oh!

github-actions bot Oct 12, 2024

Uh oh!

github-actions bot Oct 12, 2024

Uh oh!

github-actions bot Oct 12, 2024

Uh oh!

github-actions bot Oct 12, 2024

Uh oh!

github-actions bot commented Apr 11, 2025

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

2 participants

	long tz_offset() { return _tz_offset; }
	long tz_offset() const { return _tz_offset; }

	uint32_t* start_offset_array, size_t num,
	const uint32_t* start_offset_array, size_t num,

	DataTypeDateTimeV2(UInt32 scale = 0, bool is_timestamp = 0)
	DataTypeDateTimeV2(UInt32 scale = 0, bool is_timestamp = false)

	DataTypePtr create_datetimev2(UInt64 scale, bool is_timestamp = 0);
	DataTypePtr create_datetimev2(UInt64 scale, bool is_timestamp = false);

	~DateTimeV2UInt64Union() {}
	~DateTimeV2UInt64Union() = default;

Conversation

justfortaste commented Oct 12, 2024 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Proposed changes

Uh oh!

doris-robot commented Oct 12, 2024

Uh oh!

github-actions bot left a comment

Choose a reason for hiding this comment

Uh oh!

github-actions bot Oct 12, 2024

Choose a reason for hiding this comment

Uh oh!

github-actions bot Oct 12, 2024

Choose a reason for hiding this comment

Uh oh!

github-actions bot Oct 12, 2024

Choose a reason for hiding this comment

Uh oh!

github-actions bot Oct 12, 2024

Choose a reason for hiding this comment

Uh oh!

github-actions bot Oct 12, 2024

Choose a reason for hiding this comment

Uh oh!

github-actions bot Oct 12, 2024

Choose a reason for hiding this comment

Uh oh!

github-actions bot Oct 12, 2024

Choose a reason for hiding this comment

Uh oh!

github-actions bot Oct 12, 2024

Choose a reason for hiding this comment

Uh oh!

github-actions bot commented Apr 11, 2025

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

2 participants

justfortaste commented Oct 12, 2024 •

edited

Loading