Skip to content

[fix](datetime) Replace legacy from_date_str with cast function #61682

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
zclllyybb merged 4 commits into from
Mar 30, 2026
Merged

[fix](datetime) Replace legacy from_date_str with cast function #61682

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
655e7aa
[fix](datetime) Replace legacy `from_date_str` with cast function
zclllyybb Mar 24, 2026
7bd5be0
[refactor](datetime) Replace boolean template parameters with enum ty…
zclllyybb Mar 25, 2026
a4a5960
[fix](datetime) Correct parse mode in `from_decimal_batch` and add DC…
zclllyybb Mar 25, 2026
84d7746
Merge branch 'master' into cascade/datetime-parsing-casttodatev2-d2f2db
zclllyybb Mar 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions be/src/core/data_type/data_type_date.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
#include "core/string_buffer.hpp"
#include "core/types.h"
#include "core/value/vdatetime_value.h"
#include "exprs/function/cast/cast_to_date_or_datetime_impl.hpp"
#include "util/io_helper.h"

namespace doris {
Expand All @@ -47,4 +48,19 @@ MutableColumnPtr DataTypeDate::create_column() const {
return DataTypeNumberBase<PrimitiveType::TYPE_DATE>::create_column();
}

Field DataTypeDate::get_field(const TExprNode& node) const {
VecDateTimeValue value;
CastParameters params;
if (CastToDateOrDatetime::from_string_strict_mode<DatelikeParseMode::STRICT,
DatelikeTargetType::DATE>(
{node.date_literal.value.c_str(), node.date_literal.value.size()}, value, nullptr,
params)) {
value.cast_to_date();
return Field::create_field<TYPE_DATE>(std::move(value));
} else {
throw doris::Exception(doris::ErrorCode::INVALID_ARGUMENT,
"Invalid value: {} for type Date", node.date_literal.value);
}
}

} // namespace doris
11 changes: 1 addition & 10 deletions be/src/core/data_type/data_type_date.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,16 +60,7 @@ class DataTypeDate final : public DataTypeNumberBase<PrimitiveType::TYPE_DATE> {
}
#endif
static void cast_to_date(VecDateTimeValue& x);
Field get_field(const TExprNode& node) const override {
VecDateTimeValue value;
if (value.from_date_str(node.date_literal.value.c_str(), node.date_literal.value.size())) {
value.cast_to_date();
return Field::create_field<TYPE_DATE>(std::move(value));
} else {
throw doris::Exception(doris::ErrorCode::INVALID_ARGUMENT,
"Invalid value: {} for type Date", node.date_literal.value);
}
}
Field get_field(const TExprNode& node) const override;

MutableColumnPtr create_column() const override;

Expand Down
31 changes: 31 additions & 0 deletions be/src/core/data_type/data_type_date_or_datetime_v2.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@
#include "core/string_buffer.hpp"
#include "core/types.h"
#include "core/value/vdatetime_value.h"
#include "exprs/function/cast/cast_to_datetimev2_impl.hpp"
#include "exprs/function/cast/cast_to_datev2_impl.hpp"
#include "exprs/function/cast/cast_to_string.h"
#include "util/io_helper.h"

Expand All @@ -50,6 +52,35 @@ class IColumn;
#endif

namespace doris {

Field DataTypeDateV2::get_field(const TExprNode& node) const {
DateV2Value<DateV2ValueType> value;
CastParameters params;
if (CastToDateV2::from_string_strict_mode<DatelikeParseMode::STRICT>(
{node.date_literal.value.c_str(), node.date_literal.value.size()}, value, nullptr,
params)) {
return Field::create_field<TYPE_DATEV2>(std::move(value));
} else {
throw doris::Exception(doris::ErrorCode::INVALID_ARGUMENT,
"Invalid value: {} for type DateV2", node.date_literal.value);
}
}

Field DataTypeDateTimeV2::get_field(const TExprNode& node) const {
DateV2Value<DateTimeV2ValueType> value;
const int32_t scale = node.type.types.empty() ? -1 : node.type.types.front().scalar_type.scale;
CastParameters params;
if (CastToDatetimeV2::from_string_strict_mode<DatelikeParseMode::STRICT>(
{node.date_literal.value.c_str(), node.date_literal.value.size()}, value, nullptr,
scale, params)) {
return Field::create_field<TYPE_DATETIMEV2>(std::move(value));
} else {
throw doris::Exception(doris::ErrorCode::INVALID_ARGUMENT,
"Invalid value: {} for type DateTimeV2({})", node.date_literal.value,
_scale);
}
}

bool DataTypeDateV2::equals(const IDataType& rhs) const {
return typeid(rhs) == typeid(*this);
}
Expand Down
25 changes: 2 additions & 23 deletions be/src/core/data_type/data_type_date_or_datetime_v2.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,16 +66,7 @@ class DataTypeDateV2 final : public DataTypeNumberBase<PrimitiveType::TYPE_DATEV
return std::make_shared<SerDeType>(nesting_level);
}

Field get_field(const TExprNode& node) const override {
DateV2Value<DateV2ValueType> value;
if (value.from_date_str(node.date_literal.value.c_str(),
cast_set<Int32>(node.date_literal.value.size()))) {
return Field::create_field<TYPE_DATEV2>(std::move(value));
} else {
throw doris::Exception(doris::ErrorCode::INVALID_ARGUMENT,
"Invalid value: {} for type DateV2", node.date_literal.value);
}
}
Field get_field(const TExprNode& node) const override;
bool equals(const IDataType& rhs) const override;

#ifdef BE_TEST
Expand Down Expand Up @@ -147,19 +138,7 @@ class DataTypeDateTimeV2 final : public DataTypeNumberBase<PrimitiveType::TYPE_D
return std::make_shared<SerDeType>(_scale, nesting_level);
};

Field get_field(const TExprNode& node) const override {
DateV2Value<DateTimeV2ValueType> value;
const int32_t scale =
node.type.types.empty() ? -1 : node.type.types.front().scalar_type.scale;
if (value.from_date_str(node.date_literal.value.c_str(),
cast_set<int32_t>(node.date_literal.value.size()), scale)) {
return Field::create_field<TYPE_DATETIMEV2>(std::move(value));
} else {
throw doris::Exception(doris::ErrorCode::INVALID_ARGUMENT,
"Invalid value: {} for type DateTimeV2({})",
node.date_literal.value, _scale);
}
}
Field get_field(const TExprNode& node) const override;
MutableColumnPtr create_column() const override;

UInt32 get_scale() const override { return _scale; }
Expand Down
16 changes: 16 additions & 0 deletions be/src/core/data_type/data_type_date_time.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
#include "core/string_buffer.hpp"
#include "core/types.h"
#include "core/value/vdatetime_value.h"
#include "exprs/function/cast/cast_to_date_or_datetime_impl.hpp"
#include "exprs/function/cast/cast_to_string.h"
#include "util/io_helper.h"

Expand All @@ -49,4 +50,19 @@ MutableColumnPtr DataTypeDateTime::create_column() const {
return DataTypeNumberBase<PrimitiveType::TYPE_DATETIME>::create_column();
}

Field DataTypeDateTime::get_field(const TExprNode& node) const {
VecDateTimeValue value;
CastParameters params;
if (CastToDateOrDatetime::from_string_strict_mode<DatelikeParseMode::STRICT,
DatelikeTargetType::DATE_TIME>(
{node.date_literal.value.c_str(), node.date_literal.value.size()}, value, nullptr,
params)) {
value.to_datetime();
return Field::create_field<TYPE_DATETIME>(std::move(value));
} else {
throw doris::Exception(doris::ErrorCode::INVALID_ARGUMENT,
"Invalid value: {} for type DateTime", node.date_literal.value);
}
}

} // namespace doris
11 changes: 1 addition & 10 deletions be/src/core/data_type/data_type_date_time.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,16 +91,7 @@ class DataTypeDateTime final : public DataTypeNumberBase<PrimitiveType::TYPE_DAT
return std::make_shared<SerDeType>(nesting_level);
}

Field get_field(const TExprNode& node) const override {
VecDateTimeValue value;
if (value.from_date_str(node.date_literal.value.c_str(), node.date_literal.value.size())) {
value.to_datetime();
return Field::create_field<TYPE_DATETIME>(std::move(value));
} else {
throw doris::Exception(doris::ErrorCode::INVALID_ARGUMENT,
"Invalid value: {} for type DateTime", node.date_literal.value);
}
}
Field get_field(const TExprNode& node) const override;

static void cast_to_date_time(VecDateTimeValue& x);

Expand Down
4 changes: 2 additions & 2 deletions be/src/core/data_type/data_type_timestamptz.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@ namespace doris {
Field DataTypeTimeStampTz::get_field(const TExprNode& node) const {
TimestampTzValue res;
CastParameters params {.status = Status::OK(), .is_strict = true};

if (!CastToTimstampTz::from_string(
// FE pass the value with timezone info(using legacy planner), so no `local_time_zone` here is ok.
if (!CastToTimestampTz::from_string(
{node.date_literal.value.c_str(), node.date_literal.value.size()}, res, params,
nullptr, _scale)) [[unlikely]] {
throw doris::Exception(doris::ErrorCode::INVALID_ARGUMENT,
Expand Down
72 changes: 49 additions & 23 deletions be/src/core/data_type_serde/data_type_date_or_datetime_serde.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,9 @@ Status DataTypeDateSerDe<T>::_read_column_from_arrow(IColumn& column,
for (auto value_i = start; value_i < end; ++value_i) {
auto val_str = concrete_array->GetString(value_i);
VecDateTimeValue v;
v.from_date_str(val_str.c_str(), val_str.length(), ctz);
CastParameters params;
CastToDateOrDatetime::from_string_non_strict_mode<DatelikeTargetType::DATE_TIME>(
{val_str.c_str(), val_str.length()}, v, &ctz, params);
if constexpr (is_date) {
v.cast_to_date();
}
Expand Down Expand Up @@ -331,8 +333,10 @@ Status DataTypeDateSerDe<T>::from_string_batch(
// then we rely on return value to check success.
// return value only represent OK or InvalidArgument for other error(like InternalError) in parser, MUST throw
// Exception!
if (!CastToDateOrDatetime::from_string_non_strict_mode<IsDatetime>(
str, res, options.timezone, params)) [[unlikely]] {
if (!CastToDateOrDatetime::from_string_non_strict_mode < IsDatetime
? DatelikeTargetType::DATE_TIME
: DatelikeTargetType::DATE > (str, res, options.timezone, params))
[[unlikely]] {
col_nullmap.get_data()[i] = true;
//TODO: we should set `for` functions who need it then skip to set default value for null rows.
col_data.get_data()[i] = VecDateTimeValue::FIRST_DAY;
Expand Down Expand Up @@ -360,8 +364,10 @@ Status DataTypeDateSerDe<T>::from_string_strict_mode_batch(
}
auto str = col_str.get_data_at(i);
CppType res;
CastToDateOrDatetime::from_string_strict_mode<true, IsDatetime>(str, res, options.timezone,
params);
CastToDateOrDatetime::from_string_strict_mode<DatelikeParseMode::STRICT,
IsDatetime ? DatelikeTargetType::DATE_TIME
: DatelikeTargetType::DATE>(
str, res, options.timezone, params);
// only after we called something with `IS_STRICT = true`, params.status will be set
if (!params.status.ok()) [[unlikely]] {
params.status.prepend(
Expand All @@ -386,8 +392,9 @@ Status DataTypeDateSerDe<T>::from_string(StringRef& str, IColumn& column,
// then we rely on return value to check success.
// return value only represent OK or InvalidArgument for other error(like InternalError) in parser, MUST throw
// Exception!
if (!CastToDateOrDatetime::from_string_non_strict_mode<IsDatetime>(str, res, options.timezone,
params)) [[unlikely]] {
if (!CastToDateOrDatetime::from_string_non_strict_mode < IsDatetime
? DatelikeTargetType::DATE_TIME
: DatelikeTargetType::DATE > (str, res, options.timezone, params)) [[unlikely]] {
return Status::InvalidArgument("parse date or datetime fail, string: '{}'",
str.to_string());
}
Expand Down Expand Up @@ -417,8 +424,10 @@ Status DataTypeDateSerDe<T>::from_olap_string(const std::string& str, Field& fie
// then we rely on return value to check success.
// return value only represent OK or InvalidArgument for other error(like InternalError) in parser, MUST throw
// Exception!
if (!CastToDateOrDatetime::from_string_non_strict_mode<IsDatetime>(
StringRef(str), res, options.timezone, params)) [[unlikely]] {
if (!CastToDateOrDatetime::from_string_non_strict_mode < IsDatetime
? DatelikeTargetType::DATE_TIME
: DatelikeTargetType::DATE > (StringRef(str), res, options.timezone, params))
[[unlikely]] {
return Status::InvalidArgument("parse date or datetime fail, string: '{}'", str);
}
field = Field::create_field<T>(std::move(res));
Expand All @@ -433,8 +442,10 @@ Status DataTypeDateSerDe<T>::from_string_strict_mode(StringRef& str, IColumn& co
CastParameters params {.status = Status::OK(), .is_strict = true};

CppType res;
CastToDateOrDatetime::from_string_strict_mode<true, IsDatetime>(str, res, options.timezone,
params);
CastToDateOrDatetime::from_string_strict_mode<DatelikeParseMode::STRICT,
IsDatetime ? DatelikeTargetType::DATE_TIME
: DatelikeTargetType::DATE>(
str, res, options.timezone, params);
// only after we called something with `IS_STRICT = true`, params.status will be set
if (!params.status.ok()) [[unlikely]] {
params.status.prepend(fmt::format("parse {} to {} failed: ", str.to_string_view(), name()));
Expand All @@ -457,8 +468,10 @@ Status DataTypeDateSerDe<T>::from_int_batch(const typename IntDataType::ColumnTy
CastParameters params {.status = Status::OK(), .is_strict = false};
for (size_t i = 0; i < int_col.size(); ++i) {
CppType val;
if (CastToDateOrDatetime::from_integer<false, IsDatetime>(int_col.get_element(i), val,
params)) [[likely]] {
if (CastToDateOrDatetime::from_integer < DatelikeParseMode::NON_STRICT,
IsDatetime ? DatelikeTargetType::DATE_TIME
: DatelikeTargetType::DATE > (int_col.get_element(i), val, params))
[[likely]] {
// did cast_to_type in `from_integer`
col_data.get_data()[i] = val;
col_nullmap.get_data()[i] = false;
Expand All @@ -480,7 +493,10 @@ Status DataTypeDateSerDe<T>::from_int_strict_mode_batch(
CastParameters params {.status = Status::OK(), .is_strict = true};
for (size_t i = 0; i < int_col.size(); ++i) {
CppType val;
CastToDateOrDatetime::from_integer<true, IsDatetime>(int_col.get_element(i), val, params);
CastToDateOrDatetime::from_integer<DatelikeParseMode::STRICT,
IsDatetime ? DatelikeTargetType::DATE_TIME
: DatelikeTargetType::DATE>(
int_col.get_element(i), val, params);
if (!params.status.ok()) [[unlikely]] {
params.status.prepend(
fmt::format("parse {} to {} failed: ", int_col.get_element(i), name()));
Expand All @@ -504,8 +520,10 @@ Status DataTypeDateSerDe<T>::from_float_batch(const typename FloatDataType::Colu
CastParameters params {.status = Status::OK(), .is_strict = false};
for (size_t i = 0; i < float_col.size(); ++i) {
CppType val;
if (CastToDateOrDatetime::from_float<false, IsDatetime>(float_col.get_data()[i], val,
params)) [[likely]] {
if (CastToDateOrDatetime::from_float < DatelikeParseMode::NON_STRICT,
IsDatetime ? DatelikeTargetType::DATE_TIME
: DatelikeTargetType::DATE > (float_col.get_data()[i], val, 0, params))
[[likely]] {
col_data.get_data()[i] = val;
col_nullmap.get_data()[i] = false;
} else {
Expand All @@ -526,7 +544,10 @@ Status DataTypeDateSerDe<T>::from_float_strict_mode_batch(
CastParameters params {.status = Status::OK(), .is_strict = true};
for (size_t i = 0; i < float_col.size(); ++i) {
CppType val;
CastToDateOrDatetime::from_float<true, IsDatetime>(float_col.get_data()[i], val, params);
CastToDateOrDatetime::from_float<DatelikeParseMode::STRICT,
IsDatetime ? DatelikeTargetType::DATE_TIME
: DatelikeTargetType::DATE>(
float_col.get_data()[i], val, 0, params);
if (!params.status.ok()) [[unlikely]] {
params.status.prepend(
fmt::format("parse {} to {} failed: ", float_col.get_data()[i], name()));
Expand All @@ -550,9 +571,12 @@ Status DataTypeDateSerDe<T>::from_decimal_batch(
CastParameters params {.status = Status::OK(), .is_strict = false};
for (size_t i = 0; i < decimal_col.size(); ++i) {
CppType val;
if (CastToDateOrDatetime::from_decimal<true, IsDatetime>(
decimal_col.get_intergral_part(i), decimal_col.get_fractional_part(i),
decimal_col.get_scale(), val, params)) [[likely]] {
if (CastToDateOrDatetime::from_decimal < DatelikeParseMode::NON_STRICT,
IsDatetime ? DatelikeTargetType::DATE_TIME
: DatelikeTargetType::DATE > (decimal_col.get_intergral_part(i),
decimal_col.get_fractional_part(i),
decimal_col.get_scale(), val, params))
[[likely]] {
col_data.get_data()[i] = val;
col_nullmap.get_data()[i] = false;
} else {
Expand All @@ -573,9 +597,11 @@ Status DataTypeDateSerDe<T>::from_decimal_strict_mode_batch(
CastParameters params {.status = Status::OK(), .is_strict = true};
for (size_t i = 0; i < decimal_col.size(); ++i) {
CppType val;
CastToDateOrDatetime::from_decimal<true, IsDatetime>(decimal_col.get_intergral_part(i),
decimal_col.get_fractional_part(i),
decimal_col.get_scale(), val, params);
CastToDateOrDatetime::from_decimal<DatelikeParseMode::STRICT,
IsDatetime ? DatelikeTargetType::DATE_TIME
: DatelikeTargetType::DATE>(
decimal_col.get_intergral_part(i), decimal_col.get_fractional_part(i),
decimal_col.get_scale(), val, params);
if (!params.status.ok()) [[unlikely]] {
params.status.prepend(
fmt::format("parse {}.{} to {} failed: ", decimal_col.get_intergral_part(i),
Expand Down
Loading
Loading