Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: port the changes for date_to_unix_ts SQL fun from 4.4 #12668

Merged
merged 2 commits into from Mar 12, 2024
Merged

fix: port the changes for date_to_unix_ts SQL fun from 4.4 #12668

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
163d095
fix: port the changes for date_to_unix_ts SQL fun from 4.4
terry-xiaoyu Mar 8, 2024
29111a2
fix: dialyzer problems
terry-xiaoyu Mar 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion apps/emqx_rule_engine/src/emqx_rule_funcs.erl
View file
Open in desktop
Expand Up @@ -1181,7 +1181,7 @@ format_date(TimeUnit, Offset, FormatString, TimeEpoch) ->

date_to_unix_ts(TimeUnit, FormatString, InputString) ->
Unit = time_unit(TimeUnit),
emqx_utils_calendar:parse(InputString, Unit, FormatString).
emqx_utils_calendar:formatted_datetime_to_system_time(InputString, Unit, FormatString).

date_to_unix_ts(TimeUnit, Offset, FormatString, InputString) ->
Unit = time_unit(TimeUnit),
Expand Down
118 changes: 105 additions & 13 deletions apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl
View file
Open in desktop
Expand Up @@ -1143,6 +1143,50 @@ timezone_to_offset_seconds_helper(FunctionName) ->
apply_func(FunctionName, [local]),
ok.

t_date_to_unix_ts(_) ->
TestTab = [
{{"2024-03-01T10:30:38+08:00", second}, [
<<"second">>, <<"+08:00">>, <<"%Y-%m-%d %H-%M-%S">>, <<"2024-03-01 10:30:38">>
]},
{{"2024-03-01T10:30:38.333+08:00", second}, [
<<"second">>, <<"+08:00">>, <<"%Y-%m-%d %H-%M-%S.%3N">>, <<"2024-03-01 10:30:38.333">>
]},
{{"2024-03-01T10:30:38.333+08:00", millisecond}, [
<<"millisecond">>,
<<"+08:00">>,
<<"%Y-%m-%d %H-%M-%S.%3N">>,
<<"2024-03-01 10:30:38.333">>
]},
{{"2024-03-01T10:30:38.333+08:00", microsecond}, [
<<"microsecond">>,
<<"+08:00">>,
<<"%Y-%m-%d %H-%M-%S.%3N">>,
<<"2024-03-01 10:30:38.333">>
]},
{{"2024-03-01T10:30:38.333+08:00", nanosecond}, [
<<"nanosecond">>,
<<"+08:00">>,
<<"%Y-%m-%d %H-%M-%S.%3N">>,
<<"2024-03-01 10:30:38.333">>
]},
{{"2024-03-01T10:30:38.333444+08:00", microsecond}, [
<<"microsecond">>,
<<"+08:00">>,
<<"%Y-%m-%d %H-%M-%S.%6N">>,
<<"2024-03-01 10:30:38.333444">>
]}
],
lists:foreach(
fun({{DateTime3339, Unit}, DateToTsArgs}) ->
?assertEqual(
calendar:rfc3339_to_system_time(DateTime3339, [{unit, Unit}]),
apply_func(date_to_unix_ts, DateToTsArgs),
"Failed on test: " ++ DateTime3339 ++ "/" ++ atom_to_list(Unit)
)
end,
TestTab
).

t_parse_date_errors(_) ->
?assertError(
bad_formatter_or_date,
Expand All @@ -1154,6 +1198,37 @@ t_parse_date_errors(_) ->
bad_formatter_or_date,
emqx_rule_funcs:date_to_unix_ts(second, <<"%y-%m-%d %H:%M:%S">>, <<"2022-05-26 10:40:12">>)
),
%% invalid formats
?assertThrow(
{missing_date_part, month},
emqx_rule_funcs:date_to_unix_ts(
second, <<"%Y-%d %H:%M:%S">>, <<"2022-32 10:40:12">>
)
),
?assertThrow(
{missing_date_part, year},
emqx_rule_funcs:date_to_unix_ts(
second, <<"%H:%M:%S">>, <<"10:40:12">>
)
),
?assertError(
_,
emqx_rule_funcs:date_to_unix_ts(
second, <<"%Y-%m-%d %H:%M:%S">>, <<"2022-05-32 10:40:12">>
)
),
?assertError(
_,
emqx_rule_funcs:date_to_unix_ts(
second, <<"%Y-%m-%d %H:%M:%S">>, <<"2023-02-29 10:40:12">>
)
),
?assertError(
_,
emqx_rule_funcs:date_to_unix_ts(
second, <<"%Y-%m-%d %H:%M:%S">>, <<"2024-02-30 10:40:12">>
)
),

%% Compatibility test
%% UTC+0
Expand All @@ -1173,25 +1248,42 @@ t_parse_date_errors(_) ->
emqx_rule_funcs:date_to_unix_ts(second, <<"%Y-%m-%d %H:%M:%S">>, <<"2022-05-26 10-40-12">>)
),

%% UTC+0
UnixTsLeap0 = 1582986700,
%% leap year checks
?assertEqual(
UnixTsLeap0,
emqx_rule_funcs:date_to_unix_ts(second, <<"%Y-%m-%d %H:%M:%S">>, <<"2020-02-29 14:31:40">>)
%% UTC+0
1709217100,
emqx_rule_funcs:date_to_unix_ts(second, <<"%Y-%m-%d %H:%M:%S">>, <<"2024-02-29 14:31:40">>)
),

%% UTC+0
UnixTsLeap1 = 1709297071,
?assertEqual(
UnixTsLeap1,
%% UTC+0
1709297071,
emqx_rule_funcs:date_to_unix_ts(second, <<"%Y-%m-%d %H:%M:%S">>, <<"2024-03-01 12:44:31">>)
),

%% UTC+0
UnixTsLeap2 = 1709535387,
?assertEqual(
UnixTsLeap2,
emqx_rule_funcs:date_to_unix_ts(second, <<"%Y-%m-%d %H:%M:%S">>, <<"2024-03-04 06:56:27">>)
%% UTC+0
4107588271,
emqx_rule_funcs:date_to_unix_ts(second, <<"%Y-%m-%d %H:%M:%S">>, <<"2100-03-01 12:44:31">>)
),
?assertEqual(
%% UTC+8
1709188300,
emqx_rule_funcs:date_to_unix_ts(
second, <<"+08:00">>, <<"%Y-%m-%d %H:%M:%S">>, <<"2024-02-29 14:31:40">>
)
),
?assertEqual(
%% UTC+8
1709268271,
emqx_rule_funcs:date_to_unix_ts(
second, <<"+08:00">>, <<"%Y-%m-%d %H:%M:%S">>, <<"2024-03-01 12:44:31">>
)
),
?assertEqual(
%% UTC+8
4107559471,
emqx_rule_funcs:date_to_unix_ts(
second, <<"+08:00">>, <<"%Y-%m-%d %H:%M:%S">>, <<"2100-03-01 12:44:31">>
)
),

%% None zero zone shift with millisecond level precision
Expand Down
125 changes: 50 additions & 75 deletions apps/emqx_utils/src/emqx_utils_calendar.erl
View file
Open in desktop
Expand Up @@ -22,7 +22,7 @@
formatter/1,
format/3,
format/4,
parse/3,
formatted_datetime_to_system_time/3,
offset_second/1
]).

Expand All @@ -48,8 +48,9 @@
-define(DAYS_PER_YEAR, 365).
-define(DAYS_PER_LEAP_YEAR, 366).
-define(DAYS_FROM_0_TO_1970, 719528).
-define(SECONDS_FROM_0_TO_1970, (?DAYS_FROM_0_TO_1970 * ?SECONDS_PER_DAY)).

-define(DAYS_FROM_0_TO_10000, 2932897).
-define(SECONDS_FROM_0_TO_1970, ?DAYS_FROM_0_TO_1970 * ?SECONDS_PER_DAY).
-define(SECONDS_FROM_0_TO_10000, (?DAYS_FROM_0_TO_10000 * ?SECONDS_PER_DAY)).
%% the maximum value is the SECONDS_FROM_0_TO_10000 in the calendar.erl,
%% here minus SECONDS_PER_DAY to tolerate timezone time offset,
%% so the maximum date can reach 9999-12-31 which is ample.
Expand Down Expand Up @@ -171,10 +172,10 @@ format(Time, Unit, Offset, FormatterBin) when is_binary(FormatterBin) ->
format(Time, Unit, Offset, Formatter) ->
do_format(Time, time_unit(Unit), offset_second(Offset), Formatter).

parse(DateStr, Unit, FormatterBin) when is_binary(FormatterBin) ->
parse(DateStr, Unit, formatter(FormatterBin));
parse(DateStr, Unit, Formatter) ->
do_parse(DateStr, Unit, Formatter).
formatted_datetime_to_system_time(DateStr, Unit, FormatterBin) when is_binary(FormatterBin) ->
formatted_datetime_to_system_time(DateStr, Unit, formatter(FormatterBin));
formatted_datetime_to_system_time(DateStr, Unit, Formatter) ->
do_formatted_datetime_to_system_time(DateStr, Unit, Formatter).

%%--------------------------------------------------------------------
%% Time unit
Expand Down Expand Up @@ -467,56 +468,51 @@ padding(Data, _Len) ->
Data.

%%--------------------------------------------------------------------
%% internal: parse part
%% internal: formatted_datetime_to_system_time part
%%--------------------------------------------------------------------

do_parse(DateStr, Unit, Formatter) ->
do_formatted_datetime_to_system_time(DateStr, Unit, Formatter) ->
DateInfo = do_parse_date_str(DateStr, Formatter, #{}),
{Precise, PrecisionUnit} = precision(DateInfo),
Counter =
fun
(year, V, Res) ->
Res + dy(V) * ?SECONDS_PER_DAY * Precise - (?SECONDS_FROM_0_TO_1970 * Precise);
(month, V, Res) ->
Dm = dym(maps:get(year, DateInfo, 0), V),
Res + Dm * ?SECONDS_PER_DAY * Precise;
(day, V, Res) ->
Res + (V * ?SECONDS_PER_DAY * Precise);
(hour, V, Res) ->
Res + (V * ?SECONDS_PER_HOUR * Precise);
(minute, V, Res) ->
Res + (V * ?SECONDS_PER_MINUTE * Precise);
(second, V, Res) ->
Res + V * Precise;
(millisecond, V, Res) ->
case PrecisionUnit of
millisecond ->
Res + V;
microsecond ->
Res + (V * 1000);
nanosecond ->
Res + (V * 1000000)
end;
(microsecond, V, Res) ->
case PrecisionUnit of
microsecond ->
Res + V;
nanosecond ->
Res + (V * 1000)
end;
(nanosecond, V, Res) ->
Res + V;
(parsed_offset, V, Res) ->
Res - V * Precise
end,
Count = maps:fold(Counter, 0, DateInfo) - (?SECONDS_PER_DAY * Precise),
erlang:convert_time_unit(Count, PrecisionUnit, Unit).

precision(#{nanosecond := _}) -> {1000_000_000, nanosecond};
precision(#{microsecond := _}) -> {1000_000, microsecond};
precision(#{millisecond := _}) -> {1000, millisecond};
precision(#{second := _}) -> {1, second};
precision(_) -> {1, second}.
PrecisionUnit = precision(DateInfo),
ToPrecisionUnit = fun(Time, FromUnit) ->
erlang:convert_time_unit(Time, FromUnit, PrecisionUnit)
end,
GetRequiredPart = fun(Key) ->
case maps:get(Key, DateInfo, undefined) of
undefined -> throw({missing_date_part, Key});
Value -> Value
end
end,
GetOptionalPart = fun(Key) -> maps:get(Key, DateInfo, 0) end,
Year = GetRequiredPart(year),
Month = GetRequiredPart(month),
Day = GetRequiredPart(day),
Hour = GetRequiredPart(hour),
Min = GetRequiredPart(minute),
Sec = GetRequiredPart(second),
DateTime = {{Year, Month, Day}, {Hour, Min, Sec}},
TotalSecs = datetime_to_system_time(DateTime) - GetOptionalPart(parsed_offset),
check(TotalSecs, DateStr, Unit),
TotalTime =
ToPrecisionUnit(TotalSecs, second) +
ToPrecisionUnit(GetOptionalPart(millisecond), millisecond) +
ToPrecisionUnit(GetOptionalPart(microsecond), microsecond) +
ToPrecisionUnit(GetOptionalPart(nanosecond), nanosecond),
erlang:convert_time_unit(TotalTime, PrecisionUnit, Unit).

check(Secs, _, _) when Secs >= -?SECONDS_FROM_0_TO_1970, Secs < ?SECONDS_FROM_0_TO_10000 ->
ok;
check(_Secs, DateStr, Unit) ->
throw({bad_format, #{date_string => DateStr, to_unit => Unit}}).

datetime_to_system_time(DateTime) ->
calendar:datetime_to_gregorian_seconds(DateTime) - ?SECONDS_FROM_0_TO_1970.

precision(#{nanosecond := _}) -> nanosecond;
precision(#{microsecond := _}) -> microsecond;
precision(#{millisecond := _}) -> millisecond;
precision(#{second := _}) -> second;
precision(_) -> second.

do_parse_date_str(<<>>, _, Result) ->
Result;
Expand Down Expand Up @@ -564,27 +560,6 @@ date_size(timezone) -> 5;
date_size(timezone1) -> 6;
date_size(timezone2) -> 9.

dym(Y, M) ->
case is_leap_year(Y) of
true when M > 2 ->
dm(M) + 1;
_ ->
dm(M)
end.

dm(1) -> 0;
dm(2) -> 31;
dm(3) -> 59;
dm(4) -> 90;
dm(5) -> 120;
dm(6) -> 151;
dm(7) -> 181;
dm(8) -> 212;
dm(9) -> 243;
dm(10) -> 273;
dm(11) -> 304;
dm(12) -> 334.

str_to_int_or_error(Str, Error) ->
case string:to_integer(Str) of
{Int, []} ->
Expand Down
2 changes: 2 additions & 0 deletions changes/ce/fix-12668.en.md
View file
Open in desktop
@@ -0,0 +1,2 @@
Refactor the SQL function: `date_to_unix_ts()` by using `calendar:datetime_to_gregorian_seconds/1`.
This change also added validation for the input date format.