Skip to content

Commit

Permalink
fix: port the changes for date_to_unix_ts SQL fun from 4.4
Browse files Browse the repository at this point in the history
  • Loading branch information
@terry-xiaoyu
terry-xiaoyu committed Mar 8, 2024
1 parent 1f38813 commit 163d095
Show file tree
Hide file tree
Showing 4 changed files with 158 additions and 68 deletions.
2 changes: 1 addition & 1 deletion apps/emqx_rule_engine/src/emqx_rule_funcs.erl
View file
Expand Up @@ -1181,7 +1181,7 @@ format_date(TimeUnit, Offset, FormatString, TimeEpoch) ->

date_to_unix_ts(TimeUnit, FormatString, InputString) ->
Unit = time_unit(TimeUnit),
emqx_utils_calendar:parse(InputString, Unit, FormatString).
emqx_utils_calendar:formatted_datetime_to_system_time(InputString, Unit, FormatString).

date_to_unix_ts(TimeUnit, Offset, FormatString, InputString) ->
Unit = time_unit(TimeUnit),
Expand Down
118 changes: 105 additions & 13 deletions apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl
View file
Expand Up @@ -1143,6 +1143,50 @@ timezone_to_offset_seconds_helper(FunctionName) ->
apply_func(FunctionName, [local]),
ok.

t_date_to_unix_ts(_) ->
TestTab = [
{{"2024-03-01T10:30:38+08:00", second}, [
<<"second">>, <<"+08:00">>, <<"%Y-%m-%d %H-%M-%S">>, <<"2024-03-01 10:30:38">>
]},
{{"2024-03-01T10:30:38.333+08:00", second}, [
<<"second">>, <<"+08:00">>, <<"%Y-%m-%d %H-%M-%S.%3N">>, <<"2024-03-01 10:30:38.333">>
]},
{{"2024-03-01T10:30:38.333+08:00", millisecond}, [
<<"millisecond">>,
<<"+08:00">>,
<<"%Y-%m-%d %H-%M-%S.%3N">>,
<<"2024-03-01 10:30:38.333">>
]},
{{"2024-03-01T10:30:38.333+08:00", microsecond}, [
<<"microsecond">>,
<<"+08:00">>,
<<"%Y-%m-%d %H-%M-%S.%3N">>,
<<"2024-03-01 10:30:38.333">>
]},
{{"2024-03-01T10:30:38.333+08:00", nanosecond}, [
<<"nanosecond">>,
<<"+08:00">>,
<<"%Y-%m-%d %H-%M-%S.%3N">>,
<<"2024-03-01 10:30:38.333">>
]},
{{"2024-03-01T10:30:38.333444+08:00", microsecond}, [
<<"microsecond">>,
<<"+08:00">>,
<<"%Y-%m-%d %H-%M-%S.%6N">>,
<<"2024-03-01 10:30:38.333444">>
]}
],
lists:foreach(
fun({{DateTime3339, Unit}, DateToTsArgs}) ->
?assertEqual(
calendar:rfc3339_to_system_time(DateTime3339, [{unit, Unit}]),
apply_func(date_to_unix_ts, DateToTsArgs),
"Failed on test: " ++ DateTime3339 ++ "/" ++ atom_to_list(Unit)
)
end,
TestTab
).

t_parse_date_errors(_) ->
?assertError(
bad_formatter_or_date,
Expand All @@ -1154,6 +1198,37 @@ t_parse_date_errors(_) ->
bad_formatter_or_date,
emqx_rule_funcs:date_to_unix_ts(second, <<"%y-%m-%d %H:%M:%S">>, <<"2022-05-26 10:40:12">>)
),
%% invalid formats
?assertThrow(
{missing_date_part, month},
emqx_rule_funcs:date_to_unix_ts(
second, <<"%Y-%d %H:%M:%S">>, <<"2022-32 10:40:12">>
)
),
?assertThrow(
{missing_date_part, year},
emqx_rule_funcs:date_to_unix_ts(
second, <<"%H:%M:%S">>, <<"10:40:12">>
)
),
?assertError(
_,
emqx_rule_funcs:date_to_unix_ts(
second, <<"%Y-%m-%d %H:%M:%S">>, <<"2022-05-32 10:40:12">>
)
),
?assertError(
_,
emqx_rule_funcs:date_to_unix_ts(
second, <<"%Y-%m-%d %H:%M:%S">>, <<"2023-02-29 10:40:12">>
)
),
?assertError(
_,
emqx_rule_funcs:date_to_unix_ts(
second, <<"%Y-%m-%d %H:%M:%S">>, <<"2024-02-30 10:40:12">>
)
),

%% Compatibility test
%% UTC+0
Expand All @@ -1173,25 +1248,42 @@ t_parse_date_errors(_) ->
emqx_rule_funcs:date_to_unix_ts(second, <<"%Y-%m-%d %H:%M:%S">>, <<"2022-05-26 10-40-12">>)
),

%% UTC+0
UnixTsLeap0 = 1582986700,
%% leap year checks
?assertEqual(
UnixTsLeap0,
emqx_rule_funcs:date_to_unix_ts(second, <<"%Y-%m-%d %H:%M:%S">>, <<"2020-02-29 14:31:40">>)
%% UTC+0
1709217100,
emqx_rule_funcs:date_to_unix_ts(second, <<"%Y-%m-%d %H:%M:%S">>, <<"2024-02-29 14:31:40">>)
),

%% UTC+0
UnixTsLeap1 = 1709297071,
?assertEqual(
UnixTsLeap1,
%% UTC+0
1709297071,
emqx_rule_funcs:date_to_unix_ts(second, <<"%Y-%m-%d %H:%M:%S">>, <<"2024-03-01 12:44:31">>)
),

%% UTC+0
UnixTsLeap2 = 1709535387,
?assertEqual(
UnixTsLeap2,
emqx_rule_funcs:date_to_unix_ts(second, <<"%Y-%m-%d %H:%M:%S">>, <<"2024-03-04 06:56:27">>)
%% UTC+0
4107588271,
emqx_rule_funcs:date_to_unix_ts(second, <<"%Y-%m-%d %H:%M:%S">>, <<"2100-03-01 12:44:31">>)
),
?assertEqual(
%% UTC+8
1709188300,
emqx_rule_funcs:date_to_unix_ts(
second, <<"+08:00">>, <<"%Y-%m-%d %H:%M:%S">>, <<"2024-02-29 14:31:40">>
)
),
?assertEqual(
%% UTC+8
1709268271,
emqx_rule_funcs:date_to_unix_ts(
second, <<"+08:00">>, <<"%Y-%m-%d %H:%M:%S">>, <<"2024-03-01 12:44:31">>
)
),
?assertEqual(
%% UTC+8
4107559471,
emqx_rule_funcs:date_to_unix_ts(
second, <<"+08:00">>, <<"%Y-%m-%d %H:%M:%S">>, <<"2100-03-01 12:44:31">>
)
),

%% None zero zone shift with millisecond level precision
Expand Down
104 changes: 50 additions & 54 deletions apps/emqx_utils/src/emqx_utils_calendar.erl
View file
Expand Up @@ -22,7 +22,7 @@
formatter/1,
format/3,
format/4,
parse/3,
formatted_datetime_to_system_time/3,
offset_second/1
]).

Expand All @@ -48,8 +48,9 @@
-define(DAYS_PER_YEAR, 365).
-define(DAYS_PER_LEAP_YEAR, 366).
-define(DAYS_FROM_0_TO_1970, 719528).
-define(SECONDS_FROM_0_TO_1970, (?DAYS_FROM_0_TO_1970 * ?SECONDS_PER_DAY)).

-define(DAYS_FROM_0_TO_10000, 2932897).
-define(SECONDS_FROM_0_TO_1970, ?DAYS_FROM_0_TO_1970 * ?SECONDS_PER_DAY).
-define(SECONDS_FROM_0_TO_10000, (?DAYS_FROM_0_TO_10000 * ?SECONDS_PER_DAY)).
%% the maximum value is the SECONDS_FROM_0_TO_10000 in the calendar.erl,
%% here minus SECONDS_PER_DAY to tolerate timezone time offset,
%% so the maximum date can reach 9999-12-31 which is ample.
Expand Down Expand Up @@ -171,10 +172,10 @@ format(Time, Unit, Offset, FormatterBin) when is_binary(FormatterBin) ->
format(Time, Unit, Offset, Formatter) ->
do_format(Time, time_unit(Unit), offset_second(Offset), Formatter).

parse(DateStr, Unit, FormatterBin) when is_binary(FormatterBin) ->
parse(DateStr, Unit, formatter(FormatterBin));
parse(DateStr, Unit, Formatter) ->
do_parse(DateStr, Unit, Formatter).
formatted_datetime_to_system_time(DateStr, Unit, FormatterBin) when is_binary(FormatterBin) ->
formatted_datetime_to_system_time(DateStr, Unit, formatter(FormatterBin));
formatted_datetime_to_system_time(DateStr, Unit, Formatter) ->
do_formatted_datetime_to_system_time(DateStr, Unit, Formatter).

%%--------------------------------------------------------------------
%% Time unit
Expand Down Expand Up @@ -467,56 +468,51 @@ padding(Data, _Len) ->
Data.

%%--------------------------------------------------------------------
%% internal: parse part
%% internal: formatted_datetime_to_system_time part
%%--------------------------------------------------------------------

do_parse(DateStr, Unit, Formatter) ->
do_formatted_datetime_to_system_time(DateStr, Unit, Formatter) ->
DateInfo = do_parse_date_str(DateStr, Formatter, #{}),
{Precise, PrecisionUnit} = precision(DateInfo),
Counter =
fun
(year, V, Res) ->
Res + dy(V) * ?SECONDS_PER_DAY * Precise - (?SECONDS_FROM_0_TO_1970 * Precise);
(month, V, Res) ->
Dm = dym(maps:get(year, DateInfo, 0), V),
Res + Dm * ?SECONDS_PER_DAY * Precise;
(day, V, Res) ->
Res + (V * ?SECONDS_PER_DAY * Precise);
(hour, V, Res) ->
Res + (V * ?SECONDS_PER_HOUR * Precise);
(minute, V, Res) ->
Res + (V * ?SECONDS_PER_MINUTE * Precise);
(second, V, Res) ->
Res + V * Precise;
(millisecond, V, Res) ->
case PrecisionUnit of
millisecond ->
Res + V;
microsecond ->
Res + (V * 1000);
nanosecond ->
Res + (V * 1000000)
end;
(microsecond, V, Res) ->
case PrecisionUnit of
microsecond ->
Res + V;
nanosecond ->
Res + (V * 1000)
end;
(nanosecond, V, Res) ->
Res + V;
(parsed_offset, V, Res) ->
Res - V * Precise
end,
Count = maps:fold(Counter, 0, DateInfo) - (?SECONDS_PER_DAY * Precise),
erlang:convert_time_unit(Count, PrecisionUnit, Unit).

precision(#{nanosecond := _}) -> {1000_000_000, nanosecond};
precision(#{microsecond := _}) -> {1000_000, microsecond};
precision(#{millisecond := _}) -> {1000, millisecond};
precision(#{second := _}) -> {1, second};
precision(_) -> {1, second}.
PrecisionUnit = precision(DateInfo),
ToPrecisionUnit = fun(Time, FromUnit) ->
erlang:convert_time_unit(Time, FromUnit, PrecisionUnit)
end,
GetRequiredPart = fun(Key) ->
case maps:get(Key, DateInfo, undefined) of
undefined -> throw({missing_date_part, Key});
Value -> Value
end
end,
GetOptionalPart = fun(Key) -> maps:get(Key, DateInfo, 0) end,
Year = GetRequiredPart(year),
Month = GetRequiredPart(month),
Day = GetRequiredPart(day),
Hour = GetRequiredPart(hour),
Min = GetRequiredPart(minute),
Sec = GetRequiredPart(second),
DateTime = {{Year, Month, Day}, {Hour, Min, Sec}},
TotalSecs = datetime_to_system_time(DateTime) - GetOptionalPart(parsed_offset),
check(TotalSecs, DateStr, Unit),
TotalTime =
ToPrecisionUnit(TotalSecs, second) +
ToPrecisionUnit(GetOptionalPart(millisecond), millisecond) +
ToPrecisionUnit(GetOptionalPart(microsecond), microsecond) +
ToPrecisionUnit(GetOptionalPart(nanosecond), nanosecond),
erlang:convert_time_unit(TotalTime, PrecisionUnit, Unit).

check(Secs, _, _) when Secs >= -?SECONDS_FROM_0_TO_1970, Secs < ?SECONDS_FROM_0_TO_10000 ->
ok;
check(_Secs, DateStr, Unit) ->
throw({bad_format, #{date_string => DateStr, to_unit => Unit}}).

datetime_to_system_time(DateTime) ->
calendar:datetime_to_gregorian_seconds(DateTime) - ?SECONDS_FROM_0_TO_1970.

precision(#{nanosecond := _}) -> nanosecond;
precision(#{microsecond := _}) -> microsecond;
precision(#{millisecond := _}) -> millisecond;
precision(#{second := _}) -> second;
precision(_) -> second.

do_parse_date_str(<<>>, _, Result) ->
Result;
Expand Down
2 changes: 2 additions & 0 deletions changes/ce/fix-12668.en.md
View file
@@ -0,0 +1,2 @@
Refactor the SQL function: `date_to_unix_ts()` by using `calendar:datetime_to_gregorian_seconds/1`.
This change also added validation for the input date format.

0 comments on commit 163d095

Please sign in to comment.