feat(log): configurable time format

now logs can be configured to use 'epoch' or 'rfc3339' time format

apps/emqx/src/config/emqx_config_logger.erl

@@ -237,25 +237,32 @@ log_formatter(HandlerName, Conf) ->
             _ ->
                 conf_get("formatter", Conf)
         end,
+    TsFormat = timstamp_format(Conf),
     do_formatter(
-        Format, CharsLimit, SingleLine, TimeOffSet, Depth
+        Format, CharsLimit, SingleLine, TimeOffSet, Depth, TsFormat
     ).
 
+%% auto | epoch | rfc3339
+timstamp_format(Conf) ->
+    conf_get("timestamp_format", Conf).
+
 %% helpers
-do_formatter(json, CharsLimit, SingleLine, TimeOffSet, Depth) ->
+do_formatter(json, CharsLimit, SingleLine, TimeOffSet, Depth, TsFormat) ->
     {emqx_logger_jsonfmt, #{
         chars_limit => CharsLimit,
         single_line => SingleLine,
         time_offset => TimeOffSet,
-        depth => Depth
+        depth => Depth,
+        timestamp_format => TsFormat
     }};
-do_formatter(text, CharsLimit, SingleLine, TimeOffSet, Depth) ->
+do_formatter(text, CharsLimit, SingleLine, TimeOffSet, Depth, TsFormat) ->
     {emqx_logger_textfmt, #{
-        template => [time, " [", level, "] ", msg, "\n"],
+        template => ["[", level, "] ", msg, "\n"],
         chars_limit => CharsLimit,
         single_line => SingleLine,
         time_offset => TimeOffSet,
-        depth => Depth
+        depth => Depth,
+        timestamp_format => TsFormat
     }}.
 
 %% Don't record all logger message

apps/emqx/src/emqx_logger_jsonfmt.erl

@@ -30,6 +30,7 @@
 -module(emqx_logger_jsonfmt).
 
 -export([format/2]).
+-export([check_config/1]).
 
 %% For CLI HTTP API outputs
 -export([best_effort_json/1, best_effort_json/2]).
@@ -57,6 +58,8 @@
 
 -define(IS_STRING(String), (is_list(String) orelse is_binary(String))).
 
+check_config(X) -> logger_formatter:check_config(maps:without([timestamp_format], X)).
+
 %% @doc Format a list() or map() to JSON object.
 %% This is used for CLI result prints,
 %% or HTTP API result formatting.
@@ -285,9 +288,21 @@ json_obj_root(Data0, Config) ->
         ),
     lists:filter(
         fun({_, V}) -> V =/= undefined end,
-        [{time, Time}, {level, Level}, {msg, Msg}]
+        [{time, format_ts(Time, Config)}, {level, Level}, {msg, Msg}]
     ) ++ Data.
 
+format_ts(Ts, #{timestamp_format := rfc3339, time_offset := Offset}) when is_integer(Ts) ->
+    iolist_to_binary(
+        calendar:system_time_to_rfc3339(Ts, [
+            {unit, microsecond},
+            {offset, Offset},
+            {time_designator, $T}
+        ])
+    );
+format_ts(Ts, _Config) ->
+    % auto | epoch
+    Ts.
+
 json_obj(Data, Config) ->
     maps:fold(
         fun(K, V, D) ->

apps/emqx/src/emqx_logger_textfmt.erl

@@ -20,7 +20,7 @@
 -export([check_config/1]).
 -export([try_format_unicode/1]).
 
-check_config(X) -> logger_formatter:check_config(X).
+check_config(X) -> logger_formatter:check_config(maps:without([timestamp_format], X)).
 
 %% Principle here is to delegate the formatting to logger_formatter:format/2
 %% as much as possible, and only enrich the report with clientid, peername, topic, username
@@ -35,7 +35,7 @@ format(#{msg := {report, ReportMap}, meta := Meta} = Event, Config) when is_map(
             false ->
                 maps:from_list(ReportList)
         end,
-    logger_formatter:format(Event#{msg := {report, Report}}, Config);
+    fmt(Event#{msg := {report, Report}}, Config);
 format(#{msg := {string, String}} = Event, Config) ->
     %% copied from logger_formatter:format/2
     %% unsure how this case is triggered
@@ -45,7 +45,23 @@ format(#{msg := Msg0, meta := Meta} = Event, Config) ->
     %% and logger:log(Level, "message", #{key => value})
     Msg1 = enrich_client_info(Msg0, Meta),
     Msg2 = enrich_topic(Msg1, Meta),
-    logger_formatter:format(Event#{msg := Msg2}, Config).
+    fmt(Event#{msg := Msg2}, Config).
+
+fmt(#{meta := #{time := Ts}} = Data, Config) ->
+    Timestamp =
+        case Config of
+            #{timestamp_format := epoch} ->
+                integer_to_list(Ts);
+            _ ->
+                % auto | rfc3339
+                TimeOffset = maps:get(time_offset, Config, ""),
+                calendar:system_time_to_rfc3339(Ts, [
+                    {unit, microsecond},
+                    {offset, TimeOffset},
+                    {time_designator, $T}
+                ])
+        end,
+    [Timestamp, " ", logger_formatter:format(Data, Config)].
 
 %% Other report callbacks may only accept map() reports such as gen_server formatter
 is_list_report_acceptable(#{report_cb := Cb}) ->

apps/emqx_conf/src/emqx_conf_schema.erl

@@ -1277,6 +1277,15 @@ log_handler_common_confs(Handler, Default) ->
                     importance => ?IMPORTANCE_MEDIUM
                 }
             )},
+        {"timestamp_format",
+            sc(
+                hoconsc:enum([auto, epoch, rfc3339]),
+                #{
+                    default => auto,
+                    desc => ?DESC("common_handler_timestamp_format"),
+                    importance => ?IMPORTANCE_MEDIUM
+                }
+            )},
         {"time_offset",
             sc(
                 string(),

changes/ce/feat-12785.en.md

@@ -0,0 +1,10 @@
+Add `timestamp_format` config to log handers.
+
+We've added a new configuration option `timestamp_format` to the log handlers.
+This new config supports the following values:
+
+- `auto`: Automatically determines the timestamp format based on the log formatter being used.
+  Utilizes `rfc3339` format for text formatters, and `epoch` format for JSON formatters.
+- `epoch`: Represents timestamps in milliseconds precision Unix epoch format.
+- `rfc3339`: Uses RFC3339 compliant format for date-time strings. For example: `2024-03-26T11:52:19.777087+00:00`.
+

rel/i18n/emqx_conf_schema.hocon

@@ -761,6 +761,15 @@ common_handler_formatter.desc:
 common_handler_formatter.label:
 """Log Formatter"""
 
+common_handler_timestamp_format.label:
+"""Timestamp Format"""
+
+common_handler_timestamp_format.desc: """~
+    Pick a timestamp format:
+    - `auto`: automatically choose the best format based on log formatter. `epoch` for JSON and `rfc3339` for text.
+    - `epoch`: Unix epoch time in milliseconds.
+    - `rfc3339`: RFC3339 format."""
+
 rpc_async_batch_size.desc:
 """The maximum number of batch messages sent in asynchronous mode.
       Note that this configuration does not work in synchronous mode."""