diff --git a/lib/logmsg/tests/test_type_hints.c b/lib/logmsg/tests/test_type_hints.c index ae3b762dc0..30da45d33b 100644 --- a/lib/logmsg/tests/test_type_hints.c +++ b/lib/logmsg/tests/test_type_hints.c @@ -276,7 +276,11 @@ ParameterizedTestParameters(type_hints, test_datetime_cast) {"12345.5", 12345500}, {"12345.54", 12345540}, {"12345.543", 12345543}, - {"12345.54321", 12345543} + {"12345.54321", 12345543}, + {"12345.987654", 12345987}, + {"12345.987654321", 12345987}, + {"12345+05:00", 12345000}, + {"12345-05:00", 12345000}, }; return cr_make_param_array(StringUInt64Pair, string_value_pairs, @@ -290,20 +294,40 @@ ParameterizedTest(StringUInt64Pair *string_value_pair, type_hints, test_datetime cr_assert_eq(type_cast_to_datetime_msec(string_value_pair->string, &value, &error), TRUE, "Type cast of \"%s\" to msecs failed", string_value_pair->string); - cr_assert_eq(value, string_value_pair->value); + cr_assert_eq(value, string_value_pair->value, + "datetime cast failed %" G_GINT64_FORMAT " != %" G_GINT64_FORMAT, + value, string_value_pair->value); cr_assert_null(error); } -Test(type_hints, test_invalid_datetime_cast) +ParameterizedTestParameters(type_hints, test_invalid_datetime_cast) +{ + static StringUInt64Pair string_value_pairs[] = + { + {"invalid", }, + {"12345T", }, + {"12345.", }, + {"12345.1234567890", }, + {"12345+XX:YY", 12345000}, + {"12345-05", 12345000}, + {"12345-XX:YY", 12345000} + + }; + + return cr_make_param_array(StringUInt64Pair, string_value_pairs, + sizeof(string_value_pairs) / sizeof(string_value_pairs[0])); +} + + +ParameterizedTest(StringUInt64Pair *string_value_pair, type_hints, test_invalid_datetime_cast) { GError *error = NULL; gint64 value; - cr_assert_eq(type_cast_to_datetime_msec("invalid", &value, &error), FALSE, - "Type cast of invalid string to gint64 should be failed"); + cr_assert_eq(type_cast_to_datetime_msec(string_value_pair->string, &value, &error), FALSE, + "Type cast of invalid string to gint64 should have failed %s", string_value_pair->string); cr_assert_not_null(error); cr_assert_eq(error->domain, TYPE_HINTING_ERROR); cr_assert_eq(error->code, TYPE_HINTING_INVALID_CAST); - g_clear_error(&error); } diff --git a/lib/logmsg/type-hinting.c b/lib/logmsg/type-hinting.c index ebc137d2d4..9d9c1a345b 100644 --- a/lib/logmsg/type-hinting.c +++ b/lib/logmsg/type-hinting.c @@ -25,6 +25,7 @@ #include "messages.h" #include "type-hinting.h" #include "template/templates.h" +#include "timeutils/scan-timestamp.h" #include #include @@ -139,56 +140,70 @@ type_cast_to_double(const gchar *value, gdouble *out, GError **error) return success; } -gboolean -type_cast_to_datetime_msec(const gchar *value, gint64 *out, GError **error) +static gboolean +_parse_fixed_point_timestamp_in_nsec(const gchar *value, gchar **endptr, gint64 *sec, gint64 *nsec) { - gchar *endptr; - - *out = (gint64)strtoll(value, &endptr, 10) * 1000; + *nsec = 0; - if (endptr[0] == '.') + *sec = (gint64) strtoll(value, endptr, 10); + if (**endptr == '.') { - gsize len = strlen(endptr) - 1, p; - gchar *e, tmp[4]; - glong i; + const gchar *nsec_start = (*endptr) + 1; - if (len > 3) - len = 3; + *nsec = (gint64) strtoll(nsec_start, endptr, 10); + gint nsec_length = (*endptr) - nsec_start; + if (nsec_length == 0) + return FALSE; - memcpy(tmp, endptr + 1, len); - tmp[len] = '\0'; + if (nsec_length > 9) + return FALSE; - i = strtoll(tmp, &e, 10); + for (gint i = 0; i < 9 - nsec_length; i++) + *nsec *= 10; + return TRUE; + } + return TRUE; +} - if (e[0] != '\0') - { - if (error) - g_set_error(error, TYPE_HINTING_ERROR, TYPE_HINTING_INVALID_CAST, - "datetime(%s)", value); - return FALSE; - } +gboolean +type_cast_to_datetime_unixtime(const gchar *value, UnixTime *ut, GError **error) +{ + gchar *endptr; + gint64 sec, nsec; + gint tzofs = -1; - for (p = 3 - len; p > 0; p--) - i *= 10; + if (!_parse_fixed_point_timestamp_in_nsec(value, &endptr, &sec, &nsec)) + goto error; - *out += i; - } - else if (endptr[0] != '\0') + const guchar *tz_start = (guchar *) endptr; + if (*tz_start != 0) { - if (error) - g_set_error(error, TYPE_HINTING_ERROR, TYPE_HINTING_INVALID_CAST, - "datetime(%s)", value); - return FALSE; + gint tz_length = strlen(endptr); + if (!scan_iso_timezone(&tz_start, &tz_length, &tzofs)) + goto error; } + + ut->ut_sec = sec; + ut->ut_usec = nsec / 1000; + ut->ut_gmtoff = tzofs; return TRUE; -} +error: -gboolean -type_cast_to_datetime_str(const gchar *value, const char *format, - gchar **out, GError **error) -{ if (error) g_set_error(error, TYPE_HINTING_ERROR, TYPE_HINTING_INVALID_CAST, - "datetime_str is not supported yet"); + "datetime(%s)", value); return FALSE; + +} + +gboolean +type_cast_to_datetime_msec(const gchar *value, gint64 *out, GError **error) +{ + UnixTime ut; + + if (!type_cast_to_datetime_unixtime(value, &ut, error)) + return FALSE; + + *out = ut.ut_sec * 1000 + ut.ut_usec / 1000; + return TRUE; } diff --git a/lib/logmsg/type-hinting.h b/lib/logmsg/type-hinting.h index 6b6dc98f32..e91553b8f6 100644 --- a/lib/logmsg/type-hinting.h +++ b/lib/logmsg/type-hinting.h @@ -48,7 +48,6 @@ gboolean type_cast_to_int32(const gchar *value, gint32 *out, GError **error); gboolean type_cast_to_int64(const gchar *value, gint64 *out, GError **error); gboolean type_cast_to_double(const gchar *value, gdouble *out, GError **error); gboolean type_cast_to_datetime_msec(const gchar *value, gint64 *out, GError **error); -gboolean type_cast_to_datetime_str(const gchar *value, const char *format, - gchar **out, GError **error); +gboolean type_cast_to_datetime_unixtime(const gchar *value, UnixTime *ut, GError **error); #endif diff --git a/lib/timeutils/scan-timestamp.c b/lib/timeutils/scan-timestamp.c index c00d8e6a96..2088dffb95 100644 --- a/lib/timeutils/scan-timestamp.c +++ b/lib/timeutils/scan-timestamp.c @@ -382,6 +382,16 @@ __parse_iso_timezone(const guchar **data, gint *length) return tz; } +gboolean +scan_iso_timezone(const guchar **data, gint *length, gint *gmtoff) +{ + if (__has_iso_timezone(*data, *length)) + { + *gmtoff = __parse_iso_timezone(data, length); + return TRUE; + } + return FALSE; +} static gboolean __parse_iso_stamp(WallClockTime *wct, const guchar **data, gint *length) diff --git a/lib/timeutils/scan-timestamp.h b/lib/timeutils/scan-timestamp.h index bce6696e68..a1de397fa5 100644 --- a/lib/timeutils/scan-timestamp.h +++ b/lib/timeutils/scan-timestamp.h @@ -29,6 +29,8 @@ #include "timeutils/wallclocktime.h" #include "timeutils/unixtime.h" +gboolean scan_iso_timezone(const guchar **buf, gint *length, gint *gmtoff); + gboolean scan_iso_timestamp(const gchar **buf, gint *left, WallClockTime *wct); gboolean scan_pix_timestamp(const gchar **buf, gint *left, WallClockTime *wct); gboolean scan_linksys_timestamp(const gchar **buf, gint *left, WallClockTime *wct); diff --git a/modules/timestamp/date-parser.c b/modules/timestamp/date-parser.c index 5b368edbea..79f9efced4 100644 --- a/modules/timestamp/date-parser.c +++ b/modules/timestamp/date-parser.c @@ -27,6 +27,8 @@ #include "timeutils/wallclocktime.h" #include "timeutils/cache.h" #include "timeutils/conv.h" +#include "scratch-buffers.h" +#include "str-format.h" enum { @@ -41,6 +43,7 @@ typedef struct _DateParser LogMessageTimeStamp time_stamp; TimeZoneInfo *date_tz_info; guint32 flags; + NVHandle value_handle; } DateParser; void @@ -69,6 +72,14 @@ date_parser_set_time_stamp(LogParser *s, LogMessageTimeStamp time_stamp) self->time_stamp = time_stamp; } +void +date_parser_set_value(LogParser *s, const gchar *value_name) +{ + DateParser *self = (DateParser *) s; + + self->value_handle = log_msg_get_value_handle(value_name); +} + static gboolean date_parser_init(LogPipe *s) { @@ -132,6 +143,22 @@ _convert_timestamp_to_logstamp(DateParser *self, time_t now, UnixTime *target, c return TRUE; } +static void +_store_timestamp(DateParser *self, LogMessage *msg, UnixTime *time_stamp) +{ + if (!self->value_handle) + { + msg->timestamps[self->time_stamp] = *time_stamp; + return; + } + + GString *time_stamp_repr = scratch_buffers_alloc(); + format_int64_padded(time_stamp_repr, -1, ' ', 10, time_stamp->ut_sec); + g_string_append_c(time_stamp_repr, '.'); + format_uint64_padded(time_stamp_repr, 6, '0', 10, time_stamp->ut_usec); + log_msg_set_value_with_type(msg, self->value_handle, time_stamp_repr->str, time_stamp_repr->len, LM_VT_DATETIME); +} + static gboolean date_parser_process(LogParser *s, LogMessage **pmsg, @@ -141,6 +168,8 @@ date_parser_process(LogParser *s, { DateParser *self = (DateParser *) s; LogMessage *msg = log_msg_make_writable(pmsg, path_options); + UnixTime time_stamp; + msg_trace("date-parser message processing started", evt_tag_str("input", input), evt_tag_msg_reference(*pmsg)); @@ -152,8 +181,11 @@ date_parser_process(LogParser *s, APPEND_ZERO(input, input, input_len); gboolean res = _convert_timestamp_to_logstamp(self, msg->timestamps[LM_TS_RECVD].ut_sec, - &msg->timestamps[self->time_stamp], + &time_stamp, input); + if (res) + _store_timestamp(self, msg, &time_stamp); + return res; } diff --git a/modules/timestamp/date-parser.h b/modules/timestamp/date-parser.h index 75b9f8792e..fa88ea50af 100644 --- a/modules/timestamp/date-parser.h +++ b/modules/timestamp/date-parser.h @@ -32,6 +32,7 @@ void date_parser_set_offset(LogParser *s, goffset offset); void date_parser_set_formats(LogParser *s, GList *formats); void date_parser_set_timezone(LogParser *s, gchar *tz); void date_parser_set_time_stamp(LogParser *s, LogMessageTimeStamp timestamp); +void date_parser_set_value(LogParser *s, const gchar *value_name); gboolean date_parser_process_flag(LogParser *s, gchar *flag); #endif diff --git a/modules/timestamp/tests/test_format_date.c b/modules/timestamp/tests/test_format_date.c index 0effb9bd23..17617f836f 100644 --- a/modules/timestamp/tests/test_format_date.c +++ b/modules/timestamp/tests/test_format_date.c @@ -81,6 +81,11 @@ Test(format_date, test_format_date_with_timestamp_argument_formats_it_using_strf assert_template_format("$(format-date %Y-%m-%dT%H:%M:%S 1667500613)", "2022-11-03T19:36:53"); } +Test(format_date, test_format_date_with_timestamp_argument_using_fractions_and_timezone_works) +{ + assert_template_format("$(format-date %Y-%m-%dT%H:%M:%S 1667500613.613+05:00)", "2022-11-03T23:36:53"); +} + Test(format_date, test_format_date_with_time_zone_option_overrrides_timezone) { assert_template_format("$(format-date --time-zone PST8PDT %Y-%m-%dT%H:%M:%S 1667500613)", "2022-11-03T11:36:53"); diff --git a/modules/timestamp/tf-format-date.c b/modules/timestamp/tf-format-date.c index a67f4a1806..98637b1f5e 100644 --- a/modules/timestamp/tf-format-date.c +++ b/modules/timestamp/tf-format-date.c @@ -112,16 +112,16 @@ tf_format_date_call(LogTemplateFunction *self, gpointer s, const LogTemplateInvo UnixTime ut; *type = LM_VT_STRING; - gint64 msec; if (state->super.argc != 0) { const gchar *ts = args->argv[0]->str; - if (!type_cast_to_datetime_msec(ts, &msec, NULL)) - msec = 0; - ut.ut_sec = msec / 1000; - ut.ut_usec = (msec % 1000) * 1000; - ut.ut_gmtoff = -1; + if (!type_cast_to_datetime_unixtime(ts, &ut, NULL)) + { + ut.ut_sec = 0; + ut.ut_usec = 0; + ut.ut_gmtoff = -1; + } } else { diff --git a/modules/timestamp/timestamp-grammar.ym b/modules/timestamp/timestamp-grammar.ym index 7bee8c4695..846824a8f8 100644 --- a/modules/timestamp/timestamp-grammar.ym +++ b/modules/timestamp/timestamp-grammar.ym @@ -82,6 +82,7 @@ date_parser_option : KW_FORMAT '(' string_list ')' { date_parser_set_formats(last_parser, $3); } | KW_TIME_ZONE '(' string ')' { date_parser_set_timezone(last_parser, $3); free($3); } | KW_TIME_STAMP '(' date_parser_stamp ')' { date_parser_set_time_stamp(last_parser, $3); } + | KW_VALUE '(' string ')' { date_parser_set_value(last_parser, $3); free($3); } | KW_FLAGS '(' date_parser_flags ')' | parser_opt ; diff --git a/news/feature-4319.md b/news/feature-4319.md new file mode 100644 index 0000000000..a7596b3873 --- /dev/null +++ b/news/feature-4319.md @@ -0,0 +1,18 @@ +`date-parser()`: add `value()` parameter to instruct `date-parser()` to store +the resulting timestamp in a name-value pair, instead of changing the +timestamp value of the LogMessage. + +`datetime` type representation: typed values in syslog-ng are represented as +strings when stored as a part of a log message. syslog-ng simply remembers +the type it was stored as. Whenever the value is used as a specific type in +a type-aware context where we need the value of the specific type, an +automatic string parsing takes place. This parsing happens for instance +whenever syslog-ng stores a datetime value in MongoDB or when +`$(format-date)` template function takes a name-value pair as parameter. +The datetime() type has stored its value as the number of milliseconds since +the epoch (1970-01-01 00:00:00 GMT). This has now been enhanced by making +it possible to store timestamps up to nanosecond resolutions along with an +optional timezone offset. + +`$(format-date)`: when applied to name-value pairs with the `datetime` type, +use the timezone offset if one is available.