Merge pull request #4319 from bazsi/date-parser-add-value-parameter

add value() parameter to date-parser()

lib/logmsg/tests/test_type_hints.c

@@ -276,7 +276,11 @@ ParameterizedTestParameters(type_hints, test_datetime_cast)
     {"12345.5", 12345500},
     {"12345.54", 12345540},
     {"12345.543", 12345543},
-    {"12345.54321", 12345543}
+    {"12345.54321", 12345543},
+    {"12345.987654", 12345987},
+    {"12345.987654321", 12345987},
+    {"12345+05:00", 12345000},
+    {"12345-05:00", 12345000},
   };
 
   return cr_make_param_array(StringUInt64Pair, string_value_pairs,
@@ -290,20 +294,40 @@ ParameterizedTest(StringUInt64Pair *string_value_pair, type_hints, test_datetime
 
   cr_assert_eq(type_cast_to_datetime_msec(string_value_pair->string, &value, &error), TRUE,
                "Type cast of \"%s\" to msecs failed", string_value_pair->string);
-  cr_assert_eq(value, string_value_pair->value);
+  cr_assert_eq(value, string_value_pair->value,
+               "datetime cast failed %" G_GINT64_FORMAT " != %" G_GINT64_FORMAT,
+               value, string_value_pair->value);
   cr_assert_null(error);
 }
 
-Test(type_hints, test_invalid_datetime_cast)
+ParameterizedTestParameters(type_hints, test_invalid_datetime_cast)
+{
+  static StringUInt64Pair string_value_pairs[] =
+  {
+    {"invalid", },
+    {"12345T", },
+    {"12345.", },
+    {"12345.1234567890", },
+    {"12345+XX:YY", 12345000},
+    {"12345-05", 12345000},
+    {"12345-XX:YY", 12345000}
+
+  };
+
+  return cr_make_param_array(StringUInt64Pair, string_value_pairs,
+                             sizeof(string_value_pairs) / sizeof(string_value_pairs[0]));
+}
+
+
+ParameterizedTest(StringUInt64Pair *string_value_pair, type_hints, test_invalid_datetime_cast)
 {
   GError *error = NULL;
   gint64 value;
 
-  cr_assert_eq(type_cast_to_datetime_msec("invalid", &value, &error), FALSE,
-               "Type cast of invalid string to gint64 should be failed");
+  cr_assert_eq(type_cast_to_datetime_msec(string_value_pair->string, &value, &error), FALSE,
+               "Type cast of invalid string to gint64 should have failed %s", string_value_pair->string);
   cr_assert_not_null(error);
   cr_assert_eq(error->domain, TYPE_HINTING_ERROR);
   cr_assert_eq(error->code, TYPE_HINTING_INVALID_CAST);
-
   g_clear_error(&error);
 }

lib/logmsg/type-hinting.c

@@ -25,6 +25,7 @@
 #include "messages.h"
 #include "type-hinting.h"
 #include "template/templates.h"
+#include "timeutils/scan-timestamp.h"
 
 #include <errno.h>
 #include <math.h>
@@ -139,56 +140,70 @@ type_cast_to_double(const gchar *value, gdouble *out, GError **error)
   return success;
 }
 
-gboolean
-type_cast_to_datetime_msec(const gchar *value, gint64 *out, GError **error)
+static gboolean
+_parse_fixed_point_timestamp_in_nsec(const gchar *value, gchar **endptr, gint64 *sec, gint64 *nsec)
 {
-  gchar *endptr;
-
-  *out = (gint64)strtoll(value, &endptr, 10) * 1000;
+  *nsec = 0;
 
-  if (endptr[0] == '.')
+  *sec = (gint64) strtoll(value, endptr, 10);
+  if (**endptr == '.')
     {
-      gsize len = strlen(endptr) - 1, p;
-      gchar *e, tmp[4];
-      glong i;
+      const gchar *nsec_start = (*endptr) + 1;
 
-      if (len > 3)
-        len = 3;
+      *nsec = (gint64) strtoll(nsec_start, endptr, 10);
+      gint nsec_length = (*endptr) - nsec_start;
+      if (nsec_length == 0)
+        return FALSE;
 
-      memcpy(tmp, endptr + 1, len);
-      tmp[len] = '\0';
+      if (nsec_length > 9)
+        return FALSE;
 
-      i = strtoll(tmp, &e, 10);
+      for (gint i = 0; i < 9 - nsec_length; i++)
+        *nsec *= 10;
+      return TRUE;
+    }
+  return TRUE;
+}
 
-      if (e[0] != '\0')
-        {
-          if (error)
-            g_set_error(error, TYPE_HINTING_ERROR, TYPE_HINTING_INVALID_CAST,
-                        "datetime(%s)", value);
-          return FALSE;
-        }
+gboolean
+type_cast_to_datetime_unixtime(const gchar *value, UnixTime *ut, GError **error)
+{
+  gchar *endptr;
+  gint64 sec, nsec;
+  gint tzofs = -1;
 
-      for (p = 3 - len; p > 0; p--)
-        i *= 10;
+  if (!_parse_fixed_point_timestamp_in_nsec(value, &endptr, &sec, &nsec))
+    goto error;
 
-      *out += i;
-    }
-  else if (endptr[0] != '\0')
+  const guchar *tz_start = (guchar *) endptr;
+  if (*tz_start != 0)
     {
-      if (error)
-        g_set_error(error, TYPE_HINTING_ERROR, TYPE_HINTING_INVALID_CAST,
-                    "datetime(%s)", value);
-      return FALSE;
+      gint tz_length = strlen(endptr);
+      if (!scan_iso_timezone(&tz_start, &tz_length, &tzofs))
+        goto error;
     }
+
+  ut->ut_sec = sec;
+  ut->ut_usec = nsec / 1000;
+  ut->ut_gmtoff = tzofs;
   return TRUE;
-}
+error:
 
-gboolean
-type_cast_to_datetime_str(const gchar *value, const char *format,
-                          gchar **out, GError **error)
-{
   if (error)
     g_set_error(error, TYPE_HINTING_ERROR, TYPE_HINTING_INVALID_CAST,
-                "datetime_str is not supported yet");
+                "datetime(%s)", value);
   return FALSE;
+
+}
+
+gboolean
+type_cast_to_datetime_msec(const gchar *value, gint64 *out, GError **error)
+{
+  UnixTime ut;
+
+  if (!type_cast_to_datetime_unixtime(value, &ut, error))
+    return FALSE;
+
+  *out = ut.ut_sec * 1000 + ut.ut_usec / 1000;
+  return TRUE;
 }

lib/logmsg/type-hinting.h

@@ -48,7 +48,6 @@ gboolean type_cast_to_int32(const gchar *value, gint32 *out, GError **error);
 gboolean type_cast_to_int64(const gchar *value, gint64 *out, GError **error);
 gboolean type_cast_to_double(const gchar *value, gdouble *out, GError **error);
 gboolean type_cast_to_datetime_msec(const gchar *value, gint64 *out, GError **error);
-gboolean type_cast_to_datetime_str(const gchar *value, const char *format,
-                                   gchar **out, GError **error);
+gboolean type_cast_to_datetime_unixtime(const gchar *value, UnixTime *ut, GError **error);
 
 #endif

lib/timeutils/scan-timestamp.c

@@ -382,6 +382,16 @@ __parse_iso_timezone(const guchar **data, gint *length)
   return tz;
 }
 
+gboolean
+scan_iso_timezone(const guchar **data, gint *length, gint *gmtoff)
+{
+  if (__has_iso_timezone(*data, *length))
+    {
+      *gmtoff = __parse_iso_timezone(data, length);
+      return TRUE;
+    }
+  return FALSE;
+}
 
 static gboolean
 __parse_iso_stamp(WallClockTime *wct, const guchar **data, gint *length)

lib/timeutils/scan-timestamp.h

@@ -29,6 +29,8 @@
 #include "timeutils/wallclocktime.h"
 #include "timeutils/unixtime.h"
 
+gboolean scan_iso_timezone(const guchar **buf, gint *length, gint *gmtoff);
+
 gboolean scan_iso_timestamp(const gchar **buf, gint *left, WallClockTime *wct);
 gboolean scan_pix_timestamp(const gchar **buf, gint *left, WallClockTime *wct);
 gboolean scan_linksys_timestamp(const gchar **buf, gint *left, WallClockTime *wct);

modules/timestamp/date-parser.c

@@ -27,6 +27,8 @@
 #include "timeutils/wallclocktime.h"
 #include "timeutils/cache.h"
 #include "timeutils/conv.h"
+#include "scratch-buffers.h"
+#include "str-format.h"
 
 enum
 {
@@ -41,6 +43,7 @@ typedef struct _DateParser
   LogMessageTimeStamp time_stamp;
   TimeZoneInfo *date_tz_info;
   guint32 flags;
+  NVHandle value_handle;
 } DateParser;
 
 void
@@ -69,6 +72,14 @@ date_parser_set_time_stamp(LogParser *s, LogMessageTimeStamp time_stamp)
   self->time_stamp = time_stamp;
 }
 
+void
+date_parser_set_value(LogParser *s, const gchar *value_name)
+{
+  DateParser *self = (DateParser *) s;
+
+  self->value_handle = log_msg_get_value_handle(value_name);
+}
+
 static gboolean
 date_parser_init(LogPipe *s)
 {
@@ -132,6 +143,22 @@ _convert_timestamp_to_logstamp(DateParser *self, time_t now, UnixTime *target, c
   return TRUE;
 }
 
+static void
+_store_timestamp(DateParser *self, LogMessage *msg, UnixTime *time_stamp)
+{
+  if (!self->value_handle)
+    {
+      msg->timestamps[self->time_stamp] = *time_stamp;
+      return;
+    }
+
+  GString *time_stamp_repr = scratch_buffers_alloc();
+  format_int64_padded(time_stamp_repr, -1, ' ', 10, time_stamp->ut_sec);
+  g_string_append_c(time_stamp_repr, '.');
+  format_uint64_padded(time_stamp_repr, 6, '0', 10, time_stamp->ut_usec);
+  log_msg_set_value_with_type(msg, self->value_handle, time_stamp_repr->str, time_stamp_repr->len, LM_VT_DATETIME);
+}
+
 static gboolean
 date_parser_process(LogParser *s,
                     LogMessage **pmsg,
@@ -141,6 +168,8 @@ date_parser_process(LogParser *s,
 {
   DateParser *self = (DateParser *) s;
   LogMessage *msg = log_msg_make_writable(pmsg, path_options);
+  UnixTime time_stamp;
+
   msg_trace("date-parser message processing started",
             evt_tag_str("input", input),
             evt_tag_msg_reference(*pmsg));
@@ -152,8 +181,11 @@ date_parser_process(LogParser *s,
   APPEND_ZERO(input, input, input_len);
   gboolean res = _convert_timestamp_to_logstamp(self,
                                                 msg->timestamps[LM_TS_RECVD].ut_sec,
-                                                &msg->timestamps[self->time_stamp],
+                                                &time_stamp,
                                                 input);
+  if (res)
+    _store_timestamp(self, msg, &time_stamp);
+
 
   return res;
 }

modules/timestamp/date-parser.h

@@ -32,6 +32,7 @@ void date_parser_set_offset(LogParser *s, goffset offset);
 void date_parser_set_formats(LogParser *s, GList *formats);
 void date_parser_set_timezone(LogParser *s, gchar *tz);
 void date_parser_set_time_stamp(LogParser *s, LogMessageTimeStamp timestamp);
+void date_parser_set_value(LogParser *s, const gchar *value_name);
 gboolean date_parser_process_flag(LogParser *s, gchar *flag);
 
 #endif

modules/timestamp/tests/test_format_date.c

@@ -81,6 +81,11 @@ Test(format_date, test_format_date_with_timestamp_argument_formats_it_using_strf
   assert_template_format("$(format-date %Y-%m-%dT%H:%M:%S 1667500613)", "2022-11-03T19:36:53");
 }
 
+Test(format_date, test_format_date_with_timestamp_argument_using_fractions_and_timezone_works)
+{
+  assert_template_format("$(format-date %Y-%m-%dT%H:%M:%S 1667500613.613+05:00)", "2022-11-03T23:36:53");
+}
+
 Test(format_date, test_format_date_with_time_zone_option_overrrides_timezone)
 {
   assert_template_format("$(format-date --time-zone PST8PDT %Y-%m-%dT%H:%M:%S 1667500613)", "2022-11-03T11:36:53");

modules/timestamp/tf-format-date.c

@@ -112,16 +112,16 @@ tf_format_date_call(LogTemplateFunction *self, gpointer s, const LogTemplateInvo
   UnixTime ut;
 
   *type = LM_VT_STRING;
-  gint64 msec;
 
   if (state->super.argc != 0)
     {
       const gchar *ts = args->argv[0]->str;
-      if (!type_cast_to_datetime_msec(ts, &msec, NULL))
-        msec = 0;
-      ut.ut_sec = msec / 1000;
-      ut.ut_usec = (msec % 1000) * 1000;
-      ut.ut_gmtoff = -1;
+      if (!type_cast_to_datetime_unixtime(ts, &ut, NULL))
+        {
+          ut.ut_sec = 0;
+          ut.ut_usec = 0;
+          ut.ut_gmtoff = -1;
+        }
     }
   else
     {

modules/timestamp/timestamp-grammar.ym

@@ -82,6 +82,7 @@ date_parser_option
         : KW_FORMAT '(' string_list ')'             { date_parser_set_formats(last_parser, $3); }
         | KW_TIME_ZONE '(' string ')'               { date_parser_set_timezone(last_parser, $3); free($3); }
         | KW_TIME_STAMP '(' date_parser_stamp ')'   { date_parser_set_time_stamp(last_parser, $3); }
+	| KW_VALUE '(' string ')'                   { date_parser_set_value(last_parser, $3); free($3); }
         | KW_FLAGS '(' date_parser_flags ')'
         | parser_opt
         ;

news/feature-4319.md

@@ -0,0 +1,18 @@
+`date-parser()`: add `value()` parameter to instruct `date-parser()` to store
+the resulting timestamp in a name-value pair, instead of changing the
+timestamp value of the LogMessage.
+
+`datetime` type representation: typed values in syslog-ng are represented as
+strings when stored as a part of a log message.  syslog-ng simply remembers
+the type it was stored as.  Whenever the value is used as a specific type in
+a type-aware context where we need the value of the specific type, an
+automatic string parsing takes place.  This parsing happens for instance
+whenever syslog-ng stores a datetime value in MongoDB or when
+`$(format-date)` template function takes a name-value pair as parameter.
+The datetime() type has stored its value as the number of milliseconds since
+the epoch (1970-01-01 00:00:00 GMT).  This has now been enhanced by making
+it possible to store timestamps up to nanosecond resolutions along with an
+optional timezone offset.
+
+`$(format-date)`: when applied to name-value pairs with the `datetime` type,
+use the timezone offset if one is available.