tags-parser: new module to parse $TAGS values

The new tags-parser() takes a value encoded by $TAGS and parses it
back into actual tags on the message.

Example:

	tags-parser(template("$PROGRAM"));

Signed-off-by: Balazs Scheidler <balazs.scheidler@balabit.com>

modules/CMakeLists.txt

@@ -32,3 +32,4 @@ add_subdirectory(python)
 add_subdirectory(java)
 add_subdirectory(java-modules)
 add_subdirectory(http)
+add_subdirectory(tagsparser)

modules/Makefile.am

@@ -39,6 +39,7 @@ include modules/getent/Makefile.am
 include modules/map-value-pairs/Makefile.am
 include modules/stardate/Makefile.am
 include modules/snmptrapd-parser/Makefile.am
+include modules/tagsparser/Makefile.am
 
 SYSLOG_NG_CORE_JAR=$(top_builddir)/modules/java/syslog-ng-core/libs/syslog-ng-core.jar
 
@@ -51,7 +52,7 @@ SYSLOG_NG_MODULES	=	\
 	mod-redis mod-pseudofile mod-graphite mod-riemann \
 	mod-python mod-java mod-java-modules mod-kvformat mod-date \
 	mod-native mod-cef mod-add-contextual-data mod-diskq mod-getent \
-	mod-map-value-pairs mod-snmptrapd-parser
+	mod-map-value-pairs mod-snmptrapd-parser mod-tags-parser
 
 
 modules modules/: ${SYSLOG_NG_MODULES}
@@ -67,6 +68,6 @@ modules_test_subdirs	=	\
 	modules_graphite modules_riemann modules_python \
 	modules_systemd_journal modules_kvformat modules_date \
 	modules_cef modules_diskq modules-add-contextual-data modules_getent \
-	modules_map-value-pairs
+	modules_map-value-pairs modules_tagsparser
 
 .PHONY: modules modules/

modules/tagsparser/CMakeLists.txt

@@ -0,0 +1,23 @@
+set(CSVPARSER_SOURCES
+    tags-parser.c
+    tags-parser.h
+    tags-parser-parser.c
+    tags-parser-parser.h
+    tags-parser-plugin.c
+    ${CMAKE_CURRENT_BINARY_DIR}/tags-parser-grammar.h
+    ${CMAKE_CURRENT_BINARY_DIR}/tags-parser-grammar.c
+)
+
+generate_y_from_ym(modules/tags-parser/tags-parser-grammar)
+
+bison_target(CSVParserGrammar
+    ${CMAKE_CURRENT_BINARY_DIR}/tags-parser-grammar.y
+    ${CMAKE_CURRENT_BINARY_DIR}/tags-parser-grammar.c
+    COMPILE_FLAGS ${BISON_FLAGS})
+
+include_directories (${CMAKE_CURRENT_BINARY_DIR})
+include_directories (${CMAKE_CURRENT_SOURCE_DIR})
+add_library(tags-parser MODULE ${CSVPARSER_SOURCES})
+target_link_libraries(tags-parser PRIVATE syslog-ng)
+
+install(TARGETS tags-parser LIBRARY DESTINATION lib/syslog-ng/ COMPONENT tags-parser)

modules/tagsparser/Makefile.am

@@ -0,0 +1,31 @@
+module_LTLIBRARIES				+= modules/tagsparser/libtags-parser.la
+modules_tagsparser_libtags_parser_la_SOURCES	=	\
+	modules/tagsparser/tags-parser.c			\
+	modules/tagsparser/tags-parser.h			\
+	modules/tagsparser/tags-parser-grammar.y		\
+	modules/tagsparser/tags-parser-parser.c		\
+	modules/tagsparser/tags-parser-parser.h		\
+	modules/tagsparser/tags-parser-plugin.c
+
+modules_tagsparser_libtags_parser_la_CPPFLAGS	=	\
+	$(AM_CPPFLAGS)					\
+	-I$(top_srcdir)/modules/tagsparser		\
+	-I$(top_builddir)/modules/tagsparser
+modules_tagsparser_libtags_parser_la_LIBADD	=	\
+	$(MODULE_DEPS_LIBS)
+modules_tagsparser_libtags_parser_la_LDFLAGS	=	\
+	$(MODULE_LDFLAGS)
+modules_tagsparser_libtags_parser_la_DEPENDENCIES	=	\
+	$(MODULE_DEPS_LIBS)
+
+BUILT_SOURCES					+=	\
+	modules/tagsparser/tags-parser-grammar.y		\
+	modules/tagsparser/tags-parser-grammar.c		\
+	modules/tagsparser/tags-parser-grammar.h
+EXTRA_DIST					+=	\
+	modules/tagsparser/tags-parser-grammar.ym
+
+modules/tagsparser modules/tagsparser/ mod-tags-parser: modules/tagsparser/libtags-parser.la
+.PHONY: modules/tagsparser/ mod-tags-parser
+
+include modules/tagsparser/tests/Makefile.am

modules/tagsparser/tags-parser-grammar.ym

@@ -0,0 +1,83 @@
+/*
+ * Copyright (c) 2002-2017 Balabit
+ * Copyright (c) 1998-2017 Balázs Scheidler
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * As an additional exemption you are allowed to compile & link against the
+ * OpenSSL libraries as published by the OpenSSL project. See the file
+ * COPYING for details.
+ *
+ */
+
+%code top {
+#include "tags-parser-parser.h"
+
+}
+
+
+%code {
+
+#include "tags-parser.h"
+#include "cfg-parser.h"
+#include "cfg-grammar.h"
+#include "tags-parser-grammar.h"
+#include "syslog-names.h"
+#include "messages.h"
+
+}
+
+%name-prefix "tags_parser_"
+
+/* this parameter is needed in order to instruct bison to use a complete
+ * argument list for yylex/yyerror */
+
+%lex-param {CfgLexer *lexer}
+%parse-param {CfgLexer *lexer}
+%parse-param {LogParser **instance}
+%parse-param {gpointer arg}
+
+/* INCLUDE_DECLS */
+
+%token KW_TAGS_PARSER
+
+%type	<ptr> parser_expr_tags
+
+%%
+
+start
+        : LL_CONTEXT_PARSER parser_expr_tags                  { YYACCEPT; }
+        ;
+
+
+parser_expr_tags
+        : KW_TAGS_PARSER '('
+          {
+            last_parser = *instance = tags_parser_new(configuration);
+          }
+          parser_tags_opts ')'				      { $$ = last_parser; }
+        ;
+
+parser_tags_opts
+        : parser_tags_opt parser_tags_opts
+        |
+        ;
+
+parser_tags_opt
+	: parser_opt
+	;
+
+/* INCLUDE_RULES */
+
+%%

modules/tagsparser/tags-parser-parser.c

@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2002-2017 Balabit
+ * Copyright (c) 1998-2017 Balázs Scheidler
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * As an additional exemption you are allowed to compile & link against the
+ * OpenSSL libraries as published by the OpenSSL project. See the file
+ * COPYING for details.
+ *
+ */
+
+#include "tags-parser.h"
+#include "cfg-parser.h"
+#include "tags-parser-grammar.h"
+
+extern int tags_parser_debug;
+
+int tags_parser_parse(CfgLexer *lexer, LogParser **instance, gpointer arg);
+
+static CfgLexerKeyword tags_parser_keywords[] =
+{
+  { "tags_parser",  KW_TAGS_PARSER },
+  { NULL }
+};
+
+CfgParser tags_parser_parser =
+{
+#if SYSLOG_NG_ENABLE_DEBUG
+  .debug_flag = &tags_parser_debug,
+#endif
+  .name = "tags_parser",
+  .keywords = tags_parser_keywords,
+  .parse = (gint (*)(CfgLexer *, gpointer *, gpointer)) tags_parser_parse,
+  .cleanup = (void (*)(gpointer)) log_pipe_unref,
+};
+
+CFG_PARSER_IMPLEMENT_LEXER_BINDING(tags_parser_, LogParser **)

modules/tagsparser/tags-parser-parser.h

@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2002-2017 Balabit
+ * Copyright (c) 1998-2017 Balázs Scheidler
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * As an additional exemption you are allowed to compile & link against the
+ * OpenSSL libraries as published by the OpenSSL project. See the file
+ * COPYING for details.
+ *
+ */
+
+#ifndef TAGS_PARSER_PARSER_H_INCLUDED
+#define TAGS_PARSER_PARSER_H_INCLUDED
+
+#include "cfg-parser.h"
+#include "cfg-lexer.h"
+#include "parser/parser-expr.h"
+
+extern CfgParser tags_parser_parser;
+
+CFG_PARSER_DECLARE_LEXER_BINDING(tags_parser_, LogParser **)
+
+#endif

modules/tagsparser/tags-parser-plugin.c

@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2002-2017 Balabit
+ * Copyright (c) 1998-2017 Balázs Scheidler
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * As an additional exemption you are allowed to compile & link against the
+ * OpenSSL libraries as published by the OpenSSL project. See the file
+ * COPYING for details.
+ *
+ */
+
+#include "cfg-parser.h"
+#include "tags-parser.h"
+#include "plugin.h"
+#include "plugin-types.h"
+
+extern CfgParser tags_parser_parser;
+
+static Plugin tags_parser_plugins[] =
+{
+  {
+    .type = LL_CONTEXT_PARSER,
+    .name = "tags-parser",
+    .parser = &tags_parser_parser,
+  },
+};
+
+gboolean
+tags_parser_module_init(GlobalConfig *cfg, CfgArgs *args)
+{
+  plugin_register(cfg, tags_parser_plugins, G_N_ELEMENTS(tags_parser_plugins));
+  return TRUE;
+}
+
+const ModuleInfo module_info =
+{
+  .canonical_name = "tags_parser",
+  .version = SYSLOG_NG_VERSION,
+  .description = "The tags_parser module provides parsing support for restoring message tags as produced by the TAGS macro.",
+  .core_revision = SYSLOG_NG_SOURCE_REVISION,
+  .plugins = tags_parser_plugins,
+  .plugins_len = G_N_ELEMENTS(tags_parser_plugins),
+};

modules/tagsparser/tags-parser.c

@@ -0,0 +1,78 @@
+/*
+ * Copyright (c) 2002-2017 Balabit
+ * Copyright (c) 1998-2017 Balázs Scheidler
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * As an additional exemption you are allowed to compile & link against the
+ * OpenSSL libraries as published by the OpenSSL project. See the file
+ * COPYING for details.
+ *
+ */
+
+#include "tags-parser.h"
+#include "scanner/list-scanner/list-scanner.h"
+#include "parser/parser-expr.h"
+#include "scratch-buffers.h"
+
+#include <string.h>
+
+typedef struct _TagsParser
+{
+  LogParser super;
+} TagsParser;
+
+static gboolean
+_process(LogParser *s, LogMessage **pmsg, const LogPathOptions *path_options, const gchar *input,
+         gsize input_len)
+{
+  LogMessage *msg = log_msg_make_writable(pmsg, path_options);
+
+  ListScanner scanner;
+  list_scanner_init(&scanner);
+  list_scanner_input_va(&scanner, input, NULL);
+
+  while (list_scanner_scan_next(&scanner))
+    {
+      log_msg_set_tag_by_name(msg, list_scanner_get_current_value(&scanner));
+    }
+
+  list_scanner_deinit(&scanner);
+  return TRUE;
+}
+
+static LogPipe *
+tags_parser_clone(LogPipe *s)
+{
+  TagsParser *self = (TagsParser *) s;
+  TagsParser *cloned;
+
+  cloned = (TagsParser *) tags_parser_new(s->cfg);
+  cloned->super.template = log_template_ref(self->super.template);
+  return &cloned->super.super;
+}
+
+/*
+ * Parse comma-separated values from a log message.
+ */
+LogParser *
+tags_parser_new(GlobalConfig *cfg)
+{
+  TagsParser *self = g_new0(TagsParser, 1);
+
+  log_parser_init_instance(&self->super, cfg);
+  self->super.super.clone = tags_parser_clone;
+  self->super.process = _process;
+  return &self->super;
+}