Permalink
Browse files

jsonparser: New parser module for JSON parsing.

The new parser supports json-c only, it does not work with
json-glib. For this reason, the configure script was modified a
little, so that it uses --with-json only. When json-c is found or
selected, then both the parser and the template function will be
built. If json-glib is found or selected, then only the template
function will be available.

The summary after configure now lists the determined settings.

The new parser is - so far - fairly simple: it parses a JSON with
json-c (int, double and string values are supported so far, booleans,
arrays and sub-objects aren't yet), and sets the appropriate
name-value pair in the LogMessage.

Users can configure a prefix to prepend before each key.

Usage is as simple as:

source s_json { tcp(port(21514) flags(no-parse)); };
destination d_json {
  file("/tmp/test.json"
       template("$(format-json --scope dot-nv-pairs)\n"));
};
parser p_json { json-parser (prefix(".json.")); };
log { source(s_json); parser(p_json); destination(d_json); };

Signed-off-by: Gergely Nagy <algernon@balabit.hu>
  • Loading branch information...
1 parent 05065fa commit e5569687bba2551c89a78faee55bcf8b4944066f @bazsi bazsi committed Oct 29, 2011
View
@@ -25,7 +25,7 @@ EVTLOG_MIN_VERSION="0.2.12"
OPENSSL_MIN_VERSION="0.9.8"
LIBDBI_MIN_VERSION="0.8.0"
IVYKIS_MIN_VERSION="0.18"
-JSON_C_MIN_VERSION="0.7"
+JSON_C_MIN_VERSION="0.9"
JSON_GLIB_MIN_VERSION="0.12"
PCRE_MIN_VERSION="6.1"
LMC_MIN_VERSION="0.1.0"
@@ -178,10 +178,6 @@ AC_ARG_WITH(json,
Use the JSON implementation specified]
,,with_json="auto")
-AC_ARG_ENABLE(json,
- [ --enable-json Enable support for JSON template formatting (default: auto)]
- ,,enable_json="auto")
-
AC_ARG_ENABLE(systemd,
[ --enable-systemd Enable systemd support (default: auto)]
,,enable_systemd="auto")
@@ -607,8 +603,12 @@ fi
dnl ***************************************************************************
dnl json headers/libraries
dnl ***************************************************************************
-PKG_CHECK_MODULES(JSON_C, json >= $JSON_C_MIN_VERSION,, JSON_C_LIBS="")
-PKG_CHECK_MODULES(JSON_GLIB, json-glib-1.0 >= $JSON_GLIB_MIN_VERSION,, JSON_GLIB_LIBS="")
+if test "x$with_json" = "xauto" || test "x$with_json" = "json-c"; then
+ PKG_CHECK_MODULES(JSON_C, json >= $JSON_C_MIN_VERSION,, JSON_C_LIBS="")
+fi
+if test "x$with_json" = "xauto" || test "x$with_json" = "json-glib"; then
+ PKG_CHECK_MODULES(JSON_GLIB, json-glib-1.0 >= $JSON_GLIB_MIN_VERSION,, JSON_GLIB_LIBS="")
+fi
dnl ***************************************************************************
dnl pcre headers/libraries
@@ -887,14 +887,14 @@ elif test "x$with_json" = "xjson-c"; then
AC_MSG_ERROR([Cannot find json-c version >= $JSON_C_MIN_VERSION: is pkg-config in path?])
fi
-if test "x$enable_json" = "xauto"; then
- if test "x$with_json" = "xno"; then
- enable_json="no"
- else
- enable_json="yes"
- fi
-elif test "x$enable_json" = "xyes" -a "x$with_json" = "xno"; then
- AC_MSG_ERROR([Cannot find json-c version >= $JSON_C_MIN_VERSION or json-glib-1.0 >= $JSON_GLIB_MIN_VERSION: is pkg-config in path?])
+if test "x$with_json" = "xjson-c"; then
+ enable_json_parse="yes"
+ enable_json_format="yes"
+fi
+
+if test "x$with_json" = "xjson-glib"; then
+ enable_json_parse="no"
+ enable_json_format="yes"
fi
if test "x$enable_systemd" = "xauto"; then
@@ -1077,7 +1077,8 @@ AM_CONDITIONAL(ENABLE_SQL, [test "$enable_sql" = "yes"])
AM_CONDITIONAL(ENABLE_SUN_STREAMS, [test "$enable_sun_streams" = "yes"])
AM_CONDITIONAL(ENABLE_PACCT, [test "$enable_pacct" = "yes"])
AM_CONDITIONAL(ENABLE_MONGODB, [test "$enable_mongodb" = "yes"])
-AM_CONDITIONAL(ENABLE_JSON, [test "$enable_json" = "yes"])
+AM_CONDITIONAL(ENABLE_JSON_FORMAT, [test "$enable_json_format" = "yes"])
+AM_CONDITIONAL(ENABLE_JSON_PARSE, [test "$enable_json_parse" = "yes"])
AM_CONDITIONAL(WITH_LIBSYSTEMD, [test "$with_libsystemd" = "yes"])
# substitution into manual pages
@@ -1144,6 +1145,7 @@ AC_OUTPUT(dist.conf
modules/pacctformat/Makefile
modules/basicfuncs/Makefile
modules/tfjson/Makefile
+ modules/jsonparser/Makefile
scripts/Makefile
scripts/update-patterndb
doc/Makefile
@@ -1193,6 +1195,6 @@ echo " SSL support (module) : ${enable_ssl:=no}"
echo " SQL support (module) : ${enable_sql:=no}"
echo " PACCT module (EXPERIMENTAL) : ${enable_pacct:=no}"
echo " MongoDB destination (module): ${enable_mongodb:=no}"
-echo " JSON support (module) : ${enable_json:=no} (using ${with_json})"
+echo " JSON support (module) : parser=${enable_json_parse:=no}, formatter=${enable_json_format:=no} (using ${with_json})"
View
@@ -1 +1 @@
-SUBDIRS = afsocket afsql afstreams affile afprog afuser afmongodb csvparser confgen syslogformat pacctformat basicfuncs dbparser tfjson dummy
+SUBDIRS = afsocket afsql afstreams affile afprog afuser afmongodb csvparser confgen syslogformat pacctformat basicfuncs dbparser tfjson jsonparser dummy
@@ -0,0 +1,22 @@
+moduledir = @moduledir@
+AM_CPPFLAGS = -I$(top_srcdir)/lib -I../../lib
+export top_srcdir
+
+if ENABLE_JSON_PARSE
+module_LTLIBRARIES := libjsonparser.la
+libjsonparser_la_SOURCES = \
+ jsonparser.c jsonparser.h \
+ jsonparser-grammar.y \
+ jsonparser-parser.c jsonparser-parser.h \
+ jsonparser-plugin.c
+
+libjsonparser_la_CPPFLAGS = $(AM_CPPFLAGS)
+libjsonparser_la_CFLAGS = $(JSON_CFLAGS)
+libjsonparser_la_LIBADD = $(MODULE_DEPS_LIBS)
+libjsonparser_la_LDFLAGS = $(MODULE_LDFLAGS) $(JSON_LIBS)
+
+BUILT_SOURCES = jsonparser-grammar.y jsonparser-grammar.c jsonparser-grammar.h
+EXTRA_DIST = $(BUILT_SOURCES) jsonparser-grammar.ym
+endif
+
+include $(top_srcdir)/build/lex-rules.am
@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2011 BalaBit IT Ltd, Budapest, Hungary
+ * Copyright (c) 2011 Gergely Nagy <algernon@balabit.hu>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * As an additional exemption you are allowed to compile & link against the
+ * OpenSSL libraries as published by the OpenSSL project. See the file
+ * COPYING for details.
+ */
+
+%code top {
+#include "jsonparser-parser.h"
+
+}
+
+
+%code {
+
+#include "jsonparser.h"
+#include "cfg-parser.h"
+#include "jsonparser-grammar.h"
+#include "syslog-names.h"
+#include "messages.h"
+
+extern LogParser *last_parser;
+
+}
+
+%name-prefix "jsonparser_"
+
+/* this parameter is needed in order to instruct bison to use a complete
+ * argument list for yylex/yyerror */
+
+%lex-param {CfgLexer *lexer}
+%parse-param {CfgLexer *lexer}
+%parse-param {LogParser **instance}
+%parse-param {gpointer arg}
+
+/* INCLUDE_DECLS */
+
+%token KW_JSON_PARSER
+%token KW_PREFIX
+
+%type <ptr> parser_expr_json
+
+%%
+
+start
+ : LL_CONTEXT_PARSER parser_expr_json { YYACCEPT; }
+ ;
+
+
+parser_expr_json
+ : KW_JSON_PARSER '('
+ {
+ last_parser = *instance = (LogParser *) log_json_parser_new();
+ }
+ parser_json_opts
+ ')' { $$ = last_parser; }
+ ;
+
+parser_json_opts
+ : parser_json_opt parser_json_opts
+ |
+ ;
+
+parser_json_opt
+ : KW_PREFIX '(' string ')' { log_json_parser_set_prefix(last_parser, $3); free ($3); }
+ |
+ ;
+
+/* INCLUDE_RULES */
+
+%%
@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2011 BalaBit IT Ltd, Budapest, Hungary
+ * Copyright (c) 2011 Gergely Nagy <algernon@balabit.hu>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * As an additional exemption you are allowed to compile & link against the
+ * OpenSSL libraries as published by the OpenSSL project. See the file
+ * COPYING for details.
+ */
+
+#include "jsonparser.h"
+#include "cfg-parser.h"
+#include "jsonparser-grammar.h"
+
+extern int jsonparser_debug;
+
+int jsonparser_parse(CfgLexer *lexer, LogParser **instance, gpointer arg);
+
+static CfgLexerKeyword jsonparser_keywords[] =
+{
+ { "json_parser", KW_JSON_PARSER, },
+ { "prefix", KW_PREFIX, },
+ { NULL }
+};
+
+CfgParser jsonparser_parser =
+{
+#if ENABLE_DEBUG
+ .debug_flag = &jsonparser_debug,
+#endif
+ .name = "jsonparser",
+ .keywords = jsonparser_keywords,
+ .parse = (gint (*)(CfgLexer *, gpointer *, gpointer)) jsonparser_parse,
+ .cleanup = (void (*)(gpointer)) log_pipe_unref,
+};
+
+CFG_PARSER_IMPLEMENT_LEXER_BINDING(jsonparser_, LogParser **)
@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2011 BalaBit IT Ltd, Budapest, Hungary
+ * Copyright (c) 2011 Gergely Nagy <algernon@balabit.hu>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * As an additional exemption you are allowed to compile & link against the
+ * OpenSSL libraries as published by the OpenSSL project. See the file
+ * COPYING for details.
+ */
+
+#ifndef JSONPARSER_PARSER_H_INCLUDED
+#define JSONPARSER_PARSER_H_INCLUDED
+
+#include "cfg-parser.h"
+#include "cfg-lexer.h"
+#include "logparser.h"
+
+extern CfgParser jsonparser_parser;
+
+CFG_PARSER_DECLARE_LEXER_BINDING(jsonparser_, LogParser **)
+
+#endif
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2011 BalaBit IT Ltd, Budapest, Hungary
+ * Copyright (c) 2011 Gergely Nagy <algernon@balabit.hu>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * As an additional exemption you are allowed to compile & link against the
+ * OpenSSL libraries as published by the OpenSSL project. See the file
+ * COPYING for details.
+ */
+
+#include "cfg-parser.h"
+#include "plugin.h"
+#include "jsonparser.h"
+
+extern CfgParser jsonparser_parser;
+
+static Plugin jsonparser_plugins[] =
+{
+ {
+ .type = LL_CONTEXT_PARSER,
+ .name = "json-parser",
+ .parser = &jsonparser_parser,
+ },
+};
+
+gboolean
+jsonparser_module_init(GlobalConfig *cfg, CfgArgs *args)
+{
+ plugin_register(cfg, jsonparser_plugins, G_N_ELEMENTS(jsonparser_plugins));
+ return TRUE;
+}
+
+const ModuleInfo module_info =
+{
+ .canonical_name = "jsonparser",
+ .version = VERSION,
+ .description = "The jsonparser module provides JSON parsing support for syslog-ng.",
+ .core_revision = SOURCE_REVISION,
+ .plugins = jsonparser_plugins,
+ .plugins_len = G_N_ELEMENTS(jsonparser_plugins),
+};
Oops, something went wrong.

0 comments on commit e556968

Please sign in to comment.