python-dest: string compatibility for python3

.travis.yml

@@ -55,6 +55,7 @@ before_script:
       --disable-sql
       --enable-pacct
       --enable-manpages
+      --with-python=2
       --with-docbook=/usr/share/xml/docbook/stylesheet/docbook-xsl/manpages/docbook.xsl
       --enable-extra-warnings
       "

configure.ac

@@ -1308,23 +1308,29 @@ dnl ***************************************************************************
 dnl python checks
 dnl ***************************************************************************
 if test "x$enable_python" != "xno"; then
-  if test "x$with_python" != "xauto"; then
+  if test "x$with_python" = "xauto"; then
+    PKG_CHECK_MODULES(PYTHON, python3 >= 3, python_found="yes", python_found="no")
+    if test "$python_found" = "yes"; then
+      with_python="python3"
+    else
+      PKG_CHECK_MODULES(PYTHON, python2 >= 2.6, python_found="yes", python_found="no")
+      if test "$python_found" = "yes"; then
+        with_python="python2"
+      fi
+    fi
+  else
     case "$with_python" in
-      auto)
-          with_python="python"
-          ;;
       [[0-9]])
           with_python="python${with_python}"
           ;;
       [[0-9]].[[0-9]])
           with_python="python-${with_python}"
           ;;
     esac
-  else
-    with_python="python"
+
+    PKG_CHECK_MODULES(PYTHON, $with_python >= 2.6, python_found="yes", python_found="no")
   fi
 
-  PKG_CHECK_MODULES(PYTHON, $with_python >= 2.6, python_found="yes", python_found="no")
   if test "x$enable_python" = "xyes" -a "x$python_found" = "xno"; then
     AC_MSG_ERROR([Could not find the requested Python development libraries])
   fi
@@ -1659,6 +1665,8 @@ AC_DEFINE_UNQUOTED(ENABLE_ENV_WRAPPER, `enable_value $enable_env_wrapper`, [Enab
 AC_DEFINE_UNQUOTED(ENABLE_SYSTEMD, `enable_value $enable_systemd`, [Enable systemd support])
 AC_DEFINE_UNQUOTED(SYSTEMD_JOURNAL_MODE, `journald_mode`, [Systemd-journal support mode])
 AC_DEFINE_UNQUOTED(HAVE_INOTIFY, `enable_value $ac_cv_func_inotify_init`, [Have inotify])
+AC_DEFINE_UNQUOTED(ENABLE_PYTHONv2, `(echo "$with_python" | grep -Eq "python-?2.*") && echo 1 || echo 0`, [Python2 c api])
+AC_DEFINE_UNQUOTED(ENABLE_PYTHONv3, `(echo "$with_python" | grep -Eq "python-?3.*") && echo 1 || echo 0`, [Python3 c api])
 
 
 AM_CONDITIONAL(ENABLE_ENV_WRAPPER, [test "$enable_env_wrapper" = "yes"])

modules/python/Makefile.am

@@ -21,6 +21,8 @@ modules_python_libmod_python_la_LIBADD		= \
 	$(IVYKIS_LIBS)
 
 modules_python_libmod_python_la_SOURCES		= \
+	modules/python/compat/compat-python.h     \
+	modules/python/compat/compat-python.c     \
 	modules/python/python-module.h		  \
 	modules/python/python-config.h		  \
 	modules/python/python-config.c		  \
@@ -65,9 +67,10 @@ endif
 include modules/python/pylib/Makefile.am
 
 EXTRA_DIST					+= \
+	modules/python/compat/compat-python-v2.c   \
+	modules/python/compat/compat-python-v3.c   \
 	modules/python/python-grammar.ym	   \
 	modules/python/python-example.conf	   \
 	modules/python/sngexample.py
 
 .PHONY: modules/python mod-python
-

modules/python/compat/compat-python-v2.c

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2017 Balabit
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * As an additional exemption you are allowed to compile & link against the
+ * OpenSSL libraries as published by the OpenSSL project. See the file
+ * COPYING for details.
+ *
+ */
+
+gboolean
+py_is_string(PyObject *object)
+{
+  return PyBytes_Check(object);
+}
+
+const gchar *
+py_object_as_string(PyObject *object)
+{
+  return PyBytes_AsString(object);
+}

modules/python/compat/compat-python-v3.c

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2017 Balabit
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * As an additional exemption you are allowed to compile & link against the
+ * OpenSSL libraries as published by the OpenSSL project. See the file
+ * COPYING for details.
+ *
+ */
+
+gboolean
+py_is_string(PyObject *object)
+{
+  return PyUnicode_Check(object);
+}
+
+const gchar *
+py_object_as_string(PyObject *object)
+{
+  return PyUnicode_AsUTF8(object);
+}

modules/python/compat/compat-python.c

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2017 Balabit
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * As an additional exemption you are allowed to compile & link against the
+ * OpenSSL libraries as published by the OpenSSL project. See the file
+ * COPYING for details.
+ *
+ */
+
+#include "compat-python.h"
+
+#if SYSLOG_NG_ENABLE_PYTHONv2
+#include "compat-python-v2.c"
+#endif
+
+#if SYSLOG_NG_ENABLE_PYTHONv3
+#include "compat-python-v3.c"
+#endif

modules/python/compat/compat-python.h

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2017 Balabit
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * As an additional exemption you are allowed to compile & link against the
+ * OpenSSL libraries as published by the OpenSSL project. See the file
+ * COPYING for details.
+ *
+ */
+
+#ifndef _COMPAT_PYTHON_H
+#define _COMPAT_PYTHON_H
+
+#include <Python.h>
+#include "syslog-ng.h"
+
+gboolean py_is_string(PyObject *object);
+const gchar *py_object_as_string(PyObject *object);
+#endif

modules/python/python-debugger.c

@@ -128,7 +128,7 @@ python_fetch_debugger_command(void)
       Py_DECREF(ret);
       goto exit;
     }
-  command = g_strdup(PyBytes_AsString(ret));
+  command = g_strdup(py_object_as_string(ret));
   Py_DECREF(ret);
 exit:
   PyGILState_Release(gstate);

modules/python/python-helpers.c

@@ -31,7 +31,7 @@ _py_get_callable_name(PyObject *callable, gchar *buf, gsize buf_len)
 
   if (name)
     {
-      g_strlcpy(buf, PyBytes_AsString(name), buf_len);
+      g_strlcpy(buf, py_object_as_string(name), buf_len);
     }
   else
     {
@@ -58,7 +58,7 @@ _py_fetch_and_format_exception_text(gchar *buf, gsize buf_len)
   str = PyObject_Str(value);
   if (str)
     {
-      g_snprintf(buf, buf_len, "%s: %s", ((PyTypeObject *) exc)->tp_name, PyBytes_AsString(str));
+      g_snprintf(buf, buf_len, "%s: %s", ((PyTypeObject *) exc)->tp_name, py_object_as_string(str));
     }
   else
     {
@@ -280,4 +280,3 @@ _py_perform_imports(GList *imports)
 {
   g_list_foreach(imports, _foreach_import, NULL);
 }
-

modules/python/python-logmsg.c

@@ -68,10 +68,11 @@ _is_key_blacklisted(const gchar *key)
 static PyObject *
 _py_log_message_getattr(PyObject *o, PyObject *key)
 {
-  if (!PyBytes_Check(key))
+  if (!py_is_string(key))
     return NULL;
 
-  gchar *name = PyBytes_AsString(key);
+  const gchar *name = py_object_as_string(key);
+
   if (_is_key_blacklisted(name))
     {
       msg_error("Blacklisted attribute requested", evt_tag_str("key", name));
@@ -91,16 +92,16 @@ _py_log_message_getattr(PyObject *o, PyObject *key)
 static int
 _py_log_message_setattr(PyObject *o, PyObject *key, PyObject *value)
 {
-  if (!PyBytes_Check(key))
+  if (!py_is_string(key))
     return -1;
 
   PyLogMessage *py_msg = (PyLogMessage *)o;
-  gchar *name = PyBytes_AsString(key);
+  const gchar *name = py_object_as_string(key);
   NVHandle handle = log_msg_get_value_handle(name);
   PyObject *value_as_strobj = PyObject_Str(value);
   if (value_as_strobj)
     {
-      log_msg_set_value(py_msg->msg, handle, PyBytes_AsString(value_as_strobj), -1);
+      log_msg_set_value(py_msg->msg, handle, py_object_as_string(value_as_strobj), -1);
       Py_DECREF(value_as_strobj);
     }
   else

modules/python/python-module.h

@@ -62,7 +62,5 @@
 #error "_POSIX_C_SOURCE or _XOPEN_SOURCE is already defined, python-module.h should be included first. Check out the comment in python-module.h for more information"
 #endif
 
-#include <Python.h>
-#include "syslog-ng.h"
-
+#include "compat/compat-python.h"
 #endif