syslog-ng · mitzkia · Oct 26, 2022 · Oct 31, 2022 · Oct 31, 2022 · Oct 31, 2022
diff --git a/.github/workflows/devshell.yml b/.github/workflows/devshell.yml
@@ -10,7 +10,7 @@ jobs:
   general:
     runs-on: ubuntu-18.04
     container:
-      image: ghcr.io/syslog-ng/dbld-devshell:latest
+      image: ghcr.io/mitzkia/dbld-devshell:latest
       options: --privileged --ulimit core=-1
 
     strategy:
@@ -146,7 +146,7 @@ jobs:
   distcheck:
     runs-on: ubuntu-18.04
     container:
-      image: ghcr.io/syslog-ng/dbld-devshell:latest
+      image: ghcr.io/mitzkia/dbld-devshell:latest
       options: --security-opt seccomp=unconfined --ulimit core=-1
 
     steps:
@@ -187,7 +187,7 @@ jobs:
   style-check:
     runs-on: ubuntu-18.04
     container:
-      image: ghcr.io/syslog-ng/dbld-devshell:latest
+      image: ghcr.io/mitzkia/dbld-devshell:latest
 
     steps:
       - name: Checkout syslog-ng source
@@ -211,20 +211,16 @@ jobs:
         working-directory: ./build
         continue-on-error: true # Fix errors from pylint
         run: |
-          make python-pep8 && make python-pylint || (git reset --hard HEAD && exit 1)
+          make pymodules-checks || (git reset --hard HEAD && exit 1)
 
       - name: Style check (Light)
         id: light-style-check
         if: always()
         working-directory: ./build
         shell: bash
         run: |
-          python3.11 -m venv venv
-          source venv/bin/activate
-          python -m pip install -r "${GITHUB_WORKSPACE}/requirements.txt"
           make light-linters || \
             (git diff > ../light-style-problems.diff ; git reset --hard HEAD && exit 1)
-          deactivate
 
       - name: "Artifact: c-style-problems"
         uses: actions/upload-artifact@v2
@@ -243,7 +239,7 @@ jobs:
   copyright-check:
     runs-on: ubuntu-18.04
     container:
-      image: ghcr.io/syslog-ng/dbld-devshell:latest
+      image: ghcr.io/mitzkia/dbld-devshell:latest
       env:
         COPYRIGHTVERBOSITY: 1
 
@@ -264,7 +260,7 @@ jobs:
   commits-check:
     runs-on: ubuntu-18.04
     container:
-      image: ghcr.io/syslog-ng/dbld-devshell:latest
+      image: ghcr.io/mitzkia/dbld-devshell:latest
 
     steps:
       - name: Checkout syslog-ng source

diff --git a/.github/workflows/gh-tools.sh b/.github/workflows/gh-tools.sh
@@ -16,7 +16,7 @@ gh_path()
 gh_output()
 {
     while [ "x$1" != "x" ]; do
-        echo "::set-output name=$1::$(eval echo \$$1)"
+        echo "$1=$(eval echo \$$1)" >> $GITHUB_OUTPUT
         shift;
     done
 }
diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml
@@ -24,7 +24,6 @@ jobs:
         run: |
           brew update --preinstall
           brew bundle --file=contrib/Brewfile
-          pip3 install --user -r requirements.txt
 
       - name: Set ENV variables
         run: |

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -257,17 +257,7 @@ else ()
 endif ()
 
 MESSAGE(STATUS "Detected pythonlib version: ${PYTHONLIBS_VERSION_STRING}")
-
-set (PYLINT "" CACHE STRING "pylint binary")
-
-if (PYLINT)
-  set(PYLINT_BIN ${PYLINT})
-else ()
-  find_program(PYLINT_BIN "pylint3")
-    if (NOT PYLINT_BIN)
-      find_program(PYLINT_BIN "pylint")
-    endif()
-endif()
+include(python_build_venv)
 
 include(openssl_functions)
 openssl_set_defines()

diff --git a/Makefile.am b/Makefile.am
@@ -42,6 +42,7 @@ EXTRA_DIST		= ${BUILT_SOURCES} VERSION NEWS.md autogen.sh \
 			  syslog-ng-native-connector.pc.cmake	\
 			  syslog-ng-ctl/CMakeLists.txt	\
 			  requirements.txt	\
+			  dev-requirements.txt  \
 			  README.md	\
 			  .astylerc
 
@@ -212,6 +213,7 @@ install_loggenpluginLTLIBRARIES	= install-loggenpluginLTLIBRARIES
 $(install_loggenpluginLTLIBRARIES): install-libLTLIBRARIES
 
 include Mk/lex-rules.am
+include Mk/python-venv.am
 include Mk/Makefile.am
 if ENABLE_TESTING
 include libtest/Makefile.am

diff --git a/Mk/python-venv.am b/Mk/python-venv.am
@@ -0,0 +1,16 @@
+
+PYTHON_VENV_TOUCHFILE=$(abs_builddir)/.python-venv-built
+
+$(PYTHON_VENV_TOUCHFILE): Makefile
+	if [ "$(with_python_packages)" = "venv" ]; then \
+	    $(top_srcdir)/scripts/build-python-venv.sh "$(PYTHON)" "$(PYTHON_VENV_DIR)" "$(top_srcdir)"; \
+	fi
+	touch $@
+
+python-venv: $(PYTHON_VENV_TOUCHFILE)
+
+python-venv-clean:
+	rm -rf "$(PYTHON_VENV_DIR)"
+	rm -rf "$(PYTHON_VENV_TOUCHFILE)"
+
+CLEAN_HOOKS += python-venv-clean
diff --git a/cmake/python_build_venv.cmake b/cmake/python_build_venv.cmake
@@ -0,0 +1,17 @@
+
+set(PYTHON_VENV_DIR "${PROJECT_BINARY_DIR}/venv")
+set(PYTHON_VENV_TOUCHFILE "${PROJECT_BINARY_DIR}/.python-venv-built")
+set(PYTHON_VENV_EXECUTABLE "${PYTHON_VENV_DIR}/bin/python")
+
+add_custom_command(
+    OUTPUT ${PYTHON_VENV_TOUCHFILE}
+    COMMAND ${PROJECT_SOURCE_DIR}/scripts/build-python-venv.sh "${PYTHON_EXECUTABLE}" "${PROJECT_BINARY_DIR}/venv" "${PROJECT_SOURCE_DIR}" && touch "${PYTHON_VENV_TOUCHFILE}"
+    VERBATIM
+)
+
+# the touchfile is cleaned up by cmake automatically, we only need to remove
+# the venv directory
+set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES "${PYTHON_VENV_DIR}")
+
+add_custom_target(BuildPyVirtualEnv
+                  DEPENDS ${PYTHON_VENV_TOUCHFILE})
diff --git a/configure.ac b/configure.ac
@@ -331,18 +331,23 @@ AC_ARG_WITH(systemd-journal,
                                          Link against the system supplied or the wrapper library. (default: auto)]
               ,,with_systemd_journal="auto")
 
+AC_ARG_ENABLE(kafka,
+              [  --enable-kafka         Enable rdkafka support]
+              ,,enable_kafka="auto")
+
 AC_ARG_ENABLE(python,
               [  --disable-python       Disable Python module]
               ,,enable_python="auto")
 
-AC_ARG_ENABLE(kafka,
-              [  --enable-kafka         Enable rdkafka support]
-              ,,enable_kafka="auto")
 
 AC_ARG_WITH(python,
               [  --with-python=VERSION    Build with a specific version of python]
               ,,with_python="auto")
 
+AC_ARG_WITH(python-packages,
+              [  --with-python-packages=[system|venv]    Use Python packages from the system or a private virtualenv (default: venv)]
+              ,,with_python_packages="venv")
+
 AC_ARG_WITH(python3-home,
               [  --with-python3-home=path Use path as a hard-coded Python home directory that can not be overwritten by the PYTHONHOME environment variable],
               python3_home_dir=$with_python3_home)
@@ -1516,21 +1521,17 @@ else
   with_python=""
 fi
 
-dnl ***************************************************************************
-dnl pylint
-dnl ***************************************************************************
-
-if test "x$with_pylint" != "x"; then
-    PYLINT=$with_pylint
-else
-    if test "$python_found" = "yes"; then
-        AC_CHECK_PROG(PYLINT, pylint3, pylint3)
-        if test "x$PYLINT" = "x"; then
-          AC_CHECK_PROG(PYLINT, pylint, pylint)
-        fi
+if test "x$enable_python" = "xyes"; then
+    if test "x$with_python_packages" = "xvenv"; then
+        PYTHON_VENV_DIR=`pwd`/venv
+        PYTHON_VENV=${PYTHON_VENV_DIR}/bin/python
+    else
+        with_python_packages="system"
+        PYTHON_VENV=$PYTHON
     fi
 fi
 
+
 dnl ***************************************************************************
 dnl docbook
 dnl ***************************************************************************
@@ -1969,10 +1970,11 @@ AM_CONDITIONAL([HAVE_INOTIFY], [test x$ac_cv_func_inotify_init = xyes])
 AM_CONDITIONAL([HAVE_GETRANDOM], [test x$ac_cv_func_getrandom = xyes])
 AM_CONDITIONAL([HAVE_FMEMOPEN], [test x$ac_cv_func_fmemopen = xyes])
 AM_CONDITIONAL(ENABLE_IPV6, [test $enable_ipv6 = yes])
-AM_CONDITIONAL(PYLINT, [test "x$PYLINT" != "x"])
 
 
-AC_SUBST(PYLINT)
+AC_SUBST(PYTHON_VENV)
+AC_SUBST(PYTHON_VENV_DIR)
+AC_SUBST(with_python_packages)
 AC_SUBST(timezonedir)
 AC_SUBST(pidfiledir)
 AC_SUBST(moduledir)
@@ -2093,7 +2095,7 @@ echo "  GEOIP2 support (module)     : ${enable_geoip2:=no}"
 echo "  Redis support (module)      : ${enable_redis:=no}"
 echo "  Kafka support (module)      : ${enable_kafka:=no}"
 echo "  Riemann destination (module): ${enable_riemann:=no}, microseconds: ${riemann_micros:=no}"
-echo "  python                      : ${enable_python:=no} (pkg-config package: ${with_python:=none})"
+echo "  python                      : ${enable_python:=no} (pkg-config package: ${with_python:=none}, packages: ${with_python_packages})"
 echo "  java                        : ${enable_java:=no}"
 echo "  java modules                : ${enable_java_modules:=no}"
 echo "  afsnmp module               : ${enable_afsnmp:=no}"

diff --git a/dbld/rules b/dbld/rules
@@ -20,7 +20,7 @@ TARBALL_IMAGE ?= tarball
 DEFAULT_DEB_IMAGE=ubuntu-bionic
 DEFAULT_RPM_IMAGE=centos-7
 DOCKER=docker
-CONTAINER_REGISTRY ?= ghcr.io/syslog-ng
+CONTAINER_REGISTRY ?= ghcr.io/mitzkia
 MODE ?= snapshot
 VERSION ?= $(shell MODE=${MODE} scripts/version.sh)
 DOCKER_RUN_ARGS=-e USER_NAME_ON_HOST=$(shell whoami)	\

diff --git a/dev-requirements.txt b/dev-requirements.txt
@@ -0,0 +1,12 @@
+-r requirements.txt
+
+ply==3.10
+pep8==1.7.1
+isort==4.3.4
+pylint==1.8.2
+astroid==1.6.1
+logilab-common<=0.63.0
+psutil
+pytest
+pre-commit
+networkx
diff --git a/lib/cfg.h b/lib/cfg.h
@@ -216,7 +216,7 @@ __cfg_is_config_version_older(GlobalConfig *cfg, gint req)
  */
 #define __cfg_is_feature_enabled(cfg, min_version) \
   ({ \
-    /* the flip-over for min_version reached, set min_version to 0 */ G_STATIC_ASSERT(VERSION_VALUE_CURRENT < ((min_version) - 1));       \
+    /* the flip-over for min_version reached, set min_version to 0 */ G_STATIC_ASSERT(min_version == 0 || VERSION_VALUE_CURRENT < ((min_version) - 1));       \
     version_convert_from_user(cfg->user_version) >= (min_version) - 1;  \
   })
 

diff --git a/lib/template/tests/test_template.c b/lib/template/tests/test_template.c
@@ -144,6 +144,7 @@ Test(template, test_macros_v3x)
   /* pri 3, fac 19 == local3 */
 
   /* v3.x behavior */
+  cfg_set_version_without_validation(configuration, VERSION_VALUE_3_38);
 
   assert_template_format_value_and_type("$FACILITY", "local3", LM_VT_STRING);
   assert_template_format_value_and_type("$FACILITY_NUM", "19", LM_VT_STRING);
@@ -483,18 +484,20 @@ Test(template, test_single_values_and_literal_strings_are_considered_trivial)
 {
   LogMessage *msg = create_sample_message();
 
+  cfg_set_version_without_validation(configuration, VERSION_VALUE_4_0);
   assert_template_trivial_value("", msg, "", LM_VT_STRING);
   assert_template_trivial_value(" ", msg, " ", LM_VT_STRING);
   assert_template_trivial_value("literal", msg, "literal", LM_VT_STRING);
   assert_template_trivial_value("$1", msg, "first-match", LM_VT_STRING);
   assert_template_trivial_value("$MSG", msg, "árvíztűrőtükörfúrógép", LM_VT_STRING);
   assert_template_trivial_value("$HOST", msg, "bzorp", LM_VT_STRING);
   assert_template_trivial_value("${APP.VALUE}", msg, "value", LM_VT_STRING);
-  assert_template_trivial_value("${number1}", msg, "123", LM_VT_STRING);
-
-  cfg_set_version_without_validation(configuration, VERSION_VALUE_4_0);
   assert_template_trivial_value("${number1}", msg, "123", LM_VT_INTEGER);
 
+
+  cfg_set_version_without_validation(configuration, VERSION_VALUE_3_38);
+  assert_template_trivial_value("${number1}", msg, "123", LM_VT_STRING);
+
   log_msg_unref(msg);
 }
 

diff --git a/lib/tests/test_lexer.c b/lib/tests/test_lexer.c
@@ -345,40 +345,6 @@ foo\n");
                "@version parsing mismatch, value %04x expected %04x", configuration->user_version, VERSION_VALUE_CURRENT);
 }
 
-Test(lexer, feature_flip_to_typing)
-{
-  gchar buf[128];
-  parser->lexer->ignore_pragma = FALSE;
-
-  start_grabbing_messages();
-
-  cfg_set_version_without_validation(configuration, 0);
-
-  gint flip_version = FEATURE_TYPING_MIN_VERSION - 1;
-  g_snprintf(buf, sizeof(buf), "@version: %d.%d\nbar\n",
-             (flip_version & 0xFF00) >> 8,
-             flip_version & 0xFF);
-  _input(buf);
-  assert_parser_identifier("bar");
-  cr_assert(configuration->user_version == flip_version);
-  assert_grabbed_log_contains("experimental behaviors of the future syslog-ng");
-
-  cr_assert(cfg_is_config_version_older(configuration, VERSION_VALUE_4_0));
-  cr_assert(cfg_is_typing_feature_enabled(configuration));
-
-  reset_grabbed_messages();
-
-  cfg_set_version_without_validation(configuration, 0);
-  g_snprintf(buf, sizeof(buf), "@version: %d.%d\nbar\n",
-             (VERSION_VALUE_4_0 & 0xFF00) >> 8,
-             VERSION_VALUE_4_0 & 0xFF);
-  _input(buf);
-  assert_parser_identifier("bar");
-  assert_grabbed_log_contains("experimental behaviors of the future syslog-ng");
-  cr_assert(!cfg_is_config_version_older(configuration, VERSION_VALUE_4_0));
-  cr_assert(cfg_is_typing_feature_enabled(configuration));
-}
-
 Test(lexer, test_lexer_others)
 {
   _input("#This is a full line comment\nfoobar");

diff --git a/modules/dbparser/dbparser-grammar.ym b/modules/dbparser/dbparser-grammar.ym
@@ -67,6 +67,7 @@ SyntheticMessage *last_message;
 %token KW_PROGRAM_TEMPLATE
 %token KW_MESSAGE_TEMPLATE
 %token KW_SORT_KEY
+%token KW_PREFIX
 
 %type <num> stateful_parser_inject_mode
 %type <ptr> synthetic_message
@@ -107,6 +108,7 @@ parser_db_opt
              log_db_parser_set_program_template_ref(last_parser, $3);
           }
         | KW_MESSAGE_TEMPLATE '(' template_content ')'    { log_parser_set_template(last_parser, $3); }
+	| KW_PREFIX '(' string ')'				{ log_db_parser_set_prefix(((LogDBParser *) last_parser), $3); free($3); };
 	| stateful_parser_opt
         ;
 
@@ -163,6 +165,7 @@ grouping_by_opt
             CHECK_ERROR_WITHOUT_MESSAGE(cfg_parser_parse(&filter_expr_parser, lexer, (gpointer *) &filter_expr, NULL), @1);
             grouping_by_set_trigger_condition(last_parser, filter_expr);
           } ')'
+	| KW_PREFIX '(' string ')'				{ grouping_by_set_prefix(last_parser, $3); free($3); };
 	| stateful_parser_opt
 	;
 

diff --git a/modules/dbparser/dbparser-parser.c b/modules/dbparser/dbparser-parser.c
@@ -48,6 +48,7 @@ static CfgLexerKeyword dbparser_keywords[] =
   { "having",             KW_HAVING },
   { "trigger",            KW_TRIGGER },
   { "value",              KW_VALUE },
+  { "prefix",             KW_PREFIX },
   { "program_template",   KW_PROGRAM_TEMPLATE },
   { "message_template",   KW_MESSAGE_TEMPLATE },
   { NULL }