From 709c641127449113a9903cc27765c9330e5f1bb1 Mon Sep 17 00:00:00 2001 From: Ilya Kheifets Date: Fri, 26 Jan 2024 13:37:28 +0100 Subject: [PATCH] feat: SC4S + EP POC --- package/etc/conf.d/conflib/_common/t_templates.conf | 1 - .../conf.d/conflib/fallback/app-fallbackz-lastchance.conf | 1 - package/etc/conf.d/sources/source_syslog/plugin.jinja | 6 ++++++ package/etc/conf.d/sources/source_syslog/plugin.py | 1 + package/sbin/entrypoint.sh | 4 ++-- tests/docker-compose.yml | 2 ++ 6 files changed, 11 insertions(+), 4 deletions(-) diff --git a/package/etc/conf.d/conflib/_common/t_templates.conf b/package/etc/conf.d/conflib/_common/t_templates.conf index da7738bf24..3d43a5420f 100644 --- a/package/etc/conf.d/conflib/_common/t_templates.conf +++ b/package/etc/conf.d/conflib/_common/t_templates.conf @@ -138,7 +138,6 @@ template t_JSON_5424_SDATA { template t_splunk_hec { template('$(format-json - time=$(if ("${.netsource.sc4s_use_recv_time}" eq "yes") "$R_UNIXTIME" "$S_UNIXTIME") host=$(lowercase ${HOST}) source=${.splunk.source:-SC4S} sourcetype=${.splunk.sourcetype:-sc4s:fallback} diff --git a/package/etc/conf.d/conflib/fallback/app-fallbackz-lastchance.conf b/package/etc/conf.d/conflib/fallback/app-fallbackz-lastchance.conf index ea1b19219f..84bfe488c5 100644 --- a/package/etc/conf.d/conflib/fallback/app-fallbackz-lastchance.conf +++ b/package/etc/conf.d/conflib/fallback/app-fallbackz-lastchance.conf @@ -6,7 +6,6 @@ block parser app-fallbackz-lastchance() { vendor('splunk') product('sc4s') class("fallback") - template('t_fallback_kv') ); }; diff --git a/package/etc/conf.d/sources/source_syslog/plugin.jinja b/package/etc/conf.d/sources/source_syslog/plugin.jinja index 89caf04801..97be458b11 100644 --- a/package/etc/conf.d/sources/source_syslog/plugin.jinja +++ b/package/etc/conf.d/sources/source_syslog/plugin.jinja @@ -104,6 +104,7 @@ source s_{{ port_id }} { {%- endfor %} }; + {% if not ignore_parsing %} {%- if vendor and product %} parser { p_set_netsource_fields( @@ -265,6 +266,8 @@ source s_{{ port_id }} { }; }; + {% endif %} + rewrite { groupunset(values('.tmp.*')); }; @@ -426,6 +429,8 @@ source s_{{ port_id }} { rewrite { set('$(lowercase "$HOST")' value(HOST)); }; + + {% if not ignore_parsing %} {%- if not vendor or not product %} {%- if use_vpscache == True %} if { @@ -466,6 +471,7 @@ source s_{{ port_id }} { rewrite{ groupunset(values('.tmp.*')); }; + {% endif %} if { filter(f_is_source_identified); diff --git a/package/etc/conf.d/sources/source_syslog/plugin.py b/package/etc/conf.d/sources/source_syslog/plugin.py index 950b58c944..4efa2e38f6 100755 --- a/package/etc/conf.d/sources/source_syslog/plugin.py +++ b/package/etc/conf.d/sources/source_syslog/plugin.py @@ -60,6 +60,7 @@ def normalize_env_variable_input(env_variable: str): vendor=vendor, product=product, enable_ipv6=enable_ipv6, + ignore_parsing=normalize_env_variable_input("SC4S_IGNORE_PARSING"), store_raw_message=normalize_env_variable_input("SC4S_SOURCE_STORE_RAWMSG"), port_id=port_id, use_reverse_dns=normalize_env_variable_input("SC4S_USE_REVERSE_DNS"), diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh index e20ee84c78..f95e55119e 100755 --- a/package/sbin/entrypoint.sh +++ b/package/sbin/entrypoint.sh @@ -175,13 +175,13 @@ then SC4S_DEST_SPLUNK_HEC_EVENTS_INDEX=$(cat $SC4S_ETC/conf.d/local/context/splunk_metadata.csv | grep ',index,' | grep sc4s_events | cut -d, -f 3) export SC4S_DEST_SPLUNK_HEC_EVENTS_INDEX=${SC4S_DEST_SPLUNK_HEC_EVENTS_INDEX:=main} - if curl -s -S ${NO_VERIFY} "${HEC}?/index=${SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX}" -H "Authorization: Splunk ${SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN}" -d '{"event": "HEC TEST EVENT", "sourcetype": "sc4s:probe"}' 2>&1 | grep -v '{"text":"Success"' + if curl -s -S ${NO_VERIFY} "${HEC}?/index=${SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX}" -H "Authorization: Splunk ${SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN}" -d '{"event": "HEC TEST EVENT", "sourcetype": "sc4s:probe"}' 2>&1 | grep -v '{"text": "Success"' then echo -e "SC4S_ENV_CHECK_HEC: Invalid Splunk HEC URL, invalid token, or other HEC connectivity issue index=${SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX}. sourcetype=sc4s:fallback\nStartup will continue to prevent data loss if this is a transient failure." echo "" else echo -e "SC4S_ENV_CHECK_HEC: Splunk HEC connection test successful to index=${SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX} for sourcetype=sc4s:fallback..." - if curl -s -S ${NO_VERIFY} "${HEC}?/index=${SC4S_DEST_SPLUNK_HEC_EVENTS_INDEX}" -H "Authorization: Splunk ${SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN}" -d '{"event": "HEC TEST EVENT", "sourcetype": "sc4s:probe"}' 2>&1 | grep -v '{"text":"Success"' + if curl -s -S ${NO_VERIFY} "${HEC}?/index=${SC4S_DEST_SPLUNK_HEC_EVENTS_INDEX}" -H "Authorization: Splunk ${SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN}" -d '{"event": "HEC TEST EVENT", "sourcetype": "sc4s:probe"}' 2>&1 | grep -v '{"text": "Success"' then echo -e "SC4S_ENV_CHECK_HEC: Invalid Splunk HEC URL, invalid token, or other HEC connectivity issue for index=${SC4S_DEST_SPLUNK_HEC_EVENTS_INDEX}. sourcetype=sc4s:events \nStartup will continue to prevent data loss if this is a transient failure." echo "" diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index 23e6aae911..410371d5c4 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -37,6 +37,8 @@ services: - SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://splunk:8088 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=${SPLUNK_HEC_TOKEN} - SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY=no + - SC4S_DEBUG_HEC=yes + - SC4S_IGNORE_PARSING=yes #- SC4S_DEST_SPLUNK_HEC_DEFAULT_HTTP_COMPRESSION=yes - SC4S_DEST_SYSLOG_NCSYSLOG_HOST=ncsyslog - SC4S_DEST_SYSLOG_NCSYSLOG_PORT=2514