From c354851d5ec0fed41f17ab89c6c9238e04c82c3e Mon Sep 17 00:00:00 2001 From: Tom Hendrikx Date: Mon, 17 Nov 2025 23:38:38 +0100 Subject: [PATCH 1/6] Update to latest logstash version --- .github/workflows/test_config_syntax.yml | 2 +- .github/workflows/test_grok_patterns.yml | 2 +- test_config_syntax.sh | 2 +- test_pipeline.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_config_syntax.yml b/.github/workflows/test_config_syntax.yml index 38ad6f9..605db7f 100644 --- a/.github/workflows/test_config_syntax.yml +++ b/.github/workflows/test_config_syntax.yml @@ -5,7 +5,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - logstash-version: ['8.14.1', '7.17.22'] + logstash-version: ['9.2.1', '8.19.7', '7.17.28'] steps: - name: Checkout code uses: actions/checkout@v4 diff --git a/.github/workflows/test_grok_patterns.yml b/.github/workflows/test_grok_patterns.yml index 160f8ec..2a7256a 100644 --- a/.github/workflows/test_grok_patterns.yml +++ b/.github/workflows/test_grok_patterns.yml @@ -10,6 +10,6 @@ jobs: submodules: true - uses: ruby/setup-ruby@v1 with: - ruby-version: '3.2' + ruby-version: '3.4' - run: gem install jls-grok minitest - run: ruby test/test.rb diff --git a/test_config_syntax.sh b/test_config_syntax.sh index 228e42d..6f96b11 100755 --- a/test_config_syntax.sh +++ b/test_config_syntax.sh @@ -8,7 +8,7 @@ set -eu -LOGSTASH_VERSION=8.14.1 +LOGSTASH_VERSION=9.2.1 docker run \ --rm \ diff --git a/test_pipeline.sh b/test_pipeline.sh index a7ef1b9..21abe89 100755 --- a/test_pipeline.sh +++ b/test_pipeline.sh @@ -9,7 +9,7 @@ set -eu -LOGSTASH_VERSION=8.14.1 +LOGSTASH_VERSION=9.2.1 INPUT=$(mktemp tmp.logstash.in.XXXXX) OUTPUT=$(mktemp tmp.logstash.out.XXXXX) From 68cdf46795a3b1c08d80890c6b66bac7774a0938 Mon Sep 17 00:00:00 2001 From: Tom Hendrikx Date: Mon, 17 Nov 2025 23:04:18 +0100 Subject: [PATCH 2/6] Add new test files that verify extraction of 'tls=...' to key-value data These tests merely verify that the syntax used by postfix with separators like /:?! is transferred correctly to the key-value data. --- test/tls_features_0001.yaml | 6 ++++++ test/tls_features_0002.yaml | 6 ++++++ test/tls_features_0003.yaml | 6 ++++++ test/tls_features_0004.yaml | 6 ++++++ test/tls_features_0005.yaml | 6 ++++++ test/tls_features_0006.yaml | 6 ++++++ 6 files changed, 36 insertions(+) create mode 100644 test/tls_features_0001.yaml create mode 100644 test/tls_features_0002.yaml create mode 100644 test/tls_features_0003.yaml create mode 100644 test/tls_features_0004.yaml create mode 100644 test/tls_features_0005.yaml create mode 100644 test/tls_features_0006.yaml diff --git a/test/tls_features_0001.yaml b/test/tls_features_0001.yaml new file mode 100644 index 0000000..e02d705 --- /dev/null +++ b/test/tls_features_0001.yaml @@ -0,0 +1,6 @@ +pattern: "^%{POSTFIX_SMTP}$" +# TLS features: security level only (single word) +data: "7EE668039: to=, relay=mail.example.com[1.2.3.4]:25, delay=3.6, delays=0.08/0.02/0.85/0.14, tls=dane, dsn=2.1.5, status=sent (250 2.0.0 Ok: queued as 153053D)" +results: + postfix_queueid: 7EE668039 + postfix_keyvalue_data: "to=, relay=mail.example.com[1.2.3.4]:25, delay=3.6, delays=0.08/0.02/0.85/0.14, tls=dane, dsn=2.1.5," diff --git a/test/tls_features_0002.yaml b/test/tls_features_0002.yaml new file mode 100644 index 0000000..f2b4573 --- /dev/null +++ b/test/tls_features_0002.yaml @@ -0,0 +1,6 @@ +pattern: "^%{POSTFIX_SMTP}$" +# TLS features: security level undecided (with question mark) +data: "7EE668039: to=, relay=mail.example.com[1.2.3.4]:25, delay=3.6, delays=0.08/0.02/0.85/0.14, tls=may?, dsn=2.1.5, status=sent (250 2.0.0 Ok: queued as 153053D)" +results: + postfix_queueid: 7EE668039 + postfix_keyvalue_data: "to=, relay=mail.example.com[1.2.3.4]:25, delay=3.6, delays=0.08/0.02/0.85/0.14, tls=may?, dsn=2.1.5," diff --git a/test/tls_features_0003.yaml b/test/tls_features_0003.yaml new file mode 100644 index 0000000..1e09748 --- /dev/null +++ b/test/tls_features_0003.yaml @@ -0,0 +1,6 @@ +pattern: "^%{POSTFIX_SMTP}$" +# TLS features: with downgrade level (separated by colon) +data: "7EE668039: to=, relay=mail.example.com[1.2.3.4]:25, delay=3.6, delays=0.08/0.02/0.85/0.14, tls=may:none, dsn=2.1.5, status=sent (250 2.0.0 Ok: queued as 153053D)" +results: + postfix_queueid: 7EE668039 + postfix_keyvalue_data: "to=, relay=mail.example.com[1.2.3.4]:25, delay=3.6, delays=0.08/0.02/0.85/0.14, tls=may:none, dsn=2.1.5," diff --git a/test/tls_features_0004.yaml b/test/tls_features_0004.yaml new file mode 100644 index 0000000..8a525fb --- /dev/null +++ b/test/tls_features_0004.yaml @@ -0,0 +1,6 @@ +pattern: "^%{POSTFIX_SMTP}$" +# TLS features: requiretls +data: "7EE668039: to=, relay=mail.example.com[1.2.3.4]:25, delay=3.6, delays=0.08/0.02/0.85/0.14, tls=dane/requiretls, dsn=2.1.5, status=sent (250 2.0.0 Ok: queued as 153053D)" +results: + postfix_queueid: 7EE668039 + postfix_keyvalue_data: "to=, relay=mail.example.com[1.2.3.4]:25, delay=3.6, delays=0.08/0.02/0.85/0.14, tls=dane/requiretls, dsn=2.1.5," diff --git a/test/tls_features_0005.yaml b/test/tls_features_0005.yaml new file mode 100644 index 0000000..d6da13e --- /dev/null +++ b/test/tls_features_0005.yaml @@ -0,0 +1,6 @@ +pattern: "^%{POSTFIX_SMTP}$" +# TLS features: requiretls violation +data: "7EE668039: to=, relay=mail.example.com[1.2.3.4]:25, delay=3.6, delays=0.08/0.02/0.85/0.14, tls=dane/!requiretls:nocertmatch, dsn=2.1.5, status=sent (250 2.0.0 Ok: queued as 153053D)" +results: + postfix_queueid: 7EE668039 + postfix_keyvalue_data: "to=, relay=mail.example.com[1.2.3.4]:25, delay=3.6, delays=0.08/0.02/0.85/0.14, tls=dane/!requiretls:nocertmatch, dsn=2.1.5," diff --git a/test/tls_features_0006.yaml b/test/tls_features_0006.yaml new file mode 100644 index 0000000..2b448cd --- /dev/null +++ b/test/tls_features_0006.yaml @@ -0,0 +1,6 @@ +pattern: "^%{POSTFIX_SMTP}$" +# TLS features: requiretls policy undecided +data: "7EE668039: to=, relay=mail.example.com[1.2.3.4]:25, delay=3.6, delays=0.08/0.02/0.85/0.14, tls=dane/requiretls?, dsn=2.1.5, status=sent (250 2.0.0 Ok: queued as 153053D)" +results: + postfix_queueid: 7EE668039 + postfix_keyvalue_data: "to=, relay=mail.example.com[1.2.3.4]:25, delay=3.6, delays=0.08/0.02/0.85/0.14, tls=dane/requiretls?, dsn=2.1.5," From 25934f52f6658f19949e7db8e34448d38ab59cd1 Mon Sep 17 00:00:00 2001 From: Tom Hendrikx Date: Mon, 17 Nov 2025 23:12:45 +0100 Subject: [PATCH 3/6] Add tls features logging in pipeline check example --- test_pipeline.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test_pipeline.sh b/test_pipeline.sh index 21abe89..e682c97 100755 --- a/test_pipeline.sh +++ b/test_pipeline.sh @@ -23,7 +23,7 @@ perform_cleanup() { trap perform_cleanup INT TERM echo Preparing input data -echo "postfix/smtp[123]: 7EE668039: to=, relay=127.0.0.1[127.0.0.1]:2525, delay=3.6, delays=0.2/0.02/0.04/3.3, dsn=2.0.0, status=sent (250 2.0.0 Ok: queued as 153053D)" > "$INPUT" +echo "postfix/smtp[123]: 7EE668039: to=, relay=127.0.0.1[127.0.0.1]:2525, delay=3.6, delays=0.2/0.02/0.04/3.3, dsn=2.0.0, tls=dane:none/!requiretls:nostarttls, status=sent (250 2.0.0 Ok: queued as 153053D)" > "$INPUT" echo Preparing pipeline config cat > "$PIPELINE" << EOF @@ -70,10 +70,10 @@ echo if test "$(jq .tags[0] "$OUTPUT")" = '"_grok_postfix_success"'; then echo Grok processing successful! - jq . "$OUTPUT" + jq --sort-keys . "$OUTPUT" else echo "Grok processing failed :<" - jq . "$OUTPUT" + jq --sort-keys . "$OUTPUT" exit 1 fi From d18bb8ca936dad93bd1a7423b9ee3605ea848823 Mon Sep 17 00:00:00 2001 From: Tom Hendrikx Date: Sat, 29 Nov 2025 15:12:42 +0100 Subject: [PATCH 4/6] Extract TLS security and downgrade levels from 'postfix_tls' field --- 50-filter-postfix.conf | 8 ++++++++ postfix.grok | 3 +++ test/tls_features_0007.yaml | 4 ++++ test/tls_features_0008.yaml | 5 +++++ test_pipeline.sh | 2 ++ 5 files changed, 22 insertions(+) create mode 100644 test/tls_features_0007.yaml create mode 100644 test/tls_features_0008.yaml diff --git a/50-filter-postfix.conf b/50-filter-postfix.conf index b48acdb..3450f5e 100644 --- a/50-filter-postfix.conf +++ b/50-filter-postfix.conf @@ -222,6 +222,14 @@ filter { remove_field => [ "postfix_delays" ] } } + if [postfix_tls] { + grok { + patterns_dir => "/etc/logstash/patterns.d" + match => ["postfix_tls", "^%{POSTFIX_TLS_FEATURES}$"] + tag_on_failure => [ "_grok_kv_postfix_tls_nomatch" ] + remove_field => [ "postfix_tls" ] + } + } } # process command counter data if it exists diff --git a/postfix.grok b/postfix.grok index b121df6..a3e85bc 100644 --- a/postfix.grok +++ b/postfix.grok @@ -121,6 +121,9 @@ POSTFIX_VERIFY_CACHE cache %{DATA} (?(full|partial) # local patterns POSTFIX_LOCAL_DELIVERY %{POSTFIX_KEYVALUE} status=%{STATUS_WORD:postfix_status}( \(%{GREEDYDATA:postfix_local_response}\))? +# TLS features +POSTFIX_TLS_FEATURES %{STATUS_WORD:postfix_tls_security_level}(:%{STATUS_WORD:postfix_tls_downgrade_level})? + # aggregate all patterns POSTFIX_SMTPD %{POSTFIX_SMTPD_CONNECT}|%{POSTFIX_SMTPD_DISCONNECT}|%{POSTFIX_SMTPD_LOSTCONN}|%{POSTFIX_SMTPD_NOQUEUE}|%{POSTFIX_SMTPD_PIPELINING}|%{POSTFIX_TLSCONN}|%{POSTFIX_WARNING}|%{POSTFIX_SMTPD_PROXY}|%{POSTFIX_KEYVALUE} POSTFIX_CLEANUP %{POSTFIX_CLEANUP_MESSAGEID}|%{POSTFIX_CLEANUP_MILTER}|%{POSTFIX_CLEANUP_PREPEND}|%{POSTFIX_WARNING}|%{POSTFIX_KEYVALUE} diff --git a/test/tls_features_0007.yaml b/test/tls_features_0007.yaml new file mode 100644 index 0000000..e4739d5 --- /dev/null +++ b/test/tls_features_0007.yaml @@ -0,0 +1,4 @@ +pattern: "^%{POSTFIX_TLS_FEATURES}$" +data: "dane" +results: + postfix_tls_security_level: dane diff --git a/test/tls_features_0008.yaml b/test/tls_features_0008.yaml new file mode 100644 index 0000000..a566f53 --- /dev/null +++ b/test/tls_features_0008.yaml @@ -0,0 +1,5 @@ +pattern: "^%{POSTFIX_TLS_FEATURES}$" +data: "dane:none" +results: + postfix_tls_security_level: dane + postfix_tls_downgrade_level: none diff --git a/test_pipeline.sh b/test_pipeline.sh index e682c97..499766e 100755 --- a/test_pipeline.sh +++ b/test_pipeline.sh @@ -63,6 +63,8 @@ CONTAINER_ID=$(docker run --rm --detach \ printf "Waiting for output from logstash " until test -s "$OUTPUT"; do + # For debugging a crashing container (probably invalid configuration) + # docker inspect "$CONTAINER_ID" | jq '.[0].State' printf "." sleep 2 done From 44172d92c68ad93058cf905626b315596af1533e Mon Sep 17 00:00:00 2001 From: Tom Hendrikx Date: Sat, 29 Nov 2025 15:15:44 +0100 Subject: [PATCH 5/6] Extract TLS undecided policy from 'postfix_tls' --- 50-filter-postfix.conf | 5 ++++- postfix.grok | 3 ++- test/tls_features_0009.yaml | 5 +++++ test_pipeline.sh | 2 +- 4 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 test/tls_features_0009.yaml diff --git a/50-filter-postfix.conf b/50-filter-postfix.conf index 3450f5e..bffbad8 100644 --- a/50-filter-postfix.conf +++ b/50-filter-postfix.conf @@ -297,6 +297,9 @@ filter { "postfix_delay_transmission", "float", "postfix_postscreen_violation_time", "float" ] + gsub => [ + # rewrite some extracted values + "postfix_tls_policy_undecided", "\?", "true" + ] } } - diff --git a/postfix.grok b/postfix.grok index a3e85bc..43d01db 100644 --- a/postfix.grok +++ b/postfix.grok @@ -122,7 +122,8 @@ POSTFIX_VERIFY_CACHE cache %{DATA} (?(full|partial) POSTFIX_LOCAL_DELIVERY %{POSTFIX_KEYVALUE} status=%{STATUS_WORD:postfix_status}( \(%{GREEDYDATA:postfix_local_response}\))? # TLS features -POSTFIX_TLS_FEATURES %{STATUS_WORD:postfix_tls_security_level}(:%{STATUS_WORD:postfix_tls_downgrade_level})? +POSTFIX_TLS_POLICY_UNDECIDED \? +POSTFIX_TLS_FEATURES %{STATUS_WORD:postfix_tls_security_level}(:%{STATUS_WORD:postfix_tls_downgrade_level})?(%{POSTFIX_TLS_POLICY_UNDECIDED:postfix_tls_policy_undecided})? # aggregate all patterns POSTFIX_SMTPD %{POSTFIX_SMTPD_CONNECT}|%{POSTFIX_SMTPD_DISCONNECT}|%{POSTFIX_SMTPD_LOSTCONN}|%{POSTFIX_SMTPD_NOQUEUE}|%{POSTFIX_SMTPD_PIPELINING}|%{POSTFIX_TLSCONN}|%{POSTFIX_WARNING}|%{POSTFIX_SMTPD_PROXY}|%{POSTFIX_KEYVALUE} diff --git a/test/tls_features_0009.yaml b/test/tls_features_0009.yaml new file mode 100644 index 0000000..d14f41e --- /dev/null +++ b/test/tls_features_0009.yaml @@ -0,0 +1,5 @@ +pattern: "^%{POSTFIX_TLS_FEATURES}$" +data: "may?" +results: + postfix_tls_security_level: may + postfix_tls_policy_undecided: "?" diff --git a/test_pipeline.sh b/test_pipeline.sh index 499766e..6cce2c0 100755 --- a/test_pipeline.sh +++ b/test_pipeline.sh @@ -23,7 +23,7 @@ perform_cleanup() { trap perform_cleanup INT TERM echo Preparing input data -echo "postfix/smtp[123]: 7EE668039: to=, relay=127.0.0.1[127.0.0.1]:2525, delay=3.6, delays=0.2/0.02/0.04/3.3, dsn=2.0.0, tls=dane:none/!requiretls:nostarttls, status=sent (250 2.0.0 Ok: queued as 153053D)" > "$INPUT" +echo "postfix/smtp[123]: 7EE668039: to=, relay=127.0.0.1[127.0.0.1]:2525, delay=3.6, delays=0.2/0.02/0.04/3.3, dsn=2.0.0, tls=dane?, status=sent (250 2.0.0 Ok: queued as 153053D)" > "$INPUT" echo Preparing pipeline config cat > "$PIPELINE" << EOF From baab2d43d2d18166bcfa54bd32348512363c57d0 Mon Sep 17 00:00:00 2001 From: Tom Hendrikx Date: Sat, 29 Nov 2025 15:50:30 +0100 Subject: [PATCH 6/6] Extract TLS requiretls feature data from 'postfix_tls' Also switch to named captures when possible --- 50-filter-postfix.conf | 5 ++++- postfix.grok | 4 ++-- test/tls_features_0010.yaml | 5 +++++ test/tls_features_0011.yaml | 6 ++++++ test/tls_features_0012.yaml | 7 +++++++ test_pipeline.sh | 2 +- 6 files changed, 25 insertions(+), 4 deletions(-) create mode 100644 test/tls_features_0010.yaml create mode 100644 test/tls_features_0011.yaml create mode 100644 test/tls_features_0012.yaml diff --git a/50-filter-postfix.conf b/50-filter-postfix.conf index bffbad8..50ab186 100644 --- a/50-filter-postfix.conf +++ b/50-filter-postfix.conf @@ -299,7 +299,10 @@ filter { ] gsub => [ # rewrite some extracted values - "postfix_tls_policy_undecided", "\?", "true" + "postfix_tls_policy_undecided", "\?", "true", + "postfix_requiretls_policy_undecided", "\?", "true", + "postfix_requiretls_policy_violation", "\!", "true", + "postfix_requiretls", "requiretls", "true" ] } } diff --git a/postfix.grok b/postfix.grok index 43d01db..07a70e2 100644 --- a/postfix.grok +++ b/postfix.grok @@ -122,8 +122,8 @@ POSTFIX_VERIFY_CACHE cache %{DATA} (?(full|partial) POSTFIX_LOCAL_DELIVERY %{POSTFIX_KEYVALUE} status=%{STATUS_WORD:postfix_status}( \(%{GREEDYDATA:postfix_local_response}\))? # TLS features -POSTFIX_TLS_POLICY_UNDECIDED \? -POSTFIX_TLS_FEATURES %{STATUS_WORD:postfix_tls_security_level}(:%{STATUS_WORD:postfix_tls_downgrade_level})?(%{POSTFIX_TLS_POLICY_UNDECIDED:postfix_tls_policy_undecided})? +POSTFIX_TLS_FEAT_REQUIRETLS (?\!)?(?requiretls)(:(?\w+))?(?\?)? +POSTFIX_TLS_FEATURES (?\w+)(:(?\w+))?(?\?)?(/%{POSTFIX_TLS_FEAT_REQUIRETLS})? # aggregate all patterns POSTFIX_SMTPD %{POSTFIX_SMTPD_CONNECT}|%{POSTFIX_SMTPD_DISCONNECT}|%{POSTFIX_SMTPD_LOSTCONN}|%{POSTFIX_SMTPD_NOQUEUE}|%{POSTFIX_SMTPD_PIPELINING}|%{POSTFIX_TLSCONN}|%{POSTFIX_WARNING}|%{POSTFIX_SMTPD_PROXY}|%{POSTFIX_KEYVALUE} diff --git a/test/tls_features_0010.yaml b/test/tls_features_0010.yaml new file mode 100644 index 0000000..ec22bb3 --- /dev/null +++ b/test/tls_features_0010.yaml @@ -0,0 +1,5 @@ +pattern: "^%{POSTFIX_TLS_FEATURES}$" +data: "dane/requiretls" +results: + postfix_tls_security_level: dane + postfix_requiretls: requiretls diff --git a/test/tls_features_0011.yaml b/test/tls_features_0011.yaml new file mode 100644 index 0000000..02f1328 --- /dev/null +++ b/test/tls_features_0011.yaml @@ -0,0 +1,6 @@ +pattern: "^%{POSTFIX_TLS_FEATURES}$" +data: "dane/requiretls?" +results: + postfix_tls_security_level: dane + postfix_requiretls: requiretls + postfix_requiretls_policy_undecided: "?" diff --git a/test/tls_features_0012.yaml b/test/tls_features_0012.yaml new file mode 100644 index 0000000..36d925c --- /dev/null +++ b/test/tls_features_0012.yaml @@ -0,0 +1,7 @@ +pattern: "^%{POSTFIX_TLS_FEATURES}$" +data: "dane/!requiretls:nostarttls" +results: + postfix_tls_security_level: dane + postfix_requiretls: requiretls + postfix_requiretls_policy_violation: "!" + postfix_requiretls_downgrade_level: nostarttls diff --git a/test_pipeline.sh b/test_pipeline.sh index 6cce2c0..a236dcf 100755 --- a/test_pipeline.sh +++ b/test_pipeline.sh @@ -23,7 +23,7 @@ perform_cleanup() { trap perform_cleanup INT TERM echo Preparing input data -echo "postfix/smtp[123]: 7EE668039: to=, relay=127.0.0.1[127.0.0.1]:2525, delay=3.6, delays=0.2/0.02/0.04/3.3, dsn=2.0.0, tls=dane?, status=sent (250 2.0.0 Ok: queued as 153053D)" > "$INPUT" +echo "postfix/smtp[123]: 7EE668039: to=, relay=127.0.0.1[127.0.0.1]:2525, delay=3.6, delays=0.2/0.02/0.04/3.3, dsn=2.0.0, tls=dane/!requiretls:nostarttls, status=sent (250 2.0.0 Ok: queued as 153053D)" > "$INPUT" echo Preparing pipeline config cat > "$PIPELINE" << EOF