feat: add HTTP compression (#2254)

splunk · Nov 14, 2023 · a66f8c6 · a66f8c6
1 parent 0bb4f91
commit a66f8c6
Show file tree

Hide file tree

Showing 6 changed files with 32 additions and 1 deletion.
diff --git a/.github/workflows/ci-lite.yaml b/.github/workflows/ci-lite.yaml
@@ -194,6 +194,7 @@ jobs:
           SC4S_DEST_SPLUNK_HEC_DEFAULT_URL: https://splunk:8088
           SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630
           SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY: "no"
+          SC4S_DEST_SPLUNK_HEC_DEFAULT_HTTP_COMPRESSION: "yes"
           SC4S_LISTEN_PFSENSE_FIREWALL_TCP_PORT: 6000
           SC4S_LISTEN_SIMPLE_TEST_ONE_TCP_PORT: 5514
           SC4S_LISTEN_SIMPLE_TEST_ONE_UDP_PORT: 5514

diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml
@@ -194,6 +194,7 @@ jobs:
           SC4S_DEST_SPLUNK_HEC_DEFAULT_URL: https://splunk:8088
           SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630
           SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY: "no"
+          SC4S_DEST_SPLUNK_HEC_DEFAULT_HTTP_COMPRESSION: "yes"
           SC4S_LISTEN_PFSENSE_FIREWALL_TCP_PORT: 6000
           SC4S_LISTEN_SIMPLE_TEST_ONE_TCP_PORT: 5514
           SC4S_LISTEN_SIMPLE_TEST_ONE_UDP_PORT: 5514

diff --git a/docs/destinations.md b/docs/destinations.md
@@ -144,6 +144,18 @@ can be supported (defined by syslog-ng OSE)
 | SC4S_DEST_SPLUNK_HEC_&lt;ID&gt;_MODE | string | "GLOBAL" or "SELECT" |
 | SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY | yes(default) or no | verify HTTP(s) certificate |
 
+### HTTP Compression
+
+HTTP traffic compression helps to reduce network bandwidth usage. SC4S currently supports gzip for compressing transmitted traffic.\
+Using the 'gzip' compression algorithm can result in lower CPU load and increased utilization of RAM. The algorithm may also cause a decrease in performance. Tests observed a decrease in message processing speed by 6% to 7%.\
+Compression affects the content but does not affect the HTTP headers. Enable batch packet processing to make the solution particularly efficient, as this allows compression of a large number of logs at once.
+
+
+| Variable | Values        | Description |
+|----------|---------------|-------------|
+| SC4S_DEST_SPLUNK_HEC_&lt;ID&gt;_HTTP_COMPRESSION;       | yes or no(default) | compress outgoing HTTP traffic using gzip method |
+
+
 ## Syslog Standard destination.
 
 Note: in many cases destinations incorrectly assert "syslog" support. IETF standards RFC5424, RFC5425, RFC6587 define the use of "syslog" as a network protocol. Often the actual configuration required is Legacy BSD syslog which is NOT a standard and was documented "historically" in RFC3164 see BSD Destination section.
@@ -158,7 +170,7 @@ Note: in many cases destinations incorrectly assert "syslog" support. IETF stand
 
 ## BSD legacy destination (Non standard)
 
-Note: in many cases destinations incorrectly assert "syslog" support. IETF standards RFC5424, RFC5425, RFC6587 define the use of "syslog" as a network protocol. Often the actual configuration required is Legacy BSD syslog which is NOT a standard and was documented "historically" in RFC3164 see BSD Destination section.
+Note: in many cases, destinations incorrectly assert "syslog" support. Internet Engineering Task Force standards RFC5424, RFC5425, and RFC6587 define the use of "syslog" as a network protocol. Often the actual configuration required is Legacy BSD syslog which is not a standard and was documented in RFC3164.
 
 | Variable | Values        | Description |
 |----------|---------------|-------------|

diff --git a/package/etc/conf.d/destinations/dest_hec/plugin.jinja b/package/etc/conf.d/destinations/dest_hec/plugin.jinja
@@ -43,5 +43,9 @@ destination d_hec{{ dest_mode }}{{ altname }}{
         )
         use-system-cert-store(yes)
         body('{{ msg_template }}')
+    {%- if http_compression %}
+        content-compression("gzip")
+        accept-encoding("all")
+    {%- endif %}
     );
 };
diff --git a/package/etc/conf.d/destinations/dest_hec/plugin.py b/package/etc/conf.d/destinations/dest_hec/plugin.py
@@ -101,6 +101,17 @@ def hec_endpoint_collector(hec_path, url_hec):
     else:
         headers.append("Connection: keep-alive")
 
+    if os.getenv(f"SC4S_DEST_SPLUNK_HEC_{group}_HTTP_COMPRESSION", "no").lower() in [
+        "true",
+        "1",
+        "t",
+        "y",
+        "yes",
+    ]:
+        http_compression = True
+    else:
+        http_compression = False
+
     msg = tm.render(
         group=group,
         altname=altname,
@@ -136,6 +147,7 @@ def hec_endpoint_collector(hec_path, url_hec):
         peer_verify=os.getenv(f"SC4S_DEST_SPLUNK_HEC_{group}_TLS_VERIFY", "yes"),
         cipher_suite=os.getenv(f"SC4S_DEST_SPLUNK_HEC_{group}_CIPHER_SUITE"),
         ssl_version=os.getenv(f"SC4S_DEST_SPLUNK_HEC_{group}_SSL_VERSION"),
+        http_compression=http_compression
     )
 
     print(msg)
diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml
@@ -37,6 +37,7 @@ services:
       - SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://splunk:8088
       - SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=${SPLUNK_HEC_TOKEN}
       - SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY=no
+      #- SC4S_DEST_SPLUNK_HEC_DEFAULT_HTTP_COMPRESSION=yes
       - SC4S_DEST_SYSLOG_NCSYSLOG_HOST=ncsyslog
       - SC4S_DEST_SYSLOG_NCSYSLOG_PORT=2514
       - SC4S_DEST_SYSLOG_NCSYSLOG_MODE=GLOBAL