address issues with NetBox database and Logstash's NetBox cache (idah…

…olab#259); restore caching for performance reasons, but decrease TTL significantly and allow it to be specified via environment variable
mmguero-dev · Oct 27, 2023 · 92b4b34 · 92b4b34
1 parent 16d5053
commit 92b4b34
Show file tree

Hide file tree

Showing 4 changed files with 514 additions and 512 deletions.
diff --git a/Dockerfiles/logstash.Dockerfile b/Dockerfiles/logstash.Dockerfile
@@ -32,6 +32,8 @@ ARG LOGSTASH_NETBOX_ENRICHMENT=false
 ARG LOGSTASH_NETBOX_ENRICHMENT_VERBOSE=false
 ARG LOGSTASH_NETBOX_ENRICHMENT_LOOKUP_SERVICE=true
 ARG LOGSTASH_NETBOX_AUTO_POPULATE=false
+ARG LOGSTASH_NETBOX_CACHE_SIZE=1000
+ARG LOGSTASH_NETBOX_CACHE_TTL=30
 
 ENV LOGSTASH_ENRICHMENT_PIPELINE $LOGSTASH_ENRICHMENT_PIPELINE
 ENV LOGSTASH_PARSE_PIPELINE_ADDRESSES $LOGSTASH_PARSE_PIPELINE_ADDRESSES
@@ -42,6 +44,8 @@ ENV LOGSTASH_NETBOX_ENRICHMENT $LOGSTASH_NETBOX_ENRICHMENT
 ENV LOGSTASH_NETBOX_ENRICHMENT_VERBOSE $LOGSTASH_NETBOX_ENRICHMENT_VERBOSE
 ENV LOGSTASH_NETBOX_ENRICHMENT_LOOKUP_SERVICE $LOGSTASH_NETBOX_ENRICHMENT_LOOKUP_SERVICE
 ENV LOGSTASH_NETBOX_AUTO_POPULATE $LOGSTASH_NETBOX_AUTO_POPULATE
+ENV LOGSTASH_NETBOX_CACHE_SIZE $LOGSTASH_NETBOX_CACHE_SIZE
+ENV LOGSTASH_NETBOX_CACHE_TTL $LOGSTASH_NETBOX_CACHE_TTL
 
 USER root
 

diff --git a/config/logstash.env.example b/config/logstash.env.example
@@ -13,5 +13,8 @@ LOGSTASH_REVERSE_DNS=false
 LOGSTASH_NETBOX_ENRICHMENT=false
 # Whether or not unobserved network entities in Logstash data will be used to populate NetBox
 LOGSTASH_NETBOX_AUTO_POPULATE=false
+# Caching parameters for NetBox's LogStash lookups
+LOGSTASH_NETBOX_CACHE_SIZE=1000
+LOGSTASH_NETBOX_CACHE_TTL=30
 # Logstash memory allowance and other Java options
 LS_JAVA_OPTS=-server -Xms2500m -Xmx2500m -Xss1536k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true
diff --git a/logstash/pipelines/enrichment/21_netbox.conf b/logstash/pipelines/enrichment/21_netbox.conf
@@ -31,6 +31,8 @@ filter {
           "lookup_site_env" => "NETBOX_DEFAULT_SITE"
           "verbose_env" => "LOGSTASH_NETBOX_ENRICHMENT_VERBOSE"
           "netbox_token_env" => "SUPERUSER_API_TOKEN"
+          "cache_size_env" => "LOGSTASH_NETBOX_CACHE_SIZE"
+          "cache_ttl_env" => "LOGSTASH_NETBOX_CACHE_TTL"
         }
       }
       ruby {
@@ -44,6 +46,8 @@ filter {
           "lookup_service" => "false"
           "verbose_env" => "LOGSTASH_NETBOX_ENRICHMENT_VERBOSE"
           "netbox_token_env" => "SUPERUSER_API_TOKEN"
+          "cache_size_env" => "LOGSTASH_NETBOX_CACHE_SIZE"
+          "cache_ttl_env" => "LOGSTASH_NETBOX_CACHE_TTL"
           "autopopulate_env" => "LOGSTASH_NETBOX_AUTO_POPULATE"
           "default_manuf_env" => "NETBOX_DEFAULT_MANUFACTURER"
           "default_dtype_env" => "NETBOX_DEFAULT_DEVICE_TYPE"
@@ -66,6 +70,8 @@ filter {
           "lookup_site_env" => "NETBOX_DEFAULT_SITE"
           "verbose_env" => "LOGSTASH_NETBOX_ENRICHMENT_VERBOSE"
           "netbox_token_env" => "SUPERUSER_API_TOKEN"
+          "cache_size_env" => "LOGSTASH_NETBOX_CACHE_SIZE"
+          "cache_ttl_env" => "LOGSTASH_NETBOX_CACHE_TTL"
         }
       }
       ruby {
@@ -80,6 +86,8 @@ filter {
           "lookup_service_port_source" => "[destination][port]"
           "verbose_env" => "LOGSTASH_NETBOX_ENRICHMENT_VERBOSE"
           "netbox_token_env" => "SUPERUSER_API_TOKEN"
+          "cache_size_env" => "LOGSTASH_NETBOX_CACHE_SIZE"
+          "cache_ttl_env" => "LOGSTASH_NETBOX_CACHE_TTL"
           "autopopulate_env" => "LOGSTASH_NETBOX_AUTO_POPULATE"
           "default_manuf_env" => "NETBOX_DEFAULT_MANUFACTURER"
           "default_dtype_env" => "NETBOX_DEFAULT_DEVICE_TYPE"