Packetbeat protocol analyzer enhancements (#3518)

- Optionally configure protocol analyzers using dictionary and/or list - Add 'fields', 'fields_under_root' and 'tags' settings to every protocol analyzer - update sample config file to use list style configuration - add deprecated warning if dictionary style configuration is used - update docs This change allows for configuring packetbeat protocols in 2 different styles... both styles can be used at the same time,. 1.) (deprecated) dictionary style: ``` packetbeat.protocols.http: ... packetbeat.protocols.dns: ... ``` 2.) array style: ``` packetbeat.protocols: - type: http ... - type: dns ... ``` Examples (1) and (2) are equivalent. But array style allows to configure a protocol analyzer multiple times: e.g. (3) array style with multiple instances of http protocol analyzer: ``` packetbeat.protocols: - type: http ports: [80] fields.service: nginx - type: http ports: [9200] fields.service: elasticsearch ``` 4) mixed style: ``` packetbeat.protocols.http: ... packetbeat.protocols: - type: dns ... ``` Limitations: a) due to limitations in yaml parser, only capturing the last 'name' in a dictionary the key name `packetbeat.protocols` must not be used multiple times. e.g. this will result in an incompletely processed config (only DNS will be configured): ``` packetbeat.protocols: http: ... packetbeat.protocols: - type: dns ``` b) Reusing port numbers (overlapping) might result in one module not seeing any packets (this is already the case if any 2 protocols shall listen on same port number).
elastic · Feb 22, 2017 · bae62de · bae62de
1 parent b777783
commit bae62de
Show file tree

Hide file tree

Showing 12 changed files with 271 additions and 112 deletions.
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -93,6 +93,8 @@ https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff]
 - The HAProxy module is now GA, instead of experimental. {pull}3525[3525]
 
 *Packetbeat*
+- Add `fields` and `fields_under_root` to packetbeat protocols configurations. {pull}3518[3518]
+- Add list style packetbeat protocols configurations. This change supports specifying multiple configurations of the same protocol analyzer. {pull]3518[3518]
 
 *Winlogbeat*
 
@@ -111,6 +113,7 @@ https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff]
 *Metricbeat*
 
 *Packetbeat*
+- Deprecate dictionary style protocols configuration. {pull}3518[3518]
 
 *Winlogbeat*
 

diff --git a/packetbeat/_meta/beat.full.yml b/packetbeat/_meta/beat.full.yml
@@ -58,11 +58,12 @@ packetbeat.flows:
 
 #========================== Transaction protocols =============================
 
-packetbeat.protocols.icmp:
+packetbeat.protocols:
+- type: icmp
   # Enable ICMPv4 and ICMPv6 monitoring. Default: true
   #enabled: true
 
-packetbeat.protocols.amqp:
+- type: amqp
   # Enable AMQP monitoring. Default: true
   #enabled: true
 
@@ -99,7 +100,7 @@ packetbeat.protocols.amqp:
   # incoming responses, but sent to Elasticsearch immediately.
   #transaction_timeout: 10s
 
-packetbeat.protocols.cassandra:
+- type: cassandra
   #Cassandra port for traffic monitoring.
   ports: [9042]
 
@@ -126,7 +127,7 @@ packetbeat.protocols.cassandra:
   # This option indicates which Operator/Operators will be ignored.
   #ignored_ops: ["SUPPORTED","OPTIONS"]
 
-packetbeat.protocols.dns:
+- type: dns
   # Enable DNS monitoring. Default: true
   #enabled: true
 
@@ -156,7 +157,7 @@ packetbeat.protocols.dns:
   # incoming responses, but sent to Elasticsearch immediately.
   #transaction_timeout: 10s
 
-packetbeat.protocols.http:
+- type: http
   # Enable HTTP monitoring. Default: true
   #enabled: true
 
@@ -209,7 +210,7 @@ packetbeat.protocols.http:
   # be trimmed to this size. Default is 10 MB.
   #max_message_size: 10485760
 
-packetbeat.protocols.memcache:
+- type: memcache
   # Enable memcache monitoring. Default: true
   #enabled: true
 
@@ -258,7 +259,7 @@ packetbeat.protocols.memcache:
   # incoming responses, but sent to Elasticsearch immediately.
   #transaction_timeout: 10s
 
-packetbeat.protocols.mysql:
+- type: mysql
   # Enable mysql monitoring. Default: true
   #enabled: true
 
@@ -278,7 +279,7 @@ packetbeat.protocols.mysql:
   # incoming responses, but sent to Elasticsearch immediately.
   #transaction_timeout: 10s
 
-packetbeat.protocols.pgsql:
+- type: pgsql
   # Enable pgsql monitoring. Default: true
   #enabled: true
 
@@ -298,7 +299,7 @@ packetbeat.protocols.pgsql:
   # incoming responses, but sent to Elasticsearch immediately.
   #transaction_timeout: 10s
 
-packetbeat.protocols.redis:
+- type: redis
   # Enable redis monitoring. Default: true
   #enabled: true
 
@@ -318,7 +319,7 @@ packetbeat.protocols.redis:
   # incoming responses, but sent to Elasticsearch immediately.
   #transaction_timeout: 10s
 
-packetbeat.protocols.thrift:
+- type: thrift
   # Enable thrift monitoring. Default: true
   #enabled: true
 
@@ -373,7 +374,7 @@ packetbeat.protocols.thrift:
   # incoming responses, but sent to Elasticsearch immediately.
   #transaction_timeout: 10s
 
-packetbeat.protocols.mongodb:
+- type: mongodb
   # Enable mongodb monitoring. Default: true
   #enabled: true
 
@@ -403,7 +404,7 @@ packetbeat.protocols.mongodb:
   # incoming responses, but sent to Elasticsearch immediately.
   #transaction_timeout: 10s
 
-packetbeat.protocols.nfs:
+- type: nfs
   # Enable NFS monitoring. Default: true
   #enabled: true
 

diff --git a/packetbeat/_meta/beat.yml b/packetbeat/_meta/beat.yml
@@ -26,20 +26,21 @@ packetbeat.flows:
 
 #========================== Transaction protocols =============================
 
-packetbeat.protocols.icmp:
+packetbeat.protocols:
+- type: icmp
   # Enable ICMPv4 and ICMPv6 monitoring. Default: false
   enabled: true
 
-packetbeat.protocols.amqp:
+- type: amqp
   # Configure the ports where to listen for AMQP traffic. You can disable
   # the AMQP protocol by commenting out the list of ports.
   ports: [5672]
 
-packetbeat.protocols.cassandra:
+- type: cassandra
   #Cassandra port for traffic monitoring.
   ports: [9042]
 
-packetbeat.protocols.dns:
+- type: dns
   # Configure the ports where to listen for DNS traffic. You can disable
   # the DNS protocol by commenting out the list of ports.
   ports: [53]
@@ -52,42 +53,42 @@ packetbeat.protocols.dns:
   # (additional resource records) is added to messages.
   include_additionals: true
 
-packetbeat.protocols.http:
+- type: http
   # Configure the ports where to listen for HTTP traffic. You can disable
   # the HTTP protocol by commenting out the list of ports.
   ports: [80, 8080, 8000, 5000, 8002]
 
-packetbeat.protocols.memcache:
+- type: memcache
   # Configure the ports where to listen for memcache traffic. You can disable
   # the Memcache protocol by commenting out the list of ports.
   ports: [11211]
 
-packetbeat.protocols.mysql:
+- type: mysql
   # Configure the ports where to listen for MySQL traffic. You can disable
   # the MySQL protocol by commenting out the list of ports.
   ports: [3306]
 
-packetbeat.protocols.pgsql:
+- type: pgsql
   # Configure the ports where to listen for Pgsql traffic. You can disable
   # the Pgsql protocol by commenting out the list of ports.
   ports: [5432]
 
-packetbeat.protocols.redis:
+- type: redis
   # Configure the ports where to listen for Redis traffic. You can disable
   # the Redis protocol by commenting out the list of ports.
   ports: [6379]
 
-packetbeat.protocols.thrift:
+- type: thrift
   # Configure the ports where to listen for Thrift-RPC traffic. You can disable
   # the Thrift-RPC protocol by commenting out the list of ports.
   ports: [9090]
 
-packetbeat.protocols.mongodb:
+- type: mongodb
   # Configure the ports where to listen for MongoDB traffic. You can disable
   # the MongoDB protocol by commenting out the list of ports.
   ports: [27017]
 
-packetbeat.protocols.nfs:
+- type: nfs
   # Configure the ports where to listen for NFS traffic. You can disable
   # the NFS protocol by commenting out the list of ports.
   ports: [2049]
diff --git a/packetbeat/beater/packetbeat.go b/packetbeat/beater/packetbeat.go
@@ -1,6 +1,7 @@
 package beater
 
 import (
+	"errors"
 	"flag"
 	"fmt"
 	"sync"
@@ -107,7 +108,7 @@ func (pb *packetbeat) init(b *beat.Beat) error {
 	}
 
 	logp.Debug("main", "Initializing protocol plugins")
-	err = protos.Protos.Init(false, pb.pub, cfg.Protocols)
+	err = protos.Protos.Init(false, pb.pub, cfg.Protocols, cfg.ProtocolsList)
 	if err != nil {
 		return fmt.Errorf("Initializing protocol analyzers failed: %v", err)
 	}
@@ -189,8 +190,13 @@ func (pb *packetbeat) Stop() {
 func (pb *packetbeat) setupSniffer() error {
 	config := &pb.config
 
+	icmp, err := pb.icmpConfig()
+	if err != nil {
+		return err
+	}
+
 	withVlans := config.Interfaces.WithVlans
-	withICMP := config.Protocols["icmp"].Enabled()
+	withICMP := icmp.Enabled()
 
 	filter := config.Interfaces.BpfFilter
 	if filter == "" && !config.Flows.IsEnabled() {
@@ -215,7 +221,11 @@ func (pb *packetbeat) createWorker(dl layers.LinkType) (sniffer.Worker, error) {
 
 	var icmp4 icmp.ICMPv4Processor
 	var icmp6 icmp.ICMPv6Processor
-	if cfg := config.Protocols["icmp"]; cfg.Enabled() {
+	cfg, err := pb.icmpConfig()
+	if err != nil {
+		return nil, err
+	}
+	if cfg.Enabled() {
 		icmp, err := icmp.New(false, pb.pub, cfg)
 		if err != nil {
 			return nil, err
@@ -245,3 +255,32 @@ func (pb *packetbeat) createWorker(dl layers.LinkType) (sniffer.Worker, error) {
 	}
 	return worker, nil
 }
+
+func (pb *packetbeat) icmpConfig() (*common.Config, error) {
+	var icmp *common.Config
+	if pb.config.Protocols["icmp"].Enabled() {
+		icmp = pb.config.Protocols["icmp"]
+	}
+
+	for _, cfg := range pb.config.ProtocolsList {
+		info := struct {
+			Type string `config:"type" validate:"required"`
+		}{}
+
+		if err := cfg.Unpack(&info); err != nil {
+			return nil, err
+		}
+
+		if info.Type != "icmp" {
+			continue
+		}
+
+		if icmp != nil {
+			return nil, errors.New("More then one icmp confgigurations found")
+		}
+
+		icmp = cfg
+	}
+
+	return icmp, nil
+}
diff --git a/packetbeat/config/config.go b/packetbeat/config/config.go
@@ -12,6 +12,7 @@ type Config struct {
 	Interfaces     InterfacesConfig          `config:"interfaces"`
 	Flows          *Flows                    `config:"flows"`
 	Protocols      map[string]*common.Config `config:"protocols"`
+	ProtocolsList  []*common.Config          `config:"protocols"`
 	Procs          procs.ProcsConfig         `config:"procs"`
 	IgnoreOutgoing bool                      `config:"ignore_outgoing"`
 	RunOptions     droppriv.RunOptions

diff --git a/packetbeat/docs/gettingstarted.asciidoc b/packetbeat/docs/gettingstarted.asciidoc
@@ -146,34 +146,36 @@ default values should do just fine.
 +
 [source,yaml]
 ----------------------------------------------------------------------
-packetbeat.protocols.dns:
+packetbeat.protocols:
+
+- type: dns
   ports: [53]
 
   include_authorities: true
   include_additionals: true
 
-packetbeat.protocols.http:
+- type: http
   ports: [80, 8080, 8081, 5000, 8002]
 
-packetbeat.protocols.memcache:
+- type: memcache
   ports: [11211]
 
-packetbeat.protocols.mysql:
+- type: mysql
   ports: [3306]
 
-packetbeat.protocols.pgsql:
+- type: pgsql
   ports: [5432]
 
-packetbeat.protocols.redis:
+- type: redis
   ports: [6379]
 
-packetbeat.protocols.thrift:
+- type: thrift
   ports: [9090]
 
-packetbeat.protocols.mongodb:
+- type: mongodb
   ports: [27017]
 
-packetbeat.protocols.cassandra:
+- type: cassandra
   ports: [9042]
 
 ----------------------------------------------------------------------