From bae62de72d4175cd5375091ad3381aebd1a44734 Mon Sep 17 00:00:00 2001 From: Steffen Siering Date: Wed, 22 Feb 2017 09:02:58 +0100 Subject: [PATCH] Packetbeat protocol analyzer enhancements (#3518) - Optionally configure protocol analyzers using dictionary and/or list - Add 'fields', 'fields_under_root' and 'tags' settings to every protocol analyzer - update sample config file to use list style configuration - add deprecated warning if dictionary style configuration is used - update docs This change allows for configuring packetbeat protocols in 2 different styles... both styles can be used at the same time,. 1.) (deprecated) dictionary style: ``` packetbeat.protocols.http: ... packetbeat.protocols.dns: ... ``` 2.) array style: ``` packetbeat.protocols: - type: http ... - type: dns ... ``` Examples (1) and (2) are equivalent. But array style allows to configure a protocol analyzer multiple times: e.g. (3) array style with multiple instances of http protocol analyzer: ``` packetbeat.protocols: - type: http ports: [80] fields.service: nginx - type: http ports: [9200] fields.service: elasticsearch ``` 4) mixed style: ``` packetbeat.protocols.http: ... packetbeat.protocols: - type: dns ... ``` Limitations: a) due to limitations in yaml parser, only capturing the last 'name' in a dictionary the key name `packetbeat.protocols` must not be used multiple times. e.g. this will result in an incompletely processed config (only DNS will be configured): ``` packetbeat.protocols: http: ... packetbeat.protocols: - type: dns ``` b) Reusing port numbers (overlapping) might result in one module not seeing any packets (this is already the case if any 2 protocols shall listen on same port number). --- CHANGELOG.asciidoc | 3 + packetbeat/_meta/beat.full.yml | 25 ++-- packetbeat/_meta/beat.yml | 25 ++-- packetbeat/beater/packetbeat.go | 45 ++++++- packetbeat/config/config.go | 1 + packetbeat/docs/gettingstarted.asciidoc | 20 ++-- .../configuration/packetbeat-options.asciidoc | 76 +++++++++--- packetbeat/docs/thrift.asciidoc | 3 +- packetbeat/packetbeat.full.yml | 25 ++-- packetbeat/packetbeat.yml | 25 ++-- packetbeat/protos/protos.go | 110 ++++++++++++++---- .../tests/system/config/packetbeat.yml.j2 | 25 ++-- 12 files changed, 271 insertions(+), 112 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index c28f39af3c2..68251d07b3d 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -93,6 +93,8 @@ https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff] - The HAProxy module is now GA, instead of experimental. {pull}3525[3525] *Packetbeat* +- Add `fields` and `fields_under_root` to packetbeat protocols configurations. {pull}3518[3518] +- Add list style packetbeat protocols configurations. This change supports specifying multiple configurations of the same protocol analyzer. {pull]3518[3518] *Winlogbeat* @@ -111,6 +113,7 @@ https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff] *Metricbeat* *Packetbeat* +- Deprecate dictionary style protocols configuration. {pull}3518[3518] *Winlogbeat* diff --git a/packetbeat/_meta/beat.full.yml b/packetbeat/_meta/beat.full.yml index bba989ae9c7..093ea616018 100644 --- a/packetbeat/_meta/beat.full.yml +++ b/packetbeat/_meta/beat.full.yml @@ -58,11 +58,12 @@ packetbeat.flows: #========================== Transaction protocols ============================= -packetbeat.protocols.icmp: +packetbeat.protocols: +- type: icmp # Enable ICMPv4 and ICMPv6 monitoring. Default: true #enabled: true -packetbeat.protocols.amqp: +- type: amqp # Enable AMQP monitoring. Default: true #enabled: true @@ -99,7 +100,7 @@ packetbeat.protocols.amqp: # incoming responses, but sent to Elasticsearch immediately. #transaction_timeout: 10s -packetbeat.protocols.cassandra: +- type: cassandra #Cassandra port for traffic monitoring. ports: [9042] @@ -126,7 +127,7 @@ packetbeat.protocols.cassandra: # This option indicates which Operator/Operators will be ignored. #ignored_ops: ["SUPPORTED","OPTIONS"] -packetbeat.protocols.dns: +- type: dns # Enable DNS monitoring. Default: true #enabled: true @@ -156,7 +157,7 @@ packetbeat.protocols.dns: # incoming responses, but sent to Elasticsearch immediately. #transaction_timeout: 10s -packetbeat.protocols.http: +- type: http # Enable HTTP monitoring. Default: true #enabled: true @@ -209,7 +210,7 @@ packetbeat.protocols.http: # be trimmed to this size. Default is 10 MB. #max_message_size: 10485760 -packetbeat.protocols.memcache: +- type: memcache # Enable memcache monitoring. Default: true #enabled: true @@ -258,7 +259,7 @@ packetbeat.protocols.memcache: # incoming responses, but sent to Elasticsearch immediately. #transaction_timeout: 10s -packetbeat.protocols.mysql: +- type: mysql # Enable mysql monitoring. Default: true #enabled: true @@ -278,7 +279,7 @@ packetbeat.protocols.mysql: # incoming responses, but sent to Elasticsearch immediately. #transaction_timeout: 10s -packetbeat.protocols.pgsql: +- type: pgsql # Enable pgsql monitoring. Default: true #enabled: true @@ -298,7 +299,7 @@ packetbeat.protocols.pgsql: # incoming responses, but sent to Elasticsearch immediately. #transaction_timeout: 10s -packetbeat.protocols.redis: +- type: redis # Enable redis monitoring. Default: true #enabled: true @@ -318,7 +319,7 @@ packetbeat.protocols.redis: # incoming responses, but sent to Elasticsearch immediately. #transaction_timeout: 10s -packetbeat.protocols.thrift: +- type: thrift # Enable thrift monitoring. Default: true #enabled: true @@ -373,7 +374,7 @@ packetbeat.protocols.thrift: # incoming responses, but sent to Elasticsearch immediately. #transaction_timeout: 10s -packetbeat.protocols.mongodb: +- type: mongodb # Enable mongodb monitoring. Default: true #enabled: true @@ -403,7 +404,7 @@ packetbeat.protocols.mongodb: # incoming responses, but sent to Elasticsearch immediately. #transaction_timeout: 10s -packetbeat.protocols.nfs: +- type: nfs # Enable NFS monitoring. Default: true #enabled: true diff --git a/packetbeat/_meta/beat.yml b/packetbeat/_meta/beat.yml index 60b731b9954..c771cd282c4 100644 --- a/packetbeat/_meta/beat.yml +++ b/packetbeat/_meta/beat.yml @@ -26,20 +26,21 @@ packetbeat.flows: #========================== Transaction protocols ============================= -packetbeat.protocols.icmp: +packetbeat.protocols: +- type: icmp # Enable ICMPv4 and ICMPv6 monitoring. Default: false enabled: true -packetbeat.protocols.amqp: +- type: amqp # Configure the ports where to listen for AMQP traffic. You can disable # the AMQP protocol by commenting out the list of ports. ports: [5672] -packetbeat.protocols.cassandra: +- type: cassandra #Cassandra port for traffic monitoring. ports: [9042] -packetbeat.protocols.dns: +- type: dns # Configure the ports where to listen for DNS traffic. You can disable # the DNS protocol by commenting out the list of ports. ports: [53] @@ -52,42 +53,42 @@ packetbeat.protocols.dns: # (additional resource records) is added to messages. include_additionals: true -packetbeat.protocols.http: +- type: http # Configure the ports where to listen for HTTP traffic. You can disable # the HTTP protocol by commenting out the list of ports. ports: [80, 8080, 8000, 5000, 8002] -packetbeat.protocols.memcache: +- type: memcache # Configure the ports where to listen for memcache traffic. You can disable # the Memcache protocol by commenting out the list of ports. ports: [11211] -packetbeat.protocols.mysql: +- type: mysql # Configure the ports where to listen for MySQL traffic. You can disable # the MySQL protocol by commenting out the list of ports. ports: [3306] -packetbeat.protocols.pgsql: +- type: pgsql # Configure the ports where to listen for Pgsql traffic. You can disable # the Pgsql protocol by commenting out the list of ports. ports: [5432] -packetbeat.protocols.redis: +- type: redis # Configure the ports where to listen for Redis traffic. You can disable # the Redis protocol by commenting out the list of ports. ports: [6379] -packetbeat.protocols.thrift: +- type: thrift # Configure the ports where to listen for Thrift-RPC traffic. You can disable # the Thrift-RPC protocol by commenting out the list of ports. ports: [9090] -packetbeat.protocols.mongodb: +- type: mongodb # Configure the ports where to listen for MongoDB traffic. You can disable # the MongoDB protocol by commenting out the list of ports. ports: [27017] -packetbeat.protocols.nfs: +- type: nfs # Configure the ports where to listen for NFS traffic. You can disable # the NFS protocol by commenting out the list of ports. ports: [2049] diff --git a/packetbeat/beater/packetbeat.go b/packetbeat/beater/packetbeat.go index f1c8a519de6..67949c08822 100644 --- a/packetbeat/beater/packetbeat.go +++ b/packetbeat/beater/packetbeat.go @@ -1,6 +1,7 @@ package beater import ( + "errors" "flag" "fmt" "sync" @@ -107,7 +108,7 @@ func (pb *packetbeat) init(b *beat.Beat) error { } logp.Debug("main", "Initializing protocol plugins") - err = protos.Protos.Init(false, pb.pub, cfg.Protocols) + err = protos.Protos.Init(false, pb.pub, cfg.Protocols, cfg.ProtocolsList) if err != nil { return fmt.Errorf("Initializing protocol analyzers failed: %v", err) } @@ -189,8 +190,13 @@ func (pb *packetbeat) Stop() { func (pb *packetbeat) setupSniffer() error { config := &pb.config + icmp, err := pb.icmpConfig() + if err != nil { + return err + } + withVlans := config.Interfaces.WithVlans - withICMP := config.Protocols["icmp"].Enabled() + withICMP := icmp.Enabled() filter := config.Interfaces.BpfFilter if filter == "" && !config.Flows.IsEnabled() { @@ -215,7 +221,11 @@ func (pb *packetbeat) createWorker(dl layers.LinkType) (sniffer.Worker, error) { var icmp4 icmp.ICMPv4Processor var icmp6 icmp.ICMPv6Processor - if cfg := config.Protocols["icmp"]; cfg.Enabled() { + cfg, err := pb.icmpConfig() + if err != nil { + return nil, err + } + if cfg.Enabled() { icmp, err := icmp.New(false, pb.pub, cfg) if err != nil { return nil, err @@ -245,3 +255,32 @@ func (pb *packetbeat) createWorker(dl layers.LinkType) (sniffer.Worker, error) { } return worker, nil } + +func (pb *packetbeat) icmpConfig() (*common.Config, error) { + var icmp *common.Config + if pb.config.Protocols["icmp"].Enabled() { + icmp = pb.config.Protocols["icmp"] + } + + for _, cfg := range pb.config.ProtocolsList { + info := struct { + Type string `config:"type" validate:"required"` + }{} + + if err := cfg.Unpack(&info); err != nil { + return nil, err + } + + if info.Type != "icmp" { + continue + } + + if icmp != nil { + return nil, errors.New("More then one icmp confgigurations found") + } + + icmp = cfg + } + + return icmp, nil +} diff --git a/packetbeat/config/config.go b/packetbeat/config/config.go index 977e4e1a621..cbcf9834515 100644 --- a/packetbeat/config/config.go +++ b/packetbeat/config/config.go @@ -12,6 +12,7 @@ type Config struct { Interfaces InterfacesConfig `config:"interfaces"` Flows *Flows `config:"flows"` Protocols map[string]*common.Config `config:"protocols"` + ProtocolsList []*common.Config `config:"protocols"` Procs procs.ProcsConfig `config:"procs"` IgnoreOutgoing bool `config:"ignore_outgoing"` RunOptions droppriv.RunOptions diff --git a/packetbeat/docs/gettingstarted.asciidoc b/packetbeat/docs/gettingstarted.asciidoc index e11c6717a17..84cdfec14a5 100644 --- a/packetbeat/docs/gettingstarted.asciidoc +++ b/packetbeat/docs/gettingstarted.asciidoc @@ -146,34 +146,36 @@ default values should do just fine. + [source,yaml] ---------------------------------------------------------------------- -packetbeat.protocols.dns: +packetbeat.protocols: + +- type: dns ports: [53] include_authorities: true include_additionals: true -packetbeat.protocols.http: +- type: http ports: [80, 8080, 8081, 5000, 8002] -packetbeat.protocols.memcache: +- type: memcache ports: [11211] -packetbeat.protocols.mysql: +- type: mysql ports: [3306] -packetbeat.protocols.pgsql: +- type: pgsql ports: [5432] -packetbeat.protocols.redis: +- type: redis ports: [6379] -packetbeat.protocols.thrift: +- type: thrift ports: [9090] -packetbeat.protocols.mongodb: +- type: mongodb ports: [27017] -packetbeat.protocols.cassandra: +- type: cassandra ports: [9042] ---------------------------------------------------------------------- diff --git a/packetbeat/docs/reference/configuration/packetbeat-options.asciidoc b/packetbeat/docs/reference/configuration/packetbeat-options.asciidoc index 317e60b4792..0c570d9bb1d 100644 --- a/packetbeat/docs/reference/configuration/packetbeat-options.asciidoc +++ b/packetbeat/docs/reference/configuration/packetbeat-options.asciidoc @@ -249,34 +249,36 @@ Example configuration: [source,yaml] ------------------------------------------------------------------------------ -packetbeat.protocols.icmp: +packetbeat.protocols: + +- type: icmp enabled: true -packetbeat.protocols.dns: +- type: dns ports: [53] -packetbeat.protocols.http: +- type: http ports: [80, 8080, 8000, 5000, 8002] -packetbeat.protocols.amqp: +- type: amqp ports: [5672] -packetbeat.protocols.cassandra: +- type: cassandra ports: [9042] -packetbeat.protocols.memcache: +- type: memcache ports: [11211] -packetbeat.protocols.mysql: +- type: mysql ports: [3306] -packetbeat.protocols.redis: +- type: redis ports: [6379] -packetbeat.protocols.pgsql: +- type: pgsql ports: [5432] -packetbeat.protocols.thrift: +- type: thrift ports: [9090] @@ -324,6 +326,36 @@ by default, only the HTTP headers. The per protocol transaction timeout. Expired transactions will no longer be correlated to incoming responses, but sent to Elasticsearch immediately. +[[packetbeat-configuration-fields]] +===== fields + +Optional fields that you can specify to add additional information to the +output. For example, you might add fields that you can use for filtering log +data. Fields can be scalar values, arrays, dictionaries, or any nested +combination of these. By default, the fields that you specify here will be +grouped under a `fields` sub-dictionary in the output document. To store the +custom fields as top-level fields, set the `fields_under_root` option to true. +If a duplicate field is declared in the general configuration, then its value +will be overwritten by the value declared here. + +[source,yaml] +-------------------------------------------------------------------------------- +packetbeat.protocols: +- type: http + ports: [80] + fields: + service_id: nginx +-------------------------------------------------------------------------------- + +[[packetbeat-fields-under-root]] +===== fields_under_root + +If this option is set to true, the custom <> +are stored as top-level fields in the output document instead of being grouped +under a `fields` sub-dictionary. If the custom field names conflict with other +field names added by Packetbeat, then the custom fields overwrite the other +fields. + ==== ICMP Configuration Options You can specify the following options in the `icmp` section of the +{beatname_lc}.yml+ config file: @@ -340,7 +372,8 @@ The `dns` section of the +{beatname_lc}.yml+ config file specifies configuration [source,yaml] ------------------------------------------------------------------------------ -packetbeat.protocols.dns: +packetbeat.protocols: +- type: dns ports: [53] include_authorities: true include_additionals: true @@ -363,7 +396,8 @@ sample configuration for the `http` section of the +{beatname_lc}.yml+ config fi [source,yaml] ------------------------------------------------------------------------------ -packetbeat.protocols.http: +packetbeat.protocols: +- type: http ports: [80, 8080, 8000, 5000, 8002] hide_keywords: ["pass", "password", "passwd"] send_headers: ["User-Agent", "Cookie", "Set-Cookie"] @@ -423,7 +457,8 @@ In the following example, the HTML attachments of the HTTP responses are exporte [source,yml] ------------------------------------------------------------------------------ -packetbeat.protocols.http: +packetbeat.protocols: +- type: http ports: [80, 8080] send_response: true include_body_for: ["text/html"] @@ -484,7 +519,8 @@ protocol. Here is a sample configuration: [source,yaml] ------------------------------------------------------------------------------ -packetbeat.protocols.amqp: +packetbeat.protocols: +- type: amqp ports: [5672] max_body_length: 1000 parse_headers: true @@ -526,7 +562,8 @@ configuration for the `cassandra` section of the +{beatname_lc}.yml+ config file [source,yaml] ------------------------------------------------------------------------------ -packetbeat.protocols.cassandra: +packetbeat.protocols: +- type: cassandra send_request_header: true send_response_header: true compressor: "snappy" @@ -563,7 +600,8 @@ protocol. Here is a sample configuration section for memcache: [source,yaml] ------------------------------------------------------------------------------ -packetbeat.protocols.memcache: +packetbeat.protocols: +- type: memcache ports: [11211] parseunknown: false maxvalues: 0 @@ -629,7 +667,8 @@ sample configuration for the `thrift` section of the +{beatname_lc}.yml+ config [source,yaml] ------------------------------------------------------------------------------ -packetbeat.protocols.thrift: +packetbeat.protocols: +- type: thrift transport_type: socket protocol_type: binary idl_files: ["tutorial.thrift", "shared.thrift"] @@ -699,7 +738,8 @@ configuration for the `mongodb` section of the +{beatname_lc}.yml+ config file: [source,yaml] ------------------------------------------------------------------------------ -packetbeat.protocols.mongodb: +packetbeat.protocols: +- type: mongodb send_request: true send_response: true max_docs: 0 diff --git a/packetbeat/docs/thrift.asciidoc b/packetbeat/docs/thrift.asciidoc index a4f07da9aba..acec8ee0aca 100644 --- a/packetbeat/docs/thrift.asciidoc +++ b/packetbeat/docs/thrift.asciidoc @@ -45,7 +45,8 @@ Here is an example configuration section for the Thrift protocol: [source,yaml] ------------------------------------------------------------------------------ -packetbeat.protocols.thrift: +packetbeat.protocols: +- type: thrift transport_type: socket protocol_type: binary idl_files: ["tutorial.thrift", "shared.thrift"] diff --git a/packetbeat/packetbeat.full.yml b/packetbeat/packetbeat.full.yml index a663bccdd86..170bd029e1e 100644 --- a/packetbeat/packetbeat.full.yml +++ b/packetbeat/packetbeat.full.yml @@ -58,11 +58,12 @@ packetbeat.flows: #========================== Transaction protocols ============================= -packetbeat.protocols.icmp: +packetbeat.protocols: +- type: icmp # Enable ICMPv4 and ICMPv6 monitoring. Default: true #enabled: true -packetbeat.protocols.amqp: +- type: amqp # Enable AMQP monitoring. Default: true #enabled: true @@ -99,7 +100,7 @@ packetbeat.protocols.amqp: # incoming responses, but sent to Elasticsearch immediately. #transaction_timeout: 10s -packetbeat.protocols.cassandra: +- type: cassandra #Cassandra port for traffic monitoring. ports: [9042] @@ -126,7 +127,7 @@ packetbeat.protocols.cassandra: # This option indicates which Operator/Operators will be ignored. #ignored_ops: ["SUPPORTED","OPTIONS"] -packetbeat.protocols.dns: +- type: dns # Enable DNS monitoring. Default: true #enabled: true @@ -156,7 +157,7 @@ packetbeat.protocols.dns: # incoming responses, but sent to Elasticsearch immediately. #transaction_timeout: 10s -packetbeat.protocols.http: +- type: http # Enable HTTP monitoring. Default: true #enabled: true @@ -209,7 +210,7 @@ packetbeat.protocols.http: # be trimmed to this size. Default is 10 MB. #max_message_size: 10485760 -packetbeat.protocols.memcache: +- type: memcache # Enable memcache monitoring. Default: true #enabled: true @@ -258,7 +259,7 @@ packetbeat.protocols.memcache: # incoming responses, but sent to Elasticsearch immediately. #transaction_timeout: 10s -packetbeat.protocols.mysql: +- type: mysql # Enable mysql monitoring. Default: true #enabled: true @@ -278,7 +279,7 @@ packetbeat.protocols.mysql: # incoming responses, but sent to Elasticsearch immediately. #transaction_timeout: 10s -packetbeat.protocols.pgsql: +- type: pgsql # Enable pgsql monitoring. Default: true #enabled: true @@ -298,7 +299,7 @@ packetbeat.protocols.pgsql: # incoming responses, but sent to Elasticsearch immediately. #transaction_timeout: 10s -packetbeat.protocols.redis: +- type: redis # Enable redis monitoring. Default: true #enabled: true @@ -318,7 +319,7 @@ packetbeat.protocols.redis: # incoming responses, but sent to Elasticsearch immediately. #transaction_timeout: 10s -packetbeat.protocols.thrift: +- type: thrift # Enable thrift monitoring. Default: true #enabled: true @@ -373,7 +374,7 @@ packetbeat.protocols.thrift: # incoming responses, but sent to Elasticsearch immediately. #transaction_timeout: 10s -packetbeat.protocols.mongodb: +- type: mongodb # Enable mongodb monitoring. Default: true #enabled: true @@ -403,7 +404,7 @@ packetbeat.protocols.mongodb: # incoming responses, but sent to Elasticsearch immediately. #transaction_timeout: 10s -packetbeat.protocols.nfs: +- type: nfs # Enable NFS monitoring. Default: true #enabled: true diff --git a/packetbeat/packetbeat.yml b/packetbeat/packetbeat.yml index ea8e970a7e8..88f7fb05e36 100644 --- a/packetbeat/packetbeat.yml +++ b/packetbeat/packetbeat.yml @@ -26,20 +26,21 @@ packetbeat.flows: #========================== Transaction protocols ============================= -packetbeat.protocols.icmp: +packetbeat.protocols: +- type: icmp # Enable ICMPv4 and ICMPv6 monitoring. Default: false enabled: true -packetbeat.protocols.amqp: +- type: amqp # Configure the ports where to listen for AMQP traffic. You can disable # the AMQP protocol by commenting out the list of ports. ports: [5672] -packetbeat.protocols.cassandra: +- type: cassandra #Cassandra port for traffic monitoring. ports: [9042] -packetbeat.protocols.dns: +- type: dns # Configure the ports where to listen for DNS traffic. You can disable # the DNS protocol by commenting out the list of ports. ports: [53] @@ -52,42 +53,42 @@ packetbeat.protocols.dns: # (additional resource records) is added to messages. include_additionals: true -packetbeat.protocols.http: +- type: http # Configure the ports where to listen for HTTP traffic. You can disable # the HTTP protocol by commenting out the list of ports. ports: [80, 8080, 8000, 5000, 8002] -packetbeat.protocols.memcache: +- type: memcache # Configure the ports where to listen for memcache traffic. You can disable # the Memcache protocol by commenting out the list of ports. ports: [11211] -packetbeat.protocols.mysql: +- type: mysql # Configure the ports where to listen for MySQL traffic. You can disable # the MySQL protocol by commenting out the list of ports. ports: [3306] -packetbeat.protocols.pgsql: +- type: pgsql # Configure the ports where to listen for Pgsql traffic. You can disable # the Pgsql protocol by commenting out the list of ports. ports: [5432] -packetbeat.protocols.redis: +- type: redis # Configure the ports where to listen for Redis traffic. You can disable # the Redis protocol by commenting out the list of ports. ports: [6379] -packetbeat.protocols.thrift: +- type: thrift # Configure the ports where to listen for Thrift-RPC traffic. You can disable # the Thrift-RPC protocol by commenting out the list of ports. ports: [9090] -packetbeat.protocols.mongodb: +- type: mongodb # Configure the ports where to listen for MongoDB traffic. You can disable # the MongoDB protocol by commenting out the list of ports. ports: [27017] -packetbeat.protocols.nfs: +- type: nfs # Configure the ports where to listen for NFS traffic. You can disable # the NFS protocol by commenting out the list of ports. ports: [2049] diff --git a/packetbeat/protos/protos.go b/packetbeat/protos/protos.go index 8675364d99f..7c0f98af637 100644 --- a/packetbeat/protos/protos.go +++ b/packetbeat/protos/protos.go @@ -35,6 +35,11 @@ type PortsConfig struct { Ports []int } +type enhTransactions struct { + t publish.Transactions + meta common.EventMetadata +} + func (p *PortsConfig) Init(ports ...int) error { return p.Set(ports) } @@ -84,46 +89,93 @@ func (s ProtocolsStruct) Init( testMode bool, results publish.Transactions, configs map[string]*common.Config, + listConfigs []*common.Config, ) error { + if len(configs) > 0 { + logp.Warn("Deprecated: dictionary style protocols configuration has been deprecated. Please use list-style protocols configuration.") + } + for proto := range protocolSyms { logp.Info("registered protocol plugin: %v", proto) } for name, config := range configs { - // XXX: icmp is special, ignore here :/ - if name == "icmp" { - continue + if err := s.configureProtocol(testMode, results, name, config); err != nil { + return err } + } - proto, exists := protocolSyms[name] - if !exists { - logp.Err("Unknown protocol plugin: %v", name) - continue + for _, config := range listConfigs { + module := struct { + Name string `config:"type" validate:"required"` + }{} + if err := config.Unpack(&module); err != nil { + return err } - plugin, exists := protocolPlugins[proto] - if !exists { - logp.Err("Protocol plugin '%v' not registered (%v).", name, proto.String()) - continue + if err := s.configureProtocol(testMode, results, module.Name, config); err != nil { + return err } + } - if !config.Enabled() { - logp.Info("Protocol plugin '%v' disabled by config", name) - continue - } + return nil +} - inst, err := plugin(testMode, results, config) - if err != nil { - logp.Err("Failed to register protocol plugin: %v", err) - return err - } +func (s ProtocolsStruct) configureProtocol( + testMode bool, + results publish.Transactions, + name string, + config *common.Config, +) error { + // XXX: icmp is special, ignore here :/ + if name == "icmp" { + return nil + } + + proto, exists := protocolSyms[name] + if !exists { + logp.Err("Unknown protocol plugin: %v", name) + return nil + } + + plugin, exists := protocolPlugins[proto] + if !exists { + logp.Err("Protocol plugin '%v' not registered (%v).", name, proto.String()) + return nil + } + + if !config.Enabled() { + logp.Info("Protocol plugin '%v' disabled by config", name) + return nil + } + + meta := struct { + Config common.EventMetadata `config:",inline"` // Fields and tags to add to events. + }{} + if err := config.Unpack(&meta); err != nil { + return err + } - s.register(proto, inst) + inst, err := plugin(testMode, resultsChannel(results, meta.Config), config) + if err != nil { + logp.Err("Failed to register protocol plugin: %v", err) + return err } + s.register(proto, inst) return nil } +func resultsChannel( + results publish.Transactions, + meta common.EventMetadata, +) publish.Transactions { + if len(meta.Fields) == 0 && len(meta.Tags) == 0 { + return results + } + return &enhTransactions{results, meta} +} + func (s ProtocolsStruct) GetTCP(proto Protocol) TCPPlugin { plugin, exists := s.tcp[proto] if !exists { @@ -225,3 +277,19 @@ func (s ProtocolsStruct) register(proto Protocol, plugin Plugin) { logp.Warn("Protocol (%s) register failed, port: %v", proto.String(), plugin.GetPorts()) } } + +func (e *enhTransactions) PublishTransaction(event common.MapStr) bool { + err := common.MergeFields(event, e.meta.Fields, e.meta.FieldsUnderRoot) + if err != nil { + logp.Err("Failed to merge fields: %v", err) + return false + } + + err = common.AddTags(event, e.meta.Tags) + if err != nil { + logp.Err("Failed to add tags: %v", err) + return false + } + + return e.t.PublishTransaction(event) +} diff --git a/packetbeat/tests/system/config/packetbeat.yml.j2 b/packetbeat/tests/system/config/packetbeat.yml.j2 index 916f4ee672f..86be018a1ea 100644 --- a/packetbeat/tests/system/config/packetbeat.yml.j2 +++ b/packetbeat/tests/system/config/packetbeat.yml.j2 @@ -14,27 +14,28 @@ packetbeat.flows.timeout: 10s packetbeat.flows.period: -1s {% endif %} +packetbeat.protocols: # Configure which protocols to monitor and the ports where they are # running. You can disable a given protocol by commenting out its # configuration. -packetbeat.protocols.icmp: +- type: icmp enabled: true {% if icmp_send_request %} send_request: true{%- endif %} {% if icmp_send_response %} send_response: true{%- endif %} -packetbeat.protocols.dns: +- type: dns ports: [{{ dns_ports|default([53])|join(", ") }}] {% if dns_include_authorities %} include_authorities: true{%- endif %} {% if dns_include_additionals %} include_additionals: true{%- endif %} {% if dns_send_request %} send_request: true{%- endif %} {% if dns_send_response %} send_response: true{%- endif %} -packetbeat.protocols.amqp: +- type: amqp ports: [{{ amqp_ports|default([5672])|join(", ") }}] {% if amqp_send_request %} send_request: true{%- endif %} {% if amqp_send_response %} send_response: true{%- endif %} -packetbeat.protocols.cassandra: +- type: cassandra ports: [{{ cassandra_ports|default([9042])|join(", ") }}] {% if cassandra_send_request %} send_request: true{%endif %} {% if cassandra_send_response %} send_response: true{% endif %} @@ -43,7 +44,7 @@ packetbeat.protocols.cassandra: {% if cassandra_ignored_ops %} ignored_ops: {{cassandra_ignored_ops}}{% endif %} {% if cassandra_compressor %} compressor: {{cassandra_compressor}}{% endif %} -packetbeat.protocols.http: +- type: http ports: [{{ http_ports|default([80])|join(", ") }}] {% if http_send_request %} send_request: true{%- endif %} {% if http_send_response %} send_response: true{%- endif %} @@ -77,7 +78,7 @@ packetbeat.protocols.http: {%- endif %} {%- if http_max_message_size %} max_message_size: {{ http_max_message_size }} {%- endif %} -packetbeat.protocols.memcache: +- type: memcache ports: [{{ memcache_ports|default([11211])|join(", ") }}] {% if memcache_send_request %} send_request: true{%- endif %} {% if memcache_send_response %} send_response: true{%- endif %} @@ -85,29 +86,29 @@ packetbeat.protocols.memcache: {% if memcache_max_values %} maxvalues: {{ memcache_max_values }}{%- endif %} {% if memcache_udp_transaction_timeout %} udptransactiontimeout: {{ memcache_udp_transaction_timeout}}ms {%- endif %} -packetbeat.protocols.mysql: +- type: mysql ports: [{{ mysql_ports|default([3306])|join(", ") }}] {% if mysql_max_rows %} max_rows: {{mysql_max_rows}}{%- endif %} {% if mysql_max_row_length %} max_row_length: {{mysql_max_row_length}}{%- endif %} {% if mysql_send_request %} send_request: true{%- endif %} {% if mysql_send_response %} send_response: true{%- endif %} -packetbeat.protocols.pgsql: +- type: pgsql ports: [{{ pgsql_ports|default([5432])|join(", ") }}] {% if pgsql_max_rows %} max_rows: {{pgsql_max_rows}}{%- endif %} {% if pgsql_max_row_length %} max_row_length: {{pgsql_max_row_length}}{%- endif %} {% if pgsql_send_request %} send_request: true{%- endif %} {% if pgsql_send_response %} send_response: true{%- endif %} -packetbeat.protocols.redis: +- type: redis ports: [{{ redis_ports|default([6379])|join(", ") }}] {% if redis_send_request %} send_request: true{% endif %} {% if redis_send_response %} send_response: true{% endif %} -packetbeat.protocols.nfs: +- type: nfs ports: [{{ nfs_ports|default([2049])|join(", ") }}] -packetbeat.protocols.thrift: +- type: thrift ports: [{{ thrift_ports|default([9090])|join(", ") }}] transport_type: "{{ thrift_transport_type|default('socket') }}" {% if thrift_idl_files %} @@ -121,7 +122,7 @@ packetbeat.protocols.thrift: {% if thrift_send_request %} send_request: true{%- endif %} {% if thrift_send_response %} send_response: true{%- endif %} -packetbeat.protocols.mongodb: +- type: mongodb ports: [{{ mongodb_ports|default([27017])|join(", ") }}] {% if mongodb_send_request %} send_request: true{%endif %} {% if mongodb_send_response %} send_response: true{% endif %}