Add system socket MetricSet

CHANGELOG.asciidoc

@@ -56,6 +56,7 @@ https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff]
 - Add a couchbase module with metricsets for node, cluster and bucket. {pull}3081[3081]
 - Export number of cores for cpu module. {pull}3192[3192]
 - Experimental Prometheus module. {pull}3202[3202]
+- Add system socket module that reports all TCP sockets. {pull}3246[3246]
 
 *Packetbeat*
 

glide.yaml

@@ -25,7 +25,9 @@ import:
   subpackages:
   - /difflib
 - package: github.com/elastic/gosigar
-  version: 0ba14c25cb14e2d939b563ca49a504c3117680c6
+  version: 171a3c9e31dde9688c154ba94be6cd5d8a78bf64
+- package: github.com/elastic/procfs
+  version: abf152e5f3e97f2fafac028d2cc06c1feb87ffa5
 - package: github.com/samuel/go-parser
   version: ca8abbf65d0e61dedf061f98bd3850f250e27539
 - package: github.com/samuel/go-thrift

libbeat/scripts/generate_template.py

@@ -205,6 +205,18 @@ def fill_field_properties(args, field, defaults, path):
                 "ignore_above": 1024
             }
 
+    elif field["type"] == "ip":
+        if args.es2x:
+            properties[field["name"]] = {
+                "type": "string",
+                "index": "not_analyzed",
+                "ignore_above": 1024
+            }
+        else:
+            properties[field["name"]] = {
+                "type": "ip"
+            }
+
     elif field["type"] in ["geo_point", "date", "long", "integer",
                            "double", "float", "half_float", "scaled_float",
                            "boolean"]:

metricbeat/_meta/beat.full.yml

@@ -39,6 +39,9 @@ metricbeat.modules:
 
     # Per process stats
     - process
+
+    # Sockets and connection info (linux only)
+    #- socket
   enabled: true
   period: 10s
   processes: ['.*']
@@ -49,6 +52,11 @@ metricbeat.modules:
   # EXPERIMENTAL: cgroups can be enabled for the process metricset.
   #cgroups: false
 
+  # Configure reverse DNS lookup on remote IP addresses in the socket metricset.
+  #socket.reverse_lookup.enabled: false
+  #socket.reverse_lookup.success_ttl: 60s
+  #socket.reverse_lookup.failure_ttl: 60s
+
 #------------------------------- Apache Module -------------------------------
 #- module: apache
   #metricsets: ["status"]

metricbeat/_meta/beat.yml

@@ -39,6 +39,9 @@ metricbeat.modules:
 
     # Per process stats
     - process
+
+    # Sockets (linux only)
+    #- socket
   enabled: true
   period: 10s
   processes: ['.*']

metricbeat/docs/fields.asciidoc

@@ -6206,6 +6206,148 @@ type: long
 Total number of I/O operations performed on all devices by processes in the cgroup as seen by the throttling policy.
 
 
+[float]
+== socket Fields
+
+TCP sockets that are active.
+
+
+
+[float]
+=== system.socket.direction
+
+type: keyword
+
+example: incoming
+
+How the socket was initiated. Possible values are incoming, outgoing, or listening.
+
+
+[float]
+=== system.socket.family
+
+type: keyword
+
+example: ipv4
+
+Address family.
+
+
+[float]
+=== system.socket.local.ip
+
+type: ip
+
+example: 192.0.2.1 or 2001:0DB8:ABED:8536::1
+
+Local IP address. This can be an IPv4 or IPv6 address.
+
+
+[float]
+=== system.socket.local.port
+
+type: long
+
+example: 22
+
+Local port.
+
+
+[float]
+=== system.socket.remote.ip
+
+type: ip
+
+example: 192.0.2.1 or 2001:0DB8:ABED:8536::1
+
+Remote IP address. This can be an IPv4 or IPv6 address
+
+
+[float]
+=== system.socket.remote.port
+
+type: long
+
+example: 22
+
+Remote port.
+
+
+[float]
+=== system.socket.remote.host
+
+type: keyword
+
+example: 76-211-117-36.nw.example.com.
+
+PTR record associated with the remote IP. It is obtained via reverse IP lookup.
+
+
+[float]
+=== system.socket.remote.etld_plus_one
+
+type: keyword
+
+example: example.com.
+
+The effective top-level domain (eTLD) of the remote host plus one more label. For example, the eTLD+1 for "foo.bar.golang.org." is "golang.org.". The data for determining the eTLD comes from an embedded copy of the data from http://publicsuffix.org.
+
+
+[float]
+=== system.socket.remote.host_error
+
+type: keyword
+
+Error describing the cause of the reverse lookup failure.
+
+
+[float]
+=== system.socket.process.pid
+
+type: long
+
+ID of the process that opened the socket.
+
+
+[float]
+=== system.socket.process.command
+
+type: keyword
+
+Name of the command (limited to 20 chars by the OS).
+
+
+[float]
+=== system.socket.process.cmdline
+
+type: keyword
+
+
+
+[float]
+=== system.socket.process.exe
+
+type: keyword
+
+Absolute path to the executable.
+
+
+[float]
+=== system.socket.user.id
+
+type: long
+
+UID of the user running the process.
+
+
+[float]
+=== system.socket.user.name
+
+type: keyword
+
+Name of the user running the process.
+
+
 [[exported-fields-zookeeper]]
 == ZooKeeper Fields
 

metricbeat/docs/modules/system.asciidoc

@@ -93,6 +93,9 @@ metricbeat.modules:
 
     # Per process stats
     - process
+
+    # Sockets (linux only)
+    #- socket
   enabled: true
   period: 10s
   processes: ['.*']
@@ -121,6 +124,8 @@ The following metricsets are available:
 
 * <<metricbeat-metricset-system-process,process>>
 
+* <<metricbeat-metricset-system-socket,socket>>
+
 include::system/core.asciidoc[]
 
 include::system/cpu.asciidoc[]
@@ -139,3 +144,5 @@ include::system/network.asciidoc[]
 
 include::system/process.asciidoc[]
 
+include::system/socket.asciidoc[]
+

metricbeat/docs/modules/system/socket.asciidoc

@@ -0,0 +1,19 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[metricbeat-metricset-system-socket]]
+include::../../../module/system/socket/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-system,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/system/socket/_meta/data.json[]
+----

metricbeat/include/list.go

@@ -52,6 +52,7 @@ import (
 	_ "github.com/elastic/beats/metricbeat/module/system/memory"
 	_ "github.com/elastic/beats/metricbeat/module/system/network"
 	_ "github.com/elastic/beats/metricbeat/module/system/process"
+	_ "github.com/elastic/beats/metricbeat/module/system/socket"
 	_ "github.com/elastic/beats/metricbeat/module/zookeeper"
 	_ "github.com/elastic/beats/metricbeat/module/zookeeper/mntr"
 )

metricbeat/mb/testing/data_generator.go

@@ -3,6 +3,7 @@ package testing
 import (
 	"encoding/json"
 	"flag"
+	"fmt"
 	"io/ioutil"
 	"os"
 	"testing"
@@ -33,15 +34,18 @@ func WriteEvent(f mb.EventFetcher, t *testing.T) error {
 }
 
 func WriteEvents(f mb.EventsFetcher, t *testing.T) error {
-
 	if !*dataFlag {
 		t.Skip("Skip data generation tests")
 	}
+
 	events, err := f.Fetch()
 	if err != nil {
 		return err
 	}
 
+	if len(events) == 0 {
+		return fmt.Errorf("no events were generated")
+	}
 	return createEvent(events[0], f)
 }
 

metricbeat/metricbeat.full.yml

@@ -39,6 +39,9 @@ metricbeat.modules:
 
     # Per process stats
     - process
+
+    # Sockets and connection info (linux only)
+    #- socket
   enabled: true
   period: 10s
   processes: ['.*']
@@ -49,6 +52,11 @@ metricbeat.modules:
   # EXPERIMENTAL: cgroups can be enabled for the process metricset.
   #cgroups: false
 
+  # Configure reverse DNS lookup on remote IP addresses in the socket metricset.
+  #socket.reverse_lookup.enabled: false
+  #socket.reverse_lookup.success_ttl: 60s
+  #socket.reverse_lookup.failure_ttl: 60s
+
 #------------------------------- Apache Module -------------------------------
 #- module: apache
   #metricsets: ["status"]