Skip to content

Commit

Permalink
Merge pull request #2499 from DataDog/massi/network_stats_win
Browse files Browse the repository at this point in the history 
[Network] Implement the check using psutil on Windows
  • Loading branch information
@truthbk
truthbk committed Sep 6, 2016
2 parents e44e902 + 25c7ddc commit ab085ce
Show file tree
Hide file tree
Showing 3 changed files with 190 additions and 13 deletions.
104 changes: 92 additions & 12 deletions checks.d/network.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
"""
# stdlib
import re
import socket
from collections import defaultdict

# project
from checks import AgentCheck
Expand All @@ -15,6 +17,7 @@
get_subprocess_output,
SubprocessOutputEmptyError,
)
import psutil

BSD_TCP_METRICS = [
(re.compile("^\s*(\d+) data packets \(\d+ bytes\) retransmitted\s*$"), 'system.net.tcp.retrans_packs'),
Expand Down Expand Up @@ -59,22 +62,46 @@ class Network(AgentCheck):
"LAST_ACK": "closing",
"LISTEN": "listening",
"CLOSING": "closing",
},
"psutil": {
psutil.CONN_ESTABLISHED: "established",
psutil.CONN_SYN_SENT: "opening",
psutil.CONN_SYN_RECV: "opening",
psutil.CONN_FIN_WAIT1: "closing",
psutil.CONN_FIN_WAIT2: "closing",
psutil.CONN_TIME_WAIT: "time_wait",
psutil.CONN_CLOSE: "closing",
psutil.CONN_CLOSE_WAIT: "closing",
psutil.CONN_LAST_ACK: "closing",
psutil.CONN_LISTEN: "listening",
psutil.CONN_CLOSING: "closing",
psutil.CONN_NONE: "connections", # CONN_NONE is always returned for udp connections
}
}

PSUTIL_TYPE_MAPPING = {
socket.SOCK_STREAM: 'tcp',
socket.SOCK_DGRAM: 'udp',
}

PSUTIL_FAMILY_MAPPING = {
socket.AF_INET: '4',
socket.AF_INET6: '6',
}

CX_STATE_GAUGE = {
('udp4', 'connections') : 'system.net.udp4.connections',
('udp6', 'connections') : 'system.net.udp6.connections',
('tcp4', 'established') : 'system.net.tcp4.established',
('tcp4', 'opening') : 'system.net.tcp4.opening',
('tcp4', 'closing') : 'system.net.tcp4.closing',
('tcp4', 'listening') : 'system.net.tcp4.listening',
('tcp4', 'time_wait') : 'system.net.tcp4.time_wait',
('tcp6', 'established') : 'system.net.tcp6.established',
('tcp6', 'opening') : 'system.net.tcp6.opening',
('tcp6', 'closing') : 'system.net.tcp6.closing',
('tcp6', 'listening') : 'system.net.tcp6.listening',
('tcp6', 'time_wait') : 'system.net.tcp6.time_wait',
('udp4', 'connections'): 'system.net.udp4.connections',
('udp6', 'connections'): 'system.net.udp6.connections',
('tcp4', 'established'): 'system.net.tcp4.established',
('tcp4', 'opening'): 'system.net.tcp4.opening',
('tcp4', 'closing'): 'system.net.tcp4.closing',
('tcp4', 'listening'): 'system.net.tcp4.listening',
('tcp4', 'time_wait'): 'system.net.tcp4.time_wait',
('tcp6', 'established'): 'system.net.tcp6.established',
('tcp6', 'opening'): 'system.net.tcp6.opening',
('tcp6', 'closing'): 'system.net.tcp6.closing',
('tcp6', 'listening'): 'system.net.tcp6.listening',
('tcp6', 'time_wait'): 'system.net.tcp6.time_wait',
}

def __init__(self, name, init_config, agentConfig, instances=None):
Expand All @@ -101,6 +128,8 @@ def check(self, instance):
self._check_bsd(instance)
elif Platform.is_solaris():
self._check_solaris(instance)
elif Platform.is_windows():
self._check_psutil()

def _submit_devicemetrics(self, iface, vals_by_metric):
if iface in self._excluded_ifaces or (self._exclude_iface_re and self._exclude_iface_re.match(iface)):
Expand Down Expand Up @@ -515,3 +544,54 @@ def _parse_solaris_netstat(self, netstat_output):
metrics_by_interface[iface] = metrics

return metrics_by_interface

def _check_psutil(self):
"""
Gather metrics about connections states and interfaces counters
using psutil facilities
"""
if self._collect_cx_state:
self._cx_state_psutil()

self._cx_counters_psutil()

def _cx_state_psutil(self):
"""
Collect metrics about connections state using psutil
"""
metrics = defaultdict(int)
for conn in psutil.net_connections():
protocol = self._parse_protocol_psutil(conn)
status = self.TCP_STATES['psutil'].get(conn.status)
metric = self.CX_STATE_GAUGE.get((protocol, status))
if metric is None:
self.log.warning('Metric not found for: %s,%s', protocol, status)
else:
metrics[metric] += 1

for metric, value in metrics.iteritems():
self.gauge(metric, value)

def _cx_counters_psutil(self):
"""
Collect metrics about interfaces counters using psutil
"""
for iface, counters in psutil.net_io_counters(pernic=True).iteritems():
metrics = {
'bytes_rcvd': counters.bytes_recv,
'bytes_sent': counters.bytes_sent,
'packets_in.count': counters.packets_recv,
'packets_in.error': counters.errin,
'packets_out.count': counters.packets_sent,
'packets_out.error': counters.errout,
}
self._submit_devicemetrics(iface, metrics)

def _parse_protocol_psutil(self, conn):
"""
Returns a string describing the protocol for the given connection
in the form `tcp4`, 'udp4` as in `self.CX_STATE_GAUGE`
"""
protocol = self.PSUTIL_TYPE_MAPPING.get(conn.type, '')
family = self.PSUTIL_FAMILY_MAPPING.get(conn.family, '')
return '{}{}'.format(protocol, family)
98 changes: 98 additions & 0 deletions tests/checks/mock/test_network.py
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
from collections import namedtuple
import socket

# 3p
import mock

Expand Down Expand Up @@ -64,3 +67,98 @@ def test_cx_state_linux_netstat(self, mock_subprocess, mock_platform):
# Assert metrics
for metric, value in self.CX_STATE_GAUGES_VALUES.iteritems():
self.assertMetric(metric, value=value)

@mock.patch('network.Platform.is_linux', return_value=False)
@mock.patch('network.Platform.is_bsd', return_value=False)
@mock.patch('network.Platform.is_solaris', return_value=False)
@mock.patch('network.Platform.is_windows', return_value=True)
def test_win_uses_psutil(self, *args):
self.check._check_psutil = mock.MagicMock()
self.run_check({})
self.check._check_psutil.assert_called_once_with()

@mock.patch('network.Network._cx_state_psutil')
@mock.patch('network.Network._cx_counters_psutil')
def test_check_psutil(self, state, counters):
self.check._cx_state_psutil = state
self.check._cx_counters_psutil = counters

self.check._collect_cx_state = False
self.check._check_psutil()
state.assert_not_called()
counters.assert_called_once_with()

state.reset_mock()
counters.reset_mock()

self.check._collect_cx_state = True
self.check._check_psutil()
state.assert_called_once_with()
counters.assert_called_once_with()

def test_cx_state_psutil(self):
sconn = namedtuple('sconn', ['fd', 'family', 'type', 'laddr', 'raddr', 'status', 'pid'])
conn = [
sconn(fd=-1, family=socket.AF_INET, type=socket.SOCK_STREAM, laddr=('127.0.0.1', 50482), raddr=('127.0.0.1',2638), status='ESTABLISHED', pid=1416),
sconn(fd=-1, family=socket.AF_INET6, type=socket.SOCK_STREAM, laddr=('::', 50482), raddr=('::',2638), status='ESTABLISHED', pid=42),
sconn(fd=-1, family=socket.AF_INET6, type=socket.SOCK_STREAM, laddr=('::', 49163), raddr=(), status='LISTEN', pid=1416),
sconn(fd=-1, family=socket.AF_INET, type=socket.SOCK_STREAM, laddr=('0.0.0.0', 445), raddr=(), status='LISTEN', pid=4),
sconn(fd=-1, family=socket.AF_INET6, type=socket.SOCK_STREAM, laddr=('::1', 56521), raddr=('::1', 17123), status='TIME_WAIT', pid=0),
sconn(fd=-1, family=socket.AF_INET6, type=socket.SOCK_DGRAM, laddr=('::', 500), raddr=(), status='NONE', pid=892),
sconn(fd=-1, family=socket.AF_INET6, type=socket.SOCK_STREAM, laddr=('::1', 56493), raddr=('::1', 17123), status='TIME_WAIT', pid=0),
sconn(fd=-1, family=socket.AF_INET, type=socket.SOCK_STREAM, laddr=('127.0.0.1', 54541), raddr=('127.0.0.1', 54542), status='ESTABLISHED', pid=20500),
]

results = {
'system.net.tcp6.time_wait': 2,
'system.net.tcp4.listening': 1,
'system.net.tcp6.closing': 0,
'system.net.tcp4.closing': 0,
'system.net.tcp4.time_wait': 0,
'system.net.tcp6.established': 1,
'system.net.tcp4.established': 2,
'system.net.tcp6.listening': 1,
'system.net.tcp4.opening': 0,
'system.net.udp4.connections': 0,
'system.net.udp6.connections': 1,
'system.net.tcp6.opening': 0,
}

with mock.patch('network.psutil') as mock_psutil:
mock_psutil.net_connections.return_value = conn
self.check._cx_state_psutil()
for _, m in self.check.aggregator.metrics.iteritems():
self.assertEqual(results[m.name], m.value)

def test_cx_counters_psutil(self):
snetio = namedtuple('snetio', ['bytes_sent', 'bytes_recv', 'packets_sent', 'packets_recv', 'errin', 'errout', 'dropin', 'dropout'])
counters = {
'Ethernet': snetio(bytes_sent=3096403230L, bytes_recv=3280598526L, packets_sent=6777924, packets_recv=32888147, errin=0, errout=0, dropin=0, dropout=0),
'Loopback Pseudo-Interface 1': snetio(bytes_sent=0, bytes_recv=0, packets_sent=0, packets_recv=0, errin=0, errout=0, dropin=0, dropout=0),
}
with mock.patch('network.psutil') as mock_psutil:
mock_psutil.net_io_counters.return_value = counters
self.check._excluded_ifaces = ['Loopback Pseudo-Interface 1']
self.check._exclude_iface_re = ''
self.check._cx_counters_psutil()
for _, m in self.check.aggregator.metrics.iteritems():
self.assertEqual(m.device_name, 'Ethernet')
if 'bytes_rcvd' in m.name: # test just one of the metrics
self.assertEqual(m.samples[0][1], 3280598526)

def test_parse_protocol_psutil(self):
import socket
conn = mock.MagicMock()

protocol = self.check._parse_protocol_psutil(conn)
self.assertEqual(protocol, '')

conn.type = socket.SOCK_STREAM
conn.family = socket.AF_INET6
protocol = self.check._parse_protocol_psutil(conn)
self.assertEqual(protocol, 'tcp6')

conn.type = socket.SOCK_DGRAM
conn.family = socket.AF_INET
protocol = self.check._parse_protocol_psutil(conn)
self.assertEqual(protocol, 'udp4')
1 change: 0 additions & 1 deletion win32/gui.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,6 @@
'marathon',
'mcache',
'mesos',
'network',
'postfix',
'zk',
]
Expand Down

0 comments on commit ab085ce

Please sign in to comment.