From 25c7ddc4f429926bca2f693de2630c0d35c72b54 Mon Sep 17 00:00:00 2001 From: Massimiliano Pippi Date: Fri, 13 May 2016 17:34:29 +0200 Subject: [PATCH] implement check with psutil on Windows address review comments --- checks.d/network.py | 104 ++++++++++++++++++++++++++---- tests/checks/mock/test_network.py | 98 ++++++++++++++++++++++++++++ win32/gui.py | 1 - 3 files changed, 190 insertions(+), 13 deletions(-) diff --git a/checks.d/network.py b/checks.d/network.py index 16bd2f2ca1..3b921b91fc 100644 --- a/checks.d/network.py +++ b/checks.d/network.py @@ -7,6 +7,8 @@ """ # stdlib import re +import socket +from collections import defaultdict # project from checks import AgentCheck @@ -15,6 +17,7 @@ get_subprocess_output, SubprocessOutputEmptyError, ) +import psutil BSD_TCP_METRICS = [ (re.compile("^\s*(\d+) data packets \(\d+ bytes\) retransmitted\s*$"), 'system.net.tcp.retrans_packs'), @@ -59,22 +62,46 @@ class Network(AgentCheck): "LAST_ACK": "closing", "LISTEN": "listening", "CLOSING": "closing", + }, + "psutil": { + psutil.CONN_ESTABLISHED: "established", + psutil.CONN_SYN_SENT: "opening", + psutil.CONN_SYN_RECV: "opening", + psutil.CONN_FIN_WAIT1: "closing", + psutil.CONN_FIN_WAIT2: "closing", + psutil.CONN_TIME_WAIT: "time_wait", + psutil.CONN_CLOSE: "closing", + psutil.CONN_CLOSE_WAIT: "closing", + psutil.CONN_LAST_ACK: "closing", + psutil.CONN_LISTEN: "listening", + psutil.CONN_CLOSING: "closing", + psutil.CONN_NONE: "connections", # CONN_NONE is always returned for udp connections } } + PSUTIL_TYPE_MAPPING = { + socket.SOCK_STREAM: 'tcp', + socket.SOCK_DGRAM: 'udp', + } + + PSUTIL_FAMILY_MAPPING = { + socket.AF_INET: '4', + socket.AF_INET6: '6', + } + CX_STATE_GAUGE = { - ('udp4', 'connections') : 'system.net.udp4.connections', - ('udp6', 'connections') : 'system.net.udp6.connections', - ('tcp4', 'established') : 'system.net.tcp4.established', - ('tcp4', 'opening') : 'system.net.tcp4.opening', - ('tcp4', 'closing') : 'system.net.tcp4.closing', - ('tcp4', 'listening') : 'system.net.tcp4.listening', - ('tcp4', 'time_wait') : 'system.net.tcp4.time_wait', - ('tcp6', 'established') : 'system.net.tcp6.established', - ('tcp6', 'opening') : 'system.net.tcp6.opening', - ('tcp6', 'closing') : 'system.net.tcp6.closing', - ('tcp6', 'listening') : 'system.net.tcp6.listening', - ('tcp6', 'time_wait') : 'system.net.tcp6.time_wait', + ('udp4', 'connections'): 'system.net.udp4.connections', + ('udp6', 'connections'): 'system.net.udp6.connections', + ('tcp4', 'established'): 'system.net.tcp4.established', + ('tcp4', 'opening'): 'system.net.tcp4.opening', + ('tcp4', 'closing'): 'system.net.tcp4.closing', + ('tcp4', 'listening'): 'system.net.tcp4.listening', + ('tcp4', 'time_wait'): 'system.net.tcp4.time_wait', + ('tcp6', 'established'): 'system.net.tcp6.established', + ('tcp6', 'opening'): 'system.net.tcp6.opening', + ('tcp6', 'closing'): 'system.net.tcp6.closing', + ('tcp6', 'listening'): 'system.net.tcp6.listening', + ('tcp6', 'time_wait'): 'system.net.tcp6.time_wait', } def __init__(self, name, init_config, agentConfig, instances=None): @@ -101,6 +128,8 @@ def check(self, instance): self._check_bsd(instance) elif Platform.is_solaris(): self._check_solaris(instance) + elif Platform.is_windows(): + self._check_psutil() def _submit_devicemetrics(self, iface, vals_by_metric): if iface in self._excluded_ifaces or (self._exclude_iface_re and self._exclude_iface_re.match(iface)): @@ -515,3 +544,54 @@ def _parse_solaris_netstat(self, netstat_output): metrics_by_interface[iface] = metrics return metrics_by_interface + + def _check_psutil(self): + """ + Gather metrics about connections states and interfaces counters + using psutil facilities + """ + if self._collect_cx_state: + self._cx_state_psutil() + + self._cx_counters_psutil() + + def _cx_state_psutil(self): + """ + Collect metrics about connections state using psutil + """ + metrics = defaultdict(int) + for conn in psutil.net_connections(): + protocol = self._parse_protocol_psutil(conn) + status = self.TCP_STATES['psutil'].get(conn.status) + metric = self.CX_STATE_GAUGE.get((protocol, status)) + if metric is None: + self.log.warning('Metric not found for: %s,%s', protocol, status) + else: + metrics[metric] += 1 + + for metric, value in metrics.iteritems(): + self.gauge(metric, value) + + def _cx_counters_psutil(self): + """ + Collect metrics about interfaces counters using psutil + """ + for iface, counters in psutil.net_io_counters(pernic=True).iteritems(): + metrics = { + 'bytes_rcvd': counters.bytes_recv, + 'bytes_sent': counters.bytes_sent, + 'packets_in.count': counters.packets_recv, + 'packets_in.error': counters.errin, + 'packets_out.count': counters.packets_sent, + 'packets_out.error': counters.errout, + } + self._submit_devicemetrics(iface, metrics) + + def _parse_protocol_psutil(self, conn): + """ + Returns a string describing the protocol for the given connection + in the form `tcp4`, 'udp4` as in `self.CX_STATE_GAUGE` + """ + protocol = self.PSUTIL_TYPE_MAPPING.get(conn.type, '') + family = self.PSUTIL_FAMILY_MAPPING.get(conn.family, '') + return '{}{}'.format(protocol, family) diff --git a/tests/checks/mock/test_network.py b/tests/checks/mock/test_network.py index d0dfd0e2fb..d56798f955 100644 --- a/tests/checks/mock/test_network.py +++ b/tests/checks/mock/test_network.py @@ -1,3 +1,6 @@ +from collections import namedtuple +import socket + # 3p import mock @@ -64,3 +67,98 @@ def test_cx_state_linux_netstat(self, mock_subprocess, mock_platform): # Assert metrics for metric, value in self.CX_STATE_GAUGES_VALUES.iteritems(): self.assertMetric(metric, value=value) + + @mock.patch('network.Platform.is_linux', return_value=False) + @mock.patch('network.Platform.is_bsd', return_value=False) + @mock.patch('network.Platform.is_solaris', return_value=False) + @mock.patch('network.Platform.is_windows', return_value=True) + def test_win_uses_psutil(self, *args): + self.check._check_psutil = mock.MagicMock() + self.run_check({}) + self.check._check_psutil.assert_called_once_with() + + @mock.patch('network.Network._cx_state_psutil') + @mock.patch('network.Network._cx_counters_psutil') + def test_check_psutil(self, state, counters): + self.check._cx_state_psutil = state + self.check._cx_counters_psutil = counters + + self.check._collect_cx_state = False + self.check._check_psutil() + state.assert_not_called() + counters.assert_called_once_with() + + state.reset_mock() + counters.reset_mock() + + self.check._collect_cx_state = True + self.check._check_psutil() + state.assert_called_once_with() + counters.assert_called_once_with() + + def test_cx_state_psutil(self): + sconn = namedtuple('sconn', ['fd', 'family', 'type', 'laddr', 'raddr', 'status', 'pid']) + conn = [ + sconn(fd=-1, family=socket.AF_INET, type=socket.SOCK_STREAM, laddr=('127.0.0.1', 50482), raddr=('127.0.0.1',2638), status='ESTABLISHED', pid=1416), + sconn(fd=-1, family=socket.AF_INET6, type=socket.SOCK_STREAM, laddr=('::', 50482), raddr=('::',2638), status='ESTABLISHED', pid=42), + sconn(fd=-1, family=socket.AF_INET6, type=socket.SOCK_STREAM, laddr=('::', 49163), raddr=(), status='LISTEN', pid=1416), + sconn(fd=-1, family=socket.AF_INET, type=socket.SOCK_STREAM, laddr=('0.0.0.0', 445), raddr=(), status='LISTEN', pid=4), + sconn(fd=-1, family=socket.AF_INET6, type=socket.SOCK_STREAM, laddr=('::1', 56521), raddr=('::1', 17123), status='TIME_WAIT', pid=0), + sconn(fd=-1, family=socket.AF_INET6, type=socket.SOCK_DGRAM, laddr=('::', 500), raddr=(), status='NONE', pid=892), + sconn(fd=-1, family=socket.AF_INET6, type=socket.SOCK_STREAM, laddr=('::1', 56493), raddr=('::1', 17123), status='TIME_WAIT', pid=0), + sconn(fd=-1, family=socket.AF_INET, type=socket.SOCK_STREAM, laddr=('127.0.0.1', 54541), raddr=('127.0.0.1', 54542), status='ESTABLISHED', pid=20500), + ] + + results = { + 'system.net.tcp6.time_wait': 2, + 'system.net.tcp4.listening': 1, + 'system.net.tcp6.closing': 0, + 'system.net.tcp4.closing': 0, + 'system.net.tcp4.time_wait': 0, + 'system.net.tcp6.established': 1, + 'system.net.tcp4.established': 2, + 'system.net.tcp6.listening': 1, + 'system.net.tcp4.opening': 0, + 'system.net.udp4.connections': 0, + 'system.net.udp6.connections': 1, + 'system.net.tcp6.opening': 0, + } + + with mock.patch('network.psutil') as mock_psutil: + mock_psutil.net_connections.return_value = conn + self.check._cx_state_psutil() + for _, m in self.check.aggregator.metrics.iteritems(): + self.assertEqual(results[m.name], m.value) + + def test_cx_counters_psutil(self): + snetio = namedtuple('snetio', ['bytes_sent', 'bytes_recv', 'packets_sent', 'packets_recv', 'errin', 'errout', 'dropin', 'dropout']) + counters = { + 'Ethernet': snetio(bytes_sent=3096403230L, bytes_recv=3280598526L, packets_sent=6777924, packets_recv=32888147, errin=0, errout=0, dropin=0, dropout=0), + 'Loopback Pseudo-Interface 1': snetio(bytes_sent=0, bytes_recv=0, packets_sent=0, packets_recv=0, errin=0, errout=0, dropin=0, dropout=0), + } + with mock.patch('network.psutil') as mock_psutil: + mock_psutil.net_io_counters.return_value = counters + self.check._excluded_ifaces = ['Loopback Pseudo-Interface 1'] + self.check._exclude_iface_re = '' + self.check._cx_counters_psutil() + for _, m in self.check.aggregator.metrics.iteritems(): + self.assertEqual(m.device_name, 'Ethernet') + if 'bytes_rcvd' in m.name: # test just one of the metrics + self.assertEqual(m.samples[0][1], 3280598526) + + def test_parse_protocol_psutil(self): + import socket + conn = mock.MagicMock() + + protocol = self.check._parse_protocol_psutil(conn) + self.assertEqual(protocol, '') + + conn.type = socket.SOCK_STREAM + conn.family = socket.AF_INET6 + protocol = self.check._parse_protocol_psutil(conn) + self.assertEqual(protocol, 'tcp6') + + conn.type = socket.SOCK_DGRAM + conn.family = socket.AF_INET + protocol = self.check._parse_protocol_psutil(conn) + self.assertEqual(protocol, 'udp4') diff --git a/win32/gui.py b/win32/gui.py index c5d9327b51..4d0b249fa0 100644 --- a/win32/gui.py +++ b/win32/gui.py @@ -128,7 +128,6 @@ 'marathon', 'mcache', 'mesos', - 'network', 'postfix', 'zk', ]