diff --git a/mutornadomon/collectors/ioloop_util.py b/mutornadomon/collectors/ioloop_util.py new file mode 100644 index 0000000..13f60ae --- /dev/null +++ b/mutornadomon/collectors/ioloop_util.py @@ -0,0 +1,35 @@ +import time +from functools import wraps + + +class UtilizationCollector(object): + """Collects stats for overall callback durations""" + + def __init__(self, monitor, flush_interval): + self.monitor = monitor + self.flush_interval = flush_interval or 10000 + + def start(self): + self.original_add_callback = self.monitor.io_loop.add_callback + + def measure_callback(callback): + @wraps(callback) + def timed_callback(*args, **kwargs): + last_start_time = time.time() + result = callback(*args, **kwargs) + duration = (time.time() - last_start_time) + self.monitor.accumulated_kv('callback_duration', duration) + + return result + + return timed_callback + + @wraps(self.original_add_callback) + def add_timed_callback(callback, *args, **kwargs): + return self.original_add_callback(measure_callback(callback), *args, **kwargs) + + self.monitor.io_loop.add_callback = add_timed_callback + + def stop(self): + self.monitor.add_callback = self.original_add_callback + self.flush_callback.stop() diff --git a/mutornadomon/config.py b/mutornadomon/config.py index 543492a..e9b7fbc 100644 --- a/mutornadomon/config.py +++ b/mutornadomon/config.py @@ -1,9 +1,11 @@ from __future__ import absolute_import + from mutornadomon import MuTornadoMon from mutornadomon.external_interfaces.publish import PublishExternalInterface from mutornadomon.external_interfaces.http_endpoints import HTTPEndpointExternalInterface from mutornadomon.collectors.web import WebCollector +from mutornadomon.collectors.ioloop_util import UtilizationCollector def initialize_mutornadomon(tornado_app=None, publisher=None, publish_interval=None, @@ -28,4 +30,7 @@ def initialize_mutornadomon(tornado_app=None, publisher=None, publish_interval=N web_collector = WebCollector(monitor, tornado_app) web_collector.start() + utilization_collector = UtilizationCollector(monitor, publish_interval) + utilization_collector.start() + return monitor diff --git a/mutornadomon/monitor.py b/mutornadomon/monitor.py index 421773a..ba5d825 100644 --- a/mutornadomon/monitor.py +++ b/mutornadomon/monitor.py @@ -88,6 +88,7 @@ def _reset_ephemeral(self): """ self._MIN_GAUGES = {} self._MAX_GAUGES = {} + self._ACCUMULATED_GAUGES = collections.defaultdict(float) def count(self, stat, value=1): """Increment a counter by the given value""" @@ -105,6 +106,11 @@ def kv(self, stat, value): if stat not in self._MIN_GAUGES or value < self._MIN_GAUGES[stat]: self._MIN_GAUGES[stat] = value + def accumulated_kv(self, stat, value): + """Accumulate a value that will be flushed to a gauge when metrics is read""" + + self._ACCUMULATED_GAUGES[stat] += value + def start(self): for collector in self.collectors: collector.start(self) @@ -154,6 +160,8 @@ def metrics(self): create_time = me.create_time() num_threads = me.num_threads() num_fds = me.num_fds() + for key, value in self._ACCUMULATED_GAUGES.items(): + self.kv(key, value) rv = { 'process': { 'uptime': time.time() - create_time, diff --git a/tests/test_basic.py b/tests/test_basic.py index 2922593..8e39f25 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -36,10 +36,8 @@ def test_endpoint(self, mock_num_threads): resp = self.fetch('/mutornadomon') self.assertEqual(resp.code, 200) resp = json.loads(resp.body.decode('utf-8')) - self.assertEqual( - resp['counters'], - {'requests': 2, 'localhost_requests': 2, 'private_requests': 2} - ) + expected = {'requests': 2, 'localhost_requests': 2, 'private_requests': 2}.items() + self.assertTrue(all(pair in resp['counters'].items() for pair in expected)) self.assertEqual(resp['process']['cpu']['num_threads'], 5) assert resp['process']['cpu']['system_time'] < 1.0 diff --git a/tests/test_config.py b/tests/test_config.py index ae567b2..1bf9508 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -15,7 +15,8 @@ class TestInitializeMutornadomon(unittest.TestCase): @mock.patch('mutornadomon.config.MuTornadoMon') @mock.patch('mutornadomon.config.WebCollector') - def test_initialize_mutornadmon(self, web_collector_mock, mutornadomon_mock): + @mock.patch('mutornadomon.config.UtilizationCollector') + def test_initialize_mutornadmon(self, utilization_collector_mock, web_collector_mock, mutornadomon_mock): """Test initialize_mutornadomon() sets up HTTP endpoints interface""" app = sentinel.application, result = initialize_mutornadomon(app, host_limit='test') @@ -26,6 +27,7 @@ def test_initialize_mutornadmon(self, web_collector_mock, mutornadomon_mock): mutornadomon_mock.assert_called_once() web_collector_mock.assert_called_once_with(monitor_inst, app) + utilization_collector_mock.assert_called_once_with(monitor_inst, None) # MuTornadoMon was created with monitor config values arg_list = mutornadomon_mock.call_args_list @@ -39,7 +41,13 @@ def test_initialize_mutornadmon(self, web_collector_mock, mutornadomon_mock): @mock.patch('mutornadomon.config.MuTornadoMon') @mock.patch('mutornadomon.config.WebCollector') - def test_initialize_mutornadmon_passes_publisher(self, web_collector_mock, mutornadomon_mock): + @mock.patch('mutornadomon.config.UtilizationCollector') + def test_initialize_mutornadmon_passes_publisher( + self, + utilization_collector_mock, + web_collector_mock, + mutornadomon_mock + ): """Test initialize_mutornadomon() sets up publishing interface""" def publisher(monitor): @@ -55,6 +63,7 @@ def publisher(monitor): self.assertEqual(result, monitor_inst) web_collector_mock.assert_called_once_with(monitor_inst, app) + utilization_collector_mock.assert_called_once_with(monitor_inst, None) mutornadomon_mock.assert_called_once() arg_list = mutornadomon_mock.call_args_list @@ -72,8 +81,9 @@ def test_initialize_mutornadmon_works_with_publisher_and_no_app(self, mutornadom def publisher(monitor): pass - result = initialize_mutornadomon(publisher=publisher) monitor_inst = mutornadomon_mock.return_value + monitor_inst.io_loop.add_callback.__name__ = 'add_callback' + result = initialize_mutornadomon(publisher=publisher) # initialize_mutornadomon() should return the monitor instance self.assertEqual(result, monitor_inst)