Merge 365597d into 5da4bde

mutornadomon/collectors/ioloop_util.py

@@ -0,0 +1,35 @@
+import time
+from functools import wraps
+
+
+class UtilizationCollector(object):
+    """Collects stats for overall callback durations"""
+
+    def __init__(self, monitor, flush_interval):
+        self.monitor = monitor
+        self.flush_interval = flush_interval or 10000
+
+    def start(self):
+        self.original_add_callback = self.monitor.io_loop.add_callback
+
+        def measure_callback(callback):
+            @wraps(callback)
+            def timed_callback(*args, **kwargs):
+                last_start_time = time.time()
+                result = callback(*args, **kwargs)
+                duration = (time.time() - last_start_time)
+                self.monitor.accumulated_kv('callback_duration', duration)
+
+                return result
+
+            return timed_callback
+
+        @wraps(self.original_add_callback)
+        def add_timed_callback(callback, *args, **kwargs):
+            return self.original_add_callback(measure_callback(callback), *args, **kwargs)
+
+        self.monitor.io_loop.add_callback = add_timed_callback
+
+    def stop(self):
+        self.monitor.add_callback = self.original_add_callback
+        self.flush_callback.stop()

mutornadomon/config.py

@@ -1,9 +1,11 @@
 from __future__ import absolute_import
 
+
 from mutornadomon import MuTornadoMon
 from mutornadomon.external_interfaces.publish import PublishExternalInterface
 from mutornadomon.external_interfaces.http_endpoints import HTTPEndpointExternalInterface
 from mutornadomon.collectors.web import WebCollector
+from mutornadomon.collectors.ioloop_util import UtilizationCollector
 
 
 def initialize_mutornadomon(tornado_app=None, publisher=None, publish_interval=None,
@@ -28,4 +30,7 @@ def initialize_mutornadomon(tornado_app=None, publisher=None, publish_interval=N
         web_collector = WebCollector(monitor, tornado_app)
         web_collector.start()
 
+    utilization_collector = UtilizationCollector(monitor, publish_interval)
+    utilization_collector.start()
+
     return monitor

mutornadomon/monitor.py

@@ -88,6 +88,7 @@ def _reset_ephemeral(self):
         """
         self._MIN_GAUGES = {}
         self._MAX_GAUGES = {}
+        self._ACCUMULATED_GAUGES = collections.defaultdict(float)
 
     def count(self, stat, value=1):
         """Increment a counter by the given value"""
@@ -105,6 +106,11 @@ def kv(self, stat, value):
         if stat not in self._MIN_GAUGES or value < self._MIN_GAUGES[stat]:
             self._MIN_GAUGES[stat] = value
 
+    def accumulated_kv(self, stat, value):
+        """Accumulate a value that will be flushed to a gauge when metrics is read"""
+
+        self._ACCUMULATED_GAUGES[stat] += value
+
     def start(self):
         for collector in self.collectors:
             collector.start(self)
@@ -154,6 +160,8 @@ def metrics(self):
             create_time = me.create_time()
             num_threads = me.num_threads()
             num_fds = me.num_fds()
+        for key, value in self._ACCUMULATED_GAUGES.items():
+            self.kv(key, value)
         rv = {
             'process': {
                 'uptime': time.time() - create_time,

tests/test_basic.py

@@ -36,10 +36,8 @@ def test_endpoint(self, mock_num_threads):
         resp = self.fetch('/mutornadomon')
         self.assertEqual(resp.code, 200)
         resp = json.loads(resp.body.decode('utf-8'))
-        self.assertEqual(
-            resp['counters'],
-            {'requests': 2, 'localhost_requests': 2, 'private_requests': 2}
-        )
+        expected = {'requests': 2, 'localhost_requests': 2, 'private_requests': 2}.items()
+        self.assertTrue(all(pair in resp['counters'].items() for pair in expected))
         self.assertEqual(resp['process']['cpu']['num_threads'], 5)
         assert resp['process']['cpu']['system_time'] < 1.0
 

tests/test_config.py

@@ -15,7 +15,8 @@ class TestInitializeMutornadomon(unittest.TestCase):
 
     @mock.patch('mutornadomon.config.MuTornadoMon')
     @mock.patch('mutornadomon.config.WebCollector')
-    def test_initialize_mutornadmon(self, web_collector_mock, mutornadomon_mock):
+    @mock.patch('mutornadomon.config.UtilizationCollector')
+    def test_initialize_mutornadmon(self, utilization_collector_mock, web_collector_mock, mutornadomon_mock):
         """Test initialize_mutornadomon() sets up HTTP endpoints interface"""
         app = sentinel.application,
         result = initialize_mutornadomon(app, host_limit='test')
@@ -26,6 +27,7 @@ def test_initialize_mutornadmon(self, web_collector_mock, mutornadomon_mock):
 
         mutornadomon_mock.assert_called_once()
         web_collector_mock.assert_called_once_with(monitor_inst, app)
+        utilization_collector_mock.assert_called_once_with(monitor_inst, None)
 
         # MuTornadoMon was created with monitor config values
         arg_list = mutornadomon_mock.call_args_list
@@ -39,7 +41,13 @@ def test_initialize_mutornadmon(self, web_collector_mock, mutornadomon_mock):
 
     @mock.patch('mutornadomon.config.MuTornadoMon')
     @mock.patch('mutornadomon.config.WebCollector')
-    def test_initialize_mutornadmon_passes_publisher(self, web_collector_mock, mutornadomon_mock):
+    @mock.patch('mutornadomon.config.UtilizationCollector')
+    def test_initialize_mutornadmon_passes_publisher(
+        self,
+        utilization_collector_mock,
+        web_collector_mock,
+        mutornadomon_mock
+    ):
         """Test initialize_mutornadomon() sets up publishing interface"""
 
         def publisher(monitor):
@@ -55,6 +63,7 @@ def publisher(monitor):
         self.assertEqual(result, monitor_inst)
 
         web_collector_mock.assert_called_once_with(monitor_inst, app)
+        utilization_collector_mock.assert_called_once_with(monitor_inst, None)
 
         mutornadomon_mock.assert_called_once()
         arg_list = mutornadomon_mock.call_args_list
@@ -72,8 +81,9 @@ def test_initialize_mutornadmon_works_with_publisher_and_no_app(self, mutornadom
         def publisher(monitor):
             pass
 
-        result = initialize_mutornadomon(publisher=publisher)
         monitor_inst = mutornadomon_mock.return_value
+        monitor_inst.io_loop.add_callback.__name__ = 'add_callback'
+        result = initialize_mutornadomon(publisher=publisher)
 
         # initialize_mutornadomon() should return the monitor instance
         self.assertEqual(result, monitor_inst)