Permalink
Browse files

Merge pull request #1147 from twobraids/statsd

fixes Bug 853299: added statsd to legacy processor
  • Loading branch information...
2 parents f2a5693 + 5f78d0e commit 808a340e3ad3c2c9c8f581fd527b4d553c613f2e @lonnen lonnen committed Mar 25, 2013
View
@@ -0,0 +1,83 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+"""
+
+from statsd import StatsClient
+
+from configman import RequiredConfig, Namespace
+
+def str_to_list(string_list):
+ item_list = []
+ for x in string_list.split(','):
+ item_list.append(x.strip())
+ return item_list
+
+
+class StatisticsForStatsd(RequiredConfig):
+ """This class is a wrapper around statsd adding a simple configman
+ compatible interface and stats naming scheme. Code using this class
+ will distrubute `incr` calls with names associated with them. When
+ ever an `incr` call is encountered, the name will be paired with the
+ name of the statsd names and the increment action fired off.
+
+ This class will only send stats `incr` calls for names that appear in
+ the configuration parameter `active_counters_list`. This enables counters
+ to be turned on and off at configuration time."""
+
+ required_config = Namespace()
+ required_config.add_option(
+ 'statsd_host',
+ doc='the hostname of statsd',
+ default=''
+ )
+ required_config.add_option(
+ 'statsd_port',
+ doc='the port number for statsd',
+ default=''
+ )
+ required_config.add_option(
+ 'prefix',
+ doc='a string to be used as the prefix for statsd names',
+ default=''
+ )
+ required_config.add_option(
+ 'active_counters_list',
+ default='',
+ #default='restarts, jobs, criticals, errors, mdsw_failures',
+ doc='a comma delimeted list of counters',
+ from_string_converter=str_to_list
+ )
+
+ def __init__(self, config, name):
+ super(StatisticsForStatsd, self).__init__()
+ self.config = config
+ if config.prefix and name:
+ self.prefix = '.'.join((config.prefix, name))
+ elif config.prefix:
+ self.prefix = config.prefix
+ elif name:
+ self.prefix = name
+ else:
+ self.prefix = ''
+ self.statsd = StatsClient(
+ config.statsd_host,
+ config.statsd_port,
+ self.prefix
+ )
+
+ def incr(self, name):
+ if (
+ self.config.statsd_host
+ and name in self.config.active_counters_list
+ ):
+ if self.prefix and name:
+ name = '.'.join((self.prefix, name))
+ elif self.prefix:
+ name = self.prefix
+ elif not name:
+ name = 'unknown'
+ self.statsd.incr(name)
+
@@ -183,6 +183,13 @@ class LegacyCrashProcessor(RequiredConfig):
'collected',
default=True,
)
+ required_config.namespace('statistics')
+ required_config.statistics.add_option(
+ 'stats_class',
+ default='socorro.lib.statistics.StatisticsForStatsd',
+ doc='name of a class that will gather statistics',
+ from_string_converter=class_converter
+ )
#--------------------------------------------------------------------------
def __init__(self, config, quit_check_callback=None):
@@ -239,6 +246,12 @@ def __init__(self, config, quit_check_callback=None):
)
self._product_id_map = {}
self._load_product_id_map()
+ self._statistics = config.statistics.stats_class(
+ config.statistics,
+ self.config.processor_name
+ )
+ self._statistics.incr('restarts')
+
#--------------------------------------------------------------------------
def reject_raw_crash(self, crash_id, reason):
@@ -255,6 +268,7 @@ def convert_raw_crash_to_processed_crash(self, raw_crash, raw_dumps):
input parameters:
"""
+ self._statistics.incr('jobs')
processor_notes = [self.config.processor_name]
try:
self.quit_check()
@@ -314,6 +328,7 @@ def convert_raw_crash_to_processed_crash(self, raw_crash, raw_dumps):
)
processed_crash.success = False
processor_notes.append('unrecoverable processor error')
+ self._statistics.incr('errors')
processor_notes = '; '.join(processor_notes)
processed_crash.processor_notes = processor_notes
@@ -768,6 +783,7 @@ def _stackwalk_analysis(
return_code = mdsw_subprocess_handle.wait()
if return_code is not None and return_code != 0:
+ self._statistics.incr('mdsw_failures')
processor_notes.append(
"MDSW failed: %s" % mdsw_subprocess_handle.returncode
)
@@ -0,0 +1,179 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import unittest
+
+from mock import Mock, patch, call
+
+from socorro.lib.util import DotDict, SilentFakeLogger
+from socorro.lib.statistics import StatisticsForStatsd
+
+class TestStatsd(unittest.TestCase):
+
+ def setUp(self):
+ self.logger = SilentFakeLogger()
+ pass
+
+ def tearDown(self):
+ pass
+
+ def test_statistics_all_good(self):
+ d = DotDict()
+ d.statsd_host = 'localhost'
+ d.statsd_port = 666
+ d.prefix = 'a.b'
+ d.active_counters_list = ['x', 'y', 'z']
+
+ with patch('socorro.lib.statistics.StatsClient') as StatsClientMocked:
+ s = StatisticsForStatsd(d, 'processor')
+ StatsClientMocked.assert_called_with(
+ 'localhost', 666, 'a.b.processor')
+
+ s.incr('x')
+ StatsClientMocked.assert_has_calls(
+ StatsClientMocked.mock_calls,
+ [call.incr('a.b.processor.x')]
+ )
+
+ s.incr('y')
+ StatsClientMocked.assert_has_calls(
+ StatsClientMocked.mock_calls,
+ [call.incr('a.b.processor.y')]
+ )
+
+ s.incr('z')
+ StatsClientMocked.assert_has_calls(
+ StatsClientMocked.mock_calls,
+ [call.incr('a.b.processor.z')]
+ )
+
+ s.incr('w')
+ StatsClientMocked.assert_has_calls(
+ StatsClientMocked.mock_calls,
+ [
+ call.incr('a.b.processor.y'),
+ call.incr('a.b.processor.x'),
+ call.incr('a.b.processor.y')
+ ]
+ )
+
+ s.incr(None)
+ StatsClientMocked.assert_has_calls(
+ StatsClientMocked.mock_calls,
+ [
+ call.incr('a.b.processor.y'),
+ call.incr('a.b.processor.x'),
+ call.incr('a.b.processor.y'),
+ call.incr('a.b.processor.unknown')
+ ]
+ )
+
+
+
+ def test_statistics_all_missing_prefix(self):
+ d = DotDict()
+ d.statsd_host = 'localhost'
+ d.statsd_port = 666
+ d.prefix = None
+ d.active_counters_list = ['x', 'y', 'z']
+
+ with patch('socorro.lib.statistics.StatsClient') as StatsClientMocked:
+ s = StatisticsForStatsd(d, 'processor')
+ StatsClientMocked.assert_called_with(
+ 'localhost', 666, 'processor')
+
+ s.incr('x')
+ StatsClientMocked.assert_has_calls(
+ StatsClientMocked.mock_calls,
+ [call.incr('processor.x')]
+ )
+
+ s.incr('y')
+ StatsClientMocked.assert_has_calls(
+ StatsClientMocked.mock_calls,
+ [call.incr('processor.y')]
+ )
+
+ s.incr('z')
+ StatsClientMocked.assert_has_calls(
+ StatsClientMocked.mock_calls,
+ [call.incr('processor.z')]
+ )
+
+ s.incr('w')
+ StatsClientMocked.assert_has_calls(
+ StatsClientMocked.mock_calls,
+ [
+ call.incr('processor.y'),
+ call.incr('processor.x'),
+ call.incr('processor.y')
+ ]
+ )
+
+ s.incr(None)
+ StatsClientMocked.assert_has_calls(
+ StatsClientMocked.mock_calls,
+ [
+ call.incr('processor.y'),
+ call.incr('processor.x'),
+ call.incr('processor.y'),
+ call.incr('processor.unknown')
+ ]
+ )
+
+
+ def test_statistics_all_missing_prefix_and_missing_name(self):
+ d = DotDict()
+ d.statsd_host = 'localhost'
+ d.statsd_port = 666
+ d.prefix = None
+ d.active_counters_list = ['x', 'y', 'z']
+
+ with patch('socorro.lib.statistics.StatsClient') as StatsClientMocked:
+ s = StatisticsForStatsd(d, None)
+ StatsClientMocked.assert_called_with(
+ 'localhost', 666, '')
+
+ s.incr('x')
+ StatsClientMocked.assert_has_calls(
+ StatsClientMocked.mock_calls,
+ [call.incr('x')]
+ )
+
+ s.incr('y')
+ StatsClientMocked.assert_has_calls(
+ StatsClientMocked.mock_calls,
+ [call.incr('y')]
+ )
+
+ s.incr('z')
+ StatsClientMocked.assert_has_calls(
+ StatsClientMocked.mock_calls,
+ [call.incr('z')]
+ )
+
+ s.incr('w')
+ StatsClientMocked.assert_has_calls(
+ StatsClientMocked.mock_calls,
+ [
+ call.incr('y'),
+ call.incr('x'),
+ call.incr('y')
+ ]
+ )
+
+ s.incr(None)
+ StatsClientMocked.assert_has_calls(
+ StatsClientMocked.mock_calls,
+ [
+ call.incr('y'),
+ call.incr('x'),
+ call.incr('y'),
+ call.incr('unknown')
+ ]
+ )
+
+
+
+
Oops, something went wrong.

0 comments on commit 808a340

Please sign in to comment.