Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

added the stats classes

  • Loading branch information...
commit 5f78d0e1a8fd86321ff0073569bbb50cdc4bf112 1 parent 9312c22
@twobraids twobraids authored twobraids committed
View
83 socorro/lib/statistics.py
@@ -0,0 +1,83 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+"""
+
+from statsd import StatsClient
+
+from configman import RequiredConfig, Namespace
+
+def str_to_list(string_list):
+ item_list = []
+ for x in string_list.split(','):
+ item_list.append(x.strip())
+ return item_list
+
+
+class StatisticsForStatsd(RequiredConfig):
+ """This class is a wrapper around statsd adding a simple configman
+ compatible interface and stats naming scheme. Code using this class
+ will distrubute `incr` calls with names associated with them. When
+ ever an `incr` call is encountered, the name will be paired with the
+ name of the statsd names and the increment action fired off.
+
+ This class will only send stats `incr` calls for names that appear in
+ the configuration parameter `active_counters_list`. This enables counters
+ to be turned on and off at configuration time."""
+
+ required_config = Namespace()
+ required_config.add_option(
+ 'statsd_host',
+ doc='the hostname of statsd',
+ default=''
+ )
+ required_config.add_option(
+ 'statsd_port',
+ doc='the port number for statsd',
+ default=''
+ )
+ required_config.add_option(
+ 'prefix',
+ doc='a string to be used as the prefix for statsd names',
+ default=''
+ )
+ required_config.add_option(
+ 'active_counters_list',
+ default='',
+ #default='restarts, jobs, criticals, errors, mdsw_failures',
+ doc='a comma delimeted list of counters',
+ from_string_converter=str_to_list
+ )
+
+ def __init__(self, config, name):
+ super(StatisticsForStatsd, self).__init__()
+ self.config = config
+ if config.prefix and name:
+ self.prefix = '.'.join((config.prefix, name))
+ elif config.prefix:
+ self.prefix = config.prefix
+ elif name:
+ self.prefix = name
+ else:
+ self.prefix = ''
+ self.statsd = StatsClient(
+ config.statsd_host,
+ config.statsd_port,
+ self.prefix
+ )
+
+ def incr(self, name):
+ if (
+ self.config.statsd_host
+ and name in self.config.active_counters_list
+ ):
+ if self.prefix and name:
+ name = '.'.join((self.prefix, name))
+ elif self.prefix:
+ name = self.prefix
+ elif not name:
+ name = 'unknown'
+ self.statsd.incr(name)
+
View
16 socorro/processor/legacy_processor.py
@@ -183,6 +183,13 @@ class LegacyCrashProcessor(RequiredConfig):
'collected',
default=True,
)
+ required_config.namespace('statistics')
+ required_config.statistics.add_option(
+ 'stats_class',
+ default='socorro.lib.statistics.StatisticsForStatsd',
+ doc='name of a class that will gather statistics',
+ from_string_converter=class_converter
+ )
#--------------------------------------------------------------------------
def __init__(self, config, quit_check_callback=None):
@@ -239,6 +246,12 @@ def __init__(self, config, quit_check_callback=None):
)
self._product_id_map = {}
self._load_product_id_map()
+ self._statistics = config.statistics.stats_class(
+ config.statistics,
+ self.config.processor_name
+ )
+ self._statistics.incr('restarts')
+
#--------------------------------------------------------------------------
def reject_raw_crash(self, crash_id, reason):
@@ -255,6 +268,7 @@ def convert_raw_crash_to_processed_crash(self, raw_crash, raw_dumps):
input parameters:
"""
+ self._statistics.incr('jobs')
processor_notes = [self.config.processor_name]
try:
self.quit_check()
@@ -314,6 +328,7 @@ def convert_raw_crash_to_processed_crash(self, raw_crash, raw_dumps):
)
processed_crash.success = False
processor_notes.append('unrecoverable processor error')
+ self._statistics.incr('errors')
processor_notes = '; '.join(processor_notes)
processed_crash.processor_notes = processor_notes
@@ -768,6 +783,7 @@ def _stackwalk_analysis(
return_code = mdsw_subprocess_handle.wait()
if return_code is not None and return_code != 0:
+ self._statistics.incr('mdsw_failures')
processor_notes.append(
"MDSW failed: %s" % mdsw_subprocess_handle.returncode
)
View
179 socorro/unittest/lib/test_statistics.py
@@ -0,0 +1,179 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import unittest
+
+from mock import Mock, patch, call
+
+from socorro.lib.util import DotDict, SilentFakeLogger
+from socorro.lib.statistics import StatisticsForStatsd
+
+class TestStatsd(unittest.TestCase):
+
+ def setUp(self):
+ self.logger = SilentFakeLogger()
+ pass
+
+ def tearDown(self):
+ pass
+
+ def test_statistics_all_good(self):
+ d = DotDict()
+ d.statsd_host = 'localhost'
+ d.statsd_port = 666
+ d.prefix = 'a.b'
+ d.active_counters_list = ['x', 'y', 'z']
+
+ with patch('socorro.lib.statistics.StatsClient') as StatsClientMocked:
+ s = StatisticsForStatsd(d, 'processor')
+ StatsClientMocked.assert_called_with(
+ 'localhost', 666, 'a.b.processor')
+
+ s.incr('x')
+ StatsClientMocked.assert_has_calls(
+ StatsClientMocked.mock_calls,
+ [call.incr('a.b.processor.x')]
+ )
+
+ s.incr('y')
+ StatsClientMocked.assert_has_calls(
+ StatsClientMocked.mock_calls,
+ [call.incr('a.b.processor.y')]
+ )
+
+ s.incr('z')
+ StatsClientMocked.assert_has_calls(
+ StatsClientMocked.mock_calls,
+ [call.incr('a.b.processor.z')]
+ )
+
+ s.incr('w')
+ StatsClientMocked.assert_has_calls(
+ StatsClientMocked.mock_calls,
+ [
+ call.incr('a.b.processor.y'),
+ call.incr('a.b.processor.x'),
+ call.incr('a.b.processor.y')
+ ]
+ )
+
+ s.incr(None)
+ StatsClientMocked.assert_has_calls(
+ StatsClientMocked.mock_calls,
+ [
+ call.incr('a.b.processor.y'),
+ call.incr('a.b.processor.x'),
+ call.incr('a.b.processor.y'),
+ call.incr('a.b.processor.unknown')
+ ]
+ )
+
+
+
+ def test_statistics_all_missing_prefix(self):
+ d = DotDict()
+ d.statsd_host = 'localhost'
+ d.statsd_port = 666
+ d.prefix = None
+ d.active_counters_list = ['x', 'y', 'z']
+
+ with patch('socorro.lib.statistics.StatsClient') as StatsClientMocked:
+ s = StatisticsForStatsd(d, 'processor')
+ StatsClientMocked.assert_called_with(
+ 'localhost', 666, 'processor')
+
+ s.incr('x')
+ StatsClientMocked.assert_has_calls(
+ StatsClientMocked.mock_calls,
+ [call.incr('processor.x')]
+ )
+
+ s.incr('y')
+ StatsClientMocked.assert_has_calls(
+ StatsClientMocked.mock_calls,
+ [call.incr('processor.y')]
+ )
+
+ s.incr('z')
+ StatsClientMocked.assert_has_calls(
+ StatsClientMocked.mock_calls,
+ [call.incr('processor.z')]
+ )
+
+ s.incr('w')
+ StatsClientMocked.assert_has_calls(
+ StatsClientMocked.mock_calls,
+ [
+ call.incr('processor.y'),
+ call.incr('processor.x'),
+ call.incr('processor.y')
+ ]
+ )
+
+ s.incr(None)
+ StatsClientMocked.assert_has_calls(
+ StatsClientMocked.mock_calls,
+ [
+ call.incr('processor.y'),
+ call.incr('processor.x'),
+ call.incr('processor.y'),
+ call.incr('processor.unknown')
+ ]
+ )
+
+
+ def test_statistics_all_missing_prefix_and_missing_name(self):
+ d = DotDict()
+ d.statsd_host = 'localhost'
+ d.statsd_port = 666
+ d.prefix = None
+ d.active_counters_list = ['x', 'y', 'z']
+
+ with patch('socorro.lib.statistics.StatsClient') as StatsClientMocked:
+ s = StatisticsForStatsd(d, None)
+ StatsClientMocked.assert_called_with(
+ 'localhost', 666, '')
+
+ s.incr('x')
+ StatsClientMocked.assert_has_calls(
+ StatsClientMocked.mock_calls,
+ [call.incr('x')]
+ )
+
+ s.incr('y')
+ StatsClientMocked.assert_has_calls(
+ StatsClientMocked.mock_calls,
+ [call.incr('y')]
+ )
+
+ s.incr('z')
+ StatsClientMocked.assert_has_calls(
+ StatsClientMocked.mock_calls,
+ [call.incr('z')]
+ )
+
+ s.incr('w')
+ StatsClientMocked.assert_has_calls(
+ StatsClientMocked.mock_calls,
+ [
+ call.incr('y'),
+ call.incr('x'),
+ call.incr('y')
+ ]
+ )
+
+ s.incr(None)
+ StatsClientMocked.assert_has_calls(
+ StatsClientMocked.mock_calls,
+ [
+ call.incr('y'),
+ call.incr('x'),
+ call.incr('y'),
+ call.incr('unknown')
+ ]
+ )
+
+
+
+
View
47 socorro/unittest/processor/test_legacy_processor.py
@@ -48,6 +48,9 @@ def setup_config_with_mocks():
config.java_signature = DotDict()
config.java_signature.java_signature_tool_class = mock.Mock()
+ config.statistics = DotDict()
+ config.statistics.stats_class = mock.Mock()
+
return config
canonical_standard_raw_crash = DotDict({
@@ -367,6 +370,14 @@ def test_convert_raw_crash_to_processed_crash_basic(self):
dict(epc)
)
+ leg_proc._statistics.assert_has_calls(
+ [
+ mock.call.incr('jobs'),
+ mock.call.incr('restarts')
+ ],
+ any_order=True
+ )
+
def test_convert_raw_crash_to_processed_crash_unexpected_error(self):
config = setup_config_with_mocks()
mocked_transform_rules_str = \
@@ -438,8 +449,16 @@ def test_convert_raw_crash_to_processed_crash_unexpected_error(self):
'java_stack_trace': None,
'additional_minidumps': [],
}
- print processed_crash.processor_notes
self.assertEqual(e, processed_crash)
+ leg_proc._statistics.assert_has_calls(
+ [
+ mock.call.incr('jobs'),
+ mock.call.incr('restarts'),
+ mock.call.incr('errors'),
+ ],
+ any_order=True
+ )
+
def test_create_basic_processed_crash_normal(self):
config = setup_config_with_mocks()
@@ -584,8 +603,12 @@ def test_create_basic_processed_crash_normal(self):
processed_crash_with_hang_only
)
self.assertEqual(len(processor_notes), 0)
-
-
+ leg_proc._statistics.assert_has_calls(
+ [
+ mock.call.incr('restarts'),
+ ],
+ any_order=True
+ )
def test_process_list_of_addons(self):
@@ -638,6 +661,12 @@ def test_process_list_of_addons(self):
('adblockpopups@jessehakanen.net', '0:3:1'),
]
self.assertEqual(addon_list, expected_addon_list)
+ leg_proc._statistics.assert_has_calls(
+ [
+ mock.call.incr('restarts'),
+ ],
+ any_order=True
+ )
def test_add_process_type_to_processed_crash(self):
config = setup_config_with_mocks()
@@ -795,3 +824,15 @@ def _analyze_frames(self, hang_type, java_stack_trace,
self.assertEqual(e_pcu, processed_crash_update)
excess = list(m_iter)
self.assertEqual(len(excess), 0)
+ # even though we're testing _do_breakpad_stack_dump_analysis
+ # for a successful run, it technically fails because the
+ # return code of mdsw is non-zero. Well take advantage of that
+ # and see if we logged a failure in the stats package.
+ leg_proc._statistics.assert_has_calls(
+ [
+ mock.call.incr('restarts'),
+ mock.call.incr('mdsw_failures'),
+ ],
+ any_order=True
+ )
+
Please sign in to comment.
Something went wrong with that request. Please try again.