Permalink
Browse files

Add additional module for memcached

Although there is already one memcached Python module, mine approaches things
someone differently, and adds aggregated stats about the max age of items in
slabs -- metrics that are useful to us at Wikimedia and hopefully will be
elsewhere, too.
  • Loading branch information...
1 parent be072b7 commit 81ad2efec3957245fc34d0ad4c73b629c443b502 @atdt atdt committed Aug 7, 2012
View
@@ -0,0 +1,21 @@
+python-memcached-gmond
+======================
+
+
+This is a Python Gmond module for Memcached, compatible with both Python 2 and
+3. In addition to the usual datapoints provided by "stats", this module
+aggregates max age metrics from "stats items". All metrics are available in a
+"memcached" collection group.
+
+If you've installed ganglia at the standard locations, you should be able to
+install this module by copying `memcached.pyconf` to `/etc/ganglia/conf.d` and
+`memcached.py`, `memcached_metrics.py`, and 'every.py' to
+`/usr/lib/ganglia/python_modules`. The memcached server's host and port can be
+specified in the configuration in memcached.pyconf.
+
+For more information, see the section [Gmond Python metric modules][1] in the
+Ganglia documentation.
+
+Author: Ori Livneh <ori@wikimedia.org>
+
+ [1]: http://sourceforge.net/apps/trac/ganglia/wiki/ganglia_gmond_python_modules
@@ -0,0 +1,133 @@
+# Gmond configuration for memcached metric module
+# Install to /etc/ganglia/conf.d
+
+modules {
+ module {
+ name = "memcached"
+ language = "python"
+ param host {
+ value = "127.0.0.1"
+ }
+ param port {
+ value = "11211"
+ }
+ }
+}
+
+collection_group {
+ collect_every = 10
+ time_threshold = 60
+
+ metric {
+ name = "curr_items"
+ title = "curr_items"
+ }
+ metric {
+ name = "total_items"
+ title = "total_items"
+ }
+ metric {
+ name = "bytes"
+ title = "bytes"
+ }
+ metric {
+ name = "curr_connections"
+ title = "curr_connections"
+ }
+ metric {
+ name = "total_connections"
+ title = "total_connections"
+ }
+ metric {
+ name = "connection_structures"
+ title = "connection_structures"
+ }
+ metric {
+ name = "cmd_get"
+ title = "cmd_get"
+ }
+ metric {
+ name = "cmd_set"
+ title = "cmd_set"
+ }
+ metric {
+ name = "get_hits"
+ title = "get_hits"
+ }
+ metric {
+ name = "get_misses"
+ title = "get_misses"
+ }
+ metric {
+ name = "delete_hits"
+ title = "delete_hits"
+ }
+ metric {
+ name = "delete_misses"
+ title = "delete_misses"
+ }
+ metric {
+ name = "incr_hits"
+ title = "incr_hits"
+ }
+ metric {
+ name = "incr_misses"
+ title = "incr_misses"
+ }
+ metric {
+ name = "decr_hits"
+ title = "decr_hits"
+ }
+ metric {
+ name = "decr_misses"
+ title = "decr_misses"
+ }
+ metric {
+ name = "cas_hits"
+ title = "cas_hits"
+ }
+ metric {
+ name = "cas_misses"
+ title = "cas_misses"
+ }
+ metric {
+ name = "evictions"
+ title = "evictions"
+ }
+ metric {
+ name = "bytes_read"
+ title = "bytes_read"
+ }
+ metric {
+ name = "bytes_written"
+ title = "bytes_written"
+ }
+ metric {
+ name = "limit_maxbytes"
+ title = "limit_maxbytes"
+ }
+ metric {
+ name = "threads"
+ title = "threads"
+ }
+ metric {
+ name = "conn_yields"
+ title = "conn_yields"
+ }
+ metric {
+ name = "age_mean"
+ title = "age_mean"
+ }
+ metric {
+ name = "age_median"
+ title = "age_median"
+ }
+ metric {
+ name = "age_min"
+ title = "age_min"
+ }
+ metric {
+ name = "age_max"
+ title = "age_max"
+ }
+}
@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+ Every
+
+ Python decorator; decorated function is called on a set interval.
+
+ :author: Ori Livneh <ori@wikimedia.org>
+ :copyright: (c) 2012 Wikimedia Foundation
+ :license: GPL, version 2 or later
+"""
+from __future__ import division
+from datetime import timedelta
+import signal
+import sys
+import threading
+
+
+# pylint: disable=C0111, W0212, W0613, W0621
+
+
+__all__ = ('every', )
+
+
+def total_seconds(delta):
+ """
+ Get total seconds of timedelta object. Equivalent to
+ timedelta.total_seconds(), which was introduced in Python 2.7.
+ """
+ us = (delta.microseconds + (delta.seconds + delta.days * 24 * 3600) * 10**6)
+ return us / 1000000.0
+
+
+def handle_sigint(signal, frame):
+ """
+ Attempt to kill all child threads and exit. Installing this as a sigint
+ handler allows the program to run indefinitely if unmolested, but still
+ terminate gracefully on Ctrl-C.
+ """
+ for thread in threading.enumerate():
+ if thread.isAlive():
+ thread._Thread__stop()
+ sys.exit(0)
+
+
+def every(*args, **kwargs):
+ """
+ Decorator; calls decorated function on a set interval. Arguments to every()
+ are passed on to the constructor of datetime.timedelta(), which accepts the
+ following arguments: days, seconds, microseconds, milliseconds, minutes,
+ hours, weeks. This decorator is intended for functions with side effects;
+ the return value is discarded.
+ """
+ interval = total_seconds(timedelta(*args, **kwargs))
+ def decorator(func):
+ def poll():
+ func()
+ threading.Timer(interval, poll).start()
+ poll()
+ return func
+ return decorator
+
+
+def join():
+ """Pause until sigint"""
+ signal.signal(signal.SIGINT, handle_sigint)
+ signal.pause()
+
+
+every.join = join
@@ -0,0 +1,137 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+ Python Gmond Module for Memcached
+
+ This module declares a "memcached" collection group. For more information,
+ including installation instructions, see:
+
+ http://sourceforge.net/apps/trac/ganglia/wiki/ganglia_gmond_python_modules
+
+ When invoked as a standalone script, this module will attempt to use the
+ default configuration to query memcached every 10 seconds and print out the
+ results.
+
+ Based on a suggestion from Domas Mitzuas, this module also reports the min,
+ max, median and mean of the 'age' metric across slabs, as reported by the
+ "stats items" memcached command.
+
+ :copyright: (c) 2012 Wikimedia Foundation
+ :author: Ori Livneh <ori@wikimedia.org>
+ :license: GPL, v2 or later
+"""
+from __future__ import division, print_function
+
+from threading import Timer
+
+import logging
+import os
+import pprint
+import sys
+import telnetlib
+
+logging.basicConfig(level=logging.DEBUG)
+
+# Hack: load a file from the current module's directory, because gmond doesn't
+# know how to work with Python packages. (To be fair, neither does Python.)
+sys.path.insert(0, os.path.dirname(__file__))
+from memcached_metrics import descriptors
+from every import every
+sys.path.pop(0)
+
+
+# Default configuration
+config = {
+ 'host' : '127.0.0.1',
+ 'port' : 11211,
+}
+
+stats = {}
+client = telnetlib.Telnet()
+
+
+def median(values):
+ """Calculate median of series"""
+ values = sorted(values)
+ length = len(values)
+ mid = length // 2
+ if (length % 2):
+ return values[mid]
+ else:
+ return (values[mid - 1] + values[mid]) / 2
+
+
+def mean(values):
+ """Calculate mean (average) of series"""
+ return sum(values) / len(values)
+
+
+def cast(value):
+ """Cast value to float or int, if possible"""
+ try:
+ return float(value) if '.' in value else int(value)
+ except ValueError:
+ return value
+
+
+def query(command):
+ """Send `command` to memcached and stream response"""
+ client.write(command.encode('ascii') + b'\n')
+ while True:
+ line = client.read_until(b'\r\n').decode('ascii').strip()
+ if not line or line == 'END':
+ break
+ (_, metric, value) = line.split(None, 2)
+ yield metric, cast(value)
+
+
+@every(seconds=10)
+def update_stats():
+ """Refresh stats by polling memcached server"""
+ try:
+ client.open(**config)
+ stats.update(query('stats'))
+ ages = [v for k, v in query('stats items') if k.endswith('age')]
+ if not ages:
+ return {'age_min': 0, 'age_max': 0, 'age_mean': 0, 'age_median': 0}
+ stats.update({
+ 'age_min' : min(ages),
+ 'age_max' : max(ages),
+ 'age_mean' : mean(ages),
+ 'age_median' : median(ages)
+ })
+ finally:
+ client.close()
+ logging.info("Updated stats: %s", pprint.pformat(stats, indent=4))
+
+
+#
+# Gmond Interface
+#
+
+def metric_handler(name):
+ """Get the value for a particular metric; part of Gmond interface"""
+ return stats[name]
+
+
+def metric_init(params):
+ """Initialize; part of Gmond interface"""
+ print('[memcached] memcached stats')
+ config.update(params)
+ for metric in descriptors:
+ metric['call_back'] = metric_handler
+ return descriptors
+
+
+def metric_cleanup():
+ """Teardown; part of Gmond interface"""
+ client.close()
+
+
+if __name__ == '__main__':
+ # When invoked as standalone script, run a self-test by querying each
+ # metric descriptor and printing it out.
+ for metric in metric_init({}):
+ value = metric['call_back'](metric['name'])
+ print(( "%s => " + metric['format'] ) % ( metric['name'], value ))
+ every.join()
Oops, something went wrong.

0 comments on commit 81ad2ef

Please sign in to comment.