initial commit of some code, much of it from django-slow-log, some fr…

…om previously private code @ hiidef/flavorsme (with blessing)

kokuen/django/middleware.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""Kokuen django middleware(s)."""
+
+from kokuen.django import settings
+
+# TODO: how should we do this?  registration of prefabs?
+
+class KokuenMiddleware(object):
+    pass
+

kokuen/django/settings.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""Kokuen -> Django settings."""
+
+from django.conf import settings
+
+# TODO: i don't like this, should be module paths to prefab filters
+
+defaults = {
+    'ENABLE_TIMER' : True,
+    'ENABLE_COUNTER' : True,
+    'ENABLE_MEMORY' : True,
+    'ENABLE_LOAD': True,
+}
+
+KOKUEN_SETTINGS = getattr(settings, 'KOKUEN_SETTINGS', defaults)
+

kokuen/stats/counter.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""Simple counter class."""
+
+class Counter(object):
+    """Lightweight counter/accumulator."""
+    def __init__(self):
+        self.clear()
+        super(Timer, self).__init__()
+
+    def clear(self):
+        self.counters = {}
+        self.disabled = False
+
+    def increment(self, name):
+        if name in self.accumuators:
+            self.accumuators[name] += 1
+            return
+        self.counters[name] = 1
+
+    def decrement(self, name):
+        if name in self.counters:
+            self.counters[name] -=1
+            return
+        self.counters[name] -=1
+
+    def json(self):
+        return json.dumps(self.counters)
+
+    def __repr__(self):
+        return '<Counter: %r>' % (self.counters)
+
+# process-global counter
+counter = Counter()

kokuen/stats/loadavg.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""Loadavg stats gathering."""
+
+import re
+from subprocess import Popen, PIPE
+
+
+class LoadAverage(object):
+    """Fetch the current load average.  Uses /proc/loadavg in linux, falls back
+    to executing the `uptime` command, which is 240x slower than reading
+    from proc."""
+    matcher = re.compile("load average:\s*([.\d]+),\s*([.\d]+),\s*([.\d]+)")
+    uptime_fallback = False
+
+    def __init__(self):
+        uptime_fallback = not os.path.exists('/proc/loadavg')
+
+    def current(self):
+        """Returns 3 floats, (1 min, 5 min, 15 min) load averages like
+        the datetime command."""
+        if self.uptime_fallback:
+            return self.uptime_fallback_load()
+        return self.proc_load()
+
+    def proc_load(self):
+        try:
+            with open('/proc/loadavg') as f:
+                content = f.read()
+            return [float(c) for c in content.split()[:3]]
+        except:
+            return self.uptime_fallback_load()
+
+    def uptime_fallback_load(self):
+        p = Popen(['uptime'], stdout=PIPE)
+        output = p.stdout.read()
+        return map(float, self.matcher.search(output).groups())
+
+

kokuen/stats/memory.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""Memory stats gathering."""
+
+import re
+from subprocess import Popen, PIPE
+from kokuen.utils import to_bytes
+
+class MemoryStatus(object):
+    """Fetch the memory usage for a given PID.  Note that this is designed
+    mostly to read the current processes memory size;  it won't work well on
+    non-linux machines when trying to find the mem usage of a process not owned
+    by the current user.  Reading from proc is almost 600x faster than using
+    the ps fallback."""
+    matcher = re.compile('VmSize:\s*(\d+\s*\w+)')
+    ps_fallback = False
+
+    def __init__(self, pid):
+        self.pid = int(pid)
+        self.procpath = '/proc/%s/status' % pid
+        if not os.path.exists(self.procpath):
+            self.ps_fallback = True
+
+    def usage(self):
+        if self.ps_fallback:
+            return self.ps_fallback_usage()
+        return self.proc_usage()
+
+    def proc_usage(self):
+        """Memory usage for given PID."""
+        try:
+            with open(self.procpath) as f:
+                content = f.read()
+            size = self.matcher.search(content).groups()[0]
+            return to_bytes(size)
+        except:
+            return self.ps_fallback_usage()
+
+    def ps_fallback_usage(self):
+        """Memory usage for the given PID using ps instead of proc."""
+        p = Popen(['ps', 'u', '-p', str(self.pid)], stdout=PIPE)
+        output = p.stdout.read().split('\n')
+        output = filter(None, output)
+        process_line = output[-1].split()
+        vsize_in_kb = process_line[5] + ' kB'
+        return to_bytes(vsize_in_kb)
+

kokuen/stats/timer.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""Timing helpers."""
+
+import time
+
+try:
+    import simplejson as json
+except ImportError:
+    import json
+
+class Timer(threading.local):
+    """Lightweight timing class.  Stores both absolute time of given events and
+    accumulated times of one or more arbitrary start/stop pairs."""
+    def __init__(self):
+        self.clear()
+        super(Timer, self).__init__()
+
+    def clear(self):
+        self.checkpoints = []
+        self.accumulators = {}
+        self.disabled = False
+
+    def disable(self):
+        self.disabled = True
+
+    def checkpoint(self, name):
+        if self.disabled: return
+        t = time.time()
+        self.checkpoints.append((name, t))
+
+    def start(self, name):
+        if self.disabled: return
+        t0 = time.time()
+        if name in self.accumulators and self.accumulators[name].get('t0', None):
+            # XXX: we tried to start an accumulator twice without ending it..
+            # that's bad, we shouldn't allow that
+            return
+        self.accumulators.setdefault(name, {'dt': 0})['t0'] = t0
+
+    def stop(self, name):
+        if self.disabled: return
+        t1 = time.time()
+        if name not in self.accumulators or not self.accumulators[name].get('t0', 0):
+            # XXX: we are trying to stop an accumulator that doesn't exist or
+            # wasn't started, which we also should do something about
+            return
+        acc = self.accumulators[name]
+        if 'called' not in acc:
+            acc['called'] = 0
+        acc['called'] += 1
+        dt = t1 - acc['t0']
+        del acc['t0']
+        acc['dt'] += dt
+
+    def json(self):
+        acc = self.accumuators
+        return json.dumps({
+            'checkpoints' : self.checkpoints,
+            'accumulators' : dict([(key, (acc[key]['called'], acc[key]['dt'])) for key in acc]),
+        })
+
+    def as_header(self):
+        return base64.b64encode(self.json())
+
+    def __repr__(self):
+        return repr({'checkpoints' : self.checkpoints, 'accumulators' : self.accumulators})
+
+# process-global timer
+timer = Timer()
+

kokuen/utils.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""Utilities for kokuen."""
+
+def to_bytes(string):
+    """Converts a string with a human-readable byte size to a number of
+    bytes.  Takes strings like '7536 kB', in the format of proc."""
+    num, units = string.split()
+    num = int(num)
+    powers = {'kb': 10, 'mb': 20, 'gb': 30}
+    if units and units.lower() in powers:
+        num <<= powers[units.lower()]
+    return num
+
+def bytes_to_string(bytes):
+    """Converts number of bytes to a string.  Based on old code here:
+    Uses proc-like units (capital B, lowercase prefix).  This only takes a
+    few microseconds even for numbers in the terabytes.
+    """
+    units = ['B', 'kB', 'mB', 'gB', 'tB']
+    negate = bytes < 0
+    if negate: bytes = -bytes
+    factor = 0
+    while bytes/(1024.0**(factor+1)) >= 1:
+        factor += 1
+    return '%s%0.1f %s' % ('-' if negate else '', bytes/(1024.0**factor), units[factor])
+
+

setup.py

@@ -22,6 +22,8 @@
     # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers
     classifiers=[
         'Development Status :: 1 - Planning',
+        'License :: OSI Approved :: MIT License',
+        'Intended Audience :: Developers',
     ],
     keywords='django performance statsd graphite',
     author='Jason Moiron',