histogram, better test/example, percentile, median, mean and std dev …

…for histogram data

.gitignore

@@ -2,3 +2,4 @@
 build
 dist
 pymetrics.egg-info
+*.swp

README.md

@@ -3,44 +3,8 @@
 Redis backed metrics library - tries to implement the most of the famous Metrics library.
 
 # Examples:
-
-    import metrics
-    import time
-
-    rf = metrics.RetricsFactory("metrics_test")
-
-    # incremental/decr counter counter
-    counter = rf.new_counter("a_counter")
-    counter.reset()
-    for a in xrange(10):
-        counter.incr()
-    print counter.get_value() # must be 10
-
-    # a gauge, which indicate an absolute value
-    gauge = rf.new_gauge("a_gauge")
-    gauge.set(10)
-    print gauge.get()
-
-    # a meter vith 1, 5 and 15 mins avg
-    meters = rf.new_meter("a_meter")
-    # ... do something ...
-    meters.mark()
-    # do other things, etc, better check metrics_test.py for a working example
-    meters.get_value()
-
-    # returns last timestamp, 1 min, 5 min, 15 min
-
-    # a timer, which marks the begin and end of a task
-    timer = rf.new_timer("a_timer")
-    timer.start()
-    time.sleep(5)
-    timer.stop()
-    print timer.get_value()
+    check the tests/ directory    
 
 # TODO
-
-Needs to receive a Redis conn string, let the meter have the automatic timestamp
-Should use riemann optionally
-Meters needs a mean value
-Implement Histogram
+    Should use riemann optionally
 

pymetrics/__init__.py

@@ -1,6 +1,7 @@
 #encoding: utf-8
 import redis
 import time
+from pds_redis import Enum
 
 class BaseMetrics(object):
     def __init__(self, appname, name):
@@ -117,8 +118,45 @@ class RetricsHistogram(BaseMetrics):
     of data. In addition to minimum, maximum, mean, etc., it also measures
     median, 75th, 90th, 95th, 98th, 99th, and 99.9th percentiles.
     """
-    def update():
-        pass
+    def __init__(self, appname, name):
+        super(RetricsHistogram, self).__init__(appname, name)
+        self._list_name = "retrics:histogram:%s:%s" % (self._appname, self._name)
+        self._e = Enum(self._load_list())
+
+    def update(self, val):
+        assert(type(val) is int or type(val) is float)
+        self._redis.lpush(self._list_name, val)
+        self._e.reload(self._load_list())
+
+    def percentile(self, p):
+        """
+        Percentile of the Histogram list
+        """
+        assert(type(p) is float)
+        return self._e.percentile(p)
+
+    def standard_deviation(self):
+        """
+        standard deviation of the Histogram list
+        """
+        return self._e.standard_deviation()
+
+    def median(self):
+        """
+        Median (50% Percentile) of the Histogram list
+        """
+        return self._e.median()
+
+    def mean(self):
+        """
+        Mean of the Histogram list
+        """
+        return self._e.mean()
+
+    def _load_list(self):
+        l =  self._redis.lrange(self._list_name, 0, -1)
+        l = map(lambda x: float(x), l) #oh god why
+        return l
 
 class RetricsTimer(BaseMetrics):
     """

pymetrics/pds_redis.py

@@ -0,0 +1,43 @@
+# derived from https://github.com/gleicon/py_descriptive_statistics
+
+import math
+
+class Enum(object):
+    def __init__(self, l=[]):
+        self.l = l
+
+    def mean(self):
+        return self.sum()/self.number()
+
+    def median(self):
+        return self.percentile(50.0)
+
+    def number(self):
+        return float(len(self.l))
+
+    def percentile(self, p):
+        s = sorted(self.l)
+        p = float(p)
+        if p == 100.0: return float(s[-1])
+        rank = p / 100.0 * (self.number() - 1) 
+        lrank = int(math.floor(rank))
+        d = rank - lrank
+        lower = float(s[lrank])
+        upper = float(s[lrank + 1])
+        r = lower + (upper - lower) * d
+        return r
+
+    def standard_deviation(self):
+        return math.sqrt(self.variance())
+
+    def sum(self):
+        return sum(self.l)
+
+    def variance(self):
+        m = self.mean()
+        p1 = map(lambda x: (m - x) ** 2 , self.l)
+        r = sum(p1) / self.number()
+        return r
+
+    def reload(self, l):
+        self.l = l

tests/metrics_test.py

@@ -1,13 +1,18 @@
-import metrics
+import sys, os
+
+sys.path.append("..") 
+sys.path.append(os.path.join(sys.path[0], '..'))
+
+import pymetrics
 import time, random
 
-rf = metrics.RetricsFactory("metrics_test")
+rf = pymetrics.RetricsFactory("metrics_test")
 
 def test_meter(rf):
     meters = rf.new_meter("a_meter")
     last_t = time.time()
-    c = 0
-    while(True):
+    c = j = 0
+    while(j < 10):
         c = c +1
         if c == random.random() % 10:
             c = 0
@@ -20,6 +25,7 @@ def test_meter(rf):
         print "avg 1 min: %s" % one_min
         print "avg 5 min: %s" % five_min
         print "avg 15 min: %s" % fifteen_min
+        j = j + 1
 
 counter = rf.new_counter("a_counter")
 counter.reset()
@@ -40,5 +46,19 @@ def test_meter(rf):
 print timer.get_value()
 assert(timer.get_value() == 5)
 
+
+histogram = rf.new_histogram("a_histogram")
+histogram.update(10)
+histogram.update(1)
+histogram.update(50)
+histogram.update(5)
+histogram.update(28)
+histogram.update(12)
+
+print histogram.percentile(99.0)
+print histogram.mean()
+print histogram.median()
+print histogram.standard_deviation()
+
 test_meter(rf)