Permalink
Browse files

adding updates for whatrunswhere stuff, better timeperiods support, n…

…ew intervals formats, more stable graphs, more wsgi methods
  • Loading branch information...
linked committed Jun 5, 2012
1 parent 3ddb733 commit c086872ddb51188348ddbd936a5b6cd86cb373c7
Showing with 171 additions and 63 deletions.
  1. +5 −5 demo/index.html
  2. +113 −37 hailwhale/periods.py
  3. +3 −3 hailwhale/test.py
  4. +16 −13 hailwhale/whale.py
  5. +24 −2 hailwhale/wsgi.py
  6. +10 −3 js/hailwhale.js
View
@@ -27,16 +27,16 @@
$(function() {
debug = true;
hw.add_graph($('#countOne .plot'), {pk: 'countOne', autoupdate: debug,
- period:'10x300'});
+ period:'monthly'});
hw.add_graph($('#countManyNums .plot'), {pk: 'countManyNums', depth: 1,
- autoupdate: debug, period: '10x300'});
+ autoupdate: debug, period: 'monthly'});
hw.add_graph($('#countManyNums .areaplot'), {pk: 'countManyNums', depth: 1,
- autoupdate: debug, period: '10x300', area: true});
+ autoupdate: debug, period: 'monthly', area: true});
hw.add_graph($('#countManyNums .wiggleplot'), {pk: 'countManyNums', depth: 1,
- autoupdate: debug, period: '10x300', area: 'silhouette'});
+ autoupdate: debug, period: 'monthly', area: 'silhouette'});
hw.add_graph($('#manyMetrics .plot'), {pk: 'system_status',
metrics: { 'CPU percent':'hits', 'RAM (GB)':'hits' },
- period: '10x300',
+ period: 'monthly',
interval: 10000,
autoupdate: debug});
});
View
@@ -1,29 +1,79 @@
from datetime import datetime, timedelta, date
import time
import times
+import re
PERIODS = [
-{'name': 'Last year, by 14 days',
- 'length': 3600 * 24 * 365,
- 'interval': 3600 * 24 * 14,
+{'name': 'Last 3 years, by month',
+ 'length': '3y',
+ 'interval': '1mo',
+ 'nickname': 'monthly'},
+{'name': 'Last year, by week',
+ 'length': '1y',
+ 'interval': '1w',
'nickname': 'year'},
-{'name': 'Last 30 days, by day', 'length': 3600 * 24 * 30, 'interval': 3600 * 24,
+{'name': 'Last 30 days, by day', 'length': '1mo', 'interval': '1d',
'nickname': 'thirty'},
-{'name': 'Last week, by 6 hours', 'length': 3600 * 24 * 7, 'interval': 3600 * 6,
+{'name': 'Last week, by 6 hours', 'length': '1w', 'interval': '6h',
'nickname': 'seven'},
-{'name': 'Last day, by hour', 'length': 3600 * 24, 'interval': 3600,
+{'name': 'Last day, by hour', 'length': '1d', 'interval': '1h',
'nickname': '24h'},
-{'name': 'Last 6 hours, by 15 minutes', 'length': 3600 * 6, 'interval': 60 * 15},
-{'name': 'Last hour, by 2 minutes', 'length': 3600, 'interval': 60 * 2,
+{'name': 'Last hour, by 1 minutes', 'length': '1h', 'interval': '1m',
'nickname': 'hour'},
-{'name': 'Last 5 minutes, by 10 seconds', 'length': 300, 'interval': 10,
- 'nickname': 'fivemin'},
+{'name': 'Last 5 minutes, by 10 seconds', 'length': '5m', 'interval': '10s',
+ 'nickname': 'fivemin'}
]
+
+UnitMultipliers = {
+ 'seconds' : 1,
+ 'minutes' : 60,
+ 'hours' : 3600,
+ 'days' : 86400,
+ 'weeks' : 86400 * 7,
+ 'months' : 86400 * 31,
+ 'years' : 86400 * 365
+}
+
+
+def getUnitString(s):
+ if 'seconds'.startswith(s): return 'seconds'
+ if 'minutes'.startswith(s): return 'minutes'
+ if 'hours'.startswith(s): return 'hours'
+ if 'days'.startswith(s): return 'days'
+ if 'weeks'.startswith(s): return 'weeks'
+ if 'months'.startswith(s): return 'months'
+ if 'years'.startswith(s): return 'years'
+ raise ValueError("Invalid unit '%s'" % s)
+
+def parseUnit(unit):
+ if str(unit).isdigit():
+ return int(unit) * UnitMultipliers[getUnitString('s')]
+ unit_re = re.compile(r'^(\d+)([a-z]+)$')
+ match = unit_re.match(str(unit))
+ if match:
+ unit = int(match.group(1)) * UnitMultipliers[getUnitString(match.group(2))]
+ else:
+ raise ValueError("Invalid unit specification '%s'" % unit)
+ return unit
+
+def parseRetentionDef(retentionDef):
+ (precision, points) = retentionDef.strip().split(':')
+ precision = parseUnit(precision)
+
+ if points.isdigit():
+ points = int(points)
+ else:
+ points = parseUnit(points) / precision
+
+ return (precision, points)
+
class Period(object):
def __init__(self, interval, length, name=False, nickname=False):
- self.interval = int(interval)
- self.length = int(length)
+ self.interval = str(interval)
+ self.length = str(length)
self.name = name
self.nickname = nickname
+ def getUnits(self):
+ return parseUnit(self.interval), parseUnit(self.length)
@classmethod
def get_days(cls, period, at=None, tzoffset=None):
ats = False
@@ -87,28 +137,30 @@ def get_days(cls, period, at=None, tzoffset=None):
return period, list(ats), tzoffset
def start(self):
+ interval, length = self.getUnits()
dt= (times.now() -
- timedelta(seconds=self.length))
- if self.interval < 60:
- interval_seconds = self.interval
+ timedelta(seconds=length))
+ if interval < 60:
+ interval_seconds = interval
else: interval_seconds = 60
- if self.interval < 3600:
- interval_minutes = (self.interval - interval_seconds)/60
+ if interval < 3600:
+ interval_minutes = (interval - interval_seconds)/60
else: interval_minutes = 60
- if self.interval < 3600*24:
- interval_hours = (self.interval - interval_seconds -
+ if interval < 3600*24:
+ interval_hours = (interval - interval_seconds -
(60*interval_minutes))/3600
else:
interval_hours = 24
if interval_hours == 0: interval_hours = 1
if interval_minutes == 0: interval_minutes = 1
- return dt.replace(
+ new_start = dt.replace(
microsecond = 0,
second = (dt.second - dt.second%interval_seconds),
minute = (dt.minute - dt.minute%interval_minutes),
hour = (dt.hour - dt.hour%interval_hours),)
- def delta(self):
- return timedelta(seconds=self.interval)
+ if interval >= (3600*24*30):
+ new_start = new_start.replace(day=1)
+ return new_start
@staticmethod
def format_dt_str(t):
return t.strftime('%a %b %d %H:%M:%S %Y')
@@ -120,11 +172,33 @@ def parse_dt_str(t):
return None
def datetimes(self, start=False, end=False, tzoffset=None):
+ from dateutil import rrule
from util import datetimeIterator
in_range = lambda dt: (not start or start <= dt) and (
not end or end >= dt)
- return (dt for dt in datetimeIterator(
- start or self.start(), end or convert(times.now(), tzoffset), delta=self.delta()) if in_range(dt))
+ use_start = start or self.start()
+ use_end = end or convert(times.now(), tzoffset)
+ interval, length = self.getUnits()
+ if interval >= 3600*24*30:
+ rule = rrule.MONTHLY
+ step = interval / (3600*24*30)
+ elif interval >= 3600*24*7:
+ rule = rrule.WEEKLY
+ step = interval / (3600*24*7)
+ elif interval >= 3600*24:
+ rule = rrule.DAILY
+ step = interval / (3600*24)
+ elif interval >= 3600:
+ rule = rrule.HOURLY
+ step = interval / 3600
+ elif interval >= 60:
+ rule = rrule.MINUTELY
+ step = interval / 60
+ else:
+ rule = rrule.SECONDLY
+ step = interval
+ dts = rrule.rrule(rule, dtstart=use_start, until=use_end, interval=step)
+ return dts
def datetimes_strs(self, start=False, end=False, tzoffset=None):
return (Period.format_dt_str(dt) for dt in
@@ -135,14 +209,8 @@ def flatten(self, dtf=None):
dtf = times.now()
if type(dtf) in (str, unicode):
dtf = self.parse_dt_str(dtf)
- if not dtf:
- return False
- diff_delta = dtf - self.start()
- diff = diff_delta.seconds + (diff_delta.days * 86400)
- if diff < 0:
- return False
- p = int(diff / self.interval)
- flat = (self.start() + timedelta(seconds=p * self.interval)).replace(microsecond=0)
+ dts = list(self.datetimes(end=dtf))
+ flat = len(dts) and dts[-1] or False
return flat
def flatten_str(self, dtf):
@@ -152,18 +220,18 @@ def flatten_str(self, dtf):
return self.format_dt_str(f)
def __unicode__(self):
- return '%dx%d' % (self.interval, self.length)
+ return '%s:%s' % (self.interval, self.length)
def __str__(self):
- return '%dx%d' % (self.interval, self.length)
+ return '%s:%s' % (self.interval, self.length)
@staticmethod
def all_sizes():
return PERIOD_OBJS
@staticmethod
def all_sizes_dict():
- return dict(map(lambda p: ('%sx%s' % (p.interval, p.length), p),
+ return dict(map(lambda p: ('%s:%s' % (p.interval, p.length), p),
Period.all_sizes()))
@staticmethod
@@ -174,7 +242,12 @@ def get(name=None):
return PERIOD_NICKS[str(name)]
if not name or name == 'None':
name = Period.default_size()
- return Period.all_sizes_dict()[str(name)]
+ if str(name) in Period.all_sizes_dict():
+ return Period.all_sizes_dict()[str(name)]
+ try:
+ return PERIOD_INTERVALS[parseUnit(name)]
+ except:
+ raise KeyError(name)
@staticmethod
def default_size():
@@ -184,16 +257,19 @@ def convert(tz, tzo):
return convert(tz, tzo)
def friendly_name(self):
- return self.name if self.name else '%sx%s' % (
+ return self.name if self.name else '%s:%s' % (
self.interval, self.length)
PERIOD_OBJS = []
PERIOD_NICKS = {}
+PERIOD_INTERVALS = {}
for p in PERIODS:
period = Period(p['interval'], p['length'], p['name'], p.get('nickname', None))
PERIOD_OBJS.append(period)
+ PERIOD_INTERVALS[parseUnit(p['interval'])] = period
if 'nickname' in p:
PERIOD_NICKS[p['nickname']] = period
+ PERIOD_NICKS[p['interval']] = period
DEFAULT_PERIODS = Period.all_sizes()
def convert(tzs, tzoffset=None):
if tzoffset == 'system':
View
@@ -46,15 +46,15 @@ def testResetCategory(self):
self.getStandardParamsURL('/reset')
def testCountingNowCorrectly(self):
- counting = lambda n: n['10x300']['empty']['counting_now']
+ counting = lambda n: n['fivemin']['empty']['counting_now']
totals = self.getTotalsURL(metrics=['counting_now'])
for i in range(3):
self.assertEqual(self.getCountNowURL(metrics={'counting_now': 5}), 'OK')
new_totals = self.getTotalsURL(metrics=['counting_now'])
self.assertEqual(counting(new_totals), counting(totals) + 15)
def testCountingCorrectly(self):
- counting = lambda n: n['10x300']['empty']['counting']
+ counting = lambda n: n['fivemin']['empty']['counting']
totals = self.getTotalsURL(metrics=['counting'])
for i in range(3):
self.assertEqual(self.getCountURL(metrics={'counting': 5}), 'OK')
@@ -282,7 +282,7 @@ def testWhaleCacheWrapper(self):
t = str(time.time())
count = lambda: self.whale.count_now('test_cached', t)
cached_sum = lambda clear=False: sum(self.whale.cached_plotpoints('test_cached',
- t, period='10x300', unmemoize=clear)[t]['hits'].values())
+ t, period='fivemin', unmemoize=clear)[t]['hits'].values())
# Set hits to 1
count()
View
@@ -60,7 +60,7 @@ def _increment(*args, **kwargs):
def _store(redis, pk, dimension, metric, period, dt, count, method='set',
rank=False):
# Keep a list of graphs per pk
- key = keyify(pk, dimension, str(period), metric)
+ key = keyify(pk, dimension, Period.get(period).interval, metric)
# Store pk dimensions
dimension_key = keyify('dimensions', pk)
dimension_json = keyify(dimension)
@@ -86,15 +86,16 @@ def _store(redis, pk, dimension, metric, period, dt, count, method='set',
else:
tgt_pk = pk
tgt_dimension = parent(dimension)
- rank_key = keyify('rank', tgt_pk, tgt_dimension, str(period), dt, metric)
+ rank_key = keyify('rank', tgt_pk, tgt_dimension,
+ Period.get(period).interval, dt, metric)
redis.zadd(rank_key, dimension_json, new_val)
return new_val
def _ranked(redis, pk, parent_dimension, metric, period, ats, start=0, size=10,
sort_dir=None):
top, bot = parse_formula(metric)
- rank_keyify = lambda ats, met: keyify('rank', pk, parent_dimension, str(period),
- ats, met)
+ rank_keyify = lambda ats, met: keyify('rank', pk, parent_dimension,
+ Period.get(period).interval, ats, met)
final_rank_key = rank_keyify(ats, metric)
def squash_ats(met):
if len(ats) > 1:
@@ -120,10 +121,10 @@ def squash_ats(met):
def _retrieve(redis, pk, dimensions, metrics, period=None, dt=None):
nested = defaultdict(dict)
- period = str(Period.get(period))
+ interval = Period.get(period).interval
for dimension in iterate_dimensions(dimensions)+['_']:
for metric in metrics:
- hash_key = keyify(pk, dimension, period, metric)
+ hash_key = keyify(pk, dimension, interval, metric)
value_dict = redis.hgetall(hash_key)
nested[maybe_dumps(dimension)][maybe_dumps(metric)] = dict([
(k, float(v)) for k, v in value_dict.items()])
@@ -393,7 +394,7 @@ def totals(cls, pk, dimensions=None, metrics=None, periods=None):
metrics += metric.split('/')
d = {}
for p in periods:
- p_data = cls.plotpoints(pk, dimensions, metrics, period=str(p))
+ p_data = cls.plotpoints(pk, dimensions, metrics, period=p)
p_totals = dict()
for dim in p_data.keys():
p_totals[dim] = dict()
@@ -533,9 +534,9 @@ def zranked(cls, pk, parent_dimension='_', metric='hits', period=None,
@classmethod
def rank_subdimensions_scalar(cls, pk, dimension='_', metric='hits',
period=None, recursive=True, prune_parents=True, points=False):
- period = period or Period.default_size()
+ period = Period.get(period) or Period.default_size()
d_k = keyify(dimension)
- total = cls.cached_totals(pk, dimension, metric, periods=[period])[period][d_k][metric]
+ total = cls.cached_totals(pk, dimension, metric, periods=[period])[period.interval][d_k][metric]
ranked = dict()
def info(sub):
@@ -571,10 +572,12 @@ def info(sub):
def rank_subdimensions_ratio(cls, pk, numerator, denominator='hits',
dimension='_', period=None, recursive=True, points=False):
top, bottom = numerator, denominator
- period = period or Period.default_size()
+ period = Period.get(period) or Period.default_size()
d_k = keyify(dimension)
- top_total = cls.cached_totals(pk, dimension, top, periods=[period])[str(period)][d_k][top]
- bottom_total = cls.cached_totals(pk, dimension, bottom, periods=[period])[str(period)][d_k][bottom]
+ top_total = cls.cached_totals(pk, dimension, top,
+ periods=[period])[period.interval][d_k][top]
+ bottom_total = cls.cached_totals(pk, dimension, bottom,
+ periods=[period])[period.interval][d_k][bottom]
ratio_total = bottom_total and float(top_total / bottom_total) or 0
ranked = dict()
@@ -761,7 +764,7 @@ def generate_increments(metrics, periods=False, at=False):
if not dt:
continue
observations.add((period, dt))
- rr = [(str(period), dt, metric, incr_by)
+ rr = [(period, dt, metric, incr_by)
for (period, dt) in observations
for metric, incr_by in metrics.items()]
return rr
Oops, something went wrong.

0 comments on commit c086872

Please sign in to comment.