Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

adding update_count_to and cleaning up some core logic; some base fun…

…ctionality is now 2-3x as fast
  • Loading branch information...
commit 7167f09a2f5434f6f19343d9535f8ab02889c6c8 1 parent 4bc947d
Leeward Bound authored
Showing with 46 additions and 32 deletions.
  1. +17 −17 hailwhale/test.py
  2. +1 −1  hailwhale/util.py
  3. +28 −14 hailwhale/whale.py
34 hailwhale/test.py
View
@@ -117,8 +117,8 @@ def testPlotpointsDepth(self):
plotpoints = self.whale.plotpoints('test_depth', t, points_type=list,
depth=1, limit=2)
self.assertEqual(plotpoints[maybe_dumps([t, 'b'])]['hits'][-1][1], 2)
- self.assertNotIn(maybe_dumps([t, 'a']), plotpoints)
- self.assertNotIn(maybe_dumps([t, 'c']), plotpoints)
+ self.assertEqual(True, maybe_dumps([t, 'a']) not in plotpoints)
+ self.assertEqual(True, maybe_dumps([t, 'c']) not in plotpoints)
def testRatioPlotpoints(self):
t = str(time.time())
@@ -167,7 +167,7 @@ def testRankSubdimensionsRatio(self):
t, recursive=False)
all_levels = self.whale.rank_subdimensions_ratio(pk, 'value', 'visitors', t)
- self.assertNotIn(maybe_dumps([t, 'a', 'asub1']), one_level)
+ self.assertEqual(True, maybe_dumps([t, 'a', 'asub1']) not in one_level)
self.assertEqual(all_levels[maybe_dumps([t, 'a', 'asub1'])]['important'], False)
self.assertEqual(all_levels[maybe_dumps([t, 'a', 'asub2'])]['important'], True)
self.assertEqual(all_levels[maybe_dumps([t, 'b'])]['important'], True)
@@ -180,7 +180,7 @@ def testBasicDecision(self):
good, bad, test = self.whale.weighted_reasons(pk, 'random', [1,2,3])
#_print_reasons(good, bad, test)
any_one = self.whale.decide_from_reasons(good, bad, test)
- self.assertIn(any_one, [1, 2, 3])
+ self.assertEqual(True, any_one in [1, 2, 3])
# OK, now how about something somewhat informed?
# This will be easy. Slogan A makes us huge profit. Products B and C suck.
@@ -194,10 +194,10 @@ def testBasicDecision(self):
good, bad, test = self.whale.weighted_reasons(pk, decision, opts, formula='dollars/visitors')
#_print_reasons(good, bad, test)
- self.assertIn('a', good.keys())
- self.assertIn('b', bad.keys())
- self.assertIn('c', bad.keys())
- self.assertIn('d', test.keys())
+ self.assertEqual(True, 'a' in good.keys())
+ self.assertEqual(True, 'b' in bad.keys())
+ self.assertEqual(True, 'c' in bad.keys())
+ self.assertEqual(True, 'd' in test.keys())
which_one = self.whale.decide(pk, decision, opts, formula='dollars/visitors',
bad_idea_threshold=0, test_idea_threshold=0)
self.assertEqual(which_one, 'a')
@@ -217,19 +217,19 @@ def testInformedDecision(self):
# Here's a visitor with no info -- 'A' should win by far.
good, bad, test = self.whale.weighted_reasons(pk, decision, opts, formula='dollars/visitors')
#_print_reasons(good, bad, test)
- self.assertIn('a', good.keys())
- self.assertIn('b', bad.keys())
- self.assertIn('c', bad.keys())
- self.assertIn('d', test.keys())
+ self.assertEqual(True, 'a' in good.keys())
+ self.assertEqual(True, 'b' in bad.keys())
+ self.assertEqual(True, 'c' in bad.keys())
+ self.assertEqual(True, 'd' in test.keys())
# How about when we know the country is "UK"?
good, bad, test = self.whale.weighted_reasons(pk, decision, opts, formula='dollars/visitors',
known_data={'country': 'uk'})
#_print_reasons(good, bad, test)
- self.assertIn('a', good.keys())
- self.assertIn('b', good.keys())
- self.assertIn('c', bad.keys())
- self.assertIn('d', test.keys())
+ self.assertEqual(True, 'a' in good.keys())
+ self.assertEqual(True, 'b' in good.keys())
+ self.assertEqual(True, 'c' in bad.keys())
+ self.assertEqual(True, 'd' in test.keys())
chosen = {'a': 0, 'b': 0}
for k in range(100):
choose = self.whale.decide(pk, decision, opts, formula='dollars/visitors',
@@ -275,7 +275,7 @@ def justify(geo):
count('br', 'pt', 500 * k, 50 * k) # $10 JACKPOT
self.assertEqual('en', justify('us'))
- self.assertIn(justify('mx'), ['sp', 'en'])
+ self.assertEqual(True, justify('mx') in ['sp', 'en'])
self.assertEqual('pt', justify('br'))
def testWhaleCacheWrapper(self):
2  hailwhale/util.py
View
@@ -23,7 +23,7 @@ def maybe_dumps(arg, dump_dicts=True):
if isinstance(arg, basestring):
arg = try_loads(arg)
if isinstance(arg, basestring):
- return str(arg)
+ return unicode(arg)
if isinstance(arg, list):
if len(arg) == 1:
return maybe_dumps(arg[0])
42 hailwhale/whale.py
View
@@ -52,22 +52,37 @@
_added_dimensions = collections.defaultdict(list)
_added_subdimensions = collections.defaultdict(list)
-def _increment(redis, pk, dimension, metric, period, dt, count):
+
+def _increment(*args, **kwargs):
+ kwargs['method'] = 'incr'
+ return _store(*args, **kwargs)
+
+def _store(redis, pk, dimension, metric, period, dt, count, method='set',
+ rank=False):
# Keep a list of graphs per pk
key = keyify(pk, dimension, str(period), metric)
# Store pk dimensions
- dimension_key = keyify(pk, 'dimensions')
+ dimension_key = keyify('dimensions', pk)
dimension_json = keyify(dimension)
if not dimension_json in _added_dimensions[dimension_key]:
redis.sadd(dimension_key, dimension_json)
_added_dimensions[dimension_key].append(dimension_json)
# Store dimensional subdimensions
if dimension != '_':
- subdimension_key = keyify(pk, 'subdimensions', parent(dimension))
+ subdimension_key = keyify('subdimensions', pk, parent(dimension))
if not dimension_json in _added_subdimensions[subdimension_key]:
redis.sadd(subdimension_key, dimension_json)
_added_subdimensions[subdimension_key].append(dimension_json)
- return redis.hincrby(key, dt, int(count))
+
+ if method == 'set':
+ new_val = float(count)
+ redis.hset(key, dt, new_val)
+ elif method == 'incr':
+ new_val = redis.execute_command('HINCRBYFLOAT', key, dt, float(count))
+ if rank and dimension != '_':
+ rank_key = keyify('rank', pk, parent(dimension), str(period), metric)
+ redis.zadd(rank_key, dimension_json, new_val)
+ return new_val
def _retrieve(redis, pk, dimensions, metrics, period=None, dt=None):
nested = defaultdict(dict)
@@ -89,7 +104,7 @@ def curry_whale_instance_methods(self, attr='id'):
for method in ['plotpoints', 'ratio_plotpoints', 'scalar_plotpoints',
'totals', 'count_now', 'count_decided_now', 'decide',
'weighted_reasons', 'reasons_for', 'graph_tag', 'today',
- 'count_up_to', 'total']:
+ 'update_count_to', 'total']:
curry_instance_attribute(attr, method, self,
with_class_name=True)
# Currying for related models as
@@ -382,7 +397,7 @@ def get_subdimensions(cls, pk, dimension='_'):
dimension = '_'
if dimension != '_' and not isinstance(dimension, list):
dimension = [dimension]
- set_members = cls.whale_driver().smembers(keyify(pk, 'subdimensions',
+ set_members = cls.whale_driver().smembers(keyify('subdimensions', pk,
dimension))
subdimensions = []
for s in set_members:
@@ -433,23 +448,22 @@ def count_now(cls, pk, dimensions='_', metrics=None, at=False):
iterate_dimensions(pk),
iterate_dimensions(dimensions, add_root=True),
generate_increments(metrics, periods, at)):
+ if i == 0:
+ continue
_increment(cls.whale_driver(), pkk, dimension, metric, period, dt, i)
@classmethod
- def count_up_to(cls, pk, dimensions='_', metrics=None, period=False,
- at=False):
- """ Immediately count a hit, as opposed to logging it into Hail"""
+ def update_count_to(cls, pk, dimensions='_', metrics=None, period=False,
+ at=False, rank=False):
period = Period.get(period)
+ at = at or times.now()
dt = period.flatten_str(at)
-
for pkk, dimension, (metric, i) in itertools.product(
iterate_dimensions(pk),
iterate_dimensions(dimensions, add_root=True),
metrics.iteritems()):
- n = float(i) - cls.total(pk, metric, dimension, period, at)
- if not n > 0:
- continue
- _increment(cls.whale_driver(), pkk, dimension, metric, period, dt, n)
+ _store(cls.whale_driver(), pkk, dimension, metric, period, dt, i,
+ rank=rank)
@classmethod
Please sign in to comment.
Something went wrong with that request. Please try again.