A start to karma implementation with redis backend.

* Karma actions defined for answer, first answer, solution, helpful vote
* Behind 'karma' waffle switch
* /admin/karma page shows top contributors, allows user key lookup, and initializing karma.
* REDIS_TEST_BACKENDS settings used for tests
* SkipTest tests that depend on redis when the test backend(s) arent defined
* Added redis-test.conf

apps/karma/actions.py

@@ -0,0 +1,189 @@
+from datetime import date, datetime, timedelta
+
+from django.contrib.auth.models import User
+
+from celery.decorators import task
+import waffle
+
+from sumo.utils import redis_client
+
+
+KEY_PREFIX = 'karma'  # Prefix for the Redis keys used.
+
+
+class KarmaAction(object):
+    """Abstract base class for karma actions."""
+    action_type = None  # For example 'first-answer'.
+    points = 0  # Number of points the action is worth.
+
+    def __init__(self, user, day=date.today(), redis=None):
+        if not waffle.switch_is_active('karma'):
+            return
+        if isinstance(user, User):
+            self.userid = user.id
+        else:
+            self.userid = user
+        if isinstance(day, datetime):  # Gracefully handle a datetime.
+            self.date = day.date()
+        else:
+            self.date = day
+        if not redis:
+            self.redis = redis_client(name='karma')
+        else:
+            self.redis = redis
+
+    def save(self):
+        """Save the action information to redis."""
+        if waffle.switch_is_active('karma'):
+            self._save.delay(self)
+
+    @task
+    def _save(self):
+        key = hash_key(self.userid)
+
+        # Point counters:
+        # Increment total points
+        self.redis.hincrby(key, 'points:total', self.points)
+        # Increment points daily count
+        self.redis.hincrby(key, 'points:{d}'.format(
+            d=self.date), self.points)
+        # Increment points monthly count
+        self.redis.hincrby(key, 'points:{y}-{m:02d}'.format(
+            y=self.date.year, m=self.date.month), self.points)
+        # Increment points yearly count
+        self.redis.hincrby(key, 'points:{y}'.format(
+            y=self.date.year), self.points)
+
+        # Action counters:
+        # Increment action total count
+        self.redis.hincrby(key, '{t}:total'.format(t=self.action_type), 1)
+        # Increment action daily count
+        self.redis.hincrby(key, '{t}:{d}'.format(
+             t=self.action_type, d=self.date), 1)
+        # Increment action monthly count
+        self.redis.hincrby(key, '{t}:{y}-{m:02d}'.format(
+            t=self.action_type, y=self.date.year, m=self.date.month), 1)
+        # Increment action yearly count
+        self.redis.hincrby(key, '{t}:{y}'.format(
+            t=self.action_type, y=self.date.year), 1)
+
+
+# TODO: move this to it's own file?
+class KarmaManager(object):
+    """Manager for querying karma data in Redis."""
+    def __init__(self):
+        self.redis = redis_client(name='karma')
+
+    # Updaters:
+    def update_top_alltime(self):
+        """Updated the top contributors alltime sorted set."""
+        key = '{p}:points:total'.format(p=KEY_PREFIX)
+        # TODO: Maintain a user id list in Redis?
+        for userid in User.objects.values_list('id', flat=True):
+            pts = self.total_points(userid)
+            if pts:
+                self.redis.zadd(key, userid, pts)
+
+    def update_top_week(self):
+        """Updated the top contributors past week sorted set."""
+        key = '{p}:points:week'.format(p=KEY_PREFIX)
+        for userid in User.objects.values_list('id', flat=True):
+            pts = self.week_points(userid)
+            if pts:
+                self.redis.zadd(key, userid, pts)
+
+    # def update_trending...
+
+    # Getters:
+    def top_alltime(self, count=10):
+        """Returns the top users based on alltime points."""
+        return self._top_points(count, 'total')
+
+    def top_week(self, count=10):
+        """Returns the top users based on points in the last 7 days."""
+        return self._top_points(count, 'week')
+
+    def _top_points(self, count, suffix):
+        ids = self.redis.zrevrange('{p}:points:{s}'.format(
+            p=KEY_PREFIX, s=suffix), 0, count - 1)
+        users = list(User.objects.filter(id__in=ids))
+        users.sort(key=lambda user: ids.index(str(user.id)))
+        return users
+
+    def total_points(self, user):
+        """Returns the total points for a given user."""
+        count = self.redis.hget(hash_key(user), 'points:total')
+        return int(count) if count else 0
+
+    def week_points(self, user):
+        """Returns total points from the last 7 days for a given user."""
+        today = date.today()
+        days = [today - timedelta(days=d + 1) for d in range(7)]
+        counts = self.redis.hmget(hash_key(user),
+                                  ['points:{d}'.format(d=d) for d in days])
+        fn = lambda x: int(x) if x else 0
+        count = sum([fn(c) for c in counts])
+        return count
+
+    def daily_points(self, user, days_back=30):
+        """Returns a list of points from the past `days_back` days."""
+        today = date.today()
+        days = [today - timedelta(days=d) for d in range(days_back)]
+        counts = self.redis.hmget(hash_key(user),
+                                  ['points:{d}'.format(d=d) for d in days])
+        fn = lambda x: int(x) if x else 0
+        return [fn(c) for c in counts]
+
+    def monthly_points(self, user, months_back=12):
+        """Returns a list of points from the past `months_back` months."""
+        # TODO: Tricky?
+        pass
+
+    def total_count(self, action, user):
+        """Returns the total count of an action for a given user."""
+        count = self.redis.hget(
+            hash_key(user), '{t}:total'.format(t=action.action_type))
+        return int(count) if count else 0
+
+    def day_count(self, action, user, date=date.today()):
+        """Returns the total count of an action for a given user and day."""
+        count = self.redis.hget(
+            hash_key(user), '{t}:{d}'.format(d=date, t=action.action_type))
+        return int(count) if count else 0
+
+    def week_count(self, action, user):
+        """Returns total count of an action for a given user (last 7 days)."""
+        # TODO: DRY this up with week_points and daily_points.
+        today = date.today()
+        days = [today - timedelta(days=d + 1) for d in range(7)]
+        counts = self.redis.hmget(hash_key(user), ['{t}:{d}'.format(
+            t=action.action_type, d=d) for d in days])
+        fn = lambda x: int(x) if x else 0
+        count = sum([fn(c) for c in counts])
+        return count
+
+    def month_count(self, action, user, year, month):
+        """Returns the total count of an action for a given user and month."""
+        count = self.redis.hget(
+            hash_key(user),
+            '{t}:{y}-{m:02d}'.format(t=action.action_type, y=year, m=month))
+        return int(count) if count else 0
+
+    def year_count(self, action, user, year):
+        """Returns the total count of an action for a given user and year."""
+        count = self.redis.hget(
+            hash_key(user), '{t}:{y}'.format(y=year, t=action.action_type))
+        return int(count) if count else 0
+
+    def user_data(self, user):
+        """Returns all the data stored for the given user."""
+        return self.redis.hgetall(hash_key(user))
+
+
+def hash_key(user):
+    """Returns the hash key for a given user."""
+    if isinstance(user, User):
+        userid = user.id
+    else:
+        userid = user
+    return "{p}:{u}".format(p=KEY_PREFIX, u=userid)

apps/karma/admin.py

@@ -0,0 +1,71 @@
+from django.contrib import admin, messages
+from django.contrib.auth.models import User
+from django.http import HttpResponseRedirect
+from django.shortcuts import render_to_response
+from django.template import RequestContext
+
+from karma.actions import KarmaManager
+from karma.tasks import init_karma, update_top_contributors
+from questions.karma_actions import (AnswerAction, AnswerMarkedHelpfulAction,
+                                     FirstAnswerAction, SolutionAction)
+
+
+def karma(request):
+    """Admin view that displays karma related data."""
+    if request.POST.get('init'):
+        init_karma.delay()
+        messages.add_message(request, messages.SUCCESS,
+                             'init_karma task queued!')
+        return HttpResponseRedirect(request.path)
+
+    if request.POST.get('update-top'):
+        update_top_contributors.delay()
+        messages.add_message(request, messages.SUCCESS,
+                             'update_top_contributors task queued!')
+        return HttpResponseRedirect(request.path)
+
+    kmgr = KarmaManager()
+    top_alltime = [_user_karma_alltime(u, kmgr) for u in kmgr.top_alltime()]
+    top_week = [_user_karma_week(u, kmgr) for u in kmgr.top_week()]
+
+    username = request.GET.get('username')
+    user_karma = None
+    if username:
+        try:
+            user = User.objects.get(username=username)
+            d = kmgr.user_data(user)
+            user_karma = [{'key': k, 'value': d[k]} for k in sorted(d.keys())]
+        except User.DoesNotExist:
+            pass
+
+    return render_to_response('karma/admin/karma.html',
+                              {'title': 'Karma',
+                               'top_alltime': top_alltime,
+                               'top_week': top_week,
+                               'username': username,
+                               'user_karma': user_karma},
+                              RequestContext(request, {}))
+
+admin.site.register_view('karma', karma, 'Karma')
+
+
+def _user_karma_alltime(user, kmgr):
+    return {
+        'user': user,
+        'points': kmgr.total_points(user),
+        'answers': kmgr.total_count(AnswerAction, user),
+        'first_answers': kmgr.total_count(FirstAnswerAction, user),
+        'helpful_votes': kmgr.total_count(AnswerMarkedHelpfulAction, user),
+        'solutions': kmgr.total_count(SolutionAction, user),
+    }
+
+
+def _user_karma_week(user, kmgr):
+    return {
+        'user': user,
+        'points': kmgr.week_points(user),
+        'answers': kmgr.week_count(AnswerAction, user),
+        'first_answers': kmgr.week_count(FirstAnswerAction, user),
+        'helpful_votes': kmgr.week_count(AnswerMarkedHelpfulAction, user),
+        'solutions': kmgr.week_count(SolutionAction, user),
+    }

apps/karma/cron.py

@@ -0,0 +1,15 @@
+import cronjobs
+import waffle
+
+from karma.actions import KarmaManager
+
+
+@cronjobs.register
+def update_top_contributors():
+    """"Update the top contributor lists"""
+    if not waffle.switch_is_active('karma'):
+        return
+
+    kmgr = KarmaManager()
+    kmgr.update_top_alltime()
+    kmgr.update_top_week()

apps/karma/models.py

@@ -0,0 +1 @@
+# Everything is in REDIS

apps/karma/tasks.py

@@ -0,0 +1,63 @@
+from celery.decorators import task
+import waffle
+
+from karma.actions import redis_client
+from karma.cron import update_top_contributors as _update_top_contributors
+from questions.karma_actions import (AnswerAction, AnswerMarkedHelpfulAction,
+                                     FirstAnswerAction, SolutionAction)
+from questions.models import Question, AnswerVote
+from sumo.utils import chunked
+
+
+@task
+def init_karma():
+    """Flushes the karma redis backend and populates with fresh data.
+
+    Goes through all questions/answers/votes and save karma actions for them.
+    """
+    if not waffle.switch_is_active('karma'):
+        return
+
+    redis_client('karma').flushdb()
+
+    questions = Question.objects.all()
+    for chunk in chunked(questions.values_list('pk', flat=True), 200):
+        _process_question_chunk.apply_async(args=[chunk])
+
+    votes = AnswerVote.objects.filter(helpful=True)
+    for chunk in chunked(votes.values_list('pk', flat=True), 1000):
+        _process_answer_vote_chunk.apply_async(args=[chunk])
+
+
+@task
+def update_top_contributors():
+    """Updates the top contributor sorted sets."""
+    _update_top_contributors()
+
+
+@task
+def _process_question_chunk(data, **kwargs):
+    """Save karma data for a chunk of questions."""
+    redis = redis_client('karma')
+    q_qs = Question.objects.select_related('solution').defer('content')
+    for question in q_qs.filter(pk__in=data):
+        first = True
+        a_qs = question.answers.order_by('created').select_related('creator')
+        for answer in a_qs.values_list('creator', 'created'):
+            AnswerAction(answer[0], answer[1], redis).save()
+            if first:
+                FirstAnswerAction(answer[0], answer[1], redis).save()
+                first = False
+        soln = question.solution
+        if soln:
+            SolutionAction(soln.creator, soln.created, redis).save()
+
+
+@task
+def _process_answer_vote_chunk(data, **kwargs):
+    """Save karma data for a chunk of answer votes."""
+    redis = redis_client('karma')
+    v_qs = AnswerVote.objects.select_related('answer')
+    for vote in v_qs.filter(pk__in=data):
+        AnswerMarkedHelpfulAction(
+            vote.answer.creator_id, vote.created, redis).save()