Shard commentstree_q processing by Link ID.

This should better isolate large threads and increase batch effectiveness.
reddit-archive · Jun 19, 2013 · 713a8ee · 713a8ee
1 parent bcea327
commit 713a8ee
Show file tree

Hide file tree

Showing 6 changed files with 25 additions and 26 deletions.
diff --git a/r2/example.ini b/r2/example.ini
@@ -344,6 +344,10 @@ TRAFFIC_LOG_HOSTS =
 # this helps with lock contention but isn't necessary on smaller sites
 shard_link_vote_queues = false
 
+# should we split comment tree processing into shards based on the link id?
+# this helps with lock contention but isn't necessary on smaller sites
+shard_commentstree_queues = false
+
 # list of cnames allowed to render as reddit.com without a frame
 authorized_cnames = 
 

diff --git a/r2/r2/config/queues.py b/r2/r2/config/queues.py
@@ -75,8 +75,8 @@ def declare_queues(g):
     queues = Queues({
         "scraper_q": MessageQueue(),
         "newcomments_q": MessageQueue(),
-        "commentstree_q": MessageQueue(),
-        "commentstree_fastlane_q": MessageQueue(),
+        "commentstree_q": MessageQueue(bind_to_self=True),
+        "commentstree_fastlane_q": MessageQueue(bind_to_self=True),
         "vote_link_q": MessageQueue(bind_to_self=True),
         "vote_comment_q": MessageQueue(bind_to_self=True),
         "vote_fastlane_q": MessageQueue(bind_to_self=True),
@@ -92,13 +92,15 @@ def declare_queues(g):
                                for i in xrange(10)}
         queues.declare(sharded_vote_queues)
 
+    if g.shard_commentstree_queues:
+        sharded_commentstree_queues = {"commentstree_%d_q" % i :
+                                       MessageQueue(bind_to_self=True)
+                                       for i in xrange(10)}
+        queues.declare(sharded_commentstree_queues)
+
     queues.cloudsearch_changes << "search_changes"
     queues.scraper_q << "new_link"
-    queues.newcomments_q << ("new_comment",
-                             "new_fastlane_comment",
-                            )
-    queues.commentstree_q << "new_comment"
-    queues.commentstree_fastlane_q << "new_fastlane_comment"
+    queues.newcomments_q << "new_comment"
     queues.butler_q << ("new_comment",
                         "usertext_edited")
     return queues
diff --git a/r2/r2/lib/app_globals.py b/r2/r2/lib/app_globals.py
@@ -159,6 +159,7 @@ class Globals(object):
             'static_secure_pre_gzipped',
             'trust_local_proxies',
             'shard_link_vote_queues',
+            'shard_commentstree_queues',
         ],
 
         ConfigValue.tuple: [

diff --git a/r2/r2/lib/db/queries.py b/r2/r2/lib/db/queries.py
@@ -867,10 +867,15 @@ def new_comment(comment, inbox_rels):
                     not (sr.exclude_banned_modqueue and author._spam)):
                 m.insert(get_spam_filtered_comments(sr), [comment])
 
+            amqp.add_item('new_comment', comment._fullname)
+
             if utils.to36(comment.link_id) in g.live_config["fastlane_links"]:
-                amqp.add_item('new_fastlane_comment', comment._fullname)
+                amqp.add_item('commentstree_fastlane_q', comment._fullname)
+            elif g.shard_commentstree_queues:
+                amqp.add_item('commentstree_%d_q' % (comment.link_id % 10),
+                              comment._fullname)
             else:
-                amqp.add_item('new_comment', comment._fullname)
+                amqp.add_item('commentstree_q', comment._fullname)
 
             if not g.amqp_host:
                 add_comments([comment])
@@ -1385,7 +1390,7 @@ def _run_commentstree(msgs, chan):
         # messages that were put into the non-fastlane queue and are causing
         # both to back up. a full recompute of the old thread will fix these
         # missed messages.
-        if qname == "commentstree_q":
+        if qname != "commentstree_fastlane_q":
             fastlaned_links = g.live_config["fastlane_links"]
             links = Link._byID([com.link_id for com in comments], data=True)
             comments = [com for com in comments

diff --git a/upstart/reddit-consumer-commentstree_fastlane_q.conf b/upstart/reddit-consumer-commentstree_fastlane_q.conf
diff --git a/upstart/reddit-consumer-commentstree_q.conf b/upstart/reddit-consumer-commentstree_q.conf
@@ -1,6 +1,7 @@
 description "place new comments in the precomputed comment trees"
 
-instance $x
+instance $type$x
+env type=commentstree_q
 
 stop on reddit-stop or runlevel [016]
 
@@ -10,5 +11,5 @@ respawn limit 10 5
 nice 10
 script
     . /etc/default/reddit
-    wrap-job paster run --proctitle commentstree_q$x $REDDIT_INI $REDDIT_ROOT/r2/lib/db/queries.py -c 'run_commentstree()'
+    wrap-job paster run --proctitle $type$x $REDDIT_INI $REDDIT_ROOT/r2/lib/db/queries.py -c "run_commentstree(qname='$type')"
 end script