Permalink
Browse files

Shard commentstree_q processing by Link ID.

This should better isolate large threads and increase batch
effectiveness.
  • Loading branch information...
1 parent bcea327 commit 713a8ee11256353f0262077b00bffe5290affc9f @spladug spladug committed Jun 10, 2013
View
@@ -344,6 +344,10 @@ TRAFFIC_LOG_HOSTS =
# this helps with lock contention but isn't necessary on smaller sites
shard_link_vote_queues = false
+# should we split comment tree processing into shards based on the link id?
+# this helps with lock contention but isn't necessary on smaller sites
+shard_commentstree_queues = false
+
# list of cnames allowed to render as reddit.com without a frame
authorized_cnames =
View
@@ -75,8 +75,8 @@ def declare_queues(g):
queues = Queues({
"scraper_q": MessageQueue(),
"newcomments_q": MessageQueue(),
- "commentstree_q": MessageQueue(),
- "commentstree_fastlane_q": MessageQueue(),
+ "commentstree_q": MessageQueue(bind_to_self=True),
+ "commentstree_fastlane_q": MessageQueue(bind_to_self=True),
"vote_link_q": MessageQueue(bind_to_self=True),
"vote_comment_q": MessageQueue(bind_to_self=True),
"vote_fastlane_q": MessageQueue(bind_to_self=True),
@@ -92,13 +92,15 @@ def declare_queues(g):
for i in xrange(10)}
queues.declare(sharded_vote_queues)
+ if g.shard_commentstree_queues:
+ sharded_commentstree_queues = {"commentstree_%d_q" % i :
+ MessageQueue(bind_to_self=True)
+ for i in xrange(10)}
+ queues.declare(sharded_commentstree_queues)
+
queues.cloudsearch_changes << "search_changes"
queues.scraper_q << "new_link"
- queues.newcomments_q << ("new_comment",
- "new_fastlane_comment",
- )
- queues.commentstree_q << "new_comment"
- queues.commentstree_fastlane_q << "new_fastlane_comment"
+ queues.newcomments_q << "new_comment"
queues.butler_q << ("new_comment",
"usertext_edited")
return queues
View
@@ -159,6 +159,7 @@ class Globals(object):
'static_secure_pre_gzipped',
'trust_local_proxies',
'shard_link_vote_queues',
+ 'shard_commentstree_queues',
],
ConfigValue.tuple: [
View
@@ -867,10 +867,15 @@ def new_comment(comment, inbox_rels):
not (sr.exclude_banned_modqueue and author._spam)):
m.insert(get_spam_filtered_comments(sr), [comment])
+ amqp.add_item('new_comment', comment._fullname)
+
if utils.to36(comment.link_id) in g.live_config["fastlane_links"]:
- amqp.add_item('new_fastlane_comment', comment._fullname)
+ amqp.add_item('commentstree_fastlane_q', comment._fullname)
+ elif g.shard_commentstree_queues:
+ amqp.add_item('commentstree_%d_q' % (comment.link_id % 10),
+ comment._fullname)
else:
- amqp.add_item('new_comment', comment._fullname)
+ amqp.add_item('commentstree_q', comment._fullname)
if not g.amqp_host:
add_comments([comment])
@@ -1385,7 +1390,7 @@ def _run_commentstree(msgs, chan):
# messages that were put into the non-fastlane queue and are causing
# both to back up. a full recompute of the old thread will fix these
# missed messages.
- if qname == "commentstree_q":
+ if qname != "commentstree_fastlane_q":
fastlaned_links = g.live_config["fastlane_links"]
links = Link._byID([com.link_id for com in comments], data=True)
comments = [com for com in comments
@@ -1,14 +0,0 @@
-description "place new comments in the precomputed comment trees - dedicated fastlane edition"
-
-instance $x
-
-stop on reddit-stop or runlevel [016]
-
-respawn
-respawn limit 10 5
-
-nice 10
-script
- . /etc/default/reddit
- wrap-job paster run --proctitle commentstree_fastlane_q$x $REDDIT_INI $REDDIT_ROOT/r2/lib/db/queries.py -c 'run_commentstree("commentstree_fastlane_q")'
-end script
@@ -1,6 +1,7 @@
description "place new comments in the precomputed comment trees"
-instance $x
+instance $type$x
+env type=commentstree_q
stop on reddit-stop or runlevel [016]
@@ -10,5 +11,5 @@ respawn limit 10 5
nice 10
script
. /etc/default/reddit
- wrap-job paster run --proctitle commentstree_q$x $REDDIT_INI $REDDIT_ROOT/r2/lib/db/queries.py -c 'run_commentstree()'
+ wrap-job paster run --proctitle $type$x $REDDIT_INI $REDDIT_ROOT/r2/lib/db/queries.py -c "run_commentstree(qname='$type')"
end script

0 comments on commit 713a8ee

Please sign in to comment.