Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Shard vote_link_qs by last digit of subreddit ID.

This should help with permacache contention.
  • Loading branch information...
commit e412fdcde78435b61cc78854a02ec81156a54f1f 1 parent 9faa009
@spladug spladug authored
View
4 r2/example.ini
@@ -337,6 +337,10 @@ spreadshirt_test_font =
# Other magic settings
###
+# should we split link votes into separate queues based on subreddit id?
+# this helps with lock contention but isn't necessary on smaller sites
+shard_link_vote_queues = false
+
# list of cnames allowed to render as reddit.com without a frame
authorized_cnames =
View
8 r2/r2/config/queues.py
@@ -71,7 +71,7 @@ def __lshift__(self, routing_keys):
self._bind(routing_key)
-def declare_queues():
+def declare_queues(g):
queues = Queues({
"scraper_q": MessageQueue(),
"newcomments_q": MessageQueue(),
@@ -86,6 +86,12 @@ def declare_queues():
"update_promos_q": MessageQueue(bind_to_self=True),
})
+ if g.shard_link_vote_queues:
+ sharded_vote_queues = {"vote_link_%d_q" % i :
+ MessageQueue(bind_to_self=True)
+ for i in xrange(10)}
+ queues.declare(sharded_vote_queues)
+
queues.cloudsearch_changes << "search_changes"
queues.scraper_q << "new_link"
queues.newcomments_q << "new_comment"
View
3  r2/r2/lib/app_globals.py
@@ -148,6 +148,7 @@ class Globals(object):
'static_pre_gzipped',
'static_secure_pre_gzipped',
'trust_local_proxies',
+ 'shard_link_vote_queues',
],
ConfigValue.tuple: [
@@ -233,7 +234,7 @@ def __init__(self, global_conf, app_conf, paths, **extra):
self.config = ConfigValueParser(global_conf)
self.config.add_spec(self.spec)
self.plugins = PluginLoader(self.config.get("plugins", []))
- self.queues = queues.declare_queues()
+ self.queues = queues.declare_queues(self)
self.paths = paths
View
5 r2/r2/lib/db/queries.py
@@ -1371,7 +1371,10 @@ def queue_vote(user, thing, dir, ip, organic = False,
if thing._id36 in g.live_config["fastlane_links"]:
qname = vote_fastlane_q
else:
- qname = vote_link_q
+ if g.shard_link_vote_queues:
+ qname = "vote_link_%s_q" % str(thing.sr_id)[-1]
+ else:
+ qname = vote_link_q
elif isinstance(thing, Comment):
if utils.to36(thing.link_id) in g.live_config["fastlane_links"]:
Please sign in to comment.
Something went wrong with that request. Please try again.