Permalink
Browse files

Allow Cassandra models to choose a connection pool.

Note: some CFs are explicitly marked for the "main" connection
pool and others are not. This is an artifact of the current
state of reddit.com's cassandra setup and is intentional. While
cassandra_default_pool remains "main", this will not cause an issue
for you.
  • Loading branch information...
1 parent 59fb2a9 commit 532be3fe2a80a4ed47bc99ee250d159c10b8d029 @spladug spladug committed Nov 30, 2011
View
@@ -122,6 +122,8 @@ cassandra_seeds = 127.0.0.1:9160
# read/write consistency levels for Cassandra
cassandra_rcl = ONE
cassandra_wcl = ONE
+# name of default connection pool to use when _connection_pool not specified
+cassandra_default_pool = main
# -- url cache options --
url_caches = 127.0.0.1:11211
View
@@ -217,18 +217,28 @@ def setup(self, global_conf):
if not self.cassandra_seeds:
raise ValueError("cassandra_seeds not set in the .ini")
- self.cassandra = PycassaConnectionPool('reddit',
- server_list = self.cassandra_seeds,
- pool_size = len(self.cassandra_seeds),
- # TODO: .ini setting
- timeout=15, max_retries=3,
- prefill=False)
+
+
+ keyspace = "reddit"
+ self.cassandra_pools = {
+ "main":
+ PycassaConnectionPool(
+ keyspace,
+ logging_name="main",
+ server_list=self.cassandra_seeds,
+ pool_size=len(self.cassandra_seeds),
+ timeout=2,
+ max_retries=3,
+ prefill=False
+ ),
+ }
+
perma_memcache = (CMemcache(self.permacache_memcaches, num_clients = num_mc_clients)
if self.permacache_memcaches
else None)
self.permacache = CassandraCacheChain(localcache_cls(),
CassandraCache('permacache',
- self.cassandra,
+ self.cassandra_pools[self.cassandra_default_pool],
read_consistency_level = self.cassandra_rcl,
write_consistency_level = self.cassandra_wcl),
memcache = perma_memcache,
@@ -35,10 +35,11 @@
from itertools import chain
import cPickle as pickle
-cassandra = g.cassandra
-thing_cache = g.thing_cache
-seeds = g.cassandra_seeds
+connection_pools = g.cassandra_pools
+default_connection_pool = g.cassandra_default_pool
+
keyspace = 'reddit'
+thing_cache = g.thing_cache
disallow_db_writes = g.disallow_db_writes
tz = g.tz
log = g.log
@@ -93,7 +94,7 @@ def _fn(*a, **kw):
return fn(*a, **kw)
return _fn
-def get_manager():
+def get_manager(seeds):
# n.b. does not retry against multiple servers
server = seeds[0]
return SystemManager(server)
@@ -125,16 +126,20 @@ def __init__(cls, name, bases, dct):
if not getattr(cls, "_write_consistency_level", None):
cls._write_consistency_level = write_consistency_level
+ pool_name = getattr(cls, "_connection_pool", default_connection_pool)
+ connection_pool = connection_pools[pool_name]
+ cassandra_seeds = connection_pool.server_list
+
try:
- cls._cf = ColumnFamily(cassandra,
+ cls._cf = ColumnFamily(connection_pool,
cf_name,
read_consistency_level = cls._read_consistency_level,
write_consistency_level = cls._write_consistency_level)
except NotFoundException:
if not db_create_tables:
raise
- manager = get_manager()
+ manager = get_manager(cassandra_seeds)
log.warning("Creating Cassandra Column Family %s" % (cf_name,))
with make_lock('cassandra_schema'):
@@ -143,7 +148,7 @@ def __init__(cls, name, bases, dct):
log.warning("Created Cassandra Column Family %s" % (cf_name,))
# try again to look it up
- cls._cf = ColumnFamily(cassandra,
+ cls._cf = ColumnFamily(connection_pool,
cf_name,
read_consistency_level = cls._read_consistency_level,
write_consistency_level = cls._write_consistency_level)
@@ -8,7 +8,7 @@
class SubredditsByPartialName(tdb_cassandra.View):
_use_db = True
_value_type = 'pickle'
- _use_new_ring = True
+ _connection_pool = 'main'
_read_consistency_level = CL_ONE
def load_all_reddits():
View
@@ -83,7 +83,7 @@ class FlairTemplate(tdb_cassandra.Thing):
_bool_props = ('text_editable',)
_use_db = True
- _use_new_ring = True
+ _connection_pool = 'main'
@classmethod
def _new(cls, text='', css_class='', text_editable=False):
@@ -137,7 +137,7 @@ class FlairTemplateBySubredditIndex(tdb_cassandra.Thing):
_int_props = ('sr_id',)
_use_db = True
- _use_new_ring = True
+ _connection_pool = 'main'
_key_prefix = 'ft_'
View
@@ -833,6 +833,7 @@ class CommentSortsCache(tdb_cassandra.View):
the candidate order"""
_use_db = True
_value_type = 'float'
+ _connection_pool = 'main'
_read_consistency_level = tdb_cassandra.CL.ONE
class StarkComment(Comment):
@@ -1175,9 +1176,11 @@ def _uncreate(cls, user, link):
except tdb_cassandra.NotFound:
pass
+
class CassandraSave(SimpleRelation):
_use_db = True
_cf_name = 'Save'
+ _connection_pool = 'main'
# thing1_cls = Account
# thing2_cls = Link
@@ -1211,6 +1214,7 @@ class CassandraHide(SimpleRelation):
_use_db = True
_cf_name = 'Hide'
_ttl = 7*24*60*60
+ _connection_pool = 'main'
@classmethod
def _hide(cls, *a, **kw):
@@ -1227,6 +1231,7 @@ class CassandraClick(SimpleRelation):
class SavesByAccount(tdb_cassandra.View):
_use_db = True
_cf_name = 'SavesByAccount'
+ _connection_pool = 'main'
class Inbox(MultiRelation('inbox',
Relation(Account, Comment),
View
@@ -69,7 +69,7 @@ class OAuth2Client(OAuth2Token):
redirect_uri="",
)
_use_db = True
- _use_new_ring = True
+ _connection_pool = 'main'
@classmethod
def _new(cls, **kwargs):
@@ -90,7 +90,7 @@ class OAuth2AuthorizationCode(OAuth2Token):
_int_props = ("user_id",)
_warn_on_partial_ttl = False
_use_db = True
- _use_new_ring = True
+ _connection_pool = 'main'
@classmethod
def _new(cls, client_id, redirect_uri, user_id, scope):
@@ -127,7 +127,7 @@ class OAuth2AccessToken(OAuth2Token):
)
_int_props = ("user_id",)
_use_db = True
- _use_new_ring = True
+ _connection_pool = 'main'
@classmethod
def _new(cls, user_id, scope):
@@ -978,5 +978,5 @@ class SRMember(Relation(Subreddit, Account)): pass
class SubredditPopularityByLanguage(tdb_cassandra.View):
_use_db = True
_value_type = 'pickle'
- _use_new_ring = True
+ _connection_pool = 'main'
_read_consistency_level = CL_ONE
View
@@ -45,6 +45,7 @@ def score_changes(amount, old_amount):
class CassandraVote(tdb_cassandra.Relation):
_use_db = False
+ _connection_pool = 'main'
_bool_props = ('valid_user', 'valid_thing', 'organic')
_str_props = ('name', # one of '-1', '0', '1'
@@ -84,6 +85,7 @@ def _copy_from(cls, v):
class VotesByLink(tdb_cassandra.View):
_use_db = True
_type_prefix = 'VotesByLink'
+ _connection_pool = 'main'
# _view_of = CassandraLinkVote
@@ -99,6 +101,7 @@ def get_all(cls, *link_ids):
class VotesByDay(tdb_cassandra.View):
_use_db = True
_type_prefix = 'VotesByDay'
+ _connection_pool = 'main'
# _view_of = CassandraLinkVote

0 comments on commit 532be3f

Please sign in to comment.