Permalink
Browse files

Moved database configuration from databases.py to the ini.

Added support for load-balancing database reads.
  • Loading branch information...
1 parent 8626d43 commit 2a5f891abe0b193787b55d3aa2e4007e86149da7 @spez spez committed Jan 28, 2009
View
@@ -20,35 +20,49 @@ tracker_url =
adtracker_url =
clicktracker_url =
-main_db_name = reddit
-main_db_host = 127.0.0.1
-main_db_user = reddit
-main_db_pass = password
-
-comment_db_name = reddit
-comment_db_host = 127.0.0.1
-comment_db_user = reddit
-comment_db_pass = password
-
-vote_db_name = reddit
-vote_db_host = 127.0.0.1
-vote_db_user = reddit
-vote_db_pass = password
-
-change_db_name = changes
-change_db_host = 127.0.0.1
-change_db_user = reddit
-change_db_pass = password
-
-email_db_name = email
-email_db_host = 127.0.0.1
-email_db_user = reddit
-email_db_pass = password
-
-query_queue_db_name = query_queue
-query_queue_db_host = 127.0.0.1
-query_queue_db_user = reddit
-query_queue_db_pass = password
+databases = main, comment, vote, change, email, neuter, query_queue
+
+#db name db host user, pass
+main_db = newreddit, 10.96.53.79, ri, password
+comment_db = newreddit, 10.96.53.76, ri, password
+comment2_db = newreddit, 10.96.53.78, ri, password
+vote_db = newreddit, 10.96.53.77, ri, password
+change_db = changed, 10.96.53.216, ri, password
+email_db = email, 10.96.53.216, ri, password
+neuter_db = neuter, 10.96.53.216, ri, password
+query_queue_db = query_queue, 10.96.53.220, ri, password
+
+db_app_name = reddit
+db_create_tables = True
+
+type_db = main
+rel_type_db = main
+
+db_table_link = thing, main, main
+db_table_account = thing, main
+db_table_message = thing, main
+
+db_table_savehide = relation, account, link, main
+db_table_click = relation, account, link, main
+
+db_table_comment = thing, comment
+db_table_subreddit = thing, comment
+
+db_table_srmember = relation, subreddit, account, comment
+
+db_table_friend = relation, account, account, comment
+
+db_table_vote_account_link = relation, account, link, vote
+db_table_vote_account_comment = relation, account, comment, vote
+
+db_table_inbox_account_comment = relation, account, comment
+db_table_inbox_account_message = relation, account, message, main
+
+db_table_report_account_link = relation, account, link, main
+db_table_report_account_comment = relation, account, comment, comment
+db_table_report_account_message = relation, account, message, main
+db_table_report_account_subreddit = relation, account, subreddit, main
+
###
# Other magic settings
View
@@ -1,98 +0,0 @@
-# The contents of this file are subject to the Common Public Attribution
-# License Version 1.0. (the "License"); you may not use this file except in
-# compliance with the License. You may obtain a copy of the License at
-# http://code.reddit.com/LICENSE. The License is based on the Mozilla Public
-# License Version 1.1, but Sections 14 and 15 have been added to cover use of
-# software over a computer network and provide for limited attribution for the
-# Original Developer. In addition, Exhibit A has been modified to be consistent
-# with Exhibit B.
-#
-# Software distributed under the License is distributed on an "AS IS" basis,
-# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for
-# the specific language governing rights and limitations under the License.
-#
-# The Original Code is Reddit.
-#
-# The Original Developer is the Initial Developer. The Initial Developer of the
-# Original Code is CondeNet, Inc.
-#
-# All portions of the code written by CondeNet are Copyright (c) 2006-2008
-# CondeNet, Inc. All Rights Reserved.
-################################################################################
-from r2.lib.manager import db_manager
-from pylons import g
-try:
- #TODO: move?
- from r2admin.config.databases import *
-except:
- pass
-
-tz = g.tz
-
-dbm = db_manager.db_manager()
-
-main_engine = db_manager.get_engine(g.main_db_name,
- db_host = g.main_db_host,
- db_user = g.main_db_user,
- db_pass = g.main_db_pass)
-
-comment_engine = db_manager.get_engine(g.comment_db_name,
- db_host = g.comment_db_host,
- db_user = g.comment_db_user,
- db_pass = g.comment_db_pass)
-
-vote_engine = db_manager.get_engine(g.vote_db_name,
- db_host = g.vote_db_host,
- db_user = g.vote_db_user,
- db_pass = g.vote_db_pass)
-
-change_engine = db_manager.get_engine(g.change_db_name,
- db_host = g.change_db_host,
- db_user = g.change_db_user,
- db_pass = g.change_db_pass,
- pool_size = 2,
- max_overflow = 2)
-
-email_engine = db_manager.get_engine(g.email_db_name,
- db_host = g.email_db_host,
- db_user = g.email_db_user,
- db_pass = g.email_db_pass,
- pool_size = 2,
- max_overflow = 2)
-
-query_queue_engine = db_manager.get_engine(g.query_queue_db_name,
- db_host = g.query_queue_db_host,
- db_user = g.query_queue_db_user,
- db_pass = g.query_queue_db_pass,
- pool_size = 2,
- max_overflow = 2)
-
-dbm.type_db = main_engine
-dbm.relation_type_db = main_engine
-
-dbm.thing('link', main_engine, main_engine)
-dbm.thing('account', main_engine, main_engine)
-dbm.thing('message', main_engine, main_engine)
-
-dbm.relation('savehide', 'account', 'link', main_engine)
-dbm.relation('click', 'account', 'link', main_engine)
-
-dbm.thing('comment', comment_engine, comment_engine)
-
-dbm.thing('subreddit', comment_engine, comment_engine)
-dbm.relation('srmember', 'subreddit', 'account', comment_engine)
-
-dbm.relation('friend', 'account', 'account', comment_engine)
-
-dbm.relation('vote_account_link', 'account', 'link', vote_engine)
-dbm.relation('vote_account_comment', 'account', 'comment', vote_engine)
-
-dbm.relation('inbox_account_comment', 'account', 'comment', comment_engine)
-dbm.relation('inbox_account_message', 'account', 'message', main_engine)
-
-dbm.relation('report_account_link', 'account', 'link', main_engine)
-dbm.relation('report_account_comment', 'account', 'comment', comment_engine)
-dbm.relation('report_account_message', 'account', 'message', main_engine)
-dbm.relation('report_account_subreddit', 'account', 'subreddit', main_engine)
-
-
View
@@ -27,6 +27,7 @@
from r2.lib.db.stats import QueryStats
from r2.lib.translation import _get_languages
from r2.lib.lock import make_lock_factory
+from r2.lib.manager import db_manager
class Globals(object):
@@ -52,7 +53,8 @@ class Globals(object):
'enable_doquery',
'use_query_cache',
'write_query_queue',
- 'css_killswitch']
+ 'css_killswitch',
+ 'db_create_tables']
tuple_props = ['memcaches',
'rec_cache',
@@ -91,22 +93,15 @@ def __init__(self, global_conf, app_conf, paths, **extra):
"""
- def to_bool(x):
- return (x.lower() == 'true') if x else None
-
- def to_iter(name, delim = ','):
- return (x.strip() for x in global_conf.get(name, '').split(delim))
-
-
# slop over all variables to start with
for k, v in global_conf.iteritems():
if not k.startswith("_") and not hasattr(self, k):
if k in self.int_props:
v = int(v)
elif k in self.bool_props:
- v = to_bool(v)
+ v = self.to_bool(v)
elif k in self.tuple_props:
- v = tuple(to_iter(k))
+ v = tuple(self.to_iter(v))
setattr(self, k, v)
# initialize caches
@@ -121,6 +116,9 @@ def to_iter(name, delim = ','):
# set default time zone if one is not set
self.tz = pytz.timezone(global_conf.get('timezone'))
+ #load the database info
+ self.dbm = self.load_db_params(global_conf)
+
#make a query cache
self.stats_collector = QueryStats()
@@ -177,6 +175,43 @@ def to_iter(name, delim = ','):
self.reddit_host = socket.gethostname()
self.reddit_pid = os.getpid()
+ @staticmethod
+ def to_bool(x):
+ return (x.lower() == 'true') if x else None
+
+ @staticmethod
+ def to_iter(v, delim = ','):
+ return (x.strip() for x in v.split(delim))
+
+ def load_db_params(self, gc):
+ databases = self.to_iter(gc['databases'])
+ if not databases:
+ return
+
+ dbm = db_manager.db_manager()
+ db_params = ('name', 'db_host', 'db_user', 'db_pass',
+ 'pool_size', 'max_overflow')
+ for db_name in databases:
+ conf_params = self.to_iter(gc[db_name + '_db'])
+ params = dict(zip(db_params, conf_params))
+ dbm.engines[db_name] = db_manager.get_engine(**params)
+
+ dbm.type_db = dbm.engines[gc['type_db']]
+ dbm.relation_type_db = dbm.engines[gc['rel_type_db']]
+
+ prefix = 'db_table_'
+ for k, v in gc.iteritems():
+ if k.startswith(prefix):
+ params = list(self.to_iter(v))
+ name = k[len(prefix):]
+ kind = params[0]
+ if kind == 'thing':
+ dbm.add_thing(name, [dbm.engines[n] for n in params[1:]])
+ elif kind == 'relation':
+ dbm.add_relation(name, params[1], params[2],
+ [dbm.engines[n] for n in params[3:]])
+ return dbm
+
def __del__(self):
"""
Put any cleanup code to be run when the application finally exits
@@ -1,7 +1,6 @@
from __future__ import with_statement
from r2.lib.workqueue import WorkQueue
-from r2.config.databases import query_queue_engine, tz
-from r2.lib.db.tdb_sql import make_metadata, settings, create_table, index_str
+from r2.lib.db.tdb_sql import make_metadata, create_table, index_str
import cPickle as pickle
from datetime import datetime
@@ -14,14 +13,16 @@
from sqlalchemy.exceptions import SQLError
from pylons import g
+tz = g.tz
#the current
running = set()
running_lock = Lock()
def make_query_queue_table():
- metadata = make_metadata(query_queue_engine)
- table = sa.Table(settings.DB_APP_NAME + '_query_queue', metadata,
+ engine = g.dbm.engines['query_queue']
+ metadata = make_metadata(engine)
+ table = sa.Table(g.db_app_name + '_query_queue', metadata,
sa.Column('iden', sa.String, primary_key = True),
sa.Column('query', sa.Binary),
sa.Column('date', sa.DateTime(timezone = True)))
View
@@ -21,9 +21,9 @@
################################################################################
from math import log
from datetime import datetime, timedelta
-from r2.config.databases import tz
+from pylons import g
-epoch = datetime(1970, 1, 1, tzinfo = tz)
+epoch = datetime(1970, 1, 1, tzinfo = g.tz)
def epoch_seconds(date):
"""Returns the number of seconds from the epoch to date. Should match
Oops, something went wrong.

0 comments on commit 2a5f891

Please sign in to comment.