Permalink
Browse files

Refactored impl_mongodb to use full connection url

Host parameter can be a full URI see:
http://docs.mongodb.org/manual/reference/connection-string/

This allows to set replicaSet config and other options in the connection
URL without the need to have additional config values

Replaced deprecated pymongo.Connection with pymongo.MongoClient

Change-Id: I1aea1d8a48dc1bec717e7d1fbe256e2887e27039
  • Loading branch information...
Alexei Kornienko
Alexei Kornienko committed Jul 23, 2013
1 parent a4b1b54 commit 3ad779edf545199a185f0fa9a2f850bc3db7488d
Showing with 15 additions and 39 deletions.
  1. +12 −37 ceilometer/storage/impl_mongodb.py
  2. +1 −1 requirements.txt
  3. +2 −1 tests/storage/test_impl_mongodb.py
@@ -24,9 +24,7 @@
import datetime
import operator
import os
import re
import time
import urlparse
import uuid
import bson.code
@@ -36,7 +34,6 @@
from oslo.config import cfg
from ceilometer.openstack.common import log
from ceilometer.openstack.common import network_utils
from ceilometer.openstack.common import timeutils
from ceilometer import storage
from ceilometer.storage import base
@@ -147,20 +144,15 @@ class ConnectionPool(object):
def __init__(self):
self._pool = {}
def connect(self, opts):
# opts is a dict, dict are unhashable, convert to tuple
connection_pool_key = tuple(sorted(opts.items()))
if connection_pool_key not in self._pool:
LOG.info('connecting to MongoDB replicaset "%s" on %s',
opts['replica_set'],
opts['netloc'])
self._pool[connection_pool_key] = pymongo.Connection(
opts['netloc'],
replicaSet=opts['replica_set'],
def connect(self, url):
if url not in self._pool:
LOG.info('connecting to MongoDB on %s', url)
self._pool[url] = pymongo.MongoClient(
url,
use_greenlets=True,
safe=True)
return self._pool.get(connection_pool_key)
return self._pool.get(url)
class Connection(base.Connection):
@@ -249,25 +241,23 @@ class Connection(base.Connection):
}""")
def __init__(self, conf):
opts = self._parse_connection_url(conf.database.connection)
url = conf.database.connection
if opts['netloc'] == '__test__':
if url == 'mongodb://__test__':
url = os.environ.get('CEILOMETER_TEST_MONGODB_URL')
if not url:
raise RuntimeError(
"No MongoDB test URL set,"
"export CEILOMETER_TEST_MONGODB_URL environment variable")
opts = self._parse_connection_url(url)
# NOTE(jd) Use our own connection pooling on top of the Pymongo one.
# We need that otherwise we overflow the MongoDB instance with new
# connection since we instanciate a Pymongo client each time someone
# requires a new storage connection.
self.conn = self.CONNECTION_POOL.connect(opts)
self.conn = self.CONNECTION_POOL.connect(url)
self.db = getattr(self.conn, opts['dbname'])
if 'username' in opts:
self.db.authenticate(opts['username'], opts['password'])
connection_options = pymongo.uri_parser.parse_uri(url)
self.db = getattr(self.conn, connection_options['database'])
# NOTE(jd) Upgrading is just about creating index, so let's do this
# on connection to be sure at least the TTL is correcly updated if
@@ -333,21 +323,6 @@ def _is_natively_ttl_supported(self):
def clear(self):
self.conn.drop_database(self.db)
@staticmethod
def _parse_connection_url(url):
opts = {}
result = network_utils.urlsplit(url)
opts['replica_set'] = urlparse.parse_qs(
result.query).get('replica_set', [""])[0]
opts['dbtype'] = result.scheme
opts['dbname'] = result.path.replace('/', '')
netloc_match = re.match(r'(?:(\w+:\w+)@)?(.*)', result.netloc)
auth = netloc_match.group(1)
opts['netloc'] = netloc_match.group(2)
if auth:
opts['username'], opts['password'] = auth.split(':')
return opts
def record_metering_data(self, data):
"""Write the data to the backend storage system.
View
@@ -8,7 +8,7 @@ SQLAlchemy>=0.7,<=0.7.99
sqlalchemy-migrate>=0.7.2
alembic>=0.4.1
netaddr
pymongo>=2.2
pymongo>=2.4
eventlet
anyjson>=0.3.3
Flask==0.9
@@ -48,9 +48,10 @@ def test_connection_pooling(self):
impl_mongodb.Connection(cfg.CONF).conn)
def test_replica_set(self):
# FIXME(Alexei_987) should not hardcode URL here
cfg.CONF.set_override(
'connection',
'mongodb://__test__?replica_set=foobar',
'mongodb://localhost:29000/ceilometer?replicaSet=foobar',
group='database')
conn = impl_mongodb.Connection(cfg.CONF)
self.assertTrue(conn.conn)

0 comments on commit 3ad779e

Please sign in to comment.