Browse files

Ensure a functional database connection

Allow retrying database connection in get_engine() at an interval.  Resolves
the issue of nova components erroring at startup if a database connection is
unavailable, particularly at boot. Borrowed from a similar commit to glance,
(https://review.openstack.org/#change,5552).

This also fixes code duplication due to a half-backport of
commit 155ef7d

Fixes Bug #959426 for nova.

Change-Id: Ifea94da8347714887c8cae02cc48288f3fa4fa7f
  • Loading branch information...
1 parent 615087d commit ccb93c47370d774dc7b1959f2b2d038a4819855d @ttx ttx committed Apr 2, 2012
Showing with 51 additions and 18 deletions.
  1. +3 −1 etc/nova/nova.conf.sample
  2. +44 −17 nova/db/sqlalchemy/session.py
  3. +4 −0 nova/flags.py
View
4 etc/nova/nova.conf.sample
@@ -215,6 +215,8 @@
# sql_connection="sqlite:///$state_path/$sqlite_db"
###### (IntOpt) timeout before idle sql connections are reaped
# sql_idle_timeout=3600
+###### (IntOpt) maximum db connection retries during startup. (setting -1 implies an infinite retry count)
+# sql_max_retries=10
###### (IntOpt) interval between retries of opening a sql connection
# sql_retry_interval=10
###### (StrOpt) the filename to use with sqlite
@@ -1105,4 +1107,4 @@
###### (StrOpt) The ZFS path under which to create zvols for volumes.
# san_zfs_volume_base="rpool/"
-# Total option count: 466
+# Total option count: 467
View
61 nova/db/sqlalchemy/session.py
@@ -22,7 +22,7 @@
import sqlalchemy.interfaces
import sqlalchemy.orm
-from sqlalchemy.exc import DisconnectionError
+from sqlalchemy.exc import DisconnectionError, OperationalError
from sqlalchemy.pool import NullPool, StaticPool
import nova.exception
@@ -39,11 +39,11 @@
def get_session(autocommit=True, expire_on_commit=False):
"""Return a SQLAlchemy session."""
- global _ENGINE, _MAKER
+ global _MAKER
- if _MAKER is None or _ENGINE is None:
- _ENGINE = get_engine()
- _MAKER = get_maker(_ENGINE, autocommit, expire_on_commit)
+ if _MAKER is None:
+ engine = get_engine()
+ _MAKER = get_maker(engine, autocommit, expire_on_commit)
session = _MAKER()
session.query = nova.exception.wrap_db_error(session.query)
@@ -80,6 +80,17 @@ def checkout(self, dbapi_con, con_record, con_proxy):
raise
+def is_db_connection_error(args):
+ """Return True if error in connecting to db."""
+ # NOTE(adam_g): This is currently MySQL specific and needs to be extended
+ # to support Postgres and others.
+ conn_err_codes = ('2002', '2003', '2006')
+ for err_code in conn_err_codes:
+ if args.find(err_code) != -1:
+ return True
+ return False
+
+
def get_engine():
"""Return a SQLAlchemy engine."""
global _ENGINE
@@ -105,21 +116,37 @@ def get_engine():
engine_args["poolclass"] = StaticPool
engine_args["connect_args"] = {'check_same_thread': False}
- engine_args = {
- "pool_recycle": FLAGS.sql_idle_timeout,
- "echo": False,
- 'convert_unicode': True,
- }
+ if not FLAGS.sqlite_synchronous:
+ engine_args["listeners"] = [SynchronousSwitchListener()]
+
+ if 'mysql' in connection_dict.drivername:
+ engine_args['listeners'] = [MySQLPingListener()]
- if "sqlite" in connection_dict.drivername:
- engine_args["poolclass"] = sqlalchemy.pool.NullPool
- if not FLAGS.sqlite_synchronous:
- engine_args["listeners"] = [SynchronousSwitchListener()]
+ _ENGINE = sqlalchemy.create_engine(FLAGS.sql_connection, **engine_args)
- if 'mysql' in connection_dict.drivername:
- engine_args['listeners'] = [MySQLPingListener()]
+ try:
+ _ENGINE.connect()
+ except OperationalError, e:
+ if not is_db_connection_error(e.args[0]):
+ raise
- return sqlalchemy.create_engine(FLAGS.sql_connection, **engine_args)
+ remaining = FLAGS.sql_max_retries
+ if remaining == -1:
+ remaining = 'infinite'
+ while True:
+ msg = _('SQL connection failed. %s attempts left.')
+ LOG.warn(msg % remaining)
+ if remaining != 'infinite':
+ remaining -= 1
+ time.sleep(FLAGS.sql_retry_interval)
+ try:
+ _ENGINE.connect()
+ break
+ except OperationalError, e:
+ if (remaining != 'infinite' and remaining == 0) or \
+ not is_db_connection_error(e.args[0]):
+ raise
+ return _ENGINE
def get_maker(engine, autocommit=True, expire_on_commit=False):
View
4 nova/flags.py
@@ -325,6 +325,10 @@ def _get_my_ip():
cfg.IntOpt('sql_idle_timeout',
default=3600,
help='timeout before idle sql connections are reaped'),
+ cfg.IntOpt('sql_max_retries',
+ default=10,
+ help='maximum db connection retries during startup. '
+ '(setting -1 implies an infinite retry count)'),
cfg.IntOpt('sql_retry_interval',
default=10,
help='interval between retries of opening a sql connection'),

0 comments on commit ccb93c4

Please sign in to comment.