Retry failed SQL connections (LP #876663)

Change-Id: I972d8debb2e3230bccfcca68aee5b08021ba16c7

nova/db/sqlalchemy/session.py

@@ -18,14 +18,17 @@
 
 """Session Handling for SQLAlchemy backend."""
 
+import sqlalchemy.exc
 import sqlalchemy.orm
+import time
 
 import nova.exception
-import nova.flags
+import nova.flags as flags
+import nova.log as logging
 
 
-FLAGS = nova.flags.FLAGS
-
+FLAGS = flags.FLAGS
+LOG = logging.getLogger("nova.db.sqlalchemy.session")
 
 _ENGINE = None
 _MAKER = None
@@ -57,7 +60,26 @@ def get_engine():
     if "sqlite" in connection_dict.drivername:
         engine_args["poolclass"] = sqlalchemy.pool.NullPool
 
-    return sqlalchemy.create_engine(FLAGS.sql_connection, **engine_args)
+    engine = sqlalchemy.create_engine(FLAGS.sql_connection, **engine_args)
+    ensure_connection(engine)
+    return engine
+
+
+def ensure_connection(engine):
+    remaining_attempts = FLAGS.sql_max_retries
+    while True:
+        try:
+            engine.connect()
+            return
+        except sqlalchemy.exc.OperationalError:
+            if remaining_attempts == 0:
+                raise
+            LOG.warning(_('SQL connection failed (%(connstring)s). '
+                          '%(attempts)d attempts left.'),
+                           {'connstring': FLAGS.sql_connection,
+                            'attempts': remaining_attempts})
+            time.sleep(FLAGS.sql_retry_interval)
+            remaining_attempts -= 1
 
 
 def get_maker(engine, autocommit=True, expire_on_commit=False):