Expose most common QueuePoll parameters from SQLAlchemy

Fixes bug 1160442

The defualt parameters are used unless otherwise configured. This
is applicable for:
 - pool_size
 - max_overflow
 - pool_timeout

By increasing the pool_size and the max_overflow and lowering the timeout
the issue reported in the bug is resolved.

NOTE: the configuration values enable the user to make use of the default
QueuePool tuning values.

Change-Id: Ic258442531a2caf2db4f05c4ddd6fbfb0c180f17

etc/quantum/plugins/bigswitch/restproxy.ini

@@ -23,7 +23,11 @@ reconnect_interval = 2
 # Timeout in seconds before idle sql connections are reaped
 # sql_idle_timeout = 3600
 # Maximum number of SQL connections to keep open in a QueuePool in SQLAlchemy
-# sqlalchemy_pool_size = 5
+# Example sqlalchemy_pool_size = 5
+# Maximum number of overflow connections in a QueuePool in SQLAlchemy
+# Example sqlalchemy_max_overflow = 10
+# Timeout of the open connections QueuePool in SQLAlchemy
+# Example sqlalchemy_pool_timeout = 30
 
 [RESTPROXY]
 # All configuration for this plugin is in section '[restproxy]'

etc/quantum/plugins/brocade/brocade.ini

@@ -22,7 +22,11 @@
 # Timeout in seconds before idle sql connections are reaped
 # sql_idle_timeout = 3600
 # Maximum number of SQL connections to keep open in a QueuePool in SQLAlchemy
-# sqlalchemy_pool_size = 5
+# Example sqlalchemy_pool_size = 5
+# Maximum number of overflow connections in a QueuePool in SQLAlchemy
+# Example sqlalchemy_max_overflow = 10
+# Timeout of the open connections QueuePool in SQLAlchemy
+# Example sqlalchemy_pool_timeout = 30
 #
 # Example:
 # sql_connection = mysql://root:pass@localhost/brcd_quantum?charset=utf8

etc/quantum/plugins/hyperv/hyperv_quantum_plugin.ini

@@ -21,7 +21,11 @@ reconnect_interval = 2
 # Timeout in seconds before idle sql connections are reaped
 # sql_idle_timeout = 3600
 # Maximum number of SQL connections to keep open in a QueuePool in SQLAlchemy
-# sqlalchemy_pool_size = 5
+# Example sqlalchemy_pool_size = 5
+# Maximum number of overflow connections in a QueuePool in SQLAlchemy
+# Example sqlalchemy_max_overflow = 10
+# Timeout of the open connections QueuePool in SQLAlchemy
+# Example sqlalchemy_pool_timeout = 30
 
 [HYPERV]
 # (StrOpt) Type of network to allocate for tenant networks. The

etc/quantum/plugins/linuxbridge/linuxbridge_conf.ini

@@ -43,7 +43,11 @@ reconnect_interval = 2
 # Timeout in seconds before idle sql connections are reaped
 # sql_idle_timeout = 3600
 # Maximum number of SQL connections to keep open in a QueuePool in SQLAlchemy
-# sqlalchemy_pool_size = 5
+# Example sqlalchemy_pool_size = 5
+# Maximum number of overflow connections in a QueuePool in SQLAlchemy
+# Example sqlalchemy_max_overflow = 10
+# Timeout of the open connections QueuePool in SQLAlchemy
+# Example sqlalchemy_pool_timeout = 30
 
 [LINUX_BRIDGE]
 # (ListOpt) Comma-separated list of

etc/quantum/plugins/metaplugin/metaplugin.ini

@@ -23,7 +23,11 @@ reconnect_interval = 2
 # Timeout in seconds before idle sql connections are reaped
 # sql_idle_timeout = 3600
 # Maximum number of SQL connections to keep open in a QueuePool in SQLAlchemy
-# sqlalchemy_pool_size = 5
+# Example sqlalchemy_pool_size = 5
+# Maximum number of overflow connections in a QueuePool in SQLAlchemy
+# Example sqlalchemy_max_overflow = 10
+# Timeout of the open connections QueuePool in SQLAlchemy
+# Example sqlalchemy_pool_timeout = 30
 
 [META]
 ## This is list of flavor:quantum_plugins

etc/quantum/plugins/midonet/midonet.ini

@@ -21,7 +21,11 @@ reconnect_interval = 2
 # Timeout in seconds before idle sql connections are reaped
 # sql_idle_timeout = 3600
 # Maximum number of SQL connections to keep open in a QueuePool in SQLAlchemy
-# sqlalchemy_pool_size = 5
+# Example sqlalchemy_pool_size = 5
+# Maximum number of overflow connections in a QueuePool in SQLAlchemy
+# Example sqlalchemy_max_overflow = 10
+# Timeout of the open connections QueuePool in SQLAlchemy
+# Example sqlalchemy_pool_timeout = 30
 
 [MIDONET]
 # MidoNet API server URI

etc/quantum/plugins/ml2/ml2_conf.ini

@@ -40,11 +40,12 @@
 # sql_idle_timeout = 3600
 # Example: sql_idle_timeout = 6000
 
-# (IntOpt) Maximum number of SQL connections to keep open in a
-# QueuePool in SQLAlchemy.
-#
-# sqlalchemy_pool_size = 5
-# Example: sqlalchemy_pool_size = 10
+# Maximum number of SQL connections to keep open in a QueuePool in SQLAlchemy
+# Example sqlalchemy_pool_size = 5
+# Maximum number of overflow connections in a QueuePool in SQLAlchemy
+# Example sqlalchemy_max_overflow = 10
+# Timeout of the open connections QueuePool in SQLAlchemy
+# Example sqlalchemy_pool_timeout = 30
 
 [ml2]
 # (ListOpt) List of network type driver entrypoints to be loaded from

etc/quantum/plugins/nec/nec.ini

@@ -23,7 +23,11 @@ reconnect_interval = 2
 # Timeout in seconds before idle sql connections are reaped
 # sql_idle_timeout = 3600
 # Maximum number of SQL connections to keep open in a QueuePool in SQLAlchemy
-# sqlalchemy_pool_size = 5
+# Example sqlalchemy_pool_size = 5
+# Maximum number of overflow connections in a QueuePool in SQLAlchemy
+# Example sqlalchemy_max_overflow = 10
+# Timeout of the open connections QueuePool in SQLAlchemy
+# Example sqlalchemy_pool_timeout = 30
 
 [OVS]
 # Do not change this parameter unless you have a good reason to.

etc/quantum/plugins/nicira/nvp.ini

@@ -91,8 +91,13 @@ sql_connection = sqlite://
 # sql_idle_timeout = 3600
 
 # Maximum number of SQL connections to keep open in a QueuePool in SQLAlchemy
-# sqlalchemy_pool_size = 5
+# Example sqlalchemy_pool_size = 5
 
+# Maximum number of overflow connections in a QueuePool in SQLAlchemy
+# Example sqlalchemy_max_overflow = 10
+
+# Timeout of the open connections QueuePool in SQLAlchemy
+# Example sqlalchemy_pool_timeout = 30
 
 [QUOTAS]
 # number of network gateways allowed per tenant, -1 means unlimited

etc/quantum/plugins/openvswitch/ovs_quantum_plugin.ini

@@ -21,7 +21,11 @@ reconnect_interval = 2
 # Timeout in seconds before idle sql connections are reaped
 # sql_idle_timeout = 3600
 # Maximum number of SQL connections to keep open in a QueuePool in SQLAlchemy
-# sqlalchemy_pool_size = 5
+# Example sqlalchemy_pool_size = 5
+# Maximum number of overflow connections in a QueuePool in SQLAlchemy
+# Example sqlalchemy_max_overflow = 10
+# Timeout of the open connections QueuePool in SQLAlchemy
+# Example sqlalchemy_pool_timeout = 30
 
 [OVS]
 # (StrOpt) Type of network to allocate for tenant networks. The

etc/quantum/plugins/plumgrid/plumgrid.ini

@@ -23,7 +23,11 @@
 # Timeout in seconds before idle sql connections are reaped
 # sql_idle_timeout = 3600
 # Maximum number of SQL connections to keep open in a QueuePool in SQLAlchemy
-# sqlalchemy_pool_size = 5
+# Example sqlalchemy_pool_size = 5
+# Maximum number of overflow connections in a QueuePool in SQLAlchemy
+# Example sqlalchemy_max_overflow = 10
+# Timeout of the open connections QueuePool in SQLAlchemy
+# Example sqlalchemy_pool_timeout = 30
 
 [PLUMgridNOS]
 # This line should be pointing to the NOS server,

etc/quantum/plugins/ryu/ryu.ini

@@ -13,7 +13,11 @@ sql_connection = sqlite://
 # Timeout in seconds before idle sql connections are reaped
 # sql_idle_timeout = 3600
 # Maximum number of SQL connections to keep open in a QueuePool in SQLAlchemy
-# sqlalchemy_pool_size = 5
+# Example sqlalchemy_pool_size = 5
+# Maximum number of overflow connections in a QueuePool in SQLAlchemy
+# Example sqlalchemy_max_overflow = 10
+# Timeout of the open connections QueuePool in SQLAlchemy
+# Example sqlalchemy_pool_timeout = 30
 
 [OVS]
 integration_bridge = br-int

quantum/db/api.py

@@ -64,9 +64,17 @@
                 default=False,
                 help=_("Enable the use of eventlet's db_pool for MySQL")),
     cfg.IntOpt('sqlalchemy_pool_size',
-               default=5,
+               default=None,
                help=_("Maximum number of SQL connections to keep open in a "
                       "QueuePool in SQLAlchemy")),
+    cfg.IntOpt('sqlalchemy_max_overflow',
+               default=None,
+               help=_("If set, use this value for max_overflow with "
+                      "sqlalchemy")),
+    cfg.IntOpt('sqlalchemy_pool_timeout',
+               default=None,
+               help=_("If set, use this value for pool_timeout with "
+                      "sqlalchemy")),
 ]
 
 cfg.CONF.register_opts(database_opts, "DATABASE")
@@ -124,9 +132,18 @@ def configure_db():
             'pool_recycle': 3600,
             'echo': False,
             'convert_unicode': True,
-            'pool_size': cfg.CONF.DATABASE.sqlalchemy_pool_size,
         }
 
+        if cfg.CONF.DATABASE.sqlalchemy_pool_size is not None:
+            pool_size = cfg.CONF.DATABASE.sqlalchemy_pool_size
+            engine_args['pool_size'] = pool_size
+        if cfg.CONF.DATABASE.sqlalchemy_max_overflow is not None:
+            max_overflow = cfg.CONF.DATABASE.sqlalchemy_max_overflow
+            engine_args['max_overflow'] = max_overflow
+        if cfg.CONF.DATABASE.sqlalchemy_pool_timeout is not None:
+            pool_timeout = cfg.CONF.DATABASE.sqlalchemy_pool_timeout
+            engine_args['pool_timeout'] = pool_timeout
+
         if 'mysql' in connection_dict.drivername:
             engine_args['listeners'] = [MySQLPingListener()]
             if (MySQLdb is not None and