Move to oslo.db

Replace common oslo code nova.openstack.common.db by usage
of oslo.db library and remove common code.

Replaced catching of raw sqlalchemy exceptions by catches
of oslo.db exceptions(such as DBError, DBDuplicateEntry, etc).

Co-Authored-By: Eugeniya Kudryashova  <ekudryashova@mirantis.com>

Closes-Bug: #1364986
Closes-Bug: #1353131
Closes-Bug: #1283987
Closes-Bug: #1274523
Change-Id: I0649539e071b2318ec85ed5d70259c949408e64b

Author: andreykurilin

HACKING.rst

@@ -10,7 +10,7 @@ Nova Specific Commandments
 
 - ``nova.db`` imports are not allowed in ``nova/virt/*``
 - [N309] no db session in public API methods (disabled)
-  This enforces a guideline defined in ``nova.openstack.common.db.sqlalchemy.session``
+  This enforces a guideline defined in ``oslo.db.sqlalchemy.session``
 - [N310] timeutils.utcnow() wrapper must be used instead of direct calls to
   datetime.datetime.utcnow() to make it easy to override its return value in tests
 - [N311] importing code from other virt drivers forbidden

nova/cells/state.py

@@ -22,13 +22,13 @@
 import time
 
 from oslo.config import cfg
+from oslo.db import exception as db_exc
 
 from nova.cells import rpc_driver
 from nova import context
 from nova.db import base
 from nova import exception
 from nova.i18n import _
-from nova.openstack.common.db import exception as db_exc
 from nova.openstack.common import fileutils
 from nova.openstack.common import jsonutils
 from nova.openstack.common import log as logging

nova/cmd/manage.py

@@ -61,6 +61,7 @@
 import decorator
 import netaddr
 from oslo.config import cfg
+from oslo.db import exception as db_exc
 from oslo import messaging
 import six
 
@@ -75,7 +76,6 @@
 from nova.i18n import _
 from nova import objects
 from nova.openstack.common import cliutils
-from nova.openstack.common.db import exception as db_exc
 from nova.openstack.common import importutils
 from nova.openstack.common import log as logging
 from nova import quota

nova/compute/flavors.py

@@ -22,14 +22,14 @@
 import uuid
 
 from oslo.config import cfg
+from oslo.db import exception as db_exc
 import six
 
 from nova import context
 from nova import db
 from nova import exception
 from nova.i18n import _
 from nova.i18n import _LE
-from nova.openstack.common.db import exception as db_exc
 from nova.openstack.common import log as logging
 from nova.openstack.common import strutils
 from nova.pci import pci_request

nova/config.py

@@ -16,23 +16,26 @@
 #    under the License.
 
 from oslo.config import cfg
+from oslo.db import options
 
 from nova import debugger
-from nova.openstack.common.db import options
 from nova import paths
 from nova import rpc
 from nova import version
 
+
+CONF = cfg.CONF
+
 _DEFAULT_SQL_CONNECTION = 'sqlite:///' + paths.state_path_def('nova.sqlite')
 
 
 def parse_args(argv, default_config_files=None):
-    options.set_defaults(sql_connection=_DEFAULT_SQL_CONNECTION,
+    options.set_defaults(CONF, connection=_DEFAULT_SQL_CONNECTION,
                          sqlite_db='nova.sqlite')
     rpc.set_defaults(control_exchange='nova')
     debugger.register_cli_opts()
-    cfg.CONF(argv[1:],
-             project='nova',
-             version=version.version_string(),
-             default_config_files=default_config_files)
-    rpc.init(cfg.CONF)
+    CONF(argv[1:],
+         project='nova',
+         version=version.version_string(),
+         default_config_files=default_config_files)
+    rpc.init(CONF)

nova/db/api.py

@@ -27,12 +27,11 @@
 
 """
 
-from eventlet import tpool
 from oslo.config import cfg
+from oslo.db import concurrency
 
 from nova.cells import rpcapi as cells_rpcapi
 from nova.i18n import _
-from nova.openstack.common.db import api as db_api
 from nova.openstack.common import log as logging
 
 
@@ -48,51 +47,13 @@
                help='Template string to be used to generate snapshot names'),
 ]
 
-tpool_opts = [
-    cfg.BoolOpt('use_tpool',
-                default=False,
-                deprecated_name='dbapi_use_tpool',
-                deprecated_group='DEFAULT',
-                help='Enable the experimental use of thread pooling for '
-                     'all DB API calls'),
-]
-
 CONF = cfg.CONF
 CONF.register_opts(db_opts)
-CONF.register_opts(tpool_opts, 'database')
-CONF.import_opt('backend', 'nova.openstack.common.db.options',
-                group='database')
 
 _BACKEND_MAPPING = {'sqlalchemy': 'nova.db.sqlalchemy.api'}
 
 
-class NovaDBAPI(object):
-    """Nova's DB API wrapper class.
-
-    This wraps the oslo DB API with an option to be able to use eventlet's
-    thread pooling. Since the CONF variable may not be loaded at the time
-    this class is instantiated, we must look at it on the first DB API call.
-    """
-
-    def __init__(self):
-        self.__db_api = None
-
-    @property
-    def _db_api(self):
-        if not self.__db_api:
-            nova_db_api = db_api.DBAPI(CONF.database.backend,
-                                       backend_mapping=_BACKEND_MAPPING)
-            if CONF.database.use_tpool:
-                self.__db_api = tpool.Proxy(nova_db_api)
-            else:
-                self.__db_api = nova_db_api
-        return self.__db_api
-
-    def __getattr__(self, key):
-        return getattr(self._db_api, key)
-
-
-IMPL = NovaDBAPI()
+IMPL = concurrency.TpoolDbapiWrapper(CONF, backend_mapping=_BACKEND_MAPPING)
 
 LOG = logging.getLogger(__name__)
 

nova/db/sqlalchemy/api.py

@@ -22,15 +22,17 @@
 import datetime
 import functools
 import sys
+import threading
 import time
 import uuid
 
 from oslo.config import cfg
+from oslo.db import exception as db_exc
+from oslo.db.sqlalchemy import session as db_session
+from oslo.db.sqlalchemy import utils as sqlalchemyutils
 import six
 from sqlalchemy import and_
 from sqlalchemy import Boolean
-from sqlalchemy.exc import DataError
-from sqlalchemy.exc import IntegrityError
 from sqlalchemy.exc import NoSuchTableError
 from sqlalchemy import Integer
 from sqlalchemy import MetaData
@@ -56,9 +58,6 @@
 from nova.db.sqlalchemy import models
 from nova import exception
 from nova.i18n import _
-from nova.openstack.common.db import exception as db_exc
-from nova.openstack.common.db.sqlalchemy import session as db_session
-from nova.openstack.common.db.sqlalchemy import utils as sqlalchemyutils
 from nova.openstack.common import excutils
 from nova.openstack.common import log as logging
 from nova.openstack.common import timeutils
@@ -73,57 +72,34 @@
                     'Should be empty, "project" or "global".'),
 ]
 
-connection_opts = [
-    cfg.StrOpt('slave_connection',
-               secret=True,
-               help='The SQLAlchemy connection string used to connect to the '
-                    'slave database'),
-]
-
 CONF = cfg.CONF
 CONF.register_opts(db_opts)
-CONF.register_opts(connection_opts, group='database')
 CONF.import_opt('compute_topic', 'nova.compute.rpcapi')
-CONF.import_opt('connection',
-                'nova.openstack.common.db.options',
-                group='database')
 
 LOG = logging.getLogger(__name__)
 
 
-_MASTER_FACADE = None
-_SLAVE_FACADE = None
-
+_ENGINE_FACADE = None
+_LOCK = threading.Lock()
 
-def _create_facade_lazily(use_slave=False):
-    global _MASTER_FACADE
-    global _SLAVE_FACADE
 
-    return_slave = use_slave and CONF.database.slave_connection
-    if not return_slave:
-        if _MASTER_FACADE is None:
-            _MASTER_FACADE = db_session.EngineFacade(
-                CONF.database.connection,
-                **dict(CONF.database.iteritems())
-            )
-        return _MASTER_FACADE
-    else:
-        if _SLAVE_FACADE is None:
-            _SLAVE_FACADE = db_session.EngineFacade(
-                CONF.database.slave_connection,
-                **dict(CONF.database.iteritems())
-            )
-        return _SLAVE_FACADE
+def _create_facade_lazily():
+    global _LOCK, _ENGINE_FACADE
+    if _ENGINE_FACADE is None:
+        with _LOCK:
+            if _ENGINE_FACADE is None:
+                _ENGINE_FACADE = db_session.EngineFacade.from_config(CONF)
+    return _ENGINE_FACADE
 
 
 def get_engine(use_slave=False):
-    facade = _create_facade_lazily(use_slave)
-    return facade.get_engine()
+    facade = _create_facade_lazily()
+    return facade.get_engine(use_slave=use_slave)
 
 
 def get_session(use_slave=False, **kwargs):
-    facade = _create_facade_lazily(use_slave)
-    return facade.get_session(**kwargs)
+    facade = _create_facade_lazily()
+    return facade.get_session(use_slave=use_slave, **kwargs)
 
 
 _SHADOW_TABLE_PREFIX = 'shadow_'
@@ -749,7 +725,7 @@ def floating_ip_get(context, id):
 
         if not result:
             raise exception.FloatingIpNotFound(id=id)
-    except DataError:
+    except db_exc.DBError:
         msg = _("Invalid floating ip id %s in request") % id
         LOG.warn(msg)
         raise exception.InvalidID(id=id)
@@ -1003,7 +979,7 @@ def _floating_ip_get_by_address(context, address, session=None):
 
         if not result:
             raise exception.FloatingIpNotFoundForAddress(address=address)
-    except DataError:
+    except db_exc.DBError:
         msg = _("Invalid floating IP %s in request") % address
         LOG.warn(msg)
         raise exception.InvalidIpAddressError(msg)
@@ -1306,7 +1282,7 @@ def _fixed_ip_get_by_address(context, address, session=None,
             result = result.filter_by(address=address).first()
             if not result:
                 raise exception.FixedIpNotFoundForAddress(address=address)
-        except DataError:
+        except db_exc.DBError:
             msg = _("Invalid fixed IP Address %s in request") % address
             LOG.warn(msg)
             raise exception.FixedIpInvalid(msg)
@@ -1345,7 +1321,7 @@ def fixed_ip_get_by_address_detailed(context, address):
         if not result:
             raise exception.FixedIpNotFoundForAddress(address=address)
 
-    except DataError:
+    except db_exc.DBError:
         msg = _("Invalid fixed IP Address %s in request") % address
         LOG.warn(msg)
         raise exception.FixedIpInvalid(msg)
@@ -1480,7 +1456,7 @@ def virtual_interface_get_by_address(context, address):
         vif_ref = _virtual_interface_query(context).\
                           filter_by(address=address).\
                           first()
-    except DataError:
+    except db_exc.DBError:
         msg = _("Invalid virtual interface address %s in request") % address
         LOG.warn(msg)
         raise exception.InvalidIpAddressError(msg)
@@ -1735,7 +1711,7 @@ def instance_get(context, instance_id, columns_to_join=None):
             raise exception.InstanceNotFound(instance_id=instance_id)
 
         return result
-    except DataError:
+    except db_exc.DBError:
         # NOTE(sdague): catch all in case the db engine chokes on the
         # id because it's too long of an int to store.
         msg = _("Invalid instance id %s in request") % instance_id
@@ -5729,7 +5705,9 @@ def archive_deleted_rows_for_table(context, tablename, max_rows):
         with conn.begin():
             conn.execute(insert_statement)
             result_delete = conn.execute(delete_statement)
-    except IntegrityError:
+    except db_exc.DBError:
+        # TODO(ekudryashova): replace by DBReferenceError when db layer
+        # raise it.
         # A foreign key constraint keeps us from deleting some of
         # these rows until we clean up a dependent table.  Just
         # skip this table for now; we'll come back to it later.

nova/db/sqlalchemy/migrate_repo/versions/230_add_details_column_to_instance_actions_events.py

@@ -12,10 +12,10 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from oslo.db.sqlalchemy import utils
 from sqlalchemy import Column, String, Text
 
 from nova.db.sqlalchemy import api
-from nova.openstack.common.db.sqlalchemy import utils
 
 
 def upgrade(migrate_engine):

nova/db/sqlalchemy/models.py

@@ -19,15 +19,15 @@
 SQLAlchemy models for nova data.
 """
 
+from oslo.config import cfg
+from oslo.db.sqlalchemy import models
 from sqlalchemy import Column, Index, Integer, BigInteger, Enum, String, schema
 from sqlalchemy.dialects.mysql import MEDIUMTEXT
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy import orm
 from sqlalchemy import ForeignKey, DateTime, Boolean, Text, Float
-from oslo.config import cfg
 
 from nova.db.sqlalchemy import types
-from nova.openstack.common.db.sqlalchemy import models
 from nova.openstack.common import timeutils
 
 CONF = cfg.CONF
@@ -43,6 +43,16 @@ class NovaBase(models.SoftDeleteMixin,
                models.ModelBase):
     metadata = None
 
+    # TODO(ekudryashova): remove this after both nova and oslo.db
+    # will use oslo.utils library
+    # NOTE: Both projects(nova and oslo.db) use `timeutils.utcnow`, which
+    # returns specified time(if override_time is set). Time overriding is
+    # only used by unit tests, but in a lot of places, temporarily overriding
+    # this columns helps to avoid lots of calls of timeutils.set_override
+    # from different places in unit tests.
+    created_at = Column(DateTime, default=lambda: timeutils.utcnow())
+    updated_at = Column(DateTime, onupdate=lambda: timeutils.utcnow())
+
     def save(self, session=None):
         from nova.db.sqlalchemy import api