Finer control over sqlite storage locking, oid allocation and stats. …

…This makes it consistently a few orders of magnitude faster in most benchmarks.

Also expose all the pragmas in the configuration so users can tune it appropriately for their environment. This changes the config schema a bit, so since we were doing that, clean that up a bit.

Documentation still needs updated.

src/relstorage/_compat.py

@@ -37,6 +37,7 @@
     # Constants
     'PY3',
     'PY2',
+    'PY36',
     'PYPY',
     'WIN',
     'MAC',
@@ -72,6 +73,7 @@
 ]
 
 PY3 = sys.version_info[0] == 3
+PY36 = sys.version_info[:2] >= (3, 6)
 PY2 = not PY3
 PYPY = platform.python_implementation() == 'PyPy'
 WIN = sys.platform.startswith('win')

src/relstorage/adapters/_util.py

@@ -12,6 +12,8 @@
 #
 ##############################################################################
 "Internal helper utilities."
+from __future__ import print_function
+from __future__ import absolute_import
 
 from collections import namedtuple
 from functools import partial
@@ -186,7 +188,11 @@ def _rows_as_pretty_string(self, cursor):
         """
         Return the rows formatted in a way for easy human consumption.
         """
-        # This could be tabular, but its easiest just to use pprint
-        import pprint
-        rows = list(self._rows_as_dicts(cursor))
-        return pprint.pformat(rows)
+        from .._compat import NStringIO
+        out = NStringIO()
+        names = [d.name for d in self._column_descriptions(cursor)]
+        kwargs = {'sep': '\t', 'file': out}
+        print(*names, **kwargs)
+        for row in cursor:
+            print(*row, **kwargs)
+        return out.getvalue()

src/relstorage/adapters/connections.py

@@ -286,6 +286,10 @@ class StoreConnection(AbstractManagedConnection):
     def begin(self):
         self.connmanager.begin(*self.open_if_needed())
 
+class PrePackConnection(StoreConnection):
+    __slots__ = ()
+    _NEW_CONNECTION_NAME = 'open_for_pre_pack'
+
 @implementer(interfaces.IManagedDBConnection)
 class ClosedConnection(object):
     """

src/relstorage/adapters/connmanager.py

@@ -45,10 +45,14 @@ class AbstractConnectionManager(object):
     # in practice they should.)
     _ignored_exceptions = ()
 
-    isolation_load = None
-    isolation_store = None
+    # Subclasses should set these to get semantics as close
+    # as possible to these standard levels.
     isolation_serializable = None
     isolation_read_committed = None
+    # If these are not set by a subclass, they will be copied
+    # from isolation_serializable and read_committed, respectively.
+    isolation_load = None
+    isolation_store = None
 
     replica_selector = None
     ro_replica_selector = None
@@ -79,8 +83,10 @@ def __init__(self, options, driver):
         else:
             self.ro_replica_selector = self.replica_selector
 
-        self.isolation_load = self.isolation_serializable
-        self.isolation_store = self.isolation_read_committed
+        if not self.isolation_load:
+            self.isolation_load = self.isolation_serializable
+        if not self.isolation_store:
+            self.isolation_store = self.isolation_read_committed
 
         self._may_need_rollback = driver.connection_may_need_rollback
         self._may_need_commit = driver.connection_may_need_commit
@@ -301,6 +307,9 @@ def open_for_pre_pack(self):
         """
         return self.open_for_store(application_name='RS prepack')
 
+    def open_for_pack_lock(self):
+        return self.open()
+
     def _do_open_for_store(self, **open_args):
         open_args['isolation'] = self.isolation_store
         open_args['read_only'] = False

src/relstorage/adapters/drivers.py

@@ -29,6 +29,7 @@
 from .._compat import PY3
 from .._compat import casefold
 from .._util import positive_integer
+from .._util import consume
 
 from .interfaces import IDBDriver
 from .interfaces import IDBDriverFactory
@@ -117,6 +118,8 @@ class AbstractModuleDriver(ABC):
     def __init__(self):
         if PYPY and not self.AVAILABLE_ON_PYPY:
             raise DriverNotAvailableError(self.__name__)
+        if not self.STATIC_AVAILABLE:
+            raise DriverNotAvailableError(self.__name__)
         try:
             self.driver_module = mod = self.get_driver_module()
         except ImportError:
@@ -205,9 +208,8 @@ def synchronize_cursor_for_rollback(self, cursor):
         # it in certain circumstances.
 
         if cursor is not None:
-            fetchall = cursor.fetchall
             try:
-                fetchall()
+                consume(cursor)
             except Exception: # pylint:disable=broad-except
                 pass
 

src/relstorage/adapters/interfaces.py

@@ -449,6 +449,21 @@ def open_for_pre_pack():
         :return: ``(conn, cursor)``
         """
 
+    def open_for_pack_lock():
+        """
+        Open a connection to be used for the sole purpose of holding
+        the pack lock.
+
+        Use a private connection (lock_conn and lock_cursor) to hold
+        the pack lock. Have the adapter open temporary connections
+        to do the actual work, allowing the adapter to use special
+        transaction modes for packing, and to commit at will without
+        losing the lock.
+
+        If the database doesn't actually use a pack lock,
+        this may return ``(None, None)``.
+        """
+
     def cursor_for_connection(conn):
         """
         If the cursor returned by an open method was discarded

src/relstorage/adapters/mover.py

@@ -25,6 +25,7 @@
 from .._compat import metricmethod_sampled
 from ._util import noop_when_history_free
 from ._util import query_property as _query_property
+from ._util import DatabaseHelpersMixin
 from .._compat import ABC
 from .batch import RowBatcher
 from .interfaces import IObjectMover
@@ -38,7 +39,7 @@
 
 
 @implementer(IObjectMover)
-class AbstractObjectMover(ABC):
+class AbstractObjectMover(DatabaseHelpersMixin, ABC):
 
     def __init__(self, database_driver, options, runner=None,
                  version_detector=None,
@@ -335,15 +336,12 @@ def restore(self, cursor, batcher, oid, tid, data):
     ).where(
         # Some databases may be able to benefit from prev_tid <> 0, but it depends
         # on their ability to make use of indexes
-        Schema.temp_store.c.prev_tid != Schema.all_current_object_state.c.tid
-    ).order_by(
-        Schema.temp_store.c.zoid
+        Schema.all_current_object_state.c.tid != Schema.temp_store.c.prev_tid
     ).prepared()
 
     @metricmethod_sampled
     def detect_conflict(self, cursor):
-        stmt = self._detect_conflict_query
-        stmt.execute(cursor)
+        self._detect_conflict_query.execute(cursor)
         # Note that we're not transforming the state into
         # bytes; it doesn't seem to be needed here, even with sqlite3
         # on Python 2 (where it is a buffer).

src/relstorage/adapters/packundo.py

@@ -32,7 +32,7 @@
 
 from .schema import Schema
 from .connections import LoadConnection
-from .connections import StoreConnection
+from .connections import PrePackConnection
 from .interfaces import IPackUndo
 from ._util import DatabaseHelpersMixin
 from .sql import it
@@ -634,7 +634,7 @@ def pre_pack(self, pack_tid, get_references):
         much faster.
         """
         load_connection = LoadConnection(self.connmanager)
-        store_connection = StoreConnection(self.connmanager)
+        store_connection = PrePackConnection(self.connmanager)
         try:
             # The pre-pack functions are responsible for managing
             # their own commits; when they return, the transaction
@@ -1297,7 +1297,7 @@ def pre_pack(self, pack_tid, get_references):
             return
 
         load_connection = LoadConnection(self.connmanager)
-        store_connection = StoreConnection(self.connmanager)
+        store_connection = PrePackConnection(self.connmanager)
         try:
             try:
                 self._pre_pack_main(load_connection, store_connection,

src/relstorage/adapters/sqlite/__init__.py

@@ -37,7 +37,8 @@
 Locks
 =====
 
-Sqlite3 only supports database-wide locks.
+Sqlite3 only supports database-wide write locks. For details on when and how
+they are taken and managed, see connmanager.py and locker.py
 """
 
 from __future__ import absolute_import

src/relstorage/adapters/sqlite/adapter.py

@@ -47,30 +47,22 @@ class Sqlite3Adapter(AbstractAdapter):
     driver_options = drivers
     WRITING_REQUIRES_EXCLUSIVE_LOCK = True
 
-    def __init__(self, path, options=None, oidallocator=None):
-        self.path = os.path.abspath(path)
+    def __init__(self, data_dir, pragmas, options=None, oidallocator=None):
+        self.data_dir = os.path.abspath(data_dir)
+        self.pragmas = pragmas
         self.oidallocator = oidallocator
         super(Sqlite3Adapter, self).__init__(options)
 
-    def _path_for_oids(self):
-        dirname = os.path.dirname(self.path)
-        root, _ = os.path.splitext(os.path.basename(self.path))
-        oid_name = root + '.oids'
-        return os.path.join(dirname, oid_name)
-
     def _create(self):
         driver = self.driver
         options = self.options
         self.connmanager = Sqlite3ConnectionManager(
             driver,
-            path=self.path,
+            path=os.path.join(self.data_dir, 'main.sqlite3'),
+            pragmas=self.pragmas,
             options=options
         )
-        oid_connman = Sqlite3ConnectionManager(
-            driver,
-            path=self._path_for_oids(),
-            options=options,
-        )
+
         self.mover = Sqlite3ObjectMover(
             driver,
             options=options,
@@ -82,8 +74,7 @@ def _create(self):
 
         if not self.oidallocator:
             self.oidallocator = Sqlite3OIDAllocator(
-                driver,
-                oid_connman
+                os.path.join(self.data_dir, 'oids.sqlite3')
             )
 
         self.runner = Sqlite3ScriptRunner()
@@ -95,7 +86,6 @@ def _create(self):
             transactions_may_go_backwards=False
         )
 
-
         self.txncontrol = Sqlite3TransactionControl(
             connmanager=self.connmanager,
             poller=self.poller,
@@ -106,8 +96,8 @@ def _create(self):
 
         self.schema = Sqlite3SchemaInstaller(
             driver=driver,
+            oid_allocator=self.oidallocator,
             connmanager=self.connmanager,
-            oid_connmanager=oid_connman,
             runner=self.runner,
             keep_history=self.keep_history
         )
@@ -144,7 +134,8 @@ def _create(self):
 
     def new_instance(self):
         inst = type(self)(
-            self.path,
+            self.data_dir,
+            self.pragmas,
             options=self.options,
             oidallocator=self.oidallocator.new_instance())
         return inst