Merge 1c5adf8 into 3dba61e

.pylintrc

@@ -90,7 +90,7 @@ generated-members=exc_clear
 # (useful for classes with attributes dynamically set). This supports can work
 # with qualified names.
 
-ignored-classes=SectionValue
+ignored-classes=SectionValue,Lazy,_v_c
 
 # List of module names for which member attributes should not be checked
 # (useful for modules/projects where namespaces are manipulated during runtime

src/relstorage/_util.py

@@ -371,6 +371,7 @@ def __get__(self, inst, class_):
         inst.__dict__[name] = value
         return value
 
+
 class CachedIn(object):
     """Cached method with given cache attribute."""
 

src/relstorage/adapters/__init__.py

@@ -1 +1,36 @@
-"""relstorage.adapters package"""
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+# Copyright (c) 2019 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""
+Abstractions for using particular RDBMS implementations.
+
+The interfaces in this package define what operations RelStorage needs
+to work with a particular database. Most modules then have an Abstract
+(partial) implementation of one such interface. The central
+``IRelStorageAdapter`` interface ties them all together.
+
+To use a new RDBMS, create a new package named for it, containing a
+module named like each module found here to store the
+implementation-specific class that subclasses the abstract version
+found here.
+
+Decisions that need to be made include:
+
+- How to allocate OIDs.
+- How to handle locking (commit and pack)
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function

src/relstorage/adapters/adapter.py

@@ -49,6 +49,7 @@ class AbstractAdapter(object):
     txncontrol = None # type: ITransactionControl
     mover = None # type: IObjectMover
     connmanager = None # type: IConnectionManager
+    oidallocator = None # type: IOIDAllocator
 
     def __init__(self, options=None):
         if options is None:
@@ -68,6 +69,16 @@ def __init__(self, options=None):
     def _create(self):
         raise NotImplementedError
 
+    def release(self):
+        if self.oidallocator is not None:
+            self.oidallocator.release()
+            self.oidallocator = None
+
+    def close(self):
+        if self.oidallocator is not None:
+            self.oidallocator.close()
+            self.oidallocator = None
+
     def _select_driver(self, options=None):
         return _select_driver(
             options or self.options or Options(),
@@ -152,11 +163,13 @@ def lock_database_and_move(self,
         return committing_tid_int, prepared_txn_id
 
     DEFAULT_LOCK_OBJECTS_AND_DETECT_CONFLICTS_INTERLEAVABLE = True
+    WRITING_REQUIRES_EXCLUSIVE_LOCK = False
 
     # Hooks for unit tests.
     force_lock_objects_and_detect_conflicts_interleavable = False
     force_lock_readCurrent_for_share_blocking = False
 
+
     @metricmethod_sampled
     def lock_objects_and_detect_conflicts(self, cursor, read_current_oids):
         if (

src/relstorage/adapters/batch.py

@@ -19,6 +19,9 @@
 from relstorage._compat import iteritems
 from relstorage._util import parse_byte_size
 
+# A cache
+# {(placeholder, count): "string"}
+_row_schemas = {}
 
 class RowBatcher(object):
     """
@@ -38,13 +41,19 @@ class RowBatcher(object):
     # what value we have)
     size_limit = parse_byte_size('2 MB')
     delete_placeholder = '%s'
+    insert_placeholder = '%s'
     # For testing, force the delete order to be deterministic
     # when multiple columns are involved
     sorted_deletes = False
 
-    def __init__(self, cursor, row_limit=None, delete_placeholder="%s"):
+    def __init__(self, cursor, row_limit=None,
+                 delete_placeholder=None,
+                 insert_placeholder=None):
         self.cursor = cursor
-        self.delete_placeholder = delete_placeholder
+        if delete_placeholder is not None:
+            self.delete_placeholder = delete_placeholder
+        if insert_placeholder is not None:
+            self.insert_placeholder = insert_placeholder
         if row_limit is not None:
             self.row_limit = row_limit
 
@@ -93,6 +102,17 @@ def insert_into(self, header, row_schema, row, rowkey, size,
                 or self.size_added >= self.size_limit):
             self.flush()
 
+    def row_schema_of_length(self, param_count):
+        # Use as the *row_schema* parameter to insert_into
+        key = (self.insert_placeholder, param_count)
+        try:
+            return _row_schemas[key]
+        except KeyError:
+            p = [self.insert_placeholder] * param_count
+            p = ', '.join(p)
+            _row_schemas[key] = p
+            return p
+
     def select_from(self, columns, table, suffix='', **kw):
         """
         Handles a query of the ``WHERE col IN (?, ?,)`` type.
@@ -198,5 +218,6 @@ def _do_inserts(self):
             # INSERT INTO table(c1, c2)
             # VALUES (%s, %s), (%s, %s), (%s, %s)
             # <suffix>
+            __traceback_info__ = stmt
             self.cursor.execute(stmt, params)
         return count

src/relstorage/adapters/connections.py

@@ -72,8 +72,7 @@ def __bool__(self):
 
     __nonzero__ = __bool__
 
-    @Lazy
-    def cursor(self):
+    def get_cursor(self):
         if not self.active or self._cursor is None:
             # XXX: If we've been explicitly dropped, do we always want to
             # automatically re-open? Probably not; bad things could happen:
@@ -84,6 +83,8 @@ def cursor(self):
             self.active = True
         return self._cursor
 
+    cursor = Lazy(get_cursor, 'cursor')
+
     def enter_critical_phase_until_transaction_end(self):
         """
         Given an already opened connection, cause it to attempt to
@@ -153,7 +154,7 @@ def restart(self):
         Restart the connection if there is any chance that it has any associated state.
         """
         if not self:
-            assert not self.active
+            assert not self.active, self.__dict__
             return
 
         # If we've never accessed the cursor, we shouldn't have any
@@ -283,6 +284,8 @@ class StoreConnection(AbstractManagedConnection):
     _RESTART_NAME = 'restart_store'
     _ROLLBACK_NAME = 'rollback_store_quietly'
 
+    def begin(self):
+        self.connmanager.begin(*self.open_if_needed())
 
 @implementer(interfaces.IManagedDBConnection)
 class ClosedConnection(object):

src/relstorage/adapters/connmanager.py

@@ -60,6 +60,7 @@ def __init__(self, options, driver):
         """
         self.keep_history = options.keep_history
         self.driver = driver
+        self.options = options
 
         self._ignored_exceptions = tuple(set(
             driver.close_exceptions
@@ -219,6 +220,9 @@ def commit(self, conn, cursor=None, force=False):
         if self._may_need_commit(conn) or force:
             self._do_commit(conn, cursor)
 
+    def begin(self, conn, cursor):
+        pass
+
     def _do_open_for_call(self, callback): # pylint:disable=unused-argument
         return self.open()
 

src/relstorage/adapters/drivers.py

@@ -26,6 +26,7 @@
 
 from .._compat import ABC
 from .._compat import PYPY
+from .._compat import PY3
 from .._compat import casefold
 from .._util import positive_integer
 
@@ -215,6 +216,7 @@ def synchronize_cursor_for_rollback(self, cursor):
 
     # Py MySQL Connector/Python returns a bytearray, whereas
     # C MySQL Connector/Python returns bytes.
+    # sqlite uses buffer on Py2 and memoryview on Py3.
 
     # Keep these ordered with the most common at the front;
     # Python does a linear traversal of type checks.
@@ -223,10 +225,9 @@ def synchronize_cursor_for_rollback(self, cursor):
     def binary_column_as_state_type(self, data):
         if isinstance(data, self.state_types) or data is None:
             return data
-        # Nothing we know about. cx_Oracle likes to give us an object
-        # with .read(), look for that.
-        # XXX: TODO: Move this to the oracle driver.
-        return self.binary_column_as_state_type(data.read())
+        __traceback_info__ = type(data), data
+        raise TypeError("Unknown binary state column")
+
 
     def binary_column_as_bytes(self, data):
         # Take the same inputs as `as_state_type`, but turn them into
@@ -248,6 +249,36 @@ def binary_column_as_bytes(self, data):
     def enter_critical_phase_until_transaction_end(self, connection, cursor):
         """Default implementation; does nothing."""
 
+
+class MemoryViewBlobDriverMixin(object):
+    # psycopg2 is smart enough to return memoryview or buffer on
+    # Py3/Py2, respectively, for BYTEa columns. sqlite3 does exactly
+    # the same for BLOB columns (on Python 2; on Python 3 it returns
+    # bytes instead of buffer), and defines ``Binary`` that way as
+    # well.
+
+    # memoryview can't be passed to bytes() on Py2 or Py3, but it can
+    # be passed to cStringIO.StringIO() or io.BytesIO() ---
+    # unfortunately, memoryviews, at least, don't like going to
+    # io.BytesIO() on Python 3, and that's how we unpickle states. So
+    # while ideally we'd like to keep it that way, to save a copy, we
+    # are forced to make the copy. Plus there are tests that like to
+    # directly compare bytes.
+
+    if PY3:
+        def binary_column_as_state_type(self, data):
+            if data:
+                # Calling 'bytes()' on a memoryview in Python 3 does
+                # nothing useful.
+                data = data.tobytes()
+            return data
+    else:
+        def binary_column_as_state_type(self, data):
+            if data:
+                data = bytes(data)
+            return data
+
+
 @implementer(IDBDriverFactory)
 class _ClassDriverFactory(object):
 

src/relstorage/adapters/interfaces.py

@@ -349,6 +349,14 @@ def rollback_quietly(conn, cursor):
             are closed before this method returns).
         """
 
+    def begin(conn, cursor):
+        """
+        Call this on a store connection after restarting it.
+
+        This lets the store connection know that it may need to begin a
+        transaction, even if it was freshly opened.
+        """
+
     def open_and_call(callback):
         """Call a function with an open connection and cursor.
 
@@ -754,10 +762,13 @@ def restore(cursor, batcher, oid, tid, data):
 
     def detect_conflict(cursor):
         """
-        Find all conflicts in the data about to be committed.
+        Find all conflicts in the data about to be committed (as stored
+        by :meth:`store_temps`)
 
-        If there is a conflict, returns a sequence of
-        ``(oid, committed_tid, attempted_committed_tid, committed_state)``.
+        Returns a sequence of
+        ``(oid, committed_tid, attempted_committed_tid, committed_state)`` where
+        each entry refers to a conflicting object. The *committed_state* **must** be
+        returned.
 
         This method should be called during the ``tpc_vote`` phase of a transaction,
         with :meth:`ILocker.lock_current_objects` held.
@@ -857,6 +868,11 @@ def set_min_oid(cursor, oid_int):
         :meth:`new_oids`) is greater than the given *oid_int*.
         """
 
+    def reset_oid(cursor):
+        """
+        Cause the sequence of OIDs to begin again from the beginning.
+        """
+
 
 class IPackUndo(Interface):
     """Perform pack and undo operations"""
@@ -1148,10 +1164,20 @@ def new_instance():
         """
         Return an instance for use by another RelStorage instance.
 
-        Adapters that are stateless can simply return self.  Adapters
+        Adapters that are stateless can simply return self. Adapters
         that have mutable state must make a clone and return it.
         """
 
+    def release():
+        """
+        Release the resources held uniquely by this instance.
+        """
+
+    def close():
+        """
+        Release the resources held by this instance and all child instances.
+        """
+
     def __str__():
         """Return a short description of the adapter"""
 

src/relstorage/adapters/locker.py

@@ -198,7 +198,7 @@ def lock_current_objects(self, cursor, read_current_oid_ints, shared_locks_block
     def _lock_readCurrent_oids_for_share(self, cursor, current_oids, shared_locks_block):
         _, table = self._get_current_objects_query
         oids_to_lock = sorted(set(current_oids))
-        batcher = self.make_batcher(cursor, row_limit=1000)
+        batcher = self.make_batcher(cursor)
 
         locking_suffix = ' %s ' % (
             self._lock_share_clause