Cleanups

Comments about unicode for mysql connector should match reality, I hope.

Remove mysql_connection() and just pass the connection object to the one
place that needs it.

relstorage/_compat.py

@@ -78,17 +78,6 @@ def db_binary_to_bytes(data):
     return data
 
 
-# mysqlclient, a binary driver that works for Py2, Py3 and
-# PyPy (claimed), uses a connection that is a weakref. MySQLdb
-# and PyMySQL use a hard reference
-from weakref import ref as _wref
-def mysql_connection(cursor):
-    conn = cursor.connection
-    if isinstance(conn, _wref):
-        conn = conn()
-    return conn
-
-
 
 from ZODB._compat import BytesIO
 StringIO = BytesIO

relstorage/adapters/mysql/drivers.py

@@ -145,59 +145,47 @@ def escape_string(value, mapping=None):
 
     @implementer(IDBDriver)
     class MySQLConnectorDriver(AbstractDriver):
+        # See https://github.com/zodb/relstorage/issues/155
         __name__ = "mysqlconnector"
 
         disconnected_exceptions, close_exceptions, lock_exceptions = _standard_exceptions(mysql.connector)
         use_replica_exceptions = (mysql.connector.OperationalError,)
         Binary = staticmethod(mysql.connector.Binary)
-        _have_cext = mysql.connector.HAVE_CEXT
+
+        have_cext = mysql.connector.HAVE_CEXT
         _connect = staticmethod(mysql.connector.connect)
 
         def connect(self, *args, **kwargs):
             # It defaults to the (slower) pure-python version
             # NOTE: The C implementation doesn't support the prepared
             # operations.
-            # NOTE: The C implementation returns bytes when the py implementation
+            # NOTE: The C implementation returns bytes when the Py implementation
             # returns bytearray under Py2
-            # NOTE: The py implementation can return unicode strings (on OS X only?)
-            # where everything else returns native strings
-            if self._have_cext and 'gevent' not in sys.modules:
+
+            if self.have_cext:
                 kwargs['use_pure'] = False
             if PY2:
-                # The docs say that strings are returned as unicode, but this
-                # only seems to happen on OS X (the travis CI runs don't have this;
-                # maybe it got the C extension by default?)
-                # It also seems that only the C connector pays attention to this
-                # argument (despite the documentation); the python version
-                # requires the set_unicode(False) call
+                # The docs say that strings are returned as unicode by default
+                # an all platforms, but this is inconsistent. We need str anyway.
                 kwargs['use_unicode'] = False
             con = self._connect(*args, **kwargs)
-            if PY2:
-                con.set_unicode(False)
 
             return con
 
 
         def set_autocommit(self, conn, value):
-            # We use a property instead of a method
+            # This implementation uses a property instead of a method.
             conn.autocommit = value
 
         def cursor(self, conn):
             # By default, the cursor won't buffer, so we don't know
             # how many rows there are. That's fine and within the DB-API spec.
             # The Python implementation is much faster if we don't ask it to.
             # The C connection doesn't accept the 'prepared' keyword.
-            # You can't have both a buffered and prepared cursor.
+            # You can't have both a buffered and prepared cursor,
+            # but the prepared cursor doesn't gain us anything anyway.
 
             cursor = conn.cursor()
-            if not hasattr(cursor, 'connection'):
-                # We depend on the reverse mapping in some places.
-                # The python implementation keeps a weakref proxy in
-                # _connection, but if we try to access that on the C extension,
-                # we get a AttributeError, so if we then try to access 'connection',
-                # it aborts the process. So we go ahead and make a hard ref.
-                # See mysql_connection().
-                cursor.connection = conn
             return cursor
 
     del mysql.connector
@@ -208,12 +196,12 @@ def cursor(self, conn):
     if not preferred_driver_name:
         preferred_driver_name = driver.__name__
 
-    if driver._have_cext:
+    if driver.have_cext:
         driver_map['c' + driver.__name__] = driver
 
         class PyMySQLConnectorDriver(MySQLConnectorDriver):
             __name__ = 'py' + driver.__name__
-            _have_cext = False
+            have_cext = False
 
         driver = PyMySQLConnectorDriver()
         driver_map[driver.__name__] = driver

relstorage/adapters/packundo.py

@@ -27,7 +27,6 @@
 import time
 
 from relstorage._compat import db_binary_to_bytes
-from relstorage._compat import mysql_connection
 
 log = logging.getLogger(__name__)
 
@@ -142,13 +141,15 @@ def upload_batch():
         if batch:
             upload_batch()
 
-    def _pause_pack_until_lock(self, cursor, sleep):
-        """Pause until we can obtain a nowait commit lock."""
+    def _pause_pack_until_lock(self, conn, cursor, sleep):
+        """
+        Pause until we can obtain a nowait commit lock.
+        """
         if sleep is None:
             sleep = time.sleep
         delay = self.options.pack_commit_busy_delay
         while not self.locker.hold_commit_lock(cursor, nowait=True):
-            mysql_connection(cursor).rollback()
+            conn.rollback()
             log.debug('pack: commit lock busy, sleeping %.4g second(s)', delay)
             sleep(delay)
 
@@ -692,7 +693,7 @@ def pack(self, pack_tid, sleep=None, packed_func=None):
                 # We'll report on progress in at most .1% step increments
                 reportstep = max(total / 1000, 1)
 
-                self._pause_pack_until_lock(cursor, sleep)
+                self._pause_pack_until_lock(conn, cursor, sleep)
                 for tid, packed, has_removable in tid_rows:
                     self._pack_transaction(
                         cursor, pack_tid, tid, packed, has_removable,
@@ -712,7 +713,7 @@ def pack(self, pack_tid, sleep=None, packed_func=None):
                             lastreport = counter / reportstep * reportstep
                         del packed_list[:]
                         self.locker.release_commit_lock(cursor)
-                        self._pause_pack_until_lock(cursor, sleep)
+                        self._pause_pack_until_lock(conn, cursor, sleep)
                         start = time.time()
                 if packed_func is not None:
                     for oid, tid in packed_list:
@@ -806,7 +807,7 @@ def _pack_cleanup(self, conn, cursor, sleep=None):
         # We'll do it in batches of 1000 rows.
         log.debug("pack: removing empty packed transactions")
         while True:
-            self._pause_pack_until_lock(cursor, sleep)
+            self._pause_pack_until_lock(conn, cursor, sleep)
             stmt = self._script_delete_empty_transactions_batch
             self.runner.run_script_stmt(cursor, stmt)
             deleted = cursor.rowcount
@@ -1092,7 +1093,7 @@ def pack(self, pack_tid, sleep=None, packed_func=None):
                 # We'll report on progress in at most .1% step increments
                 lastreport, reportstep = 0, max(total / 1000, 1)
 
-                self._pause_pack_until_lock(cursor, sleep)
+                self._pause_pack_until_lock(conn, cursor, sleep)
                 while to_remove:
                     items = to_remove[:100]
                     del to_remove[:100]
@@ -1115,7 +1116,7 @@ def pack(self, pack_tid, sleep=None, packed_func=None):
                                      counter, counter / float(total) * 100)
                             lastreport = counter / reportstep * reportstep
                         self.locker.release_commit_lock(cursor)
-                        self._pause_pack_until_lock(cursor, sleep)
+                        self._pause_pack_until_lock(conn, cursor, sleep)
                         start = time.time()
 
                 if packed_func is not None:

relstorage/tests/testpostgresql.py

@@ -174,8 +174,8 @@ def test_suite():
     suite.addTest(unittest.makeSuite(HPPostgreSQLDestZODBConvertTests))
     suite.addTest(unittest.makeSuite(HPPostgreSQLSrcZODBConvertTests))
 
-    from .util import RUNNING_ON_CI
-    if RUNNING_ON_CI or os.environ.get("RS_PG_SMALL_BLOB"):
+    from .util import USE_SMALL_BLOBS
+    if USE_SMALL_BLOBS:
         # Avoid creating 2GB blobs to be friendly to neighbors
         # and to run fast (2GB blobs take about 4 minutes on Travis
         # CI as-of June 2016)

relstorage/tests/util.py

@@ -1,5 +1,5 @@
 import os
-
+import platform
 import unittest
 
 
@@ -37,3 +37,8 @@ def dec(f):
             CACHE_MODULE_NAME = 'memcache'
         except ImportError:
             pass
+
+USE_SMALL_BLOBS = ((RUNNING_ON_CI # slow here
+                    or platform.system() == 'Darwin' # interactive testing
+                    or os.environ.get("RS_SMALL_BLOB")) # define
+                   and not os.environ.get('RS_LARGE_BLOB'))