Make zapping MySQL much faster

.travis.yml

@@ -44,7 +44,7 @@ script:
   - if [[ $TRAVIS_PYTHON_VERSION == 3.7 ]]; then pylint --rcfile=.pylintrc relstorage -f parseable -r n; fi
   # Disable inlining right now. See https://github.com/zodb/relstorage/issues/228#issuecomment-492423284
   - if [[ $TRAVIS_PYTHON_VERSION == pypy* ]]; then python --jit inlining=0 -m zope.testrunner --test-path=src --auto-color --auto-progress; fi
-  - if [[ $TRAVIS_PYTHON_VERSION != pypy* ]]; then coverage run -m  zope.testrunner --test-path=src --auto-color --auto-progress; fi
+  - if [[ $TRAVIS_PYTHON_VERSION != pypy* ]]; then coverage run -m zope.testrunner --test-path=src --auto-color --auto-progress; fi
 after_success:
   - coveralls
 notifications:

CHANGES.rst

@@ -8,6 +8,10 @@
 - Zapping a storage now also removes any persistent cache files. See
   :issue:`241`.
 
+- Zapping a MySQL storage now issues ``DROP TABLE`` statements instead
+  of ``DELETE FROM`` statements. This is much faster on large
+  databases. See :issue:`242`.
+
 3.0a2 (2019-06-19)
 ==================
 

appveyor.yml

@@ -96,7 +96,7 @@ build_script:
 test_script:
   - if not "%GWHEEL_ONLY%"=="true" cmd /c .travis\mysql.cmd
   - if not "%GWHEEL_ONLY%"=="true" cmd /c .travis\postgres.cmd
-  - if not "%GWHEEL_ONLY%"=="true" %PYEXE% -m zope.testrunner --test-path=src --auto-color
+  - if not "%GWHEEL_ONLY%"=="true" %PYEXE% -m zope.testrunner --test-path=src -vvv --color --slow-test 3
 
 after_test:
   - "%CMD_IN_ENV% %PYEXE% setup.py bdist_wheel -d dist"

src/relstorage/adapters/mysql/schema.py

@@ -22,6 +22,7 @@
 from ..interfaces import ISchemaInstaller
 from ..schema import AbstractSchemaInstaller
 
+logger = __import__('logging').getLogger(__name__)
 
 @implementer(ISchemaInstaller)
 class MySQLSchemaInstaller(AbstractSchemaInstaller):
@@ -62,9 +63,9 @@ def _create_pack_lock(self, cursor):
     COLTYPE_BINARY_STRING = 'BLOB'
     TRANSACTIONAL_TABLE_SUFFIX = 'ENGINE = InnoDB'
 
-    # As usual, MySQL has an annoying implementation of this and we
+    # As usual, MySQL has a quirky implementation of this feature and we
     # have to re-specify *everything* about the column. MySQL 8 supports the
-    # simple 'RENAME ... TO ... syntax that everyone else does.
+    # simple 'RENAME ... TO ...' syntax that everyone else does.
     _rename_transaction_empty_stmt = (
         "ALTER TABLE transaction CHANGE empty is_empty "
         "BOOLEAN NOT NULL DEFAULT FALSE"
@@ -239,14 +240,19 @@ def _create_temp_pack_visit(self, _cursor):
     def _create_temp_undo(self, _cursor):
         return
 
-    def _init_after_create(self, cursor):
-        if self.keep_history:
-            stmt = """
-            INSERT INTO transaction (tid, username, description)
-            VALUES (0, 'system', 'special transaction for object creation');
-            """
-            self.runner.run_script(cursor, stmt)
-
     def _reset_oid(self, cursor):
         stmt = "TRUNCATE new_oid;"
         self.runner.run_script(cursor, stmt)
+
+    # We can't TRUNCATE tables that have foreign-key relationships
+    # with other tables, but we can drop them. This has to be followed up by
+    # creating them again.
+    _zap_all_tbl_stmt = 'DROP TABLE %s'
+
+    def _after_zap_all_tables(self, cursor, slow=False):
+        if not slow:
+            logger.debug("Creating tables after drop")
+            self.create(cursor)
+            logger.debug("Done creating tables after drop")
+        else:
+            super(MySQLSchemaInstaller, self)._after_zap_all_tables(cursor, slow)

src/relstorage/adapters/postgresql/adapter.py

@@ -210,15 +210,19 @@ def new_instance(self):
         return inst
 
     def __str__(self):
-        parts = [self.__class__.__name__]
+        parts = []
         if self.keep_history:
             parts.append('history preserving')
         else:
             parts.append('history free')
         dsnparts = self._dsn.split()
         s = ' '.join(p for p in dsnparts if not p.startswith('password'))
         parts.append('dsn=%r' % s)
-        return ", ".join(parts)
+        return "<%s at %x %s>" % (
+            self.__class__.__name__, id(self), ",".join(parts)
+        )
+
+    __repr__ = __str__
 
 
 

src/relstorage/adapters/schema.py

@@ -380,7 +380,8 @@ def _needs_transaction_empty_update(self, cursor):
 
     _rename_transaction_empty_stmt = 'ALTER TABLE transaction RENAME COLUMN empty TO is_empty'
 
-    _zap_all_tbl_stmt = 'DELETE FROM %s'
+    # Subclasses can redefine these.
+    _slow_zap_all_tbl_stmt = _zap_all_tbl_stmt = 'DELETE FROM %s'
 
     def zap_all(self, reset_oid=True, slow=False):
         """
@@ -391,24 +392,24 @@ def zap_all(self, reset_oid=True, slow=False):
             DELETEd. This is helpful when other connections might be open and
             holding some kind of locks.
         """
-        stmt = self._zap_all_tbl_stmt if not slow else AbstractSchemaInstaller._zap_all_tbl_stmt
+        stmt = self._zap_all_tbl_stmt if not slow else self._slow_zap_all_tbl_stmt
 
         def callback(_conn, cursor):
             existent = set(self.list_tables(cursor))
-            todo = reversed(self.all_tables)
+            todo = list(self.all_tables)
+            todo.reverse() # using reversed()  doesn't print nicely
             log.debug("Checking tables: %r", todo)
             for table in todo:
                 log.debug("Considering table %s", table)
                 if table.startswith('temp_'):
                     continue
                 if table in existent:
-                    log.debug("Deleting from table %s...", table)
-                    cursor.execute(stmt % table)
+                    table_stmt = stmt % table
+                    log.debug(table_stmt)
+                    cursor.execute(table_stmt)
             log.debug("Done deleting from tables.")
 
-            log.debug("Running init script.")
-            self._init_after_create(cursor)
-            log.debug("Done running init script.")
+            self._after_zap_all_tables(cursor, slow)
 
             if reset_oid:
                 log.debug("Running OID reset script.")
@@ -417,6 +418,11 @@ def callback(_conn, cursor):
 
         self.connmanager.open_and_call(callback)
 
+    def _after_zap_all_tables(self, cursor, slow=False):
+        log.debug("Running init script. Slow: %s", slow)
+        self._init_after_create(cursor)
+        log.debug("Done running init script.")
+
 
     def drop_all(self):
         """Drop all tables and sequences."""

src/relstorage/tests/reltestbase.py

@@ -122,7 +122,10 @@ def make_storage(self, zap=True, **kw):
             # with them? This leads to connections remaining open with
             # locks on PyPy, so on PostgreSQL we can't TRUNCATE tables
             # and have to go the slow route.
-            storage.zap_all(slow=True)
+            #
+            # As of 2019-06-20 with PyPy 7.1.1, I'm no longer able to replicate
+            # a problem like that locally, so we go back to the fast way.
+            storage.zap_all()
         return self._wrap_storage(storage)
 
 

src/relstorage/tests/testpostgresql.py

@@ -25,28 +25,28 @@
 class PostgreSQLAdapterMixin(object):
 
     def make_adapter(self, options, db=None):
-        if db is None:
-            if self.keep_history:
-                db = self.base_dbname
-            else:
-                db = self.base_dbname + '_hf'
         return PostgreSQLAdapter(
-            dsn='dbname=%s user=relstoragetest password=relstoragetest' % db,
+            dsn=self.__get_adapter_zconfig_dsn(db),
             options=options,
         )
 
     def get_adapter_class(self):
         return PostgreSQLAdapter
 
-    def __get_adapter_zconfig_dsn(self):
-        if self.keep_history:
-            dbname = self.base_dbname
-        else:
-            dbname = self.base_dbname + '_hf'
+    def __get_adapter_zconfig_dsn(self, dbname=None):
+        if dbname is None:
+            if self.keep_history:
+                dbname = self.base_dbname
+            else:
+                dbname = self.base_dbname + '_hf'
         dsn = (
             "dbname='%s' user='relstoragetest' password='relstoragetest'"
             % dbname
         )
+        # psycopg2cffi can have a unix socket path hardcoded in it,
+        # and that path may not be right
+        if 'cffi' in self.driver_name.lower():
+            dsn += " host='127.0.0.1'"
         return dsn
 
     def get_adapter_zconfig(self):

src/relstorage/tests/util.py

@@ -275,9 +275,11 @@ def create_storage(name, blob_dir,
                         **kw)
 
                     adapter_maker = self.use_adapter()
+                    adapter_maker.driver_name = driver_name
                     adapter = adapter_maker.make_adapter(options, db)
+                    __traceback_info__ = adapter, options
                     storage = RelStorage(adapter, name=name, options=options)
-                    storage.zap_all(slow=True)
+                    storage.zap_all()
                     return storage
 
                 prefix = '%s_%s%s' % (