Merge pull request #74 from zodb/external-gc

Implement IExternalGC for history-free schemas. Fixes #47.

CHANGES.rst

@@ -94,6 +94,9 @@ Other Enhancements
 - ``zodbconvert`` and ``zodbpack`` use :mod:`argparse` instead of
   :mod:`optparse` for command line handling.
 
+- Support :class:`ZODB.interfaces.IExternalGC` for history-free
+  databases, allowing multi-database garbage collection with
+  ``zc.zodbdgc``. See :issue:`47`.
 
 1.6.0 (2016-06-09)
 ==================

relstorage/adapters/packundo.py

@@ -15,6 +15,7 @@
 """
 
 from ZODB.POSException import UndoError
+from ZODB.utils import u64
 from perfmetrics import metricmethod
 from relstorage.adapters.interfaces import IPackUndo
 from relstorage.iter import fetchmany
@@ -1189,7 +1190,7 @@ def pack(self, pack_tid, sleep=None, packed_func=None):
                         counter = total - len(to_remove)
                         if counter >= lastreport + reportstep:
                             log.info("pack: removed %d (%.1f%%) state(s)",
-                                counter, counter/float(total)*100)
+                                     counter, counter/float(total)*100)
                             lastreport = counter / reportstep * reportstep
                         self.locker.release_commit_lock(cursor)
                         self._pause_pack_until_lock(cursor, sleep)
@@ -1242,6 +1243,21 @@ def _pack_cleanup(self, conn, cursor):
         """
         self.runner.run_script(cursor, stmt)
 
+    def deleteObject(self, cursor, oid, oldserial):
+        # The only things to worry about are object_state and blob_chuck.
+        # blob chunks are deleted automatically by a foreign key.
+
+        # We shouldn't *have* to verify the oldserial in the delete statement,
+        # because our only consumer is zc.zodbdgc which only calls us for
+        # unreachable objects, so they shouldn't be modified and get a new
+        # TID. But this is safer.
+        state = """
+        DELETE FROM object_state
+        WHERE zoid = %(oid)s
+        and tid = %(tid)s
+        """
+        self.runner.run_script_stmt(cursor, state, {'oid': u64(oid), 'tid': u64(oldserial)})
+        return cursor.rowcount
 
 class MySQLHistoryFreePackUndo(HistoryFreePackUndo):
 

relstorage/storage.py

@@ -22,6 +22,8 @@
 from ZODB.BaseStorage import DataRecord
 from ZODB.BaseStorage import TransactionRecord
 from ZODB.POSException import POSKeyError
+from ZODB.POSException import ReadOnlyError
+from ZODB.POSException import StorageTransactionError
 from ZODB.UndoLogCompatible import UndoLogCompatible
 from ZODB.blob import is_blob_record
 from ZODB.interfaces import StorageStopIteration
@@ -38,6 +40,7 @@
 from relstorage.options import Options
 
 from zope.interface import implementer
+from zope import interface
 
 import ZODB.interfaces
 import logging
@@ -82,10 +85,8 @@ def _from_latin1(data):
              ZODB.interfaces.IStorageUndoable,
              ZODB.interfaces.IBlobStorage,
              ZODB.interfaces.IBlobStorageRestoreable)
-class RelStorage(
-        UndoLogCompatible,
-        ConflictResolution.ConflictResolvingStorage
-        ):
+class RelStorage(UndoLogCompatible,
+                 ConflictResolution.ConflictResolvingStorage):
     """Storage to a relational database, based on invalidation polling"""
 
     _transaction = None  # Transaction that is being committed
@@ -222,6 +223,13 @@ def __init__(self, adapter, name=None, create=None,
         elif options.blob_dir:
             self.blobhelper = BlobHelper(options=options, adapter=adapter)
 
+        if hasattr(self._adapter.packundo, 'deleteObject'):
+            interface.alsoProvides(self, ZODB.interfaces.IExternalGC)
+        else:
+            def deleteObject(*args):
+                raise AttributeError("deleteObject")
+            self.deleteObject = deleteObject
+
     def new_instance(self):
         """Creates and returns another storage instance.
 
@@ -677,17 +685,50 @@ def checkCurrentSerialInTransaction(self, oid, serial, transaction):
                     oid=oid, serials=(previous_serial, serial))
         self._txn_check_serials[oid] = serial
 
+    def deleteObject(self, oid, oldserial, transaction): # pylint:disable=method-hidden
+        # NOTE: packundo.deleteObject is only defined for
+        # history-free schemas. For other schemas, the __init__ function
+        # overrides this method.
+        # This method is only expected to be called from zc.zodbdgc
+        # currently.
+        if self._is_read_only: # pragma: no cover
+            raise ReadOnlyError()
+        # This is called in a phase of two-phase-commit (tpc).
+        # This means we have a transaction, and that we are holding
+        # the commit lock as well as the regular lock.
+        # RelStorage native pack uses a separate pack lock, but
+        # unfortunately there's no way to not hold the commit lock;
+        # however, the transactions are very short.
+        if transaction is not self._transaction: # pragma: no cover
+            raise StorageTransactionError(self, transaction)
+
+        # We don't worry about anything in self._cache because
+        # by definition we are deleting objects that were
+        # not reachable and so shouldn't be in the cache (or if they
+        # were, we'll never ask for them anyway)
+
+        # We delegate the actual operation to the adapter's packundo,
+        # just like native pack
+        cursor = self._store_cursor
+        assert cursor is not None
+        # When this is done, we get a tpc_vote,
+        # and a tpc_finish.
+        # The interface doesn't specify a return value, so for testing
+        # we return the count of rows deleted (should be 1 if successful)
+        return self._adapter.packundo.deleteObject(cursor, oid, oldserial)
+
+
     @metricmethod
     def tpc_begin(self, transaction, tid=None, status=' '):
         if self._stale_error is not None:
             raise self._stale_error
         if self._is_read_only:
-            raise POSException.ReadOnlyError()
+            raise ReadOnlyError()
         self._lock.acquire()
         try:
             if self._transaction is transaction:
                 if self._options.strict_tpc:
-                    raise POSException.StorageTransactionError(
+                    raise StorageTransactionError(
                         "Duplicate tpc_begin calls for same transaction")
                 return
             self._lock.release()

relstorage/tests/hftestbase.py

@@ -17,6 +17,7 @@
 from relstorage.tests.RecoveryStorage import UndoableRecoveryStorage
 from relstorage.tests.reltestbase import GenericRelStorageTests
 from relstorage.tests.reltestbase import RelStorageTestBase
+from ZODB.DB import DB
 from ZODB.FileStorage import FileStorage
 from ZODB.serialize import referencesf
 from ZODB.tests.ConflictResolution import PCounter
@@ -249,6 +250,35 @@ def checkResolve(self):
         self.assertEqual(inst._value, 5)
 
 
+    def checkImplementsExternalGC(self):
+        from zope.interface.verify import verifyObject
+        import ZODB.interfaces
+        verifyObject(ZODB.interfaces.IExternalGC, self._storage)
+
+        # Now do it.
+        from ZODB.utils import z64
+        import transaction
+        db = ZODB.DB(self._storage) # create the root
+        conn = db.open()
+        storage = conn._storage
+        _state, tid = storage.load(z64)
+
+        # copied from zc.zodbdgc
+        t = transaction.begin()
+        storage.tpc_begin(t)
+        try:
+            count = storage.deleteObject(z64, tid, t)
+            self.assertEqual(count, 1)
+            # Doing it again will do nothing because it's already
+            # gone.
+            count = storage.deleteObject(z64, tid, t)
+            self.assertEqual(count, 0)
+        finally:
+            transaction.abort()
+            conn.close()
+            db.close()
+
+
 class HistoryFreeToFileStorage(
         RelStorageTestBase,
         BasicRecoveryStorage,

relstorage/tests/hptestbase.py

@@ -288,6 +288,10 @@ def checkHistoricalConnection(self):
 
         db.close()
 
+    def checkImplementsExternalGC(self):
+        import ZODB.interfaces
+        self.assertFalse(ZODB.interfaces.IExternalGC.providedBy(self._storage))
+        self.assertRaises(AttributeError, self._storage.deleteObject)
 
 class HistoryPreservingToFileStorage(
         RelStorageTestBase,