Let subclasses of BaseAdapterRegistry customize the data structures.

Add extensive tests for this. Fixes #224.

Also adds test for, and fixes #227

CHANGES.rst

@@ -2,10 +2,23 @@
  Changes
 =========
 
-5.2.1 (unreleased)
-==================
-
-- Nothing changed yet.
+5.3.0 (unreleased)
+==================
+
+- Allow subclasses of ``BaseAdapterRegistry`` (including
+  ``AdapterRegistry`` and ``VerifyingAdapterRegistry``) to have
+  control over the data structures. This allows persistent
+  implementations such as those based on ZODB to choose more scalable
+  options (e.g., BTrees instead of dicts). See `issue 224
+  <https://github.com/zopefoundation/zope.interface/issues/224>`_.
+
+- Fix a reference counting issue in ``BaseAdapterRegistry`` that could
+  lead to references to interfaces being kept around even when all
+  utilities/adapters/subscribers providing that interface have been
+  removed. This is mostly an issue for persistent implementations.
+  Note that this only corrects the issue moving forward, it does not
+  solve any already corrupted reference counts. See `issue 227
+  <https://github.com/zopefoundation/zope.interface/issues/227>`_.
 
 
 5.2.0 (2020-11-05)

docs/api/adapters.rst

@@ -11,3 +11,17 @@ The adapter registry's API is defined by
 .. autointerface:: zope.interface.adapter.IAdapterRegistry
    :members:
    :member-order: bysource
+
+
+The concrete implementations of ``IAdapterRegistry`` provided by this
+package allows for some customization opportunities.
+
+.. autoclass:: zope.interface.adapter.BaseAdapterRegistry
+   :members:
+   :private-members:
+
+.. autoclass:: zope.interface.adapter.AdapterRegistry
+   :members:
+
+.. autoclass:: zope.interface.adapter.VerifyingAdapterRegistry
+   :members:

src/zope/interface/adapter.py

@@ -48,6 +48,41 @@
 # All three have substantial variance.
 
 class BaseAdapterRegistry(object):
+    """
+    A basic implementation of the data storage and algorithms required
+    for a :class:`zope.interface.interfaces.IAdapterRegistry`.
+
+    Subclasses or instances can set the following attributes (before
+    calling this object's ``__init__`` method) to control how the data
+    is stored; in particular, these hooks can be helpful for ZODB
+    persistence:
+
+    _sequenceType = list
+      This is the type used for our top-level "byorder" sequences.
+      These are usually small (< 10) and are almost always accessed when
+      using this object, so it is rarely useful to change.
+    _leafSequenceType = tuple
+      This is the type used for the leaf sequences of subscribers.
+      It could be set to a ``PersistentList`` to avoid many unnecessary data
+      loads when subscribers aren't being used.
+    _mappingType = dict
+      This is the type used for the keyed mappings. A ``PersistentMapping``
+      could be used to help reduce the number of data loads when the registry is large
+      and parts of it are rarely used. Further reductions in data loads can come from
+      using a ``OOBTree``, but care is required to be sure that all required/provided
+      values are fully ordered (e.g., no required or provided values that are classes
+      can be used).
+    _providedType = dict
+      This is the type used for the ``_provided`` mapping.
+      This is separate from the generic mapping type because the values
+      are always integers, so one might choose to use a more optimized data
+      structure such as a ``OIBTree``. The same caveats apply as for
+
+
+    .. versionchanged:: 5.3.0
+        Add support for customizing the way data
+        structures are created.
+    """
 
     # List of methods copied from lookup sub-objects:
     _delegated = ('lookup', 'queryMultiAdapter', 'lookup1', 'queryAdapter',
@@ -73,15 +108,15 @@ def __init__(self, bases=()):
         # but for order == 2 (that is, self._adapters[2]), we have:
         #   {r1 -> {r2 -> {provided -> {name -> value}}}}
         #
-        self._adapters = []
+        self._adapters = self._sequenceType()
 
         # {order -> {required -> {provided -> {name -> [value]}}}}
         # where the remarks about adapters above apply
-        self._subscribers = []
+        self._subscribers = self._sequenceType()
 
         # Set, with a reference count, keeping track of the interfaces
         # for which we have provided components:
-        self._provided = {}
+        self._provided = self._providedType()
 
         # Create ``_v_lookup`` object to perform lookup.  We make this a
         # separate object to to make it easier to implement just the
@@ -106,6 +141,12 @@ def __init__(self, bases=()):
         self.__bases__ = bases
 
     def _setBases(self, bases):
+        """
+        If subclasses need to track when ``__bases__`` changes, they
+        can override this method.
+
+        Subclasses must still call this method.
+        """
         self.__dict__['__bases__'] = bases
         self.ro = ro.ro(self)
         self.changed(self)
@@ -119,6 +160,52 @@ def _createLookup(self):
         for name in self._delegated:
             self.__dict__[name] = getattr(self._v_lookup, name)
 
+    # Hooks for subclasses to define the types of objects used in
+    # our data structures.
+    # These have to be documented in the docstring, instead of local
+    # comments, because autodoc ignores the comment and just writes
+    # "alias of list"
+    _sequenceType = list
+    _leafSequenceType = tuple
+    _mappingType = dict
+    _providedType = dict
+
+    def _addValueToLeaf(self, existing_leaf_sequence, new_item):
+        """
+        Add the value *new_item* to the *existing_leaf_sequence*, which may
+        be ``None``.
+
+        If *existing_leaf_sequence* is not *None*, it will be an instance
+        of `_leafSequenceType`.
+
+        This method returns the new value to be stored. It may mutate the
+        sequence in place if it was not None and the type is mutable, but
+        it must also return it.
+
+        Subclasses that redefine `_leafSequenceType` should override this method.
+
+        .. versionadded:: 5.3.0
+        """
+        if existing_leaf_sequence is None:
+            return (new_item,)
+        return existing_leaf_sequence + (new_item,)
+
+    def _removeValueFromLeaf(self, existing_leaf_sequence, to_remove):
+        """
+        Remove the item *to_remove* from the (non-None, non-empty) *existing_leaf_sequence*
+        and return the mutated sequence.
+
+        If there is more than one item that is equal to *to_remove* they must all be
+        removed.
+
+        May mutate in place or return a new object.
+
+        Subclasses that redefine `_leafSequenceType` should override this method.
+
+        .. versionadded:: 5.3.0
+        """
+        return tuple([v for v in existing_leaf_sequence if v != to_remove])
+
     def changed(self, originally_changed):
         self._generation += 1
         self._v_lookup.changed(originally_changed)
@@ -135,14 +222,14 @@ def register(self, required, provided, name, value):
         order = len(required)
         byorder = self._adapters
         while len(byorder) <= order:
-            byorder.append({})
+            byorder.append(self._mappingType())
         components = byorder[order]
         key = required + (provided,)
 
         for k in key:
             d = components.get(k)
             if d is None:
-                d = {}
+                d = self._mappingType()
                 components[k] = d
             components = d
 
@@ -231,7 +318,7 @@ def subscribe(self, required, provided, value):
         order = len(required)
         byorder = self._subscribers
         while len(byorder) <= order:
-            byorder.append({})
+            byorder.append(self._mappingType())
         components = byorder[order]
         key = required + (provided,)
 
@@ -242,7 +329,7 @@ def subscribe(self, required, provided, value):
                 components[k] = d
             components = d
 
-        components[name] = components.get(name, ()) + (value, )
+        components[name] = self._addValueToLeaf(components.get(name), value)
 
         if provided is not None:
             n = self._provided.get(provided, 0) + 1
@@ -274,13 +361,19 @@ def unsubscribe(self, required, provided, value=None):
         if not old:
             # this is belt-and-suspenders against the failure of cleanup below
             return  # pragma: no cover
-
+        len_old = len(old)
         if value is None:
+            # Removing everything; note that the type of ``new`` won't match
+            # the ``_leafSequenceType``, but that's OK because we're about
+            # to delete the entire entry anyway.
             new = ()
         else:
-            new = tuple([v for v in old if v != value])
+            new = self._removeValueFromLeaf(old, value)
+        # new may have been mutated in place, so we cannot compare it to old
+        del old
 
-        if new == old:
+        if len(new) == len_old:
+            # No changes, so nothing could have been removed.
             return
 
         if new:
@@ -303,10 +396,12 @@ def unsubscribe(self, required, provided, value=None):
                 del byorder[-1]
 
         if provided is not None:
-            n = self._provided[provided] + len(new) - len(old)
+            n = self._provided[provided] + len(new) - len_old
             if n == 0:
                 del self._provided[provided]
                 self._v_lookup.remove_extendor(provided)
+            else:
+                self._provided[provided] = n
 
         self.changed(self)
 
@@ -630,6 +725,10 @@ class AdapterLookup(AdapterLookupBase, LookupBase):
 
 @implementer(IAdapterRegistry)
 class AdapterRegistry(BaseAdapterRegistry):
+    """
+    A full implementation of ``IAdapterRegistry`` that adds support for
+    sub-registries.
+    """
 
     LookupClass = AdapterLookup
 
@@ -670,6 +769,9 @@ class VerifyingAdapterLookup(AdapterLookupBase, VerifyingBase):
 
 @implementer(IAdapterRegistry)
 class VerifyingAdapterRegistry(BaseAdapterRegistry):
+    """
+    The most commonly-used adapter registry.
+    """
 
     LookupClass = VerifyingAdapterLookup