Restore ++apidoc++ to the root.

In the tests, use the correct root folder, not the site folder.

But also, the traverser for the apidoc namespace forces the issue.
This prevents issues with shared ZODB connections.

Doing this allows us to correctly spider all the ++apidoc++ links
without running into circular parent chains or closed connection errors.

CHANGES.rst

@@ -7,6 +7,10 @@ CHANGES
 
 - Add support for Python 3.4, 3.5, 3.6 and PyPy.
 
+- The ``++apidoc++`` namespace is now always located beneath the root.
+  This solves issues with multiple threads and concurrent requests and
+  also circular parent chains.
+
 - Modernize some of the templates.  An increment towards having
   zope.app.apidoc compatible with Chameleon.
 

src/zope/app/apidoc/README.rst

@@ -47,13 +47,12 @@ Now we can instantiate the class (which is usually done when traversing
   >>> from zope.app.apidoc.apidoc import APIDocumentation
   >>> doc = APIDocumentation(None, '++apidoc++')
 
-  >>> modules =  doc.keys()
-  >>> modules.sort()
+  >>> modules = sorted(doc.keys())
   >>> modules
   [u'Interface', u'ZCML']
 
   >>> doc['ZCML']
-  <zope.app.apidoc.zcmlmodule.ZCMLModule object at ...>
+  <zope.app.apidoc.zcmlmodule.ZCMLModule 'ZCML' at ...>
 
 
 Developing a Module

src/zope/app/apidoc/apidoc.py

@@ -14,7 +14,6 @@
 """Zope 3 API Documentation
 
 """
-from __future__ import absolute_import
 __docformat__ = 'restructuredtext'
 
 import zope.component
@@ -39,21 +38,31 @@ def __init__(self, parent, name):
         self.__parent__ = parent
         self.__name__ = name
 
+    def __locate(self, obj, name):
+        # In general, *always* doing this is not just weird (threads), it also leads to
+        # circular __parent__ chains, which causes issues with things like
+        # zope.securitypolicy
+        # (https://github.com/zopefoundation/zope.securitypolicy/issues/8), and
+        # it can lead to bad behaviour with sharing persistent objects
+        # across ZODB connections (which may close). Fortunately, our traverser
+        # takes care of this by making sure we are always located at an (equivalent)
+        # fresh root.
+        locate(obj, self, name)
+        return obj
+
     def get(self, key, default=None):
         """See zope.container.interfaces.IReadContainer"""
         utility = zope.component.queryUtility(IDocumentationModule, key, default)
-        if utility != default:
-            locate(utility, self, key)
+        if utility is not default:
+            utility = self.__locate(utility, key)
         return utility
 
     def items(self):
         """See zope.container.interfaces.IReadContainer"""
-        items = list(zope.component.getUtilitiesFor(IDocumentationModule))
-        items.sort()
+        items = sorted(zope.component.getUtilitiesFor(IDocumentationModule))
         utils = []
         for key, value in items:
-            locate(value, self, key)
-            utils.append((key, value))
+            utils.append((key, self.__locate(value, key)))
         return utils
 
 
@@ -70,4 +79,12 @@ def traverse(self, name, ignore):
 
 def handleNamespace(ob, name):
     """Used to traverse to an API Documentation."""
-    return APIDocumentation(ob, '++apidoc++'+name)
+    # Ignore the `ob` we traverse through. We always want to be
+    # located at the root, although not the *actual* root.
+    # This is because we have to reparent our children, which are
+    # shared, in-memory utilities, and in the presence of multiple threads,
+    # doing so at different times would be bad in case connections get closed.
+    # So we make a pseudo root.
+    from zope.site.folder import rootFolder
+    ob = rootFolder()
+    return APIDocumentation(ob, '++apidoc++' + name)

src/zope/app/apidoc/codemodule/README.rst

@@ -89,7 +89,7 @@ still get to them:
 
   >>> names = sorted(module['tests'].keys())
   >>> names
-  ['BrowserTestCase', 'LayerDocFileSuite', 'LayerDocTestSuite', 'Root', 'rootLocation', 'setUp', 'standard_checker', 'tearDown', 'test_suite']
+  ['BrowserTestCase', 'LayerDocFileSuite', 'LayerDocTestSuite', 'Root', 'setUp', 'standard_checker', 'tearDown', 'test_suite']
 
 
 Classes

src/zope/app/apidoc/codemodule/browser/README.rst

@@ -44,7 +44,7 @@ Return info objects for all classes in this module.
   >>> pprint(details.getClasses())
   [{'doc': 'Represent the code browser documentation root',
     'name': 'CodeModule',
-    'url': 'http://127.0.0.1/++etc++site/++apidoc++/Code/zope/app/apidoc/codemodule/codemodule/CodeModule'}]
+    'url': 'http://127.0.0.1/++apidoc++/Code/zope/app/apidoc/codemodule/codemodule/CodeModule'}]
 
 This module doesn't contain anything else.
 
@@ -75,17 +75,17 @@ way up to the root, but we just want to go to the root module.
   >>> bc.request = details.request
   >>> pprint(bc(), width=1)
   [{'name': u'[top]',
-    'url': 'http://127.0.0.1/++etc++site/++apidoc++/Code'},
+    'url': 'http://127.0.0.1/++apidoc++/Code'},
    {'name': u'zope',
-    'url': 'http://127.0.0.1/++etc++site/++apidoc++/Code/zope'},
+    'url': 'http://127.0.0.1/++apidoc++/Code/zope'},
    {'name': 'app',
-    'url': 'http://127.0.0.1/++etc++site/++apidoc++/Code/zope/app'},
+    'url': 'http://127.0.0.1/++apidoc++/Code/zope/app'},
    {'name': 'apidoc',
-    'url': 'http://127.0.0.1/++etc++site/++apidoc++/Code/zope/app/apidoc'},
+    'url': 'http://127.0.0.1/++apidoc++/Code/zope/app/apidoc'},
    {'name': 'codemodule',
-    'url': 'http://127.0.0.1/++etc++site/++apidoc++/Code/zope/app/apidoc/codemodule'},
+    'url': 'http://127.0.0.1/++apidoc++/Code/zope/app/apidoc/codemodule'},
    {'name': 'codemodule',
-    'url': 'http://127.0.0.1/++etc++site/++apidoc++/Code/zope/app/apidoc/codemodule/codemodule'}]
+    'url': 'http://127.0.0.1/++apidoc++/Code/zope/app/apidoc/codemodule/codemodule'}]
 
 
 Class Details
@@ -109,7 +109,7 @@ Get all bases of this class.
 
   >>> pprint(details.getBases())
   [{'path': 'zope.app.apidoc.codemodule.module.Module',
-    'url': 'http://127.0.0.1/++etc++site/++apidoc++/Code/zope/app/apidoc/codemodule/module/Module'}]
+    'url': 'http://127.0.0.1/++apidoc++/Code/zope/app/apidoc/codemodule/module/Module'}]
 
 `getKnownSubclasses()`
 ~~~~~~~~~~~~~~~~~~~~~~
@@ -130,9 +130,9 @@ Prepare a list of classes for presentation.
   ...       zope.app.apidoc.apidoc.APIDocumentation,
   ...       zope.app.apidoc.codemodule.codemodule.Module]))
   [{'path': 'zope.app.apidoc.apidoc.APIDocumentation',
-    'url': 'http://127.0.0.1/++etc++site/++apidoc++/Code/zope/app/apidoc/apidoc/APIDocumentation'},
+    'url': 'http://127.0.0.1/++apidoc++/Code/zope/app/apidoc/apidoc/APIDocumentation'},
    {'path': 'zope.app.apidoc.codemodule.module.Module',
-    'url': 'http://127.0.0.1/++etc++site/++apidoc++/Code/zope/app/apidoc/codemodule/module/Module'}]
+    'url': 'http://127.0.0.1/++apidoc++/Code/zope/app/apidoc/codemodule/module/Module'}]
 
 `getBaseURL()`
 ~~~~~~~~~~~~~~
@@ -143,7 +143,7 @@ Note that the following output is a bit different than usual, since
 we have not setup all path elements.
 
   >>> details.getBaseURL()
-  'http://127.0.0.1/++etc++site/++apidoc++'
+  'http://127.0.0.1/++apidoc++'
 
 `getInterfaces()`
 ~~~~~~~~~~~~~~~~~
@@ -251,7 +251,7 @@ Get all attributes of this function.
 Return the URL for the API Documentation Tool.
 
   >>> details.getBaseURL()
-  'http://127.0.0.1/++etc++site/++apidoc++'
+  'http://127.0.0.1/++apidoc++'
 
 
 Text File Details
@@ -330,7 +330,7 @@ Returns the URL of the directive docuemntation in the ZCML documentation
 module.
 
   >>> details.url()
-  u'http://127.0.0.1/++etc++site/++apidoc++/ZCML/ALL/configure/index.html'
+  u'http://127.0.0.1/++apidoc++/ZCML/ALL/configure/index.html'
 
 `objectURL(value, field, rootURL)`
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -342,7 +342,7 @@ crafts a documentation URL for it:
   >>> field = GlobalObject()
 
   >>> details.objectURL('.interfaces.IZCMLFile', field, '')
-  'http://127.0.0.1/++etc++site/++apidoc++/Interface/zope.app.apidoc.codemodule.interfaces.IZCMLFile/index.html'
+  'http://127.0.0.1/++apidoc++/Interface/zope.app.apidoc.codemodule.interfaces.IZCMLFile/index.html'
 
   >>> details.objectURL('.zcml.ZCMLFile', field, '')
   '/zope/app/apidoc/codemodule/zcml/ZCMLFile/index.html'
@@ -371,7 +371,7 @@ will be listed too.
   >>> details.context = details.context.subs[0]
   >>> pprint(details.attributes())
   [{'name': u'class',
-    'url': 'http://127.0.0.1/++etc++site/++apidoc++/Code/zope/app/apidoc/codemodule/module/Module/index.html',
+    'url': 'http://127.0.0.1/++apidoc++/Code/zope/app/apidoc/codemodule/module/Module/index.html',
     'value': u'.module.Module',
     'values': []}]
 
@@ -481,14 +481,14 @@ class. In the following section we are going to demonstrate the methods used
 to collect the data. First we need to create an object though; let's use a
 root folder:
 
-  >>> rootFolder.__parent__
+  >>> rootFolder
   <zope.site.folder.Folder object at ...>
 
 Now we instantiate the view
 
   >>> from zope.publisher.browser import TestRequest
   >>> request = TestRequest()
-  >>> inspect = introspector.Introspector(rootFolder.__parent__, request)
+  >>> inspect = introspector.Introspector(rootFolder, request)
 
 so that we can start looking at the methods. First we should note that the
 class documentation view is directly available:
@@ -585,7 +585,7 @@ and then get the sequence items:
 
 Similar functionality exists for a mapping. But we first have to add an item:
 
-  >>> rootFolder.__parent__['list'] = list
+  >>> rootFolder['list'] = list
 
 Now let's have a look:
 
@@ -609,7 +609,7 @@ object is annotatable,
 
 then we can get an annotation mapping:
 
-  >>> rootFolder.__parent__.__annotations__ = {'my.list': list}
+  >>> rootFolder.__annotations__ = {'my.list': list}
 
   >>> pprint(inspect.getAnnotationsInfo())
   [{'key': 'my.list',

src/zope/app/apidoc/codemodule/browser/class_.py

@@ -31,8 +31,8 @@
 from zope.app.apidoc.utilities import isReferencable
 
 
-def getTypeLink(type):
-    if type is type(None):
+def getTypeLink(type, _NoneType=type(None)):
+    if type is _NoneType:
         return None
     path = getPythonPath(type)
     return path.replace('.', '/') if isReferencable(path) else None
@@ -157,7 +157,8 @@ def getConstructor(self):
         if attr is None:
             return None
         attr = removeSecurityProxy(attr)
+
         return {
             'signature': getFunctionSignature(attr, ignore_self=True),
             'doc': renderText(attr.__doc__ or '', inspect.getmodule(attr)),
-            }
+        }

src/zope/app/apidoc/codemodule/browser/menu.py

@@ -65,22 +65,22 @@ def findClasses(self):
           >>> from pprint import pprint
           >>> pprint(info)
           [{'path': 'zope.app.apidoc.codemodule.browser.Foo',
-            'url': 'http://127.0.0.1/++etc++site/++apidoc++/Code/zope/app/apidoc/codemodule/browser/Foo/'},
+            'url': 'http://127.0.0.1/++apidoc++/Code/zope/app/apidoc/codemodule/browser/Foo/'},
            {'path': 'zope.app.apidoc.codemodule.browser.Foo2',
-            'url': 'http://127.0.0.1/++etc++site/++apidoc++/Code/zope/app/apidoc/codemodule/browser/Foo2/'}]
+            'url': 'http://127.0.0.1/++apidoc++/Code/zope/app/apidoc/codemodule/browser/Foo2/'}]
 
           >>> menu.request = TestRequest(form={'path': 'o2'})
           >>> info = menu.findClasses()
           >>> pprint(info)
           [{'path': 'zope.app.apidoc.codemodule.browser.Foo2',
-            'url': 'http://127.0.0.1/++etc++site/++apidoc++/Code/zope/app/apidoc/codemodule/browser/Foo2/'}]
+            'url': 'http://127.0.0.1/++apidoc++/Code/zope/app/apidoc/codemodule/browser/Foo2/'}]
 
 
           >>> menu.request = TestRequest(form={'path': 'Blah'})
           >>> info = menu.findClasses()
           >>> pprint(info)
           [{'path': 'zope.app.apidoc.codemodule.browser.Blah',
-            'url': 'http://127.0.0.1/++etc++site/++apidoc++/Code/zope/app/apidoc/codemodule/browser/Blah/'}]
+            'url': 'http://127.0.0.1/++apidoc++/Code/zope/app/apidoc/codemodule/browser/Blah/'}]
 
         """
         path = self.request.get('path', None)

src/zope/app/apidoc/interfaces.py

@@ -17,8 +17,9 @@
 
 from zope.interface import Interface
 from zope.schema import TextLine, Text
+from zope.location.interfaces import ILocation
 
-class IDocumentationModule(Interface):
+class IDocumentationModule(ILocation):
     """Zope 3 API Documentation Module
 
     A documentation module contains the documentation for one specific aspect

src/zope/app/apidoc/static.py

@@ -264,7 +264,7 @@ def processLink(self, link):
         # Retrieve the content
         try:
             self.browser.open(link.callableURL)
-        except urllib2.HTTPError, error:
+        except urllib2.HTTPError as error:
             # Something went wrong with retrieving the page.
             self.linkErrors += 1
             self.sendMessage(
@@ -279,7 +279,7 @@ def processLink(self, link):
             self.sendMessage('Bad URL: ' + link.callableURL, 2)
             self.sendMessage('+-> Reference: ' + link.referenceURL, 2)
             return
-        except Exception, error:
+        except Exception as error:
             # This should never happen outside the debug mode. We really want
             # to catch all exceptions, so that we can investigate them.
             if self.options.debug:
@@ -307,7 +307,7 @@ def processLink(self, link):
 
             try:
                 links = self.browser.links()
-            except HTMLParser.HTMLParseError, error:
+            except HTMLParser.HTMLParseError as error:
                 self.htmlErrors += 1
                 self.sendMessage('Failed to parse HTML: ' + url, 1)
                 self.sendMessage('+-> %s: line %i, column %s' % (