Merge branch 'master' into issue_401

.travis.yml

@@ -15,22 +15,17 @@ matrix:
         - python: "3.7"
           env: TOXENV=py37
           dist: xenial
-          sudo: true
         - python: "3.7-dev"
           env: TOXENV=py37
           dist: xenial
-          sudo: true
         - python: "3.8-dev"
           env: TOXENV=py38
           dist: xenial
-          sudo: true
     allow_failures:
         - python: "2.7"
           env: TOXENV=lint-py27
         - python: "3.6"
           env: TOXENV=lint-py36
-        - python: "3.8-dev"
-          env: TOXENV=py38
 
 install:
     - travis_retry pip install -U pip setuptools

CHANGES.rst

@@ -11,12 +11,23 @@ https://github.com/zopefoundation/Zope/blob/4.0a6/CHANGES.rst
 4.0b9 (unreleased)
 ------------------
 
+Features
+++++++++
+
+- Add preliminary support for Python 3.8. as of 3.8.0a1 is released.
+
 Fixes
 +++++
 
 - Fix display of ZMI breadcrumbs with non-ASCII path elements
   (`#401 <https://github.com/zopefoundation/Zope/issues/401>`_)
 
+- Make sure conflicts are always retried and not masked by exception views
+  (`#413 <https://github.com/zopefoundation/Zope/issues/413>`_)
+
+- Fix faulty ZMI links due to missing URL-quoting
+  (`#391 <https://github.com/zopefoundation/Zope/issues/391>`_)
+
 - Fix configuring the maximum number of conflict retries
   (`#413 <https://github.com/zopefoundation/Zope/issues/413>`_)
 
@@ -39,6 +50,7 @@ Other changes
 +++++++++++++
 
 - Document filesystem caching for Chameleon page templates
+  and activate it by default for new WSGI instances
   (`#291 <https://github.com/zopefoundation/Zope/issues/291>`_)
 
 - Remove obsolete environment variable "Z_DEBUG_MODE"

MANIFEST.in

@@ -1,19 +1,60 @@
 include *.txt
 include *.rst
+
 include *.cfg
+exclude pyvenv.cfg
+
+include *.yml
+exclude *.yml
+
 exclude .*.cfg
 
-recursive-include docs *
-recursive-include src *
+include .editorconfig
+exclude .editorconfig
+
+include .gitignore
+exclude .gitignore
+
+exclude MANIFEST.in
 
 prune docs/.build
 prune docs/_build
 
-global-exclude .DS_Store
-global-exclude *.dll
-global-exclude *.pyc
-global-exclude *.pyo
-global-exclude *.so
+recursive-include docs *.bat
+recursive-include docs *.css
+recursive-include docs *.jpg
+recursive-include docs *.png
+recursive-include docs *.py
+recursive-include docs *.rst
+recursive-include docs *.tgz
+recursive-include docs *.txt
+recursive-include docs Makefile
+recursive-include src *.css
+recursive-include src *.dtml
+recursive-include src *.eot
+recursive-include src *.gif
+recursive-include src *.htm
+recursive-include src *.html
+recursive-include src *.ico
+recursive-include src *.in
+recursive-include src *.js
+recursive-include src *.map
+recursive-include src *.mo
+recursive-include src *.pdf
+recursive-include src *.png
+recursive-include src *.po
+recursive-include src *.pot
+recursive-include src *.pt
+recursive-include src *.rst
+recursive-include src *.svg
+recursive-include src *.ttf
+recursive-include src *.txt
+recursive-include src *.webmanifest
+recursive-include src *.woff
+recursive-include src *.woff2
+recursive-include src *.xml
+recursive-include src *.zcml
+recursive-include src *.zpt
 
 include *.py
 include buildout.cfg

buildout.cfg

@@ -19,6 +19,7 @@ parts =
     requirements
 sources-dir = develop
 auto-checkout =
+    RestrictedPython
 
 [testenv]
 PYTHONHASHSEED = random

docs/zdgbook/ObjectPublishing.rst

@@ -34,21 +34,30 @@ web publishing.
 HTTP Publishing
 ===============
 
-When you contact Zope with a web browser, your browser sends an HTTP
-request to Zope's web server. After the request is completely
-received, it is processed by 'ZPublisher', which is Zope's object
-publisher. 'ZPublisher' is a kind of light-weight ORB (Object Request
+Zope 4 no longer ships with a builtin web server, so when you want
+to interact with Zope via browser you have to setup a WSGI server.
+
+
+.. note::
+
+    For usage on a production server you will probably want to setup a
+    reverse proxy in front of the WSGI server.
+
+
+The WSGI server receives the request and hands it over to Zope, where
+it is processed by *ZPublisher*, which is Zope's object publisher.
+**ZPublisher** is a kind of light-weight ORB (Object Request
 Broker). It takes the request and locates an object to handle the
 request. The publisher uses the request URL as a map to locate the
 published object. Finding an object to handle the request is called
-*traversal*, since the publisher moves from object to object as it
+**traversal**, since the publisher moves from object to object as it
 looks for the right one. Once the published object is found, the
 publisher calls a method on the published object, passing it
 parameters as necessary. The publisher uses information in the
-request to determine which method to call, and what parameters to
+request to determine which method to call and what parameters to
 pass. The process of extracting parameters from the request is called
-*argument marshalling*. The published object then returns a response,
-which is passed back to Zope's web server. The web server, then
+**argument marshalling**. The published object then returns a response,
+which is passed back to the WSGI server. Finally, the WSGI server
 passes the response back to your web browser.
 
 
@@ -60,23 +69,24 @@ The publishing process is summarized in [2-1]
 
 
 Typically the published object is a persistent object that the
-published module loads from the ZODB. See Chapter 4 for more
+published module loads from the ZODB. See Chapter 6 for more
 information on the ZODB.
 
 
 This chapter will cover all the steps of object publishing in detail.
+
+
 To summarize, object publishing consists of the main steps:
 
-1. The client sends a request to the publisher
+1. A request is sent to the publisher.
 
 2. The publisher locates the published object using the request
    URL as a map.
 
 3. The publisher calls the published object with arguments from
    the request.
 
-4. The publisher interprets and returns the results to the
-   client.
+4. The publisher interprets the results and passes them back.
 
 The chapter will also cover all the technical details, special cases
 and extra-steps that this list glosses over.

setup.py

@@ -54,6 +54,7 @@ def _read_file(filename):
         "Programming Language :: Python :: 3.5",
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: Implementation :: CPython",
         "Topic :: Internet :: WWW/HTTP :: Dynamic Content",
         "Topic :: Internet :: WWW/HTTP :: WSGI :: Application",

src/OFS/ObjectManager.py

@@ -50,6 +50,7 @@
 from Products.PageTemplates.PageTemplateFile import PageTemplateFile
 from six import string_types
 from six import text_type
+from six.moves.urllib.parse import quote
 from zExceptions import BadRequest
 from zExceptions import ResourceLockedError
 from zope.container.contained import notifyContainerModified
@@ -897,7 +898,7 @@ def manage_get_sortedObjects(self, sortkey, revkey):
 
         items = []
         for id, obj in self.objectItems():
-            item = {'id': id, 'obj': obj}
+            item = {'id': id, 'quoted_id': quote(id), 'obj': obj}
             if sortkey not in ['id', 'position'] and hasattr(obj, sortkey):
                 # add the attribute by which we need to sort
                 item[sortkey] = getattr(obj, sortkey)
@@ -925,6 +926,7 @@ def manage_get_sortedObjects(self, sortkey, revkey):
         return [
             {
                 'id': item['id'],
+                'quoted_id': item['quoted_id'],
                 'obj': item['obj'],
             }
             for item in sorted_items

src/OFS/tests/testObjectManager.py

@@ -16,6 +16,7 @@
 from OFS.ObjectManager import ObjectManager
 from OFS.SimpleItem import SimpleItem
 from six import PY2
+from six.moves.urllib.parse import quote
 from zExceptions import BadRequest
 from Zope2.App import zcml
 from zope.component.testing import PlacelessSetup
@@ -501,6 +502,17 @@ def test_list_imports(self):
             self.assertTrue(filename.endswith('.zexp') or
                             filename.endswith('.xml'))
 
+    def test_manage_get_sortedObjects_quote_id(self):
+        # manage_get_sortedObjects now returns a urlquoted version
+        # of the object ID to create correct links in the ZMI
+        om = self._makeOne()
+        hash_id = '#999'
+        om._setObject(hash_id, SimpleItem(hash_id))
+
+        result = om.manage_get_sortedObjects('id', 'asc')
+        self.assertEqual(len(result), 1)
+        self.assertEqual(result[0]['id'], hash_id)
+        self.assertEqual(result[0]['quoted_id'], quote(hash_id))
 
 _marker = object()
 

src/OFS/zpt/main.zpt

@@ -81,7 +81,7 @@
                 </i>
               </td>
               <td class="zmi-object-id">
-                <a tal:attributes="href python:'%s/manage_workspace'%(ob_dict['id'])">
+                <a tal:attributes="href python:'%s/manage_workspace'%(ob_dict['quoted_id'])">
                   <span tal:replace="ob_dict/id">Id</span>
                   <span class="badge badge-warning" title="This item has been locked by WebDAV" tal:condition="ob/wl_isLocked | nothing">
                     <i class="fa fa-lock"></i>

src/ZPublisher/WSGIPublisher.py

@@ -28,7 +28,6 @@
     Unauthorized,
     upgradeException,
 )
-from ZODB.POSException import ConflictError
 from zope.component import queryMultiAdapter
 from zope.event import notify
 from zope.globalrequest import setRequest, clearRequest
@@ -167,6 +166,9 @@ def transaction_pubevents(request, response, tm=transaction.manager):
             notify(pubevents.PubFailure(
                 request, exc_info, request.supports_retry()))
 
+            if isinstance(exc, TransientError) and request.supports_retry():
+                reraise(*exc_info)
+
             if not (exc_view_created or isinstance(exc, Unauthorized)):
                 reraise(*exc_info)
         finally:

src/ZPublisher/tests/test_WSGIPublisher.py

@@ -95,7 +95,7 @@ def test_listHeaders_includes_Date_header(self):
         headers = response.listHeaders()
         whenstr = time.strftime('%a, %d %b %Y %H:%M:%S GMT',
                                 time.gmtime(time.mktime(WHEN)))
-        self.assertTrue(('Date', whenstr) in headers)
+        self.assertIn(('Date', whenstr), headers)
 
     def test_setBody_IUnboundStreamIterator(self):
         from ZPublisher.Iterators import IUnboundStreamIterator
@@ -469,6 +469,50 @@ def _publish(request, module_info):
         finally:
             HTTPRequest.retry_max_count = original_retry_max_count
 
+    def testCustomExceptionViewConflictErrorHandling(self):
+        # Make sure requests are retried as often as configured
+        # even if an exception view has been registered that
+        # matches ConflictError
+        from zope.interface import directlyProvides
+        from zope.publisher.browser import IDefaultBrowserLayer
+        registerExceptionView(Exception)
+        environ = self._makeEnviron()
+        start_response = DummyCallable()
+        _publish = DummyCallable()
+        _publish._raise = ConflictError('oops')
+        _request = DummyRequest()
+        directlyProvides(_request, IDefaultBrowserLayer)
+        _request.response = DummyResponse()
+        _request.retry_count = 0
+        _request.retry_max_count = 3
+        _request.environ = {}
+
+        def _close():
+            pass
+        _request.close = _close
+
+        def _retry():
+            _request.retry_count += 1
+            return _request
+        _request.retry = _retry
+
+        def _supports_retry():
+            return _request.retry_count < _request.retry_max_count
+        _request.supports_retry = _supports_retry
+
+        def _request_factory(stdin, environ, response):
+            return _request
+
+        # At first, retry_count is zero. Request has never been retried.
+        self.assertEqual(_request.retry_count, 0)
+        app_iter = self._callFUT(environ, start_response, _publish,
+                                 _request_factory=_request_factory)
+
+        # In the end the error view is rendered, but the request should
+        # have been retried up to retry_max_count times
+        self.assertTrue(app_iter[1].startswith('Exception View: ConflictError'))
+        self.assertEqual(_request.retry_count, _request.retry_max_count)
+
     def testCustomExceptionViewUnauthorized(self):
         from AccessControl import Unauthorized
         registerExceptionView(IUnauthorized)
@@ -717,6 +761,11 @@ def setBody(self, body):
 
     body = property(lambda self: self._body, setBody)
 
+    def setStatus(self, status, reason=None, lock=None):
+        self._status = status
+
+    status = property(lambda self: self._status, setStatus)
+
 
 class DummyCallable(object):
     _called_with = _raise = _result = None

src/Zope2/utilities/skel/etc/wsgi.conf.in

@@ -48,8 +48,11 @@ instancehome $INSTANCE
 # Example:
 #
 #    <environment>
-#      CHAMELEON_CACHE $INSTANCE/var/cache
+#      MY_PRODUCT_ENVVAR foobar
 #    </environment>
+<environment>
+  CHAMELEON_CACHE $INSTANCE/var/cache
+</environment>
 
 
 # Directive: debug-mode

src/Zope2/utilities/skel/var/cache/README.txt

@@ -0,0 +1,2 @@
+This directory typically contains cache files, such as Page Template
+cache files generated by Chameleon.

tox.ini

@@ -4,6 +4,7 @@ envlist =
     py35,
     py36,
     py37,
+    py38,
     lint-py27,
     lint-py36
     coverage-report

versions-prod.cfg

@@ -25,7 +25,7 @@ pbr = 5.1.1
 Persistence = 3.0b4
 persistent = 4.4.3
 Products.BTreeFolder2 = 4.1
-Products.ZCatalog = 4.2
+Products.ZCatalog = 4.3
 Products.ZCTextIndex = 4.0.2
 pytz = 2018.7
 Record = 3.5