Merge branch 'master' into DTMLMethod-content-conversion

CHANGES.rst

@@ -59,6 +59,11 @@ Features
 Other changes
 +++++++++++++
 
+- Make Zope write a PID file again under WSGI.
+  This makes interaction with sysadmin tools easier.
+  The PID file path can be set in the Zope configuration with ``pid-filename``,
+  just like in ``ZServer``-based configurations.
+
 - Exceptions during publishing are now re-raised in a new exceptions debug
   mode to allow WSGI middleware to handle/debug it. See the `debug
   documentation <https://zope.readthedocs.io/en/latest/wsgi.html#werkzeug>`_

MANIFEST.in

@@ -1,14 +1,14 @@
-include *.txt
+include *.py
 include *.rst
-
-include *.cfg
-exclude pyvenv.cfg
+include *.txt
+include buildout.cfg
+include sources.cfg
+include versions-prod.cfg
+include versions.cfg
 
 include *.yml
 exclude *.yml
 
-exclude .*.cfg
-
 include .editorconfig
 exclude .editorconfig
 
@@ -17,9 +17,6 @@ exclude .gitignore
 
 exclude MANIFEST.in
 
-prune docs/.build
-prune docs/_build
-
 recursive-include docs *.bat
 recursive-include docs *.css
 recursive-include docs *.jpg
@@ -55,9 +52,3 @@ recursive-include src *.woff2
 recursive-include src *.xml
 recursive-include src *.zcml
 recursive-include src *.zpt
-
-include *.py
-include buildout.cfg
-include sources.cfg
-include versions-prod.cfg
-include versions.cfg

setup.cfg

@@ -14,8 +14,8 @@ ignore =
 force_single_line = True
 combine_as_imports = True
 sections = FUTURE,STDLIB,THIRDPARTY,ZOPE,FIRSTPARTY,LOCALFOLDER
-known_third_party = ipaddress, PasteDeploy, six, waitress, chameleon
-known_zope = AccessControl, Acquisition, DateTime, DocumentTemplate, ExtensionClass, MultiMapping, Persistence, persistent, RestrictedPython, Testing, transaction, ZConfig, zExceptions, ZODB, zope, Zope2
+known_third_party = ipaddress, PasteDeploy, six, waitress, chameleon, paste
+known_zope = AccessControl, Acquisition, DateTime, DocumentTemplate, ExtensionClass, MultiMapping, Persistence, persistent, RestrictedPython, Testing, transaction, ZConfig, zExceptions, ZODB, zope, Zope2, App
 default_section = ZOPE
 line_length = 79
 lines_after_imports = 2

src/ZPublisher/HTTPResponse.py

@@ -27,7 +27,6 @@
 from six import reraise
 from six import text_type
 from six.moves.urllib.parse import quote
-from six.moves.urllib.parse import unquote
 from six.moves.urllib.parse import urlparse
 from six.moves.urllib.parse import urlunparse
 
@@ -217,7 +216,14 @@ def redirect(self, location, status=302, lock=0):
         # characters in the path part are quoted correctly. This is required
         # as we now allow non-ASCII IDs
         parsed = list(urlparse(location))
-        parsed[2] = quote(unquote(parsed[2]))
+
+        # Make a hacky guess whether the path component is already
+        # URL-encoded by checking for %. If it is, we don't touch it.
+        if '%' not in parsed[2]:
+            # The list of "safe" characters is from RFC 2396 section 2.3
+            # (unreserved characters that should not be escaped) plus
+            # section 3.3 (reserved characters in path components)
+            parsed[2] = quote(parsed[2], safe="/@!*'~();,=+$")
         location = urlunparse(parsed)
 
         self.setStatus(status, lock=lock)

src/ZPublisher/WSGIPublisher.py

@@ -175,36 +175,39 @@ def transaction_pubevents(request, response, tm=transaction.manager):
             if request.environ.get('x-wsgiorg.throw_errors', False):
                 reraise(*exc_info)
 
-            # Handle exception view. Make sure an exception view that
-            # blows up doesn't leave the user e.g. unable to log in.
-            try:
-                exc_view_created = _exc_view_created_response(
-                    exc, request, response)
-            except Exception:
-                exc_view_created = False
+            # If the exception is transient and the request can be retried,
+            # shortcut further processing. It makes no sense to have an
+            # exception view registered for this type of exception.
+            retry = False
+            if isinstance(exc, TransientError) and request.supports_retry():
+                retry = True
+            else:
+                # Handle exception view. Make sure an exception view that
+                # blows up doesn't leave the user e.g. unable to log in.
+                try:
+                    exc_view_created = _exc_view_created_response(
+                        exc, request, response)
+                except Exception:
+                    exc_view_created = False
 
-            if isinstance(exc, Unauthorized):
                 # _unauthorized modifies the response in-place. If this hook
                 # is used, an exception view for Unauthorized has to merge
                 # the state of the response and the exception instance.
-                exc.setRealm(response.realm)
-                response._unauthorized()
-                response.setStatus(exc.getStatus())
-
-            retry = False
-            if isinstance(exc, TransientError) and request.supports_retry():
-                retry = True
+                if isinstance(exc, Unauthorized):
+                    exc.setRealm(response.realm)
+                    response._unauthorized()
+                    response.setStatus(exc.getStatus())
 
+            # Notify subscribers that this request is failing.
             notify(pubevents.PubBeforeAbort(request, exc_info, retry))
             tm.abort()
             notify(pubevents.PubFailure(request, exc_info, retry))
 
-            if retry:
-                reraise(*exc_info)
-
-            if not (exc_view_created or isinstance(exc, Unauthorized)) or \
+            if retry or \
+               not (exc_view_created or isinstance(exc, Unauthorized)) or \
                getattr(response, 'debug_exceptions', False):
                 reraise(*exc_info)
+
         finally:
             # Avoid traceback / exception reference cycle.
             del exc, exc_info

src/ZPublisher/tests/testHTTPResponse.py

@@ -725,29 +725,31 @@ def test_setBody_compression_no_prior_vary_header_but_forced(self):
         response.setBody(b'foo' * 100)  # body must get smaller on compression
         self.assertEqual(response.getHeader('Vary'), None)
 
-    def test_redirect_defaults(self):
-        URL = 'http://example.com'
+    def _redirectURLCheck(self, url, expected=None, lock=False, status=302):
+        if not expected:
+            expected = url
         response = self._makeOne()
-        result = response.redirect(URL)
-        self.assertEqual(result, URL)
+        result = response.redirect(url, lock=lock, status=status)
+        self.assertEqual(result, expected)
+        self.assertEqual(response.getHeader('Location'), expected)
+        return response
+
+    def test_redirect_defaults(self):
+        URL = 'http://example.com/@@login'
+        response = self._redirectURLCheck(URL)
         self.assertEqual(response.status, 302)
-        self.assertEqual(response.getHeader('Location'), URL)
         self.assertFalse(response._locked_status)
 
     def test_redirect_explicit_status(self):
         URL = 'http://example.com'
-        response = self._makeOne()
-        response.redirect(URL, status=307)
+        response = self._redirectURLCheck(URL, status=307)
         self.assertEqual(response.status, 307)
-        self.assertEqual(response.getHeader('Location'), URL)
         self.assertFalse(response._locked_status)
 
     def test_redirect_w_lock(self):
         URL = 'http://example.com'
-        response = self._makeOne()
-        response.redirect(URL, lock=True)
+        response = self._redirectURLCheck(URL, lock=True)
         self.assertEqual(response.status, 302)
-        self.assertEqual(response.getHeader('Location'), URL)
         self.assertTrue(response._locked_status)
 
     def test_redirect_nonascii(self):
@@ -756,40 +758,35 @@ def test_redirect_nonascii(self):
         ENC_URL = 'http://example.com/%C3%A4'
 
         # Pass in an unencoded string as URL
-        response = self._makeOne()
-        result = response.redirect(URL)
-        self.assertEqual(result, ENC_URL)
-        self.assertEqual(response.getHeader('Location'), ENC_URL)
+        self._redirectURLCheck(URL, expected=ENC_URL)
 
         # Pass in an encoded string as URL
-        response = self._makeOne()
-        result = response.redirect(BYTES_URL)
-        self.assertEqual(result, ENC_URL)
-        self.assertEqual(response.getHeader('Location'), ENC_URL)
+        self._redirectURLCheck(BYTES_URL, expected=ENC_URL)
 
         # Pass in a HTTPException with an unencoded string as URL
         from zExceptions import HTTPMovedPermanently
         exc = HTTPMovedPermanently(URL)
-        response = self._makeOne()
-        result = response.redirect(exc)
-        self.assertEqual(response.getHeader('Location'), ENC_URL)
-        self.assertEqual(result, ENC_URL)
+        self._redirectURLCheck(exc, expected=ENC_URL)
 
         # Pass in a HTTPException with an encoded string as URL
         from zExceptions import HTTPMovedPermanently
         exc = HTTPMovedPermanently(BYTES_URL)
-        response = self._makeOne()
-        result = response.redirect(exc)
-        self.assertEqual(response.getHeader('Location'), ENC_URL)
-        self.assertEqual(result, ENC_URL)
+        self._redirectURLCheck(exc, expected=ENC_URL)
 
     def test_redirect_alreadyquoted(self):
         # If a URL is already quoted, don't double up on the quoting
         ENC_URL = 'http://example.com/M%C3%A4H'
-        response = self._makeOne()
-        result = response.redirect(ENC_URL)
-        self.assertEqual(result, ENC_URL)
-        self.assertEqual(response.getHeader('Location'), ENC_URL)
+        self._redirectURLCheck(ENC_URL)
+
+    def test_redirect_unreserved_chars(self):
+        # RFC 2396 section 2.3, characters that should not be encoded
+        url = "http://example.com/-_.!~*'()"
+        self._redirectURLCheck(url)
+
+    def test_redirect_reserved_chars(self):
+        # RFC 2396 section 3.3, characters with reserved meaning in a path
+        url = 'http://example.com/+/$/;/,/=/?/&/@@index.html'
+        self._redirectURLCheck(url)
 
     def test__encode_unicode_no_content_type_uses_default_encoding(self):
         UNICODE = u'<h1>Tr\u0039s Bien</h1>'

src/Zope2/Startup/datatypes.py

@@ -109,6 +109,8 @@ def root_wsgi_config(section):
         section.environment = ZDaemonEnvironDict()
     if section.clienthome is None:
         section.clienthome = os.path.join(section.instancehome, "var")
+    if getattr(section, 'pid_filename', None) is None:
+        section.pid_filename = os.path.join(section.clienthome, 'Z4.pid')
 
     if not section.databases:
         section.databases = []

src/Zope2/Startup/serve.py

@@ -22,12 +22,9 @@
 
 from paste.deploy import loadapp
 from paste.deploy import loadserver
+from six.moves import configparser
 
-
-try:
-    import configparser
-except ImportError:
-    import ConfigParser as configparser
+from App.config import getConfiguration
 
 
 def parse_vars(args):
@@ -199,6 +196,8 @@ def run(self):
             self.out(msg)
 
         def serve():
+            self.makePidFile()
+
             try:
                 server(app)
             except (SystemExit, KeyboardInterrupt) as e:
@@ -209,6 +208,8 @@ def serve():
                 else:
                     msg = ''
                 self.out('Exiting%s (-v to see traceback)' % msg)
+            finally:
+                self.unlinkPidFile()
 
         serve()
 
@@ -219,6 +220,28 @@ def loadserver(self, server_spec, name, relative_to, **kw):
         return loadserver(
             server_spec, name=name, relative_to=relative_to, **kw)
 
+    def makePidFile(self):
+        options = getConfiguration()
+        try:
+            IO_ERRORS = (IOError, OSError, WindowsError)
+        except NameError:
+            IO_ERRORS = (IOError, OSError)
+
+        try:
+            if os.path.exists(options.pid_filename):
+                os.unlink(options.pid_filename)
+            with open(options.pid_filename, 'w') as fp:
+                fp.write(str(os.getpid()))
+        except IO_ERRORS:
+            pass
+
+    def unlinkPidFile(self):
+        options = getConfiguration()
+        try:
+            os.unlink(options.pid_filename)
+        except OSError:
+            pass
+
 
 def main(argv=sys.argv, quiet=False):
     command = ServeCommand(argv, quiet=quiet)

src/Zope2/Startup/tests/test_schema.py

@@ -127,3 +127,17 @@ def test_default_zpublisher_encoding(self):
         self.assertEqual(
             ZPublisher.HTTPRequest.default_encoding, 'iso-8859-15')
         self.assertEqual(type(ZPublisher.HTTPRequest.default_encoding), str)
+
+    def test_pid_filename(self):
+        conf, dummy = self.load_config_text(u"""\
+            instancehome <<INSTANCE_HOME>>
+            """)
+        default = os.path.join(conf.clienthome, 'Z4.pid')
+        self.assertEqual(conf.pid_filename, default)
+
+        conf, dummy = self.load_config_text(u"""\
+            instancehome <<INSTANCE_HOME>>
+            pid-filename <<INSTANCE_HOME>>/Z5.pid
+            """)
+        expected = os.path.join(conf.instancehome, 'Z5.pid')
+        self.assertEqual(conf.pid_filename, expected)