Merge branch 'master' into issue_580

CHANGES.rst

@@ -62,6 +62,11 @@ Features
 Other changes
 +++++++++++++
 
+- Make Zope write a PID file again under WSGI.
+  This makes interaction with sysadmin tools easier.
+  The PID file path can be set in the Zope configuration with ``pid-filename``,
+  just like in ``ZServer``-based configurations.
+
 - Exceptions during publishing are now re-raised in a new exceptions debug
   mode to allow WSGI middleware to handle/debug it. See the `debug
   documentation <https://zope.readthedocs.io/en/latest/wsgi.html#werkzeug>`_

setup.cfg

@@ -14,8 +14,8 @@ ignore =
 force_single_line = True
 combine_as_imports = True
 sections = FUTURE,STDLIB,THIRDPARTY,ZOPE,FIRSTPARTY,LOCALFOLDER
-known_third_party = ipaddress, PasteDeploy, six, waitress, chameleon
-known_zope = AccessControl, Acquisition, DateTime, DocumentTemplate, ExtensionClass, MultiMapping, Persistence, persistent, RestrictedPython, Testing, transaction, ZConfig, zExceptions, ZODB, zope, Zope2
+known_third_party = ipaddress, PasteDeploy, six, waitress, chameleon, paste
+known_zope = AccessControl, Acquisition, DateTime, DocumentTemplate, ExtensionClass, MultiMapping, Persistence, persistent, RestrictedPython, Testing, transaction, ZConfig, zExceptions, ZODB, zope, Zope2, App
 default_section = ZOPE
 line_length = 79
 lines_after_imports = 2

src/ZPublisher/HTTPResponse.py

@@ -27,7 +27,6 @@
 from six import reraise
 from six import text_type
 from six.moves.urllib.parse import quote
-from six.moves.urllib.parse import unquote
 from six.moves.urllib.parse import urlparse
 from six.moves.urllib.parse import urlunparse
 
@@ -217,7 +216,14 @@ def redirect(self, location, status=302, lock=0):
         # characters in the path part are quoted correctly. This is required
         # as we now allow non-ASCII IDs
         parsed = list(urlparse(location))
-        parsed[2] = quote(unquote(parsed[2]))
+
+        # Make a hacky guess whether the path component is already
+        # URL-encoded by checking for %. If it is, we don't touch it.
+        if '%' not in parsed[2]:
+            # The list of "safe" characters is from RFC 2396 section 2.3
+            # (unreserved characters that should not be escaped) plus
+            # section 3.3 (reserved characters in path components)
+            parsed[2] = quote(parsed[2], safe="/@!*'~();,=+$")
         location = urlunparse(parsed)
 
         self.setStatus(status, lock=lock)

src/ZPublisher/tests/testHTTPResponse.py

@@ -725,29 +725,31 @@ def test_setBody_compression_no_prior_vary_header_but_forced(self):
         response.setBody(b'foo' * 100)  # body must get smaller on compression
         self.assertEqual(response.getHeader('Vary'), None)
 
-    def test_redirect_defaults(self):
-        URL = 'http://example.com'
+    def _redirectURLCheck(self, url, expected=None, lock=False, status=302):
+        if not expected:
+            expected = url
         response = self._makeOne()
-        result = response.redirect(URL)
-        self.assertEqual(result, URL)
+        result = response.redirect(url, lock=lock, status=status)
+        self.assertEqual(result, expected)
+        self.assertEqual(response.getHeader('Location'), expected)
+        return response
+
+    def test_redirect_defaults(self):
+        URL = 'http://example.com/@@login'
+        response = self._redirectURLCheck(URL)
         self.assertEqual(response.status, 302)
-        self.assertEqual(response.getHeader('Location'), URL)
         self.assertFalse(response._locked_status)
 
     def test_redirect_explicit_status(self):
         URL = 'http://example.com'
-        response = self._makeOne()
-        response.redirect(URL, status=307)
+        response = self._redirectURLCheck(URL, status=307)
         self.assertEqual(response.status, 307)
-        self.assertEqual(response.getHeader('Location'), URL)
         self.assertFalse(response._locked_status)
 
     def test_redirect_w_lock(self):
         URL = 'http://example.com'
-        response = self._makeOne()
-        response.redirect(URL, lock=True)
+        response = self._redirectURLCheck(URL, lock=True)
         self.assertEqual(response.status, 302)
-        self.assertEqual(response.getHeader('Location'), URL)
         self.assertTrue(response._locked_status)
 
     def test_redirect_nonascii(self):
@@ -756,40 +758,35 @@ def test_redirect_nonascii(self):
         ENC_URL = 'http://example.com/%C3%A4'
 
         # Pass in an unencoded string as URL
-        response = self._makeOne()
-        result = response.redirect(URL)
-        self.assertEqual(result, ENC_URL)
-        self.assertEqual(response.getHeader('Location'), ENC_URL)
+        self._redirectURLCheck(URL, expected=ENC_URL)
 
         # Pass in an encoded string as URL
-        response = self._makeOne()
-        result = response.redirect(BYTES_URL)
-        self.assertEqual(result, ENC_URL)
-        self.assertEqual(response.getHeader('Location'), ENC_URL)
+        self._redirectURLCheck(BYTES_URL, expected=ENC_URL)
 
         # Pass in a HTTPException with an unencoded string as URL
         from zExceptions import HTTPMovedPermanently
         exc = HTTPMovedPermanently(URL)
-        response = self._makeOne()
-        result = response.redirect(exc)
-        self.assertEqual(response.getHeader('Location'), ENC_URL)
-        self.assertEqual(result, ENC_URL)
+        self._redirectURLCheck(exc, expected=ENC_URL)
 
         # Pass in a HTTPException with an encoded string as URL
         from zExceptions import HTTPMovedPermanently
         exc = HTTPMovedPermanently(BYTES_URL)
-        response = self._makeOne()
-        result = response.redirect(exc)
-        self.assertEqual(response.getHeader('Location'), ENC_URL)
-        self.assertEqual(result, ENC_URL)
+        self._redirectURLCheck(exc, expected=ENC_URL)
 
     def test_redirect_alreadyquoted(self):
         # If a URL is already quoted, don't double up on the quoting
         ENC_URL = 'http://example.com/M%C3%A4H'
-        response = self._makeOne()
-        result = response.redirect(ENC_URL)
-        self.assertEqual(result, ENC_URL)
-        self.assertEqual(response.getHeader('Location'), ENC_URL)
+        self._redirectURLCheck(ENC_URL)
+
+    def test_redirect_unreserved_chars(self):
+        # RFC 2396 section 2.3, characters that should not be encoded
+        url = "http://example.com/-_.!~*'()"
+        self._redirectURLCheck(url)
+
+    def test_redirect_reserved_chars(self):
+        # RFC 2396 section 3.3, characters with reserved meaning in a path
+        url = 'http://example.com/+/$/;/,/=/?/&/@@index.html'
+        self._redirectURLCheck(url)
 
     def test__encode_unicode_no_content_type_uses_default_encoding(self):
         UNICODE = u'<h1>Tr\u0039s Bien</h1>'

src/Zope2/Startup/datatypes.py

@@ -109,6 +109,8 @@ def root_wsgi_config(section):
         section.environment = ZDaemonEnvironDict()
     if section.clienthome is None:
         section.clienthome = os.path.join(section.instancehome, "var")
+    if getattr(section, 'pid_filename', None) is None:
+        section.pid_filename = os.path.join(section.clienthome, 'Z4.pid')
 
     if not section.databases:
         section.databases = []

src/Zope2/Startup/serve.py

@@ -22,12 +22,9 @@
 
 from paste.deploy import loadapp
 from paste.deploy import loadserver
+from six.moves import configparser
 
-
-try:
-    import configparser
-except ImportError:
-    import ConfigParser as configparser
+from App.config import getConfiguration
 
 
 def parse_vars(args):
@@ -199,6 +196,8 @@ def run(self):
             self.out(msg)
 
         def serve():
+            self.makePidFile()
+
             try:
                 server(app)
             except (SystemExit, KeyboardInterrupt) as e:
@@ -209,6 +208,8 @@ def serve():
                 else:
                     msg = ''
                 self.out('Exiting%s (-v to see traceback)' % msg)
+            finally:
+                self.unlinkPidFile()
 
         serve()
 
@@ -219,6 +220,28 @@ def loadserver(self, server_spec, name, relative_to, **kw):
         return loadserver(
             server_spec, name=name, relative_to=relative_to, **kw)
 
+    def makePidFile(self):
+        options = getConfiguration()
+        try:
+            IO_ERRORS = (IOError, OSError, WindowsError)
+        except NameError:
+            IO_ERRORS = (IOError, OSError)
+
+        try:
+            if os.path.exists(options.pid_filename):
+                os.unlink(options.pid_filename)
+            with open(options.pid_filename, 'w') as fp:
+                fp.write(str(os.getpid()))
+        except IO_ERRORS:
+            pass
+
+    def unlinkPidFile(self):
+        options = getConfiguration()
+        try:
+            os.unlink(options.pid_filename)
+        except OSError:
+            pass
+
 
 def main(argv=sys.argv, quiet=False):
     command = ServeCommand(argv, quiet=quiet)

src/Zope2/Startup/tests/test_schema.py

@@ -127,3 +127,17 @@ def test_default_zpublisher_encoding(self):
         self.assertEqual(
             ZPublisher.HTTPRequest.default_encoding, 'iso-8859-15')
         self.assertEqual(type(ZPublisher.HTTPRequest.default_encoding), str)
+
+    def test_pid_filename(self):
+        conf, dummy = self.load_config_text(u"""\
+            instancehome <<INSTANCE_HOME>>
+            """)
+        default = os.path.join(conf.clienthome, 'Z4.pid')
+        self.assertEqual(conf.pid_filename, default)
+
+        conf, dummy = self.load_config_text(u"""\
+            instancehome <<INSTANCE_HOME>>
+            pid-filename <<INSTANCE_HOME>>/Z5.pid
+            """)
+        expected = os.path.join(conf.instancehome, 'Z5.pid')
+        self.assertEqual(conf.pid_filename, expected)

src/Zope2/Startup/tests/test_serve.py

@@ -0,0 +1,116 @@
+##############################################################################
+#
+# Copyright (c) 2019 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+
+import io
+import sys
+import unittest
+
+import six
+
+
+class TestFunctions(unittest.TestCase):
+
+    def test_parse_vars(self):
+        from Zope2.Startup.serve import parse_vars
+
+        input_data = []
+        self.assertEqual(parse_vars(input_data), {})
+
+        # There is now stripping of whitespace
+        input_data = ['foo = bar', 'abc=xyz']
+        self.assertEqual(parse_vars(input_data),
+                         {'foo ': ' bar', 'abc': 'xyz'})
+
+        input_data = ['in:valid']
+        self.assertRaises(ValueError, parse_vars, input_data)
+
+    def test__getpathsec(self):
+        from Zope2.Startup.serve import _getpathsec
+
+        self.assertEqual(_getpathsec('', ''), ('', 'main'))
+        self.assertEqual(_getpathsec('name', ''), ('name', 'main'))
+        self.assertEqual(_getpathsec('name#name2', ''), ('name', 'name2'))
+        self.assertEqual(_getpathsec('name1', 'name2'), ('name1', 'name2'))
+
+
+class TestServeCommand(unittest.TestCase):
+
+    def _makeOne(self, argv=[], quiet=False):
+        from Zope2.Startup.serve import ServeCommand
+        return ServeCommand(argv, quiet=quiet)
+
+    def _getFileLike(self):
+        if six.PY2:
+            return io.BytesIO()
+        else:
+            return io.StringIO()
+
+    def test_defaults(self):
+        srv = self._makeOne()
+        self.assertEqual(srv.args, [])
+        self.assertIsNone(srv.options.app_name)
+        self.assertIsNone(srv.options.server)
+        self.assertIsNone(srv.options.server_name)
+        self.assertEqual(srv.options.verbose, 1)
+        self.assertIsNone(srv.options.debug)
+        self.assertIsNone(srv.options.debug_exceptions)
+
+    def test_nondefaults(self):
+        args = ['command', '-n', 'myapp', '-s', 'mywsgi',
+                '--server-name=myserver', '-vvvv', '-d', '-e']
+        srv = self._makeOne(args)
+        self.assertEqual(srv.args, [])
+        self.assertEqual(srv.options.app_name, 'myapp')
+        self.assertEqual(srv.options.server, 'mywsgi')
+        self.assertEqual(srv.options.server_name, 'myserver')
+        self.assertEqual(srv.options.verbose, 5)
+        self.assertTrue(srv.options.debug)
+        self.assertTrue(srv.options.debug_exceptions)
+
+    def test_quiet(self):
+        args = ['command', '-q']
+        srv = self._makeOne(args)
+        self.assertEqual(srv.options.verbose, 0)
+
+        srv = self._makeOne([], quiet=True)
+        self.assertEqual(srv.options.verbose, 0)
+
+    def test_out_notquiet(self):
+        old_stdout = sys.stdout
+
+        new_stdout = self._getFileLike()
+        sys.stdout = new_stdout
+
+        try:
+            srv = self._makeOne(quiet=False)
+            srv.out('output')
+            self.assertEqual(new_stdout.getvalue().strip(), 'output')
+        finally:
+            sys.stdout = old_stdout
+
+    def test_out_quiet(self):
+        old_stdout = sys.stdout
+        new_stdout = self._getFileLike()
+        sys.stdout = new_stdout
+        try:
+            srv = self._makeOne(quiet=True)
+            srv.out('output')
+            self.assertEqual(new_stdout.getvalue().strip(), '')
+        finally:
+            sys.stdout = old_stdout
+
+    def test_remaining_options(self):
+        args = ['command', 'somekey=somevalue', 'foo=bar']
+        srv = self._makeOne(args)
+        self.assertEqual(srv.args, ['somekey=somevalue', 'foo=bar'])

src/Zope2/Startup/wsgischema.xml

@@ -214,6 +214,14 @@
      <metadefault>off</metadefault>
   </key>
 
+  <key name="pid-filename" datatype="existing-dirpath">
+    <description>
+      The full path to which the Zope process will write its
+      OS process id at startup.
+    </description>
+    <metadefault>$clienthome/Z4.pid</metadefault>
+  </key>
+
   <multikey name="trusted-proxy" datatype="ipaddr-or-hostname"
        attribute="trusted_proxies">
      <description>

src/Zope2/utilities/skel/etc/wsgi.conf.in

@@ -131,6 +131,19 @@ instancehome $INSTANCE
 #     zmi-bookmarkable-urls on
 
 
+# Directive: pid-filename
+#
+# Description:
+#     The path to the file in which the Zope process id(s) will be written.
+#     This defaults to client-home/Z4.pid.
+#
+# Default: CLIENT_HOME/Z4.pid
+#
+# Example:
+#
+#    pid-filename /home/chrism/projects/sessions/var/Z4.pid
+
+
 # Directive: trusted-proxy
 #
 # Description: