Merge branch 'trunk' into 9716-fix-jelly-timedelta

NEWS.rst

@@ -3,6 +3,80 @@ http://twistedmatrix.com/trac/ticket/<number>
 
 .. towncrier release notes start
 
+Twisted 19.10.0 (2019-11-03)
+============================
+
+Features
+--------
+
+- twisted.trial.successResultOf, twisted.trial.failureResultOf, and
+  twisted.trial.assertNoResult accept coroutines as well as Deferreds. (#9006)
+
+
+Bugfixes
+--------
+
+- Fixed circular import in twisted.trial.reporter, introduced in Twisted 16.0.0. (#8267)
+- The POP3 server implemented by twisted.mail.pop3 now accepts passwords that contain spaces. (#9100)
+- Incoming HTTP/2 connections will now not time out if they persist for longer than one minute. (#9653)
+- The serial extra now requires pywin32 on Windows enabling use of twisted.internet.serialport without specifying the windows_platform extra. (#9700)
+
+
+Misc
+----
+
+- #8506, #9677, #9684, #9687, #9688
+
+
+Conch
+-----
+
+Bugfixes
+~~~~~~~~
+
+- twisted.conch.ssh.keys now correctly writes the "iqmp" parameter in serialized RSA private keys as q^-1 mod p rather than p^-1 mod q. (#9681)
+
+
+Misc
+~~~~
+
+- #9689
+
+
+Web
+---
+
+Features
+~~~~~~~~
+
+- twisted.web.server.Request will now use twisted.web.server.Site.getContentFile, if it exists, to get a file into which to write request content.  If getContentFile is not provided by the site, it will fall back to the previous behavior of using io.BytesIO for small requests and tempfile.TemporaryFile for large ones. (#9655)
+
+
+Bugfixes
+~~~~~~~~
+
+- twisted.web.client.FileBodyProducer will now stop producing when the Deferred returned by FileBodyProducer.startProducing is cancelled. (#9547)
+- The HTTP/2 server implementation now enforces TCP flow control on control frame messages and times out clients that send invalid data without reading responses.  This closes CVE-2019-9512 (Ping Flood), CVE-2019-9514 (Reset Flood), and CVE-2019-9515 (Settings Flood).  Thanks to Jonathan Looney and Piotr Sikora. (#9694)
+
+
+Mail
+----
+
+No significant changes.
+
+
+Words
+-----
+
+No significant changes.
+
+
+Names
+-----
+
+No significant changes.
+
+
 Twisted 19.7.0 (2019-07-28)
 ===========================
 

README.rst

@@ -5,7 +5,7 @@ Twisted
 |travis|_
 |circleci|_
 
-For information on what's new in Twisted 19.2.0, see the `NEWS <NEWS.rst>`_ file that comes with the distribution.
+For information on changes in this release, see the `NEWS <NEWS.rst>`_ file.
 
 
 What is this?

docs/core/howto/trial.rst

@@ -646,7 +646,7 @@ run your test suite using another debugger instead. To specify a debugger other
 than ``pdb`` , pass in the fully-qualified name of an
 object that provides the same interface as ``pdb`` .
 Most third-party debuggers tend to implement an interface similar to ``pdb`` , or at least provide a wrapper object that
-does. For example, invoking Trial with the extra arguments ``-debug --debugger pudb`` will open the `PuDB <http://pypi.python.org/pypi/pudb>`_ debugger instead, provided
+does. For example, invoking Trial with the extra arguments ``--debug --debugger pudb`` will open the `PuDB <http://pypi.python.org/pypi/pudb>`_ debugger instead, provided
 it is properly installed.
 
 

src/twisted/_version.py

@@ -7,5 +7,5 @@
 
 from incremental import Version
 
-__version__ = Version('Twisted', 19, 7, 0, dev=0)
+__version__ = Version('Twisted', 19, 10, 0, dev=0)
 __all__ = ["__version__"]

src/twisted/conch/newsfragments/9682.bugfix

@@ -0,0 +1 @@
+twisted.conch.keys.Key.privateBlob now returns the correct blob format for ECDSA (i.e. the same as that implemented by OpenSSH).

src/twisted/conch/ssh/keys.py

@@ -1180,9 +1180,13 @@ def privateBlob(self):
             return (common.NS(b'ssh-dss') + common.MP(data['p']) +
                     common.MP(data['q']) + common.MP(data['g']) +
                     common.MP(data['y']) + common.MP(data['x']))
-        else: # EC
-            return (common.NS(data['curve']) + common.MP(data['x']) +
-                    common.MP(data['y']) + common.MP(data['privateValue']))
+        else:  # EC
+            encPub = self._keyObject.public_key().public_bytes(
+                serialization.Encoding.X962,
+                serialization.PublicFormat.UncompressedPoint
+            )
+            return (common.NS(data['curve']) + common.NS(data['curve'][-8:]) +
+                    common.NS(encPub) + common.MP(data['privateValue']))
 
     def toString(self, type, extra=None):
         """

src/twisted/conch/test/test_keys.py

@@ -790,6 +790,8 @@ def test_fromPrivateBlobRSA(self):
 
         self.assertFalse(rsaKey.isPublic())
         self.assertEqual(keydata.RSAData, rsaKey.data())
+        self.assertEqual(
+            rsaKey, keys.Key._fromString_PRIVATE_BLOB(rsaKey.privateBlob()))
 
 
     def test_fromPrivateBlobDSA(self):
@@ -809,6 +811,8 @@ def test_fromPrivateBlobDSA(self):
 
         self.assertFalse(dsaKey.isPublic())
         self.assertEqual(keydata.DSAData, dsaKey.data())
+        self.assertEqual(
+            dsaKey, keys.Key._fromString_PRIVATE_BLOB(dsaKey.privateBlob()))
 
 
     def test_fromPrivateBlobECDSA(self):
@@ -835,6 +839,8 @@ def test_fromPrivateBlobECDSA(self):
 
         self.assertFalse(eckey.isPublic())
         self.assertEqual(keydata.ECDatanistp256, eckey.data())
+        self.assertEqual(
+            eckey, keys.Key._fromString_PRIVATE_BLOB(eckey.privateBlob()))
 
 
     def test_blobRSA(self):
@@ -935,11 +941,15 @@ def test_privateBlobEC(self):
         L{keys.Key.privateBlob} returns the SSH ptotocol-level format of EC
         private key.
         """
+        from cryptography.hazmat.primitives import serialization
         self.assertEqual(
             keys.Key(self.ecObj).privateBlob(),
             common.NS(keydata.ECDatanistp256['curve']) +
-            common.MP(self.ecObj.private_numbers().public_numbers.x) +
-            common.MP(self.ecObj.private_numbers().public_numbers.y) +
+            common.NS(keydata.ECDatanistp256['curve'][-8:]) +
+            common.NS(
+                self.ecObj.public_key().public_bytes(
+                    serialization.Encoding.X962,
+                    serialization.PublicFormat.UncompressedPoint)) +
             common.MP(self.ecObj.private_numbers().private_value)
             )
 

src/twisted/cred/checkers.py

@@ -2,6 +2,12 @@
 # Copyright (c) Twisted Matrix Laboratories.
 # See LICENSE for details.
 
+"""
+Basic credential checkers
+
+@var ANONYMOUS: An empty tuple used to represent the anonymous avatar ID.
+"""
+
 from __future__ import division, absolute_import
 
 import os
@@ -14,24 +20,29 @@
 from twisted.cred import error, credentials
 
 
+
 class ICredentialsChecker(Interface):
     """
-    An object that can check sub-interfaces of ICredentials.
+    An object that can check sub-interfaces of L{ICredentials}.
     """
 
-    credentialInterfaces = Attribute(
-        'A list of sub-interfaces of ICredentials which specifies which I may check.')
+    credentialInterfaces = Attribute((
+        'A list of sub-interfaces of L{ICredentials} which specifies which I '
+        'may check.'
+    ))
 
 
     def requestAvatarId(credentials):
         """
+        Validate credentials and produce an avatar ID.
+
         @param credentials: something which implements one of the interfaces in
-        self.credentialInterfaces.
+        C{credentialInterfaces}.
 
-        @return: a Deferred which will fire a string which identifies an
-        avatar, an empty tuple to specify an authenticated anonymous user
-        (provided as checkers.ANONYMOUS) or fire a Failure(UnauthorizedLogin).
-        Alternatively, return the result itself.
+        @return: a L{Deferred} which will fire with a L{bytes} that identifies
+        an avatar, an empty tuple to specify an authenticated anonymous user
+        (provided as L{twisted.cred.checkers.ANONYMOUS}) or fail with
+        L{UnauthorizedLogin}. Alternatively, return the result itself.
 
         @see: L{twisted.cred.credentials}
         """
@@ -51,11 +62,22 @@ def requestAvatarId(credentials):
 ANONYMOUS = ()
 
 
+
 @implementer(ICredentialsChecker)
 class AllowAnonymousAccess:
+    """
+    A credentials checker that unconditionally grants anonymous access.
+
+    @cvar credentialInterfaces: Tuple containing L{IAnonymous}.
+    """
     credentialInterfaces = credentials.IAnonymous,
 
     def requestAvatarId(self, credentials):
+        """
+        Succeed with the L{ANONYMOUS} avatar ID.
+
+        @return: L{Deferred} that fires with L{twisted.cred.checkers.ANONYMOUS}
+        """
         return defer.succeed(ANONYMOUS)
 
 
@@ -71,15 +93,45 @@ class InMemoryUsernamePasswordDatabaseDontUse(object):
     You really don't want to use this for anything else.  It is, at best, a
     toy.  If you need a simple credentials checker for a real application,
     see L{FilePasswordDB}.
+
+    @cvar credentialInterfaces: Tuple of L{IUsernamePassword} and
+    L{IUsernameHashedPassword}.
+
+    @ivar users: Mapping of usernames to passwords.
+    @type users: L{dict} mapping L{bytes} to L{bytes}
     """
     credentialInterfaces = (credentials.IUsernamePassword,
                             credentials.IUsernameHashedPassword)
 
     def __init__(self, **users):
-        self.users = {x.encode('ascii'):y for x, y in users.items()}
+        """
+        Initialize the in-memory database.
+
+        For example::
+
+            db = InMemoryUsernamePasswordDatabaseDontUse(
+                user1=b'sesame',
+                user2=b'hunter2',
+            )
+
+        @param users: Usernames and passwords to seed the database with.
+        Each username given as a keyword is encoded to L{bytes} as ASCII.
+        Passwords must be given as L{bytes}.
+        @type users: L{dict} of L{str} to L{bytes}
+        """
+        self.users = {x.encode('ascii'): y for x, y in users.items()}
 
 
     def addUser(self, username, password):
+        """
+        Set a user's password.
+
+        @param username: Name of the user.
+        @type username: L{bytes}
+
+        @param password: Password to associate with the username.
+        @type password: L{bytes}
+        """
         self.users[username] = password
 
 
@@ -111,9 +163,9 @@ class FilePasswordDB:
     by this string, as does the password.  Both fields are specifiable.  If
     the passwords are not stored plaintext, a hash function must be supplied
     to convert plaintext passwords to the form stored on disk and this
-    CredentialsChecker will only be able to check IUsernamePassword
+    CredentialsChecker will only be able to check L{IUsernamePassword}
     credentials.  If the passwords are stored plaintext,
-    IUsernameHashedPassword credentials will be checkable as well.
+    L{IUsernameHashedPassword} credentials will be checkable as well.
     """
 
     cache = False
@@ -124,22 +176,22 @@ class FilePasswordDB:
     def __init__(self, filename, delim=b':', usernameField=0, passwordField=1,
                  caseSensitive=True, hash=None, cache=False):
         """
-        @type filename: C{str}
+        @type filename: L{str}
         @param filename: The name of the file from which to read username and
         password information.
 
-        @type delim: C{str}
+        @type delim: L{bytes}
         @param delim: The field delimiter used in the file.
 
-        @type usernameField: C{int}
+        @type usernameField: L{int}
         @param usernameField: The index of the username after splitting a
         line on the delimiter.
 
-        @type passwordField: C{int}
+        @type passwordField: L{int}
         @param passwordField: The index of the password after splitting a
         line on the delimiter.
 
-        @type caseSensitive: C{bool}
+        @type caseSensitive: L{bool}
         @param caseSensitive: If true, consider the case of the username when
         performing a lookup.  Ignore it otherwise.
 
@@ -151,7 +203,7 @@ def __init__(self, filename, delim=b':', usernameField=0, passwordField=1,
         version of the password.  If the return value compares equal to the
         version stored on disk, the credentials are accepted.
 
-        @type cache: C{bool}
+        @type cache: L{bool}
         @param cache: If true, maintain an in-memory cache of the
         contents of the password file.  On lookups, the mtime of the
         file will be checked, and the file will only be re-parsed if
@@ -225,6 +277,19 @@ def _loadCredentials(self):
 
 
     def getUser(self, username):
+        """
+        Look up the credentials for a username.
+
+        @param username: The username to look up.
+        @type username: L{bytes}
+
+        @returns: Two-tuple of the canonicalicalized username (i.e. lowercase
+        if the database is not case sensitive) and the associated password
+        value, both L{bytes}.
+        @rtype: L{tuple}
+
+        @raises KeyError: When lookup of the username fails.
+        """
         if not self.caseSensitive:
             username = username.lower()
 

src/twisted/cred/credentials.py

@@ -33,7 +33,7 @@ class ICredentials(Interface):
     """
     I check credentials.
 
-    Implementors _must_ specify which sub-interfaces of ICredentials
+    Implementors I{must} specify the sub-interfaces of ICredentials
     to which it conforms, using L{zope.interface.declarations.implementer}.
     """
 
@@ -121,6 +121,8 @@ def checkPassword(password):
 class IAnonymous(ICredentials):
     """
     I am an explicitly anonymous request for access.
+
+    @see: L{twisted.cred.checkers.AllowAnonymousAccess}
     """
 
 

src/twisted/internet/testing.py

@@ -635,7 +635,7 @@ def adoptDatagramPort(self, fileno, addressFamily, protocol,
                           maxPacketSize=8192):
         """
         Fake L{IReactorSocket.adoptDatagramPort}, that logs the call and
-            returns a fake L{IListeningPort}.
+        returns a fake L{IListeningPort}.
 
         @see: L{twisted.internet.interfaces.IReactorSocket.adoptDatagramPort}
         """

src/twisted/newsfragments/9690.doc

@@ -0,0 +1 @@
+Added a missing hyphen to a reference to the ``--debug`` option of ``pdb`` in the Trial how-to.