Merge pull request #831 from tlsfuzzer/keyupdate

tls13-keyupdate: also test coalescing (combining)

docs/source/modifying-messages.rst

@@ -418,3 +418,43 @@ use the following code:
 You can find a usage example in:
 `test-large-hello.py
 <https://github.com/tomato42/tlsfuzzer/blob/master/scripts/test-large-hello.py>`_.
+
+Combining messages
+------------------
+
+While TLS allows for sending multiple messages with the same content type
+in a single record, for ease of debugging tlsfuzzer doesn't do that by default.
+
+But to verify that the other side of the connection can process such
+records (or that it rejects messages that must not be coalesced), it's
+possible to combine (coalesce) multiple messages with
+the same record_type.
+
+First, to queue a message instead of sending it, use the
+:py:func:`~tlsfuzzer.messages.queue_message` decorator:
+
+.. code:: python
+
+    node = node.add_child(queue_message(CertificateGenerator(cert_chain)))
+
+Then, to actually send the message, you can either send another message,
+of any type (the queue is flushed if the content_type of it doesn't match
+new message; and regular writes first queue a message and then flush
+the queue) or flush the queue manually using
+:py:class:`~tlsfuzzer.messages.FlushMessageQueue`:
+
+.. code:: python
+
+    node = node.add_child(FlushMessageQueue())
+
+.. note::
+
+    The ``post_send`` method is still executed right after the message is
+    queued, so if it has side effects, like updating the write state, the
+    actually sent record may be encrypted with wrong (i.e. future) keys.
+    Use ``RawMessageGenerator`` to create the message without side-effects.
+    Or use the :py:func:`~tlsfuzzer.messages.skip_post_send` to disable it.
+
+You can find a usage example in:
+`test-tls13-keyupdate.py
+<https://github.com/tomato42/tlsfuzzer/blob/master/scripts/test-tls13-keyupdate.py>`_.

scripts/test-tls13-keyupdate.py

@@ -14,7 +14,7 @@
         FinishedGenerator, ApplicationDataGenerator, AlertGenerator, \
         KeyUpdateGenerator, split_message, \
         FlushMessageList, truncate_handshake, \
-        pad_handshake
+        pad_handshake, queue_message, RawMessageGenerator, skip_post_send
 from tlsfuzzer.expect import ExpectServerHello, ExpectCertificate, \
         ExpectServerHelloDone, ExpectChangeCipherSpec, ExpectFinished, \
         ExpectAlert, ExpectApplicationData, ExpectClose, \
@@ -23,7 +23,7 @@
 
 from tlslite.constants import CipherSuite, AlertLevel, AlertDescription, \
         TLS_1_3_DRAFT, GroupName, ExtensionType, SignatureScheme, \
-        KeyUpdateMessageType, ContentType
+        KeyUpdateMessageType, ContentType, HandshakeType
 from tlslite.keyexchange import ECDHKeyExchange
 from tlsfuzzer.utils.lists import natural_sort_keys
 from tlslite.extensions import KeyShareEntry, ClientKeyShareExtension, \
@@ -32,7 +32,7 @@
 from tlsfuzzer.helpers import key_share_gen, RSA_SIG_ALL, key_share_ext_gen
 
 
-version = 4
+version = 5
 
 
 def help_msg():
@@ -324,6 +324,155 @@ def main():
     node.next_sibling.add_child(ExpectClose())
     conversations["large KeyUpdate message"] = conversation
 
+    # send two KeyUpdate messages in single record
+    conversation = Connect(host, port)
+    node = conversation
+    ciphers = [CipherSuite.TLS_AES_128_GCM_SHA256,
+               CipherSuite.TLS_EMPTY_RENEGOTIATION_INFO_SCSV]
+    ext = {}
+    groups = [GroupName.secp256r1]
+    key_shares = []
+    for group in groups:
+        key_shares.append(key_share_gen(group))
+    ext[ExtensionType.key_share] = ClientKeyShareExtension().create(key_shares)
+    ext[ExtensionType.supported_versions] = SupportedVersionsExtension()\
+        .create([TLS_1_3_DRAFT, (3, 3)])
+    ext[ExtensionType.supported_groups] = SupportedGroupsExtension()\
+        .create(groups)
+    sig_algs = [SignatureScheme.rsa_pss_rsae_sha256,
+                SignatureScheme.rsa_pss_pss_sha256]
+    ext[ExtensionType.signature_algorithms] = SignatureAlgorithmsExtension()\
+        .create(sig_algs)
+    ext[ExtensionType.signature_algorithms_cert] = SignatureAlgorithmsCertExtension()\
+        .create(RSA_SIG_ALL)
+    node = node.add_child(ClientHelloGenerator(ciphers, extensions=ext))
+    node = node.add_child(ExpectServerHello())
+    node = node.add_child(ExpectChangeCipherSpec())
+    node = node.add_child(ExpectEncryptedExtensions())
+    node = node.add_child(ExpectCertificate())
+    node = node.add_child(ExpectCertificateVerify())
+    node = node.add_child(ExpectFinished())
+    node = node.add_child(FinishedGenerator())
+    # because the post_send will fire without the message being sent
+    # both messages will be placed in a single record, but with the use
+    # of keys from the time _after_ the first KeyUpdate will be sent
+    node = node.add_child(queue_message(KeyUpdateGenerator(
+        message_type=KeyUpdateMessageType.update_not_requested)))
+    node = node.add_child(KeyUpdateGenerator(
+        message_type=KeyUpdateMessageType.update_requested))
+    node = node.add_child(ApplicationDataGenerator(
+        bytearray(b"GET / HTTP/1.0\r\n\r\n")))
+
+    # This message is optional and may show up 0 to many times
+    cycle = ExpectNewSessionTicket()
+    node = node.add_child(cycle)
+    node.add_child(cycle)
+
+    node.next_sibling = ExpectAlert(AlertLevel.fatal,
+                                    AlertDescription.bad_record_mac)
+    node.next_sibling.add_child(ExpectClose())
+    conversations["two KeyUpdates in one record with future keys"] = conversation
+
+    # send two KeyUpdate messages, one byte in one record, rest in second record
+    conversation = Connect(host, port)
+    node = conversation
+    ciphers = [CipherSuite.TLS_AES_128_GCM_SHA256,
+               CipherSuite.TLS_EMPTY_RENEGOTIATION_INFO_SCSV]
+    ext = {}
+    groups = [GroupName.secp256r1]
+    key_shares = []
+    for group in groups:
+        key_shares.append(key_share_gen(group))
+    ext[ExtensionType.key_share] = ClientKeyShareExtension().create(key_shares)
+    ext[ExtensionType.supported_versions] = SupportedVersionsExtension()\
+        .create([TLS_1_3_DRAFT, (3, 3)])
+    ext[ExtensionType.supported_groups] = SupportedGroupsExtension()\
+        .create(groups)
+    sig_algs = [SignatureScheme.rsa_pss_rsae_sha256,
+                SignatureScheme.rsa_pss_pss_sha256]
+    ext[ExtensionType.signature_algorithms] = SignatureAlgorithmsExtension()\
+        .create(sig_algs)
+    ext[ExtensionType.signature_algorithms_cert] = SignatureAlgorithmsCertExtension()\
+        .create(RSA_SIG_ALL)
+    node = node.add_child(ClientHelloGenerator(ciphers, extensions=ext))
+    node = node.add_child(ExpectServerHello())
+    node = node.add_child(ExpectChangeCipherSpec())
+    node = node.add_child(ExpectEncryptedExtensions())
+    node = node.add_child(ExpectCertificate())
+    node = node.add_child(ExpectCertificateVerify())
+    node = node.add_child(ExpectFinished())
+    node = node.add_child(FinishedGenerator())
+    fragment_list = []
+    keyupdate_gen = KeyUpdateGenerator(
+        message_type=KeyUpdateMessageType.update_not_requested)
+    node = node.add_child(split_message(keyupdate_gen, fragment_list, 1))
+    # by splitting the message 1/n, the first byte (the type - key update)
+    # will be send with old keys, but the rest of the message will be
+    # under new keys
+    node = node.add_child(queue_message(FlushMessageList(fragment_list)))
+    # because the post_send will fire without the message being sent
+    # both messages will be placed in a single record, but with the use
+    # of keys from the time _after_ the first KeyUpdate will be sent
+    node = node.add_child(KeyUpdateGenerator(
+        message_type=KeyUpdateMessageType.update_requested))
+    node = node.add_child(ApplicationDataGenerator(
+        bytearray(b"GET / HTTP/1.0\r\n\r\n")))
+
+    # This message is optional and may show up 0 to many times
+    cycle = ExpectNewSessionTicket()
+    node = node.add_child(cycle)
+    node.add_child(cycle)
+
+    node.next_sibling = ExpectAlert(AlertLevel.fatal,
+                                    AlertDescription.bad_record_mac)
+    node.next_sibling.add_child(ExpectClose())
+    conversations["two KeyUpdates, first fragmented, second fragment under new keys together with second KeyUpdate"] = conversation
+
+    conversation = Connect(host, port)
+    node = conversation
+    ciphers = [CipherSuite.TLS_AES_128_GCM_SHA256,
+               CipherSuite.TLS_EMPTY_RENEGOTIATION_INFO_SCSV]
+    ext = {}
+    groups = [GroupName.secp256r1]
+    key_shares = []
+    for group in groups:
+        key_shares.append(key_share_gen(group))
+    ext[ExtensionType.key_share] = ClientKeyShareExtension().create(key_shares)
+    ext[ExtensionType.supported_versions] = SupportedVersionsExtension()\
+        .create([TLS_1_3_DRAFT, (3, 3)])
+    ext[ExtensionType.supported_groups] = SupportedGroupsExtension()\
+        .create(groups)
+    sig_algs = [SignatureScheme.rsa_pss_rsae_sha256,
+                SignatureScheme.rsa_pss_pss_sha256]
+    ext[ExtensionType.signature_algorithms] = SignatureAlgorithmsExtension()\
+        .create(sig_algs)
+    ext[ExtensionType.signature_algorithms_cert] = SignatureAlgorithmsCertExtension()\
+        .create(RSA_SIG_ALL)
+    node = node.add_child(ClientHelloGenerator(ciphers, extensions=ext))
+    node = node.add_child(ExpectServerHello())
+    node = node.add_child(ExpectChangeCipherSpec())
+    node = node.add_child(ExpectEncryptedExtensions())
+    node = node.add_child(ExpectCertificate())
+    node = node.add_child(ExpectCertificateVerify())
+    node = node.add_child(ExpectFinished())
+    node = node.add_child(FinishedGenerator())
+    node = node.add_child(queue_message(skip_post_send(
+        KeyUpdateGenerator(KeyUpdateMessageType.update_not_requested))))
+    node = node.add_child(KeyUpdateGenerator(
+        message_type=KeyUpdateMessageType.update_requested))
+    node = node.add_child(ApplicationDataGenerator(
+        bytearray(b"GET / HTTP/1.0\r\n\r\n")))
+
+    # This message is optional and may show up 0 to many times
+    cycle = ExpectNewSessionTicket()
+    node = node.add_child(cycle)
+    node.add_child(cycle)
+
+    node.next_sibling = ExpectAlert(AlertLevel.fatal,
+                                    AlertDescription.unexpected_message)
+    node.next_sibling.add_child(ExpectClose())
+    conversations["two KeyUpdates in one record"] = conversation
+
     for msg_type in range(2, 256):
         conversation = Connect(host, port)
         node = conversation

tests/test_tlsfuzzer_runner.py

@@ -149,6 +149,7 @@ def test_run_with_generator_node(self):
         node.is_command = mock.Mock(return_value=False)
         node.is_expect = mock.Mock(return_value=False)
         node.is_generator = mock.Mock(return_value=True)
+        node.queue = False
         node.child = None
         msg = mock.MagicMock()
         msg.write = mock.Mock(return_value=bytearray(b'\x01\x00'))
@@ -322,6 +323,7 @@ def test_run_with_generate_and_unexpected_closed_socket(self, mock_print):
         node.is_command = mock.Mock(return_value=False)
         node.is_expect = mock.Mock(return_value=False)
         node.is_generator = mock.Mock(return_value=True)
+        node.queue = False
         node.child = None
 
         runner = Runner(node)

tlsfuzzer/messages.py

@@ -453,6 +453,7 @@ def __init__(self):
         """Initialize the object."""
         super(MessageGenerator, self).__init__()
         self.msg = None
+        self.queue = False
 
     def is_command(self):
         """Define object as a generator node."""
@@ -2096,6 +2097,48 @@ def new_generate(state, old_generate=generator.generate,
     return generator
 
 
+def queue_message(generator):
+    """Queue message with other ones of the same content type.
+
+    Allow coalescing of the message with other messages with the same
+    content type, this allows for sending Certificate and CertificateVerify
+    in a single record.
+    """
+    assert generator.is_generator()
+    generator.queue = True
+    return generator
+
+
+def skip_post_send(generator):
+    """Make the post_send method of generator do nothing.
+
+    This is useful when combining messages that update connection state,
+    like KeyUpdate or Finished in TLS 1.3.
+    """
+    assert generator.is_generator()
+
+    generator.post_send = lambda _: None
+    return generator
+
+
+class FlushMessageQueue(Command):
+    """Flush the record layer queue of messages."""
+
+    def __init__(self, description=None):
+        super(FlushMessageQueue, self).__init__()
+        self.description = description
+
+    def __repr__(self):
+        vals = []
+        if self.description:
+            vals.append(('description', repr(self.description)))
+        return 'FlushMessageQueue({0})'.format(
+                ', '.join("{0}={1}".format(i[0], i[1]) for i in vals))
+
+    def process(self, state):
+        state.msg_sock.flushBlocking()
+
+
 class PopMessageFromList(MessageGenerator):
     """Takes a reference to list, pops a message from it to generate one."""
 

tlsfuzzer/runner.py

@@ -247,10 +247,14 @@ def run(self):
                     # send message to peer
                     msg = node.generate(self.state)
                     try:
+                        # sendMessageBlocking is buffered and fragmenting
+                        # that means that 0-length messages would get lost
+                        # so send them directly through record layer
                         if msg.write():
-                            # sendMessageBlocking is buffered and fragmenting
-                            # that means that 0-length messages would get lost
-                            self.state.msg_sock.sendMessageBlocking(msg)
+                            if node.queue:
+                                self.state.msg_sock.queueMessageBlocking(msg)
+                            else:
+                                self.state.msg_sock.sendMessageBlocking(msg)
                         else:
                             for _ in self.state.msg_sock.sendRecord(msg):
                                 # make the method into a blocking one