conn: ensure complete_io flushes pending writes.

Previously, calls to `complete_io()` may return as if handshaking has
completed, but leave pending TLS writes queued that won't be sent until
a subsequent call to `complete_io()` is made.

This happens because `is_handshaking()` can begin to return false after
calls to `process_new_packets()` while there are final handshake packets
put in the connection's buffers, but not yet extracted to be sent to the
peer.

The end result is that calling `complete_io()` once is not
sufficient to fully complete a handshake with a peer. A second call
was required to flush the pending packets.

In this commit the `complete_io()` logic is updated to continue
processing IO when calling `process_new_packets()` has queued TLS
writes, only returning to the caller when all pending IO has been dealt
with and the handshake truly completed.

We can test this behaviour by updating the
`client_complete_io_for_handshake` and
`server_complete_io_for_handshake` unit tests to assert there are no
pending TLS writes after calling `complete_io()`. Prior to this commit
these assertions would fail, and with the updated logic they pass as
expected.

rustls/src/conn.rs

@@ -371,12 +371,13 @@ impl<Data> ConnectionCommon<Data> {
         Self: Sized,
         T: io::Read + io::Write,
     {
-        let until_handshaked = self.is_handshaking();
         let mut eof = false;
         let mut wrlen = 0;
         let mut rdlen = 0;
 
         loop {
+            let until_handshaked = self.is_handshaking();
+
             while self.wants_write() {
                 wrlen += self.write_tls(io)?;
             }
@@ -415,6 +416,13 @@ impl<Data> ConnectionCommon<Data> {
                 }
             };
 
+            // if we're doing IO until handshaked, and we believe we've finished handshaking,
+            // but process_new_packets() has queued TLS data to send, loop around again to write
+            // the queued messages.
+            if until_handshaked && !self.is_handshaking() && self.wants_write() {
+                continue;
+            }
+
             match (eof, until_handshaked, self.is_handshaking()) {
                 (_, true, false) => return Ok((rdlen, wrlen)),
                 (_, false, _) => return Ok((rdlen, wrlen)),

rustls/tests/api.rs

@@ -1315,6 +1315,7 @@ fn client_complete_io_for_handshake() {
         .unwrap();
     assert!(rdlen > 0 && wrlen > 0);
     assert!(!client.is_handshaking());
+    assert!(!client.wants_write());
 }
 
 #[test]
@@ -1390,6 +1391,7 @@ fn server_complete_io_for_handshake() {
             .unwrap();
         assert!(rdlen > 0 && wrlen > 0);
         assert!(!server.is_handshaking());
+        assert!(!server.wants_write());
     }
 }