diff --git a/src/lib-smtp/smtp-client-connection.c b/src/lib-smtp/smtp-client-connection.c
index bbb67e2a1a..3b5e0af28e 100644
--- a/src/lib-smtp/smtp-client-connection.c
+++ b/src/lib-smtp/smtp-client-connection.c
@@ -1003,8 +1003,10 @@ static void smtp_client_connection_input(struct connection *_conn)
 			i_assert(ret == 0);
 			return;
 		}
+	}
 
-		/* ready for SMTP handshake */
+	if (!conn->connect_succeeded) {
+		/* just got ready for SMTP handshake */
 		smtp_client_connection_established(conn);
 	}
 
@@ -1137,6 +1139,9 @@ static void smtp_client_connection_destroy(struct connection *_conn)
 static void
 smtp_client_connection_established(struct smtp_client_connection *conn)
 {
+	i_assert(!conn->connect_succeeded);
+	conn->connect_succeeded = TRUE;
+
 	if (conn->to_connect != NULL)
 		timeout_reset(conn->to_connect);
 
diff --git a/src/lib-smtp/smtp-client-private.h b/src/lib-smtp/smtp-client-private.h
index f70b2fd684..b0cbb5eeeb 100644
--- a/src/lib-smtp/smtp-client-private.h
+++ b/src/lib-smtp/smtp-client-private.h
@@ -163,6 +163,7 @@ struct smtp_client_connection {
 	bool authenticated:1;
 	bool initial_xclient_sent:1;
 	bool connect_failed:1;
+	bool connect_succeeded:1;
 	bool handshake_failed:1;
 	bool corked:1;
 	bool sent_quit:1;