From 82d07eb351c1a4dc25326148e6fc390f9c065d34 Mon Sep 17 00:00:00 2001 From: Toni Uhlig Date: Tue, 5 Jul 2022 13:11:41 +0200 Subject: [PATCH] Detect SMTPs w/ STARTTLS as TLS and dissect client/server hello. Fixes #1630. * FTP needs to get updated as well as it has similiar STARTTLS semantics -> follow-up Signed-off-by: Toni Uhlig --- src/lib/ndpi_main.c | 8 +++-- src/lib/protocols/mail_smtp.c | 49 ++++++++++++++++++++++------- src/lib/protocols/tls.c | 6 ++-- tests/result/ftp-start-tls.pcap.out | 4 +-- tests/result/smtp-starttls.pcap.out | 9 ++++-- 5 files changed, 56 insertions(+), 20 deletions(-) diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index 481d1fbbcde..47385de7044 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -8034,7 +8034,9 @@ u_int8_t ndpi_extra_dissection_possible(struct ndpi_detection_module_struct *ndp switch(proto) { case NDPI_PROTOCOL_TLS: case NDPI_PROTOCOL_DTLS: - if(flow->l4.tcp.tls.certificate_processed) return(0); + if(flow->l4.tcp.tls.certificate_processed || + (flow->l4.tcp.ftp_imap_pop_smtp.auth_tls == 1 && + flow->l4.tcp.ftp_imap_pop_smtp.auth_done == 1)) return(0); if(flow->l4.tcp.tls.num_tls_blocks <= ndpi_str->num_tls_blocks_to_follow) { // printf("*** %u/%u\n", flow->l4.tcp.tls.num_tls_blocks, ndpi_str->num_tls_blocks_to_follow); @@ -8058,8 +8060,8 @@ u_int8_t ndpi_extra_dissection_possible(struct ndpi_detection_module_struct *ndp case NDPI_PROTOCOL_MAIL_IMAP: case NDPI_PROTOCOL_MAIL_SMTP: if(flow->l4.tcp.ftp_imap_pop_smtp.password[0] == '\0' && - flow->l4.tcp.ftp_imap_pop_smtp.auth_tls == 0 && - flow->l4.tcp.ftp_imap_pop_smtp.auth_done == 0) + (flow->l4.tcp.ftp_imap_pop_smtp.auth_tls == 1 || + flow->l4.tcp.ftp_imap_pop_smtp.auth_done == 0)) return(1); break; diff --git a/src/lib/protocols/mail_smtp.c b/src/lib/protocols/mail_smtp.c index ee2e489df57..3d2e8d043b4 100644 --- a/src/lib/protocols/mail_smtp.c +++ b/src/lib/protocols/mail_smtp.c @@ -48,14 +48,15 @@ /* #define SMTP_DEBUG 1 */ +extern int processTLSBlock(struct ndpi_detection_module_struct *ndpi_struct, + struct ndpi_flow_struct *flow); + static void ndpi_int_mail_smtp_add_connection(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) { #ifdef SMTP_DEBUG printf("**** %s()\n", __FUNCTION__); #endif - flow->guessed_protocol_id = NDPI_PROTOCOL_MAIL_SMTP; /* Avoid SMTPS to be used s sub-protocol */ - ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_MAIL_SMTP, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI); } @@ -299,8 +300,8 @@ void ndpi_search_mail_smtp_tcp(struct ndpi_detection_module_struct *ndpi_struct, } } - if(packet->line[a].len >= 8) { - if((packet->line[a].ptr[0] == 'S' || packet->line[a].ptr[0] == 's') + if(packet->line[a].len >= 8) { + if((packet->line[a].ptr[0] == 'S' || packet->line[a].ptr[0] == 's') && (packet->line[a].ptr[1] == 'T' || packet->line[a].ptr[1] == 't') && (packet->line[a].ptr[2] == 'A' || packet->line[a].ptr[2] == 'a') && (packet->line[a].ptr[3] == 'R' || packet->line[a].ptr[3] == 'r') @@ -310,9 +311,9 @@ void ndpi_search_mail_smtp_tcp(struct ndpi_detection_module_struct *ndpi_struct, && (packet->line[a].ptr[7] == 'S' || packet->line[a].ptr[7] == 's')) { flow->l4.tcp.smtp_command_bitmask |= SMTP_BIT_STARTTLS; flow->l4.tcp.ftp_imap_pop_smtp.auth_tls = 1; - flow->l4.tcp.ftp_imap_pop_smtp.auth_done = 1; - } - } + flow->l4.tcp.ftp_imap_pop_smtp.auth_done = 0; + } + } if(packet->line[a].len >= 4) { if((packet->line[a].ptr[0] == 'D' || packet->line[a].ptr[0] == 'd') @@ -381,12 +382,38 @@ void ndpi_search_mail_smtp_tcp(struct ndpi_detection_module_struct *ndpi_struct, /* **************************************** */ int ndpi_extra_search_mail_smtp_tcp(struct ndpi_detection_module_struct *ndpi_struct, - struct ndpi_flow_struct *flow) { - int rc; + struct ndpi_flow_struct *flow) +{ + struct ndpi_packet_struct * const packet = &ndpi_struct->packet; + int rc = 0; + + if (flow->l4.tcp.smtp_command_bitmask & SMTP_BIT_STARTTLS && + packet->payload_packet_len > 5) + { + uint8_t const * const block = &packet->payload[5]; + uint8_t const * const p = &packet->payload[0]; + uint16_t const block_len = packet->payload_packet_len - 5; + uint16_t const l = packet->payload_packet_len; + + packet->payload = block; + packet->payload_packet_len = block_len; + + if (processTLSBlock(ndpi_struct, flow) != 0) { + rc = 1; + } - ndpi_search_mail_smtp_tcp(ndpi_struct, flow); + packet->payload = p; + packet->payload_packet_len = l; - rc = (flow->l4.tcp.ftp_imap_pop_smtp.password[0] == '\0') ? 1 : 0; + /* STARTTLS may be followed by a 220 - Service ready */ + if (rc == 0 && memcmp(packet->payload, "220", 3) != 0) + { + flow->l4.tcp.ftp_imap_pop_smtp.auth_done = 1; + } + } else { + ndpi_search_mail_smtp_tcp(ndpi_struct, flow); + rc = (flow->l4.tcp.ftp_imap_pop_smtp.password[0] == '\0') ? 1 : 0; + } #ifdef SMTP_DEBUG printf("**** %s() [rc: %d]\n", __FUNCTION__, rc); diff --git a/src/lib/protocols/tls.c b/src/lib/protocols/tls.c index 71e7ae50440..69bec44f585 100644 --- a/src/lib/protocols/tls.c +++ b/src/lib/protocols/tls.c @@ -28,6 +28,8 @@ #include "ndpi_encryption.h" extern char *strptime(const char *s, const char *format, struct tm *tm); +extern int processTLSBlock(struct ndpi_detection_module_struct *ndpi_struct, + struct ndpi_flow_struct *flow); extern int processClientServerHello(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow, uint32_t quic_version); extern int http_process_user_agent(struct ndpi_detection_module_struct *ndpi_struct, @@ -839,8 +841,8 @@ int processCertificate(struct ndpi_detection_module_struct *ndpi_struct, /* **************************************** */ -static int processTLSBlock(struct ndpi_detection_module_struct *ndpi_struct, - struct ndpi_flow_struct *flow) { +int processTLSBlock(struct ndpi_detection_module_struct *ndpi_struct, + struct ndpi_flow_struct *flow) { struct ndpi_packet_struct *packet = &ndpi_struct->packet; int ret; diff --git a/tests/result/ftp-start-tls.pcap.out b/tests/result/ftp-start-tls.pcap.out index 1e5ce3bdea9..6112adf25a6 100644 --- a/tests/result/ftp-start-tls.pcap.out +++ b/tests/result/ftp-start-tls.pcap.out @@ -1,6 +1,6 @@ -Guessed flow protos: 0 +Guessed flow protos: 1 -DPI Packets (TCP): 10 (10.00 pkts/flow) +DPI Packets (TCP): 51 (51.00 pkts/flow) Confidence DPI : 1 (flows) FTP_CONTROL 51 7510 1 diff --git a/tests/result/smtp-starttls.pcap.out b/tests/result/smtp-starttls.pcap.out index c031fd8af0d..cfd5ca2d83e 100644 --- a/tests/result/smtp-starttls.pcap.out +++ b/tests/result/smtp-starttls.pcap.out @@ -1,8 +1,13 @@ Guessed flow protos: 0 -DPI Packets (TCP): 9 (9.00 pkts/flow) +DPI Packets (TCP): 11 (11.00 pkts/flow) Confidence DPI : 1 (flows) Google 36 8403 1 - 1 TCP 10.0.0.1:57406 <-> 173.194.68.26:25 [proto: 3.126/SMTP.Google][ClearText][Confidence: DPI][cat: Email/3][17 pkts/2514 bytes <-> 19 pkts/5889 bytes][Goodput ratio: 55/79][0.48 sec][Hostname/SNI: mx.google.com][bytes ratio: -0.402 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 30/24 156/103 42/26][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 148/310 752/1484 168/444][PLAIN TEXT (x.google.com ESMTP s4)][Plen Bins: 23,18,13,9,4,4,4,0,0,4,0,0,0,0,0,0,0,0,0,0,0,4,0,0,0,0,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,9,0,0,0] +JA3 Host Stats: + IP Address # JA3C + 1 10.0.0.1 1 + + + 1 TCP 10.0.0.1:57406 <-> 173.194.68.26:25 [proto: 91.126/TLS.Google][Encrypted][Confidence: DPI][cat: Email/3][17 pkts/2514 bytes <-> 19 pkts/5889 bytes][Goodput ratio: 55/79][0.48 sec][Hostname/SNI: mx.google.com][bytes ratio: -0.402 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 30/24 156/103 42/26][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 148/310 752/1484 168/444][Risk: ** Obsolete TLS (v1.1 or older) **][Risk Score: 100][Risk Info: No client to server traffic / TLSv1][TLSv1][JA3C: fab507fe132c544e8a0eb7c394affeae][Plen Bins: 23,18,13,9,4,4,4,0,0,4,0,0,0,0,0,0,0,0,0,0,0,4,0,0,0,0,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,9,0,0,0]