Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add an heuristic to detect/ignore some anomalous TCP ACK packets #1948

Merged
merged 1 commit into from
Apr 25, 2023
Merged

Add an heuristic to detect/ignore some anomalous TCP ACK packets #1948

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion example/ndpiReader.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ char *_debug_protocols = NULL;
char *_disabled_protocols = NULL;
int aggressiveness[NDPI_MAX_SUPPORTED_PROTOCOLS];
static u_int8_t stats_flag = 0;
ndpi_init_prefs init_prefs = ndpi_no_prefs;
ndpi_init_prefs init_prefs = ndpi_no_prefs | ndpi_enable_tcp_ack_payload_heuristic;
u_int8_t human_readeable_string_len = 5;
u_int8_t max_num_udp_dissected_pkts = 24 /* 8 is enough for most protocols, Signal and SnapchatCall require more */, max_num_tcp_dissected_pkts = 80 /* due to telnet */;
static u_int32_t pcap_analysis_duration = (u_int32_t)-1;
Expand Down
2 changes: 1 addition & 1 deletion fuzz/fuzz_ndpi_reader.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ u_int8_t enable_protocol_guess = 1, enable_payload_analyzer = 0;
u_int8_t enable_flow_stats = 1;
u_int8_t human_readeable_string_len = 5;
u_int8_t max_num_udp_dissected_pkts = 16 /* 8 is enough for most protocols, Signal requires more */, max_num_tcp_dissected_pkts = 80 /* due to telnet */;
ndpi_init_prefs init_prefs = ndpi_track_flow_payload | ndpi_enable_ja3_plus;
ndpi_init_prefs init_prefs = ndpi_track_flow_payload | ndpi_enable_ja3_plus | ndpi_enable_tcp_ack_payload_heuristic;
int enable_malloc_bins = 1;
int malloc_size_stats = 0;
int max_malloc_bins = 0;
Expand Down
13 changes: 13 additions & 0 deletions src/include/ndpi_typedefs.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1273,6 +1273,8 @@ struct ndpi_detection_module_struct {

u_int32_t aggressiveness_ookla;

int tcp_ack_paylod_heuristic;

u_int16_t ndpi_to_user_proto_id[NDPI_MAX_NUM_CUSTOM_PROTOCOLS]; /* custom protocolId mapping */
ndpi_proto_defaults_t proto_defaults[NDPI_MAX_SUPPORTED_PROTOCOLS+NDPI_MAX_NUM_CUSTOM_PROTOCOLS];

Expand Down Expand Up @@ -1691,6 +1693,17 @@ typedef enum {
ndpi_dont_init_risk_ptree = (1 << 14),
ndpi_dont_load_cachefly_list = (1 << 15),
ndpi_track_flow_payload = (1 << 16),
/* In some networks, there are some anomalous TCP flows where
the smallest ACK packets have some kind of zero padding.
It looks like the IP and TCP headers in those frames wrongly consider the
0x00 Ethernet padding bytes as part of the TCP payload.
While this kind of packets is perfectly valid per-se, in some conditions
they might be treated by the TCP reassembler logic as (partial) overlaps,
deceiving the classification engine.
Add an heuristic to detect these packets and to ignore them, allowing
correct detection/classification.
See #1946 for other details */
ndpi_enable_tcp_ack_payload_heuristic = (1 << 17),
} ndpi_prefs;

typedef struct {
Expand Down
22 changes: 21 additions & 1 deletion src/lib/ndpi_main.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2951,6 +2951,9 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs

ndpi_str->aggressiveness_ookla = NDPI_AGGRESSIVENESS_OOKLA_TLS;

if(prefs & ndpi_enable_tcp_ack_payload_heuristic)
ndpi_str->tcp_ack_paylod_heuristic = 1;

for(i = 0; i < NUM_CUSTOM_CATEGORIES; i++)
ndpi_snprintf(ndpi_str->custom_category_labels[i], CUSTOM_CATEGORY_LABEL_LEN, "User custom category %u",
(unsigned int) (i + 1));
Expand Down Expand Up @@ -5492,6 +5495,20 @@ static u_int8_t ndpi_is_multi_or_broadcast(struct ndpi_packet_struct *packet) {

/* ************************************************ */

static int tcp_ack_padding(struct ndpi_packet_struct *packet) {
const struct ndpi_tcphdr *tcph = packet->tcp;
if(tcph && tcph->ack && !tcph->psh &&
packet->payload_packet_len < 8 &&
packet->payload_packet_len > 1 /* To avoid TCP keep-alives */) {
int i;
for(i = 0; i < packet->payload_packet_len; i++)
if(packet->payload[i] != 0)
return 0;
return 1;
}
return 0;
}

void ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str,
struct ndpi_flow_struct *flow) {
if(!flow) {
Expand Down Expand Up @@ -5580,7 +5597,10 @@ void ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str,
}
}

if(flow->next_tcp_seq_nr[0] == 0 || flow->next_tcp_seq_nr[1] == 0 ||
if(ndpi_str->tcp_ack_paylod_heuristic && tcp_ack_padding(packet)) {
NDPI_LOG_DBG2(ndpi_str, "TCP ACK with zero padding. Ignoring\n");
packet->tcp_retransmission = 1;
} else if(flow->next_tcp_seq_nr[0] == 0 || flow->next_tcp_seq_nr[1] == 0 ||
(tcph->syn && flow->packet_counter == 0)) {
/* initialize tcp sequence counters */
/* the ack flag needs to be set to get valid sequence numbers from the other
Expand Down
Binary file added tests/cfgs/default/pcap/heuristic_tcp_ack_payload.pcap
View file
Open in desktop
Binary file not shown.
40 changes: 40 additions & 0 deletions tests/cfgs/default/result/heuristic_tcp_ack_payload.pcap.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
Guessed flow protos: 0

DPI Packets (TCP): 67 (11.17 pkts/flow)
Confidence DPI : 6 (flows)
Num dissector calls: 17 (2.83 diss/flow)
LRU cache ookla: 0/0/0 (insert/search/found)
LRU cache bittorrent: 0/0/0 (insert/search/found)
LRU cache zoom: 0/0/0 (insert/search/found)
LRU cache stun: 0/0/0 (insert/search/found)
LRU cache tls_cert: 0/8/0 (insert/search/found)
LRU cache mining: 0/0/0 (insert/search/found)
LRU cache msteams: 0/0/0 (insert/search/found)
LRU cache stun_zoom: 0/0/0 (insert/search/found)
Automa host: 12/2 (search/found)
Automa domain: 12/0 (search/found)
Automa tls cert: 3/0 (search/found)
Automa risk mask: 0/0 (search/found)
Automa common alpns: 9/9 (search/found)
Patricia risk mask: 18/0 (search/found)
Patricia risk: 18/0 (search/found)
Patricia protocols: 16/2 (search/found)

TLS 196 76217 4
WindowsUpdate 19 2638 1
Pinterest 88 34448 1

JA3 Host Stats:
IP Address # JA3C
1 194.226.199.103 1
2 194.226.199.61 2
3 194.226.199.9 1
4 194.226.199.21 1


1 TCP 194.226.199.61:27453 <-> 35.241.9.150:443 [proto: 91/TLS][IP: 284/GoogleCloud][Encrypted][Confidence: DPI][DPI packets: 8][cat: Web/5][36 pkts/3477 bytes <-> 42 pkts/37330 bytes][Goodput ratio: 44/94][171.42 sec][Hostname/SNI: firefox.settings.services.mozilla.com][(Advertised) ALPNs: h2;http/1.1][(Negotiated) ALPN: h2][bytes ratio: -0.830 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 4196/3653 58250/58245 14929/14067][Pkt Len c2s/s2c min/avg/max/stddev: 56/60 97/889 375/2878 73/1070][TLSv1.2][JA3C: fb0aa01abe9d8e4037eb3473ca6e2dca][ServerNames: firefox.settings.services.mozilla.com,main-2-cdn.prod.kinto.prod.cloudops.mozgcp.net][JA3S: 9d9ce860f1b1cbef07b019450cb368d8][Issuer: C=US, O=Let's Encrypt, CN=R3][Subject: CN=main-2-cdn.prod.kinto.prod.cloudops.mozgcp.net][Certificate SHA-1: 30:0D:22:77:6E:DA:4E:99:3E:AF:8A:D0:5C:7D:97:51:8B:E6:22:11][Firefox][Validity: 2023-04-04 08:33:24 - 2023-07-03 08:33:23][Cipher: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256][Plen Bins: 49,16,2,0,3,3,1,0,1,0,2,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,7,0,0,11]
2 TCP 194.226.199.61:6946 <-> 2.22.40.186:443 [proto: 91.183/TLS.Pinterest][IP: 0/Unknown][Encrypted][Confidence: DPI][DPI packets: 10][cat: SocialNetwork/6][41 pkts/7780 bytes <-> 47 pkts/26668 bytes][Goodput ratio: 70/90][18.13 sec][Hostname/SNI: ru.pinterest.com][(Advertised) ALPNs: h2;http/1.1][TLS Supported Versions: GREASE;TLSv1.3;TLSv1.2][bytes ratio: -0.548 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 404/181 3512/2129 852/406][Pkt Len c2s/s2c min/avg/max/stddev: 56/60 190/567 1514/2974 287/678][TLSv1.3][JA3C: 598872011444709307b861ae817a4b60][JA3S: 15af977ce25de452b96affa2addb1036][Chrome][Cipher: TLS_AES_256_GCM_SHA384][Plen Bins: 50,9,1,5,0,0,0,0,2,0,0,0,0,1,0,0,1,1,0,4,0,0,0,0,1,1,2,0,0,0,0,4,1,0,0,0,1,0,0,0,0,0,0,0,0,12,0,1]
3 TCP 194.226.199.21:58155 <-> 52.18.127.189:443 [proto: 91/TLS][IP: 265/AmazonAWS][Encrypted][Confidence: DPI][DPI packets: 13][cat: Web/5][28 pkts/6789 bytes <-> 35 pkts/8995 bytes][Goodput ratio: 78/79][130.64 sec][Hostname/SNI: bitrix.info][(Advertised) ALPNs: h2;http/1.1][(Negotiated) ALPN: h2][TLS Supported Versions: GREASE;TLSv1.3;TLSv1.2][bytes ratio: -0.140 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/0 5498/4834 45102/45058 12717/11564][Pkt Len c2s/s2c min/avg/max/stddev: 56/60 242/257 1547/2974 352/535][TLSv1.2][JA3C: ab205d804ecf934209c2a1bb94f817e0][JA3S: bfc90d56141386ee83b56cda231cccfc][Chrome][Cipher: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256][Plen Bins: 46,22,6,6,0,0,0,0,0,0,1,0,6,0,1,1,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,1,1]
4 TCP 194.226.199.103:62580 <-> 217.69.139.59:443 [proto: 91/TLS][IP: 0/Unknown][Encrypted][Confidence: DPI][DPI packets: 15][cat: Web/5][22 pkts/2692 bytes <-> 16 pkts/10450 bytes][Goodput ratio: 55/92][7.28 sec][Hostname/SNI: portal.mail.ru][(Advertised) ALPNs: h2;http/1.1][(Negotiated) ALPN: http/1.1][TLS Supported Versions: TLSv1.3;TLSv1.2][bytes ratio: -0.590 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 100/27 1559/213 357/70][Pkt Len c2s/s2c min/avg/max/stddev: 56/60 122/653 623/2897 162/957][TLSv1.2][JA3C: e669667efb41c36f714c309243f41ca7][ServerNames: *.mail.ru,mail.ru][JA3S: 2b33c1374db4ddf06942f92373c0b54b][Issuer: C=BE, O=GlobalSign nv-sa, CN=GlobalSign RSA OV SSL CA 2018][Subject: C=RU, ST=Moscow, L=Moscow, O=VK LLC, CN=*.mail.ru][Certificate SHA-1: 9F:A2:43:EA:AA:62:15:13:44:0D:15:75:17:47:4C:6B:E5:8E:10:1E][Firefox][Validity: 2022-10-20 09:52:31 - 2023-11-21 09:52:30][Cipher: TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384][Plen Bins: 68,0,0,5,0,0,0,0,11,0,0,0,0,0,0,0,0,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,0,0,5]
5 TCP 194.226.199.9:49756 <-> 92.223.106.21:443 [proto: 91/TLS][IP: 0/Unknown][Encrypted][Confidence: DPI][DPI packets: 9][cat: Web/5][9 pkts/1140 bytes <-> 8 pkts/5344 bytes][Goodput ratio: 54/91][0.28 sec][Hostname/SNI: moevideo.biz][(Advertised) ALPNs: http/1.1][(Negotiated) ALPN: http/1.1][TLS Supported Versions: TLSv1.3;TLSv1.2;TLSv1.1;TLSv1;SSLv3][bytes ratio: -0.648 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 20/6 44/20 18/8][Pkt Len c2s/s2c min/avg/max/stddev: 56/60 127/668 571/2690 159/894][TLSv1.2][JA3C: 7c822e5e821268e8bd01b70e9cad0b85][ServerNames: *.moevideo.biz,moevideo.biz][JA3S: d154fcfa5bb4f0748e1dd1992c681104][Issuer: C=BE, O=GlobalSign nv-sa, CN=AlphaSSL CA - SHA256 - G4][Subject: CN=*.moevideo.biz][Certificate SHA-1: FF:0C:ED:41:2C:7C:DA:BA:89:FE:7E:09:4A:2B:62:26:A0:20:AC:53][Safari][Validity: 2023-04-04 15:59:15 - 2024-05-05 15:59:14][Cipher: TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384][Plen Bins: 51,0,8,0,0,0,0,0,8,0,0,0,0,0,0,8,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,8,0,8]
6 TCP 194.226.199.226:34101 <-> 8.247.226.126:80 [proto: 7.147/HTTP.WindowsUpdate][IP: 0/Unknown][ClearText][Confidence: DPI][DPI packets: 12][cat: SoftwareUpdate/19][7 pkts/896 bytes <-> 12 pkts/1742 bytes][Goodput ratio: 56/62][0.04 sec][Hostname/SNI: 3.tlu.dl.delivery.mp.microsoft.com][bytes ratio: -0.321 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 7/1 12/11 6/3][Pkt Len c2s/s2c min/avg/max/stddev: 56/60 128/145 550/1076 172/281][URL: 3.tlu.dl.delivery.mp.microsoft.com/filestreamingservice/files/b4f27514-1618-47a0-bcd4-5fcb469edb63?P1=1681888058&P2=404&P3=2&P4=VJ2Qv%2bUXzBGOULZmyshxlc8XXx4pLl7hoFcLgf1iS33rDGfm0tCVrTPvZN8tn8yWBSrA0idwdtOBFLQMjZCUkw%3d%3d][StatusCode: 0][User-Agent: Microsoft-Delivery-Optimization/10.0][PLAIN TEXT (GET /filestreamingservice/files)][Plen Bins: 89,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]