Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add extra entropy checks and more precise(?) analysis. #2383

Merged
merged 1 commit into from
May 9, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions src/include/ndpi_main.h
Original file line number Diff line number Diff line change
Expand Up @@ -113,8 +113,15 @@ extern "C" {
int ndpi_is_printable_buffer(u_int8_t const * const buf, size_t len);
int ndpi_normalize_printable_string(char * const str, size_t len);
int ndpi_is_valid_hostname(char * const str, size_t len);
#define NDPI_ENTROPY_ENCRYPTED_OR_RANDOM(entropy) (entropy > 7.0f)
float ndpi_entropy(u_int8_t const * const buf, size_t len);

utoni marked this conversation as resolved.
Show resolved Hide resolved
#define NDPI_ENTROPY_PLAINTEXT(entropy) (entropy < 4.941f)
#define NDPI_ENTROPY_EXECUTABLE(entropy) (entropy >= 4.941f)
#define NDPI_ENTROPY_EXECUTABLE_PACKED(entropy) (entropy >= 6.677f)
#define NDPI_ENTROPY_EXECUTABLE_ENCRYPTED(entropy) (entropy >= 7.174f)
#define NDPI_ENTROPY_ENCRYPTED_OR_RANDOM(entropy) (entropy >= 7.312f)
float ndpi_entropy(u_int8_t const * const buf, size_t len);
char *ndpi_entropy2str(float entropy, char *buf, size_t len);
void ndpi_entropy2risk(struct ndpi_flow_struct *flow);

#ifdef __cplusplus
}
Expand Down
2 changes: 1 addition & 1 deletion src/include/ndpi_typedefs.h
Original file line number Diff line number Diff line change
Expand Up @@ -1202,7 +1202,7 @@ struct ndpi_flow_struct {
/* init parameter, internal used to set up timestamp,... */
u_int16_t guessed_protocol_id, guessed_protocol_id_by_ip, guessed_category, guessed_header_category;
u_int8_t l4_proto, protocol_id_already_guessed:1, fail_with_unknown:1,
init_finished:1, client_packet_direction:1, packet_direction:1, is_ipv6:1, first_pkt_fully_encrypted:1, _pad1: 1;
init_finished:1, client_packet_direction:1, packet_direction:1, is_ipv6:1, first_pkt_fully_encrypted:1, skip_entropy_check: 1;

u_int16_t num_dissector_calls;
ndpi_confidence_t confidence; /* ndpi_confidence_t */
Expand Down
39 changes: 13 additions & 26 deletions src/lib/ndpi_main.c
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ static ndpi_risk_info ndpi_known_risks[] = {
{ NDPI_TLS_CERT_VALIDITY_TOO_LONG, NDPI_RISK_MEDIUM, CLIENT_FAIR_RISK_PERCENTAGE, NDPI_SERVER_ACCOUNTABLE },
{ NDPI_TLS_SUSPICIOUS_EXTENSION, NDPI_RISK_HIGH, CLIENT_HIGH_RISK_PERCENTAGE, NDPI_BOTH_ACCOUNTABLE },
{ NDPI_TLS_FATAL_ALERT, NDPI_RISK_LOW, CLIENT_FAIR_RISK_PERCENTAGE, NDPI_BOTH_ACCOUNTABLE },
{ NDPI_SUSPICIOUS_ENTROPY, NDPI_RISK_MEDIUM, CLIENT_FAIR_RISK_PERCENTAGE, NDPI_BOTH_ACCOUNTABLE },
{ NDPI_SUSPICIOUS_ENTROPY, NDPI_RISK_LOW, CLIENT_FAIR_RISK_PERCENTAGE, NDPI_BOTH_ACCOUNTABLE },
{ NDPI_CLEAR_TEXT_CREDENTIALS, NDPI_RISK_HIGH, CLIENT_HIGH_RISK_PERCENTAGE, NDPI_CLIENT_ACCOUNTABLE },
{ NDPI_DNS_LARGE_PACKET, NDPI_RISK_MEDIUM, CLIENT_FAIR_RISK_PERCENTAGE, NDPI_CLIENT_ACCOUNTABLE },
{ NDPI_DNS_FRAGMENTED, NDPI_RISK_MEDIUM, CLIENT_FAIR_RISK_PERCENTAGE, NDPI_CLIENT_ACCOUNTABLE },
Expand Down Expand Up @@ -4396,14 +4396,10 @@ static u_int16_t guess_protocol_id(struct ndpi_detection_module_struct *ndpi_str
ndpi_set_risk(flow, NDPI_MALFORMED_PACKET, NULL);

if(packet->payload_packet_len > sizeof(struct ndpi_icmphdr)) {
flow->entropy = ndpi_entropy(packet->payload + sizeof(struct ndpi_icmphdr),
packet->payload_packet_len - sizeof(struct ndpi_icmphdr));

if(NDPI_ENTROPY_ENCRYPTED_OR_RANDOM(flow->entropy) != 0) {
char str[32];

snprintf(str, sizeof(str), "Entropy %.2f", flow->entropy);
ndpi_set_risk(flow, NDPI_SUSPICIOUS_ENTROPY, str);
if (flow->skip_entropy_check == 0) {
flow->entropy = ndpi_entropy(packet->payload + sizeof(struct ndpi_icmphdr),
packet->payload_packet_len - sizeof(struct ndpi_icmphdr));
ndpi_entropy2risk(flow);
}

u_int16_t chksm = icmp4_checksum(packet->payload, packet->payload_packet_len);
Expand Down Expand Up @@ -8583,25 +8579,16 @@ static ndpi_protocol ndpi_internal_detection_process_packet(struct ndpi_detectio
ndpi_search_shellscript(ndpi_str, flow);
}

if(flow->first_pkt_fully_encrypted == 0 &&
ret.app_protocol == NDPI_PROTOCOL_UNKNOWN &&
NDPI_ENTROPY_ENCRYPTED_OR_RANDOM(flow->entropy) == 0 &&
flow->packet_counter < 3)
if(flow->skip_entropy_check == 0 &&
flow->first_pkt_fully_encrypted == 0 &&
flow->packet_counter < 5 &&
/* The following protocols do their own entropy calculation/classification. */
ret.app_protocol != NDPI_PROTOCOL_IP_ICMP)
{
flow->entropy = ndpi_entropy(packet->payload, packet->payload_packet_len);
if(NDPI_ENTROPY_ENCRYPTED_OR_RANDOM(flow->entropy) != 0) {
char str[32];

snprintf(str, sizeof(str), "Entropy %.2f", flow->entropy);
ndpi_set_risk(flow, NDPI_SUSPICIOUS_ENTROPY, str);
if (ret.app_protocol != NDPI_PROTOCOL_HTTP) {
flow->entropy = ndpi_entropy(packet->payload, packet->payload_packet_len);
}
}
if(ret.app_protocol != NDPI_PROTOCOL_UNKNOWN &&
ret.app_protocol != NDPI_PROTOCOL_IP_ICMP &&
flow->entropy > 0.0f)
{
flow->entropy = 0.0f;
ndpi_unset_risk(flow, NDPI_SUSPICIOUS_ENTROPY);
ndpi_entropy2risk(flow);
}

return(ret);
Expand Down
73 changes: 73 additions & 0 deletions src/lib/ndpi_utils.c
Original file line number Diff line number Diff line change
Expand Up @@ -2704,6 +2704,79 @@ float ndpi_entropy(u_int8_t const * const buf, size_t len) {
return entropy;
}

/* ******************************************************************** */

/* Losely implemented by: https://redirect.cs.umbc.edu/courses/graduate/CMSC691am/student%20talks/CMSC%20691%20Malware%20-%20Entropy%20Analysis%20Presentation.pdf */
char *ndpi_entropy2str(float entropy, char *buf, size_t len) {
if (buf == NULL) {
return NULL;
}

static const char entropy_fmtstr[] = "Entropy: %.3f (%s?)";
if (NDPI_ENTROPY_ENCRYPTED_OR_RANDOM(entropy)) {
snprintf(buf, len, entropy_fmtstr, entropy, "Encrypted or Random");
} else if (NDPI_ENTROPY_EXECUTABLE_ENCRYPTED(entropy)) {
snprintf(buf, len, entropy_fmtstr, entropy, "Encrypted Executable");
} else if (NDPI_ENTROPY_EXECUTABLE_PACKED(entropy)) {
snprintf(buf, len, entropy_fmtstr, entropy, "Compressed Executable");
} else if (NDPI_ENTROPY_EXECUTABLE(entropy)) {
snprintf(buf, len, entropy_fmtstr, entropy, "Executable");
} else {
snprintf(buf, len, entropy_fmtstr, entropy, "Unknown");
}

return buf;
}

/* ******************************************************************** */

void ndpi_entropy2risk(struct ndpi_flow_struct *flow) {
char str[64];

if (NDPI_ENTROPY_PLAINTEXT(flow->entropy))
goto reset_risk;

if (flow->detected_protocol_stack[0] == NDPI_PROTOCOL_TLS ||
flow->detected_protocol_stack[1] == NDPI_PROTOCOL_TLS ||
flow->detected_protocol_stack[0] == NDPI_PROTOCOL_QUIC ||
flow->detected_protocol_stack[1] == NDPI_PROTOCOL_QUIC ||
flow->detected_protocol_stack[0] == NDPI_PROTOCOL_DTLS ||
flow->detected_protocol_stack[1] == NDPI_PROTOCOL_DTLS)
{
flow->skip_entropy_check = 1;
goto reset_risk;
}

if (flow->confidence != NDPI_CONFIDENCE_DPI &&
flow->confidence != NDPI_CONFIDENCE_DPI_CACHE) {
ndpi_set_risk(flow, NDPI_SUSPICIOUS_ENTROPY,
ndpi_entropy2str(flow->entropy, str, sizeof(str)));
return;
}

if (ndpi_isset_risk(flow, NDPI_MALWARE_HOST_CONTACTED) ||
ndpi_isset_risk(flow, NDPI_BINARY_DATA_TRANSFER) ||
ndpi_isset_risk(flow, NDPI_BINARY_APPLICATION_TRANSFER) ||
ndpi_isset_risk(flow, NDPI_POSSIBLE_EXPLOIT) ||
ndpi_isset_risk(flow, NDPI_HTTP_SUSPICIOUS_CONTENT) ||
ndpi_isset_risk(flow, NDPI_DNS_SUSPICIOUS_TRAFFIC) ||
ndpi_isset_risk(flow, NDPI_MALFORMED_PACKET) ||
(flow->category == NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT &&
(flow->detected_protocol_stack[0] == NDPI_PROTOCOL_HTTP ||
flow->detected_protocol_stack[1] == NDPI_PROTOCOL_HTTP)) ||
flow->category == NDPI_PROTOCOL_CATEGORY_DATA_TRANSFER ||
flow->category == NDPI_PROTOCOL_CATEGORY_UNSPECIFIED ||
flow->category == NDPI_PROTOCOL_CATEGORY_WEB)
{
ndpi_set_risk(flow, NDPI_SUSPICIOUS_ENTROPY,
ndpi_entropy2str(flow->entropy, str, sizeof(str)));
return;
}

reset_risk:
ndpi_unset_risk(flow, NDPI_SUSPICIOUS_ENTROPY);
}

/* ******************************************************************** */
static inline uint16_t get_n16bit(uint8_t const * cbuf) {
uint16_t r = ((uint16_t)cbuf[0]) | (((uint16_t)cbuf[1]) << 8);
Expand Down
4 changes: 4 additions & 0 deletions src/lib/protocols/http.c
Original file line number Diff line number Diff line change
Expand Up @@ -222,8 +222,12 @@ static void ndpi_validate_http_content(struct ndpi_detection_module_struct *ndpi

if(len >= 8 /* 4 chars for \r\n\r\n and at least 4 charts for content guess */) {
double_ret += 4;
len -= 4;
utoni marked this conversation as resolved.
Show resolved Hide resolved

ndpi_http_check_human_redeable_content(ndpi_struct, flow, double_ret, len);
if (flow->skip_entropy_check == 0) {
flow->entropy = ndpi_entropy(double_ret, len);
}
}
}

Expand Down
2 changes: 1 addition & 1 deletion tests/cfgs/caches_cfg/result/teams.pcap.out
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ JA3 Host Stats:
11 TCP 192.168.1.6:60540 <-> 52.114.75.70:443 [proto: 91.250/TLS.Teams][IP: 276/Azure][Encrypted][Confidence: DPI][DPI packets: 5][cat: Collaborative/15][14 pkts/5711 bytes <-> 10 pkts/8093 bytes][Goodput ratio: 83/92][0.13 sec][Hostname/SNI: eu-prod.asyncgw.teams.microsoft.com][(Advertised) ALPNs: h2;http/1.1][bytes ratio: -0.173 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/0 8/9 32/32 13/14][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 408/809 1494/1506 517/688][TLSv1.2][JA3C: 74d5fa154a7fc0a7c655d8eaa34b89bf][JA4: t12d1312h2_8b80da21ef18_b00751acaffa][Plen Bins: 0,7,0,7,0,0,0,15,0,0,0,0,0,0,0,0,0,0,0,0,0,7,0,0,0,0,0,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,7,15,31,0,0]
12 TCP 192.168.1.6:60537 <-> 52.114.77.33:443 [proto: 91.212/TLS.Microsoft][IP: 276/Azure][Encrypted][Confidence: DPI][DPI packets: 8][cat: Cloud/13][16 pkts/8418 bytes <-> 10 pkts/5367 bytes][Goodput ratio: 87/88][0.27 sec][Hostname/SNI: mobile.pipe.aria.microsoft.com][bytes ratio: 0.221 (Upload)][IAT c2s/s2c min/avg/max/stddev: 0/0 14/27 46/46 20/20][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 526/537 1494/1506 639/623][Risk: ** TLS (probably) Not Carrying HTTPS **][Risk Score: 10][Risk Info: No ALPN][TLSv1.2][JA3C: a1674500365bdd882188db63730e69a2][JA4: t12d150700_0707305c9f76_0f3b2bcde21d][ServerNames: *.events.data.microsoft.com,events.data.microsoft.com,*.pipe.aria.microsoft.com,pipe.skype.com,*.pipe.skype.com,*.mobile.events.data.microsoft.com,mobile.events.data.microsoft.com,*.events.data.msn.com,events.data.msn.com][JA3S: ae4edc6faf64d08308082ad26be60767][Issuer: C=US, ST=Washington, L=Redmond, O=Microsoft Corporation, OU=Microsoft IT, CN=Microsoft IT TLS CA 4][Subject: CN=*.events.data.microsoft.com][Certificate SHA-1: 33:B3:B7:E9:DA:25:F5:A0:04:E9:63:87:B6:FB:54:77:DB:ED:27:EB][Safari][Validity: 2019-10-10 21:55:38 - 2021-10-10 21:55:38][Cipher: TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384][Plen Bins: 7,7,7,0,0,0,7,0,0,0,0,0,0,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,7,7,0,0,31,15,0,0]
13 TCP 192.168.1.6:60555 <-> 52.114.77.33:443 [proto: 91.212/TLS.Microsoft][IP: 276/Azure][Encrypted][Confidence: DPI][DPI packets: 11][cat: Cloud/13][18 pkts/5861 bytes <-> 13 pkts/7901 bytes][Goodput ratio: 80/89][2.79 sec][Hostname/SNI: mobile.pipe.aria.microsoft.com][bytes ratio: -0.148 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/0 192/269 2443/2490 625/741][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 326/608 1494/1506 448/617][Risk: ** TLS (probably) Not Carrying HTTPS **][Risk Score: 10][Risk Info: No ALPN][TLSv1.2][JA3C: e4d448cdfe06dc1243c1eb026c74ac9a][JA4: t12d220700_0d4ca5d4ec72_3304d8368043][ServerNames: *.events.data.microsoft.com,events.data.microsoft.com,*.pipe.aria.microsoft.com,pipe.skype.com,*.pipe.skype.com,*.mobile.events.data.microsoft.com,mobile.events.data.microsoft.com,*.events.data.msn.com,events.data.msn.com][JA3S: 986571066668055ae9481cb84fda634a][Issuer: C=US, ST=Washington, L=Redmond, O=Microsoft Corporation, OU=Microsoft IT, CN=Microsoft IT TLS CA 4][Subject: CN=*.events.data.microsoft.com][Certificate SHA-1: 33:B3:B7:E9:DA:25:F5:A0:04:E9:63:87:B6:FB:54:77:DB:ED:27:EB][Firefox][Validity: 2019-10-10 21:55:38 - 2021-10-10 21:55:38][Cipher: TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384][Plen Bins: 0,16,11,0,0,5,0,0,0,5,5,0,0,11,0,5,0,0,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,11,22,0,0]
14 UDP 192.168.1.6:51681 <-> 52.114.77.136:3478 [proto: 38/Skype_TeamsCall][IP: 276/Azure][Encrypted][Confidence: Match by port][DPI packets: 12][cat: VoIP/10][14 pkts/5838 bytes <-> 17 pkts/7907 bytes][Goodput ratio: 90/91][4.57 sec][bytes ratio: -0.151 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/0 347/256 2336/2336 693/595][Pkt Len c2s/s2c min/avg/max/stddev: 79/79 417/465 1243/1227 434/401][Risk: ** Susp Entropy **][Risk Score: 50][Risk Info: Entropy 7.83][PLAIN TEXT (TBHSWF)][Plen Bins: 0,36,0,0,0,12,6,0,3,6,0,0,0,3,0,0,0,0,0,0,0,0,0,6,6,0,0,0,0,0,3,0,3,3,0,0,0,9,0,0,0,0,0,0,0,0,0,0]
14 UDP 192.168.1.6:51681 <-> 52.114.77.136:3478 [proto: 38/Skype_TeamsCall][IP: 276/Azure][Encrypted][Confidence: Match by port][DPI packets: 12][cat: VoIP/10][14 pkts/5838 bytes <-> 17 pkts/7907 bytes][Goodput ratio: 90/91][4.57 sec][bytes ratio: -0.151 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/0 347/256 2336/2336 693/595][Pkt Len c2s/s2c min/avg/max/stddev: 79/79 417/465 1243/1227 434/401][Risk: ** Susp Entropy **][Risk Score: 10][Risk Info: Entropy: 6.927 (Compressed Executable?)][PLAIN TEXT (TBHSWF)][Plen Bins: 0,36,0,0,0,12,6,0,3,6,0,0,0,3,0,0,0,0,0,0,0,0,0,6,6,0,0,0,0,0,3,0,3,3,0,0,0,9,0,0,0,0,0,0,0,0,0,0]
15 TCP 192.168.1.6:60547 <-> 52.114.88.59:443 [proto: 91.250/TLS.Teams][IP: 276/Azure][Encrypted][Confidence: DPI][DPI packets: 14][cat: Collaborative/15][20 pkts/3926 bytes <-> 15 pkts/8828 bytes][Goodput ratio: 66/89][0.32 sec][Hostname/SNI: chatsvcagg.teams.microsoft.com][(Advertised) ALPNs: h2;http/1.1][bytes ratio: -0.384 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 13/25 91/80 23/31][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 196/589 1494/1506 320/612][TLSv1.2][JA3C: ebf5e0e525258d7a8dcb54aa1564ecbd][JA4: t12d1311h2_8b80da21ef18_77989cba1f4a][Plen Bins: 0,21,10,5,0,5,10,5,0,0,0,5,0,0,0,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,0,0,0,0,0,0,0,0,0,0,5,21,0,0]
16 TCP 192.168.1.6:60565 <-> 52.114.108.8:443 [proto: 91.250/TLS.Teams][IP: 276/Azure][Encrypted][Confidence: DPI][DPI packets: 5][cat: Collaborative/15][19 pkts/3306 bytes <-> 14 pkts/9053 bytes][Goodput ratio: 61/90][0.43 sec][Hostname/SNI: emea.ng.msg.teams.microsoft.com][(Advertised) ALPNs: h2;http/1.1][bytes ratio: -0.465 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 27/12 276/54 68/17][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 174/647 1060/1506 238/633][TLSv1.2][JA3C: ebf5e0e525258d7a8dcb54aa1564ecbd][JA4: t12d1311h2_8b80da21ef18_77989cba1f4a][Plen Bins: 0,22,16,5,0,0,5,0,0,0,0,0,0,5,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,0,0,0,0,0,0,0,0,5,0,0,0,0,22,0,0]
17 TCP 192.168.1.6:60541 <-> 52.114.75.69:443 [proto: 91.125/TLS.Skype_Teams][IP: 276/Azure][Encrypted][Confidence: DPI][DPI packets: 10][cat: VoIP/10][13 pkts/4051 bytes <-> 9 pkts/7973 bytes][Goodput ratio: 79/92][0.14 sec][Hostname/SNI: eu-api.asm.skype.com][(Advertised) ALPNs: h2;http/1.1][bytes ratio: -0.326 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 10/11 31/36 14/16][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 312/886 1494/1506 422/676][TLSv1.2][JA3C: 74d5fa154a7fc0a7c655d8eaa34b89bf][JA4: t12d1312h2_8b80da21ef18_b00751acaffa][ServerNames: *.asm.skype.com][JA3S: 986571066668055ae9481cb84fda634a][Issuer: C=US, ST=Washington, L=Redmond, O=Microsoft Corporation, OU=Microsoft IT, CN=Microsoft IT TLS CA 1][Subject: CN=*.asm.skype.com][Certificate SHA-1: B9:41:1D:AE:56:09:68:D2:07:D0:69:E1:68:00:08:2B:EF:63:1E:48][Validity: 2019-05-07 12:50:03 - 2021-05-07 12:50:03][Cipher: TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384][Plen Bins: 0,8,0,8,0,0,16,0,0,0,0,0,0,0,0,0,0,0,0,0,0,8,0,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,8,0,8,34,0,0]
Expand Down
2 changes: 1 addition & 1 deletion tests/cfgs/caches_global/result/lru_ipv6_caches.pcapng.out
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ Automa tls cert: 1/1 (search/found)
Automa risk mask: 0/0 (search/found)
Automa common alpns: 3/3 (search/found)
Patricia risk mask: 0/0 (search/found)
Patricia risk mask IPv6: 22/0 (search/found)
Patricia risk mask IPv6: 24/0 (search/found)
Patricia risk: 0/0 (search/found)
Patricia risk IPv6: 12/0 (search/found)
Patricia protocols: 0/0 (search/found)
Expand Down
Loading
Loading