Added entropy calculation to check for suspicious (encrypted) payload. (#1270)

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

Co-authored-by: Luca Deri <lucaderi@users.noreply.github.com>

5 files changed, 30 insertions, 10 deletions

diff --git a/src/include/ndpi_main.h b/src/include/ndpi_main.h
index 8c6eac83f..0a513c353 100644
--- a/src/include/ndpi_main.h
+++ b/src/include/ndpi_main.h
@@ -155,7 +155,8 @@ extern "C" {
   void ndpi_set_risk(struct ndpi_detection_module_struct *ndpi_str,
 		     struct ndpi_flow_struct *flow, ndpi_risk_enum r);
   int ndpi_is_printable_string(char const * const str, size_t len);
-  float ndpi_calculate_entropy(u_int8_t const * const buf, size_t len);
+#define NDPI_ENTROPY_ENCRYPTED_OR_RANDOM(entropy) (entropy > 7.0f)
+  float ndpi_entropy(u_int8_t const * const buf, size_t len);
   void load_common_alpns(struct ndpi_detection_module_struct *ndpi_str);
   u_int8_t is_a_common_alpn(struct ndpi_detection_module_struct *ndpi_str,
 			    const char *alpn_to_check, u_int alpn_to_check_len);    
diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h
index 12dbc4dcf..2260e95e8 100644
--- a/src/include/ndpi_typedefs.h
+++ b/src/include/ndpi_typedefs.h
@@ -105,7 +105,8 @@ typedef enum {
   NDPI_TLS_CERT_VALIDITY_TOO_LONG,
   NDPI_TLS_SUSPICIOUS_EXTENSION,
   NDPI_TLS_FATAL_ALERT,
-
+  NDPI_ENTROPY_SUSPICIOUS,
+  
   /* Leave this as last member */
   NDPI_MAX_RISK /* must be <= 63 due to (**) */
 } ndpi_risk_enum;
@@ -737,6 +738,8 @@ struct ndpi_flow_tcp_struct {
 
   /* NDPI_PROTOCOL_DIRECT_DOWNLOAD_LINK */
   u_int32_t ddlink_server_direction:1;
+
+  /* Part of the TCP header. */
   u_int32_t seen_syn:1;
   u_int32_t seen_syn_ack:1;
   u_int32_t seen_ack:1;
@@ -1293,6 +1296,9 @@ struct ndpi_flow_struct {
     struct ndpi_flow_udp_struct udp;
   } l4;
 
+  /* Some protocols calculate the entropy. */
+  float entropy;
+
   /* Place textual flow info here */
   char flow_extra_info[16];
 
diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index b3758a1aa..41caa8c40 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -105,6 +105,7 @@ static ndpi_risk_info ndpi_known_risks[] = {
   { NDPI_TLS_CERT_VALIDITY_TOO_LONG,            NDPI_RISK_MEDIUM, CLIENT_FAIR_RISK_PERCENTAGE },
   { NDPI_TLS_SUSPICIOUS_EXTENSION,              NDPI_RISK_HIGH,   CLIENT_HIGH_RISK_PERCENTAGE },
   { NDPI_TLS_FATAL_ALERT,                       NDPI_RISK_LOW,    CLIENT_FAIR_RISK_PERCENTAGE },
+  { NDPI_ENTROPY_SUSPICIOUS,                    NDPI_RISK_MEDIUM, CLIENT_FAIR_RISK_PERCENTAGE },
 
   /* Leave this as last member */
   { NDPI_MAX_RISK,                              NDPI_RISK_LOW,    CLIENT_FAIR_RISK_PERCENTAGE }
@@ -2841,6 +2842,7 @@ u_int16_t ndpi_guess_protocol_id(struct ndpi_detection_module_struct *ndpi_str,
       break;
     case NDPI_ICMP_PROTOCOL_TYPE:
       if(flow) {
+        flow->entropy = 0.0f;
 	/* Run some basic consistency tests */
 
 	if(flow->packet.payload_packet_len < sizeof(struct ndpi_icmphdr))
@@ -2853,6 +2855,15 @@ u_int16_t ndpi_guess_protocol_id(struct ndpi_detection_module_struct *ndpi_str,
 	  if(((icmp_type >= 44) && (icmp_type <= 252))
 	     || (icmp_code > 15))
 	    ndpi_set_risk(ndpi_str, flow, NDPI_MALFORMED_PACKET);
+	  if (flow->packet.payload_packet_len > sizeof(struct ndpi_icmphdr))
+	  {
+	    flow->entropy = ndpi_entropy(flow->packet.payload + sizeof(struct ndpi_icmphdr),
+	                                 flow->packet.payload_packet_len - sizeof(struct ndpi_icmphdr));
+	    if (NDPI_ENTROPY_ENCRYPTED_OR_RANDOM(flow->entropy) != 0)
+	    {
+	      ndpi_set_risk(ndpi_str, flow, NDPI_ENTROPY_SUSPICIOUS);
+	    }
+	  }
 	}
       }
       return(NDPI_PROTOCOL_IP_ICMP);
diff --git a/src/lib/ndpi_utils.c b/src/lib/ndpi_utils.c
index a753dc086..714a1f28c 100644
--- a/src/lib/ndpi_utils.c
+++ b/src/lib/ndpi_utils.c
@@ -1126,6 +1126,9 @@ int ndpi_dpi2json(struct ndpi_detection_module_struct *ndpi_struct,
 
   ndpi_serialize_start_of_block(serializer, "ndpi");
   ndpi_serialize_risk(serializer, flow);
+  if (l7_protocol.master_protocol == NDPI_PROTOCOL_IP_ICMP && flow->entropy > 0.0f) {
+    ndpi_serialize_string_float(serializer, "entropy", flow->entropy, "%.6f");
+  }
   ndpi_serialize_string_string(serializer, "proto", ndpi_protocol2name(ndpi_struct, l7_protocol, buf, sizeof(buf)));
   ndpi_protocol_breed_t breed =
       ndpi_get_proto_breed(ndpi_struct,
@@ -1777,6 +1780,9 @@ const char* ndpi_risk2str(ndpi_risk_enum risk) {
   case NDPI_TLS_FATAL_ALERT:
     return("TLS fatal alert");
 
+  case NDPI_ENTROPY_SUSPICIOUS:
+    return("Entropy suspicious");
+
   default:
     snprintf(buf, sizeof(buf), "%d", (int)risk);
     return(buf);
@@ -2129,16 +2135,14 @@ int ndpi_is_printable_string(char const * const str, size_t len) {
 
 /* ******************************************************************** */
 
-float ndpi_calculate_entropy(u_int8_t const * const buf, size_t len) {
+float ndpi_entropy(u_int8_t const * const buf, size_t len) {
   float entropy = 0.0f;
   u_int32_t byte_counters[256];
 
   memset(byte_counters, 0, sizeof(byte_counters));
 
   for (size_t i = 0; i < len; ++i) {
-    if (buf[i] == i) {
-      byte_counters[i]++;
-    }
+    byte_counters[buf[i]]++;
   }
 
   for (size_t i = 0; i < sizeof(byte_counters) / sizeof(byte_counters[0]); ++i) {
@@ -2146,11 +2150,10 @@ float ndpi_calculate_entropy(u_int8_t const * const buf, size_t len) {
       continue;
     }
 
-    float p = 1.0f * byte_counters[i] / len;
-    entropy -= p * log2f(p);
+    float const p = (float)byte_counters[i] / len;
+    entropy += p * log2f(1 / p);
   }
 
-  entropy *= -1.0f;
   return entropy;
 }
 
diff --git a/tests/result/quic_interop_V.pcapng.out b/tests/result/quic_interop_V.pcapng.out
index 12441f792..b8548ab94 100644
--- a/tests/result/quic_interop_V.pcapng.out
+++ b/tests/result/quic_interop_V.pcapng.out
@@ -13,7 +13,6 @@ JA3 Host Stats:
 	1	 2001:b07:ac9:d5ae:a4d3:fe47:691e:807d 	 1      
 	2	 192.168.1.128            	 1      
 
-
 	1	UDP 192.168.1.128:34511 -> 131.159.24.198:443 [proto: 188/QUIC][Encrypted][cat: Web/5][8 pkts/10352 bytes -> 0 pkts/0 bytes][Goodput ratio: 97/0][9.94 sec][ALPN: hq-30;h3-30;hq-29;h3-29;hq-28;h3-28;hq-27;h3-27][TLS Supported Versions: TLSv1.3][bytes ratio: 1.000 (Upload)][IAT c2s/s2c min/avg/max/stddev: 150/0 1419/0 4800/0 1551/0][Pkt Len c2s/s2c min/avg/max/stddev: 1294/0 1294/0 1294/0 0/0][TLSv1.3][Client: pandora.cm.in.tum.de][JA3C: 7d9e7f6dec1cb1dd8b79d72b1366b6cf][Firefox][PLAIN TEXT (SezYZO)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0]
 	2	UDP 192.168.1.128:37643 -> 71.202.41.169:443 [proto: 188/QUIC][Encrypted][cat: Web/5][8 pkts/10352 bytes -> 0 pkts/0 bytes][Goodput ratio: 97/0][9.98 sec][ALPN: hq-30;h3-30;hq-29;h3-29;hq-28;h3-28;hq-27;h3-27][TLS Supported Versions: TLSv1.3][bytes ratio: 1.000 (Upload)][IAT c2s/s2c min/avg/max/stddev: 243/0 1426/0 4801/0 1546/0][Pkt Len c2s/s2c min/avg/max/stddev: 1294/0 1294/0 1294/0 0/0][TLSv1.3][Client: 71.202.41.169][JA3C: 7d9e7f6dec1cb1dd8b79d72b1366b6cf][Firefox][PLAIN TEXT (tIABbj)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0]
 	3	UDP 192.168.1.128:42468 -> 138.91.188.147:4433 [proto: 188.212/QUIC.Microsoft][Encrypted][cat: Cloud/13][8 pkts/10352 bytes -> 0 pkts/0 bytes][Goodput ratio: 97/0][9.98 sec][ALPN: hq-30;h3-30;hq-29;h3-29;hq-28;h3-28;hq-27;h3-27][TLS Supported Versions: TLSv1.3][bytes ratio: 1.000 (Upload)][IAT c2s/s2c min/avg/max/stddev: 150/0 1425/0 4800/0 1548/0][Pkt Len c2s/s2c min/avg/max/stddev: 1294/0 1294/0 1294/0 0/0][Risk: ** Known protocol on non standard port **][Risk Score: 10][TLSv1.3][Client: quic.westus.cloudapp.azure.com][JA3C: 7d9e7f6dec1cb1dd8b79d72b1366b6cf][Firefox][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0]