Add extra entropy checks and more precise(?) analysis. (#2383)

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

5 files changed, 100 insertions, 29 deletions

diff --git a/src/include/ndpi_main.h b/src/include/ndpi_main.h
index 7012fd8ec..a95bf03c2 100644
--- a/src/include/ndpi_main.h
+++ b/src/include/ndpi_main.h
@@ -113,8 +113,15 @@ extern "C" {
   int ndpi_is_printable_buffer(u_int8_t const * const buf, size_t len);
   int ndpi_normalize_printable_string(char * const str, size_t len);
   int ndpi_is_valid_hostname(char * const str, size_t len);
-#define NDPI_ENTROPY_ENCRYPTED_OR_RANDOM(entropy) (entropy > 7.0f)
-  float ndpi_entropy(u_int8_t const * const buf, size_t len); 
+
+#define NDPI_ENTROPY_PLAINTEXT(entropy) (entropy < 4.941f)
+#define NDPI_ENTROPY_EXECUTABLE(entropy) (entropy >= 4.941f)
+#define NDPI_ENTROPY_EXECUTABLE_PACKED(entropy) (entropy >= 6.677f)
+#define NDPI_ENTROPY_EXECUTABLE_ENCRYPTED(entropy) (entropy >= 7.174f)
+#define NDPI_ENTROPY_ENCRYPTED_OR_RANDOM(entropy) (entropy >= 7.312f)
+  float ndpi_entropy(u_int8_t const * const buf, size_t len);
+  char *ndpi_entropy2str(float entropy, char *buf, size_t len);
+  void ndpi_entropy2risk(struct ndpi_flow_struct *flow);
 
 #ifdef __cplusplus
 }
diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h
index 51b12595b..9a65e292a 100644
--- a/src/include/ndpi_typedefs.h
+++ b/src/include/ndpi_typedefs.h
@@ -1202,7 +1202,7 @@ struct ndpi_flow_struct {
   /* init parameter, internal used to set up timestamp,... */
   u_int16_t guessed_protocol_id, guessed_protocol_id_by_ip, guessed_category, guessed_header_category;
   u_int8_t l4_proto, protocol_id_already_guessed:1, fail_with_unknown:1,
-    init_finished:1, client_packet_direction:1, packet_direction:1, is_ipv6:1, first_pkt_fully_encrypted:1, _pad1: 1;
+    init_finished:1, client_packet_direction:1, packet_direction:1, is_ipv6:1, first_pkt_fully_encrypted:1, skip_entropy_check: 1;
 
   u_int16_t num_dissector_calls;
   ndpi_confidence_t confidence; /* ndpi_confidence_t */
diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index 2fc1dc455..ada8129b2 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -175,7 +175,7 @@ static ndpi_risk_info ndpi_known_risks[] = {
   { NDPI_TLS_CERT_VALIDITY_TOO_LONG,            NDPI_RISK_MEDIUM, CLIENT_FAIR_RISK_PERCENTAGE, NDPI_SERVER_ACCOUNTABLE },
   { NDPI_TLS_SUSPICIOUS_EXTENSION,              NDPI_RISK_HIGH,   CLIENT_HIGH_RISK_PERCENTAGE, NDPI_BOTH_ACCOUNTABLE   },
   { NDPI_TLS_FATAL_ALERT,                       NDPI_RISK_LOW,    CLIENT_FAIR_RISK_PERCENTAGE, NDPI_BOTH_ACCOUNTABLE   },
-  { NDPI_SUSPICIOUS_ENTROPY,                    NDPI_RISK_MEDIUM, CLIENT_FAIR_RISK_PERCENTAGE, NDPI_BOTH_ACCOUNTABLE   },
+  { NDPI_SUSPICIOUS_ENTROPY,                    NDPI_RISK_LOW,    CLIENT_FAIR_RISK_PERCENTAGE, NDPI_BOTH_ACCOUNTABLE   },
   { NDPI_CLEAR_TEXT_CREDENTIALS,                NDPI_RISK_HIGH,   CLIENT_HIGH_RISK_PERCENTAGE, NDPI_CLIENT_ACCOUNTABLE },
   { NDPI_DNS_LARGE_PACKET,                      NDPI_RISK_MEDIUM, CLIENT_FAIR_RISK_PERCENTAGE, NDPI_CLIENT_ACCOUNTABLE },
   { NDPI_DNS_FRAGMENTED,                        NDPI_RISK_MEDIUM, CLIENT_FAIR_RISK_PERCENTAGE, NDPI_CLIENT_ACCOUNTABLE },
@@ -4396,14 +4396,10 @@ static u_int16_t guess_protocol_id(struct ndpi_detection_module_struct *ndpi_str
 	    ndpi_set_risk(flow, NDPI_MALFORMED_PACKET, NULL);
 
 	  if(packet->payload_packet_len > sizeof(struct ndpi_icmphdr)) {
-	    flow->entropy = ndpi_entropy(packet->payload + sizeof(struct ndpi_icmphdr),
-	                                 packet->payload_packet_len - sizeof(struct ndpi_icmphdr));
-
-	    if(NDPI_ENTROPY_ENCRYPTED_OR_RANDOM(flow->entropy) != 0) {
-	      char str[32];
-
-		snprintf(str, sizeof(str), "Entropy %.2f", flow->entropy);
-		ndpi_set_risk(flow, NDPI_SUSPICIOUS_ENTROPY, str);
+	    if (flow->skip_entropy_check == 0) {
+	      flow->entropy = ndpi_entropy(packet->payload + sizeof(struct ndpi_icmphdr),
+	                                   packet->payload_packet_len - sizeof(struct ndpi_icmphdr));
+	      ndpi_entropy2risk(flow);
 	    }
 
 	    u_int16_t chksm = icmp4_checksum(packet->payload, packet->payload_packet_len);
@@ -8583,25 +8579,16 @@ static ndpi_protocol ndpi_internal_detection_process_packet(struct ndpi_detectio
     ndpi_search_shellscript(ndpi_str, flow);
   }
 
-  if(flow->first_pkt_fully_encrypted == 0 &&
-     ret.app_protocol == NDPI_PROTOCOL_UNKNOWN &&
-     NDPI_ENTROPY_ENCRYPTED_OR_RANDOM(flow->entropy) == 0 &&
-     flow->packet_counter < 3)
+  if(flow->skip_entropy_check == 0 &&
+     flow->first_pkt_fully_encrypted == 0 &&
+     flow->packet_counter < 5 &&
+     /* The following protocols do their own entropy calculation/classification. */
+     ret.app_protocol != NDPI_PROTOCOL_IP_ICMP)
   {
-    flow->entropy = ndpi_entropy(packet->payload, packet->payload_packet_len);
-    if(NDPI_ENTROPY_ENCRYPTED_OR_RANDOM(flow->entropy) != 0) {
-      char str[32];
-
-      snprintf(str, sizeof(str), "Entropy %.2f", flow->entropy);
-      ndpi_set_risk(flow, NDPI_SUSPICIOUS_ENTROPY, str);
+    if (ret.app_protocol != NDPI_PROTOCOL_HTTP) {
+      flow->entropy = ndpi_entropy(packet->payload, packet->payload_packet_len);
     }
-  }
-  if(ret.app_protocol != NDPI_PROTOCOL_UNKNOWN &&
-     ret.app_protocol != NDPI_PROTOCOL_IP_ICMP &&
-     flow->entropy > 0.0f)
-  {
-    flow->entropy = 0.0f;
-    ndpi_unset_risk(flow, NDPI_SUSPICIOUS_ENTROPY);
+    ndpi_entropy2risk(flow);
   }
 
   return(ret);
diff --git a/src/lib/ndpi_utils.c b/src/lib/ndpi_utils.c
index a9f8d0a7c..9f71974c4 100644
--- a/src/lib/ndpi_utils.c
+++ b/src/lib/ndpi_utils.c
@@ -2705,6 +2705,79 @@ float ndpi_entropy(u_int8_t const * const buf, size_t len) {
 }
 
 /* ******************************************************************** */
+
+/* Losely implemented by: https://redirect.cs.umbc.edu/courses/graduate/CMSC691am/student%20talks/CMSC%20691%20Malware%20-%20Entropy%20Analysis%20Presentation.pdf */
+char *ndpi_entropy2str(float entropy, char *buf, size_t len) {
+  if (buf == NULL) {
+    return NULL;
+  }
+
+  static const char entropy_fmtstr[] = "Entropy: %.3f (%s?)";
+  if (NDPI_ENTROPY_ENCRYPTED_OR_RANDOM(entropy)) {
+    snprintf(buf, len, entropy_fmtstr, entropy, "Encrypted or Random");
+  } else if (NDPI_ENTROPY_EXECUTABLE_ENCRYPTED(entropy)) {
+    snprintf(buf, len, entropy_fmtstr, entropy, "Encrypted Executable");
+  } else if (NDPI_ENTROPY_EXECUTABLE_PACKED(entropy)) {
+    snprintf(buf, len, entropy_fmtstr, entropy, "Compressed Executable");
+  } else if (NDPI_ENTROPY_EXECUTABLE(entropy)) {
+    snprintf(buf, len, entropy_fmtstr, entropy, "Executable");
+  } else {
+    snprintf(buf, len, entropy_fmtstr, entropy, "Unknown");
+  }
+
+  return buf;
+}
+
+/* ******************************************************************** */
+
+void ndpi_entropy2risk(struct ndpi_flow_struct *flow) {
+  char str[64];
+
+  if (NDPI_ENTROPY_PLAINTEXT(flow->entropy))
+    goto reset_risk;
+
+  if (flow->detected_protocol_stack[0] == NDPI_PROTOCOL_TLS ||
+      flow->detected_protocol_stack[1] == NDPI_PROTOCOL_TLS ||
+      flow->detected_protocol_stack[0] == NDPI_PROTOCOL_QUIC ||
+      flow->detected_protocol_stack[1] == NDPI_PROTOCOL_QUIC ||
+      flow->detected_protocol_stack[0] == NDPI_PROTOCOL_DTLS ||
+      flow->detected_protocol_stack[1] == NDPI_PROTOCOL_DTLS)
+  {
+    flow->skip_entropy_check = 1;
+    goto reset_risk;
+  }
+
+  if (flow->confidence != NDPI_CONFIDENCE_DPI &&
+      flow->confidence != NDPI_CONFIDENCE_DPI_CACHE) {
+    ndpi_set_risk(flow, NDPI_SUSPICIOUS_ENTROPY,
+                  ndpi_entropy2str(flow->entropy, str, sizeof(str)));
+    return;
+  }
+
+  if (ndpi_isset_risk(flow, NDPI_MALWARE_HOST_CONTACTED) ||
+      ndpi_isset_risk(flow, NDPI_BINARY_DATA_TRANSFER) ||
+      ndpi_isset_risk(flow, NDPI_BINARY_APPLICATION_TRANSFER) ||
+      ndpi_isset_risk(flow, NDPI_POSSIBLE_EXPLOIT) ||
+      ndpi_isset_risk(flow, NDPI_HTTP_SUSPICIOUS_CONTENT) ||
+      ndpi_isset_risk(flow, NDPI_DNS_SUSPICIOUS_TRAFFIC) ||
+      ndpi_isset_risk(flow, NDPI_MALFORMED_PACKET) ||
+      (flow->category == NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT &&
+       (flow->detected_protocol_stack[0] == NDPI_PROTOCOL_HTTP ||
+        flow->detected_protocol_stack[1] == NDPI_PROTOCOL_HTTP)) ||
+      flow->category == NDPI_PROTOCOL_CATEGORY_DATA_TRANSFER ||
+      flow->category == NDPI_PROTOCOL_CATEGORY_UNSPECIFIED ||
+      flow->category == NDPI_PROTOCOL_CATEGORY_WEB)
+  {
+    ndpi_set_risk(flow, NDPI_SUSPICIOUS_ENTROPY,
+                  ndpi_entropy2str(flow->entropy, str, sizeof(str)));
+    return;
+  }
+
+reset_risk:
+  ndpi_unset_risk(flow, NDPI_SUSPICIOUS_ENTROPY);
+}
+
+/* ******************************************************************** */
 static inline uint16_t get_n16bit(uint8_t const * cbuf) {
   uint16_t r = ((uint16_t)cbuf[0]) | (((uint16_t)cbuf[1]) << 8);
   return r;
diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c
index 8fc82dd67..c73bd47fe 100644
--- a/src/lib/protocols/http.c
+++ b/src/lib/protocols/http.c
@@ -222,8 +222,12 @@ static void ndpi_validate_http_content(struct ndpi_detection_module_struct *ndpi
 
       if(len >= 8 /* 4 chars for \r\n\r\n and at least 4 charts for content guess */) {
 	double_ret += 4;
+	len -= 4;
 
 	ndpi_http_check_human_redeable_content(ndpi_struct, flow, double_ret, len);
+	if (flow->skip_entropy_check == 0) {
+	  flow->entropy = ndpi_entropy(double_ret, len);
+	}
       }
     }