Domain Classification Improvements (#2396)

* Added
size_t ndpi_compress_str(const char * in, size_t len, char * out, size_t bufsize);
size_t ndpi_decompress_str(const char * in, size_t len, char * out, size_t bufsize);

used to compress short strings such as domain names. This code is based on
https://github.com/Ed-von-Schleck/shoco

* Major code rewrite for ndpi_hash and ndpi_domain_classify

* Improvements to make sure custom categories are loaded and enabled

* Fixed string encoding

* Extended SalesForce/Cloudflare domains list

5 files changed, 17 insertions, 14 deletions

diff --git a/src/lib/protocols/dns.c b/src/lib/protocols/dns.c
index 70b8cd451..d9eaf1e84 100644
--- a/src/lib/protocols/dns.c
+++ b/src/lib/protocols/dns.c
@@ -766,7 +766,7 @@ static void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, st
     ndpi_hostname_sni_set(flow, (const u_int8_t *)_hostname, len, is_mdns ? NDPI_HOSTNAME_NORM_LC : NDPI_HOSTNAME_NORM_ALL);
 
     if (hostname_is_valid == 0)
-      ndpi_set_risk(flow, NDPI_INVALID_CHARACTERS, NULL);
+      ndpi_set_risk(flow, NDPI_INVALID_CHARACTERS, "Invalid chars detected in domain name");
 
     dot = strchr(_hostname, '.');
     if(dot) {
diff --git a/src/lib/protocols/fastcgi.c b/src/lib/protocols/fastcgi.c
index a9f9113d3..10384a13e 100644
--- a/src/lib/protocols/fastcgi.c
+++ b/src/lib/protocols/fastcgi.c
@@ -221,7 +221,7 @@ static void ndpi_search_fastcgi(struct ndpi_detection_module_struct *ndpi_struct
         ndpi_set_risk(flow, NDPI_INVALID_CHARACTERS, str);
 
         /* This looks like an attack */
-        ndpi_set_risk(flow, NDPI_POSSIBLE_EXPLOIT, NULL);
+        ndpi_set_risk(flow, NDPI_POSSIBLE_EXPLOIT, "Suspicious hostname: attack ?");
       }
       ndpi_int_fastcgi_add_connection(ndpi_struct, flow, &ret_match);
     }
diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c
index a85f1c44c..8fc82dd67 100644
--- a/src/lib/protocols/http.c
+++ b/src/lib/protocols/http.c
@@ -1007,7 +1007,7 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
 	ndpi_set_risk(flow, NDPI_INVALID_CHARACTERS, str);
 
 	/* This looks like an attack */
-	ndpi_set_risk(flow, NDPI_POSSIBLE_EXPLOIT, NULL);
+	ndpi_set_risk(flow, NDPI_POSSIBLE_EXPLOIT, "Suspicious hostname: attack ?");
       }
 
       double_col = strchr((char*)flow->host_server_name, ':');
diff --git a/src/lib/protocols/quic.c b/src/lib/protocols/quic.c
index 4734433e0..345f77c47 100644
--- a/src/lib/protocols/quic.c
+++ b/src/lib/protocols/quic.c
@@ -1475,7 +1475,7 @@ void process_chlo(struct ndpi_detection_module_struct *ndpi_struct,
 	ndpi_set_risk(flow, NDPI_INVALID_CHARACTERS, str);
 	
 	/* This looks like an attack */
-	ndpi_set_risk(flow, NDPI_POSSIBLE_EXPLOIT, NULL);
+	ndpi_set_risk(flow, NDPI_POSSIBLE_EXPLOIT, "Suspicious hostname: attack ?");
       }
       
       sni_found = 1;
@@ -1503,7 +1503,7 @@ void process_chlo(struct ndpi_detection_module_struct *ndpi_struct,
   /* Add check for missing SNI */
   if(flow->host_server_name[0] == '\0') {
     /* This is a bit suspicious */
-    ndpi_set_risk(flow, NDPI_TLS_MISSING_SNI, NULL);
+    ndpi_set_risk(flow, NDPI_TLS_MISSING_SNI, "SNI should be present all time: attack ?");
   }
 }
 
diff --git a/src/lib/protocols/tls.c b/src/lib/protocols/tls.c
index 882f463fb..54061d10c 100644
--- a/src/lib/protocols/tls.c
+++ b/src/lib/protocols/tls.c
@@ -643,7 +643,7 @@ void processCertificateElements(struct ndpi_detection_module_struct *ndpi_struct
 		      ndpi_set_risk(flow, NDPI_INVALID_CHARACTERS, dNSName);
 
 		      /* This looks like an attack */
-		      ndpi_set_risk(flow, NDPI_POSSIBLE_EXPLOIT, NULL);
+		      ndpi_set_risk(flow, NDPI_POSSIBLE_EXPLOIT, "Invalid dNSName name");
 		    }
 
 		    if(matched_name == 0) {
@@ -695,10 +695,13 @@ void processCertificateElements(struct ndpi_detection_module_struct *ndpi_struct
 
 		    i += len;
 		  } else {
+		    char buf[32];
+
+		    snprintf(buf, sizeof(buf), "Unknown extension %02X", general_name_type);
 #if DEBUG_TLS
 		    printf("[TLS] Leftover %u bytes", packet->payload_packet_len - i);
 #endif
-		    ndpi_set_risk(flow, NDPI_TLS_SUSPICIOUS_EXTENSION, NULL);
+		    ndpi_set_risk(flow, NDPI_TLS_SUSPICIOUS_EXTENSION, buf);
 		    break;
 		  }
 		} else {
@@ -781,7 +784,7 @@ int processCertificate(struct ndpi_detection_module_struct *ndpi_struct,
 
   if((packet->payload_packet_len != (length + 4 + (is_dtls ? 8 : 0))) || (packet->payload[1] != 0x0) ||
      certificates_offset >= packet->payload_packet_len) {
-    ndpi_set_risk(flow, NDPI_MALFORMED_PACKET, NULL);
+    ndpi_set_risk(flow, NDPI_MALFORMED_PACKET, "Unvalid lenght");
     return(-1); /* Invalid length */
   }
 
@@ -790,7 +793,7 @@ int processCertificate(struct ndpi_detection_module_struct *ndpi_struct,
     packet->payload[certificates_offset - 1];
 
   if((packet->payload[certificates_offset - 3] != 0x0) || ((certificates_length+3) != length)) {
-    ndpi_set_risk(flow, NDPI_MALFORMED_PACKET, NULL);
+    ndpi_set_risk(flow, NDPI_MALFORMED_PACKET, "Invalid certificate offset");
     return(-2); /* Invalid length */
   }
 
@@ -1056,7 +1059,7 @@ static int ndpi_search_tls_tcp(struct ndpi_detection_module_struct *ndpi_struct,
 	u_int8_t alert_level = message->buffer[5];
 
 	if(alert_level == 2 /* Warning (1), Fatal (2) */)
-	  ndpi_set_risk(flow, NDPI_TLS_FATAL_ALERT, NULL);
+	  ndpi_set_risk(flow, NDPI_TLS_FATAL_ALERT, "Found fatal TLS alert");
       }
 
       u_int16_t const alert_len = ntohs(*(u_int16_t const *)&message->buffer[3]);
@@ -1516,7 +1519,7 @@ static void checkExtensions(struct ndpi_detection_module_struct *ndpi_struct,
       printf("[TLS] extension length exceeds remaining packet length: %u > %u.\n",
 	     extension_len, packet->payload_packet_len - extension_payload_offset);
 #endif
-      ndpi_set_risk(flow, NDPI_TLS_SUSPICIOUS_EXTENSION, NULL);
+      ndpi_set_risk(flow, NDPI_TLS_SUSPICIOUS_EXTENSION, "Invalid extension len");
       return;
     }
 
@@ -2264,7 +2267,7 @@ int processClientServerHello(struct ndpi_detection_module_struct *ndpi_struct,
 		      ndpi_set_risk(flow, NDPI_INVALID_CHARACTERS, sni);
 
 		      /* This looks like an attack */
-		      ndpi_set_risk(flow, NDPI_POSSIBLE_EXPLOIT, NULL);
+		      ndpi_set_risk(flow, NDPI_POSSIBLE_EXPLOIT, "Invalid chars found in SNI: exploit or misconfiguration?");
 		    }
 
 		    if(!is_quic) {
@@ -2847,7 +2850,7 @@ compute_ja3c:
 	       && (flow->protos.tls_quic.encrypted_sni.esni == NULL) /* No ESNI */
 	       ) {
 	      /* This is a bit suspicious */
-	      ndpi_set_risk(flow, NDPI_TLS_MISSING_SNI, NULL);
+	      ndpi_set_risk(flow, NDPI_TLS_MISSING_SNI, "SNI should always be present");
 
 	      if(flow->protos.tls_quic.advertised_alpns != NULL) {
 		char buf[256], *tmp, *item;
@@ -2859,7 +2862,7 @@ compute_ja3c:
 		while(item != NULL) {
 		  if(item[0] == 'h') {
 		    /* Example 'h2' */
-		    ndpi_set_risk(flow, NDPI_TLS_ALPN_SNI_MISMATCH, NULL);
+		    ndpi_set_risk(flow, NDPI_TLS_ALPN_SNI_MISMATCH, item);
 		    break;
 		  } else
 		    item = strtok_r(NULL, ",", &tmp);