Fixes risk mask exception handling while improving the overall performance

5 files changed, 87 insertions, 72 deletions

diff --git a/src/include/ndpi_api.h b/src/include/ndpi_api.h
index ce124bbf6..e48fc013e 100644
--- a/src/include/ndpi_api.h
+++ b/src/include/ndpi_api.h
@@ -1082,6 +1082,8 @@ extern "C" {
   void ndpi_set_tls_cert_expire_days(struct ndpi_detection_module_struct *ndpi_str,
 				     u_int8_t days);
 
+  void ndpi_handle_risk_exceptions(struct ndpi_detection_module_struct *ndpi_str,
+				   struct ndpi_flow_struct *flow);
 
   /* Utility functions to set ndpi malloc/free/print wrappers */
   void set_ndpi_malloc(void* (*__ndpi_malloc)(size_t size));
diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h
index 373a704da..91ae93b4e 100644
--- a/src/include/ndpi_typedefs.h
+++ b/src/include/ndpi_typedefs.h
@@ -1432,7 +1432,7 @@ struct ndpi_flow_struct {
   u_int8_t initial_binary_bytes[8], initial_binary_bytes_len;
   u_int8_t risk_checked:1, ip_risk_mask_evaluated:1, host_risk_mask_evaluated:1, tree_risk_checked:1, _notused:4;
   ndpi_risk risk_mask; /* Stores the flow risk mask for flow peers */
-  ndpi_risk risk; /* Issues found with this flow [bitmask of ndpi_risk] */
+  ndpi_risk risk, risk_shadow; /* Issues found with this flow [bitmask of ndpi_risk] */
   struct ndpi_risk_information risk_infos[MAX_NUM_RISK_INFOS]; /* String that contains information about the risks found */
   u_int8_t num_risk_infos;
   
diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index 65c50b6ec..ca7bdd912 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -6079,7 +6079,6 @@ static u_int32_t make_msteams_key(struct ndpi_flow_struct *flow, u_int8_t use_cl
 static void ndpi_reconcile_msteams_udp(struct ndpi_detection_module_struct *ndpi_str,
 				       struct ndpi_flow_struct *flow,
 				       u_int16_t master) {
-
   /* This function can NOT access &ndpi_str->packet since it is called also from ndpi_detection_giveup(), via ndpi_reconcile_protocols() */
 
   if(flow->l4_proto == IPPROTO_UDP) {
@@ -6159,6 +6158,12 @@ static void ndpi_reconcile_protocols(struct ndpi_detection_module_struct *ndpi_s
 
   // printf("====>> %u.%u [%u]\n", ret->master_protocol, ret->app_protocol, flow->detected_protocol_stack[0]);
 
+  if((flow->risk != 0) && (flow->risk != flow->risk_shadow)) {
+    /* Trick to avoid evaluating exceptions when nothing changed */
+    ndpi_handle_risk_exceptions(ndpi_str, flow);
+    flow->risk_shadow = flow->risk;
+  }
+  
   switch(ret->app_protocol) {
   case NDPI_PROTOCOL_MICROSOFT_AZURE:
     ndpi_reconcile_msteams_udp(ndpi_str, flow, flow->detected_protocol_stack[1]);
diff --git a/src/lib/ndpi_utils.c b/src/lib/ndpi_utils.c
index 75201e55e..5f334081b 100644
--- a/src/lib/ndpi_utils.c
+++ b/src/lib/ndpi_utils.c
@@ -821,7 +821,7 @@ static const char* ndpi_get_flow_info_by_proto_id(struct ndpi_flow_struct const
         return flow->host_server_name;
       break;
   }
-  
+
   return NULL;
 }
 
@@ -831,8 +831,8 @@ const char* ndpi_get_flow_info(struct ndpi_flow_struct const * const flow,
                                ndpi_protocol const * const l7_protocol) {
   char const * const app_protocol_info = ndpi_get_flow_info_by_proto_id(flow, l7_protocol->app_protocol);
 
-  if(app_protocol_info != NULL)  
-    return app_protocol_info;  
+  if(app_protocol_info != NULL)
+    return app_protocol_info;
 
   return ndpi_get_flow_info_by_proto_id(flow, l7_protocol->master_protocol);
 }
@@ -2005,7 +2005,7 @@ const char* ndpi_risk2str(ndpi_risk_enum risk) {
 
   case NDPI_DNS_LARGE_PACKET:
     return("Large DNS Packet (512+ bytes)");
-    
+
   case NDPI_DNS_FRAGMENTED:
     return("Fragmented DNS Message");
 
@@ -2041,7 +2041,7 @@ const char* ndpi_risk2str(ndpi_risk_enum risk) {
 
   case NDPI_MINOR_ISSUES:
     return("Minor Issues");
-    
+
   case NDPI_TCP_ISSUES:
     return("TCP Connection Issues");
 
@@ -2069,7 +2069,7 @@ const char* ndpi_severity2str(ndpi_risk_severity s) {
 
   case NDPI_RISK_CRITICAL:
     return("Critical");
-    
+
   case NDPI_RISK_EMERGENCY:
     return("Emergency");
   }
@@ -2333,7 +2333,7 @@ u_int8_t ndpi_check_issuerdn_risk_exception(struct ndpi_detection_module_struct
 					    char *issuerDN) {
   if(issuerDN != NULL) {
     ndpi_list *head = ndpi_str->trusted_issuer_dn;
-    
+
     while(head != NULL) {
       if(strcmp(issuerDN, head->value) == 0)
 	return(1); /* This is a trusted DN */
@@ -2341,7 +2341,7 @@ u_int8_t ndpi_check_issuerdn_risk_exception(struct ndpi_detection_module_struct
 	head = head->next;
     }
   }
-  
+
   return(0 /* no exception */);
 }
 
@@ -2356,7 +2356,7 @@ static u_int8_t ndpi_check_hostname_risk_exception(struct ndpi_detection_module_
   else {
     ndpi_automa *automa = &ndpi_str->host_risk_mask_automa;
     u_int8_t ret = 0;
-    
+
     if(automa && automa->ac_automa) {
       AC_TEXT_t ac_input_text;
       AC_REP_t match;
@@ -2364,13 +2364,13 @@ static u_int8_t ndpi_check_hostname_risk_exception(struct ndpi_detection_module_
       memset(&match, 0, sizeof(match));
       ac_input_text.astring = hostname, ac_input_text.length = strlen(hostname);
       ac_input_text.option = 0;
-      
+
       if(ac_automata_search(automa->ac_automa, &ac_input_text, &match) > 0) {
 	if(flow) flow->risk_mask &= match.number64;
 	ret = 1;
       }
     }
-    
+
     return(ret);
   }
 }
@@ -2383,37 +2383,34 @@ static u_int8_t ndpi_check_ipv4_exception(struct ndpi_detection_module_struct *n
 					  u_int32_t addr) {
   struct in_addr pin;
   u_int64_t r;
-  
+
   pin.s_addr = addr;
   r = ndpi_host_ip_risk_ptree_match(ndpi_str, &pin);
-  
+
   if(flow) flow->risk_mask &= r;
-  
+
   return((r != (u_int64_t)-1) ? 1 : 0);
 }
 
 /* ********************************************************************************* */
 
-static void ndpi_handle_risk_exceptions(struct ndpi_detection_module_struct *ndpi_str,
-					struct ndpi_flow_struct *flow) {
-  char *host;
-
+void ndpi_handle_risk_exceptions(struct ndpi_detection_module_struct *ndpi_str,
+				 struct ndpi_flow_struct *flow) {
   if(flow->risk == 0) return; /* Nothing to do */
 
-  host = ndpi_get_flow_name(flow);
-
-  if((!flow->host_risk_mask_evaluated) && (!flow->ip_risk_mask_evaluated)) {
+  if((!flow->host_risk_mask_evaluated) && (!flow->ip_risk_mask_evaluated))
     flow->risk_mask = (u_int64_t)-1; /* No mask */
-  }
 
   if(!flow->host_risk_mask_evaluated) {
+    char *host = ndpi_get_flow_name(flow);
+
     if(host && (host[0] != '\0')) {
       /* Check host exception */
       ndpi_check_hostname_risk_exception(ndpi_str, flow, host);
 
       if(flow->risk_mask == 0) {
 	u_int i;
-	
+
 	/*
 	  Might be that the exception applied when some risks
 	  were already triggered: we need to clean them up
@@ -2424,10 +2421,10 @@ static void ndpi_handle_risk_exceptions(struct ndpi_detection_module_struct *ndp
 	    flow->risk_infos[i].info = NULL;
 	  }
 	}
-	
+
 	flow->num_risk_infos = 0;
       }
-      
+
       /* Used to avoid double checks (e.g. in DNS req/rsp) */
       flow->host_risk_mask_evaluated = 1;
     }
@@ -2451,14 +2448,16 @@ static void ndpi_handle_risk_exceptions(struct ndpi_detection_module_struct *ndp
 void ndpi_set_risk(struct ndpi_detection_module_struct *ndpi_str,
 		   struct ndpi_flow_struct *flow, ndpi_risk_enum r,
 		   char *risk_message) {
+
   /* Check if the risk is not yet set */
   if(!ndpi_isset_risk(ndpi_str, flow, r)) {
     ndpi_risk v = 1ull << r;
-    
+
     // NDPI_SET_BIT(flow->risk, (u_int32_t)r);
     flow->risk |= v;
-    
-    ndpi_handle_risk_exceptions(ndpi_str, flow);
+
+    /* Will be handled by ndpi_reconcile_protocols() */
+    // ndpi_handle_risk_exceptions(ndpi_str, flow);
 
     if(flow->risk != 0 /* check if it has been masked */) {
       if(risk_message != NULL) {
@@ -2574,15 +2573,15 @@ int ndpi_is_valid_hostname(char * const str, size_t len) {
        || (str[i] == '_')
        || (str[i] == ':')
        )
-      continue; /* Used in hostnames */    
-    else if((ndpi_isprint(str[i]) == 0)       
+      continue; /* Used in hostnames */
+    else if((ndpi_isprint(str[i]) == 0)
 	    || ndpi_isspace(str[i])
 	    || ndpi_ispunct(str[i])
 	    ) {
       return(0);
     }
   }
-  
+
   return(1);
 }
 
@@ -2763,7 +2762,7 @@ u_int32_t ndpi_get_flow_error_code(struct ndpi_flow_struct *flow) {
 
   case NDPI_PROTOCOL_HTTP:
     return(flow->http.response_status_code);
- 
+
   case NDPI_PROTOCOL_SNMP:
     return(flow->protos.snmp.error_status);
  }
@@ -2823,7 +2822,7 @@ char* ndpi_get_flow_risk_info(struct ndpi_flow_struct *flow,
 			      char *out, u_int out_len,
 			      u_int8_t use_json) {
   u_int i, offset = 0;
-  
+
   if((out == NULL)
      || (flow == NULL)
      || (flow->num_risk_infos == 0))
@@ -2833,15 +2832,15 @@ char* ndpi_get_flow_risk_info(struct ndpi_flow_struct *flow,
     ndpi_serializer serializer;
     u_int32_t buffer_len;
     char *buffer;
-    
+
     if(ndpi_init_serializer(&serializer, ndpi_serialization_format_json) == -1)
       return(NULL);
 
     for(i=0; i<flow->num_risk_infos; i++)
       ndpi_serialize_uint32_string(&serializer,
-				   flow->risk_infos[i].id, 
-				   flow->risk_infos[i].info);  
-    
+				   flow->risk_infos[i].id,
+				   flow->risk_infos[i].info);
+
     buffer = ndpi_serializer_get_buffer(&serializer, &buffer_len);
 
     if(buffer && (buffer_len > 0)) {
@@ -2850,28 +2849,28 @@ char* ndpi_get_flow_risk_info(struct ndpi_flow_struct *flow,
       strncpy(out, buffer, l);
       out[l] = '\0';
     }
-    
+
     ndpi_term_serializer(&serializer);
 
     return(out);
   } else {
     out[0] = '\0', out_len--;
-    
+
     for(i=0; (i<flow->num_risk_infos) && (out_len > offset); i++) {
       int rc = snprintf(&out[offset], out_len-offset, "%s%s",
 			(i == 0) ? "" : " / ",
 			flow->risk_infos[i].info);
-      
+
       if(rc <= 0)
 	break;
       else
 	offset += rc;
     }
-    
+
     if(offset > out_len) offset = out_len;
-    
+
     out[offset] = '\0';
-  
+
     return(out[0] == '\0' ? NULL : out);
   }
 }
@@ -2879,7 +2878,7 @@ char* ndpi_get_flow_risk_info(struct ndpi_flow_struct *flow,
 /* ******************************************* */
 /*
   This function checks if a flow having the specified risk
-  parameters is an exception (i.e. the flow risk should not 
+  parameters is an exception (i.e. the flow risk should not
   be triggered) or not.
 
   You can use this function to check if a flow that
@@ -2899,7 +2898,7 @@ u_int8_t ndpi_check_flow_risk_exceptions(struct ndpi_detection_module_struct *nd
       if(ndpi_check_hostname_risk_exception(ndpi_str, NULL, (char*)params[i].value))
 	return(1);
       break;
-      
+
     case NDPI_PARAM_ISSUER_DN:
       if(ndpi_check_issuerdn_risk_exception(ndpi_str, (char*)params[i].value))
 	return(1);
@@ -2920,7 +2919,7 @@ u_int8_t ndpi_check_flow_risk_exceptions(struct ndpi_detection_module_struct *nd
       break;
     }
   }
-  
+
   return(0);
 }
 
@@ -2994,7 +2993,7 @@ char* ndpi_intoav4(unsigned int addr, char* buf, u_int16_t bufLen) {
 /* Find the nearest (>=) value of x */
 u_int32_t ndpi_nearest_power_of_two(u_int32_t x) {
   x--;
-  
+
   x |= x >> 1;
   x |= x >> 2;
   x |= x >> 4;
@@ -3004,4 +3003,3 @@ u_int32_t ndpi_nearest_power_of_two(u_int32_t x) {
   x++;
   return(x);
 }
-
diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c
index 8885906d9..6951dd85c 100644
--- a/src/lib/protocols/http.c
+++ b/src/lib/protocols/http.c
@@ -741,28 +741,38 @@ static void ndpi_check_http_url(struct ndpi_detection_module_struct *ndpi_struct
 static void ndpi_check_http_server(struct ndpi_detection_module_struct *ndpi_struct,
 				   struct ndpi_flow_struct *flow,
 				   const char *server, u_int server_len) {
-  if(server_len > 7) {
-    u_int off;
-
-    if((strncasecmp(server, "Apache/", off = 7) == 0) /* X.X.X */
-	|| (strncasecmp(server, "nginx/", off = 6) == 0) /* X.X.X */) {
-      u_int i, j, a, b, c;
-      char buf[16] = { '\0' };
-
-      for(i=off, j=0; (i<server_len) && (j<sizeof(buf)-1)
-	    && (isdigit(server[i]) || (server[i] == '.')); i++)
-	buf[j++] = server[i];
-
-      if(sscanf(buf, "%d.%d.%d", &a, &b, &c) == 3) {
-	u_int32_t version = (a * 1000000) + (b * 1000) + c;
-	char msg[64];
-
-	if((off == 7) && (version < MIN_APACHE_VERSION)) {
-	  snprintf(msg, sizeof(msg), "Obsolete Apache server %s", buf);
-	  ndpi_set_risk(ndpi_struct, flow, NDPI_HTTP_OBSOLETE_SERVER, msg);
-	} else if((off == 6) && (version < MIN_NGINX_VERSION)) {
-	  snprintf(msg, sizeof(msg), "Obsolete nginx server %s", buf);
-	  ndpi_set_risk(ndpi_struct, flow, NDPI_HTTP_OBSOLETE_SERVER, msg);
+  if(server[0] != '\0') {
+    if(server_len > 7) {
+      u_int off, i;
+
+      if((strncasecmp(server, "Apache/", off = 7) == 0) /* X.X.X */
+	 || (strncasecmp(server, "nginx/", off = 6) == 0) /* X.X.X */) {
+	u_int j, a, b, c;
+	char buf[16] = { '\0' };
+
+	for(i=off, j=0; (i<server_len) && (j<sizeof(buf)-1)
+	      && (isdigit(server[i]) || (server[i] == '.')); i++)
+	  buf[j++] = server[i];
+
+	if(sscanf(buf, "%d.%d.%d", &a, &b, &c) == 3) {
+	  u_int32_t version = (a * 1000000) + (b * 1000) + c;
+	  char msg[64];
+
+	  if((off == 7) && (version < MIN_APACHE_VERSION)) {
+	    snprintf(msg, sizeof(msg), "Obsolete Apache server %s", buf);
+	    ndpi_set_risk(ndpi_struct, flow, NDPI_HTTP_OBSOLETE_SERVER, msg);
+	  } else if((off == 6) && (version < MIN_NGINX_VERSION)) {
+	    snprintf(msg, sizeof(msg), "Obsolete nginx server %s", buf);
+	    ndpi_set_risk(ndpi_struct, flow, NDPI_HTTP_OBSOLETE_SERVER, msg);
+	  }
+	}
+      }
+
+      /* Check server content */
+      for(i=0; i<server_len; i++) {
+	if(!isprint(server[i])) {
+	  ndpi_set_risk(ndpi_struct, flow, NDPI_HTTP_SUSPICIOUS_HEADER, "Suspicious Agent");
+	  break;
 	}
       }
     }