Improved WeChat support

Removed Musical.ly protocol as the service has been merged with another one and thus it is no longer used
Improved guess for UDP protocols

9 files changed, 106 insertions, 60 deletions

diff --git a/src/include/ndpi_api.h b/src/include/ndpi_api.h
index d010577a5..c7ed7cd17 100644
--- a/src/include/ndpi_api.h
+++ b/src/include/ndpi_api.h
@@ -326,6 +326,7 @@ extern "C" {
    * Search and return the protocol guessed that is undetected
    *
    * @par    ndpi_struct  = the detection module
+   * @par    flow         = the flow we're trying to guess, NULL if not available
    * @par    proto        = the l4 protocol number
    * @par    shost        = source address in host byte order
    * @par    sport        = source port number
@@ -335,6 +336,7 @@ extern "C" {
    *
    */
   ndpi_protocol ndpi_guess_undetected_protocol(struct ndpi_detection_module_struct *ndpi_struct,
+					       struct ndpi_flow_struct *flow,
 					       u_int8_t proto,
 					       u_int32_t shost,
 					       u_int16_t sport,
diff --git a/src/include/ndpi_main.h b/src/include/ndpi_main.h
index 3517d45b2..41280fb8e 100644
--- a/src/include/ndpi_main.h
+++ b/src/include/ndpi_main.h
@@ -119,6 +119,7 @@ extern "C" {
   u_int16_t ndpi_get_proto_by_name(struct ndpi_detection_module_struct *ndpi_mod, const char *name);
 
   extern u_int16_t ndpi_guess_protocol_id(struct ndpi_detection_module_struct *ndpi_struct,
+					  struct ndpi_flow_struct *flow,
 					  u_int8_t proto, u_int16_t sport, u_int16_t dport,
 					  u_int8_t *user_defined_proto);
 
diff --git a/src/include/ndpi_protocol_ids.h b/src/include/ndpi_protocol_ids.h
index 0d2712d20..aefb0fce2 100644
--- a/src/include/ndpi_protocol_ids.h
+++ b/src/include/ndpi_protocol_ids.h
@@ -73,7 +73,7 @@ typedef enum {
   NDPI_PROTOCOL_EDONKEY               = 36, /* Tomasz Bujlow <tomasz@skatnet.dk> */
   NDPI_PROTOCOL_BITTORRENT            = 37,
   NDPI_PROTOCOL_SKYPE_CALL_OUT        = 38,
-  NDPI_PROTOCOL_MUSICALLY             = 39,
+  NDPI_PROTOCOL_FREE_39               = 39, /* Free */
   NDPI_PROTOCOL_MEMCACHED             = 40, /* Memcached - Darryl Sokoloski <darryl@egloo.ca> */
   NDPI_PROTOCOL_SMBV23                = 41, /* SMB version 2/3 */  
   NDPI_PROTOCOL_MINING                = 42, /* Bitcoin, Ethereum, ZCash, Monero */
diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h
index 51681e60e..4b69c2b7b 100644
--- a/src/include/ndpi_typedefs.h
+++ b/src/include/ndpi_typedefs.h
@@ -826,7 +826,7 @@ typedef enum {
   ndpi_pref_dns_dissect_response,
   ndpi_pref_direction_detect_disable,
   ndpi_pref_disable_metadata_export,
-  ndpi_pref_enable_category_substring_match,
+  ndpi_pref_enable_category_substring_match
 } ndpi_detection_preference;
   
 /* ntop extensions */
diff --git a/src/lib/ndpi_content_match.c.inc b/src/lib/ndpi_content_match.c.inc
index 71e236d37..4bfdde983 100644
--- a/src/lib/ndpi_content_match.c.inc
+++ b/src/lib/ndpi_content_match.c.inc
@@ -21,7 +21,6 @@
 /* ****************************************************** */
 
 static ndpi_network host_protocol_list[] = {
-
   /*
     SoundCloud
   */
@@ -111,11 +110,14 @@ static ndpi_network host_protocol_list[] = {
     WeChat
     origin AS132203, AS132591, AS45090
   */
-
   { 0xCBCD93AB /* 203.205.147.171/32 */, 32, NDPI_PROTOCOL_WECHAT },
   { 0xCBCD93AD /* 203.205.147.173/32 */, 32, NDPI_PROTOCOL_WECHAT },
   { 0xCBCD97A2 /* 203.205.151.162/32 */, 32, NDPI_PROTOCOL_WECHAT },
-  { 0x67071E25 /* 103.7.30.37/32 */, 32, NDPI_PROTOCOL_WECHAT },
+  { 0x67071E25 /* 103.7.30.37/32     */, 32, NDPI_PROTOCOL_WECHAT },
+  { 0XCBCDB000 /* 203.205.176.0/20   */, 20, NDPI_PROTOCOL_WECHAT },
+  { 0XCBCDC000 /* 203.205.192.0/18   */, 18, NDPI_PROTOCOL_WECHAT },
+  
+
 
   /*
     OpenDNS, LLC
@@ -8327,10 +8329,7 @@ ndpi_protocol_match host_match[] = {
   /* Detected "slack-assets2.s3-us-west-2.amazonaws.com.". Omitted "*amazonaws.com" CDN, but no generic pattern to use on first part */
   { "slack-assets2.s3-", NULL, "slack-assets2\\.s3-",                  "Slack",            NDPI_PROTOCOL_SLACK, NDPI_PROTOCOL_CATEGORY_COLLABORATIVE, NDPI_PROTOCOL_ACCEPTABLE },
 
-  { "zhiliaoapp.com", NULL, "zhiliaoapp" TLD,                          "Musical.ly",       NDPI_PROTOCOL_MUSICALLY, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN },
-  { "muscdn.com", NULL, "muscdn" TLD,                                  "Musical.ly",       NDPI_PROTOCOL_MUSICALLY, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN },
-  { "livelycdn.com", NULL, "livelycdn" TLD,                            "Musical.ly",       NDPI_PROTOCOL_MUSICALLY, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN },
-  { "direct.ly", NULL, "direct\\.ly$",                                 "Musical.ly",       NDPI_PROTOCOL_MUSICALLY, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN },
+  { "wechat.com", NULL, "wechat\\.com" TLD,                            "WeChat",           NDPI_PROTOCOL_WECHAT, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN },
 
   { "github.com", NULL, "github" TLD,                                  "Github",           NDPI_PROTOCOL_GITHUB, NDPI_PROTOCOL_CATEGORY_COLLABORATIVE, NDPI_PROTOCOL_ACCEPTABLE },
   { ".github.com", NULL, "\\.github" TLD,                              "Github",           NDPI_PROTOCOL_GITHUB, NDPI_PROTOCOL_CATEGORY_COLLABORATIVE, NDPI_PROTOCOL_ACCEPTABLE },
diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index 3a6cd84ef..02ad31a5e 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -517,7 +517,6 @@ void ndpi_exclude_protocol(struct ndpi_detection_module_struct *ndpi_struct,
 
 		(*(ndpi_struct->ndpi_debug_printf))(protocol_id, ndpi_struct, NDPI_LOG_DEBUG,
 		 _file, _func, _line, "exclude %s\n",ndpi_get_proto_name(ndpi_struct, protocol_id));
-
 	}
 #endif
 	NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, protocol_id);
@@ -1205,9 +1204,9 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp
 			    no_master, "Teredo", NDPI_PROTOCOL_CATEGORY_NETWORK,
 			    ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
 			    ndpi_build_default_ports(ports_b, 3544, 0, 0, 0, 0) /* UDP */);
-    ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_MUSICALLY,
-			    no_master,
-			    no_master, "Musical.ly", NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK,
+    ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_WECHAT,
+			    no_master, /* wechat.com */
+			    no_master, "WeChat", NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK,
 			    ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
 			    ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
     ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_MEMCACHED,
@@ -1245,7 +1244,12 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp
 			    no_master, "Free", NDPI_PROTOCOL_CATEGORY_CUSTOM_1 /* dummy */,
 			    ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
 			    ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
-    ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_FREE_196,
+    ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_FREE_39,
+			    no_master,
+			    no_master, "Free", NDPI_PROTOCOL_CATEGORY_CUSTOM_1 /* dummy */,
+			    ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
+			    ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
+    ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_FREE_39,
 			    no_master,
 			    no_master, "Free", NDPI_PROTOCOL_CATEGORY_CUSTOM_1 /* dummy */,
 			    ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
@@ -2523,6 +2527,7 @@ static ndpi_default_ports_tree_node_t* ndpi_get_guessed_protocol_id(struct ndpi_
 /* ****************************************************** */
 
 u_int16_t ndpi_guess_protocol_id(struct ndpi_detection_module_struct *ndpi_struct,
+				 struct ndpi_flow_struct *flow,
 				 u_int8_t proto, u_int16_t sport, u_int16_t dport,
 				 u_int8_t *user_defined_proto) {
 
@@ -2531,8 +2536,17 @@ u_int16_t ndpi_guess_protocol_id(struct ndpi_detection_module_struct *ndpi_struc
     ndpi_default_ports_tree_node_t *found = ndpi_get_guessed_protocol_id(ndpi_struct, proto, sport, dport);
 
     if(found != NULL) {
-      *user_defined_proto = found->customUserProto;
-      return(found->proto->protoId);
+      u_int16_t guessed_proto = found->proto->protoId;
+
+      /* We need to check if the guessed protocol isn't excluded by nDPI */
+      if(flow
+	 && (proto == IPPROTO_UDP)
+	 && (NDPI_COMPARE_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, guessed_proto)))
+	return(NDPI_PROTOCOL_UNKNOWN);
+      else {
+	*user_defined_proto = found->customUserProto;
+	return(guessed_proto);
+      }
     }
   } else {
     /* No TCP/UDP */
@@ -3956,18 +3970,17 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st
 	 && (flow->l4.tcp.ssl_stage > 1))
 	flow->guessed_protocol_id = NDPI_PROTOCOL_SSL_NO_CERT;
 
-      guessed_protocol_id = flow->guessed_protocol_id,
-	guessed_host_protocol_id = flow->guessed_host_protocol_id;
+      guessed_protocol_id = flow->guessed_protocol_id, guessed_host_protocol_id = flow->guessed_host_protocol_id;
 
       if((guessed_host_protocol_id != NDPI_PROTOCOL_UNKNOWN)
-	 && (NDPI_ISSET(&flow->excluded_protocol_bitmask, guessed_host_protocol_id)))
-	guessed_host_protocol_id = NDPI_PROTOCOL_UNKNOWN;
-
+	 && ((flow->packet.l4_protocol == IPPROTO_UDP) && NDPI_ISSET(&flow->excluded_protocol_bitmask, guessed_host_protocol_id)))
+	flow->guessed_host_protocol_id = guessed_host_protocol_id = NDPI_PROTOCOL_UNKNOWN;
+      
       /* Ignore guessed protocol if they have been discarded */
       if((guessed_protocol_id != NDPI_PROTOCOL_UNKNOWN)
-	 && (guessed_host_protocol_id == NDPI_PROTOCOL_UNKNOWN)
-	 && (NDPI_ISSET(&flow->excluded_protocol_bitmask, guessed_protocol_id)))
-	guessed_protocol_id = NDPI_PROTOCOL_UNKNOWN;
+	 // && (guessed_host_protocol_id == NDPI_PROTOCOL_UNKNOWN)
+	 && (flow->packet.l4_protocol == IPPROTO_UDP) && NDPI_ISSET(&flow->excluded_protocol_bitmask, guessed_protocol_id))
+	flow->guessed_protocol_id = guessed_protocol_id = NDPI_PROTOCOL_UNKNOWN;
 
       if((guessed_protocol_id != NDPI_PROTOCOL_UNKNOWN)
 	 || (guessed_host_protocol_id != NDPI_PROTOCOL_UNKNOWN)) {
@@ -4394,7 +4407,7 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
     else sport = dport = 0;
 
     /* guess protocol */
-    flow->guessed_protocol_id = (int16_t) ndpi_guess_protocol_id(ndpi_struct, protocol, sport, dport, &user_defined_proto);
+    flow->guessed_protocol_id = (int16_t) ndpi_guess_protocol_id(ndpi_struct, flow, protocol, sport, dport, &user_defined_proto);
     flow->guessed_host_protocol_id = ndpi_guess_host_protocol_id(ndpi_struct, flow);
 
     if(flow->guessed_protocol_id >= (NDPI_MAX_SUPPORTED_PROTOCOLS-1)) {
@@ -5356,6 +5369,7 @@ u_int16_t ndpi_get_lower_proto(ndpi_protocol proto) {
 /* ****************************************************** */
 
 ndpi_protocol ndpi_guess_undetected_protocol(struct ndpi_detection_module_struct *ndpi_struct,
+					     struct ndpi_flow_struct *flow,
 					     u_int8_t proto,
 					     u_int32_t shost /* host byte order */, u_int16_t sport,
 					     u_int32_t dhost /* host byte order */, u_int16_t dport) {
@@ -5365,30 +5379,37 @@ ndpi_protocol ndpi_guess_undetected_protocol(struct ndpi_detection_module_struct
   u_int8_t user_defined_proto;
 
   if((proto == IPPROTO_TCP) || (proto == IPPROTO_UDP)) {
-    rc = ndpi_search_tcp_or_udp_raw(ndpi_struct, NULL, proto,
-				    shost, dhost, sport, dport);
+    rc = ndpi_search_tcp_or_udp_raw(ndpi_struct, NULL, proto, shost, dhost, sport, dport);
 
     if(rc != NDPI_PROTOCOL_UNKNOWN) {
-      ret.app_protocol = rc,
-	ret.master_protocol = ndpi_guess_protocol_id(ndpi_struct, proto, sport,
-						     dport, &user_defined_proto);
-
-      if(ret.app_protocol == ret.master_protocol)
-	ret.master_protocol = NDPI_PROTOCOL_UNKNOWN;
-
-      ret.category = ndpi_get_proto_category(ndpi_struct, ret);
-      return(ret);
+      if(flow && (proto == IPPROTO_UDP) && NDPI_COMPARE_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, rc))
+	;
+      else {
+	ret.app_protocol = rc,
+	  ret.master_protocol = ndpi_guess_protocol_id(ndpi_struct, NULL, proto, sport,
+						       dport, &user_defined_proto);
+	
+	if(ret.app_protocol == ret.master_protocol)
+	  ret.master_protocol = NDPI_PROTOCOL_UNKNOWN;
+	
+	ret.category = ndpi_get_proto_category(ndpi_struct, ret);
+	return(ret);
+      }
     }
 
-    rc = ndpi_guess_protocol_id(ndpi_struct, proto, sport, dport, &user_defined_proto);
+    rc = ndpi_guess_protocol_id(ndpi_struct, NULL, proto, sport, dport, &user_defined_proto);
     if(rc != NDPI_PROTOCOL_UNKNOWN) {
-      ret.app_protocol = rc;
-
-      if(rc == NDPI_PROTOCOL_SSL)
-	goto check_guessed_skype;
+      if(flow && (proto == IPPROTO_UDP) && NDPI_COMPARE_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, rc))
+	;
       else {
-	ret.category = ndpi_get_proto_category(ndpi_struct, ret);
-	return(ret);
+	ret.app_protocol = rc;
+	
+	if(rc == NDPI_PROTOCOL_SSL)
+	  goto check_guessed_skype;
+	else {
+	  ret.category = ndpi_get_proto_category(ndpi_struct, ret);
+	  return(ret);
+	}
       }
     }
 
@@ -5402,10 +5423,10 @@ ndpi_protocol ndpi_guess_undetected_protocol(struct ndpi_detection_module_struct
 	ret.app_protocol = NDPI_PROTOCOL_SKYPE;
     }
   } else
-    ret.app_protocol = ndpi_guess_protocol_id(ndpi_struct, proto, sport,
+    ret.app_protocol = ndpi_guess_protocol_id(ndpi_struct, NULL, proto, sport,
 					      dport, &user_defined_proto);
 
-  ret.category = ndpi_get_proto_category(ndpi_struct, ret);
+  ret.category = ndpi_get_proto_category(ndpi_struct, ret);  
   return(ret);
 }
 
diff --git a/src/lib/protocols/dropbox.c b/src/lib/protocols/dropbox.c
index d981c76b3..39bb96ff2 100644
--- a/src/lib/protocols/dropbox.c
+++ b/src/lib/protocols/dropbox.c
@@ -48,14 +48,24 @@ static void ndpi_check_dropbox(struct ndpi_detection_module_struct *ndpi_struct,
   if(packet->udp != NULL) {
     u_int16_t dropbox_port = htons(DB_LSP_PORT);
 
-    if((packet->udp->source == dropbox_port)
-       && (packet->udp->dest == dropbox_port)) {
-      if(payload_len > 2) {
-	if(strstr((const char *)packet->payload, "\"host_int\"") != NULL) {
-	  
-	  NDPI_LOG_INFO(ndpi_struct, "found dropbox\n");
-	  ndpi_int_dropbox_add_connection(ndpi_struct, flow, 0);
-	  return;
+    if(packet->udp->dest == dropbox_port) {    
+      if(packet->udp->source == dropbox_port) {	
+	if(payload_len > 10) {
+	  if(ndpi_strnstr((const char *)packet->payload, "\"host_int\"", payload_len) != NULL) {
+	    
+	    NDPI_LOG_INFO(ndpi_struct, "found dropbox\n");
+	    ndpi_int_dropbox_add_connection(ndpi_struct, flow, 0);
+	    return;
+	  }
+	}
+      } else {
+	if(payload_len > 10) {
+	  if(ndpi_strnstr((const char *)packet->payload, "Bus17Cmd", payload_len) != NULL) {
+	    
+	    NDPI_LOG_INFO(ndpi_struct, "found dropbox\n");
+	    ndpi_int_dropbox_add_connection(ndpi_struct, flow, 0);
+	    return;
+	  }
 	}
       }
     }
diff --git a/src/lib/protocols/eaq.c b/src/lib/protocols/eaq.c
index 2082b5263..8768cac6f 100644
--- a/src/lib/protocols/eaq.c
+++ b/src/lib/protocols/eaq.c
@@ -53,12 +53,15 @@ void ndpi_search_eaq(struct ndpi_detection_module_struct *ndpi_struct, struct nd
       
     if(packet->udp != NULL) {
       u_int32_t seq = (packet->payload[0] * 1000) + (packet->payload[1] * 100) + (packet->payload[2] * 10) + packet->payload[3];
-      
+
       if(flow->l4.udp.eaq_pkt_id == 0)
         flow->l4.udp.eaq_sequence = seq;
       else {
         if( (flow->l4.udp.eaq_sequence != seq) &&
-	    ((flow->l4.udp.eaq_sequence+1) != seq)) break;
+	    ((flow->l4.udp.eaq_sequence+1) != seq))
+	  break;
+	else
+	  flow->l4.udp.eaq_sequence = seq;
       }
 
       if(++flow->l4.udp.eaq_pkt_id == 4) {
@@ -66,7 +69,8 @@ void ndpi_search_eaq(struct ndpi_detection_module_struct *ndpi_struct, struct nd
         NDPI_LOG_INFO(ndpi_struct, "found eaq\n");
         ndpi_int_eaq_add_connection(ndpi_struct, flow);
         return;
-      }
+      } else
+	return;
     }
   } while(0);
 
diff --git a/src/lib/protocols/skype.c b/src/lib/protocols/skype.c
index e544724a5..8a4eafca4 100644
--- a/src/lib/protocols/skype.c
+++ b/src/lib/protocols/skype.c
@@ -31,13 +31,16 @@ static void ndpi_skype_report_protocol(struct ndpi_detection_module_struct *ndpi
   NDPI_LOG_INFO(ndpi_struct, "found skype\n");
   ndpi_set_detected_protocol(ndpi_struct, flow, proto, NDPI_PROTOCOL_SKYPE);
 }
-    
+
+static int is_port(u_int16_t a, u_int16_t b, u_int16_t c) {
+  return(((a == c) || (b == c)) ? 1 : 0);
+}
+
 static void ndpi_check_skype(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) {
   struct ndpi_packet_struct *packet = &flow->packet;
   // const u_int8_t *packet_payload = packet->payload;
   u_int32_t payload_len = packet->payload_packet_len;
 
-
   if(flow->host_server_name[0] != '\0')
     return;
 
@@ -46,10 +49,15 @@ static void ndpi_check_skype(struct ndpi_detection_module_struct *ndpi_struct, s
     flow->l4.udp.skype_packet_id++;
 
     if(flow->l4.udp.skype_packet_id < 5) {
+      u_int16_t sport = ntohs(packet->udp->source);
       u_int16_t dport = ntohs(packet->udp->dest);
 
       /* skype-to-skype */
-      if(dport != 1119) /* It can be confused with battle.net */ {
+      if(is_port(sport, dport, 1119) /* It can be confused with battle.net */
+	 || is_port(sport, dport, 80) /* No HTTP-like protocols UDP/80 */
+	 ) {
+	;
+      } else {
 	if(((payload_len == 3) && ((packet->payload[2] & 0x0F)== 0x0d)) ||
 	   ((payload_len >= 16)
 	    && (packet->payload[0] != 0x30) /* Avoid invalid SNMP detection */
@@ -57,11 +65,12 @@ static void ndpi_check_skype(struct ndpi_detection_module_struct *ndpi_struct, s
 	  ndpi_skype_report_protocol(ndpi_struct, flow);
 	}
       }
+      
       return;
     }
+    
     NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
     return;
-
     // TCP check
   } else if(packet->tcp != NULL) {
     flow->l4.tcp.skype_packet_id++;