Merge branch 'dev' of https://github.com/ntop/nDPI into dev

7 files changed, 85 insertions, 54 deletions

diff --git a/src/include/ndpi_api.h b/src/include/ndpi_api.h
index e09c91c8e..386d306fd 100644
--- a/src/include/ndpi_api.h
+++ b/src/include/ndpi_api.h
@@ -740,8 +740,11 @@ extern "C" {
 				  char *name, ndpi_protocol_category_t category);
   int ndpi_enable_loaded_categories(struct ndpi_detection_module_struct *ndpi_struct);
   int ndpi_fill_ip_protocol_category(struct ndpi_detection_module_struct *ndpi_struct,
-				 const struct ndpi_iphdr *iph,
+				 u_int32_t saddr,
+				 u_int32_t daddr,
 				 ndpi_protocol *ret);
+  int ndpi_match_custom_category(struct ndpi_detection_module_struct *ndpi_struct,
+				      char *name, unsigned long *id);
   void ndpi_fill_protocol_category(struct ndpi_detection_module_struct *ndpi_struct,
 				   struct ndpi_flow_struct *flow,
 				   ndpi_protocol *ret);
diff --git a/src/include/ndpi_protocol_ids.h b/src/include/ndpi_protocol_ids.h
index 7545de436..1d9c1d428 100644
--- a/src/include/ndpi_protocol_ids.h
+++ b/src/include/ndpi_protocol_ids.h
@@ -85,7 +85,7 @@ typedef enum {
   
   NDPI_PROTOCOL_XBOX                  = 47,
   NDPI_PROTOCOL_QQ                    = 48,
-  NDPI_PROTOCOL_FREE_49               = 49, /* Free */
+  NDPI_PROTOCOL_TIKTOK                = 49,
   NDPI_PROTOCOL_RTSP                  = 50,
   NDPI_PROTOCOL_MAIL_IMAPS            = 51,
   NDPI_PROTOCOL_ICECAST               = 52,
diff --git a/src/lib/ndpi_content_match.c.inc b/src/lib/ndpi_content_match.c.inc
index 8daa8a825..8f7fc2337 100644
--- a/src/lib/ndpi_content_match.c.inc
+++ b/src/lib/ndpi_content_match.c.inc
@@ -8420,6 +8420,8 @@ ndpi_protocol_match host_match[] = {
 
   { "signal.org", NULL, "signal\\.org" TLD,                            "Signal",           NDPI_PROTOCOL_SIGNAL, NDPI_PROTOCOL_CATEGORY_CHAT, NDPI_PROTOCOL_ACCEPTABLE },
   { "whispersystems.org", NULL, "whispersystems\\.org" TLD,            "Signal",           NDPI_PROTOCOL_SIGNAL, NDPI_PROTOCOL_CATEGORY_CHAT, NDPI_PROTOCOL_ACCEPTABLE },
+  { "musical.ly", NULL, "musical\\.ly" TLD,                            "TikTok",           NDPI_PROTOCOL_TIKTOK, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_ACCEPTABLE },
+  { "muscdn.com", NULL, "muscndl\\.com" TLD,                           "TikTok",           NDPI_PROTOCOL_TIKTOK, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_ACCEPTABLE },
   { NULL, NULL, NULL, 0 }
 };
 
diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index 01b25a1b8..936f47dac 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -1198,14 +1198,9 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp
 			    no_master, "SkypeCall", NDPI_PROTOCOL_CATEGORY_VOIP,
 			    ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
 			    ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
-    ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_FREE_49,
+    ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_TIKTOK,
 			    0 /* can_have_a_subprotocol */, no_master,
-			    no_master, "Free_49", NDPI_PROTOCOL_CATEGORY_VOIP,
-			    ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
-			    ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
-    ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_FREE_49,
-			    0 /* can_have_a_subprotocol */, no_master,
-			    no_master, "SkypeCall", NDPI_PROTOCOL_CATEGORY_VOIP,
+			    no_master, "TikTok", NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK,
 			    ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
 			    ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
     ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_TEREDO,
@@ -1991,21 +1986,42 @@ static int ac_match_handler(AC_MATCH_t *m, AC_TEXT_t *txt, AC_REP_t *match) {
 	 m->match_num, m->patterns->astring);
 #endif
 
+  {
+    char *whatfound = strstr(buf, m->patterns->astring);
+
+#ifdef MATCH_DEBUG
+    printf("[NDPI] %s() [searching=%s][pattern=%s][%s][%c]\n", 
+	   __FUNCTION__, buf,  m->patterns->astring, 
+	   whatfound ? whatfound : "<NULL>",
+	   whatfound[-1]);
+#endif
+
+    /*
+      The patch below allows in case of pattern ws.amazon.com
+      to avoid matching aws.amazon.com whereas a.ws.amazon.com
+      has to match
+     */
+    if(whatfound && (whatfound != buf)
+       && (m->patterns->astring[0] != '.')  /* The searched patter does not start with . */
+       && strchr(m->patterns->astring, '.') /* The matched pattern has a . (e.g. numeric or sym IPs) */
+       && (whatfound[-1] != '.') 
+       )
+      return(0);
+  }
+
   /*
     Return 1 for stopping to the first match.
     We might consider searching for the more
     specific match, paying more cpu cycles.
   */
-
   memcpy(match, &m->patterns[0].rep, sizeof(AC_REP_t));
 
   if(((buf_len >= min_len) && (strncmp(&buf[buf_len-min_len], m->patterns->astring, min_len) == 0))
      || (strncmp(buf, m->patterns->astring, min_len) == 0) /* begins with */
-     )
-    {
+     ) {
 #ifdef MATCH_DEBUG
       printf("Found match [%s][%s] [len: %u][proto_id: %u]\n",
-	     buf, m->patterns->astring, min_len, *matching_protocol_id);
+	     buf, m->patterns->astring, min_len , *matching_protocol_id);
 #endif
     return(1); /* If the pattern found matches the string at the beginning we stop here */
   } else
@@ -2301,8 +2317,8 @@ int ndpi_match_string(void *_automa, char *string_to_match) {
 int ndpi_match_string_id(void *_automa, char *string_to_match, unsigned long *id) {
   AC_TEXT_t ac_input_text;
   AC_AUTOMATA_t *automa = (AC_AUTOMATA_t*)_automa;
-  AC_REP_t match = { NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, NDPI_PROTOCOL_UNRATED };
-  
+  AC_REP_t match = { NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, NDPI_PROTOCOL_UNRATED }; 
+
   *id = -1;
   if((automa == NULL)
      || (string_to_match == NULL)
@@ -2338,11 +2354,14 @@ static int hyperscanCustomEventHandler(unsigned int id,
 
 /* *********************************************** */
 
-static int ndpi_match_custom_category(struct ndpi_detection_module_struct *ndpi_struct,
+int ndpi_match_custom_category(struct ndpi_detection_module_struct *ndpi_struct,
 				      char *name, unsigned long *id) {
-  /* printf("[NDPI] %s(%s)\n", __FUNCTION__, name); */
+#ifdef DEBUG
+  printf("[NDPI] %s(%s) [enable_category_substring_match: %u]\n", 
+	 __FUNCTION__, name, ndpi_struct->enable_category_substring_match);
+#endif
 
-  if(!ndpi_struct->enable_category_substring_match) {
+  if(ndpi_struct->enable_category_substring_match == 0) {
     if(ndpi_struct->custom_categories.hostnames_hash == NULL)
       return(-1);
     else {
@@ -2731,7 +2750,8 @@ int ndpi_handle_rule(struct ndpi_detection_module_struct *ndpi_mod,
       if(sscanf(value, "%u-%u", (u_int32_t *)&range.port_low, (u_int32_t *)&range.port_high) != 2)
 	range.port_low = range.port_high = atoi(&elem[4]);
       if(do_add)
-	addDefaultPort(ndpi_mod, &range, def, 1 /* Custom user proto */, is_tcp ? &ndpi_mod->tcpRoot : &ndpi_mod->udpRoot, __FUNCTION__,__LINE__);
+	addDefaultPort(ndpi_mod, &range, def, 1 /* Custom user proto */,
+		       is_tcp ? &ndpi_mod->tcpRoot : &ndpi_mod->udpRoot, __FUNCTION__,__LINE__);
       else
 	removeDefaultPort(&range, def, is_tcp ? &ndpi_mod->tcpRoot : &ndpi_mod->udpRoot);
     } else if(is_ip) {
@@ -4358,19 +4378,20 @@ int ndpi_enable_loaded_categories(struct ndpi_detection_module_struct *ndpi_str)
 /* ********************************************************************************* */
 
 int ndpi_fill_ip_protocol_category(struct ndpi_detection_module_struct *ndpi_struct,
-				 const struct ndpi_iphdr *iph,
+				 u_int32_t saddr,
+				 u_int32_t daddr,
 				 ndpi_protocol *ret) {
   if(ndpi_struct->custom_categories.categories_loaded) {
       prefix_t prefix;
       patricia_node_t *node;
-      
+
       /* Make sure all in network byte order otherwise compares wont work */
-      fill_prefix_v4(&prefix, (struct in_addr *)&iph->saddr,
+      fill_prefix_v4(&prefix, (struct in_addr *)&saddr,
 		     32, ((patricia_tree_t*)ndpi_struct->protocols_ptree)->maxbits);
       node = ndpi_patricia_search_best(ndpi_struct->custom_categories.ipAddresses, &prefix);
 
       if(!node) {
-	fill_prefix_v4(&prefix, (struct in_addr *)&iph->daddr,
+	fill_prefix_v4(&prefix, (struct in_addr *)&daddr,
 		       32, ((patricia_tree_t*)ndpi_struct->protocols_ptree)->maxbits);
 	node = ndpi_patricia_search_best(ndpi_struct->custom_categories.ipAddresses, &prefix);
       }
@@ -4385,12 +4406,14 @@ int ndpi_fill_ip_protocol_category(struct ndpi_detection_module_struct *ndpi_str
   return 0;
 }
 
+/* ********************************************************************************* */
+
 void ndpi_fill_protocol_category(struct ndpi_detection_module_struct *ndpi_struct,
 				 struct ndpi_flow_struct *flow,
 				 ndpi_protocol *ret) {
   if(ndpi_struct->custom_categories.categories_loaded) {
     if(flow->packet.iph) {
-      if(ndpi_fill_ip_protocol_category(ndpi_struct, flow->packet.iph, ret)) {
+      if(ndpi_fill_ip_protocol_category(ndpi_struct, flow->packet.iph->saddr, flow->packet.iph->daddr, ret)) {
         flow->category = ret->category;
         return;
       }
@@ -4435,7 +4458,7 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
 
   if(ndpi_struct->ndpi_log_level >= NDPI_LOG_TRACE)
   	NDPI_LOG(flow ? flow->detected_protocol_stack[0]:NDPI_PROTOCOL_UNKNOWN,
-		  ndpi_struct, NDPI_LOG_TRACE, "START packet processing\n");
+		 ndpi_struct, NDPI_LOG_TRACE, "START packet processing\n");
   if(flow == NULL)
     return(ret);
 
diff --git a/src/lib/protocols/dns.c b/src/lib/protocols/dns.c
index f77040020..e282eb4d3 100644
--- a/src/lib/protocols/dns.c
+++ b/src/lib/protocols/dns.c
@@ -63,7 +63,7 @@ static u_int getNameLength(u_int i, const u_int8_t *payload, u_int payloadLen) {
 /* *********************************************** */
 
 void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) {
-  int x;
+  int x, payload_offset;
   u_int8_t is_query;
   u_int16_t s_port = 0, d_port = 0;
 
@@ -72,16 +72,18 @@ void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, struct nd
   if(flow->packet.udp != NULL) {
     s_port = ntohs(flow->packet.udp->source);
     d_port = ntohs(flow->packet.udp->dest);
-    x = 0;
+    payload_offset = 0;
   } else if(flow->packet.tcp != NULL) /* pkt size > 512 bytes */ {
     s_port = ntohs(flow->packet.tcp->source);
     d_port = ntohs(flow->packet.tcp->dest);
-    x = 2;
+    payload_offset = 2;
   } else {
     NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
     return;
   }
 
+  x = payload_offset;
+
   if((s_port == 53 || d_port == 53 || d_port == 5355)
      && (flow->packet.payload_packet_len > sizeof(struct ndpi_dns_packet_header)+x)) {
     struct ndpi_dns_packet_header dns_header;
@@ -175,7 +177,8 @@ void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, struct nd
 	      }
 	    }
 	  }
-	}
+	} else
+	  invalid = 1;
       }
 
       if(invalid) {
@@ -184,8 +187,7 @@ void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, struct nd
       }
 
       /* extract host name server */
-      int j = 0, max_len = sizeof(flow->host_server_name)-1, off = sizeof(struct ndpi_dns_packet_header) + 1;
-
+      int j = 0, max_len = sizeof(flow->host_server_name)-1, off = sizeof(struct ndpi_dns_packet_header) + 1 + payload_offset;
       while(off < flow->packet.payload_packet_len && flow->packet.payload[off] != '\0') {
 	flow->host_server_name[j] = flow->packet.payload[off];
 	if(j < max_len) {
diff --git a/src/lib/protocols/mdns_proto.c b/src/lib/protocols/mdns_proto.c
index 00c7c8748..75eab720b 100644
--- a/src/lib/protocols/mdns_proto.c
+++ b/src/lib/protocols/mdns_proto.c
@@ -63,17 +63,15 @@ static int ndpi_int_check_mdns_payload(struct ndpi_detection_module_struct
   struct ndpi_packet_struct *packet = &flow->packet;
   struct mdns_header *h = (struct mdns_header*)packet->payload;
   u_int16_t questions = ntohs(h->questions), answers = ntohs(h->answers);
+
+  if((questions > NDPI_MAX_MDNS_REQUESTS)
+     || (answers > NDPI_MAX_MDNS_REQUESTS))
+    return(0);
   
-  if(((packet->payload[2] & 0x80) == 0)
-     && (questions <= NDPI_MAX_MDNS_REQUESTS)
-     && (answers <= NDPI_MAX_MDNS_REQUESTS)) {	
+  if((packet->payload[2] & 0x80) == 0) {
     NDPI_LOG_INFO(ndpi_struct, "found MDNS with question query\n");
     return 1;    
-  }
-  else if(((packet->payload[2] & 0x80) != 0)
-	  && (questions == 0)
-	  && (answers <= NDPI_MAX_MDNS_REQUESTS)
-	  && (answers != 0)) {
+  } else if((packet->payload[2] & 0x80) != 0) {
     char answer[256];
     int i, j, len;
 
@@ -100,8 +98,6 @@ static int ndpi_int_check_mdns_payload(struct ndpi_detection_module_struct
 void ndpi_search_mdns(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow)
 {
   struct ndpi_packet_struct *packet = &flow->packet;
-  u_int16_t dport;
-  
   NDPI_LOG_DBG(ndpi_struct, "search MDNS\n");
 
   /**
@@ -111,15 +107,13 @@ void ndpi_search_mdns(struct ndpi_detection_module_struct *ndpi_struct, struct n
   /* check if UDP packet */
   if(packet->udp != NULL) {   
     /* read destination port */
-    dport = ntohs(packet->udp->dest);
+    u_int16_t sport = ntohs(packet->udp->source);
+    u_int16_t dport = ntohs(packet->udp->dest);
 
     /* check standard MDNS ON port 5353 */
-    if(dport == 5353 && packet->payload_packet_len >= 12) {
-      /* mdns protocol must have destination address 224.0.0.251 */
-	    if(packet->iph != NULL /* && ntohl(packet->iph->daddr) == 0xe00000fb */) {
-
-	NDPI_LOG_INFO(ndpi_struct, "found MDNS with destination address 224.0.0.251 (=0xe00000fb)\n");
-	
+    if(((dport == 5353) || (sport == 5353))
+       && (packet->payload_packet_len >= 12)) {
+      if(packet->iph != NULL) {
 	if(ndpi_int_check_mdns_payload(ndpi_struct, flow) == 1) {
 	  ndpi_int_mdns_add_connection(ndpi_struct, flow);
 	  return;
@@ -141,6 +135,7 @@ void ndpi_search_mdns(struct ndpi_detection_module_struct *ndpi_struct, struct n
 #endif
     }
   }
+  
   NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
 }
 
diff --git a/src/lib/protocols/mining.c b/src/lib/protocols/mining.c
index ec094e7d3..b4361e270 100644
--- a/src/lib/protocols/mining.c
+++ b/src/lib/protocols/mining.c
@@ -44,9 +44,12 @@ void ndpi_search_mining_tcp(struct ndpi_detection_module_struct *ndpi_struct,
       if((*to_match == magic) || (*to_match == magic1)) {
 	ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_MINING, NDPI_PROTOCOL_UNKNOWN);
       }
-    } if(ndpi_strnstr((const char *)packet->payload, "\"eth1.0\"", packet->payload_packet_len)
-	 || ndpi_strnstr((const char *)packet->payload, "\"worker\"", packet->payload_packet_len)
-	 || ndpi_strnstr((const char *)packet->payload, "\"id\"", packet->payload_packet_len)) {
+    } if(ndpi_strnstr((const char *)packet->payload, "{", packet->payload_packet_len)
+	 && (
+	   ndpi_strnstr((const char *)packet->payload, "\"eth1.0\"", packet->payload_packet_len)
+	   || ndpi_strnstr((const char *)packet->payload, "\"worker\":", packet->payload_packet_len)
+	   /* || ndpi_strnstr((const char *)packet->payload, "\"id\":", packet->payload_packet_len) - Removed as too generic */
+	   )) {
       /*
 	Ethereum
 	
@@ -55,9 +58,12 @@ void ndpi_search_mining_tcp(struct ndpi_detection_module_struct *ndpi_struct,
 	{"worker": "", "jsonrpc": "2.0", "params": [], "id": 3, "method": "eth_getWork"}
       */
       ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_MINING, NDPI_PROTOCOL_UNKNOWN);
-    } else if(ndpi_strnstr((const char *)packet->payload, "\"method\"", packet->payload_packet_len)
-	      || ndpi_strnstr((const char *)packet->payload, "\"blob\"", packet->payload_packet_len)
-	      || ndpi_strnstr((const char *)packet->payload, "\"id\"", packet->payload_packet_len)) {
+    } else if(ndpi_strnstr((const char *)packet->payload, "{", packet->payload_packet_len)
+	      && (ndpi_strnstr((const char *)packet->payload, "\"method\":", packet->payload_packet_len)
+		  || ndpi_strnstr((const char *)packet->payload, "\"blob\":", packet->payload_packet_len)
+		  /* || ndpi_strnstr((const char *)packet->payload, "\"id\":", packet->payload_packet_len) - Removed as too generic */
+		)
+      ) {
       /*
 	ZCash