Improve/add several protocols (#1383)

Improve Microsoft, GMail, Likee, Whatsapp, DisneyPlus and Tiktok
detection.
Add Vimeo, Fuze, Alibaba and Firebase Crashlytics detection.

Try to differentiate between Messenger/Signal standard flows (i.e chat)
and their VOIP (video)calls (like we already do for Whatsapp and
Snapchat).

Add a partial list of some ADS/Tracking stuff.

Fix Cassandra, Radius and GTP false positives.
Fix DNS, Syslog and SIP false negatives.

Improve GTP (sub)classification: differentiate among GTP-U, GTP_C and
GTP_PRIME.

Fix 3 LGTM warnings.

8 files changed, 75 insertions, 59 deletions

diff --git a/src/lib/protocols/cassandra.c b/src/lib/protocols/cassandra.c
index f7bbccfbc..33ac1f72a 100644
--- a/src/lib/protocols/cassandra.c
+++ b/src/lib/protocols/cassandra.c
@@ -100,6 +100,11 @@ static bool ndpi_check_valid_cassandra_opcode(uint8_t opcode)
   return false;
 }
 
+static bool ndpi_check_valid_cassandra_flags(uint8_t flags)
+{
+  return (flags & 0xF0) == 0;
+}
+
 void ndpi_search_cassandra(struct ndpi_detection_module_struct *ndpi_struct,
                            struct ndpi_flow_struct *flow)
 {
@@ -108,6 +113,7 @@ void ndpi_search_cassandra(struct ndpi_detection_module_struct *ndpi_struct,
   if (packet->tcp) {
     if (packet->payload_packet_len >= CASSANDRA_HEADER_LEN &&
         ndpi_check_valid_cassandra_version(get_u_int8_t(packet->payload, 0)) &&
+        ndpi_check_valid_cassandra_flags(get_u_int8_t(packet->payload, 1)) &&
         ndpi_check_valid_cassandra_opcode(get_u_int8_t(packet->payload, 4)) &&
         get_u_int32_t(packet->payload, 5) <= CASSANDRA_MAX_BODY_SIZE &&
         get_u_int32_t(packet->payload, 5) >= (uint32_t) (packet->payload_packet_len - CASSANDRA_HEADER_LEN)) {
diff --git a/src/lib/protocols/dns.c b/src/lib/protocols/dns.c
index f32143320..c57d30ec0 100644
--- a/src/lib/protocols/dns.c
+++ b/src/lib/protocols/dns.c
@@ -206,10 +206,11 @@ static int search_valid_dns(struct ndpi_detection_module_struct *ndpi_struct,
 
   if(*is_query) {
     /* DNS Request */
-    if((dns_header->num_queries > 0) && (dns_header->num_queries <= NDPI_MAX_DNS_REQUESTS)
+    if((dns_header->num_queries <= NDPI_MAX_DNS_REQUESTS)
        //       && (dns_header->num_answers == 0)
        && (((dns_header->flags & 0x2800) == 0x2800 /* Dynamic DNS Update */)
 	   || ((dns_header->flags & 0xFCF0) == 0x00) /* Standard Query */
+	   || ((dns_header->flags & 0xFCFF) == 0x0800) /* Inverse query */
 	   || ((dns_header->num_answers == 0) && (dns_header->authority_rrs == 0)))) {
       /* This is a good query */
       while(x+2 < packet->payload_packet_len) {
diff --git a/src/lib/protocols/gtp.c b/src/lib/protocols/gtp.c
index 956ebe355..f94138baf 100644
--- a/src/lib/protocols/gtp.c
+++ b/src/lib/protocols/gtp.c
@@ -82,28 +82,33 @@ static void ndpi_check_gtp(struct ndpi_detection_module_struct *ndpi_struct, str
          (payload_len >= HEADER_LEN_GTP_U) &&
          (message_len <= (payload_len - HEADER_LEN_GTP_U))) {
         NDPI_LOG_INFO(ndpi_struct, "found gtp-u\n");
-        ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_GTP, NDPI_PROTOCOL_UNKNOWN);
+        ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_GTP_U, NDPI_PROTOCOL_GTP);
         return;
       }
     }
     if((packet->udp->source == gtp_c) || (packet->udp->dest == gtp_c)) {
       if(((version == 1) &&
           (payload_len >= HEADER_LEN_GTP_C_V1) &&
-          (message_len == (payload_len - HEADER_LEN_GTP_C_V1))) ||
+          (message_len == (payload_len - HEADER_LEN_GTP_C_V1)) &&
+          (message_len >= 4 * (!!(gtp->flags & 0x07))) &&
+          (gtp->message_type > 0 && gtp->message_type <= 129)) || /* Loose check based on TS 29.060 7.1 */
          ((version == 2) &&
           /* payload_len is always valid, because HEADER_LEN_GTP_C_V2 == sizeof(struct gtp_header_generic) */
           (message_len <= (payload_len - HEADER_LEN_GTP_C_V2)))) {
         NDPI_LOG_INFO(ndpi_struct, "found gtp-c\n");
-        ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_GTP, NDPI_PROTOCOL_UNKNOWN);
+        ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_GTP_C, NDPI_PROTOCOL_GTP);
         return;
       }
     }
     if((packet->udp->source == gtp_prime) || (packet->udp->dest == gtp_prime)) {
       if((pt == 0) &&
+         ((gtp->flags & 0x0E) >> 1 == 0x7) && /* Spare bits */
          (payload_len >= HEADER_LEN_GTP_PRIME) &&
-	 (message_len <= (payload_len - HEADER_LEN_GTP_PRIME))) {
+         (message_len <= (payload_len - HEADER_LEN_GTP_PRIME)) &&
+         ((gtp->message_type > 0 && gtp->message_type <= 7) || /* Check based on TS 32.295 6.2.1 */
+          gtp->message_type == 240 || gtp->message_type == 241)) {
         NDPI_LOG_INFO(ndpi_struct, "found gtp-prime\n");
-        ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_GTP, NDPI_PROTOCOL_UNKNOWN);
+        ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_GTP_PRIME, NDPI_PROTOCOL_GTP);
         return;
       }
     }
diff --git a/src/lib/protocols/radius.c b/src/lib/protocols/radius.c
index 97806c68e..6829047e7 100644
--- a/src/lib/protocols/radius.c
+++ b/src/lib/protocols/radius.c
@@ -24,6 +24,10 @@
 
 #include "ndpi_api.h"
 
+#define RADIUS_PORT 			1812
+#define RADIUS_PORT_ACC 		1813
+#define RADIUS_PORT_ACC_ALTERNATIVE 	18013
+
 
 struct radius_header {
   u_int8_t code;
@@ -37,7 +41,10 @@ static void ndpi_check_radius(struct ndpi_detection_module_struct *ndpi_struct,
   // const u_int8_t *packet_payload = packet->payload;
   u_int32_t payload_len = packet->payload_packet_len;
 
-  if(packet->udp != NULL) {
+  if(packet->udp != NULL &&
+     (packet->udp->dest == htons(RADIUS_PORT) || packet->udp->source == htons(RADIUS_PORT) ||
+      packet->udp->dest == htons(RADIUS_PORT_ACC) || packet->udp->source == htons(RADIUS_PORT_ACC) ||
+      packet->udp->dest == htons(RADIUS_PORT_ACC_ALTERNATIVE) || packet->udp->source == htons(RADIUS_PORT_ACC_ALTERNATIVE))) {
     struct radius_header *h = (struct radius_header*)packet->payload;
     /* RFC2865: The minimum length is 20 and maximum length is 4096. */
     if((payload_len < 20) || (payload_len > 4096)) {
diff --git a/src/lib/protocols/sip.c b/src/lib/protocols/sip.c
index 6c159afdd..bd9fb03d2 100644
--- a/src/lib/protocols/sip.c
+++ b/src/lib/protocols/sip.c
@@ -153,6 +153,20 @@ void ndpi_search_sip_handshake(struct ndpi_detection_module_struct
       ndpi_int_sip_add_connection(ndpi_struct, flow, 0);
       return;
     }
+
+    if((memcmp(packet_payload, "REFER ", 6) == 0 || memcmp(packet_payload, "refer ", 6) == 0)
+       && (memcmp(&packet_payload[6], "SIP:", 4) == 0 || memcmp(&packet_payload[6], "sip:", 4) == 0)) {
+      NDPI_LOG_INFO(ndpi_struct, "found sip REFER\n");
+      ndpi_int_sip_add_connection(ndpi_struct, flow, 0);
+      return;
+    }
+
+    if((memcmp(packet_payload, "PRACK ", 6) == 0 || memcmp(packet_payload, "prack ", 6) == 0)
+       && (memcmp(&packet_payload[6], "SIP:", 4) == 0 || memcmp(&packet_payload[6], "sip:", 4) == 0)) {
+      NDPI_LOG_INFO(ndpi_struct, "found sip REFER\n");
+      ndpi_int_sip_add_connection(ndpi_struct, flow, 0);
+      return;
+    }
   }
 
   /* add bitmask for tcp only, some stupid udp programs
diff --git a/src/lib/protocols/stun.c b/src/lib/protocols/stun.c
index eb1bc4292..8a387d8d2 100644
--- a/src/lib/protocols/stun.c
+++ b/src/lib/protocols/stun.c
@@ -264,6 +264,8 @@ static ndpi_int_stun_t ndpi_int_check_stun(struct ndpi_detection_module_struct *
 
     if(!msg_len && flow->guessed_host_protocol_id == NDPI_PROTOCOL_GOOGLE)
       flow->guessed_host_protocol_id = NDPI_PROTOCOL_HANGOUT_DUO;
+    else if(flow->guessed_host_protocol_id == NDPI_PROTOCOL_FACEBOOK)
+      flow->guessed_host_protocol_id = NDPI_PROTOCOL_FACEBOOK_VOIP;
     else
       flow->guessed_protocol_id = NDPI_PROTOCOL_STUN;
 
@@ -272,6 +274,10 @@ static ndpi_int_stun_t ndpi_int_check_stun(struct ndpi_detection_module_struct *
       return(NDPI_IS_NOT_STUN); /* This to keep analyzing STUN instead of giving up */
     }
   }
+  if(msg_type == 0x03 /* Allocate Request */) {
+    if(flow->guessed_host_protocol_id == NDPI_PROTOCOL_FACEBOOK)
+      flow->guessed_host_protocol_id = NDPI_PROTOCOL_FACEBOOK_VOIP;
+  }
 
   if(!msg_len && flow->guessed_host_protocol_id == NDPI_PROTOCOL_UNKNOWN) {
     NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
@@ -294,7 +300,8 @@ static ndpi_int_stun_t ndpi_int_check_stun(struct ndpi_detection_module_struct *
   else {
     switch(flow->guessed_protocol_id) {
     case NDPI_PROTOCOL_HANGOUT_DUO:
-    case NDPI_PROTOCOL_MESSENGER:
+    case NDPI_PROTOCOL_FACEBOOK_VOIP:
+    case NDPI_PROTOCOL_SIGNAL_VOIP:
     case NDPI_PROTOCOL_WHATSAPP_CALL:
       /* Don't overwrite the protocol with sub-STUN protocols */
       break;
@@ -360,11 +367,12 @@ static ndpi_int_stun_t ndpi_int_check_stun(struct ndpi_detection_module_struct *
 	      if(strstr(flow->host_server_name, "google.com") != NULL) {
                 flow->guessed_host_protocol_id = NDPI_PROTOCOL_HANGOUT_DUO;
                 return(NDPI_IS_STUN);
-	      } else if(strstr(flow->host_server_name, "whispersystems.org") != NULL) {
-		flow->guessed_host_protocol_id = NDPI_PROTOCOL_SIGNAL;
+	      } else if(strstr(flow->host_server_name, "whispersystems.org") != NULL ||
+	                (strstr(flow->host_server_name, "signal.org") != NULL)) {
+		flow->guessed_host_protocol_id = NDPI_PROTOCOL_SIGNAL_VOIP;
 		return(NDPI_IS_STUN);
 	      } else if(strstr(flow->host_server_name, "facebook") != NULL) {
-		flow->guessed_host_protocol_id = NDPI_PROTOCOL_MESSENGER;
+		flow->guessed_host_protocol_id = NDPI_PROTOCOL_FACEBOOK_VOIP;
 		return(NDPI_IS_STUN);
 	      }
 	    }
@@ -374,7 +382,7 @@ static ndpi_int_stun_t ndpi_int_check_stun(struct ndpi_detection_module_struct *
         case 0xC057: /* Messeger */
           if(msg_type == 0x0001) {
             if((msg_len == 100) || (msg_len == 104)) {
-              flow->guessed_host_protocol_id = NDPI_PROTOCOL_MESSENGER;
+              flow->guessed_host_protocol_id = NDPI_PROTOCOL_FACEBOOK_VOIP;
               return(NDPI_IS_STUN);
             } else if(msg_len == 76) {
 #if 0
@@ -473,7 +481,7 @@ static ndpi_int_stun_t ndpi_int_check_stun(struct ndpi_detection_module_struct *
 
   if(packet->iph) { /* TODO: ipv6 */
     if(is_messenger_ip_address(ntohl(packet->iph->saddr)) || is_messenger_ip_address(ntohl(packet->iph->daddr)))
-      flow->guessed_host_protocol_id = NDPI_PROTOCOL_MESSENGER;
+      flow->guessed_host_protocol_id = NDPI_PROTOCOL_FACEBOOK_VOIP;
     else if(is_google_ip_address(ntohl(packet->iph->saddr)) || is_google_ip_address(ntohl(packet->iph->daddr)))
       flow->guessed_host_protocol_id = NDPI_PROTOCOL_HANGOUT_DUO;
   }
diff --git a/src/lib/protocols/syslog.c b/src/lib/protocols/syslog.c
index 45b005030..a98476a11 100644
--- a/src/lib/protocols/syslog.c
+++ b/src/lib/protocols/syslog.c
@@ -42,7 +42,9 @@ void ndpi_search_syslog(struct ndpi_detection_module_struct
 
   NDPI_LOG_DBG(ndpi_struct, "search syslog\n");
 
-  if (packet->payload_packet_len > 20 && packet->payload_packet_len <= 1024 && packet->payload[0] == '<') {
+  if (packet->payload_packet_len > 20 && packet->payload[0] == '<') {
+    int j;
+
     NDPI_LOG_DBG2(ndpi_struct, "checked len>20 and <1024 and first symbol=<\n");
 
     for (i = 1; i <= 3; i++) {
@@ -55,7 +57,7 @@ void ndpi_search_syslog(struct ndpi_detection_module_struct
 
     if (packet->payload[i++] != '>') {
       NDPI_LOG_DBG(ndpi_struct, "excluded, there is no > following the number\n");
-      NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_PROTOCOL_SYSLOG);
+      NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
       return;
     } else {
       NDPI_LOG_DBG2(ndpi_struct, "a > following the number\n");
@@ -68,51 +70,23 @@ void ndpi_search_syslog(struct ndpi_detection_module_struct
       NDPI_LOG_DBG2(ndpi_struct, "no blank following the >: do nothing\n");
     }
 
-    /* check for "last message repeated" */
-    if (i + sizeof("last message") - 1 <= packet->payload_packet_len &&
-	memcmp(packet->payload + i, "last message", sizeof("last message") - 1) == 0) {
-
-      NDPI_LOG_INFO(ndpi_struct, "found syslog by 'last message' string\n");
-
-      ndpi_int_syslog_add_connection(ndpi_struct, flow);
-
-      return;
-    } else if (i + sizeof("snort: ") - 1 <= packet->payload_packet_len &&
-	       memcmp(packet->payload + i, "snort: ", sizeof("snort: ") - 1) == 0) {
-
-      /* snort events */
-
-      NDPI_LOG_INFO(ndpi_struct, "found syslog by 'snort: ' string\n");
-
-      ndpi_int_syslog_add_connection(ndpi_struct, flow);
-
-      return;
+    /* Even if there are 2 RFCs (3164, 5424), syslog format after "<NUMBER>" is
+       not standard. The only common pattern seems to be that the entire
+       payload is made by printable characters */
+    /* TODO: check only the first N bytes to avoid touching the entire payload? */
+    for (j = 0; j < packet->payload_packet_len - i; j++) {
+      if (!(ndpi_isprint(packet->payload[i + j]) ||
+            ndpi_isspace(packet->payload[i + j]))) {
+        NDPI_LOG_DBG2(ndpi_struct, "no printable char 0x%x [i/j %d/%d]\n",
+                      packet->payload[i + j], i, j);
+        NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
+        return;
+      }
     }
 
-    if (memcmp(&packet->payload[i], "Jan", 3) != 0
-	&& memcmp(&packet->payload[i], "Feb", 3) != 0
-	&& memcmp(&packet->payload[i], "Mar", 3) != 0
-	&& memcmp(&packet->payload[i], "Apr", 3) != 0
-	&& memcmp(&packet->payload[i], "May", 3) != 0
-	&& memcmp(&packet->payload[i], "Jun", 3) != 0
-	&& memcmp(&packet->payload[i], "Jul", 3) != 0
-	&& memcmp(&packet->payload[i], "Aug", 3) != 0
-	&& memcmp(&packet->payload[i], "Sep", 3) != 0
-	&& memcmp(&packet->payload[i], "Oct", 3) != 0
-	&& memcmp(&packet->payload[i], "Nov", 3) != 0 && memcmp(&packet->payload[i], "Dec", 3) != 0) {
-
-      NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
-
-      return;
-
-    } else {
-
-      NDPI_LOG_INFO(ndpi_struct, "found syslog\n");
-
-      ndpi_int_syslog_add_connection(ndpi_struct, flow);
-
-      return;
-    }
+    NDPI_LOG_INFO(ndpi_struct, "found syslog\n");
+    ndpi_int_syslog_add_connection(ndpi_struct, flow);
+    return;
   }
   NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
 }
diff --git a/src/lib/protocols/tls.c b/src/lib/protocols/tls.c
index 34af15378..87553de87 100644
--- a/src/lib/protocols/tls.c
+++ b/src/lib/protocols/tls.c
@@ -2115,7 +2115,8 @@ int processClientServerHello(struct ndpi_detection_module_struct *ndpi_struct,
 			flow->protos.tls_quic.encrypted_sni.esni = (char*)ndpi_malloc(e_sni_len*2+1);
 
 			if(flow->protos.tls_quic.encrypted_sni.esni) {
-			  u_int16_t i, off;
+			  u_int16_t off;
+			  int i;
 
 			  for(i=e_offset, off=0; i<(e_offset+e_sni_len); i++) {
 			    int rc = sprintf(&flow->protos.tls_quic.encrypted_sni.esni[off], "%02X", packet->payload[i] & 0XFF);