Extend protocols support (#1422)

Add detection of AccuWeather site/app and Google Classroom.
Improve detection of Azure, Zattoo, Whatsapp, MQTT and LDAP.

Fix some RX false positives.

Fix some "Uncommon TLS ALPN"-risk false positives.

Fix "confidence" value for some Zoom/Torrent classifications.

Minor fix in Lua script for Wireshark extcap.

Update .gitignore file.

Let GitHub correctly detect the language type of *.inc files.

Zattoo example has been provided by @subhajit-cdot in #1148.

7 files changed, 74 insertions, 23 deletions

diff --git a/src/lib/ndpi_content_match.c.inc b/src/lib/ndpi_content_match.c.inc
index 3e4198168..2ac3a4780 100644
--- a/src/lib/ndpi_content_match.c.inc
+++ b/src/lib/ndpi_content_match.c.inc
@@ -9783,6 +9783,7 @@ static ndpi_protocol_match host_match[] =
    { ".azureedge.us",                                         "Azure",        NDPI_PROTOCOL_MICROSOFT_AZURE, NDPI_PROTOCOL_CATEGORY_CLOUD, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL },
    { ".azurefd.us",                                           "Azure",        NDPI_PROTOCOL_MICROSOFT_AZURE, NDPI_PROTOCOL_CATEGORY_CLOUD, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL },
    { ".azure-automation.net",                                 "Azure",        NDPI_PROTOCOL_MICROSOFT_AZURE, NDPI_PROTOCOL_CATEGORY_NETWORK, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL },
+   { ".azureedge.net",                                        "Azure",        NDPI_PROTOCOL_MICROSOFT_AZURE, NDPI_PROTOCOL_CATEGORY_NETWORK, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL },
    
    { "teams.microsoft.com",              "Teams",         NDPI_PROTOCOL_MSTEAMS, NDPI_PROTOCOL_CATEGORY_COLLABORATIVE, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL },
    { "teams.microsoft.us",               "Teams",         NDPI_PROTOCOL_MSTEAMS, NDPI_PROTOCOL_CATEGORY_COLLABORATIVE, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL },
@@ -10091,6 +10092,16 @@ static ndpi_protocol_match host_match[] =
    { "badoocdn.com",                  "Badoo", NDPI_PROTOCOL_BADOO, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_DEFAULT_LEVEL },
    { ".badoo.app",                    "Badoo", NDPI_PROTOCOL_BADOO, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_DEFAULT_LEVEL },
 
+   { "accuweather.com",               "AccuWeather", NDPI_PROTOCOL_ACCUWEATHER, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_DEFAULT_LEVEL },
+   { ".accuweather.com",              "AccuWeather", NDPI_PROTOCOL_ACCUWEATHER, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_DEFAULT_LEVEL },
+
+   { "zattoo.com",                       "Zattoo", NDPI_PROTOCOL_ZATTOO, NDPI_PROTOCOL_CATEGORY_VIDEO, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_DEFAULT_LEVEL },
+   { ".zattoo.com",                      "Zattoo", NDPI_PROTOCOL_ZATTOO, NDPI_PROTOCOL_CATEGORY_VIDEO, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_DEFAULT_LEVEL },
+   { "zattosecurehd2-f.akamaihd.net",    "Zattoo", NDPI_PROTOCOL_ZATTOO, NDPI_PROTOCOL_CATEGORY_VIDEO, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_DEFAULT_LEVEL },
+
+   { "classroom.google.com",                "GoogleClassroom", NDPI_PROTOCOL_GOOGLE_CLASSROOM, NDPI_PROTOCOL_CATEGORY_COLLABORATIVE, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL },
+
+
    { NULL, NULL, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL }
   };
 
diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index 3cfc2b2ce..34cd6daeb 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -5255,12 +5255,14 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st
 					 flow->daddr, flow->dport)) {
       /* This looks like BitTorrent */
       ret.app_protocol = NDPI_PROTOCOL_BITTORRENT;
+      flow->confidence = NDPI_CONFIDENCE_DPI_CACHE;
     } else if((flow->l4_proto == IPPROTO_UDP) /* Zoom/UDP used for video */
 	      && (((ntohs(flow->sport) == 8801 /* Zoom port */) && ndpi_search_into_zoom_cache(ndpi_str, flow->saddr))
 		  || ((ntohs(flow->dport) == 8801 /* Zoom port */) && ndpi_search_into_zoom_cache(ndpi_str, flow->daddr))
 		  )) {
       /* This looks like Zoom */
       ret.app_protocol = NDPI_PROTOCOL_ZOOM;
+      flow->confidence = NDPI_CONFIDENCE_DPI_CACHE;
     }
   }
 
diff --git a/src/lib/ndpi_utils.c b/src/lib/ndpi_utils.c
index 115ca49bf..521e57615 100644
--- a/src/lib/ndpi_utils.c
+++ b/src/lib/ndpi_utils.c
@@ -2267,6 +2267,9 @@ void load_common_alpns(struct ndpi_detection_module_struct *ndpi_str) {
     "h3-fb-05", "h1q-fb",
     "doq-i00",
 
+    /* ApplePush */
+    "apns-security-v3", "apns-pack-v1",
+
     NULL /* end */
   };
   u_int i;
diff --git a/src/lib/protocols/ldap.c b/src/lib/protocols/ldap.c
index ae58fd71e..3462d07b8 100644
--- a/src/lib/protocols/ldap.c
+++ b/src/lib/protocols/ldap.c
@@ -62,7 +62,7 @@ void ndpi_search_ldap(struct ndpi_detection_module_struct *ndpi_struct, struct n
 			}
 		}
 		// normal type
-		if (packet->payload[1] == 0x84 && packet->payload_packet_len >= 0x84 &&
+		if (packet->payload[1] == 0x84 &&
 			packet->payload[2] == 0x00 && packet->payload[3] == 0x00 && packet->payload[6] == 0x02) {
 
 			if (packet->payload[7] == 0x01 &&
@@ -82,6 +82,15 @@ void ndpi_search_ldap(struct ndpi_detection_module_struct *ndpi_struct, struct n
 				ndpi_int_ldap_add_connection(ndpi_struct, flow);
 				return;
 			}
+
+			if (packet->payload[7] == 0x03 &&
+				(packet->payload[11] == 0x60 || packet->payload[11] == 0x61 || packet->payload[11] == 0x63 ||
+				 packet->payload[11] == 0x64 || packet->payload[11] == 0x65) && packet->payload[12] == 0x84) {
+
+				NDPI_LOG_INFO(ndpi_struct, "found ldap type 3\n");
+				ndpi_int_ldap_add_connection(ndpi_struct, flow);
+				return;
+			}
 		}
 	}
 
diff --git a/src/lib/protocols/mqtt.c b/src/lib/protocols/mqtt.c
index 23bb72cd8..4d6773abe 100644
--- a/src/lib/protocols/mqtt.c
+++ b/src/lib/protocols/mqtt.c
@@ -59,13 +59,34 @@ static void ndpi_int_mqtt_add_connection (struct ndpi_detection_module_struct *n
 	NDPI_LOG_INFO(ndpi_struct, "found Mqtt\n");
 }
 
+static int64_t get_var_int(const unsigned char *buf, int buf_len, u_int8_t *num_bytes)
+{
+	int i, multiplier = 1;
+	u_int32_t value = 0;
+	u_int8_t encodedByte;
+
+	*num_bytes= 0;
+	for (i = 0; i < 4; i++) {
+		if (i >= buf_len)
+			return -1;
+		(*num_bytes)++;
+		encodedByte = buf[i];
+		value += ((encodedByte & 127) * multiplier);
+		if ((encodedByte & 128) == 0)
+			break;
+		multiplier *= 128;
+	}
+	return value;
+}
+
 /**
  * Dissector function that searches Mqtt headers
  */
 void ndpi_search_mqtt (struct ndpi_detection_module_struct *ndpi_struct,
 		struct ndpi_flow_struct *flow)
 {
-	u_int8_t rl,pt,flags;
+	u_int8_t pt,flags, rl_len;
+	int64_t rl;
 
 	NDPI_LOG_DBG(ndpi_struct, "search Mqtt\n");
 	struct ndpi_packet_struct *packet = &ndpi_struct->packet;
@@ -89,15 +110,16 @@ void ndpi_search_mqtt (struct ndpi_detection_module_struct *ndpi_struct,
 		NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_PROTOCOL_MQTT);
 		return;
 	}
-	if (packet->payload_packet_len > 258) {
-		NDPI_LOG_DBG(ndpi_struct, "Excluding Mqtt .. maximum packet size exceeded!\n");
+	// we extract the remaining length
+	rl = get_var_int(&packet->payload[1], packet->payload_packet_len - 1, &rl_len);
+	if (rl < 0) {
+		NDPI_LOG_DBG(ndpi_struct, "Excluding Mqtt .. invalid length!\n");
 		NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_PROTOCOL_MQTT);
 		return;
 	}
-	// we extract the remaining length
-	rl = (u_int8_t) (packet->payload[1]);
-	if (packet->payload_packet_len != (rl + 2)) {
-		NDPI_LOG_DBG(ndpi_struct, "Excluding Mqtt .. packet size exceeded!\n");
+	NDPI_LOG_DBG(ndpi_struct, "Mqtt: msg_len %d\n", (unsigned long long)rl);
+	if (packet->payload_packet_len != rl + 1 + rl_len) {
+		NDPI_LOG_DBG(ndpi_struct, "Excluding Mqtt .. maximum packet size exceeded!\n");
 		NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_PROTOCOL_MQTT);
 		return;
 	}
@@ -153,15 +175,9 @@ void ndpi_search_mqtt (struct ndpi_detection_module_struct *ndpi_struct,
 	NDPI_LOG_DBG2(ndpi_struct,"====>>>> Passed second stage of identification\n");
 	// third stage verification (payload)
 	if (pt == CONNECT) {
-		if (packet->payload_packet_len >= 8 && memcmp(&(packet->payload[4]),"MQTT",4) == 0) {
-			NDPI_LOG_DBG(ndpi_struct, "found Mqtt CONNECT\n");
-			ndpi_int_mqtt_add_connection(ndpi_struct,flow);
-			return;
-		} else {
-			NDPI_LOG_DBG(ndpi_struct, "Excluding Mqtt invalid CONNECT\n");
-			NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_PROTOCOL_MQTT);
-			return;
-		}
+		NDPI_LOG_DBG(ndpi_struct, "found Mqtt CONNECT\n");
+		ndpi_int_mqtt_add_connection(ndpi_struct,flow);
+		return;
 	}
 	if (pt == PUBLISH) {
 		// payload CAN be zero bytes length (section 3.3.3 of MQTT standard)
diff --git a/src/lib/protocols/rx.c b/src/lib/protocols/rx.c
index e5bc70c5d..5022d55a7 100644
--- a/src/lib/protocols/rx.c
+++ b/src/lib/protocols/rx.c
@@ -72,6 +72,7 @@ struct ndpi_rx_header {
 #define PLUS_2             6
 #define MORE_1             9
 #define CLIENT_INIT_2     33
+#define PLUS_3            34
 
 
 
@@ -118,7 +119,7 @@ void ndpi_check_rx(struct ndpi_detection_module_struct *ndpi_struct,
      header->flags == PLUS_0 || header->flags == PLUS_1 ||
      header->flags == PLUS_2 || header->flags == REQ_ACK ||
      header->flags == MORE_1 || header->flags == CLIENT_INIT_1 ||
-     header->flags == CLIENT_INIT_2) {
+     header->flags == CLIENT_INIT_2 || header->flags == PLUS_3) {
 
     /* TYPE and FLAGS combo */
     switch(header->type)
@@ -132,7 +133,7 @@ void ndpi_check_rx(struct ndpi_detection_module_struct *ndpi_struct,
 	/* Fall-through */
       case RX_ACK:
 	if(header->flags == CLIENT_INIT_1 || header->flags == CLIENT_INIT_2 ||
-	   header->flags == EMPTY)
+	   header->flags == EMPTY || header->flags == PLUS_3)
 	  goto security;
 	/* Fall-through */
       case RX_CHALLENGE:
@@ -198,10 +199,6 @@ void ndpi_check_rx(struct ndpi_detection_module_struct *ndpi_struct,
   } else {
     flow->l4.udp.rx_conn_epoch = header->conn_epoch;
     flow->l4.udp.rx_conn_id = header->conn_id;
-    {
-      NDPI_LOG_INFO(ndpi_struct, "found RX\n");
-      ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_RX, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI);
-    }
   }
 }
 
diff --git a/src/lib/protocols/whatsapp.c b/src/lib/protocols/whatsapp.c
index 059958c18..412caf957 100644
--- a/src/lib/protocols/whatsapp.c
+++ b/src/lib/protocols/whatsapp.c
@@ -30,9 +30,22 @@ void ndpi_search_whatsapp(struct ndpi_detection_module_struct *ndpi_struct,
     0x45, 0x44, 0x0, 0x01, 0x0, 0x0, 0x02, 0x08,
     0x0, 0x57, 0x41, 0x02, 0x0, 0x0, 0x0
   };
+  static u_int8_t whatsapp_old_sequence[] = {
+    0x57, 0x41, 0x01, 0x05
+  };
 
   NDPI_LOG_DBG(ndpi_struct, "search WhatsApp\n");
 
+  /* This is a very old sequence (2015?) but we still have it in our unit tests.
+     Try to detect it, without too much effort... */
+  if(flow->l4.tcp.wa_matched_so_far == 0 &&
+     packet->payload_packet_len > sizeof(whatsapp_old_sequence) &&
+     memcmp(packet->payload, whatsapp_old_sequence, sizeof(whatsapp_old_sequence)) == 0) {
+    NDPI_LOG_INFO(ndpi_struct, "found WhatsApp (old sequence)\n");
+    ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_WHATSAPP, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI);
+    return;
+  }
+
   if(flow->l4.tcp.wa_matched_so_far < sizeof(whatsapp_sequence)) {
     size_t match_len = sizeof(whatsapp_sequence) - flow->l4.tcp.wa_matched_so_far;
     if(packet->payload_packet_len < match_len)