Increased number of protocols to 512

Implemented WhatsApp file exchage support (added test pcap)

9 files changed, 123 insertions, 28 deletions

diff --git a/src/include/ndpi_define.h.in b/src/include/ndpi_define.h.in
index c6c1f4481..a641c7b66 100644
--- a/src/include/ndpi_define.h.in
+++ b/src/include/ndpi_define.h.in
@@ -252,7 +252,7 @@
 #define NDPI_COMPARE_IPV6_ADDRESS_STRUCTS(x,y)  \
   ((((u_int64_t *)(x))[0]) < (((u_int64_t *)(y))[0]) || ( (((u_int64_t *)(x))[0]) == (((u_int64_t *)(y))[0]) && (((u_int64_t *)(x))[1]) < (((u_int64_t *)(y))[1])) )
 
-#define NDPI_NUM_BITS              256
+#define NDPI_NUM_BITS              512
 
 #define NDPI_BITS /* 32 */ (sizeof(ndpi_ndpi_mask) * 8 /* number of bits in a byte */)        /* bits per mask */
 #define howmanybits(x, y)   (((x)+((y)-1))/(y))
diff --git a/src/include/ndpi_protocol_ids.h b/src/include/ndpi_protocol_ids.h
index c8871509b..8740d372b 100644
--- a/src/include/ndpi_protocol_ids.h
+++ b/src/include/ndpi_protocol_ids.h
@@ -279,9 +279,10 @@
 #define NDPI_PROTOCOL_GOOGLE_SERVICES       239
 #define NDPI_PROTOCOL_AMAZON_VIDEO          240
 #define NDPI_PROTOCOL_GOOGLE_DOCS           241
+#define NDPI_PROTOCOL_WHATSAPP_FILES        242 /* Videos, pictures, voice messages... */
 
 /* UPDATE UPDATE UPDATE UPDATE UPDATE UPDATE UPDATE UPDATE UPDATE */
-#define NDPI_LAST_IMPLEMENTED_PROTOCOL              NDPI_PROTOCOL_GOOGLE_DOCS
+#define NDPI_LAST_IMPLEMENTED_PROTOCOL              NDPI_PROTOCOL_WHATSAPP_FILES
 
 #define NDPI_MAX_SUPPORTED_PROTOCOLS                (NDPI_LAST_IMPLEMENTED_PROTOCOL + 1)
 #define NDPI_MAX_NUM_CUSTOM_PROTOCOLS               (NDPI_NUM_BITS-NDPI_LAST_IMPLEMENTED_PROTOCOL)
diff --git a/src/include/ndpi_protocols.h b/src/include/ndpi_protocols.h
index 3268488e1..4e230a68d 100644
--- a/src/include/ndpi_protocols.h
+++ b/src/include/ndpi_protocols.h
@@ -353,5 +353,5 @@ void init_nintendo_dissector(struct ndpi_detection_module_struct *ndpi_struct, u
 void init_csgo_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask);
 void init_checkmk_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask);
 void init_apple_push_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask);
-
+void init_whatsapp_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask);
 #endif /* __NDPI_PROTOCOLS_H__ */
diff --git a/src/lib/Makefile.am b/src/lib/Makefile.am
index 33b402f6e..ec8f4e03f 100644
--- a/src/lib/Makefile.am
+++ b/src/lib/Makefile.am
@@ -166,6 +166,7 @@ libndpi_la_SOURCES = ndpi_content_match.c.inc \
 		     protocols/smpp.c \
 		     protocols/tinc.c \
 		     protocols/someip.c \
+		     protocols/whatsapp.c \
 		     third_party/include/actypes.h \
 		     third_party/include/ahocorasick.h \
 		     third_party/include/ndpi_patricia.h \
diff --git a/src/lib/ndpi_content_match.c.inc b/src/lib/ndpi_content_match.c.inc
index fee3967d8..c92a36ad3 100644
--- a/src/lib/ndpi_content_match.c.inc
+++ b/src/lib/ndpi_content_match.c.inc
@@ -82,23 +82,25 @@ static ndpi_network host_protocol_list[] = {
   /*
     WhatsApp Inc.
   */
-
-   { 0x3216C6CC /* 50.22.198.204/30 */, 30, NDPI_PROTOCOL_WHATSAPP },
-   { 0x4B7E2720 /* 75.126.39.32/27 */, 27, NDPI_PROTOCOL_WHATSAPP },
+   { 0x3216C6CC /* 50.22.198.204/30 */,  30, NDPI_PROTOCOL_WHATSAPP },
+   { 0x4B7E2720 /* 75.126.39.32/27 */,   27, NDPI_PROTOCOL_WHATSAPP },
    { 0x6CA8B460 /* 108.168.180.96/27 */, 27, NDPI_PROTOCOL_WHATSAPP },
-   { 0x9E553A00 /* 158.85.58.0/25 */, 25, NDPI_PROTOCOL_WHATSAPP },
-   { 0x9E55FE40 /* 158.85.254.64/27 */, 27, NDPI_PROTOCOL_WHATSAPP },
-   { 0xA92F2320 /* 169.47.35.32/27 */, 27, NDPI_PROTOCOL_WHATSAPP },
-   { 0xA93743E0 /* 169.55.67.224/27 */, 27, NDPI_PROTOCOL_WHATSAPP },
+   { 0x9E553A00 /* 158.85.58.0/25 */,    25, NDPI_PROTOCOL_WHATSAPP },
+   { 0x9E55FE40 /* 158.85.254.64/27 */,  27, NDPI_PROTOCOL_WHATSAPP },
+   { 0xA92F2320 /* 169.47.35.32/27 */,   27, NDPI_PROTOCOL_WHATSAPP },
+   { 0xA93C4F00 /* 169.60.79.0/24 */,    24, NDPI_PROTOCOL_WHATSAPP },
+   { 0xA93F4C11 /* 169.63.76.0/25 */,    25, NDPI_PROTOCOL_WHATSAPP },
+   { 0xA93F4940 /* 169.63.73.64/25 */,    25, NDPI_PROTOCOL_WHATSAPP },
    { 0xA93764A0 /* 169.55.100.160/27 */, 27, NDPI_PROTOCOL_WHATSAPP },
    { 0xA937EBA0 /* 169.55.235.160/27 */, 27, NDPI_PROTOCOL_WHATSAPP },
    { 0xADC0A220 /* 173.192.162.32/27 */, 27, NDPI_PROTOCOL_WHATSAPP },
    { 0xB8AD8840 /* 184.173.136.64/27 */, 27, NDPI_PROTOCOL_WHATSAPP },
-   { 0xB93CDA35 /* 185.60.218.53/32 */, 27, NDPI_PROTOCOL_WHATSAPP },
-   { 0xC60BFB20 /* 198.11.251.32/27 */, 27, NDPI_PROTOCOL_WHATSAPP },
+   { 0xB93CDA35 /* 185.60.218.53/32 */,  27, NDPI_PROTOCOL_WHATSAPP },
+   { 0xC60BFB20 /* 198.11.251.32/27 */,  27, NDPI_PROTOCOL_WHATSAPP },
    { 0xD02B73C0 /* 208.43.115.192/27 */, 27, NDPI_PROTOCOL_WHATSAPP },
    { 0xD02B7A80 /* 208.43.122.128/27 */, 27, NDPI_PROTOCOL_WHATSAPP },
-
+   /* Files */
+   { 0xB93CD835 /* 185.60.216.53/32 */,  32, NDPI_PROTOCOL_WHATSAPP_FILES },
 
   /*
     WeChat
@@ -802,7 +804,6 @@ static ndpi_network host_protocol_list[] = {
   { 0x5B6C1400 /* 91.108.20.0/22 */, 22, NDPI_PROTOCOL_TELEGRAM },
   { 0x5B6C3800 /* 91.108.56.0/22 */, 22, NDPI_PROTOCOL_TELEGRAM },
   { 0x959AA000 /* 149.154.160.0/20 */, 20, NDPI_PROTOCOL_TELEGRAM },
-  { 0xA93F4940 /* 169.63.73.64/26 */, 26, NDPI_PROTOCOL_TELEGRAM },
 
   /*
     BitTorrent
@@ -8026,7 +8027,7 @@ ndpi_protocol_match host_match[] = {
   { ".speedtest.net", NULL, NULL,           "Ookla",            NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_CATEGORY_NETWORK, NDPI_PROTOCOL_SAFE },
 
   { "docs.googleusercontent.com", NULL, NULL, "GoogleDocs",     NDPI_PROTOCOL_GOOGLE_DOCS, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_ACCEPTABLE },
-  
+
   { "drive-thirdparty.googleusercontent.com", NULL, NULL,        "GoogleDrive",      NDPI_PROTOCOL_GOOGLE_DRIVE, NDPI_PROTOCOL_CATEGORY_COLLABORATIVE, NDPI_PROTOCOL_ACCEPTABLE },
   { "drive.google.com",                       NULL, NULL,        "GoogleDrive",      NDPI_PROTOCOL_GOOGLE_DRIVE, NDPI_PROTOCOL_CATEGORY_CLOUD, NDPI_PROTOCOL_ACCEPTABLE },
 
@@ -8074,8 +8075,8 @@ ndpi_protocol_match host_match[] = {
 
   { "docs.googleusercontent.com", NULL, NULL, "GoogleDocs",     NDPI_PROTOCOL_GOOGLE_DOCS, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_ACCEPTABLE },
   { "docs.google.com",            NULL, NULL, "GoogleDocs",     NDPI_PROTOCOL_GOOGLE_DOCS, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_ACCEPTABLE },
-    
-  { "googleusercontent.com", NULL, NULL,    "Google",           NDPI_PROTOCOL_GOOGLE, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE }, 
+
+  { "googleusercontent.com", NULL, NULL,    "Google",           NDPI_PROTOCOL_GOOGLE, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE },
   { "1e100.net", NULL, NULL,                "Google",           NDPI_PROTOCOL_GOOGLE, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE },
 
   { "maps.google.", NULL, NULL,             "GoogleMaps",       NDPI_PROTOCOL_GOOGLE_MAPS, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE },
@@ -8124,6 +8125,7 @@ ndpi_protocol_match host_match[] = {
   { "mediawiki.", NULL, NULL,               "Wikipedia",        NDPI_PROTOCOL_WIKIPEDIA, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE },
   { "wikimediafoundation.", NULL, NULL,     "Wikipedia",        NDPI_PROTOCOL_WIKIPEDIA, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE },
 
+  { "mmg-fna.whatsapp.net", NULL, NULL,     "WhatsAppFiles",    NDPI_PROTOCOL_WHATSAPP_FILES, NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT, NDPI_PROTOCOL_ACCEPTABLE },
   { ".whatsapp.", NULL, NULL,               "WhatsApp",         NDPI_PROTOCOL_WHATSAPP, NDPI_PROTOCOL_CATEGORY_CHAT, NDPI_PROTOCOL_ACCEPTABLE },
 
   { ".yahoo.", NULL, NULL,                  "Yahoo",            NDPI_PROTOCOL_YAHOO, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE },
diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index 9840d8715..c598369ba 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -1320,6 +1320,16 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp
 			    no_master, "WhatsAppVoice", NDPI_PROTOCOL_CATEGORY_VOIP,
 			    ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
 			    ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
+    ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_WHATSAPP_FILES,
+			    no_master,
+			    no_master, "WhatsAppFiles", NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT,
+			    ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
+			    ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
+    ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_WHATSAPP,
+			    no_master,
+			    no_master, "WhatsApp", NDPI_PROTOCOL_CATEGORY_CHAT,
+			    ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
+			    ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
 
     custom_master[0] = NDPI_PROTOCOL_SSL_NO_CERT, custom_master[1] = NDPI_PROTOCOL_UNKNOWN;
     ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_SSL,
@@ -2931,6 +2941,9 @@ void ndpi_set_protocol_detection_bitmask2(struct ndpi_detection_module_struct *n
   /* BITTORRENT */
   init_bittorrent_dissector(ndpi_struct, &a, detection_bitmask);
 
+  /* WHATSAPP */
+  init_whatsapp_dissector(ndpi_struct, &a, detection_bitmask);
+
   /* AMQP */
   init_amqp_dissector(ndpi_struct, &a, detection_bitmask);
 
@@ -3582,6 +3595,23 @@ void check_ndpi_flow_func(struct ndpi_detection_module_struct *ndpi_struct,
 
 /* ********************************************************************************* */
 
+static u_int16_t ndpi_guess_host_protocol_id(struct ndpi_detection_module_struct *ndpi_struct,
+					     struct ndpi_flow_struct *flow) {
+  u_int16_t ret = NDPI_PROTOCOL_UNKNOWN;
+  
+  if(flow->packet.iph) {
+    /* guess host protocol */
+    ret = ndpi_network_ptree_match(ndpi_struct, (struct in_addr *)&flow->packet.iph->saddr);
+
+    if(ret == NDPI_PROTOCOL_UNKNOWN)
+      ret = ndpi_network_ptree_match(ndpi_struct, (struct in_addr *)&flow->packet.iph->daddr);
+  }
+    
+  return(ret);
+}
+
+/* ********************************************************************************* */
+
 ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_struct,
 				    struct ndpi_flow_struct *flow) {
   ndpi_protocol ret = { NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_UNKNOWN };
@@ -3598,8 +3628,8 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st
       if((flow->guessed_protocol_id == NDPI_PROTOCOL_UNKNOWN)
 	 && (flow->packet.l4_protocol == IPPROTO_TCP)
 	 && (flow->l4.tcp.ssl_stage > 1))
-	flow->guessed_protocol_id = NDPI_PROTOCOL_SSL;
-
+	flow->guessed_protocol_id = NDPI_PROTOCOL_SSL_NO_CERT;
+       
       guessed_protocol_id = flow->guessed_protocol_id,
 	guessed_host_protocol_id = flow->guessed_host_protocol_id;
 
@@ -3607,7 +3637,6 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st
 	 && (NDPI_ISSET(&flow->excluded_protocol_bitmask, guessed_host_protocol_id)))
 	guessed_host_protocol_id = NDPI_PROTOCOL_UNKNOWN;
 
-
       /* Ignore guessed protocol if they have been discarded */
       if((guessed_protocol_id != NDPI_PROTOCOL_UNKNOWN)
 	 && (guessed_host_protocol_id == NDPI_PROTOCOL_UNKNOWN)
@@ -3778,23 +3807,21 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
 
     /* guess protocol */
     flow->guessed_protocol_id = (int16_t) ndpi_guess_protocol_id(ndpi_struct, protocol, sport, dport, &user_defined_proto);
-
+    flow->guessed_host_protocol_id = ndpi_guess_host_protocol_id(ndpi_struct, flow);
+    
     if(flow->guessed_protocol_id >= (NDPI_MAX_SUPPORTED_PROTOCOLS-1)) {
       /* This is a custom protocol and it has priority over everything else */
       ret.master_protocol = NDPI_PROTOCOL_UNKNOWN, ret.app_protocol = flow->guessed_host_protocol_id;
       return(ret);
     }
 
-    if(user_defined_proto && flow->guessed_protocol_id != NDPI_PROTOCOL_UNKNOWN) {
+    if(user_defined_proto && flow->guessed_protocol_id != NDPI_PROTOCOL_UNKNOWN) {      
       if(flow->packet.iph) {
-	/* guess host protocol */
-	flow->guessed_host_protocol_id = ndpi_network_ptree_match(ndpi_struct, (struct in_addr *)&flow->packet.iph->saddr);
-	if(flow->guessed_host_protocol_id == NDPI_PROTOCOL_UNKNOWN)
-	  flow->guessed_host_protocol_id = ndpi_network_ptree_match(ndpi_struct, (struct in_addr *)&flow->packet.iph->daddr);
-	if(flow->guessed_host_protocol_id != NDPI_PROTOCOL_UNKNOWN)
+	if(flow->guessed_host_protocol_id != NDPI_PROTOCOL_UNKNOWN) {
 	  /* ret.master_protocol = flow->guessed_protocol_id , ret.app_protocol = flow->guessed_host_protocol_id; /\* ****** *\/ */
 	  ret = ndpi_detection_giveup(ndpi_struct, flow);
-
+	}
+	
 	return(ret);
       }
     } else {
diff --git a/src/lib/protocols/whatsapp.c b/src/lib/protocols/whatsapp.c
new file mode 100644
index 000000000..b079dcd01
--- /dev/null
+++ b/src/lib/protocols/whatsapp.c
@@ -0,0 +1,60 @@
+/*
+ * whatsapp.c
+ *
+ * Copyright (C) 2018 - ntop.org
+ *
+ * nDPI is free software: you can zmqtribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * nDPI is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with nDPI.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+#include "ndpi_protocol_ids.h"
+
+#ifdef NDPI_PROTOCOL_WHATSAPP
+
+#define NDPI_CURRENT_PROTO NDPI_PROTOCOL_WHATSAPP
+
+#include "ndpi_api.h"
+
+void ndpi_search_whatsapp(struct ndpi_detection_module_struct *ndpi_struct,
+			  struct ndpi_flow_struct *flow) {
+  struct ndpi_packet_struct *packet = &flow->packet;
+  u_int8_t whatsapp_sequence[] = {
+    0x45, 0x44, 0x0, 0x01, 0x0, 0x0, 0x02, 0x08,
+    0x0, 0x57, 0x41, 0x02, 0x0, 0x0, 0x0
+  };
+
+  NDPI_LOG_DBG(ndpi_struct, "search WhatsApp\n");
+
+  if((packet->payload_packet_len > 240)
+     && (memcmp(packet->payload, whatsapp_sequence, sizeof(whatsapp_sequence)) == 0)) {
+    NDPI_LOG_INFO(ndpi_struct, "found WhatsApp\n");
+    ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_WHATSAPP, NDPI_PROTOCOL_UNKNOWN);
+  } else 
+    NDPI_EXCLUDE_PROTO(ndpi_struct, flow);  
+}
+
+
+void init_whatsapp_dissector(struct ndpi_detection_module_struct *ndpi_struct,
+			     u_int32_t *id,
+			     NDPI_PROTOCOL_BITMASK *detection_bitmask) {
+  ndpi_set_bitmask_protocol_detection("WhatsApp", ndpi_struct, detection_bitmask, *id,
+				      NDPI_PROTOCOL_WHATSAPP,
+				      ndpi_search_whatsapp,
+				      NDPI_SELECTION_BITMASK_PROTOCOL_V4_V6_TCP_WITH_PAYLOAD_WITHOUT_RETRANSMISSION,
+				      SAVE_DETECTION_BITMASK_AS_UNKNOWN,
+				      ADD_TO_DETECTION_BITMASK);
+  *id += 1;
+}
+
+
+#endif
diff --git a/tests/pcap/whatsappfiles.pcap b/tests/pcap/whatsappfiles.pcap
new file mode 100644
index 000000000..c540c1c25
--- /dev/null
+++ b/tests/pcap/whatsappfiles.pcap
diff --git a/tests/result/whatsappfiles.pcap.out b/tests/result/whatsappfiles.pcap.out
new file mode 100644
index 000000000..02f393986
--- /dev/null
+++ b/tests/result/whatsappfiles.pcap.out
@@ -0,0 +1,4 @@
+WhatsAppFiles	620	452233	2
+
+	1	TCP 192.168.2.29:49698 <-> 185.60.216.53:443 [proto: 242/WhatsAppFiles][132 pkts/9906 bytes <-> 178 pkts/237405 bytes]
+	2	TCP 192.168.2.29:49674 <-> 185.60.216.53:443 [proto: 242/WhatsAppFiles][161 pkts/189194 bytes <-> 149 pkts/15728 bytes]