fixed coap.c and dropbox.c after detecting wrong detection - added test for dropbox

8 files changed, 168 insertions, 64 deletions

diff --git a/src/lib/ndpi_content_match.c.inc b/src/lib/ndpi_content_match.c.inc
index 411a3961d..87bc4c96b 100644
--- a/src/lib/ndpi_content_match.c.inc
+++ b/src/lib/ndpi_content_match.c.inc
@@ -7260,6 +7260,7 @@ ndpi_protocol_match host_match[] = {
   { ".cnn.c",				"CNN",			NDPI_SERVICE_CNN, NDPI_PROTOCOL_FUN },
   { ".cnn.net",				"CNN",			NDPI_SERVICE_CNN, NDPI_PROTOCOL_FUN },
   { ".dropbox.com",			"DropBox",		NDPI_SERVICE_DROPBOX, NDPI_PROTOCOL_SAFE },
+  { "log.getdropbox.com",		"DropBox",		NDPI_SERVICE_DROPBOX, NDPI_PROTOCOL_SAFE },
   { ".ebay.",				"eBay",			NDPI_SERVICE_EBAY, NDPI_PROTOCOL_ACCEPTABLE },
   { ".ebay.com",			"eBay",			NDPI_SERVICE_EBAY, NDPI_PROTOCOL_ACCEPTABLE },
   { ".ebaystatic.com",			"eBay",			NDPI_SERVICE_EBAY, NDPI_PROTOCOL_ACCEPTABLE },
diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index 07ae9dbab..287162a7c 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -1516,12 +1516,12 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp
     ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_VIBER,
 			    no_master,
 			    no_master, "Viber",
-			    ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0),	     /* TCP */
-			    ndpi_build_default_ports(ports_b, 7985, 7987, 0, 0, 0));    /* UDP */
-    ndpi_set_proto_defaults(ndpi_mod,NDPI_PROTOCOL_ACCEPTABLE,NDPI_PROTOCOL_COAP,
+			    ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0),	      /* TCP */
+			    ndpi_build_default_ports(ports_b, 7985, 7987, 0, 0, 0));  /* UDP */
+    ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_COAP,
 			    no_master,
 			    no_master, "COAP",
-			    ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0),   /* TCP */
+			    ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0),         /* TCP */
 			    ndpi_build_default_ports(ports_b, 5683, 5684, 0, 0, 0));  /* UDP */
 
     /* calling function for host and content matched protocols */
@@ -3287,7 +3287,7 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
 
   ndpi_connection_tracking(ndpi_struct, flow);
 
-  /* build ndpi_selction packet bitmask */
+  /* build ndpi_selection packet bitmask */
   ndpi_selection_packet = NDPI_SELECTION_BITMASK_PROTOCOL_COMPLETE_TRAFFIC;
   if(flow->packet.iph != NULL)
     ndpi_selection_packet |= NDPI_SELECTION_BITMASK_PROTOCOL_IP | NDPI_SELECTION_BITMASK_PROTOCOL_IPV4_OR_IPV6;
diff --git a/src/lib/protocols/coap.c b/src/lib/protocols/coap.c
index 8089c8159..cddf31b7e 100644
--- a/src/lib/protocols/coap.c
+++ b/src/lib/protocols/coap.c
@@ -25,78 +25,117 @@
 #include "ndpi_protocols.h"
 #ifdef NDPI_PROTOCOL_COAP
 
+#define CON     0
+#define NO_CON  1
+#define ACK     2
+#define RST     3
+
+struct ndpi_coap_hdr
+{
+#if defined(__BIG_ENDIAN__)
+  u_int8_t version:2, type:2, tkl:4;
+#elif defined(__LITTLE_ENDIAN__)
+  u_int8_t tkl:4, type:2, version:2;
+#endif
+  u_int8_t code;
+  u_int16_t message_id; //if needed, remember to convert in host number
+};
+
+
+/**
+   VALUE OF -CODE- FIELD
+
+   [0]   = "Empty",
+   [1]   = "GET",
+   [2]   = "POST",
+   [3]   = "PUT",
+   [4]   = "DELETE",
+   [65]  = "2.01 Created",
+   [66]  = "2.02 Deleted",
+   [67]  = "2.03 Valid",
+   [68]  = "2.04 Changed",
+   [69]  = "2.05 Content",
+   [128] = "4.00 Bad Request",
+   [129] = "4.01 Unauthorized",
+   [130] = "4.02 Bad Option",
+   [131] = "4.03 Forbidden",
+   [132] = "4.04 Not Found",
+   [133] = "4.05 Method Not Allowed",
+   [134] = "4.06 Not Acceptable",
+   [140] = "4.12 Precondition Failed",
+   [141] = "4.13 Request Entity Too Large",
+   [143] = "4.15 Unsupported Content-Format",
+   [160] = "5.00 Internal Server Error",
+   [161] = "5.01 Not Implemented",
+   [162] = "5.02 Bad Gateway",
+   [163] = "5.03 Service Unavailable",
+   [164] = "5.04 Gateway Timeout",
+   [165] = "5.05 Proxying Not Supported"
+**/
+  
+
 /**
  * Entry point when protocol is identified.
  */
 static void ndpi_int_coap_add_connection (struct ndpi_detection_module_struct *ndpi_struct,
-			      struct ndpi_flow_struct *flow)
+					  struct ndpi_flow_struct *flow)
 {
-  // not sure if this is accurate but coap runs on top of udp and should be connectionless
   ndpi_set_detected_protocol(ndpi_struct,flow,NDPI_PROTOCOL_COAP,NDPI_PROTOCOL_UNKNOWN);
-  NDPI_LOG(NDPI_PROTOCOL_COAP, ndpi_struct, NDPI_LOG_DEBUG, "CoAP found.\n");
 }
+
 /**
  * Dissector function that searches CoAP headers
  */
 void ndpi_search_coap (struct ndpi_detection_module_struct *ndpi_struct,
 		       struct ndpi_flow_struct *flow)
 {
-	struct ndpi_packet_struct *packet = &flow->packet;
-	if (packet->detected_protocol_stack[0] != NDPI_PROTOCOL_UNKNOWN) {
-		return;
-	}
-	NDPI_LOG(NDPI_PROTOCOL_COAP, ndpi_struct, NDPI_LOG_DEBUG, "CoAP detection...\n");
-	// searching for request
-	NDPI_LOG(NDPI_PROTOCOL_COAP, ndpi_struct, NDPI_LOG_DEBUG, "====>>>> COAP header: %04x%04x%04x%04x [len: %u]\n",
-			packet->payload[0], packet->payload[1], packet->payload[2], packet->payload[3], packet->payload_packet_len);
-	// check if we have version bits
-	if (packet->payload_packet_len < 4) {
-		NDPI_LOG(NDPI_PROTOCOL_COAP, ndpi_struct, NDPI_LOG_DEBUG, "Excluding Coap .. mandatory header not found!\n");
-		NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_PROTOCOL_COAP);
-		return;
-	}
-	// since this is always unsigned we could have spared the 0xF0 logical AND
-	// vt = version and type (version is mandatory 1; type is either 0,1,2,3 )
-	u_int8_t vt = (u_int8_t) ((packet->payload[0] & 0xF0) >> 4);
-	if ((vt == 4) || (vt == 5) || (vt == 6) || (vt == 7)) {
-		NDPI_LOG(NDPI_PROTOCOL_COAP, ndpi_struct, NDPI_LOG_DEBUG, "Continuing Coap detection \n");
-		// search for values 9 to 15 in the token length
-		u_int8_t tkl = (u_int8_t) ((packet->payload[0] & 0x0F));
-		if ((tkl >= 9) && (tkl <= 15)) {
-			NDPI_LOG(NDPI_PROTOCOL_COAP, ndpi_struct, NDPI_LOG_DEBUG, "Excluding Coap .. invalid token length found!\n");
-			NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_PROTOCOL_COAP);
-			return;
-		}
-		u_int8_t class = (u_int8_t) ((packet->payload[1] & 0xE0) >> 5);
-		u_int8_t detail = (u_int8_t) ((packet->payload[1] & 0x1F));
-		if ((class == 0) && (detail == 0) && (tkl == 0) && (packet->payload_packet_len == 4)) {
-			NDPI_LOG(NDPI_PROTOCOL_COAP, ndpi_struct, NDPI_LOG_DEBUG, "Coap found ... empty message\n");
-			ndpi_int_coap_add_connection(ndpi_struct,flow);
-			return;
-		}
-		if ((class == 0) && ((detail == 1) || (detail == 2 ) || (detail == 3 ) || (detail == 4 ))) {
-			// we should probably search for options as well and payload for deeper inspection
-			NDPI_LOG(NDPI_PROTOCOL_COAP, ndpi_struct, NDPI_LOG_DEBUG, "Coap found ... req message\n");
-			ndpi_int_coap_add_connection(ndpi_struct,flow);
-			return;
-		}
-		if ((class == 2) || (class == 4) || (class == 5)) {
-			// we should probably search for options as well and payload for deeper inspection
-			NDPI_LOG(NDPI_PROTOCOL_COAP, ndpi_struct, NDPI_LOG_DEBUG, "Coap found ... resp message\n");
-			ndpi_int_coap_add_connection(ndpi_struct,flow);
-			return;
-		}
+  struct ndpi_packet_struct *packet = &flow->packet;
+  struct ndpi_coap_hdr * h = (struct ndpi_coap_hdr*) packet->payload;
+	
+  if(packet->detected_protocol_stack[0] != NDPI_PROTOCOL_UNKNOWN) {
+    return;
+  }
+
+  // search for udp packet
+  if(packet->udp != NULL) {
+    
+    // header too short
+    if(packet->payload_packet_len < 4) {
+      
+      NDPI_LOG(NDPI_PROTOCOL_COAP, ndpi_struct, NDPI_LOG_DEBUG, "excluding Coap\n");
+      NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_PROTOCOL_COAP);
+      return;
+    }
+    
+    NDPI_LOG(NDPI_PROTOCOL_COAP, ndpi_struct, NDPI_LOG_DEBUG, "calculating coap over udp.\n");
+
+    // check values in header
+    if(h->version == 1) {
+      if(h->type == CON || h->type == NO_CON || h->type == ACK || h->type == RST ) {
+	if(h->tkl == 0 || h->tkl < 8) {
+	  if((h->code >= 0 && h->code <= 5) || (h->code >= 65 && h->code <= 69) ||
+	     (h->code >= 128  && h->code <= 134) || (h->code >= 140 && h->code <= 143) ||
+	     (h->code >= 160 && h->code <= 165)) {
+	       
+	    NDPI_LOG(NDPI_PROTOCOL_COAP, ndpi_struct, NDPI_LOG_DEBUG, "Coap found...\n");
+	    ndpi_int_coap_add_connection(ndpi_struct,flow);
+	    return;
+	  }
 	}
-	NDPI_LOG(NDPI_PROTOCOL_COAP, ndpi_struct, NDPI_LOG_DEBUG, "Excluding Coap ...\n");
-	NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_PROTOCOL_COAP);
-	return;
+      }
+    } 
+  }
+  
+  NDPI_LOG(NDPI_PROTOCOL_COAP, ndpi_struct, NDPI_LOG_DEBUG, "Excluding Coap ...\n");
+  NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_PROTOCOL_COAP);
+  return;
 
 }
 /**
  * Entry point for the ndpi library
  */
 void init_coap_dissector (struct ndpi_detection_module_struct *ndpi_struct,
-		     u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask)
+			  u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask)
 {
   ndpi_set_bitmask_protocol_detection ("COAP", ndpi_struct, detection_bitmask, *id,
 				       NDPI_PROTOCOL_COAP,
diff --git a/src/lib/protocols/dropbox.c b/src/lib/protocols/dropbox.c
index ec546d356..63a1f5531 100644
--- a/src/lib/protocols/dropbox.c
+++ b/src/lib/protocols/dropbox.c
@@ -25,6 +25,10 @@
 #include "ndpi_api.h"
 
 #ifdef NDPI_PROTOCOL_DROPBOX
+
+#define DB_LSP_PORT 17500
+
+
 static void ndpi_int_dropbox_add_connection(struct ndpi_detection_module_struct *ndpi_struct,
 					    struct ndpi_flow_struct *flow,
 					    u_int8_t due_to_correlation)
@@ -40,12 +44,25 @@ static void ndpi_check_dropbox(struct ndpi_detection_module_struct *ndpi_struct,
   u_int32_t payload_len = packet->payload_packet_len;
 
   if(packet->udp != NULL) {
-    u_int16_t dropbox_port = htons(17500);
+
+    u_int16_t dropbox_port = htons(DB_LSP_PORT);
 
     if((packet->udp->source == dropbox_port)
        && (packet->udp->dest == dropbox_port)) {
       if(payload_len > 2) {
-	if(strncmp((const char *)packet->payload, "{\"", 2) == 0) {
+	if(strncmp((const char *)packet->payload, "{\"host_int\"", 11) == 0) {
+	  
+	  NDPI_LOG(NDPI_PROTOCOL_DROPBOX, ndpi_struct, NDPI_LOG_DEBUG, "Found dropbox.\n");
+	  ndpi_int_dropbox_add_connection(ndpi_struct, flow, 0);
+	  return;
+	}
+      }
+    }
+    if((packet->udp->dest == dropbox_port)) {
+      if(payload_len > 2) {
+	char * p = (char *) packet->payload;
+	while(*p++ != '{');
+	if(strncmp(p,"\"messageType\"", 13) == 0) {
 	  NDPI_LOG(NDPI_PROTOCOL_DROPBOX, ndpi_struct, NDPI_LOG_DEBUG, "Found dropbox.\n");
 	  ndpi_int_dropbox_add_connection(ndpi_struct, flow, 0);
 	  return;
diff --git a/tests/pcap/coap.pcap b/tests/pcap/coap.pcap
index 1554fa5e7..e30a0c2ab 100644
--- a/tests/pcap/coap.pcap
+++ b/tests/pcap/coap.pcap
diff --git a/tests/pcap/dropbox.pcap b/tests/pcap/dropbox.pcap
new file mode 100644
index 000000000..6e950826e
--- /dev/null
+++ b/tests/pcap/dropbox.pcap
diff --git a/tests/result/coap.pcap.out b/tests/result/coap.pcap.out
index 7a53f9697..e17da0c77 100644
--- a/tests/result/coap.pcap.out
+++ b/tests/result/coap.pcap.out
@@ -1,6 +1,10 @@
-COAP	800	80676	4
+COAP	19	1614	8
 
-	1	UDP 192.168.56.1:50318 <-> 192.168.56.101:17500 [proto: 27/COAP][200 pkts/20220 bytes]
-	2	UDP 192.168.56.1:50312 <-> 192.168.56.101:17500 [proto: 27/COAP][200 pkts/20194 bytes]
-	3	UDP 192.168.56.1:50311 <-> 192.168.56.101:17500 [proto: 27/COAP][200 pkts/20120 bytes]
-	4	UDP 192.168.56.1:50319 <-> 192.168.56.101:17500 [proto: 27/COAP][200 pkts/20142 bytes]
+	1	UDP [2001:da8:215:1171:a10b:cb48:8f83:57f6]:5683 <-> [2001:620:8:35d9::10]:61043 [proto: 27/COAP][1 pkts/86 bytes]
+	2	UDP [2001:da8:215:1171:a10b:cb48:8f83:57f6]:5683 <-> [2001:620:8:35d9::10]:61045 [proto: 27/COAP][1 pkts/86 bytes]
+	3	UDP [2001:da8:215:1171:a10b:cb48:8f83:57f6]:5683 <-> [2001:620:8:35d9::10]:61047 [proto: 27/COAP][1 pkts/90 bytes]
+	4	UDP [bbbb::1]:33499 <-> [bbbb::3]:5683 [proto: 27/COAP][4 pkts/404 bytes]
+	5	UDP [bbbb::1]:46819 <-> [bbbb::3]:5683 [proto: 27/COAP][6 pkts/467 bytes]
+	6	UDP [2001:da8:215:1171:a10b:cb48:8f83:57f6]:5683 <-> [2001:620:8:35d9::10]:61044 [proto: 27/COAP][1 pkts/86 bytes]
+	7	UDP [2001:da8:215:1171:a10b:cb48:8f83:57f6]:5683 <-> [2001:620:8:35d9::10]:61046 [proto: 27/COAP][1 pkts/86 bytes]
+	8	UDP [bbbb::1]:50250 <-> [bbbb::3]:5683 [proto: 27/COAP][4 pkts/309 bytes]
diff --git a/tests/result/dropbox.pcap.out b/tests/result/dropbox.pcap.out
new file mode 100644
index 000000000..db5640be9
--- /dev/null
+++ b/tests/result/dropbox.pcap.out
@@ -0,0 +1,43 @@
+MDNS	16	1648	1
+SSDP	140	61108	22
+DropBox	1104	246122	16
+
+	1	UDP 192.168.1.105:33189 <-> 192.168.1.254:53 [proto: 5.121/DNS.DropBox][4 pkts/744 bytes][Host: notify.dropbox.com]
+	2	UDP 192.168.1.105:17500 <-> 192.168.1.255:17500 [proto: 121/DropBox][6 pkts/1422 bytes]
+	3	TCP 192.168.1.105:59975 <-> 108.160.172.204:443 [proto: 91.121/SSL.DropBox][34 pkts/18026 bytes][SSL client: client.dropbox.com]
+	4	UDP 192.168.1.105:36173 <-> 192.168.1.254:53 [proto: 5.121/DNS.DropBox][8 pkts/1390 bytes][Host: log.getdropbox.com]
+	5	TCP 192.168.1.105:46394 <-> 162.125.17.131:443 [proto: 91.121/SSL.DropBox][22 pkts/11392 bytes][SSL client: notify.dropbox.com]
+	6	UDP 192.168.1.105:50789 <-> 192.168.1.254:53 [proto: 5.121/DNS.DropBox][4 pkts/792 bytes][Host: d.dropbox.com]
+	7	UDP 192.168.1.105:55407 <-> 192.168.1.254:53 [proto: 5.121/DNS.DropBox][4 pkts/822 bytes][Host: client.dropbox.com]
+	8	UDP 192.168.56.1:50318 <-> 192.168.56.101:17500 [proto: 121/DropBox][200 pkts/20220 bytes]
+	9	UDP 192.168.56.1:50312 <-> 192.168.56.101:17500 [proto: 121/DropBox][200 pkts/20194 bytes]
+	10	UDP 192.168.1.101:1280 <-> 239.255.255.250:1900 [proto: 12/SSDP][2 pkts/1018 bytes]
+	11	UDP 192.168.1.101:1346 <-> 239.255.255.250:1900 [proto: 12/SSDP][2 pkts/1018 bytes]
+	12	UDP 192.168.1.101:1650 <-> 239.255.255.250:1900 [proto: 12/SSDP][6 pkts/2836 bytes]
+	13	UDP 192.168.1.101:1908 <-> 239.255.255.250:1900 [proto: 12/SSDP][2 pkts/1018 bytes]
+	14	UDP 192.168.1.101:2544 <-> 239.255.255.250:1900 [proto: 12/SSDP][2 pkts/1018 bytes]
+	15	UDP 192.168.1.101:2604 <-> 239.255.255.250:1900 [proto: 12/SSDP][2 pkts/1018 bytes]
+	16	UDP 192.168.1.101:3412 <-> 239.255.255.250:1900 [proto: 12/SSDP][6 pkts/2836 bytes]
+	17	UDP 192.168.1.101:4974 <-> 239.255.255.250:1900 [proto: 12/SSDP][6 pkts/2836 bytes]
+	18	UDP 192.168.1.105:49112 <-> 192.168.1.254:53 [proto: 5.121/DNS.DropBox][4 pkts/774 bytes][Host: client-cf.dropbox.com]
+	19	UDP 192.168.1.105:17500 <-> 255.255.255.255:17500 [proto: 121/DropBox][6 pkts/1422 bytes]
+	20	UDP 239.255.255.250:1900 <-> 192.168.1.254:50828 [proto: 12/SSDP][44 pkts/19936 bytes]
+	21	UDP 192.168.56.1:50311 <-> 192.168.56.101:17500 [proto: 121/DropBox][200 pkts/20120 bytes]
+	22	UDP 192.168.56.1:50319 <-> 192.168.56.101:17500 [proto: 121/DropBox][200 pkts/20142 bytes]
+	23	UDP 192.168.1.106:57268 <-> 239.255.255.250:1900 [proto: 12/SSDP][16 pkts/2632 bytes]
+	24	TCP 54.240.174.31:443 <-> 192.168.1.105:44949 [proto: 91.121/SSL.DropBox][138 pkts/97302 bytes][SSL client: client-cf.dropbox.com]
+	25	TCP 192.168.1.105:36226 <-> 108.160.172.195:80 [proto: 7.121/HTTP.DropBox][20 pkts/3928 bytes][Host: log.getdropbox.com]
+	26	UDP 192.168.1.101:2169 <-> 239.255.255.250:1900 [proto: 12/SSDP][2 pkts/1018 bytes]
+	27	UDP 192.168.1.101:2141 <-> 239.255.255.250:1900 [proto: 12/SSDP][6 pkts/2836 bytes]
+	28	UDP 192.168.1.101:2159 <-> 239.255.255.250:1900 [proto: 12/SSDP][2 pkts/1018 bytes]
+	29	UDP 192.168.1.101:2873 <-> 239.255.255.250:1900 [proto: 12/SSDP][6 pkts/2836 bytes]
+	30	UDP 192.168.1.101:2991 <-> 239.255.255.250:1900 [proto: 12/SSDP][2 pkts/1018 bytes]
+	31	UDP 192.168.1.101:3547 <-> 239.255.255.250:1900 [proto: 12/SSDP][6 pkts/2836 bytes]
+	32	UDP 192.168.1.101:3731 <-> 239.255.255.250:1900 [proto: 12/SSDP][2 pkts/1018 bytes]
+	33	UDP 192.168.1.101:3777 <-> 239.255.255.250:1900 [proto: 12/SSDP][2 pkts/1018 bytes]
+	34	UDP 192.168.1.101:3959 <-> 239.255.255.250:1900 [proto: 12/SSDP][6 pkts/2836 bytes]
+	35	UDP 192.168.1.101:4171 <-> 239.255.255.250:1900 [proto: 12/SSDP][6 pkts/2836 bytes]
+	36	UDP 192.168.1.101:4169 <-> 239.255.255.250:1900 [proto: 12/SSDP][6 pkts/2836 bytes]
+	37	UDP 192.168.1.106:5353 <-> 224.0.0.251:5353 [proto: 8/MDNS][16 pkts/1648 bytes]
+	38	UDP 192.168.1.101:4625 <-> 239.255.255.250:1900 [proto: 12/SSDP][6 pkts/2836 bytes]
+	39	TCP 192.168.1.105:47747 <-> 108.160.172.225:443 [proto: 91.121/SSL.DropBox][54 pkts/27432 bytes][SSL client: d.dropbox.com]