better improvement of netflix traffic detection - added netflix pcap and output - change little bit http detection behaviour

3 files changed, 98 insertions, 12 deletions

diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c
index 5ca47476a..82167e204 100644
--- a/src/lib/protocols/http.c
+++ b/src/lib/protocols/http.c
@@ -341,7 +341,8 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
 
     NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "HOST Line found %.*s\n",
 	     packet->host_line.len, packet->host_line.ptr);
-
+    
+    /* call ndpi_match_host_subprotocol to see if there is a match with known-host http subprotocol */
     if((ndpi_struct->http_dont_dissect_response) || flow->http_detected)
       ndpi_match_host_subprotocol(ndpi_struct, flow,
 				  (char*)packet->host_line.ptr,
@@ -356,25 +357,43 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
     len = ndpi_min(packet->forwarded_line.len, sizeof(flow->nat_ip)-1);
     strncpy((char*)flow->nat_ip, (char*)packet->forwarded_line.ptr, len);
     flow->nat_ip[len] = '\0';
-
+    
     if(ndpi_struct->http_dont_dissect_response)
       parseHttpSubprotocol(ndpi_struct, flow);
 
-    if((flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN)
-       && ((ndpi_struct->http_dont_dissect_response) || flow->http_detected))
-      ndpi_match_host_subprotocol(ndpi_struct, flow,
-				    (char *)flow->host_server_name,
-				    strlen((const char *)flow->host_server_name),
-				    NDPI_PROTOCOL_HTTP);
+    /**
+       check result of host subprotocol detection
+       
+       if "detected" in flow == 0 then "detected" = "guess"
+       else "guess" = "detected"
+    **/
+    if(flow->detected_protocol_stack[1] == 0) {
+      flow->detected_protocol_stack[1] = flow->guessed_protocol_id;
+      if(flow->detected_protocol_stack[0] == 0)
+    	flow->detected_protocol_stack[0] = flow->guessed_host_protocol_id;
+    }
+    else {
+      if(flow->detected_protocol_stack[1] != flow->guessed_protocol_id)
+	flow->guessed_protocol_id = flow->detected_protocol_stack[1];
+      if(flow->detected_protocol_stack[0] != flow->guessed_host_protocol_id)
+	flow->guessed_host_protocol_id = flow->detected_protocol_stack[0];
+    }
+    
+    /* if((flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN) */
+    /*    && ((ndpi_struct->http_dont_dissect_response) || flow->http_detected)) */
+    /*   ndpi_match_host_subprotocol(ndpi_struct, flow, */
+    /* 				    (char *)flow->host_server_name, */
+    /* 				    strlen((const char *)flow->host_server_name), */
+    /* 				    NDPI_PROTOCOL_HTTP); */
 
     if((flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN)
        && ((ndpi_struct->http_dont_dissect_response) || flow->http_detected)
        && (packet->http_origin.len > 0))
       ndpi_match_host_subprotocol(ndpi_struct, flow,
-				    (char *)packet->http_origin.ptr,
-				    packet->http_origin.len,
-				    NDPI_PROTOCOL_HTTP);
-
+				  (char *)packet->http_origin.ptr,
+				  packet->http_origin.len,
+				  NDPI_PROTOCOL_HTTP);
+    
     if(flow->detected_protocol_stack[0] != NDPI_PROTOCOL_UNKNOWN) {
       if(packet->detected_protocol_stack[0] != NDPI_PROTOCOL_HTTP) {
 	ndpi_int_http_add_connection(ndpi_struct, flow, packet->detected_protocol_stack[0]);
diff --git a/tests/pcap/netflix.pcap b/tests/pcap/netflix.pcap
new file mode 100644
index 000000000..5139cf5cc
--- /dev/null
+++ b/tests/pcap/netflix.pcap
diff --git a/tests/result/netflix.pcap.out b/tests/result/netflix.pcap.out
new file mode 100644
index 000000000..ec6e125c6
--- /dev/null
+++ b/tests/result/netflix.pcap.out
@@ -0,0 +1,67 @@
+DNS	4	386	2
+SSDP	16	2648	1
+IGMP	1	60	1
+NetFlix	6976	6151821	56
+Amazon	2	126	1
+
+	1	TCP 192.168.1.7:53149 <-> 184.25.204.25:80 [proto: 7.133/HTTP.NetFlix][126 pkts/128603 bytes][Host: art-2.nflximg.net]
+	2	TCP 192.168.1.7:53153 <-> 184.25.204.24:80 [proto: 7.133/HTTP.NetFlix][637 pkts/745904 bytes][Host: tp.akam.nflximg.com]
+	3	TCP 192.168.1.7:53251 <-> 184.25.204.10:80 [proto: 7.133/HTTP.NetFlix][41 pkts/34971 bytes][Host: art-1.nflximg.net]
+	4	UDP 192.168.1.1:53 <-> 192.168.1.7:51543 [proto: 5.133/DNS.NetFlix][4 pkts/806 bytes][Host: ios.nccp.netflix.com]
+	5	UDP 192.168.1.1:53 <-> 192.168.1.7:51949 [proto: 5.133/DNS.NetFlix][2 pkts/322 bytes][Host: api-global.latency.prodaa.netflix.com]
+	6	UDP 192.168.1.1:53 <-> 192.168.1.7:52095 [proto: 5.133/DNS.NetFlix][2 pkts/322 bytes][Host: api-global.latency.prodaa.netflix.com]
+	7	UDP 192.168.1.1:53 <-> 192.168.1.7:52347 [proto: 5.133/DNS.NetFlix][2 pkts/451 bytes][Host: ios.nccp.netflix.com]
+	8	UDP 192.168.1.1:53 <-> 192.168.1.7:57093 [proto: 5/DNS][2 pkts/194 bytes][Host: a1907.dscg.akamai.net]
+	9	UDP 192.168.1.1:53 <-> 192.168.1.7:57719 [proto: 5.133/DNS.NetFlix][2 pkts/222 bytes][Host: sha2.san.akam.nflximg.net]
+	10	TCP 192.168.1.7:53163 <-> 23.246.11.145:80 [proto: 7.133/HTTP.NetFlix][53 pkts/45005 bytes]
+	11	TCP 192.168.1.7:53171 <-> 23.246.3.140:80 [proto: 7.133/HTTP.NetFlix][55 pkts/47007 bytes]
+	12	TCP 192.168.1.7:53173 <-> 23.246.11.133:80 [proto: 7.133/HTTP.NetFlix][49 pkts/32105 bytes]
+	13	TCP 192.168.1.7:53175 <-> 23.246.11.141:80 [proto: 7.133/HTTP.NetFlix][53 pkts/30613 bytes]
+	14	TCP 192.168.1.7:53177 <-> 23.246.11.141:80 [proto: 7.133/HTTP.NetFlix][55 pkts/29233 bytes]
+	15	TCP 192.168.1.7:53179 <-> 23.246.11.141:80 [proto: 7.133/HTTP.NetFlix][60 pkts/40140 bytes]
+	16	TCP 192.168.1.7:53181 <-> 23.246.11.141:80 [proto: 7.133/HTTP.NetFlix][54 pkts/25252 bytes]
+	17	TCP 192.168.1.7:53183 <-> 23.246.3.140:80 [proto: 7.133/HTTP.NetFlix][1307 pkts/1242780 bytes][Host: 23.246.3.140]
+	18	TCP 192.168.1.7:53217 <-> 23.246.11.141:80 [proto: 7.133/HTTP.NetFlix][1872 pkts/1858337 bytes][Host: 23.246.11.141]
+	19	IGMP 192.168.1.7:0 <-> 239.255.255.250:0 [proto: 82/IGMP][1 pkts/60 bytes]
+	20	TCP 192.168.1.7:53152 <-> 52.89.39.139:80 [proto: 7.133/HTTP.NetFlix][27 pkts/16505 bytes][Host: api-global.netflix.com]
+	21	TCP 192.168.1.7:53116 <-> 52.32.196.36:443 [proto: 91.133/SSL.NetFlix][148 pkts/73954 bytes][client: api-global.netflix.com]
+	22	TCP 52.41.30.5:443 <-> 192.168.1.7:53239 [proto: 91.133/SSL.NetFlix][48 pkts/29661 bytes][client: api-global.netflix.com]
+	23	TCP 192.168.1.7:53133 <-> 52.89.39.139:443 [proto: 91.133/SSL.NetFlix][69 pkts/43938 bytes][client: api-global.netflix.com]
+	24	TCP 192.168.1.7:53203 <-> 52.37.36.252:443 [proto: 91.133/SSL.NetFlix][45 pkts/27952 bytes][client: ichnaea.netflix.com]
+	25	TCP 192.168.1.7:53238 <-> 52.32.22.214:443 [proto: 91.133/SSL.NetFlix][31 pkts/10934 bytes][client: ios.nccp.netflix.com]
+	26	TCP 192.168.1.7:53248 <-> 52.32.22.214:443 [proto: 91.133/SSL.NetFlix][22 pkts/10239 bytes][client: ios.nccp.netflix.com]
+	27	TCP 52.41.30.5:443 <-> 192.168.1.7:53249 [proto: 91.133/SSL.NetFlix][52 pkts/25886 bytes][client: api-global.netflix.com]
+	28	TCP 192.168.1.7:53105 <-> 54.69.204.241:443 [proto: 91.133/SSL.NetFlix][37 pkts/9285 bytes][client: ichnaea.netflix.com]
+	29	TCP 192.168.1.7:53119 <-> 54.69.204.241:443 [proto: 91.133/SSL.NetFlix][36 pkts/12874 bytes][client: ichnaea.netflix.com]
+	30	TCP 192.168.1.7:53193 <-> 54.191.17.51:443 [proto: 91.133/SSL.NetFlix][71 pkts/58161 bytes][client: ios.nccp.netflix.com]
+	31	TCP 192.168.1.7:53148 <-> 184.25.204.25:80 [proto: 7.133/HTTP.NetFlix][63 pkts/47005 bytes][Host: art-2.nflximg.net]
+	32	TCP 192.168.1.7:53150 <-> 184.25.204.25:80 [proto: 7.133/HTTP.NetFlix][21 pkts/13259 bytes][Host: art-2.nflximg.net]
+	33	TCP 192.168.1.7:53252 <-> 184.25.204.10:80 [proto: 7.133/HTTP.NetFlix][41 pkts/42239 bytes][Host: art-1.nflximg.net]
+	34	UDP 192.168.1.1:53 <-> 192.168.1.7:51622 [proto: 5.133/DNS.NetFlix][4 pkts/806 bytes][Host: ios.nccp.netflix.com]
+	35	UDP 192.168.1.1:53 <-> 192.168.1.7:51728 [proto: 5/DNS][2 pkts/192 bytes][Host: a803.dscg.akamai.net]
+	36	UDP 192.168.1.1:53 <-> 192.168.1.7:52116 [proto: 5.133/DNS.NetFlix][2 pkts/320 bytes][Host: ichnaea.us-west-2.prodaa.netflix.com]
+	37	UDP 192.168.1.1:53 <-> 192.168.1.7:58102 [proto: 5.133/DNS.NetFlix][2 pkts/271 bytes][Host: appboot.netflix.com]
+	38	UDP 192.168.1.1:53 <-> 192.168.1.7:59180 [proto: 5.133/DNS.NetFlix][2 pkts/232 bytes][Host: artwork.akam.nflximg.net]
+	39	UDP 192.168.1.1:53 <-> 192.168.1.7:60962 [proto: 5.133/DNS.NetFlix][2 pkts/331 bytes][Host: ichnaea.geo.netflix.com]
+	40	UDP 192.168.1.7:53776 <-> 239.255.255.250:1900 [proto: 12/SSDP][16 pkts/2648 bytes]
+	41	TCP 192.168.1.7:53164 <-> 23.246.10.139:80 [proto: 7.133/HTTP.NetFlix][58 pkts/47176 bytes]
+	42	TCP 192.168.1.7:53172 <-> 23.246.11.133:80 [proto: 7.133/HTTP.NetFlix][50 pkts/25032 bytes]
+	43	TCP 192.168.1.7:53174 <-> 23.246.11.141:80 [proto: 7.133/HTTP.NetFlix][54 pkts/25348 bytes]
+	44	TCP 192.168.1.7:53176 <-> 23.246.11.141:80 [proto: 7.133/HTTP.NetFlix][57 pkts/28485 bytes]
+	45	TCP 192.168.1.7:53178 <-> 23.246.11.141:80 [proto: 7.133/HTTP.NetFlix][52 pkts/28063 bytes]
+	46	TCP 192.168.1.7:53180 <-> 23.246.11.141:80 [proto: 7.133/HTTP.NetFlix][55 pkts/28320 bytes]
+	47	TCP 192.168.1.7:53182 <-> 23.246.11.141:80 [proto: 7.133/HTTP.NetFlix][58 pkts/32796 bytes]
+	48	TCP 192.168.1.7:53184 <-> 23.246.11.141:80 [proto: 7.133/HTTP.NetFlix][178 pkts/157382 bytes][Host: 23.246.11.141]
+	49	TCP 192.168.1.7:53210 <-> 23.246.11.133:80 [proto: 7.133/HTTP.NetFlix][788 pkts/759283 bytes][Host: 23.246.11.133]
+	50	TCP 52.24.87.6:443 <-> 192.168.1.7:52929 [proto: 91.178/SSL.Amazon][2 pkts/126 bytes]
+	51	TCP 192.168.1.7:53115 <-> 52.32.196.36:443 [proto: 91.133/SSL.NetFlix][28 pkts/6662 bytes][client: api-global.netflix.com]
+	52	TCP 192.168.1.7:53117 <-> 52.32.196.36:443 [proto: 91.133/SSL.NetFlix][20 pkts/3017 bytes][client: api-global.netflix.com]
+	53	TCP 192.168.1.7:53132 <-> 52.89.39.139:443 [proto: 91.133/SSL.NetFlix][40 pkts/13487 bytes][client: api-global.netflix.com]
+	54	TCP 192.168.1.7:53134 <-> 52.89.39.139:443 [proto: 91.133/SSL.NetFlix][25 pkts/8201 bytes][client: api-global.netflix.com]
+	55	TCP 192.168.1.7:53151 <-> 54.201.191.132:80 [proto: 7.133/HTTP.NetFlix][41 pkts/33170 bytes][Host: appboot.netflix.com]
+	56	TCP 52.41.30.5:443 <-> 192.168.1.7:53250 [proto: 91.133/SSL.NetFlix][17 pkts/5314 bytes][client: api-global.netflix.com]
+	57	TCP 192.168.1.7:53114 <-> 54.191.17.51:443 [proto: 91.133/SSL.NetFlix][25 pkts/8228 bytes][client: ios.nccp.netflix.com]
+	58	TCP 192.168.1.7:53118 <-> 54.69.204.241:443 [proto: 91.133/SSL.NetFlix][34 pkts/12728 bytes][client: ichnaea.netflix.com]
+	59	TCP 192.168.1.7:53162 <-> 54.191.17.51:443 [proto: 91.133/SSL.NetFlix][31 pkts/14720 bytes][client: ios.nccp.netflix.com]
+	60	TCP 192.168.1.7:53202 <-> 54.191.17.51:443 [proto: 91.133/SSL.NetFlix][38 pkts/18536 bytes][client: ios.nccp.netflix.com]
+	61	TCP 192.168.1.7:53141 <-> 104.86.97.179:443 [proto: 91.133/SSL.NetFlix][230 pkts/209948 bytes][client: art-s.nflximg.net]