Removed VEOHTV protocol and replaced with HTTPDownload

Fixed bug that prevented content type to be properly detected

8 files changed, 31 insertions, 155 deletions

diff --git a/src/include/ndpi_api.h b/src/include/ndpi_api.h
index a39acd536..5815f118c 100644
--- a/src/include/ndpi_api.h
+++ b/src/include/ndpi_api.h
@@ -172,7 +172,7 @@ extern "C" {
   ndpi_protocol ndpi_guess_undetected_protocol(struct ndpi_detection_module_struct *ndpi_struct,
 					       u_int8_t proto, u_int32_t shost, u_int16_t sport, u_int32_t dhost, u_int16_t dport);
   int ndpi_match_string_subprotocol(struct ndpi_detection_module_struct *ndpi_struct,
-				    char *string_to_match, u_int string_to_match_len);
+				    char *string_to_match, u_int string_to_match_len, u_int8_t is_host_match);
   int ndpi_match_host_subprotocol(struct ndpi_detection_module_struct *ndpi_struct,
 				  struct ndpi_flow_struct *flow, char *string_to_match, u_int string_to_match_len,
 				  u_int16_t master_protocol_id);
diff --git a/src/include/ndpi_protocol_ids.h b/src/include/ndpi_protocol_ids.h
index df213bfdd..77f50e9d4 100644
--- a/src/include/ndpi_protocol_ids.h
+++ b/src/include/ndpi_protocol_ids.h
@@ -49,7 +49,7 @@
 #define NDPI_PROTOCOL_IP_ICMPV6					102
 
 #define NDPI_PROTOCOL_HTTP					7
-#define NDPI_PROTOCOL_HTTP_APPLICATION_VEOHTV 		        60
+#define NDPI_PROTOCOL_HTTP_DOWNLOAD    		                60
 #define NDPI_PROTOCOL_SSL_NO_CERT			        64 /* SSL without certificate (Skype, Ultrasurf?) - ntop.org */
 #define NDPI_PROTOCOL_SSL					91
 #define NDPI_PROTOCOL_HTTP_APPLICATION_ACTIVESYNC		110
diff --git a/src/lib/Makefile.am b/src/lib/Makefile.am
index 3c5a69e64..eac9facad 100644
--- a/src/lib/Makefile.am
+++ b/src/lib/Makefile.am
@@ -140,7 +140,6 @@ libndpi_la_SOURCES = ndpi_content_match.c.inc \
 		     protocols/twitter.c \
 		     protocols/ubntac2.c \
 		     protocols/usenet.c \
-		     protocols/veohtv.c \
 		     protocols/viber.c \
 		     protocols/vhua.c \
 		     protocols/vmware.c \
diff --git a/src/lib/ndpi_content_match.c.inc b/src/lib/ndpi_content_match.c.inc
index 59d36b42f..8c853ebe7 100644
--- a/src/lib/ndpi_content_match.c.inc
+++ b/src/lib/ndpi_content_match.c.inc
@@ -7550,6 +7550,18 @@ ndpi_protocol_match content_match[] = {
   { "video/webm",			NULL,		        NDPI_CONTENT_WEBM, NDPI_PROTOCOL_FUN },
   { "application/x-rtsp-tunnelled",	NULL,		        NDPI_PROTOCOL_RTSP, NDPI_PROTOCOL_FUN },
   { "application/vnd.apple.mpegurl",	NULL,		        NDPI_CONTENT_MPEG, NDPI_PROTOCOL_FUN },
+  { "application/x-tar",		NULL,		        NDPI_PROTOCOL_HTTP_DOWNLOAD, NDPI_PROTOCOL_ACCEPTABLE },
+  { "application/octet-stream",		NULL,		        NDPI_PROTOCOL_HTTP_DOWNLOAD, NDPI_PROTOCOL_ACCEPTABLE },
+  { "application/mac-binary",		NULL,		        NDPI_PROTOCOL_HTTP_DOWNLOAD, NDPI_PROTOCOL_ACCEPTABLE },
+  { "/x-bzip",				NULL,		        NDPI_PROTOCOL_HTTP_DOWNLOAD, NDPI_PROTOCOL_ACCEPTABLE },
+  { "/x-gzip",				NULL,		        NDPI_PROTOCOL_HTTP_DOWNLOAD, NDPI_PROTOCOL_ACCEPTABLE },
+  { "/x-zip",				NULL,		        NDPI_PROTOCOL_HTTP_DOWNLOAD, NDPI_PROTOCOL_ACCEPTABLE },
+  { "/zip",				NULL,		        NDPI_PROTOCOL_HTTP_DOWNLOAD, NDPI_PROTOCOL_ACCEPTABLE },
+  { "binhex",				NULL,		        NDPI_PROTOCOL_HTTP_DOWNLOAD, NDPI_PROTOCOL_ACCEPTABLE },
+  { "/base64",				NULL,		        NDPI_PROTOCOL_HTTP_DOWNLOAD, NDPI_PROTOCOL_ACCEPTABLE },
+  { "application/gnutar",		NULL,		        NDPI_PROTOCOL_HTTP_DOWNLOAD, NDPI_PROTOCOL_ACCEPTABLE },
+  { "application/x-compressed",		NULL,		        NDPI_PROTOCOL_HTTP_DOWNLOAD, NDPI_PROTOCOL_ACCEPTABLE },
+  
   { NULL, 0 }
 };
 
diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index a42afa3d1..d39131dae 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -960,9 +960,9 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp
 			    no_master, "TVUplayer",
 			    ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
 			    ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
-    ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_HTTP_APPLICATION_VEOHTV,
+    ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_HTTP_DOWNLOAD,
 			    no_master,
-			    no_master, "HTTP_APPLICATION_VEOHTV",
+			    no_master, "HTTPDownload",
 			    ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
 			    ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
     ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_QQLIVE,
@@ -2273,9 +2273,6 @@ void ndpi_set_protocol_detection_bitmask2(struct ndpi_detection_module_struct *n
   /* SHOUTCAST */
   init_shoutcast_dissector(ndpi_struct, &a, detection_bitmask);
 
-  /* VEOHTV */
-  init_veohtv_dissector(ndpi_struct, &a, detection_bitmask);
-
   /* KERBEROS */
   init_kerberos_dissector(ndpi_struct, &a, detection_bitmask);
 
@@ -4313,10 +4310,11 @@ char* ndpi_strnstr(const char *s, const char *find, size_t slen) {
 /* ****************************************************** */
 
 int ndpi_match_string_subprotocol(struct ndpi_detection_module_struct *ndpi_struct,
-				  char *string_to_match, u_int string_to_match_len) {
+				  char *string_to_match, u_int string_to_match_len,
+				  u_int8_t is_host_match) {
   int matching_protocol_id = NDPI_PROTOCOL_UNKNOWN;
   AC_TEXT_t ac_input_text;
-  ndpi_automa *automa = &ndpi_struct->host_automa;
+  ndpi_automa *automa = is_host_match ? &ndpi_struct->host_automa : &ndpi_struct->content_automa;
 
   if((automa->ac_automa == NULL) || (string_to_match_len == 0)) return(NDPI_PROTOCOL_UNKNOWN);
 
@@ -4337,8 +4335,9 @@ int ndpi_match_string_subprotocol(struct ndpi_detection_module_struct *ndpi_stru
 static int ndpi_automa_match_string_subprotocol(struct ndpi_detection_module_struct *ndpi_struct,
 						struct ndpi_flow_struct *flow,
 						char *string_to_match, u_int string_to_match_len,
-						u_int16_t master_protocol_id) {
-  int matching_protocol_id = ndpi_match_string_subprotocol(ndpi_struct, string_to_match, string_to_match_len);
+						u_int16_t master_protocol_id,
+						u_int8_t is_host_match) {
+  int matching_protocol_id = ndpi_match_string_subprotocol(ndpi_struct, string_to_match, string_to_match_len, is_host_match);
   struct ndpi_packet_struct *packet = &flow->packet;
   AC_TEXT_t ac_input_text;
 
@@ -4382,7 +4381,7 @@ int ndpi_match_host_subprotocol(struct ndpi_detection_module_struct *ndpi_struct
 				u_int16_t master_protocol_id) {
   return(ndpi_automa_match_string_subprotocol(ndpi_struct,
 					      flow, string_to_match, string_to_match_len,
-					      master_protocol_id));
+					      master_protocol_id, 1));
 }
 
 /* ****************************************************** */
@@ -4393,7 +4392,7 @@ int ndpi_match_content_subprotocol(struct ndpi_detection_module_struct *ndpi_str
 				   u_int16_t master_protocol_id) {
   return(ndpi_automa_match_string_subprotocol(ndpi_struct, flow,
 					      string_to_match, string_to_match_len,
-					      master_protocol_id));
+					      master_protocol_id, 0));
 }
 
 /* ****************************************************** */
diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c
index b0fe04159..caac7390b 100644
--- a/src/lib/protocols/http.c
+++ b/src/lib/protocols/http.c
@@ -37,18 +37,14 @@ static void ndpi_int_http_add_connection(struct ndpi_detection_module_struct *nd
 
     /* If no custom protocol has been detected */
     if(flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN) {
-      if(protocol != NDPI_PROTOCOL_HTTP) {
-	ndpi_search_tcp_or_udp(ndpi_struct, flow);
-	ndpi_set_detected_protocol(ndpi_struct, flow, protocol, NDPI_PROTOCOL_UNKNOWN);
-      } else {
+      if(protocol == NDPI_PROTOCOL_HTTP)
 	ndpi_int_reset_protocol(flow);
-	ndpi_set_detected_protocol(ndpi_struct, flow, protocol, NDPI_PROTOCOL_UNKNOWN);
-      }
+
+      ndpi_set_detected_protocol(ndpi_struct, flow, protocol, NDPI_PROTOCOL_UNKNOWN);
     }
 
     flow->http_detected = 1;
   }
-
 }
 
 #ifdef NDPI_CONTENT_FLASH
@@ -202,10 +198,10 @@ static void parseHttpSubprotocol(struct ndpi_detection_module_struct *ndpi_struc
 
     /*
       NOTE
-
+      
       If http_dont_dissect_response = 1 dissection of HTTP response
       mime types won't happen
-     */
+    */
     ndpi_match_host_subprotocol(ndpi_struct, flow, (char *)flow->host_server_name,
 				strlen((const char *)flow->host_server_name),
 				NDPI_PROTOCOL_HTTP);
diff --git a/src/lib/protocols/veohtv.c b/src/lib/protocols/veohtv.c
deleted file mode 100644
index a3ab267b9..000000000
--- a/src/lib/protocols/veohtv.c
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * veohtv.c
- *
- * Copyright (C) 2009-2011 by ipoque GmbH
- * Copyright (C) 2011-15 - ntop.org
- *
- * This file is part of nDPI, an open source deep packet inspection
- * library based on the OpenDPI and PACE technology by ipoque GmbH
- *
- * nDPI is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Lesser General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * nDPI is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with nDPI.  If not, see <http://www.gnu.org/licenses/>.
- * 
- */
-
-
-#include "ndpi_api.h"
-
-
-#ifdef NDPI_PROTOCOL_HTTP_APPLICATION_VEOHTV
-
-static void ndpi_int_veohtv_add_connection(struct ndpi_detection_module_struct *ndpi_struct, 
-					   struct ndpi_flow_struct *flow/* , ndpi_protocol_type_t protocol_type */)
-{
-  ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_HTTP_APPLICATION_VEOHTV, NDPI_PROTOCOL_UNKNOWN);
-}
-
-void ndpi_search_veohtv_tcp(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow)
-{
-	struct ndpi_packet_struct *packet = &flow->packet;
-	
-//      struct ndpi_id_struct         *src=ndpi_struct->src;
-//      struct ndpi_id_struct         *dst=ndpi_struct->dst;
-
-	if (packet->detected_protocol_stack[0] == NDPI_PROTOCOL_HTTP_APPLICATION_VEOHTV)
-		return;
-
-	if (flow->l4.tcp.veoh_tv_stage == 1 || flow->l4.tcp.veoh_tv_stage == 2) {
-		if (packet->packet_direction != flow->setup_packet_direction &&
-			packet->payload_packet_len > NDPI_STATICSTRING_LEN("HTTP/1.1 20")
-			&& memcmp(packet->payload, "HTTP/1.1 ", NDPI_STATICSTRING_LEN("HTTP/1.1 ")) == 0 &&
-			(packet->payload[NDPI_STATICSTRING_LEN("HTTP/1.1 ")] == '2' ||
-			 packet->payload[NDPI_STATICSTRING_LEN("HTTP/1.1 ")] == '3' ||
-			 packet->payload[NDPI_STATICSTRING_LEN("HTTP/1.1 ")] == '4' ||
-			 packet->payload[NDPI_STATICSTRING_LEN("HTTP/1.1 ")] == '5')) {
-#ifdef NDPI_CONTENT_FLASH
-			ndpi_parse_packet_line_info(ndpi_struct, flow);
-			if (packet->detected_protocol_stack[0] == NDPI_CONTENT_FLASH &&
-				packet->server_line.ptr != NULL &&
-				packet->server_line.len > NDPI_STATICSTRING_LEN("Veoh-") &&
-				memcmp(packet->server_line.ptr, "Veoh-", NDPI_STATICSTRING_LEN("Veoh-")) == 0) {
-				NDPI_LOG(NDPI_PROTOCOL_HTTP_APPLICATION_VEOHTV, ndpi_struct, NDPI_LOG_DEBUG, "VeohTV detected.\n");
-				ndpi_int_veohtv_add_connection(ndpi_struct, flow);
-				return;
-			}
-#endif
-			if (flow->l4.tcp.veoh_tv_stage == 2) {
-				NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask,
-											   NDPI_PROTOCOL_HTTP_APPLICATION_VEOHTV);
-				return;
-			}
-			NDPI_LOG(NDPI_PROTOCOL_HTTP_APPLICATION_VEOHTV, ndpi_struct, NDPI_LOG_DEBUG, "VeohTV detected.\n");
-			ndpi_int_veohtv_add_connection(ndpi_struct, flow);
-			return;
-		} else if (flow->packet_direction_counter[(flow->setup_packet_direction == 1) ? 0 : 1] > 3) {
-			if (flow->l4.tcp.veoh_tv_stage == 2) {
-				NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask,
-											   NDPI_PROTOCOL_HTTP_APPLICATION_VEOHTV);
-				return;
-			}
-			NDPI_LOG(NDPI_PROTOCOL_HTTP_APPLICATION_VEOHTV, ndpi_struct, NDPI_LOG_DEBUG, "VeohTV detected.\n");
-			ndpi_int_veohtv_add_connection(ndpi_struct, flow);
-			return;
-		} else {
-			if (flow->packet_counter > 10) {
-				if (flow->l4.tcp.veoh_tv_stage == 2) {
-					NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask,
-												   NDPI_PROTOCOL_HTTP_APPLICATION_VEOHTV);
-					return;
-				}
-				NDPI_LOG(NDPI_PROTOCOL_HTTP_APPLICATION_VEOHTV, ndpi_struct, NDPI_LOG_DEBUG, "VeohTV detected.\n");
-				ndpi_int_veohtv_add_connection(ndpi_struct, flow);
-				return;
-			}
-			return;
-		}
-	} else if (packet->udp) {
-		/* UDP packets from Veoh Client Player
-		 *
-		 * packet starts with 16 byte random? value
-		 * then a 4 byte mode value
-		 *   values between 21 and 26 has been seen 
-		 * then a 4 byte counter */
-
-		if (packet->payload_packet_len == 28 &&
-			get_u_int32_t(packet->payload, 16) == htonl(0x00000021) &&
-			get_u_int32_t(packet->payload, 20) == htonl(0x00000000) && get_u_int32_t(packet->payload, 24) == htonl(0x01040000)) {
-			NDPI_LOG(NDPI_PROTOCOL_HTTP_APPLICATION_VEOHTV, ndpi_struct, NDPI_LOG_DEBUG, "UDP VeohTV found.\n");
-			ndpi_int_veohtv_add_connection(ndpi_struct, flow);
-			return;
-		}
-	}
-
-
-	NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_PROTOCOL_HTTP_APPLICATION_VEOHTV);
-}
-
-
-void init_veohtv_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask)
-{
-  ndpi_set_bitmask_protocol_detection("HTTP_APPLICATION_VEOHTV", ndpi_struct, detection_bitmask, *id,
-				      NDPI_PROTOCOL_HTTP_APPLICATION_VEOHTV,
-				      ndpi_search_veohtv_tcp,
-				      NDPI_SELECTION_BITMASK_PROTOCOL_V4_V6_TCP_OR_UDP_WITH_PAYLOAD_WITHOUT_RETRANSMISSION,
-				      SAVE_DETECTION_BITMASK_AS_UNKNOWN,
-				      ADD_TO_DETECTION_BITMASK);
-
-  *id += 1;
-}
-
-#endif
diff --git a/tests/result/mpeg.pcap.out b/tests/result/mpeg.pcap.out
index 2513a4e83..f6f36acbf 100644
--- a/tests/result/mpeg.pcap.out
+++ b/tests/result/mpeg.pcap.out
@@ -1,3 +1,3 @@
-HTTP	19	10643	1
+MPEG	19	10643	1
 
-	1	TCP 46.101.157.119:80 <-> 192.168.80.160:55804 [proto: 7/HTTP][19 pkts/10643 bytes][Host: luca.ntop.org]
+	1	TCP 46.101.157.119:80 <-> 192.168.80.160:55804 [proto: 7.42/HTTP.MPEG][19 pkts/10643 bytes][Host: luca.ntop.org]