Merge pull request #3 from ntop/dev

update to latest

1 files changed, 332 insertions, 439 deletions

diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c
index 795a6e5b7..187f2a84a 100644
--- a/src/lib/protocols/http.c
+++ b/src/lib/protocols/http.c
@@ -1,8 +1,7 @@
 /*
  * http.c
  *
- * Copyright (C) 2009-2011 by ipoque GmbH
- * Copyright (C) 2011-15 - ntop.org
+ * Copyright (C) 2011-17 - ntop.org
  *
  * This file is part of nDPI, an open source deep packet inspection
  * library based on the OpenDPI and PACE technology by ipoque GmbH
@@ -21,14 +20,17 @@
  * along with nDPI.  If not, see <http://www.gnu.org/licenses/>.
  *
  */
-#include "ndpi_protocols.h"
+
+#include "ndpi_protocol_ids.h"
 
 #ifdef NDPI_PROTOCOL_HTTP
 
+#define NDPI_CURRENT_PROTO NDPI_PROTOCOL_HTTP
+
+#include "ndpi_api.h"
+
 
 /* global variables used for 1kxun protocol and iqiyi service */
-static u_int16_t kxun_counter;
-static u_int16_t iqiyi_counter;
 
 static void ndpi_int_http_add_connection(struct ndpi_detection_module_struct *ndpi_struct,
 					 struct ndpi_flow_struct *flow,
@@ -40,13 +42,13 @@ static void ndpi_int_http_add_connection(struct ndpi_detection_module_struct *nd
     ndpi_search_tcp_or_udp(ndpi_struct, flow);
 
     /* If no custom protocol has been detected */
-    if(flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN) {
-      if(protocol == NDPI_PROTOCOL_HTTP)
-	ndpi_int_reset_protocol(flow);
-
-      ndpi_set_detected_protocol(ndpi_struct, flow, protocol, NDPI_PROTOCOL_UNKNOWN);
-    }
-
+    /* if(flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN) */
+    if(protocol == NDPI_PROTOCOL_HTTP) {
+      ndpi_int_reset_protocol(flow);
+      ndpi_set_detected_protocol(ndpi_struct, flow, flow->guessed_host_protocol_id, protocol);
+    } else
+      ndpi_set_detected_protocol(ndpi_struct, flow, protocol, NDPI_PROTOCOL_HTTP);
+    
     flow->http_detected = 1;
   }
 }
@@ -63,11 +65,10 @@ static void flash_check_http_payload(struct ndpi_detection_module_struct
 
   pos = &packet->payload[packet->empty_line_position] + 2;
 
-
   if(memcmp(pos, "FLV", 3) == 0 && pos[3] == 0x01 && (pos[4] == 0x01 || pos[4] == 0x04 || pos[4] == 0x05)
      && pos[5] == 0x00 && pos[6] == 0x00 && pos[7] == 0x00 && pos[8] == 0x09) {
 
-    NDPI_LOG(NDPI_CONTENT_FLASH, ndpi_struct, NDPI_LOG_DEBUG, "Flash content in http detected\n");
+    NDPI_LOG_INFO(ndpi_struct, "found Flash content in HTTP\n");
     ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_CONTENT_FLASH);
   }
 }
@@ -79,7 +80,7 @@ static void avi_check_http_payload(struct ndpi_detection_module_struct *ndpi_str
   struct ndpi_packet_struct *packet = &flow->packet;
 
 
-  NDPI_LOG(NDPI_CONTENT_AVI, ndpi_struct, NDPI_LOG_DEBUG, "called avi_check_http_payload: %u %u %u\n",
+  NDPI_LOG_DBG2(ndpi_struct, "called avi_check_http_payload: %u %u %u\n",
 	   packet->empty_line_position_set, flow->l4.tcp.http_empty_line_seen, packet->empty_line_position);
 
   if(packet->empty_line_position_set == 0 && flow->l4.tcp.http_empty_line_seen == 0)
@@ -94,7 +95,7 @@ static void avi_check_http_payload(struct ndpi_detection_module_struct *ndpi_str
   if(flow->l4.tcp.http_empty_line_seen == 1) {
     if(packet->payload_packet_len > 20 && memcmp(packet->payload, "RIFF", 4) == 0
        && memcmp(packet->payload + 8, "AVI LIST", 8) == 0) {
-      NDPI_LOG(NDPI_CONTENT_AVI, ndpi_struct, NDPI_LOG_DEBUG, "Avi content in http detected\n");
+      NDPI_LOG_INFO(ndpi_struct, "found Avi content in HTTP\n");
       ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_CONTENT_AVI);
     }
     flow->l4.tcp.http_empty_line_seen = 0;
@@ -109,11 +110,11 @@ static void avi_check_http_payload(struct ndpi_detection_module_struct *ndpi_str
     u_int32_t p = packet->empty_line_position + 2;
 
     // check for avi header
-    NDPI_LOG(NDPI_CONTENT_AVI, ndpi_struct, NDPI_LOG_DEBUG, "p = %u\n", p);
+    NDPI_LOG_DBG2(ndpi_struct, "p = %u\n", p);
 
     if((p + 16) <= packet->payload_packet_len && memcmp(&packet->payload[p], "RIFF", 4) == 0
        && memcmp(&packet->payload[p + 8], "AVI LIST", 8) == 0) {
-      NDPI_LOG(NDPI_CONTENT_AVI, ndpi_struct, NDPI_LOG_DEBUG, "Avi content in http detected\n");
+      NDPI_LOG_INFO(ndpi_struct, "found Avi content in HTTP\n");
       ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_CONTENT_AVI);
     }
   }
@@ -126,7 +127,7 @@ static void teamviewer_check_http_payload(struct ndpi_detection_module_struct *n
   struct ndpi_packet_struct *packet = &flow->packet;
   const u_int8_t *pos;
 
-  NDPI_LOG(NDPI_PROTOCOL_TEAMVIEWER, ndpi_struct, NDPI_LOG_DEBUG, "called teamviewer_check_http_payload: %u %u %u\n",
+  NDPI_LOG_DBG2(ndpi_struct, "called teamviewer_check_http_payload: %u %u %u\n",
 	   packet->empty_line_position_set, flow->l4.tcp.http_empty_line_seen, packet->empty_line_position);
 
   if(packet->empty_line_position_set == 0 || (packet->empty_line_position + 5) > (packet->payload_packet_len))
@@ -135,7 +136,7 @@ static void teamviewer_check_http_payload(struct ndpi_detection_module_struct *n
   pos = &packet->payload[packet->empty_line_position] + 2;
 
   if(pos[0] == 0x17 && pos[1] == 0x24) {
-    NDPI_LOG(NDPI_PROTOCOL_TEAMVIEWER, ndpi_struct, NDPI_LOG_DEBUG, "TeamViewer content in http detected\n");
+    NDPI_LOG_INFO(ndpi_struct, "found TeamViewer content in HTTP\n");
     ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_TEAMVIEWER);
   }
 }
@@ -149,23 +150,27 @@ static void rtsp_parse_packet_acceptline(struct ndpi_detection_module_struct
   struct ndpi_packet_struct *packet = &flow->packet;
 
   if(packet->accept_line.len >= 28 && memcmp(packet->accept_line.ptr, "application/x-rtsp-tunnelled", 28) == 0) {
-    NDPI_LOG(NDPI_PROTOCOL_RTSP, ndpi_struct, NDPI_LOG_DEBUG, "RTSP accept line detected\n");
+    NDPI_LOG_INFO(ndpi_struct, "found RTSP accept line\n");
     ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_RTSP);
   }
 }
 #endif
 
 static void setHttpUserAgent(struct ndpi_flow_struct *flow, char *ua) {
-  if(!strcmp(ua, "Windows NT 5.0")) ua = "Windows 2000";
-  else if(!strcmp(ua, "Windows NT 5.1")) ua = "Windows XP";
-  else if(!strcmp(ua, "Windows NT 5.2")) ua = "Windows Server 2003";
-  else if(!strcmp(ua, "Windows NT 6.0")) ua = "Windows Vista";
-  else if(!strcmp(ua, "Windows NT 6.1")) ua = "Windows 7";
-  else if(!strcmp(ua, "Windows NT 6.2")) ua = "Windows 8";
-  else if(!strcmp(ua, "Windows NT 6.3")) ua = "Windows 8.1";
-
-  // printf("==> %s\n", ua);
-  snprintf((char*)flow->detected_os, sizeof(flow->detected_os), "%s", ua);
+  if (    !strcmp(ua, "Windows NT 5.0"))  ua = "Windows 2000";
+  else if(!strcmp(ua, "Windows NT 5.1"))  ua = "Windows XP";
+  else if(!strcmp(ua, "Windows NT 5.2"))  ua = "Windows Server 2003";
+  else if(!strcmp(ua, "Windows NT 6.0"))  ua = "Windows Vista";
+  else if(!strcmp(ua, "Windows NT 6.1"))  ua = "Windows 7";
+  else if(!strcmp(ua, "Windows NT 6.2"))  ua = "Windows 8";
+  else if(!strcmp(ua, "Windows NT 6.3"))  ua = "Windows 8.1";
+  else if(!strcmp(ua, "Windows NT 10.0")) ua = "Windows 10";
+
+  /* Good reference for future implementations:
+   * https://github.com/ua-parser/uap-core/blob/master/regexes.yaml */
+
+  //printf("==> %s\n", ua);
+  snprintf((char*)flow->protos.http.detected_os, sizeof(flow->protos.http.detected_os), "%s", ua);
 }
 
 static void parseHttpSubprotocol(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) {
@@ -196,23 +201,26 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
   u_int8_t a;
 
 
-#ifdef NDPI_PROTOCOL_PPSTREAM
+#if defined(NDPI_PROTOCOL_1KXUN) || defined(NDPI_PROTOCOL_IQIYI)
   /* PPStream */
-  if(flow->l4.tcp.ppstream_stage > 0 && iqiyi_counter == 0) {
-    NDPI_LOG(NDPI_PROTOCOL_PPSTREAM, ndpi_struct, NDPI_LOG_DEBUG, "PPStream found.\n");
-    ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_PPSTREAM);
+  if(flow->l4.tcp.ppstream_stage > 0 && flow->iqiyi_counter == 0) {
+    NDPI_LOG_INFO(ndpi_struct, "found PPStream\n");
+    /* ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_PPSTREAM); */
+    ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_PPSTREAM, NDPI_PROTOCOL_HTTP);
   }
-  else if(iqiyi_counter > 0) {
-    NDPI_LOG(NDPI_SERVICE_IQIYI, ndpi_struct, NDPI_LOG_DEBUG, "iQiyi found.\n");
-    ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_SERVICE_IQIYI);
+  else if(flow->iqiyi_counter > 0) {
+    NDPI_LOG_INFO(ndpi_struct, "found iQiyi\n");
+    /* ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_IQIYI); */
+    ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_IQIYI, NDPI_PROTOCOL_HTTP);
   }
 #endif
 
-#ifdef NDPI_SERVICE_1KXUN
+#if defined(NDPI_PROTOCOL_1KXUN) || defined(NDPI_PROTOCOL_IQIYI)
   /* 1KXUN */
-  if(kxun_counter > 0) {
-    NDPI_LOG(NDPI_SERVICE_1KXUN, ndpi_struct, NDPI_LOG_DEBUG, "1kxun found.\n");
-    ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_SERVICE_1KXUN);
+  if(flow->kxun_counter > 0) {
+    NDPI_LOG_INFO(ndpi_struct, "found 1kxun\n");
+    /* ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_1KXUN); */
+    ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_1KXUN, NDPI_PROTOCOL_HTTP);
   }
 #endif
 
@@ -232,27 +240,27 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
       }
 
       if(flow->packet.http_method.len < 3)
-	flow->http.method = HTTP_METHOD_UNKNOWN;
+        flow->http.method = HTTP_METHOD_UNKNOWN;
       else {
-	switch(flow->packet.http_method.ptr[0]) {
-	case 'O':  flow->http.method = HTTP_METHOD_OPTIONS; break;
-	case 'G':  flow->http.method = HTTP_METHOD_GET; break;
-	case 'H':  flow->http.method = HTTP_METHOD_HEAD; break;
-
-	case 'P':
-	  switch(flow->packet.http_method.ptr[1]) {
-	  case 'O': flow->http.method = HTTP_METHOD_POST; break;
-	  case 'U': flow->http.method = HTTP_METHOD_PUT; break;
-	  }
-	  break;
-
-	case 'D':   flow->http.method = HTTP_METHOD_DELETE; break;
-	case 'T':   flow->http.method = HTTP_METHOD_TRACE; break;
-	case 'C':   flow->http.method = HTTP_METHOD_CONNECT; break;
-	default:
-	  flow->http.method = HTTP_METHOD_UNKNOWN;
-	  break;
-	}
+        switch(flow->packet.http_method.ptr[0]) {
+        case 'O':  flow->http.method = HTTP_METHOD_OPTIONS; break;
+        case 'G':  flow->http.method = HTTP_METHOD_GET; break;
+        case 'H':  flow->http.method = HTTP_METHOD_HEAD; break;
+
+        case 'P':
+          switch(flow->packet.http_method.ptr[1]) {
+          case 'O': flow->http.method = HTTP_METHOD_POST; break;
+          case 'U': flow->http.method = HTTP_METHOD_PUT; break;
+          }
+          break;
+
+        case 'D':   flow->http.method = HTTP_METHOD_DELETE; break;
+        case 'T':   flow->http.method = HTTP_METHOD_TRACE; break;
+        case 'C':   flow->http.method = HTTP_METHOD_CONNECT; break;
+        default:
+          flow->http.method = HTTP_METHOD_UNKNOWN;
+          break;
+        }
       }
     }
 
@@ -270,8 +278,9 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
 
   if(packet->user_agent_line.ptr != NULL && packet->user_agent_line.len != 0) {
     /**
-       Format:
-       Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) ....
+       Format examples:
+         Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) ....
+         Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:54.0) Gecko/20100101 Firefox/54.0
     */
     if(packet->user_agent_line.len > 7) {
       char ua[256];
@@ -301,22 +310,22 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
 	      if(token && (token[0] == ' ')) token++; /* Skip space */
 
 	      if(token
-		 && ((strcmp(token, "U") == 0)
-		     || (strncmp(token, "MSIE", 4) == 0))) {
-		token = strsep(&parent, ";");
-		if(token && (token[0] == ' ')) token++; /* Skip space */
+		     && ((strcmp(token, "U") == 0)
+		         || (strncmp(token, "MSIE", 4) == 0))) {
+		    token = strsep(&parent, ";");
+		    if(token && (token[0] == ' ')) token++; /* Skip space */
 
-		if(token && (strncmp(token, "Update", 6)  == 0)) {
-		  token = strsep(&parent, ";");
+		    if(token && (strncmp(token, "Update", 6)  == 0)) {
+		      token = strsep(&parent, ";");
 
-		  if(token && (token[0] == ' ')) token++; /* Skip space */
+		      if(token && (token[0] == ' ')) token++; /* Skip space */
 
-		  if(token && (strncmp(token, "AOL", 3)  == 0)) {
-		    token = strsep(&parent, ";");
+		      if(token && (strncmp(token, "AOL", 3)  == 0)) {
 
-		    if(token && (token[0] == ' ')) token++; /* Skip space */
-		  }
-		}
+		        token = strsep(&parent, ";");
+		        if(token && (token[0] == ' ')) token++; /* Skip space */
+		      }
+		    }
 	      }
 	    }
 
@@ -325,9 +334,14 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
 	  }
 	}
       }
+      else if(memcmp(ua, "netflix-ios-app", 15) == 0) {
+	NDPI_LOG_INFO(ndpi_struct, "found netflix\n");
+      	ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_NETFLIX);
+      	return;
+      }
     }
 
-    NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "User Agent Type Line found %.*s\n",
+    NDPI_LOG_DBG2(ndpi_struct, "User Agent Type line found %.*s\n",
 	     packet->user_agent_line.len, packet->user_agent_line.ptr);
   }
 
@@ -335,9 +349,10 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
   if(packet->host_line.ptr != NULL) {
     u_int len;
 
-    NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "HOST Line found %.*s\n",
+    NDPI_LOG_DBG2(ndpi_struct, "HOST line found %.*s\n",
 	     packet->host_line.len, packet->host_line.ptr);
 
+    /* call ndpi_match_host_subprotocol to see if there is a match with known-host HTTP subprotocol */
     if((ndpi_struct->http_dont_dissect_response) || flow->http_detected)
       ndpi_match_host_subprotocol(ndpi_struct, flow,
 				  (char*)packet->host_line.ptr,
@@ -349,30 +364,45 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
     strncpy((char*)flow->host_server_name, (char*)packet->host_line.ptr, len);
     flow->host_server_name[len] = '\0', flow->server_id = flow->dst;
 
-    len = ndpi_min(packet->forwarded_line.len, sizeof(flow->nat_ip)-1);
-    strncpy((char*)flow->nat_ip, (char*)packet->forwarded_line.ptr, len);
-    flow->nat_ip[len] = '\0';
+    if(packet->forwarded_line.ptr) {
+        len = ndpi_min(packet->forwarded_line.len, sizeof(flow->protos.http.nat_ip)-1);
+        strncpy((char*)flow->protos.http.nat_ip, (char*)packet->forwarded_line.ptr, len);
+        flow->protos.http.nat_ip[len] = '\0';
+    }
 
     if(ndpi_struct->http_dont_dissect_response)
       parseHttpSubprotocol(ndpi_struct, flow);
 
-    if((flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN)
-       && ((ndpi_struct->http_dont_dissect_response) || flow->http_detected))
-      ndpi_match_host_subprotocol(ndpi_struct, flow,
-				    (char *)flow->host_server_name,
-				    strlen((const char *)flow->host_server_name),
-				    NDPI_PROTOCOL_HTTP);
+    /**
+       check result of host subprotocol detection
+
+       if "detected" in flow == 0 then "detected" = "guess"
+       else "guess" = "detected"
+    **/
+    if(flow->detected_protocol_stack[1] == 0) {
+      flow->detected_protocol_stack[1] = flow->guessed_protocol_id;
+      if(flow->detected_protocol_stack[0] == 0)
+    	flow->detected_protocol_stack[0] = flow->guessed_host_protocol_id;
+    }
+    else {
+      if(flow->detected_protocol_stack[1] != flow->guessed_protocol_id)
+	flow->guessed_protocol_id = flow->detected_protocol_stack[1];
+      if(flow->detected_protocol_stack[0] != flow->guessed_host_protocol_id)
+	flow->guessed_host_protocol_id = flow->detected_protocol_stack[0];
+    }
 
     if((flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN)
        && ((ndpi_struct->http_dont_dissect_response) || flow->http_detected)
        && (packet->http_origin.len > 0))
       ndpi_match_host_subprotocol(ndpi_struct, flow,
-				    (char *)packet->http_origin.ptr,
-				    packet->http_origin.len,
-				    NDPI_PROTOCOL_HTTP);
+				  (char *)packet->http_origin.ptr,
+				  packet->http_origin.len,
+				  NDPI_PROTOCOL_HTTP);
 
     if(flow->detected_protocol_stack[0] != NDPI_PROTOCOL_UNKNOWN) {
       if(packet->detected_protocol_stack[0] != NDPI_PROTOCOL_HTTP) {
+	NDPI_LOG_INFO(ndpi_struct, "found HTTP/%s\n", 
+		    ndpi_get_proto_name(ndpi_struct, packet->detected_protocol_stack[0]));
 	ndpi_int_http_add_connection(ndpi_struct, flow, packet->detected_protocol_stack[0]);
 	return; /* We have identified a sub-protocol so we're done */
       }
@@ -382,11 +412,12 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
   if(!ndpi_struct->http_dont_dissect_response && flow->http_detected)
     parseHttpSubprotocol(ndpi_struct, flow);
 
-  flow->guessed_protocol_id = NDPI_PROTOCOL_HTTP;
+  if(flow->guessed_protocol_id == NDPI_PROTOCOL_UNKNOWN)
+    flow->guessed_protocol_id = NDPI_PROTOCOL_HTTP;
 
   /* check for accept line */
   if(packet->accept_line.ptr != NULL) {
-    NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "Accept Line found %.*s\n",
+    NDPI_LOG_DBG2(ndpi_struct, "Accept line found %.*s\n",
 	     packet->accept_line.len, packet->accept_line.ptr);
 #ifdef NDPI_PROTOCOL_RTSP
     if(NDPI_COMPARE_PROTOCOL_TO_BITMASK(ndpi_struct->detection_bitmask, NDPI_PROTOCOL_RTSP) != 0) {
@@ -399,7 +430,7 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
 #ifdef NDPI_CONTENT_MPEG
   for (a = 0; a < packet->parsed_lines; a++) {
     if(packet->line[a].len > 11 && memcmp(packet->line[a].ptr, "Icy-MetaData", 12) == 0) {
-      NDPI_LOG(NDPI_CONTENT_MPEG, ndpi_struct, NDPI_LOG_DEBUG, "MPEG: Icy-MetaData found.\n");
+      NDPI_LOG_INFO(ndpi_struct, "found MPEG: Icy-MetaData\n");
       ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_CONTENT_MPEG);
       return;
     }
@@ -409,7 +440,7 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
 #endif
 
   if(packet->content_line.ptr != NULL && packet->content_line.len != 0) {
-    NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "Content Type Line found %.*s\n",
+    NDPI_LOG_DBG2(ndpi_struct, "Content Type line found %.*s\n",
 	     packet->content_line.len, packet->content_line.ptr);
 
     if((ndpi_struct->http_dont_dissect_response) || flow->http_detected)
@@ -417,13 +448,11 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
 				     (char*)packet->content_line.ptr, packet->content_line.len,
 				     NDPI_PROTOCOL_HTTP);
   }
-
-  /* check user agent here too */
 }
 
 static void check_http_payload(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow)
 {
-  NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "called check_http_payload.\n");
+  NDPI_LOG_DBG2(ndpi_struct, "called check_http_payload\n");
 
 #ifdef NDPI_CONTENT_FLASH
   if(NDPI_COMPARE_PROTOCOL_TO_BITMASK(ndpi_struct->detection_bitmask, NDPI_CONTENT_FLASH) != 0)
@@ -445,54 +474,55 @@ static void check_http_payload(struct ndpi_detection_module_struct *ndpi_struct,
  * @returnvalue 0 if no valid request has been found
  * @returnvalue >0 indicates start of filename but not necessarily in packet limit
  */
+
+#define STATIC_STRING_L(a) {.str=a, .len=sizeof(a)-1 }
+
+static struct l_string {
+	const char *str;
+	size_t     len;
+} http_methods[] = {
+	STATIC_STRING_L("GET "),
+	STATIC_STRING_L("POST "),
+	STATIC_STRING_L("OPTIONS "),
+	STATIC_STRING_L("HEAD "),
+	STATIC_STRING_L("PUT "),
+	STATIC_STRING_L("DELETE "),
+	STATIC_STRING_L("CONNECT "),
+	STATIC_STRING_L("PROPFIND "),
+	STATIC_STRING_L("REPORT ") };
+static const char *http_fs = "CDGHOPR";
+
+static uint8_t non_ctrl(uint8_t c) {
+	return c < 32 ? '.':c;
+}
+
 static u_int16_t http_request_url_offset(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow)
 {
   struct ndpi_packet_struct *packet = &flow->packet;
+  int i;
 
-  NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "====>>>> HTTP: %c%c%c%c [len: %u]\n",
-	   packet->payload[0], packet->payload[1], packet->payload[2], packet->payload[3],
+  NDPI_LOG_DBG2(ndpi_struct, "====>>>> HTTP: %c%c%c%c [len: %u]\n",
+	   non_ctrl(packet->payload[0]), non_ctrl(packet->payload[1]),
+	   non_ctrl(packet->payload[2]), non_ctrl(packet->payload[3]),
 	   packet->payload_packet_len);
 
+  /* Check first char */
+  if(!strchr(http_fs,packet->payload[0])) return 0;
   /**
      FIRST PAYLOAD PACKET FROM CLIENT
   **/
-
-  /* check if the packet starts with POST or GET */
-  if(packet->payload_packet_len >= 4 && memcmp(packet->payload, "GET ", 4) == 0) {
-    NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "HTTP: GET FOUND\n");
-    return 4;
-  } else if(packet->payload_packet_len >= 5 && memcmp(packet->payload, "POST ", 5) == 0) {
-    NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "HTTP: POST FOUND\n");
-    return 5;
-  } else if(packet->payload_packet_len >= 8 && memcmp(packet->payload, "OPTIONS ", 8) == 0) {
-    NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "HTTP: OPTIONS FOUND\n");
-    return 8;
-  } else if(packet->payload_packet_len >= 5 && memcmp(packet->payload, "HEAD ", 5) == 0) {
-    NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "HTTP: HEAD FOUND\n");
-    return 5;
-  } else if(packet->payload_packet_len >= 4 && memcmp(packet->payload, "PUT ", 4) == 0) {
-    NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "HTTP: PUT FOUND\n");
-    return 4;
-  } else if(packet->payload_packet_len >= 7 && memcmp(packet->payload, "DELETE ", 7) == 0) {
-    NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "HTTP: DELETE FOUND\n");
-    return 7;
-  } else if(packet->payload_packet_len >= 8 && memcmp(packet->payload, "CONNECT ", 8) == 0) {
-    NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "HTTP: CONNECT FOUND\n");
-    return 8;
-  } else if(packet->payload_packet_len >= 9 && memcmp(packet->payload, "PROPFIND ", 9) == 0) {
-    NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "HTTP: PROFIND FOUND\n");
-    return 9;
-  } else if(packet->payload_packet_len >= 7 && memcmp(packet->payload, "REPORT ", 7) == 0) {
-    NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "HTTP: REPORT FOUND\n");
-    return 7;
+  for(i=0; i < sizeof(http_methods)/sizeof(http_methods[0]); i++) {
+	if(packet->payload_packet_len >= http_methods[i].len &&
+	   memcmp(packet->payload,http_methods[i].str,http_methods[i].len) == 0) {
+		NDPI_LOG_DBG2(ndpi_struct, "HTTP: %sFOUND\n",http_methods[i].str);
+		return http_methods[i].len;
+	}
   }
-
   return 0;
 }
 
-static void http_bitmask_exclude(struct ndpi_flow_struct *flow)
+static void http_bitmask_exclude_other(struct ndpi_flow_struct *flow)
 {
-  NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_PROTOCOL_HTTP);
 #ifdef NDPI_CONTENT_MPEG
   NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_CONTENT_MPEG);
 #endif
@@ -519,270 +549,82 @@ static void http_bitmask_exclude(struct ndpi_flow_struct *flow)
 #endif
 }
 
-void _org_ndpi_search_http_tcp(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow)
-{
-  struct ndpi_packet_struct *packet = &flow->packet;
-
-  //      struct ndpi_id_struct         *src=ndpi_struct->src;
-  //      struct ndpi_id_struct         *dst=ndpi_struct->dst;
-
-  u_int16_t filename_start;
-
-  NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "search http\n");
-
-  /* set client-server_direction */
-  if(flow->l4.tcp.http_setup_dir == 0) {
-    NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "initializes http to stage: 1 \n");
-    flow->l4.tcp.http_setup_dir = 1 + packet->packet_direction;
-  }
-
-  if(NDPI_COMPARE_PROTOCOL_TO_BITMASK
-     (ndpi_struct->generic_http_packet_bitmask, packet->detected_protocol_stack[0]) != 0) {
-    NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG,
-	     "protocol might be detected earlier as http jump to payload type detection\n");
-    goto http_parse_detection;
-  }
-
-  if(flow->l4.tcp.http_setup_dir == 1 + packet->packet_direction) {
-    NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "http stage: 1\n");
-
-    if(flow->l4.tcp.http_wait_for_retransmission) {
-      if(!packet->tcp_retransmission) {
-	if(flow->packet_counter <= 5) {
-	  NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "still waiting for retransmission\n");
-	  return;
-	} else {
-	  NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "retransmission not found, exclude\n");
-	  http_bitmask_exclude(flow);
-	  return;
-	}
-      }
-    }
-
-    if(flow->l4.tcp.http_stage == 0) {
-      filename_start = http_request_url_offset(ndpi_struct, flow);
-      if(filename_start == 0) {
-	if(packet->payload_packet_len >= 7 && memcmp(packet->payload, "HTTP/1.", 7) == 0) {
-	  NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "HTTP response found (truncated flow ?)\n");
-	  ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP);
-	  return;
-	}
-
-	NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "filename not found, exclude\n");
-	http_bitmask_exclude(flow);
-	return;
-      }
-      // parse packet
-      ndpi_parse_packet_line_info(ndpi_struct, flow);
-
-      if(packet->parsed_lines <= 1) {
-	/* parse one more packet .. */
-	NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "just one line, search next packet\n");
-
-	packet->http_method.ptr = packet->line[0].ptr;
-        packet->http_method.len = filename_start - 1;
-	flow->l4.tcp.http_stage = 1;
-	return;
-      }
-      // parsed_lines > 1 here
-      if(packet->line[0].len >= (9 + filename_start)
-	 && memcmp(&packet->line[0].ptr[packet->line[0].len - 9], " HTTP/1.", 8) == 0) {
-	u_int16_t proto_id;
-
-	packet->http_url_name.ptr = &packet->payload[filename_start];
-	packet->http_url_name.len = packet->line[0].len - (filename_start + 9);
-
-	packet->http_method.ptr = packet->line[0].ptr;
-	packet->http_method.len = filename_start - 1;
-
-	NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "http structure detected, adding\n");
-
-	if(filename_start == 8 && (memcmp(packet->payload, "CONNECT ", 8) == 0)) /* nathan@getoffmalawn.com */
-	  proto_id = NDPI_PROTOCOL_HTTP_CONNECT;
-	else {
-	  if((packet->http_url_name.len > 7) && (!strncmp((const char*)packet->http_url_name.ptr, "http://", 7)))
-	    proto_id = NDPI_PROTOCOL_HTTP_PROXY;
-	  else {
-	    proto_id = NDPI_PROTOCOL_HTTP;
-	  }
-	}
-
-	ndpi_int_http_add_connection(ndpi_struct, flow, proto_id);
-	check_content_type_and_change_protocol(ndpi_struct, flow);
-	/* HTTP found, look for host... */
-	if(packet->host_line.ptr != NULL) {
-	  /* aaahh, skip this direction and wait for a server reply here */
-	  flow->l4.tcp.http_stage = 2;
-	  NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "HTTP START HOST found\n");
-	  return;
-	}
-	NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "HTTP START HOST found\n");
-
-	/* host not found, check in next packet after */
-	flow->l4.tcp.http_stage = 1;
-	return;
-      }
-    } else if(flow->l4.tcp.http_stage == 1) {
-      /* SECOND PAYLOAD TRAFFIC FROM CLIENT, FIRST PACKET MIGHT HAVE BEEN HTTP... */
-      /* UNKNOWN TRAFFIC, HERE FOR HTTP again.. */
-      // parse packet
-      ndpi_parse_packet_line_info(ndpi_struct, flow);
-
-      if(packet->parsed_lines <= 1) {
-	/* wait some packets in case request is split over more than 2 packets */
-	if(flow->packet_counter < 5) {
-	  NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG,
-		   "line still not finished, search next packet\n");
-	  return;
-	} else {
-	  /* stop parsing here */
-	  NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG,
-		   "HTTP: PACKET DOES NOT HAVE A LINE STRUCTURE\n");
-	  http_bitmask_exclude(flow);
-	  return;
-	}
-      }
-      // http://www.slideshare.net/DSPIP/rtsp-analysis-wireshark
-      if(packet->line[0].len >= 9 && memcmp(&packet->line[0].ptr[packet->line[0].len - 9], " HTTP/1.", 8) == 0) {
-	ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP);
-	check_content_type_and_change_protocol(ndpi_struct, flow);
-	NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG,
-		 "HTTP START HTTP found in 2. packet, check host here...\n");
-	/* HTTP found, look for host... */
-	flow->l4.tcp.http_stage = 2;
-
-	return;
-      }
-    }
-  } else {
-    /* We have received a response for a previously identified partial HTTP request */
-
-    if((packet->parsed_lines == 1) && (packet->packet_direction == 1 /* server -> client */)) {
-      /*
-	In apache if you do "GET /\n\n" the response comes without any header so we can assume that
-	this can be the case
-      */
-      ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP);
-      return;
-    }
-
-  }
-
-  NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "HTTP: REQUEST NOT HTTP CONFORM\n");
-  http_bitmask_exclude(flow);
-  return;
-
- http_parse_detection:
-  if(flow->l4.tcp.http_setup_dir == 1 + packet->packet_direction) {
-    /* we have something like http here, so check for host and content type if possible */
-    if(flow->l4.tcp.http_stage == 0 || flow->l4.tcp.http_stage == 3) {
-      NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "HTTP RUN MAYBE NEXT GET/POST...\n");
-      // parse packet
-      ndpi_parse_packet_line_info(ndpi_struct, flow);
-      /* check for url here */
-      filename_start = http_request_url_offset(ndpi_struct, flow);
-      if(filename_start != 0 && packet->parsed_lines > 1 && packet->line[0].len >= (9 + filename_start)
-	 && memcmp(&packet->line[0].ptr[packet->line[0].len - 9], " HTTP/1.", 8) == 0) {
-	packet->http_url_name.ptr = &packet->payload[filename_start];
-	packet->http_url_name.len = packet->line[0].len - (filename_start + 9);
-
-	packet->http_method.ptr = packet->line[0].ptr;
-	packet->http_method.len = filename_start - 1;
-
-	NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "next http action, "
-		 "resetting to http and search for other protocols later.\n");
-	ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP);
-      }
-      check_content_type_and_change_protocol(ndpi_struct, flow);
-      /* HTTP found, look for host... */
-      if(packet->host_line.ptr != NULL) {
-	NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG,
-		 "HTTP RUN MAYBE NEXT HOST found, skipping all packets from this direction\n");
-	/* aaahh, skip this direction and wait for a server reply here */
-	flow->l4.tcp.http_stage = 2;
-	return;
-      }
-      NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG,
-	       "HTTP RUN MAYBE NEXT HOST NOT found, scanning one more packet from this direction\n");
-      flow->l4.tcp.http_stage = 1;
-    } else if(flow->l4.tcp.http_stage == 1) {
-      // parse packet and maybe find a packet info with host ptr,...
-      ndpi_parse_packet_line_info(ndpi_struct, flow);
-      check_content_type_and_change_protocol(ndpi_struct, flow);
-      NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "HTTP RUN second packet scanned\n");
-      /* HTTP found, look for host... */
-      flow->l4.tcp.http_stage = 2;
-    }
-    NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG,
-	     "HTTP skipping client packets after second packet\n");
-    return;
-  }
-  /* server response */
-  if(flow->l4.tcp.http_stage > 0) {
-    /* first packet from server direction, might have a content line */
-    ndpi_parse_packet_line_info(ndpi_struct, flow);
-    check_content_type_and_change_protocol(ndpi_struct, flow);
-
-    if(packet->empty_line_position_set != 0 || flow->l4.tcp.http_empty_line_seen == 1) {
-      NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "empty line. check_http_payload.\n");
-      check_http_payload(ndpi_struct, flow);
-    }
-
-    if(flow->l4.tcp.http_stage == 2) {
-      flow->l4.tcp.http_stage = 3;
-    } else {
-      flow->l4.tcp.http_stage = 0;
-    }
-    NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG,
-	     "HTTP response first or second packet scanned,new stage is: %u\n", flow->l4.tcp.http_stage);
-    return;
-  } else {
-    NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "HTTP response next packet skipped\n");
-  }
-}
-
 /*************************************************************************************************/
 
 static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct,
-				struct ndpi_flow_struct *flow) {
+				struct ndpi_flow_struct *flow) {       
   struct ndpi_packet_struct *packet = &flow->packet;
-  u_int16_t filename_start;
+  u_int16_t filename_start; /* the filename in the request method line, e.g., "GET filename_start..."*/
 
   packet->packet_lines_parsed_complete = 0;
 
   /* Check if we so far detected the protocol in the request or not. */
   if(flow->l4.tcp.http_stage == 0) {
+    /* Expected a request */
     flow->http_detected = 0;
 
-    NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "HTTP stage %d: \n",
-	     flow->l4.tcp.http_stage);
+    NDPI_LOG_DBG2(ndpi_struct, "HTTP stage %d: \n", flow->l4.tcp.http_stage);
 
     filename_start = http_request_url_offset(ndpi_struct, flow);
 
+    if(filename_start == 0) { /* not a regular request. In the HTTP first stage, may be a truncated flow or other protocols */
+      NDPI_LOG_DBG2(ndpi_struct, "Filename HTTP not found, we look for possible truncate flow..\n");
 
-    if(filename_start == 0) {
-      NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG,
-	       "Filename HTTP not found, we look for possible truncate flow...\n");
       if(packet->payload_packet_len >= 7 && memcmp(packet->payload, "HTTP/1.", 7) == 0) {
-        NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG,
-		 "HTTP response found (truncated flow ?)\n");
+        NDPI_LOG_INFO(ndpi_struct, "found HTTP response\n");
         ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP);
         check_content_type_and_change_protocol(ndpi_struct, flow);
         return;
       }
 
-      NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "Exclude HTTP\n");
-      http_bitmask_exclude(flow);
+      if((packet->payload_packet_len == 3) && memcmp(packet->payload, "HI\n", 3) == 0) {
+	/* This looks like Ookla: we don't give up with HTTP yet */
+        flow->l4.tcp.http_stage = 1;
+	return;
+      }
+      
+      if((packet->payload_packet_len == 40) && (flow->l4.tcp.http_stage == 0)) {
+        /*
+	  -> QR O06L0072-6L91-4O43-857J-K8OO172L6L51
+	  <- QNUUX 2.5 2017-08-15.1314.4jn12m5
+	  -> MXFWUXJM 31625365
+	*/
+
+        if((packet->payload[2] == ' ')
+	   && (packet->payload[11] == '-')
+	   && (packet->payload[16] == '-')
+	   && (packet->payload[21] == '-')
+	   && (packet->payload[26] == '-')
+	   && (packet->payload[39] == 0x0A)
+		)
+		flow->l4.tcp.http_stage = 1;
+	return;	      
+      }
+      
+      if((packet->payload_packet_len == 23) && (memcmp(packet->payload, "<policy-file-request/>", 23) == 0)) {
+        /*
+          <policy-file-request/>
+          <cross-domain-policy>
+          <allow-access-from domain="*.ookla.com" to-ports="8080"/>
+          <allow-access-from domain="*.speedtest.net" to-ports="8080"/>
+          </cross-domain-policy>
+        */
+        ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_UNKNOWN);
+        return;
+      }
+      
+      NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
+      http_bitmask_exclude_other(flow);
       return;
     }
 
-    NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG,
+    NDPI_LOG_DBG2(ndpi_struct,
 	     "Filename HTTP found: %d, we look for line info..\n", filename_start);
 
     ndpi_parse_packet_line_info(ndpi_struct, flow);
 
     if(packet->parsed_lines <= 1) {
-      NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG,
+      NDPI_LOG_DBG2(ndpi_struct,
 	       "Found just one line, we will look further for the next packet...\n");
 
       packet->http_method.ptr = packet->line[0].ptr;
@@ -793,11 +635,11 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
       return;
     }
 
-    NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG,
+    NDPI_LOG_DBG2(ndpi_struct,
 	     "Found more than one line, we look further for the next packet...\n");
 
     if(packet->line[0].len >= (9 + filename_start)
-        && memcmp(&packet->line[0].ptr[packet->line[0].len - 9], " HTTP/1.", 8) == 0) {
+        && memcmp(&packet->line[0].ptr[packet->line[0].len - 9], " HTTP/1.", 8) == 0) { /* Request line complete. Ex. "GET / HTTP/1.1" */
 
       packet->http_url_name.ptr = &packet->payload[filename_start];
       packet->http_url_name.len = packet->line[0].len - (filename_start + 9);
@@ -805,133 +647,176 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
       packet->http_method.ptr = packet->line[0].ptr;
       packet->http_method.len = filename_start - 1;
 
+      // Set the HTTP requested version: 0=HTTP/1.0 and 1=HTTP/1.1
+      if(memcmp(&packet->line[0].ptr[packet->line[0].len - 1], "1", 1) == 0)
+          flow->http.request_version = 1;
+      else
+          flow->http.request_version = 0;
+
+      /* Set the first found headers in request */
+      flow->http.num_request_headers = packet->http_num_headers;
+
+
+      /* Check for Ookla */
+      if((packet->referer_line.len > 0)
+	      && ndpi_strnstr((const char *)packet->referer_line.ptr, "www.speedtest.net", packet->referer_line.len)) {
+	    ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_HTTP);
+	    return;
+      }
+
       /* Check for additional field introduced by Steam */
       int x = 1;
       if((memcmp(packet->line[x].ptr, "x-steam-sid", 11)) == 0) {
-	ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_STEAM);
-	check_content_type_and_change_protocol(ndpi_struct, flow);
-	return;
+	    NDPI_LOG_INFO(ndpi_struct, "found STEAM\n");
+	    ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_STEAM);
+	    check_content_type_and_change_protocol(ndpi_struct, flow);
+	    return;
       }
-      
+
       /* Check for additional field introduced by Facebook */
       x = 1;
       while(packet->line[x].len != 0) {
-	if((memcmp(packet->line[x].ptr, "X-FB-SIM-HNI", 12)) == 0) {
-	  ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_SERVICE_FACEBOOK);
-	  check_content_type_and_change_protocol(ndpi_struct, flow);
-	  return;
-	}
-	x++;
+	    if(packet->line[x].len >= 12 && (memcmp(packet->line[x].ptr, "X-FB-SIM-HNI", 12)) == 0) {
+	      NDPI_LOG_INFO(ndpi_struct, "found FACEBOOK\n");
+	      ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_FACEBOOK);
+	      check_content_type_and_change_protocol(ndpi_struct, flow);
+	      return;
+	    }
+	    x++;
       }
 
+#if defined(NDPI_PROTOCOL_1KXUN) || defined(NDPI_PROTOCOL_IQIYI)
       /* check PPStream protocol or iQiyi service
-	 (iqiyi is deliverd by ppstream) */
+	 (iqiyi is delivered by ppstream) */
       // substring in url
-      if(strstr((const char*) &packet->payload[filename_start], "iqiyi.com") != NULL) {
-	if(kxun_counter == 0) {
+      if(ndpi_strnstr((const char*) &packet->payload[filename_start], "iqiyi.com", (packet->payload_packet_len - filename_start)) != NULL) {
+	if(flow->kxun_counter == 0) {
 	  flow->l4.tcp.ppstream_stage++;
-	  iqiyi_counter++;
+	  flow->iqiyi_counter++;
 	  check_content_type_and_change_protocol(ndpi_struct, flow); /* ***** CHECK ****** */
 	  return;
 	}
       }
+
       // additional field in http payload
       x = 1;
-      while(packet->line[x].len != 0) {
-	if((memcmp(packet->line[x].ptr, "qyid", 4)) == 0 &&
-	   (memcmp(packet->line[x+1].ptr, "qypid", 5)) == 0 &&
-	   (memcmp(packet->line[x+2].ptr, "qyplatform", 10)) == 0) {
+      while((packet->line[x].len >= 4) && (packet->line[x+1].len >= 5) && (packet->line[x+2].len >= 10)) {
+	if(packet->line[x].ptr && ((memcmp(packet->line[x].ptr, "qyid", 4)) == 0)
+	   && packet->line[x+1].ptr && ((memcmp(packet->line[x+1].ptr, "qypid", 5)) == 0)
+	   && packet->line[x+2].ptr && ((memcmp(packet->line[x+2].ptr, "qyplatform", 10)) == 0)
+	   ) {
 	  flow->l4.tcp.ppstream_stage++;
-	  iqiyi_counter++;
+	  flow->iqiyi_counter++;
 	  check_content_type_and_change_protocol(ndpi_struct, flow);
 	  return;
 	}
 	x++;
       }
+#endif
 
+#if defined(NDPI_PROTOCOL_1KXUN) || defined(NDPI_PROTOCOL_IQIYI)
       /* Check for 1kxun packet */
       int a;
       for (a = 0; a < packet->parsed_lines; a++) {
-	if((memcmp(packet->line[a].ptr, "Client-Source:", 14)) == 0) {
+	if(packet->line[a].len >= 14 && (memcmp(packet->line[a].ptr, "Client-Source:", 14)) == 0) {
 	  if((memcmp(packet->line[a].ptr+15, "1kxun", 5)) == 0) {
-	    kxun_counter++;
+	    flow->kxun_counter++;
 	    check_content_type_and_change_protocol(ndpi_struct, flow);
 	    return;
 	  }
 	}
       }
-
+#endif
+      
       if((packet->http_url_name.len > 7)
           && (!strncmp((const char*) packet->http_url_name.ptr, "http://", 7))) {
-        NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "HTTP_PROXY Found.\n");
+        NDPI_LOG_INFO(ndpi_struct, "found HTTP_PROXY\n");
         ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP_PROXY);
         check_content_type_and_change_protocol(ndpi_struct, flow);
       }
 
-      if(filename_start == 8 && (memcmp(packet->payload, "CONNECT ", 8) == 0)) /* nathan@getoffmalawn.com */
-      {
-        NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "HTTP_CONNECT Found.\n");
+      if(filename_start == 8 && (memcmp(packet->payload, "CONNECT ", 8) == 0)) {
+	    /* nathan@getoffmalawn.com */
+        NDPI_LOG_INFO(ndpi_struct, "found HTTP_CONNECT\n");
         ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP_CONNECT);
         check_content_type_and_change_protocol(ndpi_struct, flow);
       }
 
-      NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG,
+      NDPI_LOG_DBG2(ndpi_struct,
           "HTTP START Found, we will look for sub-protocols (content and host)...\n");
 
       if(packet->host_line.ptr != NULL) {
-	/**
-	   nDPI is pretty scrupoulous about HTTP so it waits until the
-	   HTTP response is received just to check that it conforms
-	   with the HTTP specs. However this might be a waste of time as
-	   in 99.99% of the cases is like that.
-	*/
-
-	if(ndpi_struct->http_dont_dissect_response) {
-	  if(flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN) /* No subprotocol found */
-	    ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP);
-	} else {
-	  flow->http_detected = 1;
-	  NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG,
-		   "HTTP START Found, we will look further for the response...\n");
-	  flow->l4.tcp.http_stage = packet->packet_direction + 1; // packet_direction 0: stage 1, packet_direction 1: stage 2
-	}
-
-	check_content_type_and_change_protocol(ndpi_struct, flow);
+        /**
+           nDPI is pretty scrupulous about HTTP so it waits until the
+           HTTP response is received just to check that it conforms
+           with the HTTP specs. However this might be a waste of time as
+           in 99.99% of the cases is like that.
+        */
+
+        if(ndpi_struct->http_dont_dissect_response) {
+          if(flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN) /* No subprotocol found */
+	    NDPI_LOG_INFO(ndpi_struct, "found HTTP\n");
+            ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP);
+        } else {
+          flow->http_detected = 1;
+          NDPI_LOG_DBG2(ndpi_struct,
+               "HTTP START Found, we will look further for the response...\n");
+          flow->l4.tcp.http_stage = packet->packet_direction + 1; // packet_direction 0: stage 1, packet_direction 1: stage 2
+        }
 
-	return;
+        check_content_type_and_change_protocol(ndpi_struct, flow);
+        return;
       }
     }
 
-    NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "HTTP: REQUEST NOT HTTP CONFORM\n");
-    http_bitmask_exclude(flow);
+    NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
+    http_bitmask_exclude_other(flow);
+
   } else if((flow->l4.tcp.http_stage == 1) || (flow->l4.tcp.http_stage == 2)) {
-    NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "HTTP stage %u: \n",
-	     flow->l4.tcp.http_stage);
 
+    NDPI_LOG_DBG2(ndpi_struct, "HTTP stage %u: \n", flow->l4.tcp.http_stage);
+    
+    if((packet->payload_packet_len == 34) && (flow->l4.tcp.http_stage == 1)) {
+      if((packet->payload[5] == ' ') && (packet->payload[9] == ' ')) {
+	      ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA);
+	      return;
+      }
+    }
+    
+    if((packet->payload_packet_len > 6) && memcmp(packet->payload, "HELLO ", 6) == 0) {
+       /* This looks like Ookla */
+      ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_UNKNOWN);
+      return;
+    } else
+	    NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_PROTOCOL_OOKLA);    
+    
     /**
-       At first check, if this is for sure a response packet (in another direction. If not, if http is detected do nothing now and return,
-       otherwise check the second packet for the http request . */
-    if((flow->l4.tcp.http_stage - packet->packet_direction) == 1) {
+       At first check, if this is for sure a response packet (in another direction. If not, if HTTP is detected do nothing now and return,
+       otherwise check the second packet for the HTTP request
+    */
+    if((flow->l4.tcp.http_stage - packet->packet_direction) == 1) { /* Expected a response package */
 
       if(flow->http_detected)
         return;
 
-      NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG,
+      NDPI_LOG_DBG2(ndpi_struct,
 	       " SECOND PAYLOAD TRAFFIC FROM CLIENT, FIRST PACKET MIGHT HAVE BEEN HTTP...UNKNOWN TRAFFIC, HERE FOR HTTP again.. \n");
 
       ndpi_parse_packet_line_info(ndpi_struct, flow);
 
+      // Add more found HTTP request headers.
+      flow->http.num_request_headers+=packet->http_num_headers;
+
       if(packet->parsed_lines <= 1) {
         /* wait some packets in case request is split over more than 2 packets */
         if(flow->packet_counter < 5) {
-          NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG,
-		   "line still not finished, search next packet\n");
+          NDPI_LOG_DBG2(ndpi_struct, "line still not finished, search next packet\n");
           return;
         } else {
           /* stop parsing here */
-          NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG,
-		   "HTTP: PACKET DOES NOT HAVE A LINE STRUCTURE\n");
-          http_bitmask_exclude(flow);
+          NDPI_LOG_DBG2(ndpi_struct, "exclude HTTP: PACKET DOES NOT HAVE A LINE STRUCTURE\n");
+	  NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
+          http_bitmask_exclude_other(flow);
           return;
         }
       }
@@ -939,11 +824,11 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
       if(packet->line[0].len >= 9
           && memcmp(&packet->line[0].ptr[packet->line[0].len - 9], " HTTP/1.", 8) == 0) {
 
-        NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "Found HTTP.\n");
+        NDPI_LOG_INFO(ndpi_struct, "found HTTP\n");
         ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP);
         check_content_type_and_change_protocol(ndpi_struct, flow);
 
-        NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG,
+        NDPI_LOG_DBG2(ndpi_struct,
 		 "HTTP START Found in 2. packet, we will look further for the response....\n");
         flow->http_detected = 1;
       }
@@ -956,31 +841,39 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
        We have received a response for a previously identified partial HTTP request
     */
 
+    /* response without headers
+     * TODO: Shouldn't it be below  ndpi_parse_packet_line_info, line ~825 ?
+     */
     if((packet->parsed_lines == 1) && (packet->packet_direction == 1 /* server -> client */)) {
-      /* In apache if you do "GET /\n\n" the response comes without any header */
-      NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "Found HTTP. (apache)\n");
+      /* In Apache if you do "GET /\n\n" the response comes without any header */
+      NDPI_LOG_INFO(ndpi_struct, "found HTTP. (apache)\n");
       ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP);
       check_content_type_and_change_protocol(ndpi_struct, flow);
       return;
     }
 
-    /* If we already detected the http request, we can add the connection and then check for the sub-protocol */
-    if(flow->http_detected)
+    /* If we already detected the HTTP request, we can add the connection and then check for the sub-protocol */
+    if(flow->http_detected) {
+      NDPI_LOG_INFO(ndpi_struct, "found HTTP\n");
       ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP);
+    }
 
     /* Parse packet line and we look for the subprotocols */
     ndpi_parse_packet_line_info(ndpi_struct, flow);
     check_content_type_and_change_protocol(ndpi_struct, flow);
 
+    if(packet->packet_direction == 1 /* server -> client */){
+        flow->http.num_response_headers += packet->http_num_headers; /* flow structs are initialized with zeros */
+    }
+
     if(packet->empty_line_position_set != 0 || flow->l4.tcp.http_empty_line_seen == 1) {
-      NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "empty line. check_http_payload.\n");
+      NDPI_LOG_DBG2(ndpi_struct, "empty line. check_http_payload\n");
       check_http_payload(ndpi_struct, flow);
     }
 
     flow->l4.tcp.http_stage = 0;
     return;
   }
-
 }
 
 void ndpi_search_http_tcp(struct ndpi_detection_module_struct *ndpi_struct,
@@ -989,8 +882,8 @@ void ndpi_search_http_tcp(struct ndpi_detection_module_struct *ndpi_struct,
 
   /* Break after 20 packets. */
   if(flow->packet_counter > 20) {
-    NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "Exclude HTTP.\n");
-    http_bitmask_exclude(flow);
+    NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
+    http_bitmask_exclude_other(flow);
     return;
   }
 
@@ -998,7 +891,7 @@ void ndpi_search_http_tcp(struct ndpi_detection_module_struct *ndpi_struct,
      return;
    }
 
-  NDPI_LOG(NDPI_PROTOCOL_HTTP, ndpi_struct, NDPI_LOG_DEBUG, "HTTP detection...\n");
+  NDPI_LOG_DBG(ndpi_struct, "search HTTP\n");
   ndpi_check_http_tcp(ndpi_struct, flow);
 }
 
@@ -1158,7 +1051,7 @@ void init_http_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int
   NDPI_BITMASK_SET(ndpi_struct->callback_buffer[a].excluded_protocol_bitmask,
   		   ndpi_struct->callback_buffer[a].detection_bitmask);
 
-  /*Delete protocol from exluded protocol bitmask*/
+  /*Delete protocol from excluded protocol bitmask*/
   NDPI_DEL_PROTOCOL_FROM_BITMASK(ndpi_struct->callback_buffer[a].excluded_protocol_bitmask, NDPI_PROTOCOL_UNKNOWN);
 
   NDPI_DEL_PROTOCOL_FROM_BITMASK(ndpi_struct->callback_buffer[a].excluded_protocol_bitmask, NDPI_PROTOCOL_QQ);