HTTP: improve extraction of metadata and of flow risks (#1959)

2 files changed, 260 insertions, 253 deletions

diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h
index 943de5118..aea6120df 100644
--- a/src/include/ndpi_typedefs.h
+++ b/src/include/ndpi_typedefs.h
@@ -742,7 +742,6 @@ struct ndpi_flow_tcp_struct {
 
   /* NDPI_PROTOCOL_HTTP */
   u_int32_t http_stage:2;
-  u_int32_t http_empty_line_seen:1;
 
   /* NDPI_PROTOCOL_GNUTELLA */
   u_int32_t gnutella_stage:2;		       // 0 - 2
diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c
index a54acf4bb..320fd22e4 100644
--- a/src/lib/protocols/http.c
+++ b/src/lib/protocols/http.c
@@ -51,6 +51,8 @@ extern void ookla_add_to_cache(struct ndpi_detection_module_struct *ndpi_struct,
 
 static void ndpi_search_http_tcp(struct ndpi_detection_module_struct *ndpi_struct,
 				 struct ndpi_flow_struct *flow);
+static void ndpi_check_http_header(struct ndpi_detection_module_struct *ndpi_struct,
+				   struct ndpi_flow_struct *flow);
 
 /* *********************************************** */
 
@@ -101,6 +103,9 @@ static void ndpi_analyze_content_signature(struct ndpi_detection_module_struct *
 
 static int ndpi_search_http_tcp_again(struct ndpi_detection_module_struct *ndpi_struct,
 				      struct ndpi_flow_struct *flow) {
+  struct ndpi_packet_struct *packet = &ndpi_struct->packet;
+  if(packet->payload_packet_len == 0 || packet->tcp_retransmission)
+    return 1;
 
   ndpi_search_http_tcp(ndpi_struct, flow);
 
@@ -339,8 +344,7 @@ static ndpi_protocol_category_t ndpi_http_check_content(struct ndpi_detection_mo
 
 static void ndpi_int_http_add_connection(struct ndpi_detection_module_struct *ndpi_struct,
 					 struct ndpi_flow_struct *flow,
-					 u_int16_t http_protocol,
-					 ndpi_protocol_category_t category) {
+					 u_int16_t http_protocol) {
   u_int16_t master_protocol;
 
 #ifdef HTTP_DEBUG
@@ -401,53 +405,124 @@ static void setHttpUserAgent(struct ndpi_detection_module_struct *ndpi_struct,
 /* ************************************************************* */
 
 static void ndpi_http_parse_subprotocol(struct ndpi_detection_module_struct *ndpi_struct,
-					struct ndpi_flow_struct *flow) {
+					struct ndpi_flow_struct *flow,
+					int hostname_just_set) {
   u_int16_t master_protocol;
+  struct ndpi_packet_struct *packet = &ndpi_struct->packet;
 
-  if((flow->l4.tcp.http_stage == 0) || (flow->http.url && flow->http_detected)) {
-    char *double_col = strchr((char*)flow->host_server_name, ':');
-    int a, b, c, d;
-    
-    if(double_col) double_col[0] = '\0';
+  master_protocol = NDPI_PROTOCOL_HTTP;
+  if(flow->detected_protocol_stack[1] != NDPI_PROTOCOL_UNKNOWN)
+    master_protocol = flow->detected_protocol_stack[1];
+  else if(flow->detected_protocol_stack[0] == NDPI_PROTOCOL_HTTP_CONNECT ||
+          flow->detected_protocol_stack[0] == NDPI_PROTOCOL_HTTP_PROXY)
+    master_protocol = flow->detected_protocol_stack[0];
 
-    if(ndpi_struct->packet.iph
-       && (sscanf(flow->host_server_name, "%d.%d.%d.%d", &a, &b, &c, &d) == 4)) {
-      /* IPv4 */
+  if(packet->server_line.len > 7 &&
+     strncmp((const char *)packet->server_line.ptr, "ntopng ", 7) == 0) {
+    ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_NTOP, NDPI_PROTOCOL_HTTP, NDPI_CONFIDENCE_DPI);
+    ndpi_unset_risk(ndpi_struct, flow, NDPI_KNOWN_PROTOCOL_ON_NON_STANDARD_PORT);
+  }
 
-      if(ndpi_struct->packet.iph->daddr != inet_addr(flow->host_server_name)) {
-	char buf[64], msg[128];
-	
-	snprintf(msg, sizeof(msg), "Expected %s, found %s",
-		 ndpi_intoav4(ntohl(ndpi_struct->packet.iph->daddr), buf, sizeof(buf)), flow->host_server_name);
-	ndpi_set_risk(ndpi_struct, flow, NDPI_HTTP_SUSPICIOUS_HEADER, msg);
+  /* Matching on Content-Type.
+      OCSP:  application/ocsp-request, application/ocsp-response
+  */
+  /* We overwrite any previous sub-classification (example: via hostname) */
+  if(packet->content_line.len > 17 &&
+     strncmp((const char *)packet->content_line.ptr, "application/ocsp-", 17) == 0) {
+    NDPI_LOG_DBG2(ndpi_struct, "Found OCSP\n");
+    ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OCSP, master_protocol, NDPI_CONFIDENCE_DPI);
+  }
+
+  if(flow->http.method == NDPI_HTTP_METHOD_RPC_IN_DATA ||
+     flow->http.method == NDPI_HTTP_METHOD_RPC_OUT_DATA) {
+    ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_RPC, master_protocol, NDPI_CONFIDENCE_DPI);
+  }
+
+  if(flow->detected_protocol_stack[1] == NDPI_PROTOCOL_UNKNOWN &&
+     hostname_just_set && flow->host_server_name[0] != '\0') {
+    ndpi_match_hostname_protocol(ndpi_struct, flow,
+				 master_protocol,
+				 flow->host_server_name,
+				 strlen(flow->host_server_name));
+  }
+ 
+  if(flow->detected_protocol_stack[1] == NDPI_PROTOCOL_UNKNOWN &&
+     packet->http_origin.len > 0) {
+    ndpi_protocol_match_result ret_match;
+    char *ptr, *origin_hostname;
+    size_t origin_hostname_len;
+
+    /* Origin syntax:
+        Origin: null
+        Origin: <scheme>://<hostname>
+        Origin: <scheme>://<hostname>:<port>
+    Try extracting hostname */
+
+    ptr = ndpi_strnstr((const char *)packet->http_origin.ptr, "://", packet->http_origin.len);
+    if(ptr) {
+      origin_hostname = ptr + 3;
+      origin_hostname_len = packet->http_origin.len - (ptr - (char *)packet->http_origin.ptr) - 3;
+      ptr = ndpi_strnstr(origin_hostname, ":", origin_hostname_len);
+      if(ptr) {
+        origin_hostname_len = ptr - origin_hostname;
       }
-    }
-    
-    master_protocol = NDPI_PROTOCOL_HTTP;
-    if(flow->detected_protocol_stack[1] != NDPI_PROTOCOL_UNKNOWN)
-      master_protocol = flow->detected_protocol_stack[1];
-    else if(flow->detected_protocol_stack[0] == NDPI_PROTOCOL_HTTP_CONNECT ||
-            flow->detected_protocol_stack[0] == NDPI_PROTOCOL_HTTP_PROXY)
-      master_protocol = flow->detected_protocol_stack[0];
-
-    if(ndpi_match_hostname_protocol(ndpi_struct, flow,
-				    master_protocol,
-				    flow->host_server_name,
-				    strlen(flow->host_server_name)) == 0) {
-      if(flow->http.url &&
-         ((strstr(flow->http.url, ":8080/downloading?n=0.") != NULL)
-          || (strstr(flow->http.url, ":8080/upload?n=0.") != NULL))) {
-	/* This looks like Ookla speedtest */
-	ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, master_protocol, NDPI_CONFIDENCE_DPI);
-	ookla_add_to_cache(ndpi_struct, flow);
+      NDPI_LOG_DBG2(ndpi_struct, "Origin: [%.*s] -> [%.*s]\n", packet->http_origin.len, packet->http_origin.ptr,
+		    (int)origin_hostname_len, origin_hostname);
+      /* We already checked hostname...*/
+      if(strncmp(origin_hostname, flow->host_server_name, origin_hostname_len) != 0) {
+        ndpi_match_host_subprotocol(ndpi_struct, flow,
+				    origin_hostname,
+				    origin_hostname_len,
+				    &ret_match,
+				    master_protocol);
       }
     }
+  }
 
-    if (flow->http.url != NULL &&
-        strstr(flow->http.url, "micloud.xiaomi.net") != NULL)
-    {
-      ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_XIAOMI, master_protocol, NDPI_CONFIDENCE_DPI);
-    }
+  if(flow->detected_protocol_stack[1] == NDPI_PROTOCOL_UNKNOWN &&
+     flow->http.url &&
+     ((strstr(flow->http.url, ":8080/downloading?n=0.") != NULL) ||
+      (strstr(flow->http.url, ":8080/upload?n=0.") != NULL))) {
+    /* This looks like Ookla speedtest */
+    ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, master_protocol, NDPI_CONFIDENCE_DPI);
+    ookla_add_to_cache(ndpi_struct, flow);
+  }
+
+  if(flow->detected_protocol_stack[1] == NDPI_PROTOCOL_UNKNOWN &&
+     flow->http.url != NULL &&
+     strstr(flow->http.url, "micloud.xiaomi.net") != NULL) {
+    ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_XIAOMI, master_protocol, NDPI_CONFIDENCE_DPI);
+  }
+
+  if(flow->detected_protocol_stack[1] == NDPI_PROTOCOL_UNKNOWN &&
+     packet->referer_line.len > 0 &&
+     ndpi_strnstr((const char *)packet->referer_line.ptr, "www.speedtest.net", packet->referer_line.len)) {
+    ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, master_protocol, NDPI_CONFIDENCE_DPI);
+    ookla_add_to_cache(ndpi_struct, flow);
+  }
+
+  /* WindowsUpdate over some kind of CDN */
+  if(flow->detected_protocol_stack[1] == NDPI_PROTOCOL_UNKNOWN &&
+     flow->http.user_agent && flow->http.url &&
+     (strstr(flow->http.url, "delivery.mp.microsoft.com/") ||
+      strstr(flow->http.url, "download.windowsupdate.com/")) &&
+     strstr(flow->http.user_agent, "Microsoft-Delivery-Optimization/") &&
+     ndpi_isset_risk(ndpi_struct, flow, NDPI_NUMERIC_IP_HOST)) {
+    ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_WINDOWS_UPDATE, master_protocol, NDPI_CONFIDENCE_DPI);
+  }
+
+  if(flow->detected_protocol_stack[1] == NDPI_PROTOCOL_UNKNOWN &&
+     packet->payload_packet_len >= 23 &&
+     memcmp(packet->payload, "<policy-file-request/>", 23) == 0) {
+    /*
+      <policy-file-request/>
+      <cross-domain-policy>
+      <allow-access-from domain="*.ookla.com" to-ports="8080"/>
+      <allow-access-from domain="*.speedtest.net" to-ports="8080"/>
+      </cross-domain-policy>
+     */
+    ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, master_protocol, NDPI_CONFIDENCE_DPI);
+    ookla_add_to_cache(ndpi_struct, flow);
   }
 }
 
@@ -547,9 +622,9 @@ static void ndpi_check_user_agent(struct ndpi_detection_module_struct *ndpi_stru
 
 /* ************************************************************* */
 
-int http_process_user_agent(struct ndpi_detection_module_struct *ndpi_struct,
-			    struct ndpi_flow_struct *flow,
-			    const u_int8_t *ua_ptr, u_int16_t ua_ptr_len) {
+void http_process_user_agent(struct ndpi_detection_module_struct *ndpi_struct,
+			     struct ndpi_flow_struct *flow,
+			     const u_int8_t *ua_ptr, u_int16_t ua_ptr_len) {
   /**
       Format examples:
       Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) ....
@@ -612,11 +687,10 @@ int http_process_user_agent(struct ndpi_detection_module_struct *ndpi_struct,
   if(ndpi_user_agent_set(flow, ua_ptr, ua_ptr_len) != NULL)
     ndpi_check_user_agent(ndpi_struct, flow, flow->http.user_agent, ua_ptr_len);
   else
-    NDPI_LOG_DBG2(ndpi_struct, "Could not set HTTP user agent\n");  
+    NDPI_LOG_DBG2(ndpi_struct, "Could not set HTTP user agent (already set?)\n");
 
   NDPI_LOG_DBG2(ndpi_struct, "User Agent Type line found %.*s\n",
 		ua_ptr_len, ua_ptr);
-  return 0;
 }
 
 /* ************************************************************* */
@@ -666,11 +740,8 @@ static void ndpi_check_http_server(struct ndpi_detection_module_struct *ndpi_str
   if(server_len > 7) {
     u_int off;
   
-    if(strncmp((const char *)server, "ntopng ", 7) == 0) {
-      ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_NTOP, NDPI_PROTOCOL_HTTP, NDPI_CONFIDENCE_DPI);
-      NDPI_CLR_BIT(flow->risk, NDPI_KNOWN_PROTOCOL_ON_NON_STANDARD_PORT);
-    } else if((strncasecmp(server, "Apache/", off = 7) == 0) /* X.X.X */
-	      || (strncasecmp(server, "nginx/", off = 6) == 0) /* X.X.X */) {
+    if((strncasecmp(server, "Apache/", off = 7) == 0) /* X.X.X */
+	|| (strncasecmp(server, "nginx/", off = 6) == 0) /* X.X.X */) {
       u_int i, j, a, b, c;
       char buf[16] = { '\0' };
 
@@ -703,10 +774,8 @@ static void ndpi_check_http_server(struct ndpi_detection_module_struct *ndpi_str
 static void check_content_type_and_change_protocol(struct ndpi_detection_module_struct *ndpi_struct,
 						   struct ndpi_flow_struct *flow) {
   struct ndpi_packet_struct *packet = &ndpi_struct->packet;
-  int ret;
-
-  if(flow->http_detected && (flow->http.response_status_code != 0))
-    return;
+  u_int len;
+  int hostname_just_set = 0;
 
   if((flow->http.url == NULL)
      && (packet->http_url_name.len > 0)
@@ -748,101 +817,42 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
 
       ndpi_check_http_url(ndpi_struct, flow, &flow->http.url[host_end]);
     }
+  }
 
+  if(packet->http_method.ptr != NULL)
     flow->http.method = ndpi_http_str2method((const char*)packet->http_method.ptr,
 					     (u_int16_t)packet->http_method.len);
 
-    if((flow->http.method == NDPI_HTTP_METHOD_RPC_IN_DATA)
-       || (flow->http.method == NDPI_HTTP_METHOD_RPC_OUT_DATA)) {
-      ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_RPC, flow->detected_protocol_stack[0], NDPI_CONFIDENCE_DPI);
-      check_content_type_and_change_protocol(ndpi_struct, flow);
-    }
-  }
-
   if(packet->server_line.ptr != NULL)
     ndpi_check_http_server(ndpi_struct, flow, (const char *)packet->server_line.ptr, packet->server_line.len);
  
-  if(packet->user_agent_line.ptr != NULL && packet->user_agent_line.len != 0) {
-    ret = http_process_user_agent(ndpi_struct, flow, packet->user_agent_line.ptr, packet->user_agent_line.len);
-    /* TODO: Is it correct to avoid setting ua, host_name,... if we have a (Netflix) subclassification? */
-    if(ret != 0)
-      return;
+  if(packet->user_agent_line.ptr != NULL) {
+    http_process_user_agent(ndpi_struct, flow, packet->user_agent_line.ptr, packet->user_agent_line.len);
   }
 
-  /* check for host line */
-  if(packet->host_line.ptr != NULL) {
-    u_int len;
-
-    NDPI_LOG_DBG2(ndpi_struct, "HOST line found %.*s\n",
-		  packet->host_line.len, packet->host_line.ptr);
-
-    /* Copy result for nDPI apps */
-    ndpi_hostname_sni_set(flow, packet->host_line.ptr, packet->host_line.len);
-
-    if(strlen(flow->host_server_name) > 0) {
-      if(ndpi_is_valid_hostname(flow->host_server_name,
-				strlen(flow->host_server_name)) == 0) {
-	char str[128];
-
-	snprintf(str, sizeof(str), "Invalid host %s", flow->host_server_name);
-	ndpi_set_risk(ndpi_struct, flow, NDPI_INVALID_CHARACTERS, str);
-
-	/* This looks like an attack */
-	ndpi_set_risk(ndpi_struct, flow, NDPI_POSSIBLE_EXPLOIT, NULL);
+  if(packet->forwarded_line.ptr != NULL) {
+    if(flow->http.nat_ip == NULL) {
+      len = packet->forwarded_line.len;
+      flow->http.nat_ip = ndpi_malloc(len + 1);
+      if(flow->http.nat_ip != NULL) {
+        strncpy(flow->http.nat_ip, (char*)packet->forwarded_line.ptr, len);
+        flow->http.nat_ip[len] = '\0';
       }
     }
+  }
 
-    if(packet->forwarded_line.ptr) {
-      if(flow->http.nat_ip == NULL) {
-        len = packet->forwarded_line.len;
-        flow->http.nat_ip = ndpi_malloc(len + 1);
-        if(flow->http.nat_ip != NULL) {
-          strncpy(flow->http.nat_ip, (char*)packet->forwarded_line.ptr, len);
-          flow->http.nat_ip[len] = '\0';
-        }
-      }
-    }
-
-    ndpi_http_parse_subprotocol(ndpi_struct, flow);
-
-    if(strlen(flow->host_server_name) > 0) {
-      ndpi_check_dga_name(ndpi_struct, flow, flow->host_server_name, 1, 0);
-    }
-
-    if((flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN)
-       && (flow->http_detected)
-       && (packet->http_origin.len > 0)) {
-      ndpi_protocol_match_result ret_match;
-
-      ndpi_match_host_subprotocol(ndpi_struct, flow,
-				  (char *)packet->http_origin.ptr,
-				  packet->http_origin.len,
-				  &ret_match,
-				  NDPI_PROTOCOL_HTTP);
-    }
-
-    if(flow->detected_protocol_stack[1] != NDPI_PROTOCOL_UNKNOWN) {
-      if((flow->detected_protocol_stack[1] != NDPI_PROTOCOL_HTTP)
-	 && (flow->detected_protocol_stack[1] != NDPI_PROTOCOL_HTTP_CONNECT)
-	 && (flow->detected_protocol_stack[1] != NDPI_PROTOCOL_HTTP_PROXY)) {
-	NDPI_LOG_INFO(ndpi_struct, "found HTTP/%s\n",
-		      ndpi_get_proto_name(ndpi_struct, flow->detected_protocol_stack[0]));
-	ndpi_int_http_add_connection(ndpi_struct, flow, flow->detected_protocol_stack[0], NDPI_PROTOCOL_CATEGORY_WEB);
-	return; /* We have identified a sub-protocol so we're done */
+  if(packet->server_line.ptr != NULL) {
+    if(flow->http.server == NULL) {
+      len = packet->server_line.len + 1;
+      flow->http.server = ndpi_malloc(len);
+      if(flow->http.server) {
+        strncpy(flow->http.server, (char*)packet->server_line.ptr,
+                packet->server_line.len);
+	flow->http.server[packet->server_line.len] = '\0';
       }
     }
   }
 
-  if(flow->guessed_protocol_id == NDPI_PROTOCOL_UNKNOWN)
-    flow->guessed_protocol_id = NDPI_PROTOCOL_HTTP;
-
-  /* check for accept line */
-  if(packet->accept_line.ptr != NULL) {
-    NDPI_LOG_DBG2(ndpi_struct, "Accept line found %.*s\n",
-		  packet->accept_line.len, packet->accept_line.ptr);
-  }
-
-  /* check for authorization line */
   if(packet->authorization_line.ptr != NULL) {
     NDPI_LOG_DBG2(ndpi_struct, "Authorization line found %.*s\n",
 		  packet->authorization_line.len, packet->authorization_line.ptr);
@@ -856,7 +866,7 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
     }
   }
 
-  if(packet->content_line.ptr != NULL && packet->content_line.len != 0) {
+  if(packet->content_line.ptr != NULL) {
     NDPI_LOG_DBG2(ndpi_struct, "Content Type line found %.*s\n",
 		  packet->content_line.len, packet->content_line.ptr);
 
@@ -886,54 +896,67 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
 	  flow->guessed_category = flow->category = ndpi_http_check_content(ndpi_struct, flow);
 	}
       }
+    }
+  }
 
-      if((flow->http.server == NULL) && (packet->server_line.len > 0)) {
-	int len = packet->server_line.len + 1;
-	
-	flow->http.server = ndpi_malloc(len);
-	if(flow->http.server) {
-	  strncpy(flow->http.server, (char*)packet->server_line.ptr,
-		  packet->server_line.len);
-	  flow->http.server[packet->server_line.len] = '\0';
-	}
+  /* check for host line (only if we don't already have an hostname) */
+  if(packet->host_line.ptr != NULL && flow->host_server_name[0] == '\0') {
+
+    NDPI_LOG_DBG2(ndpi_struct, "HOST line found %.*s\n",
+		  packet->host_line.len, packet->host_line.ptr);
+
+    /* Copy result for nDPI apps */
+    ndpi_hostname_sni_set(flow, packet->host_line.ptr, packet->host_line.len);
+
+    if(strlen(flow->host_server_name) > 0) {
+      char *double_col;
+      int a, b, c, d;
+
+      hostname_just_set = 1;
+
+      if(ndpi_is_valid_hostname(flow->host_server_name,
+				strlen(flow->host_server_name)) == 0) {
+	char str[128];
+
+	snprintf(str, sizeof(str), "Invalid host %s", flow->host_server_name);
+	ndpi_set_risk(ndpi_struct, flow, NDPI_INVALID_CHARACTERS, str);
+
+	/* This looks like an attack */
+	ndpi_set_risk(ndpi_struct, flow, NDPI_POSSIBLE_EXPLOIT, NULL);
       }
-    }
 
-    if(flow->http_detected && packet->content_line.ptr && *(char*)packet->content_line.ptr) {
-      /* Matching on Content-Type.
-          OCSP:  application/ocsp-request, application/ocsp-response
-       */
-      if(packet->content_line.len > 17 &&
-         strncmp((const char *)packet->content_line.ptr, "application/ocsp-", 17) == 0) {
-        NDPI_LOG_DBG2(ndpi_struct, "Found OCSP\n");
-        ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OCSP, NDPI_PROTOCOL_HTTP, NDPI_CONFIDENCE_DPI);
+      double_col = strchr((char*)flow->host_server_name, ':');
+      if(double_col) double_col[0] = '\0';
+      if(ndpi_struct->packet.iph
+         && (sscanf(flow->host_server_name, "%d.%d.%d.%d", &a, &b, &c, &d) == 4)) {
+        /* IPv4 */
+
+        if(ndpi_struct->packet.iph->daddr != inet_addr(flow->host_server_name)) {
+	  char buf[64], msg[128];
+
+	  snprintf(msg, sizeof(msg), "Expected %s, found %s",
+		   ndpi_intoav4(ntohl(ndpi_struct->packet.iph->daddr), buf, sizeof(buf)), flow->host_server_name);
+	  ndpi_set_risk(ndpi_struct, flow, NDPI_HTTP_SUSPICIOUS_HEADER, msg);
+        }
       }
     }
+
   }
 
-  if(packet->user_agent_line.ptr != NULL && packet->user_agent_line.len != 0 &&
-     flow->http.url) {
-    /* WindowsUpdate over some kind of CDN */
-    if(flow->detected_protocol_stack[1] == NDPI_PROTOCOL_UNKNOWN &&
-       flow->detected_protocol_stack[0] == NDPI_PROTOCOL_HTTP &&
-       (strstr(flow->http.url, "delivery.mp.microsoft.com/") ||
-        strstr(flow->http.url, "download.windowsupdate.com/")) &&
-       ndpi_strnstr((const char *)packet->user_agent_line.ptr, "Microsoft-Delivery-Optimization/",
-                    packet->user_agent_line.len) &&
-       ndpi_isset_risk(ndpi_struct, flow, NDPI_NUMERIC_IP_HOST)) {
-      ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_WINDOWS_UPDATE, NDPI_PROTOCOL_HTTP, NDPI_CONFIDENCE_DPI);
-    }
+  ndpi_http_parse_subprotocol(ndpi_struct, flow, hostname_just_set);
+
+  if(hostname_just_set && strlen(flow->host_server_name) > 0) {
+    ndpi_check_dga_name(ndpi_struct, flow, flow->host_server_name, 1, 0);
   }
 
+  if(flow->guessed_protocol_id == NDPI_PROTOCOL_UNKNOWN)
+    flow->guessed_protocol_id = NDPI_PROTOCOL_HTTP;
+
   if(ndpi_get_http_method(ndpi_struct, flow) != NDPI_HTTP_METHOD_UNKNOWN) {
-    ndpi_int_http_add_connection(ndpi_struct, flow, flow->detected_protocol_stack[0], NDPI_PROTOCOL_CATEGORY_WEB);
+    ndpi_int_http_add_connection(ndpi_struct, flow, flow->detected_protocol_stack[0]);
   }
-}
-
-/* ************************************************************* */
 
-static void check_http_payload(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) {
-  /* Add here your payload code check */
+  ndpi_check_http_header(ndpi_struct, flow);
 }
 
 /* ************************************************************* */
@@ -1133,6 +1156,45 @@ static void ndpi_check_http_header(struct ndpi_detection_module_struct *ndpi_str
   }
 }
 
+static void parse_response_code(struct ndpi_detection_module_struct *ndpi_struct,
+				struct ndpi_flow_struct *flow)
+{
+  struct ndpi_packet_struct *packet = &ndpi_struct->packet;
+  char buf[4];
+  char ec[48];
+
+  if(packet->payload_packet_len >= 12) {
+    /* Set server HTTP response code */
+    strncpy(buf, (char*)&packet->payload[9], 3);
+    buf[3] = '\0';
+
+    flow->http.response_status_code = atoi(buf);
+    NDPI_LOG_DBG2(ndpi_struct, "Response code %d\n", flow->http.response_status_code);
+
+    /* https://en.wikipedia.org/wiki/List_of_HTTP_status_codes */
+    if((flow->http.response_status_code < 100) || (flow->http.response_status_code > 509))
+      flow->http.response_status_code = 0; /* Out of range */
+
+    if(flow->http.response_status_code >= 400) {
+      snprintf(ec, sizeof(ec), "HTTP Error Code %u", flow->http.response_status_code);
+      ndpi_set_risk(ndpi_struct, flow, NDPI_ERROR_CODE_DETECTED, ec);
+
+      if(flow->http.url != NULL) {
+        /* Let's check for Wordpress */
+        char *slash = strchr(flow->http.url, '/');
+
+	if(slash != NULL &&
+           (((flow->http.method == NDPI_HTTP_METHOD_POST) && (strncmp(slash, "/wp-admin/", 10) == 0))
+	    || ((flow->http.method == NDPI_HTTP_METHOD_GET) && (strncmp(slash, "/wp-content/uploads/", 20) == 0))
+	   )) {
+          /* Example of popular exploits https://www.wordfence.com/blog/2022/05/millions-of-attacks-target-tatsu-builder-plugin/ */
+          ndpi_set_risk(ndpi_struct, flow, NDPI_POSSIBLE_EXPLOIT, "Possible Wordpress Exploit");
+	}
+      }
+    }
+  }
+}
+
 /*************************************************************************************************/
 
 static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct,
@@ -1148,8 +1210,7 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
   }
 
   /* Check if we so far detected the protocol in the request or not. */
-  if((packet->payload_packet_len > 0) /* Needed in case of extra packet processing */
-     && (flow->l4.tcp.http_stage == 0)) {
+  if(flow->l4.tcp.http_stage == 0) {
     /* Expected a request */
     flow->http_detected = 0;
 
@@ -1163,70 +1224,25 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
       if(packet->payload_packet_len >= 7 && strncasecmp((const char *)packet->payload, "HTTP/1.", 7) == 0) {
         NDPI_LOG_INFO(ndpi_struct, "found HTTP response\n");
 
-	if(packet->payload_packet_len >= 12) {
-	  char buf[4];
-
-	  /* Set server HTTP response code */
-	  strncpy(buf, (char*)&packet->payload[9], 3);
-	  buf[3] = '\0';
+	parse_response_code(ndpi_struct, flow);
 
-	  flow->http.response_status_code = atoi(buf);
-
-	  /* https://en.wikipedia.org/wiki/List_of_HTTP_status_codes */
-	  if((flow->http.response_status_code < 100) || (flow->http.response_status_code > 509))
-	    flow->http.response_status_code = 0; /* Out of range */
-	  else if(flow->http.response_status_code >= 400) {
-	    char ec[48];
+	if(flow->flow_payload) {
+	  char *endl;
 	    
-	    if(flow->http.url != NULL) {
-	      /* Let's check for Wordpress */
-	      char *slash = strchr(flow->http.url, '/');
-
-	      if(slash != NULL &&
-		 (((flow->http.method == NDPI_HTTP_METHOD_POST) && (strncmp(slash, "/wp-admin/", 10) == 0))
-		 || ((flow->http.method == NDPI_HTTP_METHOD_GET) && (strncmp(slash, "/wp-content/uploads/", 20) == 0))		 
-		 )) {
-		/* Example of popular exploits https://www.wordfence.com/blog/2022/05/millions-of-attacks-target-tatsu-builder-plugin/ */
-		ndpi_set_risk(ndpi_struct, flow, NDPI_POSSIBLE_EXPLOIT, "Possible Wordpress Exploit");
-	      }
-	    }
+	  flow->flow_payload[flow->flow_payload_len] = '\0';
+	  if((endl = strrchr(flow->flow_payload, '\r')) == NULL)
+	    endl = strrchr(flow->flow_payload, '\n');
 
-	    snprintf(ec, sizeof(ec), "HTTP Error Code %u", flow->http.response_status_code);
-	    ndpi_set_risk(ndpi_struct, flow, NDPI_ERROR_CODE_DETECTED, ec);
-	  }
-
-	  if(flow->flow_payload) {
-	    char *endl;
-	    
-	    flow->flow_payload[flow->flow_payload_len] = '\0';
-	    if((endl = strrchr(flow->flow_payload, '\r')) == NULL)
-	      endl = strrchr(flow->flow_payload, '\n');
-
-	    if(endl != NULL) {
-	      endl[0] = '\0';
-	      flow->flow_payload_len = endl - flow->flow_payload;
-	    }
+	  if(endl != NULL) {
+	    endl[0] = '\0';
+	    flow->flow_payload_len = endl - flow->flow_payload;
 	  }
 	}
 
 	ndpi_parse_packet_line_info(ndpi_struct, flow);
         check_content_type_and_change_protocol(ndpi_struct, flow);
 	ndpi_validate_http_content(ndpi_struct, flow);
-        ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP, NDPI_PROTOCOL_CATEGORY_WEB);
-        return;
-      }
-
-      if((packet->payload_packet_len == 23) && (memcmp(packet->payload, "<policy-file-request/>", 23) == 0)) {
-        /*
-          <policy-file-request/>
-          <cross-domain-policy>
-          <allow-access-from domain="*.ookla.com" to-ports="8080"/>
-          <allow-access-from domain="*.speedtest.net" to-ports="8080"/>
-          </cross-domain-policy>
-        */
-      ookla_found:
-        ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_CATEGORY_WEB);
-        ookla_add_to_cache(ndpi_struct, flow);
+        ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP);
         return;
       }
 
@@ -1247,7 +1263,7 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
         NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
         return;
       } else {
-        ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP, NDPI_PROTOCOL_CATEGORY_WEB);
+        ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP);
       }
     }
 
@@ -1255,15 +1271,11 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
 		  "Filename HTTP found: %d, we look for line info..\n", filename_start);
 
     ndpi_parse_packet_line_info(ndpi_struct, flow);
-    ndpi_check_http_header(ndpi_struct, flow);
 
     if(packet->parsed_lines <= 1) {
       NDPI_LOG_DBG2(ndpi_struct,
 		    "Found just one line, we will look further for the next packet...\n");
 
-      packet->http_method.ptr = packet->line[0].ptr;
-      packet->http_method.len = filename_start - 1;
-
       /* Encode the direction of the packet in the stage, so we will know when we need to look for the response packet. */
       flow->l4.tcp.http_stage = packet->packet_direction + 1; // packet_direction 0: stage 1, packet_direction 1: stage 2
       return;
@@ -1288,12 +1300,6 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
       else
 	flow->http.request_version = 0;
 
-      /* Check for Ookla */
-      if((packet->referer_line.len > 0)
-	 && ndpi_strnstr((const char *)packet->referer_line.ptr, "www.speedtest.net", packet->referer_line.len)) {
-	goto ookla_found;
-      }
-
       if((packet->http_url_name.len > 7)
 	 && (!strncasecmp((const char*) packet->http_url_name.ptr, "http://", 7))) {
         NDPI_LOG_INFO(ndpi_struct, "found HTTP_PROXY\n");
@@ -1328,7 +1334,7 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
 	  u_int proto = flow->detected_protocol_stack[1] ? flow->detected_protocol_stack[1] : flow->detected_protocol_stack[0];
 
 	  if(proto == NDPI_PROTOCOL_UNKNOWN) proto = NDPI_PROTOCOL_HTTP;
-	  ndpi_int_http_add_connection(ndpi_struct, flow, proto, NDPI_PROTOCOL_CATEGORY_WEB);
+	  ndpi_int_http_add_connection(ndpi_struct, flow, proto);
 	  flow->http_detected = 1;
 	  NDPI_LOG_DBG2(ndpi_struct,
 			"HTTP START Found, we will look further for the response...\n");
@@ -1350,10 +1356,6 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
        otherwise check the second packet for the HTTP request
     */
     if((flow->l4.tcp.http_stage - packet->packet_direction) == 1) { /* Expected a response package */
-
-      if(flow->http_detected)
-        return;
-
       NDPI_LOG_DBG2(ndpi_struct,
 		    " SECOND PAYLOAD TRAFFIC FROM CLIENT, FIRST PACKET MIGHT HAVE BEEN HTTP...UNKNOWN TRAFFIC, HERE FOR HTTP again.. \n");
 
@@ -1371,12 +1373,20 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
           return;
         }
       }
+
+      /* The previous pkt was a valid HTTP request and this one is in the same direction:
+         splitted request? Try to extract more metadata */
+      if(flow->http_detected) {
+        check_content_type_and_change_protocol(ndpi_struct, flow);
+      }
+
       // http://www.slideshare.net/DSPIP/rtsp-analysis-wireshark
-      if(packet->line[0].len >= 9
+      if(!flow->http_detected
+	 && packet->line[0].len >= 9
 	 && strncasecmp((const char *)&packet->line[0].ptr[packet->line[0].len - 9], " HTTP/1.", 8) == 0) {
 
         NDPI_LOG_INFO(ndpi_struct, "found HTTP\n");
-        ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP, NDPI_PROTOCOL_CATEGORY_WEB);
+        ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP);
         check_content_type_and_change_protocol(ndpi_struct, flow);
 
         NDPI_LOG_DBG2(ndpi_struct,
@@ -1398,19 +1408,17 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
     if((packet->parsed_lines == 1) && (packet->packet_direction == 1 /* server -> client */)) {
       /* In Apache if you do "GET /\n\n" the response comes without any header */
       NDPI_LOG_INFO(ndpi_struct, "found HTTP. (apache)\n");
-      ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP, NDPI_PROTOCOL_CATEGORY_WEB);
+      ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP);
       check_content_type_and_change_protocol(ndpi_struct, flow);
       return;
     }
 
+    parse_response_code(ndpi_struct, flow);
+
     /* Parse packet line and we look for the subprotocols */
     ndpi_parse_packet_line_info(ndpi_struct, flow);
     check_content_type_and_change_protocol(ndpi_struct, flow);
-
-    if(packet->empty_line_position_set != 0 || flow->l4.tcp.http_empty_line_seen == 1) {
-      NDPI_LOG_DBG2(ndpi_struct, "empty line. check_http_payload\n");
-      check_http_payload(ndpi_struct, flow);
-    }
+    ndpi_validate_http_content(ndpi_struct, flow);
 
     flow->l4.tcp.http_stage = 0;
     return;