HTTP: rework state machine (#1966)

The goal if to correlate the right request-response pair, exporting
metadata from only one transaction (for example, the right url & return
state pair)

As a nice side effect, the code should be much cleaner, but that is a
matter of taste.

Two differences respect to the previous code:
* as it happens in the CI, if in the flow there are only one response
(before) and one request (after), only the metadata of the response are
saved/exported
* for performance reasons, we don't call `ndpi_parse_packet_line_info()`
anymore for ALL packets triggering the HTTP dissector, but only for the
packets that we already know belong to an HTTP flow. This is the reason
for the changes in RTSP/SOAP/... code

3 files changed, 237 insertions, 229 deletions

diff --git a/src/lib/protocols/gnutella.c b/src/lib/protocols/gnutella.c
index 13b8d99c7..09190b53b 100644
--- a/src/lib/protocols/gnutella.c
+++ b/src/lib/protocols/gnutella.c
@@ -59,6 +59,10 @@ static void ndpi_search_gnutella(struct ndpi_detection_module_struct *ndpi_struc
     /* this case works asymmetrically */
     if (packet->payload_packet_len > 17 && memcmp(packet->payload, "GNUTELLA CONNECT/", 17) == 0) {
       ndpi_int_gnutella_add_connection(ndpi_struct, flow, NDPI_CONFIDENCE_DPI);
+      /* Extract some metadata HTTP-like */
+      ndpi_parse_packet_line_info(ndpi_struct, flow);
+      if(packet->user_agent_line.ptr != NULL)
+        ndpi_user_agent_set(flow, packet->user_agent_line.ptr, packet->user_agent_line.len);
       return;
     }
 
@@ -73,6 +77,9 @@ static void ndpi_search_gnutella(struct ndpi_detection_module_struct *ndpi_struc
 	    || (packet->line[c].len > 36 && memcmp(packet->line[c].ptr,
 						   "Content-Type: application/x-gnutella-", 37) == 0)) {
 	  ndpi_int_gnutella_add_connection(ndpi_struct, flow, NDPI_CONFIDENCE_DPI);
+	  /* Extract some metadata HTTP-like */
+	  if(packet->user_agent_line.ptr != NULL)
+	    ndpi_user_agent_set(flow, packet->user_agent_line.ptr, packet->user_agent_line.len);
 	  return;
 	}
       }
@@ -84,6 +91,9 @@ static void ndpi_search_gnutella(struct ndpi_detection_module_struct *ndpi_struc
 	  || (packet->accept_line.ptr != NULL && packet->accept_line.len > 24
 	      && memcmp(packet->accept_line.ptr, "application n/x-gnutella", 24) == 0)) {
 	ndpi_int_gnutella_add_connection(ndpi_struct, flow, NDPI_CONFIDENCE_DPI);
+	/* Extract some metadata HTTP-like */
+	if(packet->user_agent_line.ptr != NULL)
+	  ndpi_user_agent_set(flow, packet->user_agent_line.ptr, packet->user_agent_line.len);
       }
 
     }
diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c
index 2109f8ed3..f4cf31160 100644
--- a/src/lib/protocols/http.c
+++ b/src/lib/protocols/http.c
@@ -113,14 +113,8 @@ static int ndpi_search_http_tcp_again(struct ndpi_detection_module_struct *ndpi_
   printf("=> %s()\n", __FUNCTION__);
 #endif
 
-  if((flow->host_server_name[0] != '\0')
-     && (flow->http.response_status_code != 0)
-     ) {
-    /* stop extra processing */
-    if(flow->initial_binary_bytes_len) ndpi_analyze_content_signature(ndpi_struct, flow);
-
-    flow->extra_packets_func = NULL; /* We're good now */
-    return(0);
+  if(flow->extra_packets_func == NULL) {
+    return(0); /* We're good now */
   }
 
   /* Possibly more processing */
@@ -287,8 +281,6 @@ static ndpi_protocol_category_t ndpi_http_check_content(struct ndpi_detection_mo
 	    }
 	  }
 	}
-
-	ndpi_validate_http_content(ndpi_struct, flow);
       }
     }
 
@@ -344,42 +336,23 @@ static ndpi_protocol_category_t ndpi_http_check_content(struct ndpi_detection_mo
 
 static void ndpi_int_http_add_connection(struct ndpi_detection_module_struct *ndpi_struct,
 					 struct ndpi_flow_struct *flow,
-					 u_int16_t http_protocol) {
-  u_int16_t master_protocol;
-
+					 u_int16_t master_protocol) {
 #ifdef HTTP_DEBUG
   printf("=> %s()\n", __FUNCTION__);
 #endif
 
-  master_protocol = NDPI_PROTOCOL_HTTP;
-  if(flow->detected_protocol_stack[1] != NDPI_PROTOCOL_UNKNOWN)
-    master_protocol = flow->detected_protocol_stack[1];
-  else if(flow->detected_protocol_stack[0] == NDPI_PROTOCOL_HTTP_CONNECT ||
-          flow->detected_protocol_stack[0] == NDPI_PROTOCOL_HTTP_PROXY)
-    master_protocol = flow->detected_protocol_stack[0];
-
   /* Update the classification only if we don't already have master + app;
      for example don't change the protocols if we have already detected a
      sub-protocol via the (content-matched) subprotocols logic (i.e.
      MPEGDASH, SOAP, ....) */
-  if(flow->detected_protocol_stack[1] == 0)
-    ndpi_set_detected_protocol(ndpi_struct, flow, http_protocol,
-			       master_protocol,
-			       NDPI_CONFIDENCE_DPI);
+  if(flow->detected_protocol_stack[1] == NDPI_PROTOCOL_UNKNOWN) {
+    NDPI_LOG_DBG2(ndpi_struct, "Master: %d\n", master_protocol);
+    ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_UNKNOWN,
+			       master_protocol, NDPI_CONFIDENCE_DPI);
+  }
 
   flow->max_extra_packets_to_check = 8;
   flow->extra_packets_func = ndpi_search_http_tcp_again;
-  flow->http_detected = 1;
-
-  switch(flow->detected_protocol_stack[1]) {
-  case NDPI_PROTOCOL_HTTP_CONNECT:
-  case NDPI_PROTOCOL_HTTP_PROXY:
-    if(flow->detected_protocol_stack[0] == NDPI_PROTOCOL_HTTP) {
-      flow->detected_protocol_stack[0] = flow->detected_protocol_stack[1];
-      flow->detected_protocol_stack[1] = NDPI_PROTOCOL_UNKNOWN;
-    }
-    break;
-  }
 }
 
 /* ************************************************************* */
@@ -684,10 +657,12 @@ void http_process_user_agent(struct ndpi_detection_module_struct *ndpi_struct,
     }
   }
 
-  if(ndpi_user_agent_set(flow, ua_ptr, ua_ptr_len) != NULL)
+  if(ndpi_user_agent_set(flow, ua_ptr, ua_ptr_len) != NULL) {
+    ndpi_unset_risk(ndpi_struct, flow, NDPI_HTTP_SUSPICIOUS_USER_AGENT);
     ndpi_check_user_agent(ndpi_struct, flow, flow->http.user_agent, ua_ptr_len);
-  else
+  } else {
     NDPI_LOG_DBG2(ndpi_struct, "Could not set HTTP user agent (already set?)\n");
+  }
 
   NDPI_LOG_DBG2(ndpi_struct, "User Agent Type line found %.*s\n",
 		ua_ptr_len, ua_ptr);
@@ -952,10 +927,6 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
   if(flow->guessed_protocol_id == NDPI_PROTOCOL_UNKNOWN)
     flow->guessed_protocol_id = NDPI_PROTOCOL_HTTP;
 
-  if(ndpi_get_http_method(ndpi_struct, flow) != NDPI_HTTP_METHOD_UNKNOWN) {
-    ndpi_int_http_add_connection(ndpi_struct, flow, flow->detected_protocol_stack[0]);
-  }
-
   ndpi_check_http_header(ndpi_struct, flow);
 }
 
@@ -1195,233 +1166,248 @@ static void parse_response_code(struct ndpi_detection_module_struct *ndpi_struct
   }
 }
 
-/*************************************************************************************************/
-
-static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct,
-				struct ndpi_flow_struct *flow) {
+static int is_request(struct ndpi_detection_module_struct *ndpi_struct,
+		      struct ndpi_flow_struct *flow) {
   struct ndpi_packet_struct *packet = &ndpi_struct->packet;
-  u_int16_t filename_start; /* the filename in the request method line, e.g., "GET filename_start..."*/
-
-  packet->packet_lines_parsed_complete = 0;
-
-  if(packet->http_check_content && (packet->payload_packet_len > 0)) {
-    ndpi_http_check_human_redeable_content(ndpi_struct, flow, packet->payload, packet->payload_packet_len);
-    packet->http_check_content = 0; /* One packet is enough */
-  }
+  u_int16_t filename_start;
 
-  /* Check if we so far detected the protocol in the request or not. */
-  if(flow->l4.tcp.http_stage == 0) {
-    /* Expected a request */
-    flow->http_detected = 0;
-
-    NDPI_LOG_DBG2(ndpi_struct, "HTTP stage %d: \n", flow->l4.tcp.http_stage);
+  filename_start = http_request_url_offset(ndpi_struct, flow);
+  /* This check is required as RTSP is pretty similiar to HTTP */
+  if(filename_start > 0 &&
+     strncasecmp((const char *)packet->payload + filename_start,
+                 "rtsp://", ndpi_min(7, packet->payload_packet_len - filename_start)) == 0)
+    return 0;
+  return filename_start;
+}
 
-    filename_start = http_request_url_offset(ndpi_struct, flow);
+static int is_response(struct ndpi_detection_module_struct *ndpi_struct,
+		       struct ndpi_flow_struct *flow) {
+  struct ndpi_packet_struct *packet = &ndpi_struct->packet;
+  if(packet->payload_packet_len >= 7 &&
+     strncasecmp((const char *)packet->payload, "HTTP/1.", 7) == 0)
+    return 1;
+  return 0;
+}
 
-    if(filename_start == 0) { /* not a regular request. In the HTTP first stage, may be a truncated flow or other protocols */
-      NDPI_LOG_DBG2(ndpi_struct, "Filename HTTP not found, we look for possible truncate flow..\n");
+static void process_request(struct ndpi_detection_module_struct *ndpi_struct,
+			    struct ndpi_flow_struct *flow,
+			    u_int16_t filename_start) {
+  struct ndpi_packet_struct *packet = &ndpi_struct->packet;
+  u_int16_t master_protocol;
 
-      if(packet->payload_packet_len >= 7 && strncasecmp((const char *)packet->payload, "HTTP/1.", 7) == 0) {
-        NDPI_LOG_INFO(ndpi_struct, "found HTTP response\n");
+  ndpi_parse_packet_line_info(ndpi_struct, flow);
 
-	parse_response_code(ndpi_struct, flow);
+  master_protocol = NDPI_PROTOCOL_HTTP;
 
-	if(flow->flow_payload) {
-	  char *endl;
-	    
-	  flow->flow_payload[flow->flow_payload_len] = '\0';
-	  if((endl = strrchr(flow->flow_payload, '\r')) == NULL)
-	    endl = strrchr(flow->flow_payload, '\n');
+  if(packet->parsed_lines == 0 ||
+     !(packet->line[0].len >= (9 + filename_start) &&
+       strncasecmp((const char *)&packet->line[0].ptr[packet->line[0].len - 9], " HTTP/1.", 8) == 0)) {
+    NDPI_LOG_DBG2(ndpi_struct, "Request with an incomplete or invalid first line\n");
+    /* Since we don't save data across different packets, we will never have
+       the complete url: we can't check for HTTP_PROXY */
+    if(filename_start == 8 &&
+       strncasecmp((const char *)packet->payload, "CONNECT ", 8) == 0) {
+      master_protocol = NDPI_PROTOCOL_HTTP_CONNECT;
+    }
+  } else {
+    /* First line is complete (example: "GET / HTTP/1.1"): extract url */
 
-	  if(endl != NULL) {
-	    endl[0] = '\0';
-	    flow->flow_payload_len = endl - flow->flow_payload;
-	  }
-	}
+    packet->http_url_name.ptr = &packet->payload[filename_start];
+    packet->http_url_name.len = packet->line[0].len - (filename_start + 9);
 
-	ndpi_parse_packet_line_info(ndpi_struct, flow);
-        check_content_type_and_change_protocol(ndpi_struct, flow);
-	ndpi_validate_http_content(ndpi_struct, flow);
-        ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP);
-        return;
-      }
+    packet->http_method.ptr = packet->line[0].ptr;
+    packet->http_method.len = filename_start - 1;
 
-      /* try to get some additional request header info even if the packet may not be HTTP */
-      ndpi_parse_packet_line_info(ndpi_struct, flow);
-      if(packet->http_num_headers > 0) {
-        check_content_type_and_change_protocol(ndpi_struct, flow);
-        return;
-      }
+    /* Set the HTTP requested version: 0=HTTP/1.0 and 1=HTTP/1.1 */
+    if(memcmp(&packet->line[0].ptr[packet->line[0].len - 1], "1", 1) == 0)
+      flow->http.request_version = 1;
+    else
+      flow->http.request_version = 0;
 
-      NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
-      return;
-    } else {
-      /* This check is required as RTSP is pretty similiar to HTTP (prevent false-positives). */
-      if (strncasecmp((const char *)packet->payload + filename_start,
-                      "rtsp://", ndpi_min(7, packet->payload_packet_len - filename_start)) == 0)
-      {
-        NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
-        return;
-      } else {
-        ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP);
-      }
+    if(packet->http_url_name.len > 7 &&
+       !strncasecmp((const char*) packet->http_url_name.ptr, "http://", 7)) {
+      master_protocol = NDPI_PROTOCOL_HTTP_PROXY;
     }
+    if(filename_start == 8 &&
+       strncasecmp((const char *)packet->payload, "CONNECT ", 8) == 0) {
+      master_protocol = NDPI_PROTOCOL_HTTP_CONNECT;
+    }
+  }
+  ndpi_int_http_add_connection(ndpi_struct, flow, master_protocol);
+  check_content_type_and_change_protocol(ndpi_struct, flow);
 
-    NDPI_LOG_DBG2(ndpi_struct,
-		  "Filename HTTP found: %d, we look for line info..\n", filename_start);
-
-    ndpi_parse_packet_line_info(ndpi_struct, flow);
+  if(flow->http.user_agent == NULL ||
+     flow->http.user_agent[0] == '\0') {
+    ndpi_set_risk(ndpi_struct, flow, NDPI_HTTP_SUSPICIOUS_USER_AGENT, "Empty or missing User-Agent");
+  }
+}
 
-    if(packet->parsed_lines <= 1) {
-      NDPI_LOG_DBG2(ndpi_struct,
-		    "Found just one line, we will look further for the next packet...\n");
+static void process_response(struct ndpi_detection_module_struct *ndpi_struct,
+			     struct ndpi_flow_struct *flow) {
 
-      /* Encode the direction of the packet in the stage, so we will know when we need to look for the response packet. */
-      flow->l4.tcp.http_stage = packet->packet_direction + 1; // packet_direction 0: stage 1, packet_direction 1: stage 2
-      return;
-    }
+  ndpi_parse_packet_line_info(ndpi_struct, flow);
+  parse_response_code(ndpi_struct, flow);
+  check_content_type_and_change_protocol(ndpi_struct, flow);
 
-    NDPI_LOG_DBG2(ndpi_struct,
-		  "Found more than one line, we look further for the next packet...\n");
+  ndpi_validate_http_content(ndpi_struct, flow);
+}
 
-    if(packet->line[0].len >= (9 + filename_start)
-       && strncasecmp((const char *)&packet->line[0].ptr[packet->line[0].len - 9], " HTTP/1.", 8) == 0) {
-      /* Request line complete. Ex. "GET / HTTP/1.1" */
+static void reset(struct ndpi_detection_module_struct *ndpi_struct,
+		  struct ndpi_flow_struct *flow) {
 
-      packet->http_url_name.ptr = &packet->payload[filename_start];
-      packet->http_url_name.len = packet->line[0].len - (filename_start + 9);
+  NDPI_LOG_DBG2(ndpi_struct, "Reset status and risks\n");
 
-      packet->http_method.ptr = packet->line[0].ptr;
-      packet->http_method.len = filename_start - 1;
+  /* Reset everthing in flow->http.
+     TODO: Could we be smarter? Probably some info don't change across
+     different req-res transactions... */
 
-      // Set the HTTP requested version: 0=HTTP/1.0 and 1=HTTP/1.1
-      if(memcmp(&packet->line[0].ptr[packet->line[0].len - 1], "1", 1) == 0)
-	flow->http.request_version = 1;
-      else
-	flow->http.request_version = 0;
+  flow->http.method = 0;
+  flow->http.request_version = 0;
+  flow->http.response_status_code = 0;
+  if(flow->http.url) {
+    ndpi_free(flow->http.url);
+    flow->http.url = NULL;
+  }
+  if(flow->http.content_type) {
+    ndpi_free(flow->http.content_type);
+    flow->http.content_type = NULL;
+  }
+  if(flow->http.request_content_type) {
+    ndpi_free(flow->http.request_content_type);
+    flow->http.request_content_type = NULL;
+  }
+  if(flow->http.user_agent) {
+    ndpi_free(flow->http.user_agent);
+    flow->http.user_agent = NULL;
+  }
+  if(flow->http.server) {
+    ndpi_free(flow->http.server);
+    flow->http.server = NULL;
+  }
+  if(flow->http.detected_os) {
+    ndpi_free(flow->http.detected_os);
+    flow->http.detected_os = NULL;
+  }
+  if(flow->http.nat_ip) {
+    ndpi_free(flow->http.nat_ip);
+    flow->http.nat_ip = NULL;
+  }
 
-      if((packet->http_url_name.len > 7)
-	 && (!strncasecmp((const char*) packet->http_url_name.ptr, "http://", 7))) {
-        NDPI_LOG_INFO(ndpi_struct, "found HTTP_PROXY\n");
-	ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_HTTP_PROXY, NDPI_CONFIDENCE_DPI);
-        check_content_type_and_change_protocol(ndpi_struct, flow);
-	flow->http_detected = 1;
-	flow->l4.tcp.http_stage = packet->packet_direction + 1; // packet_direction 0: stage 1, packet_direction 1: stage 2
-	return;
-      }
+  /* Reset flow risks. We should reset only those risks triggered by
+     the previous HTTP response... */
+  /* TODO */
+  ndpi_unset_risk(ndpi_struct, flow, NDPI_BINARY_APPLICATION_TRANSFER);
+  ndpi_unset_risk(ndpi_struct, flow, NDPI_HTTP_SUSPICIOUS_CONTENT);
+  ndpi_unset_risk(ndpi_struct, flow, NDPI_POSSIBLE_EXPLOIT);
+  ndpi_unset_risk(ndpi_struct, flow, NDPI_HTTP_SUSPICIOUS_USER_AGENT);
+  ndpi_unset_risk(ndpi_struct, flow, NDPI_HTTP_CRAWLER_BOT);
+  ndpi_unset_risk(ndpi_struct, flow, NDPI_NUMERIC_IP_HOST);
+  ndpi_unset_risk(ndpi_struct, flow, NDPI_URL_POSSIBLE_RCE_INJECTION);
+  ndpi_unset_risk(ndpi_struct, flow, NDPI_HTTP_OBSOLETE_SERVER);
+  ndpi_unset_risk(ndpi_struct, flow, NDPI_CLEAR_TEXT_CREDENTIALS);
+  ndpi_unset_risk(ndpi_struct, flow, NDPI_INVALID_CHARACTERS);
+  ndpi_unset_risk(ndpi_struct, flow, NDPI_HTTP_SUSPICIOUS_HEADER);
+  ndpi_unset_risk(ndpi_struct, flow, NDPI_ERROR_CODE_DETECTED);
+  ndpi_unset_risk(ndpi_struct, flow, NDPI_MALFORMED_PACKET);
+}
 
-      if(filename_start == 8 && (strncasecmp((const char *)packet->payload, "CONNECT ", 8) == 0)) {
-        NDPI_LOG_INFO(ndpi_struct, "found HTTP_CONNECT\n");
-	ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_HTTP_CONNECT, NDPI_CONFIDENCE_DPI);
-        check_content_type_and_change_protocol(ndpi_struct, flow);
-	flow->http_detected = 1;
-	flow->l4.tcp.http_stage = packet->packet_direction + 1; // packet_direction 0: stage 1, packet_direction 1: stage 2
-	return;
-      }
+static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct,
+				struct ndpi_flow_struct *flow) {
+  struct ndpi_packet_struct *packet = &ndpi_struct->packet;
+  u_int16_t filename_start;
+
+  NDPI_LOG_DBG(ndpi_struct, "http_stage %d dir %d req/res %d/%d\n",
+	       flow->l4.tcp.http_stage, packet->packet_direction,
+	       is_request(ndpi_struct, flow), is_response(ndpi_struct, flow));
+
+  if(flow->l4.tcp.http_stage == 0) { /* Start: waiting for (the beginning of) a request */
+    filename_start = is_request(ndpi_struct, flow);
+    if(filename_start == 0) {
+      /* Flow starting with a response? */
+      if(is_response(ndpi_struct, flow)) {
+        NDPI_LOG_DBG2(ndpi_struct, "Response where a request were expected\n");
+	/* This is tricky. Two opposing goals:
+	   1) We want to correctly match request with response!! -> Skip this response
+	      and keep looking for a request.
+	   2) We want to support asymmetric detection
+	   Trade-off:
+	   a) set HTTP as master (it is a guess; we can't know it from the reply only)
+	   b) process the response(s) and save the metadata
+	   c) look for a request. If we found it, reset everything (master,
+	      classification and metadata!) */
+        ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP);
+        process_response(ndpi_struct, flow);
 
-      NDPI_LOG_DBG2(ndpi_struct,
-		    "HTTP START Found, we will look for sub-protocols (content and host)...\n");
-
-      if(packet->host_line.ptr != NULL) {
-        /**
-           nDPI is pretty scrupulous about HTTP so it waits until the
-           HTTP response is received just to check that it conforms
-           with the HTTP specs. However this might be a waste of time as
-           in 99.99% of the cases is like that.
-        */
-
-	/* if(!flow->http_detected) */ {
-	  u_int proto = flow->detected_protocol_stack[1] ? flow->detected_protocol_stack[1] : flow->detected_protocol_stack[0];
-
-	  if(proto == NDPI_PROTOCOL_UNKNOWN) proto = NDPI_PROTOCOL_HTTP;
-	  ndpi_int_http_add_connection(ndpi_struct, flow, proto);
-	  flow->http_detected = 1;
-	  NDPI_LOG_DBG2(ndpi_struct,
-			"HTTP START Found, we will look further for the response...\n");
-	  flow->l4.tcp.http_stage = packet->packet_direction + 1; // packet_direction 0: stage 1, packet_direction 1: stage 2
-	  check_content_type_and_change_protocol(ndpi_struct, flow);
-	}
+	flow->l4.tcp.http_stage = packet->packet_direction + 3; // packet_direction 0: stage 3, packet_direction 1: stage 4
         return;
       }
+      /* The first pkt is neither a request nor a response -> no http */
+      NDPI_LOG_DBG2(ndpi_struct, "Neither req nor response -> exclude\n");
+      NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
+      return;
     }
+    NDPI_LOG_DBG2(ndpi_struct, "Request where expected\n");
 
-    check_content_type_and_change_protocol(ndpi_struct, flow);
-    NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
-  } else if((flow->l4.tcp.http_stage == 1) || (flow->l4.tcp.http_stage == 2)) {
-    NDPI_LOG_DBG2(ndpi_struct, "HTTP stage %u: \n", flow->l4.tcp.http_stage);
-
-    /**
-       At first check, if this is for sure a response packet
-       (in another direction. If not, if HTTP is detected do nothing now and return,
-       otherwise check the second packet for the HTTP request
-    */
-    if((flow->l4.tcp.http_stage - packet->packet_direction) == 1) { /* Expected a response package */
-      NDPI_LOG_DBG2(ndpi_struct,
-		    " SECOND PAYLOAD TRAFFIC FROM CLIENT, FIRST PACKET MIGHT HAVE BEEN HTTP...UNKNOWN TRAFFIC, HERE FOR HTTP again.. \n");
-
-      ndpi_parse_packet_line_info(ndpi_struct, flow);
-
-      if(packet->parsed_lines <= 1) {
-        /* wait some packets in case request is split over more than 2 packets */
-        if(flow->packet_counter < 5) {
-          NDPI_LOG_DBG2(ndpi_struct, "line still not finished, search next packet\n");
-          return;
-        } else {
-          /* stop parsing here */
-          NDPI_LOG_DBG2(ndpi_struct, "exclude HTTP: PACKET DOES NOT HAVE A LINE STRUCTURE\n");
-	  NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
-          return;
-        }
-      }
-
-      /* The previous pkt was a valid HTTP request and this one is in the same direction:
-         splitted request? Try to extract more metadata */
-      if(flow->http_detected) {
-        check_content_type_and_change_protocol(ndpi_struct, flow);
-      }
-
-      // http://www.slideshare.net/DSPIP/rtsp-analysis-wireshark
-      if(!flow->http_detected
-	 && packet->line[0].len >= 9
-	 && strncasecmp((const char *)&packet->line[0].ptr[packet->line[0].len - 9], " HTTP/1.", 8) == 0) {
+    process_request(ndpi_struct, flow, filename_start);
 
-        NDPI_LOG_INFO(ndpi_struct, "found HTTP\n");
-        ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP);
-        check_content_type_and_change_protocol(ndpi_struct, flow);
+    /* Wait for the response */
+    flow->l4.tcp.http_stage = packet->packet_direction + 1; // packet_direction 0: stage 1, packet_direction 1: stage 2
 
-        NDPI_LOG_DBG2(ndpi_struct,
-		      "HTTP START Found in 2. packet, we will look further for the response....\n");
-        flow->http_detected = 1;
+    return;
+  } else if(flow->l4.tcp.http_stage == 1 || flow->l4.tcp.http_stage == 2) {
+    /* Found a request, looking for the response */
+
+    if(flow->l4.tcp.http_stage - packet->packet_direction == 1) {
+      /* Another pkt from the same direction (probably another fragment of the request)
+         Keep lookng for the response */
+      NDPI_LOG_DBG2(ndpi_struct, "Another piece of request\n");
+      filename_start = is_request(ndpi_struct, flow);
+      if(filename_start > 0) {
+        /* Probably a new, separated request (asymmetric flow or missing pkts?).
+	   What should we do? We definitely don't want to mix data from different
+	   requests. The easiest (but costly) idea is to reset the state and
+	   process it (i.e. we keep the metadata of the last request that we
+	   have processed) */
+        reset(ndpi_struct, flow);
+        process_request(ndpi_struct, flow, filename_start);
+	return;
       }
-
+      ndpi_parse_packet_line_info(ndpi_struct, flow);
+      check_content_type_and_change_protocol(ndpi_struct, flow);
       return;
-    }
+    } else if(is_response(ndpi_struct, flow)) {
+      NDPI_LOG_DBG2(ndpi_struct, "Response where expected\n");
 
-    /**
-       This is a packet in another direction. Check if we find the proper response.
-       We have received a response for a previously identified partial HTTP request
-    */
+      process_response(ndpi_struct, flow);
 
-    /* response without headers
-     * TODO: Shouldn't it be below  ndpi_parse_packet_line_info, line ~825 ?
-     */
-    if((packet->parsed_lines == 1) && (packet->packet_direction == 1 /* server -> client */)) {
-      /* In Apache if you do "GET /\n\n" the response comes without any header */
-      NDPI_LOG_INFO(ndpi_struct, "found HTTP. (apache)\n");
-      ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP);
+      flow->l4.tcp.http_stage = 0;
+    } else {
+      NDPI_LOG_DBG2(ndpi_struct, "The msg from the server doesn't look like a response...\n");
+      /* TODO */
+    }
+  } else if(flow->l4.tcp.http_stage == 3 || flow->l4.tcp.http_stage == 4) {
+    /* Found a response but we want a request */
+
+    if(flow->l4.tcp.http_stage - packet->packet_direction == 3) {
+      /* Another pkt from the same direction (probably another fragment of the response)
+         Keep lookng for the request */
+      NDPI_LOG_DBG2(ndpi_struct, "Another piece of response\n");
+      if(is_response(ndpi_struct, flow)) {
+        /* See the comment above about how we handle consecutive requests/responses */
+        reset(ndpi_struct, flow);
+        process_response(ndpi_struct, flow);
+	return;
+      }
+      ndpi_parse_packet_line_info(ndpi_struct, flow);
       check_content_type_and_change_protocol(ndpi_struct, flow);
       return;
     }
 
-    parse_response_code(ndpi_struct, flow);
-
-    /* Parse packet line and we look for the subprotocols */
-    ndpi_parse_packet_line_info(ndpi_struct, flow);
-    check_content_type_and_change_protocol(ndpi_struct, flow);
-    ndpi_validate_http_content(ndpi_struct, flow);
+    NDPI_LOG_DBG2(ndpi_struct, "Found a request. We need to reset the state!\n");
 
+    reset(ndpi_struct, flow);
     flow->l4.tcp.http_stage = 0;
-    return;
+    return ndpi_check_http_tcp(ndpi_struct, flow);
   }
 }
 
@@ -1437,6 +1423,13 @@ static void ndpi_search_http_tcp(struct ndpi_detection_module_struct *ndpi_struc
 
   NDPI_LOG_DBG(ndpi_struct, "search HTTP\n");
   ndpi_check_http_tcp(ndpi_struct, flow);
+
+  if(flow->host_server_name[0] != '\0'&&
+     flow->http.response_status_code != 0) {
+    flow->extra_packets_func = NULL; /* We're good now */
+
+    if(flow->initial_binary_bytes_len) ndpi_analyze_content_signature(ndpi_struct, flow);
+  }
 }
 
 /* ********************************* */
diff --git a/src/lib/protocols/rtsp.c b/src/lib/protocols/rtsp.c
index 1dc63f04b..9b43d5259 100644
--- a/src/lib/protocols/rtsp.c
+++ b/src/lib/protocols/rtsp.c
@@ -55,6 +55,11 @@ static void ndpi_search_rtsp_tcp_udp(struct ndpi_detection_module_struct *ndpi_s
        LINE_ENDS(packet->content_line, "application/x-rtsp-tunnelled") != 0))
   {
     ndpi_int_rtsp_add_connection(ndpi_struct, flow);
+
+    /* Extract some metadata HTTP-like */
+    if(packet->user_agent_line.ptr != NULL)
+      ndpi_user_agent_set(flow, packet->user_agent_line.ptr, packet->user_agent_line.len);
+
     return;
   }