Improved category detection with HTTP

2 files changed, 91 insertions, 63 deletions

diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index 270c8ae36..40bf9ae20 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -4699,7 +4699,7 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
     if(flow->check_extra_packets) {
       ndpi_process_extra_packet(ndpi_str, flow, packet, packetlen, current_tick_l, src, dst);
       /* Update in case of new match */
-      ret.master_protocol = flow->detected_protocol_stack[1], ret.app_protocol = flow->detected_protocol_stack[0];
+      ret.master_protocol = flow->detected_protocol_stack[1], ret.app_protocol = flow->detected_protocol_stack[0], ret.category = flow->category;;
       return(ret);
     } else
       goto ret_protocols;
diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c
index 6b26a7dc3..4382879d0 100644
--- a/src/lib/protocols/http.c
+++ b/src/lib/protocols/http.c
@@ -33,7 +33,8 @@ static void ndpi_search_http_tcp(struct ndpi_detection_module_struct *ndpi_struc
 
 /* *********************************************** */
 
-static int ndpi_search_http_tcp_again(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) {
+static int ndpi_search_http_tcp_again(struct ndpi_detection_module_struct *ndpi_struct,
+				      struct ndpi_flow_struct *flow) {
   ndpi_search_http_tcp(ndpi_struct, flow);
 
 #ifdef HTTP_DEBUG
@@ -52,9 +53,46 @@ static int ndpi_search_http_tcp_again(struct ndpi_detection_module_struct *ndpi_
 
 /* *********************************************** */
 
+/* https://www.freeformatter.com/mime-types-list.html */
+static ndpi_protocol_category_t ndpi_http_check_content(struct ndpi_detection_module_struct *ndpi_struct,
+							struct ndpi_flow_struct *flow) {
+  struct ndpi_packet_struct *packet = &flow->packet;
+
+  if(packet->content_line.len > 0) {
+    u_int app_len = sizeof("application");
+
+    if(packet->content_line.len > app_len) {
+      if(ndpi_strncasestr((const char *)&packet->content_line.ptr[app_len], "mpeg",
+			  packet->content_line.len-app_len) != NULL) {
+	flow->guessed_category = flow->category = NDPI_PROTOCOL_CATEGORY_STREAMING;
+	return(flow->category);
+      }
+    }
+
+    switch(packet->content_line.ptr[0]) {
+    case 'a':
+      if(strncasecmp((const char *)packet->content_line.ptr, "audio",
+		     ndpi_min(packet->content_line.len, 5)) == 0)
+	flow->guessed_category = flow->category = NDPI_PROTOCOL_CATEGORY_MEDIA;
+      break;
+      
+    case 'v':
+      if(strncasecmp((const char *)packet->content_line.ptr, "video",
+		     ndpi_min(packet->content_line.len, 5)) == 0)
+	flow->guessed_category = flow->category = NDPI_PROTOCOL_CATEGORY_MEDIA;
+      break;
+    }
+  }
+
+  return(flow->category);
+}
+
+/* *********************************************** */
+
 static void ndpi_int_http_add_connection(struct ndpi_detection_module_struct *ndpi_struct,
 					 struct ndpi_flow_struct *flow,
-					 u_int16_t category) {
+					 u_int16_t http_protocol,
+					 ndpi_protocol_category_t category) {
 #ifdef HTTP_DEBUG
   printf("=> %s()\n", __FUNCTION__);
 #endif
@@ -66,30 +104,36 @@ static void ndpi_int_http_add_connection(struct ndpi_detection_module_struct *nd
   ndpi_search_tcp_or_udp(ndpi_struct, flow);
 
   /* If no custom protocol has been detected */
-  if(flow->guessed_host_protocol_id != NDPI_PROTOCOL_UNKNOWN) {
-    ndpi_int_reset_protocol(flow);
-    ndpi_set_detected_protocol(ndpi_struct, flow, flow->guessed_host_protocol_id, NDPI_PROTOCOL_HTTP);
-  } else
-    ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_HTTP, NDPI_PROTOCOL_UNKNOWN);
+  if((flow->guessed_host_protocol_id == NDPI_PROTOCOL_UNKNOWN) || (http_protocol != NDPI_PROTOCOL_HTTP))
+    flow->guessed_host_protocol_id = http_protocol;
+
+  category = ndpi_http_check_content(ndpi_struct, flow);
+  ndpi_int_reset_protocol(flow);
+  ndpi_set_detected_protocol(ndpi_struct, flow, flow->guessed_host_protocol_id, NDPI_PROTOCOL_HTTP);
 
   /* This is necessary to inform the core to call this dissector again */
   flow->check_extra_packets = 1;
   flow->max_extra_packets_to_check = 5;
   flow->extra_packets_func = ndpi_search_http_tcp_again;
-  flow->http_detected = 1, flow->guessed_category = category;
+  flow->http_detected = 1, flow->guessed_category = flow->category = category;
 }
 
+/* ************************************************************* */
+
 static void rtsp_parse_packet_acceptline(struct ndpi_detection_module_struct
 					 *ndpi_struct, struct ndpi_flow_struct *flow)
 {
   struct ndpi_packet_struct *packet = &flow->packet;
 
-  if(packet->accept_line.len >= 28 && memcmp(packet->accept_line.ptr, "application/x-rtsp-tunnelled", 28) == 0) {
+  if((packet->accept_line.len >= 28)
+     && (memcmp(packet->accept_line.ptr, "application/x-rtsp-tunnelled", 28) == 0)) {
     NDPI_LOG_INFO(ndpi_struct, "found RTSP accept line\n");
-    ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_RTSP);
+    ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_RTSP, NDPI_PROTOCOL_CATEGORY_MEDIA);
   }
 }
 
+/* ************************************************************* */
+
 static void setHttpUserAgent(struct ndpi_detection_module_struct *ndpi_struct,
 			     struct ndpi_flow_struct *flow, char *ua) {
   if (    !strcmp(ua, "Windows NT 5.0"))  ua = "Windows 2000";
@@ -110,7 +154,10 @@ static void setHttpUserAgent(struct ndpi_detection_module_struct *ndpi_struct,
   }
 }
 
-static void parseHttpSubprotocol(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) {
+/* ************************************************************* */
+
+static void ndpi_http_parse_subprotocol(struct ndpi_detection_module_struct *ndpi_struct,
+				 struct ndpi_flow_struct *flow) {
   if((flow->l4.tcp.http_stage == 0) || (flow->http.url && flow->http_detected)) {
     char *double_col = strchr((char*)flow->host_server_name, ':');
     ndpi_protocol_match_result ret_match;
@@ -124,13 +171,14 @@ static void parseHttpSubprotocol(struct ndpi_detection_module_struct *ndpi_struc
   }
 }
 
+/* ************************************************************* */
+
 /**
    NOTE
    ndpi_parse_packet_line_info is in ndpi_main.c
 */
 static void check_content_type_and_change_protocol(struct ndpi_detection_module_struct *ndpi_struct,
 						   struct ndpi_flow_struct *flow) {
-
   struct ndpi_packet_struct *packet = &flow->packet;
 
   ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_HTTP, NDPI_PROTOCOL_UNKNOWN);
@@ -142,13 +190,12 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
   /* PPStream */
   if(flow->l4.tcp.ppstream_stage > 0 && flow->iqiyi_counter == 0) {
     NDPI_LOG_INFO(ndpi_struct, "found PPStream\n");
-    /* ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_PPSTREAM); */
-    ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_PPSTREAM, NDPI_PROTOCOL_HTTP);
-  }
-  else if(flow->iqiyi_counter > 0) {
+    ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_PPSTREAM,
+			       NDPI_PROTOCOL_HTTP, NDPI_PROTOCOL_CATEGORY_STREAMING);
+  } else if(flow->iqiyi_counter > 0) {
     NDPI_LOG_INFO(ndpi_struct, "found iQiyi\n");
-    /* ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_IQIYI); */
-    ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_IQIYI, NDPI_PROTOCOL_HTTP);
+    ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_IQIYI,
+			       NDPI_PROTOCOL_HTTP, NDPI_PROTOCOL_CATEGORY_STREAMING);
   }
 #endif
 
@@ -156,8 +203,7 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
   /* 1KXUN */
   if(flow->kxun_counter > 0) {
     NDPI_LOG_INFO(ndpi_struct, "found 1kxun\n");
-    /* ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_1KXUN); */
-    ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_1KXUN, NDPI_PROTOCOL_HTTP);
+    ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_1KXUN, NDPI_PROTOCOL_CATEGORY_STREAMING);
   }
 #endif
 
@@ -257,10 +303,9 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
 	      setHttpUserAgent(ndpi_struct, flow, token);
 	  }
 	}
-      }
-      else if(memcmp(ua, "netflix-ios-app", 15) == 0) {
+      } else if(memcmp(ua, "netflix-ios-app", 15) == 0) {
 	NDPI_LOG_INFO(ndpi_struct, "found netflix\n");
-      	ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_NETFLIX);
+      	ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_NETFLIX, NDPI_PROTOCOL_CATEGORY_STREAMING);
       	return;
       }
     }
@@ -294,7 +339,7 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
       flow->host_server_name[len] = '\0';
       flow->extra_packets_func = NULL; /* We're good now */
     }
-    
+
     flow->server_id = flow->dst;
 
     if(packet->forwarded_line.ptr) {
@@ -305,7 +350,7 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
       }
     }
 
-    parseHttpSubprotocol(ndpi_struct, flow);
+    ndpi_http_parse_subprotocol(ndpi_struct, flow);
 
     /**
        check result of host subprotocol detection
@@ -344,7 +389,7 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
       if(packet->detected_protocol_stack[0] != NDPI_PROTOCOL_HTTP) {
 	NDPI_LOG_INFO(ndpi_struct, "found HTTP/%s\n",
 		      ndpi_get_proto_name(ndpi_struct, packet->detected_protocol_stack[0]));
-	ndpi_int_http_add_connection(ndpi_struct, flow, packet->detected_protocol_stack[0]);
+	ndpi_int_http_add_connection(ndpi_struct, flow, packet->detected_protocol_stack[0], NDPI_PROTOCOL_CATEGORY_WEB);
 	return; /* We have identified a sub-protocol so we're done */
       }
     }
@@ -352,7 +397,7 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
 
 #if 0
   if(flow->http_detected)
-    parseHttpSubprotocol(ndpi_struct, flow);
+    ndpi_http_parse_subprotocol(ndpi_struct, flow);
 #endif
 
   if(flow->guessed_protocol_id == NDPI_PROTOCOL_UNKNOWN)
@@ -374,7 +419,7 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
 
     if((flow->http.content_type == NULL) && (packet->content_line.len > 0)) {
       int len = packet->content_line.len + 1;
-      
+
       flow->http.content_type = ndpi_malloc(len);
       if(flow->http.content_type) {
 	strncpy(flow->http.content_type, (char*)packet->content_line.ptr,
@@ -382,7 +427,7 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
 	flow->http.content_type[packet->content_line.len] = '\0';
       }
     }
-    
+
     if(flow->http_detected) {
       ndpi_protocol_match_result ret_match;
 
@@ -392,13 +437,17 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
     }
   }
 
-  ndpi_int_http_add_connection(ndpi_struct, flow, packet->detected_protocol_stack[0]);
+  ndpi_int_http_add_connection(ndpi_struct, flow, packet->detected_protocol_stack[0], NDPI_PROTOCOL_CATEGORY_WEB);
 }
 
+/* ************************************************************* */
+
 static void check_http_payload(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) {
   /* Add here your paylod code check */
 }
 
+/* ************************************************************* */
+
 /**
  * Functions to check whether the packet begins with a valid http request
  * @param ndpi_struct
@@ -529,7 +578,7 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
           </cross-domain-policy>
         */
       ookla_found:
-        ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_UNKNOWN);
+        ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_CATEGORY_WEB);
 
 	if(ndpi_struct->ookla_cache == NULL)
 	  ndpi_struct->ookla_cache = ndpi_lru_cache_init(1024);
@@ -593,34 +642,12 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
 	goto ookla_found;
       }
 
-#if OBSOLETE
-      /* Check for additional field introduced by Steam */
-      int x = 1;
-      if(packet->line[x].len >= 11 && (memcmp(packet->line[x].ptr, "x-steam-sid", 11)) == 0) {
-	NDPI_LOG_INFO(ndpi_struct, "found STEAM\n");
-	ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_STEAM);
-	check_content_type_and_change_protocol(ndpi_struct, flow);
-	return;
-      }
-
-      /* Check for additional field introduced by Facebook */
-      x = 1;
-      while(packet->line[x].len != 0) {
-	if(packet->line[x].len >= 12 && (memcmp(packet->line[x].ptr, "X-FB-SIM-HNI", 12)) == 0) {
-	  NDPI_LOG_INFO(ndpi_struct, "found FACEBOOK\n");
-	  ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_FACEBOOK);
-	  check_content_type_and_change_protocol(ndpi_struct, flow);
-	  return;
-	}
-	x++;
-      }
-#endif
-
 #if defined(NDPI_PROTOCOL_1KXUN) || defined(NDPI_PROTOCOL_IQIYI)
       /* check PPStream protocol or iQiyi service
 	 (iqiyi is delivered by ppstream) */
       // substring in url
-      if(ndpi_strnstr((const char*) &packet->payload[filename_start], "iqiyi.com", (packet->payload_packet_len - filename_start)) != NULL) {
+      if(ndpi_strnstr((const char*) &packet->payload[filename_start], "iqiyi.com",
+		      (packet->payload_packet_len - filename_start)) != NULL) {
 	if(flow->kxun_counter == 0) {
 	  flow->l4.tcp.ppstream_stage++;
 	  flow->iqiyi_counter++;
@@ -662,14 +689,14 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
       if((packet->http_url_name.len > 7)
 	 && (!strncmp((const char*) packet->http_url_name.ptr, "http://", 7))) {
         NDPI_LOG_INFO(ndpi_struct, "found HTTP_PROXY\n");
-        ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP_PROXY);
+        ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP_PROXY, NDPI_PROTOCOL_CATEGORY_WEB);
         check_content_type_and_change_protocol(ndpi_struct, flow);
       }
 
       if(filename_start == 8 && (memcmp(packet->payload, "CONNECT ", 8) == 0)) {
 	/* nathan@getoffmalawn.com */
         NDPI_LOG_INFO(ndpi_struct, "found HTTP_CONNECT\n");
-        ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP_CONNECT);
+        ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP_CONNECT, NDPI_PROTOCOL_CATEGORY_WEB);
         check_content_type_and_change_protocol(ndpi_struct, flow);
       }
 
@@ -684,7 +711,7 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
            in 99.99% of the cases is like that.
         */
 
-	ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP);
+	ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP, NDPI_PROTOCOL_CATEGORY_WEB);
 	flow->http_detected = 1;
 	NDPI_LOG_DBG2(ndpi_struct,
 		      "HTTP START Found, we will look further for the response...\n");
@@ -712,7 +739,8 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
       NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_PROTOCOL_OOKLA);
 
     /**
-       At first check, if this is for sure a response packet (in another direction. If not, if HTTP is detected do nothing now and return,
+       At first check, if this is for sure a response packet
+       (in another direction. If not, if HTTP is detected do nothing now and return,
        otherwise check the second packet for the HTTP request
     */
     if((flow->l4.tcp.http_stage - packet->packet_direction) == 1) { /* Expected a response package */
@@ -746,7 +774,7 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
 	 && memcmp(&packet->line[0].ptr[packet->line[0].len - 9], " HTTP/1.", 8) == 0) {
 
         NDPI_LOG_INFO(ndpi_struct, "found HTTP\n");
-        ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP);
+        ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP, NDPI_PROTOCOL_CATEGORY_WEB);
         check_content_type_and_change_protocol(ndpi_struct, flow);
 
         NDPI_LOG_DBG2(ndpi_struct,
@@ -768,7 +796,7 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
     if((packet->parsed_lines == 1) && (packet->packet_direction == 1 /* server -> client */)) {
       /* In Apache if you do "GET /\n\n" the response comes without any header */
       NDPI_LOG_INFO(ndpi_struct, "found HTTP. (apache)\n");
-      ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP);
+      ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP, NDPI_PROTOCOL_CATEGORY_WEB);
       check_content_type_and_change_protocol(ndpi_struct, flow);
       return;
     }
@@ -776,7 +804,7 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
     /* If we already detected the HTTP request, we can add the connection and then check for the sub-protocol */
     if(flow->http_detected) {
       NDPI_LOG_INFO(ndpi_struct, "found HTTP\n");
-      ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP);
+      ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP, NDPI_PROTOCOL_CATEGORY_WEB);
     }
 
     /* Parse packet line and we look for the subprotocols */