Merge pull request #1012 from IvanNardi/ua

QUIC: extract User Agent information

3 files changed, 197 insertions, 87 deletions

diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c
index acfa55769..983a53b1c 100644
--- a/src/lib/protocols/http.c
+++ b/src/lib/protocols/http.c
@@ -275,6 +275,90 @@ static void ndpi_check_user_agent(struct ndpi_detection_module_struct *ndpi_stru
   }
 }
 
+int http_process_user_agent(struct ndpi_detection_module_struct *ndpi_struct,
+			    struct ndpi_flow_struct *flow,
+			    const u_int8_t *ua_ptr, u_int16_t ua_ptr_len)
+{
+  /**
+      Format examples:
+      Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) ....
+      Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:54.0) Gecko/20100101 Firefox/54.0
+   */
+  if(ua_ptr_len > 7) {
+    char ua[256];
+    u_int mlen = ndpi_min(ua_ptr_len, sizeof(ua)-1);
+
+    strncpy(ua, (const char *)ua_ptr, mlen);
+    ua[mlen] = '\0';
+
+    if(strncmp(ua, "Mozilla", 7) == 0) {
+      char *parent = strchr(ua, '(');
+
+      if(parent) {
+	char *token, *end;
+
+	parent++;
+	end = strchr(parent, ')');
+	if(end) end[0] = '\0';
+
+	token = strsep(&parent, ";");
+	if(token) {
+	  if((strcmp(token, "X11") == 0)
+	     || (strcmp(token, "compatible") == 0)
+	     || (strcmp(token, "Linux") == 0)
+	     || (strcmp(token, "Macintosh") == 0)
+	     ) {
+	    token = strsep(&parent, ";");
+	    if(token && (token[0] == ' ')) token++; /* Skip space */
+
+	    if(token
+	       && ((strcmp(token, "U") == 0)
+		   || (strncmp(token, "MSIE", 4) == 0))) {
+	      token = strsep(&parent, ";");
+	      if(token && (token[0] == ' ')) token++; /* Skip space */
+
+              if(token && (strncmp(token, "Update", 6)  == 0)) {
+                token = strsep(&parent, ";");
+
+                if(token && (token[0] == ' ')) token++; /* Skip space */
+
+                if(token && (strncmp(token, "AOL", 3)  == 0)) {
+
+                  token = strsep(&parent, ";");
+                  if(token && (token[0] == ' ')) token++; /* Skip space */
+                }
+              }
+            }
+          }
+
+          if(token)
+            setHttpUserAgent(ndpi_struct, flow, token);
+	}
+      }
+    } else if((ua_ptr_len > 14) && (memcmp(ua, "netflix-ios-app", 15) == 0)) {
+      NDPI_LOG_INFO(ndpi_struct, "found netflix\n");
+      ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_NETFLIX, NDPI_PROTOCOL_CATEGORY_STREAMING);
+      return -1;
+    }
+  }
+
+  if(flow->http.user_agent == NULL) {
+    int len = ua_ptr_len + 1;
+
+    flow->http.user_agent = ndpi_malloc(len);
+    if(flow->http.user_agent) {
+      strncpy(flow->http.user_agent, (char*)ua_ptr, ua_ptr_len);
+      flow->http.user_agent[ua_ptr_len] = '\0';
+
+      ndpi_check_user_agent(ndpi_struct, flow, flow->http.user_agent);
+    }
+  }
+
+  NDPI_LOG_DBG2(ndpi_struct, "User Agent Type line found %.*s\n",
+		ua_ptr_len, ua_ptr);
+  return 0;
+}
+
 /* ************************************************************* */
 
 static void ndpi_check_numeric_ip(struct ndpi_detection_module_struct *ndpi_struct,
@@ -308,6 +392,7 @@ static void ndpi_check_http_url(struct ndpi_detection_module_struct *ndpi_struct
 static void check_content_type_and_change_protocol(struct ndpi_detection_module_struct *ndpi_struct,
 						   struct ndpi_flow_struct *flow) {
   struct ndpi_packet_struct *packet = &flow->packet;
+  int ret;
 
   ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_HTTP, NDPI_PROTOCOL_UNKNOWN);
 
@@ -342,84 +427,10 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
   }
   
   if(packet->user_agent_line.ptr != NULL && packet->user_agent_line.len != 0) {
-    /**
-       Format examples:
-       Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) ....
-       Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:54.0) Gecko/20100101 Firefox/54.0
-    */
-    if(packet->user_agent_line.len > 7) {
-      char ua[256];
-      u_int mlen = ndpi_min(packet->user_agent_line.len, sizeof(ua)-1);
-
-      strncpy(ua, (const char *)packet->user_agent_line.ptr, mlen);
-      ua[mlen] = '\0';
-
-      if(strncmp(ua, "Mozilla", 7) == 0) {
-	char *parent = strchr(ua, '(');
-
-	if(parent) {
-	  char *token, *end;
-
-	  parent++;
-	  end = strchr(parent, ')');
-	  if(end) end[0] = '\0';
-
-	  token = strsep(&parent, ";");
-	  if(token) {
-	    if((strcmp(token, "X11") == 0)
-	       || (strcmp(token, "compatible") == 0)
-	       || (strcmp(token, "Linux") == 0)
-	       || (strcmp(token, "Macintosh") == 0)
-	       ) {
-	      token = strsep(&parent, ";");
-	      if(token && (token[0] == ' ')) token++; /* Skip space */
-
-	      if(token
-		 && ((strcmp(token, "U") == 0)
-		     || (strncmp(token, "MSIE", 4) == 0))) {
-		token = strsep(&parent, ";");
-		if(token && (token[0] == ' ')) token++; /* Skip space */
-
-		if(token && (strncmp(token, "Update", 6)  == 0)) {
-		  token = strsep(&parent, ";");
-
-		  if(token && (token[0] == ' ')) token++; /* Skip space */
-
-		  if(token && (strncmp(token, "AOL", 3)  == 0)) {
-
-		    token = strsep(&parent, ";");
-		    if(token && (token[0] == ' ')) token++; /* Skip space */
-		  }
-		}
-	      }
-	    }
-
-	    if(token)
-	      setHttpUserAgent(ndpi_struct, flow, token);
-	  }
-	}
-      } else if((packet->user_agent_line.len > 14) && (memcmp(ua, "netflix-ios-app", 15) == 0)) {
-	NDPI_LOG_INFO(ndpi_struct, "found netflix\n");
-      	ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_NETFLIX, NDPI_PROTOCOL_CATEGORY_STREAMING);
-      	return;
-      }
-    }
-
-    if(flow->http.user_agent == NULL) {
-      int len = packet->user_agent_line.len + 1;
-
-      flow->http.user_agent = ndpi_malloc(len);
-      if(flow->http.user_agent) {
-	strncpy(flow->http.user_agent, (char*)packet->user_agent_line.ptr,
-		packet->user_agent_line.len);
-	flow->http.user_agent[packet->user_agent_line.len] = '\0';
-
-	ndpi_check_user_agent(ndpi_struct, flow, flow->http.user_agent);
-      }
-    }
-
-    NDPI_LOG_DBG2(ndpi_struct, "User Agent Type line found %.*s\n",
-		  packet->user_agent_line.len, packet->user_agent_line.ptr);
+    ret = http_process_user_agent(ndpi_struct, flow, packet->user_agent_line.ptr, packet->user_agent_line.len);
+    /* TODO: Is it correct to avoid setting ua, host_name,... if we have a (Netflix) subclassification? */
+    if(ret != 0)
+      return;
   }
 
   /* check for host line */
diff --git a/src/lib/protocols/quic.c b/src/lib/protocols/quic.c
index 7dda2c27b..2a4c7294b 100644
--- a/src/lib/protocols/quic.c
+++ b/src/lib/protocols/quic.c
@@ -43,7 +43,10 @@
    */
 
 extern int processClientServerHello(struct ndpi_detection_module_struct *ndpi_struct,
-                                    struct ndpi_flow_struct *flow, int is_quic);
+                                    struct ndpi_flow_struct *flow, uint32_t quic_version);
+extern int http_process_user_agent(struct ndpi_detection_module_struct *ndpi_struct,
+                                   struct ndpi_flow_struct *flow,
+                                   const u_int8_t *ua_ptr, u_int16_t ua_ptr_len);
 
 /* Versions */
 #define V_Q024		0x51303234
@@ -141,9 +144,13 @@ static int is_version_with_tls(uint32_t version)
   return is_version_quic(version) ||
     ((version & 0xFFFFFF00) == 0x54303500) /* T05X */;
 }
+int is_version_with_var_int_transport_params(uint32_t version)
+{
+  return (is_version_quic(version) && is_quic_ver_greater_than(version, 27)) ||
+    (version == V_T051);
+}
 
-
-static int quic_len(const uint8_t *buf, uint64_t *value)
+int quic_len(const uint8_t *buf, uint64_t *value)
 {
   *value = buf[0];
   switch((*value) >> 6) {
@@ -163,6 +170,22 @@ static int quic_len(const uint8_t *buf, uint64_t *value)
     return 0;
   }
 }
+int quic_len_buffer_still_required(uint8_t value)
+{
+  switch(value >> 6) {
+  case 0:
+    return 0;
+  case 1:
+    return 1;
+  case 2:
+    return 3;
+  case 3:
+    return 7;
+  default: /* No Possible */
+    return 0;
+  }
+}
+
 
 static uint16_t gquic_get_u16(const uint8_t *buf, uint32_t version)
 {
@@ -1023,7 +1046,8 @@ static uint8_t *get_clear_payload(struct ndpi_detection_module_struct *ndpi_stru
 }
 static void process_tls(struct ndpi_detection_module_struct *ndpi_struct,
 			struct ndpi_flow_struct *flow,
-			const u_int8_t *crypto_data, uint32_t crypto_data_len)
+			const u_int8_t *crypto_data, uint32_t crypto_data_len,
+			uint32_t version)
 {
   struct ndpi_packet_struct *packet = &flow->packet;
 
@@ -1035,7 +1059,7 @@ static void process_tls(struct ndpi_detection_module_struct *ndpi_struct,
   packet->payload = crypto_data;
   packet->payload_packet_len = crypto_data_len;
 
-  processClientServerHello(ndpi_struct, flow, 1);
+  processClientServerHello(ndpi_struct, flow, version);
 
   /* Restore */
   packet->payload = p;
@@ -1057,6 +1081,7 @@ static void process_chlo(struct ndpi_detection_module_struct *ndpi_struct,
   uint32_t prev_offset;
   uint32_t tag_offset_start, offset, len, sni_len;
   ndpi_protocol_match_result ret_match;
+  int sni_found = 0, ua_found = 0;
 
   if(crypto_data_len < 6)
     return;
@@ -1096,7 +1121,19 @@ static void process_chlo(struct ndpi_detection_module_struct *ndpi_struct,
                                   (char *)flow->host_server_name,
                                   strlen((const char*)flow->host_server_name),
                                   &ret_match, NDPI_PROTOCOL_QUIC);
-      return;
+      sni_found = 1;
+      if (ua_found)
+        return;
+    }
+    if((memcmp(tag, "UAID", 4) == 0) &&
+       (tag_offset_start + prev_offset + len < crypto_data_len)) {
+      NDPI_LOG_DBG2(ndpi_struct, "UA: [%.*s]\n", len, &crypto_data[tag_offset_start + prev_offset]);
+
+      http_process_user_agent(ndpi_struct, flow,
+                              &crypto_data[tag_offset_start + prev_offset], len);
+      ua_found = 1;
+      if (sni_found)
+        return;
     }
 
     prev_offset = offset;
@@ -1250,7 +1287,7 @@ void ndpi_search_quic(struct ndpi_detection_module_struct *ndpi_struct,
   if(!is_version_with_tls(version)) {
     process_chlo(ndpi_struct, flow, crypto_data, crypto_data_len);
   } else {
-    process_tls(ndpi_struct, flow, crypto_data, crypto_data_len);
+    process_tls(ndpi_struct, flow, crypto_data, crypto_data_len, version);
   }
   if(is_version_with_encrypted_header(version)) {
     ndpi_free(clear_payload);
diff --git a/src/lib/protocols/tls.c b/src/lib/protocols/tls.c
index 71fb2d5c3..134dfe614 100644
--- a/src/lib/protocols/tls.c
+++ b/src/lib/protocols/tls.c
@@ -31,7 +31,14 @@
 
 extern char *strptime(const char *s, const char *format, struct tm *tm);
 extern int processClientServerHello(struct ndpi_detection_module_struct *ndpi_struct,
-				    struct ndpi_flow_struct *flow, int is_quic);
+				    struct ndpi_flow_struct *flow, uint32_t quic_version);
+extern int http_process_user_agent(struct ndpi_detection_module_struct *ndpi_struct,
+                                   struct ndpi_flow_struct *flow,
+                                   const u_int8_t *ua_ptr, u_int16_t ua_ptr_len);
+/* QUIC/GQUIC stuff */
+extern int quic_len(const uint8_t *buf, uint64_t *value);
+extern int quic_len_buffer_still_required(uint8_t value);
+extern int is_version_with_var_int_transport_params(uint32_t version);
 
 // #define DEBUG_TLS_MEMORY       1
 // #define DEBUG_TLS              1
@@ -864,7 +871,7 @@ struct ja3_info {
 /* **************************************** */
 
 int processClientServerHello(struct ndpi_detection_module_struct *ndpi_struct,
-			     struct ndpi_flow_struct *flow, int is_quic) {
+			     struct ndpi_flow_struct *flow, uint32_t quic_version) {
   struct ndpi_packet_struct *packet = &flow->packet;
   struct ja3_info ja3;
   u_int8_t invalid_ja3 = 0;
@@ -876,6 +883,7 @@ int processClientServerHello(struct ndpi_detection_module_struct *ndpi_struct,
   u_int16_t total_len;
   u_int8_t handshake_type;
   char buffer[64] = { '\0' };
+  int is_quic = (quic_version != 0);
   int is_dtls = packet->udp && (!is_quic);
 
 #ifdef DEBUG_TLS
@@ -1366,6 +1374,60 @@ int processClientServerHello(struct ndpi_detection_module_struct *ndpi_struct,
 		    }
 		  }
 		}
+	      } else if(extension_id == 65445 /* QUIC transport parameters */) {
+		u_int16_t s_offset = offset+extension_offset;
+		uint16_t final_offset;
+		int using_var_int = is_version_with_var_int_transport_params(quic_version);
+
+		if(!using_var_int) {
+		  if(s_offset+1 >= total_len) {
+		    final_offset = 0; /* Force skipping extension */
+		  } else {
+		    u_int16_t seq_len = ntohs(*((u_int16_t*)&packet->payload[s_offset]));
+		    s_offset += 2;
+	            final_offset = MIN(total_len, s_offset + seq_len);
+		  }
+		} else {
+	          final_offset = MIN(total_len, s_offset + extension_len);
+		}
+
+		while(s_offset < final_offset) {
+		  u_int64_t param_type, param_len;
+
+                  if(!using_var_int) {
+		    if(s_offset+3 >= final_offset)
+		      break;
+		    param_type = ntohs(*((u_int16_t*)&packet->payload[s_offset]));
+		    param_len = ntohs(*((u_int16_t*)&packet->payload[s_offset + 2]));
+		    s_offset += 4;
+		  } else {
+		    if(s_offset >= final_offset ||
+		       (s_offset + quic_len_buffer_still_required(packet->payload[s_offset])) >= final_offset)
+		      break;
+		    s_offset += quic_len(&packet->payload[s_offset], &param_type);
+
+		    if(s_offset >= final_offset ||
+		       (s_offset + quic_len_buffer_still_required(packet->payload[s_offset])) >= final_offset)
+		      break;
+		    s_offset += quic_len(&packet->payload[s_offset], &param_len);
+		  }
+
+#ifdef DEBUG_TLS
+		  printf("Client SSL [QUIC TP: Param 0x%x Len %d]\n", (int)param_type, (int)param_len);
+#endif
+		  if(s_offset+param_len >= final_offset)
+		    break;
+
+		  if(param_type==0x3129) {
+#ifdef DEBUG_TLS
+		      printf("UA [%.*s]\n", (int)param_len, &packet->payload[s_offset]);
+#endif
+		      http_process_user_agent(ndpi_struct, flow,
+					      &packet->payload[s_offset], param_len);
+		      break;
+		  }
+		  s_offset += param_len;
+		}
 	      }
 
 	      extension_offset += extension_len; /* Move to the next extension */