Ookla: rework detection (#1922)

The logic of the LRU cache has been changed: once we know an ip has
connected to an Ookla server, all the following (unknown) flows (for
a short time interval) from the same ip to the port 8080 are treated
as Ookla ones.

Most of the changes in this commit are about introducing the concept of
"aggressive detection". In some cases, to properly detect a
protocol we might use some statistical/behavior logic that, from one
side, let us to identify the protocol more often but, from the other
side, might lead to some false positives.
To allow the user/application to easily detect when such logic has been
triggered, the new confidence value `NDPI_CONFIDENCE_DPI_AGGRESSIVE` has been
added.
It is always possible to disable/configure this kind of logic via the
API.

Detection of Ookla flows using plain TLS over port 8080 is the first
example of aggressive detection in nDPI.

Tested with:
* Android 9.0 with app 4.8.3
* Ubuntu 20.04 with Firefox 110
* Win 10 with app 1.15 and 1.16
* Win 10 with Chrome 108, Edge 108 and Firefox 106

4 files changed, 157 insertions, 151 deletions

diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index f4266d87d..dc5834549 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -198,6 +198,10 @@ extern void ndpi_unset_risk(struct ndpi_detection_module_struct *ndpi_str,
 			    struct ndpi_flow_struct *flow, ndpi_risk_enum r);
 extern u_int32_t make_mining_key(struct ndpi_flow_struct *flow);
 extern int stun_search_into_zoom_cache(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow);
+extern void ookla_add_to_cache(struct ndpi_detection_module_struct *ndpi_struct,
+                               struct ndpi_flow_struct *flow);
+extern int ookla_search_into_cache(struct ndpi_detection_module_struct *ndpi_struct,
+                                   struct ndpi_flow_struct *flow);
 
 /* Forward */
 static int addDefaultPort(struct ndpi_detection_module_struct *ndpi_str,
@@ -2932,7 +2936,7 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs
   ndpi_str->msteams_cache_num_entries = 1024;
   ndpi_str->stun_zoom_cache_num_entries = 1024;
 
-  ndpi_str->ookla_cache_ttl = 0;
+  ndpi_str->ookla_cache_ttl = 120; /* sec */
   ndpi_str->bittorrent_cache_ttl = 0;
   ndpi_str->zoom_cache_ttl = 0;
   ndpi_str->stun_cache_ttl = 0;
@@ -2946,6 +2950,8 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs
   ndpi_str->opportunistic_tls_pop_enabled = 1;
   ndpi_str->opportunistic_tls_ftp_enabled = 1;
 
+  ndpi_str->aggressiveness_ookla = NDPI_AGGRESSIVENESS_OOKLA_TLS;
+
   for(i = 0; i < NUM_CUSTOM_CATEGORIES; i++)
     ndpi_snprintf(ndpi_str->custom_category_labels[i], CUSTOM_CATEGORY_LABEL_LEN, "User custom category %u",
 	     (unsigned int) (i + 1));
@@ -6254,6 +6260,13 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st
     ret.app_protocol = flow->detected_protocol_stack[0];
   }
 
+  /* Does it looks like Ookla? */
+  if(ret.app_protocol == NDPI_PROTOCOL_UNKNOWN &&
+     ntohs(flow->s_port) == 8080 && ookla_search_into_cache(ndpi_str, flow)) {
+    ndpi_set_detected_protocol(ndpi_str, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI_PARTIAL_CACHE);
+    ret.app_protocol = flow->detected_protocol_stack[0];
+  }
+
   /* Classification by-port is the last resort */
   if(enable_guess && ret.app_protocol == NDPI_PROTOCOL_UNKNOWN) {
 
@@ -8052,6 +8065,9 @@ const char *ndpi_confidence_get_name(ndpi_confidence_t confidence)
   case NDPI_CONFIDENCE_MATCH_BY_IP:
     return "Match by IP";
 
+  case NDPI_CONFIDENCE_DPI_AGGRESSIVE:
+    return "DPI (aggressive)";
+
   default:
     return NULL;
   }
@@ -8572,6 +8588,11 @@ int ndpi_match_hostname_protocol(struct ndpi_detection_module_struct *ndpi_struc
     ndpi_set_detected_protocol(ndpi_struct, flow, subproto, master_protocol, NDPI_CONFIDENCE_DPI);
     if(!category_depends_on_master(master_protocol))
       ndpi_int_change_category(ndpi_struct, flow, ret_match.protocol_category);
+
+    if(subproto == NDPI_PROTOCOL_OOKLA) {
+	ookla_add_to_cache(ndpi_struct, flow);
+    }
+
     return(1);
   } else
     return(0);
@@ -9643,3 +9664,36 @@ int ndpi_get_opportunistic_tls(struct ndpi_detection_module_struct *ndpi_struct,
     return -1;
   }
 }
+
+/* ******************************************************************** */
+
+int ndpi_set_protocol_aggressiveness(struct ndpi_detection_module_struct *ndpi_struct,
+                                     u_int16_t proto, u_int32_t value)
+{
+  if(!ndpi_struct)
+    return -1;
+
+  switch(proto) {
+  case NDPI_PROTOCOL_OOKLA:
+    ndpi_struct->aggressiveness_ookla = value;
+    return 0;
+  default:
+    return -1;
+  }
+}
+
+/* ******************************************************************** */
+
+u_int32_t ndpi_get_protocol_aggressiveness(struct ndpi_detection_module_struct *ndpi_struct,
+                                           u_int16_t proto)
+{
+  if(!ndpi_struct)
+    return -1;
+
+  switch(proto) {
+  case NDPI_PROTOCOL_OOKLA:
+    return ndpi_struct->aggressiveness_ookla;
+  default:
+    return -1;
+  }
+}
diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c
index e0f56c4e8..4f139b8d3 100644
--- a/src/lib/protocols/http.c
+++ b/src/lib/protocols/http.c
@@ -46,6 +46,9 @@ static const char* binary_file_ext[] = {
 					NULL
 };
 
+extern void ookla_add_to_cache(struct ndpi_detection_module_struct *ndpi_struct,
+                               struct ndpi_flow_struct *flow);
+
 static void ndpi_search_http_tcp(struct ndpi_detection_module_struct *ndpi_struct,
 				 struct ndpi_flow_struct *flow);
 
@@ -436,6 +439,7 @@ static void ndpi_http_parse_subprotocol(struct ndpi_detection_module_struct *ndp
           || (strstr(flow->http.url, ":8080/upload?n=0.") != NULL))) {
 	/* This looks like Ookla speedtest */
 	ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, master_protocol, NDPI_CONFIDENCE_DPI);
+	ookla_add_to_cache(ndpi_struct, flow);
       }
     }
 
@@ -1217,30 +1221,6 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
         return;
       }
 
-      if((packet->payload_packet_len == 3) && memcmp(packet->payload, "HI\n", 3) == 0) {
-	/* This looks like Ookla: we don't give up with HTTP yet */
-        flow->l4.tcp.http_stage = 1;
-	return;
-      }
-
-      if((packet->payload_packet_len == 40) && (flow->l4.tcp.http_stage == 0)) {
-        /*
-	  -> QR O06L0072-6L91-4O43-857J-K8OO172L6L51
-	  <- QNUUX 2.5 2017-08-15.1314.4jn12m5
-	  -> MXFWUXJM 31625365
-	*/
-
-        if((packet->payload[2] == ' ')
-	   && (packet->payload[11] == '-')
-	   && (packet->payload[16] == '-')
-	   && (packet->payload[21] == '-')
-	   && (packet->payload[26] == '-')
-	   && (packet->payload[39] == 0x0A)
-	   )
-	  flow->l4.tcp.http_stage = 1;
-	return;
-      }
-
       if((packet->payload_packet_len == 23) && (memcmp(packet->payload, "<policy-file-request/>", 23) == 0)) {
         /*
           <policy-file-request/>
@@ -1251,25 +1231,7 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
         */
       ookla_found:
         ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_CATEGORY_WEB);
-
-	if(ndpi_struct->ookla_cache != NULL) {
-	  if(packet->iph != NULL) {
-	    if(packet->tcp->source == htons(8080))
-	      ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, packet->iph->saddr, 1 /* dummy */, ndpi_get_current_time(flow));
-	    else
-	      ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, packet->iph->daddr, 1 /* dummy */, ndpi_get_current_time(flow));
-	  } else if(packet->iphv6 != NULL) {
-	    u_int32_t h;
-
-	    if(packet->tcp->source == htons(8080))
-	      h = ndpi_quick_hash((unsigned char *)&packet->iphv6->ip6_src, sizeof(packet->iphv6->ip6_src));
-	    else
-	      h = ndpi_quick_hash((unsigned char *)&packet->iphv6->ip6_dst, sizeof(packet->iphv6->ip6_dst));
-
-	    ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, h, 1 /* dummy */, ndpi_get_current_time(flow));
-	  }
-	}
-
+        ookla_add_to_cache(ndpi_struct, flow);
         return;
       }
 
@@ -1389,18 +1351,6 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
   } else if((flow->l4.tcp.http_stage == 1) || (flow->l4.tcp.http_stage == 2)) {
     NDPI_LOG_DBG2(ndpi_struct, "HTTP stage %u: \n", flow->l4.tcp.http_stage);
 
-    if((packet->payload_packet_len == 34) && (flow->l4.tcp.http_stage == 1)) {
-      if((packet->payload[5] == ' ') && (packet->payload[9] == ' ')) {
-	goto ookla_found;
-      }
-    }
-
-    if((packet->payload_packet_len > 6) && memcmp(packet->payload, "HELLO ", 6) == 0) {
-      /* This looks like Ookla */
-      goto ookla_found;
-    } else
-      NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_PROTOCOL_OOKLA);
-
     /**
        At first check, if this is for sure a response packet
        (in another direction. If not, if HTTP is detected do nothing now and return,
diff --git a/src/lib/protocols/ookla.c b/src/lib/protocols/ookla.c
index 0da42212c..d7764e630 100644
--- a/src/lib/protocols/ookla.c
+++ b/src/lib/protocols/ookla.c
@@ -23,111 +23,92 @@
 
 #include "ndpi_api.h"
 
+/* #define DEBUG_OOKLA_LRU */
+
 const u_int16_t ookla_port = 8080;
 
 /* ************************************************************* */
 
-static void ndpi_search_ookla(struct ndpi_detection_module_struct* ndpi_struct, struct ndpi_flow_struct* flow) {
-  struct ndpi_packet_struct* packet = &ndpi_struct->packet;
-  u_int32_t addr = 0;
-  u_int16_t sport, dport;
-    
-  NDPI_LOG_DBG(ndpi_struct, "Ookla detection\n");
-
-  if(packet->tcp)
-    sport = ntohs(packet->tcp->source), dport = htons(packet->tcp->dest);
+static u_int32_t get_ookla_key(struct ndpi_flow_struct *flow)
+{
+  if(flow->is_ipv6)
+    return ndpi_quick_hash(flow->c_address.v6, 16);
   else
-    sport = ntohs(packet->udp->source), dport = htons(packet->udp->dest);
+    return ntohl(flow->c_address.v4);
+}
 
-  if((sport != ookla_port) && (dport != ookla_port)) {
-#ifdef OOKLA_DEBUG
-    printf("=>>>>>>>> [OOKLA IPv6] Skipping flow [%u -> %u]\n", sport, dport);
-#endif
-    goto ookla_exclude;
-  }
-  
-  if(packet->iphv6 != NULL) {
-    if((dport == ookla_port) && (packet->payload_packet_len >= 3)) {
-      u_int32_t h;
-      
-      if((packet->payload_packet_len == 3)
-	 && (packet->payload[0] == 0x48) /* HI\n */
-	 && (packet->payload[1] == 0x49)
-	 && (packet->payload[2] == 0x0A)) {	
-	ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI);
-	
-	if(ndpi_struct->ookla_cache != NULL) {
-	  /* In order to avoid creating an IPv6 LRU we hash the IPv6 address */
-	  h = ndpi_quick_hash((unsigned char *)&packet->iphv6->ip6_dst, sizeof(packet->iphv6->ip6_dst));
-
-#ifdef OOKLA_DEBUG
-	  printf("=>>>>>>>> [OOKLA IPv6] Adding %u\n", h);
-#endif
-	  ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, h, 1 /* dummy */, ndpi_get_current_time(flow));
-	}
-	return;
-      } else {
-	if(sport == ookla_port)
-	  h = ndpi_quick_hash((unsigned char *)&packet->iphv6->ip6_src, sizeof(packet->iphv6->ip6_src));
-	else
-	  h = ndpi_quick_hash((unsigned char *)&packet->iphv6->ip6_dst, sizeof(packet->iphv6->ip6_dst));
-	
-	if(ndpi_struct->ookla_cache != NULL) {
-	  u_int16_t dummy;
-
-#ifdef OOKLA_DEBUG
-	  printf("=>>>>>>>> [OOKLA IPv6] Searching %u\n", h);
-#endif
-	  
-	  if(ndpi_lru_find_cache(ndpi_struct->ookla_cache, h, &dummy, 0 /* Don't remove it as it can be used for other connections */,
-				 ndpi_get_current_time(flow))) {
-	    NDPI_LOG_INFO(ndpi_struct, "found ookla tcp connection\n");
-	    ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI_CACHE);
-#ifdef OOKLA_DEBUG
-	    printf("=>>>>> Found %u\n", h);
-#endif
-	    return;
-	  } else {
-#ifdef OOKLA_DEBUG
-	    printf("=>>>>> NOT Found %u\n", h);
-#endif
-	  }
-	}
-      }
-    } else {
+/* ************************************************************* */
 
-      goto ookla_exclude;
-    }
-  } else {
-    if(sport == ookla_port)
-      addr = packet->iph->saddr;
-    else
-      addr = packet->iph->daddr;
-
-#ifdef OOKLA_DEBUG
-    printf("=>>>>>>>> [OOKLA IPv4] Searching %u\n", addr);
-#endif
-    
-    if(ndpi_struct->ookla_cache != NULL) {
-      u_int16_t dummy;
-    
-      if(ndpi_lru_find_cache(ndpi_struct->ookla_cache, addr, &dummy, 0 /* Don't remove it as it can be used for other connections */,
-			     ndpi_get_current_time(flow))) {
-	NDPI_LOG_INFO(ndpi_struct, "found ookla tcp connection\n");
-	ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI_CACHE);
-#ifdef OOKLA_DEBUG
-	printf("=>>>>> Found %u\n", addr);
+int ookla_search_into_cache(struct ndpi_detection_module_struct *ndpi_struct,
+                            struct ndpi_flow_struct *flow)
+{
+  u_int32_t key;
+  u_int16_t dummy;
+
+  if(ndpi_struct->ookla_cache) {
+    key = get_ookla_key(flow);
+#ifdef DEBUG_OOKLA_LRU
+    printf("[LRU OOKLA] Search %u\n", key);
 #endif
-	return;
-      } else {
-#ifdef OOKLA_DEBUG
-	printf("=>>>>> NOT Found %u\n", addr);
+
+    if(ndpi_lru_find_cache(ndpi_struct->ookla_cache, key,
+                           &dummy, 0 /* Don't remove it as it can be used for other connections */,
+			   ndpi_get_current_time(flow))) {
+#ifdef DEBUG_OOKLA_LRU
+      printf("[LRU OOKLA] Found\n");
 #endif
-      }
+      return 1;
     }
   }
+  return 0;
+}
+
+/* ************************************************************* */
+
+void ookla_add_to_cache(struct ndpi_detection_module_struct *ndpi_struct,
+                        struct ndpi_flow_struct *flow)
+{
+  u_int32_t key;
+
+  if(ndpi_struct->ookla_cache) {
+    key = get_ookla_key(flow);
+#ifdef DEBUG_OOKLA_LRU
+    printf("[LRU OOKLA] ADDING %u\n", key);
+#endif
+    ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, key, 1 /* dummy */,
+                          ndpi_get_current_time(flow));
+  }
+
+}
+
+/* ************************************************************* */
+
+void ndpi_search_ookla(struct ndpi_detection_module_struct* ndpi_struct, struct ndpi_flow_struct* flow) {
+  struct ndpi_packet_struct *packet = &ndpi_struct->packet;
+
+  NDPI_LOG_DBG(ndpi_struct, "Ookla detection\n");
+
+  if(ntohs(flow->s_port) != ookla_port && ntohs(flow->c_port) != ookla_port) {
+    NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
+    return;
+  }
+
+  if(flow->packet_counter == 1 &&
+     packet->payload_packet_len >= NDPI_STATICSTRING_LEN("HI") &&
+     memcmp(packet->payload, "HI", NDPI_STATICSTRING_LEN("HI")) == 0) {
+    flow->ookla_stage = 1;
+    return;
+  }
+  if(flow->packet_counter == 2 &&
+     flow->ookla_stage == 1 &&
+     packet->payload_packet_len >= NDPI_STATICSTRING_LEN("HELLO") &&
+     memcmp(packet->payload, "HELLO", NDPI_STATICSTRING_LEN("HELLO")) == 0) {
+    NDPI_LOG_INFO(ndpi_struct, "found ookla (Hi + Hello)\n");
+    ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI);
+    ookla_add_to_cache(ndpi_struct, flow);
+    return;
+  }
   
- ookla_exclude:
   NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
 }
 
diff --git a/src/lib/protocols/tls.c b/src/lib/protocols/tls.c
index 8e3fbedb8..58b19d0d5 100644
--- a/src/lib/protocols/tls.c
+++ b/src/lib/protocols/tls.c
@@ -33,6 +33,8 @@ extern int processClientServerHello(struct ndpi_detection_module_struct *ndpi_st
 extern int http_process_user_agent(struct ndpi_detection_module_struct *ndpi_struct,
                                    struct ndpi_flow_struct *flow,
                                    const u_int8_t *ua_ptr, u_int16_t ua_ptr_len);
+extern int ookla_search_into_cache(struct ndpi_detection_module_struct* ndpi_struct,
+                                   struct ndpi_flow_struct* flow);
 /* QUIC/GQUIC stuff */
 extern int quic_len(const uint8_t *buf, uint64_t *value);
 extern int quic_len_buffer_still_required(uint8_t value);
@@ -1153,8 +1155,27 @@ static int ndpi_search_tls_tcp(struct ndpi_detection_module_struct *ndpi_struct,
 #ifdef DEBUG_TLS_BLOCKS
     printf("*** [TLS Block] No more blocks\n");
 #endif
-    flow->extra_packets_func = NULL;
-    return(0); /* That's all */
+    /* An ookla flow? */
+    if((ndpi_struct->aggressiveness_ookla & NDPI_AGGRESSIVENESS_OOKLA_TLS) && /* Feature enabled */
+       (!something_went_wrong &&
+        flow->tls_quic.certificate_processed == 1 &&
+        flow->protos.tls_quic.hello_processed == 1) && /* TLS handshake found without errors */
+       flow->detected_protocol_stack[0] == NDPI_PROTOCOL_TLS && /* No IMAPS/FTPS/... */
+       flow->detected_protocol_stack[1] == NDPI_PROTOCOL_UNKNOWN && /* No sub-classification */
+       ntohs(flow->s_port) == 8080 && /* Ookla port */
+       ookla_search_into_cache(ndpi_struct, flow)) {
+      NDPI_LOG_INFO(ndpi_struct, "found ookla (cache over TLS)\n");
+      /* Even if a LRU cache is involved, NDPI_CONFIDENCE_DPI_AGGRESSIVE seems more
+         suited than NDPI_CONFIDENCE_DPI_CACHE */
+      ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_TLS, NDPI_CONFIDENCE_DPI_AGGRESSIVE);
+      /* TLS over port 8080 usually triggers that risk; clear it */
+      ndpi_unset_risk(ndpi_struct, flow, NDPI_KNOWN_PROTOCOL_ON_NON_STANDARD_PORT);
+      flow->extra_packets_func = NULL;
+      return(0); /* That's all */
+    } else {
+      flow->extra_packets_func = NULL;
+      return(0); /* That's all */
+    }
   } else
     return(1);
 }