LRU caches: add a generic (optional and configurable) expiration logic (#1855)

Two caches already implemented a similar mechanism: make it generic.

7 files changed, 34 insertions, 29 deletions

diff --git a/src/lib/protocols/bittorrent.c b/src/lib/protocols/bittorrent.c
index cdb62e8db..852b7cbac 100644
--- a/src/lib/protocols/bittorrent.c
+++ b/src/lib/protocols/bittorrent.c
@@ -123,19 +123,20 @@ static void ndpi_add_connection_as_bittorrent(struct ndpi_detection_module_struc
 
     key1 = ndpi_ip_port_hash_funct(flow->c_address.v4, flow->c_port), key2 = ndpi_ip_port_hash_funct(flow->s_address.v4, flow->s_port);
     
-    ndpi_lru_add_to_cache(ndpi_struct->bittorrent_cache, key1, NDPI_PROTOCOL_BITTORRENT);
-    ndpi_lru_add_to_cache(ndpi_struct->bittorrent_cache, key2, NDPI_PROTOCOL_BITTORRENT);
+    ndpi_lru_add_to_cache(ndpi_struct->bittorrent_cache, key1, NDPI_PROTOCOL_BITTORRENT, ndpi_get_current_time(flow));
+    ndpi_lru_add_to_cache(ndpi_struct->bittorrent_cache, key2, NDPI_PROTOCOL_BITTORRENT, ndpi_get_current_time(flow));
 
     /* Now add hosts as twins */
     ndpi_lru_add_to_cache(ndpi_struct->bittorrent_cache,
 			  flow->c_address.v4 + flow->s_address.v4,
-			  NDPI_PROTOCOL_BITTORRENT);
+			  NDPI_PROTOCOL_BITTORRENT,
+			  ndpi_get_current_time(flow));
     
     /* Also add +2 ports of the sender in order to catch additional sockets open by the same client */
     for(i=0; i<2; i++) {
       key1 = ndpi_ip_port_hash_funct(flow->c_address.v4, htons(ntohs(flow->c_port)+1+i));
 
-      ndpi_lru_add_to_cache(ndpi_struct->bittorrent_cache, key1, NDPI_PROTOCOL_BITTORRENT);
+      ndpi_lru_add_to_cache(ndpi_struct->bittorrent_cache, key1, NDPI_PROTOCOL_BITTORRENT, ndpi_get_current_time(flow));
     }
     
 #ifdef BITTORRENT_CACHE_DEBUG
diff --git a/src/lib/protocols/hangout.c b/src/lib/protocols/hangout.c
index b5bde02da..576d95014 100644
--- a/src/lib/protocols/hangout.c
+++ b/src/lib/protocols/hangout.c
@@ -80,7 +80,7 @@ void ndpi_search_hangout(struct ndpi_detection_module_struct *ndpi_struct,
 	printf("[LRU] ADDING %u / %u.%u\n", key, NDPI_PROTOCOL_STUN, NDPI_PROTOCOL_HANGOUT_DUO);
 #endif
 
-	ndpi_lru_add_to_cache(ndpi_struct->stun_cache, key, NDPI_PROTOCOL_HANGOUT_DUO);
+	ndpi_lru_add_to_cache(ndpi_struct->stun_cache, key, NDPI_PROTOCOL_HANGOUT_DUO, ndpi_get_current_time(flow));
       }
       
       ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_HANGOUT_DUO,
diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c
index 50df30830..b9cc198c1 100644
--- a/src/lib/protocols/http.c
+++ b/src/lib/protocols/http.c
@@ -1255,9 +1255,9 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
 	if(ndpi_struct->ookla_cache != NULL) {
 	  if(packet->iph != NULL) {
 	    if(packet->tcp->source == htons(8080))
-	      ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, packet->iph->saddr, 1 /* dummy */);
+	      ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, packet->iph->saddr, 1 /* dummy */, ndpi_get_current_time(flow));
 	    else
-	      ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, packet->iph->daddr, 1 /* dummy */);
+	      ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, packet->iph->daddr, 1 /* dummy */, ndpi_get_current_time(flow));
 	  } else if(packet->iphv6 != NULL) {
 	    u_int32_t h;
 
@@ -1266,7 +1266,7 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
 	    else
 	      h = ndpi_quick_hash((unsigned char *)&packet->iphv6->ip6_dst, sizeof(packet->iphv6->ip6_dst));
 
-	    ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, h, 1 /* dummy */);
+	    ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, h, 1 /* dummy */, ndpi_get_current_time(flow));
 	  }
 	}
 
diff --git a/src/lib/protocols/mining.c b/src/lib/protocols/mining.c
index 4ea3e0a27..49d91e738 100644
--- a/src/lib/protocols/mining.c
+++ b/src/lib/protocols/mining.c
@@ -44,7 +44,7 @@ u_int32_t make_mining_key(struct ndpi_flow_struct *flow) {
 static void cacheMiningHostTwins(struct ndpi_detection_module_struct *ndpi_struct,
 				 struct ndpi_flow_struct *flow) {
   if(ndpi_struct->mining_cache)
-    ndpi_lru_add_to_cache(ndpi_struct->mining_cache, make_mining_key(flow), NDPI_PROTOCOL_MINING);
+    ndpi_lru_add_to_cache(ndpi_struct->mining_cache, make_mining_key(flow), NDPI_PROTOCOL_MINING, ndpi_get_current_time(flow));
 }
 
 /* ************************************************************************** */
diff --git a/src/lib/protocols/ookla.c b/src/lib/protocols/ookla.c
index 137b0a2a3..f9a7a2d94 100644
--- a/src/lib/protocols/ookla.c
+++ b/src/lib/protocols/ookla.c
@@ -63,7 +63,7 @@ void ndpi_search_ookla(struct ndpi_detection_module_struct* ndpi_struct, struct
 #ifdef OOKLA_DEBUG
 	  printf("=>>>>>>>> [OOKLA IPv6] Adding %u\n", h);
 #endif
-	  ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, h, 1 /* dummy */);
+	  ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, h, 1 /* dummy */, ndpi_get_current_time(flow));
 	}
 	return;
       } else {
@@ -79,7 +79,8 @@ void ndpi_search_ookla(struct ndpi_detection_module_struct* ndpi_struct, struct
 	  printf("=>>>>>>>> [OOKLA IPv6] Searching %u\n", h);
 #endif
 	  
-	  if(ndpi_lru_find_cache(ndpi_struct->ookla_cache, h, &dummy, 0 /* Don't remove it as it can be used for other connections */)) {
+	  if(ndpi_lru_find_cache(ndpi_struct->ookla_cache, h, &dummy, 0 /* Don't remove it as it can be used for other connections */,
+				 ndpi_get_current_time(flow))) {
 	    NDPI_LOG_INFO(ndpi_struct, "found ookla tcp connection\n");
 	    ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI_CACHE);
 #ifdef OOKLA_DEBUG
@@ -110,7 +111,8 @@ void ndpi_search_ookla(struct ndpi_detection_module_struct* ndpi_struct, struct
     if(ndpi_struct->ookla_cache != NULL) {
       u_int16_t dummy;
     
-      if(ndpi_lru_find_cache(ndpi_struct->ookla_cache, addr, &dummy, 0 /* Don't remove it as it can be used for other connections */)) {
+      if(ndpi_lru_find_cache(ndpi_struct->ookla_cache, addr, &dummy, 0 /* Don't remove it as it can be used for other connections */,
+			     ndpi_get_current_time(flow))) {
 	NDPI_LOG_INFO(ndpi_struct, "found ookla tcp connection\n");
 	ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI_CACHE);
 #ifdef OOKLA_DEBUG
diff --git a/src/lib/protocols/stun.c b/src/lib/protocols/stun.c
index 8610565ba..6792bfe8a 100644
--- a/src/lib/protocols/stun.c
+++ b/src/lib/protocols/stun.c
@@ -56,7 +56,7 @@ u_int32_t get_stun_lru_key(struct ndpi_flow_struct *flow, u_int8_t rev) {
 int stun_search_into_zoom_cache(struct ndpi_detection_module_struct *ndpi_struct,
                                 struct ndpi_flow_struct *flow)
 {
-  u_int16_t when;
+  u_int16_t dummy;
   u_int32_t key;
 
   if(ndpi_struct->stun_zoom_cache &&
@@ -67,15 +67,12 @@ int stun_search_into_zoom_cache(struct ndpi_detection_module_struct *ndpi_struct
 #endif
 
     if(ndpi_lru_find_cache(ndpi_struct->stun_zoom_cache, key,
-                           &when, 0 /* Don't remove it as it can be used for other connections */)) {
-      u_int16_t tdiff = ((flow->last_packet_time_ms /1000) & 0xFFFF) - when;
-
+                           &dummy, 0 /* Don't remove it as it can be used for other connections */,
+			   ndpi_get_current_time(flow))) {
 #ifdef DEBUG_ZOOM_LRU
-      printf("[LRU ZOOM] Found, diff %d\n", tdiff);
+      printf("[LRU ZOOM] Found");
 #endif
-
-      if(tdiff < 60 /* sec */)
-        return 1;
+      return 1;
     }
   }
   return 0;
@@ -102,7 +99,8 @@ static void ndpi_int_stun_add_connection(struct ndpi_detection_module_struct *nd
     u_int16_t cached_proto;
 
     if(ndpi_lru_find_cache(ndpi_struct->stun_cache, key,
-			   &cached_proto, 0 /* Don't remove it as it can be used for other connections */)) {
+			   &cached_proto, 0 /* Don't remove it as it can be used for other connections */,
+			   ndpi_get_current_time(flow))) {
 #ifdef DEBUG_LRU
       printf("[LRU] FOUND %u / %u: no need to cache %u.%u\n", key, cached_proto, proto, app_proto);
 #endif
@@ -114,7 +112,8 @@ static void ndpi_int_stun_add_connection(struct ndpi_detection_module_struct *nd
       u_int32_t key_rev = get_stun_lru_key(flow, 1);
 
       if(ndpi_lru_find_cache(ndpi_struct->stun_cache, key_rev,
-			     &cached_proto, 0 /* Don't remove it as it can be used for other connections */)) {
+			     &cached_proto, 0 /* Don't remove it as it can be used for other connections */,
+			     ndpi_get_current_time(flow))) {
 #ifdef DEBUG_LRU
 	printf("[LRU] FOUND %u / %u: no need to cache %u.%u\n", key_rev, cached_proto, proto, app_proto);
 #endif
@@ -131,8 +130,8 @@ static void ndpi_int_stun_add_connection(struct ndpi_detection_module_struct *nd
 		 ntohs(packet->udp->source), ntohs(packet->udp->dest));
 #endif
 
-	  ndpi_lru_add_to_cache(ndpi_struct->stun_cache, key, app_proto);
-	  ndpi_lru_add_to_cache(ndpi_struct->stun_cache, key_rev, app_proto);
+	  ndpi_lru_add_to_cache(ndpi_struct->stun_cache, key, app_proto, ndpi_get_current_time(flow));
+	  ndpi_lru_add_to_cache(ndpi_struct->stun_cache, key_rev, app_proto, ndpi_get_current_time(flow));
 	}
       }
     }
@@ -147,7 +146,7 @@ static void ndpi_int_stun_add_connection(struct ndpi_detection_module_struct *nd
     printf("[LRU ZOOM] ADDING %u [src_port %u]\n", key, ntohs(flow->c_port));
 #endif
     ndpi_lru_add_to_cache(ndpi_struct->stun_zoom_cache, key,
-                         (flow->last_packet_time_ms / 1000) & 0xFFFF /* 16 bit */);
+                          0 /* dummy */, ndpi_get_current_time(flow));
   }
 
   ndpi_set_detected_protocol(ndpi_struct, flow, app_proto, NDPI_PROTOCOL_STUN, confidence);
@@ -234,7 +233,8 @@ static ndpi_int_stun_t ndpi_int_check_stun(struct ndpi_detection_module_struct *
     u_int16_t proto;
     u_int32_t key = get_stun_lru_key(flow, 0);
     int rc = ndpi_lru_find_cache(ndpi_struct->stun_cache, key, &proto,
-                                 0 /* Don't remove it as it can be used for other connections */);
+                                 0 /* Don't remove it as it can be used for other connections */,
+				 ndpi_get_current_time(flow));
 
 #ifdef DEBUG_LRU
     printf("[LRU] Searching %u\n", key);
@@ -243,7 +243,8 @@ static ndpi_int_stun_t ndpi_int_check_stun(struct ndpi_detection_module_struct *
     if(!rc) {
       key = get_stun_lru_key(flow, 1);
       rc = ndpi_lru_find_cache(ndpi_struct->stun_cache, key, &proto,
-                               0 /* Don't remove it as it can be used for other connections */);
+                               0 /* Don't remove it as it can be used for other connections */,
+			       ndpi_get_current_time(flow));
 
 #ifdef DEBUG_LRU
       printf("[LRU] Searching %u\n", key);
diff --git a/src/lib/protocols/tls.c b/src/lib/protocols/tls.c
index 49c465bae..299e59ecb 100644
--- a/src/lib/protocols/tls.c
+++ b/src/lib/protocols/tls.c
@@ -340,7 +340,8 @@ static void checkTLSSubprotocol(struct ndpi_detection_module_struct *ndpi_struct
       key = make_tls_cert_key(packet, is_from_client);
 
       if(ndpi_lru_find_cache(ndpi_struct->tls_cert_cache, key,
-			     &cached_proto, 0 /* Don't remove it as it can be used for other connections */)) {
+			     &cached_proto, 0 /* Don't remove it as it can be used for other connections */,
+			     ndpi_get_current_time(flow))) {
 	ndpi_protocol ret = { __get_master(ndpi_struct, flow), cached_proto, NDPI_PROTOCOL_UNKNOWN /* unused */, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, NULL};
 
 	ndpi_set_detected_protocol(ndpi_struct, flow, cached_proto, __get_master(ndpi_struct, flow), NDPI_CONFIDENCE_DPI_CACHE);
@@ -730,7 +731,7 @@ static void processCertificateElements(struct ndpi_detection_module_struct *ndpi
 	if(ndpi_struct->tls_cert_cache) {
 	  u_int32_t key = make_tls_cert_key(packet, 0 /* from the server */);
 
-	  ndpi_lru_add_to_cache(ndpi_struct->tls_cert_cache, key, proto_id);
+	  ndpi_lru_add_to_cache(ndpi_struct->tls_cert_cache, key, proto_id, ndpi_get_current_time(flow));
 	}
       }
     }