Implemented (disabled by default) DNS host cache. You can set the cache size as follows:

ndpiReader --cfg=dpi.address_cache_size,1000 -i <pcap>.pcap

In the above example the cache has up to 1000 entries.

In jcase ndpiReader exports data in JSON, the cache hostname (if found) is exported in the field server_hostname

6 files changed, 257 insertions, 8 deletions

diff --git a/src/include/ndpi_api.h b/src/include/ndpi_api.h
index edfb497d4..b08b6e69f 100644
--- a/src/include/ndpi_api.h
+++ b/src/include/ndpi_api.h
@@ -2334,6 +2334,19 @@ extern "C" {
 
   /* ******************************* */
 
+  /* Address cache API */
+  struct ndpi_address_cache* ndpi_init_address_cache(u_int32_t max_num_entries);
+  void ndpi_term_address_cache(struct ndpi_address_cache *cache);
+  u_int ndpi_address_cache_flush_expired(struct ndpi_address_cache *cache, u_int32_t epoch_now);
+  struct ndpi_address_cache_item* ndpi_address_cache_find(struct ndpi_address_cache *cache, ndpi_ip_addr_t ip_addr, u_int32_t epoch_now);
+  bool ndpi_address_cache_insert(struct ndpi_address_cache *cache, ndpi_ip_addr_t ip_addr, char *hostname,
+				 u_int32_t epoch_now, u_int32_t ttl);
+
+  struct ndpi_address_cache_item* ndpi_cache_address_find(struct ndpi_detection_module_struct *ndpi_struct,
+							  ndpi_ip_addr_t ip_addr);
+  
+  /* ******************************* */
+
   const char *ndpi_lru_cache_idx_to_name(lru_cache_type idx);
 
   /**
diff --git a/src/include/ndpi_private.h b/src/include/ndpi_private.h
index ccc198cf1..54f59f652 100644
--- a/src/include/ndpi_private.h
+++ b/src/include/ndpi_private.h
@@ -199,6 +199,7 @@ struct ndpi_detection_module_config_struct {
   int libgcrypt_init;
   int guess_on_giveup;
   int compute_entropy;
+  int address_cache_size;
   int fpc_enabled;
   int guess_ip_before_port;
   int use_client_ip_in_guess;
@@ -414,6 +415,7 @@ struct ndpi_detection_module_struct {
   u_int16_t max_payload_track_len;
 
   ndpi_str_hash *public_domain_suffixes;
+  struct ndpi_address_cache *address_cache;
 };
 
 
@@ -560,10 +562,6 @@ struct ndpi_detection_module_struct {
 #define NDPI_SELECTION_BITMASK_PROTOCOL_V6_TCP_OR_UDP_WITH_PAYLOAD_WITHOUT_RETRANSMISSION	(NDPI_SELECTION_BITMASK_PROTOCOL_V6_TCP_OR_UDP | NDPI_SELECTION_BITMASK_PROTOCOL_NO_TCP_RETRANSMISSION | NDPI_SELECTION_BITMASK_PROTOCOL_HAS_PAYLOAD)
 #define NDPI_SELECTION_BITMASK_PROTOCOL_V4_V6_TCP_OR_UDP_WITH_PAYLOAD_WITHOUT_RETRANSMISSION	(NDPI_SELECTION_BITMASK_PROTOCOL_V4_V6_TCP_OR_UDP | NDPI_SELECTION_BITMASK_PROTOCOL_NO_TCP_RETRANSMISSION | NDPI_SELECTION_BITMASK_PROTOCOL_HAS_PAYLOAD)
 
-
-
-
-
 /* Generic */
 
 char *strptime(const char *s, const char *format, struct tm *tm);
@@ -635,8 +633,11 @@ int load_category_file_fd(struct ndpi_detection_module_struct *ndpi_str,
 
 u_int64_t fpc_dns_cache_key_from_dns_info(struct ndpi_flow_struct *flow);
 
+bool ndpi_cache_address(struct ndpi_detection_module_struct *ndpi_struct,
+			ndpi_ip_addr_t ip_addr, char *hostname,
+			u_int32_t epoch_now, u_int32_t ttl);
 
-/* TLS */
+  /* TLS */
 int processClientServerHello(struct ndpi_detection_module_struct *ndpi_struct,
                              struct ndpi_flow_struct *flow, uint32_t quic_version);
 void processCertificateElements(struct ndpi_detection_module_struct *ndpi_struct,
diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h
index 6116bc453..b401aad11 100644
--- a/src/include/ndpi_typedefs.h
+++ b/src/include/ndpi_typedefs.h
@@ -1862,6 +1862,21 @@ struct ndpi_des_struct {
 
 /* **************************************** */
 
+struct ndpi_address_cache_item {
+  ndpi_ip_addr_t addr; /* key */
+  char *hostname; /* value */
+  u_int32_t expire_epoch;
+  struct ndpi_address_cache_item *next; /* Linked list */
+};
+
+struct ndpi_address_cache {
+  u_int32_t num_cached_addresses, num_root_nodes;
+  u_int32_t num_entries, max_num_entries;
+  struct ndpi_address_cache_item **address_cache_root;
+};
+
+/* **************************************** */
+
 /* Prototype used to define custom DGA detection function */
 typedef int (*ndpi_custom_dga_predict_fctn)(const char* domain, int domain_length);
 
diff --git a/src/lib/ndpi_cache.c b/src/lib/ndpi_cache.c
index abc03604d..40688c384 100644
--- a/src/lib/ndpi_cache.c
+++ b/src/lib/ndpi_cache.c
@@ -115,10 +115,10 @@ u_int8_t ndpi_lru_find_cache(struct ndpi_lru_cache *c, u_int64_t key,
      now_sec >= c->entries[slot].timestamp &&
      (c->ttl == 0 || now_sec - c->entries[slot].timestamp <= c->ttl)) {
     *value = c->entries[slot].value;
-    
+
     if(clean_key_when_found)
       c->entries[slot].is_full = 0;
-    
+
     c->stats.n_found++;
     ret = 1;
   } else
@@ -215,3 +215,209 @@ int ndpi_get_lru_cache_stats(struct ndpi_global_context *g_ctx,
 }
 
 /* ******************************************************************** */
+/* ******************************************************************** */
+
+struct ndpi_address_cache* ndpi_init_address_cache(u_int32_t max_num_entries) {
+  struct ndpi_address_cache *ret = (struct ndpi_address_cache*)ndpi_malloc(sizeof(struct ndpi_address_cache));
+
+  if(ret == NULL) return(ret);
+
+  ret->num_cached_addresses = 0, ret->num_entries = 0,
+    ret->max_num_entries = max_num_entries,
+    ret->num_root_nodes = ndpi_min(NDPI_NUM_DEFAULT_ROOT_NODES, max_num_entries/16);
+  ret->address_cache_root = (struct ndpi_address_cache_item**)ndpi_calloc(ret->num_root_nodes, sizeof(struct ndpi_address_cache_item*));
+
+  if(ret->address_cache_root == NULL) {
+    ndpi_free(ret);
+    return(NULL);
+  } else
+    return(ret);
+}
+
+/* ***************************************************** */
+
+static void ndpi_free_addr_item(struct ndpi_address_cache_item *addr) {
+  ndpi_free(addr->hostname);
+  ndpi_free(addr);
+}
+
+/* ***************************************************** */
+
+void ndpi_term_address_cache(struct ndpi_address_cache *cache) {
+  u_int i;
+
+  for(i=0; i<cache->num_root_nodes; i++) {
+    struct ndpi_address_cache_item *root = cache->address_cache_root[i];
+
+    while(root != NULL) {
+      struct ndpi_address_cache_item *next = root->next;
+
+      ndpi_free_addr_item(root);
+      root = next;
+    }
+  }
+
+  ndpi_free(cache->address_cache_root);
+}
+
+/* ***************************************************** */
+
+/* Return the number of purged entries */
+u_int ndpi_address_cache_flush_expired(struct ndpi_address_cache *cache,
+				       u_int32_t epoch_now) {
+  u_int i, num_purged = 0;
+
+  for(i=0; i<cache->num_root_nodes; i++) {
+    struct ndpi_address_cache_item *root = cache->address_cache_root[i];
+    struct ndpi_address_cache_item *prev = NULL;
+
+    while(root != NULL) {
+      struct ndpi_address_cache_item *next = root->next;
+
+      if(root->expire_epoch > epoch_now) {
+	/* Time to purge */
+
+	if(prev == NULL) {
+	  /* Head element */
+	  cache->address_cache_root[i] = next;
+	} else {
+	  /* Middle element */
+	  prev->next = next;
+	}
+
+	ndpi_free_addr_item(root), num_purged++;
+      } else {
+	prev = root;
+      }
+
+      root = next;
+    } /* while */
+  } /* for */
+
+  cache->num_entries -= num_purged;
+
+  return(num_purged);
+}
+
+
+/* ***************************************************** */
+
+struct ndpi_address_cache_item* ndpi_address_cache_find(struct ndpi_address_cache *cache,
+							ndpi_ip_addr_t ip_addr, u_int32_t epoch_now) {
+  u_int32_t hash_id = ndpi_quick_hash((const unsigned char *)&ip_addr, sizeof(ip_addr)) % cache->num_root_nodes;
+  struct ndpi_address_cache_item *root = cache->address_cache_root[hash_id], *prev = NULL;
+
+  while(root != NULL) {
+    if((epoch_now != 0) && (root->expire_epoch < epoch_now)) {
+      /* Expired entry: let's remove it */
+      struct ndpi_address_cache_item *next = root->next;
+
+      if(prev == NULL)
+	cache->address_cache_root[hash_id] = next;
+      else
+	prev->next = next;
+
+      ndpi_free_addr_item(root);
+      root = next, cache->num_entries--;
+
+      continue; /* Skip this entry */
+    }
+
+    if(memcmp(&root->addr, &ip_addr, sizeof(ndpi_ip_addr_t)) == 0) {
+      return(root);
+    } else
+      root = root->next;
+  }
+
+  return(NULL);
+}
+
+/* ***************************************************** */
+
+bool ndpi_address_cache_insert(struct ndpi_address_cache *cache,
+			       ndpi_ip_addr_t ip_addr, char *hostname,
+			       u_int32_t epoch_now,
+			       u_int32_t ttl) {
+  u_int32_t hash_id = ndpi_quick_hash((const unsigned char *)&ip_addr, sizeof(ip_addr)) % cache->num_root_nodes;
+  struct ndpi_address_cache addr;
+  struct ndpi_address_cache_item *ret;
+  u_int32_t epoch_valid_until;
+
+  if(epoch_now == 0) epoch_now = (u_int32_t)time(NULL);
+  ret = ndpi_address_cache_find(cache, ip_addr, epoch_now);
+  epoch_valid_until = epoch_now + ttl;
+
+  /* printf("**** %s [%u][ttl: %u]\n", hostname, epoch_now, ttl); */
+
+  if(ret == NULL) {
+    if(cache->num_entries == cache->max_num_entries) {
+      ndpi_address_cache_flush_expired(cache, epoch_now);
+
+      if(cache->num_entries == cache->max_num_entries)
+	return(false); /* Still no room left */
+
+      /* We have room to add the new element */
+      /* Let's continue */
+    }
+
+    /* We have room to insert the new element */
+    ret = (struct ndpi_address_cache_item*)ndpi_malloc(sizeof(struct ndpi_address_cache_item));
+    if(ret == NULL)
+      return(false); /* No memory */
+
+    memcpy(&ret->addr, &ip_addr, sizeof(addr)),
+      ret->expire_epoch = epoch_valid_until,
+      ret->next = cache->address_cache_root[hash_id];
+
+    /* Create linked list */
+    cache->address_cache_root[hash_id] = ret;
+
+    if((ret->hostname = strdup(hostname)) == NULL) {
+      ndpi_free(ret);
+      return(false);
+    }
+  } else {
+    /* Element found: update TTL of the existing element */
+    ret->expire_epoch = ndpi_max(ret->expire_epoch, epoch_valid_until);
+
+    if(strcmp(ret->hostname, hostname)) {
+      /* Hostnames are different: we overwrite it */
+      char *new_hostname = ndpi_strdup(hostname);
+
+      if(new_hostname) {
+	/* Allocation ok */
+	ndpi_free(ret->hostname);
+	ret->hostname = new_hostname;
+      }
+    }
+  }
+
+  cache->num_entries++;
+  return(true);
+}
+
+/* ***************************************************** */
+/* ***************************************************** */
+
+bool ndpi_cache_address(struct ndpi_detection_module_struct *ndpi_struct,
+			ndpi_ip_addr_t ip_addr, char *hostname,
+			u_int32_t epoch_now, u_int32_t ttl) {
+  if(ndpi_struct->cfg.address_cache_size == 0) return(false);
+
+  if(ndpi_struct->address_cache == NULL)
+    ndpi_struct->address_cache = ndpi_init_address_cache(ndpi_struct->cfg.address_cache_size);
+
+  if(ndpi_struct->address_cache)
+    return(ndpi_address_cache_insert(ndpi_struct->address_cache, ip_addr, hostname, epoch_now, ttl));
+  else
+    return(false);
+}
+
+/* ***************************************************** */
+
+struct ndpi_address_cache_item* ndpi_cache_address_find(struct ndpi_detection_module_struct *ndpi_struct,
+							ndpi_ip_addr_t ip_addr) {
+  if(ndpi_struct->address_cache == NULL) return(NULL);
+
+  return(ndpi_address_cache_find(ndpi_struct->address_cache, ip_addr, 0));
+}
diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index b114f1661..464e61e6b 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -4385,6 +4385,9 @@ void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_str) {
     if(ndpi_str->public_domain_suffixes)
       ndpi_hash_free(&ndpi_str->public_domain_suffixes);
 
+    if(ndpi_str->address_cache)
+      ndpi_term_address_cache(ndpi_str->address_cache);
+    
     ndpi_free(ndpi_str);
   }
 
@@ -8539,7 +8542,6 @@ static void fpc_check_eval(struct ndpi_detection_module_struct *ndpi_str,
 {
   u_int16_t fpc_dns_cached_proto;
 
-
   if(!ndpi_str->cfg.fpc_enabled)
     return;
 
@@ -11384,6 +11386,7 @@ static const struct cfg_param {
   { NULL,            "dpi.guess_on_giveup",                     "0x3", "0", "3", CFG_PARAM_INT, __OFF(guess_on_giveup), NULL },
   { NULL,            "dpi.guess_ip_before_port",                "disable", NULL, NULL, CFG_PARAM_ENABLE_DISABLE, __OFF(guess_ip_before_port), NULL},
   { NULL,            "dpi.compute_entropy",                     "1", NULL, NULL, CFG_PARAM_ENABLE_DISABLE, __OFF(compute_entropy), NULL },
+  { NULL,            "dpi.address_cache_size",                  "0", "0", "16777215", CFG_PARAM_INT, __OFF(address_cache_size), NULL },
   { NULL,            "fpc",                                     "1", NULL, NULL, CFG_PARAM_ENABLE_DISABLE, __OFF(fpc_enabled), NULL },
 
   { NULL,            "flow_risk_lists.load",                    "1", NULL, NULL, CFG_PARAM_ENABLE_DISABLE, __OFF(flow_risk_lists_enabled), NULL },
diff --git a/src/lib/protocols/dns.c b/src/lib/protocols/dns.c
index 8a6e2d1a8..d109098d1 100644
--- a/src/lib/protocols/dns.c
+++ b/src/lib/protocols/dns.c
@@ -475,9 +475,20 @@ static int search_valid_dns(struct ndpi_detection_module_struct *ndpi_struct,
 			 || ((rsp_type == 0x1c) && (data_len == 16)) /* AAAA */
 			 )) {
 		if(found == 0) {
+		  /* Necessary for IP address comparison */
+		  memset(&flow->protos.dns.rsp_addr[flow->protos.dns.num_rsp_addr], 0, sizeof(ndpi_ip_addr_t));
+		  
 		  memcpy(&flow->protos.dns.rsp_addr[flow->protos.dns.num_rsp_addr], packet->payload + x, data_len);
 		  flow->protos.dns.is_rsp_addr_ipv6[flow->protos.dns.num_rsp_addr] = (data_len == 16) ? 1 : 0;
 		  flow->protos.dns.rsp_addr_ttl[flow->protos.dns.num_rsp_addr] = ttl;
+
+		  if(ndpi_struct->cfg.address_cache_size)
+		    ndpi_cache_address(ndpi_struct,
+				       flow->protos.dns.rsp_addr[flow->protos.dns.num_rsp_addr],
+				       flow->host_server_name,
+				       packet->current_time_ms/1000,
+				       flow->protos.dns.rsp_addr_ttl[flow->protos.dns.num_rsp_addr]);		
+		  
 		  if(++flow->protos.dns.num_rsp_addr == MAX_NUM_DNS_RSP_ADDRESSES)
 		    found = 1;
 		}