Implemented (disabled by default) DNS host cache. You can set the cache size as follows:

ndpiReader --cfg=dpi.address_cache_size,1000 -i <pcap>.pcap

In the above example the cache has up to 1000 entries.

In jcase ndpiReader exports data in JSON, the cache hostname (if found) is exported in the field server_hostname

9 files changed, 316 insertions, 12 deletions

diff --git a/example/ndpiReader.c b/example/ndpiReader.c
index 7325f71b3..b25a64e5a 100644
--- a/example/ndpiReader.c
+++ b/example/ndpiReader.c
@@ -6262,13 +6262,13 @@ void cryptDecryptUnitTest() {
   const char *test_string = "The quick brown fox jumps over the lazy dog";
   char *enc, *dec;
   u_int16_t e_len, d_len, t_len = strlen(test_string);
-  
+
   enc = ndpi_quick_encrypt(test_string, t_len, &e_len, enc_dec_key);
   assert(enc != NULL);
   dec = ndpi_quick_decrypt((const char*)enc, e_len, &d_len, enc_dec_key);
   assert(dec != NULL);
   assert(t_len == d_len);
-  
+
   assert(strncmp(dec, test_string, e_len) == 0);
 
   ndpi_free(enc);
@@ -6414,6 +6414,31 @@ void domainSearchUnitTest2() {
 
 /* *********************************************** */
 
+void domainCacheTestUnit() {
+  struct ndpi_address_cache *cache = ndpi_init_address_cache(32000);
+  ndpi_ip_addr_t ip;
+  u_int32_t epoch_now = (u_int32_t)time(NULL);
+  struct ndpi_address_cache_item *ret;
+
+  assert(cache);
+
+  memset(&ip, 0, sizeof(ip));
+  ip.ipv4 = 12345678;
+  assert(ndpi_address_cache_insert(cache, ip, "nodomain.local", epoch_now, 0) == true);
+
+  ip.ipv4 = 87654321;
+  assert(ndpi_address_cache_insert(cache, ip, "hello.local", epoch_now, 0) == true);
+
+  assert((ret = ndpi_address_cache_find(cache, ip, epoch_now)) != NULL);
+  assert(strcmp(ret->hostname, "hello.local") == 0);
+  sleep(1);
+  assert(ndpi_address_cache_find(cache, ip, 0 /* computed by the API */) == NULL);
+
+  ndpi_term_address_cache(cache);
+}
+
+/* *********************************************** */
+
 /**
    @brief MAIN FUNCTION
 **/
@@ -6443,7 +6468,7 @@ int main(int argc, char **argv) {
     printf("nDPI Library version mismatch: please make sure this code and the nDPI library are in sync\n");
     return(-1);
   }
-  
+
   if(!skip_unit_tests) {
 #ifndef DEBUG_TRACE
     /* Skip tests when debugging */
@@ -6457,6 +6482,7 @@ int main(int argc, char **argv) {
     exit(0);
 #endif
 
+    domainCacheTestUnit();
     cryptDecryptUnitTest();
     kdUnitTest();
     encodeDomainsUnitTest();
diff --git a/example/reader_util.c b/example/reader_util.c
index 11d6a24a3..5ddb9f000 100644
--- a/example/reader_util.c
+++ b/example/reader_util.c
@@ -493,6 +493,11 @@ static void ndpi_free_flow_tls_data(struct ndpi_flow_info *flow) {
     flow->dhcp_class_ident = NULL;
   }
 
+  if(flow->server_hostname) {
+    ndpi_free(flow->server_hostname);
+    flow->server_hostname = NULL;
+  }
+
   if(flow->bittorent_hash) {
     ndpi_free(flow->bittorent_hash);
     flow->bittorent_hash = NULL;
@@ -1092,6 +1097,9 @@ static void dump_flow_fingerprint(struct ndpi_workflow * workflow,
 						    flow->detected_protocol,
 						    buf, sizeof(buf)));
 
+    if(flow->server_hostname)
+      ndpi_serialize_string_string(&serializer, "server_hostname", flow->server_hostname);
+    
     buffer = ndpi_serializer_get_buffer(&serializer, &buffer_len);
     fprintf(fingerprint_fp, "%s\n", buffer);
   }
@@ -1146,7 +1154,6 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
       flow->bittorent_hash = ndpi_malloc(avail);
 
       if(flow->bittorent_hash) {
-
         for(i=0, j = 0; i < sizeof(flow->ndpi_flow->protos.bittorrent.hash); i++) {
           snprintf(&flow->bittorent_hash[j], avail-j, "%02x",
 	          flow->ndpi_flow->protos.bittorrent.hash[i]);
@@ -1423,6 +1430,24 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
                 sizeof(flow->http.user_agent),
                 "%s", (flow->ndpi_flow->http.user_agent ? flow->ndpi_flow->http.user_agent : ""));
 
+  {
+    ndpi_ip_addr_t ip_addr;
+    struct ndpi_address_cache_item *c;
+    
+    memset(&ip_addr, 0, sizeof(ip_addr));
+    
+    if(flow->ip_version == 4)
+      ip_addr.ipv4 = flow->dst_ip;
+    else
+      memcpy(&ip_addr.ipv6, &flow->dst_ip6, sizeof(struct ndpi_in6_addr));
+
+    c = ndpi_cache_address_find(workflow->ndpi_struct, ip_addr);
+				
+    if(c) {
+      flow->server_hostname = ndpi_strdup(c->hostname);
+    }
+  }
+
   if (workflow->ndpi_serialization_format != ndpi_serialization_format_unknown) {
     if (ndpi_flow2json(workflow->ndpi_struct, flow->ndpi_flow,
                        flow->ip_version, flow->protocol,
@@ -1438,6 +1463,9 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
 
     ndpi_serialize_string_uint32(&flow->ndpi_flow_serializer, "detection_completed", flow->detection_completed);
     ndpi_serialize_string_uint32(&flow->ndpi_flow_serializer, "check_extra_packets", flow->check_extra_packets);
+
+    if(flow->server_hostname)
+      ndpi_serialize_string_string(&flow->ndpi_flow_serializer, "server_hostname", flow->server_hostname);
   }
 
   if(flow->detection_completed && (!flow->check_extra_packets)) {
diff --git a/example/reader_util.h b/example/reader_util.h
index ef15053cd..956bb1955 100644
--- a/example/reader_util.h
+++ b/example/reader_util.h
@@ -259,6 +259,7 @@ typedef struct ndpi_flow_info {
   ndpi_serializer ndpi_flow_serializer;
 
   char host_server_name[80]; /* Hostname/SNI */
+  char *server_hostname;
   char *bittorent_hash;
   char *dhcp_fingerprint;
   char *dhcp_class_ident;
diff --git a/src/include/ndpi_api.h b/src/include/ndpi_api.h
index edfb497d4..b08b6e69f 100644
--- a/src/include/ndpi_api.h
+++ b/src/include/ndpi_api.h
@@ -2334,6 +2334,19 @@ extern "C" {
 
   /* ******************************* */
 
+  /* Address cache API */
+  struct ndpi_address_cache* ndpi_init_address_cache(u_int32_t max_num_entries);
+  void ndpi_term_address_cache(struct ndpi_address_cache *cache);
+  u_int ndpi_address_cache_flush_expired(struct ndpi_address_cache *cache, u_int32_t epoch_now);
+  struct ndpi_address_cache_item* ndpi_address_cache_find(struct ndpi_address_cache *cache, ndpi_ip_addr_t ip_addr, u_int32_t epoch_now);
+  bool ndpi_address_cache_insert(struct ndpi_address_cache *cache, ndpi_ip_addr_t ip_addr, char *hostname,
+				 u_int32_t epoch_now, u_int32_t ttl);
+
+  struct ndpi_address_cache_item* ndpi_cache_address_find(struct ndpi_detection_module_struct *ndpi_struct,
+							  ndpi_ip_addr_t ip_addr);
+  
+  /* ******************************* */
+
   const char *ndpi_lru_cache_idx_to_name(lru_cache_type idx);
 
   /**
diff --git a/src/include/ndpi_private.h b/src/include/ndpi_private.h
index ccc198cf1..54f59f652 100644
--- a/src/include/ndpi_private.h
+++ b/src/include/ndpi_private.h
@@ -199,6 +199,7 @@ struct ndpi_detection_module_config_struct {
   int libgcrypt_init;
   int guess_on_giveup;
   int compute_entropy;
+  int address_cache_size;
   int fpc_enabled;
   int guess_ip_before_port;
   int use_client_ip_in_guess;
@@ -414,6 +415,7 @@ struct ndpi_detection_module_struct {
   u_int16_t max_payload_track_len;
 
   ndpi_str_hash *public_domain_suffixes;
+  struct ndpi_address_cache *address_cache;
 };
 
 
@@ -560,10 +562,6 @@ struct ndpi_detection_module_struct {
 #define NDPI_SELECTION_BITMASK_PROTOCOL_V6_TCP_OR_UDP_WITH_PAYLOAD_WITHOUT_RETRANSMISSION	(NDPI_SELECTION_BITMASK_PROTOCOL_V6_TCP_OR_UDP | NDPI_SELECTION_BITMASK_PROTOCOL_NO_TCP_RETRANSMISSION | NDPI_SELECTION_BITMASK_PROTOCOL_HAS_PAYLOAD)
 #define NDPI_SELECTION_BITMASK_PROTOCOL_V4_V6_TCP_OR_UDP_WITH_PAYLOAD_WITHOUT_RETRANSMISSION	(NDPI_SELECTION_BITMASK_PROTOCOL_V4_V6_TCP_OR_UDP | NDPI_SELECTION_BITMASK_PROTOCOL_NO_TCP_RETRANSMISSION | NDPI_SELECTION_BITMASK_PROTOCOL_HAS_PAYLOAD)
 
-
-
-
-
 /* Generic */
 
 char *strptime(const char *s, const char *format, struct tm *tm);
@@ -635,8 +633,11 @@ int load_category_file_fd(struct ndpi_detection_module_struct *ndpi_str,
 
 u_int64_t fpc_dns_cache_key_from_dns_info(struct ndpi_flow_struct *flow);
 
+bool ndpi_cache_address(struct ndpi_detection_module_struct *ndpi_struct,
+			ndpi_ip_addr_t ip_addr, char *hostname,
+			u_int32_t epoch_now, u_int32_t ttl);
 
-/* TLS */
+  /* TLS */
 int processClientServerHello(struct ndpi_detection_module_struct *ndpi_struct,
                              struct ndpi_flow_struct *flow, uint32_t quic_version);
 void processCertificateElements(struct ndpi_detection_module_struct *ndpi_struct,
diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h
index 6116bc453..b401aad11 100644
--- a/src/include/ndpi_typedefs.h
+++ b/src/include/ndpi_typedefs.h
@@ -1862,6 +1862,21 @@ struct ndpi_des_struct {
 
 /* **************************************** */
 
+struct ndpi_address_cache_item {
+  ndpi_ip_addr_t addr; /* key */
+  char *hostname; /* value */
+  u_int32_t expire_epoch;
+  struct ndpi_address_cache_item *next; /* Linked list */
+};
+
+struct ndpi_address_cache {
+  u_int32_t num_cached_addresses, num_root_nodes;
+  u_int32_t num_entries, max_num_entries;
+  struct ndpi_address_cache_item **address_cache_root;
+};
+
+/* **************************************** */
+
 /* Prototype used to define custom DGA detection function */
 typedef int (*ndpi_custom_dga_predict_fctn)(const char* domain, int domain_length);
 
diff --git a/src/lib/ndpi_cache.c b/src/lib/ndpi_cache.c
index abc03604d..40688c384 100644
--- a/src/lib/ndpi_cache.c
+++ b/src/lib/ndpi_cache.c
@@ -115,10 +115,10 @@ u_int8_t ndpi_lru_find_cache(struct ndpi_lru_cache *c, u_int64_t key,
      now_sec >= c->entries[slot].timestamp &&
      (c->ttl == 0 || now_sec - c->entries[slot].timestamp <= c->ttl)) {
     *value = c->entries[slot].value;
-    
+
     if(clean_key_when_found)
       c->entries[slot].is_full = 0;
-    
+
     c->stats.n_found++;
     ret = 1;
   } else
@@ -215,3 +215,209 @@ int ndpi_get_lru_cache_stats(struct ndpi_global_context *g_ctx,
 }
 
 /* ******************************************************************** */
+/* ******************************************************************** */
+
+struct ndpi_address_cache* ndpi_init_address_cache(u_int32_t max_num_entries) {
+  struct ndpi_address_cache *ret = (struct ndpi_address_cache*)ndpi_malloc(sizeof(struct ndpi_address_cache));
+
+  if(ret == NULL) return(ret);
+
+  ret->num_cached_addresses = 0, ret->num_entries = 0,
+    ret->max_num_entries = max_num_entries,
+    ret->num_root_nodes = ndpi_min(NDPI_NUM_DEFAULT_ROOT_NODES, max_num_entries/16);
+  ret->address_cache_root = (struct ndpi_address_cache_item**)ndpi_calloc(ret->num_root_nodes, sizeof(struct ndpi_address_cache_item*));
+
+  if(ret->address_cache_root == NULL) {
+    ndpi_free(ret);
+    return(NULL);
+  } else
+    return(ret);
+}
+
+/* ***************************************************** */
+
+static void ndpi_free_addr_item(struct ndpi_address_cache_item *addr) {
+  ndpi_free(addr->hostname);
+  ndpi_free(addr);
+}
+
+/* ***************************************************** */
+
+void ndpi_term_address_cache(struct ndpi_address_cache *cache) {
+  u_int i;
+
+  for(i=0; i<cache->num_root_nodes; i++) {
+    struct ndpi_address_cache_item *root = cache->address_cache_root[i];
+
+    while(root != NULL) {
+      struct ndpi_address_cache_item *next = root->next;
+
+      ndpi_free_addr_item(root);
+      root = next;
+    }
+  }
+
+  ndpi_free(cache->address_cache_root);
+}
+
+/* ***************************************************** */
+
+/* Return the number of purged entries */
+u_int ndpi_address_cache_flush_expired(struct ndpi_address_cache *cache,
+				       u_int32_t epoch_now) {
+  u_int i, num_purged = 0;
+
+  for(i=0; i<cache->num_root_nodes; i++) {
+    struct ndpi_address_cache_item *root = cache->address_cache_root[i];
+    struct ndpi_address_cache_item *prev = NULL;
+
+    while(root != NULL) {
+      struct ndpi_address_cache_item *next = root->next;
+
+      if(root->expire_epoch > epoch_now) {
+	/* Time to purge */
+
+	if(prev == NULL) {
+	  /* Head element */
+	  cache->address_cache_root[i] = next;
+	} else {
+	  /* Middle element */
+	  prev->next = next;
+	}
+
+	ndpi_free_addr_item(root), num_purged++;
+      } else {
+	prev = root;
+      }
+
+      root = next;
+    } /* while */
+  } /* for */
+
+  cache->num_entries -= num_purged;
+
+  return(num_purged);
+}
+
+
+/* ***************************************************** */
+
+struct ndpi_address_cache_item* ndpi_address_cache_find(struct ndpi_address_cache *cache,
+							ndpi_ip_addr_t ip_addr, u_int32_t epoch_now) {
+  u_int32_t hash_id = ndpi_quick_hash((const unsigned char *)&ip_addr, sizeof(ip_addr)) % cache->num_root_nodes;
+  struct ndpi_address_cache_item *root = cache->address_cache_root[hash_id], *prev = NULL;
+
+  while(root != NULL) {
+    if((epoch_now != 0) && (root->expire_epoch < epoch_now)) {
+      /* Expired entry: let's remove it */
+      struct ndpi_address_cache_item *next = root->next;
+
+      if(prev == NULL)
+	cache->address_cache_root[hash_id] = next;
+      else
+	prev->next = next;
+
+      ndpi_free_addr_item(root);
+      root = next, cache->num_entries--;
+
+      continue; /* Skip this entry */
+    }
+
+    if(memcmp(&root->addr, &ip_addr, sizeof(ndpi_ip_addr_t)) == 0) {
+      return(root);
+    } else
+      root = root->next;
+  }
+
+  return(NULL);
+}
+
+/* ***************************************************** */
+
+bool ndpi_address_cache_insert(struct ndpi_address_cache *cache,
+			       ndpi_ip_addr_t ip_addr, char *hostname,
+			       u_int32_t epoch_now,
+			       u_int32_t ttl) {
+  u_int32_t hash_id = ndpi_quick_hash((const unsigned char *)&ip_addr, sizeof(ip_addr)) % cache->num_root_nodes;
+  struct ndpi_address_cache addr;
+  struct ndpi_address_cache_item *ret;
+  u_int32_t epoch_valid_until;
+
+  if(epoch_now == 0) epoch_now = (u_int32_t)time(NULL);
+  ret = ndpi_address_cache_find(cache, ip_addr, epoch_now);
+  epoch_valid_until = epoch_now + ttl;
+
+  /* printf("**** %s [%u][ttl: %u]\n", hostname, epoch_now, ttl); */
+
+  if(ret == NULL) {
+    if(cache->num_entries == cache->max_num_entries) {
+      ndpi_address_cache_flush_expired(cache, epoch_now);
+
+      if(cache->num_entries == cache->max_num_entries)
+	return(false); /* Still no room left */
+
+      /* We have room to add the new element */
+      /* Let's continue */
+    }
+
+    /* We have room to insert the new element */
+    ret = (struct ndpi_address_cache_item*)ndpi_malloc(sizeof(struct ndpi_address_cache_item));
+    if(ret == NULL)
+      return(false); /* No memory */
+
+    memcpy(&ret->addr, &ip_addr, sizeof(addr)),
+      ret->expire_epoch = epoch_valid_until,
+      ret->next = cache->address_cache_root[hash_id];
+
+    /* Create linked list */
+    cache->address_cache_root[hash_id] = ret;
+
+    if((ret->hostname = strdup(hostname)) == NULL) {
+      ndpi_free(ret);
+      return(false);
+    }
+  } else {
+    /* Element found: update TTL of the existing element */
+    ret->expire_epoch = ndpi_max(ret->expire_epoch, epoch_valid_until);
+
+    if(strcmp(ret->hostname, hostname)) {
+      /* Hostnames are different: we overwrite it */
+      char *new_hostname = ndpi_strdup(hostname);
+
+      if(new_hostname) {
+	/* Allocation ok */
+	ndpi_free(ret->hostname);
+	ret->hostname = new_hostname;
+      }
+    }
+  }
+
+  cache->num_entries++;
+  return(true);
+}
+
+/* ***************************************************** */
+/* ***************************************************** */
+
+bool ndpi_cache_address(struct ndpi_detection_module_struct *ndpi_struct,
+			ndpi_ip_addr_t ip_addr, char *hostname,
+			u_int32_t epoch_now, u_int32_t ttl) {
+  if(ndpi_struct->cfg.address_cache_size == 0) return(false);
+
+  if(ndpi_struct->address_cache == NULL)
+    ndpi_struct->address_cache = ndpi_init_address_cache(ndpi_struct->cfg.address_cache_size);
+
+  if(ndpi_struct->address_cache)
+    return(ndpi_address_cache_insert(ndpi_struct->address_cache, ip_addr, hostname, epoch_now, ttl));
+  else
+    return(false);
+}
+
+/* ***************************************************** */
+
+struct ndpi_address_cache_item* ndpi_cache_address_find(struct ndpi_detection_module_struct *ndpi_struct,
+							ndpi_ip_addr_t ip_addr) {
+  if(ndpi_struct->address_cache == NULL) return(NULL);
+
+  return(ndpi_address_cache_find(ndpi_struct->address_cache, ip_addr, 0));
+}
diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index b114f1661..464e61e6b 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -4385,6 +4385,9 @@ void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_str) {
     if(ndpi_str->public_domain_suffixes)
       ndpi_hash_free(&ndpi_str->public_domain_suffixes);
 
+    if(ndpi_str->address_cache)
+      ndpi_term_address_cache(ndpi_str->address_cache);
+    
     ndpi_free(ndpi_str);
   }
 
@@ -8539,7 +8542,6 @@ static void fpc_check_eval(struct ndpi_detection_module_struct *ndpi_str,
 {
   u_int16_t fpc_dns_cached_proto;
 
-
   if(!ndpi_str->cfg.fpc_enabled)
     return;
 
@@ -11384,6 +11386,7 @@ static const struct cfg_param {
   { NULL,            "dpi.guess_on_giveup",                     "0x3", "0", "3", CFG_PARAM_INT, __OFF(guess_on_giveup), NULL },
   { NULL,            "dpi.guess_ip_before_port",                "disable", NULL, NULL, CFG_PARAM_ENABLE_DISABLE, __OFF(guess_ip_before_port), NULL},
   { NULL,            "dpi.compute_entropy",                     "1", NULL, NULL, CFG_PARAM_ENABLE_DISABLE, __OFF(compute_entropy), NULL },
+  { NULL,            "dpi.address_cache_size",                  "0", "0", "16777215", CFG_PARAM_INT, __OFF(address_cache_size), NULL },
   { NULL,            "fpc",                                     "1", NULL, NULL, CFG_PARAM_ENABLE_DISABLE, __OFF(fpc_enabled), NULL },
 
   { NULL,            "flow_risk_lists.load",                    "1", NULL, NULL, CFG_PARAM_ENABLE_DISABLE, __OFF(flow_risk_lists_enabled), NULL },
diff --git a/src/lib/protocols/dns.c b/src/lib/protocols/dns.c
index 8a6e2d1a8..d109098d1 100644
--- a/src/lib/protocols/dns.c
+++ b/src/lib/protocols/dns.c
@@ -475,9 +475,20 @@ static int search_valid_dns(struct ndpi_detection_module_struct *ndpi_struct,
 			 || ((rsp_type == 0x1c) && (data_len == 16)) /* AAAA */
 			 )) {
 		if(found == 0) {
+		  /* Necessary for IP address comparison */
+		  memset(&flow->protos.dns.rsp_addr[flow->protos.dns.num_rsp_addr], 0, sizeof(ndpi_ip_addr_t));
+		  
 		  memcpy(&flow->protos.dns.rsp_addr[flow->protos.dns.num_rsp_addr], packet->payload + x, data_len);
 		  flow->protos.dns.is_rsp_addr_ipv6[flow->protos.dns.num_rsp_addr] = (data_len == 16) ? 1 : 0;
 		  flow->protos.dns.rsp_addr_ttl[flow->protos.dns.num_rsp_addr] = ttl;
+
+		  if(ndpi_struct->cfg.address_cache_size)
+		    ndpi_cache_address(ndpi_struct,
+				       flow->protos.dns.rsp_addr[flow->protos.dns.num_rsp_addr],
+				       flow->host_server_name,
+				       packet->current_time_ms/1000,
+				       flow->protos.dns.rsp_addr_ttl[flow->protos.dns.num_rsp_addr]);		
+		  
 		  if(++flow->protos.dns.num_rsp_addr == MAX_NUM_DNS_RSP_ADDRESSES)
 		    found = 1;
 		}