Removed disable_metadata_export preference that is no longer useful

since ndpi_process_extra_packet() can drive limited or full metadata export

12 files changed, 102 insertions, 143 deletions

diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h
index 11b3394da..33ca4a724 100644
--- a/src/include/ndpi_typedefs.h
+++ b/src/include/ndpi_typedefs.h
@@ -941,7 +941,6 @@ typedef enum {
 
 typedef enum {
    ndpi_pref_direction_detect_disable = 0,
-   ndpi_pref_disable_metadata_export,
 } ndpi_detection_preference;
 
 /* ntop extensions */
@@ -1119,8 +1118,7 @@ struct ndpi_detection_module_struct {
   ndpi_proto_defaults_t proto_defaults[NDPI_MAX_SUPPORTED_PROTOCOLS+NDPI_MAX_NUM_CUSTOM_PROTOCOLS];
 
   u_int8_t direction_detect_disable:1, /* disable internal detection of packet direction */
-    disable_metadata_export:1   /* No metadata is exported */
-    ;
+    _pad:7;
 
   void *hyperscan; /* Intel Hyperscan */
 };
diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index fdf01fde2..84c6883a4 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -716,10 +716,6 @@ int ndpi_set_detection_preferences(struct ndpi_detection_module_struct *ndpi_str
     ndpi_str->direction_detect_disable = (u_int8_t)value;
     break;
 
-  case ndpi_pref_disable_metadata_export:
-    ndpi_str->disable_metadata_export = (u_int8_t)value;
-    break;
-
   default:
     return(-1);
   }
diff --git a/src/lib/protocols/bittorrent.c b/src/lib/protocols/bittorrent.c
index bea7622a0..09e863bb6 100644
--- a/src/lib/protocols/bittorrent.c
+++ b/src/lib/protocols/bittorrent.c
@@ -72,9 +72,7 @@ static void ndpi_add_connection_as_bittorrent(struct ndpi_detection_module_struc
     } else
       bt_hash = (const char*)&flow->packet.payload[28];
 
-    if(!ndpi_struct->disable_metadata_export) {
-      if(bt_hash) memcpy(flow->protos.bittorrent.hash, bt_hash, 20);
-    }
+    if(bt_hash) memcpy(flow->protos.bittorrent.hash, bt_hash, 20);
   }
 
   ndpi_int_change_protocol(ndpi_struct, flow, NDPI_PROTOCOL_BITTORRENT, NDPI_PROTOCOL_UNKNOWN);
diff --git a/src/lib/protocols/dhcp.c b/src/lib/protocols/dhcp.c
index d939df1d8..1913c5997 100644
--- a/src/lib/protocols/dhcp.c
+++ b/src/lib/protocols/dhcp.c
@@ -100,42 +100,36 @@ void ndpi_search_dhcp_udp(struct ndpi_detection_module_struct *ndpi_struct, stru
 
 	    if(msg_type <= 8) foundValidMsgType = 1;
 	  } else if(id == 55 /* Parameter Request List / Fingerprint */) {
-	    if(!ndpi_struct->disable_metadata_export) {
-	      u_int idx, offset = 0;
+	    u_int idx, offset = 0;
+	    
+	    for(idx = 0; idx < len && offset < sizeof(flow->protos.dhcp.fingerprint) - 2; idx++) {
+	      int rc = snprintf((char*)&flow->protos.dhcp.fingerprint[offset],
+				sizeof(flow->protos.dhcp.fingerprint) - offset,
+				"%s%u", (idx > 0) ? "," : "",
+				(unsigned int)dhcp->options[i+2+idx] & 0xFF);
 	      
-	      for(idx = 0; idx < len && offset < sizeof(flow->protos.dhcp.fingerprint) - 2; idx++) {
-		int rc = snprintf((char*)&flow->protos.dhcp.fingerprint[offset],
-				   sizeof(flow->protos.dhcp.fingerprint) - offset,
-				   "%s%u", (idx > 0) ? "," : "",
-				   (unsigned int)dhcp->options[i+2+idx] & 0xFF);
-
-		if(rc < 0) break; else offset += rc;
-	      }
-	      
-	      flow->protos.dhcp.fingerprint[sizeof(flow->protos.dhcp.fingerprint) - 1] = '\0';
+	      if(rc < 0) break; else offset += rc;
 	    }
+	    
+	    flow->protos.dhcp.fingerprint[sizeof(flow->protos.dhcp.fingerprint) - 1] = '\0';	    
 	  } else if(id == 60 /* Class Identifier */) {
-	    if(!ndpi_struct->disable_metadata_export) {
-	      char *name = (char*)&dhcp->options[i+2];
-	      int j = 0;
-	      
-	      j = ndpi_min(len, sizeof(flow->protos.dhcp.class_ident)-1);
-	      strncpy((char*)flow->protos.dhcp.class_ident, name, j);
-	      flow->protos.dhcp.class_ident[j] = '\0';
-	    }
+	    char *name = (char*)&dhcp->options[i+2];
+	    int j = 0;
+	    
+	    j = ndpi_min(len, sizeof(flow->protos.dhcp.class_ident)-1);
+	    strncpy((char*)flow->protos.dhcp.class_ident, name, j);
+	    flow->protos.dhcp.class_ident[j] = '\0';
 	  } else if(id == 12 /* Host Name */) {
-	    if(!ndpi_struct->disable_metadata_export) {
-	      char *name = (char*)&dhcp->options[i+2];
-	      int j = 0;
-	      
+	    char *name = (char*)&dhcp->options[i+2];
+	    int j = 0;
+	    
 #ifdef DHCP_DEBUG
-	      NDPI_LOG_DBG2(ndpi_struct, "[DHCP] '%.*s'\n",name,len);
+	    NDPI_LOG_DBG2(ndpi_struct, "[DHCP] '%.*s'\n",name,len);
 	      //	    while(j < len) { printf( "%c", name[j]); j++; }; printf("\n");
 #endif
-	      j = ndpi_min(len, sizeof(flow->host_server_name)-1);
-	      strncpy((char*)flow->host_server_name, name, j);
-	      flow->host_server_name[j] = '\0';
-	    }
+	    j = ndpi_min(len, sizeof(flow->host_server_name)-1);
+	    strncpy((char*)flow->host_server_name, name, j);
+	    flow->host_server_name[j] = '\0';	    
 	  }
 
 	  i += len + 2;
diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c
index 70ca0c389..b599b82a9 100644
--- a/src/lib/protocols/http.c
+++ b/src/lib/protocols/http.c
@@ -149,9 +149,8 @@ static void setHttpUserAgent(struct ndpi_detection_module_struct *ndpi_struct,
    * https://github.com/ua-parser/uap-core/blob/master/regexes.yaml */
 
   //printf("==> %s\n", ua);
-  if(!ndpi_struct->disable_metadata_export) {
-    snprintf((char*)flow->protos.http.detected_os, sizeof(flow->protos.http.detected_os), "%s", ua);
-  }
+  snprintf((char*)flow->protos.http.detected_os,
+	   sizeof(flow->protos.http.detected_os), "%s", ua);  
 }
 
 /* ************************************************************* */
@@ -333,21 +332,17 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
 		  packet->host_line.len, packet->host_line.ptr);
 
     /* Copy result for nDPI apps */
-    if(!ndpi_struct->disable_metadata_export) {
-      len = ndpi_min(packet->host_line.len, sizeof(flow->host_server_name)-1);
-      strncpy((char*)flow->host_server_name, (char*)packet->host_line.ptr, len);
-      flow->host_server_name[len] = '\0';
-      flow->extra_packets_func = NULL; /* We're good now */
-    }
+    len = ndpi_min(packet->host_line.len, sizeof(flow->host_server_name)-1);
+    strncpy((char*)flow->host_server_name, (char*)packet->host_line.ptr, len);
+    flow->host_server_name[len] = '\0';
+    flow->extra_packets_func = NULL; /* We're good now */
 
     flow->server_id = flow->dst;
 
     if(packet->forwarded_line.ptr) {
       len = ndpi_min(packet->forwarded_line.len, sizeof(flow->protos.http.nat_ip)-1);
-      if(!ndpi_struct->disable_metadata_export) {
-	strncpy((char*)flow->protos.http.nat_ip, (char*)packet->forwarded_line.ptr, len);
-	flow->protos.http.nat_ip[len] = '\0';
-      }
+      strncpy((char*)flow->protos.http.nat_ip, (char*)packet->forwarded_line.ptr, len);
+      flow->protos.http.nat_ip[len] = '\0';
     }
 
     ndpi_http_parse_subprotocol(ndpi_struct, flow);
diff --git a/src/lib/protocols/mdns_proto.c b/src/lib/protocols/mdns_proto.c
index 6297bd4bb..f41b6de0a 100644
--- a/src/lib/protocols/mdns_proto.c
+++ b/src/lib/protocols/mdns_proto.c
@@ -82,11 +82,9 @@ static int ndpi_int_check_mdns_payload(struct ndpi_detection_module_struct
 
     /* printf("==> [%d] %s\n", j, answer);  */
 
-    if(!ndpi_struct->disable_metadata_export) {
-      len = ndpi_min(sizeof(flow->protos.mdns.answer)-1, j);
-      strncpy(flow->protos.mdns.answer, (const char *)answer, len);
-      flow->protos.mdns.answer[len] = '\0';
-    }
+    len = ndpi_min(sizeof(flow->protos.mdns.answer)-1, j);
+    strncpy(flow->protos.mdns.answer, (const char *)answer, len);
+    flow->protos.mdns.answer[len] = '\0';
     
     NDPI_LOG_INFO(ndpi_struct, "found MDNS with answer query\n");
     return 1;
diff --git a/src/lib/protocols/netbios.c b/src/lib/protocols/netbios.c
index 09666366a..bc33a5885 100644
--- a/src/lib/protocols/netbios.c
+++ b/src/lib/protocols/netbios.c
@@ -73,13 +73,10 @@ int ndpi_netbios_name_interpret(char *in, char *out, u_int out_len) {
 static void ndpi_int_netbios_add_connection(struct ndpi_detection_module_struct
 					    *ndpi_struct, struct ndpi_flow_struct *flow) { 
   char name[64];
-
-  if(!ndpi_struct->disable_metadata_export) {
-    u_int off = flow->packet.payload[12] == 0x20 ? 12 : 14;
-    
-    if(ndpi_netbios_name_interpret((char*)&flow->packet.payload[off], name, sizeof(name)) > 0)
-      snprintf((char*)flow->host_server_name, sizeof(flow->host_server_name)-1, "%s", name);    
-  }
+  u_int off = flow->packet.payload[12] == 0x20 ? 12 : 14;
+  
+  if(ndpi_netbios_name_interpret((char*)&flow->packet.payload[off], name, sizeof(name)) > 0)
+    snprintf((char*)flow->host_server_name, sizeof(flow->host_server_name)-1, "%s", name);    
   
   ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_NETBIOS, NDPI_PROTOCOL_UNKNOWN);
 }
diff --git a/src/lib/protocols/quic.c b/src/lib/protocols/quic.c
index 86464ddbd..d0fd1e599 100644
--- a/src/lib/protocols/quic.c
+++ b/src/lib/protocols/quic.c
@@ -129,23 +129,21 @@ void ndpi_search_quic(struct ndpi_detection_module_struct *ndpi_struct,
 	      sni_offset++;
 
 	    if((sni_offset+len) < udp_len) {
-	      if(!ndpi_struct->disable_metadata_export) {
-		int max_len = sizeof(flow->host_server_name)-1, j = 0;
-		ndpi_protocol_match_result ret_match;
-		
-		if(len > max_len) len = max_len;
-		
-		while((len > 0) && (sni_offset < udp_len)) {
-		  flow->host_server_name[j++] = packet->payload[sni_offset];
-		  sni_offset++, len--;
-		}
-		
-		ndpi_match_host_subprotocol(ndpi_struct, flow, 
-					    (char *)flow->host_server_name,
-					    strlen((const char*)flow->host_server_name),
-					    &ret_match,
-					    NDPI_PROTOCOL_QUIC);
+	      int max_len = sizeof(flow->host_server_name)-1, j = 0;
+	      ndpi_protocol_match_result ret_match;
+	      
+	      if(len > max_len) len = max_len;
+	      
+	      while((len > 0) && (sni_offset < udp_len)) {
+		flow->host_server_name[j++] = packet->payload[sni_offset];
+		sni_offset++, len--;
 	      }
+	      
+	      ndpi_match_host_subprotocol(ndpi_struct, flow, 
+					  (char *)flow->host_server_name,
+					  strlen((const char*)flow->host_server_name),
+					  &ret_match,
+					  NDPI_PROTOCOL_QUIC);	      
 	    }
 
 	    break;
diff --git a/src/lib/protocols/ssh.c b/src/lib/protocols/ssh.c
index 068d2c345..1e1671c9e 100644
--- a/src/lib/protocols/ssh.c
+++ b/src/lib/protocols/ssh.c
@@ -251,18 +251,16 @@ static void ndpi_search_ssh_tcp(struct ndpi_detection_module_struct *ndpi_struct
   if(flow->l4.tcp.ssh_stage == 0) {
     if(packet->payload_packet_len > 7 && packet->payload_packet_len < 100
 	&& memcmp(packet->payload, "SSH-", 4) == 0) {
-      if(!ndpi_struct->disable_metadata_export) {
-        int len = ndpi_min(sizeof(flow->protos.ssh.client_signature)-1, packet->payload_packet_len);
-
-	strncpy(flow->protos.ssh.client_signature, (const char *)packet->payload, len);
-	flow->protos.ssh.client_signature[len] = '\0';
-	ndpi_ssh_zap_cr(flow->protos.ssh.client_signature, len);
-
+      int len = ndpi_min(sizeof(flow->protos.ssh.client_signature)-1, packet->payload_packet_len);
+      
+      strncpy(flow->protos.ssh.client_signature, (const char *)packet->payload, len);
+      flow->protos.ssh.client_signature[len] = '\0';
+      ndpi_ssh_zap_cr(flow->protos.ssh.client_signature, len);
+      
 #ifdef SSH_DEBUG
-	printf("[SSH] [client_signature: %s]\n", flow->protos.ssh.client_signature);
-#endif
-      }
-
+      printf("[SSH] [client_signature: %s]\n", flow->protos.ssh.client_signature);
+#endif      
+      
       NDPI_LOG_DBG2(ndpi_struct, "ssh stage 0 passed\n");
       flow->l4.tcp.ssh_stage = 1 + packet->packet_direction;
       ndpi_int_ssh_add_connection(ndpi_struct, flow);
@@ -271,24 +269,19 @@ static void ndpi_search_ssh_tcp(struct ndpi_detection_module_struct *ndpi_struct
   } else if(flow->l4.tcp.ssh_stage == (2 - packet->packet_direction)) {
     if(packet->payload_packet_len > 7 && packet->payload_packet_len < 500
 	&& memcmp(packet->payload, "SSH-", 4) == 0) {
-      if(!ndpi_struct->disable_metadata_export) {
-	int len = ndpi_min(sizeof(flow->protos.ssh.server_signature)-1, packet->payload_packet_len);
-
-	strncpy(flow->protos.ssh.server_signature, (const char *)packet->payload, len);
-	flow->protos.ssh.server_signature[len] = '\0';
-	ndpi_ssh_zap_cr(flow->protos.ssh.server_signature, len);
-
+      int len = ndpi_min(sizeof(flow->protos.ssh.server_signature)-1, packet->payload_packet_len);
+      
+      strncpy(flow->protos.ssh.server_signature, (const char *)packet->payload, len);
+      flow->protos.ssh.server_signature[len] = '\0';
+      ndpi_ssh_zap_cr(flow->protos.ssh.server_signature, len);
+      
 #ifdef SSH_DEBUG
-	printf("[SSH] [server_signature: %s]\n", flow->protos.ssh.server_signature);
+      printf("[SSH] [server_signature: %s]\n", flow->protos.ssh.server_signature);
 #endif
-
-	NDPI_LOG_DBG2(ndpi_struct, "ssh stage 1 passed\n");
-	flow->guessed_host_protocol_id = flow->guessed_protocol_id = NDPI_PROTOCOL_SSH;
-      } else {
-	NDPI_LOG_INFO(ndpi_struct, "found ssh\n");
-	ndpi_int_ssh_add_connection(ndpi_struct, flow);
-      }
-
+      
+      NDPI_LOG_DBG2(ndpi_struct, "ssh stage 1 passed\n");
+      flow->guessed_host_protocol_id = flow->guessed_protocol_id = NDPI_PROTOCOL_SSH;
+      
 #ifdef SSH_DEBUG
       printf("[SSH] [completed stage: %u]\n", flow->l4.tcp.ssh_stage);
 #endif
diff --git a/src/lib/protocols/tls.c b/src/lib/protocols/tls.c
index 79ef6cab7..1d7d2a02b 100644
--- a/src/lib/protocols/tls.c
+++ b/src/lib/protocols/tls.c
@@ -443,28 +443,25 @@ int getTLScertificate(struct ndpi_detection_module_struct *ndpi_struct,
 	      }
 
 	      if(num_dots >= 1) {
-		if(!ndpi_struct->disable_metadata_export) {
-		  ndpi_protocol_match_result ret_match;
-		  u_int16_t subproto;
-		  
-		  stripCertificateTrailer(buffer, buffer_len);
-		  snprintf(flow->protos.stun_ssl.ssl.server_certificate,
-			   sizeof(flow->protos.stun_ssl.ssl.server_certificate), "%s", buffer);
-		  
+		ndpi_protocol_match_result ret_match;
+		u_int16_t subproto;
+		
+		stripCertificateTrailer(buffer, buffer_len);
+		snprintf(flow->protos.stun_ssl.ssl.server_certificate,
+			 sizeof(flow->protos.stun_ssl.ssl.server_certificate), "%s", buffer);
+		
 #ifdef DEBUG_TLS
-		  printf("[server_certificate: %s]\n", flow->protos.stun_ssl.ssl.server_certificate);
+		printf("[server_certificate: %s]\n", flow->protos.stun_ssl.ssl.server_certificate);
 #endif
-		  
-		  subproto = ndpi_match_host_subprotocol(ndpi_struct, flow,
-							 flow->protos.stun_ssl.ssl.server_certificate,
-							 strlen(flow->protos.stun_ssl.ssl.server_certificate),
-							 &ret_match,
-							 NDPI_PROTOCOL_TLS);
-
-		  if(subproto != NDPI_PROTOCOL_UNKNOWN)
-		    ndpi_set_detected_protocol(ndpi_struct, flow, subproto, NDPI_PROTOCOL_TLS);
-		}
-
+		
+		subproto = ndpi_match_host_subprotocol(ndpi_struct, flow,
+						       flow->protos.stun_ssl.ssl.server_certificate,
+						       strlen(flow->protos.stun_ssl.ssl.server_certificate),
+						       &ret_match,
+						       NDPI_PROTOCOL_TLS);
+		
+		if(subproto != NDPI_PROTOCOL_UNKNOWN)
+		  ndpi_set_detected_protocol(ndpi_struct, flow, subproto, NDPI_PROTOCOL_TLS);				
 		return(1 /* Server Certificate */);
 	      }
 	    }
@@ -608,10 +605,9 @@ int getTLScertificate(struct ndpi_detection_module_struct *ndpi_struct,
 			
 			stripCertificateTrailer(buffer, buffer_len);
 			
-			if(!ndpi_struct->disable_metadata_export) {
-			  snprintf(flow->protos.stun_ssl.ssl.client_certificate,
-				   sizeof(flow->protos.stun_ssl.ssl.client_certificate), "%s", buffer);
-			}
+			snprintf(flow->protos.stun_ssl.ssl.client_certificate,
+				 sizeof(flow->protos.stun_ssl.ssl.client_certificate),
+				 "%s", buffer);
 		      }
 		    } else if(extension_id == 10 /* supported groups */) {
 		      u_int16_t s_offset = offset+extension_offset + 2;
diff --git a/src/lib/protocols/ubntac2.c b/src/lib/protocols/ubntac2.c
index 6fc004228..49a63ed0a 100644
--- a/src/lib/protocols/ubntac2.c
+++ b/src/lib/protocols/ubntac2.c
@@ -64,11 +64,9 @@ void ndpi_search_ubntac2(struct ndpi_detection_module_struct *ndpi_struct, struc
 	  
 	  version[j] = '\0';
 
-	  if(!ndpi_struct->disable_metadata_export) {
-	    len = ndpi_min(sizeof(flow->protos.ubntac2.version)-1, j);
-	    strncpy(flow->protos.ubntac2.version, (const char *)version, len);
-	    flow->protos.ubntac2.version[len] = '\0';
-	  }
+	  len = ndpi_min(sizeof(flow->protos.ubntac2.version)-1, j);
+	  strncpy(flow->protos.ubntac2.version, (const char *)version, len);
+	  flow->protos.ubntac2.version[len] = '\0';
 	}
 	
 	NDPI_LOG_INFO(ndpi_struct, "UBNT AirControl 2 request\n");
diff --git a/src/lib/protocols/whoisdas.c b/src/lib/protocols/whoisdas.c
index 381acc981..5bc5df0e8 100644
--- a/src/lib/protocols/whoisdas.c
+++ b/src/lib/protocols/whoisdas.c
@@ -40,15 +40,13 @@ void ndpi_search_whois_das(struct ndpi_detection_module_struct *ndpi_struct, str
 	u_int max_len = sizeof(flow->host_server_name) - 1;
 	u_int i, j;
 
-	if(!ndpi_struct->disable_metadata_export) {
-	  for(i=strlen((const char *)flow->host_server_name), j=0; (i<max_len) && (j<packet->payload_packet_len); i++, j++) {
-	    if((packet->payload[j] == '\n') || (packet->payload[j] == '\r')) break;	  
-	    flow->host_server_name[i] = packet->payload[j];
-	  }
-	  
-	  flow->host_server_name[i] = '\0';
+	for(i=strlen((const char *)flow->host_server_name), j=0; (i<max_len) && (j<packet->payload_packet_len); i++, j++) {
+	  if((packet->payload[j] == '\n') || (packet->payload[j] == '\r')) break;	  
+	  flow->host_server_name[i] = packet->payload[j];
 	}
 	
+	flow->host_server_name[i] = '\0';
+	
 	flow->server_id = ((sport == 43) || (sport == 4343)) ? flow->src : flow->dst;
 	
 	NDPI_LOG_INFO(ndpi_struct, "[WHOIS/DAS] %s\n", flow->host_server_name);