Reworked categories handling

Removed GenericProtocol and replaced with categories
Removed ndpi_pref_enable_category_substring_match option: substring matching is now default

2 files changed, 64 insertions, 59 deletions

diff --git a/example/ndpiReader.c b/example/ndpiReader.c
index 7a13b90e2..1a54b59d4 100644
--- a/example/ndpiReader.c
+++ b/example/ndpiReader.c
@@ -1851,9 +1851,7 @@ static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) {
 				 ndpi_pref_http_dont_dissect_response, 0);
   ndpi_set_detection_preferences(ndpi_thread_info[thread_id].workflow->ndpi_struct,
 				 ndpi_pref_dns_dont_dissect_response, 0);
-  ndpi_set_detection_preferences(ndpi_thread_info[thread_id].workflow->ndpi_struct,
-				 ndpi_pref_enable_category_substring_match, 1);
-
+  
   ndpi_workflow_set_flow_detected_callback(ndpi_thread_info[thread_id].workflow,
 					   on_protocol_discovered,
 					   (void *)(uintptr_t)thread_id);
@@ -1906,7 +1904,7 @@ static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) {
 	      /* TODO free the strdup */
 	      ndpi_load_hostname_category(ndpi_thread_info[thread_id].workflow->ndpi_struct,
 					  strdup(name), (ndpi_protocol_category_t)atoi(category));
-      }
+	    }
 	  }
 	}
       }
diff --git a/example/reader_util.c b/example/reader_util.c
index 1a8fb2a1d..db25099a2 100644
--- a/example/reader_util.c
+++ b/example/reader_util.c
@@ -171,7 +171,7 @@ void ndpi_analyze_payload(struct ndpi_flow_info *flow,
     if((p = (struct packet_id_stats*)calloc(1, sizeof(struct packet_id_stats))) == NULL)
       return; /* OOM */
     p->packet_id = packet_id;
-    
+
     HASH_ADD_INT(ret->packets, packet_id, p);
   }
 }
@@ -406,21 +406,27 @@ static int _debug_protocols_ok = 0;
 
 struct ndpi_workflow* ndpi_workflow_init(const struct ndpi_workflow_prefs * prefs,
 					 pcap_t * pcap_handle) {
+  struct ndpi_detection_module_struct * module;
+  struct ndpi_workflow * workflow;
+
   set_ndpi_malloc(malloc_wrapper), set_ndpi_free(free_wrapper);
   set_ndpi_flow_malloc(NULL), set_ndpi_flow_free(NULL);
+
   /* TODO: just needed here to init ndpi malloc wrapper */
-  struct ndpi_detection_module_struct * module = ndpi_init_detection_module();
-  if (module == NULL) {
-      NDPI_LOG(0, NULL, NDPI_LOG_ERROR, "global structure initialization failed\n");
-      exit(-1);
+  module = ndpi_init_detection_module();
+
+  if(module == NULL) {
+    NDPI_LOG(0, NULL, NDPI_LOG_ERROR, "global structure initialization failed\n");
+    exit(-1);
   }
 
-  struct ndpi_workflow * workflow = ndpi_calloc(1, sizeof(struct ndpi_workflow));
-  if (workflow == NULL) {
-      NDPI_LOG(0, NULL, NDPI_LOG_ERROR, "global structure initialization failed\n");
-      ndpi_free(module);
-      exit(-1);
+  workflow = ndpi_calloc(1, sizeof(struct ndpi_workflow));
+  if(workflow == NULL) {
+    NDPI_LOG(0, NULL, NDPI_LOG_ERROR, "global structure initialization failed\n");
+    ndpi_free(module);
+    exit(-1);
   }
+
   workflow->pcap_handle = pcap_handle;
   workflow->prefs       = *prefs;
   workflow->ndpi_struct = module;
@@ -435,6 +441,7 @@ struct ndpi_workflow* ndpi_workflow_init(const struct ndpi_workflow_prefs * pref
 
 #ifdef NDPI_ENABLE_DEBUG_MESSAGES
   NDPI_BITMASK_RESET(module->debug_bitmask);
+
   if(_debug_protocols_ok)
     module->debug_bitmask = debug_bitmask;
 #endif
@@ -458,7 +465,7 @@ void ndpi_flow_info_freer(void *node) {
   if(flow->pktlen_s_to_c) ndpi_free_data_analysis(flow->pktlen_s_to_c);
 
   if(flow->iat_flow) ndpi_free_data_analysis(flow->iat_flow);
-  
+
   ndpi_free(flow);
 }
 
@@ -489,19 +496,19 @@ int ndpi_workflow_node_cmp(const void *a, const void *b) {
   if(fa->protocol  < fb->protocol  ) return(-1); else { if(fa->protocol   > fb->protocol  ) return(1); }
 
   if(
-    (
-     (fa->src_ip      == fb->src_ip  )
-     && (fa->src_port == fb->src_port)
-     && (fa->dst_ip   == fb->dst_ip  )
-     && (fa->dst_port == fb->dst_port)
-     )
-    ||
-    (
-     (fa->src_ip      == fb->dst_ip  )
-     && (fa->src_port == fb->dst_port)
-     && (fa->dst_ip   == fb->src_ip  )
-     && (fa->dst_port == fb->src_port)
-     )
+     (
+      (fa->src_ip      == fb->src_ip  )
+      && (fa->src_port == fb->src_port)
+      && (fa->dst_ip   == fb->dst_ip  )
+      && (fa->dst_port == fb->dst_port)
+      )
+     ||
+     (
+      (fa->src_ip      == fb->dst_ip  )
+      && (fa->src_port == fb->dst_port)
+      && (fa->dst_ip   == fb->src_ip  )
+      && (fa->dst_port == fb->src_port)
+      )
      )
     return(0);
 
@@ -537,21 +544,21 @@ ndpi_flow_update_byte_count(struct ndpi_flow_info *flow, const void *x,
    */
 
   /* octet count was already incremented before processing this payload */
-  if (src_to_dst_direction) {
+  if(src_to_dst_direction) {
     current_count = flow->src2dst_l4_bytes - len;
   } else {
     current_count = flow->dst2src_l4_bytes - len;
   }
 
-  if (current_count < ETTA_MIN_OCTETS) {
-    for (i=0; i<len; i++) {
-      if (src_to_dst_direction) {
+  if(current_count < ETTA_MIN_OCTETS) {
+    for(i=0; i<len; i++) {
+      if(src_to_dst_direction) {
         flow->src2dst_byte_count[data[i]]++;
       } else {
         flow->dst2src_byte_count[data[i]]++;
       }
       current_count++;
-      if (current_count >= ETTA_MIN_OCTETS) {
+      if(current_count >= ETTA_MIN_OCTETS) {
         break;
       }
     }
@@ -574,8 +581,8 @@ ndpi_flow_update_byte_dist_mean_var(ndpi_flow_info_t *flow, const void *x,
   double delta;
   unsigned int i;
 
-  for (i=0; i<len; i++) {
-    if (src_to_dst_direction) {
+  for(i=0; i<len; i++) {
+    if(src_to_dst_direction) {
       flow->src2dst_num_bytes += 1;
       delta = ((double)data[i] - flow->src2dst_bd_mean);
       flow->src2dst_bd_mean += delta/((double)flow->src2dst_num_bytes);
@@ -597,9 +604,9 @@ float ndpi_flow_get_byte_count_entropy(const uint32_t byte_count[256],
   int i;
   float tmp, sum = 0.0;
 
-  for (i=0; i<256; i++) {
+  for(i=0; i<256; i++) {
     tmp = (float) byte_count[i] / (float) num_bytes;
-    if (tmp > FLT_EPSILON) {
+    if(tmp > FLT_EPSILON) {
       sum -= tmp * logf(tmp);
     }
   }
@@ -714,7 +721,7 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
     *payload_len = (l4_packet_len > sizeof(struct ndpi_icmphdr)) ? l4_packet_len-sizeof(struct ndpi_icmphdr) : 0;
     l4_data_len = l4_packet_len - sizeof(struct ndpi_icmphdr);
     *sport = *dport = 0;
-  } else if (*proto == IPPROTO_ICMPV6) {
+  } else if(*proto == IPPROTO_ICMPV6) {
     *payload = (u_int8_t*)&l4[sizeof(struct ndpi_icmp6hdr)];
     *payload_len = (l4_packet_len > sizeof(struct ndpi_icmp6hdr)) ? l4_packet_len-sizeof(struct ndpi_icmp6hdr) : 0;
     l4_data_len = l4_packet_len - sizeof(struct ndpi_icmp6hdr);
@@ -734,7 +741,7 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
   printf("hashval=%u [%u][%u][%u:%u][%u:%u]\n", hashval, flow.protocol, flow.vlan_id,
 	 flow.src_ip, flow.src_port, flow.dst_ip, flow.dst_port);
 #endif
-  
+
   idx = hashval % workflow->prefs.num_roots;
   ret = ndpi_tfind(&flow, &workflow->ndpi_flows_root[idx], ndpi_workflow_node_cmp);
 
@@ -821,11 +828,11 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
       *src = newflow->src_id, *dst = newflow->dst_id;
       newflow->src2dst_pkt_len[newflow->src2dst_pkt_count] = l4_packet_len;
       newflow->src2dst_pkt_time[newflow->src2dst_pkt_count] = when;
-      if (newflow->src2dst_pkt_count == 0) {
+      if(newflow->src2dst_pkt_count == 0) {
         newflow->src2dst_start = when;
       }
       newflow->src2dst_pkt_count++;
-      if (l4_data_len != 0XFEEDFACE) {
+      if(l4_data_len != 0XFEEDFACE) {
         newflow->src2dst_opackets++;
       }
       return newflow;
@@ -838,7 +845,7 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
 	 && flow->dst_ip == iph->daddr
 	 && flow->src_port == htons(*sport)
 	 && flow->dst_port == htons(*dport)
-	)
+	 )
 	*src = flow->dst_id, *dst = flow->src_id, *src_to_dst_direction = 0, flow->bidirectional = 1;
       else
 	*src = flow->src_id, *dst = flow->dst_id, *src_to_dst_direction = 1;
@@ -848,30 +855,30 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
 	 && flow->dst_ip == iph->daddr
 	 && flow->src_port == htons(*sport)
 	 && flow->dst_port == htons(*dport)
-	)
+	 )
 	*src = flow->src_id, *dst = flow->dst_id, *src_to_dst_direction = 1;
       else
 	*src = flow->dst_id, *dst = flow->src_id, *src_to_dst_direction = 0, flow->bidirectional = 1;
     }
-    if (src_to_dst_direction) {
-      if (flow->src2dst_pkt_count < MAX_NUM_PKTS) {
+    if(src_to_dst_direction) {
+      if(flow->src2dst_pkt_count < MAX_NUM_PKTS) {
         flow->src2dst_pkt_len[flow->src2dst_pkt_count] = l4_packet_len;
         flow->src2dst_pkt_time[flow->src2dst_pkt_count] = when;
         flow->src2dst_pkt_count++;
       }
-      if (l4_data_len != 0XFEEDFACE) {
+      if(l4_data_len != 0XFEEDFACE) {
         flow->src2dst_opackets++;
       }
     } else {
-      if (flow->dst2src_pkt_count < MAX_NUM_PKTS) {
+      if(flow->dst2src_pkt_count < MAX_NUM_PKTS) {
         flow->dst2src_pkt_len[flow->dst2src_pkt_count] = l4_packet_len;
         flow->dst2src_pkt_time[flow->dst2src_pkt_count] = when;
-        if (flow->dst2src_pkt_count == 0) {
+        if(flow->dst2src_pkt_count == 0) {
           flow->dst2src_start = when;
         }
         flow->dst2src_pkt_count++;
       }
-      if (l4_data_len != 0XFEEDFACE) {
+      if(l4_data_len != 0XFEEDFACE) {
         flow->dst2src_opackets++;
       }
     }
@@ -985,7 +992,7 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
     else if((flow->detected_protocol.app_protocol == NDPI_PROTOCOL_TLS)
 	    || (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_TLS)
 	    || (flow->ndpi_flow->protos.stun_ssl.ssl.ja3_client[0] != '\0')
-      ) {
+	    ) {
       flow->ssh_tls.ssl_version = flow->ndpi_flow->protos.stun_ssl.ssl.ssl_version;
       snprintf(flow->ssh_tls.client_info, sizeof(flow->ssh_tls.client_info), "%s",
 	       flow->ndpi_flow->protos.stun_ssl.ssl.client_certificate);
@@ -1073,7 +1080,7 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
 
     if((tcph != NULL) && (tcph->fin || tcph->rst || tcph->syn))
       begin_or_end_tcp = 1;
-    
+
     if(flow->flow_last_pkt_time.tv_sec) {
       ndpi_timer_sub(&when, &flow->flow_last_pkt_time, &tdiff);
 
@@ -1121,8 +1128,8 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
     if(enable_payload_analyzer && (payload_len > 0))
       ndpi_payload_analyzer(flow, src_to_dst_direction,
 			    payload, payload_len,
-			    workflow->stats.ip_packet_count);    
-    
+			    workflow->stats.ip_packet_count);
+
     if(enable_joy_stats) {
       /* Update BD, distribution and mean. */
       ndpi_flow_update_byte_count(flow, payload, payload_len, src_to_dst_direction);
@@ -1131,7 +1138,7 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
 
     if(flow->first_seen == 0)
       flow->first_seen = time;
-    
+
     flow->last_seen = time;
 
     if(!flow->has_human_readeable_strings) {
@@ -1280,7 +1287,7 @@ struct ndpi_proto ndpi_workflow_process_packet(struct ndpi_workflow * workflow,
   datalink_type = (int)pcap_datalink(workflow->pcap_handle);
 #endif
 
-datalink_check:
+ datalink_check:
   switch(datalink_type) {
   case DLT_NULL:
     if(ntohl(*((u_int32_t*)&packet[eth_offset])) == 2)
@@ -1416,7 +1423,7 @@ datalink_check:
 
   workflow->stats.vlan_count += vlan_packet;
 
-iph_check:
+ iph_check:
   /* Check and set IP header size and total packet length */
   iph = (struct ndpi_iphdr *) &packet[ip_offset];
 
@@ -1610,7 +1617,7 @@ u_int32_t ethernet_crc32(const void* data, size_t n_bytes) {
 #ifdef USE_DPDK
 
 static const struct rte_eth_conf port_conf_default = {
-  .rxmode = { .max_rx_pkt_len = ETHER_MAX_LEN }
+						      .rxmode = { .max_rx_pkt_len = ETHER_MAX_LEN }
 };
 
 /* ************************************ */
@@ -1627,13 +1634,13 @@ int dpdk_port_init(int port, struct rte_mempool *mbuf_pool) {
   if(retval != 0)
     return retval;
 
-  for (q = 0; q < rx_rings; q++) {
+  for(q = 0; q < rx_rings; q++) {
     retval = rte_eth_rx_queue_setup(port, q, RX_RING_SIZE, rte_eth_dev_socket_id(port), NULL, mbuf_pool);
     if(retval < 0)
       return retval;
   }
 
-  for (q = 0; q < tx_rings; q++) {
+  for(q = 0; q < tx_rings; q++) {
     retval = rte_eth_tx_queue_setup(port, q, TX_RING_SIZE, rte_eth_dev_socket_id(port), NULL);
     if(retval < 0)
       return retval;