Integrate root flows free and little changes

3 files changed, 79 insertions, 87 deletions

diff --git a/example/ndpiReader.c b/example/ndpiReader.c
index 0a1747088..43ce6d0c5 100644
--- a/example/ndpiReader.c
+++ b/example/ndpiReader.c
@@ -125,11 +125,6 @@ typedef struct ndpi_id {
   struct ndpi_id_struct *ndpi_id;		// nDpi worker structure
 } ndpi_id_t;
 
-static u_int32_t size_id_struct = 0;		// ID tracking structure size
-
-
-static u_int32_t size_flow_struct = 0;
-
 static void help(u_int long_help) {
   printf("ndpiReader -i <file|device> [-f <filter>][-s <duration>]\n"
 	 "          [-p <protos>][-l <loops> [-q][-d][-h][-t][-v <level>]\n"
@@ -383,11 +378,11 @@ static void printFlow(u_int16_t thread_id, struct ndpi_flow_info *flow) {
     fprintf(out, "\t%s %s%s%s:%u <-> %s%s%s:%u ",
 	    ipProto2Name(flow->protocol),
 	    (flow->ip_version == 6) ? "[" : "",
-	    flow->lower_name, 
+	    flow->lower_name,
 	    (flow->ip_version == 6) ? "]" : "",
 	    ntohs(flow->lower_port),
 	    (flow->ip_version == 6) ? "[" : "",
-	    flow->upper_name, 
+	    flow->upper_name,
 	    (flow->ip_version == 6) ? "]" : "",
 	    ntohs(flow->upper_port));
 
@@ -472,24 +467,6 @@ static void printFlow(u_int16_t thread_id, struct ndpi_flow_info *flow) {
 
 /* ***************************************************** */
 
-static void free_ndpi_flow_info(struct ndpi_flow_info *flow) {
-  if(flow->ndpi_flow) { ndpi_free_flow(flow->ndpi_flow); flow->ndpi_flow = NULL; }
-  if(flow->src_id)    { ndpi_free(flow->src_id); flow->src_id = NULL; }
-  if(flow->dst_id)    { ndpi_free(flow->dst_id); flow->dst_id = NULL; }
-
-}
-
-/* ***************************************************** */
-
-static void ndpi_flow_info_freer(void *node) {
-  struct ndpi_flow_info *flow = (struct ndpi_flow_info*)node;
-
-  free_ndpi_flow_info(flow);
-  ndpi_free(flow);
-}
-
-/* ***************************************************** */
-
 static void node_print_unknown_proto_walker(const void *node, ndpi_VISIT which, int depth, void *user_data) {
   struct ndpi_flow_info *flow = *(struct ndpi_flow_info**)node;
   u_int16_t thread_id = *((u_int16_t*)user_data);
@@ -569,7 +546,7 @@ static void node_idle_scan_walker(const void *node, ndpi_VISIT which, int depth,
       if((flow->detected_protocol.protocol == NDPI_PROTOCOL_UNKNOWN) && !undetected_flows_deleted)
         undetected_flows_deleted = 1;
 
-      free_ndpi_flow_info(flow);
+      ndpi_free_flow_info_half(flow);
       ndpi_thread_info[thread_id].workflow->stats.ndpi_flow_count--;
 
       /* adding to a queue (we can't delete it from the tree inline ) */
@@ -584,7 +561,7 @@ static void on_protocol_discovered(struct ndpi_workflow * workflow,
         struct ndpi_flow_info * flow,
         void * udata) {
   const u_int16_t thread_id = *((u_int16_t *)udata);
-  
+
   if(verbose > 1) {
     if(enable_protocol_guess) {
       if(flow->detected_protocol.protocol == NDPI_PROTOCOL_UNKNOWN) {
@@ -638,7 +615,7 @@ static void debug_printf(u_int32_t protocol, void *id_struct,
 
 static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) {
   NDPI_PROTOCOL_BITMASK all;
-  
+
   struct ndpi_workflow_prefs prefs;
   memset(&prefs, 0, sizeof(prefs));
   prefs.decode_tunnels = decode_tunnels;
@@ -650,7 +627,7 @@ static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) {
   memset(&ndpi_thread_info[thread_id], 0, sizeof(ndpi_thread_info[thread_id]));
   ndpi_thread_info[thread_id].workflow = ndpi_workflow_init(&prefs, pcap_handle, malloc_wrapper, free_wrapper, debug_printf);
   /* ndpi_thread_info[thread_id].workflow->ndpi_struct->http_dont_dissect_response = 1; */
-  
+
   ndpi_workflow_set_flow_detected_callback(ndpi_thread_info[thread_id].workflow, on_protocol_discovered, (void *)&thread_id);
 
   // enable all protocols
@@ -669,13 +646,6 @@ static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) {
 /* ***************************************************** */
 
 static void terminateDetection(u_int16_t thread_id) {
-  int i;
-
-  for(i=0; i<NUM_ROOTS; i++) {
-    ndpi_tdestroy(ndpi_thread_info[thread_id].workflow->ndpi_flows_root[i], ndpi_flow_info_freer);
-    ndpi_thread_info[thread_id].workflow->ndpi_flows_root[i] = NULL;
-  }
-
   ndpi_workflow_free(ndpi_thread_info[thread_id].workflow);
 }
 
@@ -810,7 +780,7 @@ static void printResults(u_int64_t tot_usec) {
   if(!quiet_mode) {
     printf("\nnDPI Memory statistics:\n");
     printf("\tnDPI Memory (once):      %-13s\n", formatBytes(sizeof(struct ndpi_detection_module_struct), buf, sizeof(buf)));
-    printf("\tFlow Memory (per flow):  %-13s\n", formatBytes(size_flow_struct, buf, sizeof(buf)));
+    printf("\tFlow Memory (per flow):  %-13s\n", formatBytes(sizeof(struct ndpi_flow_struct), buf, sizeof(buf)));
     printf("\tActual Memory:           %-13s\n", formatBytes(current_ndpi_memory, buf, sizeof(buf)));
     printf("\tPeak Memory:             %-13s\n", formatBytes(max_ndpi_memory, buf, sizeof(buf)));
 
@@ -1112,7 +1082,7 @@ static pcap_t * openPcapFileOrDevice(u_int16_t thread_id, const u_char * pcap_fi
     signal(SIGALRM, sigproc);
 #endif
   }
-  
+
   return pcap_handle;
 }
 
@@ -1122,24 +1092,24 @@ static void pcap_packet_callback_checked(u_char *args,
 				 const struct pcap_pkthdr *header,
 				 const u_char *packet) {
   u_int16_t thread_id = *((u_int16_t*)args);
-           
+
   /* allocate an exact size buffer to check overflows */
   uint8_t *packet_checked = malloc(header->caplen);
   memcpy(packet_checked, packet, header->caplen);
   ndpi_workflow_process_packet(ndpi_thread_info[thread_id].workflow, header, packet_checked);
-  
+
   if((capture_until != 0) && (header->ts.tv_sec >= capture_until)) {
     if(ndpi_thread_info[thread_id].workflow->pcap_handle != NULL)
       pcap_breakloop(ndpi_thread_info[thread_id].workflow->pcap_handle);
     return;
   }
-  
+
   /* Check if capture is live or not */
   if (!live_capture) {
     if (!pcap_start.tv_sec) pcap_start.tv_sec = header->ts.tv_sec, pcap_start.tv_usec = header->ts.tv_usec;
     pcap_end.tv_sec = header->ts.tv_sec, pcap_end.tv_usec = header->ts.tv_usec;
   }
-  
+
   /* Idle flows cleanup */
   if(live_capture) {
     if(ndpi_thread_info[thread_id].last_idle_scan_time + IDLE_SCAN_PERIOD < ndpi_thread_info[thread_id].workflow->last_time) {
@@ -1148,22 +1118,22 @@ static void pcap_packet_callback_checked(u_char *args,
 
       /* remove idle flows (unfortunately we cannot do this inline) */
       while (ndpi_thread_info[thread_id].num_idle_flows > 0) {
-	
+
 	/* search and delete the idle flow from the "ndpi_flow_root" (see struct reader thread) - here flows are the node of a b-tree */
 	ndpi_tdelete(ndpi_thread_info[thread_id].idle_flows[--ndpi_thread_info[thread_id].num_idle_flows],
         &ndpi_thread_info[thread_id].workflow->ndpi_flows_root[ndpi_thread_info[thread_id].idle_scan_idx],
         ndpi_workflow_node_cmp);
-	
+
 	/* free the memory associated to idle flow in "idle_flows" - (see struct reader thread)*/
-	free_ndpi_flow_info(ndpi_thread_info[thread_id].idle_flows[ndpi_thread_info[thread_id].num_idle_flows]);
+	ndpi_free_flow_info_half(ndpi_thread_info[thread_id].idle_flows[ndpi_thread_info[thread_id].num_idle_flows]);
 	ndpi_free(ndpi_thread_info[thread_id].idle_flows[ndpi_thread_info[thread_id].num_idle_flows]);
       }
-      
+
       if(++ndpi_thread_info[thread_id].idle_scan_idx == ndpi_thread_info[thread_id].workflow->prefs.num_roots) ndpi_thread_info[thread_id].idle_scan_idx = 0;
       ndpi_thread_info[thread_id].last_idle_scan_time = ndpi_thread_info[thread_id].workflow->last_time;
     }
   }
-  
+
   /* check for buffer changes */
   if(memcmp(packet, packet_checked, header->caplen) != 0)
     printf("INTERNAL ERROR: ingress packet was nodified by nDPI: this should not happen [thread_id=%u, packetId=%lu]\n",
@@ -1211,7 +1181,7 @@ void *processing_thread(void *_thread_id) {
       goto pcap_loop;
     }
   }
-  
+
   return NULL;
 }
 
diff --git a/src/include/ndpi_util.h b/src/include/ndpi_util.h
index a23c5a0da..44f7be03d 100644
--- a/src/include/ndpi_util.h
+++ b/src/include/ndpi_util.h
@@ -24,13 +24,13 @@
 
 /**
  * This module contains routines to help setup a simple nDPI program.
- * 
+ *
  * If you concern about performance or have to integrate nDPI in your
  * application, you could need to reimplement them yourself.
- * 
+ *
  * WARNING: this API is unstable! Use it at your own risk!
  */
- 
+
 #ifndef __NDPI_UTIL_H__
 #define __NDPI_UTIL_H__
 
@@ -93,15 +93,15 @@ typedef void (*ndpi_workflow_callback_ptr) (struct ndpi_workflow *, struct ndpi_
 
 typedef struct ndpi_workflow {
   u_int64_t last_time;
-  
+
   struct ndpi_workflow_prefs prefs;
   struct ndpi_stats stats;
-  
+
   ndpi_workflow_callback_ptr __flow_detected_callback;
   void * __flow_detected_udata;
   ndpi_workflow_callback_ptr __flow_giveup_callback;
   void * __flow_giveup_udata;
-  
+
   /* outside referencies */
   pcap_t *pcap_handle;
 
@@ -116,14 +116,20 @@ struct ndpi_workflow * ndpi_workflow_init(const struct ndpi_workflow_prefs * pre
         void * (*malloc_wrapper)(size_t),
         void (*free_wrapper)(void*),
         ndpi_debug_function_ptr ndpi_debug_printf);
-        
+
 void ndpi_workflow_free(struct ndpi_workflow * workflow);
 
+/** Free flow_info ndpi support structures but not the flow_info itself
+ *
+ *  TODO remove! Half freeing things is bad!
+ */
+void ndpi_free_flow_info_half(struct ndpi_flow_info *flow);
+
 /** Process a @packet and update the @workflow.  */
 void ndpi_workflow_process_packet (struct ndpi_workflow * workflow,
 				 const struct pcap_pkthdr *header,
 				 const u_char *packet);
-         
+
 /* flow callbacks: ndpi_flow_info will be freed right after */
 static inline void ndpi_workflow_set_flow_detected_callback(struct ndpi_workflow * workflow,
         ndpi_workflow_callback_ptr callback,
diff --git a/src/lib/ndpi_util.c b/src/lib/ndpi_util.c
index 51d6757e1..36897cf2d 100644
--- a/src/lib/ndpi_util.c
+++ b/src/lib/ndpi_util.c
@@ -21,7 +21,7 @@
  * along with nDPI.  If not, see <http://www.gnu.org/licenses/>.
  *
  */
- 
+
 #include <stdlib.h>
 
 #ifdef WIN32
@@ -71,7 +71,7 @@
 
 /* ***************************************************** */
 
-static void free_ndpi_flow_info(struct ndpi_flow_info *flow) {
+void ndpi_free_flow_info_half(struct ndpi_flow_info *flow) {
   if(flow->ndpi_flow) { ndpi_free_flow(flow->ndpi_flow); flow->ndpi_flow = NULL; }
   if(flow->src_id)    { ndpi_free(flow->src_id); flow->src_id = NULL; }
   if(flow->dst_id)    { ndpi_free(flow->dst_id); flow->dst_id = NULL; }
@@ -91,29 +91,45 @@ struct ndpi_workflow * ndpi_workflow_init(const struct ndpi_workflow_prefs * pre
         void * (*malloc_wrapper)(size_t),
         void (*free_wrapper)(void*),
         ndpi_debug_function_ptr ndpi_debug_printf) {
-            
+
   /* TODO: just needed here to init ndpi malloc wrapper */
   struct ndpi_detection_module_struct * module = ndpi_init_detection_module(
      prefs->detection_tick_resolution, malloc_wrapper, free_wrapper, ndpi_debug_printf);
-      
+
   struct ndpi_workflow * workflow = ndpi_calloc(1, sizeof(struct ndpi_workflow));
-  
+
   removeme_free_wrapper = free_wrapper;
-  
+
   workflow->pcap_handle = pcap_handle;
   workflow->prefs = *prefs;
   workflow->ndpi_struct = module;
-      
+
   if(workflow->ndpi_struct == NULL) {
     NDPI_LOG(0, NULL, NDPI_LOG_ERROR, "global structure initialization failed\n");
     exit(-1);
   }
-  
+
   workflow->ndpi_flows_root = ndpi_calloc(workflow->prefs.num_roots, sizeof(void *));
   return workflow;
 }
 
+/* ***************************************************** */
+
+static void ndpi_flow_info_freer(void *node) {
+  struct ndpi_flow_info *flow = (struct ndpi_flow_info*)node;
+
+  ndpi_free_flow_info_half(flow);
+  ndpi_free(flow);
+}
+
+/* ***************************************************** */
+
 void ndpi_workflow_free(struct ndpi_workflow * workflow) {
+  int i;
+
+  for(i=0; i<workflow->prefs.num_roots; i++)
+    ndpi_tdestroy(workflow->ndpi_flows_root[i], ndpi_flow_info_freer);
+
   ndpi_exit_detection_module(workflow->ndpi_struct, removeme_free_wrapper);
   free(workflow->ndpi_flows_root);
   free(workflow);
@@ -217,7 +233,7 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
     // tcp
     *tcph = (struct ndpi_tcphdr *)l4;
     *sport = ntohs((*tcph)->source), *dport = ntohs((*tcph)->dest);
-    
+
     if(iph->saddr < iph->daddr) {
       lower_port = (*tcph)->source, upper_port = (*tcph)->dest;
       *src_to_dst_direction = 1;
@@ -235,7 +251,7 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
 	}
       }
     }
-    
+
     tcp_len = ndpi_min(4*(*tcph)->doff, l4_packet_len);
     *payload = &l4[tcp_len];
     *payload_len = ndpi_max(0, l4_packet_len-4*(*tcph)->doff);
@@ -244,7 +260,7 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
     workflow->stats.udp_count++;
 
     *udph = (struct ndpi_udphdr *)l4;
-    *sport = ntohs((*udph)->source), *dport = ntohs((*udph)->dest);    
+    *sport = ntohs((*udph)->source), *dport = ntohs((*udph)->dest);
     *payload = &l4[sizeof(struct ndpi_udphdr)];
     *payload_len = ndpi_max(0, l4_packet_len-sizeof(struct ndpi_udphdr));
 
@@ -259,7 +275,7 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
       if(iph->saddr == iph->daddr) {
 	if(lower_port > upper_port) {
 	  u_int16_t p = lower_port;
-	  
+
 	  lower_port = upper_port;
 	  upper_port = p;
 	}
@@ -406,17 +422,17 @@ static unsigned int packet_processing(struct ndpi_workflow * workflow,
   u_int16_t sport, dport, payload_len;
   u_int8_t *payload;
   u_int8_t src_to_dst_direction= 1;
-  
+
   if(iph)
     flow = get_ndpi_flow_info(workflow, 4, vlan_id, iph, NULL,
 			 ip_offset, ipsize,
 			 ntohs(iph->tot_len) - (iph->ihl * 4),
-			 &tcph, &udph, &sport, &dport,			
+			 &tcph, &udph, &sport, &dport,
 			 &src, &dst, &proto,
 			 &payload, &payload_len, &src_to_dst_direction);
   else
     flow = get_ndpi_flow_info6(workflow, vlan_id, iph6, ip_offset,
-			  &tcph, &udph, &sport, &dport,			
+			  &tcph, &udph, &sport, &dport,
 			  &src, &dst, &proto,
 			  &payload, &payload_len, &src_to_dst_direction);
 
@@ -436,7 +452,7 @@ static unsigned int packet_processing(struct ndpi_workflow * workflow,
   flow->detected_protocol = ndpi_detection_process_packet(workflow->ndpi_struct, ndpi_flow,
 							  iph ? (uint8_t *)iph : (uint8_t *)iph6,
 							  ipsize, time, src, dst);
-  
+
   if((flow->detected_protocol.protocol != NDPI_PROTOCOL_UNKNOWN)
      || ((proto == IPPROTO_UDP) && (flow->packets > 8))
      || ((proto == IPPROTO_TCP) && (flow->packets > 10))) {
@@ -450,15 +466,15 @@ static unsigned int packet_processing(struct ndpi_workflow * workflow,
 
     if(flow->detected_protocol.protocol == NDPI_PROTOCOL_BITTORRENT) {
       int i, j, n = 0;
-      
+
       for(i=0, j = 0; i<20; i++) {
 	sprintf(&flow->bittorent_hash[j], "%02x", flow->ndpi_flow->bittorent_hash[i]);
 	j += 2,	n += flow->ndpi_flow->bittorent_hash[i];
       }
-      
+
       if(n == 0) flow->bittorent_hash[0] = '\0';
     }
-      
+
     if((proto == IPPROTO_TCP) && (flow->detected_protocol.protocol != NDPI_PROTOCOL_DNS)) {
       snprintf(flow->ssl.client_certificate, sizeof(flow->ssl.client_certificate), "%s", flow->ndpi_flow->protos.ssl.client_certificate);
       snprintf(flow->ssl.server_certificate, sizeof(flow->ssl.server_certificate), "%s", flow->ndpi_flow->protos.ssl.server_certificate);
@@ -466,17 +482,17 @@ static unsigned int packet_processing(struct ndpi_workflow * workflow,
 
     if(flow->detected_protocol.protocol == NDPI_PROTOCOL_UNKNOWN) {
       flow->detected_protocol = ndpi_detection_giveup(workflow->ndpi_struct, flow->ndpi_flow);
-      
+
       if (workflow->__flow_giveup_callback != NULL)
         workflow->__flow_giveup_callback(workflow, flow, workflow->__flow_giveup_udata);
     } else {
       if (workflow->__flow_detected_callback != NULL)
         workflow->__flow_detected_callback(workflow, flow, workflow->__flow_detected_udata);
     }
-    
-    free_ndpi_flow_info(flow);
+
+    ndpi_free_flow_info_half(flow);
   }
-  
+
   return 0;
 }
 
@@ -487,7 +503,7 @@ void ndpi_workflow_process_packet (struct ndpi_workflow * workflow,
   /*
    * Declare pointers to packet headers
    */
-  
+
   /* --- Ethernet header --- */
   const struct ndpi_ethhdr *ethernet;
   /* --- Ethernet II header --- */
@@ -576,7 +592,7 @@ void ndpi_workflow_process_packet (struct ndpi_workflow * workflow,
       ip_offset = sizeof(struct ndpi_chdlc); /* CHDLC_OFF = 4 */
       type = ntohs(chdlc->proto_code);
       break;
-      
+
       /* IEEE 802.3 Ethernet - 1 */
     case DLT_EN10MB :
       ethernet = (struct ndpi_ethhdr *) &packet[eth_offset];
@@ -654,13 +670,13 @@ void ndpi_workflow_process_packet (struct ndpi_workflow * workflow,
     type = (packet[ip_offset+2] << 8) + packet[ip_offset+3];
     ip_offset += 4;
     vlan_packet = 1;
-  } else if(type == MPLS_UNI || type == MPLS_MULTI) {    
+  } else if(type == MPLS_UNI || type == MPLS_MULTI) {
     mpls = (struct ndpi_mpls_header *) &packet[ip_offset];
     label = ntohl(mpls->label);
     /* label = ntohl(*((u_int32_t*)&packet[ip_offset])); */
     workflow->stats.mpls_count++;
     type = ETH_P_IP, ip_offset += 4;
-    
+
     while((label & 0x100) != 0x100) {
       ip_offset += 4;
       label = ntohl(mpls->label);
@@ -676,13 +692,13 @@ void ndpi_workflow_process_packet (struct ndpi_workflow * workflow,
   else if(type == CISCO_D_PROTO) {
     cdp = (struct ndpi_cdp *) &packet[ip_offset];
     cdp_pkts++;
-  }    
+  }
   else if(type == PPPoE) {
     workflow->stats.pppoe_count++;
     type = ETH_P_IP;
     ip_offset += 8;
   }
-  
+
   workflow->stats.vlan_count += vlan_packet;
 
  iph_check:
@@ -787,13 +803,13 @@ void ndpi_workflow_process_packet (struct ndpi_workflow * workflow,
 
       if((version == 1) && (type == 0) && (encapsulates == 1)) {
 	u_int8_t stop = 0;
-	
+
 	offset += 4;
 
 	while((!stop) && (offset < header->caplen)) {
 	  u_int8_t tag_type = packet[offset];
 	  u_int8_t tag_len;
-	  
+
 	  switch(tag_type) {
 	  case 0: /* PADDING Tag */
 	    tag_len = 1;