1 files changed, 124 insertions, 102 deletions

diff --git a/example/reader_util.c b/example/reader_util.c
index 1cfedf1a6..a5a40905d 100644
--- a/example/reader_util.c
+++ b/example/reader_util.c
@@ -109,7 +109,7 @@ struct payload_stats {
 
 
 struct payload_stats *pstats = NULL;
-u_int32_t max_num_packets_per_flow      = 32;
+u_int32_t max_num_packets_per_flow      = 10; /* ETTA requires min 10 pkts for record. */
 u_int32_t max_packet_payload_dissection = 128;
 u_int32_t max_num_reported_top_payloads = 25;
 u_int16_t min_pattern_len = 4;
@@ -185,7 +185,7 @@ void ndpi_payload_analyzer(struct ndpi_flow_info *flow,
   u_int16_t i, j;
   u_int16_t scan_len = ndpi_min(max_packet_payload_dissection, payload_len);
 
-  if((flow->src2dst_pkt_count+flow->dst2src_pkt_count) < max_num_packets_per_flow) {
+  if((flow->entropy.src2dst_pkt_count+flow->entropy.dst2src_pkt_count) <= max_num_packets_per_flow) {
 #ifdef DEBUG_PAYLOAD
     printf("[hashval: %u][proto: %u][vlan: %u][%s:%u <-> %s:%u][direction: %s][payload_len: %u]\n",
 	   flow->hashval, flow->protocol, flow->vlan_id,
@@ -543,23 +543,25 @@ ndpi_flow_update_byte_count(struct ndpi_flow_info *flow, const void *x,
    * the 4000th octet has been seen for a flow.
    */
 
-  /* octet count was already incremented before processing this payload */
-  if(src_to_dst_direction) {
-    current_count = flow->src2dst_l4_bytes - len;
-  } else {
-    current_count = flow->dst2src_l4_bytes - len;
-  }
+  if((flow->entropy.src2dst_pkt_count+flow->entropy.dst2src_pkt_count) <= max_num_packets_per_flow) {
+    /* octet count was already incremented before processing this payload */
+    if(src_to_dst_direction) {
+      current_count = flow->entropy.src2dst_l4_bytes - len;
+    } else {
+      current_count = flow->entropy.dst2src_l4_bytes - len;
+    }
 
-  if(current_count < ETTA_MIN_OCTETS) {
-    for(i=0; i<len; i++) {
-      if(src_to_dst_direction) {
-        flow->src2dst_byte_count[data[i]]++;
-      } else {
-        flow->dst2src_byte_count[data[i]]++;
-      }
-      current_count++;
-      if(current_count >= ETTA_MIN_OCTETS) {
-        break;
+    if(current_count < ETTA_MIN_OCTETS) {
+      for(i=0; i<len; i++) {
+        if(src_to_dst_direction) {
+          flow->entropy.src2dst_byte_count[data[i]]++;
+        } else {
+          flow->entropy.dst2src_byte_count[data[i]]++;
+        }
+        current_count++;
+        if(current_count >= ETTA_MIN_OCTETS) {
+          break;
+        }
       }
     }
   }
@@ -581,17 +583,19 @@ ndpi_flow_update_byte_dist_mean_var(ndpi_flow_info_t *flow, const void *x,
   double delta;
   unsigned int i;
 
-  for(i=0; i<len; i++) {
-    if(src_to_dst_direction) {
-      flow->src2dst_num_bytes += 1;
-      delta = ((double)data[i] - flow->src2dst_bd_mean);
-      flow->src2dst_bd_mean += delta/((double)flow->src2dst_num_bytes);
-      flow->src2dst_bd_variance += delta*((double)data[i] - flow->src2dst_bd_mean);
-    } else {
-      flow->dst2src_num_bytes += 1;
-      delta = ((double)data[i] - flow->dst2src_bd_mean);
-      flow->dst2src_bd_mean += delta/((double)flow->dst2src_num_bytes);
-      flow->dst2src_bd_variance += delta*((double)data[i] - flow->dst2src_bd_mean);
+  if((flow->entropy.src2dst_pkt_count+flow->entropy.dst2src_pkt_count) <= max_num_packets_per_flow) {
+    for(i=0; i<len; i++) {
+      if(src_to_dst_direction) {
+        flow->entropy.src2dst_num_bytes += 1;
+        delta = ((double)data[i] - flow->entropy.src2dst_bd_mean);
+        flow->entropy.src2dst_bd_mean += delta/((double)flow->entropy.src2dst_num_bytes);
+        flow->entropy.src2dst_bd_variance += delta*((double)data[i] - flow->entropy.src2dst_bd_mean);
+      } else {
+        flow->entropy.dst2src_num_bytes += 1;
+        delta = ((double)data[i] - flow->entropy.dst2src_bd_mean);
+        flow->entropy.dst2src_bd_mean += delta/((double)flow->entropy.dst2src_num_bytes);
+        flow->entropy.dst2src_bd_variance += delta*((double)data[i] - flow->entropy.dst2src_bd_mean);
+      }
     }
   }
 }
@@ -826,63 +830,70 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
       workflow->stats.ndpi_flow_count++;
 
       *src = newflow->src_id, *dst = newflow->dst_id;
-      newflow->src2dst_pkt_len[newflow->src2dst_pkt_count] = l4_packet_len;
-      newflow->src2dst_pkt_time[newflow->src2dst_pkt_count] = when;
-      if(newflow->src2dst_pkt_count == 0) {
-        newflow->src2dst_start = when;
+      newflow->entropy.src2dst_pkt_len[newflow->entropy.src2dst_pkt_count] = l4_data_len;
+      newflow->entropy.src2dst_pkt_time[newflow->entropy.src2dst_pkt_count] = when;
+      if (newflow->entropy.src2dst_pkt_count == 0) {
+        newflow->entropy.src2dst_start = when;
       }
-      newflow->src2dst_pkt_count++;
-      if(l4_data_len != 0XFEEDFACE) {
-        newflow->src2dst_opackets++;
+      newflow->entropy.src2dst_pkt_count++;
+      // Non zero app data.
+      if (l4_data_len != 0XFEEDFACE && l4_data_len != 0) {
+        newflow->entropy.src2dst_opackets++;
+        newflow->entropy.src2dst_l4_bytes += l4_data_len;
       }
       return newflow;
     }
   } else {
-    struct ndpi_flow_info *flow = *(struct ndpi_flow_info**)ret;
+    struct ndpi_flow_info *rflow = *(struct ndpi_flow_info**)ret;
 
     if(is_changed) {
-      if(flow->src_ip == iph->saddr
-	 && flow->dst_ip == iph->daddr
-	 && flow->src_port == htons(*sport)
-	 && flow->dst_port == htons(*dport)
+      if(rflow->src_ip == iph->saddr
+	 && rflow->dst_ip == iph->daddr
+	 && rflow->src_port == htons(*sport)
+	 && rflow->dst_port == htons(*dport)
 	 )
-	*src = flow->dst_id, *dst = flow->src_id, *src_to_dst_direction = 0, flow->bidirectional = 1;
+	*src = rflow->dst_id, *dst = rflow->src_id, *src_to_dst_direction = 0, rflow->bidirectional = 1;
       else
-	*src = flow->src_id, *dst = flow->dst_id, *src_to_dst_direction = 1;
+	*src = rflow->src_id, *dst = rflow->dst_id, *src_to_dst_direction = 1;
     }
     else {
-      if(flow->src_ip == iph->saddr
-	 && flow->dst_ip == iph->daddr
-	 && flow->src_port == htons(*sport)
-	 && flow->dst_port == htons(*dport)
+      if(rflow->src_ip == iph->saddr
+	 && rflow->dst_ip == iph->daddr
+	 && rflow->src_port == htons(*sport)
+	 && rflow->dst_port == htons(*dport)
 	 )
-	*src = flow->src_id, *dst = flow->dst_id, *src_to_dst_direction = 1;
+	*src = rflow->src_id, *dst = rflow->dst_id, *src_to_dst_direction = 1;
       else
-	*src = flow->dst_id, *dst = flow->src_id, *src_to_dst_direction = 0, flow->bidirectional = 1;
+	*src = rflow->dst_id, *dst = rflow->src_id, *src_to_dst_direction = 0, rflow->bidirectional = 1;
     }
-    if(src_to_dst_direction) {
-      if(flow->src2dst_pkt_count < MAX_NUM_PKTS) {
-        flow->src2dst_pkt_len[flow->src2dst_pkt_count] = l4_packet_len;
-        flow->src2dst_pkt_time[flow->src2dst_pkt_count] = when;
-        flow->src2dst_pkt_count++;
+    if (src_to_dst_direction) {
+      if (rflow->entropy.src2dst_pkt_count < max_num_packets_per_flow) {
+        rflow->entropy.src2dst_pkt_len[rflow->entropy.src2dst_pkt_count] = l4_data_len;
+        rflow->entropy.src2dst_pkt_time[rflow->entropy.src2dst_pkt_count] = when;
+        rflow->entropy.src2dst_l4_bytes += l4_data_len;
+        rflow->entropy.src2dst_pkt_count++;
       }
-      if(l4_data_len != 0XFEEDFACE) {
-        flow->src2dst_opackets++;
+      // Non zero app data.
+      if (l4_data_len != 0XFEEDFACE && l4_data_len != 0) {
+        rflow->entropy.src2dst_opackets++;
       }
     } else {
-      if(flow->dst2src_pkt_count < MAX_NUM_PKTS) {
-        flow->dst2src_pkt_len[flow->dst2src_pkt_count] = l4_packet_len;
-        flow->dst2src_pkt_time[flow->dst2src_pkt_count] = when;
-        if(flow->dst2src_pkt_count == 0) {
-          flow->dst2src_start = when;
+      if (rflow->entropy.dst2src_pkt_count < max_num_packets_per_flow) {
+        rflow->entropy.dst2src_pkt_len[rflow->entropy.dst2src_pkt_count] = l4_data_len;
+        rflow->entropy.dst2src_pkt_time[rflow->entropy.dst2src_pkt_count] = when;
+        if (rflow->entropy.dst2src_pkt_count == 0) {
+          rflow->entropy.dst2src_start = when;
         }
-        flow->dst2src_pkt_count++;
+        rflow->entropy.dst2src_l4_bytes += l4_data_len;
+        rflow->entropy.dst2src_pkt_count++;
       }
-      if(l4_data_len != 0XFEEDFACE) {
-        flow->dst2src_opackets++;
+      // Non zero app data.
+      if (l4_data_len != 0XFEEDFACE && l4_data_len != 0) {
+        rflow->entropy.dst2src_opackets++;
       }
     }
-    return flow;
+    
+    return(rflow);
   }
 }
 
@@ -927,27 +938,6 @@ static struct ndpi_flow_info *get_ndpi_flow_info6(struct ndpi_workflow * workflo
 /* ****************************************************** */
 
 void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_flow_info *flow) {
-  if(enable_joy_stats) {
-    /* Update SPLT scores. */
-
-    if(flow->bidirectional)
-      flow->score = ndpi_classify(flow->src2dst_pkt_len, flow->src2dst_pkt_time,
-				  flow->dst2src_pkt_len, flow->dst2src_pkt_time,
-				  flow->src2dst_start, flow->dst2src_start,
-				  MAX_NUM_PKTS, flow->src_port, flow->dst_port,
-				  flow->src2dst_packets, flow->dst2src_packets,
-				  flow->src2dst_opackets, flow->dst2src_opackets,
-				  flow->src2dst_l4_bytes, flow->dst2src_l4_bytes, 1,
-				  flow->src2dst_byte_count, flow->dst2src_byte_count);
-    else
-      flow->score = ndpi_classify(flow->src2dst_pkt_len, flow->src2dst_pkt_time,
-				  NULL, NULL, flow->src2dst_start, flow->src2dst_start,
-				  MAX_NUM_PKTS, flow->src_port, flow->dst_port,
-				  flow->src2dst_packets, 0,
-				  flow->src2dst_opackets, 0,
-				  flow->src2dst_l4_bytes, 0, 1,
-				  flow->src2dst_byte_count, NULL);
-  }
 
   if(!flow->ndpi_flow) return;
 
@@ -1036,6 +1026,19 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
 /* ****************************************************** */
 
 /**
+ * @brief Clear entropy stats if it meets prereq.
+ */
+static void
+ndpi_clear_entropy_stats(struct ndpi_flow_info *flow)
+{
+  if(flow->entropy.src2dst_pkt_count + flow->entropy.dst2src_pkt_count == max_num_packets_per_flow) {
+    memcpy(&flow->last_entropy, &flow->entropy,  sizeof(struct ndpi_entropy));
+    memset(&flow->entropy, 0x00, sizeof(struct ndpi_entropy));
+  }
+}
+
+/* ****************************************************** */
+/**
    Function to process the packet:
    determine the flow of a packet and try to decode it
    @return: 0 if success; else != 0
@@ -1088,9 +1091,9 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
     if((tcph != NULL) && (tcph->fin || tcph->rst || tcph->syn))
       begin_or_end_tcp = 1;
 
-    if(flow->flow_last_pkt_time.tv_sec) {
-      ndpi_timer_sub(&when, &flow->flow_last_pkt_time, &tdiff);
-
+    if(flow->entropy.flow_last_pkt_time.tv_sec) {
+      ndpi_timer_sub(&when, &flow->entropy.flow_last_pkt_time, &tdiff);
+    
       if(flow->iat_flow) {
 	u_int32_t ms = ndpi_timeval_to_milliseconds(tdiff);
 
@@ -1098,11 +1101,11 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
 	  ndpi_data_add_value(flow->iat_flow, ms);
       }
     }
-    memcpy(&flow->flow_last_pkt_time, &when, sizeof(when));
+    memcpy(&flow->entropy.flow_last_pkt_time, &when, sizeof(when));
 
     if(src_to_dst_direction) {
-      if(flow->src2dst_last_pkt_time.tv_sec && (!begin_or_end_tcp)) {
-	ndpi_timer_sub(&when, &flow->src2dst_last_pkt_time, &tdiff);
+      if(flow->entropy.src2dst_last_pkt_time.tv_sec) {
+	ndpi_timer_sub(&when, &flow->entropy.src2dst_last_pkt_time, &tdiff);
 
 	if(flow->iat_c_to_s) {
 	  u_int32_t ms = ndpi_timeval_to_milliseconds(tdiff);
@@ -1113,11 +1116,10 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
 
       ndpi_data_add_value(flow->pktlen_c_to_s, rawsize);
       flow->src2dst_packets++, flow->src2dst_bytes += rawsize;
-      flow->src2dst_l4_bytes += payload_len;
-      memcpy(&flow->src2dst_last_pkt_time, &when, sizeof(when));
+      memcpy(&flow->entropy.src2dst_last_pkt_time, &when, sizeof(when));
     } else {
-      if(flow->dst2src_last_pkt_time.tv_sec && (!begin_or_end_tcp)) {
-	ndpi_timer_sub(&when, &flow->dst2src_last_pkt_time, &tdiff);
+      if(flow->entropy.dst2src_last_pkt_time.tv_sec && (!begin_or_end_tcp)) {
+	ndpi_timer_sub(&when, &flow->entropy.dst2src_last_pkt_time, &tdiff);
 
 	if(flow->iat_s_to_c) {
 	  u_int32_t ms = ndpi_timeval_to_milliseconds(tdiff);
@@ -1128,8 +1130,7 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
 
       ndpi_data_add_value(flow->pktlen_s_to_c, rawsize);
       flow->dst2src_packets++, flow->dst2src_bytes += rawsize;
-      flow->dst2src_l4_bytes += payload_len;
-      memcpy(&flow->dst2src_last_pkt_time, &when, sizeof(when));
+      memcpy(&flow->entropy.dst2src_last_pkt_time, &when, sizeof(when));
     }
 
     if(enable_payload_analyzer && (payload_len > 0))
@@ -1141,6 +1142,26 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
       /* Update BD, distribution and mean. */
       ndpi_flow_update_byte_count(flow, payload, payload_len, src_to_dst_direction);
       ndpi_flow_update_byte_dist_mean_var(flow, payload, payload_len, src_to_dst_direction);
+      /* Update SPLT scores for first 32 packets. */
+      if((flow->entropy.src2dst_pkt_count+flow->entropy.dst2src_pkt_count) <= max_num_packets_per_flow) {
+        if(flow->bidirectional)
+          flow->entropy.score = ndpi_classify(flow->entropy.src2dst_pkt_len, flow->entropy.src2dst_pkt_time,
+				     flow->entropy.dst2src_pkt_len, flow->entropy.dst2src_pkt_time,
+				     flow->entropy.src2dst_start, flow->entropy.dst2src_start,
+				     max_num_packets_per_flow, flow->src_port, flow->dst_port,
+				     flow->src2dst_packets, flow->dst2src_packets,
+				     flow->entropy.src2dst_opackets, flow->entropy.dst2src_opackets,
+				     flow->entropy.src2dst_l4_bytes, flow->entropy.dst2src_l4_bytes, 1,
+				     flow->entropy.src2dst_byte_count, flow->entropy.dst2src_byte_count);
+       else
+         flow->entropy.score = ndpi_classify(flow->entropy.src2dst_pkt_len, flow->entropy.src2dst_pkt_time,
+				     NULL, NULL, flow->entropy.src2dst_start, flow->entropy.src2dst_start,
+				     max_num_packets_per_flow, flow->src_port, flow->dst_port,
+				     flow->src2dst_packets, 0,
+				     flow->entropy.src2dst_opackets, 0,
+				     flow->entropy.src2dst_l4_bytes, 0, 1,
+				     flow->entropy.src2dst_byte_count, NULL);
+      }
     }
 
     if(flow->first_seen == 0)
@@ -1148,6 +1169,9 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
 
     flow->last_seen = time;
 
+    /* Copy packets entropy if num packets count == 10 */
+    ndpi_clear_entropy_stats(flow);
+
     if(!flow->has_human_readeable_strings) {
       u_int8_t skip = 0;
 
@@ -1163,8 +1187,6 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
       }
 
       if(!skip) {
-	char outbuf[64] = { '\0' };
-
 	if(ndpi_has_human_readeable_string(workflow->ndpi_struct, (char*)packet, header->caplen,
 					   human_readeable_string_len,
 					   flow->human_readeable_string_buffer,
@@ -1527,10 +1549,10 @@ struct ndpi_proto ndpi_workflow_process_packet(struct ndpi_workflow * workflow,
       /* https://en.wikipedia.org/wiki/TZSP */
       u_int offset = ip_offset+ip_len+sizeof(struct ndpi_udphdr);
       u_int8_t version = packet[offset];
-      u_int8_t type    = packet[offset+1];
+      u_int8_t ts_type    = packet[offset+1];
       u_int16_t encapsulates = ntohs(*((u_int16_t*)&packet[offset+2]));
 
-      if((version == 1) && (type == 0) && (encapsulates == 1)) {
+      if((version == 1) && (ts_type == 0) && (encapsulates == 1)) {
 	u_int8_t stop = 0;
 
 	offset += 4;