Improvements for dta dissection

5 files changed, 92 insertions, 70 deletions

diff --git a/example/ndpiReader.c b/example/ndpiReader.c
index e4dde12c6..12a88a1c5 100644
--- a/example/ndpiReader.c
+++ b/example/ndpiReader.c
@@ -82,7 +82,7 @@ static u_int8_t live_capture = 0;
 static u_int8_t undetected_flows_deleted = 0;
 /** User preferences **/
 u_int8_t enable_protocol_guess = 1;
-static u_int8_t verbose = 0, json_flag = 0;
+u_int8_t verbose = 0, json_flag = 0;
 int nDPI_LogLevel = 0;
 char *_debug_protocols = NULL;
 static u_int8_t stats_flag = 0, bpf_filter_flag = 0;
@@ -833,15 +833,8 @@ static void printFlow(u_int16_t id, struct ndpi_flow_info *flow, u_int16_t threa
     if(flow->bittorent_hash[0] != '\0') fprintf(out, "[BT Hash: %s]", flow->bittorent_hash);
     if(flow->dhcp_fingerprint[0] != '\0') fprintf(out, "[DHCP Fingerprint: %s]", flow->dhcp_fingerprint);
 
-    //fprintf(out, "[Num_Packt_Human_Readable_String: %d]", flow->n_pckt_human_readable_string);
-
-
-    //if( (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_HTTP ) && (flow->n_pckt_human_readable_string == 0) ) printf("!WARNING!");
-    //if( (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_FTP_CONTROL) && (flow->n_pckt_human_readable_string == 0) ) printf("!WARNING!");
-    //if( (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_NTP ) && (flow->n_pckt_human_readable_string != 0) ) printf("!WARNING!");
-    //if( (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_IP_ICMP ) && (flow->n_pckt_human_readable_string != 0) ) printf("!WARNING!");
-    //if( (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_VNC ) && (flow->n_pckt_human_readable_string != 0 ) ) printf("!WARNING!");
-
+    if(flow->has_human_readeable_strings) fprintf(out, "[PLAIN TEXT (%s)]", flow->human_readeable_string_buffer);
+    
     fprintf(out, "\n");
   } else {
 #ifdef HAVE_JSON_C
@@ -2804,20 +2797,9 @@ static void ndpi_process_packet(u_char *args,
   /* allocate an exact size buffer to check overflows */
   uint8_t *packet_checked = malloc(header->caplen);
 
-  ndpi_thread_info[thread_id].workflow->hrs = 0;
-  if (ndpi_has_human_readeable_string(ndpi_info_mod, (char*)packet, header->caplen) == 1) {
-    ndpi_thread_info[thread_id].workflow->hrs = 1;
-  }
-
   memcpy(packet_checked, packet, header->caplen);
   p = ndpi_workflow_process_packet(ndpi_thread_info[thread_id].workflow, header, packet_checked);
 
-  if((capture_until != 0) && (header->ts.tv_sec >= capture_until)) {
-    if(ndpi_thread_info[thread_id].workflow->pcap_handle != NULL)
-      pcap_breakloop(ndpi_thread_info[thread_id].workflow->pcap_handle);
-    return;
-  }
-
   if(!pcap_start.tv_sec) pcap_start.tv_sec = header->ts.tv_sec, pcap_start.tv_usec = header->ts.tv_usec;
   pcap_end.tv_sec = header->ts.tv_sec, pcap_end.tv_usec = header->ts.tv_usec;
 
@@ -2840,7 +2822,9 @@ static void ndpi_process_packet(u_char *args,
 	ndpi_free(ndpi_thread_info[thread_id].idle_flows[ndpi_thread_info[thread_id].num_idle_flows]);
       }
 
-      if(++ndpi_thread_info[thread_id].idle_scan_idx == ndpi_thread_info[thread_id].workflow->prefs.num_roots) ndpi_thread_info[thread_id].idle_scan_idx = 0;
+      if(++ndpi_thread_info[thread_id].idle_scan_idx == ndpi_thread_info[thread_id].workflow->prefs.num_roots)
+	ndpi_thread_info[thread_id].idle_scan_idx = 0;
+      
       ndpi_thread_info[thread_id].last_idle_scan_time = ndpi_thread_info[thread_id].workflow->last_time;
     }
   }
@@ -2874,8 +2858,7 @@ static void ndpi_process_packet(u_char *args,
     ndpi_protocol2name(ndpi_thread_info[thread_id].workflow->ndpi_struct, p, trailer->name, sizeof(trailer->name));
     crc = (uint32_t*)&extcap_buf[h.caplen+sizeof(struct ndpi_packet_trailer)];
     *crc = ethernet_crc32((const void*)extcap_buf, h.caplen+sizeof(struct ndpi_packet_trailer));
-    h.caplen += delta;
-    h.len += delta;
+    h.caplen += delta, h.len += delta;
 
 #ifdef DEBUG_TRACE
     if(trace) fprintf(trace, "Dumping %u bytes packet\n", h.caplen);
@@ -2921,7 +2904,6 @@ static void ndpi_process_packet(u_char *args,
   free(packet_checked);
 }
 
-
 /**
  * @brief Call pcap_loop() to process packets from a live capture or savefile
  */
diff --git a/example/ndpi_util.c b/example/ndpi_util.c
index 97ac16790..695028364 100644
--- a/example/ndpi_util.c
+++ b/example/ndpi_util.c
@@ -75,6 +75,7 @@
 #include "ndpi_util.h"
 
 extern u_int8_t enable_protocol_guess;
+extern u_int8_t verbose;
 
 /* ***************************************************** */
 
@@ -459,32 +460,11 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
 
       *src = newflow->src_id, *dst = newflow->dst_id;
 
-      if(workflow->hrs == 1){
-       /* count if no SSL protocol */
-        if((newflow->detected_protocol.app_protocol != NDPI_PROTOCOL_SSL)
-            && (newflow->detected_protocol.master_protocol != NDPI_PROTOCOL_SSL)){
-                newflow->n_pckt_human_readable_string++;
-        }
-      }
-
       return newflow;
     }
   } else {
     struct ndpi_flow_info *flow = *(struct ndpi_flow_info**)ret;
 
-    if(workflow->hrs == 1){
-    /* count if no SSL protocol */
-        if((flow->detected_protocol.app_protocol != NDPI_PROTOCOL_SSL)
-            && (flow->detected_protocol.master_protocol != NDPI_PROTOCOL_SSL)){
-                flow->n_pckt_human_readable_string++;
-        }
-
-    }
-    if((flow->detected_protocol.app_protocol == NDPI_PROTOCOL_SSL)
-            || (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_SSL)){
-                flow->n_pckt_human_readable_string = 0;
-    }
-
     if(is_changed) {
       if(flow->src_ip == iph->saddr
 	 && flow->dst_ip == iph->daddr
@@ -631,7 +611,9 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
 					   const struct ndpi_iphdr *iph,
 					   struct ndpi_ipv6hdr *iph6,
 					   u_int16_t ip_offset,
-					   u_int16_t ipsize, u_int16_t rawsize) {
+					   u_int16_t ipsize, u_int16_t rawsize,
+					   const struct pcap_pkthdr *header,
+					   const u_char *packet) {
   struct ndpi_id_struct *src, *dst;
   struct ndpi_flow_info *flow = NULL;
   struct ndpi_flow_struct *ndpi_flow = NULL;
@@ -668,6 +650,42 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
       flow->dst2src_packets++, flow->dst2src_bytes += rawsize;
 
     flow->last_seen = time;
+
+    if(verbose) {
+    }
+      
+    if(!flow->has_human_readeable_strings) {
+      u_int8_t skip = 0;
+      
+      if((iph->protocol == IPPROTO_TCP)
+	 && (
+	     (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_SSL)
+	     || (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_SSL)
+	     || (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_SSH)
+	     || (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_SSH))	     
+	 ) {
+	if((flow->src2dst_packets+flow->dst2src_packets) < 10 /* MIN_NUM_ENCRYPT_SKIP_PACKETS */)
+	  skip = 1;
+      }
+
+      if(!skip) {
+	char outbuf[64] = { '\0' };
+	
+	if(ndpi_has_human_readeable_string(workflow->ndpi_struct, (char*)packet, header->caplen, 8,
+					   flow->human_readeable_string_buffer,
+					   sizeof(flow->human_readeable_string_buffer)) == 1)
+	  flow->has_human_readeable_strings = 1;
+      }
+    } else {
+      if((iph->protocol == IPPROTO_TCP)
+	 && (
+	     (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_SSL)
+	     || (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_SSL)
+	     || (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_SSH)
+	     || (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_SSH))	     
+	 )
+	flow->has_human_readeable_strings = 0;
+    }
   } else { // flow is NULL
     workflow->stats.total_discarded_bytes++;
     return(nproto);
@@ -965,20 +983,20 @@ iph_check:
     ip_len = sizeof(struct ndpi_ipv6hdr);
 
     if(proto == IPPROTO_DSTOPTS /* IPv6 destination option */) {
-
       u_int8_t *options = (u_int8_t*)&packet[ip_offset+ip_len];
       proto = options[0];
       ip_len += 8 * (options[1] + 1);
     }
+    
     iph = NULL;
-
   } else {
     static u_int8_t ipv4_warning_used = 0;
 
   v4_warning:
     if(ipv4_warning_used == 0) {
       if(!workflow->prefs.quiet_mode)
-        NDPI_LOG(0, workflow->ndpi_struct, NDPI_LOG_DEBUG, "\n\nWARNING: only IPv4/IPv6 packets are supported in this demo (nDPI supports both IPv4 and IPv6), all other packets will be discarded\n\n");
+        NDPI_LOG(0, workflow->ndpi_struct, NDPI_LOG_DEBUG,
+		 "\n\nWARNING: only IPv4/IPv6 packets are supported in this demo (nDPI supports both IPv4 and IPv6), all other packets will be discarded\n\n");
       ipv4_warning_used = 1;
     }
     workflow->stats.total_discarded_bytes +=  header->len;
@@ -1053,7 +1071,8 @@ iph_check:
 
   /* process the packet */
   return(packet_processing(workflow, time, vlan_id, iph, iph6,
-			   ip_offset, header->caplen - ip_offset, header->caplen));
+			   ip_offset, header->caplen - ip_offset,
+			   header->caplen, header, packet));
 }
 
 /* ********************************************************** */
diff --git a/example/ndpi_util.h b/example/ndpi_util.h
index aed63efe3..b006fd8d3 100644
--- a/example/ndpi_util.h
+++ b/example/ndpi_util.h
@@ -128,8 +128,9 @@ typedef struct ndpi_flow_info {
   u_int64_t last_seen;
   u_int64_t src2dst_bytes, dst2src_bytes;
   u_int32_t src2dst_packets, dst2src_packets;
-  u_int32_t n_pckt_human_readable_string;
-
+  u_int32_t has_human_readeable_strings;
+  char human_readeable_string_buffer[32];
+  
   // result only, not used for flow identification
   ndpi_protocol detected_protocol;
 
@@ -200,10 +201,7 @@ typedef struct ndpi_workflow {
   void **ndpi_flows_root;
   struct ndpi_detection_module_struct *ndpi_struct;
   u_int32_t num_allocated_flows;
-  
-  int hrs;
-  
-} ndpi_workflow_t;
+ } ndpi_workflow_t;
 
 
 /* TODO: remove wrappers parameters and use ndpi global, when their initialization will be fixed... */
diff --git a/src/include/ndpi_api.h b/src/include/ndpi_api.h
index f2b03774f..d5cc9ad15 100644
--- a/src/include/ndpi_api.h
+++ b/src/include/ndpi_api.h
@@ -812,7 +812,9 @@ extern "C" {
   u_int16_t ndpi_guess_host_protocol_id(struct ndpi_detection_module_struct *ndpi_struct,
 					struct ndpi_flow_struct *flow);
   int ndpi_has_human_readeable_string(struct ndpi_detection_module_struct *ndpi_struct,
-				      char *buffer, u_int buffer_size);
+				      char *buffer, u_int buffer_size,
+				      u_int8_t min_string_match_len, /* Will return 0 if no string > min_string_match_len have been found */
+				      char *outbuf, u_int outbuf_len);
   char* ndpi_ssl_version2str(u_int16_t version);
 
   /* Serializer */
diff --git a/src/lib/ndpi_utils.c b/src/lib/ndpi_utils.c
index 02876d6f1..55c626c27 100644
--- a/src/lib/ndpi_utils.c
+++ b/src/lib/ndpi_utils.c
@@ -608,16 +608,21 @@ const char* ndpi_cipher2str(u_int32_t cipher) {
 
 static int ndpi_is_other_char(char c) {
   return((c == '.')
+	 || (c == ' ')
 	 || (c == '@')
+	 || (c == '/')
 	 );
 }
 
 /* ******************************************************************** */
 
 static int ndpi_is_valid_char(char c) {
-  return(isdigit(c)
-	 || isalpha(c)
-	 || ndpi_is_other_char(c));
+  if(ispunct(c) && (!ndpi_is_other_char(c)))
+    return(0);
+  else
+    return(isdigit(c)
+	   || isalpha(c)
+	   || ndpi_is_other_char(c));
 }
 
 /* ******************************************************************** */
@@ -628,11 +633,11 @@ static int ndpi_find_non_eng_bigrams(struct ndpi_detection_module_struct *ndpi_s
   char s[3];
 
   if((isdigit(str[0]) && isdigit(str[1]))
-     || (ndpi_is_other_char(str[0]) || ndpi_is_other_char(str[1]))
-     || (ndpi_is_other_char(str[0]) || ndpi_is_other_char(str[1]))
+     || ndpi_is_other_char(str[0])
+     || ndpi_is_other_char(str[1])
      )
     return(1);
-
+  
   s[0] = tolower(str[0]), s[1] = tolower(str[1]), s[2] = '\0';
 
   return(ndpi_match_bigram(ndpi_struct, &ndpi_struct->bigrams_automa, s));
@@ -643,13 +648,17 @@ static int ndpi_find_non_eng_bigrams(struct ndpi_detection_module_struct *ndpi_s
 /* #define PRINT_STRINGS 1 */
 
 int ndpi_has_human_readeable_string(struct ndpi_detection_module_struct *ndpi_struct,
-				    char *buffer, u_int buffer_size) {
-  u_int ret = 0, i = 0, do_cr = 0, len = 0;
-  const u_int8_t NDPI_MIN_VALID_STRING_LEN = 4; /* Will return 0 if no string > NDPI_MIN_VALID_STRING_LEN have been found */
+				    char *buffer, u_int buffer_size,
+				    u_int8_t min_string_match_len,
+				    char *outbuf, u_int outbuf_len) {
+  u_int ret = 0, i = 0, do_cr = 0, len = 0, o_idx = 0, being_o_idx = 0;
 
   if(buffer_size <= 0)
     return(0);
 
+  outbuf_len--;
+  outbuf[outbuf_len] = '\0';
+  
   for(i=0; i<buffer_size-2; i++) {
     if(ndpi_is_valid_char(buffer[i])
        && ndpi_is_valid_char(buffer[i+1])
@@ -657,23 +666,35 @@ int ndpi_has_human_readeable_string(struct ndpi_detection_module_struct *ndpi_st
 #ifdef PRINT_STRINGS
       printf("%c%c", buffer[i], buffer[i+1]);
 #endif
+      if(o_idx < outbuf_len) outbuf[o_idx++] = buffer[i];
+      if(o_idx < outbuf_len) outbuf[o_idx++] = buffer[i+1];
       do_cr = 1, i += 1, len += 2;
     } else {
       if(ndpi_is_valid_char(buffer[i]) && do_cr) {
 #ifdef PRINT_STRINGS
 	printf("%c", buffer[i]);
 #endif
+	if(o_idx < outbuf_len) outbuf[o_idx++] = buffer[i];
 	len += 1;
       }
 
       // printf("->> %c%c\n", isprint(buffer[i]) ? buffer[i] : '.', isprint(buffer[i+1]) ? buffer[i+1] : '.');
       if(do_cr) {
+	if(len > min_string_match_len)
+	  ret = 1;
+	else {
+	  o_idx = being_o_idx;
+	  being_o_idx = o_idx;
+	  outbuf[o_idx] = '\0';
+	}
+	
 #ifdef PRINT_STRINGS
-	printf(" [len: %u]\n", len);
+	printf(" [len: %u]%s\n", len, ret ? "<-- HIT" : "");
 #endif
-	if(len > NDPI_MIN_VALID_STRING_LEN)
-	  ret = 1;
 
+	if(ret)
+	  break;
+	
 	do_cr = 0, len = 0;
       }
     }