Added extra processing for POP and SMTP

5 files changed, 121 insertions, 41 deletions

diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index 995bc4c26..8e3f704e4 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -4651,18 +4651,10 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
   ret.master_protocol = flow->detected_protocol_stack[1], ret.app_protocol = flow->detected_protocol_stack[0];
 
   if(flow->server_id == NULL) flow->server_id = dst; /* Default */
+
   if(flow->detected_protocol_stack[0] != NDPI_PROTOCOL_UNKNOWN) {
-    /*
-      With SSL we might want to dissect further packets to decode
-      the certificate type for instance
-    */
-    if(flow->check_extra_packets
-       /*
-	 && (flow->detected_protocol_stack[0] == NDPI_PROTOCOL_TLS)
-       */
-       ) {
+    if(flow->check_extra_packets) {
       ndpi_process_extra_packet(ndpi_str, flow, packet, packetlen, current_tick_l, src, dst);
-
       return(ret);
     } else
       goto ret_protocols;
@@ -6467,7 +6459,7 @@ int ndpi_flowv6_flow_hash(u_int8_t l4_proto, struct ndpi_in6_addr *src_ip, struc
 */
 u_int8_t ndpi_extra_dissection_possible(struct ndpi_detection_module_struct *ndpi_str,
 					struct ndpi_flow_struct *flow) {
-    u_int16_t proto = flow->detected_protocol_stack[1] ? flow->detected_protocol_stack[1] : flow->detected_protocol_stack[0];
+  u_int16_t proto = flow->detected_protocol_stack[1] ? flow->detected_protocol_stack[1] : flow->detected_protocol_stack[0];
 
 #if 0
   printf("[DEBUG] %s(%u.%u): %u\n", __FUNCTION__,
@@ -6493,6 +6485,14 @@ u_int8_t ndpi_extra_dissection_possible(struct ndpi_detection_module_struct *ndp
       return(1);
     break;
 
+  case NDPI_PROTOCOL_FTP_CONTROL:
+  case NDPI_PROTOCOL_MAIL_POP:
+  case NDPI_PROTOCOL_MAIL_IMAP:
+  case NDPI_PROTOCOL_MAIL_SMTP:
+    if(flow->protos.ftp_imap_pop_smtp.password[0] == '\0')
+      return(1);
+    break;
+
   case NDPI_PROTOCOL_SSH:
     if((flow->protos.ssh.hassh_client[0] == '\0')
        || (flow->protos.ssh.hassh_server[0] == '\0'))
diff --git a/src/lib/protocols/dns.c b/src/lib/protocols/dns.c
index 91598f47c..f41eb65ee 100644
--- a/src/lib/protocols/dns.c
+++ b/src/lib/protocols/dns.c
@@ -60,18 +60,19 @@ static u_int getNameLength(u_int i, const u_int8_t *payload, u_int payloadLen) {
   }
 }
 /*
-   allowed chars for dns names A-Z 0-9 _ -
-   Perl script for generation map:
-   my @M;
-   for(my $ch=0; $ch < 256; $ch++) {
-   $M[$ch >> 5] |= 1 << ($ch & 0x1f) if chr($ch) =~ /[a-z0-9_-]/i;
-   }
-   print join(',', map { sprintf "0x%08x",$_ } @M),"\n";
+  allowed chars for dns names A-Z 0-9 _ -
+  Perl script for generation map:
+  my @M;
+  for(my $ch=0; $ch < 256; $ch++) {
+  $M[$ch >> 5] |= 1 << ($ch & 0x1f) if chr($ch) =~ /[a-z0-9_-]/i;
+  }
+  print join(',', map { sprintf "0x%08x",$_ } @M),"\n";
 */
 
-static uint32_t dns_validchar[8] = {
-				    0x00000000,0x03ff2000,0x87fffffe,0x07fffffe,0,0,0,0
-};
+static uint32_t dns_validchar[8] =
+  {
+   0x00000000,0x03ff2000,0x87fffffe,0x07fffffe,0,0,0,0
+  };
 
 /* *********************************************** */
 
@@ -239,7 +240,7 @@ void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, struct nd
 
     ret.master_protocol   = NDPI_PROTOCOL_UNKNOWN;
     ret.app_protocol      = (d_port == 5355) ? NDPI_PROTOCOL_LLMNR : NDPI_PROTOCOL_DNS;
-      
+
     if(invalid) {
       NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
       return;
@@ -262,7 +263,7 @@ void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, struct nd
 
       while(j < max_len && cl != 0) {
 	c = flow->packet.payload[off++];
-	flow->host_server_name[j++] = (dns_validchar[c >> 5] & (1 << (c & 0x1f))) ? c : '_';
+	flow->host_server_name[j++] = tolower((dns_validchar[c >> 5] & (1 << (c & 0x1f))) ? c : '_');
 	cl--;
       }
     }
@@ -270,13 +271,13 @@ void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, struct nd
 
     if(j > 0) {
       ndpi_protocol_match_result ret_match;
-      
+
       ret.app_protocol = ndpi_match_host_subprotocol(ndpi_struct, flow,
 						     (char *)flow->host_server_name,
 						     strlen((const char*)flow->host_server_name),
 						     &ret_match,
 						     NDPI_PROTOCOL_DNS);
-      
+
       if(ret_match.protocol_category != NDPI_PROTOCOL_CATEGORY_UNSPECIFIED)
 	flow->category = ret_match.protocol_category;
 
@@ -288,7 +289,7 @@ void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, struct nd
 
     /* Report if this is a DNS query or reply */
     flow->protos.dns.is_query = is_query;
-    
+
     if(is_query && (ndpi_struct->dns_dont_dissect_response == 0) && (flow->check_extra_packets == 0)) {
       /* In this case we say that the protocol has been detected just to let apps carry on with their activities */
       ndpi_set_detected_protocol(ndpi_struct, flow, ret.app_protocol, ret.master_protocol);
diff --git a/src/lib/protocols/mail_pop.c b/src/lib/protocols/mail_pop.c
index c3ab9ffe0..d12ded666 100644
--- a/src/lib/protocols/mail_pop.c
+++ b/src/lib/protocols/mail_pop.c
@@ -48,6 +48,11 @@ static void ndpi_int_mail_pop_add_connection(struct ndpi_detection_module_struct
   ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_MAIL_POP, NDPI_PROTOCOL_UNKNOWN);
 }
 
+/* **************************************** */
+
+static void popInitExtraPacketProcessing(struct ndpi_flow_struct *flow);
+
+/* **************************************** */
 
 static int ndpi_int_mail_pop_check_for_client_commands(struct ndpi_detection_module_struct
 						       *ndpi_struct, struct ndpi_flow_struct *flow) {
@@ -174,8 +179,10 @@ void ndpi_search_mail_pop_tcp(struct ndpi_detection_module_struct
 	NDPI_LOG_INFO(ndpi_struct, "mail_pop identified\n");
 	
 	if((flow->protos.ftp_imap_pop_smtp.password[0] != '\0')
-	   || (flow->l4.tcp.mail_pop_stage > 3))
+	   || (flow->l4.tcp.mail_pop_stage > 3)) {
 	  ndpi_int_mail_pop_add_connection(ndpi_struct, flow);
+	  popInitExtraPacketProcessing(flow);
+	}
       }
       
       return;      
@@ -203,6 +210,38 @@ void ndpi_search_mail_pop_tcp(struct ndpi_detection_module_struct
 }
 
 
+/* **************************************** */
+
+int ndpi_extra_search_mail_pop_tcp(struct ndpi_detection_module_struct *ndpi_struct,
+				    struct ndpi_flow_struct *flow) {
+  int rc;
+  
+  ndpi_search_mail_pop_tcp(ndpi_struct, flow);
+
+  rc = (flow->protos.ftp_imap_pop_smtp.password[0] == '\0') ? 1 : 0;
+  
+#ifdef POP_DEBUG
+  printf("**** %s() [rc: %d]\n", __FUNCTION__, rc);
+#endif  
+
+  return(rc);
+}
+
+/* **************************************** */
+
+static void popInitExtraPacketProcessing(struct ndpi_flow_struct *flow) {
+#ifdef POP_DEBUG
+  printf("**** %s()\n", __FUNCTION__);
+#endif
+  
+  flow->check_extra_packets = 1;
+  /* At most 7 packets should almost always be enough */
+  flow->max_extra_packets_to_check = 7;
+  flow->extra_packets_func = ndpi_extra_search_mail_pop_tcp;
+}
+
+/* **************************************** */
+
 void init_mail_pop_dissector(struct ndpi_detection_module_struct *ndpi_struct,
 			     u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask) {
   ndpi_set_bitmask_protocol_detection("MAIL_POP", ndpi_struct, detection_bitmask, *id,
diff --git a/src/lib/protocols/mail_smtp.c b/src/lib/protocols/mail_smtp.c
index 5899027b9..fcb390229 100644
--- a/src/lib/protocols/mail_smtp.c
+++ b/src/lib/protocols/mail_smtp.c
@@ -49,10 +49,20 @@
 
 static void ndpi_int_mail_smtp_add_connection(struct ndpi_detection_module_struct
 					      *ndpi_struct, struct ndpi_flow_struct *flow) {
+#ifdef SMTP_DEBUG
+  printf("**** %s()\n", __FUNCTION__);
+#endif
+  
   ndpi_set_detected_protocol(ndpi_struct, flow,
 			     NDPI_PROTOCOL_MAIL_SMTP, NDPI_PROTOCOL_UNKNOWN);
 }
 
+/* **************************************** */
+
+static void smtpInitExtraPacketProcessing(struct ndpi_flow_struct *flow);
+
+/* **************************************** */
+
 void ndpi_search_mail_smtp_tcp(struct ndpi_detection_module_struct *ndpi_struct,
 			       struct ndpi_flow_struct *flow) {
   struct ndpi_packet_struct *packet = &flow->packet;
@@ -66,7 +76,7 @@ void ndpi_search_mail_smtp_tcp(struct ndpi_detection_module_struct *ndpi_struct,
     u_int8_t a;
     u_int8_t bit_count = 0;
 
-    NDPI_PARSE_PACKET_LINE_INFO(ndpi_struct, flow,packet);
+    NDPI_PARSE_PACKET_LINE_INFO(ndpi_struct, flow, packet);
 
     for(a = 0; a < packet->parsed_lines; a++) {
       // expected server responses
@@ -225,12 +235,12 @@ void ndpi_search_mail_smtp_tcp(struct ndpi_detection_module_struct *ndpi_struct,
       NDPI_LOG_INFO(ndpi_struct, "mail smtp identified\n");
 
 #ifdef SMTP_DEBUG
-      printf("%s() [bit_count: %u][%s]\n", __FUNCTION__, bit_count, flow->protos.ftp_imap_pop_smtp.password);
+      printf("%s() [bit_count: %u][%s]\n", __FUNCTION__,
+	     bit_count, flow->protos.ftp_imap_pop_smtp.password);
 #endif
 
-      if(flow->protos.ftp_imap_pop_smtp.password[0] != '\0')
-	ndpi_int_mail_smtp_add_connection(ndpi_struct, flow);
-      
+      ndpi_int_mail_smtp_add_connection(ndpi_struct, flow);
+      smtpInitExtraPacketProcessing(flow);      
       return;
     }
 
@@ -248,10 +258,42 @@ void ndpi_search_mail_smtp_tcp(struct ndpi_detection_module_struct *ndpi_struct,
     return;
   }
 
-  NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
+  if(!flow->check_extra_packets)
+    NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
+}
+
+/* **************************************** */
+
+int ndpi_extra_search_mail_smtp_tcp(struct ndpi_detection_module_struct *ndpi_struct,
+				    struct ndpi_flow_struct *flow) {
+  int rc;
+  
+  ndpi_search_mail_smtp_tcp(ndpi_struct, flow);
 
+  rc = (flow->protos.ftp_imap_pop_smtp.password[0] == '\0') ? 1 : 0;
+  
+#ifdef SMTP_DEBUG
+  printf("**** %s() [rc: %d]\n", __FUNCTION__, rc);
+#endif  
+
+  return(rc);
+}
+
+/* **************************************** */
+
+static void smtpInitExtraPacketProcessing(struct ndpi_flow_struct *flow) {
+#ifdef SMTP_DEBUG
+  printf("**** %s()\n", __FUNCTION__);
+#endif
+  
+  flow->check_extra_packets = 1;
+  /* At most 7 packets should almost always be enough */
+  flow->max_extra_packets_to_check = 7;
+  flow->extra_packets_func = ndpi_extra_search_mail_smtp_tcp;
 }
 
+/* **************************************** */
+
 void init_mail_smtp_dissector(struct ndpi_detection_module_struct *ndpi_struct,
 			      u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask) {
   ndpi_set_bitmask_protocol_detection("MAIL_SMTP", ndpi_struct, detection_bitmask, *id,
diff --git a/src/lib/protocols/tls.c b/src/lib/protocols/tls.c
index a849fba46..cce5e0471 100644
--- a/src/lib/protocols/tls.c
+++ b/src/lib/protocols/tls.c
@@ -1070,14 +1070,12 @@ int sslTryAndRetrieveServerCertificate(struct ndpi_detection_module_struct *ndpi
 
 /* **************************************** */
 
-void sslInitExtraPacketProcessing(int caseNum, struct ndpi_flow_struct *flow) {
+static void sslInitExtraPacketProcessing(struct ndpi_flow_struct *flow) {
   flow->check_extra_packets = 1;
-  /* 0 is the case for waiting for the server certificate */
-  if(caseNum == 0) {
-    /* At most 7 packets should almost always be enough to find the server certificate if it's there */
-    flow->max_extra_packets_to_check = 7;
-    flow->extra_packets_func = sslTryAndRetrieveServerCertificate;
-  }
+
+  /* At most 7 packets should almost always be enough to find the server certificate if it's there */
+  flow->max_extra_packets_to_check = 7;
+  flow->extra_packets_func = sslTryAndRetrieveServerCertificate;
 }
 
 /* **************************************** */
@@ -1120,7 +1118,7 @@ int tlsDetectProtocolFromCertificate(struct ndpi_detection_module_struct *ndpi_s
 	   * a few more packets. */
 	  if(((flow->l4.tcp.tls_seen_client_cert == 1) && (flow->protos.stun_ssl.ssl.client_certificate[0] != '\0'))
 	     && ((flow->l4.tcp.tls_seen_server_cert != 1) && (flow->protos.stun_ssl.ssl.server_certificate[0] == '\0'))) {
-	    sslInitExtraPacketProcessing(0, flow);
+	    sslInitExtraPacketProcessing(flow);
 	  }
 
 	  ndpi_set_detected_protocol(ndpi_struct, flow, subproto,