Implemented IMAP data extraction

3 files changed, 103 insertions, 46 deletions

diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h
index 9535136e1..4ca640d43 100644
--- a/src/include/ndpi_typedefs.h
+++ b/src/include/ndpi_typedefs.h
@@ -1239,6 +1239,10 @@ struct ndpi_flow_struct {
     } ftp;
     
     struct {
+      char username[16], password[16];
+    } imap;
+    
+    struct {
       /* Bittorrent hash */
       u_char hash[20];
     } bittorrent;
diff --git a/src/lib/ndpi_utils.c b/src/lib/ndpi_utils.c
index 4d98aa6d2..20d231db8 100644
--- a/src/lib/ndpi_utils.c
+++ b/src/lib/ndpi_utils.c
@@ -628,7 +628,6 @@ static int ndpi_is_valid_char(char c) {
 
 /* ******************************************************************** */
 
-
 static int ndpi_find_non_eng_bigrams(struct ndpi_detection_module_struct *ndpi_struct,
 				     char *str) {
   char s[3];
@@ -881,6 +880,13 @@ int ndpi_flow2json(struct ndpi_detection_module_struct *ndpi_struct,
     ndpi_serialize_end_of_block(serializer);
     break;
 
+  case NDPI_PROTOCOL_MAIL_IMAP:
+    ndpi_serialize_start_of_block(serializer, "imap");
+    ndpi_serialize_string_string(serializer, "user", flow->protos.imap.username);
+    ndpi_serialize_string_string(serializer, "password", flow->protos.imap.password);
+    ndpi_serialize_end_of_block(serializer);
+    break;
+      
   case NDPI_PROTOCOL_FTP_CONTROL:
     ndpi_serialize_start_of_block(serializer, "ftp");
     ndpi_serialize_string_string(serializer, "user", flow->protos.ftp.username);
diff --git a/src/lib/protocols/mail_imap.c b/src/lib/protocols/mail_imap.c
index 4d87275b7..1ebb18487 100644
--- a/src/lib/protocols/mail_imap.c
+++ b/src/lib/protocols/mail_imap.c
@@ -28,6 +28,7 @@
 
 #include "ndpi_api.h"
 
+/* #define IMAP_DEBUG 1*/
 
 static void ndpi_int_mail_imap_add_connection(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow)
 {
@@ -44,17 +45,21 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
   /* const u_int8_t *command = 0; */
 
   NDPI_LOG_DBG(ndpi_struct, "search IMAP_IMAP\n");
-  
-  if (flow->l4.tcp.mail_imap_starttls == 2) {
+
+#ifdef IMAP_DEBUG
+  printf("%s() [%s]\n", __FUNCTION__, packet->payload);
+#endif
+
+  if(flow->l4.tcp.mail_imap_starttls == 2) {
     NDPI_LOG_DBG2(ndpi_struct, "starttls detected\n");
     NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_PROTOCOL_MAIL_IMAP);
     NDPI_DEL_PROTOCOL_FROM_BITMASK(flow->excluded_protocol_bitmask, NDPI_PROTOCOL_TLS);
     return;
   }
 
-  if (packet->payload_packet_len >= 4 && ntohs(get_u_int16_t(packet->payload, packet->payload_packet_len - 2)) == 0x0d0a) {
+  if(packet->payload_packet_len >= 4 && ntohs(get_u_int16_t(packet->payload, packet->payload_packet_len - 2)) == 0x0d0a) {
     // the DONE command appears without a tag
-    if (packet->payload_packet_len == 6 && ((packet->payload[0] == 'D' || packet->payload[0] == 'd')
+    if(packet->payload_packet_len == 6 && ((packet->payload[0] == 'D' || packet->payload[0] == 'd')
 					    && (packet->payload[1] == 'O' || packet->payload[1] == 'o')
 					    && (packet->payload[2] == 'N' || packet->payload[2] == 'n')
 					    && (packet->payload[3] == 'E' || packet->payload[3] == 'e'))) {
@@ -62,37 +67,37 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
       saw_command = 1;
     } else {
 
-      if (flow->l4.tcp.mail_imap_stage < 4) {
+      if(flow->l4.tcp.mail_imap_stage < 4) {
 	// search for the first space character (end of the tag)
 	while (i < 20 && i < packet->payload_packet_len) {
-	  if (i > 0 && packet->payload[i] == ' ') {
+	  if(i > 0 && packet->payload[i] == ' ') {
 	    space_pos = i;
 	    break;
 	  }
-	  if (!((packet->payload[i] >= 'a' && packet->payload[i] <= 'z') ||
+	  if(!((packet->payload[i] >= 'a' && packet->payload[i] <= 'z') ||
 		(packet->payload[i] >= 'A' && packet->payload[i] <= 'Z') ||
 		(packet->payload[i] >= '0' && packet->payload[i] <= '9') || packet->payload[i] == '*' || packet->payload[i] == '.')) {
 	    goto imap_excluded;
 	  }
 	  i++;
 	}
-	if (space_pos == 0 || space_pos == (packet->payload_packet_len - 1)) {
+	if(space_pos == 0 || space_pos == (packet->payload_packet_len - 1)) {
 	  goto imap_excluded;
 	}
 	// now walk over a possible mail number to the next space
 	i++;
-	if (i < packet->payload_packet_len && (packet->payload[i] >= '0' && packet->payload[i] <= '9')) {
+	if(i < packet->payload_packet_len && (packet->payload[i] >= '0' && packet->payload[i] <= '9')) {
 	  while (i < 20 && i < packet->payload_packet_len) {
-	    if (i > 0 && packet->payload[i] == ' ') {
+	    if(i > 0 && packet->payload[i] == ' ') {
 	      space_pos = i;
 	      break;
 	    }
-	    if (!(packet->payload[i] >= '0' && packet->payload[i] <= '9')) {
+	    if(!(packet->payload[i] >= '0' && packet->payload[i] <= '9')) {
 	      goto imap_excluded;
 	    }
 	    i++;
 	  }
-	  if (space_pos == 0 || space_pos == (packet->payload_packet_len - 1)) {
+	  if(space_pos == 0 || space_pos == (packet->payload_packet_len - 1)) {
 	    goto imap_excluded;
 	  }
 	}
@@ -103,23 +108,23 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
 	/* command = &(packet->payload[command_start]); */
       }
 
-      if ((command_start + 3) < packet->payload_packet_len) {
-	if ((packet->payload[command_start] == 'O' || packet->payload[command_start] == 'o')
+      if((command_start + 3) < packet->payload_packet_len) {
+	if((packet->payload[command_start] == 'O' || packet->payload[command_start] == 'o')
 	    && (packet->payload[command_start + 1] == 'K' || packet->payload[command_start + 1] == 'k')
 	    && packet->payload[command_start + 2] == ' ') {
 	  flow->l4.tcp.mail_imap_stage += 1;
-	  if (flow->l4.tcp.mail_imap_starttls == 1)
+	  if(flow->l4.tcp.mail_imap_starttls == 1)
 	    flow->l4.tcp.mail_imap_starttls = 2;
 	  saw_command = 1;
-	} else if ((packet->payload[command_start] == 'U' || packet->payload[command_start] == 'u')
+	} else if((packet->payload[command_start] == 'U' || packet->payload[command_start] == 'u')
 		   && (packet->payload[command_start + 1] == 'I' || packet->payload[command_start + 1] == 'i')
 		   && (packet->payload[command_start + 2] == 'D' || packet->payload[command_start + 2] == 'd')) {
 	  flow->l4.tcp.mail_imap_stage += 1;
 	  saw_command = 1;
 	}
       }
-      if ((command_start + 10) < packet->payload_packet_len) {
-	if ((packet->payload[command_start] == 'C' || packet->payload[command_start] == 'c')
+      if((command_start + 10) < packet->payload_packet_len) {
+	if((packet->payload[command_start] == 'C' || packet->payload[command_start] == 'c')
 	    && (packet->payload[command_start + 1] == 'A' || packet->payload[command_start + 1] == 'a')
 	    && (packet->payload[command_start + 2] == 'P' || packet->payload[command_start + 2] == 'p')
 	    && (packet->payload[command_start + 3] == 'A' || packet->payload[command_start + 3] == 'a')
@@ -133,8 +138,8 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
 	  saw_command = 1;
 	}
       }
-      if ((command_start + 8) < packet->payload_packet_len) {
-	if ((packet->payload[command_start] == 'S' || packet->payload[command_start] == 's')
+      if((command_start + 8) < packet->payload_packet_len) {
+	if((packet->payload[command_start] == 'S' || packet->payload[command_start] == 's')
 	    && (packet->payload[command_start + 1] == 'T' || packet->payload[command_start + 1] == 't')
 	    && (packet->payload[command_start + 2] == 'A' || packet->payload[command_start + 2] == 'a')
 	    && (packet->payload[command_start + 3] == 'R' || packet->payload[command_start + 3] == 'r')
@@ -148,36 +153,71 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
         saw_command = 1;
 	}
       }
-      if ((command_start + 5) < packet->payload_packet_len) {
-	if ((packet->payload[command_start] == 'L' || packet->payload[command_start] == 'l')
+      if((command_start + 5) < packet->payload_packet_len) {
+	if((packet->payload[command_start] == 'L' || packet->payload[command_start] == 'l')
 	    && (packet->payload[command_start + 1] == 'O' || packet->payload[command_start + 1] == 'o')
 	    && (packet->payload[command_start + 2] == 'G' || packet->payload[command_start + 2] == 'g')
 	    && (packet->payload[command_start + 3] == 'I' || packet->payload[command_start + 3] == 'i')
 	    && (packet->payload[command_start + 4] == 'N' || packet->payload[command_start + 4] == 'n')) {
+	  /* xxxx LOGIN "username" "password" */
+	  char str[256], *item;
+	  u_int len = packet->payload_packet_len > sizeof(str) ? sizeof(str) : packet->payload_packet_len;
+	  
+	  strncpy(str, (const char*)packet->payload, len);
+	  str[len] = '\0';
+
+	  item = strchr(str, '"');
+	  if(item) {
+	    char *column;
+	    
+	    item++;
+	    column = strchr(item, '"');
+
+	    if(column) {
+	      column[0] = '\0';
+	      snprintf(flow->protos.imap.username,
+		       sizeof(flow->protos.imap.username),
+		       "%s", item);
+
+	      column = strchr(&column[1], '"');
+	      if(column) {
+		item = &column[1];
+		column = strchr(item, '"');
+
+		if(column) {
+		  column[0] = '\0';
+		  snprintf(flow->protos.imap.password,
+			   sizeof(flow->protos.imap.password),
+			   "%s", item);
+		}
+	      }
+	    }
+	  }
+	  
 	  flow->l4.tcp.mail_imap_stage += 1;
 	  saw_command = 1;
-	} else if ((packet->payload[command_start] == 'F' || packet->payload[command_start] == 'f')
+	} else if((packet->payload[command_start] == 'F' || packet->payload[command_start] == 'f')
 		   && (packet->payload[command_start + 1] == 'E' || packet->payload[command_start + 1] == 'e')
 		   && (packet->payload[command_start + 2] == 'T' || packet->payload[command_start + 2] == 't')
 		   && (packet->payload[command_start + 3] == 'C' || packet->payload[command_start + 3] == 'c')
 		   && (packet->payload[command_start + 4] == 'H' || packet->payload[command_start + 4] == 'h')) {
 	  flow->l4.tcp.mail_imap_stage += 1;
 	  saw_command = 1;
-	} else if ((packet->payload[command_start] == 'F' || packet->payload[command_start] == 'f')
+	} else if((packet->payload[command_start] == 'F' || packet->payload[command_start] == 'f')
 		   && (packet->payload[command_start + 1] == 'L' || packet->payload[command_start + 1] == 'l')
 		   && (packet->payload[command_start + 2] == 'A' || packet->payload[command_start + 2] == 'a')
 		   && (packet->payload[command_start + 3] == 'G' || packet->payload[command_start + 3] == 'g')
 		   && (packet->payload[command_start + 4] == 'S' || packet->payload[command_start + 4] == 's')) {
 	  flow->l4.tcp.mail_imap_stage += 1;
 	  saw_command = 1;
-	} else if ((packet->payload[command_start] == 'C' || packet->payload[command_start] == 'c')
+	} else if((packet->payload[command_start] == 'C' || packet->payload[command_start] == 'c')
 		   && (packet->payload[command_start + 1] == 'H' || packet->payload[command_start + 1] == 'h')
 		   && (packet->payload[command_start + 2] == 'E' || packet->payload[command_start + 2] == 'e')
 		   && (packet->payload[command_start + 3] == 'C' || packet->payload[command_start + 3] == 'c')
 		   && (packet->payload[command_start + 4] == 'K' || packet->payload[command_start + 4] == 'k')) {
 	  flow->l4.tcp.mail_imap_stage += 1;
 	  saw_command = 1;
-	} else if ((packet->payload[command_start] == 'S' || packet->payload[command_start] == 's')
+	} else if((packet->payload[command_start] == 'S' || packet->payload[command_start] == 's')
 		   && (packet->payload[command_start + 1] == 'T' || packet->payload[command_start + 1] == 't')
 		   && (packet->payload[command_start + 2] == 'O' || packet->payload[command_start + 2] == 'o')
 		   && (packet->payload[command_start + 3] == 'R' || packet->payload[command_start + 3] == 'r')
@@ -186,8 +226,8 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
 	  saw_command = 1;
 	}
       }
-      if ((command_start + 12) < packet->payload_packet_len) {
-	if ((packet->payload[command_start] == 'A' || packet->payload[command_start] == 'a')
+      if((command_start + 12) < packet->payload_packet_len) {
+	if((packet->payload[command_start] == 'A' || packet->payload[command_start] == 'a')
 	    && (packet->payload[command_start + 1] == 'U' || packet->payload[command_start + 1] == 'u')
 	    && (packet->payload[command_start + 2] == 'T' || packet->payload[command_start + 2] == 't')
 	    && (packet->payload[command_start + 3] == 'H' || packet->payload[command_start + 3] == 'h')
@@ -203,8 +243,8 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
 	  saw_command = 1;
 	}
       }
-      if ((command_start + 9) < packet->payload_packet_len) {
-	if ((packet->payload[command_start] == 'N' || packet->payload[command_start] == 'n')
+      if((command_start + 9) < packet->payload_packet_len) {
+	if((packet->payload[command_start] == 'N' || packet->payload[command_start] == 'n')
 	    && (packet->payload[command_start + 1] == 'A' || packet->payload[command_start + 1] == 'a')
 	    && (packet->payload[command_start + 2] == 'M' || packet->payload[command_start + 2] == 'm')
 	    && (packet->payload[command_start + 3] == 'E' || packet->payload[command_start + 3] == 'e')
@@ -217,26 +257,26 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
 	  saw_command = 1;
 	}
       }
-      if ((command_start + 4) < packet->payload_packet_len) {
-	if ((packet->payload[command_start] == 'L' || packet->payload[command_start] == 'l')
+      if((command_start + 4) < packet->payload_packet_len) {
+	if((packet->payload[command_start] == 'L' || packet->payload[command_start] == 'l')
 	    && (packet->payload[command_start + 1] == 'S' || packet->payload[command_start + 1] == 's')
 	    && (packet->payload[command_start + 2] == 'U' || packet->payload[command_start + 2] == 'u')
 	    && (packet->payload[command_start + 3] == 'B' || packet->payload[command_start + 3] == 'b')) {
 	  flow->l4.tcp.mail_imap_stage += 1;
 	  saw_command = 1;
-	} else if ((packet->payload[command_start] == 'L' || packet->payload[command_start] == 'l')
+	} else if((packet->payload[command_start] == 'L' || packet->payload[command_start] == 'l')
 		   && (packet->payload[command_start + 1] == 'I' || packet->payload[command_start + 1] == 'i')
 		   && (packet->payload[command_start + 2] == 'S' || packet->payload[command_start + 2] == 's')
 		   && (packet->payload[command_start + 3] == 'T' || packet->payload[command_start + 3] == 't')) {
 	  flow->l4.tcp.mail_imap_stage += 1;
 	  saw_command = 1;
-	} else if ((packet->payload[command_start] == 'N' || packet->payload[command_start] == 'n')
+	} else if((packet->payload[command_start] == 'N' || packet->payload[command_start] == 'n')
 		   && (packet->payload[command_start + 1] == 'O' || packet->payload[command_start + 1] == 'o')
 		   && (packet->payload[command_start + 2] == 'O' || packet->payload[command_start + 2] == 'o')
 		   && (packet->payload[command_start + 3] == 'P' || packet->payload[command_start + 3] == 'p')) {
 	  flow->l4.tcp.mail_imap_stage += 1;
 	  saw_command = 1;
-	} else if ((packet->payload[command_start] == 'I' || packet->payload[command_start] == 'i')
+	} else if((packet->payload[command_start] == 'I' || packet->payload[command_start] == 'i')
 		   && (packet->payload[command_start + 1] == 'D' || packet->payload[command_start + 1] == 'd')
 		   && (packet->payload[command_start + 2] == 'L' || packet->payload[command_start + 2] == 'l')
 		   && (packet->payload[command_start + 3] == 'E' || packet->payload[command_start + 3] == 'e')) {
@@ -244,8 +284,8 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
 	  saw_command = 1;
 	}
       }
-      if ((command_start + 6) < packet->payload_packet_len) {
-	if ((packet->payload[command_start] == 'S' || packet->payload[command_start] == 's')
+      if((command_start + 6) < packet->payload_packet_len) {
+	if((packet->payload[command_start] == 'S' || packet->payload[command_start] == 's')
 	    && (packet->payload[command_start + 1] == 'E' || packet->payload[command_start + 1] == 'e')
 	    && (packet->payload[command_start + 2] == 'L' || packet->payload[command_start + 2] == 'l')
 	    && (packet->payload[command_start + 3] == 'E' || packet->payload[command_start + 3] == 'e')
@@ -253,7 +293,7 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
 	    && (packet->payload[command_start + 5] == 'T' || packet->payload[command_start + 5] == 't')) {
 	  flow->l4.tcp.mail_imap_stage += 1;
 	  saw_command = 1;
-	} else if ((packet->payload[command_start] == 'E' || packet->payload[command_start] == 'e')
+	} else if((packet->payload[command_start] == 'E' || packet->payload[command_start] == 'e')
 		   && (packet->payload[command_start + 1] == 'X' || packet->payload[command_start + 1] == 'x')
 		   && (packet->payload[command_start + 2] == 'I' || packet->payload[command_start + 2] == 'i')
 		   && (packet->payload[command_start + 3] == 'S' || packet->payload[command_start + 3] == 's')
@@ -261,7 +301,7 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
 		   && (packet->payload[command_start + 5] == 'S' || packet->payload[command_start + 5] == 's')) {
 	  flow->l4.tcp.mail_imap_stage += 1;
 	  saw_command = 1;
-	} else if ((packet->payload[command_start] == 'A' || packet->payload[command_start] == 'a')
+	} else if((packet->payload[command_start] == 'A' || packet->payload[command_start] == 'a')
 		   && (packet->payload[command_start + 1] == 'P' || packet->payload[command_start + 1] == 'p')
 		   && (packet->payload[command_start + 2] == 'P' || packet->payload[command_start + 2] == 'p')
 		   && (packet->payload[command_start + 3] == 'E' || packet->payload[command_start + 3] == 'e')
@@ -274,16 +314,23 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
 
     }
 
-    if (saw_command == 1) {
-      if (flow->l4.tcp.mail_imap_stage == 3 || flow->l4.tcp.mail_imap_stage == 5) {
-	NDPI_LOG_INFO(ndpi_struct, "found MAIL_IMAP\n");
-	ndpi_int_mail_imap_add_connection(ndpi_struct, flow);
+    if(saw_command == 1) {
+      if((flow->l4.tcp.mail_imap_stage == 3)
+	 || (flow->l4.tcp.mail_imap_stage == 5)
+	 || (flow->l4.tcp.mail_imap_stage == 7)
+	 ) {
+	if((flow->protos.imap.username[0] != '\0')
+	   || (flow->l4.tcp.mail_imap_stage >= 7)) {
+	  NDPI_LOG_INFO(ndpi_struct, "found MAIL_IMAP\n");
+	  ndpi_int_mail_imap_add_connection(ndpi_struct, flow);
+	}
+	
 	return;
       }
     }
   }
 
-  if (packet->payload_packet_len > 1 && packet->payload[packet->payload_packet_len - 1] == ' ') {
+  if(packet->payload_packet_len > 1 && packet->payload[packet->payload_packet_len - 1] == ' ') {
     NDPI_LOG_DBG2(ndpi_struct,
 	     "maybe a split imap command -> need next packet and imap_stage is set to 4.\n");
     flow->l4.tcp.mail_imap_stage = 4;
@@ -294,7 +341,7 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
 
   // skip over possible authentication hashes etc. that cannot be identified as imap commands or responses
   // if the packet count is low enough and at least one command or response was seen before
-  if ((packet->payload_packet_len >= 2 && ntohs(get_u_int16_t(packet->payload, packet->payload_packet_len - 2)) == 0x0d0a)
+  if((packet->payload_packet_len >= 2 && ntohs(get_u_int16_t(packet->payload, packet->payload_packet_len - 2)) == 0x0d0a)
       && flow->packet_counter < 6 && flow->l4.tcp.mail_imap_stage >= 1) {
     NDPI_LOG_DBG2(ndpi_struct,
 	     "no imap command or response but packet count < 6 and imap stage >= 1 -> skip\n");