Add support to opportunistic TLS

A lot of protocols provide the feature to upgrade their plain text
connections to an encrypted one, via some kind of "STARTTLS" command.

Add generic code to support this extension, and allow dissection of the
entire TLS handshake.

As examples, SMTP, POP, IMAP and FTP dissectors have been updated.

Since this feature requires to process more packets per flow, add the
possibility to disable it.

Fix some log messages.

Slight improvement on TCP sequence number tracking.

As a side effect, this commit fix also a memory leak found by
oss-fuzzer
```
==108966==ERROR: LeakSanitizer: detected memory leaks

Direct leak of 22 byte(s) in 1 object(s) allocated from:
    #0 0x55f8b367a0be in malloc (/home/ivan/svnrepos/nDPI/fuzz/fuzz_ndpi_reader_with_main+0x5480be) (BuildId: 94debacb4a6784c30420ab748c8bf3cc59621063)
    #1 0x55f8b36e1345 in ndpi_malloc_wrapper /home/ivan/svnrepos/nDPI/example/reader_util.c:321:10
    #2 0x55f8b379c7d2 in ndpi_malloc /home/ivan/svnrepos/nDPI/src/lib/ndpi_main.c:212:25
    #3 0x55f8b379cb18 in ndpi_strdup /home/ivan/svnrepos/nDPI/src/lib/ndpi_main.c:279:13
    #4 0x55f8b386ce46 in processClientServerHello /home/ivan/svnrepos/nDPI/src/lib/protocols/tls.c:2153:34
    #5 0x55f8b385ebf7 in processTLSBlock /home/ivan/svnrepos/nDPI/src/lib/protocols/tls.c:867:5
    #6 0x55f8b39e708c in ndpi_extra_search_mail_smtp_tcp /home/ivan/svnrepos/nDPI/src/lib/protocols/mail_smtp.c:422:9
    #7 0x55f8b37e636c in ndpi_process_extra_packet /home/ivan/svnrepos/nDPI/src/lib/ndpi_main.c:5884:9
    #8 0x55f8b37edc05 in ndpi_detection_process_packet /home/ivan/svnrepos/nDPI/src/lib/ndpi_main.c:6276:5
    #9 0x55f8b3701ffc in packet_processing /home/ivan/svnrepos/nDPI/example/reader_util.c:1619:31
    #10 0x55f8b36faf14 in ndpi_workflow_process_packet /home/ivan/svnrepos/nDPI/example/reader_util.c:2189:10
    #11 0x55f8b36b6a50 in LLVMFuzzerTestOneInput /home/ivan/svnrepos/nDPI/fuzz/fuzz_ndpi_reader.c:107:7

```
See: https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=50765

5 files changed, 115 insertions, 50 deletions

diff --git a/src/lib/protocols/ftp_control.c b/src/lib/protocols/ftp_control.c
index a0bec3864..44911b2d2 100644
--- a/src/lib/protocols/ftp_control.c
+++ b/src/lib/protocols/ftp_control.c
@@ -29,6 +29,9 @@
 
 // #define FTP_DEBUG
 
+extern void switch_extra_dissection_to_tls(struct ndpi_detection_module_struct *ndpi_struct,
+					   struct ndpi_flow_struct *flow);
+
 /* *************************************************************** */
 
 static void ndpi_int_ftp_control_add_connection(struct ndpi_detection_module_struct *ndpi_struct,
@@ -643,10 +646,21 @@ static void ndpi_check_ftp_control(struct ndpi_detection_module_struct *ndpi_str
 
 	if(flow->l4.tcp.ftp_imap_pop_smtp.password[0] == '\0' &&
 	   flow->l4.tcp.ftp_imap_pop_smtp.auth_done == 0 &&
-	   flow->l4.tcp.ftp_imap_pop_smtp.auth_tls == 0) /* TODO: any values on dissecting TLS handshake? */
+	   flow->l4.tcp.ftp_imap_pop_smtp.auth_tls == 0) {
 	  flow->ftp_control_stage = 0;
-	else
+	} else if (flow->l4.tcp.ftp_imap_pop_smtp.auth_tls == 1 &&
+		   ndpi_struct->opportunistic_tls_ftp_enabled) {
+	  flow->host_server_name[0] = '\0'; /* Remove any data set by other dissectors (eg. SMTP) */
+	  /* Switch classification to FTPS */
+	  ndpi_set_detected_protocol(ndpi_struct, flow,
+				     NDPI_PROTOCOL_FTPS, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI);
+	  NDPI_LOG_DBG(ndpi_struct, "Switching to [%d/%d]\n",
+		       flow->detected_protocol_stack[0], flow->detected_protocol_stack[1]);
+	  /* We are done (in FTP dissector): delegating TLS... */
+	  switch_extra_dissection_to_tls(ndpi_struct, flow);
+	} else {
 	  ndpi_int_ftp_control_add_connection(ndpi_struct, flow);
+	}
       } else {
 	NDPI_LOG_DBG2(ndpi_struct, "The reply did not seem to belong to FTP_CONTROL, "
 		      "resetting the stage to 0\n");
diff --git a/src/lib/protocols/mail_imap.c b/src/lib/protocols/mail_imap.c
index a6809b454..2ae04f24b 100644
--- a/src/lib/protocols/mail_imap.c
+++ b/src/lib/protocols/mail_imap.c
@@ -30,6 +30,9 @@
 
 /* #define IMAP_DEBUG 1*/
 
+extern void switch_extra_dissection_to_tls(struct ndpi_detection_module_struct *ndpi_struct,
+					   struct ndpi_flow_struct *flow);
+
 static void ndpi_int_mail_imap_add_connection(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow,
 					      u_int16_t protocol) {
   flow->guessed_protocol_id = NDPI_PROTOCOL_UNKNOWN; /* Avoid IMAPS to be used s sub-protocol */
@@ -51,13 +54,6 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
   printf("%s() [%.*s]\n", __FUNCTION__, packet->payload_packet_len, packet->payload);
 #endif
 
-  if(flow->l4.tcp.mail_imap_starttls == 2) {
-    NDPI_LOG_DBG2(ndpi_struct, "starttls detected\n");
-    NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_PROTOCOL_MAIL_IMAP);
-    NDPI_DEL_PROTOCOL_FROM_BITMASK(flow->excluded_protocol_bitmask, NDPI_PROTOCOL_TLS);
-    return;
-  }
-
   if(packet->payload_packet_len >= 4 && ntohs(get_u_int16_t(packet->payload, packet->payload_packet_len - 2)) == 0x0d0a) {
     // the DONE command appears without a tag
     if(packet->payload_packet_len == 6 && ((packet->payload[0] == 'D' || packet->payload[0] == 'd')
@@ -113,8 +109,17 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
 	    && (packet->payload[command_start + 1] == 'K' || packet->payload[command_start + 1] == 'k')
 	    && packet->payload[command_start + 2] == ' ') {
 	  flow->l4.tcp.mail_imap_stage += 1;
-	  if(flow->l4.tcp.mail_imap_starttls == 1)
-	    flow->l4.tcp.mail_imap_starttls = 2;
+	  if(flow->l4.tcp.mail_imap_starttls == 1) {
+	    NDPI_LOG_DBG2(ndpi_struct, "starttls detected\n");
+	    ndpi_int_mail_imap_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_MAIL_IMAPS);
+	    if(ndpi_struct->opportunistic_tls_imap_enabled) {
+	      NDPI_LOG_DBG(ndpi_struct, "Switching to [%d/%d]\n",
+			   flow->detected_protocol_stack[0], flow->detected_protocol_stack[1]);
+	      /* We are done (in IMAP dissector): delegating TLS... */
+	      switch_extra_dissection_to_tls(ndpi_struct, flow);
+	      return;
+	    }
+	  }
 	  saw_command = 1;
 	} else if((packet->payload[command_start] == 'U' || packet->payload[command_start] == 'u')
 		   && (packet->payload[command_start + 1] == 'I' || packet->payload[command_start + 1] == 'i')
@@ -126,7 +131,7 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
 	    && packet->payload[command_start + 2] == ' ') {
 	  flow->l4.tcp.mail_imap_stage += 1;
 	  if(flow->l4.tcp.mail_imap_starttls == 1)
-	    flow->l4.tcp.mail_imap_starttls = 2;
+	    flow->l4.tcp.mail_imap_starttls = 0;
 	  saw_command = 1;
 	}
       }
@@ -156,7 +161,6 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
 	    && (packet->payload[command_start + 7] == 'S' || packet->payload[command_start + 7] == 's')) {
         flow->l4.tcp.mail_imap_stage += 1;
         flow->l4.tcp.mail_imap_starttls = 1;
-        ndpi_int_mail_imap_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_MAIL_IMAPS);
         saw_command = 1;
 	}
       }
@@ -242,7 +246,6 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
 	  flow->l4.tcp.mail_imap_stage += 1;
 	  /* Authenticate phase may have multiple messages. Ignore them since they are
 	     somehow encrypted anyway. */
-	  flow->l4.tcp.mail_imap_starttls = 2;
           ndpi_int_mail_imap_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_MAIL_IMAPS);
 	  saw_command = 1;
 	}
diff --git a/src/lib/protocols/mail_pop.c b/src/lib/protocols/mail_pop.c
index f0a1731b3..1474af1be 100644
--- a/src/lib/protocols/mail_pop.c
+++ b/src/lib/protocols/mail_pop.c
@@ -43,12 +43,16 @@
 #define POP_BIT_STLS		0x0400
 
 
+extern void switch_extra_dissection_to_tls(struct ndpi_detection_module_struct *ndpi_struct,
+					   struct ndpi_flow_struct *flow);
+
 static void ndpi_int_mail_pop_add_connection(struct ndpi_detection_module_struct
-					     *ndpi_struct, struct ndpi_flow_struct *flow) {
+					     *ndpi_struct, struct ndpi_flow_struct *flow,
+					     u_int16_t protocol) {
 
   NDPI_LOG_INFO(ndpi_struct, "mail_pop identified\n");
   flow->guessed_protocol_id = NDPI_PROTOCOL_UNKNOWN; /* Avoid POP3S to be used s sub-protocol */
-  ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_MAIL_POP, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI);
+  ndpi_set_detected_protocol(ndpi_struct, flow, protocol, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI);
 }
 
 /* **************************************** */
@@ -142,6 +146,7 @@ static int ndpi_int_mail_pop_check_for_client_commands(struct ndpi_detection_mod
 	       && (packet->payload[2] == 'L' || packet->payload[2] == 'l')
 	       && (packet->payload[3] == 'S' || packet->payload[3] == 's')) {
       flow->l4.tcp.pop_command_bitmask |= POP_BIT_STLS;
+      flow->l4.tcp.mail_imap_starttls = 1;
       return 1;
     }
   }
@@ -168,6 +173,19 @@ void ndpi_search_mail_pop_tcp(struct ndpi_detection_module_struct
 	      && (packet->payload[3] == 'R' || packet->payload[3] == 'r')))) {
     // +OK or -ERR seen
     flow->l4.tcp.mail_pop_stage += 1;
+    if(packet->payload[0] == '+' && flow->l4.tcp.mail_imap_starttls == 1) {
+      NDPI_LOG_DBG2(ndpi_struct, "starttls detected\n");
+      ndpi_int_mail_pop_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_MAIL_POPS);
+      if(ndpi_struct->opportunistic_tls_pop_enabled) {
+        NDPI_LOG_DBG(ndpi_struct, "Switching to [%d/%d]\n",
+		     flow->detected_protocol_stack[0], flow->detected_protocol_stack[1]);
+	/* We are done (in POP dissector): delegating TLS... */
+	switch_extra_dissection_to_tls(ndpi_struct, flow);
+	return;
+      }
+    }
+    if(packet->payload[0] == '-' && flow->l4.tcp.mail_imap_starttls == 1)
+      flow->l4.tcp.mail_imap_starttls = 0;
   } else if(!ndpi_int_mail_pop_check_for_client_commands(ndpi_struct, flow)) {
     goto maybe_split_pop;
   }
@@ -189,7 +207,7 @@ void ndpi_search_mail_pop_tcp(struct ndpi_detection_module_struct
 	
 	if((flow->l4.tcp.ftp_imap_pop_smtp.password[0] != '\0')
 	   || (flow->l4.tcp.mail_pop_stage > 3)) {
-	  ndpi_int_mail_pop_add_connection(ndpi_struct, flow);
+	  ndpi_int_mail_pop_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_MAIL_POP);
 	  if(flow->l4.tcp.ftp_imap_pop_smtp.password[0] == '\0')
 	    popInitExtraPacketProcessing(flow);
 	}
diff --git a/src/lib/protocols/mail_smtp.c b/src/lib/protocols/mail_smtp.c
index 31f07c1c0..f6b0af060 100644
--- a/src/lib/protocols/mail_smtp.c
+++ b/src/lib/protocols/mail_smtp.c
@@ -48,8 +48,8 @@
 
 /* #define SMTP_DEBUG 1 */
 
-extern int processTLSBlock(struct ndpi_detection_module_struct *ndpi_struct,
-                           struct ndpi_flow_struct *flow);
+extern void switch_extra_dissection_to_tls(struct ndpi_detection_module_struct *ndpi_struct,
+					   struct ndpi_flow_struct *flow);
 
 static void ndpi_int_mail_smtp_add_connection(struct ndpi_detection_module_struct
 					      *ndpi_struct, struct ndpi_flow_struct *flow) {
@@ -153,6 +153,7 @@ void ndpi_search_mail_smtp_tcp(struct ndpi_detection_module_struct *ndpi_struct,
 		  len = i-4;
 		  /* Copy result for nDPI apps */
 		  ndpi_hostname_sni_set(flow, &packet->line[a].ptr[4], len);
+		  NDPI_LOG_DBG(ndpi_struct, "SMTP: hostname [%s]\n", flow->host_server_name);
 
 		  if (ndpi_match_hostname_protocol(ndpi_struct, flow, NDPI_PROTOCOL_MAIL_SMTP,
 						   flow->host_server_name,
@@ -406,38 +407,40 @@ int ndpi_extra_search_mail_smtp_tcp(struct ndpi_detection_module_struct *ndpi_st
 				    struct ndpi_flow_struct *flow)
 {
   struct ndpi_packet_struct * const packet = &ndpi_struct->packet;
-  int rc = 0;
+  int rc;
 
-  if (flow->l4.tcp.smtp_command_bitmask & SMTP_BIT_STARTTLS &&
-      packet->payload_packet_len > 5)
-  {
-    uint8_t const * const block = &packet->payload[5];
-    uint8_t const * const p = &packet->payload[0];
-    uint16_t const block_len = packet->payload_packet_len - 5;
-    uint16_t const l = packet->payload_packet_len;
+  if(flow->l4.tcp.smtp_command_bitmask & SMTP_BIT_STARTTLS) {
 
-    packet->payload = block;
-    packet->payload_packet_len = block_len;
+    /* RFC 3207:
+       "After the client gives the STARTTLS command, the server responds with
+        one of the following reply codes:
+         220 Ready to start TLS
+         501 Syntax error (no parameters allowed)
+         454 TLS not available due to temporary reason"
+    */
 
-    if (processTLSBlock(ndpi_struct, flow) != 0) {
+    if(ndpi_struct->opportunistic_tls_smtp_enabled &&
+       packet->payload_packet_len > 3 && memcmp(packet->payload, "220", 3) == 0) {
       rc = 1;
-    }
-
-    packet->payload = p;
-    packet->payload_packet_len = l;
-
-    /* STARTTLS may be followed by a 220 - Service ready */
-    if (rc == 0 && memcmp(packet->payload, "220", 3) != 0)
-    {
-      flow->l4.tcp.ftp_imap_pop_smtp.auth_done = 1;
-      if (flow->guessed_host_protocol_id == NDPI_PROTOCOL_UNKNOWN) {
-        ndpi_set_detected_protocol(ndpi_struct, flow,
-                                   NDPI_PROTOCOL_MAIL_SMTPS, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI);
+      /* Switch classification to SMTPS, keeping the hostname sub-classification (if any) */
+      if(flow->detected_protocol_stack[0] != NDPI_PROTOCOL_UNKNOWN &&
+	 flow->detected_protocol_stack[0] != NDPI_PROTOCOL_MAIL_SMTP) {
+	ndpi_set_detected_protocol(ndpi_struct, flow,
+                                   flow->detected_protocol_stack[0], NDPI_PROTOCOL_MAIL_SMTPS, NDPI_CONFIDENCE_DPI);
+	/* Now it is safe to write to `flow->protos.tls_quic` union */
+	flow->protos.tls_quic.subprotocol_detected = 1;
       } else {
         ndpi_set_detected_protocol(ndpi_struct, flow,
-                                   flow->guessed_host_protocol_id, NDPI_PROTOCOL_MAIL_SMTPS, NDPI_CONFIDENCE_DPI);
+                                   NDPI_PROTOCOL_MAIL_SMTPS, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI);
       }
+      NDPI_LOG_DBG(ndpi_struct, "Switching to [%d/%d]\n",
+                   flow->detected_protocol_stack[0], flow->detected_protocol_stack[1]);
+      /* We are done (in SMTP dissector): delegating TLS... */
+      switch_extra_dissection_to_tls(ndpi_struct, flow);
+    } else {
+      rc = 0; /* Something went wrong. Stop extra dissection */
     }
+
   } else {
     ndpi_search_mail_smtp_tcp(ndpi_struct, flow);
     rc = ((flow->l4.tcp.ftp_imap_pop_smtp.password[0] == '\0') &&
diff --git a/src/lib/protocols/tls.c b/src/lib/protocols/tls.c
index 53245a21e..98a8d8208 100644
--- a/src/lib/protocols/tls.c
+++ b/src/lib/protocols/tls.c
@@ -28,8 +28,6 @@
 #include "ndpi_encryption.h"
 
 extern char *strptime(const char *s, const char *format, struct tm *tm);
-extern int processTLSBlock(struct ndpi_detection_module_struct *ndpi_struct,
-                           struct ndpi_flow_struct *flow);
 extern int processClientServerHello(struct ndpi_detection_module_struct *ndpi_struct,
 				    struct ndpi_flow_struct *flow, uint32_t quic_version);
 extern int http_process_user_agent(struct ndpi_detection_module_struct *ndpi_struct,
@@ -221,7 +219,7 @@ void ndpi_search_tls_tcp_memory(struct ndpi_detection_module_struct *ndpi_struct
 	     message->buffer_len,
 	     packet->packet_direction,
 	     ntohl(packet->tcp->seq),
-	     ntohl(packet->tcp->seq)+packet->payload_packet_len);
+	     message->next_seq);
 #endif
     }
   }
@@ -852,8 +850,8 @@ int processCertificate(struct ndpi_detection_module_struct *ndpi_struct,
 
 /* **************************************** */
 
-int processTLSBlock(struct ndpi_detection_module_struct *ndpi_struct,
-                    struct ndpi_flow_struct *flow) {
+static int processTLSBlock(struct ndpi_detection_module_struct *ndpi_struct,
+                           struct ndpi_flow_struct *flow) {
   struct ndpi_packet_struct *packet = &ndpi_struct->packet;
   int ret;
 
@@ -924,8 +922,17 @@ static int ndpi_search_tls_tcp(struct ndpi_detection_module_struct *ndpi_struct,
 	 packet->payload_packet_len);
 #endif
 
-  if(packet->payload_packet_len == 0)
-    return(1); /* Keep working */
+  /* This function is also called by "extra dissection" data path. Unfortunately,
+     generic "extra function" code doesn't honour protocol bitmask.
+     TODO: handle that in ndpi_main.c for all the protocols */
+  if(packet->payload_packet_len == 0 ||
+     packet->tcp_retransmission) {
+#ifdef DEBUG_TLS_MEMORY
+    printf("[TLS Mem] Ack or retransmission %d/%d. Skip\n",
+           packet->payload_packet_len, packet->tcp_retransmission);
+#endif
+    return 1; /* Keep working */
+  }
 
   ndpi_search_tls_tcp_memory(ndpi_struct, flow);
   message = &flow->l4.tcp.tls.message[packet->packet_direction];
@@ -1224,6 +1231,26 @@ static void tlsInitExtraPacketProcessing(struct ndpi_detection_module_struct *nd
 
 /* **************************************** */
 
+void switch_extra_dissection_to_tls(struct ndpi_detection_module_struct *ndpi_struct,
+				    struct ndpi_flow_struct *flow)
+{
+#ifdef DEBUG_TLS
+  printf("Switching to TLS extra dissection\n");
+#endif
+
+  /* Reset reassemblers */
+  if(flow->l4.tcp.tls.message[0].buffer)
+    ndpi_free(flow->l4.tcp.tls.message[0].buffer);
+  memset(&flow->l4.tcp.tls.message[0], '\0', sizeof(flow->l4.tcp.tls.message[0]));
+  if(flow->l4.tcp.tls.message[1].buffer)
+    ndpi_free(flow->l4.tcp.tls.message[1].buffer);
+  memset(&flow->l4.tcp.tls.message[1], '\0', sizeof(flow->l4.tcp.tls.message[1]));
+
+  tlsInitExtraPacketProcessing(ndpi_struct, flow);
+}
+
+/* **************************************** */
+
 static void tlsCheckUncommonALPN(struct ndpi_detection_module_struct *ndpi_struct,
 				 struct ndpi_flow_struct *flow) {
   char * alpn_start = flow->protos.tls_quic.alpn;