Fix writes to `flow->protos` union fields (#1354)

We can write to `flow->protos` only after a proper classification.

This issue has been found in Kerberos, DHCP, HTTP, STUN, IMO, FTP,
SMTP, IMAP and POP code.
There are two kinds of fixes:
 * write to `flow->protos` only if a final protocol has been detected
 * move protocol state out of `flow->protos`
The hard part is to find, for each protocol, the right tradeoff between
memory usage and code complexity.

Handle Kerberos like DNS: if we find a request, we set the protocol
and an extra callback to further parsing the reply.

For all the other protocols, move the state out of `flow->protos`. This
is an issue only for the FTP/MAIL stuff.

Add DHCP Class Identification value to the output of ndpiReader and to
the Jason serialization.

Extend code coverage of fuzz tests.

Close #1343
Close #1342

2 files changed, 35 insertions, 46 deletions

diff --git a/python/ndpi.py b/python/ndpi.py
index 48103e777..a31a41a17 100644
--- a/python/ndpi.py
+++ b/python/ndpi.py
@@ -644,6 +644,13 @@ struct ndpi_flow_udp_struct {
   /* NDPI_PROTOCOL_CSGO */
   uint8_t csgo_strid[18],csgo_state,csgo_s2;
   uint32_t csgo_id2;
+
+  /* NDPI_PROTOCOL_RDP */
+  u_int8_t rdp_to_srv[3], rdp_from_srv[3], rdp_to_srv_pkts, rdp_from_srv_pkts;
+
+  /* NDPI_PROTOCOL_IMO */
+  uint8_t imo_last_one_byte_pkt, imo_last_byte;
+
 };
 
 struct ndpi_int_one_line_struct {
@@ -999,6 +1006,7 @@ struct ndpi_flow_struct {
     uint8_t request_version; /* 0=1.0 and 1=1.1. Create an enum for this? */
     uint16_t response_status_code; /* 200, 404, etc. */
     uint8_t detected_os[32]; /* Via HTTP/QUIC User-Agent */
+    uint8_t nat_ip[24];
 
   } http;
 
@@ -1011,6 +1019,18 @@ struct ndpi_flow_struct {
     char *pktbuf;
     uint16_t pktbuf_maxlen, pktbuf_currlen;
   } kerberos_buf;
+
+  struct {
+    u_int8_t num_udp_pkts, num_binding_requests;
+    u_int16_t num_processed_pkts;
+  } stun;
+
+  /* TODO: something clever to save memory */
+  struct {
+      uint8_t auth_found:1, auth_failed:1, auth_tls:1, auth_done:1, _pad:4;
+      char username[32], password[16];
+  } ftp_imap_pop_smtp;
+
   union {
     /* the only fields useful for nDPI and ntopng */
     struct {
@@ -1029,7 +1049,6 @@ struct ndpi_flow_struct {
     } kerberos;
 
     struct {
-      struct {
       char ssl_version_str[12];
       uint16_t ssl_version, server_names_len;
       char client_requested_server_name[64], *server_names,
@@ -1044,14 +1063,7 @@ struct ndpi_flow_struct {
         char *esni;
       } encrypted_sni;
       ndpi_cipher_weakness server_unsafe_cipher;
-      } ssl;
-
-      struct {
-      uint8_t num_udp_pkts, num_processed_pkts, num_binding_requests;
-      } stun;
-
-      /* We can have STUN over SSL/TLS thus they need to live together */
-    } stun_ssl;
+    } tls_quic;
 
     struct {
       char client_signature[48], server_signature[48];
@@ -1059,10 +1071,6 @@ struct ndpi_flow_struct {
     } ssh;
 
     struct {
-      uint8_t last_one_byte_pkt, last_byte;
-    } imo;
-
-    struct {
       uint8_t username_detected:1, username_found:1,
       password_detected:1, password_found:1,
       pad:4;
@@ -1079,16 +1087,6 @@ struct ndpi_flow_struct {
     } ubntac2;
 
     struct {
-      /* Via HTTP X-Forwarded-For */
-      uint8_t nat_ip[24];
-    } http;
-
-    struct {
-      uint8_t auth_found:1, auth_failed:1, auth_tls:1, _pad:5;
-      char username[16], password[16];
-    } ftp_imap_pop_smtp;
-
-    struct {
       /* Bittorrent hash */
       uint8_t hash[20];
     } bittorrent;
diff --git a/python/ndpi_typestruct.py b/python/ndpi_typestruct.py
index 606bf6576..09c1f7d7e 100644
--- a/python/ndpi_typestruct.py
+++ b/python/ndpi_typestruct.py
@@ -406,6 +406,12 @@ class NDPIFlowUdpStruct(Structure):
         ('csgo_state', c_uint8),
         ('csgo_s2', c_uint8),
         ('csgo_id2', c_uint32),
+        ('rdp_to_srv', c_uint8 * 3),
+        ('rdp_from_srv', c_uint8 * 3),
+        ('rdp_to_srv_pkts,', c_uint8),
+        ('rdp_from_srv_pkts', c_uint8),
+        ('imo_last_one_byte_pkt,', c_uint8),
+        ('imo_last_byte', c_uint8),
     ]
 
 
@@ -423,6 +429,7 @@ class Http(Structure):
         ("request_version", c_uint8),
         ("response_status_code", c_uint16),
         ("detected_os", c_char * 32),
+        ("nat_ip", c_char * 24),
     ]
 
 
@@ -449,7 +456,7 @@ class Kerberos(Structure):
                 ("realm", c_char * 24)]
 
 
-class Ssl(Structure):
+class QuicSsl(Structure):
     _fields_ = [
         ("ssl_version", c_uint16),
         ("client_certificate", c_char * 64),
@@ -472,10 +479,6 @@ class Stun(Structure):
     ]
 
 
-class StunSsl(Structure):
-    _fields_ = [("ssl", Ssl), ("stun", Stun)]
-
-
 class Ssh(Structure):
     _fields_ = [
         ("client_signature", c_char * 48),
@@ -485,13 +488,6 @@ class Ssh(Structure):
     ]
 
 
-class Imo(Structure):
-    _fields_ = [
-        ("last_one_byte_pkt", c_uint8),
-        ("last_byte", c_uint8)
-    ]
-
-
 class Mdns(Structure):
     _fields_ = [("answer", c_char * 96)]
 
@@ -500,18 +496,14 @@ class Ubntac2(Structure):
     _fields_ = [("version", c_char * 32)]
 
 
-class Http2(Structure):
-    _fields_ = [
-        ("nat_ip", c_char * 24)
-    ]
-
 class FtpImapPopSmtp(Structure):
     _fields_ = [
         ("auth_found", c_uint8, 1),
         ("auth_failed", c_uint8, 1),
         ("auth_tls", c_uint8, 1),
-        ("_pad", c_uint8, 5),
-        ("username", c_char * 16),
+        ("auth_done", c_uint8, 1),
+        ("_pad", c_uint8, 4),
+        ("username", c_char * 32),
         ("password", c_char * 16)
     ]
 
@@ -530,13 +522,10 @@ class Protos(Union):
     _fields_ = [
         ("dns", Dns),
         ("kerberos", Kerberos),
-        ("stun_ssl", StunSsl),
+        ("quic_ssl", QuicSsl),
         ("ssh", Ssh),
-        ("imo", Imo),
         ("mdns", Mdns),
         ("ubntac2", Ubntac2),
-        ("http", Http2),
-        ("ftp_imap_pop_smtp", FtpImapPopSmtp),
         ("bittorrent", Bittorrent),
         ("dhcp", Dhcp)
     ]
@@ -699,6 +688,8 @@ NDPIFlowStruct._fields_ = [
     ("l4", L4),
     ("host_server_name", c_ubyte * 256),
     ("http", Http),
+    ("stun", Stun),
+    ("ftp_imap_pop_smtp", FtpImapPopSmtp),
     ("protos", Protos),
     ("excluded_protocol_bitmask", NDPIProtocolBitMask),
     ("category", c_int),