A (tiny) effort to reduce memory usage per flow (#1304)

Follow-up of 22241a1d
Only trivial changes:
* remove completely unused fields
* remove fields only written (but never read)
* CSGO protocol only handles UDP traffic

2 files changed, 6 insertions, 119 deletions

diff --git a/python/ndpi.py b/python/ndpi.py
index db87633ac..a7b568451 100644
--- a/python/ndpi.py
+++ b/python/ndpi.py
@@ -441,8 +441,6 @@ struct ndpi_id_struct {
      to compare this, use:
   **/
   NDPI_PROTOCOL_BITMASK detected_protocol_bitmask;
-  /* NDPI_PROTOCOL_RTSP */
-  ndpi_ip_addr_t rtsp_ip_address;
 
   /* NDPI_PROTOCOL_IRC_MAXPORT % 2 must be 0 */
   /* NDPI_PROTOCOL_IRC */
@@ -457,9 +455,6 @@ struct ndpi_id_struct {
   /* NDPI_PROTOCOL_THUNDER */
   uint32_t thunder_ts;
 
-  /* NDPI_PROTOCOL_RTSP */
-  uint32_t rtsp_timer;
-
   /* NDPI_PROTOCOL_ZATTOO */
   uint32_t zattoo_ts;
 
@@ -494,8 +489,6 @@ struct ndpi_id_struct {
 
   /* NDPI_PROTOCOL_UNENCRYPTED_JABBER */
   uint8_t jabber_voice_stun_used_ports;
-
-  uint8_t rtsp_ts_set:1;
 };
 
 struct ndpi_flow_tcp_struct {
@@ -505,15 +498,9 @@ struct ndpi_flow_tcp_struct {
   /* NDPI_PROTOCOL_MAIL_POP */
   uint16_t pop_command_bitmask;
 
-  /* NDPI_PROTOCOL_QQ */
-  uint16_t qq_nxt_len;
-
   /* NDPI_PROTOCOL_WHATSAPP */
   uint8_t wa_matched_so_far;
 
-  /* NDPI_PROTOCOL_TDS */
-  uint8_t tds_login_version;
-
   /* NDPI_PROTOCOL_IRC */
   uint8_t irc_stage;
   uint8_t irc_port;
@@ -533,35 +520,16 @@ struct ndpi_flow_tcp_struct {
   /* NDPI_PROTOCOL_SOULSEEK */
   uint32_t soulseek_stage:2;
 
-  /* NDPI_PROTOCOL_TDS */
-  uint32_t tds_stage:3;
-
   /* NDPI_PROTOCOL_USENET */
   uint32_t usenet_stage:2;
 
-  /* NDPI_PROTOCOL_IMESH */
-  uint32_t imesh_stage:4;
-
   /* NDPI_PROTOCOL_HTTP */
-  uint32_t http_setup_dir:2;
   uint32_t http_stage:2;
   uint32_t http_empty_line_seen:1;
-  uint32_t http_wait_for_retransmission:1;
 
   /* NDPI_PROTOCOL_GNUTELLA */
   uint32_t gnutella_stage:2;		       // 0 - 2
 
-  /* NDPI_CONTENT_MMS */
-  uint32_t mms_stage:2;
-
-  /* NDPI_PROTOCOL_YAHOO */
-  uint32_t yahoo_sip_comm:1;
-  uint32_t yahoo_http_proxy_stage:2;
-
-  /* NDPI_PROTOCOL_MSN */
-  uint32_t msn_stage:3;
-  uint32_t msn_ssl_ft:2;
-
   /* NDPI_PROTOCOL_SSH */
   uint32_t ssh_stage:3;
 
@@ -577,8 +545,6 @@ struct ndpi_flow_tcp_struct {
       unsigned buffer_len, buffer_used;
     } message;
 
-    void* srv_cert_fingerprint_ctx; /* SHA-1 */
-
     /* NDPI_PROTOCOL_TLS */
     uint8_t certificate_processed:1, fingerprint_set:1, _pad:6;
     uint8_t sha1_certificate_fingerprint[20], num_tls_blocks;
@@ -588,8 +554,7 @@ struct ndpi_flow_tcp_struct {
   /* NDPI_PROTOCOL_POSTGRES */
   uint32_t postgres_stage:3;
 
-  /* NDPI_PROTOCOL_DIRECT_DOWNLOAD_LINK */
-  uint32_t ddlink_server_direction:1;
+  /* Part of the TCP header. */
   uint32_t seen_syn:1;
   uint32_t seen_syn_ack:1;
   uint32_t seen_ack:1;
@@ -606,9 +571,6 @@ struct ndpi_flow_tcp_struct {
   /* NDPI_PROTOCOL_WORLDOFWARCRAFT */
   uint32_t wow_stage:2;
 
-  /* NDPI_PROTOCOL_HTTP_APPLICATION_VEOHTV */
-  uint32_t veoh_tv_stage:2;
-
   /* NDPI_PROTOCOL_SHOUTCAST */
   uint32_t shoutcast_stage:2;
 
@@ -648,12 +610,6 @@ struct ndpi_flow_tcp_struct {
 };
 
 struct ndpi_flow_udp_struct {
-  /* NDPI_PROTOCOL_SNMP */
-  uint32_t snmp_msg_id;
-
-  /* NDPI_PROTOCOL_SNMP */
-  uint32_t snmp_stage:2;
-
   /* NDPI_PROTOCOL_PPSTREAM */
   uint32_t ppstream_stage:3;		  // 0 - 7
 
@@ -669,9 +625,6 @@ struct ndpi_flow_udp_struct {
   /* NDPI_PROTOCOL_XBOX */
   uint32_t xbox_stage:1;
 
-  /* NDPI_PROTOCOL_WINDOWS_UPDATE */
-  uint32_t wsus_stage:1;
-
   /* NDPI_PROTOCOL_SKYPE */
   uint8_t skype_packet_id;
 
@@ -710,7 +663,6 @@ struct ndpi_packet_struct {
   uint64_t current_time_ms;
 
   uint16_t detected_protocol_stack[2];
-  uint16_t protocol_stack_info;
 
   struct ndpi_int_one_line_struct line[64];
   /* HTTP headers */
@@ -739,14 +691,12 @@ struct ndpi_packet_struct {
   uint16_t actual_payload_len;
   uint16_t num_retried_bytes;
   uint16_t parsed_lines;
-  uint16_t parsed_unix_lines;
   uint16_t empty_line_position;
   uint8_t tcp_retransmission;
   uint8_t l4_protocol;
 
-  uint8_t tls_certificate_detected:4, tls_certificate_num_checks:4;
   uint8_t packet_lines_parsed_complete:1,
-  packet_direction:1, empty_line_position_set:1, pad:5;
+  packet_direction:1, empty_line_position_set:1, http_check_content:1, pad:4;
 };
 
 struct ndpi_detection_module_struct;
@@ -962,13 +912,8 @@ struct ndpi_detection_module_struct {
   uint32_t thunder_timeout;
   /* SoulSeek parameters */
   uint32_t soulseek_connection_ip_tick_timeout;
-  /* rtsp parameters */
-  uint32_t rtsp_connection_timeout;
   /* rstp */
   uint32_t orb_rstp_ts_timeout;
-  /* yahoo */
-  uint8_t yahoo_detect_http_connections;
-  uint32_t yahoo_lan_video_timeout;
   uint32_t zattoo_connection_timeout;
   uint32_t jabber_stun_timeout;
   uint32_t jabber_file_transfer_timeout;
@@ -1010,7 +955,6 @@ typedef enum {
 
 struct ndpi_flow_struct {
   uint16_t detected_protocol_stack[2];
-  uint16_t protocol_stack_info;
   /* init parameter, internal used to set up timestamp,... */
   uint16_t guessed_protocol_id, guessed_host_protocol_id, guessed_category, guessed_header_category;
   uint8_t l4_proto, protocol_id_already_guessed:1, host_already_guessed:1,
@@ -1037,11 +981,6 @@ struct ndpi_flow_struct {
   /* Place textual flow info here */
   char flow_extra_info[16];
 
-  /*
-    Pointer to src or dst that identifies the
-    server of this connection
-  */
-  struct ndpi_id_struct *server_id;
   /* HTTP host or DNS query */
   uint8_t host_server_name[240];
   uint8_t initial_binary_bytes[8], initial_binary_bytes_len;
@@ -1181,24 +1120,15 @@ struct ndpi_flow_struct {
   /* NDPI_PROTOCOL_DIRECTCONNECT */
   uint8_t directconnect_stage:2;	      // 0 - 1
 
-  /* NDPI_PROTOCOL_YAHOO */
-  uint8_t sip_yahoo_voice:1;
-
   /* NDPI_PROTOCOL_HTTP */
   uint8_t http_detected:1;
 
   /* NDPI_PROTOCOL_RTSP */
-  uint8_t rtsprdt_stage:2, rtsp_control_flow:1;
-
-  /* NDPI_PROTOCOL_YAHOO */
-  uint8_t yahoo_detection_finished:2;
+  uint8_t rtsprdt_stage:2;
 
   /* NDPI_PROTOCOL_ZATTOO */
   uint8_t zattoo_stage:3;
 
-  /* NDPI_PROTOCOL_QQ */
-  uint8_t qq_stage:3;
-
   /* NDPI_PROTOCOL_THUNDER */
   uint8_t thunder_stage:2;		        // 0 - 3
 
@@ -1242,7 +1172,6 @@ struct ndpi_flow_struct {
   uint32_t csgo_id2;
   /* internal structures to save functions calls */
   struct ndpi_packet_struct packet;
-  struct ndpi_flow_struct *flow;
   struct ndpi_id_struct *src;
   struct ndpi_id_struct *dst;
 };
diff --git a/python/ndpi_typestruct.py b/python/ndpi_typestruct.py
index 20f77120b..9413ca039 100644
--- a/python/ndpi_typestruct.py
+++ b/python/ndpi_typestruct.py
@@ -262,11 +262,8 @@ NDPIDetectionModuleStruct._fields_ = [
     ("battlefield_timeout", c_uint32),
     ("thunder_timeout", c_uint32),
     ("soulseek_connection_ip_tick_timeout", c_uint32),
-    ("rtsp_connection_timeout", c_uint32),
     ("tvants_connection_timeout", c_uint32),
     ("orb_rstp_ts_timeout", c_uint32),
-    ("yahoo_detect_http_connections", c_uint8),
-    ("yahoo_lan_video_timeout", c_uint32),
     ("zattoo_connection_timeout", c_uint32),
     ("jabber_stun_timeout", c_uint32),
     ("jabber_file_transfer_timeout", c_uint32),
@@ -312,7 +309,6 @@ class NDPIIpAddrT(Union):
 class NDPIIdStruct(Structure):
     _fields_ = [
         ('detected_protocol_bitmask', NDPIProtocolBitMask),
-        ('rtsp_ip_address', NDPIIpAddrT),
         ('irc_port', c_uint16 * 8),
         ('last_time_port_used', c_uint32 * 8),
         ('irc_ts', c_uint32),
@@ -336,7 +332,6 @@ class NDPIIdStruct(Structure):
         ('irc_number_of_port', c_uint8),
         ('oscar_ssl_session_id', c_uint8 * 33),
         ('jabber_voice_stun_used_ports', c_uint8),
-        ('rtsp_ts_set', c_uint8, 1),
     ]
 
 
@@ -345,9 +340,7 @@ class NDPIFlowTcpStruct(Structure):
     _fields_ = [
         ('smtp_command_bitmask', c_uint16),
         ('pop_command_bitmask', c_uint16),
-        ('qq_nxt_len', c_uint16),
         ('wa_matched_so_far', c_uint8),
-        ('tds_login_version', c_uint8),
         ('irc_stage', c_uint8),
         ('irc_port', c_uint8),
         ('h323_valid_packets', c_uint8),
@@ -357,23 +350,13 @@ class NDPIFlowTcpStruct(Structure):
         ('irc_direction', c_uint32, 2),
         ('irc_0x1000_full', c_uint32, 1),
         ('soulseek_stage', c_uint32, 2),
-        ('tds_stage', c_uint32, 3),
         ('usenet_stage', c_uint32, 2),
-        ('imesh_stage', c_uint32, 4),
-        ('http_setup_dir', c_uint32, 2),
         ('http_stage', c_uint32, 2),
         ('http_empty_line_seen', c_uint32, 1),
-        ('http_wait_for_retransmission', c_uint32, 1),
         ('gnutella_stage', c_uint32, 2),
-        ('mms_stage', c_uint32, 2),
-        ('yahoo_sip_comm', c_uint32, 1),
-        ('yahoo_http_proxy_stage', c_uint32, 2),
-        ('msn_stage', c_uint32, 3),
-        ('msn_ssl_ft', c_uint32, 2),
         ('ssh_stage', c_uint32, 3),
         ('vnc_stage', c_uint32, 2),
         ('telnet_stage', c_uint32, 2),
-        ('tls_srv_cert_fingerprint_ctx', c_void_p),
         ('tls_seen_client_cert', c_uint8, 1),
         ('tls_seen_server_cert', c_uint8, 1),
         ('tls_seen_certificate', c_uint8, 1),
@@ -384,7 +367,6 @@ class NDPIFlowTcpStruct(Structure):
         ('tls_fingerprint_len', c_int16),
         ('tls_sha1_certificate_fingerprint', c_uint8 * 20),
         ('postgres_stage', c_uint32, 3),
-        ('ddlink_server_direction', c_uint32, 1),
         ('seen_syn', c_uint32, 1),
         ('seen_syn_ack', c_uint32, 1),
         ('seen_ack', c_uint32, 1),
@@ -392,7 +374,6 @@ class NDPIFlowTcpStruct(Structure):
         ('dofus_stage', c_uint32, 1),
         ('fiesta_stage', c_uint32, 2),
         ('wow_stage', c_uint32, 2),
-        ('veoh_tv_stage', c_uint32, 2),
         ('shoutcast_stage', c_uint32, 2),
         ('rtp_special_packets_seen', c_uint32, 1),
         ('mail_pop_stage', c_uint32, 2),
@@ -413,16 +394,11 @@ class NDPIFlowTcpStruct(Structure):
 class NDPIFlowUdpStruct(Structure):
     _pack_ = 1
     _fields_ = [
-        ('battlefield_msg_id', c_uint32),
-        ('snmp_msg_id', c_uint32),
-        ('battlefield_stage', c_uint32, 3),
-        ('snmp_stage', c_uint32, 2),
         ('ppstream_stage', c_uint32, 3),
         ('halflife2_stage', c_uint32, 2),
         ('tftp_stage', c_uint32, 2),
         ('aimini_stage', c_uint32, 5),
         ('xbox_stage', c_uint32, 1),
-        ('wsus_stage', c_uint32, 1),
         ('skype_packet_id', c_uint8),
         ('teamviewer_stage', c_uint8),
         ('eaq_pkt_id', c_uint8),
@@ -642,13 +618,6 @@ class NDPIUdpHdr(Structure):
     ]
 
 
-class NDPIPacketStructStack(Structure):
-    _pack_ = 1
-    _fields_ = [
-        ('protocol_stack_info', c_uint16)
-    ]
-
-
 class NDPIPacketStruct(Structure):
     _fields_ = [
         ('iph', POINTER(NDPIIphdr)),
@@ -657,10 +626,8 @@ class NDPIPacketStruct(Structure):
         ('udp', POINTER(NDPIUdpHdr)),
         ('generic_l4_ptr', POINTER(c_uint8)),
         ('payload', POINTER(c_uint8)),
-        ('tick_timestamp', c_uint32),
-        ('tick_timestamp_l', c_uint64),
+        ('current_timestamp_ms', c_uint64),
         ('detected_protocol_stack', c_uint16 * ndpi.ndpi_wrap_ndpi_procol_size()),
-        ('ndpi_packet_stack', NDPIPacketStructStack),
         ('line', NDPIIntOneLineStruct * 64),
         ('host_line', NDPIIntOneLineStruct),
         ('forwarded_line', NDPIIntOneLineStruct),
@@ -685,16 +652,14 @@ class NDPIPacketStruct(Structure):
         ('actual_payload_len', c_uint16),
         ('num_retried_bytes', c_uint16),
         ('parsed_lines', c_uint16),
-        ('parsed_unix_lines', c_uint16),
         ('empty_line_position', c_uint16),
         ('tcp_retransmission', c_uint8),
         ('l4_protocol', c_uint8),
-        ('ssl_certificate_detected', c_uint8, 4),
-        ('ssl_certificate_num_checks', c_uint8, 4),
         ('packet_lines_parsed_complete', c_uint8, 1),
         ('packet_direction', c_uint8, 1),
         ('empty_line_position_set', c_uint8, 1),
-        ('pad', c_uint8, 5),
+        ('http_check_content', c_uint8, 1),
+        ('pad', c_uint8, 4),
     ]
 
 
@@ -702,7 +667,6 @@ class NDPIFlowStructStack(Structure):
     _pack_ = 1
     _fields_ = [
         ("detected_protocol_stack", c_uint16 * ndpi.ndpi_wrap_ndpi_procol_size()),
-        ("protocol_stack_info", c_uint16)
     ]
 
 
@@ -725,7 +689,6 @@ NDPIFlowStruct._fields_ = [
     ("num_processed_pkts", c_uint8),
     ("extra_packets_func", CFUNCTYPE(c_int, POINTER(NDPIDetectionModuleStruct), POINTER(NDPIFlowStruct))),
     ("l4", L4),
-    ("server_id", POINTER(NDPIIdStruct)),
     ("host_server_name", c_ubyte * 256),
     ("http", Http),
     ("protos", Protos),
@@ -738,15 +701,11 @@ NDPIFlowStruct._fields_ = [
     ('byte_counter', c_uint16 * 2),
     ('bittorrent_stage', c_uint8),
     ('directconnect_stage', c_uint8, 2),
-    ('sip_yahoo_voice', c_uint8, 1),
     ('http_detected', c_uint8, 1),
     ('http_upper_protocol', c_uint16),
     ('http_lower_protocol', c_uint16),
     ('rtsprdt_stage', c_uint8, 2),
-    ('rtsp_control_flow', c_uint8, 1),
-    ('yahoo_detection_finished', c_uint8, 2),
     ('zattoo_stage', c_uint8, 3),
-    ('qq_stage', c_uint8, 3),
     ('thunder_stage', c_uint8, 2),
     ('oscar_ssl_voice_stage', c_uint8, 3),
     ('oscar_video_voice', c_uint8, 1),
@@ -775,7 +734,6 @@ NDPIFlowStruct._fields_ = [
     ('kxun_counter', c_uint16),
     ('iqiyi_counter', c_uint16),
     ('packet', NDPIPacketStruct),
-    ('flow', POINTER(NDPIFlowStruct)),
     ('src', POINTER(NDPIIdStruct)),
     ('dst', POINTER(NDPIIdStruct))
 ]