From 91bb77a8806ee2987e856f66674cf3aa8b1d60db Mon Sep 17 00:00:00 2001
From: Ivan Nardi <12729895+IvanNardi@users.noreply.github.com>
Date: Wed, 22 Dec 2021 19:54:06 +0100
Subject: A final(?) effort to reduce memory usage per flow (#1389)

Remove some unused fields and re-organize other ones.
In particular:
* Update the parameters of `ndpi_ssl_version2str()` function
* Zattoo, Thunder: these timestamps aren't really used.
* Ftp/mail: these protocols are dissected only over TCP.
* Attention must be paid to TLS.Bittorrent flows to avoid invalid
read/write to `flow->protos.bittorrent.hash` field.

This is the last(?) commit of a long series (see 22241a1d, 227e586e,
730c2360, a8ffcd8b) aiming to reduce library memory consumption.

Before, at nDPI 4.0 (more precisly, at a6b10cf7, because memory stats
were wrong until that commit):
```
nDPI Memory statistics:
	nDPI Memory (once):      221.15 KB
	Flow Memory (per flow):  2.94 KB
```
Now:
```
nDPI Memory statistics:
	nDPI Memory (once):      231.71 KB
	Flow Memory (per flow):  1008 B       <---------
```
i.e. memory usage per flow has been reduced by 66%, dropping below the
psychological threshold of 1 KB.

To further reduce this value, we probably need to look into #1279:
let's fight this battle another day.
---
 python/ndpi.py            | 21 ++++-----------------
 python/ndpi_typestruct.py | 19 +++++--------------
 2 files changed, 9 insertions(+), 31 deletions(-)

(limited to 'python')

diff --git a/python/ndpi.py b/python/ndpi.py
index ba081acb9..d3a1f6389 100644
--- a/python/ndpi.py
+++ b/python/ndpi.py
@@ -452,12 +452,6 @@ struct ndpi_id_struct {
   /* NDPI_PROTOCOL_GNUTELLA */
   uint32_t gnutella_ts;
 
-  /* NDPI_PROTOCOL_THUNDER */
-  uint32_t thunder_ts;
-
-  /* NDPI_PROTOCOL_ZATTOO */
-  uint32_t zattoo_ts;
-
   /* NDPI_PROTOCOL_UNENCRYPTED_JABBER */
   uint32_t jabber_stun_or_ft_ts;
 
@@ -497,7 +491,6 @@ struct ndpi_flow_tcp_struct {
 
   /* NDPI_PROTOCOL_IRC */
   uint8_t irc_stage;
-  uint8_t irc_port;
 
   /* NDPI_PROTOCOL_H323 */
   uint8_t h323_valid_packets;
@@ -912,11 +905,7 @@ struct ndpi_detection_module_struct {
   uint32_t irc_timeout;
   /* gnutella parameters */
   uint32_t gnutella_timeout;
-  /* thunder parameters */
-  uint32_t thunder_timeout;
   /* rstp */
-  uint32_t orb_rstp_ts_timeout;
-  uint32_t zattoo_connection_timeout;
   uint32_t jabber_stun_timeout;
   uint32_t jabber_file_transfer_timeout;
   uint8_t ip_version_limit;
@@ -1001,12 +990,11 @@ struct ndpi_flow_struct {
   */
   struct {
     ndpi_http_method method;
-    char *url, *content_type, *user_agent;
-    uint8_t num_request_headers, num_response_headers;
     uint8_t request_version; /* 0=1.0 and 1=1.1. Create an enum for this? */
     uint16_t response_status_code; /* 200, 404, etc. */
-    uint8_t detected_os[32]; /* Via HTTP/QUIC User-Agent */
-    uint8_t nat_ip[24];
+    char *url, *content_type, *user_agent;
+    char *detected_os; /* Via HTTP/QUIC User-Agent */
+    char *nat_ip; /* Via HTTP X-Forwarded-For */
 
   } http;
 
@@ -1049,8 +1037,6 @@ struct ndpi_flow_struct {
     } kerberos;
 
     struct {
-      char ssl_version_str[12];
-      uint16_t ssl_version, server_names_len;
       char *server_names, *alpn, *tls_supported_versions, *issuerDN, *subjectDN;
       uint32_t notBefore, notAfter;
       char ja3_client[33], ja3_server[33];
@@ -1062,6 +1048,7 @@ struct ndpi_flow_struct {
         char *esni;
       } encrypted_sni;
       ndpi_cipher_weakness server_unsafe_cipher;
+      uint16_t ssl_version, server_names_len;
     } tls_quic;
 
     struct {
diff --git a/python/ndpi_typestruct.py b/python/ndpi_typestruct.py
index 3909897ee..8b16f8af0 100644
--- a/python/ndpi_typestruct.py
+++ b/python/ndpi_typestruct.py
@@ -257,10 +257,6 @@ NDPIDetectionModuleStruct._fields_ = [
     ("protocols_ptree", c_void_p),
     ("irc_timeout", c_uint32),
     ("gnutella_timeout", c_uint32),
-    ("battlefield_timeout", c_uint32),
-    ("thunder_timeout", c_uint32),
-    ("tvants_connection_timeout", c_uint32),
-    ("orb_rstp_ts_timeout", c_uint32),
     ("zattoo_connection_timeout", c_uint32),
     ("jabber_stun_timeout", c_uint32),
     ("jabber_file_transfer_timeout", c_uint32),
@@ -309,10 +305,7 @@ class NDPIIdStruct(Structure):
         ('irc_ts', c_uint32),
         ('gnutella_ts', c_uint32),
         ('battlefield_ts', c_uint32),
-        ('thunder_ts', c_uint32),
         ('rtsp_timer', c_uint32),
-        ('oscar_last_safe_access_time', c_uint32),
-        ('zattoo_ts', c_uint32),
         ('jabber_stun_or_ft_ts', c_uint32),
         ('directconnect_last_safe_access_time', c_uint32),
         ('detected_directconnect_port', c_uint16),
@@ -335,7 +328,6 @@ class NDPIFlowTcpStruct(Structure):
         ('pop_command_bitmask', c_uint16),
         ('wa_matched_so_far', c_uint8),
         ('irc_stage', c_uint8),
-        ('irc_port', c_uint8),
         ('h323_valid_packets', c_uint8),
         ('gnutella_msg_id', c_uint8 * 3),
         ('irc_3a_counter', c_uint32, 3),
@@ -422,14 +414,13 @@ class L4(Union):
 class Http(Structure):
     _fields_ = [
         ("method", c_int),
-        ("url", c_char_p),
-        ("content_type", c_char_p),
-        ("num_request_headers", c_uint8),
-        ("num_response_headers", c_uint8),
         ("request_version", c_uint8),
         ("response_status_code", c_uint16),
-        ("detected_os", c_char * 32),
-        ("nat_ip", c_char * 24),
+        ("url", c_char_p),
+        ("content_type", c_char_p),
+        ("user_agent", c_char_p),
+        ("detected_os", c_char_p),
+        ("nat_ip", c_char_p),
     ]
 
 
-- 
cgit v1.2.3