Add an heuristic to detect/ignore some anomalous TCP ACK packets (#1948)

In some networks, there are some anomalous TCP flows where the smallest
ACK packets have some kind of zero padding.
It looks like the IP and TCP headers in those frames wrongly consider the
0x00 Ethernet padding bytes as part of the TCP payload.
While this kind of packets is perfectly valid per-se, in some conditions
they might be treated by the TCP reassembler logic as (partial) overlaps,
deceiving the classification engine.
Add an heuristic to detect these packets and to ignore them, allowing
correct detection/classification.

This heuristic is configurable. Default value:
* in the library, it is disabled
* in `ndpiReader` and in the fuzzers, it is enabled (to ease testing)

Credit to @vel21ripn for the initial patch.

Close #1946

6 files changed, 76 insertions, 3 deletions

diff --git a/example/ndpiReader.c b/example/ndpiReader.c
index 72408409e..768ec4354 100644
--- a/example/ndpiReader.c
+++ b/example/ndpiReader.c
@@ -98,7 +98,7 @@ char *_debug_protocols = NULL;
 char *_disabled_protocols = NULL;
 int aggressiveness[NDPI_MAX_SUPPORTED_PROTOCOLS];
 static u_int8_t stats_flag = 0;
-ndpi_init_prefs init_prefs = ndpi_no_prefs;
+ndpi_init_prefs init_prefs = ndpi_no_prefs | ndpi_enable_tcp_ack_payload_heuristic;
 u_int8_t human_readeable_string_len = 5;
 u_int8_t max_num_udp_dissected_pkts = 24 /* 8 is enough for most protocols, Signal and SnapchatCall require more */, max_num_tcp_dissected_pkts = 80 /* due to telnet */;
 static u_int32_t pcap_analysis_duration = (u_int32_t)-1;
diff --git a/fuzz/fuzz_ndpi_reader.c b/fuzz/fuzz_ndpi_reader.c
index 4ae6a8246..f210611a4 100644
--- a/fuzz/fuzz_ndpi_reader.c
+++ b/fuzz/fuzz_ndpi_reader.c
@@ -18,7 +18,7 @@ u_int8_t enable_protocol_guess = 1, enable_payload_analyzer = 0;
 u_int8_t enable_flow_stats = 1;
 u_int8_t human_readeable_string_len = 5;
 u_int8_t max_num_udp_dissected_pkts = 16 /* 8 is enough for most protocols, Signal requires more */, max_num_tcp_dissected_pkts = 80 /* due to telnet */;
-ndpi_init_prefs init_prefs = ndpi_track_flow_payload | ndpi_enable_ja3_plus;
+ndpi_init_prefs init_prefs = ndpi_track_flow_payload | ndpi_enable_ja3_plus | ndpi_enable_tcp_ack_payload_heuristic;
 int enable_malloc_bins = 1;
 int malloc_size_stats = 0;
 int max_malloc_bins = 14;
diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h
index d3ccd208c..2e8c76d26 100644
--- a/src/include/ndpi_typedefs.h
+++ b/src/include/ndpi_typedefs.h
@@ -1273,6 +1273,8 @@ struct ndpi_detection_module_struct {
 
   u_int32_t aggressiveness_ookla;
 
+  int tcp_ack_paylod_heuristic;
+
   u_int16_t ndpi_to_user_proto_id[NDPI_MAX_NUM_CUSTOM_PROTOCOLS]; /* custom protocolId mapping */
   ndpi_proto_defaults_t proto_defaults[NDPI_MAX_SUPPORTED_PROTOCOLS+NDPI_MAX_NUM_CUSTOM_PROTOCOLS];
 
@@ -1691,6 +1693,17 @@ typedef enum {
     ndpi_dont_init_risk_ptree      = (1 << 14),
     ndpi_dont_load_cachefly_list   = (1 << 15),
     ndpi_track_flow_payload        = (1 << 16),
+    /* In some networks, there are some anomalous TCP flows where
+       the smallest ACK packets have some kind of zero padding.
+       It looks like the IP and TCP headers in those frames wrongly consider the
+       0x00 Ethernet padding bytes as part of the TCP payload.
+       While this kind of packets is perfectly valid per-se, in some conditions
+       they might be treated by the TCP reassembler logic as (partial) overlaps,
+       deceiving the classification engine.
+       Add an heuristic to detect these packets and to ignore them, allowing
+       correct detection/classification.
+       See #1946 for other details */
+    ndpi_enable_tcp_ack_payload_heuristic = (1 << 17),
   } ndpi_prefs;
 
 typedef struct {
diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index 4d83def4d..cab17cc5f 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -2962,6 +2962,9 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs
 
   ndpi_str->aggressiveness_ookla = NDPI_AGGRESSIVENESS_OOKLA_TLS;
 
+  if(prefs & ndpi_enable_tcp_ack_payload_heuristic)
+    ndpi_str->tcp_ack_paylod_heuristic = 1;
+
   for(i = 0; i < NUM_CUSTOM_CATEGORIES; i++)
     ndpi_snprintf(ndpi_str->custom_category_labels[i], CUSTOM_CATEGORY_LABEL_LEN, "User custom category %u",
 	     (unsigned int) (i + 1));
@@ -5509,6 +5512,20 @@ static u_int8_t ndpi_is_multi_or_broadcast(struct ndpi_packet_struct *packet) {
 
 /* ************************************************ */
 
+static int tcp_ack_padding(struct ndpi_packet_struct *packet) {
+  const struct ndpi_tcphdr *tcph = packet->tcp;
+  if(tcph && tcph->ack && !tcph->psh &&
+     packet->payload_packet_len < 8 &&
+     packet->payload_packet_len > 1 /* To avoid TCP keep-alives */) {
+    int i;
+    for(i = 0; i < packet->payload_packet_len; i++)
+      if(packet->payload[i] != 0)
+        return 0;
+    return 1;
+  }
+  return 0;
+}
+
 void ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str,
 			      struct ndpi_flow_struct *flow) {
   if(!flow) {
@@ -5597,7 +5614,10 @@ void ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str,
 	}
       }
 
-      if(flow->next_tcp_seq_nr[0] == 0 || flow->next_tcp_seq_nr[1] == 0 ||
+      if(ndpi_str->tcp_ack_paylod_heuristic && tcp_ack_padding(packet)) {
+        NDPI_LOG_DBG2(ndpi_str, "TCP ACK with zero padding. Ignoring\n");
+        packet->tcp_retransmission = 1;
+      } else if(flow->next_tcp_seq_nr[0] == 0 || flow->next_tcp_seq_nr[1] == 0 ||
 	 (tcph->syn && flow->packet_counter == 0)) {
 	/* initialize tcp sequence counters */
 	/* the ack flag needs to be set to get valid sequence numbers from the other
diff --git a/tests/cfgs/default/pcap/heuristic_tcp_ack_payload.pcap b/tests/cfgs/default/pcap/heuristic_tcp_ack_payload.pcap
new file mode 100644
index 000000000..241efeac1
--- /dev/null
+++ b/tests/cfgs/default/pcap/heuristic_tcp_ack_payload.pcap
diff --git a/tests/cfgs/default/result/heuristic_tcp_ack_payload.pcap.out b/tests/cfgs/default/result/heuristic_tcp_ack_payload.pcap.out
new file mode 100644
index 000000000..bb44ee95d
--- /dev/null
+++ b/tests/cfgs/default/result/heuristic_tcp_ack_payload.pcap.out
@@ -0,0 +1,40 @@
+Guessed flow protos:	0
+
+DPI Packets (TCP):	67	(11.17 pkts/flow)
+Confidence DPI              : 6 (flows)
+Num dissector calls: 17 (2.83 diss/flow)
+LRU cache ookla:      0/0/0 (insert/search/found)
+LRU cache bittorrent: 0/0/0 (insert/search/found)
+LRU cache zoom:       0/0/0 (insert/search/found)
+LRU cache stun:       0/0/0 (insert/search/found)
+LRU cache tls_cert:   0/8/0 (insert/search/found)
+LRU cache mining:     0/0/0 (insert/search/found)
+LRU cache msteams:    0/0/0 (insert/search/found)
+LRU cache stun_zoom:  0/0/0 (insert/search/found)
+Automa host:          12/2 (search/found)
+Automa domain:        12/0 (search/found)
+Automa tls cert:      3/0 (search/found)
+Automa risk mask:     0/0 (search/found)
+Automa common alpns:  9/9 (search/found)
+Patricia risk mask:   18/0 (search/found)
+Patricia risk:        18/0 (search/found)
+Patricia protocols:   16/2 (search/found)
+
+TLS	196	76217	4
+WindowsUpdate	19	2638	1
+Pinterest	88	34448	1
+
+JA3 Host Stats: 
+		 IP Address                  	 # JA3C     
+	1	 194.226.199.103          	 1      
+	2	 194.226.199.61           	 2      
+	3	 194.226.199.9            	 1      
+	4	 194.226.199.21           	 1      
+
+
+	1	TCP 194.226.199.61:27453 <-> 35.241.9.150:443 [proto: 91/TLS][IP: 284/GoogleCloud][Encrypted][Confidence: DPI][DPI packets: 8][cat: Web/5][36 pkts/3477 bytes <-> 42 pkts/37330 bytes][Goodput ratio: 44/94][171.42 sec][Hostname/SNI: firefox.settings.services.mozilla.com][(Advertised) ALPNs: h2;http/1.1][(Negotiated) ALPN: h2][bytes ratio: -0.830 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 4196/3653 58250/58245 14929/14067][Pkt Len c2s/s2c min/avg/max/stddev: 56/60 97/889 375/2878 73/1070][TLSv1.2][JA3C: fb0aa01abe9d8e4037eb3473ca6e2dca][ServerNames: firefox.settings.services.mozilla.com,main-2-cdn.prod.kinto.prod.cloudops.mozgcp.net][JA3S: 9d9ce860f1b1cbef07b019450cb368d8][Issuer: C=US, O=Let's Encrypt, CN=R3][Subject: CN=main-2-cdn.prod.kinto.prod.cloudops.mozgcp.net][Certificate SHA-1: 30:0D:22:77:6E:DA:4E:99:3E:AF:8A:D0:5C:7D:97:51:8B:E6:22:11][Firefox][Validity: 2023-04-04 08:33:24 - 2023-07-03 08:33:23][Cipher: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256][Plen Bins: 49,16,2,0,3,3,1,0,1,0,2,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,7,0,0,11]
+	2	TCP 194.226.199.61:6946 <-> 2.22.40.186:443 [proto: 91.183/TLS.Pinterest][IP: 0/Unknown][Encrypted][Confidence: DPI][DPI packets: 10][cat: SocialNetwork/6][41 pkts/7780 bytes <-> 47 pkts/26668 bytes][Goodput ratio: 70/90][18.13 sec][Hostname/SNI: ru.pinterest.com][(Advertised) ALPNs: h2;http/1.1][TLS Supported Versions: GREASE;TLSv1.3;TLSv1.2][bytes ratio: -0.548 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 404/181 3512/2129 852/406][Pkt Len c2s/s2c min/avg/max/stddev: 56/60 190/567 1514/2974 287/678][TLSv1.3][JA3C: 598872011444709307b861ae817a4b60][JA3S: 15af977ce25de452b96affa2addb1036][Chrome][Cipher: TLS_AES_256_GCM_SHA384][Plen Bins: 50,9,1,5,0,0,0,0,2,0,0,0,0,1,0,0,1,1,0,4,0,0,0,0,1,1,2,0,0,0,0,4,1,0,0,0,1,0,0,0,0,0,0,0,0,12,0,1]
+	3	TCP 194.226.199.21:58155 <-> 52.18.127.189:443 [proto: 91/TLS][IP: 265/AmazonAWS][Encrypted][Confidence: DPI][DPI packets: 13][cat: Web/5][28 pkts/6789 bytes <-> 35 pkts/8995 bytes][Goodput ratio: 78/79][130.64 sec][Hostname/SNI: bitrix.info][(Advertised) ALPNs: h2;http/1.1][(Negotiated) ALPN: h2][TLS Supported Versions: GREASE;TLSv1.3;TLSv1.2][bytes ratio: -0.140 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/0 5498/4834 45102/45058 12717/11564][Pkt Len c2s/s2c min/avg/max/stddev: 56/60 242/257 1547/2974 352/535][TLSv1.2][JA3C: ab205d804ecf934209c2a1bb94f817e0][JA3S: bfc90d56141386ee83b56cda231cccfc][Chrome][Cipher: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256][Plen Bins: 46,22,6,6,0,0,0,0,0,0,1,0,6,0,1,1,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,1,1]
+	4	TCP 194.226.199.103:62580 <-> 217.69.139.59:443 [proto: 91/TLS][IP: 0/Unknown][Encrypted][Confidence: DPI][DPI packets: 15][cat: Web/5][22 pkts/2692 bytes <-> 16 pkts/10450 bytes][Goodput ratio: 55/92][7.28 sec][Hostname/SNI: portal.mail.ru][(Advertised) ALPNs: h2;http/1.1][(Negotiated) ALPN: http/1.1][TLS Supported Versions: TLSv1.3;TLSv1.2][bytes ratio: -0.590 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 100/27 1559/213 357/70][Pkt Len c2s/s2c min/avg/max/stddev: 56/60 122/653 623/2897 162/957][TLSv1.2][JA3C: e669667efb41c36f714c309243f41ca7][ServerNames: *.mail.ru,mail.ru][JA3S: 2b33c1374db4ddf06942f92373c0b54b][Issuer: C=BE, O=GlobalSign nv-sa, CN=GlobalSign RSA OV SSL CA 2018][Subject: C=RU, ST=Moscow, L=Moscow, O=VK LLC, CN=*.mail.ru][Certificate SHA-1: 9F:A2:43:EA:AA:62:15:13:44:0D:15:75:17:47:4C:6B:E5:8E:10:1E][Firefox][Validity: 2022-10-20 09:52:31 - 2023-11-21 09:52:30][Cipher: TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384][Plen Bins: 68,0,0,5,0,0,0,0,11,0,0,0,0,0,0,0,0,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,0,0,5]
+	5	TCP 194.226.199.9:49756 <-> 92.223.106.21:443 [proto: 91/TLS][IP: 0/Unknown][Encrypted][Confidence: DPI][DPI packets: 9][cat: Web/5][9 pkts/1140 bytes <-> 8 pkts/5344 bytes][Goodput ratio: 54/91][0.28 sec][Hostname/SNI: moevideo.biz][(Advertised) ALPNs: http/1.1][(Negotiated) ALPN: http/1.1][TLS Supported Versions: TLSv1.3;TLSv1.2;TLSv1.1;TLSv1;SSLv3][bytes ratio: -0.648 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 20/6 44/20 18/8][Pkt Len c2s/s2c min/avg/max/stddev: 56/60 127/668 571/2690 159/894][TLSv1.2][JA3C: 7c822e5e821268e8bd01b70e9cad0b85][ServerNames: *.moevideo.biz,moevideo.biz][JA3S: d154fcfa5bb4f0748e1dd1992c681104][Issuer: C=BE, O=GlobalSign nv-sa, CN=AlphaSSL CA - SHA256 - G4][Subject: CN=*.moevideo.biz][Certificate SHA-1: FF:0C:ED:41:2C:7C:DA:BA:89:FE:7E:09:4A:2B:62:26:A0:20:AC:53][Safari][Validity: 2023-04-04 15:59:15 - 2024-05-05 15:59:14][Cipher: TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384][Plen Bins: 51,0,8,0,0,0,0,0,8,0,0,0,0,0,0,8,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,8,0,8]
+	6	TCP 194.226.199.226:34101 <-> 8.247.226.126:80 [proto: 7.147/HTTP.WindowsUpdate][IP: 0/Unknown][ClearText][Confidence: DPI][DPI packets: 12][cat: SoftwareUpdate/19][7 pkts/896 bytes <-> 12 pkts/1742 bytes][Goodput ratio: 56/62][0.04 sec][Hostname/SNI: 3.tlu.dl.delivery.mp.microsoft.com][bytes ratio: -0.321 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 7/1 12/11 6/3][Pkt Len c2s/s2c min/avg/max/stddev: 56/60 128/145 550/1076 172/281][URL: 3.tlu.dl.delivery.mp.microsoft.com/filestreamingservice/files/b4f27514-1618-47a0-bcd4-5fcb469edb63?P1=1681888058&P2=404&P3=2&P4=VJ2Qv%2bUXzBGOULZmyshxlc8XXx4pLl7hoFcLgf1iS33rDGfm0tCVrTPvZN8tn8yWBSrA0idwdtOBFLQMjZCUkw%3d%3d][StatusCode: 0][User-Agent: Microsoft-Delivery-Optimization/10.0][PLAIN TEXT (GET /filestreamingservice/files)][Plen Bins: 89,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]