config: HTTP: enable/disable processing of HTTP responses

12 files changed, 111 insertions, 14 deletions

diff --git a/doc/configuration_parameters.md b/doc/configuration_parameters.md
index 9bfa944ea..c92969038 100644
--- a/doc/configuration_parameters.md
+++ b/doc/configuration_parameters.md
@@ -26,5 +26,6 @@ TODO
 | "pop"        | "tls_dissection.enable"                   | 1             | NULL      | NULL      | Enable/disable dissection of TLS packets in cleartext POP flows (because of opportunistic TLS, via STARTTLS msg) |
 | "ftp"        | "tls_dissection.enable"                   | 1             | NULL      | NULL      | Enable/disable dissection of TLS packets in cleartext FTP flows (because of opportunistic TLS, via AUTH TLS msg) |
 | "stun"       | "tls_dissection.enable"                   | 1             | NULL      | NULL      | Enable/disable dissection of TLS packets multiplexed into STUN flows |
+| "http"       | "process_response.enable"                 | 1             | NULL      | NULL      | Enable/disable processing of HTTP responses. By default, HTTP flows are usually fully classified after the first request/response pair. If this parameter is disabled, the flows are fully classified after the first request (or after the first response, if the request is missing); in that case, some flow risks are not checked and some metadata are not exported |
 | "ookla"      | "aggressiveness",                         | 0x01          | 0x00      | 0x01      | Detection aggressiveness for Ookla. The value is a bitmask. Values: 0x0 = disabled; 0x01 = enable heuristic for detection over TLS (via Ookla LRU cache) |
 | $PROTO_NAME   | "ip_list.load"                           | 1             | NULL      | NULL      | Enable/disable loading of internal list of IP addresses (used for (sub)classification) specific to that protocol. Use "any" as protocol name if you want to easily enable/disable all lists. This knob is valid only for the following protocols: Alibaba, Amazon AWS, Apple, Avast, Bloomberg, Cachefly, Cloudflare, Discord, Disney+, Dropbox, Edgecast, EpicGames, Ethereum, Facebook, Github, Google, Google Cloud, GoTo, Hotspot Shield, Hulu, Line, Microsoft 365, Microsoft Azure, Microsoft One Drive, Microsoft Outlook, Mullvad, Netflix, Nvidia, OpenDNS, ProtonVPN, RiotGames, Roblox, Skype/Teams, Starcraft, Steam, Teamviewer, Telegram, Tencent, Threema, TOR, Twitch, Twitter, UbuntuONE, VK, Yandex, Yandex Cloud, Webex, Whatsapp, Zoom |
diff --git a/fuzz/fuzz_config.cpp b/fuzz/fuzz_config.cpp
index 13ee51e01..955948255 100644
--- a/fuzz/fuzz_config.cpp
+++ b/fuzz/fuzz_config.cpp
@@ -118,6 +118,11 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
     ndpi_set_config(ndpi_info_mod, "stun", "tls_dissection.enable", cfg_value);
   }
   if(fuzzed_data.ConsumeBool()) {
+    value = fuzzed_data.ConsumeIntegralInRange(0, 1 + 1);
+    sprintf(cfg_value, "%d", value);
+    ndpi_set_config(ndpi_info_mod, "http", "process_response.enable", cfg_value);
+  }
+  if(fuzzed_data.ConsumeBool()) {
     value = fuzzed_data.ConsumeIntegralInRange(0, 0x01 + 1);
     sprintf(cfg_value, "%d", value);
     ndpi_set_config(ndpi_info_mod, "ookla", "aggressiveness", cfg_value);
diff --git a/src/include/ndpi_private.h b/src/include/ndpi_private.h
index 8f7208a05..cf857be23 100644
--- a/src/include/ndpi_private.h
+++ b/src/include/ndpi_private.h
@@ -205,6 +205,8 @@ struct ndpi_detection_module_config_struct {
 
   int stun_opportunistic_tls_enabled;
 
+  int http_parse_response_enabled;
+
   int ookla_aggressiveness;
 
   NDPI_PROTOCOL_BITMASK ip_list_bitmask;
@@ -327,14 +329,9 @@ struct ndpi_detection_module_struct {
 
   /* *** If you add a new LRU cache, please update lru_cache_type above! *** */
 
-  int tcp_ack_paylod_heuristic;
-  int fully_encrypted_based_on_first_pkt_heuristic;
-
   u_int16_t ndpi_to_user_proto_id[NDPI_MAX_NUM_CUSTOM_PROTOCOLS]; /* custom protocolId mapping */
   ndpi_proto_defaults_t proto_defaults[NDPI_MAX_SUPPORTED_PROTOCOLS+NDPI_MAX_NUM_CUSTOM_PROTOCOLS];
 
-  u_int8_t direction_detect_disable:1, /* disable internal detection of packet direction */ _pad:7;
-
 #ifdef CUSTOM_NDPI_PROTOCOLS
   #include "../../../nDPI-custom/custom_ndpi_typedefs.h"
 #endif
diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index 33278c490..d3282c7da 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -10790,6 +10790,8 @@ static const struct cfg_param {
 
   { "stun",          "tls_dissection.enable",                   "1", NULL, NULL, CFG_PARAM_ENABLE_DISABLE, __OFF(stun_opportunistic_tls_enabled) },
 
+  { "http",          "process_response.enable",                 "1", NULL, NULL, CFG_PARAM_ENABLE_DISABLE, __OFF(http_parse_response_enabled) },
+
   { "ookla",         "aggressiveness",                          "0x01", "0", "1", CFG_PARAM_INT, __OFF(ookla_aggressiveness) },
 
   { "$PROTO_NAME_OR_ID", "ip_list.load",                        "1", NULL, NULL, CFG_PARAM_PROTOCOL_ENABLE_DISABLE, __OFF(ip_list_bitmask)},
diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c
index b0bbd30ca..673345b93 100644
--- a/src/lib/protocols/http.c
+++ b/src/lib/protocols/http.c
@@ -1493,16 +1493,20 @@ static void ndpi_search_http_tcp(struct ndpi_detection_module_struct *ndpi_struc
   NDPI_LOG_DBG(ndpi_struct, "search HTTP\n");
   ndpi_check_http_tcp(ndpi_struct, flow);
 
-  if((flow->host_server_name[0] != '\0'&&
+  if((ndpi_struct->cfg.http_parse_response_enabled &&
+      flow->host_server_name[0] != '\0'&&
       flow->http.response_status_code != 0) ||
-    /* We have found 3 consecutive requests (without the reply) or 3
-       consecutive replies (without the request). If the traffic is really
-       asymmetric, stop here, because we will never find the metadata from
-       both the request and the reply. We wait for 3 events (instead of 2)
-       to avoid false positives triggered by missing/dropped packets */
-    (flow->l4.tcp.http_asymmetric_stage == 2 &&
-     (flow->packet_direction_complete_counter[0] == 0 ||
-      flow->packet_direction_complete_counter[1] == 0))) {
+     (!ndpi_struct->cfg.http_parse_response_enabled &&
+      (flow->host_server_name[0] != '\0' ||
+       flow->http.response_status_code != 0)) ||
+     /* We have found 3 consecutive requests (without the reply) or 3
+        consecutive replies (without the request). If the traffic is really
+        asymmetric, stop here, because we will never find the metadata from
+        both the request and the reply. We wait for 3 events (instead of 2)
+        to avoid false positives triggered by missing/dropped packets */
+     (flow->l4.tcp.http_asymmetric_stage == 2 &&
+      (flow->packet_direction_complete_counter[0] == 0 ||
+       flow->packet_direction_complete_counter[1] == 0))) {
     flow->extra_packets_func = NULL; /* We're good now */
 
     if(flow->initial_binary_bytes_len) ndpi_analyze_content_signature(ndpi_struct, flow);
diff --git a/tests/cfgs/default/pcap/http.pcapng b/tests/cfgs/default/pcap/http.pcapng
new file mode 100644
index 000000000..6e665b5f2
--- /dev/null
+++ b/tests/cfgs/default/pcap/http.pcapng
diff --git a/tests/cfgs/default/result/http.pcapng.out b/tests/cfgs/default/result/http.pcapng.out
new file mode 100644
index 000000000..1e30ce9da
--- /dev/null
+++ b/tests/cfgs/default/result/http.pcapng.out
@@ -0,0 +1,28 @@
+DPI Packets (TCP):	6	(6.00 pkts/flow)
+Confidence DPI              : 1 (flows)
+Num dissector calls: 15 (15.00 diss/flow)
+LRU cache ookla:      0/0/0 (insert/search/found)
+LRU cache bittorrent: 0/0/0 (insert/search/found)
+LRU cache zoom:       0/0/0 (insert/search/found)
+LRU cache stun:       0/0/0 (insert/search/found)
+LRU cache tls_cert:   0/0/0 (insert/search/found)
+LRU cache mining:     0/0/0 (insert/search/found)
+LRU cache msteams:    0/0/0 (insert/search/found)
+LRU cache stun_zoom:  0/0/0 (insert/search/found)
+Automa host:          1/1 (search/found)
+Automa domain:        1/0 (search/found)
+Automa tls cert:      0/0 (search/found)
+Automa risk mask:     0/0 (search/found)
+Automa common alpns:  0/0 (search/found)
+Patricia risk mask:   0/0 (search/found)
+Patricia risk mask IPv6: 0/0 (search/found)
+Patricia risk:        0/0 (search/found)
+Patricia risk IPv6:   0/0 (search/found)
+Patricia protocols:   1/1 (search/found)
+Patricia protocols IPv6: 0/0 (search/found)
+
+Google	10	1278	1
+
+Acceptable                      10 1278          1            
+
+	1	TCP 192.168.1.128:42170 <-> 216.58.208.142:80 [proto: 7.126/HTTP.Google][IP: 126/Google][ClearText][Confidence: DPI][DPI packets: 6][cat: Web/5][6 pkts/478 bytes <-> 4 pkts/800 bytes][Goodput ratio: 15/66][0.04 sec][Hostname/SNI: google.com][bytes ratio: -0.252 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/7 8/14 28/20 10/6][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 80/200 140/594 27/227][URL: google.com/][StatusCode: 301][Content-Type: text/html][Server: gws][User-Agent: curl/7.68.0][PLAIN TEXT (GET / HTTP/1.1)][Plen Bins: 0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
diff --git a/tests/cfgs/http_process_response_disable/config.txt b/tests/cfgs/http_process_response_disable/config.txt
new file mode 100644
index 000000000..8ca478992
--- /dev/null
+++ b/tests/cfgs/http_process_response_disable/config.txt
@@ -0,0 +1 @@
+--cfg=http,process_response.enable,0
diff --git a/tests/cfgs/http_process_response_disable/pcap/http.pcapng b/tests/cfgs/http_process_response_disable/pcap/http.pcapng
new file mode 120000
index 000000000..9e909a49d
--- /dev/null
+++ b/tests/cfgs/http_process_response_disable/pcap/http.pcapng
@@ -0,0 +1 @@
+../../default/pcap/http.pcapng
\ No newline at end of file
diff --git a/tests/cfgs/http_process_response_disable/pcap/http_asymmetric.pcapng b/tests/cfgs/http_process_response_disable/pcap/http_asymmetric.pcapng
new file mode 120000
index 000000000..1f6c090a0
--- /dev/null
+++ b/tests/cfgs/http_process_response_disable/pcap/http_asymmetric.pcapng
@@ -0,0 +1 @@
+../../default/pcap/http_asymmetric.pcapng
\ No newline at end of file
diff --git a/tests/cfgs/http_process_response_disable/result/http.pcapng.out b/tests/cfgs/http_process_response_disable/result/http.pcapng.out
new file mode 100644
index 000000000..691ee0835
--- /dev/null
+++ b/tests/cfgs/http_process_response_disable/result/http.pcapng.out
@@ -0,0 +1,28 @@
+DPI Packets (TCP):	4	(4.00 pkts/flow)
+Confidence DPI              : 1 (flows)
+Num dissector calls: 15 (15.00 diss/flow)
+LRU cache ookla:      0/0/0 (insert/search/found)
+LRU cache bittorrent: 0/0/0 (insert/search/found)
+LRU cache zoom:       0/0/0 (insert/search/found)
+LRU cache stun:       0/0/0 (insert/search/found)
+LRU cache tls_cert:   0/0/0 (insert/search/found)
+LRU cache mining:     0/0/0 (insert/search/found)
+LRU cache msteams:    0/0/0 (insert/search/found)
+LRU cache stun_zoom:  0/0/0 (insert/search/found)
+Automa host:          1/1 (search/found)
+Automa domain:        1/0 (search/found)
+Automa tls cert:      0/0 (search/found)
+Automa risk mask:     0/0 (search/found)
+Automa common alpns:  0/0 (search/found)
+Patricia risk mask:   0/0 (search/found)
+Patricia risk mask IPv6: 0/0 (search/found)
+Patricia risk:        0/0 (search/found)
+Patricia risk IPv6:   0/0 (search/found)
+Patricia protocols:   1/1 (search/found)
+Patricia protocols IPv6: 0/0 (search/found)
+
+Google	10	1278	1
+
+Acceptable                      10 1278          1            
+
+	1	TCP 192.168.1.128:42170 <-> 216.58.208.142:80 [proto: 7.126/HTTP.Google][IP: 126/Google][ClearText][Confidence: DPI][DPI packets: 4][cat: Web/5][6 pkts/478 bytes <-> 4 pkts/800 bytes][Goodput ratio: 15/66][0.04 sec][Hostname/SNI: google.com][bytes ratio: -0.252 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/7 8/14 28/20 10/6][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 80/200 140/594 27/227][URL: google.com/][User-Agent: curl/7.68.0][PLAIN TEXT (GET / HTTP/1.1)][Plen Bins: 0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
diff --git a/tests/cfgs/http_process_response_disable/result/http_asymmetric.pcapng.out b/tests/cfgs/http_process_response_disable/result/http_asymmetric.pcapng.out
new file mode 100644
index 000000000..df351c7ec
--- /dev/null
+++ b/tests/cfgs/http_process_response_disable/result/http_asymmetric.pcapng.out
@@ -0,0 +1,29 @@
+DPI Packets (TCP):	6	(3.00 pkts/flow)
+Confidence DPI              : 2 (flows)
+Num dissector calls: 30 (15.00 diss/flow)
+LRU cache ookla:      0/0/0 (insert/search/found)
+LRU cache bittorrent: 0/0/0 (insert/search/found)
+LRU cache zoom:       0/0/0 (insert/search/found)
+LRU cache stun:       0/0/0 (insert/search/found)
+LRU cache tls_cert:   0/0/0 (insert/search/found)
+LRU cache mining:     0/0/0 (insert/search/found)
+LRU cache msteams:    0/0/0 (insert/search/found)
+LRU cache stun_zoom:  0/0/0 (insert/search/found)
+Automa host:          1/0 (search/found)
+Automa domain:        1/0 (search/found)
+Automa tls cert:      0/0 (search/found)
+Automa risk mask:     0/0 (search/found)
+Automa common alpns:  0/0 (search/found)
+Patricia risk mask:   4/0 (search/found)
+Patricia risk mask IPv6: 0/0 (search/found)
+Patricia risk:        0/0 (search/found)
+Patricia risk IPv6:   0/0 (search/found)
+Patricia protocols:   4/0 (search/found)
+Patricia protocols IPv6: 0/0 (search/found)
+
+HTTP	23	9961	2
+
+Acceptable                      23 9961          2            
+
+	1	TCP 192.168.1.146:80 -> 192.168.1.103:1044 [proto: 7/HTTP][IP: 0/Unknown][ClearText][Confidence: DPI][DPI packets: 3][cat: Web/5][13 pkts/8357 bytes -> 0 pkts/0 bytes][Goodput ratio: 91/0][5.11 sec][bytes ratio: 1.000 (Upload)][IAT c2s/s2c min/avg/max/stddev: 0/0 464/0 5000/0 1435/0][Pkt Len c2s/s2c min/avg/max/stddev: 60/0 643/0 1514/0 626/0][StatusCode: 200][Content-Type: text/html][Server: Apache/2.4.41 (Ubuntu)][Risk: ** HTTP Susp User-Agent **** Unidirectional Traffic **][Risk Score: 110][Risk Info: No client to server traffic / Empty or missing User-Agent][PLAIN TEXT (HTTP/1.1 200 OK)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,14,14,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,57,0,0]
+	2	TCP 192.168.0.1:1044 -> 10.10.10.1:80 [proto: 7/HTTP][IP: 0/Unknown][ClearText][Confidence: DPI][DPI packets: 3][cat: Web/5][10 pkts/1604 bytes -> 0 pkts/0 bytes][Goodput ratio: 66/0][5.11 sec][Hostname/SNI: proxy.wiresharkfest.acropolis.local][bytes ratio: 1.000 (Upload)][IAT c2s/s2c min/avg/max/stddev: 0/0 567/0 4951/0 1550/0][Pkt Len c2s/s2c min/avg/max/stddev: 54/0 160/0 418/0 160/0][URL: proxy.wiresharkfest.acropolis.local/][User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:91.0) Gecko/20100101 Firefox/91.0][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][PLAIN TEXT (GET / HTTP/1.1)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,66,33,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]