aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorIvan Nardi <12729895+IvanNardi@users.noreply.github.com>2021-12-22 19:54:06 +0100
committerGitHub <noreply@github.com>2021-12-22 19:54:06 +0100
commit91bb77a8806ee2987e856f66674cf3aa8b1d60db (patch)
tree0f2a06cc9b64a8af221552e87ae771ac882ab4b5 /src
parent02da143e4567cbfe32b139561ec3a702ce380fc7 (diff)
A final(?) effort to reduce memory usage per flow (#1389)
Remove some unused fields and re-organize other ones. In particular: * Update the parameters of `ndpi_ssl_version2str()` function * Zattoo, Thunder: these timestamps aren't really used. * Ftp/mail: these protocols are dissected only over TCP. * Attention must be paid to TLS.Bittorrent flows to avoid invalid read/write to `flow->protos.bittorrent.hash` field. This is the last(?) commit of a long series (see 22241a1d, 227e586e, 730c2360, a8ffcd8b) aiming to reduce library memory consumption. Before, at nDPI 4.0 (more precisly, at a6b10cf7, because memory stats were wrong until that commit): ``` nDPI Memory statistics: nDPI Memory (once): 221.15 KB Flow Memory (per flow): 2.94 KB ``` Now: ``` nDPI Memory statistics: nDPI Memory (once): 231.71 KB Flow Memory (per flow): 1008 B <--------- ``` i.e. memory usage per flow has been reduced by 66%, dropping below the psychological threshold of 1 KB. To further reduce this value, we probably need to look into #1279: let's fight this battle another day.
Diffstat (limited to 'src')
-rw-r--r--src/include/ndpi_api.h.in2
-rw-r--r--src/include/ndpi_typedefs.h49
-rw-r--r--src/lib/ndpi_main.c18
-rw-r--r--src/lib/ndpi_utils.c64
-rw-r--r--src/lib/protocols/ftp_control.c22
-rw-r--r--src/lib/protocols/http.c25
-rw-r--r--src/lib/protocols/mail_imap.c10
-rw-r--r--src/lib/protocols/mail_pop.c12
-rw-r--r--src/lib/protocols/mail_smtp.c58
-rw-r--r--src/lib/protocols/thunder.c27
-rw-r--r--src/lib/protocols/tls.c5
-rw-r--r--src/lib/protocols/zattoo.c14
12 files changed, 133 insertions, 173 deletions
diff --git a/src/include/ndpi_api.h.in b/src/include/ndpi_api.h.in
index 82010ece5..28f305193 100644
--- a/src/include/ndpi_api.h.in
+++ b/src/include/ndpi_api.h.in
@@ -975,7 +975,7 @@ extern "C" {
/* Return a flow info string (summarized). Does only work for DNS/HTTP/TLS/QUIC. */
const char* ndpi_get_flow_info(struct ndpi_flow_struct const * const flow,
ndpi_protocol const * const l7_protocol);
- char* ndpi_ssl_version2str(struct ndpi_flow_struct *flow,
+ char* ndpi_ssl_version2str(char *buf, int buf_len,
u_int16_t version, u_int8_t *unknown_tls_version);
int ndpi_netbios_name_interpret(u_char *in, u_int in_len, u_char *out, u_int out_len);
void ndpi_patchIPv6Address(char *str);
diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h
index d61bd9c34..d7748ad8b 100644
--- a/src/include/ndpi_typedefs.h
+++ b/src/include/ndpi_typedefs.h
@@ -489,7 +489,7 @@ struct ndpi_vxlanhdr {
typedef struct message {
u_int8_t *buffer;
- u_int buffer_len, buffer_used, max_expected;
+ u_int buffer_len, buffer_used;
u_int32_t next_seq[2]; /* Directions */
} message_t;
@@ -557,12 +557,6 @@ struct ndpi_id_struct {
/* NDPI_PROTOCOL_GNUTELLA */
u_int32_t gnutella_ts;
- /* NDPI_PROTOCOL_THUNDER */
- u_int32_t thunder_ts;
-
- /* NDPI_PROTOCOL_ZATTOO */
- u_int32_t zattoo_ts;
-
/* NDPI_PROTOCOL_JABBER */
u_int32_t jabber_stun_or_ft_ts;
@@ -593,6 +587,17 @@ struct ndpi_id_struct {
/* ************************************************** */
struct ndpi_flow_tcp_struct {
+
+ /* NDPI_PROTOCOL_MAIL_SMTP */
+ /* NDPI_PROTOCOL_MAIL_POP */
+ /* NDPI_PROTOCOL_MAIL_IMAP */
+ /* NDPI_PROTOCOL_MAIL_FTP */
+ /* TODO: something clever to save memory */
+ struct {
+ u_int8_t auth_found:1, auth_failed:1, auth_tls:1, auth_done:1, _pad:4;
+ char username[32], password[16];
+ } ftp_imap_pop_smtp;
+
/* NDPI_PROTOCOL_MAIL_SMTP */
u_int16_t smtp_command_bitmask;
@@ -604,7 +609,6 @@ struct ndpi_flow_tcp_struct {
/* NDPI_PROTOCOL_IRC */
u_int8_t irc_stage;
- u_int8_t irc_port;
/* NDPI_PROTOCOL_H323 */
u_int8_t h323_valid_packets;
@@ -1078,11 +1082,7 @@ struct ndpi_detection_module_struct {
u_int32_t irc_timeout;
/* gnutella parameters */
u_int32_t gnutella_timeout;
- /* thunder parameters */
- u_int32_t thunder_timeout;
/* rstp */
- u_int32_t orb_rstp_ts_timeout;
- u_int32_t zattoo_connection_timeout;
u_int32_t jabber_stun_timeout;
u_int32_t jabber_file_transfer_timeout;
u_int8_t ip_version_limit;
@@ -1219,12 +1219,11 @@ struct ndpi_flow_struct {
*/
struct {
ndpi_http_method method;
- char *url, *content_type /* response */, *request_content_type /* e.g. for POST */, *user_agent;
- u_int8_t num_request_headers, num_response_headers;
u_int8_t request_version; /* 0=1.0 and 1=1.1. Create an enum for this? */
u_int16_t response_status_code; /* 200, 404, etc. */
- u_char detected_os[32]; /* Via HTTP/QUIC User-Agent */
- u_char nat_ip[24]; /* Via HTTP X-Forwarded-For */
+ char *url, *content_type /* response */, *request_content_type /* e.g. for POST */, *user_agent;
+ char *detected_os; /* Via HTTP/QUIC User-Agent */
+ char *nat_ip; /* Via HTTP X-Forwarded-For */
} http;
/*
@@ -1242,16 +1241,6 @@ struct ndpi_flow_struct {
u_int16_t num_processed_pkts;
} stun;
- /* TODO: something clever to save memory */
- struct {
- u_int8_t auth_found:1, auth_failed:1, auth_tls:1, auth_done:1, _pad:4;
- char username[32], password[16];
- } ftp_imap_pop_smtp;
-
- struct {
- u_int8_t bt_check_performed;
- } bittorrent;
-
union {
/* the only fields useful for nDPI and ntopng */
struct {
@@ -1270,8 +1259,6 @@ struct ndpi_flow_struct {
} kerberos;
struct {
- char ssl_version_str[12];
- u_int16_t ssl_version, server_names_len;
char *server_names, *alpn, *tls_supported_versions, *issuerDN, *subjectDN;
u_int32_t notBefore, notAfter;
char ja3_client[33], ja3_server[33];
@@ -1287,6 +1274,8 @@ struct ndpi_flow_struct {
struct tls_heuristics browser_heuristics;
+ u_int16_t ssl_version, server_names_len;
+
struct {
u_int16_t cipher_suite;
char *esni;
@@ -1311,6 +1300,9 @@ struct ndpi_flow_struct {
char version[32];
} ubntac2;
+ /* In TLS.Bittorent flows there is no hash.
+ Nonetheless, we must pay attention to NOT write to /read from this field
+ with these flows */
struct {
/* Bittorrent hash */
u_char hash[20];
@@ -1337,6 +1329,7 @@ struct ndpi_flow_struct {
u_int16_t byte_counter[2];
/* NDPI_PROTOCOL_BITTORRENT */
u_int8_t bittorrent_stage; // can be 0 - 255
+ u_int8_t bt_check_performed : 1;
/* NDPI_PROTOCOL_DIRECTCONNECT */
u_int8_t directconnect_stage:2; // 0 - 1
diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
index ad70e59c6..7e1829b22 100644
--- a/src/lib/ndpi_main.c
+++ b/src/lib/ndpi_main.c
@@ -2377,8 +2377,6 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs
ndpi_str->irc_timeout = NDPI_IRC_CONNECTION_TIMEOUT * ndpi_str->ticks_per_second;
ndpi_str->gnutella_timeout = NDPI_GNUTELLA_CONNECTION_TIMEOUT * ndpi_str->ticks_per_second;
- ndpi_str->thunder_timeout = NDPI_THUNDER_CONNECTION_TIMEOUT * ndpi_str->ticks_per_second;
- ndpi_str->zattoo_connection_timeout = NDPI_ZATTOO_CONNECTION_TIMEOUT * ndpi_str->ticks_per_second;
ndpi_str->jabber_stun_timeout = NDPI_JABBER_STUN_TIMEOUT * ndpi_str->ticks_per_second;
ndpi_str->jabber_file_transfer_timeout = NDPI_JABBER_FT_TIMEOUT * ndpi_str->ticks_per_second;
@@ -4322,6 +4320,12 @@ void ndpi_free_flow_data(struct ndpi_flow_struct* flow) {
if(flow->http.user_agent)
ndpi_free(flow->http.user_agent);
+ if(flow->http.nat_ip)
+ ndpi_free(flow->http.nat_ip);
+
+ if(flow->http.detected_os)
+ ndpi_free(flow->http.detected_os);
+
if(flow->kerberos_buf.pktbuf)
ndpi_free(flow->kerberos_buf.pktbuf);
@@ -4918,12 +4922,12 @@ int ndpi_search_into_bittorrent_cache(struct ndpi_detection_module_struct *ndpi_
struct ndpi_flow_struct *flow,
/* Parameters below need to be in network byte order */
u_int32_t saddr, u_int16_t sport, u_int32_t daddr, u_int16_t dport) {
- if((!flow->bittorrent.bt_check_performed /* Do the check once */) && ndpi_struct->bittorrent_cache) {
+ if((!flow->bt_check_performed /* Do the check once */) && ndpi_struct->bittorrent_cache) {
u_int16_t cached_proto;
u_int8_t found = 0;
u_int32_t key1, key2;
- flow->bittorrent.bt_check_performed = 1;
+ flow->bt_check_performed = 1;
/* Check cached communications */
key1 = ndpi_bittorrent_hash_funct(saddr, sport), key2 = ndpi_bittorrent_hash_funct(daddr, dport);
@@ -7286,9 +7290,9 @@ u_int8_t ndpi_extra_dissection_possible(struct ndpi_detection_module_struct *ndp
case NDPI_PROTOCOL_MAIL_POP:
case NDPI_PROTOCOL_MAIL_IMAP:
case NDPI_PROTOCOL_MAIL_SMTP:
- if(flow->ftp_imap_pop_smtp.password[0] == '\0' &&
- flow->ftp_imap_pop_smtp.auth_tls == 0 &&
- flow->ftp_imap_pop_smtp.auth_done == 0)
+ if(flow->l4.tcp.ftp_imap_pop_smtp.password[0] == '\0' &&
+ flow->l4.tcp.ftp_imap_pop_smtp.auth_tls == 0 &&
+ flow->l4.tcp.ftp_imap_pop_smtp.auth_done == 0)
return(1);
break;
diff --git a/src/lib/ndpi_utils.c b/src/lib/ndpi_utils.c
index 0a2eba307..e24405d73 100644
--- a/src/lib/ndpi_utils.c
+++ b/src/lib/ndpi_utils.c
@@ -890,21 +890,24 @@ const char* ndpi_get_flow_info(struct ndpi_flow_struct const * const flow,
/* ********************************** */
-char* ndpi_ssl_version2str(struct ndpi_flow_struct *flow,
+char* ndpi_ssl_version2str(char *buf, int buf_len,
u_int16_t version, u_int8_t *unknown_tls_version) {
if(unknown_tls_version)
*unknown_tls_version = 0;
+ if(buf == NULL || buf_len <= 1)
+ return NULL;
+
switch(version) {
- case 0x0300: return("SSLv3");
- case 0x0301: return("TLSv1");
- case 0x0302: return("TLSv1.1");
- case 0x0303: return("TLSv1.2");
- case 0x0304: return("TLSv1.3");
- case 0XFB1A: return("TLSv1.3 (Fizz)"); /* https://engineering.fb.com/security/fizz/ */
- case 0XFEFF: return("DTLSv1.0");
- case 0XFEFD: return("DTLSv1.2");
+ case 0x0300: strncpy(buf, "SSLv3", buf_len); buf[buf_len - 1] = '\0'; return buf;
+ case 0x0301: strncpy(buf, "TLSv1", buf_len); buf[buf_len - 1] = '\0'; return buf;
+ case 0x0302: strncpy(buf, "TLSv1.1", buf_len); buf[buf_len - 1] = '\0'; return buf;
+ case 0x0303: strncpy(buf, "TLSv1.2", buf_len); buf[buf_len - 1] = '\0'; return buf;
+ case 0x0304: strncpy(buf, "TLSv1.3", buf_len); buf[buf_len - 1] = '\0'; return buf;
+ case 0XFB1A: strncpy(buf, "TLSv1.3 (Fizz)", buf_len); buf[buf_len - 1] = '\0'; return buf; /* https://engineering.fb.com/security/fizz/ */
+ case 0XFEFF: strncpy(buf, "DTLSv1.0", buf_len); buf[buf_len - 1] = '\0'; return buf;
+ case 0XFEFD: strncpy(buf, "DTLSv1.2", buf_len); buf[buf_len - 1] = '\0'; return buf;
case 0x0A0A:
case 0x1A1A:
case 0x2A2A:
@@ -920,22 +923,21 @@ char* ndpi_ssl_version2str(struct ndpi_flow_struct *flow,
case 0xCACA:
case 0xDADA:
case 0xEAEA:
- case 0xFAFA: return("GREASE");
+ case 0xFAFA: strncpy(buf, "GREASE", buf_len); buf[buf_len - 1] = '\0'; return buf;
}
- if((version >= 0x7f00) && (version <= 0x7fff))
- return("TLSv1.3 (draft)");
+ if((version >= 0x7f00) && (version <= 0x7fff)) {
+ strncpy(buf, "TLSv1.3 (draft)", buf_len);
+ buf[buf_len - 1] = '\0';
+ return buf;
+ }
if(unknown_tls_version)
*unknown_tls_version = 1;
- if(flow != NULL) {
- snprintf(flow->protos.tls_quic.ssl_version_str,
- sizeof(flow->protos.tls_quic.ssl_version_str), "TLS (%04X)", version);
+ snprintf(buf, buf_len, "TLS (%04X)", version);
- return(flow->protos.tls_quic.ssl_version_str);
- } else
- return("");
+ return buf;
}
/* ***************************************************** */
@@ -1261,7 +1263,9 @@ int ndpi_dpi2json(struct ndpi_detection_module_struct *ndpi_struct,
ndpi_serialize_string_string(serializer, "user_agent", flow->http.user_agent);
if(flow->protos.tls_quic.ssl_version) {
u_int8_t unknown_tls_version;
- char *version = ndpi_ssl_version2str(flow, flow->protos.tls_quic.ssl_version, &unknown_tls_version);
+ char version[16];
+
+ ndpi_ssl_version2str(version, sizeof(version), flow->protos.tls_quic.ssl_version, &unknown_tls_version);
if(!unknown_tls_version)
ndpi_serialize_string_string(serializer, "version", version);
@@ -1276,30 +1280,30 @@ int ndpi_dpi2json(struct ndpi_detection_module_struct *ndpi_struct,
case NDPI_PROTOCOL_MAIL_IMAP:
ndpi_serialize_start_of_block(serializer, "imap");
- ndpi_serialize_string_string(serializer, "user", flow->ftp_imap_pop_smtp.username);
- ndpi_serialize_string_string(serializer, "password", flow->ftp_imap_pop_smtp.password);
+ ndpi_serialize_string_string(serializer, "user", flow->l4.tcp.ftp_imap_pop_smtp.username);
+ ndpi_serialize_string_string(serializer, "password", flow->l4.tcp.ftp_imap_pop_smtp.password);
ndpi_serialize_end_of_block(serializer);
break;
case NDPI_PROTOCOL_MAIL_POP:
ndpi_serialize_start_of_block(serializer, "pop");
- ndpi_serialize_string_string(serializer, "user", flow->ftp_imap_pop_smtp.username);
- ndpi_serialize_string_string(serializer, "password", flow->ftp_imap_pop_smtp.password);
+ ndpi_serialize_string_string(serializer, "user", flow->l4.tcp.ftp_imap_pop_smtp.username);
+ ndpi_serialize_string_string(serializer, "password", flow->l4.tcp.ftp_imap_pop_smtp.password);
ndpi_serialize_end_of_block(serializer);
break;
case NDPI_PROTOCOL_MAIL_SMTP:
ndpi_serialize_start_of_block(serializer, "smtp");
- ndpi_serialize_string_string(serializer, "user", flow->ftp_imap_pop_smtp.username);
- ndpi_serialize_string_string(serializer, "password", flow->ftp_imap_pop_smtp.password);
+ ndpi_serialize_string_string(serializer, "user", flow->l4.tcp.ftp_imap_pop_smtp.username);
+ ndpi_serialize_string_string(serializer, "password", flow->l4.tcp.ftp_imap_pop_smtp.password);
ndpi_serialize_end_of_block(serializer);
break;
case NDPI_PROTOCOL_FTP_CONTROL:
ndpi_serialize_start_of_block(serializer, "ftp");
- ndpi_serialize_string_string(serializer, "user", flow->ftp_imap_pop_smtp.username);
- ndpi_serialize_string_string(serializer, "password", flow->ftp_imap_pop_smtp.password);
- ndpi_serialize_string_uint32(serializer, "auth_failed", flow->ftp_imap_pop_smtp.auth_failed);
+ ndpi_serialize_string_string(serializer, "user", flow->l4.tcp.ftp_imap_pop_smtp.username);
+ ndpi_serialize_string_string(serializer, "password", flow->l4.tcp.ftp_imap_pop_smtp.password);
+ ndpi_serialize_string_uint32(serializer, "auth_failed", flow->l4.tcp.ftp_imap_pop_smtp.auth_failed);
ndpi_serialize_end_of_block(serializer);
break;
@@ -1319,7 +1323,9 @@ int ndpi_dpi2json(struct ndpi_detection_module_struct *ndpi_struct,
struct tm a, b, *before = NULL, *after = NULL;
u_int i, off;
u_int8_t unknown_tls_version;
- char *version = ndpi_ssl_version2str(flow, flow->protos.tls_quic.ssl_version, &unknown_tls_version);
+ char version[16];
+
+ ndpi_ssl_version2str(version, sizeof(version), flow->protos.tls_quic.ssl_version, &unknown_tls_version);
if(flow->protos.tls_quic.notBefore)
before = gmtime_r((const time_t *)&flow->protos.tls_quic.notBefore, &a);
diff --git a/src/lib/protocols/ftp_control.c b/src/lib/protocols/ftp_control.c
index 2b6f1396f..81e475041 100644
--- a/src/lib/protocols/ftp_control.c
+++ b/src/lib/protocols/ftp_control.c
@@ -50,23 +50,23 @@ static int ndpi_ftp_control_check_request(struct ndpi_detection_module_struct *n
#endif
if(ndpi_match_strprefix(payload, payload_len, "USER")) {
- ndpi_user_pwd_payload_copy((u_int8_t*)flow->ftp_imap_pop_smtp.username,
- sizeof(flow->ftp_imap_pop_smtp.username), 5,
+ ndpi_user_pwd_payload_copy((u_int8_t*)flow->l4.tcp.ftp_imap_pop_smtp.username,
+ sizeof(flow->l4.tcp.ftp_imap_pop_smtp.username), 5,
payload, payload_len);
ndpi_set_risk(ndpi_struct, flow, NDPI_CLEAR_TEXT_CREDENTIALS);
return 1;
}
if(ndpi_match_strprefix(payload, payload_len, "PASS")) {
- ndpi_user_pwd_payload_copy((u_int8_t*)flow->ftp_imap_pop_smtp.password,
- sizeof(flow->ftp_imap_pop_smtp.password), 5,
+ ndpi_user_pwd_payload_copy((u_int8_t*)flow->l4.tcp.ftp_imap_pop_smtp.password,
+ sizeof(flow->l4.tcp.ftp_imap_pop_smtp.password), 5,
payload, payload_len);
return 1;
}
if(ndpi_match_strprefix(payload, payload_len, "AUTH") ||
ndpi_match_strprefix(payload, payload_len, "auth")) {
- flow->ftp_imap_pop_smtp.auth_found = 1;
+ flow->l4.tcp.ftp_imap_pop_smtp.auth_found = 1;
return 1;
}
/* ***************************************************** */
@@ -562,14 +562,14 @@ static int ndpi_ftp_control_check_response(struct ndpi_flow_struct *flow,
case '2':
case '3':
case '6':
- if(flow->ftp_imap_pop_smtp.auth_found == 1)
- flow->ftp_imap_pop_smtp.auth_tls = 1;
+ if(flow->l4.tcp.ftp_imap_pop_smtp.auth_found == 1)
+ flow->l4.tcp.ftp_imap_pop_smtp.auth_tls = 1;
return(1);
break;
case '4':
case '5':
- flow->ftp_imap_pop_smtp.auth_failed = 1;
+ flow->l4.tcp.ftp_imap_pop_smtp.auth_failed = 1;
return(1);
break;
}
@@ -632,11 +632,11 @@ static void ndpi_check_ftp_control(struct ndpi_detection_module_struct *ndpi_str
#ifdef FTP_DEBUG
printf("%s() [user: %s][pwd: %s]\n", __FUNCTION__,
- flow->ftp_imap_pop_smtp.username, flow->ftp_imap_pop_smtp.password);
+ flow->l4.tcp.ftp_imap_pop_smtp.username, flow->l4.tcp.ftp_imap_pop_smtp.password);
#endif
- if(flow->ftp_imap_pop_smtp.password[0] == '\0' &&
- flow->ftp_imap_pop_smtp.auth_tls == 0) /* TODO: any values on dissecting TLS handshake? */
+ if(flow->l4.tcp.ftp_imap_pop_smtp.password[0] == '\0' &&
+ flow->l4.tcp.ftp_imap_pop_smtp.auth_tls == 0) /* TODO: any values on dissecting TLS handshake? */
flow->ftp_control_stage = 0;
else
ndpi_int_ftp_control_add_connection(ndpi_struct, flow);
diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c
index 105ef1bca..b2116a8fe 100644
--- a/src/lib/protocols/http.c
+++ b/src/lib/protocols/http.c
@@ -370,8 +370,9 @@ static void setHttpUserAgent(struct ndpi_detection_module_struct *ndpi_struct,
/* Good reference for future implementations:
* https://github.com/ua-parser/uap-core/blob/master/regexes.yaml */
- snprintf((char*)flow->http.detected_os,
- sizeof(flow->http.detected_os), "%s", ua);
+ if(flow->http.detected_os == NULL) {
+ flow->http.detected_os = ndpi_strdup(ua);
+ }
}
/* ************************************************************* */
@@ -606,9 +607,14 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
if(strlen(flow->host_server_name) > 0) ndpi_check_dga_name(ndpi_struct, flow, flow->host_server_name, 1);
if(packet->forwarded_line.ptr) {
- len = ndpi_min(packet->forwarded_line.len, sizeof(flow->http.nat_ip)-1);
- strncpy((char*)flow->http.nat_ip, (char*)packet->forwarded_line.ptr, len);
- flow->http.nat_ip[len] = '\0';
+ if(flow->http.nat_ip == NULL) {
+ len = packet->forwarded_line.len;
+ flow->http.nat_ip = ndpi_malloc(len + 1);
+ if(flow->http.nat_ip == NULL) {
+ strncpy(flow->http.nat_ip, (char*)packet->forwarded_line.ptr, len);
+ flow->http.nat_ip[len] = '\0';
+ }
+ }
}
ndpi_http_parse_subprotocol(ndpi_struct, flow);
@@ -1074,9 +1080,6 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
else
flow->http.request_version = 0;
- /* Set the first found headers in request */
- flow->http.num_request_headers = packet->http_num_headers;
-
/* Check for Ookla */
if((packet->referer_line.len > 0)
&& ndpi_strnstr((const char *)packet->referer_line.ptr, "www.speedtest.net", packet->referer_line.len)) {
@@ -1155,9 +1158,6 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
ndpi_parse_packet_line_info(ndpi_struct, flow);
- // Add more found HTTP request headers.
- flow->http.num_request_headers+=packet->http_num_headers;
-
if(packet->parsed_lines <= 1) {
/* wait some packets in case request is split over more than 2 packets */
if(flow->packet_counter < 5) {
@@ -1213,9 +1213,6 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
ndpi_parse_packet_line_info(ndpi_struct, flow);
check_content_type_and_change_protocol(ndpi_struct, flow);
- if(packet->packet_direction == 1 /* server -> client */)
- flow->http.num_response_headers += packet->http_num_headers; /* flow structs are initialized with zeros */
-
if(packet->empty_line_position_set != 0 || flow->l4.tcp.http_empty_line_seen == 1) {
NDPI_LOG_DBG2(ndpi_struct, "empty line. check_http_payload\n");
check_http_payload(ndpi_struct, flow);
diff --git a/src/lib/protocols/mail_imap.c b/src/lib/protocols/mail_imap.c
index 8a4e8fa2e..317ac1848 100644
--- a/src/lib/protocols/mail_imap.c
+++ b/src/lib/protocols/mail_imap.c
@@ -177,16 +177,16 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
if(user) {
char *pwd;
- snprintf(flow->ftp_imap_pop_smtp.username,
- sizeof(flow->ftp_imap_pop_smtp.username),
+ snprintf(flow->l4.tcp.ftp_imap_pop_smtp.username,
+ sizeof(flow->l4.tcp.ftp_imap_pop_smtp.username),
"%s", user);
ndpi_set_risk(ndpi_struct, flow, NDPI_CLEAR_TEXT_CREDENTIALS);
pwd = strtok_r(NULL, " \"\r\n", &saveptr);
if(pwd) {
- snprintf(flow->ftp_imap_pop_smtp.password,
- sizeof(flow->ftp_imap_pop_smtp.password),
+ snprintf(flow->l4.tcp.ftp_imap_pop_smtp.password,
+ sizeof(flow->l4.tcp.ftp_imap_pop_smtp.password),
"%s", pwd);
}
}
@@ -320,7 +320,7 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
|| (flow->l4.tcp.mail_imap_stage == 5)
|| (flow->l4.tcp.mail_imap_stage == 7)
) {
- if((flow->ftp_imap_pop_smtp.username[0] != '\0')
+ if((flow->l4.tcp.ftp_imap_pop_smtp.username[0] != '\0')
|| (flow->l4.tcp.mail_imap_stage >= 7)) {
NDPI_LOG_INFO(ndpi_struct, "found MAIL_IMAP\n");
ndpi_int_mail_imap_add_connection(ndpi_struct, flow);
diff --git a/src/lib/protocols/mail_pop.c b/src/lib/protocols/mail_pop.c
index 483c4da35..125fe5da1 100644
--- a/src/lib/protocols/mail_pop.c
+++ b/src/lib/protocols/mail_pop.c
@@ -77,8 +77,8 @@ static int ndpi_int_mail_pop_check_for_client_commands(struct ndpi_detection_mod
&& (packet->payload[1] == 'S' || packet->payload[1] == 's')
&& (packet->payload[2] == 'E' || packet->payload[2] == 'e')
&& (packet->payload[3] == 'R' || packet->payload[3] == 'r')) {
- ndpi_user_pwd_payload_copy((u_int8_t*)flow->ftp_imap_pop_smtp.username,
- sizeof(flow->ftp_imap_pop_smtp.username), 5,
+ ndpi_user_pwd_payload_copy((u_int8_t*)flow->l4.tcp.ftp_imap_pop_smtp.username,
+ sizeof(flow->l4.tcp.ftp_imap_pop_smtp.username), 5,
packet->payload, packet->payload_packet_len);
ndpi_set_risk(ndpi_struct, flow, NDPI_CLEAR_TEXT_CREDENTIALS);
@@ -88,8 +88,8 @@ static int ndpi_int_mail_pop_check_for_client_commands(struct ndpi_detection_mod
&& (packet->payload[1] == 'A' || packet->payload[1] == 'a')
&& (packet->payload[2] == 'S' || packet->payload[2] == 's')
&& (packet->payload[3] == 'S' || packet->payload[3] == 's')) {
- ndpi_user_pwd_payload_copy((u_int8_t*)flow->ftp_imap_pop_smtp.password,
- sizeof(flow->ftp_imap_pop_smtp.password), 5,
+ ndpi_user_pwd_payload_copy((u_int8_t*)flow->l4.tcp.ftp_imap_pop_smtp.password,
+ sizeof(flow->l4.tcp.ftp_imap_pop_smtp.password), 5,
packet->payload, packet->payload_packet_len);
ndpi_set_risk(ndpi_struct, flow, NDPI_CLEAR_TEXT_CREDENTIALS);
@@ -182,7 +182,7 @@ void ndpi_search_mail_pop_tcp(struct ndpi_detection_module_struct
if(flow->l4.tcp.mail_pop_stage > 0) {
NDPI_LOG_INFO(ndpi_struct, "mail_pop identified\n");
- if((flow->ftp_imap_pop_smtp.password[0] != '\0')
+ if((flow->l4.tcp.ftp_imap_pop_smtp.password[0] != '\0')
|| (flow->l4.tcp.mail_pop_stage > 3)) {
ndpi_int_mail_pop_add_connection(ndpi_struct, flow);
popInitExtraPacketProcessing(flow);
@@ -222,7 +222,7 @@ int ndpi_extra_search_mail_pop_tcp(struct ndpi_detection_module_struct *ndpi_str
ndpi_search_mail_pop_tcp(ndpi_struct, flow);
- rc = (flow->ftp_imap_pop_smtp.password[0] == '\0') ? 1 : 0;
+ rc = (flow->l4.tcp.ftp_imap_pop_smtp.password[0] == '\0') ? 1 : 0;
#ifdef POP_DEBUG
printf("**** %s() [rc: %d]\n", __FUNCTION__, rc);
diff --git a/src/lib/protocols/mail_smtp.c b/src/lib/protocols/mail_smtp.c
index e1d98dd35..ce3ba4058 100644
--- a/src/lib/protocols/mail_smtp.c
+++ b/src/lib/protocols/mail_smtp.c
@@ -93,19 +93,19 @@ static void get_credentials_auth_plain(struct ndpi_detection_module_struct *ndpi
user_len = i - 1;
}
if(user_len > 0) {
- user_len = ndpi_min(user_len, sizeof(flow->ftp_imap_pop_smtp.username) - 1);
+ user_len = ndpi_min(user_len, sizeof(flow->l4.tcp.ftp_imap_pop_smtp.username) - 1);
- memcpy(flow->ftp_imap_pop_smtp.username, out + 1, user_len);
- flow->ftp_imap_pop_smtp.username[user_len] = '\0';
+ memcpy(flow->l4.tcp.ftp_imap_pop_smtp.username, out + 1, user_len);
+ flow->l4.tcp.ftp_imap_pop_smtp.username[user_len] = '\0';
ndpi_set_risk(ndpi_struct, flow, NDPI_CLEAR_TEXT_CREDENTIALS);
if(1 + user_len + 1 < out_len) {
unsigned int pwd_len;
- pwd_len = ndpi_min(out_len - (1 + user_len + 1), sizeof(flow->ftp_imap_pop_smtp.password) - 1);
- memcpy(flow->ftp_imap_pop_smtp.password, out + 1 + user_len + 1, pwd_len);
- flow->ftp_imap_pop_smtp.password[pwd_len] = '\0';
+ pwd_len = ndpi_min(out_len - (1 + user_len + 1), sizeof(flow->l4.tcp.ftp_imap_pop_smtp.password) - 1);
+ memcpy(flow->l4.tcp.ftp_imap_pop_smtp.password, out + 1 + user_len + 1, pwd_len);
+ flow->l4.tcp.ftp_imap_pop_smtp.password[pwd_len] = '\0';
}
}
ndpi_free(out);
@@ -182,25 +182,25 @@ void ndpi_search_mail_smtp_tcp(struct ndpi_detection_module_struct *ndpi_struct,
&& (packet->line[a].ptr[3] == 'O' || packet->line[a].ptr[3] == 'o')
&& packet->line[a].ptr[4] == ' ') {
flow->l4.tcp.smtp_command_bitmask |= SMTP_BIT_HELO_EHLO;
- flow->ftp_imap_pop_smtp.auth_found = 0;
+ flow->l4.tcp.ftp_imap_pop_smtp.auth_found = 0;
} else if((packet->line[a].ptr[0] == 'M' || packet->line[a].ptr[0] == 'm')
&& (packet->line[a].ptr[1] == 'A' || packet->line[a].ptr[1] == 'a')
&& (packet->line[a].ptr[2] == 'I' || packet->line[a].ptr[2] == 'i')
&& (packet->line[a].ptr[3] == 'L' || packet->line[a].ptr[3] == 'l')
&& packet->line[a].ptr[4] == ' ') {
flow->l4.tcp.smtp_command_bitmask |= SMTP_BIT_MAIL;
- flow->ftp_imap_pop_smtp.auth_found = 0;
+ flow->l4.tcp.ftp_imap_pop_smtp.auth_found = 0;
/* We shouldn't be here if there are credentials */
- flow->ftp_imap_pop_smtp.auth_done = 1;
+ flow->l4.tcp.ftp_imap_pop_smtp.auth_done = 1;
} else if((packet->line[a].ptr[0] == 'R' || packet->line[a].ptr[0] == 'r')
&& (packet->line[a].ptr[1] == 'C' || packet->line[a].ptr[1] == 'c')
&& (packet->line[a].ptr[2] == 'P' || packet->line[a].ptr[2] == 'p')
&& (packet->line[a].ptr[3] == 'T' || packet->line[a].ptr[3] == 't')
&& packet->line[a].ptr[4] == ' ') {
flow->l4.tcp.smtp_command_bitmask |= SMTP_BIT_RCPT;
- flow->ftp_imap_pop_smtp.auth_found = 0;
+ flow->l4.tcp.ftp_imap_pop_smtp.auth_found = 0;
/* We shouldn't be here if there are credentials */
- flow->ftp_imap_pop_smtp.auth_done = 1;
+ flow->l4.tcp.ftp_imap_pop_smtp.auth_done = 1;
} else if((packet->line[a].ptr[0] == 'A' || packet->line[a].ptr[0] == 'a')
&& (packet->line[a].ptr[1] == 'U' || packet->line[a].ptr[1] == 'u')
&& (packet->line[a].ptr[2] == 'T' || packet->line[a].ptr[2] == 't')
@@ -209,7 +209,7 @@ void ndpi_search_mail_smtp_tcp(struct ndpi_detection_module_struct *ndpi_struct,
#ifdef SMTP_DEBUG
printf("%s() AUTH [%.*s]\n", __FUNCTION__, packet->line[a].len, packet->line[a].ptr);
#endif
- flow->ftp_imap_pop_smtp.auth_found = 1;
+ flow->l4.tcp.ftp_imap_pop_smtp.auth_found = 1;
if(packet->line[a].len >= 6) {
if(packet->line[a].ptr[5] == 'L' || packet->line[a].ptr[5] == 'l') {
flow->l4.tcp.smtp_command_bitmask |= SMTP_BIT_AUTH_LOGIN;
@@ -219,7 +219,7 @@ void ndpi_search_mail_smtp_tcp(struct ndpi_detection_module_struct *ndpi_struct,
/* AUTH PLAIN: username and pwd here */
get_credentials_auth_plain(ndpi_struct, flow,
packet->line[a].ptr, packet->line[a].len);
- flow->ftp_imap_pop_smtp.auth_done = 1;
+ flow->l4.tcp.ftp_imap_pop_smtp.auth_done = 1;
}
}
} else {
@@ -228,9 +228,9 @@ void ndpi_search_mail_smtp_tcp(struct ndpi_detection_module_struct *ndpi_struct,
printf("%s() => [%.*s]\n", __FUNCTION__, packet->line[a].len, packet->line[a].ptr);
#endif
- if(flow->ftp_imap_pop_smtp.auth_found &&
+ if(flow->l4.tcp.ftp_imap_pop_smtp.auth_found &&
(flow->l4.tcp.smtp_command_bitmask & SMTP_BIT_AUTH_LOGIN)) {
- if(flow->ftp_imap_pop_smtp.username[0] == '\0') {
+ if(flow->l4.tcp.ftp_imap_pop_smtp.username[0] == '\0') {
/* Username */
u_int8_t buf[48];
u_char *out;
@@ -240,22 +240,22 @@ void ndpi_search_mail_smtp_tcp(struct ndpi_detection_module_struct *ndpi_struct,
packet->line[a].ptr, packet->line[a].len);
#ifdef SMTP_DEBUG
- printf("%s() => [auth: %u] (username) [%s]\n", __FUNCTION__, flow->ftp_imap_pop_smtp.auth_found, buf);
+ printf("%s() => [auth: %u] (username) [%s]\n", __FUNCTION__, flow->l4.tcp.ftp_imap_pop_smtp.auth_found, buf);
#endif
out = ndpi_base64_decode((const u_char*)buf, (size_t)strlen((const char*)buf), &out_len);
if(out) {
- size_t len = ndpi_min(out_len, sizeof(flow->ftp_imap_pop_smtp.username) - 1);
+ size_t len = ndpi_min(out_len, sizeof(flow->l4.tcp.ftp_imap_pop_smtp.username) - 1);
- memcpy(flow->ftp_imap_pop_smtp.username, out, len);
- flow->ftp_imap_pop_smtp.username[len] = '\0';
+ memcpy(flow->l4.tcp.ftp_imap_pop_smtp.username, out, len);
+ flow->l4.tcp.ftp_imap_pop_smtp.username[len] = '\0';
ndpi_free(out);
}
ndpi_set_risk(ndpi_struct, flow, NDPI_CLEAR_TEXT_CREDENTIALS);
- } else if(flow->ftp_imap_pop_smtp.password[0] == '\0') {
+ } else if(flow->l4.tcp.ftp_imap_pop_smtp.password[0] == '\0') {
/* Password */
u_int8_t buf[48];
u_char *out;
@@ -265,23 +265,23 @@ void ndpi_search_mail_smtp_tcp(struct ndpi_detection_module_struct *ndpi_struct,
packet->line[a].ptr, packet->line[a].len);
#ifdef SMTP_DEBUG
- printf("%s() => [auth: %u] (password) [%s]\n", __FUNCTION__, flow->ftp_imap_pop_smtp.auth_found, buf);
+ printf("%s() => [auth: %u] (password) [%s]\n", __FUNCTION__, flow->l4.tcp.ftp_imap_pop_smtp.auth_found, buf);
#endif
out = ndpi_base64_decode((const u_char*)buf, (size_t)strlen((const char*)buf), &out_len);
if(out) {
- size_t len = ndpi_min(out_len, sizeof(flow->ftp_imap_pop_smtp.password) - 1);
+ size_t len = ndpi_min(out_len, sizeof(flow->l4.tcp.ftp_imap_pop_smtp.password) - 1);
- memcpy(flow->ftp_imap_pop_smtp.password, out, len);
- flow->ftp_imap_pop_smtp.password[len] = '\0';
+ memcpy(flow->l4.tcp.ftp_imap_pop_smtp.password, out, len);
+ flow->l4.tcp.ftp_imap_pop_smtp.password[len] = '\0';
ndpi_free(out);
}
ndpi_set_risk(ndpi_struct, flow, NDPI_CLEAR_TEXT_CREDENTIALS);
- flow->ftp_imap_pop_smtp.auth_done = 1;
+ flow->l4.tcp.ftp_imap_pop_smtp.auth_done = 1;
} else {
flow->host_server_name[0] = '\0';
NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
@@ -302,8 +302,8 @@ void ndpi_search_mail_smtp_tcp(struct ndpi_detection_module_struct *ndpi_struct,
&& (packet->line[a].ptr[6] == 'L' || packet->line[a].ptr[6] == 'l')
&& (packet->line[a].ptr[7] == 'S' || packet->line[a].ptr[7] == 's')) {
flow->l4.tcp.smtp_command_bitmask |= SMTP_BIT_STARTTLS;
- flow->ftp_imap_pop_smtp.auth_tls = 1;
- flow->ftp_imap_pop_smtp.auth_done = 1;
+ flow->l4.tcp.ftp_imap_pop_smtp.auth_tls = 1;
+ flow->l4.tcp.ftp_imap_pop_smtp.auth_done = 1;
}
}
@@ -341,7 +341,7 @@ void ndpi_search_mail_smtp_tcp(struct ndpi_detection_module_struct *ndpi_struct,
#ifdef SMTP_DEBUG
printf("%s() [bit_count: %u][%s]\n", __FUNCTION__,
- bit_count, flow->ftp_imap_pop_smtp.password);
+ bit_count, flow->l4.tcp.ftp_imap_pop_smtp.password);
#endif
/* Only if we don't have already set the protocol via hostname matching */
@@ -379,7 +379,7 @@ int ndpi_extra_search_mail_smtp_tcp(struct ndpi_detection_module_struct *ndpi_st
ndpi_search_mail_smtp_tcp(ndpi_struct, flow);
- rc = (flow->ftp_imap_pop_smtp.password[0] == '\0') ? 1 : 0;
+ rc = (flow->l4.tcp.ftp_imap_pop_smtp.password[0] == '\0') ? 1 : 0;
#ifdef SMTP_DEBUG
printf("**** %s() [rc: %d]\n", __FUNCTION__, rc);
diff --git a/src/lib/protocols/thunder.c b/src/lib/protocols/thunder.c
index 31fa720d4..1b33c41d7 100644
--- a/src/lib/protocols/thunder.c
+++ b/src/lib/protocols/thunder.c
@@ -32,18 +32,7 @@
static void ndpi_int_thunder_add_connection(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow/* , ndpi_protocol_type_t protocol_type */)
{
- struct ndpi_packet_struct *packet = &ndpi_struct->packet;
- struct ndpi_id_struct *src = flow->src;
- struct ndpi_id_struct *dst = flow->dst;
-
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_THUNDER, NDPI_PROTOCOL_UNKNOWN);
-
- if (src != NULL) {
- src->thunder_ts = packet->current_time_ms;
- }
- if (dst != NULL) {
- dst->thunder_ts = packet->current_time_ms;
- }
}
@@ -149,22 +138,6 @@ void ndpi_int_search_thunder_http(struct ndpi_detection_module_struct
struct ndpi_id_struct *src = flow->src;
struct ndpi_id_struct *dst = flow->dst;
-
- if (flow->detected_protocol_stack[0] == NDPI_PROTOCOL_THUNDER) {
- if (src != NULL && ((u_int32_t)
- (packet->current_time_ms - src->thunder_ts) < ndpi_struct->thunder_timeout)) {
- NDPI_LOG_DBG2(ndpi_struct,
- "thunder : save src connection packet detected\n");
- src->thunder_ts = packet->current_time_ms;
- } else if (dst != NULL && ((u_int32_t)
- (packet->current_time_ms - dst->thunder_ts) < ndpi_struct->thunder_timeout)) {
- NDPI_LOG_DBG2(ndpi_struct,
- "thunder : save dst connection packet detected\n");
- dst->thunder_ts = packet->current_time_ms;
- }
- return;
- }
-
if (packet->payload_packet_len > 5
&& memcmp(packet->payload, "GET /", 5) == 0 && NDPI_SRC_OR_DST_HAS_PROTOCOL(src, dst, NDPI_PROTOCOL_THUNDER)) {
NDPI_LOG_DBG2(ndpi_struct, "HTTP packet detected\n");
diff --git a/src/lib/protocols/tls.c b/src/lib/protocols/tls.c
index 87553de87..b8f8159e5 100644
--- a/src/lib/protocols/tls.c
+++ b/src/lib/protocols/tls.c
@@ -2030,6 +2030,7 @@ int processClientServerHello(struct ndpi_detection_module_struct *ndpi_struct,
u_int16_t s_offset = offset+extension_offset;
u_int8_t version_len = packet->payload[s_offset];
char version_str[256];
+ char buf_ver_tmp[16];
size_t version_str_len = 0;
version_str[0] = 0;
#ifdef DEBUG_TLS
@@ -2049,14 +2050,14 @@ int processClientServerHello(struct ndpi_detection_module_struct *ndpi_struct,
#ifdef DEBUG_TLS
printf("Client TLS [TLS version: %s/0x%04X]\n",
- ndpi_ssl_version2str(flow, tls_version, &unknown_tls_version), tls_version);
+ ndpi_ssl_version2str(buf_ver_tmp, sizeof(buf_ver_tmp), tls_version, &unknown_tls_version), tls_version);
#endif
if((version_str_len+8) < sizeof(version_str)) {
int rc = snprintf(&version_str[version_str_len],
sizeof(version_str) - version_str_len, "%s%s",
(version_str_len > 0) ? "," : "",
- ndpi_ssl_version2str(flow, tls_version, &unknown_tls_version));
+ ndpi_ssl_version2str(buf_ver_tmp, sizeof(buf_ver_tmp), tls_version, &unknown_tls_version));
if(rc <= 0)
break;
else
diff --git a/src/lib/protocols/zattoo.c b/src/lib/protocols/zattoo.c
index 6b3e9625c..b5b305ecb 100644
--- a/src/lib/protocols/zattoo.c
+++ b/src/lib/protocols/zattoo.c
@@ -47,30 +47,16 @@ u_int8_t ndpi_int_zattoo_user_agent_set(struct ndpi_detection_module_struct *ndp
}
#define ZATTOO_DETECTED \
- if (src != NULL) \
- src->zattoo_ts = packet->current_time_ms; \
- if (dst != NULL) \
- dst->zattoo_ts = packet->current_time_ms; \
- \
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_ZATTOO, NDPI_PROTOCOL_UNKNOWN)
void ndpi_search_zattoo(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow)
{
struct ndpi_packet_struct *packet = &ndpi_struct->packet;
- struct ndpi_id_struct *src = flow->src;
- struct ndpi_id_struct *dst = flow->dst;
u_int16_t i;
NDPI_LOG_DBG(ndpi_struct, "search ZATTOO\n");
- if(flow->detected_protocol_stack[0] == NDPI_PROTOCOL_ZATTOO) {
- if(src != NULL && ((u_int32_t) (packet->current_time_ms - src->zattoo_ts) < ndpi_struct->zattoo_connection_timeout))
- src->zattoo_ts = packet->current_time_ms;
- if (dst != NULL && ((u_int32_t) (packet->current_time_ms - dst->zattoo_ts) < ndpi_struct->zattoo_connection_timeout))
- dst->zattoo_ts = packet->current_time_ms;
- return;
- }
/* search over TCP */
if(packet->tcp != NULL) {
if(packet->payload_packet_len > 50 && memcmp(packet->payload, "GET /frontdoor/fd?brand=Zattoo&v=", 33) == 0) {