diff options
author | Ivan Nardi <12729895+IvanNardi@users.noreply.github.com> | 2021-12-22 19:54:06 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-12-22 19:54:06 +0100 |
commit | 91bb77a8806ee2987e856f66674cf3aa8b1d60db (patch) | |
tree | 0f2a06cc9b64a8af221552e87ae771ac882ab4b5 /src | |
parent | 02da143e4567cbfe32b139561ec3a702ce380fc7 (diff) |
A final(?) effort to reduce memory usage per flow (#1389)
Remove some unused fields and re-organize other ones.
In particular:
* Update the parameters of `ndpi_ssl_version2str()` function
* Zattoo, Thunder: these timestamps aren't really used.
* Ftp/mail: these protocols are dissected only over TCP.
* Attention must be paid to TLS.Bittorrent flows to avoid invalid
read/write to `flow->protos.bittorrent.hash` field.
This is the last(?) commit of a long series (see 22241a1d, 227e586e,
730c2360, a8ffcd8b) aiming to reduce library memory consumption.
Before, at nDPI 4.0 (more precisly, at a6b10cf7, because memory stats
were wrong until that commit):
```
nDPI Memory statistics:
nDPI Memory (once): 221.15 KB
Flow Memory (per flow): 2.94 KB
```
Now:
```
nDPI Memory statistics:
nDPI Memory (once): 231.71 KB
Flow Memory (per flow): 1008 B <---------
```
i.e. memory usage per flow has been reduced by 66%, dropping below the
psychological threshold of 1 KB.
To further reduce this value, we probably need to look into #1279:
let's fight this battle another day.
Diffstat (limited to 'src')
-rw-r--r-- | src/include/ndpi_api.h.in | 2 | ||||
-rw-r--r-- | src/include/ndpi_typedefs.h | 49 | ||||
-rw-r--r-- | src/lib/ndpi_main.c | 18 | ||||
-rw-r--r-- | src/lib/ndpi_utils.c | 64 | ||||
-rw-r--r-- | src/lib/protocols/ftp_control.c | 22 | ||||
-rw-r--r-- | src/lib/protocols/http.c | 25 | ||||
-rw-r--r-- | src/lib/protocols/mail_imap.c | 10 | ||||
-rw-r--r-- | src/lib/protocols/mail_pop.c | 12 | ||||
-rw-r--r-- | src/lib/protocols/mail_smtp.c | 58 | ||||
-rw-r--r-- | src/lib/protocols/thunder.c | 27 | ||||
-rw-r--r-- | src/lib/protocols/tls.c | 5 | ||||
-rw-r--r-- | src/lib/protocols/zattoo.c | 14 |
12 files changed, 133 insertions, 173 deletions
diff --git a/src/include/ndpi_api.h.in b/src/include/ndpi_api.h.in index 82010ece5..28f305193 100644 --- a/src/include/ndpi_api.h.in +++ b/src/include/ndpi_api.h.in @@ -975,7 +975,7 @@ extern "C" { /* Return a flow info string (summarized). Does only work for DNS/HTTP/TLS/QUIC. */ const char* ndpi_get_flow_info(struct ndpi_flow_struct const * const flow, ndpi_protocol const * const l7_protocol); - char* ndpi_ssl_version2str(struct ndpi_flow_struct *flow, + char* ndpi_ssl_version2str(char *buf, int buf_len, u_int16_t version, u_int8_t *unknown_tls_version); int ndpi_netbios_name_interpret(u_char *in, u_int in_len, u_char *out, u_int out_len); void ndpi_patchIPv6Address(char *str); diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h index d61bd9c34..d7748ad8b 100644 --- a/src/include/ndpi_typedefs.h +++ b/src/include/ndpi_typedefs.h @@ -489,7 +489,7 @@ struct ndpi_vxlanhdr { typedef struct message { u_int8_t *buffer; - u_int buffer_len, buffer_used, max_expected; + u_int buffer_len, buffer_used; u_int32_t next_seq[2]; /* Directions */ } message_t; @@ -557,12 +557,6 @@ struct ndpi_id_struct { /* NDPI_PROTOCOL_GNUTELLA */ u_int32_t gnutella_ts; - /* NDPI_PROTOCOL_THUNDER */ - u_int32_t thunder_ts; - - /* NDPI_PROTOCOL_ZATTOO */ - u_int32_t zattoo_ts; - /* NDPI_PROTOCOL_JABBER */ u_int32_t jabber_stun_or_ft_ts; @@ -593,6 +587,17 @@ struct ndpi_id_struct { /* ************************************************** */ struct ndpi_flow_tcp_struct { + + /* NDPI_PROTOCOL_MAIL_SMTP */ + /* NDPI_PROTOCOL_MAIL_POP */ + /* NDPI_PROTOCOL_MAIL_IMAP */ + /* NDPI_PROTOCOL_MAIL_FTP */ + /* TODO: something clever to save memory */ + struct { + u_int8_t auth_found:1, auth_failed:1, auth_tls:1, auth_done:1, _pad:4; + char username[32], password[16]; + } ftp_imap_pop_smtp; + /* NDPI_PROTOCOL_MAIL_SMTP */ u_int16_t smtp_command_bitmask; @@ -604,7 +609,6 @@ struct ndpi_flow_tcp_struct { /* NDPI_PROTOCOL_IRC */ u_int8_t irc_stage; - u_int8_t irc_port; /* NDPI_PROTOCOL_H323 */ u_int8_t h323_valid_packets; @@ -1078,11 +1082,7 @@ struct ndpi_detection_module_struct { u_int32_t irc_timeout; /* gnutella parameters */ u_int32_t gnutella_timeout; - /* thunder parameters */ - u_int32_t thunder_timeout; /* rstp */ - u_int32_t orb_rstp_ts_timeout; - u_int32_t zattoo_connection_timeout; u_int32_t jabber_stun_timeout; u_int32_t jabber_file_transfer_timeout; u_int8_t ip_version_limit; @@ -1219,12 +1219,11 @@ struct ndpi_flow_struct { */ struct { ndpi_http_method method; - char *url, *content_type /* response */, *request_content_type /* e.g. for POST */, *user_agent; - u_int8_t num_request_headers, num_response_headers; u_int8_t request_version; /* 0=1.0 and 1=1.1. Create an enum for this? */ u_int16_t response_status_code; /* 200, 404, etc. */ - u_char detected_os[32]; /* Via HTTP/QUIC User-Agent */ - u_char nat_ip[24]; /* Via HTTP X-Forwarded-For */ + char *url, *content_type /* response */, *request_content_type /* e.g. for POST */, *user_agent; + char *detected_os; /* Via HTTP/QUIC User-Agent */ + char *nat_ip; /* Via HTTP X-Forwarded-For */ } http; /* @@ -1242,16 +1241,6 @@ struct ndpi_flow_struct { u_int16_t num_processed_pkts; } stun; - /* TODO: something clever to save memory */ - struct { - u_int8_t auth_found:1, auth_failed:1, auth_tls:1, auth_done:1, _pad:4; - char username[32], password[16]; - } ftp_imap_pop_smtp; - - struct { - u_int8_t bt_check_performed; - } bittorrent; - union { /* the only fields useful for nDPI and ntopng */ struct { @@ -1270,8 +1259,6 @@ struct ndpi_flow_struct { } kerberos; struct { - char ssl_version_str[12]; - u_int16_t ssl_version, server_names_len; char *server_names, *alpn, *tls_supported_versions, *issuerDN, *subjectDN; u_int32_t notBefore, notAfter; char ja3_client[33], ja3_server[33]; @@ -1287,6 +1274,8 @@ struct ndpi_flow_struct { struct tls_heuristics browser_heuristics; + u_int16_t ssl_version, server_names_len; + struct { u_int16_t cipher_suite; char *esni; @@ -1311,6 +1300,9 @@ struct ndpi_flow_struct { char version[32]; } ubntac2; + /* In TLS.Bittorent flows there is no hash. + Nonetheless, we must pay attention to NOT write to /read from this field + with these flows */ struct { /* Bittorrent hash */ u_char hash[20]; @@ -1337,6 +1329,7 @@ struct ndpi_flow_struct { u_int16_t byte_counter[2]; /* NDPI_PROTOCOL_BITTORRENT */ u_int8_t bittorrent_stage; // can be 0 - 255 + u_int8_t bt_check_performed : 1; /* NDPI_PROTOCOL_DIRECTCONNECT */ u_int8_t directconnect_stage:2; // 0 - 1 diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index ad70e59c6..7e1829b22 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -2377,8 +2377,6 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs ndpi_str->irc_timeout = NDPI_IRC_CONNECTION_TIMEOUT * ndpi_str->ticks_per_second; ndpi_str->gnutella_timeout = NDPI_GNUTELLA_CONNECTION_TIMEOUT * ndpi_str->ticks_per_second; - ndpi_str->thunder_timeout = NDPI_THUNDER_CONNECTION_TIMEOUT * ndpi_str->ticks_per_second; - ndpi_str->zattoo_connection_timeout = NDPI_ZATTOO_CONNECTION_TIMEOUT * ndpi_str->ticks_per_second; ndpi_str->jabber_stun_timeout = NDPI_JABBER_STUN_TIMEOUT * ndpi_str->ticks_per_second; ndpi_str->jabber_file_transfer_timeout = NDPI_JABBER_FT_TIMEOUT * ndpi_str->ticks_per_second; @@ -4322,6 +4320,12 @@ void ndpi_free_flow_data(struct ndpi_flow_struct* flow) { if(flow->http.user_agent) ndpi_free(flow->http.user_agent); + if(flow->http.nat_ip) + ndpi_free(flow->http.nat_ip); + + if(flow->http.detected_os) + ndpi_free(flow->http.detected_os); + if(flow->kerberos_buf.pktbuf) ndpi_free(flow->kerberos_buf.pktbuf); @@ -4918,12 +4922,12 @@ int ndpi_search_into_bittorrent_cache(struct ndpi_detection_module_struct *ndpi_ struct ndpi_flow_struct *flow, /* Parameters below need to be in network byte order */ u_int32_t saddr, u_int16_t sport, u_int32_t daddr, u_int16_t dport) { - if((!flow->bittorrent.bt_check_performed /* Do the check once */) && ndpi_struct->bittorrent_cache) { + if((!flow->bt_check_performed /* Do the check once */) && ndpi_struct->bittorrent_cache) { u_int16_t cached_proto; u_int8_t found = 0; u_int32_t key1, key2; - flow->bittorrent.bt_check_performed = 1; + flow->bt_check_performed = 1; /* Check cached communications */ key1 = ndpi_bittorrent_hash_funct(saddr, sport), key2 = ndpi_bittorrent_hash_funct(daddr, dport); @@ -7286,9 +7290,9 @@ u_int8_t ndpi_extra_dissection_possible(struct ndpi_detection_module_struct *ndp case NDPI_PROTOCOL_MAIL_POP: case NDPI_PROTOCOL_MAIL_IMAP: case NDPI_PROTOCOL_MAIL_SMTP: - if(flow->ftp_imap_pop_smtp.password[0] == '\0' && - flow->ftp_imap_pop_smtp.auth_tls == 0 && - flow->ftp_imap_pop_smtp.auth_done == 0) + if(flow->l4.tcp.ftp_imap_pop_smtp.password[0] == '\0' && + flow->l4.tcp.ftp_imap_pop_smtp.auth_tls == 0 && + flow->l4.tcp.ftp_imap_pop_smtp.auth_done == 0) return(1); break; diff --git a/src/lib/ndpi_utils.c b/src/lib/ndpi_utils.c index 0a2eba307..e24405d73 100644 --- a/src/lib/ndpi_utils.c +++ b/src/lib/ndpi_utils.c @@ -890,21 +890,24 @@ const char* ndpi_get_flow_info(struct ndpi_flow_struct const * const flow, /* ********************************** */ -char* ndpi_ssl_version2str(struct ndpi_flow_struct *flow, +char* ndpi_ssl_version2str(char *buf, int buf_len, u_int16_t version, u_int8_t *unknown_tls_version) { if(unknown_tls_version) *unknown_tls_version = 0; + if(buf == NULL || buf_len <= 1) + return NULL; + switch(version) { - case 0x0300: return("SSLv3"); - case 0x0301: return("TLSv1"); - case 0x0302: return("TLSv1.1"); - case 0x0303: return("TLSv1.2"); - case 0x0304: return("TLSv1.3"); - case 0XFB1A: return("TLSv1.3 (Fizz)"); /* https://engineering.fb.com/security/fizz/ */ - case 0XFEFF: return("DTLSv1.0"); - case 0XFEFD: return("DTLSv1.2"); + case 0x0300: strncpy(buf, "SSLv3", buf_len); buf[buf_len - 1] = '\0'; return buf; + case 0x0301: strncpy(buf, "TLSv1", buf_len); buf[buf_len - 1] = '\0'; return buf; + case 0x0302: strncpy(buf, "TLSv1.1", buf_len); buf[buf_len - 1] = '\0'; return buf; + case 0x0303: strncpy(buf, "TLSv1.2", buf_len); buf[buf_len - 1] = '\0'; return buf; + case 0x0304: strncpy(buf, "TLSv1.3", buf_len); buf[buf_len - 1] = '\0'; return buf; + case 0XFB1A: strncpy(buf, "TLSv1.3 (Fizz)", buf_len); buf[buf_len - 1] = '\0'; return buf; /* https://engineering.fb.com/security/fizz/ */ + case 0XFEFF: strncpy(buf, "DTLSv1.0", buf_len); buf[buf_len - 1] = '\0'; return buf; + case 0XFEFD: strncpy(buf, "DTLSv1.2", buf_len); buf[buf_len - 1] = '\0'; return buf; case 0x0A0A: case 0x1A1A: case 0x2A2A: @@ -920,22 +923,21 @@ char* ndpi_ssl_version2str(struct ndpi_flow_struct *flow, case 0xCACA: case 0xDADA: case 0xEAEA: - case 0xFAFA: return("GREASE"); + case 0xFAFA: strncpy(buf, "GREASE", buf_len); buf[buf_len - 1] = '\0'; return buf; } - if((version >= 0x7f00) && (version <= 0x7fff)) - return("TLSv1.3 (draft)"); + if((version >= 0x7f00) && (version <= 0x7fff)) { + strncpy(buf, "TLSv1.3 (draft)", buf_len); + buf[buf_len - 1] = '\0'; + return buf; + } if(unknown_tls_version) *unknown_tls_version = 1; - if(flow != NULL) { - snprintf(flow->protos.tls_quic.ssl_version_str, - sizeof(flow->protos.tls_quic.ssl_version_str), "TLS (%04X)", version); + snprintf(buf, buf_len, "TLS (%04X)", version); - return(flow->protos.tls_quic.ssl_version_str); - } else - return(""); + return buf; } /* ***************************************************** */ @@ -1261,7 +1263,9 @@ int ndpi_dpi2json(struct ndpi_detection_module_struct *ndpi_struct, ndpi_serialize_string_string(serializer, "user_agent", flow->http.user_agent); if(flow->protos.tls_quic.ssl_version) { u_int8_t unknown_tls_version; - char *version = ndpi_ssl_version2str(flow, flow->protos.tls_quic.ssl_version, &unknown_tls_version); + char version[16]; + + ndpi_ssl_version2str(version, sizeof(version), flow->protos.tls_quic.ssl_version, &unknown_tls_version); if(!unknown_tls_version) ndpi_serialize_string_string(serializer, "version", version); @@ -1276,30 +1280,30 @@ int ndpi_dpi2json(struct ndpi_detection_module_struct *ndpi_struct, case NDPI_PROTOCOL_MAIL_IMAP: ndpi_serialize_start_of_block(serializer, "imap"); - ndpi_serialize_string_string(serializer, "user", flow->ftp_imap_pop_smtp.username); - ndpi_serialize_string_string(serializer, "password", flow->ftp_imap_pop_smtp.password); + ndpi_serialize_string_string(serializer, "user", flow->l4.tcp.ftp_imap_pop_smtp.username); + ndpi_serialize_string_string(serializer, "password", flow->l4.tcp.ftp_imap_pop_smtp.password); ndpi_serialize_end_of_block(serializer); break; case NDPI_PROTOCOL_MAIL_POP: ndpi_serialize_start_of_block(serializer, "pop"); - ndpi_serialize_string_string(serializer, "user", flow->ftp_imap_pop_smtp.username); - ndpi_serialize_string_string(serializer, "password", flow->ftp_imap_pop_smtp.password); + ndpi_serialize_string_string(serializer, "user", flow->l4.tcp.ftp_imap_pop_smtp.username); + ndpi_serialize_string_string(serializer, "password", flow->l4.tcp.ftp_imap_pop_smtp.password); ndpi_serialize_end_of_block(serializer); break; case NDPI_PROTOCOL_MAIL_SMTP: ndpi_serialize_start_of_block(serializer, "smtp"); - ndpi_serialize_string_string(serializer, "user", flow->ftp_imap_pop_smtp.username); - ndpi_serialize_string_string(serializer, "password", flow->ftp_imap_pop_smtp.password); + ndpi_serialize_string_string(serializer, "user", flow->l4.tcp.ftp_imap_pop_smtp.username); + ndpi_serialize_string_string(serializer, "password", flow->l4.tcp.ftp_imap_pop_smtp.password); ndpi_serialize_end_of_block(serializer); break; case NDPI_PROTOCOL_FTP_CONTROL: ndpi_serialize_start_of_block(serializer, "ftp"); - ndpi_serialize_string_string(serializer, "user", flow->ftp_imap_pop_smtp.username); - ndpi_serialize_string_string(serializer, "password", flow->ftp_imap_pop_smtp.password); - ndpi_serialize_string_uint32(serializer, "auth_failed", flow->ftp_imap_pop_smtp.auth_failed); + ndpi_serialize_string_string(serializer, "user", flow->l4.tcp.ftp_imap_pop_smtp.username); + ndpi_serialize_string_string(serializer, "password", flow->l4.tcp.ftp_imap_pop_smtp.password); + ndpi_serialize_string_uint32(serializer, "auth_failed", flow->l4.tcp.ftp_imap_pop_smtp.auth_failed); ndpi_serialize_end_of_block(serializer); break; @@ -1319,7 +1323,9 @@ int ndpi_dpi2json(struct ndpi_detection_module_struct *ndpi_struct, struct tm a, b, *before = NULL, *after = NULL; u_int i, off; u_int8_t unknown_tls_version; - char *version = ndpi_ssl_version2str(flow, flow->protos.tls_quic.ssl_version, &unknown_tls_version); + char version[16]; + + ndpi_ssl_version2str(version, sizeof(version), flow->protos.tls_quic.ssl_version, &unknown_tls_version); if(flow->protos.tls_quic.notBefore) before = gmtime_r((const time_t *)&flow->protos.tls_quic.notBefore, &a); diff --git a/src/lib/protocols/ftp_control.c b/src/lib/protocols/ftp_control.c index 2b6f1396f..81e475041 100644 --- a/src/lib/protocols/ftp_control.c +++ b/src/lib/protocols/ftp_control.c @@ -50,23 +50,23 @@ static int ndpi_ftp_control_check_request(struct ndpi_detection_module_struct *n #endif if(ndpi_match_strprefix(payload, payload_len, "USER")) { - ndpi_user_pwd_payload_copy((u_int8_t*)flow->ftp_imap_pop_smtp.username, - sizeof(flow->ftp_imap_pop_smtp.username), 5, + ndpi_user_pwd_payload_copy((u_int8_t*)flow->l4.tcp.ftp_imap_pop_smtp.username, + sizeof(flow->l4.tcp.ftp_imap_pop_smtp.username), 5, payload, payload_len); ndpi_set_risk(ndpi_struct, flow, NDPI_CLEAR_TEXT_CREDENTIALS); return 1; } if(ndpi_match_strprefix(payload, payload_len, "PASS")) { - ndpi_user_pwd_payload_copy((u_int8_t*)flow->ftp_imap_pop_smtp.password, - sizeof(flow->ftp_imap_pop_smtp.password), 5, + ndpi_user_pwd_payload_copy((u_int8_t*)flow->l4.tcp.ftp_imap_pop_smtp.password, + sizeof(flow->l4.tcp.ftp_imap_pop_smtp.password), 5, payload, payload_len); return 1; } if(ndpi_match_strprefix(payload, payload_len, "AUTH") || ndpi_match_strprefix(payload, payload_len, "auth")) { - flow->ftp_imap_pop_smtp.auth_found = 1; + flow->l4.tcp.ftp_imap_pop_smtp.auth_found = 1; return 1; } /* ***************************************************** */ @@ -562,14 +562,14 @@ static int ndpi_ftp_control_check_response(struct ndpi_flow_struct *flow, case '2': case '3': case '6': - if(flow->ftp_imap_pop_smtp.auth_found == 1) - flow->ftp_imap_pop_smtp.auth_tls = 1; + if(flow->l4.tcp.ftp_imap_pop_smtp.auth_found == 1) + flow->l4.tcp.ftp_imap_pop_smtp.auth_tls = 1; return(1); break; case '4': case '5': - flow->ftp_imap_pop_smtp.auth_failed = 1; + flow->l4.tcp.ftp_imap_pop_smtp.auth_failed = 1; return(1); break; } @@ -632,11 +632,11 @@ static void ndpi_check_ftp_control(struct ndpi_detection_module_struct *ndpi_str #ifdef FTP_DEBUG printf("%s() [user: %s][pwd: %s]\n", __FUNCTION__, - flow->ftp_imap_pop_smtp.username, flow->ftp_imap_pop_smtp.password); + flow->l4.tcp.ftp_imap_pop_smtp.username, flow->l4.tcp.ftp_imap_pop_smtp.password); #endif - if(flow->ftp_imap_pop_smtp.password[0] == '\0' && - flow->ftp_imap_pop_smtp.auth_tls == 0) /* TODO: any values on dissecting TLS handshake? */ + if(flow->l4.tcp.ftp_imap_pop_smtp.password[0] == '\0' && + flow->l4.tcp.ftp_imap_pop_smtp.auth_tls == 0) /* TODO: any values on dissecting TLS handshake? */ flow->ftp_control_stage = 0; else ndpi_int_ftp_control_add_connection(ndpi_struct, flow); diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c index 105ef1bca..b2116a8fe 100644 --- a/src/lib/protocols/http.c +++ b/src/lib/protocols/http.c @@ -370,8 +370,9 @@ static void setHttpUserAgent(struct ndpi_detection_module_struct *ndpi_struct, /* Good reference for future implementations: * https://github.com/ua-parser/uap-core/blob/master/regexes.yaml */ - snprintf((char*)flow->http.detected_os, - sizeof(flow->http.detected_os), "%s", ua); + if(flow->http.detected_os == NULL) { + flow->http.detected_os = ndpi_strdup(ua); + } } /* ************************************************************* */ @@ -606,9 +607,14 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ if(strlen(flow->host_server_name) > 0) ndpi_check_dga_name(ndpi_struct, flow, flow->host_server_name, 1); if(packet->forwarded_line.ptr) { - len = ndpi_min(packet->forwarded_line.len, sizeof(flow->http.nat_ip)-1); - strncpy((char*)flow->http.nat_ip, (char*)packet->forwarded_line.ptr, len); - flow->http.nat_ip[len] = '\0'; + if(flow->http.nat_ip == NULL) { + len = packet->forwarded_line.len; + flow->http.nat_ip = ndpi_malloc(len + 1); + if(flow->http.nat_ip == NULL) { + strncpy(flow->http.nat_ip, (char*)packet->forwarded_line.ptr, len); + flow->http.nat_ip[len] = '\0'; + } + } } ndpi_http_parse_subprotocol(ndpi_struct, flow); @@ -1074,9 +1080,6 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct else flow->http.request_version = 0; - /* Set the first found headers in request */ - flow->http.num_request_headers = packet->http_num_headers; - /* Check for Ookla */ if((packet->referer_line.len > 0) && ndpi_strnstr((const char *)packet->referer_line.ptr, "www.speedtest.net", packet->referer_line.len)) { @@ -1155,9 +1158,6 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct ndpi_parse_packet_line_info(ndpi_struct, flow); - // Add more found HTTP request headers. - flow->http.num_request_headers+=packet->http_num_headers; - if(packet->parsed_lines <= 1) { /* wait some packets in case request is split over more than 2 packets */ if(flow->packet_counter < 5) { @@ -1213,9 +1213,6 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct ndpi_parse_packet_line_info(ndpi_struct, flow); check_content_type_and_change_protocol(ndpi_struct, flow); - if(packet->packet_direction == 1 /* server -> client */) - flow->http.num_response_headers += packet->http_num_headers; /* flow structs are initialized with zeros */ - if(packet->empty_line_position_set != 0 || flow->l4.tcp.http_empty_line_seen == 1) { NDPI_LOG_DBG2(ndpi_struct, "empty line. check_http_payload\n"); check_http_payload(ndpi_struct, flow); diff --git a/src/lib/protocols/mail_imap.c b/src/lib/protocols/mail_imap.c index 8a4e8fa2e..317ac1848 100644 --- a/src/lib/protocols/mail_imap.c +++ b/src/lib/protocols/mail_imap.c @@ -177,16 +177,16 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct, if(user) { char *pwd; - snprintf(flow->ftp_imap_pop_smtp.username, - sizeof(flow->ftp_imap_pop_smtp.username), + snprintf(flow->l4.tcp.ftp_imap_pop_smtp.username, + sizeof(flow->l4.tcp.ftp_imap_pop_smtp.username), "%s", user); ndpi_set_risk(ndpi_struct, flow, NDPI_CLEAR_TEXT_CREDENTIALS); pwd = strtok_r(NULL, " \"\r\n", &saveptr); if(pwd) { - snprintf(flow->ftp_imap_pop_smtp.password, - sizeof(flow->ftp_imap_pop_smtp.password), + snprintf(flow->l4.tcp.ftp_imap_pop_smtp.password, + sizeof(flow->l4.tcp.ftp_imap_pop_smtp.password), "%s", pwd); } } @@ -320,7 +320,7 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct, || (flow->l4.tcp.mail_imap_stage == 5) || (flow->l4.tcp.mail_imap_stage == 7) ) { - if((flow->ftp_imap_pop_smtp.username[0] != '\0') + if((flow->l4.tcp.ftp_imap_pop_smtp.username[0] != '\0') || (flow->l4.tcp.mail_imap_stage >= 7)) { NDPI_LOG_INFO(ndpi_struct, "found MAIL_IMAP\n"); ndpi_int_mail_imap_add_connection(ndpi_struct, flow); diff --git a/src/lib/protocols/mail_pop.c b/src/lib/protocols/mail_pop.c index 483c4da35..125fe5da1 100644 --- a/src/lib/protocols/mail_pop.c +++ b/src/lib/protocols/mail_pop.c @@ -77,8 +77,8 @@ static int ndpi_int_mail_pop_check_for_client_commands(struct ndpi_detection_mod && (packet->payload[1] == 'S' || packet->payload[1] == 's') && (packet->payload[2] == 'E' || packet->payload[2] == 'e') && (packet->payload[3] == 'R' || packet->payload[3] == 'r')) { - ndpi_user_pwd_payload_copy((u_int8_t*)flow->ftp_imap_pop_smtp.username, - sizeof(flow->ftp_imap_pop_smtp.username), 5, + ndpi_user_pwd_payload_copy((u_int8_t*)flow->l4.tcp.ftp_imap_pop_smtp.username, + sizeof(flow->l4.tcp.ftp_imap_pop_smtp.username), 5, packet->payload, packet->payload_packet_len); ndpi_set_risk(ndpi_struct, flow, NDPI_CLEAR_TEXT_CREDENTIALS); @@ -88,8 +88,8 @@ static int ndpi_int_mail_pop_check_for_client_commands(struct ndpi_detection_mod && (packet->payload[1] == 'A' || packet->payload[1] == 'a') && (packet->payload[2] == 'S' || packet->payload[2] == 's') && (packet->payload[3] == 'S' || packet->payload[3] == 's')) { - ndpi_user_pwd_payload_copy((u_int8_t*)flow->ftp_imap_pop_smtp.password, - sizeof(flow->ftp_imap_pop_smtp.password), 5, + ndpi_user_pwd_payload_copy((u_int8_t*)flow->l4.tcp.ftp_imap_pop_smtp.password, + sizeof(flow->l4.tcp.ftp_imap_pop_smtp.password), 5, packet->payload, packet->payload_packet_len); ndpi_set_risk(ndpi_struct, flow, NDPI_CLEAR_TEXT_CREDENTIALS); @@ -182,7 +182,7 @@ void ndpi_search_mail_pop_tcp(struct ndpi_detection_module_struct if(flow->l4.tcp.mail_pop_stage > 0) { NDPI_LOG_INFO(ndpi_struct, "mail_pop identified\n"); - if((flow->ftp_imap_pop_smtp.password[0] != '\0') + if((flow->l4.tcp.ftp_imap_pop_smtp.password[0] != '\0') || (flow->l4.tcp.mail_pop_stage > 3)) { ndpi_int_mail_pop_add_connection(ndpi_struct, flow); popInitExtraPacketProcessing(flow); @@ -222,7 +222,7 @@ int ndpi_extra_search_mail_pop_tcp(struct ndpi_detection_module_struct *ndpi_str ndpi_search_mail_pop_tcp(ndpi_struct, flow); - rc = (flow->ftp_imap_pop_smtp.password[0] == '\0') ? 1 : 0; + rc = (flow->l4.tcp.ftp_imap_pop_smtp.password[0] == '\0') ? 1 : 0; #ifdef POP_DEBUG printf("**** %s() [rc: %d]\n", __FUNCTION__, rc); diff --git a/src/lib/protocols/mail_smtp.c b/src/lib/protocols/mail_smtp.c index e1d98dd35..ce3ba4058 100644 --- a/src/lib/protocols/mail_smtp.c +++ b/src/lib/protocols/mail_smtp.c @@ -93,19 +93,19 @@ static void get_credentials_auth_plain(struct ndpi_detection_module_struct *ndpi user_len = i - 1; } if(user_len > 0) { - user_len = ndpi_min(user_len, sizeof(flow->ftp_imap_pop_smtp.username) - 1); + user_len = ndpi_min(user_len, sizeof(flow->l4.tcp.ftp_imap_pop_smtp.username) - 1); - memcpy(flow->ftp_imap_pop_smtp.username, out + 1, user_len); - flow->ftp_imap_pop_smtp.username[user_len] = '\0'; + memcpy(flow->l4.tcp.ftp_imap_pop_smtp.username, out + 1, user_len); + flow->l4.tcp.ftp_imap_pop_smtp.username[user_len] = '\0'; ndpi_set_risk(ndpi_struct, flow, NDPI_CLEAR_TEXT_CREDENTIALS); if(1 + user_len + 1 < out_len) { unsigned int pwd_len; - pwd_len = ndpi_min(out_len - (1 + user_len + 1), sizeof(flow->ftp_imap_pop_smtp.password) - 1); - memcpy(flow->ftp_imap_pop_smtp.password, out + 1 + user_len + 1, pwd_len); - flow->ftp_imap_pop_smtp.password[pwd_len] = '\0'; + pwd_len = ndpi_min(out_len - (1 + user_len + 1), sizeof(flow->l4.tcp.ftp_imap_pop_smtp.password) - 1); + memcpy(flow->l4.tcp.ftp_imap_pop_smtp.password, out + 1 + user_len + 1, pwd_len); + flow->l4.tcp.ftp_imap_pop_smtp.password[pwd_len] = '\0'; } } ndpi_free(out); @@ -182,25 +182,25 @@ void ndpi_search_mail_smtp_tcp(struct ndpi_detection_module_struct *ndpi_struct, && (packet->line[a].ptr[3] == 'O' || packet->line[a].ptr[3] == 'o') && packet->line[a].ptr[4] == ' ') { flow->l4.tcp.smtp_command_bitmask |= SMTP_BIT_HELO_EHLO; - flow->ftp_imap_pop_smtp.auth_found = 0; + flow->l4.tcp.ftp_imap_pop_smtp.auth_found = 0; } else if((packet->line[a].ptr[0] == 'M' || packet->line[a].ptr[0] == 'm') && (packet->line[a].ptr[1] == 'A' || packet->line[a].ptr[1] == 'a') && (packet->line[a].ptr[2] == 'I' || packet->line[a].ptr[2] == 'i') && (packet->line[a].ptr[3] == 'L' || packet->line[a].ptr[3] == 'l') && packet->line[a].ptr[4] == ' ') { flow->l4.tcp.smtp_command_bitmask |= SMTP_BIT_MAIL; - flow->ftp_imap_pop_smtp.auth_found = 0; + flow->l4.tcp.ftp_imap_pop_smtp.auth_found = 0; /* We shouldn't be here if there are credentials */ - flow->ftp_imap_pop_smtp.auth_done = 1; + flow->l4.tcp.ftp_imap_pop_smtp.auth_done = 1; } else if((packet->line[a].ptr[0] == 'R' || packet->line[a].ptr[0] == 'r') && (packet->line[a].ptr[1] == 'C' || packet->line[a].ptr[1] == 'c') && (packet->line[a].ptr[2] == 'P' || packet->line[a].ptr[2] == 'p') && (packet->line[a].ptr[3] == 'T' || packet->line[a].ptr[3] == 't') && packet->line[a].ptr[4] == ' ') { flow->l4.tcp.smtp_command_bitmask |= SMTP_BIT_RCPT; - flow->ftp_imap_pop_smtp.auth_found = 0; + flow->l4.tcp.ftp_imap_pop_smtp.auth_found = 0; /* We shouldn't be here if there are credentials */ - flow->ftp_imap_pop_smtp.auth_done = 1; + flow->l4.tcp.ftp_imap_pop_smtp.auth_done = 1; } else if((packet->line[a].ptr[0] == 'A' || packet->line[a].ptr[0] == 'a') && (packet->line[a].ptr[1] == 'U' || packet->line[a].ptr[1] == 'u') && (packet->line[a].ptr[2] == 'T' || packet->line[a].ptr[2] == 't') @@ -209,7 +209,7 @@ void ndpi_search_mail_smtp_tcp(struct ndpi_detection_module_struct *ndpi_struct, #ifdef SMTP_DEBUG printf("%s() AUTH [%.*s]\n", __FUNCTION__, packet->line[a].len, packet->line[a].ptr); #endif - flow->ftp_imap_pop_smtp.auth_found = 1; + flow->l4.tcp.ftp_imap_pop_smtp.auth_found = 1; if(packet->line[a].len >= 6) { if(packet->line[a].ptr[5] == 'L' || packet->line[a].ptr[5] == 'l') { flow->l4.tcp.smtp_command_bitmask |= SMTP_BIT_AUTH_LOGIN; @@ -219,7 +219,7 @@ void ndpi_search_mail_smtp_tcp(struct ndpi_detection_module_struct *ndpi_struct, /* AUTH PLAIN: username and pwd here */ get_credentials_auth_plain(ndpi_struct, flow, packet->line[a].ptr, packet->line[a].len); - flow->ftp_imap_pop_smtp.auth_done = 1; + flow->l4.tcp.ftp_imap_pop_smtp.auth_done = 1; } } } else { @@ -228,9 +228,9 @@ void ndpi_search_mail_smtp_tcp(struct ndpi_detection_module_struct *ndpi_struct, printf("%s() => [%.*s]\n", __FUNCTION__, packet->line[a].len, packet->line[a].ptr); #endif - if(flow->ftp_imap_pop_smtp.auth_found && + if(flow->l4.tcp.ftp_imap_pop_smtp.auth_found && (flow->l4.tcp.smtp_command_bitmask & SMTP_BIT_AUTH_LOGIN)) { - if(flow->ftp_imap_pop_smtp.username[0] == '\0') { + if(flow->l4.tcp.ftp_imap_pop_smtp.username[0] == '\0') { /* Username */ u_int8_t buf[48]; u_char *out; @@ -240,22 +240,22 @@ void ndpi_search_mail_smtp_tcp(struct ndpi_detection_module_struct *ndpi_struct, packet->line[a].ptr, packet->line[a].len); #ifdef SMTP_DEBUG - printf("%s() => [auth: %u] (username) [%s]\n", __FUNCTION__, flow->ftp_imap_pop_smtp.auth_found, buf); + printf("%s() => [auth: %u] (username) [%s]\n", __FUNCTION__, flow->l4.tcp.ftp_imap_pop_smtp.auth_found, buf); #endif out = ndpi_base64_decode((const u_char*)buf, (size_t)strlen((const char*)buf), &out_len); if(out) { - size_t len = ndpi_min(out_len, sizeof(flow->ftp_imap_pop_smtp.username) - 1); + size_t len = ndpi_min(out_len, sizeof(flow->l4.tcp.ftp_imap_pop_smtp.username) - 1); - memcpy(flow->ftp_imap_pop_smtp.username, out, len); - flow->ftp_imap_pop_smtp.username[len] = '\0'; + memcpy(flow->l4.tcp.ftp_imap_pop_smtp.username, out, len); + flow->l4.tcp.ftp_imap_pop_smtp.username[len] = '\0'; ndpi_free(out); } ndpi_set_risk(ndpi_struct, flow, NDPI_CLEAR_TEXT_CREDENTIALS); - } else if(flow->ftp_imap_pop_smtp.password[0] == '\0') { + } else if(flow->l4.tcp.ftp_imap_pop_smtp.password[0] == '\0') { /* Password */ u_int8_t buf[48]; u_char *out; @@ -265,23 +265,23 @@ void ndpi_search_mail_smtp_tcp(struct ndpi_detection_module_struct *ndpi_struct, packet->line[a].ptr, packet->line[a].len); #ifdef SMTP_DEBUG - printf("%s() => [auth: %u] (password) [%s]\n", __FUNCTION__, flow->ftp_imap_pop_smtp.auth_found, buf); + printf("%s() => [auth: %u] (password) [%s]\n", __FUNCTION__, flow->l4.tcp.ftp_imap_pop_smtp.auth_found, buf); #endif out = ndpi_base64_decode((const u_char*)buf, (size_t)strlen((const char*)buf), &out_len); if(out) { - size_t len = ndpi_min(out_len, sizeof(flow->ftp_imap_pop_smtp.password) - 1); + size_t len = ndpi_min(out_len, sizeof(flow->l4.tcp.ftp_imap_pop_smtp.password) - 1); - memcpy(flow->ftp_imap_pop_smtp.password, out, len); - flow->ftp_imap_pop_smtp.password[len] = '\0'; + memcpy(flow->l4.tcp.ftp_imap_pop_smtp.password, out, len); + flow->l4.tcp.ftp_imap_pop_smtp.password[len] = '\0'; ndpi_free(out); } ndpi_set_risk(ndpi_struct, flow, NDPI_CLEAR_TEXT_CREDENTIALS); - flow->ftp_imap_pop_smtp.auth_done = 1; + flow->l4.tcp.ftp_imap_pop_smtp.auth_done = 1; } else { flow->host_server_name[0] = '\0'; NDPI_EXCLUDE_PROTO(ndpi_struct, flow); @@ -302,8 +302,8 @@ void ndpi_search_mail_smtp_tcp(struct ndpi_detection_module_struct *ndpi_struct, && (packet->line[a].ptr[6] == 'L' || packet->line[a].ptr[6] == 'l') && (packet->line[a].ptr[7] == 'S' || packet->line[a].ptr[7] == 's')) { flow->l4.tcp.smtp_command_bitmask |= SMTP_BIT_STARTTLS; - flow->ftp_imap_pop_smtp.auth_tls = 1; - flow->ftp_imap_pop_smtp.auth_done = 1; + flow->l4.tcp.ftp_imap_pop_smtp.auth_tls = 1; + flow->l4.tcp.ftp_imap_pop_smtp.auth_done = 1; } } @@ -341,7 +341,7 @@ void ndpi_search_mail_smtp_tcp(struct ndpi_detection_module_struct *ndpi_struct, #ifdef SMTP_DEBUG printf("%s() [bit_count: %u][%s]\n", __FUNCTION__, - bit_count, flow->ftp_imap_pop_smtp.password); + bit_count, flow->l4.tcp.ftp_imap_pop_smtp.password); #endif /* Only if we don't have already set the protocol via hostname matching */ @@ -379,7 +379,7 @@ int ndpi_extra_search_mail_smtp_tcp(struct ndpi_detection_module_struct *ndpi_st ndpi_search_mail_smtp_tcp(ndpi_struct, flow); - rc = (flow->ftp_imap_pop_smtp.password[0] == '\0') ? 1 : 0; + rc = (flow->l4.tcp.ftp_imap_pop_smtp.password[0] == '\0') ? 1 : 0; #ifdef SMTP_DEBUG printf("**** %s() [rc: %d]\n", __FUNCTION__, rc); diff --git a/src/lib/protocols/thunder.c b/src/lib/protocols/thunder.c index 31fa720d4..1b33c41d7 100644 --- a/src/lib/protocols/thunder.c +++ b/src/lib/protocols/thunder.c @@ -32,18 +32,7 @@ static void ndpi_int_thunder_add_connection(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow/* , ndpi_protocol_type_t protocol_type */) { - struct ndpi_packet_struct *packet = &ndpi_struct->packet; - struct ndpi_id_struct *src = flow->src; - struct ndpi_id_struct *dst = flow->dst; - ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_THUNDER, NDPI_PROTOCOL_UNKNOWN); - - if (src != NULL) { - src->thunder_ts = packet->current_time_ms; - } - if (dst != NULL) { - dst->thunder_ts = packet->current_time_ms; - } } @@ -149,22 +138,6 @@ void ndpi_int_search_thunder_http(struct ndpi_detection_module_struct struct ndpi_id_struct *src = flow->src; struct ndpi_id_struct *dst = flow->dst; - - if (flow->detected_protocol_stack[0] == NDPI_PROTOCOL_THUNDER) { - if (src != NULL && ((u_int32_t) - (packet->current_time_ms - src->thunder_ts) < ndpi_struct->thunder_timeout)) { - NDPI_LOG_DBG2(ndpi_struct, - "thunder : save src connection packet detected\n"); - src->thunder_ts = packet->current_time_ms; - } else if (dst != NULL && ((u_int32_t) - (packet->current_time_ms - dst->thunder_ts) < ndpi_struct->thunder_timeout)) { - NDPI_LOG_DBG2(ndpi_struct, - "thunder : save dst connection packet detected\n"); - dst->thunder_ts = packet->current_time_ms; - } - return; - } - if (packet->payload_packet_len > 5 && memcmp(packet->payload, "GET /", 5) == 0 && NDPI_SRC_OR_DST_HAS_PROTOCOL(src, dst, NDPI_PROTOCOL_THUNDER)) { NDPI_LOG_DBG2(ndpi_struct, "HTTP packet detected\n"); diff --git a/src/lib/protocols/tls.c b/src/lib/protocols/tls.c index 87553de87..b8f8159e5 100644 --- a/src/lib/protocols/tls.c +++ b/src/lib/protocols/tls.c @@ -2030,6 +2030,7 @@ int processClientServerHello(struct ndpi_detection_module_struct *ndpi_struct, u_int16_t s_offset = offset+extension_offset; u_int8_t version_len = packet->payload[s_offset]; char version_str[256]; + char buf_ver_tmp[16]; size_t version_str_len = 0; version_str[0] = 0; #ifdef DEBUG_TLS @@ -2049,14 +2050,14 @@ int processClientServerHello(struct ndpi_detection_module_struct *ndpi_struct, #ifdef DEBUG_TLS printf("Client TLS [TLS version: %s/0x%04X]\n", - ndpi_ssl_version2str(flow, tls_version, &unknown_tls_version), tls_version); + ndpi_ssl_version2str(buf_ver_tmp, sizeof(buf_ver_tmp), tls_version, &unknown_tls_version), tls_version); #endif if((version_str_len+8) < sizeof(version_str)) { int rc = snprintf(&version_str[version_str_len], sizeof(version_str) - version_str_len, "%s%s", (version_str_len > 0) ? "," : "", - ndpi_ssl_version2str(flow, tls_version, &unknown_tls_version)); + ndpi_ssl_version2str(buf_ver_tmp, sizeof(buf_ver_tmp), tls_version, &unknown_tls_version)); if(rc <= 0) break; else diff --git a/src/lib/protocols/zattoo.c b/src/lib/protocols/zattoo.c index 6b3e9625c..b5b305ecb 100644 --- a/src/lib/protocols/zattoo.c +++ b/src/lib/protocols/zattoo.c @@ -47,30 +47,16 @@ u_int8_t ndpi_int_zattoo_user_agent_set(struct ndpi_detection_module_struct *ndp } #define ZATTOO_DETECTED \ - if (src != NULL) \ - src->zattoo_ts = packet->current_time_ms; \ - if (dst != NULL) \ - dst->zattoo_ts = packet->current_time_ms; \ - \ ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_ZATTOO, NDPI_PROTOCOL_UNKNOWN) void ndpi_search_zattoo(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) { struct ndpi_packet_struct *packet = &ndpi_struct->packet; - struct ndpi_id_struct *src = flow->src; - struct ndpi_id_struct *dst = flow->dst; u_int16_t i; NDPI_LOG_DBG(ndpi_struct, "search ZATTOO\n"); - if(flow->detected_protocol_stack[0] == NDPI_PROTOCOL_ZATTOO) { - if(src != NULL && ((u_int32_t) (packet->current_time_ms - src->zattoo_ts) < ndpi_struct->zattoo_connection_timeout)) - src->zattoo_ts = packet->current_time_ms; - if (dst != NULL && ((u_int32_t) (packet->current_time_ms - dst->zattoo_ts) < ndpi_struct->zattoo_connection_timeout)) - dst->zattoo_ts = packet->current_time_ms; - return; - } /* search over TCP */ if(packet->tcp != NULL) { if(packet->payload_packet_len > 50 && memcmp(packet->payload, "GET /frontdoor/fd?brand=Zattoo&v=", 33) == 0) { |