diff options
author | Ivan Nardi <12729895+IvanNardi@users.noreply.github.com> | 2021-11-24 10:46:48 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-11-24 10:46:48 +0100 |
commit | a8ffcd8bb0273d59600c6310a80b81206096c113 (patch) | |
tree | 2a62911824363509ea5e7c69afa189e98556e495 /src/lib/protocols | |
parent | fd02e1b3043eecc5711eb8254aadaa3f43ca7503 (diff) |
Rework how hostname/SNI info is saved (#1330)
Looking at `struct ndpi_flow_struct` the two bigger fields are
`host_server_name[240]` (mainly for HTTP hostnames and DNS domains) and
`protos.tls_quic.client_requested_server_name[256]`
(for TLS/QUIC SNIs).
This commit aims to reduce `struct ndpi_flow_struct` size, according to
two simple observations:
1) maximum one of these two fields is used for each flow. So it seems safe
to merge them;
2) even if hostnames/SNIs might be very long, in practice they are rarely
longer than a fews tens of bytes. So, using a (single) large buffer is a
waste of memory for all kinds of flows. If we need to truncate the name,
we keep the *last* characters, easing domain matching.
Analyzing some real traffic, it seems safe to assume that the vast
majority of hostnames/SNIs is shorter than 80 bytes.
Hostnames/SNIs are always converted to lowercase.
Attention was given so as to be sure that unit-tests outputs are not
affected by this change.
Because of a bug, TLS/QUIC SNI were always truncated to 64 bytes (the
*first* 64 ones): as a consequence, there were some "Suspicious DGA
domain name" and "TLS Certificate Mismatch" false positives.
Diffstat (limited to 'src/lib/protocols')
-rw-r--r-- | src/lib/protocols/dhcp.c | 11 | ||||
-rw-r--r-- | src/lib/protocols/dns.c | 54 | ||||
-rw-r--r-- | src/lib/protocols/http.c | 10 | ||||
-rw-r--r-- | src/lib/protocols/mail_smtp.c | 11 | ||||
-rw-r--r-- | src/lib/protocols/netbios.c | 4 | ||||
-rw-r--r-- | src/lib/protocols/quic.c | 20 | ||||
-rw-r--r-- | src/lib/protocols/stun.c | 14 | ||||
-rw-r--r-- | src/lib/protocols/tls.c | 71 | ||||
-rw-r--r-- | src/lib/protocols/whoisdas.c | 5 |
9 files changed, 88 insertions, 112 deletions
diff --git a/src/lib/protocols/dhcp.c b/src/lib/protocols/dhcp.c index d40bb5c35..23d358b74 100644 --- a/src/lib/protocols/dhcp.c +++ b/src/lib/protocols/dhcp.c @@ -159,17 +159,14 @@ void ndpi_search_dhcp_udp(struct ndpi_detection_module_struct *ndpi_struct, stru strncpy((char*)flow->protos.dhcp.class_ident, name, j); flow->protos.dhcp.class_ident[j] = '\0'; } else if(id == 12 /* Host Name */) { - char *name = (char*)&dhcp->options[i+2]; - int j = 0; - + u_int8_t *name = &dhcp->options[i+2]; + #ifdef DHCP_DEBUG NDPI_LOG_DBG2(ndpi_struct, "[DHCP] '%.*s'\n",name,len); // while(j < len) { printf( "%c", name[j]); j++; }; printf("\n"); #endif - j = ndpi_min(len, sizeof(flow->host_server_name)-1); - strncpy((char*)flow->host_server_name, name, j); - flow->host_server_name[j] = '\0'; - } + ndpi_hostname_sni_set(flow, name, len); + } i += len + 2; } diff --git a/src/lib/protocols/dns.c b/src/lib/protocols/dns.c index 98c6bf142..eb5af5a34 100644 --- a/src/lib/protocols/dns.c +++ b/src/lib/protocols/dns.c @@ -368,6 +368,7 @@ static void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, st int invalid = search_valid_dns(ndpi_struct, flow, &dns_header, payload_offset, &is_query); ndpi_protocol ret; u_int num_queries, idx; + char _hostname[256]; ret.master_protocol = NDPI_PROTOCOL_UNKNOWN; ret.app_protocol = (d_port == LLMNR_PORT) ? NDPI_PROTOCOL_LLMNR : ((d_port == MDNS_PORT) ? NDPI_PROTOCOL_MDNS : NDPI_PROTOCOL_DNS); @@ -378,7 +379,6 @@ static void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, st } /* extract host name server */ - max_len = sizeof(flow->host_server_name)-1; off = sizeof(struct ndpi_dns_packet_header) + payload_offset; /* Before continuing let's dissect the following queries to see if they are valid */ @@ -434,6 +434,7 @@ static void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, st } /* for */ u_int8_t hostname_is_valid = 1; + max_len = sizeof(_hostname)-1; while((j < max_len) && (off < packet->payload_packet_len) && (packet->payload[off] != '\0')) { uint8_t c, cl = packet->payload[off++]; @@ -443,40 +444,43 @@ static void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, st break; } - if(j && (j < max_len)) flow->host_server_name[j++] = '.'; - - while((j < max_len) && (cl != 0)) { - u_int32_t shift; - - c = packet->payload[off++]; - shift = ((u_int32_t) 1) << (c & 0x1f); - if ((dns_validchar[c >> 5] & shift)) { - flow->host_server_name[j++] = tolower(c); - } else { - if (isprint(c) == 0) { - hostname_is_valid = 0; - flow->host_server_name[j++] = '?'; - } else { - flow->host_server_name[j++] = '_'; - } - } - cl--; - } + if(j && (j < max_len)) _hostname[j++] = '.'; + + while((j < max_len) && (cl != 0)) { + u_int32_t shift; + + c = packet->payload[off++]; + shift = ((u_int32_t) 1) << (c & 0x1f); + if((dns_validchar[c >> 5] & shift)) { + _hostname[j++] = tolower(c); + } else { + if (isprint(c) == 0) { + hostname_is_valid = 0; + _hostname[j++] = '?'; + } else { + _hostname[j++] = '_'; + } + } + cl--; + } } + + _hostname[j] = '\0'; + + ndpi_hostname_sni_set(flow, (const u_int8_t *)_hostname, j); + if (hostname_is_valid == 0) { ndpi_set_risk(ndpi_struct, flow, NDPI_INVALID_CHARACTERS); } - flow->host_server_name[j] = '\0'; - if(j > 0) { ndpi_protocol_match_result ret_match; - ndpi_check_dga_name(ndpi_struct, flow, (char*)flow->host_server_name, 1); + ndpi_check_dga_name(ndpi_struct, flow, flow->host_server_name, 1); ret.app_protocol = ndpi_match_host_subprotocol(ndpi_struct, flow, - (char *)flow->host_server_name, - strlen((const char*)flow->host_server_name), + flow->host_server_name, + strlen(flow->host_server_name), &ret_match, NDPI_PROTOCOL_DNS); diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c index cef8b3cfc..bc0ca1fc2 100644 --- a/src/lib/protocols/http.c +++ b/src/lib/protocols/http.c @@ -371,8 +371,8 @@ static void ndpi_http_parse_subprotocol(struct ndpi_detection_module_struct *ndp if(double_col) double_col[0] = '\0'; if(ndpi_match_hostname_protocol(ndpi_struct, flow, NDPI_PROTOCOL_HTTP, - (char *)flow->host_server_name, - strlen((const char *)flow->host_server_name)) == 0) { + flow->host_server_name, + strlen(flow->host_server_name)) == 0) { if(flow->http.url && ((strstr(flow->http.url, ":8080/downloading?n=0.") != NULL) || (strstr(flow->http.url, ":8080/upload?n=0.") != NULL))) { @@ -566,12 +566,10 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ packet->host_line.len, packet->host_line.ptr); /* Copy result for nDPI apps */ - len = ndpi_min(packet->host_line.len, sizeof(flow->host_server_name)-1); - strncpy((char*)flow->host_server_name, (char*)packet->host_line.ptr, len); - flow->host_server_name[len] = '\0'; + ndpi_hostname_sni_set(flow, packet->host_line.ptr, packet->host_line.len); flow->extra_packets_func = NULL; /* We're good now */ - if(len > 0) ndpi_check_dga_name(ndpi_struct, flow, (char*)flow->host_server_name, 1); + if(strlen(flow->host_server_name) > 0) ndpi_check_dga_name(ndpi_struct, flow, flow->host_server_name, 1); if(packet->forwarded_line.ptr) { len = ndpi_min(packet->forwarded_line.len, sizeof(flow->http.nat_ip)-1); diff --git a/src/lib/protocols/mail_smtp.c b/src/lib/protocols/mail_smtp.c index 31310202b..e1d98dd35 100644 --- a/src/lib/protocols/mail_smtp.c +++ b/src/lib/protocols/mail_smtp.c @@ -147,12 +147,11 @@ void ndpi_search_mail_smtp_tcp(struct ndpi_detection_module_struct *ndpi_struct, && (packet->line[a].ptr[i+1] != '\n')) { len = i-4; /* Copy result for nDPI apps */ - len = ndpi_min(len, sizeof(flow->host_server_name)-1); - strncpy((char*)flow->host_server_name, (char*)&packet->line[a].ptr[4], len); - flow->host_server_name[len] = '\0'; - if(ndpi_match_hostname_protocol(ndpi_struct, flow, NDPI_PROTOCOL_MAIL_SMTP, - (char *)flow->host_server_name, - strlen((const char *)flow->host_server_name))) { + ndpi_hostname_sni_set(flow, &packet->line[a].ptr[4], len); + + if (ndpi_match_hostname_protocol(ndpi_struct, flow, NDPI_PROTOCOL_MAIL_SMTP, + flow->host_server_name, + strlen(flow->host_server_name))) { /* We set the protocols; we need to initialize extra dissection to search for credentials */ NDPI_LOG_DBG(ndpi_struct, "SMTP: hostname matched\n"); diff --git a/src/lib/protocols/netbios.c b/src/lib/protocols/netbios.c index ccf910489..88c211551 100644 --- a/src/lib/protocols/netbios.c +++ b/src/lib/protocols/netbios.c @@ -102,9 +102,9 @@ static void ndpi_int_netbios_add_connection(struct ndpi_detection_module_struct if((off < packet->payload_packet_len) && ndpi_netbios_name_interpret((unsigned char*)&packet->payload[off], (u_int)(packet->payload_packet_len - off), name, sizeof(name)-1) > 0) { - snprintf((char*)flow->host_server_name, sizeof(flow->host_server_name)-1, "%s", name); + ndpi_hostname_sni_set(flow, (const u_int8_t *)name, strlen((char *)name)); - ndpi_check_dga_name(ndpi_struct, flow, (char*)flow->host_server_name, 1); + ndpi_check_dga_name(ndpi_struct, flow, flow->host_server_name, 1); } if(sub_protocol == NDPI_PROTOCOL_UNKNOWN) diff --git a/src/lib/protocols/quic.c b/src/lib/protocols/quic.c index 358edb064..6bae70524 100644 --- a/src/lib/protocols/quic.c +++ b/src/lib/protocols/quic.c @@ -1324,7 +1324,7 @@ static void process_chlo(struct ndpi_detection_module_struct *ndpi_struct, uint32_t i; uint16_t num_tags; uint32_t prev_offset; - uint32_t tag_offset_start, offset, len, sni_len; + uint32_t tag_offset_start, offset, len; ndpi_protocol_match_result ret_match; int sni_found = 0, ua_found = 0; @@ -1356,22 +1356,20 @@ static void process_chlo(struct ndpi_detection_module_struct *ndpi_struct, crypto_data_len, tag_offset_start, prev_offset, offset, len); #endif if(memcmp(tag, "SNI\0", 4) == 0) { - sni_len = MIN(len, sizeof(flow->protos.tls_quic.client_requested_server_name) - 1); - memcpy(flow->protos.tls_quic.client_requested_server_name, - &crypto_data[tag_offset_start + prev_offset], sni_len); - flow->protos.tls_quic.client_requested_server_name[sni_len] = '\0'; + + ndpi_hostname_sni_set(flow, &crypto_data[tag_offset_start + prev_offset], len); NDPI_LOG_DBG2(ndpi_struct, "SNI: [%s]\n", - flow->protos.tls_quic.client_requested_server_name); + flow->host_server_name); ndpi_match_host_subprotocol(ndpi_struct, flow, - (char *)flow->protos.tls_quic.client_requested_server_name, - strlen((const char*)flow->protos.tls_quic.client_requested_server_name), + flow->host_server_name, + strlen(flow->host_server_name), &ret_match, NDPI_PROTOCOL_QUIC); flow->protos.tls_quic.hello_processed = 1; /* Allow matching of custom categories */ ndpi_check_dga_name(ndpi_struct, flow, - flow->protos.tls_quic.client_requested_server_name, 1); + flow->host_server_name, 1); sni_found = 1; if (ua_found) @@ -1396,7 +1394,7 @@ static void process_chlo(struct ndpi_detection_module_struct *ndpi_struct, NDPI_LOG_DBG(ndpi_struct, "Something went wrong in tags iteration\n"); /* Add check for missing SNI */ - if(flow->protos.tls_quic.client_requested_server_name[0] == '\0') { + if(flow->host_server_name[0] == '\0') { /* This is a bit suspicious */ ndpi_set_risk(ndpi_struct, flow, NDPI_TLS_MISSING_SNI); } @@ -1508,7 +1506,7 @@ static int eval_extra_processing(struct ndpi_detection_module_struct *ndpi_struc */ if((version == V_Q046 && - flow->protos.tls_quic.client_requested_server_name[0] == '\0') || + flow->host_server_name[0] == '\0') || is_ch_reassembler_pending(flow)) { NDPI_LOG_DBG2(ndpi_struct, "We have further work to do\n"); return 1; diff --git a/src/lib/protocols/stun.c b/src/lib/protocols/stun.c index f0884fae3..1ffa8afef 100644 --- a/src/lib/protocols/stun.c +++ b/src/lib/protocols/stun.c @@ -349,25 +349,21 @@ static ndpi_int_stun_t ndpi_int_check_stun(struct ndpi_detection_module_struct * u_int16_t realm_len = ntohs(*((u_int16_t*)&payload[offset+2])); if(flow->host_server_name[0] == '\0') { - u_int i; u_int k = offset+4; - i = ndpi_min(realm_len, sizeof(flow->host_server_name) - 1); - i = ndpi_min(i, payload_length - k); - memcpy(flow->host_server_name, payload + k, i); - flow->host_server_name[i] = '\0'; - + ndpi_hostname_sni_set(flow, payload + k, ndpi_min(realm_len, payload_length - k)); + #ifdef DEBUG_STUN printf("==> [%s]\n", flow->host_server_name); #endif - if(strstr((char*) flow->host_server_name, "google.com") != NULL) { + if(strstr(flow->host_server_name, "google.com") != NULL) { flow->guessed_host_protocol_id = NDPI_PROTOCOL_HANGOUT_DUO; return(NDPI_IS_STUN); - } else if(strstr((char*) flow->host_server_name, "whispersystems.org") != NULL) { + } else if(strstr(flow->host_server_name, "whispersystems.org") != NULL) { flow->guessed_host_protocol_id = NDPI_PROTOCOL_SIGNAL; return(NDPI_IS_STUN); - } else if(strstr((char*) flow->host_server_name, "facebook") != NULL) { + } else if(strstr(flow->host_server_name, "facebook") != NULL) { flow->guessed_host_protocol_id = NDPI_PROTOCOL_MESSENGER; return(NDPI_IS_STUN); } diff --git a/src/lib/protocols/tls.c b/src/lib/protocols/tls.c index d912ae947..a75e26eb2 100644 --- a/src/lib/protocols/tls.c +++ b/src/lib/protocols/tls.c @@ -478,7 +478,7 @@ static void processCertificateElements(struct ndpi_detection_module_struct *ndpi u_int8_t matched_name = 0; /* If the client hello was not observed or the requested name was missing, there is no need to trigger an alert */ - if(flow->protos.tls_quic.client_requested_server_name[0] == '\0') + if(flow->host_server_name[0] == '\0') matched_name = 1; #ifdef DEBUG_TLS @@ -520,7 +520,7 @@ static void processCertificateElements(struct ndpi_detection_module_struct *ndpi #if DEBUG_TLS printf("[TLS] dNSName %s [%s][len: %u][leftover: %d]\n", dNSName, - flow->protos.tls_quic.client_requested_server_name, len, + flow->host_server_name, len, packet->payload_packet_len-i-len); #endif if (ndpi_is_printable_string(dNSName, len) == 0) { @@ -530,27 +530,24 @@ static void processCertificateElements(struct ndpi_detection_module_struct *ndpi if(matched_name == 0) { #if DEBUG_TLS printf("[TLS] Trying to match '%s' with '%s'\n", - flow->protos.tls_quic.client_requested_server_name, + flow->host_server_name, dNSName); #endif - if(flow->protos.tls_quic.client_requested_server_name[0] == '\0') + if(flow->host_server_name[0] == '\0') { matched_name = 1; /* No SNI */ - else if (dNSName[0] == '*') - { - char * label = strstr(flow->protos.tls_quic.client_requested_server_name, &dNSName[1]); - - if (label != NULL) - { - char * first_dot = strchr(flow->protos.tls_quic.client_requested_server_name, '.'); - - if (first_dot == NULL || first_dot >= label) - { - matched_name = 1; - } - } - } - else if(strcmp(flow->protos.tls_quic.client_requested_server_name, dNSName) == 0) { + } else if (dNSName[0] == '*') { + char * label = strstr(flow->host_server_name, &dNSName[1]); + + if (label != NULL) { + char * first_dot = strchr(flow->host_server_name, '.'); + + if (first_dot == NULL || first_dot >= label) { + matched_name = 1; + } + } + } + else if(strcmp(flow->host_server_name, dNSName) == 0) { matched_name = 1; } } @@ -1244,7 +1241,6 @@ int processClientServerHello(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t i, j; u_int16_t total_len; u_int8_t handshake_type; - char buffer[64] = { '\0' }; int is_quic = (quic_version != 0); int is_dtls = packet->udp && (!is_quic); @@ -1721,50 +1717,41 @@ int processClientServerHello(struct ndpi_detection_module_struct *ndpi_struct, if((offset+extension_offset+4) < packet->payload_packet_len) { len = (packet->payload[offset+extension_offset+3] << 8) + packet->payload[offset+extension_offset+4]; - len = (u_int)ndpi_min(len, sizeof(buffer)-1); if((offset+extension_offset+5+len) <= packet->payload_packet_len) { - strncpy(buffer, (char*)&packet->payload[offset+extension_offset+5], len); - buffer[len] = '\0'; - cleanupServerName(buffer, sizeof(buffer)); - - snprintf(flow->protos.tls_quic.client_requested_server_name, - sizeof(flow->protos.tls_quic.client_requested_server_name), - "%s", buffer); + char *sni = ndpi_hostname_sni_set(flow, &packet->payload[offset+extension_offset+5], len); + int sni_len = strlen(sni); #ifdef DEBUG_TLS - printf("[TLS] SNI: [%s]\n", buffer); + printf("[TLS] SNI: [%s]\n", sni); #endif - if (ndpi_is_printable_string(buffer, len) == 0) + if (ndpi_is_printable_string(sni, sni_len) == 0) { ndpi_set_risk(ndpi_struct, flow, NDPI_INVALID_CHARACTERS); } if(!is_quic) { - if(ndpi_match_hostname_protocol(ndpi_struct, flow, NDPI_PROTOCOL_TLS, buffer, strlen(buffer))) + if(ndpi_match_hostname_protocol(ndpi_struct, flow, NDPI_PROTOCOL_TLS, sni, sni_len)) flow->protos.tls_quic.subprotocol_detected = 1; } else { - if(ndpi_match_hostname_protocol(ndpi_struct, flow, NDPI_PROTOCOL_QUIC, buffer, strlen(buffer))) + if(ndpi_match_hostname_protocol(ndpi_struct, flow, NDPI_PROTOCOL_QUIC, sni, sni_len)) flow->protos.tls_quic.subprotocol_detected = 1; } if(ndpi_check_dga_name(ndpi_struct, flow, - flow->protos.tls_quic.client_requested_server_name, 1)) { - char *sni = flow->protos.tls_quic.client_requested_server_name; - int len = strlen(sni); - + sni, 1)) { #ifdef DEBUG_TLS - printf("[TLS] SNI: (DGA) [%s]\n", flow->protos.tls_quic.client_requested_server_name); + printf("[TLS] SNI: (DGA) [%s]\n", sni); #endif - if((len >= 4) + if((sni_len >= 4) /* Check if it ends in .com or .net */ - && ((strcmp(&sni[len-4], ".com") == 0) || (strcmp(&sni[len-4], ".net") == 0)) + && ((strcmp(&sni[sni_len-4], ".com") == 0) || (strcmp(&sni[sni_len-4], ".net") == 0)) && (strncmp(sni, "www.", 4) == 0)) /* Not starting with www.... */ ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_TOR, NDPI_PROTOCOL_TLS); } else { #ifdef DEBUG_TLS - printf("[TLS] SNI: (NO DGA) [%s]\n", flow->protos.tls_quic.client_requested_server_name); + printf("[TLS] SNI: (NO DGA) [%s]\n", sni); #endif } } else { @@ -2268,12 +2255,12 @@ int processClientServerHello(struct ndpi_detection_module_struct *ndpi_struct, /* Suspicious Domain Fronting: https://github.com/SixGenInc/Noctilucent/blob/master/docs/ */ if(flow->protos.tls_quic.encrypted_sni.esni && - flow->protos.tls_quic.client_requested_server_name[0] != '\0') { + flow->host_server_name[0] != '\0') { ndpi_set_risk(ndpi_struct, flow, NDPI_TLS_SUSPICIOUS_ESNI_USAGE); } /* Add check for missing SNI */ - if((flow->protos.tls_quic.client_requested_server_name[0] == 0) + if(flow->host_server_name[0] == '\0' && (flow->protos.tls_quic.ssl_version >= 0x0302) /* TLSv1.1 */ && (flow->protos.tls_quic.encrypted_sni.esni == NULL) /* No ESNI */ ) { diff --git a/src/lib/protocols/whoisdas.c b/src/lib/protocols/whoisdas.c index 7321626d2..6f6f2a06b 100644 --- a/src/lib/protocols/whoisdas.c +++ b/src/lib/protocols/whoisdas.c @@ -41,10 +41,7 @@ void ndpi_search_whois_das(struct ndpi_detection_module_struct *ndpi_struct, str ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_WHOIS_DAS, NDPI_PROTOCOL_UNKNOWN); if((dport == 43) || (dport == 4343)) { /* Request */ - u_int hostname_len = ndpi_min(sizeof(flow->host_server_name) - 1, (long unsigned int)packet->payload_packet_len - 2); /* Skip \r\n */ - - memcpy(flow->host_server_name, &packet->payload[0], hostname_len); - flow->host_server_name[hostname_len] = '\0'; + ndpi_hostname_sni_set(flow, &packet->payload[0], packet->payload_packet_len - 2); /* Skip \r\n */ NDPI_LOG_INFO(ndpi_struct, "[WHOIS/DAS] %s\n", flow->host_server_name); } return; |