aboutsummaryrefslogtreecommitdiff
path: root/src/lib/protocols
diff options
context:
space:
mode:
authorIvan Nardi <12729895+IvanNardi@users.noreply.github.com>2021-11-24 10:46:48 +0100
committerGitHub <noreply@github.com>2021-11-24 10:46:48 +0100
commita8ffcd8bb0273d59600c6310a80b81206096c113 (patch)
tree2a62911824363509ea5e7c69afa189e98556e495 /src/lib/protocols
parentfd02e1b3043eecc5711eb8254aadaa3f43ca7503 (diff)
Rework how hostname/SNI info is saved (#1330)
Looking at `struct ndpi_flow_struct` the two bigger fields are `host_server_name[240]` (mainly for HTTP hostnames and DNS domains) and `protos.tls_quic.client_requested_server_name[256]` (for TLS/QUIC SNIs). This commit aims to reduce `struct ndpi_flow_struct` size, according to two simple observations: 1) maximum one of these two fields is used for each flow. So it seems safe to merge them; 2) even if hostnames/SNIs might be very long, in practice they are rarely longer than a fews tens of bytes. So, using a (single) large buffer is a waste of memory for all kinds of flows. If we need to truncate the name, we keep the *last* characters, easing domain matching. Analyzing some real traffic, it seems safe to assume that the vast majority of hostnames/SNIs is shorter than 80 bytes. Hostnames/SNIs are always converted to lowercase. Attention was given so as to be sure that unit-tests outputs are not affected by this change. Because of a bug, TLS/QUIC SNI were always truncated to 64 bytes (the *first* 64 ones): as a consequence, there were some "Suspicious DGA domain name" and "TLS Certificate Mismatch" false positives.
Diffstat (limited to 'src/lib/protocols')
-rw-r--r--src/lib/protocols/dhcp.c11
-rw-r--r--src/lib/protocols/dns.c54
-rw-r--r--src/lib/protocols/http.c10
-rw-r--r--src/lib/protocols/mail_smtp.c11
-rw-r--r--src/lib/protocols/netbios.c4
-rw-r--r--src/lib/protocols/quic.c20
-rw-r--r--src/lib/protocols/stun.c14
-rw-r--r--src/lib/protocols/tls.c71
-rw-r--r--src/lib/protocols/whoisdas.c5
9 files changed, 88 insertions, 112 deletions
diff --git a/src/lib/protocols/dhcp.c b/src/lib/protocols/dhcp.c
index d40bb5c35..23d358b74 100644
--- a/src/lib/protocols/dhcp.c
+++ b/src/lib/protocols/dhcp.c
@@ -159,17 +159,14 @@ void ndpi_search_dhcp_udp(struct ndpi_detection_module_struct *ndpi_struct, stru
strncpy((char*)flow->protos.dhcp.class_ident, name, j);
flow->protos.dhcp.class_ident[j] = '\0';
} else if(id == 12 /* Host Name */) {
- char *name = (char*)&dhcp->options[i+2];
- int j = 0;
-
+ u_int8_t *name = &dhcp->options[i+2];
+
#ifdef DHCP_DEBUG
NDPI_LOG_DBG2(ndpi_struct, "[DHCP] '%.*s'\n",name,len);
// while(j < len) { printf( "%c", name[j]); j++; }; printf("\n");
#endif
- j = ndpi_min(len, sizeof(flow->host_server_name)-1);
- strncpy((char*)flow->host_server_name, name, j);
- flow->host_server_name[j] = '\0';
- }
+ ndpi_hostname_sni_set(flow, name, len);
+ }
i += len + 2;
}
diff --git a/src/lib/protocols/dns.c b/src/lib/protocols/dns.c
index 98c6bf142..eb5af5a34 100644
--- a/src/lib/protocols/dns.c
+++ b/src/lib/protocols/dns.c
@@ -368,6 +368,7 @@ static void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, st
int invalid = search_valid_dns(ndpi_struct, flow, &dns_header, payload_offset, &is_query);
ndpi_protocol ret;
u_int num_queries, idx;
+ char _hostname[256];
ret.master_protocol = NDPI_PROTOCOL_UNKNOWN;
ret.app_protocol = (d_port == LLMNR_PORT) ? NDPI_PROTOCOL_LLMNR : ((d_port == MDNS_PORT) ? NDPI_PROTOCOL_MDNS : NDPI_PROTOCOL_DNS);
@@ -378,7 +379,6 @@ static void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, st
}
/* extract host name server */
- max_len = sizeof(flow->host_server_name)-1;
off = sizeof(struct ndpi_dns_packet_header) + payload_offset;
/* Before continuing let's dissect the following queries to see if they are valid */
@@ -434,6 +434,7 @@ static void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, st
} /* for */
u_int8_t hostname_is_valid = 1;
+ max_len = sizeof(_hostname)-1;
while((j < max_len) && (off < packet->payload_packet_len) && (packet->payload[off] != '\0')) {
uint8_t c, cl = packet->payload[off++];
@@ -443,40 +444,43 @@ static void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, st
break;
}
- if(j && (j < max_len)) flow->host_server_name[j++] = '.';
-
- while((j < max_len) && (cl != 0)) {
- u_int32_t shift;
-
- c = packet->payload[off++];
- shift = ((u_int32_t) 1) << (c & 0x1f);
- if ((dns_validchar[c >> 5] & shift)) {
- flow->host_server_name[j++] = tolower(c);
- } else {
- if (isprint(c) == 0) {
- hostname_is_valid = 0;
- flow->host_server_name[j++] = '?';
- } else {
- flow->host_server_name[j++] = '_';
- }
- }
- cl--;
- }
+ if(j && (j < max_len)) _hostname[j++] = '.';
+
+ while((j < max_len) && (cl != 0)) {
+ u_int32_t shift;
+
+ c = packet->payload[off++];
+ shift = ((u_int32_t) 1) << (c & 0x1f);
+ if((dns_validchar[c >> 5] & shift)) {
+ _hostname[j++] = tolower(c);
+ } else {
+ if (isprint(c) == 0) {
+ hostname_is_valid = 0;
+ _hostname[j++] = '?';
+ } else {
+ _hostname[j++] = '_';
+ }
+ }
+ cl--;
+ }
}
+
+ _hostname[j] = '\0';
+
+ ndpi_hostname_sni_set(flow, (const u_int8_t *)_hostname, j);
+
if (hostname_is_valid == 0) {
ndpi_set_risk(ndpi_struct, flow, NDPI_INVALID_CHARACTERS);
}
- flow->host_server_name[j] = '\0';
-
if(j > 0) {
ndpi_protocol_match_result ret_match;
- ndpi_check_dga_name(ndpi_struct, flow, (char*)flow->host_server_name, 1);
+ ndpi_check_dga_name(ndpi_struct, flow, flow->host_server_name, 1);
ret.app_protocol = ndpi_match_host_subprotocol(ndpi_struct, flow,
- (char *)flow->host_server_name,
- strlen((const char*)flow->host_server_name),
+ flow->host_server_name,
+ strlen(flow->host_server_name),
&ret_match,
NDPI_PROTOCOL_DNS);
diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c
index cef8b3cfc..bc0ca1fc2 100644
--- a/src/lib/protocols/http.c
+++ b/src/lib/protocols/http.c
@@ -371,8 +371,8 @@ static void ndpi_http_parse_subprotocol(struct ndpi_detection_module_struct *ndp
if(double_col) double_col[0] = '\0';
if(ndpi_match_hostname_protocol(ndpi_struct, flow, NDPI_PROTOCOL_HTTP,
- (char *)flow->host_server_name,
- strlen((const char *)flow->host_server_name)) == 0) {
+ flow->host_server_name,
+ strlen(flow->host_server_name)) == 0) {
if(flow->http.url &&
((strstr(flow->http.url, ":8080/downloading?n=0.") != NULL)
|| (strstr(flow->http.url, ":8080/upload?n=0.") != NULL))) {
@@ -566,12 +566,10 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
packet->host_line.len, packet->host_line.ptr);
/* Copy result for nDPI apps */
- len = ndpi_min(packet->host_line.len, sizeof(flow->host_server_name)-1);
- strncpy((char*)flow->host_server_name, (char*)packet->host_line.ptr, len);
- flow->host_server_name[len] = '\0';
+ ndpi_hostname_sni_set(flow, packet->host_line.ptr, packet->host_line.len);
flow->extra_packets_func = NULL; /* We're good now */
- if(len > 0) ndpi_check_dga_name(ndpi_struct, flow, (char*)flow->host_server_name, 1);
+ if(strlen(flow->host_server_name) > 0) ndpi_check_dga_name(ndpi_struct, flow, flow->host_server_name, 1);
if(packet->forwarded_line.ptr) {
len = ndpi_min(packet->forwarded_line.len, sizeof(flow->http.nat_ip)-1);
diff --git a/src/lib/protocols/mail_smtp.c b/src/lib/protocols/mail_smtp.c
index 31310202b..e1d98dd35 100644
--- a/src/lib/protocols/mail_smtp.c
+++ b/src/lib/protocols/mail_smtp.c
@@ -147,12 +147,11 @@ void ndpi_search_mail_smtp_tcp(struct ndpi_detection_module_struct *ndpi_struct,
&& (packet->line[a].ptr[i+1] != '\n')) {
len = i-4;
/* Copy result for nDPI apps */
- len = ndpi_min(len, sizeof(flow->host_server_name)-1);
- strncpy((char*)flow->host_server_name, (char*)&packet->line[a].ptr[4], len);
- flow->host_server_name[len] = '\0';
- if(ndpi_match_hostname_protocol(ndpi_struct, flow, NDPI_PROTOCOL_MAIL_SMTP,
- (char *)flow->host_server_name,
- strlen((const char *)flow->host_server_name))) {
+ ndpi_hostname_sni_set(flow, &packet->line[a].ptr[4], len);
+
+ if (ndpi_match_hostname_protocol(ndpi_struct, flow, NDPI_PROTOCOL_MAIL_SMTP,
+ flow->host_server_name,
+ strlen(flow->host_server_name))) {
/* We set the protocols; we need to initialize extra dissection
to search for credentials */
NDPI_LOG_DBG(ndpi_struct, "SMTP: hostname matched\n");
diff --git a/src/lib/protocols/netbios.c b/src/lib/protocols/netbios.c
index ccf910489..88c211551 100644
--- a/src/lib/protocols/netbios.c
+++ b/src/lib/protocols/netbios.c
@@ -102,9 +102,9 @@ static void ndpi_int_netbios_add_connection(struct ndpi_detection_module_struct
if((off < packet->payload_packet_len)
&& ndpi_netbios_name_interpret((unsigned char*)&packet->payload[off],
(u_int)(packet->payload_packet_len - off), name, sizeof(name)-1) > 0) {
- snprintf((char*)flow->host_server_name, sizeof(flow->host_server_name)-1, "%s", name);
+ ndpi_hostname_sni_set(flow, (const u_int8_t *)name, strlen((char *)name));
- ndpi_check_dga_name(ndpi_struct, flow, (char*)flow->host_server_name, 1);
+ ndpi_check_dga_name(ndpi_struct, flow, flow->host_server_name, 1);
}
if(sub_protocol == NDPI_PROTOCOL_UNKNOWN)
diff --git a/src/lib/protocols/quic.c b/src/lib/protocols/quic.c
index 358edb064..6bae70524 100644
--- a/src/lib/protocols/quic.c
+++ b/src/lib/protocols/quic.c
@@ -1324,7 +1324,7 @@ static void process_chlo(struct ndpi_detection_module_struct *ndpi_struct,
uint32_t i;
uint16_t num_tags;
uint32_t prev_offset;
- uint32_t tag_offset_start, offset, len, sni_len;
+ uint32_t tag_offset_start, offset, len;
ndpi_protocol_match_result ret_match;
int sni_found = 0, ua_found = 0;
@@ -1356,22 +1356,20 @@ static void process_chlo(struct ndpi_detection_module_struct *ndpi_struct,
crypto_data_len, tag_offset_start, prev_offset, offset, len);
#endif
if(memcmp(tag, "SNI\0", 4) == 0) {
- sni_len = MIN(len, sizeof(flow->protos.tls_quic.client_requested_server_name) - 1);
- memcpy(flow->protos.tls_quic.client_requested_server_name,
- &crypto_data[tag_offset_start + prev_offset], sni_len);
- flow->protos.tls_quic.client_requested_server_name[sni_len] = '\0';
+
+ ndpi_hostname_sni_set(flow, &crypto_data[tag_offset_start + prev_offset], len);
NDPI_LOG_DBG2(ndpi_struct, "SNI: [%s]\n",
- flow->protos.tls_quic.client_requested_server_name);
+ flow->host_server_name);
ndpi_match_host_subprotocol(ndpi_struct, flow,
- (char *)flow->protos.tls_quic.client_requested_server_name,
- strlen((const char*)flow->protos.tls_quic.client_requested_server_name),
+ flow->host_server_name,
+ strlen(flow->host_server_name),
&ret_match, NDPI_PROTOCOL_QUIC);
flow->protos.tls_quic.hello_processed = 1; /* Allow matching of custom categories */
ndpi_check_dga_name(ndpi_struct, flow,
- flow->protos.tls_quic.client_requested_server_name, 1);
+ flow->host_server_name, 1);
sni_found = 1;
if (ua_found)
@@ -1396,7 +1394,7 @@ static void process_chlo(struct ndpi_detection_module_struct *ndpi_struct,
NDPI_LOG_DBG(ndpi_struct, "Something went wrong in tags iteration\n");
/* Add check for missing SNI */
- if(flow->protos.tls_quic.client_requested_server_name[0] == '\0') {
+ if(flow->host_server_name[0] == '\0') {
/* This is a bit suspicious */
ndpi_set_risk(ndpi_struct, flow, NDPI_TLS_MISSING_SNI);
}
@@ -1508,7 +1506,7 @@ static int eval_extra_processing(struct ndpi_detection_module_struct *ndpi_struc
*/
if((version == V_Q046 &&
- flow->protos.tls_quic.client_requested_server_name[0] == '\0') ||
+ flow->host_server_name[0] == '\0') ||
is_ch_reassembler_pending(flow)) {
NDPI_LOG_DBG2(ndpi_struct, "We have further work to do\n");
return 1;
diff --git a/src/lib/protocols/stun.c b/src/lib/protocols/stun.c
index f0884fae3..1ffa8afef 100644
--- a/src/lib/protocols/stun.c
+++ b/src/lib/protocols/stun.c
@@ -349,25 +349,21 @@ static ndpi_int_stun_t ndpi_int_check_stun(struct ndpi_detection_module_struct *
u_int16_t realm_len = ntohs(*((u_int16_t*)&payload[offset+2]));
if(flow->host_server_name[0] == '\0') {
- u_int i;
u_int k = offset+4;
- i = ndpi_min(realm_len, sizeof(flow->host_server_name) - 1);
- i = ndpi_min(i, payload_length - k);
- memcpy(flow->host_server_name, payload + k, i);
- flow->host_server_name[i] = '\0';
-
+ ndpi_hostname_sni_set(flow, payload + k, ndpi_min(realm_len, payload_length - k));
+
#ifdef DEBUG_STUN
printf("==> [%s]\n", flow->host_server_name);
#endif
- if(strstr((char*) flow->host_server_name, "google.com") != NULL) {
+ if(strstr(flow->host_server_name, "google.com") != NULL) {
flow->guessed_host_protocol_id = NDPI_PROTOCOL_HANGOUT_DUO;
return(NDPI_IS_STUN);
- } else if(strstr((char*) flow->host_server_name, "whispersystems.org") != NULL) {
+ } else if(strstr(flow->host_server_name, "whispersystems.org") != NULL) {
flow->guessed_host_protocol_id = NDPI_PROTOCOL_SIGNAL;
return(NDPI_IS_STUN);
- } else if(strstr((char*) flow->host_server_name, "facebook") != NULL) {
+ } else if(strstr(flow->host_server_name, "facebook") != NULL) {
flow->guessed_host_protocol_id = NDPI_PROTOCOL_MESSENGER;
return(NDPI_IS_STUN);
}
diff --git a/src/lib/protocols/tls.c b/src/lib/protocols/tls.c
index d912ae947..a75e26eb2 100644
--- a/src/lib/protocols/tls.c
+++ b/src/lib/protocols/tls.c
@@ -478,7 +478,7 @@ static void processCertificateElements(struct ndpi_detection_module_struct *ndpi
u_int8_t matched_name = 0;
/* If the client hello was not observed or the requested name was missing, there is no need to trigger an alert */
- if(flow->protos.tls_quic.client_requested_server_name[0] == '\0')
+ if(flow->host_server_name[0] == '\0')
matched_name = 1;
#ifdef DEBUG_TLS
@@ -520,7 +520,7 @@ static void processCertificateElements(struct ndpi_detection_module_struct *ndpi
#if DEBUG_TLS
printf("[TLS] dNSName %s [%s][len: %u][leftover: %d]\n", dNSName,
- flow->protos.tls_quic.client_requested_server_name, len,
+ flow->host_server_name, len,
packet->payload_packet_len-i-len);
#endif
if (ndpi_is_printable_string(dNSName, len) == 0) {
@@ -530,27 +530,24 @@ static void processCertificateElements(struct ndpi_detection_module_struct *ndpi
if(matched_name == 0) {
#if DEBUG_TLS
printf("[TLS] Trying to match '%s' with '%s'\n",
- flow->protos.tls_quic.client_requested_server_name,
+ flow->host_server_name,
dNSName);
#endif
- if(flow->protos.tls_quic.client_requested_server_name[0] == '\0')
+ if(flow->host_server_name[0] == '\0') {
matched_name = 1; /* No SNI */
- else if (dNSName[0] == '*')
- {
- char * label = strstr(flow->protos.tls_quic.client_requested_server_name, &dNSName[1]);
-
- if (label != NULL)
- {
- char * first_dot = strchr(flow->protos.tls_quic.client_requested_server_name, '.');
-
- if (first_dot == NULL || first_dot >= label)
- {
- matched_name = 1;
- }
- }
- }
- else if(strcmp(flow->protos.tls_quic.client_requested_server_name, dNSName) == 0) {
+ } else if (dNSName[0] == '*') {
+ char * label = strstr(flow->host_server_name, &dNSName[1]);
+
+ if (label != NULL) {
+ char * first_dot = strchr(flow->host_server_name, '.');
+
+ if (first_dot == NULL || first_dot >= label) {
+ matched_name = 1;
+ }
+ }
+ }
+ else if(strcmp(flow->host_server_name, dNSName) == 0) {
matched_name = 1;
}
}
@@ -1244,7 +1241,6 @@ int processClientServerHello(struct ndpi_detection_module_struct *ndpi_struct,
u_int32_t i, j;
u_int16_t total_len;
u_int8_t handshake_type;
- char buffer[64] = { '\0' };
int is_quic = (quic_version != 0);
int is_dtls = packet->udp && (!is_quic);
@@ -1721,50 +1717,41 @@ int processClientServerHello(struct ndpi_detection_module_struct *ndpi_struct,
if((offset+extension_offset+4) < packet->payload_packet_len) {
len = (packet->payload[offset+extension_offset+3] << 8) + packet->payload[offset+extension_offset+4];
- len = (u_int)ndpi_min(len, sizeof(buffer)-1);
if((offset+extension_offset+5+len) <= packet->payload_packet_len) {
- strncpy(buffer, (char*)&packet->payload[offset+extension_offset+5], len);
- buffer[len] = '\0';
- cleanupServerName(buffer, sizeof(buffer));
-
- snprintf(flow->protos.tls_quic.client_requested_server_name,
- sizeof(flow->protos.tls_quic.client_requested_server_name),
- "%s", buffer);
+ char *sni = ndpi_hostname_sni_set(flow, &packet->payload[offset+extension_offset+5], len);
+ int sni_len = strlen(sni);
#ifdef DEBUG_TLS
- printf("[TLS] SNI: [%s]\n", buffer);
+ printf("[TLS] SNI: [%s]\n", sni);
#endif
- if (ndpi_is_printable_string(buffer, len) == 0)
+ if (ndpi_is_printable_string(sni, sni_len) == 0)
{
ndpi_set_risk(ndpi_struct, flow, NDPI_INVALID_CHARACTERS);
}
if(!is_quic) {
- if(ndpi_match_hostname_protocol(ndpi_struct, flow, NDPI_PROTOCOL_TLS, buffer, strlen(buffer)))
+ if(ndpi_match_hostname_protocol(ndpi_struct, flow, NDPI_PROTOCOL_TLS, sni, sni_len))
flow->protos.tls_quic.subprotocol_detected = 1;
} else {
- if(ndpi_match_hostname_protocol(ndpi_struct, flow, NDPI_PROTOCOL_QUIC, buffer, strlen(buffer)))
+ if(ndpi_match_hostname_protocol(ndpi_struct, flow, NDPI_PROTOCOL_QUIC, sni, sni_len))
flow->protos.tls_quic.subprotocol_detected = 1;
}
if(ndpi_check_dga_name(ndpi_struct, flow,
- flow->protos.tls_quic.client_requested_server_name, 1)) {
- char *sni = flow->protos.tls_quic.client_requested_server_name;
- int len = strlen(sni);
-
+ sni, 1)) {
#ifdef DEBUG_TLS
- printf("[TLS] SNI: (DGA) [%s]\n", flow->protos.tls_quic.client_requested_server_name);
+ printf("[TLS] SNI: (DGA) [%s]\n", sni);
#endif
- if((len >= 4)
+ if((sni_len >= 4)
/* Check if it ends in .com or .net */
- && ((strcmp(&sni[len-4], ".com") == 0) || (strcmp(&sni[len-4], ".net") == 0))
+ && ((strcmp(&sni[sni_len-4], ".com") == 0) || (strcmp(&sni[sni_len-4], ".net") == 0))
&& (strncmp(sni, "www.", 4) == 0)) /* Not starting with www.... */
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_TOR, NDPI_PROTOCOL_TLS);
} else {
#ifdef DEBUG_TLS
- printf("[TLS] SNI: (NO DGA) [%s]\n", flow->protos.tls_quic.client_requested_server_name);
+ printf("[TLS] SNI: (NO DGA) [%s]\n", sni);
#endif
}
} else {
@@ -2268,12 +2255,12 @@ int processClientServerHello(struct ndpi_detection_module_struct *ndpi_struct,
/* Suspicious Domain Fronting:
https://github.com/SixGenInc/Noctilucent/blob/master/docs/ */
if(flow->protos.tls_quic.encrypted_sni.esni &&
- flow->protos.tls_quic.client_requested_server_name[0] != '\0') {
+ flow->host_server_name[0] != '\0') {
ndpi_set_risk(ndpi_struct, flow, NDPI_TLS_SUSPICIOUS_ESNI_USAGE);
}
/* Add check for missing SNI */
- if((flow->protos.tls_quic.client_requested_server_name[0] == 0)
+ if(flow->host_server_name[0] == '\0'
&& (flow->protos.tls_quic.ssl_version >= 0x0302) /* TLSv1.1 */
&& (flow->protos.tls_quic.encrypted_sni.esni == NULL) /* No ESNI */
) {
diff --git a/src/lib/protocols/whoisdas.c b/src/lib/protocols/whoisdas.c
index 7321626d2..6f6f2a06b 100644
--- a/src/lib/protocols/whoisdas.c
+++ b/src/lib/protocols/whoisdas.c
@@ -41,10 +41,7 @@ void ndpi_search_whois_das(struct ndpi_detection_module_struct *ndpi_struct, str
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_WHOIS_DAS, NDPI_PROTOCOL_UNKNOWN);
if((dport == 43) || (dport == 4343)) { /* Request */
- u_int hostname_len = ndpi_min(sizeof(flow->host_server_name) - 1, (long unsigned int)packet->payload_packet_len - 2); /* Skip \r\n */
-
- memcpy(flow->host_server_name, &packet->payload[0], hostname_len);
- flow->host_server_name[hostname_len] = '\0';
+ ndpi_hostname_sni_set(flow, &packet->payload[0], packet->payload_packet_len - 2); /* Skip \r\n */
NDPI_LOG_INFO(ndpi_struct, "[WHOIS/DAS] %s\n", flow->host_server_name);
}
return;