From 476fbbfe4e141c199f0cd79258ab9085ee536e9e Mon Sep 17 00:00:00 2001 From: Vitaly Lavrov Date: Thu, 31 Jan 2019 11:27:40 +0300 Subject: Fix whatsapp signature detecting. --- src/lib/protocols/whatsapp.c | 34 +++++++++++++--------------------- 1 file changed, 13 insertions(+), 21 deletions(-) (limited to 'src/lib/protocols') diff --git a/src/lib/protocols/whatsapp.c b/src/lib/protocols/whatsapp.c index 6964a8e0e..608e6576e 100644 --- a/src/lib/protocols/whatsapp.c +++ b/src/lib/protocols/whatsapp.c @@ -26,34 +26,26 @@ void ndpi_search_whatsapp(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) { struct ndpi_packet_struct *packet = &flow->packet; - u_int8_t whatsapp_sequence[] = { + static u_int8_t whatsapp_sequence[] = { 0x45, 0x44, 0x0, 0x01, 0x0, 0x0, 0x02, 0x08, 0x0, 0x57, 0x41, 0x02, 0x0, 0x0, 0x0 }; NDPI_LOG_DBG(ndpi_struct, "search WhatsApp\n"); - if(flow->l4.tcp.wa_matched_so_far == 0) { - if(memcmp(packet->payload, whatsapp_sequence, packet->payload_packet_len)) { - NDPI_EXCLUDE_PROTO(ndpi_struct, flow); - } else - flow->l4.tcp.wa_matched_so_far = packet->payload_packet_len; + if(flow->l4.tcp.wa_matched_so_far < sizeof(whatsapp_sequence)) { + size_t match_len = sizeof(whatsapp_sequence) - flow->l4.tcp.wa_matched_so_far; + if(packet->payload_packet_len < match_len) + match_len = packet->payload_packet_len; - return; - } else { - if(memcmp(packet->payload, &whatsapp_sequence[flow->l4.tcp.wa_matched_so_far], - sizeof(whatsapp_sequence)-flow->l4.tcp.wa_matched_so_far)) - NDPI_EXCLUDE_PROTO(ndpi_struct, flow); - else - ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_WHATSAPP, NDPI_PROTOCOL_UNKNOWN); - - return; - } - - if((packet->payload_packet_len > 240) - && (memcmp(packet->payload, whatsapp_sequence, sizeof(whatsapp_sequence)) == 0)) { - NDPI_LOG_INFO(ndpi_struct, "found WhatsApp\n"); - ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_WHATSAPP, NDPI_PROTOCOL_UNKNOWN); + if(!memcmp(packet->payload, &whatsapp_sequence[flow->l4.tcp.wa_matched_so_far], match_len)) { + flow->l4.tcp.wa_matched_so_far += match_len; + if(flow->l4.tcp.wa_matched_so_far == sizeof(whatsapp_sequence)) { + NDPI_LOG_INFO(ndpi_struct, "found WhatsApp\n"); + ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_WHATSAPP, NDPI_PROTOCOL_UNKNOWN); + } + return; + } } NDPI_EXCLUDE_PROTO(ndpi_struct, flow); -- cgit v1.2.3 From a99754d88bd1386536162d434304b61048da3f22 Mon Sep 17 00:00:00 2001 From: Vitaly Lavrov Date: Wed, 27 Feb 2019 12:21:33 +0300 Subject: More accurate hostname verification in a DNS query. --- src/lib/protocols/dns.c | 44 ++++++++++++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 14 deletions(-) (limited to 'src/lib/protocols') diff --git a/src/lib/protocols/dns.c b/src/lib/protocols/dns.c index b636f3e89..57213b889 100644 --- a/src/lib/protocols/dns.c +++ b/src/lib/protocols/dns.c @@ -59,7 +59,19 @@ static u_int getNameLength(u_int i, const u_int8_t *payload, u_int payloadLen) { return(off + getNameLength(i+off, payload, payloadLen)); } } +/* + allowed chars for dns names A-Z 0-9 _ - + Perl script for generation map: + my @M; + for(my $ch=0; $ch < 256; $ch++) { + $M[$ch >> 5] |= 1 << ($ch & 0x1f) if chr($ch) =~ /[a-z0-9_-]/i; + } + print join(',', map { sprintf "0x%08x",$_ } @M),"\n"; + */ +static uint32_t dns_validchar[8] = { + 0x00000000,0x03ff2000,0x87fffffe,0x07fffffe,0,0,0,0 +}; /* *********************************************** */ void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) { @@ -108,6 +120,7 @@ void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, struct nd invalid = 1; if(!invalid) { + int j = 0, max_len, off; if(is_query) { /* DNS Request */ if((dns_header.num_queries > 0) && (dns_header.num_queries <= NDPI_MAX_DNS_REQUESTS) @@ -186,28 +199,31 @@ void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, struct nd } /* extract host name server */ - int j = 0, max_len = sizeof(flow->host_server_name)-1, off = sizeof(struct ndpi_dns_packet_header) + 1 + payload_offset; - while(off < flow->packet.payload_packet_len && flow->packet.payload[off] != '\0') { - flow->host_server_name[j] = flow->packet.payload[off]; - if(j < max_len) { - if(flow->host_server_name[j] < ' ') - flow->host_server_name[j] = '.'; - j++; - } else - break; - - off++; + max_len = sizeof(flow->host_server_name)-1; + off = sizeof(struct ndpi_dns_packet_header) + payload_offset; + + while(j < max_len && off < flow->packet.payload_packet_len && flow->packet.payload[off] != '\0') { + uint8_t c,cl = flow->packet.payload[off++]; + if( (cl & 0xc0) != 0 || // we not support compressed names in query + off + cl >= flow->packet.payload_packet_len) { + j = 0; break; + } + if(j && j < max_len) flow->host_server_name[j++] = '.'; + while(j < max_len && cl != 0) { + c = flow->packet.payload[off++]; + flow->host_server_name[j++] = dns_validchar[c >> 5] & (1 << (c & 0x1f)) ? c:'_'; + cl--; + } } + flow->host_server_name[j] = '\0'; if(is_query && (ndpi_struct->dns_dont_dissect_response == 0)) { // dpi_set_detected_protocol(ndpi_struct, flow, (d_port == 5355) ? NDPI_PROTOCOL_LLMNR : NDPI_PROTOCOL_DNS, NDPI_PROTOCOL_UNKNOWN); return; /* The response will set the verdict */ } - - flow->host_server_name[j] = '\0'; flow->protos.dns.num_queries = (u_int8_t)dns_header.num_queries, - flow->protos.dns.num_answers = (u_int8_t) (dns_header.num_answers + dns_header.authority_rrs + dns_header.additional_rrs); + flow->protos.dns.num_answers = (u_int8_t) (dns_header.num_answers + dns_header.authority_rrs + dns_header.additional_rrs); if(j > 0) { ndpi_protocol_match_result ret_match; -- cgit v1.2.3 From b1bd7d6f8c7a18beabaf8224056f13c2193dce03 Mon Sep 17 00:00:00 2001 From: chiehminw Date: Tue, 12 Mar 2019 16:11:33 +0800 Subject: Add NDPI prefix for HTTP_METHOD enum to avoid name collisions ref: https://stackoverflow.com/questions/35380279/avoid-name-collisions-with-enum-in-c-c99 Signed-off-by: chiehminw --- src/include/ndpi_typedefs.h | 18 +++++++++--------- src/lib/protocols/http.c | 22 +++++++++++----------- 2 files changed, 20 insertions(+), 20 deletions(-) (limited to 'src/lib/protocols') diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h index a5f1864e0..e561deb47 100644 --- a/src/include/ndpi_typedefs.h +++ b/src/include/ndpi_typedefs.h @@ -356,15 +356,15 @@ PACK_ON struct tinc_cache_entry { } PACK_OFF; typedef enum { - HTTP_METHOD_UNKNOWN = 0, - HTTP_METHOD_OPTIONS, - HTTP_METHOD_GET, - HTTP_METHOD_HEAD, - HTTP_METHOD_POST, - HTTP_METHOD_PUT, - HTTP_METHOD_DELETE, - HTTP_METHOD_TRACE, - HTTP_METHOD_CONNECT + NDPI_HTTP_METHOD_UNKNOWN = 0, + NDPI_HTTP_METHOD_OPTIONS, + NDPI_HTTP_METHOD_GET, + NDPI_HTTP_METHOD_HEAD, + NDPI_HTTP_METHOD_POST, + NDPI_HTTP_METHOD_PUT, + NDPI_HTTP_METHOD_DELETE, + NDPI_HTTP_METHOD_TRACE, + NDPI_HTTP_METHOD_CONNECT } ndpi_http_method; struct ndpi_lru_cache { diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c index 4ce80f9c9..661e43bf2 100644 --- a/src/lib/protocols/http.c +++ b/src/lib/protocols/http.c @@ -260,25 +260,25 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ } if(flow->packet.http_method.len < 3) - flow->http.method = HTTP_METHOD_UNKNOWN; + flow->http.method = NDPI_HTTP_METHOD_UNKNOWN; else { switch(flow->packet.http_method.ptr[0]) { - case 'O': flow->http.method = HTTP_METHOD_OPTIONS; break; - case 'G': flow->http.method = HTTP_METHOD_GET; break; - case 'H': flow->http.method = HTTP_METHOD_HEAD; break; + case 'O': flow->http.method = NDPI_HTTP_METHOD_OPTIONS; break; + case 'G': flow->http.method = NDPI_HTTP_METHOD_GET; break; + case 'H': flow->http.method = NDPI_HTTP_METHOD_HEAD; break; case 'P': switch(flow->packet.http_method.ptr[1]) { - case 'O': flow->http.method = HTTP_METHOD_POST; break; - case 'U': flow->http.method = HTTP_METHOD_PUT; break; + case 'O': flow->http.method = NDPI_HTTP_METHOD_POST; break; + case 'U': flow->http.method = NDPI_HTTP_METHOD_PUT; break; } break; - case 'D': flow->http.method = HTTP_METHOD_DELETE; break; - case 'T': flow->http.method = HTTP_METHOD_TRACE; break; - case 'C': flow->http.method = HTTP_METHOD_CONNECT; break; + case 'D': flow->http.method = NDPI_HTTP_METHOD_DELETE; break; + case 'T': flow->http.method = NDPI_HTTP_METHOD_TRACE; break; + case 'C': flow->http.method = NDPI_HTTP_METHOD_CONNECT; break; default: - flow->http.method = HTTP_METHOD_UNKNOWN; + flow->http.method = NDPI_HTTP_METHOD_UNKNOWN; break; } } @@ -949,7 +949,7 @@ void ndpi_search_http_tcp(struct ndpi_detection_module_struct *ndpi_struct, ndpi_http_method ndpi_get_http_method(struct ndpi_detection_module_struct *ndpi_mod, struct ndpi_flow_struct *flow) { if(!flow) - return(HTTP_METHOD_UNKNOWN); + return(NDPI_HTTP_METHOD_UNKNOWN); else return(flow->http.method); } -- cgit v1.2.3 From b1282da5424cfc7fbe331ac5256148780a5fa2fb Mon Sep 17 00:00:00 2001 From: Guido Falsi Date: Tue, 12 Mar 2019 18:12:18 +0100 Subject: The le32toh() function used in some places on BSD OSes require sys/endian.h to be included. --- src/lib/ndpi_main.c | 4 ++++ src/lib/protocols/quic.c | 4 ++++ 2 files changed, 8 insertions(+) (limited to 'src/lib/protocols') diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index 2471a0eb1..4e9f37248 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -41,6 +41,10 @@ #include #endif +#if defined __FreeBSD__ || defined __NetBSD__ || defined __OpenBSD__ +#include +#endif + #include "ndpi_content_match.c.inc" #include "third_party/include/ndpi_patricia.h" #include "third_party/include/ht_hash.h" diff --git a/src/lib/protocols/quic.c b/src/lib/protocols/quic.c index 322eb9be7..d14538e0d 100644 --- a/src/lib/protocols/quic.c +++ b/src/lib/protocols/quic.c @@ -22,6 +22,10 @@ * */ +#if defined __FreeBSD__ || defined __NetBSD__ || defined __OpenBSD__ +#include +#endif + #include "ndpi_protocol_ids.h" #define NDPI_CURRENT_PROTO NDPI_PROTOCOL_QUIC -- cgit v1.2.3 From db91837647737b3d57359dd6d7bc8474644f8043 Mon Sep 17 00:00:00 2001 From: chiehminw Date: Wed, 20 Mar 2019 17:22:37 +0800 Subject: retreive ssl certificate origanization Signed-off-by: chiehminw --- src/include/ndpi_typedefs.h | 2 +- src/lib/protocols/ssl.c | 69 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) (limited to 'src/lib/protocols') diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h index 39995d662..0c71f9de4 100644 --- a/src/include/ndpi_typedefs.h +++ b/src/include/ndpi_typedefs.h @@ -1085,7 +1085,7 @@ struct ndpi_flow_struct { struct { struct { - char client_certificate[64], server_certificate[64]; + char client_certificate[64], server_certificate[64], server_organization[64]; } ssl; struct { diff --git a/src/lib/protocols/ssl.c b/src/lib/protocols/ssl.c index ff6b47a0f..845f20de6 100644 --- a/src/lib/protocols/ssl.c +++ b/src/lib/protocols/ssl.c @@ -335,18 +335,87 @@ int getSSLcertificate(struct ndpi_detection_module_struct *ndpi_struct, return(0); /* Not found */ } +void getSSLorganization(struct ndpi_detection_module_struct *ndpi_struct, + struct ndpi_flow_struct *flow, + char *buffer, int buffer_len) { + struct ndpi_packet_struct *packet = &flow->packet; + + if(packet->payload[0] != 0x16 /* Handshake */) + return; + + u_int16_t total_len = (packet->payload[3] << 8) + packet->payload[4] + 5 /* SSL Header */; + u_int8_t handshake_protocol = packet->payload[5]; /* handshake protocol a bit misleading, it is message type according TLS specs */ + + if(handshake_protocol != 0x02 && handshake_protocol != 0xb /* Server Hello and Certificate message types are interesting for us */) + return; + + /* Truncate total len, search at least in incomplete packet */ + if(total_len > packet->payload_packet_len) + total_len = packet->payload_packet_len; + + memset(buffer, 0, buffer_len); + + /* Check after handshake protocol header (5 bytes) and message header (4 bytes) */ + u_int num_found = 0; + u_int i, j; + for(i = 9; i < packet->payload_packet_len-4; i++) { + /* Organization OID: 2.5.4.10 */ + if((packet->payload[i] == 0x55) && (packet->payload[i+1] == 0x04) && (packet->payload[i+2] == 0x0a)) { + u_int8_t type_tag = packet->payload[i+3]; // 0x0c: utf8string / 0x13: printable_string + u_int8_t server_len = packet->payload[i+4]; + + num_found++; + /* what we want is subject certificate, so we bypass the issuer certificate */ + if (num_found != 2) continue; + + // packet is truncated... further inspection is not needed + if(i+4+server_len >= packet->payload_packet_len) { + break; + } + + char *server_org = (char*)&packet->payload[i+5]; + + u_int len = (u_int)ndpi_min(server_len, buffer_len-1); + strncpy(buffer, server_org, len); + buffer[len] = '\0'; + + // check if organization string are all printable + u_int8_t is_printable = 1; + for (j = 0; j < len; j++) { + if(!ndpi_isprint(buffer[j])) { + is_printable = 0; + break; + } + } + + if (is_printable == 1) { + snprintf(flow->protos.stun_ssl.ssl.server_organization, + sizeof(flow->protos.stun_ssl.ssl.server_organization), "%s", buffer); +#ifdef CERTIFICATE_DEBUG + printf("Certificate origanization: %s\n", flow->protos.stun_ssl.ssl.server_organization); +#endif + } + } + } +} + int sslTryAndRetrieveServerCertificate(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) { struct ndpi_packet_struct *packet = &flow->packet; /* consider only specific SSL packets (handshake) */ if((packet->payload_packet_len > 9) && (packet->payload[0] == 0x16)) { char certificate[64]; + char organization[64]; int rc; certificate[0] = '\0'; rc = getSSLcertificate(ndpi_struct, flow, certificate, sizeof(certificate)); packet->ssl_certificate_num_checks++; if (rc > 0) { + // try fetch server organization once server certificate is found + organization[0] = '\0'; + getSSLorganization(ndpi_struct, flow, organization, sizeof(organization)); + packet->ssl_certificate_detected++; if ((flow->l4.tcp.ssl_seen_server_cert == 1) && (flow->protos.stun_ssl.ssl.server_certificate[0] != '\0')) /* 0 means we're done processing extra packets (since we found what we wanted) */ -- cgit v1.2.3 From 96ed31874dd62caef9e6f4dfe5c3825e88151c20 Mon Sep 17 00:00:00 2001 From: chiehminw Date: Wed, 20 Mar 2019 17:25:39 +0800 Subject: we only need one dot to confirm it is a valid domain name github.com is valid but can not be retreive by server certificate parsing Signed-off-by: chiehminw --- src/lib/protocols/ssl.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/lib/protocols') diff --git a/src/lib/protocols/ssl.c b/src/lib/protocols/ssl.c index 845f20de6..e6cc48bba 100644 --- a/src/lib/protocols/ssl.c +++ b/src/lib/protocols/ssl.c @@ -216,11 +216,11 @@ int getSSLcertificate(struct ndpi_detection_module_struct *ndpi_struct, break; } else if(buffer[j] == '.') { num_dots++; - if(num_dots >=2) break; + if(num_dots >=1) break; } } - if(num_dots >= 2) { + if(num_dots >= 1) { if(!ndpi_struct->disable_metadata_export) { stripCertificateTrailer(buffer, buffer_len); snprintf(flow->protos.stun_ssl.ssl.server_certificate, -- cgit v1.2.3 From 0b5e5a5f82ce87c14a5ecd588ed1dd0b08ff5e18 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Mon, 25 Mar 2019 22:08:39 +0100 Subject: SSL cleanup --- src/include/ndpi_typedefs.h | 5 ++- src/lib/protocols/ssl.c | 89 +++++++++++++++++++++++++-------------------- 2 files changed, 53 insertions(+), 41 deletions(-) (limited to 'src/lib/protocols') diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h index c49768f64..92c011b3d 100644 --- a/src/include/ndpi_typedefs.h +++ b/src/include/ndpi_typedefs.h @@ -541,7 +541,10 @@ struct ndpi_flow_tcp_struct { u_int32_t telnet_stage:2; // 0 - 2 /* NDPI_PROTOCOL_SSL */ - u_int8_t ssl_seen_client_cert:1, ssl_seen_server_cert:1, ssl_stage:2; // 0 - 5 + u_int8_t ssl_seen_client_cert:1, + ssl_seen_server_cert:1, + ssl_seen_certificate:1, + ssl_stage:2; // 0 - 5 /* NDPI_PROTOCOL_POSTGRES */ u_int32_t postgres_stage:3; diff --git a/src/lib/protocols/ssl.c b/src/lib/protocols/ssl.c index e6cc48bba..05988a8d4 100644 --- a/src/lib/protocols/ssl.c +++ b/src/lib/protocols/ssl.c @@ -27,7 +27,7 @@ #include "ndpi_api.h" -//#define CERTIFICATE_DEBUG 1 +// #define CERTIFICATE_DEBUG 1 #define NDPI_MAX_SSL_REQUEST_SIZE 10000 @@ -152,8 +152,9 @@ int getSSLcertificate(struct ndpi_detection_module_struct *ndpi_struct, #ifdef CERTIFICATE_DEBUG { u_int16_t ssl_version = (packet->payload[1] << 8) + packet->payload[2]; - - printf("SSL [version: %u]\n", ssl_version); + u_int16_t ssl_len = (packet->payload[3] << 8) + packet->payload[4]; + + printf("SSL Record [version: 0x%02X][len: %u]\n", ssl_version, ssl_len); } #endif @@ -175,10 +176,18 @@ int getSSLcertificate(struct ndpi_detection_module_struct *ndpi_struct, if(total_len > 4) { int i; - if(handshake_protocol == 0x02 || handshake_protocol == 0xb /* Server Hello and Certificate message types are interesting for us */) { +#ifdef CERTIFICATE_DEBUG + printf("SSL [len: %u][handshake_protocol: %02X]\n", packet->payload_packet_len, handshake_protocol); +#endif + + if((handshake_protocol == 0x02) + || (handshake_protocol == 0xb) /* Server Hello and Certificate message types are interesting for us */) { u_int num_found = 0; - flow->l4.tcp.ssl_seen_server_cert = 1; + if(handshake_protocol == 0x02) + flow->l4.tcp.ssl_seen_server_cert = 1; + else + flow->l4.tcp.ssl_seen_certificate = 1; /* Check after handshake protocol header (5 bytes) and message header (4 bytes) */ for(i = 9; i < packet->payload_packet_len-3; i++) { @@ -233,7 +242,7 @@ int getSSLcertificate(struct ndpi_detection_module_struct *ndpi_struct, } } else if(handshake_protocol == 0x01 /* Client Hello */) { u_int offset, base_offset = 43; - if (base_offset + 2 <= packet->payload_packet_len) { + if(base_offset + 2 <= packet->payload_packet_len) { u_int16_t session_id_len = packet->payload[base_offset]; if((session_id_len+base_offset+2) <= total_len) { @@ -366,7 +375,7 @@ void getSSLorganization(struct ndpi_detection_module_struct *ndpi_struct, num_found++; /* what we want is subject certificate, so we bypass the issuer certificate */ - if (num_found != 2) continue; + if(num_found != 2) continue; // packet is truncated... further inspection is not needed if(i+4+server_len >= packet->payload_packet_len) { @@ -388,7 +397,7 @@ void getSSLorganization(struct ndpi_detection_module_struct *ndpi_struct, } } - if (is_printable == 1) { + if(is_printable == 1) { snprintf(flow->protos.stun_ssl.ssl.server_organization, sizeof(flow->protos.stun_ssl.ssl.server_organization), "%s", buffer); #ifdef CERTIFICATE_DEBUG @@ -411,18 +420,18 @@ int sslTryAndRetrieveServerCertificate(struct ndpi_detection_module_struct *ndpi certificate[0] = '\0'; rc = getSSLcertificate(ndpi_struct, flow, certificate, sizeof(certificate)); packet->ssl_certificate_num_checks++; - if (rc > 0) { + if(rc > 0) { // try fetch server organization once server certificate is found organization[0] = '\0'; getSSLorganization(ndpi_struct, flow, organization, sizeof(organization)); packet->ssl_certificate_detected++; - if ((flow->l4.tcp.ssl_seen_server_cert == 1) && (flow->protos.stun_ssl.ssl.server_certificate[0] != '\0')) + if((flow->l4.tcp.ssl_seen_server_cert == 1) && (flow->protos.stun_ssl.ssl.server_certificate[0] != '\0')) /* 0 means we're done processing extra packets (since we found what we wanted) */ return 0; } /* Client hello, Server Hello, and certificate packets probably all checked in this case */ - if ((packet->ssl_certificate_num_checks >= 3) + if((packet->ssl_certificate_num_checks >= 3) && (flow->l4.tcp.seen_syn) && (flow->l4.tcp.seen_syn_ack) && (flow->l4.tcp.seen_ack) /* We have seen the 3-way handshake */) @@ -438,7 +447,7 @@ int sslTryAndRetrieveServerCertificate(struct ndpi_detection_module_struct *ndpi void sslInitExtraPacketProcessing(int caseNum, struct ndpi_flow_struct *flow) { flow->check_extra_packets = 1; /* 0 is the case for waiting for the server certificate */ - if (caseNum == 0) { + if(caseNum == 0) { /* At most 7 packets should almost always be enough to find the server certificate if it's there */ flow->max_extra_packets_to_check = 7; flow->extra_packets_func = sslTryAndRetrieveServerCertificate; @@ -474,7 +483,8 @@ int sslDetectProtocolFromCertificate(struct ndpi_detection_module_struct *ndpi_s /* If we've detected the subprotocol from client certificate but haven't had a chance * to see the server certificate yet, set up extra packet processing to wait * a few more packets. */ - if(((flow->l4.tcp.ssl_seen_client_cert == 1) && (flow->protos.stun_ssl.ssl.client_certificate[0] != '\0')) && ((flow->l4.tcp.ssl_seen_server_cert != 1) && (flow->protos.stun_ssl.ssl.server_certificate[0] == '\0'))) { + if(((flow->l4.tcp.ssl_seen_client_cert == 1) && (flow->protos.stun_ssl.ssl.client_certificate[0] != '\0')) + && ((flow->l4.tcp.ssl_seen_server_cert != 1) && (flow->protos.stun_ssl.ssl.server_certificate[0] == '\0'))) { sslInitExtraPacketProcessing(0, flow); } @@ -487,11 +497,13 @@ int sslDetectProtocolFromCertificate(struct ndpi_detection_module_struct *ndpi_s return(rc); } - if(((packet->ssl_certificate_num_checks >= 2) + if(((packet->ssl_certificate_num_checks >= 3) && flow->l4.tcp.seen_syn && flow->l4.tcp.seen_syn_ack && flow->l4.tcp.seen_ack /* We have seen the 3-way handshake */) - || ((flow->l4.tcp.ssl_seen_server_cert == 1) && (flow->protos.stun_ssl.ssl.server_certificate[0] != '\0')) + || ((flow->l4.tcp.ssl_seen_certificate == 1) + && (flow->l4.tcp.ssl_seen_server_cert == 1) + && (flow->protos.stun_ssl.ssl.server_certificate[0] != '\0')) /* || ((flow->l4.tcp.ssl_seen_client_cert == 1) && (flow->protos.stun_ssl.ssl.client_certificate[0] != '\0')) */ ) { ndpi_int_ssl_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_SSL); @@ -501,8 +513,7 @@ int sslDetectProtocolFromCertificate(struct ndpi_detection_module_struct *ndpi_s return(0); } -static void ssl_mark_and_payload_search_for_other_protocols(struct - ndpi_detection_module_struct +static void ssl_mark_and_payload_search_for_other_protocols(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) { struct ndpi_packet_struct *packet = &flow->packet; @@ -579,16 +590,16 @@ static void ssl_mark_and_payload_search_for_other_protocols(struct /* SSL without certificate (Skype, Ultrasurf?) */ NDPI_LOG_INFO(ndpi_struct, "found ssl NO_CERT\n"); ndpi_int_ssl_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_SSL_NO_CERT); - } else + } else if(packet->ssl_certificate_num_checks >= 3) { NDPI_LOG_INFO(ndpi_struct, "found ssl\n"); - ndpi_int_ssl_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_SSL); + ndpi_int_ssl_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_SSL); + } } } static u_int8_t ndpi_search_sslv3_direction1(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) { - struct ndpi_packet_struct *packet = &flow->packet; if((packet->payload_packet_len >= 5) @@ -716,27 +727,25 @@ void ndpi_search_ssl_tcp(struct ndpi_detection_module_struct *ndpi_struct, struc NDPI_LOG_DBG(ndpi_struct, "search ssl\n"); - { - /* Check if this is whatsapp first (this proto runs over port 443) */ - if((packet->payload_packet_len > 5) - && ((packet->payload[0] == 'W') - && (packet->payload[1] == 'A') - && (packet->payload[4] == 0) - && (packet->payload[2] <= 9) - && (packet->payload[3] <= 9))) { - ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_WHATSAPP, NDPI_PROTOCOL_UNKNOWN); - return; - } else if((packet->payload_packet_len == 4) - && (packet->payload[0] == 'W') - && (packet->payload[1] == 'A')) { - ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_WHATSAPP, NDPI_PROTOCOL_UNKNOWN); + /* Check if this is whatsapp first (this proto runs over port 443) */ + if((packet->payload_packet_len > 5) + && ((packet->payload[0] == 'W') + && (packet->payload[1] == 'A') + && (packet->payload[4] == 0) + && (packet->payload[2] <= 9) + && (packet->payload[3] <= 9))) { + ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_WHATSAPP, NDPI_PROTOCOL_UNKNOWN); + return; + } else if((packet->payload_packet_len == 4) + && (packet->payload[0] == 'W') + && (packet->payload[1] == 'A')) { + ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_WHATSAPP, NDPI_PROTOCOL_UNKNOWN); + return; + } else { + /* No whatsapp, let's try SSL */ + if(sslDetectProtocolFromCertificate(ndpi_struct, flow) > 0) return; - } else { - /* No whatsapp, let's try SSL */ - if(sslDetectProtocolFromCertificate(ndpi_struct, flow) > 0) - return; - } - } + } if(packet->payload_packet_len > 40 && flow->l4.tcp.ssl_stage == 0) { NDPI_LOG_DBG2(ndpi_struct, "first ssl packet\n"); -- cgit v1.2.3 From c496c7975f5d95fb4e1b9774c3592713cda78206 Mon Sep 17 00:00:00 2001 From: emanuele-f Date: Fri, 29 Mar 2019 19:58:10 +0100 Subject: Add DNS response address --- src/include/ndpi_typedefs.h | 1 + src/lib/protocols/dns.c | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) (limited to 'src/lib/protocols') diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h index 92c011b3d..1bd8fd2db 100644 --- a/src/include/ndpi_typedefs.h +++ b/src/include/ndpi_typedefs.h @@ -1079,6 +1079,7 @@ struct ndpi_flow_struct { struct { u_int8_t num_queries, num_answers, reply_code; u_int16_t query_type, query_class, rsp_type; + ndpi_ip_addr_t rsp_addr; /* The first address in a DNS response packet */ } dns; struct { diff --git a/src/lib/protocols/dns.c b/src/lib/protocols/dns.c index d17acd2bd..b99e5a5da 100644 --- a/src/lib/protocols/dns.c +++ b/src/lib/protocols/dns.c @@ -186,6 +186,22 @@ void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, struct nd rsp_type = get16(&x, flow->packet.payload); flow->protos.dns.rsp_type = rsp_type; + + /* here x points to the response "class" field */ + if((x+12) < flow->packet.payload_packet_len) { + x += 6; + data_len = get16(&x, flow->packet.payload); + + if(((x + data_len) < flow->packet.payload_packet_len) + && (((rsp_type == 0x1) && (data_len == 4)) /* A */ +#ifdef NDPI_DETECTION_SUPPORT_IPV6 + || ((rsp_type == 0x1c) && (data_len == 16)) /* AAAA */ +#endif + )) { + memcpy(&flow->protos.dns.rsp_addr, flow->packet.payload + x, data_len); + } + } + break; } } -- cgit v1.2.3 From 1915a63cf29fbe3d2b0a983b2875929518d242ad Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Mon, 1 Apr 2019 12:30:51 +0200 Subject: Implemented ndpi_process_partial_detection() API call to handle partial matches due to the nDPI specified configuration --- example/ndpiReader.c | 31 ++++----- src/include/ndpi_api.h | 15 +++- src/lib/ndpi_main.c | 176 ++++++++++++++++++++++++++++++----------------- src/lib/protocols/dns.c | 6 +- src/lib/protocols/http.c | 4 +- 5 files changed, 149 insertions(+), 83 deletions(-) (limited to 'src/lib/protocols') diff --git a/example/ndpiReader.c b/example/ndpiReader.c index 4244e3745..72eaecca4 100644 --- a/example/ndpiReader.c +++ b/example/ndpiReader.c @@ -914,11 +914,10 @@ static void node_proto_guess_walker(const void *node, ndpi_VISIT which, int dept struct ndpi_flow_info *flow = *(struct ndpi_flow_info **) node; u_int16_t thread_id = *((u_int16_t *) user_data); - if((which == ndpi_preorder) || (which == ndpi_leaf)) { /* Avoid walking the same node multiple times */ - if((!flow->detection_completed) && flow->ndpi_flow) { + if((which == ndpi_preorder) || (which == ndpi_leaf)) { /* Avoid walking the same node multiple times */ + if((!flow->detection_completed) && flow->ndpi_flow) flow->detected_protocol = ndpi_detection_giveup(ndpi_thread_info[0].workflow->ndpi_struct, flow->ndpi_flow, enable_protocol_guess); - } - + process_ndpi_collected_info(ndpi_thread_info[thread_id].workflow, flow); ndpi_thread_info[thread_id].workflow->stats.protocol_counter[flow->detected_protocol.app_protocol] += flow->src2dst_packets + flow->dst2src_packets; @@ -1956,7 +1955,7 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us printf("\tPeak Memory: %-13s\n", formatBytes(max_ndpi_memory, buf, sizeof(buf))); printf("\tSetup Time: %lu msec\n", (unsigned long)(setup_time_usec/1000)); printf("\tPacket Processing Time: %lu msec\n", (unsigned long)(processing_time_usec/1000)); - + if(!json_flag) { printf("\nTraffic statistics:\n"); printf("\tEthernet bytes: %-13llu (includes ethernet CRC/IFC/trailer)\n", @@ -1992,10 +1991,10 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us float t = (float)(cumulative_stats.ip_packet_count*1000000)/(float)processing_time_usec; float b = (float)(cumulative_stats.total_wire_bytes * 8 *1000000)/(float)processing_time_usec; float traffic_duration; - + if(live_capture) traffic_duration = processing_time_usec; else traffic_duration = (pcap_end.tv_sec*1000000 + pcap_end.tv_usec) - (pcap_start.tv_sec*1000000 + pcap_start.tv_usec); - + printf("\tnDPI throughput: %s pps / %s/sec\n", formatPackets(t, buf), formatTraffic(b, 1, buf1)); t = (float)(cumulative_stats.ip_packet_count*1000000)/(float)traffic_duration; b = (float)(cumulative_stats.total_wire_bytes * 8 *1000000)/(float)traffic_duration; @@ -2005,7 +2004,7 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us strftime(when, sizeof(when), "%d/%b/%Y %H:%M:%S", localtime(&pcap_end.tv_sec)); printf("\tAnalysis end: %s\n", when); printf("\tTraffic throughput: %s pps / %s/sec\n", formatPackets(t, buf), formatTraffic(b, 1, buf1)); - printf("\tTraffic duration: %.3f sec\n", traffic_duration/1000000); + printf("\tTraffic duration: %.3f sec\n", traffic_duration/1000000); } if(enable_protocol_guess) @@ -2490,7 +2489,7 @@ static void ndpi_process_packet(u_char *args, if(memcmp(packet, packet_checked, header->caplen) != 0) printf("INTERNAL ERROR: ingress packet was modified by nDPI: this should not happen [thread_id=%u, packetId=%lu, caplen=%u]\n", thread_id, (unsigned long)ndpi_thread_info[thread_id].workflow->stats.raw_packet_count, header->caplen); - + if((pcap_end.tv_sec-pcap_start.tv_sec) > pcap_analysis_duration) { int i; u_int64_t processing_time_usec, setup_time_usec; @@ -2498,7 +2497,7 @@ static void ndpi_process_packet(u_char *args, gettimeofday(&end, NULL); processing_time_usec = end.tv_sec*1000000 + end.tv_usec - (begin.tv_sec*1000000 + begin.tv_usec); setup_time_usec = begin.tv_sec*1000000 + begin.tv_usec - (startup_time.tv_sec*1000000 + startup_time.tv_usec); - + printResults(processing_time_usec, setup_time_usec); for(i=0; iprefs.num_roots; i++) { @@ -2515,7 +2514,7 @@ static void ndpi_process_packet(u_char *args, memcpy(&pcap_start, &pcap_end, sizeof(pcap_start)); } - /* + /* Leave the free as last statement to avoid crashes when ndpi_detection_giveup() is called above by printResults() */ @@ -2541,7 +2540,7 @@ void * processing_thread(void *_thread_id) { #if defined(linux) && defined(HAVE_PTHREAD_SETAFFINITY_NP) if(core_affinity[thread_id] >= 0) { cpu_set_t cpuset; - + CPU_ZERO(&cpuset); CPU_SET(core_affinity[thread_id], &cpuset); @@ -2559,7 +2558,7 @@ void * processing_thread(void *_thread_id) { struct rte_mbuf *bufs[BURST_SIZE]; u_int16_t num = rte_eth_rx_burst(dpdk_port_id, 0, bufs, BURST_SIZE); u_int i; - + if(num == 0) { usleep(1); continue; @@ -3273,8 +3272,8 @@ int orginal_main(int argc, char **argv) { #else int main(int argc, char **argv) { #endif - int i; - + int i; + if(ndpi_get_api_version() != NDPI_API_VERSION) { printf("nDPI Library version mismatch: please make sure this code and the nDPI library are in sync\n"); return(-1); @@ -3284,7 +3283,7 @@ int orginal_main(int argc, char **argv) { gettimeofday(&startup_time, NULL); ndpi_info_mod = ndpi_init_detection_module(); - + if(ndpi_info_mod == NULL) return -1; memset(ndpi_thread_info, 0, sizeof(ndpi_thread_info)); diff --git a/src/include/ndpi_api.h b/src/include/ndpi_api.h index 386d306fd..84633c3ae 100644 --- a/src/include/ndpi_api.h +++ b/src/include/ndpi_api.h @@ -1,7 +1,7 @@ /* * ndpi_api.h * - * Copyright (C) 2011-17 - ntop.org + * Copyright (C) 2011-19 - ntop.org * * This file is part of nDPI, an open source deep packet inspection * library based on the OpenDPI and PACE technology by ipoque GmbH @@ -206,7 +206,18 @@ extern "C" { */ void ndpi_set_protocol_detection_bitmask2(struct ndpi_detection_module_struct *ndpi_struct, const NDPI_PROTOCOL_BITMASK * detection_bitmask); - + + /** + * Function to be called to see in case of unknown match to see if there is + * a partial match that has been prevented by the current nDPI preferences configuration + * + * @par ndpi_struct = the detection module + * @par flow = the flow given for the detection module + * @return the detected protocol even if the flow is not completed; + * + */ + ndpi_protocol ndpi_get_partial_detection(struct ndpi_detection_module_struct *ndpi_struct, + struct ndpi_flow_struct *flow); /** * Function to be called before we give up with detection for a given flow. * This function reduces the NDPI_UNKNOWN_PROTOCOL detection diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index 59de7a763..acbe5b8ec 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -669,7 +669,7 @@ void ndpi_set_proto_defaults(struct ndpi_detection_module_struct *ndpi_mod, ndpi_mod->proto_defaults[protoId].protoId = protoId, ndpi_mod->proto_defaults[protoId].protoBreed = breed; ndpi_mod->proto_defaults[protoId].can_have_a_subprotocol = can_have_a_subprotocol; - + memcpy(&ndpi_mod->proto_defaults[protoId].master_tcp_protoId, tcp_master_protoId, 2*sizeof(u_int16_t)); memcpy(&ndpi_mod->proto_defaults[protoId].master_udp_protoId, udp_master_protoId, 2*sizeof(u_int16_t)); @@ -677,7 +677,7 @@ void ndpi_set_proto_defaults(struct ndpi_detection_module_struct *ndpi_mod, if(udpDefPorts[j].port_low != 0) addDefaultPort(ndpi_mod, &udpDefPorts[j], &ndpi_mod->proto_defaults[protoId], 0, &ndpi_mod->udpRoot, __FUNCTION__,__LINE__); - + if(tcpDefPorts[j].port_low != 0) addDefaultPort(ndpi_mod, &tcpDefPorts[j], &ndpi_mod->proto_defaults[protoId], 0, &ndpi_mod->tcpRoot, __FUNCTION__,__LINE__); @@ -790,15 +790,15 @@ static int ndpi_string_to_automa(struct ndpi_detection_module_struct *ndpi_struc if(automa->ac_automa == NULL) return(-2); ac_pattern.astring = value, - ac_pattern.rep.number = protocol_id, + ac_pattern.rep.number = protocol_id, ac_pattern.rep.category = (u_int16_t)category, ac_pattern.rep.breed = (u_int16_t)breed; - + #ifdef MATCH_DEBUG printf("Adding to automa [%s][protocol_id: %u][category: %u][breed: %u]\n", value, protocol_id, category, breed); #endif - + if(value == NULL) ac_pattern.length = 0; else @@ -856,14 +856,14 @@ void ndpi_init_protocol_match(struct ndpi_detection_module_struct *ndpi_mod, u_int16_t no_master[2] = { NDPI_PROTOCOL_NO_MASTER_PROTO, NDPI_PROTOCOL_NO_MASTER_PROTO }; ndpi_port_range ports_a[MAX_DEFAULT_PORTS], ports_b[MAX_DEFAULT_PORTS]; static u_int16_t generic_id = NDPI_LAST_IMPLEMENTED_PROTOCOL; - + if(ndpi_mod->proto_defaults[match->protocol_id].protoName == NULL) { if(match->protocol_id == NDPI_PROTOCOL_GENERIC) ndpi_mod->proto_defaults[match->protocol_id].protoName = ndpi_strdup(NDPI_CONST_GENERIC_PROTOCOL_NAME); else ndpi_mod->proto_defaults[match->protocol_id].protoName = ndpi_strdup(match->proto_name); - - ndpi_mod->proto_defaults[match->protocol_id].protoId = match->protocol_id; + + ndpi_mod->proto_defaults[match->protocol_id].protoId = match->protocol_id; ndpi_mod->proto_defaults[match->protocol_id].protoCategory = match->protocol_category; ndpi_mod->proto_defaults[match->protocol_id].protoBreed = match->protocol_breed; @@ -875,7 +875,7 @@ void ndpi_init_protocol_match(struct ndpi_detection_module_struct *ndpi_mod, ndpi_mod->proto_defaults[match->protocol_id].protoName, ndpi_mod->proto_defaults[match->protocol_id].protoCategory, ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */, - ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); + ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); } ndpi_add_host_url_subprotocol(ndpi_mod, @@ -1025,7 +1025,7 @@ static void init_string_based_protocols(struct ndpi_detection_module_struct *ndp for(i=0; host_match[i].string_to_match != NULL; i++) ndpi_init_protocol_match(ndpi_mod, &host_match[i]); -#ifdef MATCH_DEBUG +#ifdef MATCH_DEBUG // ac_automata_display(ndpi_mod->host_automa.ac_automa, 'n'); #endif @@ -1065,7 +1065,7 @@ int ndpi_set_detection_preferences(struct ndpi_detection_module_struct *ndpi_mod case ndpi_pref_enable_category_substring_match: ndpi_mod->enable_category_substring_match = (u_int8_t)value; break; - + default: return(-1); } @@ -2089,7 +2089,7 @@ static int ac_match_handler(AC_MATCH_t *m, AC_TEXT_t *txt, AC_REP_t *match) { char buf[64] = { '\0' }; int min_buf_len = (txt->length > 63 /* sizeof(buf)-1 */) ? 63 : txt->length; u_int buf_len = strlen(buf); - + strncpy(buf, txt->astring, min_buf_len); buf[min_buf_len] = '\0'; @@ -2103,8 +2103,8 @@ static int ac_match_handler(AC_MATCH_t *m, AC_TEXT_t *txt, AC_REP_t *match) { char *whatfound = strstr(buf, m->patterns->astring); #ifdef MATCH_DEBUG - printf("[NDPI] %s() [searching=%s][pattern=%s][%s][%c]\n", - __FUNCTION__, buf, m->patterns->astring, + printf("[NDPI] %s() [searching=%s][pattern=%s][%s][%c]\n", + __FUNCTION__, buf, m->patterns->astring, whatfound ? whatfound : "", whatfound[-1]); #endif @@ -2117,7 +2117,7 @@ static int ac_match_handler(AC_MATCH_t *m, AC_TEXT_t *txt, AC_REP_t *match) { if(whatfound && (whatfound != buf) && (m->patterns->astring[0] != '.') /* The searched patter does not start with . */ && strchr(m->patterns->astring, '.') /* The matched pattern has a . (e.g. numeric or sym IPs) */ - && (whatfound[-1] != '.') + && (whatfound[-1] != '.') ) return(0); } @@ -2368,7 +2368,7 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(void) { #endif ndpi_str->custom_categories.hostnames_hash = NULL; - + ndpi_str->custom_categories.ipAddresses = ndpi_New_Patricia(32 /* IPv4 */); ndpi_str->custom_categories.ipAddresses_shadow = ndpi_New_Patricia(32 /* IPv4 */); @@ -2436,7 +2436,7 @@ int ndpi_match_string(void *_automa, char *string_to_match) { int ndpi_match_string_id(void *_automa, char *string_to_match, unsigned long *id) { AC_TEXT_t ac_input_text; AC_AUTOMATA_t *automa = (AC_AUTOMATA_t*)_automa; - AC_REP_t match = { NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, NDPI_PROTOCOL_UNRATED }; + AC_REP_t match = { NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, NDPI_PROTOCOL_UNRATED }; *id = -1; if((automa == NULL) @@ -2449,7 +2449,7 @@ int ndpi_match_string_id(void *_automa, char *string_to_match, unsigned long *id ac_automata_reset(automa); *id = match.number; - + return(*id != NDPI_PROTOCOL_UNKNOWN ? 0 : -1); } @@ -2476,7 +2476,7 @@ static int hyperscanCustomEventHandler(unsigned int id, int ndpi_match_custom_category(struct ndpi_detection_module_struct *ndpi_struct, char *name, unsigned long *id) { #ifdef DEBUG - printf("[NDPI] %s(%s) [enable_category_substring_match: %u]\n", + printf("[NDPI] %s(%s) [enable_category_substring_match: %u]\n", __FUNCTION__, name, ndpi_struct->enable_category_substring_match); #endif @@ -2575,7 +2575,7 @@ void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_struct if(ndpi_struct->protocols_ptree) ndpi_Destroy_Patricia((patricia_tree_t*)ndpi_struct->protocols_ptree, free_ptree_data); - + if(ndpi_struct->udpRoot != NULL) ndpi_tdestroy(ndpi_struct->udpRoot, ndpi_free); if(ndpi_struct->tcpRoot != NULL) @@ -2621,7 +2621,7 @@ void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_struct if(ndpi_struct->custom_categories.hostnames_hash) ht_free((hashtable_t*)ndpi_struct->custom_categories.hostnames_hash); - + ndpi_free(ndpi_struct); } } @@ -2634,7 +2634,7 @@ int ndpi_get_protocol_id_master_proto(struct ndpi_detection_module_struct *ndpi_ u_int16_t** udp_master_proto) { if(protocol_id >= (NDPI_MAX_SUPPORTED_PROTOCOLS+NDPI_MAX_NUM_CUSTOM_PROTOCOLS)) { *tcp_master_proto = ndpi_struct->proto_defaults[NDPI_PROTOCOL_UNKNOWN].master_tcp_protoId, - *udp_master_proto = ndpi_struct->proto_defaults[NDPI_PROTOCOL_UNKNOWN].master_udp_protoId; + *udp_master_proto = ndpi_struct->proto_defaults[NDPI_PROTOCOL_UNKNOWN].master_udp_protoId; return(-1); } @@ -2782,7 +2782,7 @@ char * strsep(char **sp, char *sep) /* ******************************************************************** */ -int ndpi_handle_rule(struct ndpi_detection_module_struct *ndpi_mod, +int ndpi_handle_rule(struct ndpi_detection_module_struct *ndpi_mod, char* rule, u_int8_t do_add) { char *at, *proto, *elem; ndpi_proto_defaults_t *def; @@ -2917,7 +2917,7 @@ int ndpi_load_protocols_file(struct ndpi_detection_module_struct *ndpi_mod, char if(buffer == NULL) { NDPI_LOG_ERR(ndpi_mod, "Memory allocation failure"); - goto close_fd; + goto close_fd; } while(fd) { @@ -3820,18 +3820,18 @@ static int ndpi_init_packet_header(struct ndpi_detection_module_struct *ndpi_str && flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN) { u_int8_t backup; u_int16_t backup1, backup2; - + if(flow->http.url) ndpi_free(flow->http.url); if(flow->http.content_type) ndpi_free(flow->http.content_type); backup = flow->num_processed_pkts; backup1 = flow->guessed_protocol_id; - backup2 = flow->guessed_host_protocol_id; + backup2 = flow->guessed_host_protocol_id; memset(flow, 0, sizeof(*(flow))); flow->num_processed_pkts = backup; flow->guessed_protocol_id = backup1; flow->guessed_host_protocol_id = backup2; - + NDPI_LOG_DBG(ndpi_struct, "tcp syn packet for unknown protocol, reset detection state\n"); } @@ -4165,6 +4165,52 @@ static u_int16_t ndpi_guess_host_protocol_id(struct ndpi_detection_module_struct /* ********************************************************************************* */ +static ndpi_protocol ndpi_process_partial_detection(struct ndpi_detection_module_struct *ndpi_struct, + struct ndpi_flow_struct *flow) { + ndpi_protocol ret; + ndpi_protocol_match_result ret_match; + + ret.master_protocol = flow->guessed_protocol_id; + ret.app_protocol = ndpi_match_host_subprotocol(ndpi_struct, flow, + (char *)flow->host_server_name, + strlen((const char*)flow->host_server_name), + &ret_match, + flow->guessed_protocol_id); + ret.category = ret_match.protocol_category; + + if(ret.app_protocol == NDPI_PROTOCOL_UNKNOWN) + ret.app_protocol = ret.master_protocol; + + ndpi_int_change_protocol(ndpi_struct, flow, ret.app_protocol, ret.master_protocol); + return(ret); +} + +/* ********************************************************************************* */ + +/* + You can call this function at any time in case of unknown match to see if there is + a partial match that has been prevented by the current nDPI preferences configuration + */ +ndpi_protocol ndpi_get_partial_detection(struct ndpi_detection_module_struct *ndpi_struct, + struct ndpi_flow_struct *flow) { + if((flow->guessed_protocol_id == NDPI_PROTOCOL_HTTP) + && (ndpi_struct->http_dont_dissect_response == 0) + && (flow->host_server_name[0] != '\0') + && (!NDPI_ISSET(&flow->excluded_protocol_bitmask, flow->guessed_host_protocol_id))) + return(ndpi_process_partial_detection(ndpi_struct, flow)); + else if((flow->guessed_protocol_id == NDPI_PROTOCOL_DNS) + && (ndpi_struct->dns_dont_dissect_response == 0) + && (flow->host_server_name[0] != '\0') + && (!NDPI_ISSET(&flow->excluded_protocol_bitmask, flow->guessed_host_protocol_id))) + return(ndpi_process_partial_detection(ndpi_struct, flow)); + else { + ndpi_protocol ret = { NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED }; + return(ret); + } +} + +/* ********************************************************************************* */ + ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow, u_int8_t enable_guess) { ndpi_protocol ret = { NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED }; @@ -4177,12 +4223,19 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st if(flow->guessed_protocol_id == NDPI_PROTOCOL_STUN) goto check_stun_export; - else if((flow->l4.tcp.ssl_seen_client_cert == 1) && (flow->protos.stun_ssl.ssl.client_certificate[0] != '\0')) { + else if((flow->l4.tcp.ssl_seen_client_cert == 1) + && (flow->protos.stun_ssl.ssl.client_certificate[0] != '\0')) { ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_SSL, NDPI_PROTOCOL_UNKNOWN); } else { - if(!enable_guess) - return(ret); - + ndpi_protocol ret_g = ndpi_get_partial_detection(ndpi_struct, flow); + + if(ret_g.master_protocol != NDPI_PROTOCOL_UNKNOWN) + return(ret_g); + else { + if(!enable_guess) + return(ret); + } + if((flow->guessed_protocol_id == NDPI_PROTOCOL_UNKNOWN) && (flow->packet.l4_protocol == IPPROTO_TCP) && (flow->l4.tcp.ssl_stage > 1)) @@ -4196,7 +4249,7 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st && is_udp_guessable_protocol(guessed_host_protocol_id) )) flow->guessed_host_protocol_id = guessed_host_protocol_id = NDPI_PROTOCOL_UNKNOWN; - + /* Ignore guessed protocol if they have been discarded */ if((guessed_protocol_id != NDPI_PROTOCOL_UNKNOWN) // && (guessed_host_protocol_id == NDPI_PROTOCOL_UNKNOWN) @@ -4207,7 +4260,6 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st if((guessed_protocol_id != NDPI_PROTOCOL_UNKNOWN) || (guessed_host_protocol_id != NDPI_PROTOCOL_UNKNOWN)) { - if((guessed_protocol_id == 0) && (flow->protos.stun_ssl.stun.num_binding_requests > 0) && (flow->protos.stun_ssl.stun.num_processed_pkts > 0)) @@ -4246,7 +4298,7 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st else if(ret.app_protocol == NDPI_PROTOCOL_GOOGLE) ret.app_protocol = NDPI_PROTOCOL_HANGOUT; } - + if(enable_guess && (ret.app_protocol == NDPI_PROTOCOL_UNKNOWN) && flow->packet.iph /* Guess only IPv4 */ @@ -4260,7 +4312,7 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st ntohl(flow->packet.iph->daddr), ntohs(flow->packet.udp ? flow->packet.udp->dest : flow->packet.tcp->dest) ); - + ndpi_fill_protocol_category(ndpi_struct, flow, &ret); return(ret); @@ -4353,15 +4405,15 @@ int ndpi_load_hostname_category(struct ndpi_detection_module_struct *ndpi_struct if(ndpi_struct->custom_categories.hostnames_hash) ht_set((hashtable_t*)ndpi_struct->custom_categories.hostnames_hash, name, (u_int16_t)category); - - return(0); + + return(0); } else { AC_PATTERN_t ac_pattern; - + /* printf("===> Loading %s as %u\n", name, category); */ memset(&ac_pattern, 0, sizeof(ac_pattern)); - + #ifdef HAVE_HYPERSCAN { struct hs_list *h = (struct hs_list*)malloc(sizeof(struct hs_list)); @@ -4401,7 +4453,7 @@ int ndpi_load_hostname_category(struct ndpi_detection_module_struct *ndpi_struct ac_automata_add(ndpi_struct->custom_categories.hostnames_shadow.ac_automa, &ac_pattern); #endif } - + return(0); } @@ -4481,16 +4533,16 @@ int ndpi_enable_loaded_categories(struct ndpi_detection_module_struct *ndpi_str) ndpi_str->custom_categories.hostnames_shadow.ac_automa = ac_automata_init(ac_match_handler); #endif } - + if(ndpi_str->custom_categories.ipAddresses != NULL) - ndpi_Destroy_Patricia((patricia_tree_t*)ndpi_str->custom_categories.ipAddresses, + ndpi_Destroy_Patricia((patricia_tree_t*)ndpi_str->custom_categories.ipAddresses, free_ptree_data); ndpi_str->custom_categories.ipAddresses = ndpi_str->custom_categories.ipAddresses_shadow; ndpi_str->custom_categories.ipAddresses_shadow = ndpi_New_Patricia(32 /* IPv4 */); ndpi_str->custom_categories.categories_loaded = 1; - + return(0); } @@ -4582,7 +4634,7 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct return(ret); flow->num_processed_pkts++; - + if(flow->server_id == NULL) flow->server_id = dst; /* Default */ if(flow->detected_protocol_stack[0] != NDPI_PROTOCOL_UNKNOWN) goto ret_protocols; @@ -4705,7 +4757,7 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct break; } } - + ndpi_check_flow_func(ndpi_struct, flow, &ndpi_selection_packet); ndpi_fill_protocol_category(ndpi_struct, flow, &ret); @@ -4969,22 +5021,22 @@ void ndpi_parse_packet_line_info(struct ndpi_detection_module_struct *ndpi_struc packet->http_response.ptr = &packet->line[0].ptr[NDPI_STATICSTRING_LEN("HTTP/1.1 ")]; packet->http_response.len = packet->line[0].len - NDPI_STATICSTRING_LEN("HTTP/1.1 "); packet->http_num_headers++; - + /* Set server HTTP response code */ if(packet->payload_packet_len >= 12) { char buf[4]; - + /* Set server HTTP response code */ strncpy(buf, (char*)&packet->payload[9], 3); buf[3] = '\0'; - + flow->http.response_status_code = atoi(buf); /* https://en.wikipedia.org/wiki/List_of_HTTP_status_codes */ if((flow->http.response_status_code < 100) || (flow->http.response_status_code > 509)) flow->http.response_status_code = 0; /* Out of range */ } } - + /* "Server:" header line in HTTP response */ if(packet->line[packet->parsed_lines].len > NDPI_STATICSTRING_LEN("Server:") + 1 && strncasecmp((const char *)packet->line[packet->parsed_lines].ptr, "Server:", NDPI_STATICSTRING_LEN("Server:")) == 0) { @@ -5403,7 +5455,7 @@ void ndpi_int_change_protocol(struct ndpi_detection_module_struct *ndpi_struct, } } } - + ndpi_int_change_flow_protocol(ndpi_struct, flow, upper_detected_protocol, lower_detected_protocol); ndpi_int_change_packet_protocol(ndpi_struct, flow, @@ -5435,7 +5487,7 @@ void ndpi_int_reset_protocol(struct ndpi_flow_struct *flow) { int a; for(a = 0; a < NDPI_PROTOCOL_SIZE; a++) - flow->detected_protocol_stack[a] = NDPI_PROTOCOL_UNKNOWN; + flow->detected_protocol_stack[a] = NDPI_PROTOCOL_UNKNOWN; } } @@ -5657,10 +5709,10 @@ ndpi_protocol ndpi_guess_undetected_protocol(struct ndpi_detection_module_struct ret.app_protocol = rc, ret.master_protocol = ndpi_guess_protocol_id(ndpi_struct, NULL, proto, sport, dport, &user_defined_proto); - + if(ret.app_protocol == ret.master_protocol) ret.master_protocol = NDPI_PROTOCOL_UNKNOWN; - + ret.category = ndpi_get_proto_category(ndpi_struct, ret); return(ret); } @@ -5674,7 +5726,7 @@ ndpi_protocol ndpi_guess_undetected_protocol(struct ndpi_detection_module_struct ; else { ret.app_protocol = rc; - + if(rc == NDPI_PROTOCOL_SSL) goto check_guessed_skype; else { @@ -5697,7 +5749,7 @@ ndpi_protocol ndpi_guess_undetected_protocol(struct ndpi_detection_module_struct ret.app_protocol = ndpi_guess_protocol_id(ndpi_struct, NULL, proto, sport, dport, &user_defined_proto); - ret.category = ndpi_get_proto_category(ndpi_struct, ret); + ret.category = ndpi_get_proto_category(ndpi_struct, ret); return(ret); } @@ -5939,7 +5991,7 @@ char* ndpi_strnstr(const char *s, const char *find, size_t slen) { } while(strncmp(s, find, len) != 0); s--; } - + return((char *)s); } @@ -5987,7 +6039,7 @@ int ndpi_match_string_subprotocol(struct ndpi_detection_module_struct *ndpi_stru ndpi_automa *automa = is_host_match ? &ndpi_struct->host_automa : &ndpi_struct->content_automa; AC_REP_t match = { NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, NDPI_PROTOCOL_UNRATED }; - + if((automa->ac_automa == NULL) || (string_to_match_len == 0)) return(NDPI_PROTOCOL_UNKNOWN); @@ -5995,15 +6047,15 @@ int ndpi_match_string_subprotocol(struct ndpi_detection_module_struct *ndpi_stru ac_automata_finalize((AC_AUTOMATA_t*)automa->ac_automa); automa->ac_automa_finalized = 1; } - + ac_input_text.astring = string_to_match, ac_input_text.length = string_to_match_len; ac_automata_search(((AC_AUTOMATA_t*)automa->ac_automa), &ac_input_text, &match); ac_automata_reset(((AC_AUTOMATA_t*)automa->ac_automa)); - + ret_match->protocol_id = match.number, ret_match->protocol_category = match.category, ret_match->protocol_breed = match.breed; - + return(match.number); } @@ -6252,9 +6304,9 @@ struct ndpi_lru_cache* ndpi_lru_cache_init(u_int32_t num_entries) { if(!c->entries) { free(c); return(NULL); - } else + } else c->num_entries = num_entries; - + return(c); } diff --git a/src/lib/protocols/dns.c b/src/lib/protocols/dns.c index b99e5a5da..1c2593feb 100644 --- a/src/lib/protocols/dns.c +++ b/src/lib/protocols/dns.c @@ -153,8 +153,10 @@ void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, struct nd || ((dns_header.authority_rrs > 0) && (dns_header.authority_rrs <= NDPI_MAX_DNS_REQUESTS)) || ((dns_header.additional_rrs > 0) && (dns_header.additional_rrs <= NDPI_MAX_DNS_REQUESTS))) ) { - /* This is a good reply */ - if(ndpi_struct->dns_dont_dissect_response == 0) { + /* This is a good reply: we dissect it both for request and response */ + + /* Leave the statement below commented necessary in case of call to ndpi_get_partial_detection() */ + /* if(ndpi_struct->dns_dont_dissect_response == 0) */ { x++; if(flow->packet.payload[x] != '\0') { diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c index fc392c2b7..33ef9e2ed 100644 --- a/src/lib/protocols/http.c +++ b/src/lib/protocols/http.c @@ -157,7 +157,9 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_ } #endif - if(!ndpi_struct->http_dont_dissect_response) { + /* Leave the statement below commented necessary in case of call to ndpi_get_partial_detection() */ + + /* if(!ndpi_struct->http_dont_dissect_response) */ { if((flow->http.url == NULL) && (packet->http_url_name.len > 0) && (packet->host_line.len > 0)) { -- cgit v1.2.3