diff options
author | Luca Deri <deri@ntop.org> | 2019-09-14 12:38:58 +0200 |
---|---|---|
committer | Luca Deri <deri@ntop.org> | 2019-09-14 12:38:58 +0200 |
commit | 659f75138c2a95e5823608a545b9a3d3ced223bc (patch) | |
tree | a2677cb7b266db448beaed31c427b75d7269a4ad /src | |
parent | d7c2ced475603228c8999b416719f0cd84b971c7 (diff) |
TLS cerficate hash calculation
Diffstat (limited to 'src')
-rw-r--r-- | src/include/ndpi_typedefs.h | 17 | ||||
-rw-r--r-- | src/lib/ndpi_main.c | 10 | ||||
-rw-r--r-- | src/lib/protocols/skype.c | 4 | ||||
-rw-r--r-- | src/lib/protocols/tls.c | 305 | ||||
-rw-r--r-- | src/lib/protocols/tor.c | 2 | ||||
-rw-r--r-- | src/lib/third_party/include/ndpi_sha1.h | 17 | ||||
-rw-r--r-- | src/lib/third_party/src/ndpi_sha1.c | 276 |
7 files changed, 552 insertions, 79 deletions
diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h index b67a22a14..0f3aee9f2 100644 --- a/src/include/ndpi_typedefs.h +++ b/src/include/ndpi_typedefs.h @@ -602,12 +602,17 @@ struct ndpi_flow_tcp_struct { /* NDPI_PROTOCOL_TELNET */ u_int32_t telnet_stage:2; // 0 - 2 + void* tls_srv_cert_fingerprint_ctx; + /* NDPI_PROTOCOL_TLS */ - u_int8_t ssl_seen_client_cert:1, - ssl_seen_server_cert:1, - ssl_seen_certificate:1, - ssl_stage:2; // 0 - 5 - + u_int8_t tls_seen_client_cert:1, + tls_seen_server_cert:1, + tls_seen_certificate:1, + tls_srv_cert_fingerprint_found:1, + tls_srv_cert_fingerprint_processed:1, + tls_stage:2, _pad:1; // 0 - 5 + int16_t tls_record_offset, tls_fingerprint_len; /* Need to be signed */ + /* NDPI_PROTOCOL_POSTGRES */ u_int32_t postgres_stage:3; @@ -793,7 +798,7 @@ struct ndpi_packet_struct { u_int8_t tcp_retransmission; u_int8_t l4_protocol; - u_int8_t ssl_certificate_detected:4, ssl_certificate_num_checks:4; + u_int8_t tls_certificate_detected:4, tls_certificate_num_checks:4; u_int8_t packet_lines_parsed_complete:1, packet_direction:1, empty_line_position_set:1, pad:5; }; diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index 4c286a754..b485fe631 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -4059,7 +4059,7 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st || (flow->guessed_protocol_id == NDPI_PROTOCOL_MESSENGER) || (flow->guessed_protocol_id == NDPI_PROTOCOL_WHATSAPP_VOICE)) ndpi_set_detected_protocol(ndpi_struct, flow, flow->guessed_protocol_id, NDPI_PROTOCOL_UNKNOWN); - else if((flow->l4.tcp.ssl_seen_client_cert == 1) + else if((flow->l4.tcp.tls_seen_client_cert == 1) && (flow->protos.stun_ssl.ssl.client_certificate[0] != '\0')) { ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_TLS, NDPI_PROTOCOL_UNKNOWN); } else { @@ -4074,7 +4074,7 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st if((flow->guessed_protocol_id == NDPI_PROTOCOL_UNKNOWN) && (flow->packet.l4_protocol == IPPROTO_TCP) - && (flow->l4.tcp.ssl_stage > 1)) + && (flow->l4.tcp.tls_stage > 1)) flow->guessed_protocol_id = NDPI_PROTOCOL_TLS_NO_CERT; guessed_protocol_id = flow->guessed_protocol_id, guessed_host_protocol_id = flow->guessed_host_protocol_id; @@ -4448,7 +4448,7 @@ void ndpi_fill_protocol_category(struct ndpi_detection_module_struct *ndpi_struc } } - if((flow->l4.tcp.ssl_seen_client_cert == 1) && (flow->protos.stun_ssl.ssl.client_certificate[0] != '\0')) { + if((flow->l4.tcp.tls_seen_client_cert == 1) && (flow->protos.stun_ssl.ssl.client_certificate[0] != '\0')) { unsigned long id; int rc = ndpi_match_custom_category(ndpi_struct, (char *)flow->protos.stun_ssl.ssl.client_certificate, @@ -4501,7 +4501,9 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct */ ) { ndpi_process_extra_packet(ndpi_struct, flow, packet, packetlen, current_tick_l, src, dst); - ret.master_protocol = flow->detected_protocol_stack[1], ret.app_protocol = flow->detected_protocol_stack[0]; + if(flow->check_extra_packets == 0) + ret.master_protocol = flow->detected_protocol_stack[1], ret.app_protocol = flow->detected_protocol_stack[0]; + return(ret); } else goto ret_protocols; diff --git a/src/lib/protocols/skype.c b/src/lib/protocols/skype.c index b36e03277..45a86f05f 100644 --- a/src/lib/protocols/skype.c +++ b/src/lib/protocols/skype.c @@ -83,7 +83,8 @@ static void ndpi_check_skype(struct ndpi_detection_module_struct *ndpi_struct, s && flow->l4.tcp.seen_syn && flow->l4.tcp.seen_syn_ack && flow->l4.tcp.seen_ack) { - + /* Disabled this logic as it's too weak and leads to false positives */ +#if 0 if((payload_len == 8) || (payload_len == 3) || (payload_len == 17)) { // printf("[SKYPE] payload_len=%u\n", payload_len); /* printf("[SKYPE] %u/%u\n", ntohs(packet->tcp->source), ntohs(packet->tcp->dest)); */ @@ -95,6 +96,7 @@ static void ndpi_check_skype(struct ndpi_detection_module_struct *ndpi_struct, s } /* printf("[SKYPE] [id: %u][len: %d]\n", flow->l4.tcp.skype_packet_id, payload_len); */ +#endif } else { NDPI_EXCLUDE_PROTO(ndpi_struct, flow); } diff --git a/src/lib/protocols/tls.c b/src/lib/protocols/tls.c index 5c8e2b18b..a6d510160 100644 --- a/src/lib/protocols/tls.c +++ b/src/lib/protocols/tls.c @@ -27,8 +27,13 @@ #include "ndpi_api.h" #include "ndpi_md5.h" +#include "ndpi_sha1.h" -// #define DEBUG_TLS 1 +extern char *strptime(const char *s, const char *format, struct tm *tm); + +/* #define DEBUG_TLS 1 */ + +#define DEBUG_FINGERPRINT 1 /* NOTE @@ -41,6 +46,8 @@ 3. openssl x509 -noout -fingerprint -sha1 -inform pem -in certificate.der SHA1 Fingerprint=15:9A:76.... + $ shasum -a 1 www.grc.com.bin + 159a76..... */ #define NDPI_MAX_TLS_REQUEST_SIZE 10000 @@ -58,8 +65,8 @@ static u_int32_t ndpi_tls_refine_master_protocol(struct ndpi_detection_module_st struct ndpi_flow_struct *flow, u_int32_t protocol) { struct ndpi_packet_struct *packet = &flow->packet; - if(((flow->l4.tcp.ssl_seen_client_cert == 1) && (flow->protos.stun_ssl.ssl.ja3_client[0] != '\0')) - || ((flow->l4.tcp.ssl_seen_server_cert == 1) && (flow->protos.stun_ssl.ssl.ja3_server[0] != '\0')) + if(((flow->l4.tcp.tls_seen_client_cert == 1) && (flow->protos.stun_ssl.ssl.ja3_client[0] != '\0')) + || ((flow->l4.tcp.tls_seen_server_cert == 1) && (flow->protos.stun_ssl.ssl.ja3_server[0] != '\0')) // || (flow->host_server_name[0] != '\0') ) protocol = NDPI_PROTOCOL_TLS; @@ -352,9 +359,9 @@ int getTLScertificate(struct ndpi_detection_module_struct *ndpi_struct, printf("[JA3] Server: %s \n", flow->protos.stun_ssl.ssl.ja3_server); #endif - flow->l4.tcp.ssl_seen_server_cert = 1; + flow->l4.tcp.tls_seen_server_cert = 1; } else - flow->l4.tcp.ssl_seen_certificate = 1; + flow->l4.tcp.tls_seen_certificate = 1; /* Check after handshake protocol header (5 bytes) and message header (4 bytes) */ for(i = 9; i < packet->payload_packet_len-3; i++) { @@ -477,7 +484,7 @@ int getTLScertificate(struct ndpi_detection_module_struct *ndpi_struct, offset = base_offset + session_id_len + cipher_len + 2; - flow->l4.tcp.ssl_seen_client_cert = 1; + flow->l4.tcp.tls_seen_client_cert = 1; if(offset < total_len) { u_int16_t compression_len; @@ -684,21 +691,149 @@ int getTLScertificate(struct ndpi_detection_module_struct *ndpi_struct, /* **************************************** */ /* See https://blog.catchpoint.com/2017/05/12/dissecting-tls-using-wireshark/ */ +int getSSCertificateFingerprint(struct ndpi_detection_module_struct *ndpi_struct, + struct ndpi_flow_struct *flow) { + struct ndpi_packet_struct *packet = &flow->packet; + u_int8_t multiple_messages; + +#ifdef DEBUG_TLS + printf("=>> [TLS] %s() [tls_record_offset=%d][payload_packet_len=%u][direction: %u][%02X %02X %02X...]\n", + __FUNCTION__, flow->l4.tcp.tls_record_offset, packet->payload_packet_len, + packet->packet_direction, + packet->payload[0], packet->payload[1], packet->payload[2]); +#endif + + if((packet->packet_direction == 0) /* Client -> Server */ + || (packet->payload_packet_len == 0)) + return(1); /* More packets please */ + else if(flow->l4.tcp.tls_srv_cert_fingerprint_processed) + return(0); /* We're good */ + + if(flow->l4.tcp.tls_fingerprint_len > 0) { + unsigned char sha1[20]; + unsigned int i, avail = packet->payload_packet_len - flow->l4.tcp.tls_record_offset; + + if(avail > flow->l4.tcp.tls_fingerprint_len) + avail = flow->l4.tcp.tls_fingerprint_len; + +#ifdef DEBUG_TLS + printf("=>> [TLS] Certificate record [%02X %02X %02X...][missing: %u][offset: %u][avail: %u] (B)\n", + packet->payload[flow->l4.tcp.tls_record_offset], + packet->payload[flow->l4.tcp.tls_record_offset+1], + packet->payload[flow->l4.tcp.tls_record_offset+2], + flow->l4.tcp.tls_fingerprint_len, flow->l4.tcp.tls_record_offset, avail + ); +#endif + +#ifdef DEBUG_CERTIFICATE_HASH + for(i=0;i<avail;i++) + printf("%02X ", packet->payload[flow->l4.tcp.tls_record_offset+i]); + printf("\n"); +#endif + + SHA1Update(flow->l4.tcp.tls_srv_cert_fingerprint_ctx, + &packet->payload[flow->l4.tcp.tls_record_offset], + avail); + + flow->l4.tcp.tls_fingerprint_len -= avail; + + if(flow->l4.tcp.tls_fingerprint_len == 0) { + SHA1Final(sha1, flow->l4.tcp.tls_srv_cert_fingerprint_ctx); + +#ifdef DEBUG_TLS + printf("=>> [TLS] SHA-1: "); + for(i=0;i<20;i++) + printf("%s%02X", (i > 0) ? ":" : "", sha1[i]); + printf("\n"); +#endif + + flow->l4.tcp.tls_srv_cert_fingerprint_processed = 1; + return(0); /* We're good */ + } else { + flow->l4.tcp.tls_record_offset = 0; +#ifdef DEBUG_TLS + printf("=>> [TLS] Certificate record: still missing %u bytes\n", flow->l4.tcp.tls_fingerprint_len); +#endif + return(1); /* More packets please */ + } + } + + multiple_messages = (packet->payload[flow->l4.tcp.tls_record_offset] == 0x16 /* Handshake */) ? 0 : 1; + +#ifdef DEBUG_TLS + printf("=>> [TLS] [multiple_messages: %d]\n", multiple_messages); +#endif + + if((!multiple_messages) && (packet->payload[flow->l4.tcp.tls_record_offset] != 0x16 /* Handshake */)) + return(1); + else if(((!multiple_messages) && (packet->payload[flow->l4.tcp.tls_record_offset+5] == 0xb) /* Certificate */) + || (packet->payload[flow->l4.tcp.tls_record_offset] == 0xb) /* Certificate */) { + /* TODO: Do not take into account all certificates but only the first one */ +#ifdef DEBUG_TLS + printf("=>> [TLS] Certificate found\n"); +#endif + flow->l4.tcp.tls_srv_cert_fingerprint_ctx = (void*)ndpi_malloc(sizeof(SHA1_CTX)); + + if(flow->l4.tcp.tls_srv_cert_fingerprint_ctx) { + SHA1Init(flow->l4.tcp.tls_srv_cert_fingerprint_ctx); + flow->l4.tcp.tls_srv_cert_fingerprint_found = 1; + flow->l4.tcp.tls_record_offset += (!multiple_messages) ? 13 : 8; + flow->l4.tcp.tls_fingerprint_len = ntohs(*(u_int16_t*)&packet->payload[flow->l4.tcp.tls_record_offset]); + flow->l4.tcp.tls_record_offset = flow->l4.tcp.tls_record_offset+2; +#ifdef DEBUG_TLS + printf("=>> [TLS] Certificate [total certificate len: %u][certificate initial offset: %u]\n", + flow->l4.tcp.tls_fingerprint_len, flow->l4.tcp.tls_record_offset); +#endif + return(getSSCertificateFingerprint(ndpi_struct, flow)); + } else + return(0); /* That's all */ + } else if(flow->l4.tcp.tls_seen_certificate) + return(0); /* That's all */ + else { + /* This is a handshake but not a certificate record */ + u_int16_t len = ntohs(*(u_int16_t*)&packet->payload[flow->l4.tcp.tls_record_offset+7]); + +#ifdef DEBUG_TLS + printf("=>> [TLS] Found record %02X [len: %u]\n", + packet->payload[flow->l4.tcp.tls_record_offset+5], len); +#endif + + flow->l4.tcp.tls_record_offset += len + 9; + + if(flow->l4.tcp.tls_record_offset < packet->payload_packet_len) + return(getSSCertificateFingerprint(ndpi_struct, flow)); + else { + flow->l4.tcp.tls_record_offset -= packet->payload_packet_len; + } + } + + return(1); +} + +/* **************************************** */ + +/* See https://blog.catchpoint.com/2017/05/12/dissecting-tls-using-wireshark/ */ void getSSLorganization(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow, char *buffer, int buffer_len) { struct ndpi_packet_struct *packet = &flow->packet; - + u_int16_t total_len; + u_int8_t handshake_protocol; + if(packet->payload[0] != 0x16 /* Handshake */) return; - u_int16_t total_len = (packet->payload[3] << 8) + packet->payload[4] + 5 /* SSL Header */; - u_int8_t handshake_protocol = packet->payload[5]; /* handshake protocol a bit misleading, it is message type according TLS specs */ - + total_len = (packet->payload[3] << 8) + packet->payload[4] + 5 /* SSL Header */; + handshake_protocol = packet->payload[5]; /* handshake protocol a bit misleading, it is message type according TLS specs */ + if((handshake_protocol != 0x02) && (handshake_protocol != 0xb) /* Server Hello and Certificate message types are interesting for us */) return; +#ifdef DEBUG_TLS + printf("=>> [TLS] Certificate [total_len: %u/%u]\n", ntohs(*(u_int16_t*)&packet->payload[3]), total_len); +#endif + /* Truncate total len, search at least in incomplete packet */ if(total_len > packet->payload_packet_len) total_len = packet->payload_packet_len; @@ -759,19 +894,20 @@ void getSSLorganization(struct ndpi_detection_module_struct *ndpi_struct, if(len < (sizeof(utcDate)-1)) { struct tm utc; - + strncpy(utcDate, (const char*)&packet->payload[i+4], len); utcDate[len] = '\0'; - /* 141021000000Z */ + /* 141021000000Z */ if(strptime(utcDate, "%y%m%d%H%M%SZ", &utc) != NULL) { #ifdef DEBUG_TLS - printf("[CERTIFICATE] notBefore %u [%s]\n", mktime(&utc), utcDate); + printf("[CERTIFICATE] notBefore %u [%s]\n", + (unsigned int)mktime(&utc), utcDate); #endif flow->protos.stun_ssl.ssl.notBefore = timegm(&utc); } } - + offset += len; if((offset+1) < packet->payload_packet_len) { @@ -788,14 +924,15 @@ void getSSLorganization(struct ndpi_detection_module_struct *ndpi_struct, if(len < (sizeof(utcDate)-1)) { struct tm utc; - + strncpy(utcDate, (const char*)&packet->payload[offset], len); utcDate[len] = '\0'; - - /* 141021000000Z */ + + /* 141021000000Z */ if(strptime(utcDate, "%y%m%d%H%M%SZ", &utc) != NULL) { #ifdef DEBUG_TLS - printf("[CERTIFICATE] notAfter %u [%s]\n", mktime(&utc), utcDate); + printf("[CERTIFICATE] notAfter %u [%s]\n", + (unsigned int)mktime(&utc), utcDate); #endif flow->protos.stun_ssl.ssl.notAfter = timegm(&utc); } @@ -812,7 +949,14 @@ void getSSLorganization(struct ndpi_detection_module_struct *ndpi_struct, int sslTryAndRetrieveServerCertificate(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) { struct ndpi_packet_struct *packet = &flow->packet; - + int rc = 1; + + if(packet->tcp) { + if(!flow->l4.tcp.tls_srv_cert_fingerprint_processed) + getSSCertificateFingerprint(ndpi_struct, flow); + } + +#if 1 /* consider only specific SSL packets (handshake) */ if((packet->payload_packet_len > 9) && (packet->payload[0] == 0x16)) { char certificate[64]; @@ -820,7 +964,7 @@ int sslTryAndRetrieveServerCertificate(struct ndpi_detection_module_struct *ndpi certificate[0] = '\0'; rc = getTLScertificate(ndpi_struct, flow, certificate, sizeof(certificate)); - packet->ssl_certificate_num_checks++; + packet->tls_certificate_num_checks++; if(rc > 0) { char organization[64]; @@ -829,26 +973,32 @@ int sslTryAndRetrieveServerCertificate(struct ndpi_detection_module_struct *ndpi organization[0] = '\0'; getSSLorganization(ndpi_struct, flow, organization, sizeof(organization)); - packet->ssl_certificate_detected++; - if((flow->l4.tcp.ssl_seen_server_cert == 1) && (flow->protos.stun_ssl.ssl.server_certificate[0] != '\0')) - /* 0 means we're done processing extra packets (since we found what we wanted) */ + packet->tls_certificate_detected++; +#if 0 + if((flow->l4.tcp.tls_seen_server_cert == 1) + && (flow->protos.stun_ssl.ssl.server_certificate[0] != '\0')) + /* 0 means we've done processing extra packets (since we found what we wanted) */ return 0; +#endif } + if(flow->l4.tcp.tls_record_offset == 0) { /* Client hello, Server Hello, and certificate packets probably all checked in this case */ - if(((packet->ssl_certificate_num_checks >= 3) - && (flow->l4.tcp.seen_syn) - && (flow->l4.tcp.seen_syn_ack) - && (flow->l4.tcp.seen_ack) /* We have seen the 3-way handshake */) - || (flow->protos.stun_ssl.ssl.ja3_server[0] != '\0') - ) { - /* We're done processing extra packets since we've probably checked all possible cert packets */ - return 0; + if(((packet->tls_certificate_num_checks >= 3) + && (flow->l4.tcp.seen_syn) + && (flow->l4.tcp.seen_syn_ack) + && (flow->l4.tcp.seen_ack) /* We have seen the 3-way handshake */) + || (flow->protos.stun_ssl.ssl.ja3_server[0] != '\0') + ) { + /* We're done processing extra packets since we've probably checked all possible cert packets */ + return(rc); } + } } - +#endif + /* 1 means keep looking for more packets */ - return 1; + return((!flow->l4.tcp.tls_srv_cert_fingerprint_processed) ? 1 : rc); } /* **************************************** */ @@ -866,9 +1016,15 @@ void sslInitExtraPacketProcessing(int caseNum, struct ndpi_flow_struct *flow) { /* **************************************** */ int tlsDetectProtocolFromCertificate(struct ndpi_detection_module_struct *ndpi_struct, - struct ndpi_flow_struct *flow) { + struct ndpi_flow_struct *flow, + u_int8_t skip_cert_processing) { struct ndpi_packet_struct *packet = &flow->packet; + if((!skip_cert_processing) && packet->tcp) { + if(!flow->l4.tcp.tls_srv_cert_fingerprint_processed) + getSSCertificateFingerprint(ndpi_struct, flow); + } + if((packet->payload_packet_len > 9) && (packet->payload[0] == 0x16 /* consider only specific SSL packets (handshake) */)) { if((packet->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN) @@ -878,10 +1034,10 @@ int tlsDetectProtocolFromCertificate(struct ndpi_detection_module_struct *ndpi_s certificate[0] = '\0'; rc = getTLScertificate(ndpi_struct, flow, certificate, sizeof(certificate)); - packet->ssl_certificate_num_checks++; + packet->tls_certificate_num_checks++; if(rc > 0) { - packet->ssl_certificate_detected++; + packet->tls_certificate_detected++; #ifdef DEBUG_TLS NDPI_LOG_DBG2(ndpi_struct, "***** [SSL] %s\n", certificate); #endif @@ -895,8 +1051,8 @@ int tlsDetectProtocolFromCertificate(struct ndpi_detection_module_struct *ndpi_s /* If we've detected the subprotocol from client certificate but haven't had a chance * to see the server certificate yet, set up extra packet processing to wait * a few more packets. */ - if(((flow->l4.tcp.ssl_seen_client_cert == 1) && (flow->protos.stun_ssl.ssl.client_certificate[0] != '\0')) - && ((flow->l4.tcp.ssl_seen_server_cert != 1) && (flow->protos.stun_ssl.ssl.server_certificate[0] == '\0'))) { + if(((flow->l4.tcp.tls_seen_client_cert == 1) && (flow->protos.stun_ssl.ssl.client_certificate[0] != '\0')) + && ((flow->l4.tcp.tls_seen_server_cert != 1) && (flow->protos.stun_ssl.ssl.server_certificate[0] == '\0'))) { sslInitExtraPacketProcessing(0, flow); } @@ -909,30 +1065,40 @@ int tlsDetectProtocolFromCertificate(struct ndpi_detection_module_struct *ndpi_s return(rc); } - if(((packet->ssl_certificate_num_checks >= 3) + if(((packet->tls_certificate_num_checks >= 3) && flow->l4.tcp.seen_syn && flow->l4.tcp.seen_syn_ack - && flow->l4.tcp.seen_ack /* We have seen the 3-way handshake */) - || ((flow->l4.tcp.ssl_seen_certificate == 1) - && (flow->l4.tcp.ssl_seen_server_cert == 1) + && flow->l4.tcp.seen_ack /* We have seen the 3-way handshake */ + && flow->l4.tcp.tls_srv_cert_fingerprint_processed + ) + /* + || ((flow->l4.tcp.tls_seen_certificate == 1) + && (flow->l4.tcp.tls_seen_server_cert == 1) && (flow->protos.stun_ssl.ssl.server_certificate[0] != '\0')) - /* || ((flow->l4.tcp.ssl_seen_client_cert == 1) && (flow->protos.stun_ssl.ssl.client_certificate[0] != '\0')) */ + */ + /* || ((flow->l4.tcp.tls_seen_client_cert == 1) && (flow->protos.stun_ssl.ssl.client_certificate[0] != '\0')) */ ) { ndpi_int_tls_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_TLS); } } } + return(0); } /* **************************************** */ -static void tls_mark_and_payload_search_for_other_protocols(struct ndpi_detection_module_struct - *ndpi_struct, struct ndpi_flow_struct *flow) { +static void tls_mark_and_payload_search(struct ndpi_detection_module_struct + *ndpi_struct, struct ndpi_flow_struct *flow, + u_int8_t skip_cert_processing) { struct ndpi_packet_struct *packet = &flow->packet; u_int32_t a; u_int32_t end; +#ifdef DEBUG_TLS + printf("[TLS] %s()\n", __FUNCTION__); +#endif + if(NDPI_COMPARE_PROTOCOL_TO_BITMASK(ndpi_struct->detection_bitmask, NDPI_PROTOCOL_UNENCRYPTED_JABBER) != 0) goto check_for_tls_payload; @@ -996,14 +1162,15 @@ static void tls_mark_and_payload_search_for_other_protocols(struct ndpi_detectio no_check_for_tls_payload: if(packet->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN) { NDPI_LOG_DBG(ndpi_struct, "found ssl connection\n"); - tlsDetectProtocolFromCertificate(ndpi_struct, flow); - - if(!packet->ssl_certificate_detected - && (!(flow->l4.tcp.ssl_seen_client_cert && flow->l4.tcp.ssl_seen_server_cert))) { + tlsDetectProtocolFromCertificate(ndpi_struct, flow, skip_cert_processing); + + if(!packet->tls_certificate_detected + && (!(flow->l4.tcp.tls_seen_client_cert && flow->l4.tcp.tls_seen_server_cert))) { /* SSL without certificate (Skype, Ultrasurf?) */ NDPI_LOG_INFO(ndpi_struct, "found ssl NO_CERT\n"); ndpi_int_tls_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_TLS_NO_CERT); - } else if(packet->ssl_certificate_num_checks >= 3) { + } else if((packet->tls_certificate_num_checks >= 3) + && flow->l4.tcp.tls_srv_cert_fingerprint_processed) { NDPI_LOG_INFO(ndpi_struct, "found ssl\n"); ndpi_int_tls_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_TLS); } @@ -1120,7 +1287,7 @@ static u_int8_t ndpi_search_tlsv3_direction1(struct ndpi_detection_module_struct void ndpi_search_tls_tcp_udp(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) { struct ndpi_packet_struct *packet = &flow->packet; - u_int8_t ret; + u_int8_t ret, skip_cert_processing = 0; if(packet->udp != NULL) { /* DTLS dissector */ @@ -1142,8 +1309,10 @@ void ndpi_search_tls_tcp_udp(struct ndpi_detection_module_struct *ndpi_struct, ndpi_struct->stun_cache = ndpi_lru_cache_init(1024); ndpi_lru_add_to_cache(ndpi_struct->stun_cache, key, NDPI_PROTOCOL_SIGNAL); +#ifdef DEBUG_TLS printf("[LRU] Adding Signal cached key %u\n", key); - +#endif + /* In Signal protocol STUN turns into DTLS... */ ndpi_int_tls_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_SIGNAL); } else if(flow->protos.stun_ssl.ssl.ja3_server[0] != '\0') { @@ -1156,13 +1325,13 @@ void ndpi_search_tls_tcp_udp(struct ndpi_detection_module_struct *ndpi_struct, } if(packet->detected_protocol_stack[0] == NDPI_PROTOCOL_TLS) { - if(flow->l4.tcp.ssl_stage == 3 && packet->payload_packet_len > 20 && flow->packet_counter < 5) { + if(flow->l4.tcp.tls_stage == 3 && packet->payload_packet_len > 20 && flow->packet_counter < 5) { /* this should only happen, when we detected SSL with a packet that had parts of the certificate in subsequent packets * so go on checking for certificate patterns for a couple more packets */ NDPI_LOG_DBG2(ndpi_struct, "ssl flow but check another packet for patterns\n"); - tls_mark_and_payload_search_for_other_protocols(ndpi_struct, flow); + tls_mark_and_payload_search(ndpi_struct, flow, skip_cert_processing); if(packet->detected_protocol_stack[0] == NDPI_PROTOCOL_TLS) { /* still ssl so check another packet */ @@ -1194,18 +1363,20 @@ void ndpi_search_tls_tcp_udp(struct ndpi_detection_module_struct *ndpi_struct, return; } else { /* No whatsapp, let's try SSL */ - if(tlsDetectProtocolFromCertificate(ndpi_struct, flow) > 0) + if(tlsDetectProtocolFromCertificate(ndpi_struct, flow, skip_cert_processing) > 0) return; + else + skip_cert_processing = 1; } - if(packet->payload_packet_len > 40 && flow->l4.tcp.ssl_stage == 0) { + if(packet->payload_packet_len > 40 && flow->l4.tcp.tls_stage == 0) { NDPI_LOG_DBG2(ndpi_struct, "first ssl packet\n"); // SSLv2 Record if(packet->payload[2] == 0x01 && packet->payload[3] == 0x03 && (packet->payload[4] == 0x00 || packet->payload[4] == 0x01 || packet->payload[4] == 0x02) && (packet->payload_packet_len - packet->payload[1] == 2)) { NDPI_LOG_DBG2(ndpi_struct, "sslv2 len match\n"); - flow->l4.tcp.ssl_stage = 1 + packet->packet_direction; + flow->l4.tcp.tls_stage = 1 + packet->packet_direction; return; } @@ -1214,7 +1385,7 @@ void ndpi_search_tls_tcp_udp(struct ndpi_detection_module_struct *ndpi_struct, && (packet->payload_packet_len - ntohs(get_u_int16_t(packet->payload, 3)) == 5)) { // SSLv3 Record NDPI_LOG_DBG2(ndpi_struct, "sslv3 len match\n"); - flow->l4.tcp.ssl_stage = 1 + packet->packet_direction; + flow->l4.tcp.tls_stage = 1 + packet->packet_direction; return; } @@ -1224,41 +1395,41 @@ void ndpi_search_tls_tcp_udp(struct ndpi_detection_module_struct *ndpi_struct, packet->payload[2] == 0x02 || packet->payload[2] == 0x03)) { if(packet->payload_packet_len - ntohs(get_u_int16_t(packet->payload, 3)) == 5) { NDPI_LOG_DBG2(ndpi_struct, "TLS len match\n"); - flow->l4.tcp.ssl_stage = 1 + packet->packet_direction; + flow->l4.tcp.tls_stage = 1 + packet->packet_direction; return; } } } if(packet->payload_packet_len > 40 && - flow->l4.tcp.ssl_stage == 1 + packet->packet_direction + flow->l4.tcp.tls_stage == 1 + packet->packet_direction && flow->packet_direction_counter[packet->packet_direction] < 5) { return; } - if(packet->payload_packet_len > 40 && flow->l4.tcp.ssl_stage == 2 - packet->packet_direction) { + if(packet->payload_packet_len > 40 && flow->l4.tcp.tls_stage == 2 - packet->packet_direction) { NDPI_LOG_DBG2(ndpi_struct, "second ssl packet\n"); // SSLv2 Record if(packet->payload[2] == 0x01 && packet->payload[3] == 0x03 && (packet->payload[4] == 0x00 || packet->payload[4] == 0x01 || packet->payload[4] == 0x02) && (packet->payload_packet_len - 2) >= packet->payload[1]) { NDPI_LOG_DBG2(ndpi_struct, "sslv2 server len match\n"); - tls_mark_and_payload_search_for_other_protocols(ndpi_struct, flow); + tls_mark_and_payload_search(ndpi_struct, flow, skip_cert_processing); return; } ret = ndpi_search_tlsv3_direction1(ndpi_struct, flow); if(ret == 1) { NDPI_LOG_DBG2(ndpi_struct, "sslv3 server len match\n"); - tls_mark_and_payload_search_for_other_protocols(ndpi_struct, flow); + tls_mark_and_payload_search(ndpi_struct, flow, skip_cert_processing); return; } else if(ret == 2) { NDPI_LOG_DBG2(ndpi_struct, "sslv3 server len match with split packet -> check some more packets for SSL patterns\n"); - tls_mark_and_payload_search_for_other_protocols(ndpi_struct, flow); - if(packet->detected_protocol_stack[0] == NDPI_PROTOCOL_TLS) { - flow->l4.tcp.ssl_stage = 3; - } + tls_mark_and_payload_search(ndpi_struct, flow, skip_cert_processing); + + if(packet->detected_protocol_stack[0] == NDPI_PROTOCOL_TLS) + flow->l4.tcp.tls_stage = 3; return; } diff --git a/src/lib/protocols/tor.c b/src/lib/protocols/tor.c index b3070d0b9..9dd14046e 100644 --- a/src/lib/protocols/tor.c +++ b/src/lib/protocols/tor.c @@ -101,7 +101,7 @@ void ndpi_search_tor(struct ndpi_detection_module_struct *ndpi_struct, struct nd NDPI_LOG_DBG(ndpi_struct, "search for TOR\n"); if((packet->tcp != NULL) - && (!packet->ssl_certificate_detected)) { + && (!packet->tls_certificate_detected)) { u_int16_t dport, sport; sport = ntohs(packet->tcp->source), dport = ntohs(packet->tcp->dest); diff --git a/src/lib/third_party/include/ndpi_sha1.h b/src/lib/third_party/include/ndpi_sha1.h new file mode 100644 index 000000000..11eab1e51 --- /dev/null +++ b/src/lib/third_party/include/ndpi_sha1.h @@ -0,0 +1,17 @@ +/* ================ sha1.h ================ */ +/* +SHA-1 in C +By Steve Reid <steve@edmweb.com> +100% Public Domain +*/ + +typedef struct { + u_int32_t state[5]; + u_int32_t count[2]; + unsigned char buffer[64]; +} SHA1_CTX; + +void SHA1Transform(u_int32_t state[5], const unsigned char buffer[64]); +void SHA1Init(SHA1_CTX* context); +void SHA1Update(SHA1_CTX* context, const unsigned char* data, u_int32_t len); +void SHA1Final(unsigned char digest[20], SHA1_CTX* context); diff --git a/src/lib/third_party/src/ndpi_sha1.c b/src/lib/third_party/src/ndpi_sha1.c new file mode 100644 index 000000000..09cb9b906 --- /dev/null +++ b/src/lib/third_party/src/ndpi_sha1.c @@ -0,0 +1,276 @@ + +/* from valgrind tests */ + +/* ================ sha1.c ================ */ +/* +SHA-1 in C +By Steve Reid <steve@edmweb.com> +100% Public Domain + +Test Vectors (from FIPS PUB 180-1) +"abc" + A9993E36 4706816A BA3E2571 7850C26C 9CD0D89D +"abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" + 84983E44 1C3BD26E BAAE4AA1 F95129E5 E54670F1 +A million repetitions of "a" + 34AA973C D4C4DAA4 F61EEB2B DBAD2731 6534016F +*/ + +/* #define LITTLE_ENDIAN * This should be #define'd already, if true. */ +/* #define SHA1HANDSOFF * Copies data before messing with it. */ + +#define SHA1HANDSOFF + +#include <stdio.h> +#include <string.h> +#include <sys/types.h> /* for u_int*_t */ +#if defined(__sun) +#include "solarisfixes.h" +#endif +#include "ndpi_sha1.h" + +#ifndef BYTE_ORDER +#if (BSD >= 199103) +# include <machine/endian.h> +#else +#if defined(linux) || defined(__linux__) +# include <endian.h> +#else +#define LITTLE_ENDIAN 1234 /* least-significant byte first (vax, pc) */ +#define BIG_ENDIAN 4321 /* most-significant byte first (IBM, net) */ +#define PDP_ENDIAN 3412 /* LSB first in word, MSW first in long (pdp)*/ + +#if defined(vax) || defined(ns32000) || defined(sun386) || defined(__i386__) || \ + defined(MIPSEL) || defined(_MIPSEL) || defined(BIT_ZERO_ON_RIGHT) || \ + defined(__alpha__) || defined(__alpha) +#define BYTE_ORDER LITTLE_ENDIAN +#endif + +#if defined(sel) || defined(pyr) || defined(mc68000) || defined(sparc) || \ + defined(is68k) || defined(tahoe) || defined(ibm032) || defined(ibm370) || \ + defined(MIPSEB) || defined(_MIPSEB) || defined(_IBMR2) || defined(DGUX) ||\ + defined(apollo) || defined(__convex__) || defined(_CRAY) || \ + defined(__hppa) || defined(__hp9000) || \ + defined(__hp9000s300) || defined(__hp9000s700) || \ + defined (BIT_ZERO_ON_LEFT) || defined(m68k) || defined(__sparc) +#define BYTE_ORDER BIG_ENDIAN +#endif +#endif /* linux */ +#endif /* BSD */ +#endif /* BYTE_ORDER */ + +#if defined(__BYTE_ORDER) && !defined(BYTE_ORDER) +#if (__BYTE_ORDER == __LITTLE_ENDIAN) +#define BYTE_ORDER LITTLE_ENDIAN +#else +#define BYTE_ORDER BIG_ENDIAN +#endif +#endif + +#if !defined(BYTE_ORDER) || \ + (BYTE_ORDER != BIG_ENDIAN && BYTE_ORDER != LITTLE_ENDIAN && \ + BYTE_ORDER != PDP_ENDIAN) + /* you must determine what the correct bit order is for + * your compiler - the next line is an intentional error + * which will force your compiles to bomb until you fix + * the above macros. + */ +#error "Undefined or invalid BYTE_ORDER" +#endif + +#define rol(value, bits) (((value) << (bits)) | ((value) >> (32 - (bits)))) + +/* blk0() and blk() perform the initial expand. */ +/* I got the idea of expanding during the round function from SSLeay */ +#if BYTE_ORDER == LITTLE_ENDIAN +#define blk0(i) (block->l[i] = (rol(block->l[i],24)&0xFF00FF00) \ + |(rol(block->l[i],8)&0x00FF00FF)) +#elif BYTE_ORDER == BIG_ENDIAN +#define blk0(i) block->l[i] +#else +#error "Endianness not defined!" +#endif +#define blk(i) (block->l[i&15] = rol(block->l[(i+13)&15]^block->l[(i+8)&15] \ + ^block->l[(i+2)&15]^block->l[i&15],1)) + +/* (R0+R1), R2, R3, R4 are the different operations used in SHA1 */ +#define R0(v,w,x,y,z,i) z+=((w&(x^y))^y)+blk0(i)+0x5A827999+rol(v,5);w=rol(w,30); +#define R1(v,w,x,y,z,i) z+=((w&(x^y))^y)+blk(i)+0x5A827999+rol(v,5);w=rol(w,30); +#define R2(v,w,x,y,z,i) z+=(w^x^y)+blk(i)+0x6ED9EBA1+rol(v,5);w=rol(w,30); +#define R3(v,w,x,y,z,i) z+=(((w|x)&y)|(w&x))+blk(i)+0x8F1BBCDC+rol(v,5);w=rol(w,30); +#define R4(v,w,x,y,z,i) z+=(w^x^y)+blk(i)+0xCA62C1D6+rol(v,5);w=rol(w,30); + + +/* Hash a single 512-bit block. This is the core of the algorithm. */ + +void SHA1Transform(u_int32_t state[5], const unsigned char buffer[64]) +{ +u_int32_t a, b, c, d, e; +typedef union { + unsigned char c[64]; + u_int32_t l[16]; +} CHAR64LONG16; +#ifdef SHA1HANDSOFF +CHAR64LONG16 block[1]; /* use array to appear as a pointer */ + memcpy(block, buffer, 64); +#else + /* The following had better never be used because it causes the + * pointer-to-const buffer to be cast into a pointer to non-const. + * And the result is written through. I threw a "const" in, hoping + * this will cause a diagnostic. + */ +CHAR64LONG16* block = (const CHAR64LONG16*)buffer; +#endif + /* Copy context->state[] to working vars */ + a = state[0]; + b = state[1]; + c = state[2]; + d = state[3]; + e = state[4]; + /* 4 rounds of 20 operations each. Loop unrolled. */ + R0(a,b,c,d,e, 0); R0(e,a,b,c,d, 1); R0(d,e,a,b,c, 2); R0(c,d,e,a,b, 3); + R0(b,c,d,e,a, 4); R0(a,b,c,d,e, 5); R0(e,a,b,c,d, 6); R0(d,e,a,b,c, 7); + R0(c,d,e,a,b, 8); R0(b,c,d,e,a, 9); R0(a,b,c,d,e,10); R0(e,a,b,c,d,11); + R0(d,e,a,b,c,12); R0(c,d,e,a,b,13); R0(b,c,d,e,a,14); R0(a,b,c,d,e,15); + R1(e,a,b,c,d,16); R1(d,e,a,b,c,17); R1(c,d,e,a,b,18); R1(b,c,d,e,a,19); + R2(a,b,c,d,e,20); R2(e,a,b,c,d,21); R2(d,e,a,b,c,22); R2(c,d,e,a,b,23); + R2(b,c,d,e,a,24); R2(a,b,c,d,e,25); R2(e,a,b,c,d,26); R2(d,e,a,b,c,27); + R2(c,d,e,a,b,28); R2(b,c,d,e,a,29); R2(a,b,c,d,e,30); R2(e,a,b,c,d,31); + R2(d,e,a,b,c,32); R2(c,d,e,a,b,33); R2(b,c,d,e,a,34); R2(a,b,c,d,e,35); + R2(e,a,b,c,d,36); R2(d,e,a,b,c,37); R2(c,d,e,a,b,38); R2(b,c,d,e,a,39); + R3(a,b,c,d,e,40); R3(e,a,b,c,d,41); R3(d,e,a,b,c,42); R3(c,d,e,a,b,43); + R3(b,c,d,e,a,44); R3(a,b,c,d,e,45); R3(e,a,b,c,d,46); R3(d,e,a,b,c,47); + R3(c,d,e,a,b,48); R3(b,c,d,e,a,49); R3(a,b,c,d,e,50); R3(e,a,b,c,d,51); + R3(d,e,a,b,c,52); R3(c,d,e,a,b,53); R3(b,c,d,e,a,54); R3(a,b,c,d,e,55); + R3(e,a,b,c,d,56); R3(d,e,a,b,c,57); R3(c,d,e,a,b,58); R3(b,c,d,e,a,59); + R4(a,b,c,d,e,60); R4(e,a,b,c,d,61); R4(d,e,a,b,c,62); R4(c,d,e,a,b,63); + R4(b,c,d,e,a,64); R4(a,b,c,d,e,65); R4(e,a,b,c,d,66); R4(d,e,a,b,c,67); + R4(c,d,e,a,b,68); R4(b,c,d,e,a,69); R4(a,b,c,d,e,70); R4(e,a,b,c,d,71); + R4(d,e,a,b,c,72); R4(c,d,e,a,b,73); R4(b,c,d,e,a,74); R4(a,b,c,d,e,75); + R4(e,a,b,c,d,76); R4(d,e,a,b,c,77); R4(c,d,e,a,b,78); R4(b,c,d,e,a,79); + /* Add the working vars back into context.state[] */ + state[0] += a; + state[1] += b; + state[2] += c; + state[3] += d; + state[4] += e; + /* Wipe variables */ + a = b = c = d = e = 0; +#ifdef SHA1HANDSOFF + memset(block, '\0', sizeof(block)); +#endif +} + + +/* SHA1Init - Initialize new context */ + +void SHA1Init(SHA1_CTX* context) +{ + /* SHA1 initialization constants */ + context->state[0] = 0x67452301; + context->state[1] = 0xEFCDAB89; + context->state[2] = 0x98BADCFE; + context->state[3] = 0x10325476; + context->state[4] = 0xC3D2E1F0; + context->count[0] = context->count[1] = 0; +} + + +/* Run your data through this. */ + +void SHA1Update(SHA1_CTX* context, const unsigned char* data, u_int32_t len) +{ +u_int32_t i; +u_int32_t j; + + j = context->count[0]; + if ((context->count[0] += len << 3) < j) + context->count[1]++; + context->count[1] += (len>>29); + j = (j >> 3) & 63; + if ((j + len) > 63) { + memcpy(&context->buffer[j], data, (i = 64-j)); + SHA1Transform(context->state, context->buffer); + for ( ; i + 63 < len; i += 64) { + SHA1Transform(context->state, &data[i]); + } + j = 0; + } + else i = 0; + memcpy(&context->buffer[j], &data[i], len - i); +} + + +/* Add padding and return the message digest. */ + +void SHA1Final(unsigned char digest[20], SHA1_CTX* context) +{ +unsigned i; +unsigned char finalcount[8]; +unsigned char c; + +#if 0 /* untested "improvement" by DHR */ + /* Convert context->count to a sequence of bytes + * in finalcount. Second element first, but + * big-endian order within element. + * But we do it all backwards. + */ + unsigned char *fcp = &finalcount[8]; + + for (i = 0; i < 2; i++) + { + u_int32_t t = context->count[i]; + int j; + + for (j = 0; j < 4; t >>= 8, j++) + *--fcp = (unsigned char) t + } +#else + for (i = 0; i < 8; i++) { + finalcount[i] = (unsigned char)((context->count[(i >= 4 ? 0 : 1)] + >> ((3-(i & 3)) * 8) ) & 255); /* Endian independent */ + } +#endif + c = 0200; + SHA1Update(context, &c, 1); + while ((context->count[0] & 504) != 448) { + c = 0000; + SHA1Update(context, &c, 1); + } + SHA1Update(context, finalcount, 8); /* Should cause a SHA1Transform() */ + for (i = 0; i < 20; i++) { + digest[i] = (unsigned char) + ((context->state[i>>2] >> ((3-(i & 3)) * 8) ) & 255); + } + /* Wipe variables */ + memset(context, '\0', sizeof(*context)); + memset(&finalcount, '\0', sizeof(finalcount)); +} +/* ================ end of sha1.c ================ */ + +#if 0 +#define BUFSIZE 4096 + +int +main(int argc, char **argv) +{ + SHA1_CTX ctx; + unsigned char hash[20], buf[BUFSIZE]; + int i; + + for(i=0;i<BUFSIZE;i++) + buf[i] = i; + + SHA1Init(&ctx); + for(i=0;i<1000;i++) + SHA1Update(&ctx, buf, BUFSIZE); + SHA1Final(hash, &ctx); + + printf("SHA1="); + for(i=0;i<20;i++) + printf("%02x", hash[i]); + printf("\n"); + return 0; +} + +#endif |