From 582da8d4bb00ed6be656451fbd6ebde6719c5ce2 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Mon, 19 Nov 2018 18:24:33 +0100 Subject: Improved WeChat support Removed Musical.ly protocol as the service has been merged with another one and thus it is no longer used Improved guess for UDP protocols --- src/include/ndpi_api.h | 2 + src/include/ndpi_main.h | 1 + src/include/ndpi_protocol_ids.h | 2 +- src/include/ndpi_typedefs.h | 2 +- src/lib/ndpi_content_match.c.inc | 13 +++--- src/lib/ndpi_main.c | 93 ++++++++++++++++++++++++---------------- src/lib/protocols/dropbox.c | 26 +++++++---- src/lib/protocols/eaq.c | 10 +++-- src/lib/protocols/skype.c | 17 ++++++-- 9 files changed, 106 insertions(+), 60 deletions(-) (limited to 'src') diff --git a/src/include/ndpi_api.h b/src/include/ndpi_api.h index d010577a5..c7ed7cd17 100644 --- a/src/include/ndpi_api.h +++ b/src/include/ndpi_api.h @@ -326,6 +326,7 @@ extern "C" { * Search and return the protocol guessed that is undetected * * @par ndpi_struct = the detection module + * @par flow = the flow we're trying to guess, NULL if not available * @par proto = the l4 protocol number * @par shost = source address in host byte order * @par sport = source port number @@ -335,6 +336,7 @@ extern "C" { * */ ndpi_protocol ndpi_guess_undetected_protocol(struct ndpi_detection_module_struct *ndpi_struct, + struct ndpi_flow_struct *flow, u_int8_t proto, u_int32_t shost, u_int16_t sport, diff --git a/src/include/ndpi_main.h b/src/include/ndpi_main.h index 3517d45b2..41280fb8e 100644 --- a/src/include/ndpi_main.h +++ b/src/include/ndpi_main.h @@ -119,6 +119,7 @@ extern "C" { u_int16_t ndpi_get_proto_by_name(struct ndpi_detection_module_struct *ndpi_mod, const char *name); extern u_int16_t ndpi_guess_protocol_id(struct ndpi_detection_module_struct *ndpi_struct, + struct ndpi_flow_struct *flow, u_int8_t proto, u_int16_t sport, u_int16_t dport, u_int8_t *user_defined_proto); diff --git a/src/include/ndpi_protocol_ids.h b/src/include/ndpi_protocol_ids.h index 0d2712d20..aefb0fce2 100644 --- a/src/include/ndpi_protocol_ids.h +++ b/src/include/ndpi_protocol_ids.h @@ -73,7 +73,7 @@ typedef enum { NDPI_PROTOCOL_EDONKEY = 36, /* Tomasz Bujlow */ NDPI_PROTOCOL_BITTORRENT = 37, NDPI_PROTOCOL_SKYPE_CALL_OUT = 38, - NDPI_PROTOCOL_MUSICALLY = 39, + NDPI_PROTOCOL_FREE_39 = 39, /* Free */ NDPI_PROTOCOL_MEMCACHED = 40, /* Memcached - Darryl Sokoloski */ NDPI_PROTOCOL_SMBV23 = 41, /* SMB version 2/3 */ NDPI_PROTOCOL_MINING = 42, /* Bitcoin, Ethereum, ZCash, Monero */ diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h index 51681e60e..4b69c2b7b 100644 --- a/src/include/ndpi_typedefs.h +++ b/src/include/ndpi_typedefs.h @@ -826,7 +826,7 @@ typedef enum { ndpi_pref_dns_dissect_response, ndpi_pref_direction_detect_disable, ndpi_pref_disable_metadata_export, - ndpi_pref_enable_category_substring_match, + ndpi_pref_enable_category_substring_match } ndpi_detection_preference; /* ntop extensions */ diff --git a/src/lib/ndpi_content_match.c.inc b/src/lib/ndpi_content_match.c.inc index 71e236d37..4bfdde983 100644 --- a/src/lib/ndpi_content_match.c.inc +++ b/src/lib/ndpi_content_match.c.inc @@ -21,7 +21,6 @@ /* ****************************************************** */ static ndpi_network host_protocol_list[] = { - /* SoundCloud */ @@ -111,11 +110,14 @@ static ndpi_network host_protocol_list[] = { WeChat origin AS132203, AS132591, AS45090 */ - { 0xCBCD93AB /* 203.205.147.171/32 */, 32, NDPI_PROTOCOL_WECHAT }, { 0xCBCD93AD /* 203.205.147.173/32 */, 32, NDPI_PROTOCOL_WECHAT }, { 0xCBCD97A2 /* 203.205.151.162/32 */, 32, NDPI_PROTOCOL_WECHAT }, - { 0x67071E25 /* 103.7.30.37/32 */, 32, NDPI_PROTOCOL_WECHAT }, + { 0x67071E25 /* 103.7.30.37/32 */, 32, NDPI_PROTOCOL_WECHAT }, + { 0XCBCDB000 /* 203.205.176.0/20 */, 20, NDPI_PROTOCOL_WECHAT }, + { 0XCBCDC000 /* 203.205.192.0/18 */, 18, NDPI_PROTOCOL_WECHAT }, + + /* OpenDNS, LLC @@ -8327,10 +8329,7 @@ ndpi_protocol_match host_match[] = { /* Detected "slack-assets2.s3-us-west-2.amazonaws.com.". Omitted "*amazonaws.com" CDN, but no generic pattern to use on first part */ { "slack-assets2.s3-", NULL, "slack-assets2\\.s3-", "Slack", NDPI_PROTOCOL_SLACK, NDPI_PROTOCOL_CATEGORY_COLLABORATIVE, NDPI_PROTOCOL_ACCEPTABLE }, - { "zhiliaoapp.com", NULL, "zhiliaoapp" TLD, "Musical.ly", NDPI_PROTOCOL_MUSICALLY, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN }, - { "muscdn.com", NULL, "muscdn" TLD, "Musical.ly", NDPI_PROTOCOL_MUSICALLY, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN }, - { "livelycdn.com", NULL, "livelycdn" TLD, "Musical.ly", NDPI_PROTOCOL_MUSICALLY, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN }, - { "direct.ly", NULL, "direct\\.ly$", "Musical.ly", NDPI_PROTOCOL_MUSICALLY, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN }, + { "wechat.com", NULL, "wechat\\.com" TLD, "WeChat", NDPI_PROTOCOL_WECHAT, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN }, { "github.com", NULL, "github" TLD, "Github", NDPI_PROTOCOL_GITHUB, NDPI_PROTOCOL_CATEGORY_COLLABORATIVE, NDPI_PROTOCOL_ACCEPTABLE }, { ".github.com", NULL, "\\.github" TLD, "Github", NDPI_PROTOCOL_GITHUB, NDPI_PROTOCOL_CATEGORY_COLLABORATIVE, NDPI_PROTOCOL_ACCEPTABLE }, diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index 3a6cd84ef..02ad31a5e 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -517,7 +517,6 @@ void ndpi_exclude_protocol(struct ndpi_detection_module_struct *ndpi_struct, (*(ndpi_struct->ndpi_debug_printf))(protocol_id, ndpi_struct, NDPI_LOG_DEBUG, _file, _func, _line, "exclude %s\n",ndpi_get_proto_name(ndpi_struct, protocol_id)); - } #endif NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, protocol_id); @@ -1205,9 +1204,9 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp no_master, "Teredo", NDPI_PROTOCOL_CATEGORY_NETWORK, ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */, ndpi_build_default_ports(ports_b, 3544, 0, 0, 0, 0) /* UDP */); - ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_MUSICALLY, - no_master, - no_master, "Musical.ly", NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, + ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_WECHAT, + no_master, /* wechat.com */ + no_master, "WeChat", NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */, ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_MEMCACHED, @@ -1245,7 +1244,12 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp no_master, "Free", NDPI_PROTOCOL_CATEGORY_CUSTOM_1 /* dummy */, ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */, ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); - ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_FREE_196, + ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_FREE_39, + no_master, + no_master, "Free", NDPI_PROTOCOL_CATEGORY_CUSTOM_1 /* dummy */, + ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */, + ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); + ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_FREE_39, no_master, no_master, "Free", NDPI_PROTOCOL_CATEGORY_CUSTOM_1 /* dummy */, ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */, @@ -2523,6 +2527,7 @@ static ndpi_default_ports_tree_node_t* ndpi_get_guessed_protocol_id(struct ndpi_ /* ****************************************************** */ u_int16_t ndpi_guess_protocol_id(struct ndpi_detection_module_struct *ndpi_struct, + struct ndpi_flow_struct *flow, u_int8_t proto, u_int16_t sport, u_int16_t dport, u_int8_t *user_defined_proto) { @@ -2531,8 +2536,17 @@ u_int16_t ndpi_guess_protocol_id(struct ndpi_detection_module_struct *ndpi_struc ndpi_default_ports_tree_node_t *found = ndpi_get_guessed_protocol_id(ndpi_struct, proto, sport, dport); if(found != NULL) { - *user_defined_proto = found->customUserProto; - return(found->proto->protoId); + u_int16_t guessed_proto = found->proto->protoId; + + /* We need to check if the guessed protocol isn't excluded by nDPI */ + if(flow + && (proto == IPPROTO_UDP) + && (NDPI_COMPARE_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, guessed_proto))) + return(NDPI_PROTOCOL_UNKNOWN); + else { + *user_defined_proto = found->customUserProto; + return(guessed_proto); + } } } else { /* No TCP/UDP */ @@ -3956,18 +3970,17 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st && (flow->l4.tcp.ssl_stage > 1)) flow->guessed_protocol_id = NDPI_PROTOCOL_SSL_NO_CERT; - guessed_protocol_id = flow->guessed_protocol_id, - guessed_host_protocol_id = flow->guessed_host_protocol_id; + guessed_protocol_id = flow->guessed_protocol_id, guessed_host_protocol_id = flow->guessed_host_protocol_id; if((guessed_host_protocol_id != NDPI_PROTOCOL_UNKNOWN) - && (NDPI_ISSET(&flow->excluded_protocol_bitmask, guessed_host_protocol_id))) - guessed_host_protocol_id = NDPI_PROTOCOL_UNKNOWN; - + && ((flow->packet.l4_protocol == IPPROTO_UDP) && NDPI_ISSET(&flow->excluded_protocol_bitmask, guessed_host_protocol_id))) + flow->guessed_host_protocol_id = guessed_host_protocol_id = NDPI_PROTOCOL_UNKNOWN; + /* Ignore guessed protocol if they have been discarded */ if((guessed_protocol_id != NDPI_PROTOCOL_UNKNOWN) - && (guessed_host_protocol_id == NDPI_PROTOCOL_UNKNOWN) - && (NDPI_ISSET(&flow->excluded_protocol_bitmask, guessed_protocol_id))) - guessed_protocol_id = NDPI_PROTOCOL_UNKNOWN; + // && (guessed_host_protocol_id == NDPI_PROTOCOL_UNKNOWN) + && (flow->packet.l4_protocol == IPPROTO_UDP) && NDPI_ISSET(&flow->excluded_protocol_bitmask, guessed_protocol_id)) + flow->guessed_protocol_id = guessed_protocol_id = NDPI_PROTOCOL_UNKNOWN; if((guessed_protocol_id != NDPI_PROTOCOL_UNKNOWN) || (guessed_host_protocol_id != NDPI_PROTOCOL_UNKNOWN)) { @@ -4394,7 +4407,7 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct else sport = dport = 0; /* guess protocol */ - flow->guessed_protocol_id = (int16_t) ndpi_guess_protocol_id(ndpi_struct, protocol, sport, dport, &user_defined_proto); + flow->guessed_protocol_id = (int16_t) ndpi_guess_protocol_id(ndpi_struct, flow, protocol, sport, dport, &user_defined_proto); flow->guessed_host_protocol_id = ndpi_guess_host_protocol_id(ndpi_struct, flow); if(flow->guessed_protocol_id >= (NDPI_MAX_SUPPORTED_PROTOCOLS-1)) { @@ -5356,6 +5369,7 @@ u_int16_t ndpi_get_lower_proto(ndpi_protocol proto) { /* ****************************************************** */ ndpi_protocol ndpi_guess_undetected_protocol(struct ndpi_detection_module_struct *ndpi_struct, + struct ndpi_flow_struct *flow, u_int8_t proto, u_int32_t shost /* host byte order */, u_int16_t sport, u_int32_t dhost /* host byte order */, u_int16_t dport) { @@ -5365,30 +5379,37 @@ ndpi_protocol ndpi_guess_undetected_protocol(struct ndpi_detection_module_struct u_int8_t user_defined_proto; if((proto == IPPROTO_TCP) || (proto == IPPROTO_UDP)) { - rc = ndpi_search_tcp_or_udp_raw(ndpi_struct, NULL, proto, - shost, dhost, sport, dport); + rc = ndpi_search_tcp_or_udp_raw(ndpi_struct, NULL, proto, shost, dhost, sport, dport); if(rc != NDPI_PROTOCOL_UNKNOWN) { - ret.app_protocol = rc, - ret.master_protocol = ndpi_guess_protocol_id(ndpi_struct, proto, sport, - dport, &user_defined_proto); - - if(ret.app_protocol == ret.master_protocol) - ret.master_protocol = NDPI_PROTOCOL_UNKNOWN; - - ret.category = ndpi_get_proto_category(ndpi_struct, ret); - return(ret); + if(flow && (proto == IPPROTO_UDP) && NDPI_COMPARE_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, rc)) + ; + else { + ret.app_protocol = rc, + ret.master_protocol = ndpi_guess_protocol_id(ndpi_struct, NULL, proto, sport, + dport, &user_defined_proto); + + if(ret.app_protocol == ret.master_protocol) + ret.master_protocol = NDPI_PROTOCOL_UNKNOWN; + + ret.category = ndpi_get_proto_category(ndpi_struct, ret); + return(ret); + } } - rc = ndpi_guess_protocol_id(ndpi_struct, proto, sport, dport, &user_defined_proto); + rc = ndpi_guess_protocol_id(ndpi_struct, NULL, proto, sport, dport, &user_defined_proto); if(rc != NDPI_PROTOCOL_UNKNOWN) { - ret.app_protocol = rc; - - if(rc == NDPI_PROTOCOL_SSL) - goto check_guessed_skype; + if(flow && (proto == IPPROTO_UDP) && NDPI_COMPARE_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, rc)) + ; else { - ret.category = ndpi_get_proto_category(ndpi_struct, ret); - return(ret); + ret.app_protocol = rc; + + if(rc == NDPI_PROTOCOL_SSL) + goto check_guessed_skype; + else { + ret.category = ndpi_get_proto_category(ndpi_struct, ret); + return(ret); + } } } @@ -5402,10 +5423,10 @@ ndpi_protocol ndpi_guess_undetected_protocol(struct ndpi_detection_module_struct ret.app_protocol = NDPI_PROTOCOL_SKYPE; } } else - ret.app_protocol = ndpi_guess_protocol_id(ndpi_struct, proto, sport, + ret.app_protocol = ndpi_guess_protocol_id(ndpi_struct, NULL, proto, sport, dport, &user_defined_proto); - ret.category = ndpi_get_proto_category(ndpi_struct, ret); + ret.category = ndpi_get_proto_category(ndpi_struct, ret); return(ret); } diff --git a/src/lib/protocols/dropbox.c b/src/lib/protocols/dropbox.c index d981c76b3..39bb96ff2 100644 --- a/src/lib/protocols/dropbox.c +++ b/src/lib/protocols/dropbox.c @@ -48,14 +48,24 @@ static void ndpi_check_dropbox(struct ndpi_detection_module_struct *ndpi_struct, if(packet->udp != NULL) { u_int16_t dropbox_port = htons(DB_LSP_PORT); - if((packet->udp->source == dropbox_port) - && (packet->udp->dest == dropbox_port)) { - if(payload_len > 2) { - if(strstr((const char *)packet->payload, "\"host_int\"") != NULL) { - - NDPI_LOG_INFO(ndpi_struct, "found dropbox\n"); - ndpi_int_dropbox_add_connection(ndpi_struct, flow, 0); - return; + if(packet->udp->dest == dropbox_port) { + if(packet->udp->source == dropbox_port) { + if(payload_len > 10) { + if(ndpi_strnstr((const char *)packet->payload, "\"host_int\"", payload_len) != NULL) { + + NDPI_LOG_INFO(ndpi_struct, "found dropbox\n"); + ndpi_int_dropbox_add_connection(ndpi_struct, flow, 0); + return; + } + } + } else { + if(payload_len > 10) { + if(ndpi_strnstr((const char *)packet->payload, "Bus17Cmd", payload_len) != NULL) { + + NDPI_LOG_INFO(ndpi_struct, "found dropbox\n"); + ndpi_int_dropbox_add_connection(ndpi_struct, flow, 0); + return; + } } } } diff --git a/src/lib/protocols/eaq.c b/src/lib/protocols/eaq.c index 2082b5263..8768cac6f 100644 --- a/src/lib/protocols/eaq.c +++ b/src/lib/protocols/eaq.c @@ -53,12 +53,15 @@ void ndpi_search_eaq(struct ndpi_detection_module_struct *ndpi_struct, struct nd if(packet->udp != NULL) { u_int32_t seq = (packet->payload[0] * 1000) + (packet->payload[1] * 100) + (packet->payload[2] * 10) + packet->payload[3]; - + if(flow->l4.udp.eaq_pkt_id == 0) flow->l4.udp.eaq_sequence = seq; else { if( (flow->l4.udp.eaq_sequence != seq) && - ((flow->l4.udp.eaq_sequence+1) != seq)) break; + ((flow->l4.udp.eaq_sequence+1) != seq)) + break; + else + flow->l4.udp.eaq_sequence = seq; } if(++flow->l4.udp.eaq_pkt_id == 4) { @@ -66,7 +69,8 @@ void ndpi_search_eaq(struct ndpi_detection_module_struct *ndpi_struct, struct nd NDPI_LOG_INFO(ndpi_struct, "found eaq\n"); ndpi_int_eaq_add_connection(ndpi_struct, flow); return; - } + } else + return; } } while(0); diff --git a/src/lib/protocols/skype.c b/src/lib/protocols/skype.c index e544724a5..8a4eafca4 100644 --- a/src/lib/protocols/skype.c +++ b/src/lib/protocols/skype.c @@ -31,13 +31,16 @@ static void ndpi_skype_report_protocol(struct ndpi_detection_module_struct *ndpi NDPI_LOG_INFO(ndpi_struct, "found skype\n"); ndpi_set_detected_protocol(ndpi_struct, flow, proto, NDPI_PROTOCOL_SKYPE); } - + +static int is_port(u_int16_t a, u_int16_t b, u_int16_t c) { + return(((a == c) || (b == c)) ? 1 : 0); +} + static void ndpi_check_skype(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) { struct ndpi_packet_struct *packet = &flow->packet; // const u_int8_t *packet_payload = packet->payload; u_int32_t payload_len = packet->payload_packet_len; - if(flow->host_server_name[0] != '\0') return; @@ -46,10 +49,15 @@ static void ndpi_check_skype(struct ndpi_detection_module_struct *ndpi_struct, s flow->l4.udp.skype_packet_id++; if(flow->l4.udp.skype_packet_id < 5) { + u_int16_t sport = ntohs(packet->udp->source); u_int16_t dport = ntohs(packet->udp->dest); /* skype-to-skype */ - if(dport != 1119) /* It can be confused with battle.net */ { + if(is_port(sport, dport, 1119) /* It can be confused with battle.net */ + || is_port(sport, dport, 80) /* No HTTP-like protocols UDP/80 */ + ) { + ; + } else { if(((payload_len == 3) && ((packet->payload[2] & 0x0F)== 0x0d)) || ((payload_len >= 16) && (packet->payload[0] != 0x30) /* Avoid invalid SNMP detection */ @@ -57,11 +65,12 @@ static void ndpi_check_skype(struct ndpi_detection_module_struct *ndpi_struct, s ndpi_skype_report_protocol(ndpi_struct, flow); } } + return; } + NDPI_EXCLUDE_PROTO(ndpi_struct, flow); return; - // TCP check } else if(packet->tcp != NULL) { flow->l4.tcp.skype_packet_id++; -- cgit v1.2.3